@opena2a/oasb 0.1.0

package/LICENSE

@@ -0,0 +1,98 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work.
+
+      "Contributor" shall mean any Legal Entity on behalf of whom a
+      Contribution has been received by the Licensor and subsequently
+      incorporated within the Work.
+
+      "Contribution" shall mean any work of authorship, including the
+      original version of the Work and any modifications or additions
+      to that Work, that is intentionally submitted to the Licensor for
+      inclusion in the Work by the copyright owner or by an individual
+      or Legal Entity authorized to submit on behalf of the copyright
+      owner.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      patent license to make, have made, use, offer to sell, sell,
+      import, and otherwise transfer the Work.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file.
+
+   5. Submission of Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND.
+
+   8. Limitation of Liability. In no event shall any Contributor be
+      liable to You for damages.
+
+   9. Accepting Warranty or Additional Liability.
+
+   Copyright 2025 OpenA2A
+   Licensed under the Apache License, Version 2.0

package/README.md

@@ -0,0 +1,287 @@
+# OASB — Open Agent Security Benchmark
+
+[![License: Apache-2.0](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0)
+[![Tests](https://img.shields.io/badge/tests-182%20passing-brightgreen)](https://github.com/opena2a-org/oasb)
+[![MITRE ATLAS](https://img.shields.io/badge/MITRE%20ATLAS-10%20techniques-teal)](https://atlas.mitre.org/)
+
+**MITRE ATT&CK Evaluations, but for AI agent security products.**
+
+182 standardized attack scenarios that evaluate whether a runtime security product can detect and respond to threats against AI agents. Each test is mapped to MITRE ATLAS and OWASP Agentic Top 10. Plug in your product, run the suite, get a detection coverage scorecard.
+
+[OASB Website](https://oasb.ai) | [OpenA2A](https://opena2a.org) | [MITRE ATLAS Coverage](#mitre-atlas-coverage) | [ARP (Reference Adapter)](https://github.com/opena2a-org/arp)
+
+---
+
+## What OASB Is (and Isn't)
+
+OASB evaluates **security products**, not agents. It answers: "does your runtime protection actually catch these attacks?"
+
+| | OASB | [HackMyAgent](https://github.com/opena2a-org/hackmyagent) |
+|---|---|---|
+| **Purpose** | Evaluate security *products* | Pentest AI *agents* |
+| **Tests** | "Does your EDR catch this exfiltration?" | "Is your agent leaking credentials?" |
+| **Audience** | Security product vendors, evaluators | Agent developers, red teams |
+| **Analogous to** | [MITRE ATT&CK Evaluations](https://attackevals.mitre-engenuity.org/) | [OWASP ZAP](https://www.zaproxy.org/) / Burp Suite |
+| **Method** | Controlled lab — inject attacks, measure detection | Active scanning + adversarial payloads against live targets |
+| **Output** | Detection coverage scorecard | Vulnerability report + auto-fix |
+
+Use both together: **HackMyAgent** finds vulnerabilities in your agent, **OASB** proves your security product catches real attacks.
+
+---
+
+## Table of Contents
+
+- [Quick Start](#quick-start)
+- [What Gets Tested](#what-gets-tested)
+- [Test Categories](#test-categories)
+  - [Atomic Tests](#atomic-tests-srcatomic) — 25 discrete detection tests
+  - [Integration Tests](#integration-tests-srcintegration) — 8 multi-step attack chains
+  - [Baseline Tests](#baseline-tests-srcbaseline) — 3 false positive validations
+  - [E2E Tests](#e2e-tests-srce2e) — 6 real OS-level detection tests
+- [MITRE ATLAS Coverage](#mitre-atlas-coverage)
+- [Test Harness](#test-harness)
+- [Known Detection Gaps](#known-detection-gaps)
+- [License](#license)
+
+---
+
+## Quick Start
+
+Currently ships with [ARP](https://github.com/opena2a-org/arp) as the reference adapter. Vendor adapter interface coming soon — implement the adapter for your product and run the same 182 tests.
+
+```bash
+git clone https://github.com/opena2a-org/arp.git
+git clone https://github.com/opena2a-org/oasb.git
+
+cd arp && npm install && npm run build && cd ..
+cd oasb && npm install
+```
+
+### Run the Evaluation
+
+```bash
+npm test                    # Full evaluation (182 tests)
+npm run test:atomic         # 25 atomic tests (no external deps)
+npm run test:integration    # 8 integration scenarios
+npm run test:baseline       # 3 baseline tests
+npx vitest run src/e2e/     # 6 E2E tests (real OS detection)
+```
+
+---
+
+## What Gets Tested
+
+Each test simulates a specific attack technique and checks whether the security product under evaluation detects it, classifies it correctly, and responds appropriately.
+
+| Category | Tests | What It Evaluates |
+|----------|-------|-------------------|
+| Process detection | 25 | Child process spawns, suspicious binaries, privilege escalation, CPU anomalies |
+| Network detection | 20 | Outbound connections, suspicious hosts, exfiltration, subdomain bypass |
+| Filesystem detection | 28 | Sensitive path access, credential files, dotfile persistence, mass file DoS |
+| Intelligence layers | 21 | Rule matching, anomaly scoring, LLM escalation, budget exhaustion |
+| Enforcement actions | 18 | Logging, alerting, process pause (SIGSTOP), kill (SIGTERM/SIGKILL), resume |
+| Multi-step attacks | 33 | Data exfiltration chains, MCP tool abuse, prompt injection, A2A trust exploitation |
+| Baseline behavior | 13 | False positive rates, anomaly injection, baseline persistence |
+| Real OS detection | 14 | Live filesystem watches, process polling, network monitoring |
+| Application-level hooks | 14 | Pre-execution interception of spawn, connect, read/write |
+| **Total** | **182** | **10 MITRE ATLAS techniques** |
+
+---
+
+## Test Categories
+
+### Atomic Tests (`src/atomic/`)
+
+Discrete tests that exercise individual detection capabilities. Each test injects a single attack event and verifies the product detects it with the correct classification and severity.
+
+<details>
+<summary><strong>Process Detection</strong> — 5 files</summary>
+
+| Test | ATLAS | What the Product Should Detect |
+|------|-------|-------------------------------|
+| AT-PROC-001 | AML.T0046 | Child process spawn |
+| AT-PROC-002 | AML.T0046 | Suspicious binary execution (curl, wget, nc) |
+| AT-PROC-003 | AML.T0029 | High CPU anomaly |
+| AT-PROC-004 | AML.T0046 | Privilege escalation (root user) |
+| AT-PROC-005 | AML.TA0006 | Process termination |
+
+</details>
+
+<details>
+<summary><strong>Network Detection</strong> — 5 files</summary>
+
+| Test | ATLAS | What the Product Should Detect |
+|------|-------|-------------------------------|
+| AT-NET-001 | AML.T0024 | New outbound connection |
+| AT-NET-002 | AML.T0057 | Connection to suspicious host (webhook.site, ngrok) |
+| AT-NET-003 | AML.T0029 | Connection burst |
+| AT-NET-004 | AML.T0024 | Subdomain bypass of allowlist |
+| AT-NET-005 | AML.T0057 | Exfiltration destination |
+
+</details>
+
+<details>
+<summary><strong>Filesystem Detection</strong> — 5 files</summary>
+
+| Test | ATLAS | What the Product Should Detect |
+|------|-------|-------------------------------|
+| AT-FS-001 | AML.T0057 | Sensitive path access (.ssh, .aws, .gnupg) |
+| AT-FS-002 | AML.T0046 | Access outside allowed paths |
+| AT-FS-003 | AML.T0057 | Credential file access (.npmrc, .pypirc, .netrc) |
+| AT-FS-004 | AML.T0029 | Mass file creation (DoS) |
+| AT-FS-005 | AML.T0018 | Shell config modification (.bashrc, .zshrc) |
+
+</details>
+
+<details>
+<summary><strong>Intelligence</strong> — 5 files</summary>
+
+| Test | ATLAS | What the Product Should Do |
+|------|-------|---------------------------|
+| AT-INT-001 | AML.T0054 | Match rules and trigger enforcement |
+| AT-INT-002 | AML.T0015 | Score statistical anomalies (z-score) |
+| AT-INT-003 | AML.T0054 | Escalate to LLM-assisted assessment |
+| AT-INT-004 | AML.T0029 | Handle budget exhaustion gracefully |
+| AT-INT-005 | AML.T0015 | Learn and reset behavioral baselines |
+
+</details>
+
+<details>
+<summary><strong>Enforcement</strong> — 5 files</summary>
+
+| Test | ATLAS | What the Product Should Do |
+|------|-------|---------------------------|
+| AT-ENF-001 | AML.TA0006 | Execute log action |
+| AT-ENF-002 | AML.TA0006 | Fire alert callback |
+| AT-ENF-003 | AML.TA0006 | Pause process (SIGSTOP) |
+| AT-ENF-004 | AML.TA0006 | Kill process (SIGTERM/SIGKILL) |
+| AT-ENF-005 | AML.TA0006 | Resume paused process (SIGCONT) |
+
+</details>
+
+---
+
+### Integration Tests (`src/integration/`)
+
+Multi-step attack chains that combine multiple techniques. Tests whether the product can detect coordinated attacks, not just isolated events. Optionally validates against live [DVAA](https://github.com/opena2a-org/damn-vulnerable-ai-agent) agents.
+
+| Test | ATLAS | Attack Chain |
+|------|-------|-------------|
+| INT-001 | AML.T0057 | Data exfiltration: internal contact lookup → credential harvest → webhook.site POST |
+| INT-002 | AML.T0056 | MCP tool abuse: path traversal + command injection via tool arguments |
+| INT-003 | AML.T0051 | Prompt injection: establish baseline → inject malicious prompt → measure detection |
+| INT-004 | AML.T0024 | A2A trust exploitation: spoofed agent identity → unauthorized data access |
+| INT-005 | AML.T0015 | Evasion: 5 minutes normal traffic → sudden attack burst → verify anomaly detection |
+| INT-006 | AML.T0046 | Multi-monitor correlation: single attack triggers process + network + filesystem events |
+| INT-007 | AML.T0029 | Budget exhaustion: noise flood drains LLM budget → real attack goes unanalyzed |
+| INT-008 | AML.TA0006 | Kill switch: critical threat → product kills agent → verify death → recovery |
+
+---
+
+### Baseline Tests (`src/baseline/`)
+
+Every security product must avoid false positives. These tests verify the product stays quiet during normal operations.
+
+| Test | What It Proves |
+|------|----------------|
+| BL-001 | Zero false positives from normal agent activity |
+| BL-002 | Controlled anomaly injection triggers detection (not silent) |
+| BL-003 | Baseline persistence across product restarts |
+
+---
+
+### E2E Tests (`src/e2e/`)
+
+Real OS-level detection — no mocks, no event injection. These tests spawn real processes, open real connections, and write real files, then verify the product detects them.
+
+<details>
+<summary><strong>Live Monitors</strong> — OS-level polling</summary>
+
+| Test | Latency | What the Product Should Detect |
+|------|---------|-------------------------------|
+| E2E-001 | ~200ms | fs.watch detects .env, .ssh, .bashrc, .npmrc writes |
+| E2E-002 | ~1000ms | ps polling detects child processes, suspicious binaries |
+| E2E-003 | ~1000ms | lsof detects outbound TCP (skips if unavailable) |
+
+</details>
+
+<details>
+<summary><strong>Interceptors</strong> — application-level hooks</summary>
+
+| Test | Latency | What the Product Should Intercept |
+|------|---------|----------------------------------|
+| E2E-004 | <1ms | child_process.spawn/exec intercepted before execution |
+| E2E-005 | <1ms | net.Socket.connect intercepted before connection |
+| E2E-006 | <1ms | fs.writeFileSync/readFileSync intercepted before I/O |
+
+</details>
+
+---
+
+## MITRE ATLAS Coverage
+
+10 unique techniques across 42 test files:
+
+| Technique | ID | Tests |
+|-----------|----|-------|
+| Unsafe ML Inference | AML.T0046 | AT-PROC-001/002/004, AT-FS-002, INT-006, E2E-002/004 |
+| Data Leakage | AML.T0057 | AT-NET-002/005, AT-FS-001/003, INT-001, E2E-001/006 |
+| Exfiltration | AML.T0024 | AT-NET-001/004, INT-004, E2E-003/005 |
+| Persistence | AML.T0018 | AT-FS-005, E2E-001/006 |
+| Denial of Service | AML.T0029 | AT-PROC-003, AT-NET-003, AT-INT-004, INT-007 |
+| Evasion | AML.T0015 | AT-INT-002/005, INT-005, BL-002/003 |
+| Jailbreak | AML.T0054 | AT-INT-001/003 |
+| MCP Compromise | AML.T0056 | INT-002 |
+| Prompt Injection | AML.T0051 | INT-003 |
+| Defense Response | AML.TA0006 | AT-ENF-001-005, AT-PROC-005, INT-008 |
+
+---
+
+## Test Harness
+
+The harness wraps a security product via an adapter interface and provides event collection, injection, and metrics.
+
+| File | Purpose |
+|------|---------|
+| `arp-wrapper.ts` | Reference adapter — wraps ARP with temp dataDir, event collection, injection helpers |
+| `event-collector.ts` | Captures events with async `waitForEvent(predicate, timeout)` |
+| `mock-llm-adapter.ts` | Deterministic LLM for intelligence layer testing (pattern-based responses) |
+| `dvaa-client.ts` | HTTP client for DVAA vulnerable agent endpoints |
+| `dvaa-manager.ts` | DVAA process lifecycle (spawn, health check, teardown) |
+| `metrics.ts` | Detection rate, false positive rate, P95 latency computation |
+
+To evaluate your own product: implement an adapter that translates OASB events into your product's API, then run the full suite. Vendor adapter interface spec coming soon.
+
+---
+
+## Known Detection Gaps
+
+OASB documents what the reference product (ARP) does and doesn't catch. Other products may have different gap profiles — that's the point of running the benchmark.
+
+| Gap | Severity | Test |
+|-----|----------|------|
+| Anomaly baselines not persisted across restarts | Medium | BL-003 |
+| No connection rate anomaly detection | Medium | AT-NET-003 |
+| No HTTP response/output monitoring | Architectural | INT-003 |
+| No cross-monitor event correlation | Architectural | INT-006 |
+
+---
+
+## License
+
+Apache-2.0
+
+---
+
+## OpenA2A Ecosystem
+
+| Project | What it does |
+|---------|-------------|
+| [**OASB**](https://github.com/opena2a-org/oasb) | Evaluate security products — 182 attack scenarios, MITRE ATLAS mapped |
+| [**HackMyAgent**](https://github.com/opena2a-org/hackmyagent) | Pentest AI agents — 147 checks, 55 attack payloads, auto-fix |
+| [**ARP**](https://github.com/opena2a-org/arp) | Runtime security monitoring for AI agents |
+| [**AIM**](https://github.com/opena2a-org/agent-identity-management) | Identity and access management for AI agents |
+| [**Secretless AI**](https://github.com/opena2a-org/secretless-ai) | Keep credentials out of AI context windows |
+| [**DVAA**](https://github.com/opena2a-org/damn-vulnerable-ai-agent) | Deliberately vulnerable AI agents for security training |
+
+[Website](https://opena2a.org) · [OASB](https://oasb.ai) · [Discord](https://discord.gg/uRZa3KXgEn) · [Email](mailto:info@opena2a.org)

package/config/arp-lab-default.yaml

@@ -0,0 +1,54 @@
+agentName: arp-lab-target
+agentDescription: "Test target agent for ARP security testing"
+declaredCapabilities:
+  - "file read/write in workspace"
+  - "HTTP requests to allowed APIs"
+  - "process management"
+
+monitors:
+  process:
+    enabled: true
+    intervalMs: 1000
+  network:
+    enabled: true
+    intervalMs: 2000
+    allowedHosts:
+      - "localhost"
+      - "127.0.0.1"
+  filesystem:
+    enabled: true
+
+rules:
+  - name: critical-threat
+    condition:
+      category: threat
+      minSeverity: critical
+    action: kill
+    requireLlmConfirmation: true
+
+  - name: high-violation
+    condition:
+      category: violation
+      minSeverity: high
+    action: alert
+
+  - name: suspicious-binary
+    condition:
+      source: process
+      category: violation
+      fieldMatch:
+        data.binary: "*"
+    action: alert
+
+  - name: anomaly-burst
+    condition:
+      category: anomaly
+      threshold:
+        count: 5
+        windowMs: 30000
+    action: alert
+    requireLlmConfirmation: true
+
+intelligence:
+  enabled: false
+  budgetUsd: 0

package/config/dvaa-targets.ts

@@ -0,0 +1,97 @@
+/** DVAA agent port and endpoint configuration */
+
+export interface DVAATarget {
+  name: string;
+  port: number;
+  protocol: 'api' | 'mcp' | 'a2a';
+  securityLevel: 'hardened' | 'weak' | 'vulnerable' | 'critical' | 'standard';
+  vulnerabilities: string[];
+}
+
+export const DVAA_TARGETS: DVAATarget[] = [
+  // API Agents (OpenAI-compatible)
+  {
+    name: 'SecureBot',
+    port: 3001,
+    protocol: 'api',
+    securityLevel: 'hardened',
+    vulnerabilities: [],
+  },
+  {
+    name: 'HelperBot',
+    port: 3002,
+    protocol: 'api',
+    securityLevel: 'weak',
+    vulnerabilities: ['promptInjection', 'dataExfiltration', 'contextManipulation'],
+  },
+  {
+    name: 'LegacyBot',
+    port: 3003,
+    protocol: 'api',
+    securityLevel: 'critical',
+    vulnerabilities: ['promptInjection', 'jailbreak', 'dataExfiltration', 'capabilityAbuse', 'contextManipulation'],
+  },
+  {
+    name: 'CodeBot',
+    port: 3004,
+    protocol: 'api',
+    securityLevel: 'vulnerable',
+    vulnerabilities: ['capabilityAbuse'],
+  },
+  {
+    name: 'RAGBot',
+    port: 3005,
+    protocol: 'api',
+    securityLevel: 'weak',
+    vulnerabilities: ['dataExfiltration'],
+  },
+  {
+    name: 'VisionBot',
+    port: 3006,
+    protocol: 'api',
+    securityLevel: 'weak',
+    vulnerabilities: ['promptInjection'],
+  },
+
+  // MCP Agents
+  {
+    name: 'ToolBot',
+    port: 3010,
+    protocol: 'mcp',
+    securityLevel: 'vulnerable',
+    vulnerabilities: ['capabilityAbuse', 'mcpToolExploitation'],
+  },
+  {
+    name: 'DataBot',
+    port: 3011,
+    protocol: 'mcp',
+    securityLevel: 'weak',
+    vulnerabilities: ['dataExfiltration'],
+  },
+
+  // A2A Agents
+  {
+    name: 'Orchestrator',
+    port: 3020,
+    protocol: 'a2a',
+    securityLevel: 'standard',
+    vulnerabilities: ['agentToAgent'],
+  },
+  {
+    name: 'Worker',
+    port: 3021,
+    protocol: 'a2a',
+    securityLevel: 'weak',
+    vulnerabilities: ['agentToAgent', 'capabilityAbuse'],
+  },
+];
+
+export const DVAA_DASHBOARD_PORT = 3000;
+
+export function getTarget(name: string): DVAATarget | undefined {
+  return DVAA_TARGETS.find((t) => t.name === name);
+}
+
+export function getTargetsByProtocol(protocol: 'api' | 'mcp' | 'a2a'): DVAATarget[] {
+  return DVAA_TARGETS.filter((t) => t.protocol === protocol);
+}

package/dist/harness/arp-wrapper.d.ts

@@ -0,0 +1,28 @@
+import { AgentRuntimeProtection, EventEngine, EnforcementEngine, type ARPEvent } from '@opena2a/arp';
+import { EventCollector } from './event-collector';
+import type { LabConfig } from './types';
+/**
+ * Wraps AgentRuntimeProtection for controlled testing.
+ * Creates temp dataDir per test, registers EventCollector,
+ * and provides injection + assertion helpers.
+ */
+export declare class ArpWrapper {
+    private arp;
+    private _dataDir;
+    readonly collector: EventCollector;
+    constructor(labConfig?: LabConfig);
+    start(): Promise<void>;
+    stop(): Promise<void>;
+    /** Get the underlying ARP instance */
+    getInstance(): AgentRuntimeProtection;
+    /** Get the event engine for direct event injection */
+    getEngine(): EventEngine;
+    /** Get the enforcement engine */
+    getEnforcement(): EnforcementEngine;
+    /** Inject a synthetic event into the ARP engine (for testing without real OS activity) */
+    injectEvent(event: Omit<ARPEvent, 'id' | 'timestamp' | 'classifiedBy'>): Promise<ARPEvent>;
+    /** Wait for an event matching a predicate */
+    waitForEvent(predicate: (event: ARPEvent) => boolean, timeoutMs?: number): Promise<ARPEvent>;
+    /** Get the data directory */
+    get dataDir(): string;
+}