@opena2a/oasb 0.1.0

package/dist/harness/mock-llm-adapter.js

@@ -0,0 +1,68 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.MockLLMAdapter = void 0;
+/**
+ * Deterministic LLM adapter for testing L2 intelligence layer.
+ * Returns structured responses based on input patterns.
+ */
+class MockLLMAdapter {
+    constructor(options) {
+        this.name = 'mock';
+        this.calls = [];
+        this.latencyMs = options?.latencyMs ?? 10;
+        this.costPerCall = options?.costPerCall ?? 0.001;
+    }
+    async assess(prompt, maxTokens) {
+        this.calls.push({ prompt, maxTokens, timestamp: Date.now() });
+        if (this.latencyMs > 0) {
+            await new Promise((r) => setTimeout(r, this.latencyMs));
+        }
+        const response = this.generateResponse(prompt);
+        return {
+            content: response,
+            inputTokens: Math.ceil(prompt.length / 4),
+            outputTokens: Math.ceil(response.length / 4),
+            model: 'mock-llm',
+        };
+    }
+    estimateCost(inputTokens, outputTokens) {
+        return this.costPerCall;
+    }
+    async healthCheck() {
+        return true;
+    }
+    /** Get number of calls made */
+    getCallCount() {
+        return this.calls.length;
+    }
+    /** Get all calls for assertions */
+    getCalls() {
+        return [...this.calls];
+    }
+    /** Get the most recent call */
+    getLastCall() {
+        return this.calls[this.calls.length - 1];
+    }
+    /** Reset call history */
+    reset() {
+        this.calls = [];
+    }
+    generateResponse(prompt) {
+        const lower = prompt.toLowerCase();
+        // Threat patterns - always flag
+        if (lower.includes('threat') || lower.includes('critical') || lower.includes('kill')) {
+            return 'CONSISTENT: NO\nCONFIDENCE: 0.95\nREASONING: Activity matches known attack pattern\nACTION: KILL';
+        }
+        // Violation patterns - flag as suspicious
+        if (lower.includes('violation') || lower.includes('suspicious') || lower.includes('high')) {
+            return 'CONSISTENT: NO\nCONFIDENCE: 0.80\nREASONING: Behavior deviates from declared capabilities\nACTION: ALERT';
+        }
+        // Anomaly patterns - moderate concern
+        if (lower.includes('anomaly') || lower.includes('unusual')) {
+            return 'CONSISTENT: NO\nCONFIDENCE: 0.60\nREASONING: Statistical anomaly detected but uncertain\nACTION: ALERT';
+        }
+        // Default - allow
+        return 'CONSISTENT: YES\nCONFIDENCE: 0.90\nREASONING: Activity consistent with agent purpose\nACTION: ALLOW';
+    }
+}
+exports.MockLLMAdapter = MockLLMAdapter;

package/dist/harness/types.d.ts

@@ -0,0 +1,73 @@
+import type { ARPEvent, EnforcementResult } from '@opena2a/arp';
+/** Annotation metadata for test cases */
+export interface TestAnnotation {
+    /** Is this scenario an actual attack? */
+    isAttack: boolean;
+    /** MITRE ATLAS technique ID */
+    atlasId?: string;
+    /** OWASP Agentic Top 10 category */
+    owaspId?: string;
+    /** Whether ARP should detect this */
+    expectedDetection: boolean;
+    /** Expected minimum severity if detected */
+    expectedSeverity?: 'info' | 'low' | 'medium' | 'high' | 'critical';
+    /** Timestamp when the attack was initiated */
+    attackTimestamp?: number;
+}
+/** Collected test result with timing info */
+export interface TestResult {
+    testId: string;
+    annotation: TestAnnotation;
+    detected: boolean;
+    detectionTimeMs?: number;
+    events: ARPEvent[];
+    enforcements: EnforcementResult[];
+}
+/** Suite-level metrics */
+export interface SuiteMetrics {
+    totalTests: number;
+    attacks: number;
+    benign: number;
+    truePositives: number;
+    falsePositives: number;
+    trueNegatives: number;
+    falseNegatives: number;
+    detectionRate: number;
+    falsePositiveRate: number;
+    meanDetectionTimeMs: number;
+    p95DetectionTimeMs: number;
+}
+/** ARP wrapper configuration for tests */
+export interface LabConfig {
+    monitors?: {
+        process?: boolean;
+        network?: boolean;
+        filesystem?: boolean;
+    };
+    rules?: import('@opena2a/arp').AlertRule[];
+    intelligence?: {
+        enabled?: boolean;
+    };
+    /** Temp data dir (auto-created per test) */
+    dataDir?: string;
+    /** Filesystem paths to watch (for real FilesystemMonitor) */
+    filesystemWatchPaths?: string[];
+    /** Filesystem allowed paths (for real FilesystemMonitor) */
+    filesystemAllowedPaths?: string[];
+    /** Network allowed hosts (for real NetworkMonitor) */
+    networkAllowedHosts?: string[];
+    /** Process monitor poll interval in ms */
+    processIntervalMs?: number;
+    /** Network monitor poll interval in ms */
+    networkIntervalMs?: number;
+    /** Application-level interceptors (zero-latency hooks) */
+    interceptors?: {
+        process?: boolean;
+        network?: boolean;
+        filesystem?: boolean;
+    };
+    /** Interceptor network allowed hosts */
+    interceptorNetworkAllowedHosts?: string[];
+    /** Interceptor filesystem allowed paths */
+    interceptorFilesystemAllowedPaths?: string[];
+}

package/dist/harness/types.js

@@ -0,0 +1,2 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });

package/package.json

@@ -0,0 +1,39 @@
+{
+  "name": "@opena2a/oasb",
+  "version": "0.1.0",
+  "description": "Open Agent Security Benchmark — 182 attack scenarios mapped to MITRE ATLAS and OWASP Agentic Top 10",
+  "main": "dist/index.js",
+  "types": "dist/index.d.ts",
+  "files": ["dist", "src", "config", "README.md", "LICENSE"],
+  "scripts": {
+    "build": "tsc",
+    "test": "vitest run",
+    "test:atomic": "vitest run src/atomic/",
+    "test:integration": "vitest run src/integration/",
+    "test:baseline": "vitest run src/baseline/",
+    "test:watch": "vitest",
+    "report": "npx tsx scripts/generate-report.ts"
+  },
+  "dependencies": {
+    "@opena2a/arp": "^0.1.0"
+  },
+  "devDependencies": {
+    "@types/node": "^20.0.0",
+    "typescript": "^5.3.3",
+    "vitest": "^3.0.0"
+  },
+  "engines": {
+    "node": ">=18.0.0"
+  },
+  "keywords": ["ai", "agent", "security", "benchmark", "oasb", "mitre-atlas", "evaluation", "runtime-protection", "opena2a"],
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/opena2a-org/oasb.git"
+  },
+  "homepage": "https://oasb.ai/eval",
+  "bugs": {
+    "url": "https://github.com/opena2a-org/oasb/issues"
+  },
+  "author": "OpenA2A",
+  "license": "Apache-2.0"
+}

package/src/atomic/enforcement/AT-ENF-001.log-action.test.ts

@@ -0,0 +1,89 @@
+// AT-ENF-001: Log Enforcement Action
+// Tests that 'log' action records event without process manipulation.
+// Verifies the simplest enforcement path: event in, log result out.
+
+import { describe, it, expect, beforeEach, afterEach } from 'vitest';
+import { ArpWrapper } from '../../harness/arp-wrapper';
+import type { ARPEvent } from '@opena2a/arp';
+
+describe('AT-ENF-001: Log Enforcement Action', () => {
+  let arp: ArpWrapper;
+
+  beforeEach(async () => {
+    arp = new ArpWrapper({
+      monitors: { process: false, network: false, filesystem: false },
+      rules: [
+        {
+          name: 'log-rule',
+          condition: { category: 'anomaly' },
+          action: 'log',
+        },
+      ],
+    });
+    await arp.start();
+  });
+
+  afterEach(async () => {
+    await arp.stop();
+  });
+
+  it('should return success when executing log action', async () => {
+    const mockEvent: ARPEvent = {
+      id: 'test-enf-001-1',
+      timestamp: new Date().toISOString(),
+      source: 'process',
+      category: 'anomaly',
+      severity: 'medium',
+      description: 'Anomalous process behavior detected',
+      data: {},
+      classifiedBy: 'L0-rules',
+    };
+
+    const enforcement = arp.getEnforcement();
+    const result = await enforcement.execute('log', mockEvent);
+
+    expect(result.action).toBe('log');
+    expect(result.success).toBe(true);
+    expect(result.event).toBe(mockEvent);
+  });
+
+  it('should not set a targetPid for log actions', async () => {
+    const mockEvent: ARPEvent = {
+      id: 'test-enf-001-2',
+      timestamp: new Date().toISOString(),
+      source: 'process',
+      category: 'anomaly',
+      severity: 'high',
+      description: 'Suspicious activity logged',
+      data: { pid: 99999 },
+      classifiedBy: 'L0-rules',
+    };
+
+    const enforcement = arp.getEnforcement();
+    const result = await enforcement.execute('log', mockEvent);
+
+    expect(result.action).toBe('log');
+    expect(result.success).toBe(true);
+    expect(result.targetPid).toBeUndefined();
+  });
+
+  it('should include a reason string in the result', async () => {
+    const mockEvent: ARPEvent = {
+      id: 'test-enf-001-3',
+      timestamp: new Date().toISOString(),
+      source: 'network',
+      category: 'anomaly',
+      severity: 'low',
+      description: 'Unexpected outbound connection',
+      data: {},
+      classifiedBy: 'L0-rules',
+    };
+
+    const enforcement = arp.getEnforcement();
+    const result = await enforcement.execute('log', mockEvent);
+
+    expect(result.reason).toBeDefined();
+    expect(typeof result.reason).toBe('string');
+    expect(result.reason.length).toBeGreaterThan(0);
+  });
+});

package/src/atomic/enforcement/AT-ENF-002.alert-callback.test.ts

@@ -0,0 +1,120 @@
+// AT-ENF-002: Alert Callback Execution
+// Tests the alert callback mechanism in EnforcementEngine.
+// Verifies that registered callbacks fire on alert actions and
+// that alerts succeed even without a callback registered.
+
+import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest';
+import { ArpWrapper } from '../../harness/arp-wrapper';
+import type { ARPEvent, EnforcementResult } from '@opena2a/arp';
+
+describe('AT-ENF-002: Alert Callback Execution', () => {
+  let arp: ArpWrapper;
+
+  beforeEach(async () => {
+    arp = new ArpWrapper({
+      monitors: { process: false, network: false, filesystem: false },
+    });
+    await arp.start();
+  });
+
+  afterEach(async () => {
+    await arp.stop();
+  });
+
+  it('should invoke the alert callback with event and result', async () => {
+    const callbackFn = vi.fn();
+    const enforcement = arp.getEnforcement();
+    enforcement.setAlertCallback(callbackFn);
+
+    const mockEvent: ARPEvent = {
+      id: 'test-enf-002-1',
+      timestamp: new Date().toISOString(),
+      source: 'process',
+      category: 'violation',
+      severity: 'high',
+      description: 'Unauthorized process spawn',
+      data: {},
+      classifiedBy: 'L0-rules',
+    };
+
+    const result = await enforcement.execute('alert', mockEvent);
+
+    expect(result.action).toBe('alert');
+    expect(result.success).toBe(true);
+    expect(callbackFn).toHaveBeenCalledOnce();
+    expect(callbackFn).toHaveBeenCalledWith(mockEvent, expect.objectContaining({
+      action: 'alert',
+      success: true,
+    }));
+  });
+
+  it('should succeed without a callback registered', async () => {
+    const enforcement = arp.getEnforcement();
+    // No callback set
+
+    const mockEvent: ARPEvent = {
+      id: 'test-enf-002-2',
+      timestamp: new Date().toISOString(),
+      source: 'network',
+      category: 'anomaly',
+      severity: 'medium',
+      description: 'Suspicious outbound connection',
+      data: {},
+      classifiedBy: 'L0-rules',
+    };
+
+    const result = await enforcement.execute('alert', mockEvent);
+
+    expect(result.action).toBe('alert');
+    expect(result.success).toBe(true);
+  });
+
+  it('should succeed even if the callback throws an error', async () => {
+    const enforcement = arp.getEnforcement();
+    enforcement.setAlertCallback(() => {
+      throw new Error('Callback failure');
+    });
+
+    const mockEvent: ARPEvent = {
+      id: 'test-enf-002-3',
+      timestamp: new Date().toISOString(),
+      source: 'filesystem',
+      category: 'threat',
+      severity: 'critical',
+      description: 'Credential file access attempt',
+      data: {},
+      classifiedBy: 'L0-rules',
+    };
+
+    const result = await enforcement.execute('alert', mockEvent);
+
+    // Callback errors should not block enforcement
+    expect(result.action).toBe('alert');
+    expect(result.success).toBe(true);
+  });
+
+  it('should replace a previous callback when setAlertCallback is called again', async () => {
+    const firstCallback = vi.fn();
+    const secondCallback = vi.fn();
+    const enforcement = arp.getEnforcement();
+
+    enforcement.setAlertCallback(firstCallback);
+    enforcement.setAlertCallback(secondCallback);
+
+    const mockEvent: ARPEvent = {
+      id: 'test-enf-002-4',
+      timestamp: new Date().toISOString(),
+      source: 'process',
+      category: 'violation',
+      severity: 'high',
+      description: 'Process violation detected',
+      data: {},
+      classifiedBy: 'L0-rules',
+    };
+
+    await enforcement.execute('alert', mockEvent);
+
+    expect(firstCallback).not.toHaveBeenCalled();
+    expect(secondCallback).toHaveBeenCalledOnce();
+  });
+});

package/src/atomic/enforcement/AT-ENF-003.pause-sigstop.test.ts

@@ -0,0 +1,104 @@
+// AT-ENF-003: Process Pause via SIGSTOP
+// Tests enforcement engine's ability to pause a running process.
+// Spawns a real child process and uses SIGSTOP to suspend it,
+// then verifies the process appears in the paused PID list.
+
+import { describe, it, expect, beforeEach, afterEach } from 'vitest';
+import { spawn, type ChildProcess } from 'child_process';
+import { ArpWrapper } from '../../harness/arp-wrapper';
+import type { ARPEvent } from '@opena2a/arp';
+
+describe('AT-ENF-003: Process Pause via SIGSTOP', () => {
+  let arp: ArpWrapper;
+  let child: ChildProcess | null = null;
+
+  beforeEach(async () => {
+    arp = new ArpWrapper({
+      monitors: { process: false, network: false, filesystem: false },
+    });
+    await arp.start();
+  });
+
+  afterEach(async () => {
+    if (child && child.pid) {
+      // Resume if paused, then kill
+      try {
+        arp.getEnforcement().resume(child.pid);
+      } catch { /* already resumed or dead */ }
+      try {
+        process.kill(child.pid, 'SIGKILL');
+      } catch { /* already dead */ }
+      child = null;
+    }
+    await arp.stop();
+  });
+
+  it('should pause a running process and track its PID', async () => {
+    child = spawn('node', ['-e', 'setTimeout(()=>{},30000)'], {
+      stdio: 'ignore',
+      detached: false,
+    });
+    const pid = child.pid!;
+    expect(pid).toBeDefined();
+
+    const mockEvent: ARPEvent = {
+      id: 'test-enf-003-1',
+      timestamp: new Date().toISOString(),
+      source: 'process',
+      category: 'violation',
+      severity: 'high',
+      description: 'Suspicious process activity',
+      data: { pid },
+      classifiedBy: 'L0-rules',
+    };
+
+    const enforcement = arp.getEnforcement();
+    const result = await enforcement.execute('pause', mockEvent, pid);
+
+    expect(result.action).toBe('pause');
+    expect(result.success).toBe(true);
+    expect(result.targetPid).toBe(pid);
+    expect(enforcement.getPausedPids()).toContain(pid);
+  }, 10000);
+
+  it('should fail to pause when no PID is provided', async () => {
+    const mockEvent: ARPEvent = {
+      id: 'test-enf-003-2',
+      timestamp: new Date().toISOString(),
+      source: 'process',
+      category: 'violation',
+      severity: 'high',
+      description: 'No PID available',
+      data: {},
+      classifiedBy: 'L0-rules',
+    };
+
+    const enforcement = arp.getEnforcement();
+    const result = await enforcement.execute('pause', mockEvent);
+
+    expect(result.action).toBe('pause');
+    expect(result.success).toBe(false);
+    expect(result.reason).toContain('No PID');
+  });
+
+  it('should fail to pause a non-existent process', async () => {
+    const fakePid = 999999;
+    const mockEvent: ARPEvent = {
+      id: 'test-enf-003-3',
+      timestamp: new Date().toISOString(),
+      source: 'process',
+      category: 'violation',
+      severity: 'high',
+      description: 'Attempt to pause non-existent process',
+      data: {},
+      classifiedBy: 'L0-rules',
+    };
+
+    const enforcement = arp.getEnforcement();
+    const result = await enforcement.execute('pause', mockEvent, fakePid);
+
+    expect(result.action).toBe('pause');
+    expect(result.success).toBe(false);
+    expect(enforcement.getPausedPids()).not.toContain(fakePid);
+  });
+});

package/src/atomic/enforcement/AT-ENF-004.kill-sigterm.test.ts

@@ -0,0 +1,153 @@
+// AT-ENF-004: Process Kill via SIGTERM
+// Tests enforcement engine's ability to terminate a running process.
+// Spawns a real child process, sends SIGTERM via the enforcement engine,
+// then verifies the process is no longer alive.
+
+import { describe, it, expect, beforeEach, afterEach } from 'vitest';
+import { spawn, type ChildProcess } from 'child_process';
+import { ArpWrapper } from '../../harness/arp-wrapper';
+import type { ARPEvent } from '@opena2a/arp';
+
+/** Wait for a specified number of milliseconds */
+function sleep(ms: number): Promise<void> {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+/** Check if a process is still alive */
+function isProcessAlive(pid: number): boolean {
+  try {
+    process.kill(pid, 0);
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+describe('AT-ENF-004: Process Kill via SIGTERM', () => {
+  let arp: ArpWrapper;
+  let child: ChildProcess | null = null;
+
+  beforeEach(async () => {
+    arp = new ArpWrapper({
+      monitors: { process: false, network: false, filesystem: false },
+    });
+    await arp.start();
+  });
+
+  afterEach(async () => {
+    if (child && child.pid) {
+      try {
+        process.kill(child.pid, 'SIGKILL');
+      } catch { /* already dead */ }
+      child = null;
+    }
+    await arp.stop();
+  });
+
+  it('should kill a running process and return success', async () => {
+    child = spawn('node', ['-e', 'setTimeout(()=>{},30000)'], {
+      stdio: 'ignore',
+      detached: false,
+    });
+    const pid = child.pid!;
+    expect(pid).toBeDefined();
+    expect(isProcessAlive(pid)).toBe(true);
+
+    const mockEvent: ARPEvent = {
+      id: 'test-enf-004-1',
+      timestamp: new Date().toISOString(),
+      source: 'process',
+      category: 'threat',
+      severity: 'critical',
+      description: 'Malicious process detected — terminating',
+      data: { pid },
+      classifiedBy: 'L0-rules',
+    };
+
+    const enforcement = arp.getEnforcement();
+    const result = await enforcement.execute('kill', mockEvent, pid);
+
+    expect(result.action).toBe('kill');
+    expect(result.success).toBe(true);
+    expect(result.targetPid).toBe(pid);
+
+    // Wait for SIGTERM to take effect
+    await sleep(2000);
+
+    expect(isProcessAlive(pid)).toBe(false);
+  }, 10000);
+
+  it('should fail to kill when no PID is provided', async () => {
+    const mockEvent: ARPEvent = {
+      id: 'test-enf-004-2',
+      timestamp: new Date().toISOString(),
+      source: 'process',
+      category: 'threat',
+      severity: 'critical',
+      description: 'No PID available for kill',
+      data: {},
+      classifiedBy: 'L0-rules',
+    };
+
+    const enforcement = arp.getEnforcement();
+    const result = await enforcement.execute('kill', mockEvent);
+
+    expect(result.action).toBe('kill');
+    expect(result.success).toBe(false);
+    expect(result.reason).toContain('No PID');
+  });
+
+  it('should fail to kill a non-existent process', async () => {
+    const fakePid = 999999;
+    const mockEvent: ARPEvent = {
+      id: 'test-enf-004-3',
+      timestamp: new Date().toISOString(),
+      source: 'process',
+      category: 'threat',
+      severity: 'critical',
+      description: 'Attempt to kill non-existent process',
+      data: {},
+      classifiedBy: 'L0-rules',
+    };
+
+    const enforcement = arp.getEnforcement();
+    const result = await enforcement.execute('kill', mockEvent, fakePid);
+
+    expect(result.action).toBe('kill');
+    expect(result.success).toBe(false);
+  });
+
+  it('should remove a killed PID from paused list if it was paused', async () => {
+    child = spawn('node', ['-e', 'setTimeout(()=>{},30000)'], {
+      stdio: 'ignore',
+      detached: false,
+    });
+    const pid = child.pid!;
+    expect(pid).toBeDefined();
+
+    const mockEvent: ARPEvent = {
+      id: 'test-enf-004-4',
+      timestamp: new Date().toISOString(),
+      source: 'process',
+      category: 'threat',
+      severity: 'critical',
+      description: 'Paused process escalated to kill',
+      data: { pid },
+      classifiedBy: 'L0-rules',
+    };
+
+    const enforcement = arp.getEnforcement();
+
+    // First pause it
+    await enforcement.execute('pause', mockEvent, pid);
+    expect(enforcement.getPausedPids()).toContain(pid);
+
+    // Then kill it
+    const result = await enforcement.execute('kill', mockEvent, pid);
+    expect(result.success).toBe(true);
+    expect(enforcement.getPausedPids()).not.toContain(pid);
+
+    await sleep(1000);
+    expect(isProcessAlive(pid)).toBe(false);
+  }, 10000);
+});