@opena2a/oasb 0.1.0

package/src/harness/dvaa-client.ts

@@ -0,0 +1,130 @@
+import http from 'http';
+
+interface ChatResponse {
+  id: string;
+  choices: Array<{
+    message: { role: string; content: string };
+    finish_reason: string;
+  }>;
+}
+
+interface MCPToolResponse {
+  success: boolean;
+  content?: string;
+  output?: string;
+  results?: unknown[];
+  note?: string;
+}
+
+interface HealthResponse {
+  status: string;
+  agent: string;
+  port: number;
+}
+
+interface StatsResponse {
+  totalRequests: number;
+  attacksDetected: number;
+  attacksSuccessful: number;
+}
+
+/**
+ * HTTP client for DVAA agent endpoints.
+ */
+export class DVAAClient {
+  /** Send a chat message to an API agent */
+  async chat(port: number, message: string): Promise<ChatResponse> {
+    return this.post<ChatResponse>(port, '/v1/chat/completions', {
+      messages: [{ role: 'user', content: message }],
+    });
+  }
+
+  /** Execute an MCP tool on an MCP agent */
+  async mcpExecute(port: number, tool: string, args: Record<string, unknown>): Promise<MCPToolResponse> {
+    return this.post<MCPToolResponse>(port, '/mcp/execute', {
+      tool,
+      arguments: args,
+    });
+  }
+
+  /** Send an A2A message */
+  async a2aMessage(port: number, from: string, message: string): Promise<ChatResponse> {
+    return this.post<ChatResponse>(port, '/v1/chat/completions', {
+      messages: [
+        { role: 'system', content: `Message from agent: ${from}` },
+        { role: 'user', content: message },
+      ],
+    });
+  }
+
+  /** Health check */
+  async health(port: number): Promise<HealthResponse> {
+    return this.get<HealthResponse>(port, '/health');
+  }
+
+  /** Get stats */
+  async stats(port: number): Promise<StatsResponse> {
+    return this.get<StatsResponse>(port, '/stats');
+  }
+
+  private get<T>(port: number, path: string): Promise<T> {
+    return new Promise((resolve, reject) => {
+      const req = http.get(`http://localhost:${port}${path}`, (res) => {
+        let body = '';
+        res.on('data', (chunk) => { body += chunk; });
+        res.on('end', () => {
+          try {
+            resolve(JSON.parse(body) as T);
+          } catch {
+            reject(new Error(`Invalid JSON from port ${port}${path}: ${body.slice(0, 200)}`));
+          }
+        });
+      });
+
+      req.on('error', reject);
+      req.setTimeout(10000, () => {
+        req.destroy();
+        reject(new Error(`Request to port ${port}${path} timed out`));
+      });
+    });
+  }
+
+  private post<T>(port: number, path: string, body: unknown): Promise<T> {
+    const payload = JSON.stringify(body);
+
+    return new Promise((resolve, reject) => {
+      const req = http.request(
+        {
+          hostname: 'localhost',
+          port,
+          path,
+          method: 'POST',
+          headers: {
+            'Content-Type': 'application/json',
+            'Content-Length': Buffer.byteLength(payload),
+          },
+        },
+        (res) => {
+          let data = '';
+          res.on('data', (chunk) => { data += chunk; });
+          res.on('end', () => {
+            try {
+              resolve(JSON.parse(data) as T);
+            } catch {
+              reject(new Error(`Invalid JSON from port ${port}${path}: ${data.slice(0, 200)}`));
+            }
+          });
+        },
+      );
+
+      req.on('error', reject);
+      req.setTimeout(10000, () => {
+        req.destroy();
+        reject(new Error(`POST to port ${port}${path} timed out`));
+      });
+
+      req.write(payload);
+      req.end();
+    });
+  }
+}

package/src/harness/dvaa-manager.ts

@@ -0,0 +1,106 @@
+import { fork, type ChildProcess } from 'child_process';
+import * as path from 'path';
+import http from 'http';
+
+const DVAA_PATH = path.resolve(__dirname, '../../../damn-vulnerable-ai-agent');
+const HEALTH_CHECK_TIMEOUT = 30000;
+const HEALTH_CHECK_INTERVAL = 500;
+
+/**
+ * Manages the DVAA (Damn Vulnerable AI Agent) process lifecycle for integration tests.
+ */
+export class DVAAManager {
+  private process: ChildProcess | null = null;
+  private started = false;
+
+  /** Start DVAA with all agents */
+  async start(): Promise<void> {
+    if (this.started) return;
+
+    const entryPoint = path.join(DVAA_PATH, 'src', 'index.js');
+
+    this.process = fork(entryPoint, [], {
+      cwd: DVAA_PATH,
+      stdio: 'pipe',
+      env: { ...process.env, NODE_ENV: 'test' },
+    });
+
+    this.process.on('error', (err) => {
+      console.error('DVAA process error:', err.message);
+    });
+
+    // Wait for health checks on key ports
+    await this.waitForHealth(3000); // Dashboard
+    await this.waitForHealth(3001); // SecureBot
+    await this.waitForHealth(3003); // LegacyBot
+
+    this.started = true;
+  }
+
+  /** Stop DVAA gracefully */
+  async stop(): Promise<void> {
+    if (!this.process || !this.started) return;
+
+    return new Promise<void>((resolve) => {
+      const timeout = setTimeout(() => {
+        if (this.process) {
+          this.process.kill('SIGKILL');
+        }
+        resolve();
+      }, 5000);
+
+      this.process!.once('exit', () => {
+        clearTimeout(timeout);
+        resolve();
+      });
+
+      this.process!.kill('SIGTERM');
+      this.started = false;
+      this.process = null;
+    });
+  }
+
+  /** Get the DVAA process PID (for ARP to monitor) */
+  getPid(): number | undefined {
+    return this.process?.pid;
+  }
+
+  /** Check if DVAA is running */
+  isRunning(): boolean {
+    return this.started && this.process !== null;
+  }
+
+  private waitForHealth(port: number): Promise<void> {
+    const deadline = Date.now() + HEALTH_CHECK_TIMEOUT;
+
+    return new Promise((resolve, reject) => {
+      const check = () => {
+        if (Date.now() > deadline) {
+          reject(new Error(`DVAA health check timed out on port ${port}`));
+          return;
+        }
+
+        const req = http.get(`http://localhost:${port}/health`, (res) => {
+          if (res.statusCode === 200) {
+            res.resume();
+            resolve();
+          } else {
+            res.resume();
+            setTimeout(check, HEALTH_CHECK_INTERVAL);
+          }
+        });
+
+        req.on('error', () => {
+          setTimeout(check, HEALTH_CHECK_INTERVAL);
+        });
+
+        req.setTimeout(2000, () => {
+          req.destroy();
+          setTimeout(check, HEALTH_CHECK_INTERVAL);
+        });
+      };
+
+      check();
+    });
+  }
+}

package/src/harness/event-collector.ts

@@ -0,0 +1,100 @@
+import type { ARPEvent, EnforcementResult } from '@opena2a/arp';
+
+/**
+ * Collects ARP events and enforcement results for test assertions.
+ * Supports async waiting for specific events with timeout.
+ */
+export class EventCollector {
+  private events: ARPEvent[] = [];
+  private enforcements: EnforcementResult[] = [];
+  private waiters: Array<{
+    predicate: (event: ARPEvent) => boolean;
+    resolve: (event: ARPEvent) => void;
+    timer: ReturnType<typeof setTimeout>;
+  }> = [];
+
+  /** Handler to register on ARP's onEvent */
+  readonly eventHandler = (event: ARPEvent): void => {
+    this.events.push(event);
+
+    // Check if any waiters match
+    for (let i = this.waiters.length - 1; i >= 0; i--) {
+      const waiter = this.waiters[i];
+      if (waiter.predicate(event)) {
+        clearTimeout(waiter.timer);
+        waiter.resolve(event);
+        this.waiters.splice(i, 1);
+      }
+    }
+  };
+
+  /** Handler to register on ARP's onEnforcement */
+  readonly enforcementHandler = (result: EnforcementResult): void => {
+    this.enforcements.push(result);
+  };
+
+  /** Wait for an event matching a predicate, with timeout */
+  waitForEvent(
+    predicate: (event: ARPEvent) => boolean,
+    timeoutMs: number = 10000,
+  ): Promise<ARPEvent> {
+    // Check existing events first
+    const existing = this.events.find(predicate);
+    if (existing) return Promise.resolve(existing);
+
+    return new Promise((resolve, reject) => {
+      const timer = setTimeout(() => {
+        const idx = this.waiters.findIndex((w) => w.resolve === resolve);
+        if (idx >= 0) this.waiters.splice(idx, 1);
+        reject(new Error(`Timed out after ${timeoutMs}ms waiting for event`));
+      }, timeoutMs);
+
+      this.waiters.push({ predicate, resolve, timer });
+    });
+  }
+
+  /** Check if any event matches a predicate */
+  hasEvent(predicate: (event: ARPEvent) => boolean): boolean {
+    return this.events.some(predicate);
+  }
+
+  /** Get all events */
+  getEvents(): ARPEvent[] {
+    return [...this.events];
+  }
+
+  /** Get events by category */
+  eventsByCategory(category: string): ARPEvent[] {
+    return this.events.filter((e) => e.category === category);
+  }
+
+  /** Get events by severity */
+  eventsBySeverity(severity: string): ARPEvent[] {
+    return this.events.filter((e) => e.severity === severity);
+  }
+
+  /** Get events by source */
+  eventsBySource(source: string): ARPEvent[] {
+    return this.events.filter((e) => e.source === source);
+  }
+
+  /** Get all enforcement results */
+  getEnforcements(): EnforcementResult[] {
+    return [...this.enforcements];
+  }
+
+  /** Get enforcement results by action */
+  enforcementsByAction(action: string): EnforcementResult[] {
+    return this.enforcements.filter((e) => e.action === action);
+  }
+
+  /** Reset all collected data */
+  reset(): void {
+    this.events = [];
+    this.enforcements = [];
+    for (const waiter of this.waiters) {
+      clearTimeout(waiter.timer);
+    }
+    this.waiters = [];
+  }
+}

package/src/harness/metrics.ts

@@ -0,0 +1,64 @@
+import type { TestResult, TestAnnotation, SuiteMetrics } from './types';
+
+/**
+ * Computes detection effectiveness metrics from test results.
+ */
+export function computeMetrics(results: TestResult[]): SuiteMetrics {
+  const attacks = results.filter((r) => r.annotation.isAttack);
+  const benign = results.filter((r) => !r.annotation.isAttack);
+
+  const truePositives = attacks.filter((r) => r.detected).length;
+  const falseNegatives = attacks.filter((r) => !r.detected).length;
+  const trueNegatives = benign.filter((r) => !r.detected).length;
+  const falsePositives = benign.filter((r) => r.detected).length;
+
+  const detectionTimes = attacks
+    .filter((r) => r.detected && r.detectionTimeMs !== undefined)
+    .map((r) => r.detectionTimeMs!);
+
+  detectionTimes.sort((a, b) => a - b);
+
+  const meanDetectionTimeMs = detectionTimes.length > 0
+    ? detectionTimes.reduce((sum, t) => sum + t, 0) / detectionTimes.length
+    : 0;
+
+  const p95Index = Math.ceil(detectionTimes.length * 0.95) - 1;
+  const p95DetectionTimeMs = detectionTimes.length > 0
+    ? detectionTimes[Math.max(0, p95Index)]
+    : 0;
+
+  return {
+    totalTests: results.length,
+    attacks: attacks.length,
+    benign: benign.length,
+    truePositives,
+    falsePositives,
+    trueNegatives,
+    falseNegatives,
+    detectionRate: attacks.length > 0 ? truePositives / attacks.length : 1,
+    falsePositiveRate: benign.length > 0 ? falsePositives / benign.length : 0,
+    meanDetectionTimeMs,
+    p95DetectionTimeMs,
+  };
+}
+
+/** Create a test annotation for attack scenarios */
+export function attackAnnotation(opts: {
+  atlasId?: string;
+  owaspId?: string;
+  expectedSeverity?: 'info' | 'low' | 'medium' | 'high' | 'critical';
+}): TestAnnotation {
+  return {
+    isAttack: true,
+    expectedDetection: true,
+    ...opts,
+  };
+}
+
+/** Create a test annotation for benign scenarios */
+export function benignAnnotation(): TestAnnotation {
+  return {
+    isAttack: false,
+    expectedDetection: false,
+  };
+}

package/src/harness/mock-llm-adapter.ts

@@ -0,0 +1,90 @@
+import type { LLMAdapter, LLMResponse } from '@opena2a/arp';
+
+interface MockCall {
+  prompt: string;
+  maxTokens: number;
+  timestamp: number;
+}
+
+/**
+ * Deterministic LLM adapter for testing L2 intelligence layer.
+ * Returns structured responses based on input patterns.
+ */
+export class MockLLMAdapter implements LLMAdapter {
+  readonly name = 'mock';
+  private calls: MockCall[] = [];
+  private latencyMs: number;
+  private costPerCall: number;
+
+  constructor(options?: { latencyMs?: number; costPerCall?: number }) {
+    this.latencyMs = options?.latencyMs ?? 10;
+    this.costPerCall = options?.costPerCall ?? 0.001;
+  }
+
+  async assess(prompt: string, maxTokens: number): Promise<LLMResponse> {
+    this.calls.push({ prompt, maxTokens, timestamp: Date.now() });
+
+    if (this.latencyMs > 0) {
+      await new Promise((r) => setTimeout(r, this.latencyMs));
+    }
+
+    const response = this.generateResponse(prompt);
+
+    return {
+      content: response,
+      inputTokens: Math.ceil(prompt.length / 4),
+      outputTokens: Math.ceil(response.length / 4),
+      model: 'mock-llm',
+    };
+  }
+
+  estimateCost(inputTokens: number, outputTokens: number): number {
+    return this.costPerCall;
+  }
+
+  async healthCheck(): Promise<boolean> {
+    return true;
+  }
+
+  /** Get number of calls made */
+  getCallCount(): number {
+    return this.calls.length;
+  }
+
+  /** Get all calls for assertions */
+  getCalls(): MockCall[] {
+    return [...this.calls];
+  }
+
+  /** Get the most recent call */
+  getLastCall(): MockCall | undefined {
+    return this.calls[this.calls.length - 1];
+  }
+
+  /** Reset call history */
+  reset(): void {
+    this.calls = [];
+  }
+
+  private generateResponse(prompt: string): string {
+    const lower = prompt.toLowerCase();
+
+    // Threat patterns - always flag
+    if (lower.includes('threat') || lower.includes('critical') || lower.includes('kill')) {
+      return 'CONSISTENT: NO\nCONFIDENCE: 0.95\nREASONING: Activity matches known attack pattern\nACTION: KILL';
+    }
+
+    // Violation patterns - flag as suspicious
+    if (lower.includes('violation') || lower.includes('suspicious') || lower.includes('high')) {
+      return 'CONSISTENT: NO\nCONFIDENCE: 0.80\nREASONING: Behavior deviates from declared capabilities\nACTION: ALERT';
+    }
+
+    // Anomaly patterns - moderate concern
+    if (lower.includes('anomaly') || lower.includes('unusual')) {
+      return 'CONSISTENT: NO\nCONFIDENCE: 0.60\nREASONING: Statistical anomaly detected but uncertain\nACTION: ALERT';
+    }
+
+    // Default - allow
+    return 'CONSISTENT: YES\nCONFIDENCE: 0.90\nREASONING: Activity consistent with agent purpose\nACTION: ALLOW';
+  }
+}

package/src/harness/types.ts

@@ -0,0 +1,77 @@
+import type { ARPEvent, EnforcementResult } from '@opena2a/arp';
+
+/** Annotation metadata for test cases */
+export interface TestAnnotation {
+  /** Is this scenario an actual attack? */
+  isAttack: boolean;
+  /** MITRE ATLAS technique ID */
+  atlasId?: string;
+  /** OWASP Agentic Top 10 category */
+  owaspId?: string;
+  /** Whether ARP should detect this */
+  expectedDetection: boolean;
+  /** Expected minimum severity if detected */
+  expectedSeverity?: 'info' | 'low' | 'medium' | 'high' | 'critical';
+  /** Timestamp when the attack was initiated */
+  attackTimestamp?: number;
+}
+
+/** Collected test result with timing info */
+export interface TestResult {
+  testId: string;
+  annotation: TestAnnotation;
+  detected: boolean;
+  detectionTimeMs?: number;
+  events: ARPEvent[];
+  enforcements: EnforcementResult[];
+}
+
+/** Suite-level metrics */
+export interface SuiteMetrics {
+  totalTests: number;
+  attacks: number;
+  benign: number;
+  truePositives: number;
+  falsePositives: number;
+  trueNegatives: number;
+  falseNegatives: number;
+  detectionRate: number;
+  falsePositiveRate: number;
+  meanDetectionTimeMs: number;
+  p95DetectionTimeMs: number;
+}
+
+/** ARP wrapper configuration for tests */
+export interface LabConfig {
+  monitors?: {
+    process?: boolean;
+    network?: boolean;
+    filesystem?: boolean;
+  };
+  rules?: import('@opena2a/arp').AlertRule[];
+  intelligence?: {
+    enabled?: boolean;
+  };
+  /** Temp data dir (auto-created per test) */
+  dataDir?: string;
+  /** Filesystem paths to watch (for real FilesystemMonitor) */
+  filesystemWatchPaths?: string[];
+  /** Filesystem allowed paths (for real FilesystemMonitor) */
+  filesystemAllowedPaths?: string[];
+  /** Network allowed hosts (for real NetworkMonitor) */
+  networkAllowedHosts?: string[];
+  /** Process monitor poll interval in ms */
+  processIntervalMs?: number;
+  /** Network monitor poll interval in ms */
+  networkIntervalMs?: number;
+  /** Application-level interceptors (zero-latency hooks) */
+  interceptors?: {
+    process?: boolean;
+    network?: boolean;
+    filesystem?: boolean;
+  };
+  /** Interceptor network allowed hosts */
+  interceptorNetworkAllowedHosts?: string[];
+  /** Interceptor filesystem allowed paths */
+  interceptorFilesystemAllowedPaths?: string[];
+}