@opena2a/oasb 0.2.0 → 0.3.1

package/src/harness/capabilities.ts

@@ -0,0 +1,79 @@
+/**
+ * Capability-aware test helpers.
+ *
+ * Tests call requireCapability() to skip gracefully when the
+ * adapter under test doesn't support a given feature. This produces
+ * an honest scorecard: N/A instead of FAIL.
+ *
+ * @example
+ *   import { requireCapability } from '../harness/capabilities';
+ *
+ *   describe('MCP Tool Scanning', () => {
+ *     requireCapability('mcp-scanning');
+ *     // tests only run if adapter has mcp-scanning
+ *   });
+ */
+import { describe } from 'vitest';
+import { createAdapter } from './create-adapter';
+import type { Capability, CapabilityMatrix } from './adapter';
+
+let _matrix: CapabilityMatrix | null = null;
+
+function getMatrix(): CapabilityMatrix {
+  if (!_matrix) {
+    const adapter = createAdapter();
+    _matrix = adapter.getCapabilities();
+  }
+  return _matrix;
+}
+
+/**
+ * Check if the current adapter has a capability.
+ */
+export function hasCapability(cap: Capability): boolean {
+  return getMatrix().capabilities.has(cap);
+}
+
+/**
+ * Call at the top of a describe() block to skip the entire suite
+ * if the adapter lacks the required capability.
+ *
+ * Uses describe.skipIf() so the tests show as skipped, not failed.
+ */
+export function requireCapability(cap: Capability): void {
+  const has = hasCapability(cap);
+  if (!has) {
+    // Can't use describe.skipIf at this point, but we can use
+    // a beforeAll that throws a skip. The caller should use
+    // describeWithCapability instead for cleaner skip behavior.
+  }
+}
+
+/**
+ * A describe() wrapper that skips the entire suite if the adapter
+ * lacks the required capability. Produces N/A in the scorecard.
+ *
+ * @example
+ *   describeWithCapability('mcp-scanning', 'MCP Tool Scanning', () => {
+ *     it('should detect path traversal', () => { ... });
+ *   });
+ */
+export const describeWithCapability = (
+  cap: Capability,
+  name: string,
+  fn: () => void,
+) => {
+  const has = hasCapability(cap);
+  if (has) {
+    describe(name, fn);
+  } else {
+    describe.skip(`${name} [requires: ${cap}]`, fn);
+  }
+};
+
+/**
+ * Get the full capability matrix for reporting.
+ */
+export function getCapabilityMatrix(): CapabilityMatrix {
+  return getMatrix();
+}

package/src/harness/create-adapter.ts

@@ -0,0 +1,53 @@
+/**
+ * Adapter factory — selects which security product adapter to use.
+ *
+ * Set OASB_ADAPTER env var to choose:
+ *   - "arp" (default) — uses arp-guard (must be installed)
+ *   - "llm-guard" — uses theRizwan/llm-guard
+ *   - path to a JS/TS module that exports a class implementing SecurityProductAdapter
+ *
+ * All test files import from here instead of instantiating adapters directly.
+ */
+import type { SecurityProductAdapter, LabConfig } from './adapter';
+
+// Eagerly resolve the adapter class at import time.
+// This file is only imported by tests that need the adapter,
+// so the cost is acceptable. Each wrapper handles lazy loading internally.
+import { ArpWrapper } from './arp-wrapper';
+import { LLMGuardWrapper } from './llm-guard-wrapper';
+import { RebuffWrapper } from './rebuff-wrapper';
+
+let AdapterClass: new (config?: LabConfig) => SecurityProductAdapter;
+
+const adapterName = process.env.OASB_ADAPTER || 'arp';
+
+switch (adapterName) {
+  case 'arp':
+    AdapterClass = ArpWrapper;
+    break;
+  case 'llm-guard':
+    AdapterClass = LLMGuardWrapper;
+    break;
+  case 'rebuff':
+    AdapterClass = RebuffWrapper;
+    break;
+  default: {
+    // Custom adapter — loaded at module level
+    // eslint-disable-next-line @typescript-eslint/no-var-requires
+    const mod = require(adapterName);
+    const Cls = mod.default || mod.Adapter || mod[Object.keys(mod)[0]];
+    if (!Cls || typeof Cls !== 'function') {
+      throw new Error(`Module "${adapterName}" does not export an adapter class`);
+    }
+    AdapterClass = Cls;
+    break;
+  }
+}
+
+/**
+ * Create a configured adapter instance.
+ * Uses OASB_ADAPTER env var to select the product under test.
+ */
+export function createAdapter(config?: LabConfig): SecurityProductAdapter {
+  return new AdapterClass(config);
+}

package/src/harness/event-collector.ts

@@ -1,4 +1,4 @@
-import type { ARPEvent, EnforcementResult } from '@opena2a/arp';
+import type { SecurityEvent as ARPEvent, EnforcementResult } from './adapter';
 
 /**
  * Collects ARP events and enforcement results for test assertions.

package/src/harness/llm-guard-wrapper.ts

@@ -0,0 +1,345 @@
+/**
+ * llm-guard Adapter — Third-party benchmark comparison
+ *
+ * Wraps theRizwan/llm-guard (npm: llm-guard) for OASB evaluation.
+ * This is a prompt-level scanner only — it does NOT provide:
+ * - Process/network/filesystem monitoring
+ * - MCP tool call validation
+ * - A2A message scanning
+ * - Anomaly detection / intelligence layers
+ * - Enforcement actions (pause/kill/resume)
+ *
+ * Tests that require these capabilities will get no-op implementations
+ * that return empty/negative results, documenting the coverage gap.
+ */
+import * as fs from 'fs';
+import * as os from 'os';
+import * as path from 'path';
+import { EventCollector } from './event-collector';
+import type {
+  SecurityProductAdapter,
+  SecurityEvent,
+  EnforcementResult,
+  EnforcementAction,
+  LabConfig,
+  PromptScanner,
+  MCPScanner,
+  A2AScanner,
+  PatternScanner,
+  BudgetManager,
+  AnomalyScorer,
+  EventEngine,
+  EnforcementEngine,
+  ScanResult,
+  ThreatPattern,
+  AlertRule,
+  CapabilityMatrix,
+} from './adapter';
+
+// Lazy-loaded llm-guard
+let _LLMGuard: any;
+function getLLMGuard(): any {
+  if (!_LLMGuard) {
+    _LLMGuard = require('llm-guard').LLMGuard;
+  }
+  return _LLMGuard;
+}
+
+/** Convert llm-guard result to OASB ScanResult */
+function toScanResult(guardResult: any): ScanResult {
+  const matches: ScanResult['matches'] = [];
+
+  if (guardResult.results) {
+    for (const r of guardResult.results) {
+      if (!r.valid && r.details) {
+        for (const d of r.details) {
+          matches.push({
+            pattern: {
+              id: d.rule || 'LLM-GUARD',
+              category: d.rule?.includes('jailbreak') ? 'jailbreak'
+                : d.rule?.includes('pii') ? 'data-exfiltration'
+                : d.rule?.includes('injection') ? 'prompt-injection'
+                : 'unknown',
+              description: d.message || '',
+              pattern: /./,
+              severity: guardResult.score <= 0.3 ? 'high' : 'medium',
+            },
+            matchedText: d.matched || '',
+          });
+        }
+      }
+    }
+  }
+
+  return {
+    detected: !guardResult.isValid,
+    matches,
+  };
+}
+
+/** Simple event engine that stores and emits events */
+class SimpleEventEngine implements EventEngine {
+  private handlers: Array<(event: SecurityEvent) => void | Promise<void>> = [];
+  private idCounter = 0;
+
+  emit(event: Omit<SecurityEvent, 'id' | 'timestamp' | 'classifiedBy'>): SecurityEvent {
+    const full: SecurityEvent = {
+      ...event,
+      id: `llmg-${++this.idCounter}`,
+      timestamp: new Date().toISOString(),
+      classifiedBy: 'llm-guard',
+    };
+    for (const h of this.handlers) {
+      h(full);
+    }
+    return full;
+  }
+
+  onEvent(handler: (event: SecurityEvent) => void | Promise<void>): void {
+    this.handlers.push(handler);
+  }
+}
+
+/** Simple enforcement engine — llm-guard doesn't have enforcement */
+class SimpleEnforcementEngine implements EnforcementEngine {
+  private pausedPids = new Set<number>();
+  private alertCallback?: (event: SecurityEvent, rule: AlertRule) => void;
+
+  async execute(action: EnforcementAction, event: SecurityEvent): Promise<EnforcementResult> {
+    return { action, success: true, reason: 'llm-guard-enforcement', event };
+  }
+
+  pause(pid: number): boolean {
+    this.pausedPids.add(pid);
+    return true;
+  }
+
+  resume(pid: number): boolean {
+    return this.pausedPids.delete(pid);
+  }
+
+  kill(pid: number): boolean {
+    this.pausedPids.delete(pid);
+    return true;
+  }
+
+  getPausedPids(): number[] {
+    return [...this.pausedPids];
+  }
+
+  setAlertCallback(callback: (event: SecurityEvent, rule: AlertRule) => void): void {
+    this.alertCallback = callback;
+  }
+}
+
+export class LLMGuardWrapper implements SecurityProductAdapter {
+  private _dataDir: string;
+  private engine: SimpleEventEngine;
+  private enforcement: SimpleEnforcementEngine;
+  private rules: AlertRule[];
+  readonly collector: EventCollector;
+
+  constructor(labConfig?: LabConfig) {
+    this._dataDir = labConfig?.dataDir ?? fs.mkdtempSync(path.join(os.tmpdir(), 'llmg-lab-'));
+    this.engine = new SimpleEventEngine();
+    this.enforcement = new SimpleEnforcementEngine();
+    this.rules = labConfig?.rules ?? [];
+    this.collector = new EventCollector();
+
+    this.engine.onEvent(async (event) => {
+      this.collector.eventHandler(event);
+
+      // Check rules for enforcement
+      for (const rule of this.rules) {
+        const cond = rule.condition;
+        if (cond.category && cond.category !== event.category) continue;
+        if (cond.source && cond.source !== event.source) continue;
+        if (cond.minSeverity) {
+          const sevOrder = ['info', 'low', 'medium', 'high', 'critical'];
+          if (sevOrder.indexOf(event.severity) < sevOrder.indexOf(cond.minSeverity)) continue;
+        }
+        const result = await this.enforcement.execute(rule.action, event);
+        result.reason = rule.name;
+        this.collector.enforcementHandler(result);
+      }
+    });
+  }
+
+  getCapabilities(): CapabilityMatrix {
+    return {
+      product: 'llm-guard',
+      version: '0.1.8',
+      capabilities: new Set([
+        'prompt-input-scanning',
+        'pattern-scanning',
+      ]),
+    };
+  }
+
+  async start(): Promise<void> {}
+
+  async stop(): Promise<void> {
+    this.collector.reset();
+    try {
+      fs.rmSync(this._dataDir, { recursive: true, force: true });
+    } catch {}
+  }
+
+  async injectEvent(event: Omit<SecurityEvent, 'id' | 'timestamp' | 'classifiedBy'>): Promise<SecurityEvent> {
+    return this.engine.emit(event);
+  }
+
+  waitForEvent(predicate: (event: SecurityEvent) => boolean, timeoutMs: number = 10000): Promise<SecurityEvent> {
+    return this.collector.waitForEvent(predicate, timeoutMs);
+  }
+
+  getEvents(): SecurityEvent[] { return this.collector.getEvents(); }
+  getEventsByCategory(category: string): SecurityEvent[] { return this.collector.eventsByCategory(category); }
+  getEnforcements(): EnforcementResult[] { return this.collector.getEnforcements() as EnforcementResult[]; }
+  getEnforcementsByAction(action: string): EnforcementResult[] { return this.collector.enforcementsByAction(action) as EnforcementResult[]; }
+  resetCollector(): void { this.collector.reset(); }
+
+  getEventEngine(): EventEngine { return this.engine; }
+  getEnforcementEngine(): EnforcementEngine { return this.enforcement; }
+
+  get dataDir(): string { return this._dataDir; }
+
+  // ─── Factory Methods ────────────────────────────────────────────
+
+  createPromptScanner(): PromptScanner {
+    const LLMGuard = getLLMGuard();
+    const guard = new LLMGuard({
+      promptInjection: { enabled: true },
+      jailbreak: { enabled: true },
+      pii: { enabled: true },
+    });
+
+    return {
+      start: async () => {},
+      stop: async () => {},
+      scanInput: (text: string) => {
+        // llm-guard is async, but OASB scanner interface is sync.
+        // We run synchronously by checking patterns manually.
+        // This is a limitation — real usage would be async.
+        const result = scanWithPatterns(text, 'input');
+        return result;
+      },
+      scanOutput: (text: string) => {
+        return scanWithPatterns(text, 'output');
+      },
+    };
+  }
+
+  createMCPScanner(_allowedTools?: string[]): MCPScanner {
+    // llm-guard has no MCP scanning capability
+    return {
+      start: async () => {},
+      stop: async () => {},
+      scanToolCall: () => ({ detected: false, matches: [] }),
+    };
+  }
+
+  createA2AScanner(_trustedAgents?: string[]): A2AScanner {
+    // llm-guard has no A2A scanning capability
+    return {
+      start: async () => {},
+      stop: async () => {},
+      scanMessage: () => ({ detected: false, matches: [] }),
+    };
+  }
+
+  createPatternScanner(): PatternScanner {
+    // llm-guard uses its own internal patterns, not the OASB ThreatPattern format.
+    // We expose what we can via regex approximation.
+    const patterns = getLLMGuardPatterns();
+    return {
+      scanText: (text: string, pats: readonly ThreatPattern[]) => scanWithPatterns(text, 'input'),
+      getAllPatterns: () => patterns,
+      getPatternSets: () => ({
+        inputPatterns: patterns.filter(p => p.category !== 'output-leak'),
+        outputPatterns: patterns.filter(p => p.category === 'output-leak'),
+        mcpPatterns: [],
+        a2aPatterns: [],
+      }),
+    };
+  }
+
+  createBudgetManager(dataDir: string, config?: { budgetUsd?: number; maxCallsPerHour?: number }): BudgetManager {
+    // llm-guard has no budget management — implement a simple one
+    let spent = 0;
+    let totalCalls = 0;
+    let callsThisHour = 0;
+    const budgetUsd = config?.budgetUsd ?? 5;
+    const maxCallsPerHour = config?.maxCallsPerHour ?? 20;
+
+    return {
+      canAfford: (cost: number) => spent + cost <= budgetUsd && callsThisHour < maxCallsPerHour,
+      record: (cost: number, _tokens: number) => { spent += cost; totalCalls++; callsThisHour++; },
+      getStatus: () => ({
+        spent,
+        budget: budgetUsd,
+        remaining: budgetUsd - spent,
+        percentUsed: Math.round((spent / budgetUsd) * 100),
+        callsThisHour,
+        maxCallsPerHour,
+        totalCalls,
+      }),
+      reset: () => { spent = 0; totalCalls = 0; callsThisHour = 0; },
+    };
+  }
+
+  createAnomalyScorer(): AnomalyScorer {
+    // llm-guard has no anomaly detection — implement a stub
+    const baselines = new Map<string, { mean: number; stddev: number; count: number }>();
+    const observations = new Map<string, number[]>();
+
+    return {
+      score: () => 0,
+      record: (event: SecurityEvent) => {
+        const key = event.source;
+        if (!observations.has(key)) observations.set(key, []);
+        observations.get(key)!.push(1);
+        const vals = observations.get(key)!;
+        const mean = vals.length;
+        baselines.set(key, { mean, stddev: 0, count: 1 });
+      },
+      getBaseline: (source: string) => baselines.get(source) ?? null,
+      reset: () => { baselines.clear(); observations.clear(); },
+    };
+  }
+}
+
+// ─── Internal pattern-based scanning (sync approximation of llm-guard) ───
+
+function getLLMGuardPatterns(): ThreatPattern[] {
+  return [
+    { id: 'LLMG-PI-001', category: 'prompt-injection', description: 'Instruction override', pattern: /(?:ignore|disregard|forget)\s+(?:all\s+)?(?:previous\s+)?(?:instructions?|prompts?|rules?)/i, severity: 'high' },
+    { id: 'LLMG-PI-002', category: 'prompt-injection', description: 'System prompt extraction', pattern: /(?:system\s+prompt|repeat\s+(?:your|the)\s+(?:instructions?|prompt))/i, severity: 'high' },
+    { id: 'LLMG-PI-003', category: 'prompt-injection', description: 'Persona override', pattern: /(?:you\s+are\s+now|pretend\s+you\s+are|act\s+as\s+if)/i, severity: 'medium' },
+    { id: 'LLMG-JB-001', category: 'jailbreak', description: 'DAN jailbreak', pattern: /(?:DAN|do\s+anything\s+now)/i, severity: 'high' },
+    { id: 'LLMG-JB-002', category: 'jailbreak', description: 'Roleplay bypass', pattern: /(?:pretend|imagine|roleplay)\s+(?:you\s+are|as)\s+(?:an?\s+)?(?:evil|unrestricted|unfiltered)/i, severity: 'high' },
+    { id: 'LLMG-PII-001', category: 'data-exfiltration', description: 'SSN detection', pattern: /\b\d{3}-\d{2}-\d{4}\b/, severity: 'high' },
+    { id: 'LLMG-PII-002', category: 'data-exfiltration', description: 'Credit card detection', pattern: /\b(?:\d{4}[- ]?){3}\d{4}\b/, severity: 'high' },
+    { id: 'LLMG-PII-003', category: 'data-exfiltration', description: 'API key detection', pattern: /(?:sk-[a-zA-Z0-9]{20,}|AKIA[A-Z0-9]{12,})/i, severity: 'critical' },
+  ];
+}
+
+function scanWithPatterns(text: string, _direction: 'input' | 'output'): ScanResult {
+  const patterns = getLLMGuardPatterns();
+  const matches: ScanResult['matches'] = [];
+
+  for (const pattern of patterns) {
+    const match = pattern.pattern.exec(text);
+    if (match) {
+      matches.push({
+        pattern,
+        matchedText: match[0].slice(0, 200),
+      });
+    }
+  }
+
+  return {
+    detected: matches.length > 0,
+    matches,
+  };
+}

package/src/harness/mock-llm-adapter.ts

@@ -1,4 +1,4 @@
-import type { LLMAdapter, LLMResponse } from '@opena2a/arp';
+import type { LLMAdapter, LLMResponse } from './adapter';
 
 interface MockCall {
   prompt: string;
@@ -21,8 +21,8 @@ export class MockLLMAdapter implements LLMAdapter {
     this.costPerCall = options?.costPerCall ?? 0.001;
   }
 
-  async assess(prompt: string, maxTokens: number): Promise<LLMResponse> {
-    this.calls.push({ prompt, maxTokens, timestamp: Date.now() });
+  async assess(prompt: string): Promise<LLMResponse> {
+    this.calls.push({ prompt, maxTokens: 300, timestamp: Date.now() });
 
     if (this.latencyMs > 0) {
       await new Promise((r) => setTimeout(r, this.latencyMs));
@@ -32,9 +32,10 @@ export class MockLLMAdapter implements LLMAdapter {
 
     return {
       content: response,
-      inputTokens: Math.ceil(prompt.length / 4),
-      outputTokens: Math.ceil(response.length / 4),
-      model: 'mock-llm',
+      usage: {
+        inputTokens: Math.ceil(prompt.length / 4),
+        outputTokens: Math.ceil(response.length / 4),
+      },
     };
   }