@oculum/scanner 1.0.11 → 1.0.13

package/src/__tests__/benchmark/fixtures/layer2/phase5-excessive-agency.ts

@@ -0,0 +1,580 @@
+/**
+ * Phase 5: Excessive Agency Detection Test Fixtures
+ * Tests for detecting AI-specific agency vulnerabilities
+ *
+ * Tasks covered:
+ * - Task 1: LLM Output in Tool Names/Paths
+ * - Task 2: Tool Permission Accumulation
+ * - Task 3: Database Writes Without User Scoping
+ * - Task 4: Recursive Agent Patterns
+ * - Task 5: MCP Schema Validation
+ * - Task 6: Human-in-the-Loop for Destructive Operations
+ */
+
+import type { TestGroup } from '../../types'
+
+export const phase5ExcessiveAgencyTests: TestGroup = {
+  name: 'Phase 5: Excessive Agency Detection',
+  tier: 'A',
+  layer: 2,
+  description: 'Detection of AI-specific agency vulnerabilities (Phase 5)',
+
+  truePositives: [
+    {
+      name: 'Task 1: LLM Output in Tool Names/Paths',
+      expectFindings: true,
+      expectedCategories: ['ai_excessive_agency'],
+      description: 'LLM-generated content used in dangerous operations',
+      file: {
+        path: 'src/agents/llm-output-flow.ts',
+        content: `
+import { OpenAI } from 'openai'
+import * as fs from 'fs'
+import { exec } from 'child_process'
+
+// DANGEROUS: LLM output used as tool name
+async function callToolFromLLM(tools: Record<string, Function>, response: OpenAI.ChatCompletion) {
+  const toolName = response.content
+  // Attacker could make the model suggest calling arbitrary tools
+  await tools[toolName]()
+}
+
+// DANGEROUS: LLM output in file path
+async function writeToLLMPath(response: { path: string, data: string }) {
+  // Path traversal attack via prompt injection
+  await fs.writeFile(response.path, response.data)
+}
+
+// DANGEROUS: LLM output in shell command
+async function runLLMCommand(completion: { command: string }) {
+  // Remote code execution via prompt injection
+  exec(completion.command)
+}
+
+// DANGEROUS: LLM output in URL
+async function fetchLLMUrl(result: { url: string }) {
+  // SSRF via prompt injection
+  await fetch(result.url)
+}
+
+// DANGEROUS: Tool name destructured from LLM response
+async function dynamicToolCall(response: any) {
+  const { toolName, args } = response
+  // Tool selection controlled by LLM
+  return executeTool(toolName, args)
+}
+
+// DANGEROUS: Dynamic property access with LLM output
+function dispatchTool(handlers: Record<string, Function>, llm: { tool: string }) {
+  // Arbitrary property access
+  handlers[llm.tool]()
+}
+`,
+        language: 'typescript',
+        size: 1100,
+      },
+    },
+    {
+      name: 'Task 2: Tool Permission Accumulation',
+      expectFindings: true,
+      expectedCategories: ['ai_excessive_agency'],
+      description: 'Unbounded tool registration and permission growth',
+      file: {
+        path: 'src/agents/tool-accumulation.ts',
+        content: `
+import { Agent } from './agent'
+
+// DANGEROUS: Tool registration from user input without bounds
+async function registerUserTool(agent: Agent, request: { tool: any }) {
+  // User can register unlimited tools
+  agent.registerTool(request.tool)
+}
+
+// DANGEROUS: Tools pushed without limit check
+class ToolRegistry {
+  private tools: any[] = []
+
+  addTool(tool: any) {
+    // No limit on number of tools
+    this.tools.push(tool)
+  }
+}
+
+// DANGEROUS: Dynamic tool loading from user config
+async function loadPlugins(config: { tool: string }) {
+  // Arbitrary module loading
+  const module = await import(config.tool)
+  return module
+}
+
+// DANGEROUS: Permission grant without authorization
+class PermissionManager {
+  grantPermission(userId: string, permission: string) {
+    // No auth check - anyone can grant permissions
+    this.permissions.set(userId, permission)
+  }
+
+  addCapability(agentId: string, capability: string) {
+    // No authorization check before adding
+    this.capabilities.get(agentId).add(capability)
+  }
+}
+
+// DANGEROUS: Inheriting all tools without filtering
+function createChildAgent(parent: Agent) {
+  const child = new Agent()
+  // Blindly inherits all parent capabilities
+  inheritTools(parent.tools)
+  return child
+}
+`,
+        language: 'typescript',
+        size: 1000,
+      },
+    },
+    {
+      name: 'Task 3: Database Writes Without User Scoping',
+      expectFindings: true,
+      expectedCategories: ['ai_excessive_agency'],
+      description: 'AI-generated content stored without user attribution',
+      file: {
+        path: 'src/agents/db-writes.ts',
+        content: `
+import { db } from '../database'
+
+// DANGEROUS: Insert without userId
+async function saveAIResponse(aiContent: string) {
+  // No user attribution - content orphaned
+  await db.insert({ content: aiContent, timestamp: Date.now() })
+}
+
+// DANGEROUS: AI content inserted without user scoping
+async function storeCompletion(response: { content: string }) {
+  // AI output stored without user context
+  await prisma.create({ data: response.content })
+}
+
+// DANGEROUS: Bulk insert without tenant filter
+async function batchInsert(records: any[]) {
+  // Multi-tenant data isolation at risk
+  await db.insertMany(records)
+}
+
+// DANGEROUS: Update by ID without ownership check
+async function updateRecord(id: string, data: any) {
+  // Could update other users' data
+  await supabase.update({ id, data })
+}
+
+// DANGEROUS: Delete by ID without user scoping
+async function deleteItem(id: string) {
+  // Could delete other users' data
+  await mongoose.delete({ id })
+}
+
+// SAFE EXAMPLE for comparison
+async function safeInsert(content: string, ctx: { user: { id: string } }) {
+  await db.insert({ content, userId: ctx.user.id, createdAt: Date.now() })
+}
+`,
+        language: 'typescript',
+        size: 900,
+      },
+    },
+    {
+      name: 'Task 4: Recursive Agent Patterns',
+      expectFindings: true,
+      expectedCategories: ['ai_excessive_agency'],
+      description: 'Unbounded agent recursion and self-spawning',
+      file: {
+        path: 'src/agents/recursive-agents.ts',
+        content: `
+// DANGEROUS: Recursive agent call without depth limit
+async function runAgent(task: string) {
+  const result = await executeTask(task)
+  if (result.needsSubtask) {
+    // No depth parameter - could recurse forever
+    await runAgent(result.subtask)
+  }
+  return result
+}
+
+// DANGEROUS: Sub-agent spawning without limit
+async function processTask(task: Task) {
+  for (const subtask of task.children) {
+    // No count/depth tracking
+    spawnAgent(subtask)
+  }
+}
+
+// DANGEROUS: Recursive task processing from agent output
+async function handleAgentOutput(result: { subtasks: Task[] }) {
+  // Agent can generate unlimited subtasks
+  result.subtasks.forEach(task => processTask(task))
+}
+
+// DANGEROUS: Self-improvement loop without termination
+async function improveAgent(agent: Agent) {
+  while (agent.needsImprovement) {
+    // No iteration limit - could run forever
+    await agent.improve()
+  }
+}
+
+// DANGEROUS: CrewAI delegation without depth limit
+const delegatingAgent = {
+  async execute(task: string) {
+    const result = await this.run(task)
+    if (result.needsHelp) {
+      // No max_delegation_depth
+      this.delegate(result.subTask)
+    }
+  }
+}
+
+// DANGEROUS: LangGraph self-loop
+const graph = {
+  nodes: ['process'],
+  edges: []
+}
+graph.add_edge('process', 'process') // Self-referential edge
+`,
+        language: 'typescript',
+        size: 1100,
+      },
+    },
+    {
+      name: 'Task 5: MCP Schema Validation Bypass',
+      expectFindings: true,
+      expectedCategories: ['ai_mcp_tool_poisoning'],
+      description: 'MCP tools using arguments without validation',
+      file: {
+        path: 'src/mcp/unvalidated-tools.ts',
+        content: `
+import { McpServer } from '@modelcontextprotocol/sdk'
+
+const server = new McpServer({ name: 'unvalidated-server' })
+
+// DANGEROUS: Args used directly without schema validation
+server.tool('execute', async (args) => {
+  // No validation - args.command could be anything
+  return runCommand(args.command)
+})
+
+// DANGEROUS: Dangerous property used without validation
+server.tool('query', async (params) => {
+  // params.sql used directly
+  return db.execute(params.sql)
+})
+
+// DANGEROUS: Args spread into function call
+server.tool('process', async (args) => {
+  // All fields pass through unvalidated
+  return processData(...args)
+})
+
+// DANGEROUS: Dynamic property access on args
+server.tool('lookup', async (args) => {
+  const key = args[fieldName]  // Arbitrary property access
+  return storage.get(key)
+})
+`,
+        language: 'typescript',
+        size: 700,
+      },
+    },
+    {
+      name: 'Task 6: Destructive Operations Without Confirmation',
+      expectFindings: true,
+      expectedCategories: ['ai_excessive_agency', 'ai_mcp_tool_poisoning'],
+      description: 'MCP tools performing destructive operations without human approval',
+      file: {
+        path: 'src/mcp/destructive-tools.ts',
+        content: `
+import { McpServer } from '@modelcontextprotocol/sdk'
+import * as fs from 'fs'
+
+const server = new McpServer({ name: 'destructive-server' })
+
+// DANGEROUS: File deletion without confirmation
+server.tool('delete_file', async (args) => {
+  // No confirmation mechanism
+  await fs.rm(args.path)
+  return { deleted: true }
+})
+
+// DANGEROUS: Database deletion without confirmation
+server.tool('delete_records', async (args) => {
+  // No human approval required
+  await db.delete(args.id)
+  return { success: true }
+})
+
+// DANGEROUS: Recursive deletion without confirmation
+server.tool('cleanup', async (args) => {
+  // Extremely dangerous - no confirmation
+  await fs.rm(args.path, { recursive: true })
+  return { cleaned: true }
+})
+
+// DANGEROUS: Shell execution without confirmation
+server.tool('run_command', async (args) => {
+  // No approval for shell commands
+  exec(args.command)
+  return { executed: true }
+})
+
+// DANGEROUS: Send operation without confirmation
+server.tool('send_email', async (args) => {
+  // No draft/preview/confirmation
+  await emailService.send(args.to, args.subject, args.body)
+  return { sent: true }
+})
+
+// DANGEROUS: Payment without confirmation
+server.tool('charge_card', async (args) => {
+  // Financial operation without human approval
+  await payments.charge(args.amount, args.cardId)
+  return { charged: true }
+})
+
+// DANGEROUS: Credential deletion without confirmation
+server.tool('revoke_api_key', async (args) => {
+  // Could cause service disruption
+  await apiKeys.delete(args.keyId)
+  return { revoked: true }
+})
+
+// SAFE EXAMPLE for comparison
+server.tool('safe_delete', async (args) => {
+  if (!args.confirmed) {
+    return { needsConfirmation: true, action: 'delete', path: args.path }
+  }
+  await fs.rm(args.path)
+  return { deleted: true }
+})
+`,
+        language: 'typescript',
+        size: 1400,
+      },
+    },
+  ],
+
+  falseNegatives: [
+    {
+      name: 'Phase 5: Safe LLM Tool Patterns',
+      expectFindings: false,
+      description: 'Safe patterns with proper validation that should NOT be flagged',
+      allowedInfoFindings: [
+        {
+          category: 'ai_excessive_agency',
+          maxCount: 3,
+          reason: 'Some validation patterns may still flag at info level',
+        },
+      ],
+      file: {
+        path: 'src/agents/safe-llm-tools.ts',
+        content: `
+const ALLOWED_TOOLS = ['search', 'calculate', 'lookup'] as const
+
+// SAFE: Tool name validated against allowlist
+async function safeToolCall(response: { toolName: string }) {
+  if (!ALLOWED_TOOLS.includes(response.toolName)) {
+    throw new Error('Invalid tool')
+  }
+  return executeTool(response.toolName)
+}
+
+// SAFE: Path validated against allowed directory
+async function safeFilePath(response: { path: string }) {
+  const resolved = path.resolve(response.path)
+  if (!resolved.startsWith(ALLOWED_BASE_DIR)) {
+    throw new Error('Invalid path')
+  }
+  return fs.readFile(resolved)
+}
+
+// SAFE: Command validated against allowlist
+const SAFE_COMMANDS = ['ls', 'pwd', 'whoami']
+async function safeCommand(response: { command: string }) {
+  if (!SAFE_COMMANDS.includes(response.command)) {
+    throw new Error('Command not allowed')
+  }
+  return exec(response.command)
+}
+`,
+        language: 'typescript',
+        size: 700,
+      },
+    },
+    {
+      name: 'Phase 5: Safe Database Operations',
+      expectFindings: false,
+      description: 'Database operations with proper user scoping',
+      file: {
+        path: 'src/agents/safe-db-ops.ts',
+        content: `
+// SAFE: Insert with userId
+async function safeInsert(content: string, ctx: { user: { id: string } }) {
+  await db.insert({
+    content,
+    userId: ctx.user.id,
+    createdBy: ctx.user.id,
+    timestamp: Date.now()
+  })
+}
+
+// SAFE: Update with ownership check
+async function safeUpdate(id: string, data: any, ctx: { user: { id: string } }) {
+  await db.update({
+    where: { id, userId: ctx.user.id },  // Scoped to user
+    data
+  })
+}
+
+// SAFE: Delete with user scoping
+async function safeDelete(id: string, ctx: { user: { id: string } }) {
+  await db.delete({
+    where: { id, ownerId: ctx.user.id }  // Owner check
+  })
+}
+
+// SAFE: Bulk insert with tenant scoping
+async function safeBulkInsert(records: any[], ctx: { tenant: { id: string } }) {
+  const scoped = records.map(r => ({ ...r, tenantId: ctx.tenant.id }))
+  await db.insertMany(scoped)
+}
+`,
+        language: 'typescript',
+        size: 800,
+      },
+    },
+    {
+      name: 'Phase 5: Safe Recursive Patterns',
+      expectFindings: false,
+      description: 'Agent patterns with proper depth limits',
+      file: {
+        path: 'src/agents/safe-recursion.ts',
+        content: `
+const MAX_DEPTH = 5
+const MAX_AGENTS = 10
+
+// SAFE: Recursive agent with depth limit
+async function safeRunAgent(task: string, depth = 0) {
+  if (depth > MAX_DEPTH) {
+    throw new Error('Max depth exceeded')
+  }
+  const result = await executeTask(task)
+  if (result.needsSubtask) {
+    await safeRunAgent(result.subtask, depth + 1)
+  }
+  return result
+}
+
+// SAFE: Agent spawning with count limit
+let agentCount = 0
+async function safeSpawnAgent(task: Task) {
+  if (agentCount >= MAX_AGENTS) {
+    throw new Error('Agent limit reached')
+  }
+  agentCount++
+  try {
+    await processTask(task)
+  } finally {
+    agentCount--
+  }
+}
+
+// SAFE: Subtask processing with limit
+async function safeSubtasks(result: { subtasks: Task[] }) {
+  const maxSubtasks = 5
+  const limited = result.subtasks.slice(0, maxSubtasks)
+  for (const task of limited) {
+    await processTask(task)
+  }
+}
+
+// SAFE: Self-improvement with iteration limit
+async function safeImprove(agent: Agent) {
+  const maxIterations = 10
+  let iterations = 0
+  while (agent.needsImprovement && iterations < maxIterations) {
+    await agent.improve()
+    iterations++
+  }
+}
+`,
+        language: 'typescript',
+        size: 1000,
+      },
+    },
+    {
+      name: 'Phase 5: Safe MCP Tool Patterns',
+      expectFindings: false,
+      description: 'MCP tools with proper validation and confirmation',
+      allowedInfoFindings: [
+        {
+          category: 'ai_mcp_tool_poisoning',
+          maxCount: 2,
+          reason: 'Schema patterns may flag at info level',
+        },
+        {
+          category: 'ai_excessive_agency',
+          maxCount: 2,
+          reason: 'HITL patterns may flag at info level',
+        },
+      ],
+      file: {
+        path: 'src/mcp/safe-tools.ts',
+        content: `
+import { McpServer } from '@modelcontextprotocol/sdk'
+import { z } from 'zod'
+
+const server = new McpServer({ name: 'safe-server' })
+
+// SAFE: Schema validated input
+const commandSchema = z.object({
+  command: z.enum(['ls', 'pwd', 'whoami'])
+})
+
+server.tool('safe_execute', async (args) => {
+  const validated = commandSchema.parse(args)
+  return runCommand(validated.command)
+})
+
+// SAFE: Delete with confirmation
+server.tool('safe_delete', async (args) => {
+  if (!args.confirmed) {
+    return { needsConfirmation: true, path: args.path }
+  }
+  await fs.rm(args.path)
+  return { deleted: true }
+})
+
+// SAFE: Payment with human approval
+server.tool('safe_charge', async (args) => {
+  if (!args.approved) {
+    return { needsConfirmation: true, amount: args.amount }
+  }
+  await payments.charge(args.amount)
+  return { charged: true }
+})
+
+// SAFE: Send with draft/preview
+server.tool('safe_send', async (args) => {
+  if (args.draft) {
+    return { preview: formatEmail(args) }
+  }
+  if (!args.confirmed) {
+    return { needsConfirmation: true, preview: formatEmail(args) }
+  }
+  await sendEmail(args)
+  return { sent: true }
+})
+`,
+        language: 'typescript',
+        size: 900,
+      },
+    },
+  ],
+}