@oculum/scanner 1.0.0

package/src/__tests__/benchmark/fixtures/layer2/ai-schema-validation.ts

@@ -0,0 +1,434 @@
+/**
+ * AI Schema Validation Test Fixtures
+ * Tests for detecting missing/weak validation of AI-generated outputs
+ */
+
+import type { TestGroup } from '../../types'
+
+export const aiSchemaValidationTests: TestGroup = {
+  name: 'AI Schema Validation',
+  tier: 'B',
+  layer: 2,
+  description: 'Detection of missing or weak schema validation on AI-generated outputs',
+
+  truePositives: [
+    {
+      name: 'Schema Validation - True Positives',
+      expectFindings: true,
+      expectedCategories: ['ai_schema_mismatch'],
+      description: 'Unvalidated AI output parsing that MUST be detected',
+      file: {
+        path: 'src/ai/chat-handler.ts',
+        content: `
+import OpenAI from 'openai'
+
+const openai = new OpenAI()
+
+// Unvalidated JSON.parse of AI response - MEDIUM
+export async function getStructuredResponse(prompt: string) {
+  const completion = await openai.chat.completions.create({
+    model: 'gpt-4',
+    messages: [{ role: 'user', content: prompt }],
+  })
+
+  // UNSAFE: No schema validation
+  const data = JSON.parse(completion.choices[0].message.content)
+  return data
+}
+
+// Tool call arguments parsed without validation - MEDIUM
+export async function handleToolCall(toolCall: any) {
+  const args = JSON.parse(toolCall.function.arguments)
+  // Using args directly without validation
+  return processArgs(args)
+}
+
+// Anthropic response parsed directly - MEDIUM
+export async function handleAnthropicResponse(response: any) {
+  const parsed = JSON.parse(response.content[0].text)
+  return parsed.action
+}
+
+// AI response typed as any - LOW
+export async function processAIOutput() {
+  const response: any = await getAICompletion()
+  // Using untyped response
+  return response.result
+}
+`,
+        language: 'typescript',
+        size: 900,
+      },
+    },
+    {
+      name: 'Weak Schema Patterns',
+      expectFindings: true,
+      expectedCategories: ['ai_schema_mismatch'],
+      description: 'Permissive schema patterns that provide inadequate validation',
+      file: {
+        path: 'src/ai/weak-schemas.ts',
+        content: `
+import { z } from 'zod'
+import OpenAI from 'openai'
+
+const openai = new OpenAI()
+
+// Using z.any() defeats schema validation - LOW
+const responseSchema = z.any()
+const aiResponse = responseSchema.parse(completion.choices[0].message.content)
+
+// Passthrough allowing extra properties - INFO
+const dataSchema = z.object({
+  action: z.string(),
+}).passthrough()
+
+// Record<string, any> for AI data - LOW
+interface AIData {
+  aiOutput: Record<string, any>
+}
+
+// Generic object type - INFO
+function processResponse(completion: object) {
+  return completion
+}
+
+// Any typed response at API boundary - MEDIUM (elevated)
+export async function handler(req: Request) {
+  const response: any = await openai.chat.completions.create({
+    model: 'gpt-4',
+    messages: [{ role: 'user', content: 'test' }],
+  })
+  return Response.json(response)
+}
+`,
+        language: 'typescript',
+        size: 700,
+      },
+    },
+    {
+      name: 'Tool Parameters in Security Sinks',
+      expectFindings: true,
+      expectedCategories: ['ai_schema_mismatch'],
+      description: 'AI-generated tool parameters used in dangerous operations',
+      file: {
+        path: 'src/ai/tool-execution.ts',
+        content: `
+import { exec } from 'child_process'
+import * as fs from 'fs'
+
+// Tool parameter in shell command - CRITICAL
+async function executeToolCommand(toolArgs: any) {
+  const command = toolArgs.command
+  exec(command, (error, stdout) => {
+    console.log(stdout)
+  })
+}
+
+// Tool parameter in file path - HIGH
+async function readToolFile(args: { path: string }) {
+  const filePath = args.path
+  const content = fs.readFileSync(filePath, 'utf-8')
+  return content
+}
+
+// Tool parameter in database query - HIGH
+async function queryFromTool(parameters: any, db: any) {
+  const tableName = parameters.table
+  const results = await db.query(\`SELECT * FROM \${tableName}\`)
+  return results
+}
+
+// Tool parameter in URL - HIGH
+async function fetchToolUrl(args: any) {
+  const endpoint = args.url
+  const response = await fetch(endpoint)
+  return response.json()
+}
+
+// Function name routing without validation - MEDIUM
+async function routeToolCall(tool_call: any) {
+  const name = tool_call.function.name
+  switch (name) {
+    case 'search': return search()
+    default: return functions[name]() // Dynamic access
+  }
+}
+`,
+        language: 'typescript',
+        size: 950,
+      },
+    },
+  ],
+
+  falseNegatives: [
+    {
+      name: 'Schema Validation - False Negatives',
+      expectFindings: false,
+      description: 'Properly validated AI outputs that should NOT be flagged',
+      allowedInfoFindings: [
+        {
+          category: 'ai_schema_mismatch',
+          maxCount: 2,
+          reason: 'Some patterns may generate info-level findings even when properly handled',
+        },
+        {
+          category: 'dangerous_function',
+          maxCount: 2,
+          reason: 'JSON.parse with schema validation nearby may still trigger info-level findings',
+        },
+      ],
+      file: {
+        path: 'src/ai/validated-handler.ts',
+        content: `
+import OpenAI from 'openai'
+import { z } from 'zod'
+
+const openai = new OpenAI()
+
+// Proper schema definition
+const ResponseSchema = z.object({
+  action: z.enum(['search', 'create', 'delete']),
+  target: z.string().min(1),
+  params: z.record(z.string()).optional(),
+}).strict()
+
+// Validated AI response parsing - SAFE
+export async function getValidatedResponse(prompt: string) {
+  const completion = await openai.chat.completions.create({
+    model: 'gpt-4',
+    messages: [{ role: 'user', content: prompt }],
+  })
+
+  // Schema validation with zod
+  const validated = ResponseSchema.parse(
+    JSON.parse(completion.choices[0].message.content!)
+  )
+
+  return validated
+}
+
+// safeParse with error handling - SAFE
+export async function getSafeResponse(prompt: string) {
+  const raw = await getAICompletion(prompt)
+  const result = ResponseSchema.safeParse(JSON.parse(raw))
+
+  if (!result.success) {
+    throw new Error('Invalid AI response: ' + result.error.message)
+  }
+
+  return result.data
+}
+
+// Tool arguments with schema validation - SAFE
+const ToolArgsSchema = z.object({
+  query: z.string(),
+  limit: z.number().int().positive().max(100),
+})
+
+export async function handleToolWithValidation(toolCall: any) {
+  const args = ToolArgsSchema.parse(
+    JSON.parse(toolCall.function.arguments)
+  )
+  return search(args.query, args.limit)
+}
+
+// OpenAI Structured Outputs - SAFE
+export async function getStructuredOutput(prompt: string) {
+  const completion = await openai.chat.completions.create({
+    model: 'gpt-4',
+    messages: [{ role: 'user', content: prompt }],
+    response_format: {
+      type: 'json_schema',
+      json_schema: {
+        name: 'response',
+        schema: {
+          type: 'object',
+          properties: {
+            action: { type: 'string' },
+            target: { type: 'string' },
+          },
+          required: ['action', 'target'],
+        },
+      },
+    },
+  })
+
+  return JSON.parse(completion.choices[0].message.content!)
+}
+`,
+        language: 'typescript',
+        size: 1700,
+      },
+    },
+    {
+      name: 'Tool Parameters with Allowlist',
+      expectFindings: false,
+      description: 'Tool parameters validated against allowlists',
+      allowedInfoFindings: [
+        {
+          category: 'ai_schema_mismatch',
+          maxCount: 1,
+          reason: 'May flag routing pattern at info level',
+        },
+        {
+          category: 'dangerous_function',
+          maxCount: 2,
+          reason: 'fs operations may trigger info-level findings',
+        },
+        {
+          category: 'ai_unsafe_execution',
+          maxCount: 2,
+          reason: 'File operations with proper validation may still flag at low severity',
+        },
+      ],
+      file: {
+        path: 'src/ai/safe-tool-execution.ts',
+        content: `
+import * as path from 'path'
+import * as fs from 'fs'
+import { z } from 'zod'
+
+// Allowlisted commands - SAFE
+const ALLOWED_COMMANDS = ['ls', 'cat', 'echo'] as const
+const CommandSchema = z.enum(ALLOWED_COMMANDS)
+
+async function executeAllowlistedCommand(args: { cmd: string }) {
+  const validated = CommandSchema.parse(args.cmd)
+  // Only allowed commands can execute
+  return runCommand(validated)
+}
+
+// Path validation with allowlist - SAFE
+const ALLOWED_PATHS = ['/app/data', '/app/uploads']
+
+async function readAllowlistedFile(args: { filePath: string }) {
+  const resolved = path.resolve(args.filePath)
+
+  // Validate path against allowlist
+  if (!ALLOWED_PATHS.some(p => resolved.startsWith(p))) {
+    throw new Error('Path not allowed')
+  }
+
+  return fs.readFileSync(resolved, 'utf-8')
+}
+
+// URL validation against allowed hosts - SAFE
+const ALLOWED_HOSTS = ['api.example.com', 'data.example.com']
+
+async function fetchAllowlistedUrl(args: { url: string }) {
+  const parsed = new URL(args.url)
+
+  if (!ALLOWED_HOSTS.includes(parsed.host)) {
+    throw new Error('Host not allowed')
+  }
+
+  return fetch(args.url)
+}
+
+// Function routing with explicit allowlist - SAFE
+const ALLOWED_FUNCTIONS = ['search', 'calculate', 'format'] as const
+type AllowedFunction = typeof ALLOWED_FUNCTIONS[number]
+
+const handlers: Record<AllowedFunction, () => void> = {
+  search: () => {},
+  calculate: () => {},
+  format: () => {},
+}
+
+async function routeWithAllowlist(toolCall: { name: string }) {
+  const name = toolCall.name as AllowedFunction
+
+  if (!ALLOWED_FUNCTIONS.includes(name)) {
+    throw new Error('Function not allowed: ' + name)
+  }
+
+  return handlers[name]()
+}
+`,
+        language: 'typescript',
+        size: 1400,
+      },
+    },
+    {
+      name: 'Strongly Typed AI Interfaces',
+      expectFindings: false,
+      description: 'Properly typed AI responses with strict interfaces',
+      allowedInfoFindings: [
+        {
+          category: 'ai_schema_mismatch',
+          maxCount: 1,
+          reason: 'Type-only patterns may still flag for runtime validation',
+        },
+        {
+          category: 'dangerous_function',
+          maxCount: 2,
+          reason: 'JSON.parse with schema validation nearby may still trigger info-level findings',
+        },
+      ],
+      file: {
+        path: 'src/ai/typed-responses.ts',
+        content: `
+import OpenAI from 'openai'
+import { z } from 'zod'
+
+// Strict interface definition
+interface AISearchResult {
+  query: string
+  results: Array<{
+    id: string
+    title: string
+    score: number
+  }>
+  metadata: {
+    processingTime: number
+    model: string
+  }
+}
+
+// Runtime validation with zod matching interface
+const AISearchResultSchema = z.object({
+  query: z.string(),
+  results: z.array(z.object({
+    id: z.string(),
+    title: z.string(),
+    score: z.number().min(0).max(1),
+  })),
+  metadata: z.object({
+    processingTime: z.number(),
+    model: z.string(),
+  }),
+}).strict()
+
+// Type-safe handler - SAFE
+export async function searchWithAI(
+  query: string,
+  openai: OpenAI
+): Promise<AISearchResult> {
+  const completion = await openai.chat.completions.create({
+    model: 'gpt-4',
+    messages: [{ role: 'user', content: query }],
+  })
+
+  // Runtime validation ensures type safety
+  const validated = AISearchResultSchema.parse(
+    JSON.parse(completion.choices[0].message.content!)
+  )
+
+  return validated
+}
+
+// Generic with constraint - SAFE
+export async function getTypedResponse<T>(
+  schema: z.ZodSchema<T>,
+  prompt: string
+): Promise<T> {
+  const raw = await getCompletion(prompt)
+  return schema.parse(JSON.parse(raw))
+}
+`,
+        language: 'typescript',
+        size: 1200,
+      },
+    },
+  ],
+}

package/src/__tests__/benchmark/fixtures/layer2/auth-antipatterns.ts

@@ -0,0 +1,159 @@
+/**
+ * Auth Anti-Patterns Test Fixtures
+ * Tests for detecting missing or weak authentication patterns
+ */
+
+import type { TestGroup } from '../../types'
+
+export const authAntipatternsTests: TestGroup = {
+  name: 'Auth Anti-Patterns',
+  tier: 'B',
+  layer: 2,
+  description: 'Detection of missing or weak authentication patterns in API routes',
+  
+  truePositives: [
+    {
+      name: 'Auth Anti-Patterns - True Positives',
+      expectFindings: true,
+      expectedCategories: ['missing_auth'],
+      description: 'Missing auth patterns that MUST be detected',
+      file: {
+        path: 'src/api/admin/route.ts',
+        content: `
+import { NextRequest, NextResponse } from 'next/server'
+import { db } from '@/lib/db'
+
+// No authentication on sensitive endpoint - HIGH
+export async function GET(request: NextRequest) {
+  // No auth check!
+  const users = await db.user.findMany({
+    include: { apiKeys: true }  // Exposing sensitive data
+  })
+  return NextResponse.json(users)
+}
+
+// No authentication on data mutation - CRITICAL
+export async function POST(request: NextRequest) {
+  // No auth check!
+  const body = await request.json()
+  await db.user.update({
+    where: { id: body.userId },
+    data: { role: 'admin' }  // Anyone can make themselves admin!
+  })
+  return NextResponse.json({ success: true })
+}
+
+// No authentication on delete - CRITICAL
+export async function DELETE(request: NextRequest) {
+  const { searchParams } = new URL(request.url)
+  const userId = searchParams.get('userId')
+  // No auth check!
+  await db.user.delete({ where: { id: userId } })
+  return NextResponse.json({ success: true })
+}
+`,
+        language: 'typescript',
+        size: 900,
+      },
+    },
+    {
+      name: 'Auth Anti-Patterns - IDOR Vulnerabilities',
+      expectFindings: true,
+      expectedCategories: ['missing_auth'],
+      description: 'Insecure Direct Object Reference patterns',
+      file: {
+        path: 'src/api/users/[id]/route.ts',
+        content: `
+import { NextRequest, NextResponse } from 'next/server'
+import { db } from '@/lib/db'
+
+// IDOR - accessing any user's data without ownership check
+export async function GET(
+  request: NextRequest,
+  { params }: { params: { id: string } }
+) {
+  // No check that current user owns this resource!
+  const user = await db.user.findUnique({
+    where: { id: params.id },
+    include: { 
+      apiKeys: true,
+      sessions: true,
+      billingInfo: true
+    }
+  })
+  return NextResponse.json(user)
+}
+
+// IDOR - updating any user's data
+export async function PUT(
+  request: NextRequest,
+  { params }: { params: { id: string } }
+) {
+  const body = await request.json()
+  // No ownership verification!
+  await db.user.update({
+    where: { id: params.id },
+    data: body
+  })
+  return NextResponse.json({ success: true })
+}
+`,
+        language: 'typescript',
+        size: 800,
+      },
+    },
+  ],
+  
+  falseNegatives: [
+    {
+      name: 'Auth Anti-Patterns - False Negatives',
+      expectFindings: false,
+      description: 'Properly authenticated patterns that should NOT be flagged',
+      file: {
+        path: 'src/api/protected/route.ts',
+        content: `
+import { NextRequest, NextResponse } from 'next/server'
+import { auth } from '@/lib/auth'
+import { getCurrentUserId } from '@/lib/auth-helpers'
+import { db } from '@/lib/db'
+
+// Authenticated with session check - SAFE
+export async function GET(request: NextRequest) {
+  const session = await auth()
+  if (!session?.user) {
+    return NextResponse.json({ error: 'Unauthorized' }, { status: 401 })
+  }
+
+  const userData = await db.user.findUnique({
+    where: { id: session.user.id }
+  })
+  return NextResponse.json(userData)
+}
+
+// Using throwing auth helper - SAFE
+export async function POST(request: NextRequest) {
+  // This throws if not authenticated
+  const userId = await getCurrentUserId()
+
+  // userId is guaranteed to exist here
+  const body = await request.json()
+  await db.post.create({
+    data: {
+      ...body,
+      authorId: userId
+    }
+  })
+  return NextResponse.json({ success: true })
+}
+
+// Public health check - SAFE (intentionally public)
+export async function HEAD() {
+  return new Response(null, { status: 200 })
+}
+`,
+        language: 'typescript',
+        size: 1000,
+      },
+    },
+  ],
+}

package/src/__tests__/benchmark/fixtures/layer2/byok-patterns.ts

@@ -0,0 +1,112 @@
+/**
+ * BYOK (Bring Your Own Key) Test Fixtures
+ * Tests for detecting insecure handling of user-provided API keys
+ */
+
+import type { TestGroup } from '../../types'
+
+export const byokTests: TestGroup = {
+  name: 'BYOK Patterns',
+  tier: 'A',
+  layer: 2,
+  description: 'Detection of insecure BYOK (Bring Your Own Key) patterns',
+  
+  truePositives: [
+    {
+      name: 'BYOK - True Positives',
+      expectFindings: true,
+      expectedCategories: ['ai_pattern'],
+      description: 'Insecure BYOK patterns that MUST be detected',
+      file: {
+        path: 'src/api/ai/byok-storage.ts',
+        content: `
+import { db } from '@/lib/db'
+
+// BYOK key stored in database (Medium - should encrypt)
+export async function saveUserApiKey(userId: string, apiKey: string) {
+  await db.user.update({
+    where: { id: userId },
+    data: { openaiApiKey: apiKey }  // Stored without encryption!
+  })
+}
+
+// BYOK key logged (Medium - never log secrets)
+export async function useUserKey(userId: string) {
+  const user = await db.user.findUnique({ where: { id: userId } })
+  console.log('Using API key:', user.openaiApiKey)  // Logged!
+  return callOpenAI(user.openaiApiKey)
+}
+
+// BYOK key stored in plain text file
+export function saveKeyToFile(apiKey: string) {
+  fs.writeFileSync('/data/api-keys.txt', apiKey)
+}
+
+// Cross-tenant key access (High - data isolation risk)
+export async function getAnyUserKey(targetUserId: string) {
+  // No check that current user owns this key!
+  const user = await db.user.findUnique({ where: { id: targetUserId } })
+  return user.apiKey
+}
+
+// BYOK on unauthenticated endpoint (Medium - abuse risk)
+export async function POST(request: Request) {
+  // No auth check!
+  const { apiKey, prompt } = await request.json()
+  return callOpenAI(apiKey, prompt)
+}
+`,
+        language: 'typescript',
+        size: 1200,
+      },
+    },
+  ],
+  
+  falseNegatives: [
+    {
+      name: 'BYOK - False Negatives',
+      expectFindings: false,
+      description: 'Secure BYOK patterns that should NOT be flagged',
+      file: {
+        path: 'src/api/ai/byok-transient.ts',
+        content: `
+import { auth } from '@/lib/auth'
+import OpenAI from 'openai'
+
+// Transient BYOK usage (authenticated) - SAFE (info at most)
+export async function POST(request: Request) {
+  // Auth check
+  const session = await auth()
+  if (!session?.user) {
+    return Response.json({ error: 'Unauthorized' }, { status: 401 })
+  }
+
+  // Key from request body, used transiently
+  const { apiKey, prompt } = await request.json()
+
+  // Key only exists in memory for this request
+  const openai = new OpenAI({ apiKey })
+  const response = await openai.chat.completions.create({
+    model: 'gpt-4',
+    messages: [{ role: 'user', content: prompt }]
+  })
+
+  // Key is NOT stored or logged - this is the IDEAL BYOK pattern
+  return Response.json({ result: response.choices[0].message.content })
+}
+
+// User-scoped encrypted key storage - SAFE
+export async function saveEncryptedKey(userId: string, apiKey: string) {
+  const encryptedKey = await encrypt(apiKey, process.env.ENCRYPTION_KEY!)
+  await db.user.update({
+    where: { id: userId },
+    data: { encryptedApiKey: encryptedKey }
+  })
+}
+`,
+        language: 'typescript',
+        size: 1100,
+      },
+    },
+  ],
+}