@oculum/scanner 1.0.9 → 1.0.11

package/src/__tests__/benchmark/fixtures/layer2/ai-package-hallucination.ts

@@ -0,0 +1,255 @@
+/**
+ * AI Package Hallucination Test Fixtures
+ * Tests for detecting AI-hallucinated/fake package names in dependencies
+ *
+ * Background: USENIX research shows ~20% of AI-generated code references
+ * packages that don't exist, creating supply chain attack vectors.
+ */
+
+import type { TestGroup } from '../../types'
+
+export const aiPackageHallucinationTests: TestGroup = {
+  name: 'AI Package Hallucination Detection',
+  tier: 'A',
+  layer: 2,
+  description: 'Detection of AI-hallucinated and potentially fake package names in imports and dependencies',
+
+  truePositives: [
+    {
+      name: 'Package Hallucination - True Positives (Known Fakes)',
+      expectFindings: true,
+      expectedCategories: ['ai_package_hallucination'],
+      description: 'Known hallucinated packages that MUST be detected',
+      file: {
+        path: 'src/utils/helpers.ts',
+        content: `
+// Known AI-hallucinated package names (verified fakes from research)
+import { Chart } from 'react-charts'  // Fake - react-chartjs-2 or recharts are real
+import { MongoClient } from 'mongo-client'  // Fake - mongodb is real
+import { PostgresClient } from 'postgres-client'  // Fake - pg is real
+import fastJson from 'fast-json'  // Fake - fast-json-stringify or ajv are real
+import helpers from 'node-helpers'  // Fake - doesn't exist
+
+// Import with destructuring
+const { utils } = require('easy-utils')  // Fake
+const tools = require('simple-tools')  // Fake
+
+// Dynamic import
+const csvParser = await import('csv-parser-pro')  // Fake - csv-parser or papaparse are real
+`,
+        language: 'typescript',
+        size: 600,
+      },
+    },
+    {
+      name: 'Package Hallucination - True Positives (Suspicious Patterns)',
+      expectFindings: true,
+      expectedCategories: ['ai_package_hallucination'],
+      description: 'Suspicious package naming patterns that suggest hallucination',
+      file: {
+        path: 'src/api/services.ts',
+        content: `
+// Unscoped generic utility names (frequently hallucinated)
+import utils from 'utils'  // Too generic - likely hallucinated
+import helpers from 'helpers'  // Too generic - likely hallucinated
+import common from 'common'  // Too generic - likely hallucinated
+import tools from 'tools'  // Too generic - likely hallucinated
+
+// easy-* pattern (frequently hallucinated)
+import easyConfig from 'easy-config'  // Suspicious pattern
+import easyDb from 'easy-database'  // Suspicious pattern
+import easyAuth from 'easy-auth'  // Suspicious pattern
+
+// simple-* pattern (frequently hallucinated)
+import simpleApi from 'simple-api'  // Suspicious pattern
+import simpleServer from 'simple-server'  // Suspicious pattern
+import simpleCache from 'simple-cache'  // Suspicious pattern
+
+// node-* patterns for non-core modules (frequently hallucinated)
+import nodeUtils from 'node-utils'  // Suspicious - not a core module
+import nodeHelpers from 'node-helpers'  // Suspicious - not a core module
+
+// fast-* patterns without real packages
+import fastParser from 'fast-parser'  // Suspicious pattern
+import fastValidator from 'fast-validator'  // Suspicious pattern
+`,
+        language: 'typescript',
+        size: 900,
+      },
+    },
+    {
+      name: 'Package Hallucination - True Positives (package.json)',
+      expectFindings: true,
+      expectedCategories: ['ai_package_hallucination'],
+      description: 'Hallucinated packages in package.json dependencies',
+      file: {
+        path: 'package.json',
+        content: `{
+  "name": "my-app",
+  "dependencies": {
+    "react": "^18.0.0",
+    "react-charts": "^1.0.0",
+    "mongo-client": "^3.0.0",
+    "easy-config": "^2.0.0",
+    "utils": "^1.0.0",
+    "helpers": "^1.0.0",
+    "node-helpers": "^1.0.0"
+  },
+  "devDependencies": {
+    "simple-test": "^1.0.0",
+    "fast-bundler": "^1.0.0"
+  }
+}`,
+        language: 'json',
+        size: 400,
+      },
+    },
+    {
+      name: 'Package Hallucination - True Positives (requirements.txt)',
+      expectFindings: true,
+      expectedCategories: ['ai_package_hallucination'],
+      description: 'Hallucinated packages in Python requirements',
+      file: {
+        path: 'requirements.txt',
+        content: `# Real packages
+flask==2.0.0
+requests==2.28.0
+
+# Likely hallucinated packages
+easy-flask==1.0.0
+simple-api==1.0.0
+fast-db==2.0.0
+python-helpers==1.0.0
+utils==1.0.0
+common==1.0.0
+`,
+        language: 'text',
+        size: 250,
+      },
+    },
+  ],
+
+  falseNegatives: [
+    {
+      name: 'Package Hallucination - False Negatives (Real Packages)',
+      expectFindings: false,
+      description: 'Real, well-known packages that should NOT be flagged',
+      allowedInfoFindings: [
+        {
+          category: 'suspicious_package',
+          maxCount: 2,
+          reason: 'Some real packages may trigger info-level findings from risky-imports detector',
+        },
+        {
+          category: 'ai_package_hallucination',
+          maxCount: 1,
+          reason: 'Edge cases may generate info-level findings',
+        },
+      ],
+      file: {
+        path: 'src/app/real-packages.ts',
+        content: `
+// Popular, well-known real packages - SHOULD NOT FLAG
+import React from 'react'
+import { useState } from 'react'
+import express from 'express'
+import axios from 'axios'
+import lodash from 'lodash'
+import { z } from 'zod'
+import dayjs from 'dayjs'
+import { MongoClient } from 'mongodb'  // Real (not 'mongo-client')
+import pg from 'pg'  // Real postgres client
+import Fastify from 'fastify'  // Real
+import csvParser from 'csv-parser'  // Real (not 'csv-parser-pro')
+
+// Scoped packages are generally real
+import { Hono } from '@hono/hono'
+import { createClient } from '@supabase/supabase-js'
+import { OpenAI } from 'openai'
+import Anthropic from '@anthropic-ai/sdk'
+import { PrismaClient } from '@prisma/client'
+
+// These are real 'simple-' packages that exist
+import SimpleSchema from 'simpl-schema'  // Real package
+import simpleGit from 'simple-git'  // Real package (has >10M weekly downloads)
+
+// These are real 'fast-' packages that exist
+import fastJson from 'fast-json-stringify'  // Real package
+import fastify from 'fastify'  // Real package
+import fastGlob from 'fast-glob'  // Real package
+
+// Real utility packages with specific names
+import _ from 'underscore'
+import __ from 'radash'
+import clsx from 'clsx'
+import nanoid from 'nanoid'
+`,
+        language: 'typescript',
+        size: 1000,
+      },
+    },
+    {
+      name: 'Package Hallucination - False Negatives (Internal/Monorepo)',
+      expectFindings: false,
+      description: 'Internal monorepo packages that should NOT be flagged',
+      file: {
+        path: 'apps/web/src/index.ts',
+        content: `
+// Internal monorepo packages - these are workspace packages, not external
+import { scanner } from '@oculum/scanner'
+import { shared } from '@oculum/shared'
+import { Button } from '@mycompany/ui'
+import { utils } from '@internal/utils'  // Scoped internal is fine
+
+// Relative imports should never be flagged
+import { helper } from './utils'
+import { common } from '../shared/common'
+import api from '../../api'
+
+// Aliased paths should not be flagged
+import { config } from '@/config'
+import { utils } from '~/utils'
+import { helpers } from '#utils/helpers'
+`,
+        language: 'typescript',
+        size: 500,
+      },
+    },
+    {
+      name: 'Package Hallucination - False Negatives (Test Files)',
+      expectFindings: false,
+      description: 'Test files may use mocks that should be downgraded',
+      allowedInfoFindings: [
+        {
+          category: 'ai_package_hallucination',
+          maxCount: 3,
+          reason: 'Test file mock packages may generate info-level findings',
+        },
+      ],
+      file: {
+        path: 'src/__tests__/mocks/packages.test.ts',
+        content: `
+// Test mocks - these might look suspicious but are test-only
+import mockUtils from 'utils'
+import mockHelpers from 'helpers'
+
+jest.mock('utils', () => ({
+  format: jest.fn()
+}))
+
+jest.mock('helpers', () => ({
+  parse: jest.fn()
+}))
+
+describe('mock tests', () => {
+  it('should work', () => {
+    expect(mockUtils).toBeDefined()
+  })
+})
+`,
+        language: 'typescript',
+        size: 400,
+      },
+    },
+  ],
+}

package/src/__tests__/benchmark/fixtures/layer2/ai-prompt-hygiene.ts

@@ -10,7 +10,7 @@ export const aiPromptHygieneTests: TestGroup = {
   tier: 'B',
   layer: 2,
   description: 'Detection of prompt injection vulnerabilities and unsafe prompt practices',
-  
+
   truePositives: [
     {
       name: 'AI Prompt Hygiene - True Positives',
@@ -87,6 +87,204 @@ export async function concatenatedPrompt(name: string, task: string) {
         size: 1600,
       },
     },
+    // ========== NEW: Secrets in Prompts True Positives ==========
+    {
+      name: 'Secrets in Prompts - Provider Tokens',
+      expectFindings: true,
+      expectedCategories: ['hardcoded_secret'],
+      description: 'Various provider API keys/tokens in LLM prompt code that MUST be detected',
+      file: {
+        path: 'src/api/ai/secrets-in-prompts.ts',
+        content: `
+import OpenAI from 'openai'
+import Anthropic from '@anthropic-ai/sdk'
+
+const openai = new OpenAI()
+const anthropic = new Anthropic()
+
+// 1. OpenAI API key in prompt context - CRITICAL
+export async function openaiKeyInPrompt() {
+  const apiKey = "sk-1234567890abcdefghijklmnopqrstuvwxyz"
+  return openai.chat.completions.create({
+    model: 'gpt-4',
+    messages: [
+      { role: 'system', content: \`Use API key \${apiKey} to call the service\` }
+    ]
+  })
+}
+
+// 2. GitHub token in prompt - CRITICAL
+export async function githubTokenInPrompt() {
+  const token = "ghp_abcdefghijklmnopqrstuvwxyz1234567890"
+  return openai.chat.completions.create({
+    model: 'gpt-4',
+    messages: [
+      { role: 'system', content: \`Access the repo using token: \${token}\` }
+    ]
+  })
+}
+
+// 3. Stripe key in prompt - CRITICAL
+export async function stripeKeyInPrompt() {
+  const stripeKey = "sk_live_1234567890abcdefghijklmnop"
+  return openai.chat.completions.create({
+    model: 'gpt-4',
+    messages: [
+      { role: 'system', content: \`Process payment with Stripe key: \${stripeKey}\` }
+    ]
+  })
+}
+
+// 4. AWS credentials in prompt - CRITICAL
+export async function awsCredentialsInPrompt() {
+  const awsKey = "AKIAIOSFODNN7EXAMPLE"
+  return openai.chat.completions.create({
+    model: 'gpt-4',
+    messages: [
+      { role: 'system', content: \`AWS access key: \${awsKey}\` }
+    ]
+  })
+}
+
+// 5. Slack token in prompt - CRITICAL
+export async function slackTokenInPrompt() {
+  const slackToken = "xoxb-123456789012-1234567890123-abcdefghijklmnopqrstuvwx"
+  return anthropic.messages.create({
+    model: 'claude-3-opus-20240229',
+    messages: [
+      { role: 'user', content: \`Post to Slack using token: \${slackToken}\` }
+    ],
+    max_tokens: 1024
+  })
+}
+
+// 6. SendGrid key in prompt - CRITICAL
+export async function sendgridKeyInPrompt() {
+  const sendgridKey = "SG.abcdefghijklmnopqrstuv.wxyzABCDEFGHIJKLMNOPQRSTUVWXYZ123456789012"
+  return openai.chat.completions.create({
+    model: 'gpt-4',
+    messages: [
+      { role: 'system', content: \`Send email using SendGrid: \${sendgridKey}\` }
+    ]
+  })
+}
+
+// 7. Database URL with credentials - CRITICAL
+export async function dbUrlInPrompt() {
+  const dbUrl = "postgres://admin:secretpassword123@db.example.com:5432/mydb"
+  return openai.chat.completions.create({
+    model: 'gpt-4',
+    messages: [
+      { role: 'system', content: \`Connect to database: \${dbUrl}\` }
+    ]
+  })
+}
+
+// 8. JWT token in prompt - HIGH
+export async function jwtInPrompt() {
+  const jwt = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6IkpvaG4gRG9lIiwiaWF0IjoxNTE2MjM5MDIyfQ.SflKxwRJSMeKKF2QT4fwpMeJf36POk6yJV_adQssw5c"
+  return openai.chat.completions.create({
+    model: 'gpt-4',
+    messages: [
+      { role: 'system', content: \`User session: \${jwt}\` }
+    ]
+  })
+}
+
+// 9. Private key in prompt - CRITICAL
+export async function privateKeyInPrompt() {
+  const privateKey = \`-----BEGIN RSA PRIVATE KEY-----
+MIIEowIBAAKCAQEA0Z3Vx...
+-----END RSA PRIVATE KEY-----\`
+  return openai.chat.completions.create({
+    model: 'gpt-4',
+    messages: [
+      { role: 'system', content: \`Sign with key: \${privateKey}\` }
+    ]
+  })
+}
+
+// 10. HuggingFace token - CRITICAL
+export async function hfTokenInPrompt() {
+  const hfToken = "hf_abcdefghijklmnopqrstuvwxyz1234567890"
+  return openai.chat.completions.create({
+    model: 'gpt-4',
+    messages: [
+      { role: 'system', content: \`Download model with HF token: \${hfToken}\` }
+    ]
+  })
+}
+`,
+        language: 'typescript',
+        size: 3500,
+      },
+    },
+    // ========== Variable Indirection - Secrets flowing to prompts ==========
+    {
+      name: 'Secrets in Prompts - Variable Flow',
+      expectFindings: true,
+      expectedCategories: ['hardcoded_secret'],
+      description: 'Secrets assigned to variables that flow into prompts',
+      file: {
+        path: 'src/api/ai/secrets-variable-flow.ts',
+        content: `
+import OpenAI from 'openai'
+
+const openai = new OpenAI()
+
+// Secret assigned to variable, then used in prompt
+export async function variableFlowSecret() {
+  const apiKey = "sk-abcdefghijklmnopqrstuvwxyz1234"
+  const authToken = "ghp_1234567890abcdefghijklmnopqrstuvwxyz"
+
+  // These secrets flow into the prompt via template interpolation
+  const systemPrompt = \`You have access to external APIs.
+  OpenAI key: \${apiKey}
+  GitHub token: \${authToken}\`
+
+  return openai.chat.completions.create({
+    model: 'gpt-4',
+    messages: [
+      { role: 'system', content: systemPrompt }
+    ]
+  })
+}
+
+// Secret via concatenation
+export async function concatFlowSecret() {
+  const secretKey = "sk_live_abcdefghijklmnopqrstuvwx"
+
+  // Secret flows via string concatenation
+  const prompt = "Process payment with key: " + secretKey
+
+  return openai.chat.completions.create({
+    model: 'gpt-4',
+    messages: [
+      { role: 'system', content: prompt }
+    ]
+  })
+}
+
+// Multi-hop variable flow
+export async function multiHopSecret() {
+  const credentials = {
+    apiToken: "xoxb-123456789012-1234567890123-abcdefghijkl"
+  }
+
+  const token = credentials.apiToken
+
+  return openai.chat.completions.create({
+    model: 'gpt-4',
+    messages: [
+      { role: 'system', content: \`Slack token for notifications: \${token}\` }
+    ]
+  })
+}
+`,
+        language: 'typescript',
+        size: 1500,
+      },
+    },
   ],
   
   falseNegatives: [
@@ -174,5 +372,106 @@ export async function allowedModel(model: string, prompt: string) {
         size: 1500,
       },
     },
+    // ========== Safe Secrets Patterns ==========
+    {
+      name: 'Secrets in Prompts - Safe Patterns',
+      expectFindings: false,
+      description: 'Safe patterns for handling secrets near LLM code that should NOT be flagged',
+      allowedInfoFindings: [
+        {
+          category: 'hardcoded_secret',
+          maxCount: 1,
+          reason: 'Info-level for test variable names or placeholders',
+        },
+        {
+          category: 'ai_pattern',
+          maxCount: 1,
+          reason: 'Info-level BYOK pattern detection for user-provided API keys (feature, not vulnerability)',
+        },
+      ],
+      file: {
+        path: 'src/api/ai/safe-secrets.ts',
+        content: `
+import OpenAI from 'openai'
+
+const openai = new OpenAI()
+
+// 1. Using environment variables (SAFE)
+export async function envVarSecret() {
+  const apiKey = process.env.OPENAI_API_KEY
+  const client = new OpenAI({ apiKey })
+
+  return client.chat.completions.create({
+    model: 'gpt-4',
+    messages: [
+      { role: 'system', content: 'You are a helpful assistant.' }
+    ]
+  })
+}
+
+// 2. Server-side API call without exposing key to prompt (SAFE)
+export async function serverSideCall() {
+  // Key is used for client auth, NOT exposed to prompt
+  const stripe = require('stripe')(process.env.STRIPE_SECRET_KEY)
+
+  const result = await openai.chat.completions.create({
+    model: 'gpt-4',
+    messages: [
+      { role: 'user', content: 'What products should I buy?' }
+    ]
+  })
+
+  // Use AI response to make server-side Stripe call
+  const products = await stripe.products.list()
+  return { aiResponse: result, products }
+}
+
+// 3. Placeholder/example values (SAFE)
+export async function examplePlaceholder() {
+  // These are clearly placeholders, not real secrets
+  const EXAMPLE_API_KEY = "your-api-key-here"
+  const TEST_TOKEN = "test_token_placeholder"
+  const DEMO_KEY = "demo_sk_1234"
+
+  // Documentation example
+  console.log("Replace EXAMPLE_API_KEY with your actual key")
+}
+
+// 4. Test/mock variable names (SAFE)
+export async function testVariables() {
+  const TEST_API_KEY = "sk-test1234567890abcdefghijklmnop"
+  const MOCK_SECRET = "mock_secret_for_testing"
+  const DUMMY_TOKEN = "dummy_ghp_1234567890"
+
+  // Used in unit tests, not production
+}
+
+// 5. Import.meta.env reference (SAFE)
+export async function importMetaEnv() {
+  const key = import.meta.env.VITE_API_KEY
+  return openai.chat.completions.create({
+    model: 'gpt-4',
+    messages: [{ role: 'user', content: 'Hello' }]
+  })
+}
+
+// 6. Python os.environ (SAFE)
+// const apiKey = os.environ.get('API_KEY')
+// const token = os.getenv('AUTH_TOKEN')
+
+// 7. Referencing user's own key (BYOK pattern - SAFE)
+export async function byokPattern(userApiKey: string) {
+  // User provides their own key - this is the BYOK feature
+  const client = new OpenAI({ apiKey: userApiKey })
+  return client.chat.completions.create({
+    model: 'gpt-4',
+    messages: [{ role: 'user', content: 'Hello' }]
+  })
+}
+`,
+        language: 'typescript',
+        size: 2200,
+      },
+    },
   ],
 }