@dotsetlabs/bellwether 0.10.0

package/dist/persona/builtins.js

@@ -0,0 +1,219 @@
+/**
+ * Built-in persona definitions.
+ */
+/**
+ * Technical Writer persona - balanced documentation focus.
+ */
+export const technicalWriterPersona = {
+    id: 'technical_writer',
+    name: 'Technical Writer',
+    description: 'Creates comprehensive API documentation with realistic examples',
+    systemPrompt: `You are a technical documentation specialist creating API reference documentation.
+Your goal is to generate helpful, realistic examples that developers can use as templates.
+Focus on demonstrating the full range of tool capabilities with practical use cases.
+Be thorough but concise. Prioritize clarity and usefulness over edge cases.`,
+    questionBias: {
+        happyPath: 0.5,
+        edgeCase: 0.2,
+        errorHandling: 0.2,
+        boundary: 0.1,
+    },
+    categories: ['happy_path', 'edge_case', 'error_handling'],
+    builtin: true,
+};
+/**
+ * Security Tester persona - vulnerability-focused.
+ *
+ * Note: Prompts are designed to avoid triggering LLM safety filters while still
+ * enabling meaningful security testing. We describe test categories rather than
+ * including specific payloads, letting the LLM generate appropriate test inputs.
+ */
+export const securityTesterPersona = {
+    id: 'security_tester',
+    name: 'Security Tester',
+    description: 'Probes for security vulnerabilities and unsafe behaviors',
+    systemPrompt: `You are a security documentation specialist creating API security test documentation.
+Your goal is to generate test cases that verify proper input validation and error handling.
+
+Generate test cases for these security validation categories:
+- Path handling: Test how the API handles relative paths, parent directory references, and encoded path characters
+- Input sanitization: Test how the API handles special characters that could be interpreted as code or commands
+- URL validation: Test how the API validates and restricts URL inputs
+- Numeric boundaries: Test extreme values, negative numbers, and special numeric values
+- Empty and null handling: Test missing, empty, and null inputs
+
+For each test:
+1. Use realistic but clearly test-oriented inputs (e.g., paths like "/test/../safe" not actual system paths)
+2. Document what security property is being validated
+3. Note whether the API properly rejects or sanitizes potentially dangerous inputs
+4. Observe error message content for information disclosure
+
+Focus on testing INPUT VALIDATION behaviors, not exploitation.
+Your test cases help API developers understand their security posture.`,
+    questionBias: {
+        happyPath: 0.1,
+        edgeCase: 0.2,
+        errorHandling: 0.2,
+        boundary: 0.2,
+        security: 0.3,
+    },
+    categories: ['security', 'boundary', 'error_handling'],
+    additionalContext: `Security test input patterns (use variations appropriate to the tool):
+
+Path validation tests:
+- Relative paths with parent references (test path traversal handling)
+- URL-encoded path characters (test encoding handling)
+- Paths outside expected directories (test directory restrictions)
+
+String validation tests:
+- Strings with SQL-like syntax (test SQL injection prevention)
+- Strings with markup syntax (test XSS prevention)
+- Strings with shell metacharacters (test command injection prevention)
+
+URL validation tests:
+- Internal/private network addresses (test SSRF prevention)
+- Non-HTTP protocols (test protocol validation)
+- Localhost and loopback variations (test internal access restrictions)
+
+Numeric validation tests:
+- Zero, negative numbers, and boundary values
+- Very large numbers and overflow values
+- Non-numeric strings where numbers expected
+
+Generate realistic test inputs that verify these security controls work correctly.`,
+    builtin: true,
+};
+/**
+ * QA Engineer persona - edge case and error focus.
+ */
+export const qaEngineerPersona = {
+    id: 'qa_engineer',
+    name: 'QA Engineer',
+    description: 'Tests edge cases, error conditions, and unexpected inputs',
+    systemPrompt: `You are a quality assurance engineer testing an API for robustness.
+Your goal is to find edge cases, error conditions, and unexpected behaviors.
+Focus on:
+- Boundary values (min, max, just over/under limits)
+- Type coercion issues (strings vs numbers, null handling)
+- Empty and missing values
+- Unicode and special characters
+- Concurrent/timing issues
+- State corruption scenarios
+
+Generate test cases that stress the tool's error handling and validation.
+Document any crashes, hangs, or unexpected error messages.`,
+    questionBias: {
+        happyPath: 0.1,
+        edgeCase: 0.35,
+        errorHandling: 0.35,
+        boundary: 0.2,
+    },
+    categories: ['edge_case', 'error_handling', 'boundary'],
+    additionalContext: `Edge cases to test:
+- Empty strings, whitespace-only strings
+- Very long strings (1000+ chars)
+- Unicode: emoji, RTL text, zero-width chars
+- Numbers: 0, -0, negative, floats for ints
+- Arrays: empty, single item, thousands of items
+- Objects: empty, deeply nested, circular (if possible)`,
+    builtin: true,
+};
+/**
+ * Novice User persona - usability and error message focus.
+ */
+export const noviceUserPersona = {
+    id: 'novice_user',
+    name: 'Novice User',
+    description: 'Tests from the perspective of a new user making common mistakes',
+    systemPrompt: `You are a new developer using this API for the first time.
+Your goal is to test how the API handles common mistakes and misunderstandings.
+Focus on:
+- Missing required parameters
+- Wrong parameter types (string instead of number, etc.)
+- Misspelled parameter names
+- Incorrect formats (dates, URLs, emails)
+- Reasonable but wrong assumptions
+
+Evaluate the quality of error messages:
+- Are they clear and actionable?
+- Do they help the user fix the problem?
+- Do they expose implementation details?
+
+Generate test cases that a confused beginner might try.`,
+    questionBias: {
+        happyPath: 0.2,
+        edgeCase: 0.2,
+        errorHandling: 0.5,
+        boundary: 0.1,
+    },
+    categories: ['error_handling', 'happy_path', 'edge_case'],
+    additionalContext: `Common novice mistakes:
+- Omitting required parameters
+- Using wrong case (userId vs userid)
+- Wrong types (passing "123" instead of 123)
+- Incomplete data (partial objects)
+- Obvious typos in enum values
+- Mixing up similar parameters`,
+    builtin: true,
+};
+/**
+ * Map of built-in persona IDs to definitions.
+ */
+export const BUILTIN_PERSONAS = {
+    technical_writer: technicalWriterPersona,
+    security_tester: securityTesterPersona,
+    qa_engineer: qaEngineerPersona,
+    novice_user: noviceUserPersona,
+};
+/**
+ * Get a built-in persona by ID.
+ */
+export function getBuiltinPersona(id) {
+    const persona = BUILTIN_PERSONAS[id];
+    if (!persona) {
+        throw new Error(`Unknown built-in persona: ${id}`);
+    }
+    return persona;
+}
+/**
+ * Check if a persona ID is a built-in.
+ */
+export function isBuiltinPersona(id) {
+    return id in BUILTIN_PERSONAS;
+}
+/**
+ * Get all built-in persona IDs.
+ */
+export function getBuiltinPersonaIds() {
+    return Object.keys(BUILTIN_PERSONAS);
+}
+/**
+ * Default persona for interviews.
+ */
+export const DEFAULT_PERSONA = technicalWriterPersona;
+/**
+ * Parse persona list from string array of persona IDs.
+ * Returns the DEFAULT_PERSONA if the list is empty or contains no valid personas.
+ *
+ * @param personaList - Array of persona ID strings
+ * @param warnOnUnknown - Optional callback for unknown persona warnings
+ * @returns Array of resolved Persona objects
+ */
+export function parsePersonas(personaList, warnOnUnknown) {
+    if (personaList.length === 0) {
+        return [DEFAULT_PERSONA];
+    }
+    const personas = [];
+    const validNames = Object.keys(BUILTIN_PERSONAS);
+    for (const name of personaList) {
+        const persona = BUILTIN_PERSONAS[name];
+        if (persona) {
+            personas.push(persona);
+        }
+        else if (warnOnUnknown) {
+            warnOnUnknown(name, validNames);
+        }
+    }
+    return personas.length > 0 ? personas : [DEFAULT_PERSONA];
+}
+//# sourceMappingURL=builtins.js.map

package/dist/persona/index.d.ts

@@ -0,0 +1,8 @@
+/**
+ * Persona module - configurable interviewer personalities.
+ */
+export * from './types.js';
+export * from './builtins.js';
+export * from './loader.js';
+export * from './validation.js';
+//# sourceMappingURL=index.d.ts.map

package/dist/persona/index.js

@@ -0,0 +1,8 @@
+/**
+ * Persona module - configurable interviewer personalities.
+ */
+export * from './types.js';
+export * from './builtins.js';
+export * from './loader.js';
+export * from './validation.js';
+//# sourceMappingURL=index.js.map

package/dist/persona/loader.d.ts

@@ -0,0 +1,30 @@
+/**
+ * Persona loader - resolves persona IDs and loads custom personas from YAML.
+ */
+import type { Persona } from './types.js';
+/**
+ * Options for loading personas.
+ */
+export interface LoadPersonaOptions {
+    /** Persona ID(s) - can be built-in IDs or paths to YAML files */
+    personas?: string | string[];
+    /** Explicit path to a persona YAML file */
+    personaFile?: string;
+}
+/**
+ * Load personas from IDs or file paths.
+ */
+export declare function loadPersonas(options?: LoadPersonaOptions): Persona[];
+/**
+ * Resolve a persona from ID or file path.
+ */
+export declare function resolvePersona(personaSpec: string): Persona;
+/**
+ * Load a persona from a YAML file.
+ */
+export declare function loadPersonaFromFile(path: string): Persona;
+/**
+ * Generate a sample custom persona YAML.
+ */
+export declare function generateSamplePersonaYaml(): string;
+//# sourceMappingURL=loader.d.ts.map

package/dist/persona/loader.js

@@ -0,0 +1,190 @@
+/**
+ * Persona loader - resolves persona IDs and loads custom personas from YAML.
+ */
+import { readFileSync, existsSync } from 'fs';
+import { parseYamlSecure } from '../utils/yaml-parser.js';
+import { BUILTIN_PERSONAS, isBuiltinPersona, DEFAULT_PERSONA } from './builtins.js';
+import { validatePersona, formatValidationErrors, normalizeBiasWeights } from './validation.js';
+import { getLogger } from '../logging/logger.js';
+import { MATH_FACTORS } from '../constants.js';
+/**
+ * Load personas from IDs or file paths.
+ */
+export function loadPersonas(options = {}) {
+    const result = [];
+    // Handle explicit persona file
+    if (options.personaFile) {
+        const persona = loadPersonaFromFile(options.personaFile);
+        result.push(persona);
+    }
+    // Handle persona IDs/paths
+    if (options.personas) {
+        const personaList = Array.isArray(options.personas)
+            ? options.personas
+            : options.personas.split(',').map(s => s.trim());
+        for (const personaSpec of personaList) {
+            const persona = resolvePersona(personaSpec);
+            // Avoid duplicates
+            if (!result.some(p => p.id === persona.id)) {
+                result.push(persona);
+            }
+        }
+    }
+    // Default to technical_writer if nothing specified
+    if (result.length === 0) {
+        result.push(DEFAULT_PERSONA);
+    }
+    return result;
+}
+/**
+ * Resolve a persona from ID or file path.
+ */
+export function resolvePersona(personaSpec) {
+    // Check if it's a built-in persona ID
+    if (isBuiltinPersona(personaSpec)) {
+        return BUILTIN_PERSONAS[personaSpec];
+    }
+    // Check common aliases
+    const aliases = {
+        'writer': 'technical_writer',
+        'security': 'security_tester',
+        'qa': 'qa_engineer',
+        'novice': 'novice_user',
+        'beginner': 'novice_user',
+    };
+    if (personaSpec in aliases) {
+        const aliasedId = aliases[personaSpec];
+        if (isBuiltinPersona(aliasedId)) {
+            return BUILTIN_PERSONAS[aliasedId];
+        }
+    }
+    // Try to load as a file path
+    if (existsSync(personaSpec)) {
+        return loadPersonaFromFile(personaSpec);
+    }
+    // Check with common extensions
+    const extensions = ['.yaml', '.yml'];
+    for (const ext of extensions) {
+        const pathWithExt = personaSpec + ext;
+        if (existsSync(pathWithExt)) {
+            return loadPersonaFromFile(pathWithExt);
+        }
+    }
+    throw new Error(`Unknown persona: "${personaSpec}". ` +
+        `Available built-in personas: technical_writer, security_tester, qa_engineer, novice_user`);
+}
+/**
+ * Load a persona from a YAML file.
+ */
+export function loadPersonaFromFile(path) {
+    if (!existsSync(path)) {
+        throw new Error(`Persona file not found: ${path}`);
+    }
+    const content = readFileSync(path, 'utf-8');
+    const parsed = parseYamlSecure(content);
+    return validateAndNormalizePersona(parsed, path);
+}
+/**
+ * Validate and normalize a persona definition.
+ */
+function validateAndNormalizePersona(data, source) {
+    // Required fields - basic checks first
+    if (!data.id || typeof data.id !== 'string') {
+        throw new Error(`Persona from ${source} missing required field: id`);
+    }
+    if (!data.name || typeof data.name !== 'string') {
+        throw new Error(`Persona from ${source} missing required field: name`);
+    }
+    if (!data.systemPrompt || typeof data.systemPrompt !== 'string') {
+        throw new Error(`Persona from ${source} missing required field: systemPrompt`);
+    }
+    // Normalize question bias with defaults
+    const defaultBias = {
+        happyPath: MATH_FACTORS.DEFAULT_QUESTION_BIAS,
+        edgeCase: MATH_FACTORS.DEFAULT_QUESTION_BIAS,
+        errorHandling: MATH_FACTORS.DEFAULT_QUESTION_BIAS,
+        boundary: MATH_FACTORS.DEFAULT_QUESTION_BIAS,
+    };
+    let questionBias = {
+        ...defaultBias,
+        ...(data.questionBias ?? {}),
+    };
+    // Normalize categories
+    const defaultCategories = ['happy_path', 'edge_case', 'error_handling'];
+    const categories = data.categories ?? defaultCategories;
+    // Create the persona object
+    const persona = {
+        id: data.id,
+        name: data.name,
+        description: data.description ?? `Custom persona: ${data.name}`,
+        systemPrompt: data.systemPrompt,
+        questionBias,
+        categories,
+        additionalContext: data.additionalContext,
+        builtin: false,
+    };
+    // Run comprehensive validation
+    const validationResult = validatePersona(persona, {
+        // Allow some flexibility for custom personas
+        warnUnusedBiases: true,
+        allowMissingSecurity: false,
+    });
+    if (!validationResult.valid) {
+        throw new Error(formatValidationErrors(validationResult, source));
+    }
+    // Log warnings if any
+    if (validationResult.warnings.length > 0) {
+        const logger = getLogger('persona-loader');
+        for (const warning of validationResult.warnings) {
+            logger.warn({ source, warning }, 'Persona validation warning');
+        }
+    }
+    // Normalize bias weights if they don't sum to 1.0
+    const sum = Object.values(questionBias).filter(v => typeof v === 'number').reduce((a, b) => a + b, 0);
+    if (Math.abs(sum - 1.0) > 0.01) {
+        questionBias = normalizeBiasWeights(questionBias);
+        persona.questionBias = questionBias;
+    }
+    return persona;
+}
+/**
+ * Generate a sample custom persona YAML.
+ */
+export function generateSamplePersonaYaml() {
+    return `# Custom Persona Definition
+# Save this file and reference it with: --persona-file ./my-persona.yaml
+
+id: custom_auditor
+name: Compliance Auditor
+description: Tests for compliance with security and data handling requirements
+
+systemPrompt: |
+  You are a compliance auditor testing an API for regulatory requirements.
+  Focus on data handling, privacy, and security compliance.
+  Test for:
+  - Sensitive data exposure in responses
+  - Proper error handling without information leakage
+  - Input validation and sanitization
+  - Access control boundaries
+
+questionBias:
+  happyPath: 0.2
+  edgeCase: 0.2
+  errorHandling: 0.3
+  boundary: 0.15
+  security: 0.15
+
+categories:
+  - error_handling
+  - security
+  - boundary
+
+additionalContext: |
+  Compliance areas to verify:
+  - PII handling and masking
+  - Error message sanitization
+  - Rate limiting behavior
+  - Authentication requirements
+`;
+}
+//# sourceMappingURL=loader.js.map

package/dist/persona/types.d.ts

@@ -0,0 +1,144 @@
+/**
+ * Persona types for configurable interviewer personalities.
+ */
+import type { InterviewQuestion } from '../interview/types.js';
+/**
+ * Question categories that can be weighted by personas.
+ */
+export type QuestionCategory = 'happy_path' | 'edge_case' | 'error_handling' | 'boundary' | 'security';
+/**
+ * Weight distribution for question categories.
+ * Values should be 0-1 and represent relative likelihood.
+ */
+export interface QuestionBias {
+    /** Normal, expected usage patterns */
+    happyPath: number;
+    /** Boundary values and unusual but valid inputs */
+    edgeCase: number;
+    /** Invalid inputs and error conditions */
+    errorHandling: number;
+    /** Limits, extremes, and constraints */
+    boundary: number;
+    /** Security-focused tests (injection, traversal, etc.) */
+    security?: number;
+}
+/**
+ * Built-in persona identifiers.
+ */
+export type BuiltInPersonaId = 'technical_writer' | 'security_tester' | 'qa_engineer' | 'novice_user';
+/**
+ * A persona defines an interviewer's personality and focus.
+ */
+export interface Persona {
+    /** Unique identifier */
+    id: string;
+    /** Human-readable name */
+    name: string;
+    /** Description of the persona's focus */
+    description: string;
+    /** System prompt that shapes LLM behavior */
+    systemPrompt: string;
+    /** Weight distribution for question categories */
+    questionBias: QuestionBias;
+    /** Categories this persona focuses on */
+    categories: QuestionCategory[];
+    /** Additional context to include in prompts */
+    additionalContext?: string;
+    /** Whether this is a built-in persona */
+    builtin?: boolean;
+}
+/**
+ * Result of interviewing with a specific persona.
+ */
+export interface PersonaInterviewResult {
+    /** Persona used for this interview */
+    persona: Persona;
+    /** Questions generated by this persona */
+    questions: InterviewQuestion[];
+    /** Findings specific to this persona's focus */
+    findings: PersonaFinding[];
+    /** Security issues found (security persona only) */
+    securityIssues?: SecurityIssue[];
+}
+/**
+ * A finding from a persona-focused interview.
+ */
+export interface PersonaFinding {
+    /** Tool this finding relates to */
+    tool: string;
+    /** Category of finding */
+    category: QuestionCategory;
+    /** Severity level */
+    severity: 'info' | 'low' | 'medium' | 'high' | 'critical';
+    /** Short title */
+    title: string;
+    /** Detailed description */
+    description: string;
+    /** Evidence supporting this finding */
+    evidence?: string;
+    /** Suggested remediation */
+    recommendation?: string;
+}
+/**
+ * Security issue found during security persona testing.
+ */
+export interface SecurityIssue {
+    /** Type of security issue */
+    type: SecurityIssueType;
+    /** Tool affected */
+    tool: string;
+    /** Severity */
+    severity: 'low' | 'medium' | 'high' | 'critical';
+    /** Description of the issue */
+    description: string;
+    /** Payload or input that triggered the issue */
+    payload?: string;
+    /** Response that indicates vulnerability */
+    response?: string;
+    /** CWE ID if applicable */
+    cweId?: string;
+}
+/**
+ * Types of security issues to detect.
+ */
+export type SecurityIssueType = 'path_traversal' | 'command_injection' | 'sql_injection' | 'xss' | 'ssrf' | 'information_disclosure' | 'authentication_bypass' | 'authorization_bypass' | 'dos' | 'other';
+/**
+ * Configuration for loading personas.
+ */
+export interface PersonaConfig {
+    /** Persona ID or path to YAML file */
+    persona: string | string[];
+    /** Path to custom persona file */
+    personaFile?: string;
+}
+/**
+ * YAML schema for custom persona definitions.
+ */
+export interface PersonaYAML {
+    id: string;
+    name: string;
+    description: string;
+    systemPrompt: string;
+    questionBias: QuestionBias;
+    categories: QuestionCategory[];
+    additionalContext?: string;
+}
+/**
+ * Aggregated results from multiple personas.
+ */
+export interface AggregatedPersonaResults {
+    /** Results from each persona */
+    byPersona: Map<string, PersonaInterviewResult>;
+    /** Findings that appeared across multiple personas */
+    commonFindings: PersonaFinding[];
+    /** Unique findings per persona */
+    uniqueFindings: Map<string, PersonaFinding[]>;
+    /** Overall security assessment */
+    securitySummary?: {
+        issueCount: number;
+        criticalCount: number;
+        highCount: number;
+        issues: SecurityIssue[];
+    };
+}
+//# sourceMappingURL=types.d.ts.map

package/dist/persona/types.js

@@ -0,0 +1,5 @@
+/**
+ * Persona types for configurable interviewer personalities.
+ */
+export {};
+//# sourceMappingURL=types.js.map