@dotsetlabs/bellwether 0.10.0

package/dist/scenarios/loader.js

@@ -0,0 +1,285 @@
+/**
+ * Scenario loader - loads test scenarios from YAML files.
+ */
+import { readFileSync, existsSync } from 'fs';
+import { parseYamlSecure } from '../utils/yaml-parser.js';
+import { PATHS } from '../constants.js';
+/** Default file name for test scenarios */
+export const DEFAULT_SCENARIOS_FILE = PATHS.DEFAULT_SCENARIOS_FILE;
+/** Valid question categories */
+const VALID_CATEGORIES = [
+    'happy_path',
+    'edge_case',
+    'error_handling',
+    'boundary',
+    'security',
+];
+/** Valid assertion conditions */
+const VALID_CONDITIONS = [
+    'exists',
+    'equals',
+    'contains',
+    'truthy',
+    'type',
+    'not_error',
+];
+/**
+ * Load test scenarios from a YAML file.
+ */
+export function loadScenariosFromFile(path) {
+    if (!existsSync(path)) {
+        throw new Error(`Test scenarios file not found: ${path}`);
+    }
+    const content = readFileSync(path, 'utf-8');
+    const parsed = parseYamlSecure(content);
+    if (!parsed || typeof parsed !== 'object') {
+        throw new Error(`Invalid test scenarios file: ${path} (expected object)`);
+    }
+    const globalTags = parsed.tags ?? [];
+    const toolScenarios = [];
+    const promptScenarios = [];
+    // Parse tool scenarios
+    if (parsed.scenarios && Array.isArray(parsed.scenarios)) {
+        for (let i = 0; i < parsed.scenarios.length; i++) {
+            const raw = parsed.scenarios[i];
+            try {
+                const scenario = validateToolScenario(raw, i, path, globalTags);
+                toolScenarios.push(scenario);
+            }
+            catch (error) {
+                throw new Error(`Error in scenario ${i + 1} of ${path}: ${error.message}`);
+            }
+        }
+    }
+    // Parse prompt scenarios
+    if (parsed.prompts && Array.isArray(parsed.prompts)) {
+        for (let i = 0; i < parsed.prompts.length; i++) {
+            const raw = parsed.prompts[i];
+            try {
+                const scenario = validatePromptScenario(raw, i, path, globalTags);
+                promptScenarios.push(scenario);
+            }
+            catch (error) {
+                throw new Error(`Error in prompt scenario ${i + 1} of ${path}: ${error.message}`);
+            }
+        }
+    }
+    return {
+        source: path,
+        toolScenarios,
+        promptScenarios,
+        description: parsed.description,
+        version: parsed.version,
+    };
+}
+/**
+ * Try to load scenarios from the default file in a directory.
+ * Returns null if file doesn't exist.
+ */
+export function tryLoadDefaultScenarios(directory) {
+    const path = `${directory}/${DEFAULT_SCENARIOS_FILE}`;
+    if (!existsSync(path)) {
+        return null;
+    }
+    return loadScenariosFromFile(path);
+}
+/**
+ * Validate and normalize a tool test scenario.
+ */
+function validateToolScenario(data, _index, _source, globalTags) {
+    // Required: tool
+    if (!data.tool || typeof data.tool !== 'string') {
+        throw new Error('missing required field: tool');
+    }
+    // Optional: category (default: happy_path)
+    let category = 'happy_path';
+    if (data.category) {
+        if (!VALID_CATEGORIES.includes(data.category)) {
+            throw new Error(`invalid category "${data.category}". Valid categories: ${VALID_CATEGORIES.join(', ')}`);
+        }
+        category = data.category;
+    }
+    // Optional: args (default: empty)
+    const args = data.args ?? {};
+    if (typeof args !== 'object' || Array.isArray(args)) {
+        throw new Error('args must be an object');
+    }
+    // Optional: assertions
+    const assertions = data.assertions?.map((a, i) => validateAssertion(a, i));
+    // Merge global tags with scenario tags
+    const tags = [...globalTags, ...(data.tags ?? [])];
+    return {
+        tool: data.tool,
+        description: data.description ?? `Test ${data.tool} with ${category} scenario`,
+        category,
+        args,
+        assertions,
+        skip: data.skip ?? false,
+        tags: tags.length > 0 ? tags : undefined,
+    };
+}
+/**
+ * Validate and normalize a prompt test scenario.
+ */
+function validatePromptScenario(data, _index, _source, globalTags) {
+    // Required: prompt
+    if (!data.prompt || typeof data.prompt !== 'string') {
+        throw new Error('missing required field: prompt');
+    }
+    // Optional: args (default: empty)
+    const args = data.args ?? {};
+    if (typeof args !== 'object' || Array.isArray(args)) {
+        throw new Error('args must be an object');
+    }
+    // Validate that all arg values are strings
+    for (const [key, value] of Object.entries(args)) {
+        if (typeof value !== 'string') {
+            throw new Error(`prompt arg "${key}" must be a string, got ${typeof value}`);
+        }
+    }
+    // Optional: assertions
+    const assertions = data.assertions?.map((a, i) => validateAssertion(a, i));
+    // Merge global tags with scenario tags
+    const tags = [...globalTags, ...(data.tags ?? [])];
+    return {
+        prompt: data.prompt,
+        description: data.description ?? `Test prompt ${data.prompt}`,
+        args,
+        assertions,
+        skip: data.skip ?? false,
+        tags: tags.length > 0 ? tags : undefined,
+    };
+}
+/**
+ * Validate and normalize an assertion.
+ */
+function validateAssertion(data, index) {
+    // Required: path
+    if (!data.path || typeof data.path !== 'string') {
+        throw new Error(`assertion ${index + 1}: missing required field "path"`);
+    }
+    // Required: condition
+    if (!data.condition || typeof data.condition !== 'string') {
+        throw new Error(`assertion ${index + 1}: missing required field "condition"`);
+    }
+    if (!VALID_CONDITIONS.includes(data.condition)) {
+        throw new Error(`assertion ${index + 1}: invalid condition "${data.condition}". ` +
+            `Valid conditions: ${VALID_CONDITIONS.join(', ')}`);
+    }
+    // Validate that 'value' is provided for conditions that require it
+    const conditionsRequiringValue = ['equals', 'contains', 'type'];
+    if (conditionsRequiringValue.includes(data.condition) && data.value === undefined) {
+        throw new Error(`assertion ${index + 1}: condition "${data.condition}" requires a "value" field`);
+    }
+    return {
+        path: data.path,
+        condition: data.condition,
+        value: data.value,
+        message: data.message,
+    };
+}
+/**
+ * Generate a sample YAML template for test scenarios.
+ */
+export function generateSampleScenariosYaml() {
+    return `# Bellwether Test Scenarios
+# Save as: bellwether-tests.yaml in your project root
+# Docs: https://docs.bellwether.sh/guides/custom-scenarios
+
+version: "1"
+description: Custom test scenarios for my MCP server
+
+# Global tags applied to all scenarios (optional)
+tags:
+  - custom
+
+# Tool test scenarios
+scenarios:
+  # Happy path test
+  - tool: read_file
+    description: Read a valid file
+    category: happy_path
+    args:
+      path: "/tmp/test.txt"
+    assertions:
+      - path: content
+        condition: exists
+        message: File content should be returned
+
+  # Edge case test
+  - tool: read_file
+    description: Read file with special characters in name
+    category: edge_case
+    args:
+      path: "/tmp/file with spaces.txt"
+    assertions:
+      - path: content
+        condition: exists
+
+  # Error handling test
+  - tool: read_file
+    description: Handle missing file gracefully
+    category: error_handling
+    args:
+      path: "/nonexistent/file.txt"
+    assertions:
+      - path: error
+        condition: exists
+        message: Should return error for missing file
+
+  # Security test
+  - tool: read_file
+    description: Reject path traversal attempt
+    category: security
+    args:
+      path: "../../etc/passwd"
+    tags:
+      - security
+      - critical
+
+  # Skip a test (won't run)
+  - tool: dangerous_operation
+    description: Skipped test
+    skip: true
+    args:
+      action: delete_all
+
+# Prompt test scenarios
+prompts:
+  - prompt: summarize
+    description: Test summarize prompt with sample text
+    args:
+      text: "This is a long document that needs to be summarized..."
+    assertions:
+      - path: messages
+        condition: exists
+      - path: messages[0].content
+        condition: truthy
+
+  - prompt: translate
+    description: Test translation prompt
+    args:
+      text: "Hello, world!"
+      language: "Spanish"
+    assertions:
+      - path: messages[0].content.text
+        condition: contains
+        value: "Hola"
+
+# Assertion conditions:
+# - exists: Path exists (value is not undefined)
+# - equals: Value equals expected
+# - contains: String/array contains value
+# - truthy: Value is truthy
+# - type: Value is of type (string, number, boolean, object, array)
+# - not_error: Response is not an error
+
+# Categories:
+# - happy_path: Normal usage
+# - edge_case: Boundary conditions
+# - error_handling: Invalid inputs
+# - boundary: Limits and extremes
+# - security: Security-related tests
+`;
+}
+//# sourceMappingURL=loader.js.map

package/dist/scenarios/types.d.ts

@@ -0,0 +1,153 @@
+/**
+ * Types for custom YAML-defined test scenarios.
+ *
+ * These types define the schema for user-defined test cases
+ * that can be provided via bellwether-tests.yaml files.
+ */
+import type { QuestionCategory } from '../persona/types.js';
+/**
+ * Valid assertion conditions for scenario expectations.
+ */
+export type AssertionCondition = 'exists' | 'equals' | 'contains' | 'truthy' | 'type' | 'not_error';
+/**
+ * An assertion/expectation for a test scenario.
+ */
+export interface ScenarioAssertion {
+    /** JSONPath to the value to check */
+    path: string;
+    /** Condition to evaluate */
+    condition: AssertionCondition;
+    /** Expected value (for equals, contains, type) */
+    value?: unknown;
+    /** Custom error message on failure */
+    message?: string;
+}
+/**
+ * A single test scenario for a tool.
+ */
+export interface TestScenario {
+    /** Tool to test */
+    tool: string;
+    /** Description of what this test verifies */
+    description: string;
+    /** Category of test (default: happy_path) */
+    category: QuestionCategory;
+    /** Arguments to pass to the tool */
+    args: Record<string, unknown>;
+    /** Assertions to verify after execution */
+    assertions?: ScenarioAssertion[];
+    /** Whether this scenario should be skipped */
+    skip?: boolean;
+    /** Tags for filtering scenarios */
+    tags?: string[];
+}
+/**
+ * A single test scenario for a prompt.
+ */
+export interface PromptScenario {
+    /** Prompt to test */
+    prompt: string;
+    /** Description of what this test verifies */
+    description: string;
+    /** Arguments to pass to the prompt */
+    args: Record<string, string>;
+    /** Assertions to verify on the rendered output */
+    assertions?: ScenarioAssertion[];
+    /** Whether this scenario should be skipped */
+    skip?: boolean;
+    /** Tags for filtering scenarios */
+    tags?: string[];
+}
+/**
+ * YAML file structure for test scenarios.
+ */
+export interface TestScenariosYAML {
+    /** Version of the schema (for future compatibility) */
+    version?: string;
+    /** Description of this test file */
+    description?: string;
+    /** Tool test scenarios */
+    scenarios?: TestScenarioYAML[];
+    /** Prompt test scenarios */
+    prompts?: PromptScenarioYAML[];
+    /** Global tags applied to all scenarios */
+    tags?: string[];
+}
+/**
+ * YAML representation of a test scenario (looser types for parsing).
+ */
+export interface TestScenarioYAML {
+    tool: string;
+    description?: string;
+    category?: string;
+    args?: Record<string, unknown>;
+    assertions?: ScenarioAssertionYAML[];
+    skip?: boolean;
+    tags?: string[];
+}
+/**
+ * YAML representation of a prompt scenario.
+ */
+export interface PromptScenarioYAML {
+    prompt: string;
+    description?: string;
+    args?: Record<string, string>;
+    assertions?: ScenarioAssertionYAML[];
+    skip?: boolean;
+    tags?: string[];
+}
+/**
+ * YAML representation of an assertion.
+ */
+export interface ScenarioAssertionYAML {
+    path?: string;
+    condition?: string;
+    value?: unknown;
+    message?: string;
+}
+/**
+ * Loaded test scenarios file.
+ */
+export interface LoadedScenarios {
+    /** Source file path */
+    source: string;
+    /** Tool test scenarios */
+    toolScenarios: TestScenario[];
+    /** Prompt test scenarios */
+    promptScenarios: PromptScenario[];
+    /** File description */
+    description?: string;
+    /** Schema version */
+    version?: string;
+}
+/**
+ * Result of running a scenario assertion.
+ */
+export interface AssertionResult {
+    /** The assertion that was checked */
+    assertion: ScenarioAssertion;
+    /** Whether the assertion passed */
+    passed: boolean;
+    /** Actual value found */
+    actualValue?: unknown;
+    /** Error message if failed */
+    error?: string;
+}
+/**
+ * Result of running a single test scenario.
+ */
+export interface ScenarioResult {
+    /** The scenario that was run */
+    scenario: TestScenario | PromptScenario;
+    /** Whether the scenario passed (all assertions passed and no error) */
+    passed: boolean;
+    /** Assertion results */
+    assertionResults: AssertionResult[];
+    /** Error if execution failed */
+    error?: string;
+    /** Response from the tool/prompt */
+    response?: unknown;
+    /** Execution duration in ms */
+    durationMs: number;
+}
+//# sourceMappingURL=types.d.ts.map

package/dist/scenarios/types.js

@@ -0,0 +1,8 @@
+/**
+ * Types for custom YAML-defined test scenarios.
+ *
+ * These types define the schema for user-defined test cases
+ * that can be provided via bellwether-tests.yaml files.
+ */
+export {};
+//# sourceMappingURL=types.js.map

package/dist/security/index.d.ts

@@ -0,0 +1,17 @@
+/**
+ * Security testing module for bellwether check mode.
+ *
+ * This module provides deterministic security testing capabilities that can
+ * detect common vulnerability patterns in MCP tools without requiring LLM.
+ *
+ * Usage:
+ *   bellwether check --security           # Run security tests with default categories
+ *   bellwether check --security --security-categories sql_injection,xss
+ *
+ * The security baseline is stored in the baseline file and can be compared
+ * across runs to detect security posture changes.
+ */
+export type { SecurityCategory, RiskLevel, SecurityPayload, SecurityTestResult, SecurityFinding, SecurityFingerprint, SecurityDiff, SecurityTestOptions, SecurityTestContext, SecurityToolCallResult, SecurityReport, } from './types.js';
+export { SQL_INJECTION_PAYLOADS, XSS_PAYLOADS, PATH_TRAVERSAL_PAYLOADS, COMMAND_INJECTION_PAYLOADS, SSRF_PAYLOADS, getPayloadsForCategory, getAllSecurityPayloads, getAllSecurityCategories, } from './payloads.js';
+export { runSecurityTests, compareSecurityFingerprints, getRiskLevelFromScore, parseSecurityCategories, } from './security-tester.js';
+//# sourceMappingURL=index.d.ts.map

package/dist/security/index.js

@@ -0,0 +1,18 @@
+/**
+ * Security testing module for bellwether check mode.
+ *
+ * This module provides deterministic security testing capabilities that can
+ * detect common vulnerability patterns in MCP tools without requiring LLM.
+ *
+ * Usage:
+ *   bellwether check --security           # Run security tests with default categories
+ *   bellwether check --security --security-categories sql_injection,xss
+ *
+ * The security baseline is stored in the baseline file and can be compared
+ * across runs to detect security posture changes.
+ */
+// Payload exports
+export { SQL_INJECTION_PAYLOADS, XSS_PAYLOADS, PATH_TRAVERSAL_PAYLOADS, COMMAND_INJECTION_PAYLOADS, SSRF_PAYLOADS, getPayloadsForCategory, getAllSecurityPayloads, getAllSecurityCategories, } from './payloads.js';
+// Security tester exports
+export { runSecurityTests, compareSecurityFingerprints, getRiskLevelFromScore, parseSecurityCategories, } from './security-tester.js';
+//# sourceMappingURL=index.js.map

package/dist/security/payloads.d.ts

@@ -0,0 +1,61 @@
+/**
+ * Deterministic security test payloads.
+ *
+ * These are well-known test patterns used for security testing that don't
+ * require LLM generation. Each payload is designed to test a specific
+ * vulnerability class.
+ *
+ * IMPORTANT: These payloads are for defensive security testing only.
+ * They help developers identify input validation gaps in their MCP servers.
+ */
+import type { SecurityPayload, SecurityCategory } from './types.js';
+/**
+ * SQL injection test payloads.
+ * Tests for improper handling of SQL metacharacters in user input.
+ * CWE-89: Improper Neutralization of Special Elements used in an SQL Command
+ */
+export declare const SQL_INJECTION_PAYLOADS: SecurityPayload[];
+/**
+ * Cross-Site Scripting (XSS) test payloads.
+ * Tests for improper handling of HTML/JavaScript in user input.
+ * CWE-79: Improper Neutralization of Input During Web Page Generation
+ */
+export declare const XSS_PAYLOADS: SecurityPayload[];
+/**
+ * Path traversal test payloads.
+ * Tests for improper handling of file path inputs.
+ * CWE-22: Improper Limitation of a Pathname to a Restricted Directory
+ */
+export declare const PATH_TRAVERSAL_PAYLOADS: SecurityPayload[];
+/**
+ * Command injection test payloads.
+ * Tests for improper handling of shell metacharacters.
+ * CWE-78: Improper Neutralization of Special Elements used in an OS Command
+ */
+export declare const COMMAND_INJECTION_PAYLOADS: SecurityPayload[];
+/**
+ * Server-Side Request Forgery (SSRF) test payloads.
+ * Tests for improper URL validation allowing internal network access.
+ * CWE-918: Server-Side Request Forgery (SSRF)
+ */
+export declare const SSRF_PAYLOADS: SecurityPayload[];
+/**
+ * Get payloads for a specific security category.
+ *
+ * @param category - The security category to get payloads for
+ * @returns Array of payloads for that category
+ */
+export declare function getPayloadsForCategory(category: SecurityCategory): SecurityPayload[];
+/**
+ * Get all security payloads across all categories.
+ *
+ * @returns Array of all security payloads
+ */
+export declare function getAllSecurityPayloads(): SecurityPayload[];
+/**
+ * Get all available security categories.
+ *
+ * @returns Array of all security category identifiers
+ */
+export declare function getAllSecurityCategories(): SecurityCategory[];
+//# sourceMappingURL=payloads.d.ts.map