@dotsetlabs/bellwether 0.10.0

package/dist/utils/timeout.js

@@ -0,0 +1,205 @@
+/**
+ * Centralized timeout management utilities.
+ *
+ * Provides consistent timeout handling across all async operations
+ * to prevent indefinite hangs and improve reliability.
+ */
+import { getLogger } from '../logging/logger.js';
+import { TIMEOUTS, INTERVIEW, WORKFLOW } from '../constants.js';
+const logger = getLogger('timeout');
+/**
+ * Default timeout values in milliseconds.
+ *
+ * These reference the centralized constants where applicable,
+ * ensuring consistency across the codebase.
+ */
+export const DEFAULT_TIMEOUTS = {
+    /** Timeout for individual tool calls - uses INTERVIEW.TOOL_TIMEOUT */
+    toolCall: INTERVIEW.TOOL_TIMEOUT,
+    /** Timeout for LLM API calls - uses TIMEOUTS.INTERVIEW */
+    llmCall: TIMEOUTS.INTERVIEW,
+    /** Timeout for state snapshots - uses WORKFLOW.STATE_SNAPSHOT_TIMEOUT */
+    stateSnapshot: WORKFLOW.STATE_SNAPSHOT_TIMEOUT,
+    /** Timeout for individual probe tool calls - uses WORKFLOW.PROBE_TOOL_TIMEOUT */
+    probeTool: WORKFLOW.PROBE_TOOL_TIMEOUT,
+    /** Timeout for resource reads - uses INTERVIEW.RESOURCE_TIMEOUT */
+    resourceRead: INTERVIEW.RESOURCE_TIMEOUT,
+    /** Timeout for HTTP requests - uses TIMEOUTS.CLOUD_API */
+    httpRequest: TIMEOUTS.CLOUD_API,
+    /** Timeout for SSE connection establishment */
+    sseConnect: 10000,
+    /** Timeout for interview question generation (longer for local models) */
+    questionGeneration: 120000,
+    /** Timeout for response analysis */
+    responseAnalysis: 60000,
+    /** Timeout for profile synthesis */
+    profileSynthesis: 120000,
+};
+/**
+ * Error class for timeout errors.
+ */
+export class TimeoutError extends Error {
+    operationName;
+    timeoutMs;
+    constructor(operationName, timeoutMs, message) {
+        super(message ?? `${operationName} timed out after ${timeoutMs}ms`);
+        this.name = 'TimeoutError';
+        this.operationName = operationName;
+        this.timeoutMs = timeoutMs;
+    }
+}
+/**
+ * Error class for aborted operations.
+ */
+export class AbortError extends Error {
+    operationName;
+    constructor(operationName, message) {
+        super(message ?? `${operationName} was aborted`);
+        this.name = 'AbortError';
+        this.operationName = operationName;
+    }
+}
+/**
+ * Check if an AbortSignal is aborted and throw AbortError if so.
+ *
+ * @param signal - The AbortSignal to check
+ * @param operationName - Name of the operation for error messages
+ * @throws AbortError if the signal is aborted
+ */
+export function checkAborted(signal, operationName) {
+    if (signal?.aborted) {
+        throw new AbortError(operationName);
+    }
+}
+/**
+ * Wrap a promise with a timeout.
+ *
+ * @param promise - The promise to wrap
+ * @param timeoutMs - Timeout in milliseconds
+ * @param operationName - Name of the operation for error messages
+ * @returns The promise result or throws TimeoutError
+ */
+export async function withTimeout(promise, timeoutMs, operationName) {
+    let timeoutId;
+    const timeoutPromise = new Promise((_, reject) => {
+        timeoutId = setTimeout(() => {
+            logger.warn({ operationName, timeoutMs }, 'Operation timed out');
+            reject(new TimeoutError(operationName, timeoutMs));
+        }, timeoutMs);
+    });
+    try {
+        return await Promise.race([promise, timeoutPromise]);
+    }
+    finally {
+        clearTimeout(timeoutId);
+    }
+}
+/**
+ * Wrap a promise with a timeout and return a result object instead of throwing.
+ *
+ * @param promise - The promise to wrap
+ * @param timeoutMs - Timeout in milliseconds
+ * @param operationName - Name of the operation
+ * @returns Object with either result or error
+ */
+export async function withTimeoutResult(promise, timeoutMs, operationName) {
+    try {
+        const result = await withTimeout(promise, timeoutMs, operationName);
+        return { success: true, result };
+    }
+    catch (error) {
+        return {
+            success: false,
+            error: error instanceof Error ? error : new Error(String(error)),
+        };
+    }
+}
+/**
+ * Create an abort controller that automatically aborts after a timeout.
+ *
+ * @param timeoutMs - Timeout in milliseconds
+ * @param operationName - Name of the operation
+ * @returns AbortController and cleanup function
+ */
+export function createTimeoutAbortController(timeoutMs, operationName) {
+    const controller = new AbortController();
+    const timeoutId = setTimeout(() => {
+        logger.debug({ operationName, timeoutMs }, 'Aborting due to timeout');
+        controller.abort(new TimeoutError(operationName, timeoutMs));
+    }, timeoutMs);
+    return {
+        controller,
+        cleanup: () => clearTimeout(timeoutId),
+    };
+}
+/**
+ * Execute multiple promises with individual timeouts.
+ * Returns results for all, including those that timed out.
+ *
+ * @param operations - Array of operations with their timeouts
+ * @returns Array of results (either success with value or failure with error)
+ */
+export async function withTimeoutAll(operations) {
+    return Promise.all(operations.map(async ({ promise, timeoutMs, operationName }) => {
+        try {
+            const result = await withTimeout(promise, timeoutMs, operationName);
+            return { success: true, result };
+        }
+        catch (error) {
+            return {
+                success: false,
+                error: error instanceof Error ? error : new Error(String(error)),
+                operationName,
+            };
+        }
+    }));
+}
+/**
+ * Execute a function with a timeout, retrying on timeout up to maxRetries.
+ *
+ * @param fn - Function to execute
+ * @param timeoutMs - Timeout per attempt
+ * @param operationName - Name of the operation
+ * @param maxRetries - Maximum number of retries (default: 1)
+ * @returns The function result
+ */
+export async function withTimeoutRetry(fn, timeoutMs, operationName, maxRetries = 1) {
+    let lastError;
+    for (let attempt = 0; attempt <= maxRetries; attempt++) {
+        try {
+            return await withTimeout(fn(), timeoutMs, operationName);
+        }
+        catch (error) {
+            lastError = error instanceof Error ? error : new Error(String(error));
+            if (error instanceof TimeoutError && attempt < maxRetries) {
+                logger.debug({ operationName, attempt: attempt + 1, maxRetries }, 'Retrying after timeout');
+                continue;
+            }
+            throw lastError;
+        }
+    }
+    throw lastError ?? new Error('Unexpected retry loop exit');
+}
+/**
+ * Create a deadline-based timeout manager.
+ * Useful for operations with multiple steps that should complete within a total time.
+ *
+ * @param totalTimeoutMs - Total time allowed for all operations
+ * @param operationName - Name of the overall operation
+ * @returns Deadline manager
+ */
+export function createDeadline(totalTimeoutMs, operationName) {
+    const startTime = Date.now();
+    const deadline = startTime + totalTimeoutMs;
+    return {
+        getRemainingMs: () => Math.max(0, deadline - Date.now()),
+        isExpired: () => Date.now() >= deadline,
+        getTimeoutFor: (maxMs) => Math.min(maxMs, Math.max(0, deadline - Date.now())),
+        checkDeadline: () => {
+            if (Date.now() >= deadline) {
+                throw new TimeoutError(operationName, totalTimeoutMs, `${operationName} exceeded total deadline of ${totalTimeoutMs}ms`);
+            }
+        },
+    };
+}
+//# sourceMappingURL=timeout.js.map

package/dist/utils/yaml-parser.d.ts

@@ -0,0 +1,58 @@
+/**
+ * Secure YAML parsing utility.
+ *
+ * Provides safe YAML parsing with protection against:
+ * - Alias bombs (billion laughs attack) via maxAliasCount
+ * - Excessive nesting via depth validation
+ * - Circular references
+ */
+import { type ParseOptions } from 'yaml';
+/**
+ * Default security limits for YAML parsing.
+ */
+export declare const YAML_SECURITY_LIMITS: {
+    /** Maximum number of aliases to resolve (prevents billion laughs attack) */
+    MAX_ALIAS_COUNT: number;
+    /** Maximum nesting depth for parsed structures */
+    MAX_DEPTH: number;
+    /** Maximum size of input in characters */
+    MAX_INPUT_SIZE: number;
+};
+/**
+ * Options for secure YAML parsing.
+ */
+export interface SecureYamlOptions {
+    /** Maximum number of aliases to resolve (default: 100) */
+    maxAliasCount?: number;
+    /** Maximum nesting depth (default: 50) */
+    maxDepth?: number;
+    /** Maximum input size in characters (default: 10MB) */
+    maxInputSize?: number;
+    /** Additional yaml parse options */
+    parseOptions?: ParseOptions;
+}
+/**
+ * Parse YAML with security protections.
+ *
+ * @param content - YAML content to parse
+ * @param options - Security options
+ * @returns Parsed YAML value
+ * @throws Error if parsing fails or security limits are exceeded
+ *
+ * @example
+ * ```typescript
+ * const data = parseYamlSecure(content);
+ * const config = parseYamlSecure(content, { maxDepth: 10 });
+ * ```
+ */
+export declare function parseYamlSecure<T = unknown>(content: string, options?: SecureYamlOptions): T;
+/**
+ * Parse YAML with strict security settings.
+ * Use this for parsing untrusted input.
+ *
+ * @param content - YAML content to parse
+ * @returns Parsed YAML value
+ * @throws Error if parsing fails or security limits are exceeded
+ */
+export declare function parseYamlStrict<T = unknown>(content: string): T;
+//# sourceMappingURL=yaml-parser.d.ts.map

package/dist/utils/yaml-parser.js

@@ -0,0 +1,86 @@
+/**
+ * Secure YAML parsing utility.
+ *
+ * Provides safe YAML parsing with protection against:
+ * - Alias bombs (billion laughs attack) via maxAliasCount
+ * - Excessive nesting via depth validation
+ * - Circular references
+ */
+import { parse as yamlParse } from 'yaml';
+/**
+ * Default security limits for YAML parsing.
+ */
+export const YAML_SECURITY_LIMITS = {
+    /** Maximum number of aliases to resolve (prevents billion laughs attack) */
+    MAX_ALIAS_COUNT: 100,
+    /** Maximum nesting depth for parsed structures */
+    MAX_DEPTH: 50,
+    /** Maximum size of input in characters */
+    MAX_INPUT_SIZE: 10 * 1024 * 1024, // 10MB
+};
+/**
+ * Check the depth of a nested structure.
+ * Throws if depth exceeds the limit.
+ */
+function validateDepth(value, maxDepth, currentDepth = 0) {
+    if (currentDepth > maxDepth) {
+        throw new Error(`YAML nesting depth exceeds maximum of ${maxDepth}`);
+    }
+    if (Array.isArray(value)) {
+        for (const item of value) {
+            validateDepth(item, maxDepth, currentDepth + 1);
+        }
+    }
+    else if (value !== null && typeof value === 'object') {
+        for (const key of Object.keys(value)) {
+            validateDepth(value[key], maxDepth, currentDepth + 1);
+        }
+    }
+}
+/**
+ * Parse YAML with security protections.
+ *
+ * @param content - YAML content to parse
+ * @param options - Security options
+ * @returns Parsed YAML value
+ * @throws Error if parsing fails or security limits are exceeded
+ *
+ * @example
+ * ```typescript
+ * const data = parseYamlSecure(content);
+ * const config = parseYamlSecure(content, { maxDepth: 10 });
+ * ```
+ */
+export function parseYamlSecure(content, options) {
+    const maxAliasCount = options?.maxAliasCount ?? YAML_SECURITY_LIMITS.MAX_ALIAS_COUNT;
+    const maxDepth = options?.maxDepth ?? YAML_SECURITY_LIMITS.MAX_DEPTH;
+    const maxInputSize = options?.maxInputSize ?? YAML_SECURITY_LIMITS.MAX_INPUT_SIZE;
+    // Check input size
+    if (content.length > maxInputSize) {
+        throw new Error(`YAML input size (${content.length} bytes) exceeds maximum of ${maxInputSize} bytes`);
+    }
+    // Parse with alias protection
+    const parsed = yamlParse(content, {
+        maxAliasCount,
+        ...options?.parseOptions,
+    });
+    // Validate nesting depth
+    validateDepth(parsed, maxDepth);
+    return parsed;
+}
+/**
+ * Parse YAML with strict security settings.
+ * Use this for parsing untrusted input.
+ *
+ * @param content - YAML content to parse
+ * @returns Parsed YAML value
+ * @throws Error if parsing fails or security limits are exceeded
+ */
+export function parseYamlStrict(content) {
+    return parseYamlSecure(content, {
+        maxAliasCount: 10,
+        maxDepth: 20,
+        maxInputSize: 1024 * 1024, // 1MB
+    });
+}
+//# sourceMappingURL=yaml-parser.js.map

package/dist/validation/index.d.ts

@@ -0,0 +1,32 @@
+/**
+ * Semantic validation module for Bellwether.
+ *
+ * This module provides semantic type inference and validation for MCP tool
+ * parameters, enabling more intelligent testing of input validation.
+ *
+ * @example
+ * ```typescript
+ * import {
+ *   inferSemanticType,
+ *   validateSemanticValue,
+ *   generateSemanticTests,
+ * } from './validation/index.js';
+ *
+ * // Infer semantic type from parameter name and description
+ * const inference = inferSemanticType('email', 'User email address');
+ * // { paramName: 'email', inferredType: 'email', confidence: 0.9, ... }
+ *
+ * // Validate a value against the inferred type
+ * const result = validateSemanticValue('email', 'invalid', 'email');
+ * // { isValid: false, issue: 'Invalid email format: invalid' }
+ *
+ * // Generate semantic validation tests for a tool
+ * const { tests, inferences } = generateSemanticTests(tool);
+ * ```
+ */
+export type { SemanticType, SemanticInference, SemanticValidationResult, SemanticPatternDefinition, } from './semantic-types.js';
+export { SEMANTIC_PATTERNS, getAllSemanticTypes, isSemanticType, } from './semantic-types.js';
+export { inferSemanticType, validateSemanticValue, validateAllParameters, } from './semantic-validator.js';
+export type { SemanticTestResult, SemanticTestOptions, } from './semantic-test-generator.js';
+export { generateSemanticTests, getInvalidValuesForType, getTestableSemanticTypes, } from './semantic-test-generator.js';
+//# sourceMappingURL=index.d.ts.map

package/dist/validation/index.js

@@ -0,0 +1,32 @@
+/**
+ * Semantic validation module for Bellwether.
+ *
+ * This module provides semantic type inference and validation for MCP tool
+ * parameters, enabling more intelligent testing of input validation.
+ *
+ * @example
+ * ```typescript
+ * import {
+ *   inferSemanticType,
+ *   validateSemanticValue,
+ *   generateSemanticTests,
+ * } from './validation/index.js';
+ *
+ * // Infer semantic type from parameter name and description
+ * const inference = inferSemanticType('email', 'User email address');
+ * // { paramName: 'email', inferredType: 'email', confidence: 0.9, ... }
+ *
+ * // Validate a value against the inferred type
+ * const result = validateSemanticValue('email', 'invalid', 'email');
+ * // { isValid: false, issue: 'Invalid email format: invalid' }
+ *
+ * // Generate semantic validation tests for a tool
+ * const { tests, inferences } = generateSemanticTests(tool);
+ * ```
+ */
+// Constants and utilities
+export { SEMANTIC_PATTERNS, getAllSemanticTypes, isSemanticType, } from './semantic-types.js';
+// Validation functions
+export { inferSemanticType, validateSemanticValue, validateAllParameters, } from './semantic-validator.js';
+export { generateSemanticTests, getInvalidValuesForType, getTestableSemanticTypes, } from './semantic-test-generator.js';
+//# sourceMappingURL=index.js.map

package/dist/validation/semantic-test-generator.d.ts

@@ -0,0 +1,50 @@
+/**
+ * Generate test cases based on semantic type inference.
+ *
+ * This module generates validation tests for parameters with inferred
+ * semantic types, testing that tools properly reject invalid values.
+ */
+import type { MCPTool } from '../transport/types.js';
+import type { InterviewQuestion } from '../interview/types.js';
+import type { SemanticType, SemanticInference } from './semantic-types.js';
+/**
+ * Result of generating semantic tests for a tool.
+ */
+export interface SemanticTestResult {
+    /** Generated test cases */
+    tests: InterviewQuestion[];
+    /** Semantic inferences for each parameter */
+    inferences: SemanticInference[];
+}
+/**
+ * Options for semantic test generation.
+ */
+export interface SemanticTestOptions {
+    /** Minimum confidence threshold for generating tests (0-1, default: 0.5) */
+    minConfidence?: number;
+    /** Maximum invalid values to test per parameter (default: 2) */
+    maxInvalidValuesPerParam?: number;
+    /** Skip semantic tests entirely */
+    skipSemanticTests?: boolean;
+}
+/**
+ * Generate semantic validation tests for a tool.
+ *
+ * Analyzes the tool's input schema, infers semantic types for parameters,
+ * and generates test cases with invalid values to verify proper validation.
+ *
+ * @param tool - The MCP tool to generate tests for
+ * @param options - Configuration options
+ * @returns Generated tests and semantic inferences
+ */
+export declare function generateSemanticTests(tool: MCPTool, options?: SemanticTestOptions): SemanticTestResult;
+/**
+ * Get the invalid test values for a semantic type.
+ * Useful for external modules that need access to the test values.
+ */
+export declare function getInvalidValuesForType(type: SemanticType): readonly string[];
+/**
+ * Get all semantic types that have invalid test values defined.
+ */
+export declare function getTestableSemanticTypes(): SemanticType[];
+//# sourceMappingURL=semantic-test-generator.d.ts.map

package/dist/validation/semantic-test-generator.js

@@ -0,0 +1,176 @@
+/**
+ * Generate test cases based on semantic type inference.
+ *
+ * This module generates validation tests for parameters with inferred
+ * semantic types, testing that tools properly reject invalid values.
+ */
+import { inferSemanticType } from './semantic-validator.js';
+import { SEMANTIC_VALIDATION } from '../constants.js';
+/**
+ * Invalid test values for each semantic type.
+ * These values should be rejected by tools that properly validate input.
+ */
+const INVALID_VALUES = {
+    date_iso8601: ['not-a-date', '2024/01/15', '01-15-2024', '2024-13-45', ''],
+    date_month: ['not-a-month', '2024/01', '01-2024', '2024-13', ''],
+    datetime: ['not-a-datetime', '2024-01-15 10:30', 'tomorrow', 'now', ''],
+    timestamp: ['not-a-timestamp', '-12345', 'abc', '1.5.3', ''],
+    amount_currency: ['not-an-amount', 'free', 'hundred dollars', 'N/A', ''],
+    percentage: ['not-a-percentage', 'half', 'none', 'all', ''],
+    identifier: ['', '   ', '\n\t', '\x00'],
+    email: ['not-an-email', 'missing@domain', '@nodomain.com', 'no-at-sign', ''],
+    url: ['not-a-url', 'missing-protocol.com', '://no-scheme', 'http://', ''],
+    phone: ['not-a-phone', 'abc', '123', 'call-me', ''],
+    ip_address: ['not-an-ip', '999.999.999.999', 'localhost', '1.2.3.4.5', ''],
+    file_path: [], // Don't test - too OS-specific
+    json: ['not-json', '{invalid}', '{"missing": }', '{key: "no-quotes"}', ''],
+    base64: ['not-base64!!!', '====', 'has spaces', '@#$%', ''],
+    regex: ['[invalid', '(unclosed', '*invalid', '\\', ''],
+    unknown: [],
+};
+/**
+ * Generate semantic validation tests for a tool.
+ *
+ * Analyzes the tool's input schema, infers semantic types for parameters,
+ * and generates test cases with invalid values to verify proper validation.
+ *
+ * @param tool - The MCP tool to generate tests for
+ * @param options - Configuration options
+ * @returns Generated tests and semantic inferences
+ */
+export function generateSemanticTests(tool, options = {}) {
+    const { minConfidence = SEMANTIC_VALIDATION.MIN_CONFIDENCE_THRESHOLD, maxInvalidValuesPerParam = SEMANTIC_VALIDATION.MAX_INVALID_VALUES_PER_PARAM, skipSemanticTests = false, } = options;
+    const tests = [];
+    const inferences = [];
+    if (skipSemanticTests) {
+        return { tests, inferences };
+    }
+    const schema = tool.inputSchema;
+    if (!schema?.properties) {
+        return { tests, inferences };
+    }
+    const requiredParams = schema.required ?? [];
+    for (const [paramName, propSchema] of Object.entries(schema.properties)) {
+        // Only infer semantic types for string parameters
+        if (propSchema.type !== 'string' && propSchema.type !== undefined) {
+            continue;
+        }
+        // Infer semantic type
+        const inference = inferSemanticType(paramName, propSchema.description, propSchema.format);
+        // Only generate tests for high-confidence inferences
+        if (inference.confidence < minConfidence || inference.inferredType === 'unknown') {
+            continue;
+        }
+        inferences.push(inference);
+        // Get invalid values for this semantic type
+        const invalidValues = INVALID_VALUES[inference.inferredType];
+        if (invalidValues.length === 0)
+            continue;
+        // Build base args with required params
+        const baseArgs = {};
+        for (const req of requiredParams) {
+            if (req !== paramName) {
+                // Provide a valid placeholder for other required params
+                baseArgs[req] = generatePlaceholderValue(req, schema.properties[req]);
+            }
+        }
+        // Generate tests with invalid semantic values
+        const valuesToTest = invalidValues.slice(0, maxInvalidValuesPerParam);
+        for (const invalidValue of valuesToTest) {
+            tests.push({
+                description: `Semantic validation: invalid ${formatSemanticType(inference.inferredType)} for "${paramName}"`,
+                category: 'error_handling',
+                args: {
+                    ...baseArgs,
+                    [paramName]: invalidValue,
+                },
+                metadata: {
+                    semanticType: inference.inferredType,
+                    expectedBehavior: 'reject',
+                    confidence: inference.confidence,
+                },
+            });
+        }
+    }
+    return { tests, inferences };
+}
+/**
+ * Generate a placeholder value for a required parameter.
+ * Used to provide valid values for other params when testing one param.
+ */
+function generatePlaceholderValue(paramName, propSchema) {
+    if (!propSchema)
+        return 'test';
+    const type = propSchema.type;
+    switch (type) {
+        case 'string':
+            return generateStringPlaceholder(paramName, propSchema);
+        case 'number':
+        case 'integer':
+            return 1;
+        case 'boolean':
+            return true;
+        case 'array':
+            return [];
+        case 'object':
+            return {};
+        default:
+            return 'test';
+    }
+}
+/**
+ * Generate a placeholder string value based on parameter name and format.
+ */
+function generateStringPlaceholder(paramName, propSchema) {
+    // Use format hint if available
+    if (propSchema.format === 'date')
+        return '2024-01-15';
+    if (propSchema.format === 'date-time')
+        return '2024-01-15T12:00:00Z';
+    if (propSchema.format === 'email')
+        return 'test@example.com';
+    if (propSchema.format === 'uri' || propSchema.format === 'url')
+        return 'https://example.com';
+    if (propSchema.format === 'uuid')
+        return '550e8400-e29b-41d4-a716-446655440000';
+    // Infer from name
+    const lowerName = paramName.toLowerCase();
+    if (lowerName.includes('date'))
+        return '2024-01-15';
+    if (lowerName.includes('email'))
+        return 'test@example.com';
+    if (lowerName.includes('url') || lowerName.includes('uri'))
+        return 'https://example.com';
+    if (lowerName.includes('id'))
+        return 'test-id-123';
+    if (lowerName.includes('path') || lowerName.includes('file'))
+        return '/tmp/test';
+    if (lowerName.includes('phone'))
+        return '+1234567890';
+    return 'test';
+}
+/**
+ * Format semantic type for display in test descriptions.
+ */
+function formatSemanticType(type) {
+    return type
+        .split('_')
+        .map((word) => word.charAt(0).toUpperCase() + word.slice(1))
+        .join(' ');
+}
+/**
+ * Get the invalid test values for a semantic type.
+ * Useful for external modules that need access to the test values.
+ */
+export function getInvalidValuesForType(type) {
+    return INVALID_VALUES[type] ?? [];
+}
+/**
+ * Get all semantic types that have invalid test values defined.
+ */
+export function getTestableSemanticTypes() {
+    return Object.entries(INVALID_VALUES)
+        .filter(([, values]) => values.length > 0)
+        .map(([type]) => type);
+}
+//# sourceMappingURL=semantic-test-generator.js.map