@syrin/cli 1.3.1 → 1.4.0

package/dist/runtime/test/input-generator.js

@@ -1,498 +0,0 @@
-/**
- * Synthetic input generator.
- * Generates test inputs from JSON Schema definitions.
- */
-/**
- * Generate test inputs from a JSON Schema.
- * Includes both predictable patterns and agent-like unpredictable variations.
- * @param schema - JSON Schema definition
- * @returns Array of generated input objects
- */
-export function generateInputsFromSchema(schema) {
-    const inputs = [];
-    // 1. Predictable patterns (for correctness testing)
-    // Generate minimal valid input (only required fields)
-    inputs.push(generateMinimalInput(schema));
-    // Generate input with defaults (optional fields with default values)
-    inputs.push(generateWithDefaults(schema));
-    // Generate edge cases
-    inputs.push(...generateEdgeCases(schema));
-    // 2. Agent-like unpredictable patterns (for agent safety testing)
-    // Generate random variations (agent-like inputs)
-    inputs.push(...generateAgentLikeInputs(schema));
-    // Generate invalid-but-close inputs (type coercion attempts)
-    inputs.push(...generateInvalidButCloseInputs(schema));
-    // Generate extreme values (very long strings, very large numbers)
-    inputs.push(...generateExtremeValueInputs(schema));
-    // Remove duplicates (by JSON string comparison)
-    const uniqueInputs = removeDuplicates(inputs);
-    return uniqueInputs;
-}
-/**
- * Type guard for JSONSchemaField.
- */
-function isJSONSchemaField(obj) {
-    return (typeof obj === 'object' &&
-        obj !== null &&
-        ('type' in obj || 'properties' in obj || 'items' in obj || '$ref' in obj));
-}
-/**
- * Generate minimal input (only required fields).
- */
-function generateMinimalInput(schema) {
-    const input = {};
-    if (!isJSONSchemaField(schema)) {
-        return input;
-    }
-    const properties = schema.properties;
-    const required = schema.required;
-    if (!properties) {
-        return input;
-    }
-    // Only include required fields
-    if (required) {
-        for (const fieldName of required) {
-            const fieldSchema = properties[fieldName];
-            if (fieldSchema) {
-                input[fieldName] = generateValue(fieldSchema);
-            }
-        }
-    }
-    return input;
-}
-/**
- * Generate input with default values.
- */
-function generateWithDefaults(schema) {
-    const minimal = generateMinimalInput(schema);
-    if (!isJSONSchemaField(schema)) {
-        return minimal;
-    }
-    const properties = schema.properties;
-    if (!properties) {
-        return minimal;
-    }
-    // Add optional fields with defaults if they exist
-    for (const [fieldName, fieldSchema] of Object.entries(properties)) {
-        if (!(fieldName in minimal)) {
-            if (isJSONSchemaField(fieldSchema) && 'default' in fieldSchema) {
-                minimal[fieldName] = fieldSchema.default;
-            }
-        }
-    }
-    return minimal;
-}
-/**
- * Generate edge case inputs.
- */
-function generateEdgeCases(schema) {
-    const cases = [];
-    if (!isJSONSchemaField(schema)) {
-        return cases;
-    }
-    const properties = schema.properties;
-    const required = schema.required;
-    if (!properties) {
-        return cases;
-    }
-    // Generate edge cases for each field
-    for (const [fieldName, fieldSchema] of Object.entries(properties)) {
-        const isRequired = required?.includes(fieldName) ?? false;
-        // Generate edge case: empty string (if field is string)
-        if (isStringType(fieldSchema)) {
-            const edgeCase = {};
-            if (isRequired) {
-                edgeCase[fieldName] = '';
-                cases.push(edgeCase);
-            }
-        }
-        // Generate edge case: null (if nullable)
-        if (isNullable(fieldSchema)) {
-            const edgeCase = {};
-            if (isRequired) {
-                edgeCase[fieldName] = null;
-                cases.push(edgeCase);
-            }
-        }
-        // Generate edge case: empty array (if field is array)
-        if (isArrayType(fieldSchema)) {
-            const edgeCase = {};
-            if (isRequired) {
-                edgeCase[fieldName] = [];
-                cases.push(edgeCase);
-            }
-        }
-    }
-    return cases;
-}
-/**
- * Generate a value from a field schema.
- */
-function generateValue(schema) {
-    if (!isJSONSchemaField(schema)) {
-        return null;
-    }
-    // Check for enum
-    if (Array.isArray(schema.enum) && schema.enum.length > 0) {
-        return schema.enum[0];
-    }
-    // Check for default value
-    if ('default' in schema) {
-        return schema.default;
-    }
-    // Generate based on type
-    const type = schema.type;
-    if (Array.isArray(type)) {
-        // Use first non-null type
-        const firstType = type.find(t => t !== 'null') || type[0];
-        if (firstType) {
-            return generateValueByType(firstType, schema);
-        }
-    }
-    if (typeof type === 'string') {
-        return generateValueByType(type, schema);
-    }
-    // If no type specified, try to infer from other properties
-    if (isJSONSchemaField(schema)) {
-        if (schema.properties) {
-            return {};
-        }
-        if (schema.items) {
-            return [];
-        }
-    }
-    return null;
-}
-/**
- * Generate value by type.
- */
-function generateValueByType(type, schema) {
-    switch (type) {
-        case 'string':
-            if (typeof schema === 'object' &&
-                schema !== null &&
-                'pattern' in schema) {
-                // For patterns, return a simple string (pattern matching is complex)
-                return 'test_string';
-            }
-            return 'test_string';
-        case 'number':
-        case 'integer':
-            if (schema.minimum !== undefined) {
-                return schema.minimum;
-            }
-            if (schema.maximum !== undefined) {
-                return schema.maximum;
-            }
-            return 0;
-        case 'boolean':
-            return false;
-        case 'array':
-            return [];
-        case 'object':
-            return {};
-        case 'null':
-            return null;
-        default:
-            return null;
-    }
-}
-/**
- * Check if schema is string type.
- */
-function isStringType(schema) {
-    if (!isJSONSchemaField(schema)) {
-        return false;
-    }
-    const type = schema.type;
-    if (typeof type === 'string') {
-        return type === 'string';
-    }
-    if (Array.isArray(type)) {
-        return type.includes('string');
-    }
-    return false;
-}
-/**
- * Check if schema is array type.
- */
-function isArrayType(schema) {
-    if (!isJSONSchemaField(schema)) {
-        return false;
-    }
-    const type = schema.type;
-    if (typeof type === 'string') {
-        return type === 'array';
-    }
-    if (Array.isArray(type)) {
-        return type.includes('array');
-    }
-    return false;
-}
-/**
- * Check if schema is nullable.
- */
-function isNullable(schema) {
-    if (!isJSONSchemaField(schema)) {
-        return false;
-    }
-    if (schema.nullable === true) {
-        return true;
-    }
-    const type = schema.type;
-    if (Array.isArray(type)) {
-        return type.includes('null');
-    }
-    return false;
-}
-/**
- * Generate agent-like unpredictable inputs (random variations).
- * Simulates how agents might generate inputs.
- */
-function generateAgentLikeInputs(schema) {
-    const inputs = [];
-    if (!isJSONSchemaField(schema)) {
-        return inputs;
-    }
-    const properties = schema.properties;
-    const required = schema.required;
-    if (!properties) {
-        return inputs;
-    }
-    // Generate 3 random variations with random values
-    const numVariations = 3;
-    for (let i = 0; i < numVariations; i++) {
-        const input = {};
-        // Include all required fields with random values
-        if (required) {
-            for (const fieldName of required) {
-                const fieldSchema = properties[fieldName];
-                if (fieldSchema) {
-                    input[fieldName] = generateRandomValue(fieldSchema);
-                }
-            }
-        }
-        // Randomly include some optional fields (agent-like behavior)
-        for (const [fieldName, fieldSchema] of Object.entries(properties)) {
-            if (!required?.includes(fieldName) && Math.random() > 0.5) {
-                input[fieldName] = generateRandomValue(fieldSchema);
-            }
-        }
-        if (Object.keys(input).length > 0) {
-            inputs.push(input);
-        }
-    }
-    return inputs;
-}
-/**
- * Generate invalid-but-close inputs (type coercion attempts).
- * Simulates agents trying to pass slightly wrong types.
- */
-function generateInvalidButCloseInputs(schema) {
-    const inputs = [];
-    if (!isJSONSchemaField(schema)) {
-        return inputs;
-    }
-    const properties = schema.properties;
-    const required = schema.required;
-    if (!properties) {
-        return inputs;
-    }
-    // For each required string field, try passing a number (agent might do type coercion)
-    if (required) {
-        for (const fieldName of required) {
-            const fieldSchema = properties[fieldName];
-            if (isStringType(fieldSchema)) {
-                const input = {};
-                // Try number instead of string (common agent mistake)
-                input[fieldName] = Math.floor(Math.random() * 1000);
-                // Fill other required fields with valid values
-                for (const otherField of required) {
-                    if (otherField !== fieldName) {
-                        const prop = properties[otherField];
-                        if (!prop) {
-                            throw new Error(`Malformed schema: required field "${otherField}" not found in properties`);
-                        }
-                        input[otherField] = generateValue(prop);
-                    }
-                }
-                inputs.push(input);
-            }
-            // Try null for required non-nullable fields
-            if (!isNullable(fieldSchema)) {
-                const input = {};
-                input[fieldName] = null;
-                for (const otherField of required) {
-                    if (otherField !== fieldName) {
-                        const prop = properties[otherField];
-                        if (!prop) {
-                            throw new Error(`Malformed schema: required field "${otherField}" not found in properties`);
-                        }
-                        input[otherField] = generateValue(prop);
-                    }
-                }
-                inputs.push(input);
-            }
-        }
-    }
-    return inputs;
-}
-/**
- * Generate extreme value inputs (very long strings, very large numbers).
- * Tests tools against agent-generated extreme inputs.
- */
-function generateExtremeValueInputs(schema) {
-    const inputs = [];
-    if (!isJSONSchemaField(schema)) {
-        return inputs;
-    }
-    const properties = schema.properties;
-    const required = schema.required;
-    if (!properties) {
-        return inputs;
-    }
-    // Generate extreme string (1000+ characters)
-    if (required) {
-        for (const fieldName of required) {
-            const fieldSchema = properties[fieldName];
-            if (isStringType(fieldSchema)) {
-                const input = {};
-                // Very long string (agent might paste large text)
-                input[fieldName] =
-                    'A'.repeat(1000) + ' very long string that agents might generate';
-                for (const otherField of required) {
-                    if (otherField !== fieldName) {
-                        const prop = properties[otherField];
-                        if (!prop) {
-                            throw new Error(`Malformed schema: required field "${otherField}" not found in properties`);
-                        }
-                        input[otherField] = generateValue(prop);
-                    }
-                }
-                inputs.push(input);
-            }
-            // Very large number
-            if (isNumberType(fieldSchema)) {
-                const input = {};
-                input[fieldName] = Number.MAX_SAFE_INTEGER;
-                for (const otherField of required) {
-                    if (otherField !== fieldName) {
-                        const prop = properties[otherField];
-                        if (!prop) {
-                            throw new Error(`Malformed schema: required field "${otherField}" not found in properties`);
-                        }
-                        input[otherField] = generateValue(prop);
-                    }
-                }
-                inputs.push(input);
-                // Very small number
-                const input2 = {};
-                input2[fieldName] = Number.MIN_SAFE_INTEGER;
-                for (const otherField of required) {
-                    if (otherField !== fieldName) {
-                        const prop = properties[otherField];
-                        if (!prop) {
-                            throw new Error(`Malformed schema: required field "${otherField}" not found in properties`);
-                        }
-                        input2[otherField] = generateValue(prop);
-                    }
-                }
-                inputs.push(input2);
-            }
-        }
-    }
-    return inputs;
-}
-/**
- * Generate a random value from a field schema (for agent-like inputs).
- */
-function generateRandomValue(schema) {
-    if (!isJSONSchemaField(schema)) {
-        return null;
-    }
-    // Check for enum - pick random enum value
-    if (Array.isArray(schema.enum) && schema.enum.length > 0) {
-        return schema.enum[Math.floor(Math.random() * schema.enum.length)];
-    }
-    // Generate based on type with randomness
-    const type = schema.type;
-    if (typeof type === 'string') {
-        return generateRandomValueByType(type, schema);
-    }
-    if (Array.isArray(type)) {
-        const firstType = type.find(t => t !== 'null') || type[0];
-        if (firstType) {
-            return generateRandomValueByType(firstType, schema);
-        }
-    }
-    return null;
-}
-/**
- * Generate random value by type.
- */
-function generateRandomValueByType(type, schema) {
-    switch (type) {
-        case 'string': {
-            // Random string with various patterns (agent-like)
-            const patterns = [
-                `random_${Math.random().toString(36).substring(7)}`,
-                `Test String ${Date.now()}`,
-                `Mixed123String${Math.floor(Math.random() * 100)}`,
-                `special-chars-!@#$%^&*()`,
-                `unicode-测试-☀️🌧️`,
-            ];
-            return patterns[Math.floor(Math.random() * patterns.length)];
-        }
-        case 'number':
-        case 'integer': {
-            const min = schema.minimum ?? -1000;
-            const max = schema.maximum ?? 1000;
-            return Math.floor(Math.random() * (max - min + 1)) + min;
-        }
-        case 'boolean':
-            return Math.random() > 0.5;
-        case 'array': {
-            // Random array with 0-5 elements
-            const length = Math.floor(Math.random() * 6);
-            return Array.from({ length }, () => generateRandomValue(schema.items || {}));
-        }
-        case 'object':
-            return {};
-        case 'null':
-            return null;
-        default:
-            return null;
-    }
-}
-/**
- * Check if schema is number type.
- */
-function isNumberType(schema) {
-    if (!isJSONSchemaField(schema)) {
-        return false;
-    }
-    const type = schema.type;
-    if (typeof type === 'string') {
-        return type === 'number' || type === 'integer';
-    }
-    if (Array.isArray(type)) {
-        return type.includes('number') || type.includes('integer');
-    }
-    return false;
-}
-/**
- * Remove duplicate inputs.
- */
-function removeDuplicates(inputs) {
-    const seen = new Set();
-    const unique = [];
-    for (const input of inputs) {
-        const key = JSON.stringify(input, Object.keys(input).sort());
-        if (!seen.has(key)) {
-            seen.add(key);
-            unique.push(input);
-        }
-    }
-    return unique;
-}
-//# sourceMappingURL=input-generator.js.map

package/dist/runtime/test/mcp-root-detector.d.ts

@@ -1,31 +0,0 @@
-/**
- * MCP root detection utility.
- * Detects the MCP server root directory using priority: CLI flag > config > auto-detect.
- */
-/**
- * Options for MCP root detection.
- */
-export interface MCPRootDetectionOptions {
-    /** Explicit MCP root from CLI flag */
-    cliMCPRoot?: string;
-    /** MCP root from config file */
-    configMCPRoot?: string;
-    /** Current working directory (defaults to process.cwd()) */
-    cwd?: string;
-}
-/**
- * Detect MCP root directory using priority order.
- * Priority: CLI flag > Config > Auto-detect > Current working directory
- *
- * @param options - Detection options
- * @returns Resolved MCP root directory (absolute path)
- */
-export declare function detectMCPRoot(options?: MCPRootDetectionOptions): string;
-/**
- * Resolve tools directory path relative to MCP root.
- * @param mcpRoot - MCP root directory
- * @param toolsDir - Tools directory name (default: "tools")
- * @returns Absolute path to tools directory
- */
-export declare function resolveToolsDir(mcpRoot: string, toolsDir?: string): string;
-//# sourceMappingURL=mcp-root-detector.d.ts.map

package/dist/runtime/test/mcp-root-detector.js

@@ -1,105 +0,0 @@
-/**
- * MCP root detection utility.
- * Detects the MCP server root directory using priority: CLI flag > config > auto-detect.
- */
-import * as fs from 'fs';
-import * as path from 'path';
-/**
- * Auto-detect MCP root by looking for common indicators.
- * @param startDir - Directory to start searching from
- * @returns Detected MCP root, or null if not found
- */
-function autoDetectMCPRoot(startDir) {
-    let currentDir = path.resolve(startDir);
-    // Look up the directory tree for indicators
-    while (currentDir !== path.dirname(currentDir)) {
-        // Check for tools directory
-        const toolsDir = path.join(currentDir, 'tools');
-        try {
-            if (fs.existsSync(toolsDir) && fs.statSync(toolsDir).isDirectory()) {
-                return currentDir;
-            }
-        }
-        catch (_error) {
-            // If fs.existsSync or fs.statSync throws, treat as not present
-            // Continue searching
-        }
-        // Check for common MCP server files
-        const serverFiles = [
-            'server.py',
-            'server.ts',
-            'server.js',
-            'package.json',
-            'requirements.txt',
-        ];
-        for (const file of serverFiles) {
-            const filePath = path.join(currentDir, file);
-            try {
-                if (fs.existsSync(filePath)) {
-                    // Server file found - return currentDir (tools directory check already handled above)
-                    return currentDir;
-                }
-            }
-            catch (_error) {
-                // Ignore errors and continue
-            }
-        }
-        // Move up one directory
-        currentDir = path.dirname(currentDir);
-    }
-    return null;
-}
-/**
- * Validate MCP root path and throw descriptive errors if invalid.
- */
-function validateMCPRootPath(sourcePath, resolvedPath, source) {
-    if (!fs.existsSync(resolvedPath)) {
-        const prefix = source === 'CLI flag' ? '' : 'from config ';
-        throw new Error(`MCP root directory ${prefix}not found: ${sourcePath} (resolved to: ${resolvedPath})`);
-    }
-    // Ensure it's a directory, not a file
-    const stats = fs.statSync(resolvedPath);
-    if (!stats.isDirectory()) {
-        const prefix = source === 'CLI flag' ? '' : 'from config ';
-        throw new Error(`MCP root path ${prefix}is not a directory: ${sourcePath} (resolved to: ${resolvedPath})`);
-    }
-}
-/**
- * Detect MCP root directory using priority order.
- * Priority: CLI flag > Config > Auto-detect > Current working directory
- *
- * @param options - Detection options
- * @returns Resolved MCP root directory (absolute path)
- */
-export function detectMCPRoot(options = {}) {
-    const cwd = options.cwd || process.cwd();
-    // Priority 1: CLI flag (explicit)
-    if (options.cliMCPRoot) {
-        const resolved = path.resolve(cwd, options.cliMCPRoot);
-        validateMCPRootPath(options.cliMCPRoot, resolved, 'from CLI flag');
-        return resolved;
-    }
-    // Priority 2: Config file
-    if (options.configMCPRoot) {
-        const resolved = path.resolve(cwd, options.configMCPRoot);
-        validateMCPRootPath(options.configMCPRoot, resolved, 'from config');
-        return resolved;
-    }
-    // Priority 3: Auto-detect
-    const autoDetected = autoDetectMCPRoot(cwd);
-    if (autoDetected) {
-        return autoDetected;
-    }
-    // Priority 4: Default to current working directory
-    return path.resolve(cwd);
-}
-/**
- * Resolve tools directory path relative to MCP root.
- * @param mcpRoot - MCP root directory
- * @param toolsDir - Tools directory name (default: "tools")
- * @returns Absolute path to tools directory
- */
-export function resolveToolsDir(mcpRoot, toolsDir = 'tools') {
-    return path.resolve(mcpRoot, toolsDir);
-}
-//# sourceMappingURL=mcp-root-detector.js.map

package/dist/runtime/test/retry-tester.d.ts

@@ -1,44 +0,0 @@
-/**
- * Retry scenario tester.
- * Tests tools against agent retry patterns (retry on failure, timeout, error).
- */
-import type { SandboxExecutor, ToolExecutionResult } from '../../runtime/sandbox/index.js';
-/**
- * Retry test configuration.
- */
-export interface RetryTestConfig {
-    /** Maximum number of retries to test (default: 3) */
-    maxRetries?: number;
-    /** Retry delay in milliseconds (default: 100) */
-    retryDelayMs?: number;
-    /** Test retry on failure (default: true) */
-    testRetryOnFailure?: boolean;
-    /** Test retry on timeout (default: true) */
-    testRetryOnTimeout?: boolean;
-    /** Test retry on connection error (default: true) */
-    testRetryOnConnectionError?: boolean;
-}
-/**
- * Retry test result.
- */
-export interface RetryTestResult {
-    /** Input tested */
-    testInput: Record<string, unknown>;
-    /** Retry attempts made */
-    attempts: number;
-    /** Whether tool eventually succeeded after retries */
-    eventuallySucceeded: boolean;
-    /** Results from all retry attempts */
-    results: ToolExecutionResult[];
-    /** Issues detected */
-    issues: string[];
-}
-/**
- * Test tool retry behavior.
- * Simulates agent retry patterns to ensure tools handle retries gracefully.
- *
- * Note: Idempotency (output consistency across retries) is checked separately
- * This tester focuses on retry reliability.
- */
-export declare function testRetryBehavior(executor: SandboxExecutor, toolName: string, testInput: Record<string, unknown>, config?: RetryTestConfig, toolTimeoutMs?: number): Promise<RetryTestResult>;
-//# sourceMappingURL=retry-tester.d.ts.map