@syrin/cli 1.3.2 → 1.4.0

package/dist/runtime/test/retry-tester.js

@@ -1,103 +0,0 @@
-/**
- * Retry scenario tester.
- * Tests tools against agent retry patterns (retry on failure, timeout, error).
- */
-import { ToolExecutionErrorType } from '../../runtime/sandbox/index.js';
-/**
- * Get error type from an unknown error object.
- * @param error - Error object (may be ToolExecutionError or unknown)
- * @returns ToolExecutionErrorType if error has errorType, undefined otherwise
- */
-function getErrorType(error) {
-    if (error && typeof error === 'object' && 'errorType' in error) {
-        return error.errorType;
-    }
-    return undefined;
-}
-/**
- * Test tool retry behavior.
- * Simulates agent retry patterns to ensure tools handle retries gracefully.
- *
- * Note: Idempotency (output consistency across retries) is checked separately
- * This tester focuses on retry reliability.
- */
-export async function testRetryBehavior(executor, toolName, testInput, config = {}, toolTimeoutMs) {
-    const maxRetries = config.maxRetries ?? 3;
-    const retryDelayMs = config.retryDelayMs ?? 100;
-    const testRetryOnFailure = config.testRetryOnFailure ?? true;
-    const testRetryOnTimeout = config.testRetryOnTimeout ?? true;
-    const testRetryOnConnectionError = config.testRetryOnConnectionError ?? true;
-    const results = [];
-    const issues = [];
-    let eventuallySucceeded = false;
-    // Run initial execution
-    let executionResults = await executor.executeTool(toolName, [testInput], toolTimeoutMs);
-    let lastResult = executionResults[0];
-    if (lastResult) {
-        results.push(lastResult);
-    }
-    // If first attempt succeeded, test retry behavior
-    if (lastResult && lastResult.success) {
-        eventuallySucceeded = true;
-        // Retry to check reliability
-        for (let attempt = 1; attempt < maxRetries; attempt++) {
-            await new Promise(resolve => setTimeout(resolve, retryDelayMs));
-            executionResults = await executor.executeTool(toolName, [testInput], toolTimeoutMs);
-            const retryResult = executionResults[0];
-            if (retryResult) {
-                results.push(retryResult);
-                // Check reliability (retry shouldn't fail if first attempt succeeded)
-                if (!retryResult.success) {
-                    issues.push(`Retry ${attempt + 1} failed (was successful on first attempt) - tool may be unreliable under retry scenarios`);
-                }
-                // Note: We don't report idempotency issues here as tools can be idempotent by design
-            }
-        }
-    }
-    else if (lastResult && !lastResult.success) {
-        // First attempt failed - test retry on failure
-        if (testRetryOnFailure) {
-            const errorType = getErrorType(lastResult.error);
-            const shouldRetry = errorType === ToolExecutionErrorType.EXECUTION_ERROR ||
-                errorType === ToolExecutionErrorType.UNKNOWN;
-            if (shouldRetry) {
-                // Retry on failure
-                for (let attempt = 1; attempt < maxRetries; attempt++) {
-                    await new Promise(resolve => setTimeout(resolve, retryDelayMs));
-                    executionResults = await executor.executeTool(toolName, [testInput], toolTimeoutMs);
-                    const retryResult = executionResults[0];
-                    if (retryResult) {
-                        results.push(retryResult);
-                        if (retryResult.success) {
-                            eventuallySucceeded = true;
-                            issues.push(`Tool recovered after ${attempt + 1} retry attempts (initially failed)`);
-                            break;
-                        }
-                    }
-                }
-                if (!eventuallySucceeded) {
-                    issues.push(`Tool failed consistently after ${maxRetries} retry attempts`);
-                }
-            }
-        }
-        // Test retry on timeout
-        if (testRetryOnTimeout && lastResult.timedOut === true) {
-            issues.push(`Tool timed out - agents may retry, causing delays`);
-        }
-        // Test retry on connection error
-        if (testRetryOnConnectionError && !lastResult.success) {
-            const errorType = getErrorType(lastResult.error);
-            if (errorType === ToolExecutionErrorType.CONNECTION_ERROR) {
-                issues.push(`Connection error detected - agents may retry, causing delays`);
-            }
-        }
-    }
-    return {
-        testInput,
-        attempts: results.length,
-        eventuallySucceeded,
-        results,
-        issues,
-    };
-}
-//# sourceMappingURL=retry-tester.js.map

package/dist/runtime/test/synthetic-input-generator.d.ts

@@ -1,11 +0,0 @@
-/**
- * Generate system-level synthetic test inputs from tool input schema.
- * These inputs test edge cases, type validations, and boundary conditions.
- */
-/**
- * Generate synthetic test inputs from a JSON schema.
- * @param schema - Input schema (JSON Schema format)
- * @returns Array of synthetic test inputs
- */
-export declare function generateSyntheticInputs(schema: unknown): Array<Record<string, unknown>>;
-//# sourceMappingURL=synthetic-input-generator.d.ts.map

package/dist/runtime/test/synthetic-input-generator.js

@@ -1,154 +0,0 @@
-/**
- * Generate system-level synthetic test inputs from tool input schema.
- * These inputs test edge cases, type validations, and boundary conditions.
- */
-/**
- * Generate synthetic test inputs from a JSON schema.
- * @param schema - Input schema (JSON Schema format)
- * @returns Array of synthetic test inputs
- */
-export function generateSyntheticInputs(schema) {
-    const inputs = [];
-    if (!schema || typeof schema !== 'object') {
-        return inputs;
-    }
-    const jsonSchema = schema;
-    // Handle object schemas with properties
-    if (jsonSchema.type === 'object' && jsonSchema.properties) {
-        const properties = jsonSchema.properties;
-        const required = jsonSchema.required || [];
-        // Generate edge case inputs for each property
-        for (const [propName, propSchema] of Object.entries(properties)) {
-            const prop = propSchema;
-            const propType = Array.isArray(prop.type) ? prop.type[0] : prop.type;
-            // Generate edge case values based on property type
-            const edgeCases = generateEdgeCasesForProperty(propName, prop, propType);
-            for (const edgeCase of edgeCases) {
-                inputs.push(edgeCase);
-            }
-        }
-        // Generate input with all required fields (minimal valid input)
-        if (required.length > 0) {
-            const minimalInput = {};
-            for (const reqField of required) {
-                const propSchema = properties[reqField];
-                const propType = Array.isArray(propSchema?.type)
-                    ? propSchema.type[0]
-                    : propSchema?.type;
-                minimalInput[reqField] = getDefaultValue(propType);
-            }
-            inputs.push(minimalInput);
-        }
-    }
-    // Remove duplicates
-    return removeDuplicateInputs(inputs);
-}
-/**
- * Generate edge case values for a specific property.
- */
-function generateEdgeCasesForProperty(propName, propSchema, propType) {
-    const edgeCases = [];
-    if (propSchema.enum) {
-        // For enums, test each enum value
-        for (const enumValue of propSchema.enum) {
-            edgeCases.push({ [propName]: enumValue });
-        }
-    }
-    else {
-        switch (propType) {
-            case 'string':
-                edgeCases.push({ [propName]: '' }, // Empty string
-                { [propName]: 'test' }, // Normal string
-                ...(propSchema.minLength !== undefined ? [] : [
-                    { [propName]: 'a'.repeat(100) }, // Long string
-                ]));
-                break;
-            case 'number':
-            case 'integer':
-                edgeCases.push({ [propName]: 0 }, { [propName]: 1 }, { [propName]: -1 }, ...(propSchema.minimum !== undefined ? [] : [
-                    { [propName]: Number.MAX_SAFE_INTEGER },
-                ]), ...(propSchema.maximum !== undefined ? [] : [
-                    { [propName]: Number.MIN_SAFE_INTEGER },
-                ]));
-                break;
-            case 'boolean':
-                edgeCases.push({ [propName]: true }, { [propName]: false });
-                break;
-            case 'array':
-                edgeCases.push({ [propName]: [] }, // Empty array
-                { [propName]: [1, 2, 3] } // Array with values
-                );
-                break;
-            case 'object':
-                edgeCases.push({ [propName]: {} }, // Empty object
-                { [propName]: { key: 'value' } } // Object with values
-                );
-                break;
-            default:
-                // For unknown types, generate a basic test value
-                edgeCases.push({ [propName]: null });
-                break;
-        }
-    }
-    return edgeCases;
-}
-/**
- * Get a default value for a property type.
- */
-function getDefaultValue(propType) {
-    switch (propType) {
-        case 'string':
-            return 'test';
-        case 'number':
-        case 'integer':
-            return 0;
-        case 'boolean':
-            return false;
-        case 'array':
-            return [];
-        case 'object':
-            return {};
-        default:
-            return null;
-    }
-}
-/**
- * Remove duplicate inputs by normalizing and comparing JSON representation.
- */
-function removeDuplicateInputs(inputs) {
-    const seen = new Set();
-    const unique = [];
-    for (const input of inputs) {
-        const normalized = deepSortObject(input);
-        const key = JSON.stringify(normalized);
-        if (!seen.has(key)) {
-            seen.add(key);
-            unique.push(input);
-        }
-    }
-    return unique;
-}
-/**
- * Recursively sort object keys for deterministic comparison.
- */
-function deepSortObject(obj) {
-    if (obj === null || obj === undefined) {
-        return obj;
-    }
-    if (Array.isArray(obj)) {
-        return obj.map(item => deepSortObject(item));
-    }
-    if (typeof obj === 'object') {
-        const sorted = {};
-        const keys = Object.keys(obj).sort();
-        for (const key of keys) {
-            const value = obj[key];
-            if (value !== undefined) {
-                sorted[key] = deepSortObject(value);
-            }
-        }
-        return sorted;
-    }
-    return obj;
-}
-//# sourceMappingURL=synthetic-input-generator.js.map

package/dist/runtime/test/test-runner.d.ts

@@ -1,28 +0,0 @@
-/**
- * Test input runner.
- * Executes tools with contract-defined test inputs.
- */
-import type { ToolContract } from './contract-types.js';
-import type { SandboxExecutor, ToolExecutionResult } from '../../runtime/sandbox/index.js';
-/**
- * Test execution result with metadata.
- */
-export type TestExecutionResult = ToolExecutionResult & {
-    /** Test case name */
-    testName: string;
-    /** Test input used */
-    testInput: Record<string, unknown>;
-    /** Test expectation */
-    expectation?: import('./contract-types.js').TestExpectation;
-    /** Expected output schema name (optional, for tests expecting different output schema) */
-    expectedOutputSchema?: string;
-};
-/**
- * Run contract-defined test cases.
- * @param executor - Sandbox executor
- * @param contract - Tool contract
- * @param determinismRuns - Number of runs for deterministic tests (default: 1)
- * @returns Array of test execution results
- */
-export declare function runContractTests(executor: SandboxExecutor, contract: ToolContract, toolTimeoutMs?: number): Promise<TestExecutionResult[]>;
-//# sourceMappingURL=test-runner.d.ts.map

package/dist/runtime/test/test-runner.js

@@ -1,55 +0,0 @@
-/**
- * Test input runner.
- * Executes tools with contract-defined test inputs.
- */
-/**
- * Run contract-defined test cases.
- * @param executor - Sandbox executor
- * @param contract - Tool contract
- * @param determinismRuns - Number of runs for deterministic tests (default: 1)
- * @returns Array of test execution results
- */
-export async function runContractTests(executor, contract, toolTimeoutMs) {
-    if (!contract.tests || contract.tests.length === 0) {
-        return [];
-    }
-    const results = [];
-    for (const test of contract.tests) {
-        // Set environment variables if specified
-        const originalEnv = { ...process.env };
-        if (test.env) {
-            for (const [key, value] of Object.entries(test.env)) {
-                process.env[key] = value;
-            }
-        }
-        try {
-            // Execute tool with test input (run once)
-            const executionResults = await executor.executeTool(contract.tool, [test.input], toolTimeoutMs);
-            // Map to test execution results
-            for (const result of executionResults) {
-                results.push({
-                    ...result,
-                    testName: test.name,
-                    testInput: test.input,
-                    expectation: test.expect,
-                });
-            }
-        }
-        finally {
-            // Restore environment properly (avoid reassigning process.env which breaks Node's proxy)
-            // Remove any keys that were added
-            const currentKeys = Object.keys(process.env);
-            for (const key of currentKeys) {
-                if (!(key in originalEnv)) {
-                    delete process.env[key];
-                }
-            }
-            // Restore any keys that were changed
-            for (const [key, value] of Object.entries(originalEnv)) {
-                process.env[key] = value;
-            }
-        }
-    }
-    return results;
-}
-//# sourceMappingURL=test-runner.js.map