@syrin/cli 1.3.0

package/dist/runtime/test/output-validator.d.ts

@@ -0,0 +1,44 @@
+/**
+ * Output schema validator.
+ * Validates that tool outputs match declared schema structure.
+ */
+/**
+ * JSON Schema field interface.
+ */
+interface JSONSchemaField {
+    type?: string | string[];
+    description?: string;
+    required?: string[];
+    properties?: Record<string, JSONSchemaField>;
+    items?: JSONSchemaField | JSONSchemaField[];
+    $ref?: string;
+    enum?: unknown[];
+    pattern?: string;
+    example?: unknown;
+    examples?: unknown[];
+    nullable?: boolean;
+}
+type JSONSchema = JSONSchemaField | Record<string, unknown> | unknown;
+/**
+ * Validation result.
+ */
+export interface OutputValidationResult {
+    /** Whether output is valid */
+    valid: boolean;
+    /** Error message if invalid */
+    error?: string;
+    /** Validation details */
+    details?: {
+        /** Missing required fields */
+        missingFields?: string[];
+        /** Fields with type mismatches */
+        typeMismatches?: Array<{
+            field: string;
+            expected: string;
+            actual: string;
+        }>;
+    };
+}
+export declare function validateOutputStructure(output: unknown, schema: JSONSchema): OutputValidationResult;
+export {};
+//# sourceMappingURL=output-validator.d.ts.map

package/dist/runtime/test/output-validator.js

@@ -0,0 +1,262 @@
+/**
+ * Output schema validator.
+ * Validates that tool outputs match declared schema structure.
+ */
+/**
+ * Validate output structure against schema.
+ * @param output - Tool output to validate
+ * @param schema - Expected output schema
+ * @returns Validation result
+ */
+/**
+ * Type guard for JSONSchemaField.
+ */
+function isJSONSchemaField(obj) {
+    return (typeof obj === 'object' &&
+        obj !== null &&
+        ('type' in obj || 'properties' in obj || 'items' in obj || '$ref' in obj));
+}
+export function validateOutputStructure(output, schema) {
+    try {
+        // Basic type check
+        if (!isJSONSchemaField(schema)) {
+            return {
+                valid: false,
+                error: 'Invalid schema: schema must be an object',
+            };
+        }
+        // If output is not an object, check if schema expects object
+        if (typeof output !== 'object' || output === null) {
+            const schemaType = getSchemaType(schema);
+            if (schemaType === 'object' && !isNullable(schema)) {
+                return {
+                    valid: false,
+                    error: `Output type mismatch: expected object, got ${typeof output}`,
+                };
+            }
+            // If schema allows null or other types, it might be valid
+            return { valid: true };
+        }
+        // Validate object structure
+        return validateObjectStructure(output, schema);
+    }
+    catch (error) {
+        return {
+            valid: false,
+            error: `Validation error: ${error instanceof Error ? error.message : String(error)}`,
+        };
+    }
+}
+/**
+ * Validate object structure against schema.
+ */
+function validateObjectStructure(output, schema) {
+    if (!isJSONSchemaField(schema)) {
+        return { valid: false, error: 'Invalid schema' };
+    }
+    const properties = schema.properties;
+    const required = schema.required;
+    const missingFields = [];
+    const typeMismatches = [];
+    // Check required fields
+    if (required && properties) {
+        for (const fieldName of required) {
+            if (!(fieldName in output)) {
+                missingFields.push(fieldName);
+            }
+        }
+    }
+    // Check field types (recursively for nested objects and arrays)
+    if (properties) {
+        for (const [fieldName, fieldValue] of Object.entries(output)) {
+            const fieldSchema = properties[fieldName];
+            if (fieldSchema) {
+                const expectedType = getSchemaType(fieldSchema);
+                const actualType = getValueType(fieldValue);
+                if (!isTypeCompatible(actualType, expectedType, fieldSchema, fieldValue)) {
+                    typeMismatches.push({
+                        field: fieldName,
+                        expected: expectedType,
+                        actual: actualType,
+                    });
+                }
+                else {
+                    // Recursively validate nested structures
+                    if (isJSONSchemaField(fieldSchema)) {
+                        // Validate nested object
+                        if (expectedType === 'object' &&
+                            actualType === 'object' &&
+                            fieldSchema.properties &&
+                            fieldValue !== null &&
+                            typeof fieldValue === 'object' &&
+                            !Array.isArray(fieldValue)) {
+                            const nestedResult = validateObjectStructure(fieldValue, fieldSchema);
+                            if (!nestedResult.valid) {
+                                // Add field prefix to nested errors
+                                const prefix = fieldName + '.';
+                                if (nestedResult.details?.missingFields) {
+                                    missingFields.push(...nestedResult.details.missingFields.map(f => prefix + f));
+                                }
+                                if (nestedResult.details?.typeMismatches) {
+                                    typeMismatches.push(...nestedResult.details.typeMismatches.map(m => ({
+                                        field: prefix + m.field,
+                                        expected: m.expected,
+                                        actual: m.actual,
+                                    })));
+                                }
+                            }
+                        }
+                        // Validate array items
+                        else if (expectedType === 'array' &&
+                            actualType === 'array' &&
+                            fieldSchema.items &&
+                            Array.isArray(fieldValue)) {
+                            const itemsSchema = Array.isArray(fieldSchema.items)
+                                ? fieldSchema.items[0]
+                                : fieldSchema.items;
+                            if (itemsSchema && isJSONSchemaField(itemsSchema)) {
+                                for (let i = 0; i < fieldValue.length; i++) {
+                                    const item = fieldValue[i];
+                                    const itemType = getValueType(item);
+                                    const expectedItemType = getSchemaType(itemsSchema);
+                                    if (!isTypeCompatible(itemType, expectedItemType, itemsSchema, item)) {
+                                        typeMismatches.push({
+                                            field: `${fieldName}[${i}]`,
+                                            expected: expectedItemType,
+                                            actual: itemType,
+                                        });
+                                    }
+                                    else if (expectedItemType === 'object' &&
+                                        itemType === 'object' &&
+                                        itemsSchema.properties &&
+                                        item !== null &&
+                                        typeof item === 'object' &&
+                                        !Array.isArray(item)) {
+                                        // Recursively validate array item objects
+                                        const itemResult = validateObjectStructure(item, itemsSchema);
+                                        if (!itemResult.valid) {
+                                            const prefix = `${fieldName}[${i}].`;
+                                            if (itemResult.details?.missingFields) {
+                                                missingFields.push(...itemResult.details.missingFields.map(f => prefix + f));
+                                            }
+                                            if (itemResult.details?.typeMismatches) {
+                                                typeMismatches.push(...itemResult.details.typeMismatches.map(m => ({
+                                                    field: prefix + m.field,
+                                                    expected: m.expected,
+                                                    actual: m.actual,
+                                                })));
+                                            }
+                                        }
+                                    }
+                                }
+                            }
+                        }
+                    }
+                }
+            }
+        }
+    }
+    if (missingFields.length > 0 || typeMismatches.length > 0) {
+        return {
+            valid: false,
+            error: 'Output structure does not match schema',
+            details: {
+                missingFields: missingFields.length > 0 ? missingFields : undefined,
+                typeMismatches: typeMismatches.length > 0 ? typeMismatches : undefined,
+            },
+        };
+    }
+    return { valid: true };
+}
+/**
+ * Get schema type as string.
+ */
+function getSchemaType(schema) {
+    if (!isJSONSchemaField(schema)) {
+        return 'unknown';
+    }
+    const type = schema.type;
+    if (typeof type === 'string') {
+        return type;
+    }
+    if (Array.isArray(type)) {
+        // Return first non-null type, or 'any' if all null
+        const nonNull = type.find(t => t !== 'null');
+        return nonNull || 'null';
+    }
+    // Infer from properties
+    if (schema.properties) {
+        return 'object';
+    }
+    if (schema.items) {
+        return 'array';
+    }
+    return 'any';
+}
+/**
+ * Get value type as string.
+ */
+function getValueType(value) {
+    if (value === null) {
+        return 'null';
+    }
+    if (Array.isArray(value)) {
+        return 'array';
+    }
+    return typeof value;
+}
+/**
+ * Check if actual type is compatible with expected type.
+ */
+function isTypeCompatible(actualType, expectedType, schema, value) {
+    // Exact match
+    if (actualType === expectedType) {
+        // For integer type, verify it's actually an integer (not just a number)
+        if (expectedType === 'integer' && typeof value === 'number') {
+            return Number.isInteger(value);
+        }
+        return true;
+    }
+    // Null compatibility
+    if (actualType === 'null' && isNullable(schema)) {
+        return true;
+    }
+    // Number/integer compatibility
+    if (expectedType === 'number' && actualType === 'number') {
+        return true;
+    }
+    if (expectedType === 'integer' && actualType === 'number') {
+        // Integer requires the value to be a whole number
+        return Number.isInteger(value);
+    }
+    // Array compatibility
+    if (expectedType === 'array' && actualType === 'array') {
+        return true;
+    }
+    // Object compatibility
+    if (expectedType === 'object' && actualType === 'object') {
+        return true;
+    }
+    // Any type accepts anything
+    if (expectedType === 'any') {
+        return true;
+    }
+    return false;
+}
+/**
+ * Check if schema is nullable.
+ */
+function isNullable(schema) {
+    if (!isJSONSchemaField(schema)) {
+        return false;
+    }
+    if (schema.nullable === true) {
+        return true;
+    }
+    const type = schema.type;
+    if (Array.isArray(type)) {
+        return type.includes('null');
+    }
+    return false;
+}
+//# sourceMappingURL=output-validator.js.map

package/dist/runtime/test/retry-tester.d.ts

@@ -0,0 +1,44 @@
+/**
+ * Retry scenario tester.
+ * Tests tools against agent retry patterns (retry on failure, timeout, error).
+ */
+import type { SandboxExecutor, ToolExecutionResult } from '../../runtime/sandbox/index.js';
+/**
+ * Retry test configuration.
+ */
+export interface RetryTestConfig {
+    /** Maximum number of retries to test (default: 3) */
+    maxRetries?: number;
+    /** Retry delay in milliseconds (default: 100) */
+    retryDelayMs?: number;
+    /** Test retry on failure (default: true) */
+    testRetryOnFailure?: boolean;
+    /** Test retry on timeout (default: true) */
+    testRetryOnTimeout?: boolean;
+    /** Test retry on connection error (default: true) */
+    testRetryOnConnectionError?: boolean;
+}
+/**
+ * Retry test result.
+ */
+export interface RetryTestResult {
+    /** Input tested */
+    testInput: Record<string, unknown>;
+    /** Retry attempts made */
+    attempts: number;
+    /** Whether tool eventually succeeded after retries */
+    eventuallySucceeded: boolean;
+    /** Results from all retry attempts */
+    results: ToolExecutionResult[];
+    /** Issues detected */
+    issues: string[];
+}
+/**
+ * Test tool retry behavior.
+ * Simulates agent retry patterns to ensure tools handle retries gracefully.
+ *
+ * Note: Idempotency (output consistency across retries) is checked separately
+ * This tester focuses on retry reliability.
+ */
+export declare function testRetryBehavior(executor: SandboxExecutor, toolName: string, testInput: Record<string, unknown>, config?: RetryTestConfig, toolTimeoutMs?: number): Promise<RetryTestResult>;
+//# sourceMappingURL=retry-tester.d.ts.map

package/dist/runtime/test/retry-tester.js

@@ -0,0 +1,103 @@
+/**
+ * Retry scenario tester.
+ * Tests tools against agent retry patterns (retry on failure, timeout, error).
+ */
+import { ToolExecutionErrorType } from '../../runtime/sandbox/index.js';
+/**
+ * Get error type from an unknown error object.
+ * @param error - Error object (may be ToolExecutionError or unknown)
+ * @returns ToolExecutionErrorType if error has errorType, undefined otherwise
+ */
+function getErrorType(error) {
+    if (error && typeof error === 'object' && 'errorType' in error) {
+        return error.errorType;
+    }
+    return undefined;
+}
+/**
+ * Test tool retry behavior.
+ * Simulates agent retry patterns to ensure tools handle retries gracefully.
+ *
+ * Note: Idempotency (output consistency across retries) is checked separately
+ * This tester focuses on retry reliability.
+ */
+export async function testRetryBehavior(executor, toolName, testInput, config = {}, toolTimeoutMs) {
+    const maxRetries = config.maxRetries ?? 3;
+    const retryDelayMs = config.retryDelayMs ?? 100;
+    const testRetryOnFailure = config.testRetryOnFailure ?? true;
+    const testRetryOnTimeout = config.testRetryOnTimeout ?? true;
+    const testRetryOnConnectionError = config.testRetryOnConnectionError ?? true;
+    const results = [];
+    const issues = [];
+    let eventuallySucceeded = false;
+    // Run initial execution
+    let executionResults = await executor.executeTool(toolName, [testInput], toolTimeoutMs);
+    let lastResult = executionResults[0];
+    if (lastResult) {
+        results.push(lastResult);
+    }
+    // If first attempt succeeded, test retry behavior
+    if (lastResult && lastResult.success) {
+        eventuallySucceeded = true;
+        // Retry to check reliability
+        for (let attempt = 1; attempt < maxRetries; attempt++) {
+            await new Promise(resolve => setTimeout(resolve, retryDelayMs));
+            executionResults = await executor.executeTool(toolName, [testInput], toolTimeoutMs);
+            const retryResult = executionResults[0];
+            if (retryResult) {
+                results.push(retryResult);
+                // Check reliability (retry shouldn't fail if first attempt succeeded)
+                if (!retryResult.success) {
+                    issues.push(`Retry ${attempt + 1} failed (was successful on first attempt) - tool may be unreliable under retry scenarios`);
+                }
+                // Note: We don't report idempotency issues here as tools can be idempotent by design
+            }
+        }
+    }
+    else if (lastResult && !lastResult.success) {
+        // First attempt failed - test retry on failure
+        if (testRetryOnFailure) {
+            const errorType = getErrorType(lastResult.error);
+            const shouldRetry = errorType === ToolExecutionErrorType.EXECUTION_ERROR ||
+                errorType === ToolExecutionErrorType.UNKNOWN;
+            if (shouldRetry) {
+                // Retry on failure
+                for (let attempt = 1; attempt < maxRetries; attempt++) {
+                    await new Promise(resolve => setTimeout(resolve, retryDelayMs));
+                    executionResults = await executor.executeTool(toolName, [testInput], toolTimeoutMs);
+                    const retryResult = executionResults[0];
+                    if (retryResult) {
+                        results.push(retryResult);
+                        if (retryResult.success) {
+                            eventuallySucceeded = true;
+                            issues.push(`Tool recovered after ${attempt + 1} retry attempts (initially failed)`);
+                            break;
+                        }
+                    }
+                }
+                if (!eventuallySucceeded) {
+                    issues.push(`Tool failed consistently after ${maxRetries} retry attempts`);
+                }
+            }
+        }
+        // Test retry on timeout
+        if (testRetryOnTimeout && lastResult.timedOut === true) {
+            issues.push(`Tool timed out - agents may retry, causing delays`);
+        }
+        // Test retry on connection error
+        if (testRetryOnConnectionError && !lastResult.success) {
+            const errorType = getErrorType(lastResult.error);
+            if (errorType === ToolExecutionErrorType.CONNECTION_ERROR) {
+                issues.push(`Connection error detected - agents may retry, causing delays`);
+            }
+        }
+    }
+    return {
+        testInput,
+        attempts: results.length,
+        eventuallySucceeded,
+        results,
+        issues,
+    };
+}
+//# sourceMappingURL=retry-tester.js.map

package/dist/runtime/test/runner.d.ts

@@ -0,0 +1,28 @@
+/**
+ * Test input runner.
+ * Executes tools with contract-defined test inputs.
+ */
+import type { ToolContract } from './contract-types.js';
+import type { SandboxExecutor, ToolExecutionResult } from '../../runtime/sandbox/index.js';
+/**
+ * Test execution result with metadata.
+ */
+export type TestExecutionResult = ToolExecutionResult & {
+    /** Test case name */
+    testName: string;
+    /** Test input used */
+    testInput: Record<string, unknown>;
+    /** Test expectation */
+    expectation?: import('./contract-types.js').TestExpectation;
+    /** Expected output schema name (optional, for tests expecting different output schema) */
+    expectedOutputSchema?: string;
+};
+/**
+ * Run contract-defined test cases.
+ * @param executor - Sandbox executor
+ * @param contract - Tool contract
+ * @param toolTimeoutMs - Optional timeout for tool execution in milliseconds
+ * @returns Array of test execution results
+ */
+export declare function runContractTests(executor: SandboxExecutor, contract: ToolContract, toolTimeoutMs?: number): Promise<TestExecutionResult[]>;
+//# sourceMappingURL=runner.d.ts.map

package/dist/runtime/test/runner.js

@@ -0,0 +1,55 @@
+/**
+ * Test input runner.
+ * Executes tools with contract-defined test inputs.
+ */
+/**
+ * Run contract-defined test cases.
+ * @param executor - Sandbox executor
+ * @param contract - Tool contract
+ * @param toolTimeoutMs - Optional timeout for tool execution in milliseconds
+ * @returns Array of test execution results
+ */
+export async function runContractTests(executor, contract, toolTimeoutMs) {
+    if (!contract.tests || contract.tests.length === 0) {
+        return [];
+    }
+    const results = [];
+    for (const test of contract.tests) {
+        // Set environment variables if specified
+        const originalEnv = { ...process.env };
+        if (test.env) {
+            for (const [key, value] of Object.entries(test.env)) {
+                process.env[key] = value;
+            }
+        }
+        try {
+            // Execute tool with test input (run once)
+            const executionResults = await executor.executeTool(contract.tool, [test.input], toolTimeoutMs);
+            // Map to test execution results
+            for (const result of executionResults) {
+                results.push({
+                    ...result,
+                    testName: test.name,
+                    testInput: test.input,
+                    expectation: test.expect,
+                });
+            }
+        }
+        finally {
+            // Restore environment properly (avoid reassigning process.env which breaks Node's proxy)
+            // Remove any keys that were added
+            const currentKeys = Object.keys(process.env);
+            for (const key of currentKeys) {
+                if (!(key in originalEnv)) {
+                    delete process.env[key];
+                }
+            }
+            // Restore any keys that were changed
+            for (const [key, value] of Object.entries(originalEnv)) {
+                process.env[key] = value;
+            }
+        }
+    }
+    return results;
+}
+//# sourceMappingURL=runner.js.map

package/dist/runtime/test/synthetic-input-generator.d.ts

@@ -0,0 +1,11 @@
+/**
+ * Generate system-level synthetic test inputs from tool input schema.
+ * These inputs test edge cases, type validations, and boundary conditions.
+ */
+/**
+ * Generate synthetic test inputs from a JSON schema.
+ * @param schema - Input schema (JSON Schema format)
+ * @returns Array of synthetic test inputs
+ */
+export declare function generateSyntheticInputs(schema: unknown): Array<Record<string, unknown>>;
+//# sourceMappingURL=synthetic-input-generator.d.ts.map