@syrin/cli 1.3.0

package/dist/runtime/test/contract-schema.d.ts

@@ -0,0 +1,46 @@
+/**
+ * Contract schema validation using Zod.
+ * Provides runtime validation for tool contract YAML files.
+ */
+import { z } from 'zod';
+import type { ToolContract } from './contract-types.js';
+/**
+ * Schema for contract test cases.
+ * Note: Inline import is intentional to avoid circular dependency with contract-types.
+ */
+import type { ContractTest } from './contract-types.js';
+/**
+ * Schema for tool unit contract.
+ */
+export declare const ToolContractSchema: z.ZodObject<{
+    version: z.ZodLiteral<1>;
+    tool: z.ZodString;
+    contract: z.ZodObject<{
+        input_schema: z.ZodString;
+        output_schema: z.ZodString;
+    }, z.core.$strip>;
+    guarantees: z.ZodObject<{
+        deterministic: z.ZodOptional<z.ZodBoolean>;
+        side_effects: z.ZodOptional<z.ZodEnum<{
+            none: "none";
+            filesystem: "filesystem";
+        }>>;
+        max_output_size: z.ZodOptional<z.ZodString>;
+        max_execution_time: z.ZodOptional<z.ZodString>;
+    }, z.core.$strip>;
+    tests: z.ZodOptional<z.ZodArray<z.ZodType<ContractTest, unknown, z.core.$ZodTypeInternals<ContractTest, unknown>>>>;
+}, z.core.$strict>;
+/**
+ * Validate a contract object against the schema.
+ * @param data - Contract data to validate
+ * @returns Validated contract
+ * @throws {z.ZodError} If validation fails
+ */
+export declare function validateContract(data: unknown): ToolContract;
+/**
+ * Type guard to check if an object is a valid contract.
+ * @param data - Object to check
+ * @returns true if object is a valid contract
+ */
+export declare function isContract(data: unknown): data is ToolContract;
+//# sourceMappingURL=contract-schema.d.ts.map

package/dist/runtime/test/contract-schema.js

@@ -0,0 +1,107 @@
+/**
+ * Contract schema validation using Zod.
+ * Provides runtime validation for tool contract YAML files.
+ */
+import { z } from 'zod';
+/**
+ * Schema for contract guarantees.
+ */
+const ContractGuaranteesSchema = z.object({
+    deterministic: z.boolean().optional(),
+    side_effects: z.enum(['none', 'filesystem']).optional(),
+    max_output_size: z
+        .string()
+        .regex(/^\d+[kmgt]?b$/i, 'max_output_size must be in format: <number>[unit] (e.g., "100kb", "1mb", "2gb")')
+        .optional(),
+    max_execution_time: z
+        .string()
+        .regex(/^\d+[smhd]$/i, 'max_execution_time must be in format: <number><unit> (e.g., "30s", "5m", "2h")')
+        .optional(),
+});
+/**
+ * Schema for error expectation details.
+ */
+const ErrorExpectationDetailsSchema = z
+    .record(z.string(), z.unknown())
+    .optional();
+/**
+ * Schema for error expectation.
+ */
+const ErrorExpectationSchema = z
+    .object({
+    code: z.string().optional(), // Made optional - match by type/details instead
+    type: z
+        .string()
+        .optional()
+        .refine(val => !val ||
+        [
+            'input_validation',
+            'output_validation',
+            'execution_error',
+            'output_explosion',
+            'unbounded_execution',
+            'side_effect',
+        ].includes(val), {
+        message: 'Error type must be one of: input_validation, output_validation, execution_error, output_explosion, unbounded_execution, side_effect',
+    }),
+    details: ErrorExpectationDetailsSchema,
+})
+    .refine(data => data.code || data.type || data.details, {
+    message: 'Error expectation must specify at least one of: code, type, or details',
+});
+/**
+ * Schema for test expectation.
+ */
+const TestExpectationSchema = z
+    .object({
+    success: z.boolean().optional(),
+    output_schema: z.string().optional(),
+    error: ErrorExpectationSchema.optional(),
+})
+    .refine(data => !(data.success === true && data.error), {
+    message: 'Cannot specify both success=true and error expectation',
+    path: ['success'],
+})
+    .refine(data => !(data.success === false && !data.error), {
+    message: 'If success=false, must specify error expectation',
+    path: ['success'],
+});
+const ContractTestSchema = z.object({
+    name: z.string().min(1, 'Test name is required'),
+    input: z.record(z.string(), z.unknown()),
+    expect: TestExpectationSchema.optional(),
+    env: z.record(z.string(), z.string()).optional(),
+});
+/**
+ * Schema for tool unit contract.
+ */
+export const ToolContractSchema = z
+    .object({
+    version: z.literal(1),
+    tool: z.string().min(1, 'Tool name is required'),
+    contract: z.object({
+        input_schema: z.string().min(1, 'Input schema name is required'),
+        output_schema: z.string().min(1, 'Output schema name is required'),
+    }),
+    guarantees: ContractGuaranteesSchema,
+    tests: z.array(ContractTestSchema).optional(),
+})
+    .strict();
+/**
+ * Validate a contract object against the schema.
+ * @param data - Contract data to validate
+ * @returns Validated contract
+ * @throws {z.ZodError} If validation fails
+ */
+export function validateContract(data) {
+    return ToolContractSchema.parse(data);
+}
+/**
+ * Type guard to check if an object is a valid contract.
+ * @param data - Object to check
+ * @returns true if object is a valid contract
+ */
+export function isContract(data) {
+    return ToolContractSchema.safeParse(data).success;
+}
+//# sourceMappingURL=contract-schema.js.map

package/dist/runtime/test/contract-types.d.ts

@@ -0,0 +1,106 @@
+/**
+ * Contract type definitions for tool unit contracts.
+ * These types represent the structure of tool contract YAML files.
+ */
+/**
+ * Tool unit contract version.
+ * Currently only version 1 is supported.
+ */
+export type ContractVersion = 1;
+/**
+ * Guarantees that a tool must satisfy.
+ */
+export interface ContractGuarantees {
+    /** Whether the tool produces deterministic output (same input → same output) */
+    deterministic?: boolean;
+    /** Side effects allowed: 'none' means no side effects, 'filesystem' allows temp dir writes */
+    side_effects?: 'none' | 'filesystem';
+    /** Maximum output size in bytes (e.g., '50kb', '1mb') */
+    max_output_size?: string;
+    /** Maximum execution time (e.g., '30s', '5m', '2h'). If not specified, uses global default timeout */
+    max_execution_time?: string;
+}
+/**
+ * Error expectation details for test cases.
+ */
+export interface ErrorExpectationDetails {
+    /** Pydantic error type (e.g., 'missing_argument', 'string_type') */
+    error_type?: string;
+    /** Field name that caused the error */
+    field?: string;
+    /** Expected type/value */
+    expected?: string;
+    /** Actual type/value received */
+    received?: string;
+    /** Error message pattern to match */
+    message?: string;
+    /** Additional error details (flexible for different error types) */
+    [key: string]: unknown;
+}
+/**
+ * Error expectation for test cases.
+ */
+export interface ErrorExpectation {
+    /** Error code (optional - for backward compatibility, but not required) */
+    code?: string;
+    /** Error category (e.g., 'input_validation', 'execution_error', 'output_validation', 'side_effect', 'output_explosion', 'unbounded_execution') */
+    type?: string;
+    /** Detailed error information */
+    details?: ErrorExpectationDetails;
+}
+/**
+ * Expected test outcome.
+ */
+export interface TestExpectation {
+    /** Whether test should succeed (default: true if no error specified) */
+    success?: boolean;
+    /** Output schema name (optional, inherits from contract.output_schema) */
+    output_schema?: string;
+    /** Error expectation (if test is expected to fail) */
+    error?: ErrorExpectation;
+}
+/**
+ * Test case for a tool.
+ */
+export interface ContractTest {
+    /** Test case name */
+    name: string;
+    /** Test input values */
+    input: Record<string, unknown>;
+    /** Expected test outcome */
+    expect?: TestExpectation;
+    /** Environment variables for this test (optional) */
+    env?: Record<string, string>;
+}
+/**
+ * Tool unit contract structure.
+ */
+export interface ToolContract {
+    /** Contract version (must be 1 for v1.3.0) */
+    version: ContractVersion;
+    /** Tool name (must match MCP tool name) */
+    tool: string;
+    /** Contract specifications */
+    contract: {
+        /** Input schema name (references MCP tool's input schema) */
+        input_schema: string;
+        /** Output schema name (references MCP tool's output schema) */
+        output_schema: string;
+    };
+    /** Behavioral guarantees */
+    guarantees: ContractGuarantees;
+    /** Test cases */
+    tests?: ContractTest[];
+}
+/**
+ * Parsed contract file with metadata.
+ */
+export interface ParsedContract extends ToolContract {
+    /** Path to the contract file */
+    filePath: string;
+    /** Directory containing the contract file */
+    directory: string;
+    /** Filename without extension */
+    toolName: string;
+}
+//# sourceMappingURL=contract-types.d.ts.map

package/dist/runtime/test/contract-types.js

@@ -0,0 +1,6 @@
+/**
+ * Contract type definitions for tool unit contracts.
+ * These types represent the structure of tool contract YAML files.
+ */
+export {};
+//# sourceMappingURL=contract-types.js.map

package/dist/runtime/test/dependency-tracker.d.ts

@@ -0,0 +1,66 @@
+/**
+ * Hidden dependency tracker.
+ * Tracks tool calls during execution to detect hidden dependencies.
+ */
+/**
+ * Tool call record during execution.
+ */
+export interface ToolCallRecord {
+    /** Tool name that was called */
+    toolName: string;
+    /** Timestamp */
+    timestamp: number;
+    /** Arguments passed */
+    arguments?: Record<string, unknown>;
+}
+/**
+ * Dependency tracking result.
+ */
+export interface DependencyTrackingResult {
+    /** Whether hidden dependencies were detected */
+    detected: boolean;
+    /** List of tool calls that are not declared in contract */
+    hiddenDependencies: ToolCallRecord[];
+    /** List of declared dependencies that don't exist in the MCP server */
+    missingDependencies: string[];
+    /** Declared dependencies from contract */
+    declaredDependencies: string[];
+}
+/**
+ * Dependency tracker for monitoring tool calls.
+ * Note: This requires integration with MCP client to intercept tool calls.
+ * For v1.3.0, we'll track via a wrapper or client extension.
+ */
+export declare class DependencyTracker {
+    private toolCalls;
+    private readonly declaredDependencies;
+    constructor(declaredDependencies?: string[]);
+    /**
+     * Record a tool call.
+     */
+    recordToolCall(toolName: string, arguments_?: Record<string, unknown>): void;
+    /**
+     * Get all tool calls.
+     */
+    getToolCalls(): readonly ToolCallRecord[];
+    /**
+     * Detect hidden dependencies (tool calls not declared in contract).
+     * Also validates that declared dependencies actually exist.
+     * @param availableTools - Set of available tool names from MCP server (optional, for validation)
+     */
+    detectHiddenDependencies(availableTools?: Set<string> | string[]): DependencyTrackingResult;
+    /**
+     * Clear all recorded tool calls.
+     */
+    clear(): void;
+    /**
+     * Get summary of dependencies.
+     */
+    getSummary(availableTools?: Set<string> | string[]): {
+        totalCalls: number;
+        uniqueTools: number;
+        hiddenDependencies: number;
+        missingDependencies: number;
+    };
+}
+//# sourceMappingURL=dependency-tracker.d.ts.map

package/dist/runtime/test/dependency-tracker.js

@@ -0,0 +1,80 @@
+/**
+ * Hidden dependency tracker.
+ * Tracks tool calls during execution to detect hidden dependencies.
+ */
+/**
+ * Dependency tracker for monitoring tool calls.
+ * Note: This requires integration with MCP client to intercept tool calls.
+ * For v1.3.0, we'll track via a wrapper or client extension.
+ */
+export class DependencyTracker {
+    toolCalls = [];
+    declaredDependencies;
+    constructor(declaredDependencies = []) {
+        this.declaredDependencies = declaredDependencies;
+    }
+    /**
+     * Record a tool call.
+     */
+    recordToolCall(toolName, arguments_) {
+        this.toolCalls.push({
+            toolName,
+            timestamp: Date.now(),
+            arguments: arguments_,
+        });
+    }
+    /**
+     * Get all tool calls.
+     */
+    getToolCalls() {
+        return this.toolCalls;
+    }
+    /**
+     * Detect hidden dependencies (tool calls not declared in contract).
+     * Also validates that declared dependencies actually exist.
+     * @param availableTools - Set of available tool names from MCP server (optional, for validation)
+     */
+    detectHiddenDependencies(availableTools) {
+        // Check for hidden dependencies (tools called but not declared)
+        const hiddenDependencies = this.toolCalls.filter(call => !this.declaredDependencies.includes(call.toolName));
+        // Check for missing dependencies (declared but don't exist in MCP server)
+        const missingDependencies = [];
+        if (availableTools) {
+            const availableSet = Array.isArray(availableTools)
+                ? new Set(availableTools)
+                : availableTools;
+            for (const declaredDep of this.declaredDependencies) {
+                if (!availableSet.has(declaredDep)) {
+                    missingDependencies.push(declaredDep);
+                }
+            }
+        }
+        const detected = hiddenDependencies.length > 0 || missingDependencies.length > 0;
+        return {
+            detected,
+            hiddenDependencies,
+            missingDependencies,
+            declaredDependencies: this.declaredDependencies,
+        };
+    }
+    /**
+     * Clear all recorded tool calls.
+     */
+    clear() {
+        this.toolCalls = [];
+    }
+    /**
+     * Get summary of dependencies.
+     */
+    getSummary(availableTools) {
+        const uniqueTools = new Set(this.toolCalls.map(c => c.toolName));
+        const result = this.detectHiddenDependencies(availableTools);
+        return {
+            totalCalls: this.toolCalls.length,
+            uniqueTools: uniqueTools.size,
+            hiddenDependencies: result.hiddenDependencies.length,
+            missingDependencies: result.missingDependencies.length,
+        };
+    }
+}
+//# sourceMappingURL=dependency-tracker.js.map

package/dist/runtime/test/formatters.d.ts

@@ -0,0 +1,18 @@
+/**
+ * Test result formatters.
+ * Formats test results for CLI and JSON output.
+ */
+import type { TestOrchestratorResult } from './orchestrator.js';
+/**
+ * Format test results for CI output (minimal).
+ */
+export declare function formatCIResults(result: TestOrchestratorResult): void;
+/**
+ * Format test results for CLI output.
+ */
+export declare function formatCLIResults(result: TestOrchestratorResult): void;
+/**
+ * Format test results as JSON.
+ */
+export declare function formatJSONResults(result: TestOrchestratorResult): string;
+//# sourceMappingURL=formatters.d.ts.map

package/dist/runtime/test/formatters.js

@@ -0,0 +1,172 @@
+/**
+ * Test result formatters.
+ * Formats test results for CLI and JSON output.
+ */
+import { log } from '../../utils/logger.js';
+/**
+ * Format test results for CI output (minimal).
+ */
+export function formatCIResults(result) {
+    const errors = result.diagnostics.filter(d => d.severity === 'error');
+    const warnings = result.diagnostics.filter(d => d.severity === 'warning');
+    // CI mode: minimal output, exit codes handled by caller
+    if (result.verdict === 'fail') {
+        console.log(`✗ Test failed: ${result.toolsFailed} tool(s) failed, ${errors.length} error(s), ${warnings.length} warning(s)`);
+        // Print errors only (no warnings in CI)
+        for (const error of errors) {
+            console.log(`${error.code}: ${error.message}`);
+        }
+    }
+    else {
+        console.log(`✓ Test passed: ${result.toolsTested} tool(s) tested, ${result.toolsPassed} passed`);
+        if (warnings.length > 0) {
+            console.log(`  ${warnings.length} warning(s) (non-blocking)`);
+        }
+    }
+}
+/**
+ * Format test results for CLI output.
+ */
+export function formatCLIResults(result) {
+    log.blank();
+    log.heading('Tool Validation Results');
+    log.blank();
+    // Overall verdict
+    if (result.verdict === 'pass') {
+        log.checkmark('All tools passed validation');
+    }
+    else if (result.verdict === 'pass-with-warnings') {
+        log.warn('Tools passed with warnings');
+    }
+    else {
+        log.xmark('Tool validation failed');
+    }
+    log.blank();
+    // Summary
+    log.heading('Summary:');
+    log.labelValue('  Tools tested:', String(result.toolsTested));
+    log.labelValue('  Tools passed:', String(result.toolsPassed));
+    log.labelValue('  Tools failed:', String(result.toolsFailed));
+    const errors = result.diagnostics.filter(d => d.severity === 'error');
+    const warnings = result.diagnostics.filter(d => d.severity === 'warning');
+    if (errors.length > 0) {
+        log.labelValue('  Errors:', String(errors.length));
+    }
+    if (warnings.length > 0) {
+        log.labelValue('  Warnings:', String(warnings.length));
+    }
+    log.blank();
+    // Tool results
+    if (result.toolResults.length > 0) {
+        log.heading('Tool Results:');
+        for (const toolResult of result.toolResults) {
+            log.blank();
+            const status = toolResult.passed ? log.tick() : log.cross();
+            log.plain(`  ${status} ${log.styleText(toolResult.toolName, 'bold')}`);
+            // Summary
+            log.labelValue('    Executions:', `${toolResult.summary.successfulExecutions}/${toolResult.summary.totalExecutions} successful`);
+            if (toolResult.summary.testsPassed !== undefined &&
+                toolResult.summary.testsFailed !== undefined) {
+                const totalTests = toolResult.summary.testsPassed + toolResult.summary.testsFailed;
+                log.labelValue('    Tests:', `${toolResult.summary.testsPassed}/${totalTests} passed (expectations matched)`);
+            }
+            if (toolResult.summary.failedExecutions > 0) {
+                log.labelValue('    Failed:', String(toolResult.summary.failedExecutions));
+            }
+            if (toolResult.summary.timedOutExecutions > 0) {
+                log.labelValue('    Timed out:', String(toolResult.summary.timedOutExecutions));
+            }
+            // Diagnostics
+            if (toolResult.diagnostics.length > 0) {
+                const toolErrors = toolResult.diagnostics.filter(d => d.severity === 'error');
+                const toolWarnings = toolResult.diagnostics.filter(d => d.severity === 'warning');
+                if (toolErrors.length > 0) {
+                    log.plain('    Errors:');
+                    for (const error of toolErrors) {
+                        log.error(`      ${error.code}: ${error.message}`);
+                        // Show test context if available
+                        const context = error.context;
+                        if (context?.testName) {
+                            log.plain(`        Test: ${context.testName}`);
+                        }
+                        if (context?.testInput &&
+                            Object.keys(context.testInput).length > 0) {
+                            const inputStr = JSON.stringify(context.testInput, null, 2)
+                                .split('\n')
+                                .map((line, idx) => (idx === 0 ? line : `        ${line}`))
+                                .join('\n');
+                            log.plain(`        Input: ${inputStr}`);
+                        }
+                        if (context?.expectedOutputSchema) {
+                            log.plain(`        Expected output schema: ${context.expectedOutputSchema}`);
+                        }
+                        // Show parsed error details if available
+                        if (context?.parsedError) {
+                            const parsed = context.parsedError;
+                            if (parsed.field) {
+                                if (parsed.errorType === 'missing_argument') {
+                                    log.plain(`        Field "${parsed.field}" is required but was not provided`);
+                                }
+                                else if (parsed.inputType) {
+                                    log.plain(`        Field "${parsed.field}" received ${parsed.inputType}${parsed.inputValue && parsed.inputValue !== '{}' ? ` (${parsed.inputValue})` : ''}`);
+                                }
+                            }
+                        }
+                        if (error.suggestion) {
+                            log.plain(`        Suggestion: ${error.suggestion}`);
+                        }
+                        log.blank();
+                    }
+                }
+                if (toolWarnings.length > 0) {
+                    log.plain('    Warnings:');
+                    for (const warning of toolWarnings) {
+                        log.warn(`      ${warning.code}: ${warning.message}`);
+                        if (warning.suggestion) {
+                            log.plain(`        Suggestion: ${warning.suggestion}`);
+                        }
+                    }
+                }
+            }
+        }
+    }
+    log.blank();
+}
+/**
+ * Format test results as JSON.
+ */
+export function formatJSONResults(result) {
+    const json = {
+        verdict: result.verdict,
+        summary: {
+            toolsTested: result.toolsTested,
+            toolsPassed: result.toolsPassed,
+            toolsFailed: result.toolsFailed,
+            errors: result.diagnostics.filter(d => d.severity === 'error').length,
+            warnings: result.diagnostics.filter(d => d.severity === 'warning').length,
+        },
+        diagnostics: result.diagnostics.map(d => ({
+            code: d.code,
+            severity: d.severity,
+            message: d.message,
+            tool: d.tool,
+            field: d.field,
+            suggestion: d.suggestion,
+            context: d.context,
+        })),
+        tools: result.toolResults.map(tr => ({
+            toolName: tr.toolName,
+            passed: tr.passed,
+            summary: tr.summary,
+            diagnostics: tr.diagnostics.map(d => ({
+                code: d.code,
+                severity: d.severity,
+                message: d.message,
+                suggestion: d.suggestion,
+                context: d.context,
+            })),
+        })),
+    };
+    return JSON.stringify(json, null, 2);
+}
+//# sourceMappingURL=formatters.js.map

package/dist/runtime/test/index.d.ts

@@ -0,0 +1,12 @@
+/**
+ * Test execution engine exports.
+ * v1.3.0: Tool-level structural safety validation.
+ */
+export * from './contract-types.js';
+export * from './contract-schema.js';
+export * from './contract-loader.js';
+export * from './behavior-observer.js';
+export * from './output-validator.js';
+export * from './runner.js';
+export * from './orchestrator.js';
+//# sourceMappingURL=index.d.ts.map

package/dist/runtime/test/index.js

@@ -0,0 +1,13 @@
+/**
+ * Test execution engine exports.
+ * v1.3.0: Tool-level structural safety validation.
+ */
+export * from './contract-types.js';
+export * from './contract-schema.js';
+export * from './contract-loader.js';
+// Sandbox exports are now in @/runtime/sandbox
+export * from './behavior-observer.js';
+export * from './output-validator.js';
+export * from './runner.js';
+export * from './orchestrator.js';
+//# sourceMappingURL=index.js.map

package/dist/runtime/test/input-generator.d.ts

@@ -0,0 +1,33 @@
+/**
+ * Synthetic input generator.
+ * Generates test inputs from JSON Schema definitions.
+ */
+/**
+ * JSON Schema field interface.
+ */
+interface JSONSchemaField {
+    type?: string | string[];
+    description?: string;
+    required?: string[];
+    properties?: Record<string, JSONSchemaField>;
+    items?: JSONSchemaField | JSONSchemaField[];
+    $ref?: string;
+    enum?: unknown[];
+    pattern?: string;
+    example?: unknown;
+    examples?: unknown[];
+    nullable?: boolean;
+    default?: unknown;
+    minimum?: number;
+    maximum?: number;
+}
+type JSONSchema = JSONSchemaField | Record<string, unknown> | unknown;
+/**
+ * Generate test inputs from a JSON Schema.
+ * Includes both predictable patterns and agent-like unpredictable variations.
+ * @param schema - JSON Schema definition
+ * @returns Array of generated input objects
+ */
+export declare function generateInputsFromSchema(schema: JSONSchema): Array<Record<string, unknown>>;
+export {};
+//# sourceMappingURL=input-generator.d.ts.map