@syrin/cli 1.3.0

package/dist/runtime/test/orchestrator.d.ts

@@ -0,0 +1,131 @@
+/**
+ * Test orchestrator.
+ * Coordinates contract loading, sandboxed execution, and rule evaluation.
+ */
+import type { Diagnostic } from '../../runtime/analysis/types.js';
+import type { ToolContract } from './contract-types.js';
+import type { Command } from '../../types/ids.js';
+/**
+ * Test execution options.
+ */
+export interface TestOrchestratorOptions {
+    /** Project root directory */
+    projectRoot: string;
+    /** Tools directory (overrides config) */
+    toolsDir?: string;
+    /** Specific tool to test (if not provided, tests all) */
+    toolName?: string;
+    /** Path to filter tools (relative to tools directory, e.g., "weather" or "server") */
+    toolPath?: string;
+    /** Timeout in milliseconds (overrides config) */
+    timeout?: number;
+    /** Memory limit in MB (overrides config) */
+    memoryLimitMB?: number;
+    /** Max output size in KB (overrides config) */
+    maxOutputSizeKB?: number;
+    /** Number of determinism runs (overrides config) */
+    determinismRuns?: number;
+    /** Strict mode (treat warnings as errors) */
+    strictMode?: boolean;
+    /** CI mode (suppress verbose logging) */
+    ci?: boolean;
+    /** Show sandbox process error output (stderr from MCP server) */
+    showErrors?: boolean;
+    /** MCP command for stdio transport */
+    mcpCommand?: Command | string;
+    /** Environment variables */
+    env?: Record<string, string>;
+}
+/**
+ * Tool test result.
+ */
+export interface ToolTestResult {
+    /** Tool name */
+    toolName: string;
+    /** Contract used */
+    contract: ToolContract;
+    /** Diagnostics from rules */
+    diagnostics: Diagnostic[];
+    /** Whether test passed */
+    passed: boolean;
+    /** Execution summary */
+    summary: {
+        /** Total number of executions (contract tests) */
+        totalExecutions: number;
+        /** Number of successful executions */
+        successfulExecutions: number;
+        /** Number of failed executions */
+        failedExecutions: number;
+        /** Number of timed out executions */
+        timedOutExecutions: number;
+        /** Number of contract tests that passed (matched expectations) */
+        testsPassed?: number;
+        /** Number of contract tests that failed (didn't match expectations) */
+        testsFailed?: number;
+    };
+}
+/**
+ * Test orchestrator result.
+ */
+export interface TestOrchestratorResult {
+    /** Overall verdict */
+    verdict: 'pass' | 'fail' | 'pass-with-warnings';
+    /** All diagnostics */
+    diagnostics: Diagnostic[];
+    /** Tool test results */
+    toolResults: ToolTestResult[];
+    /** Number of tools tested */
+    toolsTested: number;
+    /** Number of tools passed */
+    toolsPassed: number;
+    /** Number of tools failed */
+    toolsFailed: number;
+}
+/**
+ * Test orchestrator for tool validation.
+ */
+export declare class TestOrchestrator {
+    private readonly options;
+    constructor(options: TestOrchestratorOptions);
+    /**
+     * Extract expectedOutputSchema from TestExecutionResult.
+     * @param result - Test execution result
+     * @returns Expected output schema name, or undefined if not present
+     */
+    private getExpectedOutputSchema;
+    /**
+     * Run tool validation tests.
+     */
+    run(): Promise<TestOrchestratorResult>;
+    /**
+     * Test a single tool.
+     */
+    private testTool;
+    /**
+     * Parse Pydantic validation error message to extract key information.
+     */
+    private parseValidationError;
+    /**
+     * Match actual result against test expectation.
+     */
+    private matchExpectation;
+    /**
+     * Match expectation details against actual error.
+     */
+    private matchExpectationDetails;
+    /**
+     * Resolve schema by name from JSON Schema $defs.
+     *
+     * Schema names in contracts are string references to named schemas defined in the
+     * tool's schema $defs section (e.g., "Username", "UserResponse").
+     *
+     * This function looks up the named schema from the root schema's $defs dictionary.
+     * If the schema name is not found in $defs, it falls back to the defaultSchema.
+     *
+     * @param schemaName - Schema name to resolve (e.g., "Username")
+     * @param defaultSchema - The root schema containing $defs (e.g., tool outputSchema)
+     * @returns The resolved schema from $defs, or defaultSchema if not found
+     */
+    private resolveSchemaByName;
+}
+//# sourceMappingURL=orchestrator.d.ts.map