@orchagent/cli 0.3.62 → 0.3.64

package/dist/commands/test.js

@@ -12,11 +12,12 @@ const yaml_1 = __importDefault(require("yaml"));
 const fast_deep_equal_1 = __importDefault(require("fast-deep-equal"));
 const chokidar_1 = __importDefault(require("chokidar"));
 const errors_1 = require("../lib/errors");
+const output_1 = require("../lib/output");
+const dotenv_1 = require("../lib/dotenv");
 const config_1 = require("../lib/config");
 const llm_1 = require("../lib/llm");
-/**
- * Validate a fixture and return helpful errors
- */
+const bundle_1 = require("../lib/bundle");
+// ─── Utility functions ───────────────────────────────────────────────────────
 function validateFixture(data, fixturePath) {
     const fileName = path_1.default.basename(fixturePath);
     if (typeof data !== 'object' || data === null) {
@@ -35,9 +36,6 @@ function validateFixture(data, fixturePath) {
     }
     return data;
 }
-/**
- * Parse SKILL.md frontmatter
- */
 async function parseSkillMd(filePath) {
     try {
         const content = await promises_1.default.readFile(filePath, 'utf-8');
@@ -55,7 +53,8 @@ async function parseSkillMd(filePath) {
     }
 }
 /**
- * Run a command and return the result
+ * Run a shell command (for test runners like pytest/vitest)
+ * Uses shell: true because test runner commands may need PATH resolution
  */
 function runCommand(command, args, cwd, verbose) {
     return new Promise((resolve) => {
@@ -87,9 +86,6 @@ function runCommand(command, args, cwd, verbose) {
         });
     });
 }
-/**
- * Check if a command exists
- */
 async function commandExists(command) {
     const isWindows = process.platform === 'win32';
     const checker = isWindows ? 'where' : 'which';
@@ -104,9 +100,6 @@ async function commandExists(command) {
         return false;
     }
 }
-/**
- * Check if a file exists
- */
 async function fileExists(filePath) {
     try {
         await promises_1.default.access(filePath);
@@ -116,62 +109,278 @@ async function fileExists(filePath) {
         return false;
     }
 }
-/**
- * Detect the agent type from the directory structure
- */
-async function detectAgentType(agentDir) {
-    // Check for SKILL.md first
-    if (await fileExists(path_1.default.join(agentDir, 'SKILL.md'))) {
-        return 'skill';
+function extractTemplateVariables(template) {
+    const seen = new Set();
+    const result = [];
+    const pattern = /\{\{(\w+)\}\}/g;
+    let match;
+    while ((match = pattern.exec(template)) !== null) {
+        if (!seen.has(match[1])) {
+            seen.add(match[1]);
+            result.push(match[1]);
+        }
     }
-    // Check for prompt.md (prompt agent)
-    if (await fileExists(path_1.default.join(agentDir, 'prompt.md'))) {
-        return 'prompt';
+    return result;
+}
+// ─── Validation ──────────────────────────────────────────────────────────────
+function inferEngine(manifest, rawType) {
+    const hasRuntimeCommand = Boolean(manifest.runtime?.command?.trim());
+    const hasLoop = Boolean(manifest.loop && Object.keys(manifest.loop).length > 0);
+    if (hasRuntimeCommand)
+        return 'code_runtime';
+    if (hasLoop)
+        return 'managed_loop';
+    if (rawType === 'tool' || rawType === 'code')
+        return 'code_runtime';
+    if (rawType === 'agentic')
+        return 'managed_loop';
+    if (rawType === 'agent') {
+        if (manifest.custom_tools?.length || manifest.max_turns)
+            return 'managed_loop';
+        return 'managed_loop';
     }
-    // Check for orchagent.json
+    return 'direct_llm';
+}
+function engineLabel(engine) {
+    switch (engine) {
+        case 'direct_llm': return 'prompt';
+        case 'managed_loop': return 'agent loop';
+        case 'code_runtime': return 'code runtime';
+    }
+}
+async function validateAgent(agentDir) {
+    const msgs = [];
+    const err = (text) => msgs.push({ level: 'error', text });
+    const warn = (text) => msgs.push({ level: 'warning', text });
+    const info = (text) => msgs.push({ level: 'info', text });
+    // Check for SKILL.md (skills are a separate path)
+    const skillPath = path_1.default.join(agentDir, 'SKILL.md');
+    const skillData = await parseSkillMd(skillPath);
+    if (skillData) {
+        info(`Skill: ${skillData.frontmatter.name}`);
+        if (!skillData.frontmatter.description) {
+            err('SKILL.md frontmatter missing "description"');
+        }
+        return { messages: msgs, isSkill: true, agentName: skillData.frontmatter.name };
+    }
+    // If SKILL.md exists but is invalid
+    if (await fileExists(skillPath)) {
+        warn('SKILL.md found but has invalid frontmatter (needs name + description in YAML)');
+    }
+    // Read orchagent.json
     const manifestPath = path_1.default.join(agentDir, 'orchagent.json');
-    if (await fileExists(manifestPath)) {
+    let manifest;
+    try {
+        const raw = await promises_1.default.readFile(manifestPath, 'utf-8');
+        manifest = JSON.parse(raw);
+    }
+    catch (e) {
+        if (e.code === 'ENOENT') {
+            err('orchagent.json not found — create one with: orch init');
+        }
+        else {
+            err(`orchagent.json is not valid JSON: ${e.message}`);
+        }
+        return { messages: msgs, isSkill: false };
+    }
+    // Name
+    if (!manifest.name) {
+        err("'name' field is required in orchagent.json");
+    }
+    // Type
+    const rawType = (manifest.type || 'agent').trim().toLowerCase();
+    const validTypes = ['prompt', 'tool', 'agent', 'skill', 'agentic', 'code'];
+    if (!validTypes.includes(rawType)) {
+        err(`Invalid type '${manifest.type}' — use: prompt, tool, or agent`);
+    }
+    if (rawType === 'agentic')
+        warn("Type 'agentic' is deprecated — use 'agent'");
+    if (rawType === 'code')
+        warn("Type 'code' is deprecated — use 'tool'");
+    if (rawType === 'skill')
+        err("Skills should use SKILL.md, not orchagent.json with type='skill'");
+    // Engine inference
+    const hasRuntimeCommand = Boolean(manifest.runtime?.command?.trim());
+    const hasLoop = Boolean(manifest.loop && Object.keys(manifest.loop).length > 0);
+    if (hasRuntimeCommand && hasLoop) {
+        err('runtime.command and loop cannot both be set — choose one execution model');
+    }
+    const engine = inferEngine(manifest, rawType);
+    // run_mode
+    const runMode = (manifest.run_mode || 'on_demand').trim().toLowerCase();
+    if (runMode !== 'on_demand' && runMode !== 'always_on') {
+        err("run_mode must be 'on_demand' or 'always_on'");
+    }
+    if (runMode === 'always_on' && engine === 'direct_llm') {
+        err('run_mode=always_on requires runtime.command or loop configuration');
+    }
+    // File structure: prompt.md
+    const hasPrompt = await fileExists(path_1.default.join(agentDir, 'prompt.md'));
+    if ((engine === 'direct_llm' || engine === 'managed_loop') && !hasPrompt) {
+        const label = engine === 'direct_llm' ? 'prompt agents' : 'agent-type agents';
+        err(`prompt.md not found (required for ${label})`);
+    }
+    // File structure: entrypoint for code_runtime
+    let entrypoint;
+    if (engine === 'code_runtime') {
+        entrypoint = manifest.entrypoint || await (0, bundle_1.detectEntrypoint)(agentDir) || undefined;
+        if (!entrypoint && !manifest.runtime?.command && !manifest.source_url) {
+            err('No entrypoint found — create main.py, app.py, or set entrypoint in orchagent.json');
+        }
+        else if (entrypoint && !await fileExists(path_1.default.join(agentDir, entrypoint))) {
+            err(`Entrypoint '${entrypoint}' declared but file not found`);
+        }
+    }
+    // schema.json validity
+    const schemaPath = path_1.default.join(agentDir, 'schema.json');
+    const hasSchema = await fileExists(schemaPath);
+    if (hasSchema) {
         try {
-            const raw = await promises_1.default.readFile(manifestPath, 'utf-8');
-            const manifest = JSON.parse(raw);
-            if (manifest.type === 'prompt')
-                return 'prompt';
-            if (manifest.type === 'skill')
-                return 'skill';
-            if (manifest.type === 'tool') {
-                // Detect language
-                if (await fileExists(path_1.default.join(agentDir, 'requirements.txt')))
-                    return 'code-python';
-                if (await fileExists(path_1.default.join(agentDir, 'pyproject.toml')))
-                    return 'code-python';
-                if (await fileExists(path_1.default.join(agentDir, 'package.json')))
-                    return 'code-js';
-                // Default to Python for tool agents
-                return 'code-python';
+            const raw = await promises_1.default.readFile(schemaPath, 'utf-8');
+            JSON.parse(raw);
+        }
+        catch {
+            err('schema.json contains invalid JSON');
+        }
+    }
+    // Deprecated fields
+    if (manifest.prompt) {
+        warn("'prompt' field in orchagent.json is ignored — use prompt.md file");
+    }
+    if (manifest.input_schema && hasSchema) {
+        warn("'input_schema' in orchagent.json is ignored — schema.json takes priority");
+    }
+    if (manifest.output_schema && hasSchema) {
+        warn("'output_schema' in orchagent.json is ignored — schema.json takes priority");
+    }
+    // Misplaced manifest fields (common error: dependencies at top level instead of under manifest)
+    const orchestrationFields = ['manifest_version', 'dependencies', 'max_hops', 'timeout_ms', 'per_call_downstream_cap'];
+    const misplaced = orchestrationFields.filter(f => (f in manifest) && !manifest.manifest);
+    if (misplaced.length > 0) {
+        err(`Orchestration fields (${misplaced.join(', ')}) must be nested under a "manifest" key`);
+    }
+    // required_secrets
+    if (manifest.required_secrets !== undefined) {
+        if (!Array.isArray(manifest.required_secrets)) {
+            err('required_secrets must be an array of strings');
+        }
+        else {
+            if (manifest.required_secrets.includes('ORCHAGENT_SERVICE_KEY')) {
+                warn('ORCHAGENT_SERVICE_KEY in required_secrets is not needed — the gateway auto-injects it for orchestrator agents');
+            }
+            // Check if secrets are available in local environment
+            const missingSecrets = manifest.required_secrets.filter(s => !process.env[s]);
+            if (missingSecrets.length > 0) {
+                warn(`Required secrets not in local environment: ${missingSecrets.join(', ')} — fixture tests may fail`);
+            }
+        }
+    }
+    // requirements.txt: orchagent vs orchagent-sdk
+    const reqPath = path_1.default.join(agentDir, 'requirements.txt');
+    if (await fileExists(reqPath)) {
+        try {
+            const reqContent = await promises_1.default.readFile(reqPath, 'utf-8');
+            if (/^orchagent\b/m.test(reqContent) && !/^orchagent-sdk\b/m.test(reqContent)) {
+                warn("requirements.txt has 'orchagent' — did you mean 'orchagent-sdk'?");
             }
         }
         catch {
-            // Invalid manifest, continue detection
+            // Can't read, skip
         }
     }
-    // Fallback: detect by file presence
-    if (await fileExists(path_1.default.join(agentDir, 'requirements.txt')))
-        return 'code-python';
-    if (await fileExists(path_1.default.join(agentDir, 'pyproject.toml')))
-        return 'code-python';
-    if (await fileExists(path_1.default.join(agentDir, 'package.json')))
-        return 'code-js';
-    return 'unknown';
+    // custom_tools validation
+    if (engine === 'managed_loop' && manifest.custom_tools) {
+        const reservedNames = new Set(['bash', 'read_file', 'write_file', 'list_files', 'submit_result']);
+        const seenNames = new Set();
+        for (const tool of manifest.custom_tools) {
+            if (!tool.name || !tool.command) {
+                err(`Custom tool missing 'name' or 'command': ${JSON.stringify(tool)}`);
+            }
+            if (tool.name && reservedNames.has(tool.name)) {
+                err(`Custom tool '${tool.name}' conflicts with built-in tool name`);
+            }
+            if (tool.name && seenNames.has(tool.name)) {
+                err(`Duplicate custom tool name: '${tool.name}'`);
+            }
+            if (tool.name)
+                seenNames.add(tool.name);
+        }
+    }
+    // max_turns range
+    if (manifest.max_turns !== undefined) {
+        if (typeof manifest.max_turns !== 'number' || manifest.max_turns < 1 || manifest.max_turns > 50) {
+            err('max_turns must be a number between 1 and 50');
+        }
+    }
+    // Template variable mismatch (prompt.md vars vs schema.json)
+    if ((engine === 'direct_llm' || engine === 'managed_loop') && hasPrompt && hasSchema) {
+        try {
+            const prompt = await promises_1.default.readFile(path_1.default.join(agentDir, 'prompt.md'), 'utf-8');
+            const schemaRaw = await promises_1.default.readFile(schemaPath, 'utf-8');
+            const schemas = JSON.parse(schemaRaw);
+            const templateVars = extractTemplateVariables(prompt);
+            if (templateVars.length > 0 && schemas.input?.properties) {
+                const schemaProps = Object.keys(schemas.input.properties);
+                const missing = templateVars.filter(v => !schemaProps.includes(v));
+                if (missing.length > 0) {
+                    warn(`prompt.md uses {{${missing.join('}}, {{')}}} but schema.json doesn't define ${missing.length === 1 ? 'it' : 'them'}`);
+                }
+            }
+        }
+        catch {
+            // Already caught above
+        }
+    }
+    return {
+        messages: msgs,
+        executionEngine: engine,
+        entrypoint,
+        agentName: manifest.name || undefined,
+        agentType: rawType,
+        isSkill: false,
+    };
 }
-/**
- * Recursively walk a directory and return all files
- */
+function printValidation(validation) {
+    const errors = validation.messages.filter(m => m.level === 'error');
+    const warnings = validation.messages.filter(m => m.level === 'warning');
+    const infos = validation.messages.filter(m => m.level === 'info');
+    process.stderr.write(chalk_1.default.bold('\nValidating agent...\n'));
+    // Agent summary line
+    if (validation.isSkill) {
+        process.stderr.write(`  ${chalk_1.default.bold('Type:')} skill\n`);
+    }
+    else if (validation.agentType && validation.executionEngine) {
+        process.stderr.write(`  ${chalk_1.default.bold('Type:')} ${validation.agentType} (${engineLabel(validation.executionEngine)})\n`);
+        if (validation.entrypoint) {
+            process.stderr.write(`  ${chalk_1.default.bold('Entrypoint:')} ${validation.entrypoint}\n`);
+        }
+    }
+    if (validation.agentName) {
+        process.stderr.write(`  ${chalk_1.default.bold('Name:')} ${validation.agentName}\n`);
+    }
+    process.stderr.write('\n');
+    // Messages
+    for (const msg of errors) {
+        process.stderr.write(chalk_1.default.red(`  ✗ ${msg.text}\n`));
+    }
+    for (const msg of warnings) {
+        process.stderr.write(chalk_1.default.yellow(`  ⚠ ${msg.text}\n`));
+    }
+    for (const msg of infos) {
+        process.stderr.write(chalk_1.default.gray(`  ℹ ${msg.text}\n`));
+    }
+    if (errors.length === 0) {
+        process.stderr.write(chalk_1.default.green('  ✓ Configuration valid\n'));
+    }
+    process.stderr.write('\n');
+    return errors.length === 0;
+}
+// ─── Test discovery ──────────────────────────────────────────────────────────
 async function walkDir(dir, files = []) {
     try {
         const entries = await promises_1.default.readdir(dir, { withFileTypes: true });
         for (const entry of entries) {
             const fullPath = path_1.default.join(dir, entry.name);
-            // Skip common non-source directories
             if (entry.isDirectory()) {
                 if (['node_modules', '__pycache__', '.git', 'dist', 'build', '.venv', 'venv'].includes(entry.name)) {
                     continue;
@@ -188,16 +397,12 @@ async function walkDir(dir, files = []) {
     }
     return files;
 }
-/**
- * Discover test files in the agent directory
- */
 async function discoverTests(agentDir) {
     const result = {
         python: [],
         javascript: [],
         fixtures: [],
     };
-    // Get all files recursively
     const allFiles = await walkDir(agentDir);
     for (const file of allFiles) {
         const basename = path_1.default.basename(file);
@@ -213,7 +418,7 @@ async function discoverTests(agentDir) {
             basename.endsWith('.spec.ts') || basename.endsWith('.spec.js')) {
             result.javascript.push(file);
         }
-        // Fixture patterns: tests/fixture*.json or fixture*.json in tests/ subdirs
+        // Fixture patterns: tests/fixture*.json
         if (basename.endsWith('.json') && basename.startsWith('fixture')) {
             if (relPath.includes('tests' + path_1.default.sep) || relPath.startsWith('tests' + path_1.default.sep)) {
                 result.fixtures.push(file);
@@ -222,19 +427,15 @@ async function discoverTests(agentDir) {
     }
     return result;
 }
-/**
- * Run Python tests using pytest
- */
+// ─── Test runners ────────────────────────────────────────────────────────────
 async function runPythonTests(agentDir, verbose) {
     process.stderr.write(chalk_1.default.blue('\nRunning Python tests...\n\n'));
-    // Check if pytest is available directly
     const hasPytest = await commandExists('pytest');
     if (hasPytest) {
         const args = verbose ? ['-v'] : [];
         const { code } = await runCommand('pytest', args, agentDir, verbose);
         return code;
     }
-    // Try Python commands in order of preference
     const pythonCommands = process.platform === 'win32'
         ? ['python', 'py', 'python3']
         : ['python3', 'python'];
@@ -250,12 +451,8 @@ async function runPythonTests(agentDir, verbose) {
     process.stderr.write(chalk_1.default.red('No Python interpreter found. Install Python and pytest.\n'));
     return 1;
 }
-/**
- * Run JavaScript/TypeScript tests
- */
 async function runJsTests(agentDir, verbose) {
     process.stderr.write(chalk_1.default.blue('\nRunning JavaScript/TypeScript tests...\n\n'));
-    // Check for vitest first
     const hasVitest = await fileExists(path_1.default.join(agentDir, 'node_modules', '.bin', 'vitest'));
     if (hasVitest) {
         const args = ['run'];
@@ -264,7 +461,6 @@ async function runJsTests(agentDir, verbose) {
         const { code } = await runCommand('npx', ['vitest', ...args], agentDir, verbose);
         return code;
     }
-    // Fall back to npm test
     const packageJsonPath = path_1.default.join(agentDir, 'package.json');
     if (await fileExists(packageJsonPath)) {
         try {
@@ -283,15 +479,14 @@ async function runJsTests(agentDir, verbose) {
     return 1;
 }
 /**
- * Run fixture-based tests for prompt agents
+ * Run fixture tests for prompt/skill/managed_loop agents using LLM calls
  */
-async function runFixtureTests(agentDir, fixtures, verbose, config) {
+async function runPromptFixtureTests(agentDir, fixtures, verbose, config) {
     process.stderr.write(chalk_1.default.blue('\nRunning fixture tests...\n\n'));
     // Read prompt
     let prompt;
     const promptPath = path_1.default.join(agentDir, 'prompt.md');
     const skillPath = path_1.default.join(agentDir, 'SKILL.md');
-    // Check if this is a skill
     const skillData = await parseSkillMd(skillPath);
     if (skillData) {
         prompt = skillData.body;
@@ -327,7 +522,8 @@ async function runFixtureTests(agentDir, fixtures, verbose, config) {
     let failed = 0;
     for (const fixturePath of fixtures) {
         const fixtureName = path_1.default.basename(fixturePath);
-        process.stderr.write(`  ${fixtureName}: `);
+        const description = await getFixtureDescription(fixturePath);
+        process.stderr.write(`  ${fixtureName}${description ? ` (${description})` : ''}: `);
         try {
             const raw = await promises_1.default.readFile(fixturePath, 'utf-8');
             let parsed;
@@ -338,10 +534,8 @@ async function runFixtureTests(agentDir, fixtures, verbose, config) {
                 throw new errors_1.CliError(`Invalid JSON in ${path_1.default.basename(fixturePath)}: ${e.message}`);
             }
             const fixture = validateFixture(parsed, fixturePath);
-            // Build and call LLM
             const fullPrompt = (0, llm_1.buildPrompt)(prompt, fixture.input);
             const result = await (0, llm_1.callLlm)(provider, key, model, fullPrompt, outputSchema);
-            // Validate result
             let testPassed = true;
             const failures = [];
             if (fixture.expected_output) {
@@ -351,7 +545,6 @@ async function runFixtureTests(agentDir, fixtures, verbose, config) {
                 }
             }
             if (fixture.expected_contains) {
-                // Check if output contains expected strings
                 const resultStr = JSON.stringify(result);
                 for (const expected of fixture.expected_contains) {
                     if (!resultStr.includes(expected)) {
@@ -387,73 +580,230 @@ async function runFixtureTests(agentDir, fixtures, verbose, config) {
     process.stderr.write(`Fixtures: ${passed} passed, ${failed} failed\n`);
     return failed > 0 ? 1 : 0;
 }
+async function getFixtureDescription(fixturePath) {
+    try {
+        const raw = await promises_1.default.readFile(fixturePath, 'utf-8');
+        const data = JSON.parse(raw);
+        return data.description || null;
+    }
+    catch {
+        return null;
+    }
+}
 /**
- * Watch mode: re-run tests on file changes
+ * Run a code_runtime entrypoint with JSON input on stdin, capture JSON output.
+ * Uses spawn with array args (no shell) to avoid injection risks.
  */
-async function watchTests(agentDir, agentType, verbose, config) {
-    process.stderr.write(chalk_1.default.cyan('\nWatching for file changes... (press Ctrl+C to exit)\n\n'));
-    const runTests = async () => {
-        process.stderr.write(chalk_1.default.dim(`\n[${new Date().toLocaleTimeString()}] Running tests...\n`));
-        // Re-discover tests each time to pick up new files
-        const testFiles = await discoverTests(agentDir);
-        await executeTests(agentDir, agentType, testFiles, verbose, config);
-    };
-    // Initial run
-    await runTests();
-    // Set up chokidar watcher
-    let debounceTimer = null;
-    const onChange = (filePath) => {
-        if (debounceTimer)
-            clearTimeout(debounceTimer);
-        if (verbose) {
-            process.stderr.write(chalk_1.default.dim(`  Changed: ${path_1.default.relative(agentDir, filePath)}\n`));
-        }
-        debounceTimer = setTimeout(runTests, 300);
-    };
-    const watcher = chokidar_1.default.watch(agentDir, {
-        ignored: /(node_modules|__pycache__|\.git|dist|build|\.venv|venv)/,
-        persistent: true,
-        ignoreInitial: true,
-    });
-    watcher
-        .on('change', onChange)
-        .on('add', onChange)
-        .on('unlink', onChange)
-        .on('error', (error) => {
-        const message = error instanceof Error ? error.message : String(error);
-        process.stderr.write(chalk_1.default.red(`Watcher error: ${message}\n`));
+function runEntrypointWithInput(agentDir, entrypoint, stdinData, verbose) {
+    return new Promise((resolve) => {
+        const isJs = entrypoint.endsWith('.js') || entrypoint.endsWith('.ts') ||
+            entrypoint.endsWith('.mjs') || entrypoint.endsWith('.cjs');
+        const cmd = isJs ? 'node' : 'python3';
+        const proc = (0, child_process_1.spawn)(cmd, [entrypoint], {
+            cwd: agentDir,
+            stdio: ['pipe', 'pipe', 'pipe'],
+            env: { ...process.env, ORCHAGENT_LOCAL_EXECUTION: 'true' },
+        });
+        let stdout = '';
+        let stderr = '';
+        proc.stdout?.on('data', (data) => {
+            stdout += data.toString();
+        });
+        proc.stderr?.on('data', (data) => {
+            const text = data.toString();
+            stderr += text;
+            if (verbose) {
+                process.stderr.write(chalk_1.default.gray(text));
+            }
+        });
+        // Write input to stdin and close
+        proc.stdin?.write(stdinData);
+        proc.stdin?.end();
+        proc.on('close', (code) => {
+            resolve({ code: code ?? 1, stdout, stderr });
+        });
+        proc.on('error', (err) => {
+            resolve({ code: 1, stdout, stderr: err.message });
+        });
     });
-    // Keep process alive
-    await new Promise(() => { });
 }
 /**
- * Execute tests based on agent type and discovered test files
+ * Run fixture tests for code_runtime agents by executing the entrypoint
+ * with fixture input as stdin and validating the JSON output.
+ * Same interface as E2B: python main.py < input.json
  */
-async function executeTests(agentDir, agentType, testFiles, verbose, config) {
+async function runCodeRuntimeFixtureTests(agentDir, fixtures, entrypoint, verbose) {
+    process.stderr.write(chalk_1.default.blue('\nRunning fixture tests (code runtime)...\n\n'));
+    let passed = 0;
+    let failed = 0;
+    for (const fixturePath of fixtures) {
+        const fixtureName = path_1.default.basename(fixturePath);
+        const description = await getFixtureDescription(fixturePath);
+        process.stderr.write(`  ${fixtureName}${description ? ` (${description})` : ''}: `);
+        try {
+            const raw = await promises_1.default.readFile(fixturePath, 'utf-8');
+            let parsed;
+            try {
+                parsed = JSON.parse(raw);
+            }
+            catch (e) {
+                throw new errors_1.CliError(`Invalid JSON in ${fixtureName}: ${e.message}`);
+            }
+            const fixture = validateFixture(parsed, fixturePath);
+            // Run entrypoint with fixture input as stdin (same as E2B: python main.py < input.json)
+            const inputJson = JSON.stringify(fixture.input);
+            const result = await runEntrypointWithInput(agentDir, entrypoint, inputJson, verbose);
+            if (result.code !== 0) {
+                throw new Error(`Entrypoint exited with code ${result.code}` +
+                    (result.stderr ? `\n    stderr: ${result.stderr.trim().split('\n').join('\n    stderr: ')}` : ''));
+            }
+            // Parse stdout as JSON
+            const trimmedOutput = result.stdout.trim();
+            let output;
+            try {
+                output = JSON.parse(trimmedOutput);
+            }
+            catch {
+                throw new Error(`Entrypoint output is not valid JSON.\n` +
+                    `    stdout: ${trimmedOutput.slice(0, 200)}${trimmedOutput.length > 200 ? '...' : ''}`);
+            }
+            // Validate result
+            let testPassed = true;
+            const failures = [];
+            if (fixture.expected_output) {
+                if (!(0, fast_deep_equal_1.default)(output, fixture.expected_output)) {
+                    testPassed = false;
+                    failures.push(`Expected: ${JSON.stringify(fixture.expected_output, null, 2)}\n` +
+                        `    Got: ${JSON.stringify(output, null, 2)}`);
+                }
+            }
+            if (fixture.expected_contains) {
+                const outputStr = JSON.stringify(output);
+                for (const expected of fixture.expected_contains) {
+                    if (!outputStr.includes(expected)) {
+                        testPassed = false;
+                        failures.push(`Expected output to contain: "${expected}"`);
+                    }
+                }
+            }
+            if (testPassed) {
+                process.stderr.write(chalk_1.default.green('PASS\n'));
+                passed++;
+                if (verbose) {
+                    process.stderr.write(chalk_1.default.gray(`    Input: ${JSON.stringify(fixture.input)}\n`));
+                    process.stderr.write(chalk_1.default.gray(`    Output: ${JSON.stringify(output)}\n`));
+                }
+            }
+            else {
+                process.stderr.write(chalk_1.default.red('FAIL\n'));
+                failed++;
+                for (const f of failures) {
+                    process.stderr.write(chalk_1.default.red(`    ${f}\n`));
+                }
+            }
+        }
+        catch (err) {
+            process.stderr.write(chalk_1.default.red('ERROR\n'));
+            failed++;
+            const message = err instanceof Error ? err.message : String(err);
+            process.stderr.write(chalk_1.default.red(`    ${message}\n`));
+        }
+    }
+    process.stderr.write('\n');
+    process.stderr.write(`Fixtures: ${passed} passed, ${failed} failed\n`);
+    return failed > 0 ? 1 : 0;
+}
+// ─── Agent type detection ────────────────────────────────────────────────────
+async function detectAgentType(agentDir) {
+    // Check for SKILL.md first
+    if (await fileExists(path_1.default.join(agentDir, 'SKILL.md'))) {
+        return 'skill';
+    }
+    // Check for orchagent.json
+    const manifestPath = path_1.default.join(agentDir, 'orchagent.json');
+    if (await fileExists(manifestPath)) {
+        try {
+            const raw = await promises_1.default.readFile(manifestPath, 'utf-8');
+            const manifest = JSON.parse(raw);
+            if (manifest.type === 'prompt')
+                return 'prompt';
+            if (manifest.type === 'skill')
+                return 'skill';
+            if (manifest.type === 'tool' || manifest.type === 'code') {
+                if (await fileExists(path_1.default.join(agentDir, 'package.json')))
+                    return 'code-js';
+                return 'code-python';
+            }
+            if (manifest.type === 'agent' || manifest.type === 'agentic') {
+                // Agent with runtime.command is code-based
+                if (manifest.runtime?.command) {
+                    if (await fileExists(path_1.default.join(agentDir, 'package.json')))
+                        return 'code-js';
+                    return 'code-python';
+                }
+                // Managed loop agent (uses prompt.md like prompt agents)
+                return 'prompt';
+            }
+        }
+        catch {
+            // Invalid manifest, continue detection
+        }
+    }
+    // Check for prompt.md (prompt agent)
+    if (await fileExists(path_1.default.join(agentDir, 'prompt.md'))) {
+        return 'prompt';
+    }
+    // Fallback: detect by file presence
+    if (await fileExists(path_1.default.join(agentDir, 'requirements.txt')))
+        return 'code-python';
+    if (await fileExists(path_1.default.join(agentDir, 'pyproject.toml')))
+        return 'code-python';
+    if (await fileExists(path_1.default.join(agentDir, 'package.json')))
+        return 'code-js';
+    return 'unknown';
+}
+// ─── Main test execution ─────────────────────────────────────────────────────
+async function executeTests(agentDir, validation, testFiles, verbose, config) {
     let exitCode = 0;
-    // Run tests based on what's available
     const hasTests = testFiles.python.length > 0 ||
         testFiles.javascript.length > 0 ||
         testFiles.fixtures.length > 0;
     if (!hasTests) {
-        // For prompt agents/skills, suggest creating fixtures
-        if (agentType === 'prompt' || agentType === 'skill') {
-            process.stderr.write(chalk_1.default.yellow('No test files found.\n\n'));
-            process.stderr.write('For prompt agents, create fixture files in tests/:\n');
-            process.stderr.write(chalk_1.default.gray('  tests/fixture-1.json:\n'));
+        // Suggest appropriate test types based on agent
+        process.stderr.write(chalk_1.default.yellow('No test files found.\n\n'));
+        if (validation.executionEngine === 'code_runtime' && validation.entrypoint) {
+            process.stderr.write('Create fixture tests to dry-run your code:\n');
+            process.stderr.write(chalk_1.default.gray('  mkdir tests\n'));
+            process.stderr.write(chalk_1.default.gray(`  # tests/fixture-basic.json — runs: ${validation.entrypoint} < input\n`));
+            process.stderr.write(chalk_1.default.gray('  {\n'));
+            process.stderr.write(chalk_1.default.gray('    "description": "Basic test",\n'));
+            process.stderr.write(chalk_1.default.gray('    "input": {"key": "value"},\n'));
+            process.stderr.write(chalk_1.default.gray('    "expected_contains": ["result"]\n'));
+            process.stderr.write(chalk_1.default.gray('  }\n\n'));
+            process.stderr.write('Or test interactively:\n');
+            process.stderr.write(chalk_1.default.gray(`  orch run . --local --data '{"key": "value"}'\n\n`));
+        }
+        else if (validation.isSkill || validation.executionEngine === 'direct_llm' || validation.executionEngine === 'managed_loop') {
+            process.stderr.write('Create fixture tests in tests/:\n');
+            process.stderr.write(chalk_1.default.gray('  mkdir tests\n'));
+            process.stderr.write(chalk_1.default.gray('  # tests/fixture-basic.json — calls LLM with your prompt + input\n'));
             process.stderr.write(chalk_1.default.gray('  {\n'));
+            process.stderr.write(chalk_1.default.gray('    "description": "Basic test",\n'));
             process.stderr.write(chalk_1.default.gray('    "input": {"text": "Hello world"},\n'));
             process.stderr.write(chalk_1.default.gray('    "expected_contains": ["response"]\n'));
             process.stderr.write(chalk_1.default.gray('  }\n\n'));
+            if (validation.executionEngine === 'managed_loop') {
+                process.stderr.write('Or test the full agent loop:\n');
+                process.stderr.write(chalk_1.default.gray(`  orch run . --local --data '{"task": "..."}'\n\n`));
+            }
         }
         else {
-            process.stderr.write(chalk_1.default.yellow('No test files found.\n\n'));
             process.stderr.write('Supported test file patterns:\n');
             process.stderr.write(chalk_1.default.gray('  Python: test_*.py, *_test.py, tests/test_*.py\n'));
             process.stderr.write(chalk_1.default.gray('  JS/TS:  *.test.ts, *.spec.ts, tests/*.test.ts\n'));
             process.stderr.write(chalk_1.default.gray('  Fixtures: tests/fixture-*.json\n\n'));
         }
-        return 1;
+        return 0; // Validation passed, no tests is OK
     }
     // Run Python tests if found
     if (testFiles.python.length > 0) {
@@ -473,42 +823,226 @@ async function executeTests(agentDir, agentType, testFiles, verbose, config) {
         if (code !== 0)
             exitCode = 1;
     }
-    // Run fixture tests if found (for prompt agents)
+    // Run fixture tests — route by execution engine
     if (testFiles.fixtures.length > 0) {
         if (verbose) {
             process.stderr.write(chalk_1.default.gray(`Found ${testFiles.fixtures.length} fixture file(s)\n`));
         }
-        const code = await runFixtureTests(agentDir, testFiles.fixtures, verbose, config);
-        if (code !== 0)
-            exitCode = 1;
+        if (validation.executionEngine === 'code_runtime' && validation.entrypoint) {
+            const code = await runCodeRuntimeFixtureTests(agentDir, testFiles.fixtures, validation.entrypoint, verbose);
+            if (code !== 0)
+                exitCode = 1;
+        }
+        else {
+            // Prompt, skill, and managed_loop agents: LLM-based fixture tests
+            const code = await runPromptFixtureTests(agentDir, testFiles.fixtures, verbose, config);
+            if (code !== 0)
+                exitCode = 1;
+        }
     }
     return exitCode;
 }
+/**
+ * Run validation + all tests in sequence
+ */
+async function runAllChecks(agentDir, verbose, config) {
+    // Load .env from agent directory (existing env vars take precedence)
+    const dotEnvVars = await (0, dotenv_1.loadDotEnv)(agentDir);
+    const dotEnvCount = Object.keys(dotEnvVars).length;
+    if (dotEnvCount > 0) {
+        for (const [key, value] of Object.entries(dotEnvVars)) {
+            if (!(key in process.env) || process.env[key] === undefined) {
+                process.env[key] = value;
+            }
+        }
+        process.stderr.write(chalk_1.default.gray(`Loaded ${dotEnvCount} variable${dotEnvCount === 1 ? '' : 's'} from .env\n`));
+    }
+    // Step 1: Validate
+    const validation = await validateAgent(agentDir);
+    const validationPassed = printValidation(validation);
+    if (!validationPassed) {
+        process.stderr.write(chalk_1.default.red('Fix validation errors above before publishing.\n'));
+        return 1;
+    }
+    // Step 2: Discover tests
+    const testFiles = await discoverTests(agentDir);
+    if (verbose) {
+        const totalTests = testFiles.python.length + testFiles.javascript.length + testFiles.fixtures.length;
+        process.stderr.write(chalk_1.default.gray(`Discovered ${totalTests} test file(s)\n`));
+    }
+    // Step 3: Run tests
+    return await executeTests(agentDir, validation, testFiles, verbose, config);
+}
+// ─── Watch mode ──────────────────────────────────────────────────────────────
+async function watchTests(agentDir, verbose, config) {
+    process.stderr.write(chalk_1.default.cyan('\nWatching for file changes... (press Ctrl+C to exit)\n'));
+    const runTests = async () => {
+        process.stderr.write(chalk_1.default.dim(`\n[${new Date().toLocaleTimeString()}] Running checks...\n`));
+        await runAllChecks(agentDir, verbose, config);
+    };
+    // Initial run
+    await runTests();
+    // Set up chokidar watcher
+    let debounceTimer = null;
+    const onChange = (filePath) => {
+        if (debounceTimer)
+            clearTimeout(debounceTimer);
+        if (verbose) {
+            process.stderr.write(chalk_1.default.dim(`  Changed: ${path_1.default.relative(agentDir, filePath)}\n`));
+        }
+        debounceTimer = setTimeout(runTests, 300);
+    };
+    const watcher = chokidar_1.default.watch(agentDir, {
+        ignored: /(node_modules|__pycache__|\.git|dist|build|\.venv|venv)/,
+        persistent: true,
+        ignoreInitial: true,
+    });
+    watcher
+        .on('change', onChange)
+        .on('add', onChange)
+        .on('unlink', onChange)
+        .on('error', (error) => {
+        const message = error instanceof Error ? error.message : String(error);
+        process.stderr.write(chalk_1.default.red(`Watcher error: ${message}\n`));
+    });
+    // Keep process alive
+    await new Promise(() => { });
+}
+// ─── Single run mode ─────────────────────────────────────────────────────────
+/**
+ * Validate, then run the agent once with the given input.
+ * code_runtime: executes entrypoint with data as stdin.
+ * direct_llm / managed_loop: calls LLM with prompt + data.
+ */
+async function runOnce(agentDir, dataJson, verbose, config) {
+    // Load .env from agent directory
+    const dotEnvVars = await (0, dotenv_1.loadDotEnv)(agentDir);
+    const dotEnvCount = Object.keys(dotEnvVars).length;
+    if (dotEnvCount > 0) {
+        for (const [key, value] of Object.entries(dotEnvVars)) {
+            if (!(key in process.env) || process.env[key] === undefined) {
+                process.env[key] = value;
+            }
+        }
+        process.stderr.write(chalk_1.default.gray(`Loaded ${dotEnvCount} variable${dotEnvCount === 1 ? '' : 's'} from .env\n`));
+    }
+    // Validate first
+    const validation = await validateAgent(agentDir);
+    const valid = printValidation(validation);
+    if (!valid) {
+        process.stderr.write(chalk_1.default.red('Fix validation errors before running.\n'));
+        return 1;
+    }
+    // Parse input
+    let inputData;
+    try {
+        inputData = JSON.parse(dataJson);
+    }
+    catch {
+        throw new errors_1.CliError(`Invalid JSON in --data: ${dataJson.slice(0, 100)}`);
+    }
+    if (validation.executionEngine === 'code_runtime' && validation.entrypoint) {
+        // Run the entrypoint with data as stdin (same as E2B sandbox)
+        process.stderr.write(`\nRunning: ${validation.entrypoint}\n\n`);
+        const result = await runEntrypointWithInput(agentDir, validation.entrypoint, dataJson, verbose);
+        // Show stderr if not already shown in verbose mode
+        if (!verbose && result.stderr.trim()) {
+            process.stderr.write(chalk_1.default.gray(result.stderr));
+        }
+        if (result.code !== 0) {
+            process.stderr.write(chalk_1.default.red(`\nExited with code ${result.code}\n`));
+            return 1;
+        }
+        // Print stdout (the agent's JSON output)
+        const trimmed = result.stdout.trim();
+        if (trimmed) {
+            try {
+                const parsed = JSON.parse(trimmed);
+                (0, output_1.printJson)(parsed);
+            }
+            catch {
+                // Not JSON — print raw
+                process.stdout.write(trimmed + '\n');
+            }
+        }
+        return 0;
+    }
+    // Prompt / managed_loop: call LLM with prompt + input
+    const promptPath = path_1.default.join(agentDir, 'prompt.md');
+    const skillPath = path_1.default.join(agentDir, 'SKILL.md');
+    let prompt;
+    const skillData = await parseSkillMd(skillPath);
+    if (skillData) {
+        prompt = skillData.body;
+    }
+    else {
+        try {
+            prompt = await promises_1.default.readFile(promptPath, 'utf-8');
+        }
+        catch {
+            throw new errors_1.CliError('No prompt.md or SKILL.md found');
+        }
+    }
+    // Read output schema if available
+    let outputSchema;
+    try {
+        const raw = await promises_1.default.readFile(path_1.default.join(agentDir, 'schema.json'), 'utf-8');
+        const schemas = JSON.parse(raw);
+        outputSchema = schemas.output;
+    }
+    catch {
+        // Optional
+    }
+    const detected = await (0, llm_1.detectLlmKey)(['any'], config);
+    if (!detected) {
+        throw new errors_1.CliError('No LLM key found.\n' +
+            'Set an environment variable (e.g., OPENAI_API_KEY) or add one to .env');
+    }
+    const { provider, key, model: serverModel } = detected;
+    const model = serverModel ?? (0, llm_1.getDefaultModel)(provider);
+    process.stderr.write(`\nRunning with ${provider} (${model})...\n\n`);
+    const fullPrompt = (0, llm_1.buildPrompt)(prompt, inputData);
+    const result = await (0, llm_1.callLlm)(provider, key, model, fullPrompt, outputSchema);
+    (0, output_1.printJson)(result);
+    return 0;
+}
+// ─── Command registration ────────────────────────────────────────────────────
 function registerTestCommand(program) {
     program
         .command('test [path]')
-        .description('Run agent test suite locally')
+        .description('Validate agent configuration and run test suite')
         .option('-v, --verbose', 'Show detailed test output')
         .option('-w, --watch', 'Watch for file changes and re-run tests')
+        .option('-r, --run', 'Run the agent once with --data input (validate first)')
+        .option('-d, --data <json>', 'JSON input for --run mode')
         .addHelpText('after', `
 Examples:
-  orch test                    Run tests in current directory
-  orch test ./my-agent         Run tests in specified directory
+  orch test                    Validate + run tests in current directory
+  orch test ./my-agent         Validate + run tests in specified directory
   orch test --verbose          Show detailed test output
-  orch test --watch            Watch mode - re-run on file changes
+  orch test --watch            Watch mode — re-run on file changes
+  orch test --run --data '{"task": "hello"}'   Validate, then run once
 
-Test Discovery:
-  Python:   test_*.py, *_test.py, tests/test_*.py, tests/*_test.py
-  JS/TS:    *.test.ts, *.test.js, *.spec.ts, *.spec.js, tests/*.test.*
-  Fixtures: tests/fixture-*.json (for prompt agents)
+What it checks:
+  1. Validates orchagent.json (type, engine, required files, secrets, etc.)
+  2. Runs Python tests (pytest): test_*.py, *_test.py
+  3. Runs JS/TS tests (vitest): *.test.ts, *.spec.ts
+  4. Runs fixture tests: tests/fixture-*.json
 
-Fixture Format (tests/fixture-1.json):
+Fixture Format (tests/fixture-basic.json):
   {
+    "description": "Test description",
     "input": {"key": "value"},
     "expected_output": {"result": "expected"},
-    "expected_contains": ["substring"],
-    "description": "Test description"
+    "expected_contains": ["substring"]
   }
+
+  For code_runtime agents, fixtures run your entrypoint with input as stdin.
+  For prompt/agent types, fixtures call the LLM with your prompt + input.
+
+Run mode (--run):
+  Validates the agent, then executes it once with the provided --data.
+  Loads .env automatically. Same interface as: orch run . --local --data '...'
 `)
         .action(async (agentPath, options) => {
         const agentDir = agentPath
@@ -527,18 +1061,7 @@ Fixture Format (tests/fixture-1.json):
             }
             throw err;
         }
-        // Detect agent type
-        const agentType = await detectAgentType(agentDir);
-        if (options.verbose) {
-            process.stderr.write(chalk_1.default.gray(`Detected agent type: ${agentType}\n`));
-        }
-        // Discover test files
-        const testFiles = await discoverTests(agentDir);
-        if (options.verbose) {
-            const totalTests = testFiles.python.length + testFiles.javascript.length + testFiles.fixtures.length;
-            process.stderr.write(chalk_1.default.gray(`Discovered ${totalTests} test file(s)\n`));
-        }
-        // Get config for LLM access (needed for fixture tests)
+        // Get config for LLM access (needed for fixture tests and run mode)
         let config;
         try {
             config = await (0, config_1.getResolvedConfig)();
@@ -546,18 +1069,27 @@ Fixture Format (tests/fixture-1.json):
         catch {
             // Config not available, fixture tests will use env vars only
         }
+        // Run mode: validate then execute once
+        if (options.run) {
+            if (!options.data) {
+                throw new errors_1.CliError('Missing --data for run mode.\n\n' +
+                    `Usage: orch test --run --data '{"key": "value"}'`);
+            }
+            const exitCode = await runOnce(agentDir, options.data, !!options.verbose, config);
+            process.exit(exitCode);
+        }
         // Watch mode
         if (options.watch) {
-            await watchTests(agentDir, agentType, !!options.verbose, config);
+            await watchTests(agentDir, !!options.verbose, config);
             return;
         }
-        // Run tests
-        const exitCode = await executeTests(agentDir, agentType, testFiles, !!options.verbose, config);
+        // Single run: validate + tests
+        const exitCode = await runAllChecks(agentDir, !!options.verbose, config);
         if (exitCode === 0) {
-            process.stderr.write(chalk_1.default.green('\nAll tests passed.\n'));
+            process.stderr.write(chalk_1.default.green('\nAll checks passed.\n'));
         }
         else {
-            process.stderr.write(chalk_1.default.red('\nSome tests failed.\n'));
+            process.stderr.write(chalk_1.default.red('\nSome checks failed.\n'));
         }
         process.exit(exitCode);
     });