@orchagent/cli 0.3.23 → 0.3.25

package/dist/commands/test.js

@@ -9,9 +9,32 @@ const path_1 = __importDefault(require("path"));
 const child_process_1 = require("child_process");
 const chalk_1 = __importDefault(require("chalk"));
 const yaml_1 = __importDefault(require("yaml"));
+const fast_deep_equal_1 = __importDefault(require("fast-deep-equal"));
+const chokidar_1 = __importDefault(require("chokidar"));
 const errors_1 = require("../lib/errors");
 const config_1 = require("../lib/config");
 const llm_1 = require("../lib/llm");
+/**
+ * Validate a fixture and return helpful errors
+ */
+function validateFixture(data, fixturePath) {
+    const fileName = path_1.default.basename(fixturePath);
+    if (typeof data !== 'object' || data === null) {
+        throw new errors_1.CliError(`Invalid fixture ${fileName}: must be a JSON object`);
+    }
+    const obj = data;
+    if (!obj.input || typeof obj.input !== 'object') {
+        throw new errors_1.CliError(`Invalid fixture ${fileName}: missing required "input" field.\n` +
+            `  Expected format: { "input": {...}, "expected_output": {...} }`);
+    }
+    if (!obj.expected_output && !obj.expected_contains) {
+        throw new errors_1.CliError(`Invalid fixture ${fileName}: must have "expected_output" or "expected_contains".\n` +
+            `  Add one of:\n` +
+            `    "expected_output": {"key": "exact value to match"}\n` +
+            `    "expected_contains": ["substring to find"]`);
+    }
+    return data;
+}
 /**
  * Parse SKILL.md frontmatter
  */
@@ -68,8 +91,10 @@ function runCommand(command, args, cwd, verbose) {
  * Check if a command exists
  */
 async function commandExists(command) {
+    const isWindows = process.platform === 'win32';
+    const checker = isWindows ? 'where' : 'which';
     try {
-        const proc = (0, child_process_1.spawn)('which', [command], { shell: true });
+        const proc = (0, child_process_1.spawn)(checker, [command], { shell: true, stdio: 'ignore' });
         return new Promise((resolve) => {
             proc.on('close', (code) => resolve(code === 0));
             proc.on('error', () => resolve(false));
@@ -202,22 +227,28 @@ async function discoverTests(agentDir) {
  */
 async function runPythonTests(agentDir, verbose) {
     process.stderr.write(chalk_1.default.blue('\nRunning Python tests...\n\n'));
-    // Check if pytest is available
+    // Check if pytest is available directly
     const hasPytest = await commandExists('pytest');
-    let command;
-    let args;
     if (hasPytest) {
-        command = 'pytest';
-        args = verbose ? ['-v'] : [];
+        const args = verbose ? ['-v'] : [];
+        const { code } = await runCommand('pytest', args, agentDir, verbose);
+        return code;
     }
-    else {
-        command = 'python3';
-        args = ['-m', 'pytest'];
-        if (verbose)
-            args.push('-v');
+    // Try Python commands in order of preference
+    const pythonCommands = process.platform === 'win32'
+        ? ['python', 'py', 'python3']
+        : ['python3', 'python'];
+    for (const pythonCmd of pythonCommands) {
+        if (await commandExists(pythonCmd)) {
+            const args = ['-m', 'pytest'];
+            if (verbose)
+                args.push('-v');
+            const { code } = await runCommand(pythonCmd, args, agentDir, verbose);
+            return code;
+        }
     }
-    const { code } = await runCommand(command, args, agentDir, verbose);
-    return code;
+    process.stderr.write(chalk_1.default.red('No Python interpreter found. Install Python and pytest.\n'));
+    return 1;
 }
 /**
  * Run JavaScript/TypeScript tests
@@ -290,8 +321,8 @@ async function runFixtureTests(agentDir, fixtures, verbose, config) {
         throw new errors_1.CliError('No LLM key found for fixture tests.\n' +
             'Set an environment variable (e.g., OPENAI_API_KEY) or run `orchagent keys add <provider>`');
     }
-    const { provider, key } = detected;
-    const model = (0, llm_1.getDefaultModel)(provider);
+    const { provider, key, model: serverModel } = detected;
+    const model = serverModel ?? (0, llm_1.getDefaultModel)(provider);
     let passed = 0;
     let failed = 0;
     for (const fixturePath of fixtures) {
@@ -299,7 +330,14 @@ async function runFixtureTests(agentDir, fixtures, verbose, config) {
         process.stderr.write(`  ${fixtureName}: `);
         try {
             const raw = await promises_1.default.readFile(fixturePath, 'utf-8');
-            const fixture = JSON.parse(raw);
+            let parsed;
+            try {
+                parsed = JSON.parse(raw);
+            }
+            catch (e) {
+                throw new errors_1.CliError(`Invalid JSON in ${path_1.default.basename(fixturePath)}: ${e.message}`);
+            }
+            const fixture = validateFixture(parsed, fixturePath);
             // Build and call LLM
             const fullPrompt = (0, llm_1.buildPrompt)(prompt, fixture.input);
             const result = await (0, llm_1.callLlm)(provider, key, model, fullPrompt, outputSchema);
@@ -307,12 +345,9 @@ async function runFixtureTests(agentDir, fixtures, verbose, config) {
             let testPassed = true;
             const failures = [];
             if (fixture.expected_output) {
-                // Exact match comparison
-                const resultStr = JSON.stringify(result, Object.keys(result).sort());
-                const expectedStr = JSON.stringify(fixture.expected_output, Object.keys(fixture.expected_output).sort());
-                if (resultStr !== expectedStr) {
+                if (!(0, fast_deep_equal_1.default)(result, fixture.expected_output)) {
                     testPassed = false;
-                    failures.push(`Expected: ${expectedStr}\nGot: ${resultStr}`);
+                    failures.push(`Expected: ${JSON.stringify(fixture.expected_output, null, 2)}\nGot: ${JSON.stringify(result, null, 2)}`);
                 }
             }
             if (fixture.expected_contains) {
@@ -355,46 +390,39 @@ async function runFixtureTests(agentDir, fixtures, verbose, config) {
 /**
  * Watch mode: re-run tests on file changes
  */
-async function watchTests(agentDir, agentType, testFiles, verbose, config) {
+async function watchTests(agentDir, agentType, verbose, config) {
     process.stderr.write(chalk_1.default.cyan('\nWatching for file changes... (press Ctrl+C to exit)\n\n'));
-    // Collect all files to watch
-    const watchPaths = [agentDir];
     const runTests = async () => {
         process.stderr.write(chalk_1.default.dim(`\n[${new Date().toLocaleTimeString()}] Running tests...\n`));
+        // Re-discover tests each time to pick up new files
+        const testFiles = await discoverTests(agentDir);
         await executeTests(agentDir, agentType, testFiles, verbose, config);
     };
     // Initial run
     await runTests();
-    // Set up file watcher using fs.watch (basic implementation)
-    const watchers = [];
+    // Set up chokidar watcher
     let debounceTimer = null;
-    const onChange = () => {
+    const onChange = (filePath) => {
         if (debounceTimer)
             clearTimeout(debounceTimer);
-        debounceTimer = setTimeout(runTests, 500);
-    };
-    for (const watchPath of watchPaths) {
-        try {
-            // Use recursive watch on the directory
-            const ac = new AbortController();
-            (async () => {
-                try {
-                    const watcher = promises_1.default.watch(watchPath, { recursive: true, signal: ac.signal });
-                    for await (const _event of watcher) {
-                        onChange();
-                    }
-                }
-                catch (err) {
-                    if (err.name !== 'AbortError') {
-                        // Watcher error, ignore
-                    }
-                }
-            })();
-        }
-        catch {
-            // Watch not supported, fall back to polling
+        if (verbose) {
+            process.stderr.write(chalk_1.default.dim(`  Changed: ${path_1.default.relative(agentDir, filePath)}\n`));
         }
-    }
+        debounceTimer = setTimeout(runTests, 300);
+    };
+    const watcher = chokidar_1.default.watch(agentDir, {
+        ignored: /(node_modules|__pycache__|\.git|dist|build|\.venv|venv)/,
+        persistent: true,
+        ignoreInitial: true,
+    });
+    watcher
+        .on('change', onChange)
+        .on('add', onChange)
+        .on('unlink', onChange)
+        .on('error', (error) => {
+        const message = error instanceof Error ? error.message : String(error);
+        process.stderr.write(chalk_1.default.red(`Watcher error: ${message}\n`));
+    });
     // Keep process alive
     await new Promise(() => { });
 }
@@ -520,7 +548,7 @@ Fixture Format (tests/fixture-1.json):
         }
         // Watch mode
         if (options.watch) {
-            await watchTests(agentDir, agentType, testFiles, !!options.verbose, config);
+            await watchTests(agentDir, agentType, !!options.verbose, config);
             return;
         }
         // Run tests

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@orchagent/cli",
-  "version": "0.3.23",
+  "version": "0.3.25",
   "description": "Command-line interface for the orchagent AI agent marketplace",
   "license": "MIT",
   "author": "orchagent <hello@orchagent.io>",
@@ -46,8 +46,10 @@
     "@sentry/node": "^9.3.0",
     "archiver": "^7.0.0",
     "chalk": "^4.1.2",
+    "chokidar": "^4.0.0",
     "cli-table3": "^0.6.3",
     "commander": "^11.1.0",
+    "fast-deep-equal": "^3.1.3",
     "open": "^8.4.2",
     "ora": "^9.1.0",
     "posthog-node": "^4.0.0",