@orchagent/cli 0.3.86 → 0.3.88

package/dist/lib/test-mock-runner.js

@@ -0,0 +1,334 @@
+"use strict";
+/**
+ * Mock Agent Runner — executes managed_loop agents with mocked sub-agent responses.
+ *
+ * Used by `orch test` to test orchestration chains in CI without live sub-agents.
+ * The LLM still runs the full tool-use loop, but custom tool calls return
+ * deterministic mock responses instead of calling real sub-agents.
+ */
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.validateMockedFixture = validateMockedFixture;
+exports.runMockedAgentFixtureTests = runMockedAgentFixtureTests;
+const promises_1 = __importDefault(require("fs/promises"));
+const path_1 = __importDefault(require("path"));
+const os_1 = __importDefault(require("os"));
+const child_process_1 = require("child_process");
+const chalk_1 = __importDefault(require("chalk"));
+const fast_deep_equal_1 = __importDefault(require("fast-deep-equal"));
+const errors_1 = require("./errors");
+const llm_1 = require("./llm");
+// SDK packages needed by agent_runner.py per provider
+const SDK_PACKAGES = {
+    anthropic: 'anthropic',
+    openai: 'openai',
+    gemini: 'google-genai',
+};
+// ─── Validation ──────────────────────────────────────────────────────────────
+function validateMockedFixture(data, fixturePath, customToolNames) {
+    const fileName = path_1.default.basename(fixturePath);
+    if (typeof data !== 'object' || data === null) {
+        throw new errors_1.CliError(`Invalid fixture ${fileName}: must be a JSON object`);
+    }
+    const obj = data;
+    if (!obj.input || typeof obj.input !== 'object') {
+        throw new errors_1.CliError(`Invalid fixture ${fileName}: missing required "input" field`);
+    }
+    if (!obj.mocks || typeof obj.mocks !== 'object' || Array.isArray(obj.mocks)) {
+        throw new errors_1.CliError(`Invalid fixture ${fileName}: "mocks" must be an object mapping tool names to responses`);
+    }
+    if (!obj.expected_output && !obj.expected_contains) {
+        throw new errors_1.CliError(`Invalid fixture ${fileName}: must have "expected_output" or "expected_contains"`);
+    }
+    // Warn about mock keys that don't match any custom tool
+    const mockKeys = Object.keys(obj.mocks);
+    const unknownMocks = mockKeys.filter(k => !customToolNames.includes(k));
+    if (unknownMocks.length > 0) {
+        process.stderr.write(chalk_1.default.yellow(`  Warning: ${fileName} mocks unknown tool(s): ${unknownMocks.join(', ')}\n`));
+    }
+    return data;
+}
+// ─── Runner ──────────────────────────────────────────────────────────────────
+function runCommand(command, args) {
+    return new Promise((resolve) => {
+        const proc = (0, child_process_1.spawn)(command, args, {
+            stdio: ['pipe', 'pipe', 'pipe'],
+            shell: true,
+        });
+        let stdout = '';
+        let stderr = '';
+        proc.stdout?.on('data', (d) => { stdout += d.toString(); });
+        proc.stderr?.on('data', (d) => { stderr += d.toString(); });
+        proc.on('close', (code) => resolve({ code: code ?? 1, stdout, stderr }));
+        proc.on('error', (err) => resolve({ code: 1, stdout, stderr: err.message }));
+    });
+}
+async function runAgentWithMocks(tempDir, env, maxTurns, verbose) {
+    return new Promise((resolve) => {
+        const args = [
+            'agent_runner.py',
+            '--max-turns', String(maxTurns),
+            '--mock-tools', 'mock_tools.json',
+        ];
+        if (verbose)
+            args.push('--verbose');
+        const proc = (0, child_process_1.spawn)('python3', args, {
+            cwd: tempDir,
+            stdio: ['pipe', 'pipe', 'pipe'],
+            env,
+        });
+        proc.stdin.end();
+        let stdout = '';
+        let stderr = '';
+        proc.stdout?.on('data', (data) => {
+            stdout += data.toString();
+        });
+        proc.stderr?.on('data', (data) => {
+            const text = data.toString();
+            stderr += text;
+            if (verbose) {
+                // Filter out heartbeat dots and orchagent events
+                for (const line of text.split('\n')) {
+                    if (line.startsWith('@@ORCHAGENT_EVENT:'))
+                        continue;
+                    if (line.trim() === '.' || line.trim() === '')
+                        continue;
+                    process.stderr.write(chalk_1.default.gray(`    ${line}\n`));
+                }
+            }
+        });
+        proc.on('close', (code) => {
+            resolve({ exitCode: code ?? 1, stdout, stderr });
+        });
+        proc.on('error', (err) => {
+            resolve({ exitCode: 1, stdout, stderr: err.message });
+        });
+    });
+}
+// ─── Public API ──────────────────────────────────────────────────────────────
+async function runMockedAgentFixtureTests(agentDir, fixtures, manifest, verbose, config) {
+    process.stderr.write(chalk_1.default.blue('\nRunning mocked orchestration tests...\n\n'));
+    // Read prompt.md
+    let prompt;
+    try {
+        prompt = await promises_1.default.readFile(path_1.default.join(agentDir, 'prompt.md'), 'utf-8');
+    }
+    catch {
+        throw new errors_1.CliError('prompt.md not found (required for mocked orchestration tests)');
+    }
+    // Read output schema if available
+    let outputSchema;
+    try {
+        const raw = await promises_1.default.readFile(path_1.default.join(agentDir, 'schema.json'), 'utf-8');
+        const schemas = JSON.parse(raw);
+        outputSchema = schemas.output;
+    }
+    catch {
+        // Optional
+    }
+    // Get custom tools from manifest
+    const customTools = manifest.loop?.custom_tools ||
+        manifest.custom_tools ||
+        [];
+    const customToolNames = customTools.map((t) => t.name);
+    if (customTools.length === 0) {
+        process.stderr.write(chalk_1.default.yellow('  Warning: No custom_tools defined — mocks will have no effect\n\n'));
+    }
+    // Detect LLM key
+    const supportedProviders = (manifest.supported_providers || ['any']);
+    const detected = await (0, llm_1.detectLlmKey)(supportedProviders, config);
+    if (!detected) {
+        throw new errors_1.CliError('No LLM key found for mocked orchestration tests.\n' +
+            'Set an environment variable (e.g., ANTHROPIC_API_KEY) or run `orch secrets set <PROVIDER>_API_KEY <key>`');
+    }
+    const { provider, key, model: serverModel } = detected;
+    const model = serverModel ?? (0, llm_1.getDefaultModel)(provider);
+    const apiKeyEnvVar = llm_1.PROVIDER_ENV_VARS[provider];
+    // Check Python 3 available
+    try {
+        const { code } = await runCommand('python3', ['--version']);
+        if (code !== 0)
+            throw new Error();
+    }
+    catch {
+        throw new errors_1.CliError('Python 3 is required for mocked orchestration tests.\n' +
+            'Install Python 3: https://python.org/downloads');
+    }
+    // Check LLM SDK installed
+    const sdkPackage = SDK_PACKAGES[provider] || 'anthropic';
+    const sdkImportName = provider === 'gemini' ? 'google.genai' : sdkPackage;
+    try {
+        const { code } = await runCommand('python3', ['-c', `import ${sdkImportName}`]);
+        if (code !== 0) {
+            process.stderr.write(`  Installing ${sdkPackage} Python SDK...\n`);
+            const install = await runCommand('python3', ['-m', 'pip', 'install', '-q', sdkPackage]);
+            if (install.code !== 0) {
+                throw new errors_1.CliError(`Failed to install ${sdkPackage} SDK. Install manually: pip install ${sdkPackage}`);
+            }
+        }
+    }
+    catch (err) {
+        if (err instanceof errors_1.CliError)
+            throw err;
+        throw new errors_1.CliError(`Failed to check Python SDK: ${err}`);
+    }
+    // Find agent_runner.py
+    const runnerPaths = [
+        path_1.default.join(__dirname, '..', 'resources', 'agent_runner.py'),
+        path_1.default.join(__dirname, '..', '..', 'src', 'resources', 'agent_runner.py'),
+    ];
+    let runnerContent;
+    for (const p of runnerPaths) {
+        try {
+            runnerContent = await promises_1.default.readFile(p, 'utf-8');
+            break;
+        }
+        catch {
+            continue;
+        }
+    }
+    if (!runnerContent) {
+        throw new errors_1.CliError('Agent runner script not found. Reinstall the CLI: npm install -g @orchagent/cli');
+    }
+    const maxTurns = manifest.max_turns ??
+        manifest.loop?.max_turns ?? 25;
+    process.stderr.write(`  Provider: ${provider} (${model})\n`);
+    process.stderr.write(`  Custom tools: ${customToolNames.join(', ') || '(none)'}\n`);
+    process.stderr.write(`  Max turns: ${maxTurns}\n\n`);
+    let passed = 0;
+    let failed = 0;
+    for (const fixturePath of fixtures) {
+        const fixtureName = path_1.default.basename(fixturePath);
+        const raw = await promises_1.default.readFile(fixturePath, 'utf-8');
+        let parsed;
+        try {
+            parsed = JSON.parse(raw);
+        }
+        catch (e) {
+            process.stderr.write(chalk_1.default.red(`  ${fixtureName}: ERROR\n`));
+            process.stderr.write(chalk_1.default.red(`    Invalid JSON: ${e.message}\n`));
+            failed++;
+            continue;
+        }
+        const desc = parsed.description;
+        process.stderr.write(`  ${fixtureName}${desc ? ` (${desc})` : ''}: `);
+        let fixture;
+        try {
+            fixture = validateMockedFixture(parsed, fixturePath, customToolNames);
+        }
+        catch (err) {
+            process.stderr.write(chalk_1.default.red('ERROR\n'));
+            process.stderr.write(chalk_1.default.red(`    ${err.message}\n`));
+            failed++;
+            continue;
+        }
+        // Create temp dir for this fixture
+        const tempDir = path_1.default.join(os_1.default.tmpdir(), `orchagent-mock-test-${Date.now()}`);
+        await promises_1.default.mkdir(tempDir, { recursive: true });
+        try {
+            // Write all files the agent runner needs
+            await Promise.all([
+                promises_1.default.writeFile(path_1.default.join(tempDir, 'agent_runner.py'), runnerContent),
+                promises_1.default.writeFile(path_1.default.join(tempDir, 'prompt.md'), prompt),
+                promises_1.default.writeFile(path_1.default.join(tempDir, 'input.json'), JSON.stringify(fixture.input, null, 2)),
+                promises_1.default.writeFile(path_1.default.join(tempDir, 'mock_tools.json'), JSON.stringify(fixture.mocks)),
+                customTools.length > 0
+                    ? promises_1.default.writeFile(path_1.default.join(tempDir, 'custom_tools.json'), JSON.stringify(customTools))
+                    : Promise.resolve(),
+                outputSchema
+                    ? promises_1.default.writeFile(path_1.default.join(tempDir, 'output_schema.json'), JSON.stringify(outputSchema))
+                    : Promise.resolve(),
+            ]);
+            // Build env
+            const subprocessEnv = { ...process.env };
+            subprocessEnv.LOCAL_MODE = '1';
+            subprocessEnv.LLM_PROVIDER = provider;
+            subprocessEnv.LLM_MODEL = model;
+            if (apiKeyEnvVar && key) {
+                subprocessEnv[apiKeyEnvVar] = key;
+            }
+            // Run the agent loop with mocked tools
+            const result = await runAgentWithMocks(tempDir, subprocessEnv, maxTurns, verbose);
+            if (result.exitCode !== 0 || !result.stdout.trim()) {
+                process.stderr.write(chalk_1.default.red('ERROR\n'));
+                if (result.stdout.trim()) {
+                    try {
+                        const errJson = JSON.parse(result.stdout.trim());
+                        if (errJson.error) {
+                            process.stderr.write(chalk_1.default.red(`    ${errJson.error}\n`));
+                        }
+                    }
+                    catch {
+                        process.stderr.write(chalk_1.default.red(`    Agent exited with code ${result.exitCode}\n`));
+                    }
+                }
+                else {
+                    process.stderr.write(chalk_1.default.red(`    Agent exited with code ${result.exitCode} (no output)\n`));
+                }
+                failed++;
+                continue;
+            }
+            // Parse output
+            let output;
+            try {
+                output = JSON.parse(result.stdout.trim());
+            }
+            catch {
+                process.stderr.write(chalk_1.default.red('ERROR\n'));
+                process.stderr.write(chalk_1.default.red(`    Agent output is not valid JSON\n`));
+                if (verbose) {
+                    process.stderr.write(chalk_1.default.gray(`    stdout: ${result.stdout.trim().slice(0, 200)}\n`));
+                }
+                failed++;
+                continue;
+            }
+            // Validate against expectations
+            let testPassed = true;
+            const failures = [];
+            if (fixture.expected_output) {
+                if (!(0, fast_deep_equal_1.default)(output, fixture.expected_output)) {
+                    testPassed = false;
+                    failures.push(`Expected: ${JSON.stringify(fixture.expected_output, null, 2)}\n` +
+                        `    Got: ${JSON.stringify(output, null, 2)}`);
+                }
+            }
+            if (fixture.expected_contains) {
+                const outputStr = JSON.stringify(output);
+                for (const expected of fixture.expected_contains) {
+                    if (!outputStr.includes(expected)) {
+                        testPassed = false;
+                        failures.push(`Expected output to contain: "${expected}"`);
+                    }
+                }
+            }
+            if (testPassed) {
+                process.stderr.write(chalk_1.default.green('PASS\n'));
+                passed++;
+                if (verbose) {
+                    process.stderr.write(chalk_1.default.gray(`    Input: ${JSON.stringify(fixture.input)}\n`));
+                    process.stderr.write(chalk_1.default.gray(`    Output: ${JSON.stringify(output)}\n`));
+                }
+            }
+            else {
+                process.stderr.write(chalk_1.default.red('FAIL\n'));
+                failed++;
+                for (const f of failures) {
+                    process.stderr.write(chalk_1.default.red(`    ${f}\n`));
+                }
+            }
+        }
+        finally {
+            try {
+                await promises_1.default.rm(tempDir, { recursive: true, force: true });
+            }
+            catch {
+                // Ignore cleanup errors
+            }
+        }
+    }
+    process.stderr.write('\n');
+    process.stderr.write(`Mocked orchestration tests: ${passed} passed, ${failed} failed\n`);
+    return failed > 0 ? 1 : 0;
+}

package/dist/lib/update-notifier.js

@@ -141,7 +141,7 @@ function printUpdateNotification() {
         const current = package_json_1.default.version;
         if (isNewer(cachedLatest, current)) {
             process.stderr.write(`\nUpdate available: v${current} → v${cachedLatest}\n` +
-                `Run \`npm update -g @orchagent/cli\` to update\n`);
+                `Run \`npm install -g @orchagent/cli@latest\` to update\n`);
         }
     }
     catch {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@orchagent/cli",
-  "version": "0.3.86",
+  "version": "0.3.88",
   "description": "Command-line interface for orchagent — deploy and run AI agents for your team",
   "license": "MIT",
   "author": "orchagent <hello@orchagent.io>",

package/src/resources/agent_runner.py

@@ -352,10 +352,14 @@ def execute_custom_tool(command_template, params):
     return execute_bash(command)
 
 
-def dispatch_tool(tool_name, tool_input, custom_tools_config):
+def dispatch_tool(tool_name, tool_input, custom_tools_config, mock_tools=None):
     """
     Dispatch a tool call. Returns (result_text, is_submit).
     is_submit is True only when tool_name == "submit_result".
+
+    When mock_tools is provided, custom tools with matching names return
+    the mock response instead of executing the real command. Built-in
+    tools (bash, read_file, etc.) are never mocked.
     """
     if tool_name == "bash":
         return execute_bash(tool_input.get("command", "")), False
@@ -374,6 +378,12 @@ def dispatch_tool(tool_name, tool_input, custom_tools_config):
     elif tool_name == "submit_result":
         return json.dumps(tool_input), True
     else:
+        # Check mock_tools first — return mock response if available
+        if mock_tools and tool_name in mock_tools:
+            mock_response = mock_tools[tool_name]
+            if isinstance(mock_response, str):
+                return mock_response, False
+            return json.dumps(mock_response), False
         for ct in custom_tools_config:
             if ct["name"] == tool_name:
                 return execute_custom_tool(ct["command"], tool_input), False
@@ -685,10 +695,27 @@ def main():
     parser = argparse.ArgumentParser()
     parser.add_argument("--max-turns", type=int, default=25)
     parser.add_argument("--verbose", action="store_true", help="Log tool calls to stderr")
+    parser.add_argument("--mock-tools", type=str, default=None,
+                        help="Path to JSON file mapping tool names to mock responses")
     args = parser.parse_args()
 
     _VERBOSE = args.verbose
 
+    # Load mock tool responses if provided (for testing orchestration chains)
+    mock_tools = {}
+    if args.mock_tools:
+        try:
+            with open(args.mock_tools, "r") as f:
+                mock_tools = json.load(f)
+            if _VERBOSE:
+                print("[agent] Loaded %d mock tool(s): %s" % (
+                    len(mock_tools), ", ".join(mock_tools.keys())
+                ), file=sys.stderr, flush=True)
+        except FileNotFoundError:
+            error_exit("Mock tools file not found: %s" % args.mock_tools)
+        except json.JSONDecodeError as e:
+            error_exit("Invalid JSON in mock tools file: %s" % e)
+
     with open("prompt.md", "r") as f:
         author_prompt = f.read()
 
@@ -761,7 +788,7 @@ def main():
         for call_id, name, input_args in provider.extract_tool_calls(response):
             verbose_log(name, input_args)
             emit_event("tool_call", turn=turn + 1, tool=name, args_brief=_brief_args(name, input_args))
-            result_text, is_submit = dispatch_tool(name, input_args, custom_tools_config)
+            result_text, is_submit = dispatch_tool(name, input_args, custom_tools_config, mock_tools)
             emit_event("tool_result", turn=turn + 1, tool=name, status="error" if result_text.startswith("[ERROR]") else "ok")
 
             if is_submit: