@orchagent/cli 0.3.85 → 0.3.87

package/dist/commands/test.js

@@ -17,6 +17,7 @@ const dotenv_1 = require("../lib/dotenv");
 const config_1 = require("../lib/config");
 const llm_1 = require("../lib/llm");
 const bundle_1 = require("../lib/bundle");
+const test_mock_runner_1 = require("../lib/test-mock-runner");
 // ─── Utility functions ───────────────────────────────────────────────────────
 function validateFixture(data, fixturePath) {
     const fileName = path_1.default.basename(fixturePath);
@@ -793,6 +794,13 @@ async function executeTests(agentDir, validation, testFiles, verbose, config) {
             process.stderr.write(chalk_1.default.gray('    "expected_contains": ["response"]\n'));
             process.stderr.write(chalk_1.default.gray('  }\n\n'));
             if (validation.executionEngine === 'managed_loop') {
+                process.stderr.write('For orchestrators with sub-agents, add mocked fixtures:\n');
+                process.stderr.write(chalk_1.default.gray('  # tests/fixture-mock-basic.json\n'));
+                process.stderr.write(chalk_1.default.gray('  {\n'));
+                process.stderr.write(chalk_1.default.gray('    "input": {"task": "..."},\n'));
+                process.stderr.write(chalk_1.default.gray('    "mocks": {"tool_name": {"key": "mock response"}},\n'));
+                process.stderr.write(chalk_1.default.gray('    "expected_contains": ["expected"]\n'));
+                process.stderr.write(chalk_1.default.gray('  }\n\n'));
                 process.stderr.write('Or test the full agent loop:\n');
                 process.stderr.write(chalk_1.default.gray(`  orch run . --local --data '{"task": "..."}'\n\n`));
             }
@@ -833,8 +841,51 @@ async function executeTests(agentDir, validation, testFiles, verbose, config) {
             if (code !== 0)
                 exitCode = 1;
         }
+        else if (validation.executionEngine === 'managed_loop') {
+            // For managed_loop agents, split fixtures: mocked vs regular
+            const mockedFixtures = [];
+            const regularFixtures = [];
+            for (const fixturePath of testFiles.fixtures) {
+                try {
+                    const raw = await promises_1.default.readFile(fixturePath, 'utf-8');
+                    const data = JSON.parse(raw);
+                    if (data.mocks && typeof data.mocks === 'object' && !Array.isArray(data.mocks)) {
+                        mockedFixtures.push(fixturePath);
+                    }
+                    else {
+                        regularFixtures.push(fixturePath);
+                    }
+                }
+                catch {
+                    regularFixtures.push(fixturePath); // Let downstream validation handle errors
+                }
+            }
+            // Run mocked orchestration tests (full agent loop with mock sub-agents)
+            if (mockedFixtures.length > 0) {
+                try {
+                    const manifestPath = path_1.default.join(agentDir, 'orchagent.json');
+                    const manifestRaw = await promises_1.default.readFile(manifestPath, 'utf-8');
+                    const manifest = JSON.parse(manifestRaw);
+                    const code = await (0, test_mock_runner_1.runMockedAgentFixtureTests)(agentDir, mockedFixtures, manifest, verbose, config);
+                    if (code !== 0)
+                        exitCode = 1;
+                }
+                catch (err) {
+                    if (err instanceof errors_1.CliError)
+                        throw err;
+                    process.stderr.write(chalk_1.default.red(`  Error running mocked tests: ${err.message}\n`));
+                    exitCode = 1;
+                }
+            }
+            // Run regular (non-mocked) fixtures as LLM-based prompt tests
+            if (regularFixtures.length > 0) {
+                const code = await runPromptFixtureTests(agentDir, regularFixtures, verbose, config);
+                if (code !== 0)
+                    exitCode = 1;
+            }
+        }
         else {
-            // Prompt, skill, and managed_loop agents: LLM-based fixture tests
+            // Prompt, skill agents: LLM-based fixture tests
             const code = await runPromptFixtureTests(agentDir, testFiles.fixtures, verbose, config);
             if (code !== 0)
                 exitCode = 1;
@@ -1040,6 +1091,22 @@ Fixture Format (tests/fixture-basic.json):
   For code_runtime agents, fixtures run your entrypoint with input as stdin.
   For prompt/agent types, fixtures call the LLM with your prompt + input.
 
+Mocked Orchestration Tests (tests/fixture-mock-*.json):
+  For agent-type (managed_loop) orchestrators, add "mocks" to test the full
+  agent loop with deterministic sub-agent responses:
+  {
+    "description": "Orchestrator handles scan results",
+    "input": {"code": "import os"},
+    "mocks": {
+      "scan_secrets": {"findings": [{"type": "code_injection"}]},
+      "scan_deps": {"vulnerabilities": []}
+    },
+    "expected_contains": ["code_injection"]
+  }
+
+  The LLM runs the full tool-use loop, but custom tool calls return mock
+  responses instead of calling real sub-agents. Great for CI testing.
+
 Run mode (--run):
   Validates the agent, then executes it once with the provided --data.
   Loads .env automatically. Same interface as: orch run . --local --data '...'

package/dist/lib/api.js

@@ -44,6 +44,7 @@ exports.publicRequest = publicRequest;
 exports.getOrg = getOrg;
 exports.updateOrg = updateOrg;
 exports.getPublicAgent = getPublicAgent;
+exports.getAgentCostEstimate = getAgentCostEstimate;
 exports.listMyAgents = listMyAgents;
 exports.createAgent = createAgent;
 exports.downloadCodeBundle = downloadCodeBundle;
@@ -58,6 +59,7 @@ exports.forkAgent = forkAgent;
 exports.checkAgentTransfer = checkAgentTransfer;
 exports.transferAgent = transferAgent;
 exports.previewAgentVersion = previewAgentVersion;
+exports.validateAgentPublish = validateAgentPublish;
 exports.reportInstall = reportInstall;
 exports.fetchUserProfile = fetchUserProfile;
 exports.listEnvironments = listEnvironments;
@@ -288,6 +290,9 @@ async function updateOrg(config, payload) {
 async function getPublicAgent(config, org, agent, version) {
     return publicRequest(config, `/public/agents/${org}/${agent}/${version}`);
 }
+async function getAgentCostEstimate(config, org, agent, version) {
+    return publicRequest(config, `/public/agents/${org}/${agent}/${version}/cost-estimate`);
+}
 async function listMyAgents(config, workspaceId) {
     const headers = {};
     if (workspaceId)
@@ -430,15 +435,21 @@ async function downloadCodeBundleAuthenticated(config, agentId) {
 /**
  * Check if an agent requires confirmation for deletion.
  */
-async function checkAgentDelete(config, agentId) {
-    return request(config, 'GET', `/agents/${agentId}/delete-check`);
+async function checkAgentDelete(config, agentId, workspaceId) {
+    const headers = {};
+    if (workspaceId)
+        headers['X-Workspace-Id'] = workspaceId;
+    return request(config, 'GET', `/agents/${agentId}/delete-check`, { headers });
 }
 /**
  * Soft delete an agent.
  */
-async function deleteAgent(config, agentId, confirmationName) {
+async function deleteAgent(config, agentId, confirmationName, workspaceId) {
     const params = confirmationName ? `?confirmation_name=${encodeURIComponent(confirmationName)}` : '';
-    return request(config, 'DELETE', `/agents/${agentId}${params}`);
+    const headers = {};
+    if (workspaceId)
+        headers['X-Workspace-Id'] = workspaceId;
+    return request(config, 'DELETE', `/agents/${agentId}${params}`, { headers });
 }
 /**
  * Fork a public agent into the caller's workspace (or an explicit workspace_id).
@@ -473,6 +484,20 @@ async function previewAgentVersion(config, agentName, workspaceId) {
         headers['X-Workspace-Id'] = workspaceId;
     return request(config, 'GET', `/agents/preview?name=${encodeURIComponent(agentName)}`, { headers });
 }
+/**
+ * Validate an agent publish payload without actually creating the agent.
+ * Runs server-side validation (name, tier limits, manifest, dependencies, etc.)
+ * and returns a validation report. Used by --dry-run.
+ */
+async function validateAgentPublish(config, data, workspaceId) {
+    const headers = { 'Content-Type': 'application/json' };
+    if (workspaceId)
+        headers['X-Workspace-Id'] = workspaceId;
+    return request(config, 'POST', '/agents/validate', {
+        body: JSON.stringify(data),
+        headers,
+    });
+}
 /**
  * Report a skill installation to the backend.
  * Only tracks authenticated installs (requires API key).

package/dist/lib/batch-publish.js

@@ -0,0 +1,223 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.discoverAgents = discoverAgents;
+exports.topoSort = topoSort;
+exports.formatPublishPlan = formatPublishPlan;
+const promises_1 = __importDefault(require("fs/promises"));
+const path_1 = __importDefault(require("path"));
+const chalk_1 = __importDefault(require("chalk"));
+/**
+ * Scan immediate subdirectories for orchagent.json or SKILL.md files.
+ * Does NOT recurse deeper than one level (matching GitHub App import behavior).
+ */
+async function discoverAgents(rootDir) {
+    const agents = [];
+    let entries;
+    try {
+        entries = await promises_1.default.readdir(rootDir, { withFileTypes: true });
+    }
+    catch {
+        return [];
+    }
+    // Also check the root directory itself
+    const dirsToCheck = [];
+    for (const entry of entries) {
+        if (!entry.isDirectory())
+            continue;
+        if (entry.name.startsWith('.') || entry.name === 'node_modules' || entry.name === '__pycache__')
+            continue;
+        dirsToCheck.push({
+            dir: path_1.default.join(rootDir, entry.name),
+            dirName: entry.name,
+        });
+    }
+    for (const { dir, dirName } of dirsToCheck) {
+        const agent = await tryParseAgentDir(dir, dirName);
+        if (agent)
+            agents.push(agent);
+    }
+    return agents;
+}
+/**
+ * Try to parse a directory as an agent. Returns null if no orchagent.json or SKILL.md found.
+ */
+async function tryParseAgentDir(dir, dirName) {
+    // Check for SKILL.md first (takes precedence, matching publish.ts behavior)
+    const skillMdPath = path_1.default.join(dir, 'SKILL.md');
+    try {
+        const content = await promises_1.default.readFile(skillMdPath, 'utf-8');
+        const match = content.match(/^---\n([\s\S]*?)\n---\n([\s\S]*)$/);
+        if (match) {
+            // Parse YAML frontmatter to get name
+            const yamlContent = match[1];
+            const nameMatch = yamlContent.match(/^name:\s*(.+)$/m);
+            const name = nameMatch ? nameMatch[1].trim().replace(/^['"]|['"]$/g, '') : dirName;
+            return {
+                dir,
+                dirName,
+                name,
+                isSkill: true,
+                dependencyRefs: [], // Skills have no dependencies
+            };
+        }
+    }
+    catch {
+        // No SKILL.md, try orchagent.json
+    }
+    // Check for orchagent.json
+    const manifestPath = path_1.default.join(dir, 'orchagent.json');
+    try {
+        const raw = await promises_1.default.readFile(manifestPath, 'utf-8');
+        const manifest = JSON.parse(raw);
+        if (!manifest.name)
+            return null;
+        // Extract dependency refs from manifest.manifest.dependencies
+        const dependencyRefs = [];
+        const deps = manifest.manifest?.dependencies;
+        if (deps && Array.isArray(deps)) {
+            for (const dep of deps) {
+                if (dep.id && typeof dep.id === 'string') {
+                    dependencyRefs.push(dep.id);
+                }
+            }
+        }
+        // Also extract from custom_tools that reference other agents via orch_call
+        // Pattern: orch_call.py org/agent@version or orch_call.js org/agent@version
+        const loopTools = manifest.loop?.custom_tools || manifest.custom_tools;
+        if (Array.isArray(loopTools)) {
+            for (const tool of loopTools) {
+                if (tool.command) {
+                    const orchCallMatch = tool.command.match(/orch_call(?:\.py|\.js)?\s+([a-z0-9-]+\/[a-z0-9-]+)/);
+                    if (orchCallMatch) {
+                        const ref = orchCallMatch[1];
+                        if (!dependencyRefs.includes(ref)) {
+                            dependencyRefs.push(ref);
+                        }
+                    }
+                }
+            }
+        }
+        return {
+            dir,
+            dirName,
+            name: manifest.name,
+            isSkill: false,
+            dependencyRefs,
+        };
+    }
+    catch {
+        return null;
+    }
+}
+/**
+ * Topologically sort agents so dependencies are published first (leaf-first).
+ *
+ * Only considers intra-project dependencies (agents whose dependency refs
+ * match another discovered agent's name). Cross-org or external deps are
+ * ignored since they must already be published.
+ */
+function topoSort(agents) {
+    // Build name → agent lookup (using agent name, not dir name)
+    const byName = new Map();
+    for (const agent of agents) {
+        byName.set(agent.name, agent);
+    }
+    // Build adjacency: agent → set of local agents it depends on
+    // "depends on" means: must be published BEFORE this agent
+    const adjList = new Map();
+    for (const agent of agents) {
+        const localDeps = new Set();
+        for (const ref of agent.dependencyRefs) {
+            // ref is "org/name" — extract just the name part for local matching
+            const depName = ref.includes('/') ? ref.split('/')[1] : ref;
+            if (byName.has(depName) && depName !== agent.name) {
+                localDeps.add(depName);
+            }
+        }
+        adjList.set(agent.name, localDeps);
+    }
+    // Kahn's algorithm for topological sort
+    const inDegree = new Map();
+    for (const agent of agents) {
+        inDegree.set(agent.name, 0);
+    }
+    for (const [, deps] of adjList) {
+        for (const dep of deps) {
+            inDegree.set(dep, (inDegree.get(dep) || 0) + 1);
+        }
+    }
+    // Note: inDegree counts how many agents depend ON this agent.
+    // We want to publish agents with no dependents-that-haven't-been-published first.
+    // Actually, let me re-think: we want leaf-first ordering.
+    // A "leaf" is an agent with NO dependencies (inDegree in the "depends-on" graph = 0 outgoing).
+    // Let's use Kahn's on the reverse: process nodes whose dependencies are all satisfied.
+    // Reset: inDegree = number of local deps this agent has (not yet satisfied)
+    const remaining = new Map();
+    for (const agent of agents) {
+        remaining.set(agent.name, adjList.get(agent.name)?.size || 0);
+    }
+    const queue = [];
+    for (const [name, count] of remaining) {
+        if (count === 0)
+            queue.push(name);
+    }
+    // Sort queue alphabetically for deterministic ordering among peers
+    queue.sort();
+    const sorted = [];
+    const visited = new Set();
+    while (queue.length > 0) {
+        const name = queue.shift();
+        if (visited.has(name))
+            continue;
+        visited.add(name);
+        const agent = byName.get(name);
+        sorted.push(agent);
+        // For each agent that depends on this one, decrement their remaining count
+        for (const [otherName, deps] of adjList) {
+            if (deps.has(name) && !visited.has(otherName)) {
+                const newCount = (remaining.get(otherName) || 1) - 1;
+                remaining.set(otherName, newCount);
+                if (newCount === 0) {
+                    // Insert in sorted position for determinism
+                    const insertIdx = queue.findIndex(q => q > otherName);
+                    if (insertIdx === -1)
+                        queue.push(otherName);
+                    else
+                        queue.splice(insertIdx, 0, otherName);
+                }
+            }
+        }
+    }
+    // If not all agents were visited, there's a cycle
+    if (sorted.length < agents.length) {
+        const inCycle = agents
+            .filter(a => !visited.has(a.name))
+            .map(a => a.name);
+        return { ok: false, cycle: inCycle };
+    }
+    return { ok: true, sorted };
+}
+/**
+ * Format the publish plan for display (used by both dry-run and normal mode).
+ */
+function formatPublishPlan(sorted, orgSlug) {
+    const lines = [];
+    lines.push('');
+    lines.push(`  Found ${sorted.length} agent${sorted.length === 1 ? '' : 's'} to publish:`);
+    lines.push('');
+    for (let i = 0; i < sorted.length; i++) {
+        const agent = sorted[i];
+        const type = agent.isSkill ? 'skill' : 'agent';
+        const deps = agent.dependencyRefs.length > 0
+            ? ` (depends on: ${agent.dependencyRefs.map(r => r.split('/').pop()).join(', ')})`
+            : '';
+        const prefix = orgSlug ? `${orgSlug}/` : '';
+        lines.push(`  ${i + 1}. ${prefix}${agent.name} [${type}]${deps}`);
+        lines.push(`     ${chalk_1.default.gray(agent.dirName + '/')}`);
+    }
+    lines.push('');
+    return lines.join('\n');
+}