@mrrlin-dev/external-agents 0.1.0

package/cli.js

@@ -0,0 +1,203 @@
+#!/usr/bin/env node
+// external-agents CLI — thin argv wrapper over the same primitives the MCP
+// server exposes. Used by shell wrappers (kilo-executor.sh, consensus-reviewer.sh)
+// that need to reach the registry from bash without speaking MCP JSON-RPC.
+//
+// Subcommands:
+//   pick [--tier T] [--n N] [--min-distinct-providers M] [--exclude ID,ID]
+//     → prints one agent id per line (up to N), or exits 3 if no candidates
+//   dispatch <agent-id> [--pro] "<prompt>"
+//     → runs the agent, prints stdout of the child, exits with:
+//        0 success  |  2 usage  |  3 unknown agent  |  4 quota exhausted
+//        1 real error
+//     → prints a JSON-RPC-style summary trailer to stderr for callers that want it:
+//        {"outcome":..., "exit_code":..., "duration_ms":..., "workdir":...}
+//   status [--json]  → table of every registry entry with state (or JSON)
+//   probe <agent-id> → probes one agent, prints new state JSON
+import { loadRegistry } from "./lib/registry.js";
+import { readState, writeState, probeInstalled } from "./lib/state.js";
+import { runAny, resolveEscalation, parseExhaustionSignal, getStats } from "./lib/dispatch.js";
+import { pickAgents } from "./lib/pick.js";
+import fs from "node:fs";
+import os from "node:os";
+import path from "node:path";
+
+// bootEnv duplicated from server.js so CLI + server load the same env context.
+function bootEnv() {
+  try {
+    const kiloAuthPath = path.join(os.homedir(), ".local/share/kilo/auth.json");
+    if (fs.existsSync(kiloAuthPath)) {
+      const a = JSON.parse(fs.readFileSync(kiloAuthPath, "utf-8"));
+      if (!process.env.DEEPSEEK_API_KEY && a.deepseek?.key) process.env.DEEPSEEK_API_KEY = a.deepseek.key;
+    }
+    const llmKeysPath = path.join(os.homedir(), "Library/Application Support/io.datasette.llm/keys.json");
+    if (!process.env.GEMINI_API_KEY || !process.env.GEMINI_API_KEY.startsWith("AIza")) {
+      if (fs.existsSync(llmKeysPath)) {
+        const k = JSON.parse(fs.readFileSync(llmKeysPath, "utf-8"));
+        if (k.gemini) process.env.GEMINI_API_KEY = k.gemini;
+      }
+    }
+  } catch {}
+}
+bootEnv();
+
+const REGISTRY_PATH = path.join(path.dirname(new URL(import.meta.url).pathname), "agents.yaml");
+const REGISTRY = loadRegistry(REGISTRY_PATH);
+
+// --- argv parsing helpers -----------------------------------------
+function parseArgs(argv) {
+  const args = [];
+  const flags = {};
+  for (let i = 0; i < argv.length; i++) {
+    const a = argv[i];
+    if (a.startsWith("--")) {
+      const key = a.slice(2);
+      const nxt = argv[i + 1];
+      if (nxt !== undefined && !nxt.startsWith("--")) { flags[key] = nxt; i++; }
+      else flags[key] = true;
+    } else args.push(a);
+  }
+  return { args, flags };
+}
+function die(msg, code = 2) { console.error(msg); process.exit(code); }
+function findAgent(id) { return REGISTRY.agents.find((a) => a.id === id); }
+
+// --- subcommands --------------------------------------------------
+function cmdPick(flags) {
+  const n = parseInt(flags.n || "1", 10);
+  const filter = {};
+  if (flags.tier) filter.tier = flags.tier;
+  if (flags.tags) filter.tags = String(flags.tags).split(",").filter(Boolean);
+  if (flags.exclude) filter.exclude_ids = String(flags.exclude).split(",").filter(Boolean);
+  if (flags.transport) filter.transport = flags.transport;
+  const picked = pickAgents(REGISTRY, readState(), {
+    n,
+    filter,
+    min_distinct_providers: flags["min-distinct-providers"] ? parseInt(flags["min-distinct-providers"], 10) : undefined,
+  });
+  if (picked.length === 0) process.exit(3);
+  for (const id of picked) console.log(id);
+}
+
+async function cmdDispatch(args, flags) {
+  const [agentId, ...promptParts] = args;
+  const prompt = promptParts.join(" ");
+  if (!agentId) die("usage: cli.js dispatch <agent-id> [--pro] \"<prompt>\"", 2);
+  if (!prompt) die("dispatch: missing prompt", 2);
+
+  const src = findAgent(agentId);
+  if (!src) die(`unknown agent: ${agentId}`, 3);
+
+  let entry = src;
+  let escalatedFrom;
+  if (flags.pro) {
+    const esc = resolveEscalation(REGISTRY, agentId);
+    if (!esc) {
+      console.error(JSON.stringify({ outcome: "no_escalation_candidate", requested: agentId }));
+      process.exit(4);
+    }
+    entry = esc;
+    escalatedFrom = agentId;
+  }
+
+  const cur = readState();
+  writeState({ [entry.id]: { ...(cur[entry.id] || {}), last_used_at: Math.floor(Date.now() / 1000) } });
+
+  const transport = flags.transport;  // "generate_new" | "edit_exists" | undefined
+  const result = await runAny(entry, prompt, { transport });
+  const now = Math.floor(Date.now() / 1000);
+
+  let outcome;
+  let statePatch;
+  if (result.exitCode === 0) {
+    statePatch = { [entry.id]: { state: "healthy", checked: now, last_used_at: now } };
+    outcome = "success";
+  } else {
+    const sig = parseExhaustionSignal(result.stderr + "\n" + result.output);
+    if (sig.detected) {
+      const cooldown_until = sig.reset_at != null ? sig.reset_at : now + 3600;
+      statePatch = { [entry.id]: { state: "quota_exhausted", cooldown_until, source: sig.reset_at != null ? "error_body" : "fallback_ttl", checked: now } };
+      outcome = "quota_exhausted";
+    } else outcome = "error";
+  }
+  if (statePatch) writeState(statePatch);
+
+  process.stdout.write(result.output);
+  const trailer = { agent_id: entry.id, outcome, exit_code: result.exitCode, duration_ms: result.durationMs, workdir: result.workdir, files: result.files };
+  if (escalatedFrom) trailer.escalated_from = escalatedFrom;
+  console.error("__EXTERNAL_AGENTS_TRAILER__ " + JSON.stringify(trailer));
+
+  process.exit(outcome === "success" ? 0 : (outcome === "quota_exhausted" ? 4 : 1));
+}
+
+function cmdStatus(flags) {
+  const state = readState();
+  const rows = REGISTRY.agents.map((e) => ({ ...e, ...(state[e.id] || { state: "healthy" }) }));
+  if (flags.json) {
+    console.log(JSON.stringify(rows, null, 2));
+    return;
+  }
+  const w = 42;
+  console.log(`${"agent".padEnd(w)} ${"state".padEnd(18)} ${"tier".padEnd(7)} tags`);
+  console.log("-".repeat(96));
+  for (const r of rows) {
+    const tagsStr = (r.tags || []).join(",");
+    console.log(`${r.id.padEnd(w)} ${(r.state || "?").padEnd(18)} ${(r.tier || "-").padEnd(7)} ${tagsStr}`);
+  }
+}
+
+function cmdStats(flags) {
+  const s = getStats(flags.since);
+  if (flags.json) {
+    console.log(JSON.stringify(s, null, 2));
+    return;
+  }
+  console.log(`total dispatches: ${s.total}${s.span.first_ts ? `  (from ${new Date(s.span.first_ts*1000).toISOString()} to ${new Date(s.span.last_ts*1000).toISOString()})` : ""}`);
+  console.log();
+  console.log("by transport:");
+  for (const [t, v] of Object.entries(s.by_transport)) {
+    console.log(`  ${t.padEnd(10)} count=${v.count} tokens_in=${v.tokens_in} tokens_out=${v.tokens_out}`);
+  }
+  console.log();
+  console.log("by agent:");
+  const rows = Object.entries(s.by_agent).sort((a,b) => b[1].count - a[1].count);
+  for (const [id, v] of rows) {
+    const okCount = v.outcomes.success || 0;
+    const successRate = v.count ? Math.round(100 * okCount / v.count) : 0;
+    console.log(`  ${id.padEnd(40)} count=${v.count} success=${successRate}% avg_dur=${Math.round(v.duration_ms/(v.count||1))}ms tokens=${v.tokens_in}/${v.tokens_out}`);
+  }
+}
+
+function cmdProbe(args) {
+  const [agentId] = args;
+  if (!agentId) die("usage: cli.js probe <agent-id>", 2);
+  const entry = findAgent(agentId);
+  if (!entry) die(`unknown agent: ${agentId}`, 3);
+  const result = probeInstalled(entry);
+  const checked = Math.floor(Date.now() / 1000);
+  writeState({ [agentId]: { ...result, checked } });
+  console.log(JSON.stringify({ id: agentId, ...result, checked }));
+}
+
+// --- entrypoint ---------------------------------------------------
+const [, , subcmd, ...rest] = process.argv;
+const { args, flags } = parseArgs(rest);
+
+switch (subcmd) {
+  case "pick":     cmdPick(flags); break;
+  case "dispatch": cmdDispatch(args, flags); break;
+  case "status":   cmdStatus(flags); break;
+  case "probe":    cmdProbe(args); break;
+  case "stats":    cmdStats(flags); break;
+  case "help":
+  case "--help":
+  case undefined:
+    console.error(`external-agents CLI — subcommands:
+  pick [--tier T] [--n N] [--min-distinct-providers M] [--exclude id,id] [--tags a,b] [--transport generate_new|edit_exists]
+  dispatch <agent-id> [--pro] [--transport generate_new|edit_exists] "<prompt>"
+  status [--json]
+  probe <agent-id>
+  stats [--since ISO] [--json]`);
+    process.exit(subcmd ? 0 : 2);
+  default: die(`unknown subcommand: ${subcmd}`, 2);
+}

package/docs/adding-a-provider.md

@@ -0,0 +1,139 @@
+# Adding a Provider
+
+This guide walks you through registering a new LLM provider or model in the `agents.yaml` registry for `@mrrlin-dev/external-agents`. Adding a model allows the local UI, MCP server, and CLI tools to discover it, track its health, and route agentic or generation tasks to it.
+
+## Prereq
+
+Before starting, ensure your local environment is set up for the transport you plan to use:
+
+- **For the `aider` transport**: This transport runs iterative, multi-file agentic loops. It requires the `aider` CLI to be globally available. Install it using `uv`:
+  ```bash
+  uv tool install aider-chat
+  ```
+- **For the `generate` transport**: No external dependencies are needed. This transport runs direct, lightweight HTTPS requests to any OpenAI-compatible completions endpoint.
+
+## Step 1: Find the provider's endpoint / prefix
+
+To configure the model, you need to identify its identifier and connection strings depending on the target transport:
+
+- **For `aider`**: Aider utilizes LiteLLM under the hood. Locate your provider's prefix format in the [Aider LLM Documentation](https://aider.chat/docs/llms.html). For example, Groq models use the prefix `groq/`, while OpenRouter uses `openrouter/`.
+- **For `generate`**: Locate the provider's OpenAI-compatible base URL in their developer docs. For example, DeepSeek uses `https://api.deepseek.com/v1/chat/completions`. Note down the exact model ID (e.g., `deepseek-chat`).
+
+## Step 2: Add a registry entry to agents.yaml
+
+Open your `agents.yaml` configuration file and append your new agent entry.
+
+```yaml
+- id: aider-groq-llama-3.3-70b
+  provider: groq
+  model: llama-3.3-70b-versatile
+  quota_scope: shared
+  tier: weak
+  tags: [quick, fast, free]
+  auth: "env:GROQ_API_KEY"
+  preference_order: 8
+  usage_url: "https://console.groq.com/settings/usage"
+  transports:
+    cli: "aider --model groq/llama-3.3-70b-versatile"
+
+- id: gen-deepseek-chat
+  provider: deepseek
+  model: deepseek-chat
+  quota_scope: shared
+  tier: strong
+  tags: [cheap, code]
+  auth: "env:DEEPSEEK_API_KEY"
+  preference_order: 9
+  transports:
+    generate:
+      url: "https://api.deepseek.com/v1/chat/completions"
+      env: DEEPSEEK_API_KEY
+      model: deepseek-chat
+```
+
+### Registry Fields Definition
+
+- **`id`**: Unique identifier for this specific provider-model-transport configuration.
+- **`provider`**: The lowercase name of the hosting provider.
+- **`model`**: The target model ID expected by the provider.
+- **`quota_scope`**: Set to `shared` if the rate limits are shared across all models on that provider account, or `individual` if the model has its own dedicated limit bucket.
+- **`tier`**: Either `strong` (complex reasoning, coding) or `weak` (fast, lightweight tasks).
+- **`tags`**: Metadata array for filtering within the UI and MCP routers.
+- **`auth`**: Specifies the environment variable containing the API key (e.g., `env:GROQ_API_KEY`). For custom or complex setups, you can override variables inline:
+  ```yaml
+  env:
+    OPENAI_API_KEY: "@file:~/.claude/state/zai.key"
+  ```
+- **`preference_order`**: Integer representing the fallback priority. Higher numbers are tried first.
+- **`transports`**: 
+  - `cli`: Command string to launch the agentic loop via aider.
+  - `generate`: Mapping configuring direct HTTPS requests to an OpenAI-compatible endpoint.
+
+## Step 3: Set the env var
+
+The agent needs the credentials declared in the `auth` or `transports.generate.env` fields. You can export these directly into your environment or apply them via the local UI's settings panel.
+
+```bash
+export GROQ_API_KEY="gsk_y0urS3cr3tKeyH3r3..."
+export DEEPSEEK_API_KEY="sk-d33ps33kKey..."
+```
+
+## Step 4: Verify
+
+Validate that your new registry entry is parsed correctly and that the external-agents daemon can communicate with the provider's endpoint.
+
+```bash
+external-agents probe aider-groq-llama-3.3-70b
+```
+
+If successful, the console will print a confirmation showing `state:healthy` alongside latency statistics.
+
+## Step 5: Test dispatch
+
+Run a direct generation test to confirm that the model returns coherent responses over your selected transport.
+
+```bash
+external-agents dispatch gen-deepseek-chat "Reply OK if you can read this."
+```
+
+The terminal should output the raw response text from the LLM.
+
+## Choosing between transports
+
+| Feature | `aider` | `generate` |
+| :--- | :--- | :--- |
+| **Iteration** | Excellent (interactive, git-aware loops) | Limited (single shot) |
+| **New-file creation** | Good | Excellent (clean write, low latency) |
+| **Multi-file edits** | Supported out-of-the-box | Manual processing required |
+| **Tool use** | Native via agent command line | No tool calling (pure text) |
+
+Choose `aider` when you want an agent to interactively edit codebases, run tests, and fix bugs. Choose `generate` for fast, zero-dependency generation jobs where you simply need text or a single file generated from scratch without overhead.
+
+## Common gotchas
+
+- **Model ID Prefixing**: When using `aider` (via LiteLLM), you often need prefixes like `groq/llama-3.3-70b-versatile` in the `cli` execution string. However, for the `generate` transport, you must use the raw model name expected by the OpenAI-compatible endpoint (e.g., `llama-3.3-70b-versatile`) in the `model` parameter, otherwise the API will return a 404.
+- **Quota Scopes**: Setting `quota_scope: shared` tells the router to avoid hammering other models under the same provider if one model returns a `429 Too Many Requests` error. Make sure to set this correctly for providers like Groq or Anthropic where your tier's limits apply across the entire account.
+- **Pricing Fields**: Pricing structures (`input_cost_per_m`, `output_cost_per_m`) are deferred to the v1-deferred schema implementation. Do not manually add price fields to your `agents.yaml` entry; they are fetched dynamically.
+- **Aider's SEARCH/REPLACE limits**: The `aider` transport relies heavily on search-and-replace blocks. If you are generating a brand-new file from scratch, aider may occasionally fail if there is no pre-existing code to target. Use the `generate` transport to bootstrap empty files, then hand them off to `aider` for iterative edits.
+
+## When it's more than one entry
+
+Certain providers require multiple entries within `agents.yaml` for what seems like a single model family.
+
+For instance, Google Gemini models have distinct quota buckets for Flash vs. Pro tiers, as well as distinct endpoint characteristics. In these scenarios, declare them as individual agent blocks so the router can gracefully fall back from a rate-limited Pro endpoint to a highly-available Flash endpoint:
+
+```yaml
+- id: gen-gemini-2.5-pro
+  provider: google
+  model: gemini-2.5-pro
+  quota_scope: individual
+  tier: strong
+  # ...
+
+- id: gen-gemini-2.5-flash
+  provider: google
+  model: gemini-2.5-flash
+  quota_scope: individual
+  tier: weak
+  # ...
+```