@tonyclaw/llm-inspector 1.18.2 → 1.19.0

package/src/cli/onboard.ts

@@ -0,0 +1,229 @@
+/**
+ * `llm-inspector onboard` subcommand.
+ *
+ * Generates a Claude Code skill + slash command into the user's home dir so
+ * the user can run `/llm-inspector:onboard` to walk through a guided setup.
+ *
+ * Idempotent by default — re-runs without `--force` are no-ops. `--force`
+ * overwrites, `--dry-run` writes nothing. Designed to be safe to invoke
+ * from npm's `postinstall` script (any error is logged to stderr, not
+ * thrown — the postinstall must not fail the install).
+ */
+
+import { mkdirSync, writeFileSync, existsSync, readFileSync } from "node:fs";
+import { homedir } from "node:os";
+import { dirname, join } from "node:path";
+import { fileURLToPath } from "node:url";
+
+import { detectAll, detectFirst } from "./detect-tools.js";
+import { renderCommandOnboard } from "./templates/command-onboard.js";
+import { renderSkillOnboard, REQUIRED_PHASE_HEADINGS } from "./templates/skill-onboard.js";
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = dirname(__filename);
+
+const DEFAULT_PORT = 25947;
+const SKILL_DIR_NAME = "llm-inspector-onboard";
+const SKILL_FILE_NAME = "SKILL.md";
+// Windows reserves `:` for NTFS alternate data streams, so a file named
+// `llm-inspector:onboard.md` becomes an empty `llm-inspector` on disk.
+// Use `-` on Windows and `:` on Unix to match the slash-command convention
+// (`/llm-inspector:onboard` vs `/llm-inspector-onboard`) within the
+// constraints of each platform.
+const COMMAND_FILE_NAME =
+  process.platform === "win32" ? "llm-inspector-onboard.md" : "llm-inspector:onboard.md";
+
+export type OnboardFlags = {
+  force: boolean;
+  dryRun: boolean;
+  skipProvider: boolean;
+  skipToolWire: boolean;
+  skillDir: string | null;
+};
+
+function parseFlags(argv: readonly string[]): OnboardFlags {
+  const flags: OnboardFlags = {
+    force: false,
+    dryRun: false,
+    skipProvider: false,
+    skipToolWire: false,
+    skillDir: null,
+  };
+  for (let i = 0; i < argv.length; i++) {
+    const arg = argv[i];
+    switch (arg) {
+      case undefined:
+        continue;
+      case "--force":
+        flags.force = true;
+        break;
+      case "--dry-run":
+        flags.dryRun = true;
+        break;
+      case "--skip-provider":
+        flags.skipProvider = true;
+        break;
+      case "--skip-tool-wire":
+        flags.skipToolWire = true;
+        break;
+      case "--skill-dir": {
+        const next = argv[i + 1];
+        if (next === undefined) {
+          process.stderr.write("llm-inspector onboard: --skill-dir requires a path argument\n");
+          process.exit(2);
+        }
+        flags.skillDir = next;
+        i++;
+        break;
+      }
+      case "--help":
+      case "-h":
+        printHelp();
+        process.exit(0);
+        break;
+      default:
+        process.stderr.write(`llm-inspector onboard: unknown flag: ${arg}\n`);
+        process.exit(2);
+    }
+  }
+  return flags;
+}
+
+function printHelp(): void {
+  process.stdout.write(`llm-inspector onboard — install the llm-inspector Claude Code skill
+
+Usage:
+  llm-inspector onboard [options]
+
+Options:
+  --force              Overwrite the existing skill and slash command if they exist
+  --dry-run            Print target paths and a template preview, write nothing
+  --skip-provider      Skip the provider-setup phase in the skill body
+  --skip-tool-wire     Skip the wire-tool phase in the skill body
+  --skill-dir <path>   Override the target skill directory (default: ~/.claude/skills)
+  -h, --help           Show this help
+
+Exit codes:
+  0  success (or already installed)
+  2  invalid arguments
+  1  write failed
+`);
+}
+
+function resolveTargets(flags: OnboardFlags): {
+  skillFile: string;
+  commandFile: string;
+} {
+  const claudeRoot = flags.skillDir ?? join(homedir(), ".claude");
+  const skillDir = join(claudeRoot, "skills", SKILL_DIR_NAME);
+  const commandsDir = join(claudeRoot, "commands");
+  return {
+    skillFile: join(skillDir, SKILL_FILE_NAME),
+    commandFile: join(commandsDir, COMMAND_FILE_NAME),
+  };
+}
+
+function isObject(value: unknown): value is Record<string, unknown> {
+  return typeof value === "object" && value !== null;
+}
+
+function buildDetectedSummary(): string {
+  const entries = detectAll();
+  const present = entries.filter((e) => e.result.found);
+  const absent = entries.filter((e) => !e.result.found);
+  const presentList = present.map((e) => e.displayName).join(", ") || "(none)";
+  const absentList = absent.map((e) => e.displayName).join(", ") || "(none)";
+  return `  - Detected: ${presentList}\n  - Not detected: ${absentList}`;
+}
+
+export function runOnboard(argv: readonly string[]): Promise<number> {
+  try {
+    return Promise.resolve(runOnboardSync(argv));
+  } catch (err) {
+    // Last-resort safety net. The postinstall hook relies on this never
+    // surfacing — log to stderr and exit 0 so npm install keeps working.
+    const msg = err instanceof Error ? err.message : String(err);
+    process.stderr.write(`llm-inspector onboard: ${msg}\n`);
+    process.stderr.write(
+      "(postinstall skill install skipped — run `llm-inspector onboard` later)\n",
+    );
+    return Promise.resolve(0);
+  }
+}
+
+function runOnboardSync(argv: readonly string[]): number {
+  const flags = parseFlags(argv);
+  const { skillFile, commandFile } = resolveTargets(flags);
+
+  // Idempotency check — only relevant for the skill file. If the slash
+  // command file is missing but the skill is present, that's weird; treat
+  // it the same as "already installed" so a partial state doesn't get
+  // half-rewritten on the next run.
+  if (!flags.force && existsSync(skillFile)) {
+    process.stdout.write(`llm-inspector onboard: already installed at ${skillFile}\n`);
+    process.stdout.write("Re-run with --force to refresh.\n");
+    return 0;
+  }
+
+  // Version stamp — read from package.json so the skill and the published
+  // package can never disagree about which version wrote them.
+  let version = "0.0.0";
+  try {
+    const raw: unknown = JSON.parse(readFileSync(join(__dirname, "..", "package.json"), "utf8"));
+    if (isObject(raw) && typeof raw["version"] === "string") {
+      version = raw["version"];
+    }
+  } catch {
+    // best-effort: leave version as the fallback
+  }
+
+  const detectedSummary = buildDetectedSummary();
+  const skillBody = renderSkillOnboard({
+    version,
+    port: DEFAULT_PORT,
+    detectedSummary,
+  });
+  const commandBody = renderCommandOnboard();
+
+  if (flags.dryRun) {
+    process.stdout.write(`llm-inspector onboard --dry-run\n\n`);
+    process.stdout.write(`Skill target:     ${skillFile}\n`);
+    process.stdout.write(`Command target:   ${commandFile}\n\n`);
+    process.stdout.write(`Skill preview (first 5 lines + headings):\n`);
+    const previewLines = skillBody.split("\n").slice(0, 5);
+    process.stdout.write(`${previewLines.join("\n")}\n`);
+    process.stdout.write(`...\n`);
+    for (const heading of REQUIRED_PHASE_HEADINGS) {
+      process.stdout.write(`  - ${heading}\n`);
+    }
+    process.stdout.write(`\nDetected tools:\n${detectedSummary}\n`);
+    process.stdout.write(`\nNo files were written.\n`);
+    return 0;
+  }
+
+  // Ensure target dirs exist. mkdirSync with recursive: true is a no-op if
+  // the dir already exists, and on failure it throws — caught by the
+  // outer try/catch and reported as a soft postinstall skip.
+  mkdirSync(join(skillFile, ".."), { recursive: true });
+  mkdirSync(join(commandFile, ".."), { recursive: true });
+
+  try {
+    writeFileSync(skillFile, skillBody, "utf8");
+    writeFileSync(commandFile, commandBody, "utf8");
+  } catch (err) {
+    const msg = err instanceof Error ? err.message : String(err);
+    process.stderr.write(`llm-inspector onboard: failed to write files: ${msg}\n`);
+    return 1;
+  }
+
+  const firstTool = detectFirst();
+  const toolHint = firstTool !== null ? firstTool.displayName : "no known tool detected";
+
+  process.stdout.write(`Installed skill to:   ${skillFile}\n`);
+  process.stdout.write(`Installed command to: ${commandFile}\n`);
+  process.stdout.write(`\nNext steps:\n`);
+  process.stdout.write(`  - Open Claude Code and run: /llm-inspector:onboard\n`);
+  process.stdout.write(`  - Or refresh later: llm-inspector onboard --force\n`);
+  process.stdout.write(`  - Detected primary tool: ${toolHint}\n`);
+  return 0;
+}

package/src/cli/templates/command-onboard.ts

@@ -0,0 +1,17 @@
+/**
+ * Renders `~/.claude/commands/llm-inspector:onboard.md` — a thin slash
+ * command that loads the `llm-inspector-onboard` skill. Claude Code
+ * auto-discovers both the skill and the command once they're written,
+ * so the command body just references the skill by name.
+ */
+
+export function renderCommandOnboard(): string {
+  return `---
+description: Walk through llm-inspector setup — start the proxy, wire your AI tool, capture your first request.
+---
+
+Invoke the \`llm-inspector-onboard\` skill and follow its phases.
+
+The user wants to set up llm-inspector. If they have specific context (e.g. "I'm on Windows", "I already added my API key"), honor it — but otherwise just run the skill end-to-end and let the \`EXPLAIN / DO / PAUSE\` markers drive the conversation.
+`;
+}

package/src/cli/templates/skill-onboard.ts

@@ -0,0 +1,325 @@
+/**
+ * Renders `~/.claude/skills/llm-inspector-onboard/SKILL.md` for the
+ * `llm-inspector onboard` subcommand. The body is a 7-phase teaching
+ * workflow with `EXPLAIN / DO / PAUSE` markers — Claude reads it and walks
+ * the user through real actions (start the proxy, hit /api/health, post a
+ * test request, poll /api/logs) instead of just printing a static tutorial.
+ *
+ * The template embeds platform-aware shell snippets inline (bash + PowerShell
+ * side-by-side). Skills are pure Markdown — they can't `require()` from the
+ * package at runtime, so the duplication is the price of portability.
+ */
+
+export type SkillOnboardContext = {
+  /** Package version (from `package.json`), stamped into the frontmatter. */
+  readonly version: string;
+  /** The default proxy port (mirrors `DEFAULT_PORT` in `src/cli.ts`). */
+  readonly port: number;
+  /**
+   * One-line "detected tools" summary written into the Preflight phase so the
+   * skill's instructions match the user's actual environment. Empty string
+   * means "no known tool detected — fall through to the curl example".
+   */
+  readonly detectedSummary: string;
+};
+
+/** The 8 phase headings the spec requires. The validator checks for these. */
+export const REQUIRED_PHASE_HEADINGS = [
+  "Preflight",
+  "Phase 1: Welcome",
+  "Phase 2: Provider setup",
+  "Phase 3: Start proxy",
+  "Phase 4: Wire tool",
+  "Phase 4.5: Wire MCP server",
+  "Phase 5: First capture",
+  "Phase 6: Tour & wrap",
+] as const;
+
+export function renderSkillOnboard(ctx: SkillOnboardContext): string {
+  const { version, port, detectedSummary } = ctx;
+  return `---
+name: llm-inspector-onboard
+description: Guided setup for llm-inspector v${version} — start the proxy, wire your AI coding tool, capture your first request, and learn the Web UI.
+metadata:
+  author: llm-inspector
+  version: ${version}
+---
+
+# llm-inspector onboard
+
+Guide the user from "I just installed llm-inspector" to "I can see my AI tool's traffic in the Web UI". This is a teaching experience — you'll do real work in their environment while explaining each step.
+
+Environment detected by the installer:
+${detectedSummary || "  (no known AI tool detected — the user can still use the generic curl example in Phase 4)"}
+
+Default proxy port: \`${port}\` (override with \`PORT=<n> llm-inspector\` or \`--port <n>\`).
+
+---
+
+## Preflight
+
+Before starting, verify the environment.
+
+**EXPLAIN:** "Let's make sure everything we need is in place. Two quick checks."
+
+**DO:** Run the platform-appropriate commands below. The user can copy-paste, or you can run them yourself if you have shell access.
+
+\`\`\`bash
+# Unix / macOS / WSL
+node --version                  # expect >= 18
+test -d "$HOME/.claude" && echo "claude-code: present" || echo "claude-code: not detected"
+\`\`\`
+
+\`\`\`powershell
+# Windows PowerShell
+node --version                  # expect >= 18
+if (Test-Path "$env:USERPROFILE\\.claude") { Write-Host "claude-code: present" } else { Write-Host "claude-code: not detected" }
+\`\`\`
+
+**PAUSE** — if Node is older than 18, ask the user to install a newer version (https://nodejs.org) before continuing.
+
+---
+
+## Phase 1: Welcome
+
+**EXPLAIN:**
+
+\`\`\`
+## Welcome to llm-inspector!
+
+llm-inspector is a transparent HTTP proxy + Web UI for AI coding tools. Point your tool at it, and you'll see every request and response — system prompts, tool definitions, message history, SSE streaming chunks, and token counts — captured live in a browser tab.
+
+**What we'll do in the next ~10 minutes:**
+1. Add your first LLM provider (Anthropic or OpenAI key)
+2. Start the proxy
+3. Wire your AI tool to use it
+4. Capture a first request end-to-end
+5. Tour the key UI affordances
+
+Ready? Let's start with the provider.
+\`\`\`
+
+**PAUSE** — wait for the user to confirm.
+
+---
+
+## Phase 2: Provider setup
+
+**EXPLAIN:** "A 'provider' is an upstream LLM endpoint — Anthropic, OpenAI, MiniMax, etc. llm-inspector routes each request to the right upstream based on the model name. You need at least one provider configured for the proxy to forward traffic."
+
+**DO:** Open (or create) \`~/.llm-inspector/config.json\` in the user's editor. If the file doesn't exist, create it with the structure below. Walk the user through filling in their API key for their provider of choice.
+
+\`\`\`json
+{
+  "providers": [
+    {
+      "id": "anthropic",
+      "type": "anthropic",
+      "apiKey": "sk-ant-...",
+      "baseUrl": "https://api.anthropic.com"
+    }
+  ]
+}
+\`\`\`
+
+Alternative (avoid touching the config file): point the user at the Web UI → top-right Settings button → Providers tab, which has a form-driven flow.
+
+**PAUSE** — wait for the user to confirm they have at least one provider with a key.
+
+---
+
+## Phase 3: Start proxy
+
+**EXPLAIN:** "Time to start the proxy. It binds to port ${port} by default, kills any process already on that port, and prints the URL."
+
+**DO:** Start the proxy in the background so you can keep working.
+
+\`\`\`bash
+# Unix / macOS / WSL
+nohup llm-inspector --no-open > /tmp/llm-inspector.log 2>&1 &
+\`\`\`
+
+\`\`\`powershell
+# Windows PowerShell
+Start-Process -FilePath "llm-inspector" -ArgumentList "--no-open" -RedirectStandardOutput "$env:TEMP\\llm-inspector.log" -RedirectStandardError "$env:TEMP\\llm-inspector.err.log" -WindowStyle Hidden
+\`\`\`
+
+Then wait for the port to be ready:
+
+\`\`\`bash
+# Wait up to 10s for the port to come up
+for i in $(seq 1 20); do
+  curl -fsS "http://localhost:${port}/api/health" >/dev/null 2>&1 && echo "ready" && break
+  sleep 0.5
+done
+\`\`\`
+
+**DO:** Hit the health endpoint to confirm the proxy is alive:
+
+\`\`\`bash
+curl -sS "http://localhost:${port}/api/health"
+\`\`\`
+
+**PAUSE** — if the health check fails, show the user the log file (\`/tmp/llm-inspector.log\` or \`%TEMP%\\\\llm-inspector.log\`) and diagnose. Common issues: another process on the port, firewall, missing providers.
+
+---
+
+## Phase 4: Wire tool
+
+**EXPLAIN:** "Now we tell your AI tool to send traffic through the proxy instead of directly to the upstream API. The exact env var depends on which tool you have."
+
+**DO:** Based on the \`Environment detected\` block at the top of this skill, print the matching wiring command. Examples for each supported tool:
+
+\`\`\`bash
+# Claude Code
+export ANTHROPIC_BASE_URL=http://localhost:${port}/proxy
+claude
+
+# OpenCode
+export LLM_BASE_URL=http://localhost:${port}/proxy
+opencode
+
+# MiMo Code
+export OPENAI_BASE_URL=http://localhost:${port}/proxy
+mimo
+
+# Cursor / Cody — set the OpenAI base URL in each tool's settings panel to http://localhost:${port}/proxy
+\`\`\`
+
+For a tool that wasn't auto-detected, fall through to the generic curl test in the next phase — the user can wire their tool later.
+
+**PAUSE** — wait for the user to confirm they've set the env var (or that they're going to use the curl test instead).
+
+---
+
+## Phase 4.5: Wire MCP server
+
+**EXPLAIN:** "The proxy also exposes an MCP server at \`http://localhost:${port}/api/mcp\`. Your AI agent can query logs, replay requests, and test providers through it — no need to leave the editor. We'll add an \`mcpServers\` entry so your agent picks it up automatically."
+
+**DO:** Add (or merge) an \`mcpServers\` entry in the user's Claude Code config. The file is \`~/.claude.json\` on all platforms; the MCP spec entry uses HTTP Streamable transport:
+
+\`\`\`json
+// ~/.claude.json  (merge into existing mcpServers)
+{
+  "mcpServers": {
+    "llm-inspector": {
+      "type": "http",
+      "url": "http://localhost:${port}/api/mcp"
+    }
+  }
+}
+\`\`\`
+
+If \`mcpServers\` already exists, add \`llm-inspector\` as a new key — don't clobber existing entries.
+
+PowerShell one-liner that creates the file if missing, or merges if present:
+
+\`\`\`powershell
+$cfg = Join-Path $env:USERPROFILE ".claude.json"
+if (Test-Path $cfg) {
+  $doc = Get-Content $cfg -Raw | ConvertFrom-Json
+} else {
+  $doc = [pscustomobject]@{ mcpServers = [pscustomobject]@{} }
+}
+if (-not ($doc.PSObject.Properties.Name -contains "mcpServers")) {
+  $doc | Add-Member -NotePropertyName mcpServers -NotePropertyValue ([pscustomobject]@{})
+}
+$doc.mcpServers | Add-Member -NotePropertyName "llm-inspector" -NotePropertyValue ([pscustomobject]@{
+  type = "http"
+  url  = "http://localhost:${port}/api/mcp"
+}) -Force
+$doc | ConvertTo-Json -Depth 10 | Set-Content $cfg -Encoding UTF8
+\`\`\`
+
+**DO:** Verify the handshake. The MCP \`initialize\` request should return 200 with a \`serverInfo\` payload — that proves the server is mounted and reachable:
+
+\`\`\`bash
+curl -sS -X POST "http://localhost:${port}/api/mcp" \\
+  -H "Content-Type: application/json" \\
+  -H "Accept: application/json, text/event-stream" \\
+  -d '{
+    "jsonrpc": "2.0",
+    "id": 1,
+    "method": "initialize",
+    "params": {
+      "protocolVersion": "2025-03-26",
+      "capabilities": {},
+      "clientInfo": { "name": "onboard-check", "version": "0" }
+    }
+  }'
+# expect: HTTP 200, body contains "result" with serverInfo.name = "llm-inspector"
+\`\`\`
+
+After the handshake, issue a \`tools/list\` to confirm the tool catalog is reachable:
+
+\`\`\`bash
+SESSION=<session-id-from-initialize-response>
+curl -sS -X POST "http://localhost:${port}/api/mcp" \\
+  -H "Content-Type: application/json" \\
+  -H "Accept: application/json, text/event-stream" \\
+  -H "mcp-session-id: $SESSION" \\
+  -d '{"jsonrpc":"2.0","id":2,"method":"tools/list","params":{}}'
+# expect: a result.tools array with at least 1 entry
+\`\`\`
+
+**PAUSE** — if \`initialize\` returns non-200, show the user the proxy log and re-check the JSON syntax. If it returns 200 but \`tools/list\` fails, the server is up but the session wasn't carried over — re-use the \`mcp-session-id\` header from the first response.
+
+---
+
+## Phase 5: First capture
+
+**EXPLAIN:** "Let's prove the proxy works end-to-end. We'll send one real request through it and confirm the log shows up in the API. A 401/403 from the upstream is fine — the point is that the *request* reaches the proxy."
+
+**DO:** Fire a minimal Anthropic-format request through the proxy. This works regardless of which tool the user wired up:
+
+\`\`\`bash
+curl -sS -X POST "http://localhost:${port}/proxy/v1/messages" \\
+  -H "Content-Type: application/json" \\
+  -H "anthropic-version: 2023-06-01" \\
+  -H "x-api-key: \${LLM_INSPECTOR_API_KEY:-sk-no-key-needed-for-routing}" \\
+  -d '{"model":"claude-3-5-sonnet-20241022","max_tokens":1,"messages":[{"role":"user","content":"ping"}]}'
+\`\`\`
+
+**DO:** Poll the logs API for up to 5 seconds. A 200 with at least one entry means success:
+
+\`\`\`bash
+for i in $(seq 1 10); do
+  resp=$(curl -sS "http://localhost:${port}/api/logs?limit=1")
+  count=$(echo "$resp" | grep -o '"total":[0-9]*' | head -1 | grep -o '[0-9]*$')
+  if [ "\${count:-0}" -ge 1 ]; then
+    echo "captured"
+    break
+  fi
+  sleep 0.5
+done
+\`\`\`
+
+**PAUSE** — show the user the captured log entry (id, status, model, elapsed). If the count never reached 1, the proxy didn't see the request — re-check the env var and the proxy log.
+
+---
+
+## Phase 6: Tour & wrap
+
+**EXPLAIN:** "Everything's working. Here's the cheat sheet for the Web UI and the supporting surfaces:"
+
+- **Web UI**: \`http://localhost:${port}/\` — collapsible log rows, per-tab Copy/Expand in the log header, Diff with Raw (request body), Diff with Previous (compare adjacent requests), Replay (re-send a request), Export (JSON ZIP).
+- **MCP server**: \`http://localhost:${port}/api/mcp\` — connect from your coding agent to query logs, replay, and test providers without leaving the editor. Look for the "MCP Ready" badge in the Web UI header.
+- **REST API**: \`/api/logs\`, \`/api/sessions\`, \`/api/providers\` — for scripting and shell-based inspection.
+- **Stop the proxy**:
+
+  \`\`\`bash
+  # Unix / macOS
+  lsof -ti:${port} | xargs -r kill -9
+
+  # Windows PowerShell
+  Get-NetTCPConnection -LocalPort ${port} | ForEach-Object { Stop-Process -Id \\$_.OwningProcess -Force }
+  \`\`\`
+
+- **Re-run onboard**: \`llm-inspector onboard --force\` refreshes this skill.
+- **Full docs**: see the project README (linked from the Web UI footer).
+
+**PAUSE** — let the user know they can come back to this skill at any time via \`/llm-inspector:onboard\` if they want a refresher, and call out that \`/llm-inspector:onboard --skip-tool-wire\` is the way to re-run later phases without re-detecting the tool.
+
+You're done. Happy inspecting.
+`;
+}