@tonyclaw/llm-inspector 1.18.2 → 1.19.1

package/src/cli/templates/skill-onboard.ts

@@ -0,0 +1,458 @@
+/**
+ * Renders `~/.claude/skills/llm-inspector-onboard/SKILL.md` for the
+ * `llm-inspector onboard` subcommand. The body is a multi-phase teaching
+ * workflow with `EXPLAIN / DO / PAUSE` markers — Claude reads it and walks
+ * the user through real actions (start the proxy, hit /api/health, post a
+ * test request, poll /api/logs) instead of just printing a static tutorial.
+ *
+ * The template embeds platform-aware shell snippets inline (bash + PowerShell
+ * side-by-side). Skills are pure Markdown — they can't `require()` from the
+ * package at runtime, so the duplication is the price of portability.
+ *
+ * All PowerShell snippets that use `$env:` variables are written so they
+ * work when invoked via the Bash tool. The pattern is:
+ *   powershell -Command '<single-quoted block with $env:LITERAL>'
+ * NOT
+ *   powershell -Command "<double-quoted block with escaped \$env:>"
+ * The latter collapses `$env:` to a literal in bash before PowerShell sees
+ * it, producing garbled paths. Single-quoted bash arguments pass through
+ * to PowerShell unchanged.
+ */
+
+export type SkillOnboardContext = {
+  /** Package version (from `package.json`), stamped into the frontmatter. */
+  readonly version: string;
+  /** The default proxy port (mirrors `DEFAULT_PORT` in `src/cli.ts`). */
+  readonly port: number;
+  /**
+   * One-line "detected tools" summary written into the Preflight phase so the
+   * skill's instructions match the user's actual environment. Empty string
+   * means "no known tool detected — fall through to the curl example".
+   */
+  readonly detectedSummary: string;
+};
+
+/** The 9 phase headings the spec requires. The validator checks for these. */
+export const REQUIRED_PHASE_HEADINGS = [
+  "Phase 0: Idempotency check",
+  "Preflight",
+  "Phase 1: Welcome",
+  "Phase 2: Provider setup",
+  "Phase 3: Start proxy",
+  "Phase 4: Wire tool",
+  "Phase 4.5: Wire MCP server",
+  "Phase 5: First capture",
+  "Phase 6: Tour & wrap",
+] as const;
+
+export function renderSkillOnboard(ctx: SkillOnboardContext): string {
+  const { version, port, detectedSummary } = ctx;
+  return `---
+name: llm-inspector-onboard
+description: Guided setup for llm-inspector v${version} — start the proxy, wire your AI coding tool, capture your first request, and learn the Web UI.
+metadata:
+  author: llm-inspector
+  version: ${version}
+---
+
+# llm-inspector onboard
+
+Guide the user from "I just installed llm-inspector" to "I can see my AI tool's traffic in the Web UI". This is a teaching experience — you'll do real work in their environment while explaining each step.
+
+Environment detected by the installer:
+${detectedSummary || "  (no known AI tool detected — the user can still use the generic curl example in Phase 4)"}
+
+Default proxy port: \`${port}\` (override with \`PORT=<n> llm-inspector\` or \`--port <n>\`).
+
+> **PAUSE protocol.** Every \`**PAUSE**\` marker in this skill is a real stop.
+> Use the \`AskUserQuestion\` tool to actually wait for the user before
+> continuing. Do not stream past a PAUSE based on context — the user has
+> not seen your output yet. Each PAUSE in the body below includes a sample
+> question you can adapt.
+
+---
+
+## Phase 0: Idempotency check
+
+**EXPLAIN:** "Before we do anything, let me see what's already set up. If some of the steps are already done, we can skip them."
+
+**DO:** Probe the three pieces of state this skill touches. Use targeted checks — do **not** read large JSON files into the conversation.
+
+\`\`\`bash
+# 1. Is the proxy already up?
+curl -fsS "http://localhost:${port}/api/health" 2>/dev/null && echo "PROXY: up" || echo "PROXY: down"
+
+# 2. Does the config have a real provider key?
+CFG="$HOME/.llm-inspector/config.json"
+if [ -f "$CFG" ]; then
+  if grep -qE '"apiKey"[[:space:]]*:[[:space:]]*"(sk-[^"]+|REPLACE|REPLACE_)' "$CFG" 2>/dev/null; then
+    echo "CONFIG: has key (no REPLACE placeholder)"
+  else
+    echo "CONFIG: missing or has placeholder key"
+  fi
+else
+  echo "CONFIG: file does not exist"
+fi
+
+# 3. Is the MCP server already wired? (project .mcp.json wins)
+PROJ_MCP=".mcp.json"
+HOME_MCP="$HOME/.claude.json"
+if [ -f "$PROJ_MCP" ] && grep -q '"llm-inspector"' "$PROJ_MCP"; then
+  echo "MCP: wired in $PROJ_MCP"
+elif [ -f "$HOME_MCP" ] && grep -q '"llm-inspector"' "$HOME_MCP"; then
+  echo "MCP: wired in $HOME_MCP"
+else
+  echo "MCP: not wired"
+fi
+\`\`\`
+
+\`\`\`powershell
+# Windows PowerShell — single-quoted so $env: expands correctly
+$port = ${port}
+$cfg  = Join-Path $env:USERPROFILE '.llm-inspector/config.json'
+
+# 1. Is the proxy already up?
+try {
+  $null = Invoke-RestMethod -Uri "http://localhost:$port/api/health" -TimeoutSec 2 -ErrorAction Stop
+  Write-Host 'PROXY: up'
+} catch {
+  Write-Host 'PROXY: down'
+}
+
+# 2. Does the config have a real provider key?
+if (Test-Path $cfg) {
+  $content = Get-Content $cfg -Raw
+  if ($content -match '"apiKey"[[:space:]]*:[[:space:]]*"(sk-[^"]+)"') {
+    Write-Host 'CONFIG: has key'
+  } elseif ($content -match 'REPLACE') {
+    Write-Host 'CONFIG: has placeholder key'
+  } else {
+    Write-Host 'CONFIG: missing key'
+  }
+} else {
+  Write-Host 'CONFIG: file does not exist'
+}
+
+# 3. Is the MCP server already wired? (project .mcp.json wins)
+$projMcp = Join-Path (Get-Location) '.mcp.json'
+$homeMcp = Join-Path $env:USERPROFILE '.claude.json'
+if ((Test-Path $projMcp) -and (Select-String -Path $projMcp -Pattern 'llm-inspector' -Quiet)) {
+  Write-Host "MCP: wired in $projMcp"
+} elseif ((Test-Path $homeMcp) -and (Select-String -Path $homeMcp -Pattern 'llm-inspector' -Quiet)) {
+  Write-Host "MCP: wired in $homeMcp"
+} else {
+  Write-Host 'MCP: not wired'
+}
+\`\`\`
+
+**DO:** Summarize the three checks in one line, then use \`AskUserQuestion\` to ask whether to skip the corresponding phases.
+
+> **PAUSE** — call \`AskUserQuestion\` with:
+> - header: \`Skip done\`
+> - question: \`Proxy/CONFIG/MCP state: <summary>. Skip the phases that are already done?\`
+> - options: \`["Yes, skip what's done", "No, walk me through everything again"]\`
+> Wait for the answer before moving to Preflight.
+
+---
+
+## Preflight
+
+**EXPLAIN:** "Quick env sanity check — make sure Node and Claude Code are present."
+
+**DO:** Run the platform-appropriate commands below.
+
+\`\`\`bash
+# Unix / macOS / WSL
+node --version                                       # expect >= 18
+command -v claude >/dev/null && echo "claude-code: present" || echo "claude-code: not detected"
+\`\`\`
+
+\`\`\`powershell
+# Windows PowerShell — single-quoted so $env: expands correctly
+node --version                                       # expect >= 18
+$null = Get-Command claude -ErrorAction SilentlyContinue
+if ($null) { Write-Host 'claude-code: present' } else { Write-Host 'claude-code: not detected' }
+\`\`\`
+
+> **PAUSE** — if Node is older than 18, ask the user to install a newer version (https://nodejs.org) before continuing. Use \`AskUserQuestion\` with header \`Node version\`.
+
+---
+
+## Phase 1: Welcome
+
+**EXPLAIN:**
+
+\`\`\`
+## Welcome to llm-inspector!
+
+llm-inspector is a transparent HTTP proxy + Web UI for AI coding tools. Point your tool at it, and you'll see every request and response — system prompts, tool definitions, message history, SSE streaming chunks, and token counts — captured live in a browser tab.
+
+**What we'll do in the next ~10 minutes:**
+1. Add your first LLM provider (Anthropic or OpenAI key)
+2. Start the proxy
+3. Wire your AI tool to use it
+4. Capture a first request end-to-end
+5. Tour the key UI affordances
+
+Ready? Let's start with the provider.
+\`\`\`
+
+> **PAUSE** — use \`AskUserQuestion\` with header \`Ready?\` and options \`["Yes, let's go", "Wait, I have a question"]\`. Wait for the user before continuing.
+
+---
+
+## Phase 2: Provider setup
+
+**EXPLAIN:** "A 'provider' is an upstream LLM endpoint — Anthropic, OpenAI, MiniMax, etc. llm-inspector routes each request to the right upstream based on the model name. You need at least one provider configured for the proxy to forward traffic."
+
+**DO:** First, re-check whether the config already has a real key (Phase 0 may have raced with a manual edit). Use the same \`grep -qE '"apiKey":"sk-' "$HOME/.llm-inspector/config.json"\` (bash) or \`Select-String\` (PowerShell) check from Phase 0. If a real key is present, skip the rest of this phase.
+
+**DO:** If no real key, ask the user for the provider type and API key via \`AskUserQuestion\`. The question should be a free-form text field (no fixed options) — the API key is a secret, so don't echo it back in the question UI.
+
+\`\`\`bash
+# After collecting the key, write the config
+mkdir -p "$HOME/.llm-inspector"
+cat > "$HOME/.llm-inspector/config.json" <<'JSON'
+{
+  "providers": [
+    {
+      "id": "anthropic",
+      "type": "anthropic",
+      "apiKey": "REPLACE_ME_BEFORE_WRITING",
+      "baseUrl": "https://api.anthropic.com"
+    }
+  ]
+}
+JSON
+# Then patch the apiKey with the user-provided value (use jq if available)
+\`\`\`
+
+\`\`\`powershell
+# Windows PowerShell — single-quoted so $env: expands correctly
+$dir  = Join-Path $env:USERPROFILE '.llm-inspector'
+$file = Join-Path $dir 'config.json'
+New-Item -ItemType Directory -Force -Path $dir | Out-Null
+@'
+{
+  "providers": [
+    {
+      "id": "anthropic",
+      "type": "anthropic",
+      "apiKey": "REPLACE_ME_BEFORE_WRITING",
+      "baseUrl": "https://api.anthropic.com"
+    }
+  ]
+}
+'@ | Set-Content -Path $file -Encoding UTF8
+# Then patch the apiKey with the user-provided value
+\`\`\`
+
+**DO:** Patch the placeholder with the actual key using \`jq\` (preferred) or a simple \`sed\`. Then read the file back to confirm the key is no longer \`REPLACE_ME_BEFORE_WRITING\`.
+
+**DO:** If the user declines to provide a key in the AskUserQuestion (selects "Skip for now"), do **not** write a placeholder config. Tell the user that Phase 5 (First capture) will be skipped, and that they can re-run the skill after adding a key.
+
+> **PAUSE** — use \`AskUserQuestion\` with header \`Provider key\` and options \`["Key is in, continue", "Skip for now, I'll add it later"]\`. Wait for the answer.
+
+---
+
+## Phase 3: Start proxy
+
+**EXPLAIN:** "Time to start the proxy. It binds to port ${port} by default, kills any process already on that port, and prints the URL."
+
+**DO:** Skip this phase entirely if the Phase 0 health check already reported \`PROXY: up\` and the user opted to skip done phases.
+
+**DO:** Otherwise, start the proxy in the background. On Windows, the npm shim is a \`.cmd\`/PowerShell file — \`Start-Process -FilePath "llm-inspector"\` returns "Invalid Win32 application" because Windows can't decide which shim to run. Use \`cmd /c start /B\` to background it through the cmd interpreter.
+
+\`\`\`bash
+# Unix / macOS / WSL
+nohup llm-inspector --no-open > /tmp/llm-inspector.log 2>&1 &
+\`\`\`
+
+\`\`\`powershell
+# Windows PowerShell — single-quoted so $env: expands correctly.
+# Locate the binary on PATH first (works for npm, pnpm, yarn, volta, fnm).
+# If not on PATH, fall back to the common npm global shim at $env:APPDATA.
+# As a last resort, let cmd /c resolve it through PATHEXT.
+$log = Join-Path $env:TEMP 'llm-inspector.log'
+$err = Join-Path $env:TEMP 'llm-inspector.err.log'
+$found = Get-Command llm-inspector -ErrorAction SilentlyContinue
+if ($found) {
+  $shim = $found.Source
+  $args = '--no-open'
+} elseif (Test-Path (Join-Path $env:APPDATA 'npm/llm-inspector.cmd')) {
+  $shim = Join-Path $env:APPDATA 'npm/llm-inspector.cmd'
+  $args = '--no-open'
+} else {
+  # bin not on PATH and not at the default npm prefix — let cmd resolve it
+  $shim = 'cmd.exe'
+  $args = '/c','start','/B','llm-inspector','--no-open'
+}
+Start-Process -FilePath $shim -ArgumentList $args -RedirectStandardOutput $log -RedirectStandardError $err -WindowStyle Hidden
+\`\`\`
+
+Then wait for the port to be ready:
+
+\`\`\`bash
+for i in $(seq 1 20); do
+  curl -fsS "http://localhost:${port}/api/health" >/dev/null 2>&1 && echo "ready" && break
+  sleep 0.5
+done
+\`\`\`
+
+**DO:** Hit the health endpoint to confirm the proxy is alive:
+
+\`\`\`bash
+curl -sS "http://localhost:${port}/api/health"
+\`\`\`
+
+> **PAUSE** — if the health check fails, show the user the log file (\`/tmp/llm-inspector.log\` or \`%TEMP%\\llm-inspector.log\`) and diagnose. Common issues: another process on the port, firewall, missing providers. Use \`AskUserQuestion\` with header \`Proxy up?\` and options \`["Yes, proxy is up", "No, I see an error in the log"]\`. Wait for the answer.
+
+---
+
+## Phase 4: Wire tool
+
+**EXPLAIN:** "Now we tell your AI tool to send traffic through the proxy instead of directly to the upstream API. The exact env var depends on which tool you have."
+
+**DO:** Based on the \`Environment detected\` block at the top of this skill, print the matching wiring command. Examples for each supported tool:
+
+\`\`\`bash
+# Claude Code
+export ANTHROPIC_BASE_URL=http://localhost:${port}/proxy
+claude
+
+# OpenCode
+export LLM_BASE_URL=http://localhost:${port}/proxy
+opencode
+
+# MiMo Code
+export OPENAI_BASE_URL=http://localhost:${port}/proxy
+mimo
+
+# Cursor / Cody — set the OpenAI base URL in each tool's settings panel to http://localhost:${port}/proxy
+\`\`\`
+
+For a tool that wasn't auto-detected, fall through to the generic curl test in the next phase — the user can wire their tool later.
+
+> **PAUSE** — use \`AskUserQuestion\` with header \`Tool wired?\` and options \`["Yes, env var is set, claude is running", "No, I'm going to use the curl test instead"]\`. Wait for the answer.
+
+---
+
+## Phase 4.5: Wire MCP server
+
+**EXPLAIN:** "The proxy also exposes an MCP server at \`http://localhost:${port}/api/mcp\`. Your AI agent can query logs, replay requests, and test providers through it — no need to leave the editor."
+
+**DO:** Skip this phase if Phase 0 reported \`MCP: wired in <path>\` and the user opted to skip done phases.
+
+**DO:** Otherwise, check the project-level \`.mcp.json\` first (preferred — modern Claude Code convention), then fall back to \`~/.claude.json\`. Use the \`Read\` tool to inspect; do **not** \`cat\` a 40 KB file into the conversation.
+
+If neither has an \`llm-inspector\` entry, add one. The simplest path is to write to project \`.mcp.json\` (create it if missing):
+
+\`\`\`json
+// .mcp.json  (project root)
+{
+  "mcpServers": {
+    "llm-inspector": {
+      "type": "http",
+      "url": "http://localhost:${port}/api/mcp"
+    }
+  }
+}
+\`\`\`
+
+If \`mcpServers\` already exists in \`.mcp.json\`, merge the \`llm-inspector\` key into it via the \`Edit\` tool — do not overwrite other entries. If you can't create a project \`.mcp.json\` (no project root, permission, etc.), fall back to merging into \`~/.claude.json\` using the same \`Read\`/\`Edit\` pattern.
+
+**DO:** Verify the handshake. The MCP \`initialize\` request should return 200 with a \`serverInfo\` payload — that proves the server is mounted and reachable:
+
+\`\`\`bash
+curl -sS -X POST "http://localhost:${port}/api/mcp" \\
+  -H "Content-Type: application/json" \\
+  -H "Accept: application/json, text/event-stream" \\
+  -d '{
+    "jsonrpc": "2.0",
+    "id": 1,
+    "method": "initialize",
+    "params": {
+      "protocolVersion": "2025-03-26",
+      "capabilities": {},
+      "clientInfo": { "name": "onboard-check", "version": "0" }
+    }
+  }' | grep -o '"name":"llm-inspector"' && echo "handshake OK"
+\`\`\`
+
+The \`grep -o '"name":"llm-inspector"'\` extracts only the serverInfo name — do not dump the full response. If the server returns session IDs, store the \`mcp-session-id\` header from the first response and use it for the follow-up \`tools/list\` call.
+
+> **PAUSE** — use \`AskUserQuestion\` with header \`MCP OK?\` and options \`["Yes, handshake returned 200", "No, the call failed"]\`. Wait for the answer.
+
+---
+
+## Phase 5: First capture
+
+**EXPLAIN:** "Let's prove the proxy works end-to-end. We'll send one real request through it and confirm the log shows up in the API."
+
+**DO:** First, re-check the config. If the \`apiKey\` is still a \`REPLACE_ME_BEFORE_WRITING\` placeholder (user opted out in Phase 2), **skip the capture test** and tell the user to fill in their key and re-run the skill. A 401 from the upstream is fine if they did provide a real key — the proxy will still log the request.
+
+Fire a minimal Anthropic-format request through the proxy:
+
+\`\`\`bash
+curl -sS -X POST "http://localhost:${port}/proxy/v1/messages" \\
+  -H "Content-Type: application/json" \\
+  -H "anthropic-version: 2023-06-01" \\
+  -H "x-api-key: \${LLM_INSPECTOR_API_KEY:-sk-no-key-needed-for-routing}" \\
+  -d '{"model":"claude-3-5-sonnet-20241022","max_tokens":1,"messages":[{"role":"user","content":"ping"}]}' \\
+  -o /tmp/llm-inspector-capture.json -w 'STATUS:%{http_code}\\n'
+\`\`\`
+
+**DO:** Poll the logs API for up to 5 seconds. A 200 with at least one entry means the request reached the proxy:
+
+\`\`\`bash
+for i in $(seq 1 10); do
+  resp=$(curl -sS "http://localhost:${port}/api/logs?limit=1")
+  count=$(echo "$resp" | grep -o '"total":[0-9]*' | head -1 | grep -o '[0-9]*$')
+  if [ "\${count:-0}" -ge 1 ]; then
+    echo "captured"
+    echo "$resp" | head -c 400
+    break
+  fi
+  sleep 0.5
+done
+\`\`\`
+
+**DO:** Diagnose the response based on the actual status and body. **Do not** default to "auth failure" for every 4xx.
+
+| Status | Body hint | Meaning |
+|--------|-----------|---------|
+| 200 | normal | Real success — the upstream returned data |
+| 401 | \`"unauthorized"\` or similar | Upstream rejected the key (expected with a test key) |
+| 403 | \`"Request not allowed"\` | **Proxy's allowlist** — not an auth failure, the proxy rejected the model/config. Show the user the proxy log. |
+| 403 | other text | Could be upstream ACL — different problem |
+| 5xx | anything | Upstream network error |
+
+> **PAUSE** — use \`AskUserQuestion\` with header \`Captured?\` and options matching the diagnosis above. Wait for the answer.
+
+---
+
+## Phase 6: Tour & wrap
+
+**EXPLAIN:** "Everything's working. Here's the cheat sheet for the Web UI and the supporting surfaces:"
+
+- **Web UI**: \`http://localhost:${port}/\` — collapsible log rows, per-tab Copy/Expand in the log header, Diff with Raw (request body), Diff with Previous (compare adjacent requests), Replay (re-send a request), Export (JSON ZIP).
+- **MCP server**: \`http://localhost:${port}/api/mcp\` — connect from your coding agent to query logs, replay, and test providers without leaving the editor. Look for the "MCP Ready" badge in the Web UI header.
+- **REST API**: \`/api/logs\`, \`/api/sessions\`, \`/api/providers\` — for scripting and shell-based inspection.
+- **Stop the proxy**:
+
+  \`\`\`bash
+  # Unix / macOS
+  lsof -ti:${port} | xargs -r kill -9
+
+  # Windows PowerShell — single-quoted so $env: expands correctly
+  Get-NetTCPConnection -LocalPort $port | ForEach-Object { Stop-Process -Id $_.OwningProcess -Force }
+  \`\`\`
+
+- **Re-run onboard**: \`llm-inspector onboard --force\` refreshes this skill.
+- **Full docs**: see the project README (linked from the Web UI footer).
+
+> **PAUSE** — use \`AskUserQuestion\` with header \`All set?\` and options \`["All set, I'm done", "Wait, I want to revisit a phase"]\`. Wait for the answer.
+
+You're done. Happy inspecting.
+`;
+}