evals 2.2.8 → 2.4.0

package/cli.js

@@ -1,246 +1,379 @@
-#!/usr/bin/env node
-
-import React, { useState, useEffect } from 'react';
-import { render, Box, Text, useInput, useApp, Static } from 'ink';
-import Gradient from 'ink-gradient';
-import { exec } from 'child_process';
-
-const e = React.createElement;
-
-// Diagnostic: Uncomment to verify script is running
-console.error('CLI script started, platform:', process.platform, 'isTTY:', process.stdin.isTTY);
-
-// "ARIZE EVALS" - EXACTLY matched width (both 44 chars)
-const largeLogo = `
- █████╗   ██████╗    ██╗  ███████╗  ███████╗
-██╔══██╗  ██╔══██╗   ██║  ╚══███╔╝  ██╔════╝
-███████║  ██████╔╝   ██║    ███╔╝   █████╗
-██╔══██║  ██╔══██╗   ██║   ███╔╝    ██╔══╝
-██║  ██║  ██║  ██║   ██║  ███████╗  ███████╗
-╚═╝  ╚═╝  ╚═╝  ╚═╝   ╚═╝  ╚══════╝  ╚══════╝
-███████╗ ██╗   ██╗  █████╗  ██╗     ███████╗
-██╔════╝ ██║   ██║ ██╔══██╗ ██║     ██╔════╝
-█████╗   ██║   ██║ ███████║ ██║     ███████╗
-██╔══╝   ╚██╗ ██╔╝ ██╔══██║ ██║     ╚════██║
-███████╗  ╚████╔╝  ██║  ██║ ███████╗███████║
-╚══════╝   ╚═══╝   ╚═╝  ╚═╝ ╚══════╝╚══════╝`;
-
-const mediumLogo = `
- █████╗   ██████╗    ██╗  ███████╗  ███████╗
-██╔══██╗  ██╔══██╗   ██║  ╚══███╔╝  ██╔════╝
-███████║  ██████╔╝   ██║    ███╔╝   █████╗
-██╔══██║  ██╔══██╗   ██║   ███╔╝    ██╔══╝
-██║  ██║  ██║  ██║   ██║  ███████╗  ███████╗
-╚═╝  ╚═╝  ╚═╝  ╚═╝   ╚═╝  ╚══════╝  ╚══════╝
-█▀▀ █ █ ▄▀█ █   █▀
-██▄ ▀▄▀ █▀█ █▄▄ ▄█`;
-
-const smallLogo = `ARIZE
-EVALS`;
-
-const OPTIONS = [
-  {
-    name: 'Evals Primer',
-    subtext: 'Learn the Craft',
-    url: 'https://arize.com/llm-evaluation/?utm_source=npmevals'
-  },
-  {
-    name: 'Arize AX SaaS',
-    subtext: 'Enterprise Agent Evaluation, Best in Class',
-    url: 'https://arize.com/docs/ax?utm_source=npmevals'
-  },
-  {
-    name: 'Arize Phoenix',
-    subtext: 'OSS Agent Evals & Traces, Fully Local',
-    url: 'https://arize.com/docs/phoenix?utm_source=npmevals'
-  },
-  {
-    name: 'Book an Eval Assessment',
-    subtext: 'Improve Your Agent Today',
-    url: 'https://arize.com/get-an-eval-assessment?utm_source=npmevals'
-  }
-];
-
-function openUrlInBrowser(url) {
-  const command = process.platform === 'win32'
-    ? `start ${url}`
-    : process.platform === 'darwin'
-      ? `open "${url}"`
-      : `xdg-open "${url}"`;
-
-  exec(command, (error) => {
-    if (error) {
-      console.error(`Failed to open browser: ${error.message}`);
-    }
-    process.exit(0);
-  });
-}
-
-// Header with hot pink to dark blue gradient
-function Header() {
-  const terminalWidth = process.stdout.columns || 80;
-
-  let logo;
-  if (terminalWidth >= 45) {
-    logo = largeLogo;
-  } else if (terminalWidth >= 35) {
-    logo = mediumLogo;
-  } else {
-    logo = smallLogo;
-  }
-
-  return e(Box, { flexDirection: 'column', marginBottom: 1 },
-    e(Gradient, { colors: ['#FF008C', '#FF1493', '#8B5CF6', '#1E3A8A'] },
-      e(Text, null, logo)
-    ),
-    e(Text, { color: 'gray', italic: true }, 'Evals and Observability for Agentic AI')
-  );
-}
-
-// Character-by-character shimmer component
-function ShimmerText({ text, shimmerPos }) {
-  const baseColor = '#FF1493';
-  const shimmerColors = ['#FF5AA7', '#FF85C0', '#FFB8D9', '#FFE0EE', '#FFB8D9', '#FF85C0', '#FF5AA7'];
-  const shimmerWidth = shimmerColors.length;
-
-  const chars = text.split('').map((char, i) => {
-    const distanceFromShimmer = i - shimmerPos;
-
-    let color = baseColor;
-    if (distanceFromShimmer >= 0 && distanceFromShimmer < shimmerWidth) {
-      color = shimmerColors[distanceFromShimmer];
-    }
-
-    return e(Text, { key: i, color, bold: true }, char);
-  });
-
-  return e(Box, null, ...chars);
-}
-
-// Menu item component - single line, hot pink selected with character shimmer
-function MenuItem({ name, subtext, isSelected, shimmerPos }) {
-  if (isSelected) {
-    return e(Box, null,
-      e(Text, { bold: true, color: '#FF1493' }, '❯ '),
-      e(ShimmerText, { text: name, shimmerPos }),
-      e(Text, { color: 'gray' }, ' — ' + subtext)
-    );
-  }
-
-  return e(Box, null,
-    e(Text, { color: 'white' }, '  ' + name),
-    e(Text, { color: 'gray' }, ' — ' + subtext)
-  );
-}
-
-// Main App component
-function App() {
-  const [selectedIndex, setSelectedIndex] = useState(0);
-  const [shimmerPos, setShimmerPos] = useState(-7); // Start off-screen
-  const { exit } = useApp();
-
-  const selectedName = OPTIONS[selectedIndex].name;
-
-  // Character-by-character shimmer animation
-  useEffect(() => {
-    const textLength = selectedName.length;
-    const shimmerWidth = 7;
-    const totalPositions = textLength + shimmerWidth + 5; // Extra padding for smooth loop
-
-    const timer = setInterval(() => {
-      setShimmerPos(prev => {
-        const next = prev + 1;
-        return next > totalPositions ? -shimmerWidth : next;
-      });
-    }, 120); // Speed of shimmer movement
-
-    return () => clearInterval(timer);
-  }, [selectedName]);
-
-  // Reset shimmer position when selection changes
-  useEffect(() => {
-    setShimmerPos(-7);
-  }, [selectedIndex]);
-
-  useInput((input, key) => {
-    if (input === 'q' || key.escape) {
-      exit();
-      return;
-    }
-
-    if (key.upArrow || input === 'k') {
-      setSelectedIndex(prev => (prev > 0 ? prev - 1 : OPTIONS.length - 1));
-    }
-
-    if (key.downArrow || input === 'j') {
-      setSelectedIndex(prev => (prev < OPTIONS.length - 1 ? prev + 1 : 0));
-    }
-
-    if (key.return) {
-      const selected = OPTIONS[selectedIndex];
-      exit();
-      setTimeout(() => {
-        console.log(`\nOpening ${selected.url} in your browser...\n`);
-        openUrlInBrowser(selected.url);
-      }, 100);
-    }
-  });
-
-  return e(Box, { flexDirection: 'column', padding: 1 },
-    e(Static, { items: ['header'] }, () => e(Header)),
-    e(Box, { marginBottom: 1 },
-      e(Text, { bold: true, color: 'white' }, 'Select an option:')
-    ),
-    e(Box, { flexDirection: 'column', paddingX: 1 },
-      ...OPTIONS.map((option, index) =>
-        e(MenuItem, {
-          key: index.toString(),
-          name: option.name,
-          subtext: option.subtext,
-          isSelected: index === selectedIndex,
-          shimmerPos: shimmerPos
-        })
-      )
-    ),
-    e(Box, { marginTop: 1 },
-      e(Text, { dimColor: true }, 'Enter to select · ↑↓ to navigate · Esc to cancel')
-    )
-  );
-}
-
-// Run the app
-// Check if we have an interactive terminal (required for Ink)
-if (!process.stdin.isTTY || !process.stdout.isTTY) {
-  console.error('Error: This command requires an interactive terminal.');
-  console.error('Please run `npx evals` from your terminal (not from a script or non-interactive environment).');
-  process.exit(1);
-}
-
-// Handle uncaught errors
-process.on('uncaughtException', (error) => {
-  console.error('Uncaught exception:', error.message);
-  console.error(error.stack);
-  process.exit(1);
-});
-
-process.on('unhandledRejection', (reason, promise) => {
-  console.error('Unhandled rejection at:', promise);
-  console.error('Reason:', reason);
-  process.exit(1);
-});
-
-// Ensure stdout is not buffered (important for Windows)
-if (process.stdout.isTTY) {
-  process.stdout.setEncoding('utf8');
-}
-
-try {
-  render(e(App), { patchConsole: false });
-} catch (error) {
-  console.error('Failed to render interactive CLI:', error.message);
-  console.error(error.stack);
-  if (process.platform === 'win32') {
-    console.error('\nNote: If you\'re using cmd.exe, try running in PowerShell or Git Bash instead.');
-  }
-  process.exit(1);
-}
+#!/usr/bin/env node
+
+import React, { useState, useEffect } from 'react';
+import { render, Box, Text, useInput, useApp, Static } from 'ink';
+import Gradient from 'ink-gradient';
+import { exec, spawn } from 'child_process';
+import { existsSync, readFileSync, writeFileSync, mkdtempSync, realpathSync } from 'fs';
+import { join } from 'path';
+import { tmpdir } from 'os';
+import { fileURLToPath } from 'url';
+
+const e = React.createElement;
+
+// The onboarding prompt is bundled with this package (onboarding-prompt.md, a
+// copy of the docs landing-page prompt). We read it, write it to a temp file,
+// and tell the agent to read that file — a ~27 KB prompt is too large to pass
+// reliably as a command-line argument.
+// TODO: sync this copy with the docs source (arize.com/docs) later.
+const BUNDLED_PROMPT_PATH = fileURLToPath(new URL('./onboarding-prompt.md', import.meta.url));
+
+// Write the bundled prompt to a temp file and return a short seed instruction
+// that points the agent at it.
+function prepareSeedPrompt() {
+  const promptText = readFileSync(BUNDLED_PROMPT_PATH, 'utf8');
+  const dir = mkdtempSync(join(tmpdir(), 'arize-onboarding-'));
+  const promptFile = join(dir, 'onboarding-prompt.md');
+  writeFileSync(promptFile, promptText, 'utf8');
+  return `Read the file ${promptFile} and follow it to set up Arize AX tracing in this project, walking me through each step and asking me questions as needed.`;
+}
+
+// Coding agents we can launch interactively, seeded with the prompt.
+// `args(seed)` returns the argv that starts the agent's REPL pre-loaded with `seed`.
+export const AGENTS = [
+  { id: 'claude', label: 'Claude Code', bin: 'claude', args: (s) => [s], installUrl: 'https://docs.claude.com/en/docs/claude-code' },
+  { id: 'codex', label: 'OpenAI Codex', bin: 'codex', args: (s) => [s], installUrl: 'https://developers.openai.com/codex/cli' },
+  { id: 'cursor-agent', label: 'Cursor', bin: 'cursor-agent', args: (s) => [s], installUrl: 'https://docs.cursor.com/en/cli/overview' },
+  { id: 'copilot', label: 'GitHub Copilot', bin: 'copilot', args: (s) => ['-i', s], installUrl: 'https://github.com/features/copilot/cli' },
+  { id: 'gemini', label: 'Gemini CLI', bin: 'gemini', args: (s) => ['-i', s], installUrl: 'https://github.com/google-gemini/gemini-cli' }
+];
+
+// PATH scan — detects an agent without executing it (running it could hang).
+export function isInstalled(bin) {
+  const path = process.env.PATH || '';
+  const dirs = path.split(process.platform === 'win32' ? ';' : ':');
+  const exts = process.platform === 'win32'
+    ? (process.env.PATHEXT || '.EXE;.CMD;.BAT').split(';')
+    : [''];
+  for (const dir of dirs) {
+    if (!dir) continue;
+    for (const ext of exts) {
+      if (existsSync(join(dir, bin + ext)) || existsSync(join(dir, bin + ext.toLowerCase()))) {
+        return true;
+      }
+    }
+  }
+  return false;
+}
+
+// "ARIZE EVALS" - EXACTLY matched width (both 44 chars)
+const largeLogo = `
+ █████╗   ██████╗    ██╗  ███████╗  ███████╗
+██╔══██╗  ██╔══██╗   ██║  ╚══███╔╝  ██╔════╝
+███████║  ██████╔╝   ██║    ███╔╝   █████╗
+██╔══██║  ██╔══██╗   ██║   ███╔╝    ██╔══╝
+██║  ██║  ██║  ██║   ██║  ███████╗  ███████╗
+╚═╝  ╚═╝  ╚═╝  ╚═╝   ╚═╝  ╚══════╝  ╚══════╝
+███████╗ ██╗   ██╗  █████╗  ██╗     ███████╗
+██╔════╝ ██║   ██║ ██╔══██╗ ██║     ██╔════╝
+█████╗   ██║   ██║ ███████║ ██║     ███████╗
+██╔══╝   ╚██╗ ██╔╝ ██╔══██║ ██║     ╚════██║
+███████╗  ╚████╔╝  ██║  ██║ ███████╗███████║
+╚══════╝   ╚═══╝   ╚═╝  ╚═╝ ╚══════╝╚══════╝`;
+
+const mediumLogo = `
+ █████╗   ██████╗    ██╗  ███████╗  ███████╗
+██╔══██╗  ██╔══██╗   ██║  ╚══███╔╝  ██╔════╝
+███████║  ██████╔╝   ██║    ███╔╝   █████╗
+██╔══██║  ██╔══██╗   ██║   ███╔╝    ██╔══╝
+██║  ██║  ██║  ██║   ██║  ███████╗  ███████╗
+╚═╝  ╚═╝  ╚═╝  ╚═╝   ╚═╝  ╚══════╝  ╚══════╝
+█▀▀ █ █ ▄▀█ █   █▀
+██▄ ▀▄▀ █▀█ █▄▄ ▄█`;
+
+const smallLogo = `ARIZE
+EVALS`;
+
+// Tag every URL the app opens with utm_source=npmevals (idempotent). Applied
+// centrally in openUrlInBrowser so all opened links (the agent install links)
+// carry it. Does not touch URLs inside the onboarding prompt.
+export function withUtm(url) {
+  try {
+    const u = new URL(url);
+    if (!u.searchParams.has('utm_source')) {
+      u.searchParams.set('utm_source', 'npmevals');
+    }
+    return u.toString();
+  } catch {
+    return url;
+  }
+}
+
+function openUrlInBrowser(rawUrl) {
+  const url = withUtm(rawUrl);
+  const command = process.platform === 'win32'
+    ? `start ${url}`
+    : process.platform === 'darwin'
+      ? `open "${url}"`
+      : `xdg-open "${url}"`;
+
+  exec(command, (error) => {
+    if (error) {
+      console.error(`Failed to open browser: ${error.message}`);
+    }
+    process.exit(0);
+  });
+}
+
+// Header with hot pink to dark blue gradient
+function Header() {
+  const terminalWidth = process.stdout.columns || 80;
+
+  let logo;
+  if (terminalWidth >= 45) {
+    logo = largeLogo;
+  } else if (terminalWidth >= 35) {
+    logo = mediumLogo;
+  } else {
+    logo = smallLogo;
+  }
+
+  return e(Box, { flexDirection: 'column', marginBottom: 1 },
+    e(Gradient, { colors: ['#FF008C', '#FF1493', '#8B5CF6', '#1E3A8A'] },
+      e(Text, null, logo)
+    ),
+    e(Text, { color: 'gray', italic: true }, 'Evals and Observability for Agentic AI')
+  );
+}
+
+// Character-by-character shimmer component
+function ShimmerText({ text, shimmerPos }) {
+  const baseColor = '#FF1493';
+  const shimmerColors = ['#FF5AA7', '#FF85C0', '#FFB8D9', '#FFE0EE', '#FFB8D9', '#FF85C0', '#FF5AA7'];
+  const shimmerWidth = shimmerColors.length;
+
+  const chars = text.split('').map((char, i) => {
+    const distanceFromShimmer = i - shimmerPos;
+
+    let color = baseColor;
+    if (distanceFromShimmer >= 0 && distanceFromShimmer < shimmerWidth) {
+      color = shimmerColors[distanceFromShimmer];
+    }
+
+    return e(Text, { key: i, color, bold: true }, char);
+  });
+
+  return e(Box, null, ...chars);
+}
+
+// Menu item component - single line, hot pink selected with character shimmer
+function MenuItem({ name, subtext, isSelected, shimmerPos }) {
+  if (isSelected) {
+    return e(Box, null,
+      e(Text, { bold: true, color: '#FF1493' }, '❯ '),
+      e(ShimmerText, { text: name, shimmerPos }),
+      e(Text, { color: 'gray' }, ' — ' + subtext)
+    );
+  }
+
+  return e(Box, null,
+    e(Text, { color: 'white' }, '  ' + name),
+    e(Text, { color: 'gray' }, ' — ' + subtext)
+  );
+}
+
+// Build the coding-agent picker items — the entry screen for `npx evals`.
+// Detected agents become launch items; if none are found we offer install
+// links instead so the screen is never a dead end.
+export function buildAgentItems() {
+  const detected = AGENTS.filter(a => isInstalled(a.bin));
+  if (detected.length > 0) {
+    return {
+      items: detected.map(a => ({
+        name: a.label,
+        subtext: 'Launch and walk me through setup',
+        launch: a
+      })),
+      none: false
+    };
+  }
+  return {
+    items: AGENTS.map(a => ({
+      name: `Install ${a.label}`,
+      subtext: 'No supported agent detected — open install docs',
+      url: a.installUrl
+    })),
+    none: true
+  };
+}
+
+// Main App component — goes straight into the onboarding flow: pick a coding
+// agent and launch it seeded with the prompt (no top-level menu).
+function App({ onDone }) {
+  const [agentState] = useState(() => buildAgentItems());
+  const [selectedIndex, setSelectedIndex] = useState(0);
+  const [shimmerPos, setShimmerPos] = useState(-7); // Start off-screen
+  const { exit } = useApp();
+
+  const items = agentState.items;
+  const selectedName = items[selectedIndex].name;
+
+  // Character-by-character shimmer animation
+  useEffect(() => {
+    const textLength = selectedName.length;
+    const shimmerWidth = 7;
+    const totalPositions = textLength + shimmerWidth + 5; // Extra padding for smooth loop
+
+    const timer = setInterval(() => {
+      setShimmerPos(prev => {
+        const next = prev + 1;
+        return next > totalPositions ? -shimmerWidth : next;
+      });
+    }, 120); // Speed of shimmer movement
+
+    return () => clearInterval(timer);
+  }, [selectedName]);
+
+  // Reset shimmer position when selection changes
+  useEffect(() => {
+    setShimmerPos(-7);
+  }, [selectedIndex]);
+
+  useInput((input, key) => {
+    if (input === 'q' || key.escape) {
+      exit();
+      return;
+    }
+
+    if (key.upArrow || input === 'k') {
+      setSelectedIndex(prev => (prev > 0 ? prev - 1 : items.length - 1));
+    }
+
+    if (key.downArrow || input === 'j') {
+      setSelectedIndex(prev => (prev < items.length - 1 ? prev + 1 : 0));
+    }
+
+    if (key.return) {
+      const selected = items[selectedIndex];
+
+      if (selected.launch) {
+        onDone({ type: 'launch', agent: selected.launch });
+        exit();
+        return;
+      }
+
+      if (selected.url) {
+        onDone({ type: 'url', url: selected.url });
+        exit();
+        return;
+      }
+    }
+  });
+
+  const heading = agentState.none
+    ? 'No coding agent found on your PATH. Install one, then re-run:'
+    : 'Choose your coding agent to instrument your app:';
+
+  return e(Box, { flexDirection: 'column', padding: 1 },
+    e(Static, { items: ['header'] }, (item) => e(Header, { key: item })),
+    e(Box, { marginBottom: 1 },
+      e(Text, { bold: true, color: 'white' }, heading)
+    ),
+    e(Box, { flexDirection: 'column', paddingX: 1 },
+      ...items.map((option, index) =>
+        e(MenuItem, {
+          key: index.toString(),
+          name: option.name,
+          subtext: option.subtext,
+          isSelected: index === selectedIndex,
+          shimmerPos: shimmerPos
+        })
+      )
+    ),
+    e(Box, { marginTop: 1 },
+      e(Text, { dimColor: true }, 'Enter to select · ↑↓ to navigate · Esc to quit')
+    )
+  );
+}
+
+// Launch a coding agent on the real terminal, seeded with the onboarding prompt.
+// Ink has fully unmounted by this point, so the child inherits a clean TTY.
+function launchAgent(agent) {
+  const isWin = process.platform === 'win32';
+
+  let seed;
+  try {
+    seed = prepareSeedPrompt();
+  } catch (err) {
+    console.error(`Could not read the onboarding prompt: ${err.message}`);
+    process.exit(1);
+  }
+
+  console.log(`\nLaunching ${agent.label}…\n`);
+
+  const child = spawn(agent.bin, agent.args(seed), {
+    stdio: 'inherit',
+    shell: isWin // .cmd/.bat shims on Windows need the shell to resolve
+  });
+
+  child.on('error', (err) => {
+    console.error(`Could not launch ${agent.bin}: ${err.message}`);
+    console.error(`Make sure "${agent.bin}" is on your PATH, then try again.`);
+    process.exit(1);
+  });
+
+  child.on('close', (code) => {
+    process.exit(code || 0);
+  });
+}
+
+// Render the interactive app and act on the user's choice. Exported so it can
+// be driven explicitly; only auto-runs when this file is the entry point (see
+// the guard below), so importing it in tests doesn't launch the TUI.
+export async function main() {
+  // Ink needs an interactive terminal.
+  if (!process.stdin.isTTY || !process.stdout.isTTY) {
+    console.error('Error: This command requires an interactive terminal.');
+    console.error('Please run `npx evals` from your terminal (not from a script or non-interactive environment).');
+    process.exit(1);
+  }
+
+  // Handle uncaught errors
+  process.on('uncaughtException', (error) => {
+    console.error('Uncaught exception:', error.message);
+    console.error(error.stack);
+    process.exit(1);
+  });
+
+  process.on('unhandledRejection', (reason, promise) => {
+    console.error('Unhandled rejection at:', promise);
+    console.error('Reason:', reason);
+    process.exit(1);
+  });
+
+  // Ensure stdout is not buffered (important for Windows)
+  if (process.stdout.isTTY) {
+    process.stdout.setEncoding('utf8');
+  }
+
+  try {
+    let result = null;
+    const app = render(e(App, { onDone: (r) => { result = r; } }), { patchConsole: false });
+
+    await app.waitUntilExit();
+
+    if (!result) {
+      process.exit(0); // user quit / cancelled
+    } else if (result.type === 'url') {
+      console.log(`\nOpening ${result.url} in your browser...\n`);
+      openUrlInBrowser(result.url);
+    } else if (result.type === 'launch') {
+      launchAgent(result.agent);
+    }
+  } catch (error) {
+    console.error('Failed to render interactive CLI:', error.message);
+    console.error(error.stack);
+    if (process.platform === 'win32') {
+      console.error('\nNote: If you\'re using cmd.exe, try running in PowerShell or Git Bash instead.');
+    }
+    process.exit(1);
+  }
+}
+
+// Only run when invoked directly (as `evals`/`node cli.js`), not when imported.
+const invokedDirectly =
+  process.argv[1] &&
+  realpathSync(process.argv[1]) === realpathSync(fileURLToPath(import.meta.url));
+if (invokedDirectly) {
+  await main();
+}