evals 2.2.8 → 2.4.0

package/start.sh

@@ -0,0 +1,186 @@
+#!/usr/bin/env bash
+#
+# Arize AX onboarding launcher.
+#
+# Lets you pick an installed coding agent and launches it seeded with the
+# onboarding prompt to walk you through signup → instrument → first trace.
+# This is the no-npm path; if you have Node, `npx evals` gives the same thing
+# with a nicer UI.
+#
+# Run it (do NOT use `curl | bash` — that hijacks stdin and breaks the menu
+# and the launched agent). Use process substitution so the menu and the agent
+# both get a real terminal:
+#
+#   bash <(curl -fsSL https://cdn.jsdelivr.net/npm/evals/start.sh)
+#
+# Non-interactive: ARIZE_AGENT=claude bash <(curl -fsSL .../start.sh)
+#             or:  bash <(curl -fsSL .../start.sh) --agent claude
+
+set -euo pipefail
+
+# The prompt ships inside the public `evals` npm package; jsDelivr serves package
+# files over HTTP, so it's fetchable even though the source repo is private.
+# Override with ARIZE_PROMPT_URL if needed.
+PROMPT_URL="${ARIZE_PROMPT_URL:-https://cdn.jsdelivr.net/npm/evals/onboarding-prompt.md}"
+
+# Prefer the richer, bundled-prompt `npx evals` experience when Node is present.
+# This shell script is the no-npm fallback; if npx exists, hand off to it.
+# Set ARIZE_SKIP_NPX=1 to force this shell path even when npx is available.
+if [ -z "${ARIZE_SKIP_NPX:-}" ] && command -v npx >/dev/null 2>&1; then
+  exec npx --yes evals
+fi
+
+# Supported agents: id | display label | argv to start the REPL seeded with a prompt.
+# The seed prompt is appended as the final argument at launch.
+AGENT_IDS=(claude codex cursor-agent copilot gemini)
+agent_label() {
+  case "$1" in
+    claude)       echo "Claude Code" ;;
+    codex)        echo "OpenAI Codex" ;;
+    cursor-agent) echo "Cursor" ;;
+    copilot)      echo "GitHub Copilot" ;;
+    gemini)       echo "Gemini CLI" ;;
+    *)            echo "$1" ;;
+  esac
+}
+agent_install_url() {
+  case "$1" in
+    claude)       echo "https://docs.claude.com/en/docs/claude-code" ;;
+    codex)        echo "https://developers.openai.com/codex/cli" ;;
+    cursor-agent) echo "https://docs.cursor.com/en/cli/overview" ;;
+    copilot)      echo "https://github.com/features/copilot/cli" ;;
+    gemini)       echo "https://github.com/google-gemini/gemini-cli" ;;
+  esac
+}
+
+# Parse --agent flag
+AGENT="${ARIZE_AGENT:-}"
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --agent) AGENT="${2:-}"; shift 2 ;;
+    --agent=*) AGENT="${1#*=}"; shift ;;
+    -h|--help)
+      echo "Usage: start.sh [--agent <claude|codex|cursor-agent|copilot|gemini>]"
+      exit 0 ;;
+    *) echo "Unknown argument: $1" >&2; exit 1 ;;
+  esac
+done
+
+# --- Welcome ---
+if [ -t 1 ] && [ "${TERM:-}" != "dumb" ]; then
+  _pink=$'\033[1;38;2;255;0;140m'; _dim=$'\033[2m'; _reset=$'\033[0m'
+else
+  _pink=""; _dim=""; _reset=""
+fi
+printf '\n%sArize AX%s %s— Evals & Observability for Agentic AI%s\n\n' "$_pink" "$_reset" "$_dim" "$_reset"
+printf "Let's get you tracing. I'll launch your coding agent with a guided prompt that\nwalks you through signup, instrumenting your app, and seeing your first traces.\n\n"
+
+# Detect installed agents (PATH lookup only — never executes them).
+DETECTED=()
+for id in "${AGENT_IDS[@]}"; do
+  if command -v "$id" >/dev/null 2>&1; then
+    DETECTED+=("$id")
+  fi
+done
+
+# Resolve which agent to launch.
+choose_agent() {
+  # Explicit choice wins if it's actually installed.
+  if [[ -n "$AGENT" ]]; then
+    if command -v "$AGENT" >/dev/null 2>&1; then
+      echo "$AGENT"; return 0
+    fi
+    echo "Requested agent '$AGENT' is not on your PATH." >&2
+    return 1
+  fi
+
+  if [[ ${#DETECTED[@]} -eq 0 ]]; then
+    echo "No supported coding agent found on your PATH." >&2
+    echo "Install one of these, then re-run:" >&2
+    for id in "${AGENT_IDS[@]}"; do
+      printf '  %-14s %s\n' "$(agent_label "$id")" "$(agent_install_url "$id")" >&2
+    done
+    return 1
+  fi
+
+  if [[ ${#DETECTED[@]} -eq 1 ]]; then
+    echo "${DETECTED[0]}"; return 0
+  fi
+
+  # Interactive menu — read from the real terminal, not the (piped) stdin.
+  if [[ ! -r /dev/tty ]]; then
+    echo "Multiple agents found but no terminal to prompt on. Set ARIZE_AGENT=<id>." >&2
+    return 1
+  fi
+  {
+    echo "Choose your coding agent:"
+    local i=1
+    for id in "${DETECTED[@]}"; do
+      printf '  %d) %s\n' "$i" "$(agent_label "$id")"
+      i=$((i + 1))
+    done
+  } >/dev/tty
+  local choice
+  while true; do
+    printf 'Enter a number [1-%d]: ' "${#DETECTED[@]}" >/dev/tty
+    read -r choice </dev/tty || return 1
+    if [[ "$choice" =~ ^[0-9]+$ ]] && (( choice >= 1 && choice <= ${#DETECTED[@]} )); then
+      echo "${DETECTED[$((choice - 1))]}"; return 0
+    fi
+    echo "Invalid choice." >/dev/tty
+  done
+}
+
+CHOSEN="$(choose_agent)" || exit 1
+
+# Download the prompt to a temp file. (mktemp templates must end in X's on
+# macOS, so add the .md extension afterwards.)
+PROMPT_FILE="$(mktemp "${TMPDIR:-/tmp}/arize-onboarding-XXXXXX")"
+mv "$PROMPT_FILE" "$PROMPT_FILE.md"
+PROMPT_FILE="$PROMPT_FILE.md"
+cleanup() { rm -f "$PROMPT_FILE"; }
+trap cleanup EXIT
+
+echo "Fetching the onboarding prompt…"
+download_failed() {
+  echo "Failed to download the onboarding prompt from $PROMPT_URL" >&2
+  echo "Check your connection and try again." >&2
+  exit 1
+}
+if command -v curl >/dev/null 2>&1; then
+  curl -fsSL "$PROMPT_URL" -o "$PROMPT_FILE" || download_failed
+elif command -v wget >/dev/null 2>&1; then
+  wget -qO "$PROMPT_FILE" "$PROMPT_URL" || download_failed
+else
+  echo "Need curl or wget to download the prompt." >&2
+  exit 1
+fi
+
+SEED="Read the file $PROMPT_FILE and follow it to set up Arize AX tracing in this project, walking me through each step and asking me questions as needed."
+
+echo "Launching $(agent_label "$CHOSEN")…"
+echo
+
+# Launch interactive + seeded, on the real terminal. No skip-permissions —
+# the agent's approval model and the prompt's own approval gate must stay intact.
+# Run as a child (not exec) so the EXIT trap cleans up the temp prompt file.
+run_agent() {
+  case "$CHOSEN" in
+    copilot|gemini) "$CHOSEN" -i "$SEED" ;;
+    *)              "$CHOSEN" "$SEED" ;;
+  esac
+}
+
+# Prefer inheriting stdin: TUI agents (e.g. codex/crossterm) need a read-write
+# terminal on fd 0. A `< /dev/tty` redirect opens fd 0 read-only and makes
+# crossterm panic ("reader source not set"). Only reach for /dev/tty when stdin
+# isn't already a terminal (the discouraged `curl | bash`), and open it
+# read-write with `<>`.
+if [ -t 0 ]; then
+  run_agent
+elif [ -r /dev/tty ]; then
+  run_agent <>/dev/tty
+else
+  echo "No interactive terminal available to launch the agent." >&2
+  exit 1
+fi

package/spawn-interactive.js

@@ -1,51 +0,0 @@
-import { spawn } from 'child_process';
-import { openSync } from 'fs';
-
-function tryOpenTTY() {
-  if (process.platform === 'win32') {
-    // On Windows, avoid CON device (issues in Node.js 22)
-    // Use isTTY check instead, and stdio: 'inherit' for spawning
-    if (process.stdin.isTTY && process.stdout.isTTY && process.stderr.isTTY) {
-      return { available: true, useInherit: true };
-    }
-    return { available: false };
-  } else {
-    // On macOS/Linux, use direct /dev/tty access
-    try {
-      const device = '/dev/tty';
-      openSync(device, 'r');
-      return { device, available: true, useInherit: false };
-    } catch (err) {
-      return { available: false };
-    }
-  }
-}
-
-const tty = tryOpenTTY();
-
-if (tty.available) {
-  const spawnOptions = tty.useInherit
-    ? { stdio: 'inherit', shell: true }
-    : {
-        stdio: [
-          openSync(tty.device, 'r'),
-          openSync(tty.device, 'w'),
-          openSync(tty.device, 'w')
-        ]
-      };
-
-  const child = spawn('node', ['./cli.js'], spawnOptions);
-
-  child.on('close', (code) => {
-    process.exit(code || 0);
-  });
-
-  child.on('error', (err) => {
-    console.error('Failed to start interactive CLI:', err.message);
-    console.log('Run `npx evals` to complete setup.');
-    // process.exit(1);
-  });
-} else {
-  console.log('Run `npx evals` to complete setup.');
-  // process.exit(1);
-}