evals 2.3.0 → 2.5.0

package/README.md

@@ -1,6 +1,6 @@
 # evals
 
-Go from zero to your first [Arize AX](https://arize.com/docs/ax) traces in one command. `evals` launches your coding agent, seeded with a guided onboarding prompt that walks you through signing up, instrumenting your app, and confirming traces land.
+Go from zero to your first [Arize AX](https://arize.com/docs/ax) traces in one command. `evals` launches your coding agent, seeded with a guided onboarding prompt that walks you through signing up, adding tracing, and confirming traces land — for your app, or for the coding agent itself.
 
 ## Quick start
 
@@ -32,7 +32,9 @@ When Node **is** present, these hand off to `npx evals` for the richer UI.
 
 1. Detects an installed coding agent (never executes it — just a PATH lookup).
 2. Launches the agent in your current directory, seeded with the bundled onboarding prompt ([`onboarding-prompt.md`](./onboarding-prompt.md)).
-3. The agent walks you through: create/sign in to Arize AX → detect your stack → instrument it → verify your first traces.
+3. The agent walks you through: create/sign in to Arize AX → pick what to trace → instrument it → verify your first traces.
+
+You can trace an app in the current folder, a starter app the agent creates, or **the coding agent itself** — so every session you run, in any project, shows up in Arize AX. Agent tracing is machine-wide and captures prompts and tool output by default, so the prompt asks for that explicitly and offers per-category opt-outs.
 
 The agent runs with its **normal permission model** — `evals` never passes skip-permissions, so you approve each step, and the prompt itself gates real changes on your confirmation.
 

package/cli.js

@@ -4,7 +4,18 @@ import React, { useState, useEffect } from 'react';
 import { render, Box, Text, useInput, useApp, Static } from 'ink';
 import Gradient from 'ink-gradient';
 import { exec, spawn } from 'child_process';
-import { existsSync, readFileSync, writeFileSync, mkdtempSync, realpathSync } from 'fs';
+import {
+  existsSync,
+  readFileSync,
+  writeFileSync,
+  mkdtempSync,
+  realpathSync,
+  readdirSync,
+  copyFileSync,
+  chmodSync,
+  renameSync,
+  rmSync,
+} from 'fs';
 import { join } from 'path';
 import { tmpdir } from 'os';
 import { fileURLToPath } from 'url';
@@ -18,6 +29,54 @@ const e = React.createElement;
 // TODO: sync this copy with the docs source (arize.com/docs) later.
 const BUNDLED_PROMPT_PATH = fileURLToPath(new URL('./onboarding-prompt.md', import.meta.url));
 
+// Wheels for the coding-agent tracing harness, built by scripts/build-harness-wheel.mjs.
+// Staging these next to the prompt lets Step 4A install with no network and no
+// remote code execution — which is what keeps it working under permission
+// classifiers that block piping a downloaded script into a shell.
+const VENDOR_DIR = fileURLToPath(new URL('./vendor', import.meta.url));
+const OFFLINE_DIR_NAME = 'arize-offline';
+
+// Files the offline install needs. Absent or incomplete vendor/ means we stage
+// nothing — the prompt keys off the directory existing, so a half-populated one
+// would send the agent down the offline path with no wheel to install.
+const OFFLINE_FILES = ['harness-install.sh', 'harness-install.bat', 'LICENSE-coding-harness-tracing'];
+
+// Copy the bundled wheels next to the prompt. Returns the staged directory, or
+// null when this package has no usable vendor/ (a git checkout that hasn't run
+// the build). The shell launchers stage the same bundle themselves, fetching it
+// from jsDelivr via vendor/MANIFEST.
+//
+// Never throws, and never leaves a partial directory behind. Both matter: the
+// prompt decides which install path to take purely on whether this directory
+// exists, so a half-copied one would send the agent offline with no wheel to
+// install — and this is an optional enhancement, so a broken vendor/ must not
+// take the launcher down with it. Either outcome degrades to the network path.
+export function stageOfflineHarness(dir, vendorDir = VENDOR_DIR) {
+  if (!existsSync(vendorDir)) return null;
+
+  let staging;
+  try {
+    const wheels = readdirSync(vendorDir).filter((f) => f.endsWith('.whl'));
+    const hasHarnessWheel = wheels.some((f) => f.startsWith('coding_harness_tracing-'));
+    if (!hasHarnessWheel || !OFFLINE_FILES.every((f) => existsSync(join(vendorDir, f)))) return null;
+
+    // Build under a temp name and rename into place, so the directory the prompt
+    // looks for only ever appears complete. Same parent, so the rename is atomic.
+    staging = mkdtempSync(join(dir, '.staging-'));
+    for (const name of [...wheels, ...OFFLINE_FILES]) {
+      copyFileSync(join(vendorDir, name), join(staging, name));
+    }
+    chmodSync(join(staging, 'harness-install.sh'), 0o755);
+
+    const target = join(dir, OFFLINE_DIR_NAME);
+    renameSync(staging, target);
+    return target;
+  } catch {
+    if (staging) rmSync(staging, { recursive: true, force: true });
+    return null;
+  }
+}
+
 // Write the bundled prompt to a temp file and return a short seed instruction
 // that points the agent at it.
 function prepareSeedPrompt() {
@@ -25,6 +84,7 @@ function prepareSeedPrompt() {
   const dir = mkdtempSync(join(tmpdir(), 'arize-onboarding-'));
   const promptFile = join(dir, 'onboarding-prompt.md');
   writeFileSync(promptFile, promptText, 'utf8');
+  stageOfflineHarness(dir);
   return `Read the file ${promptFile} and follow it to set up Arize AX tracing in this project, walking me through each step and asking me questions as needed.`;
 }
 

package/onboarding-prompt.md

@@ -13,8 +13,8 @@ Welcome to Arize AX. I'll get you set up with tracing. Here's what I'll do:
 
 1. Install the AX CLI and Arize skills
 2. Create a free Arize AX account or sign you in
-3. Instrument this project or a new app
-4. Capture traces from your app
+3. Add tracing to your app, a new starter app, or this coding agent itself
+4. Capture your first traces
 
 Shall I proceed?
 ```
@@ -23,7 +23,7 @@ Do not proceed until the user approves.
 
 ## Prerequisites
 
-The AX CLI must be **arize-ax-cli `0.28.0` or newer** (Step 1 installs the latest) and needs **Python 3.11+** — a hard requirement. If Python is missing, stop and have the user install it from https://www.python.org/downloads/ and re-run. **Node.js 18+ with npx** is optional but installs the Arize skills; without it, don't stop — continue and use the docs paths in Step 6 (the Vercel AI SDK v7 starter needs Node.js 22+).
+The AX CLI must be **arize-ax-cli `0.29.0` or newer** (Step 1 installs the latest) and needs **Python 3.11+** — a hard requirement. If Python is missing, stop and have the user install it from https://www.python.org/downloads/ and re-run. **Node.js 18+ with npx** is optional but installs the Arize skills; without it, don't stop — continue and use the docs paths in Step 6 (the Vercel AI SDK v7 starter needs Node.js 22+).
 
 ## Step 1: Install or update the AX CLI and Arize skills
 
@@ -77,9 +77,11 @@ If a `default` profile exists but the probe failed, it's signed out or expired,
 - **No key anywhere** — sign up or sign in with browser OAuth:
 
   ```bash
-  ax profiles create default --auth-method oauth
+  ax profiles create default --auth-method oauth --utm-params "utm_source=npmevals&utm_medium=cli&utm_campaign=prompt-first-onboarding"
   ```
 
+The `--utm-params` flag tags a new sign-up with onboarding attribution. It's already included on the three browser OAuth commands that take it — the two `ax profiles create --auth-method oauth` calls and the `ax auth login` fallback. Pass those verbatim; don't drop it, change the values, or add the flag to the api-key `profiles create` or any other `ax` command.
+
 Always pass the positional profile name `default`. Without it, the CLI prompts `profile name [default]:`, receives EOF from an agent-run command, and exits with `Goodbye!` without creating a profile.
 
 The rest of this step applies only to the **browser OAuth** branch.
@@ -105,7 +107,7 @@ Did you just sign up for a new account with email and password?
 **If the user says yes (new email/password signup):** the localhost callback will never fire and the command will wait forever, so this is the one time you break the "keep it alive" rule. Once they confirm they've clicked the validation link and their account exists, terminate the waiting OAuth command (send Ctrl-C / kill that process), then re-run it:
 
 ```bash
-ax profiles create default --auth-method oauth
+ax profiles create default --auth-method oauth --utm-params "utm_source=npmevals&utm_medium=cli&utm_campaign=prompt-first-onboarding"
 ```
 
 The second run is now a plain sign-in for the validated account. Its callback lands normally, so handle it with all the standard rules below — keep it alive and wait for it to exit on its own.
@@ -121,7 +123,7 @@ ax spaces list --limit 1 --output json
 If the probe succeeds, continue. If the default OAuth profile already existed and the probe returns an authentication error, the profile is signed out or expired — run the fallback:
 
 ```bash
-ax auth login
+ax auth login --utm-params "utm_source=npmevals&utm_medium=cli&utm_campaign=prompt-first-onboarding"
 ```
 
 Handle `ax auth login` with the same rules: keep its callback server alive, wait for it to exit or print a success line, then rerun the probe. Do not run `ax auth login` immediately after creating an OAuth profile; the profile creation already performed login.
@@ -142,7 +144,21 @@ If `ARIZE_SPACE_ID` is already set (environment or `.env` / `.env.local`), use i
 
 Capture the space's **ID** (not its display name) for `ARIZE_SPACE_ID`. The `arize-otel` tracing config requires the space ID; a name will silently fail to route traces.
 
-## Step 4: Inspect the folder and choose a path
+## Step 4: Choose what to trace
+
+Ask what the user wants to trace before inspecting anything — the third option has nothing to do with what is in the current folder:
+
+```text
+What would you like to trace?
+
+1. An app in this folder
+2. A new starter app I create for you
+3. This coding agent itself — every session you run, in any project
+
+Which one?
+```
+
+If they pick **3**, go to [Step 4A](#step-4a-trace-this-coding-agent) and skip the folder inspection entirely. For **1** or **2**, continue below.
 
 Inspect the current folder to decide whether an app already exists. Do not change files during inspection. Look for:
 
@@ -154,7 +170,7 @@ Inspect the current folder to decide whether an app already exists. Do not chang
 
 In a monorepo, check the git root to get oriented, but only instrument apps in or below the current working directory. If the project spans more than one language, instrument each one (route each through its own integration page in Step 6).
 
-Then branch on what you found:
+Then branch on what you found — a folder with no app means option 2:
 
 ### If an app exists in the current folder
 
@@ -178,8 +194,138 @@ Go straight to the starter-app choice — do not ask whether to instrument the e
 
 If the user picks an unsupported pairing, explain the supported options and ask again.
 
+## Step 4A: Trace this coding agent
+
+Only for option 3. This traces the **agent harness**, not an app: every session the user runs, in every project on this machine. It edits files under their home directory, nothing in the current repo. When done, go to Step 8 — Steps 5 to 7 are app-only.
+
+Installer harness names: `claude`, `codex`, `cursor`, `copilot`, `gemini`, `kiro`, `opencode`, `omp` — note Cursor is `cursor`, not `cursor-agent`. Default to the agent you are running inside: state which you are and confirm, rather than asking.
+
+Per-agent setup, including the Claude Code and Cursor marketplace-plugin routes, is at `https://arize.com/docs/ax/integrations/platforms/<agent>/<agent>-tracing` (e.g. `.../claude-code/claude-code-tracing`) — the source of truth if anything below fails.
+
+### Get approval — this needs its own explicit yes
+
+Wider scope than an app install, so disclose it and wait:
+
+```text
+This traces <agent> itself. Three things first:
+
+- It applies to EVERY session you run, in every project on this machine.
+- Unless you say otherwise I'll turn on all three capture categories: your
+  prompts, what tools were asked to do (commands, file paths, URLs), and what
+  tools returned (file contents, command output).
+- It writes to ~/.arize/harness and <agent>'s settings file. Nothing in this
+  project changes.
+
+Want to skip any capture category — prompts, tool commands, or tool output?
+Shall I go ahead?
+```
+
+Do not proceed without a yes. Note which categories they accepted — you enable those explicitly below, and anything you leave out stays off.
+
+### Write the config file
+
+The installer reads credentials from a dotenv file, keeping the API key out of the command line, shell history, and this chat. Write it outside the project so it cannot be committed:
+
+```bash
+mkdir -p ~/.arize && : > ~/.arize/onboarding.env && chmod 600 ~/.arize/onboarding.env
+```
+
+Add the Step 3 space ID as file contents, plus one `true` line per category the user **accepted**. Unattended installs capture nothing unless asked to, so a category you omit is off — omit the line for anything they declined:
+
+```dotenv
+ARIZE_SPACE_ID=<space-id-from-step-3>
+ARIZE_LOG_PROMPTS=true
+ARIZE_LOG_TOOL_DETAILS=true
+ARIZE_LOG_TOOL_CONTENT=true
+```
+
+Do **not** set `ARIZE_PROJECT_NAME` — each harness defaults to its own project (`claude-code`, `codex`, …), which keeps two traced agents apart, and you read the real name back after installing. Then create the key into the same file, written atomically and never printed:
+
+```bash
+ax api-keys create --name "Coding agent tracing" --env-file ~/.arize/onboarding.env
+```
+
+**Skip that if `ARIZE_API_KEY` already existed in Step 2** — copy the existing value in without echoing it. At most one key, ever.
+
+### Install, then delete the file
+
+If a directory named `arize-offline/` sits beside this prompt file, install from it — no download, no remote script, so it is faster and far less likely to be refused:
+
+```bash
+# keep `< /dev/null`: an installer too old for --non-interactive then fails
+# fast instead of hanging on a prompt you cannot answer
+ARIZE_ENV_FILE=~/.arize/onboarding.env \
+  bash <prompt-dir>/arize-offline/harness-install.sh \
+  <harness> --wheel-dir <prompt-dir>/arize-offline --non-interactive < /dev/null
+```
+
+Otherwise fetch it (the harness name must come first, before any flag):
+
+```bash
+ARIZE_ENV_FILE=~/.arize/onboarding.env \
+  bash <(curl -fsSL https://raw.githubusercontent.com/Arize-ai/coding-harness-tracing/main/install.sh) \
+  <harness> --non-interactive < /dev/null
+```
+
+Delete the env file only once the install has actually run — a retry needs it, and so does the user if they end up running the command:
+
+```bash
+rm -f ~/.arize/onboarding.env
+```
+
+On Windows, use `harness-install.bat` from the same directory — `cmd` cannot run the `.sh`:
+
+```powershell
+$env:ARIZE_ENV_FILE = "$HOME\.arize\onboarding.env"
+& <prompt-dir>\arize-offline\harness-install.bat <harness> --wheel-dir <prompt-dir>\arize-offline --non-interactive
+```
+
+Without `arize-offline/`, use the install command from the agent's docs page with `$env:ARIZE_ENV_FILE` set first.
+
+An `EOFError` or hang means the installer predates `--non-interactive`: have the user run it without that flag and without `< /dev/null`, answering its prompts. On any other failure, report what it printed and stop — never retry with a different backend or type the key into a prompt.
+
+### If your permission layer refuses to run it
+
+Expected on the fetched command — auto-approval modes hold back piping a downloaded script into a shell. Retry **once** with the installer downloaded to a file first; if still refused, stop and do not hunt for a phrasing that slips through. Hand it over instead:
+
+```text
+My permission settings won't let me run the installer. It creates
+~/.arize/harness and adds hooks to <harness>'s settings file. Either run it
+yourself — prefix with ! so I see the output — or tell me to go ahead and
+I'll retry.
+
+<the command, one line>
+```
+
+A go-ahead authorizes that one command, nothing wider. **Never** frame this as bypassing a safety check, and never suggest an approval flag on the agent. **Keep the env file** until they confirm — they need those credentials.
+
+### Verify before asking the user for anything
+
+```bash
+~/.arize/harness/install.sh status --json
+```
+
+Exit `0` means configured **and** hooks wired up; `1` nothing configured; `2` configured but hooks missing (the `unregistered` list names which to re-install). Continue only on `0`. Take the project name for polling from `harnesses[].project_name` in the payload rather than guessing it. The payload holds no secrets.
+
+### Get the first traces
+
+Hooks load at session start, so **this session will never emit traces**. Do not restart or kill yourself — that would end the setup. Use a second session so you stay alive to poll:
+
+```text
+Tracing is installed. This session won't be traced — it started before the
+hooks existed. So:
+
+1. Open a new terminal and start <agent> there.
+2. Ask it something small, like "list the files in this folder".
+3. Tell me when you have, and I'll check for traces.
+```
+
+Then poll with the Step 7 command and counter, passing that project name and space ID, every ~15 seconds for up to ~3 minutes. On a non-zero count go to Step 8. On timeout say so plainly and give the likely causes: the new session started before the install finished, the agent was never asked to do anything, `ARIZE_TRACE_ENABLED` is `false` in the agent's settings, or the wrong space. Never fabricate trace results.
+
 ## Step 5: Present the plan and get approval
 
+Steps 5 to 7 cover the app paths (options 1 and 2). If you took Step 4A, its own approval gate and verification replace them — go straight to Step 8.
+
 Before creating any remote resource, writing files, or installing dependencies, present one consolidated plan and wait for approval. This is the gate the intro refers to — nothing so far has modified the app or created AX resources.
 
 For an existing app, cover: detected language and framework, package manager, LLM provider or agent framework, any existing tracing to preserve, the env file that will be updated, the instrumentation packages and files that will change, whether a new AX user API key will be created or the existing `ARIZE_API_KEY` reused, and the project name that will be used.
@@ -343,6 +489,15 @@ You're set up. To go further:
 - AX CLI: https://arize.com/docs/api-clients/cli/overview
 ```
 
+If you took Step 4A, add the controls that matter for agent tracing:
+
+```text
+- Your agent's tracing page: https://arize.com/docs/ax/integrations/platforms/<agent>/<agent>-tracing
+- Check it's still wired up: ~/.arize/harness/install.sh status
+- Pause it: set ARIZE_TRACE_ENABLED=false in <agent>'s settings
+- Remove it: ~/.arize/harness/install.sh uninstall <installer-name>
+```
+
 ## Critical rules
 
 - Get the user's approval (Step 5) before creating AX resources, editing files, or installing dependencies.
@@ -351,5 +506,6 @@ You're set up. To go further:
 - Create at most one AX API key: skip if `ARIZE_API_KEY` already exists (reuse it), otherwise a **single** `ax api-keys create --env-file <file>`. Git-ignore that file before creating the key; never create a second key.
 - Write the space **ID** (not its name) to `ARIZE_SPACE_ID`, or traces won't route; never create an AX project explicitly (it's made on first ingestion).
 - Initialize tracing before LLM clients are created, and flush/shut down the tracer before short-lived scripts exit. Vercel AI SDK v7 also needs Node.js 22+, `@ai-sdk/otel` registered, and `experimental_telemetry: { isEnabled: true }` per call.
+- Tracing the coding agent (Step 4A) is machine-wide and captures prompts and tool output by default, so it needs its own explicit yes — never fold it into another approval. Run its installer with `--non-interactive` and `< /dev/null` so it can never sit waiting on a prompt, verify with `status --json` before claiming success, and never restart or kill the session you are running in.
 
 Docs: https://arize.com/docs/llms.txt

package/package.json

@@ -1,13 +1,15 @@
 {
   "name": "evals",
-  "version": "2.3.0",
+  "version": "2.5.0",
   "description": "Arize AX onboarding — instrument your app with tracing via your coding agent",
   "type": "module",
   "main": "cli.js",
   "scripts": {
     "postinstall": "node -e \"console.log('Arize evals installed — run: npx evals')\"",
     "start": "node cli.js",
-    "test": "node --test"
+    "test": "node --test",
+    "build:wheel": "node scripts/build-harness-wheel.mjs",
+    "prepack": "npm run build:wheel"
   },
   "dependencies": {
     "ink": "^6.0.0",
@@ -21,7 +23,8 @@
     "cli.js",
     "onboarding-prompt.md",
     "start.sh",
-    "start.ps1"
+    "start.ps1",
+    "vendor/"
   ],
   "keywords": [
     "arize",

package/start.ps1

@@ -26,6 +26,8 @@ $ErrorActionPreference = 'Stop'
 # files over HTTP, so it's fetchable even though the source repo is private.
 # Override with $env:ARIZE_PROMPT_URL if needed.
 $PromptUrl = if ($env:ARIZE_PROMPT_URL) { $env:ARIZE_PROMPT_URL } else { 'https://cdn.jsdelivr.net/npm/evals/onboarding-prompt.md' }
+# Where the bundled harness wheels live, same package. Override for testing.
+$VendorUrl = if ($env:ARIZE_VENDOR_URL) { $env:ARIZE_VENDOR_URL } else { 'https://cdn.jsdelivr.net/npm/evals/vendor' }
 
 # Prefer the richer, bundled-prompt `npx evals` experience when Node is present.
 # This script is the no-npm fallback; if npx exists, hand off to it.
@@ -94,11 +96,11 @@ Write-Host ""
 $chosen = Resolve-Agent
 if (-not $chosen) { exit 1 }
 
-# Download the prompt to a temp file (rename the .tmp to .md rather than
-# leaving both behind).
-$tmp = New-TemporaryFile
-$promptFile = [System.IO.Path]::ChangeExtension($tmp.FullName, 'md')
-Rename-Item -Path $tmp.FullName -NewName (Split-Path $promptFile -Leaf)
+# A directory, not a bare temp file: Step 4A looks for the offline bundle *beside*
+# the prompt, so the prompt needs a directory of its own.
+$promptDir = Join-Path ([System.IO.Path]::GetTempPath()) ("arize-onboarding-" + [System.Guid]::NewGuid().ToString('N').Substring(0, 8))
+New-Item -ItemType Directory -Path $promptDir -Force | Out-Null
+$promptFile = Join-Path $promptDir 'onboarding-prompt.md'
 try {
   Write-Host "Fetching the onboarding prompt..."
   try {
@@ -109,6 +111,42 @@ try {
     exit 1
   }
 
+  # Fetch the bundled tracing harness so Step 4A can install it without piping a
+  # downloaded script into a shell — the thing auto-approval permission
+  # classifiers refuse. `npx evals` ships these files; this path pulls them from
+  # the CDN, using vendor/MANIFEST because a launcher cannot glob a CDN and must
+  # not hardcode versioned wheel names.
+  #
+  # Best-effort throughout. Any failure leaves no arize-offline directory and the
+  # prompt takes its documented network path instead. Staged under a temp name and
+  # renamed, so the directory Step 4A keys off only ever appears complete.
+  Write-Host "Fetching the tracing harness..."
+  $staging = Join-Path $promptDir '.staging'
+  try {
+    New-Item -ItemType Directory -Path $staging -Force | Out-Null
+    $manifestPath = Join-Path $staging 'MANIFEST'
+    Invoke-RestMethod -Uri "$VendorUrl/MANIFEST" -OutFile $manifestPath
+
+    foreach ($line in Get-Content $manifestPath) {
+      if (-not $line.Trim()) { continue }
+      # MANIFEST is `shasum -a 256` format: "<sha256>  <filename>".
+      $parts = $line -split '\s+', 2
+      $expected = $parts[0]
+      $name = $parts[1].Trim()
+      $dest = Join-Path $staging $name
+      Invoke-RestMethod -Uri "$VendorUrl/$name" -OutFile $dest
+      $actual = (Get-FileHash -Path $dest -Algorithm SHA256).Hash.ToLower()
+      if ($actual -ne $expected.ToLower()) {
+        throw "checksum mismatch for $name"
+      }
+    }
+
+    Move-Item -Path $staging -Destination (Join-Path $promptDir 'arize-offline')
+  } catch {
+    Remove-Item $staging -Recurse -Force -ErrorAction SilentlyContinue
+    Write-Host "  (not available — Step 4A will download the installer instead)"
+  }
+
   $seed = "Read the file $promptFile and follow it to set up Arize AX tracing in this project, walking me through each step and asking me questions as needed."
 
   Write-Host ("Launching {0}..." -f $Agents[$chosen].Label)
@@ -124,5 +162,5 @@ try {
     & $chosen $seed
   }
 } finally {
-  Remove-Item $promptFile -ErrorAction SilentlyContinue
+  Remove-Item $promptDir -Recurse -Force -ErrorAction SilentlyContinue
 }

package/start.sh

@@ -22,6 +22,8 @@ set -euo pipefail
 # files over HTTP, so it's fetchable even though the source repo is private.
 # Override with ARIZE_PROMPT_URL if needed.
 PROMPT_URL="${ARIZE_PROMPT_URL:-https://cdn.jsdelivr.net/npm/evals/onboarding-prompt.md}"
+# Where the bundled harness wheels live, same package. Override for testing.
+VENDOR_URL="${ARIZE_VENDOR_URL:-https://cdn.jsdelivr.net/npm/evals/vendor}"
 
 # Prefer the richer, bundled-prompt `npx evals` experience when Node is present.
 # This shell script is the no-npm fallback; if npx exists, hand off to it.
@@ -133,29 +135,69 @@ choose_agent() {
 
 CHOSEN="$(choose_agent)" || exit 1
 
-# Download the prompt to a temp file. (mktemp templates must end in X's on
-# macOS, so add the .md extension afterwards.)
-PROMPT_FILE="$(mktemp "${TMPDIR:-/tmp}/arize-onboarding-XXXXXX")"
-mv "$PROMPT_FILE" "$PROMPT_FILE.md"
-PROMPT_FILE="$PROMPT_FILE.md"
-cleanup() { rm -f "$PROMPT_FILE"; }
+# A directory, not a bare temp file: Step 4A looks for the offline bundle *beside*
+# the prompt, so the prompt needs a directory of its own rather than sharing
+# /tmp with everything else.
+PROMPT_DIR="$(mktemp -d "${TMPDIR:-/tmp}/arize-onboarding-XXXXXX")"
+PROMPT_FILE="$PROMPT_DIR/onboarding-prompt.md"
+cleanup() { rm -rf "$PROMPT_DIR"; }
 trap cleanup EXIT
 
-echo "Fetching the onboarding prompt…"
-download_failed() {
-  echo "Failed to download the onboarding prompt from $PROMPT_URL" >&2
-  echo "Check your connection and try again." >&2
-  exit 1
-}
 if command -v curl >/dev/null 2>&1; then
-  curl -fsSL "$PROMPT_URL" -o "$PROMPT_FILE" || download_failed
+  fetch() { curl -fsSL "$1" -o "$2"; }
 elif command -v wget >/dev/null 2>&1; then
-  wget -qO "$PROMPT_FILE" "$PROMPT_URL" || download_failed
+  fetch() { wget -qO "$2" "$1"; }
 else
   echo "Need curl or wget to download the prompt." >&2
   exit 1
 fi
 
+echo "Fetching the onboarding prompt…"
+if ! fetch "$PROMPT_URL" "$PROMPT_FILE"; then
+  echo "Failed to download the onboarding prompt from $PROMPT_URL" >&2
+  echo "Check your connection and try again." >&2
+  exit 1
+fi
+
+# Fetch the bundled tracing harness so Step 4A can install it without piping a
+# downloaded script into a shell — the thing auto-approval permission classifiers
+# refuse. `npx evals` ships these files in the package; this path has to pull them
+# from the CDN, which is why the build writes vendor/MANIFEST: a launcher cannot
+# glob a CDN and must not hardcode versioned wheel names.
+#
+# Entirely best-effort. Any failure leaves no arize-offline/ directory, and the
+# prompt then takes its documented network path. Staged under a temp name and
+# renamed, so the directory Step 4A keys off only ever appears complete.
+stage_offline_bundle() {
+  local staging="$PROMPT_DIR/.staging" line file
+  mkdir -p "$staging" || return 1
+  fetch "$VENDOR_URL/MANIFEST" "$staging/MANIFEST" 2>/dev/null || return 1
+
+  while IFS= read -r line; do
+    file="${line#*  }"
+    [ -n "$file" ] || continue
+    fetch "$VENDOR_URL/$file" "$staging/$file" 2>/dev/null || return 1
+  done < "$staging/MANIFEST"
+
+  # MANIFEST is standard `shasum -a 256` output, so the check is the standard
+  # tool. A truncated CDN response would otherwise surface as a confusing pip
+  # error much later. Skipped, not fatal, when neither tool exists.
+  if command -v shasum >/dev/null 2>&1; then
+    ( cd "$staging" && shasum -a 256 -c MANIFEST >/dev/null 2>&1 ) || return 1
+  elif command -v sha256sum >/dev/null 2>&1; then
+    ( cd "$staging" && sha256sum -c MANIFEST >/dev/null 2>&1 ) || return 1
+  fi
+
+  chmod +x "$staging/harness-install.sh" 2>/dev/null || true
+  mv "$staging" "$PROMPT_DIR/arize-offline"
+}
+
+echo "Fetching the tracing harness…"
+if ! stage_offline_bundle; then
+  rm -rf "$PROMPT_DIR/.staging"
+  echo "  (not available — Step 4A will download the installer instead)"
+fi
+
 SEED="Read the file $PROMPT_FILE and follow it to set up Arize AX tracing in this project, walking me through each step and asking me questions as needed."
 
 echo "Launching $(agent_label "$CHOSEN")…"