evals 2.9.0 → 2.11.0

package/README.md

@@ -48,7 +48,7 @@ The agent runs with its **normal permission model** — `evals` never passes ski
 |----------|-----------|--------|
 | `ARIZE_AGENT=<id>` | `start.sh` / `start.ps1` | Skip the picker and use this agent (`claude`, `codex`, `cursor-agent`, `copilot`, `agy`). |
 | `ARIZE_SKIP_NPX=1` | `start.sh` / `start.ps1` | Force the shell path even when Node/npx is available. |
-| `ARIZE_PROMPT_URL=<url>` | `start.sh` / `start.ps1` | Fetch the onboarding prompt from a custom URL (supports `file://`). |
+| `ARIZE_PROMPT_MANIFEST_URL=<url>` | `start.sh` / `start.ps1` | Fetch the onboarding prompt manifest from a custom URL. |
 | `ARIZE_ONBOARDING_DIR=<dir>` | all three | Stage the prompt somewhere other than `~/.arize/onboarding`. |
 
 The shell launchers also accept `--agent <id>`.

package/analytics.js

@@ -0,0 +1,93 @@
+/**
+ * Anonymous npx evals TUI funnel events → app-server → Pendo Track.
+ *
+ * No PII — only a stable anon id under ~/.arize and allowlisted event names.
+ * Fire-and-forget: never blocks or fails the CLI if tracking is down.
+ *
+ * Events:
+ *   npx_evals_agent_selected       — coding agent picked in the TUI
+ *   npx_evals_agent_install_click  — install-docs link when no agent on PATH
+ *   npx_evals_skills_installed     — Arize skills/CLI install attempt
+ *   npx_evals_auth_opened          — signup/sign-in browser about to open (OAuth)
+ */
+
+import { randomUUID } from 'crypto';
+import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'fs';
+import { homedir } from 'os';
+import { join } from 'path';
+
+export const DEFAULT_ANALYTICS_URL =
+  'https://app.arize.com/api/analytics/npx-evals';
+
+/** Bound client timeout; server holds the Pendo key. */
+export const TRACK_TIMEOUT_MS = 2500;
+
+export const ALLOWED_EVENTS = new Set([
+  'npx_evals_agent_selected',
+  'npx_evals_agent_install_click',
+  'npx_evals_skills_installed',
+  'npx_evals_auth_opened',
+]);
+
+function analyticsUrl() {
+  return process.env.ARIZE_EVALS_ANALYTICS_URL || DEFAULT_ANALYTICS_URL;
+}
+
+function anonIdPath() {
+  return join(homedir(), '.arize', 'anon_client_id');
+}
+
+/** Stable anon id so Pendo can stitch steps for one machine without knowing who. */
+export function getAnonClientId() {
+  const path = anonIdPath();
+  try {
+    if (existsSync(path)) {
+      const existing = readFileSync(path, 'utf8').trim();
+      if (existing) return existing;
+    }
+  } catch {
+    // fall through and mint a new id
+  }
+
+  const id = randomUUID();
+  try {
+    mkdirSync(join(homedir(), '.arize'), { recursive: true });
+    writeFileSync(path, id, 'utf8');
+  } catch {
+    // still send with in-memory id if home isn't writable
+  }
+  return id;
+}
+
+/**
+ * Send one anon funnel event. Resolves quickly; network errors are swallowed.
+ * @param {string} name allowlisted event name
+ * @returns {Promise<void>}
+ */
+export async function trackEvent(name) {
+  if (!ALLOWED_EVENTS.has(name)) return;
+
+  const body = {
+    event: name,
+    anonymousId: getAnonClientId(),
+  };
+
+  try {
+    const controller = new AbortController();
+    const timer = setTimeout(() => controller.abort(), TRACK_TIMEOUT_MS);
+    await fetch(analyticsUrl(), {
+      method: 'POST',
+      headers: { 'content-type': 'application/json' },
+      body: JSON.stringify(body),
+      signal: controller.signal,
+    });
+    clearTimeout(timer);
+  } catch {
+    // never surface tracking failures to the user
+  }
+}
+
+/** Fire without awaiting — safe to call before launch / exit. */
+export function trackEventBackground(name) {
+  void trackEvent(name);
+}

package/bin.js

@@ -34,10 +34,19 @@ if (!Number.isInteger(major) || major < MIN_MAJOR) {
   process.exit(1);
 }
 
-// cli.js only auto-runs when it is the entry point, so call main() explicitly.
-import('./cli.js')
-  .then((cli) => cli.main())
-  .catch((err) => {
-    console.error(`Could not start evals: ${err && err.message ? err.message : err}`);
-    process.exit(1);
-  });
+// `evals track <event>` — fire-and-forget funnel ping, no TUI. Used for local/debug;
+// the onboarding prompt stages track-event.mjs instead (must not name npx evals).
+const args = process.argv.slice(2);
+if (args[0] === 'track') {
+  import('./track-event.mjs')
+    .then((m) => m.run(args.slice(1)))
+    .finally(() => process.exit(0));
+} else {
+  // cli.js only auto-runs when it is the entry point, so call main() explicitly.
+  import('./cli.js')
+    .then((cli) => cli.main())
+    .catch((err) => {
+      console.error(`Could not start evals: ${err && err.message ? err.message : err}`);
+      process.exit(1);
+    });
+}

package/cli.js

@@ -21,6 +21,7 @@ import {
 import { join, isAbsolute, resolve } from 'path';
 import { tmpdir, homedir } from 'os';
 import { fileURLToPath } from 'url';
+import { trackEventBackground } from './analytics.js';
 
 const e = React.createElement;
 
@@ -37,6 +38,8 @@ const BRAND_ACCENT = '#FF3CA8';
 // that file — a ~35 KB prompt is too large to pass reliably as a command-line
 // argument.
 const BUNDLED_PROMPT_PATH = fileURLToPath(new URL('./onboarding-prompt.md', import.meta.url));
+const BUNDLED_ANALYTICS_PATH = fileURLToPath(new URL('./analytics.js', import.meta.url));
+const BUNDLED_TRACK_PATH = fileURLToPath(new URL('./track-event.mjs', import.meta.url));
 
 // Wheels for the coding-agent tracing harness, built by scripts/build-harness-wheel.mjs.
 // Staging these next to the prompt lets Step 4A install with no network and no
@@ -57,6 +60,18 @@ const PROMPT_FILE_NAME = 'onboarding-prompt.md';
 // gets cleared at the next launch instead of sitting around holding a live key.
 const ENV_FILE_NAME = 'harness.env';
 
+// Anon funnel helper staged beside the prompt so the agent can fire auth-opened
+// before OAuth without naming `npx evals` (prompt contract forbids that).
+const TRACK_FILE_NAME = 'track-event.mjs';
+const ANALYTICS_FILE_NAME = 'analytics.js';
+
+const STAGED_ROOT_FILES = [
+  PROMPT_FILE_NAME,
+  ENV_FILE_NAME,
+  TRACK_FILE_NAME,
+  ANALYTICS_FILE_NAME,
+];
+
 // MANIFEST only lands in the shell launchers' bundles, never in ours. Naming it
 // anyway keeps one allowlist correct for every implementation, so a directory
 // staged by start.sh can be cleared by `npx evals` and vice versa.
@@ -135,7 +150,7 @@ export function clearOnboardingDir(dir) {
         continue;
       }
 
-      if (entry.name !== PROMPT_FILE_NAME && entry.name !== ENV_FILE_NAME) return false;
+      if (!STAGED_ROOT_FILES.includes(entry.name)) return false;
       unlinkSync(path);
     }
     return true;
@@ -210,6 +225,8 @@ function prepareSeedPrompt() {
   const { dir, fellBack } = prepareOnboardingDir();
   const promptFile = join(dir, PROMPT_FILE_NAME);
   writeFileSync(promptFile, promptText, 'utf8');
+  copyFileSync(BUNDLED_ANALYTICS_PATH, join(dir, ANALYTICS_FILE_NAME));
+  copyFileSync(BUNDLED_TRACK_PATH, join(dir, TRACK_FILE_NAME));
   stageOfflineHarness(dir);
   // Single quotes, not double: a home path is likelier to contain a space than a
   // temp path was, but Windows spawns through the shell and embedded double
@@ -217,7 +234,7 @@ function prepareSeedPrompt() {
   // No "in this project": the launcher may well be run from a home directory, and
   // Step 4 is what scopes the work — an app here, an app at another path, a starter
   // app, or the coding agent itself. Keep this wording in step with start.{sh,ps1}.
-  const seed = `Read the file '${promptFile}' in full with your file-reading tool, not a shell command, then follow it to set up Arize AX tracing, walking me through each step and asking me questions as needed.`;
+  const seed = `Read the file '${promptFile}' in full before doing anything else. Prefer your file-reading tool. If none is available, use read-only shell commands with bounded line ranges solely to print this exact file, repeating with successive ranges until EOF. Do not inspect or change anything else, and do not invoke a skill until a step tells you to. Then follow its steps in order from Step 0, walking me through each step and asking me questions as needed.`;
   return { seed, dir, fellBack };
 }
 
@@ -415,7 +432,8 @@ export function buildAgentItems() {
     items: AGENTS.map(a => ({
       name: `Install ${a.label}`,
       subtext: 'No supported agent detected — open install docs',
-      url: a.installUrl
+      url: a.installUrl,
+      agentId: a.id,
     })),
     none: true
   };
@@ -477,7 +495,7 @@ function App({ onDone }) {
       }
 
       if (selected.url) {
-        onDone({ type: 'url', url: selected.url });
+        onDone({ type: 'url', url: selected.url, agentId: selected.agentId });
         exit();
         return;
       }
@@ -541,6 +559,8 @@ function launchAgent(agent) {
     ? '  Done.'
     : '  Skipped — the setup will follow the Arize docs instead.');
 
+  trackEventBackground('npx_evals_skills_installed');
+
   console.log(`\nLaunching ${agent.label}…\n`);
 
   const child = spawn(agent.bin, agent.args(prepared.seed), {
@@ -649,9 +669,11 @@ export async function main() {
     if (!result) {
       process.exit(0); // user quit / cancelled
     } else if (result.type === 'url') {
+      trackEventBackground('npx_evals_agent_install_click');
       console.log(`\nOpening ${result.url} in your browser...\n`);
       openUrlInBrowser(result.url);
     } else if (result.type === 'launch') {
+      trackEventBackground('npx_evals_agent_selected');
       launchAgent(result.agent);
     }
   } catch (error) {

package/onboarding-prompt.md

@@ -4,7 +4,7 @@ Guide the user from zero to their first traces in Arize AX. Use the AX CLI, the
 
 **Read all of this file before you act.** If your tools return only part of it, keep reading to the last line. Rules you need are at the end.
 
-Work through the flow in order. Installing the AX CLI and Arize skills, authenticating, and listing spaces are fine before approval — that is your own tooling. Do not create AX resources (like API keys), edit application files, or install the app's tracing dependencies before the user approves the plan in Step 5.
+Work through the flow in order. Before approval, install the AX CLI and Arize skills, authenticate, list spaces, and create a space if the user asks in Step 3. Create no other AX resource (like an API key), edit no application file, and install no tracing dependency before the user approves the plan in Step 5.
 
 ## How to write to the user
 
@@ -94,7 +94,9 @@ python3 -m pip install --upgrade arize-ax-cli
 
 Try `uv` first, even if `pipx` is installed.
 
-**Check your skill list for `arize-instrumentation` and `arize-link` first.** They are usually installed already. If they are listed, do not run the install — invoke them by name when needed and go to Step 2.
+**Check your skill list for `arize-instrumentation` and `arize-link` first.** They are usually installed already. If they are listed, do not run the install — go to Step 2, and invoke them only where a step says to: `arize-instrumentation` at Step 6, `arize-link` at Step 8.
+
+A skill you loaded before you read this file does not own this flow — this file does. Drop that skill's plan and start at Step 0.
 
 Install them only if they are absent, or you cannot see your skill list, and npx is available:
 
@@ -139,11 +141,14 @@ If a `default` profile exists but the probe failed, it is signed out or expired,
 
 - **No key anywhere** — sign up or sign in with browser OAuth:
 
+  Immediately before the OAuth command, fire the anon funnel ping (ignore failures; never let it block auth), then open the browser flow:
+
   ```bash
+  node ~/.arize/onboarding/track-event.mjs auth-opened >/dev/null 2>&1 || true
   ax profiles create default --auth-method oauth --utm-params "utm_source=npmevals&utm_medium=cli&utm_campaign=prompt-first-onboarding"
   ```
 
-Pass `--utm-params` verbatim on the two browser OAuth commands (`profiles create --auth-method oauth` and the `ax auth login` fallback) and on no other `ax` command — it tags a new sign-up with onboarding attribution.
+Pass `--utm-params` verbatim on the two browser OAuth commands (`profiles create --auth-method oauth` and the `ax auth login` fallback) and on no other `ax` command — it tags a new sign-up with onboarding attribution. Always run `track-event.mjs auth-opened` immediately before each of those two commands.
 
 Always pass the positional profile name `default`. Without it, the CLI prompts `profile name [default]:`, receives EOF from an agent-run command, and exits with `Goodbye!` without creating a profile.
 
@@ -170,6 +175,7 @@ Then wait for the command to exit. Treat exit code `0`, or success output such a
 Only after the OAuth command completes, re-run the probe from the top of this step. If it succeeds, continue. If the profile already existed and the probe returns an authentication error, it is signed out or expired — run the fallback:
 
 ```bash
+node ~/.arize/onboarding/track-event.mjs auth-opened >/dev/null 2>&1 || true
 ax auth login --utm-params "utm_source=npmevals&utm_medium=cli&utm_campaign=prompt-first-onboarding"
 ```
 
@@ -181,13 +187,41 @@ Run no other remote `ax` command (creating keys, listing all spaces, inspecting
 
 ## Step 3: Select the AX space
 
-List the spaces the authenticated profile can access:
+List every space the authenticated profile can access. The default limit is low, so raise it, and page with `--cursor <cursor>` while the response reports `pagination.has_more` as true:
+
+```bash
+ax spaces list --limit 100 --output json
+```
+
+**No spaces returned** — the account is new. Create a space (see below). Do not ask the user to choose.
+
+**One or more spaces** — ask the user which space to use. Ask this even when the list holds one space. Offer every space by display name, in the order returned, and offer a new space as the last choice:
+
+```text
+Which Arize AX space do you want to use?
+
+1. <space name>
+2. <space name>
+3. Create a new space
+```
+
+If `ARIZE_SPACE_ID` is already set (environment or `.env` / `.env.local`) **and** it appears in that list, mark that space as the one the app uses now and make it the default choice — the list confirms the active profile can reach it, since the app's key and your CLI profile can point at different spaces. If it is set but absent from the list, leave it out of the question.
+
+### Create a new space
+
+Ask the user for a name, and offer `arize-tracing` as the default. A new space needs an organization ID:
+
+```bash
+ax organizations list --output json
+```
+
+One organization, use it; several, ask the user which one. Then create the space:
 
 ```bash
-ax spaces list --output json
+ax spaces create --name "<name>" --organization-id "<organization-id>" --output json
 ```
 
-If `ARIZE_SPACE_ID` is already set (environment or `.env` / `.env.local`), use it **only if it appears in that list** — that confirms the active profile can reach it, since the app's key and your CLI profile may point at different spaces. If it is set but absent from the list, select from the list instead. Otherwise: one space returned, use it; several, ask the user which; none, guide the user to create one in the Arize AX UI (or with an organization ID if available), then re-list.
+A space the user asked for is its own approval — it is the only AX resource you create before Step 5. Take the space **ID** from the command output.
 
 Capture the space's **ID**, not its display name, for `ARIZE_SPACE_ID`. `arize-otel` requires the ID; a name silently fails to route traces.
 
@@ -598,7 +632,7 @@ https://arize.com/docs/ax/integrations/platforms/<agent>/<agent>-tracing
 ## Critical rules
 
 - Keep your task list current if you have one (see "Track progress"): one task per milestone, exactly one in progress, complete only when done, never mentioned in chat.
-- Get the user's approval (Step 5) before creating AX resources, editing files, or installing dependencies.
+- Get the user's approval (Step 5) before creating AX resources, editing files, or installing dependencies — except a space the user asked for in Step 3.
 - Authenticate the CLI (Step 2) before any other `ax` call; env vars alone don't. Never interrupt a waiting OAuth command — no stdin close, Ctrl-C, `pkill`, or probe. A slow wait is not a hang.
 - Never print, log, or summarize secrets in chat — API keys, env-file contents/values, or span bodies (prompts, completions, tool args, user data) — and never read env files into chat. Only report traces a command actually returned.
 - One AX API key, ever: reuse a **validated** `ARIZE_API_KEY`, else a **single** `ax api-keys create --env-file <file>` into an already-git-ignored file.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "evals",
-  "version": "2.9.0",
+  "version": "2.11.0",
   "description": "Arize AX onboarding — instrument your app with tracing via your coding agent",
   "type": "module",
   "main": "cli.js",
@@ -10,8 +10,10 @@
   "scripts": {
     "postinstall": "node -e \"console.log('Arize evals installed — run: npx evals')\"",
     "start": "node cli.js",
-    "test": "node --test",
+    "test": "node --test && npm run verify:prompt-manifest",
     "build:wheel": "node scripts/build-harness-wheel.mjs",
+    "build:prompt-manifest": "node scripts/generate-prompt-manifest.mjs",
+    "verify:prompt-manifest": "node scripts/verify-prompt-manifest.mjs",
     "prepack": "npm run build:wheel"
   },
   "dependencies": {
@@ -24,7 +26,10 @@
   "files": [
     "bin.js",
     "cli.js",
+    "analytics.js",
+    "track-event.mjs",
     "onboarding-prompt.md",
+    "prompt-manifest.json",
     "start.sh",
     "start.ps1",
     "vendor/"

package/prompt-manifest.json

@@ -0,0 +1,9 @@
+{
+  "key_id": "2026-09-18",
+  "minimum_ax_cli_version": "0.35.0",
+  "prompt_sha256": "fd0582dd4fb125a2b948d901d76f1e80b775599d1e8b23a886db4a9fbbfdc056",
+  "prompt_url": "https://cdn.jsdelivr.net/npm/evals@2.11.0/onboarding-prompt.md",
+  "revision": "evals@2.11.0",
+  "schema_version": 1,
+  "signature": "H6yB6gs8+8y30/EYWnjQrJy6OV68AM7ktosqpJJqr/udmRwuSNzndcXmVGGHrhDN/nwcCB/Z0C6pZ8f2GJJ6Cw=="
+}

package/start.ps1

@@ -24,8 +24,11 @@ $ErrorActionPreference = 'Stop'
 
 # The prompt ships inside the public `evals` npm package; jsDelivr serves package
 # files over HTTP, so it's fetchable even though the source repo is private.
-# Override with $env:ARIZE_PROMPT_URL if needed.
-$PromptUrl = if ($env:ARIZE_PROMPT_URL) { $env:ARIZE_PROMPT_URL } else { 'https://cdn.jsdelivr.net/npm/evals/onboarding-prompt.md' }
+# Override the manifest URL with $env:ARIZE_PROMPT_MANIFEST_URL if needed.
+$PromptManifestUrl = if ($env:ARIZE_PROMPT_MANIFEST_URL) { $env:ARIZE_PROMPT_MANIFEST_URL } else { 'https://cdn.jsdelivr.net/npm/evals/prompt-manifest.json' }
+$TrackUrl = if ($env:ARIZE_TRACK_URL) { $env:ARIZE_TRACK_URL } else { 'https://cdn.jsdelivr.net/npm/evals/track-event.mjs' }
+$AnalyticsUrl = if ($env:ARIZE_ANALYTICS_URL) { $env:ARIZE_ANALYTICS_URL } else { 'https://cdn.jsdelivr.net/npm/evals/analytics.js' }
+$AnalyticsEndpoint = if ($env:ARIZE_EVALS_ANALYTICS_URL) { $env:ARIZE_EVALS_ANALYTICS_URL } else { 'https://app.arize.com/api/analytics/npx-evals' }
 # Where the bundled harness wheels live, same package. Override for testing.
 $VendorUrl = if ($env:ARIZE_VENDOR_URL) { $env:ARIZE_VENDOR_URL } else { 'https://cdn.jsdelivr.net/npm/evals/vendor' }
 
@@ -99,6 +102,32 @@ function Resolve-Agent {
   }
 }
 
+
+function Send-EvalsFunnelEvent {
+  param([string]$Name)
+  $anonFile = Join-Path $HOME '.arize/anon_client_id'
+  $clientId = $null
+  if (Test-Path -LiteralPath $anonFile) {
+    $clientId = (Get-Content -LiteralPath $anonFile -Raw -ErrorAction SilentlyContinue).Trim()
+  }
+  if (-not $clientId) {
+    $clientId = [guid]::NewGuid().ToString()
+    try {
+      New-Item -ItemType Directory -Force -Path (Join-Path $HOME '.arize') | Out-Null
+      Set-Content -LiteralPath $anonFile -Value $clientId -NoNewline
+    } catch {}
+  }
+  $body = @{ event = $Name; anonymousId = $clientId } | ConvertTo-Json -Compress
+  try {
+    Start-Job -ScriptBlock {
+      param($u, $j)
+      try {
+        Invoke-RestMethod -Method Post -Uri $u -ContentType 'application/json' -Body $j -TimeoutSec 2 | Out-Null
+      } catch {}
+    } -ArgumentList $AnalyticsEndpoint, $body | Out-Null
+  } catch {}
+}
+
 # --- Welcome ---
 Write-Host ""
 Write-Host "Arize AX" -ForegroundColor Magenta -NoNewline
@@ -116,6 +145,7 @@ if ([Console]::IsOutputRedirected) {
 } else {
   $chosen = Resolve-Agent
   if (-not $chosen) { exit 1 }
+  Send-EvalsFunnelEvent -Name 'npx_evals_agent_selected'
 }
 
 # A directory, not a bare temp file: Step 4A looks for the offline bundle *beside*
@@ -154,7 +184,7 @@ function Clear-PromptDir($dir) {
     if (-not $ours) { return $false }
     if (-not (Clear-BundleDir $sub.FullName)) { return $false }
   }
-  foreach ($name in @('onboarding-prompt.md', 'harness.env')) {
+  foreach ($name in @('onboarding-prompt.md', 'harness.env', '.prompt-manifest', 'track-event.mjs', 'analytics.js')) {
     Remove-Item -LiteralPath (Join-Path $dir $name) -Force -ErrorAction SilentlyContinue
   }
   # Anything left is not ours.
@@ -192,13 +222,52 @@ if (-not $promptDir) {
 $promptFile = Join-Path $promptDir 'onboarding-prompt.md'
 
 Write-Host "Fetching the onboarding prompt..."
+# This bootstrap verifies the prompt digest but cannot independently verify the
+# manifest signature on every supported PowerShell runtime. Its trust root is TLS to jsDelivr.
 try {
-  Invoke-RestMethod -Uri $PromptUrl -OutFile $promptFile
+  $manifestRaw = (Invoke-WebRequest -Uri $PromptManifestUrl -UseBasicParsing).Content
 } catch {
-  Write-Host "Failed to download the onboarding prompt from $PromptUrl" -ForegroundColor Red
+  Write-Host "Failed to download the onboarding prompt manifest." -ForegroundColor Red
   Write-Host "Check your connection and try again."
   exit 1
 }
+try {
+  if ([regex]::Matches($manifestRaw, '"prompt_url"\s*:').Count -ne 1 -or
+      [regex]::Matches($manifestRaw, '"prompt_sha256"\s*:').Count -ne 1) {
+    throw 'duplicate manifest field'
+  }
+  $promptManifest = $manifestRaw | ConvertFrom-Json
+  $allowLocalPromptTest = $env:ARIZE_ALLOW_LOCAL_PROMPT_TEST -eq '1' -and $env:ARIZE_PROMPT_MANIFEST_URL
+  $validPromptUrl = if ($allowLocalPromptTest) {
+    $promptManifest.prompt_url -match '^http://127\.0\.0\.1:[0-9]+/onboarding-prompt\.md$'
+  } else {
+    $promptManifest.prompt_url -match '^https://cdn\.jsdelivr\.net/npm/evals@[^/]+/onboarding-prompt\.md$'
+  }
+  if ($promptManifest.schema_version -ne 1 -or
+      -not $validPromptUrl -or
+      $promptManifest.prompt_sha256 -notmatch '^[0-9a-fA-F]{64}$') {
+    throw 'invalid onboarding prompt manifest'
+  }
+} catch {
+  Write-Host "The onboarding prompt manifest is invalid." -ForegroundColor Red
+  exit 1
+}
+try {
+  Invoke-RestMethod -Uri $promptManifest.prompt_url -OutFile $promptFile
+} catch {
+  Write-Host "Failed to download the onboarding prompt." -ForegroundColor Red
+  Write-Host "Check your connection and try again."
+  exit 1
+}
+$actualPromptSha256 = (Get-FileHash -Path $promptFile -Algorithm SHA256).Hash.ToLower()
+if ($actualPromptSha256 -ne $promptManifest.prompt_sha256.ToLower()) {
+  Write-Host "The onboarding prompt digest did not match its manifest." -ForegroundColor Red
+  exit 1
+}
+
+# Best-effort: agent runs these before OAuth. Missing files just skip the ping.
+try { Invoke-RestMethod -Uri $TrackUrl -OutFile (Join-Path $promptDir 'track-event.mjs') } catch {}
+try { Invoke-RestMethod -Uri $AnalyticsUrl -OutFile (Join-Path $promptDir 'analytics.js') } catch {}
 
 # Fetch the bundled tracing harness so Step 4A can install it without piping a
 # downloaded script into a shell — the thing auto-approval permission
@@ -241,7 +310,7 @@ try {
 # where embedded double quotes would not.
 # No "in this project": this may well be run from a home directory, and Step 4 is
 # what scopes the work. Keep this wording in step with cli.js and start.sh.
-$seed = "Read the file '$promptFile' in full with your file-reading tool, not a shell command, then follow it to set up Arize AX tracing, walking me through each step and asking me questions as needed."
+$seed = "Read the file '$promptFile' in full before doing anything else. Prefer your file-reading tool. If none is available, use read-only shell commands with bounded line ranges solely to print this exact file, repeating with successive ranges until EOF. Do not inspect or change anything else, and do not invoke a skill until a step tells you to. Then follow its steps in order from Step 0, walking me through each step and asking me questions as needed."
 
 if (-not $chosen) {
   # Addressed to the person, two lines, recommendation first - see the note in cli.js.

package/start.sh

@@ -20,8 +20,35 @@ set -euo pipefail
 
 # The prompt ships inside the public `evals` npm package; jsDelivr serves package
 # files over HTTP, so it's fetchable even though the source repo is private.
-# Override with ARIZE_PROMPT_URL if needed.
-PROMPT_URL="${ARIZE_PROMPT_URL:-https://cdn.jsdelivr.net/npm/evals/onboarding-prompt.md}"
+# Override the manifest URL with ARIZE_PROMPT_MANIFEST_URL if needed.
+PROMPT_MANIFEST_URL="${ARIZE_PROMPT_MANIFEST_URL:-https://cdn.jsdelivr.net/npm/evals/prompt-manifest.json}"
+TRACK_URL="${ARIZE_TRACK_URL:-https://cdn.jsdelivr.net/npm/evals/track-event.mjs}"
+ANALYTICS_URL="${ARIZE_ANALYTICS_URL:-https://cdn.jsdelivr.net/npm/evals/analytics.js}"
+ANALYTICS_ENDPOINT="${ARIZE_EVALS_ANALYTICS_URL:-https://app.arize.com/api/analytics/npx-evals}"
+
+# Fire-and-forget funnel ping (same events as cli.js). Never blocks the launcher.
+track_evals_event() {
+  local name="$1"
+  local anon_file="${HOME:-}/.arize/anon_client_id"
+  local client_id=""
+  if [[ -f "$anon_file" ]]; then
+    client_id="$(tr -d '[:space:]' < "$anon_file" 2>/dev/null || true)"
+  fi
+  if [[ -z "$client_id" ]]; then
+    if command -v uuidgen >/dev/null 2>&1; then
+      client_id="$(uuidgen | tr '[:upper:]' '[:lower:]')"
+    else
+      client_id="$(od -An -N16 -tx1 /dev/urandom 2>/dev/null | tr -d ' \n')"
+      client_id="${client_id:0:8}-${client_id:8:4}-${client_id:12:4}-${client_id:16:4}-${client_id:20:12}"
+    fi
+    [[ -n "${HOME:-}" ]] && mkdir -p "${HOME}/.arize" 2>/dev/null || true
+    [[ -n "${HOME:-}" ]] && printf '%s' "$client_id" > "$anon_file" 2>/dev/null || true
+  fi
+  local body="{\"event\":\"$name\",\"anonymousId\":\"$client_id\"}"
+  if command -v curl >/dev/null 2>&1; then
+    curl -fsS -m 2 -X POST -H 'content-type: application/json' -d "$body" "$ANALYTICS_ENDPOINT" >/dev/null 2>&1 &
+  fi
+}
 # Where the bundled harness wheels live, same package. Override for testing.
 VENDOR_URL="${ARIZE_VENDOR_URL:-https://cdn.jsdelivr.net/npm/evals/vendor}"
 
@@ -155,6 +182,7 @@ if [[ ! -t 1 ]]; then
   CHOSEN=""
 else
   CHOSEN="$(choose_agent)" || exit 1
+  track_evals_event npx_evals_agent_selected
 fi
 
 # A directory, not a bare temp file: Step 4A looks for the offline bundle *beside*
@@ -186,7 +214,8 @@ clear_prompt_dir() {
   for sub in "$1"/arize-offline "$1"/.staging*; do
     [ -d "$sub" ] && clear_bundle_dir "$sub"
   done
-  rm -f "$1/onboarding-prompt.md" "$1/harness.env"
+  rm -f "$1/onboarding-prompt.md" "$1/harness.env" "$1/.prompt-manifest" \
+        "$1/track-event.mjs" "$1/analytics.js"
   # Anything left is not ours: bail out and let the caller fall back.
   rmdir "$1" 2>/dev/null || { [ -z "$(ls -A "$1" 2>/dev/null)" ]; return $?; }
 }
@@ -224,12 +253,65 @@ else
   exit 1
 fi
 
+MANIFEST_FILE="$PROMPT_DIR/.prompt-manifest"
+# This bootstrap verifies the prompt digest but cannot independently verify the
+# manifest signature on every supported shell. Its trust root is TLS to jsDelivr.
 echo "Fetching the onboarding prompt…"
+if ! fetch "$PROMPT_MANIFEST_URL" "$MANIFEST_FILE"; then
+  echo "Failed to download the onboarding prompt manifest." >&2
+  echo "Check your connection and try again." >&2
+  exit 1
+fi
+
+manifest_value() {
+  values="$(sed -n "s/^[[:space:]]*\"$1\"[[:space:]]*:[[:space:]]*\"\([^\"]*\)\"[[:space:]]*,\{0,1\}[[:space:]]*$/\1/p" "$MANIFEST_FILE")"
+  [ "$(printf '%s\n' "$values" | sed '/^$/d' | wc -l | tr -d ' ')" -eq 1 ] || return 1
+  printf '%s' "$values"
+}
+
+PROMPT_URL="$(manifest_value prompt_url)" || { echo "The onboarding prompt manifest is invalid." >&2; exit 1; }
+PROMPT_SHA256="$(manifest_value prompt_sha256)" || { echo "The onboarding prompt manifest is invalid." >&2; exit 1; }
+rm -f "$MANIFEST_FILE"
+if [ "${ARIZE_ALLOW_LOCAL_PROMPT_TEST:-}" = "1" ] \
+  && [ -n "${ARIZE_PROMPT_MANIFEST_URL:-}" ]; then
+  case "$PROMPT_URL" in
+    http://127.0.0.1:*/onboarding-prompt.md) ;;
+    *) echo "The local prompt test URL must use 127.0.0.1 and onboarding-prompt.md." >&2; exit 1 ;;
+  esac
+else
+  case "$PROMPT_URL" in
+    https://cdn.jsdelivr.net/npm/evals@*/onboarding-prompt.md) ;;
+    *) echo "The onboarding prompt manifest has an invalid prompt URL." >&2; exit 1 ;;
+  esac
+fi
+case "$PROMPT_SHA256" in
+  *[!0-9a-fA-F]*) echo "The onboarding prompt manifest has an invalid digest." >&2; exit 1 ;;
+esac
+if [ "${#PROMPT_SHA256}" -ne 64 ]; then
+  echo "The onboarding prompt manifest has an invalid digest." >&2
+  exit 1
+fi
 if ! fetch "$PROMPT_URL" "$PROMPT_FILE"; then
-  echo "Failed to download the onboarding prompt from $PROMPT_URL" >&2
+  echo "Failed to download the onboarding prompt." >&2
   echo "Check your connection and try again." >&2
   exit 1
 fi
+if command -v shasum >/dev/null 2>&1; then
+  ACTUAL_SHA256="$(shasum -a 256 "$PROMPT_FILE" | awk '{print $1}')"
+elif command -v sha256sum >/dev/null 2>&1; then
+  ACTUAL_SHA256="$(sha256sum "$PROMPT_FILE" | awk '{print $1}')"
+else
+  echo "Need shasum or sha256sum to verify the onboarding prompt." >&2
+  exit 1
+fi
+if [ "$ACTUAL_SHA256" != "$PROMPT_SHA256" ]; then
+  echo "The onboarding prompt digest did not match its manifest." >&2
+  exit 1
+fi
+
+# Best-effort: agent runs these before OAuth. Missing files just skip the ping.
+fetch "$TRACK_URL" "$PROMPT_DIR/track-event.mjs" 2>/dev/null || true
+fetch "$ANALYTICS_URL" "$PROMPT_DIR/analytics.js" 2>/dev/null || true
 
 # Fetch the bundled tracing harness so Step 4A can install it without piping a
 # downloaded script into a shell — the thing auto-approval permission classifiers
@@ -272,7 +354,7 @@ fi
 
 # No "in this project": this may well be run from a home directory, and Step 4 is
 # what scopes the work. Keep this wording in step with cli.js and start.ps1.
-SEED="Read the file '$PROMPT_FILE' in full with your file-reading tool, not a shell command, then follow it to set up Arize AX tracing, walking me through each step and asking me questions as needed."
+SEED="Read the file '$PROMPT_FILE' in full before doing anything else. Prefer your file-reading tool. If none is available, use read-only shell commands with bounded line ranges solely to print this exact file, repeating with successive ranges until EOF. Do not inspect or change anything else, and do not invoke a skill until a step tells you to. Then follow its steps in order from Step 0, walking me through each step and asking me questions as needed."
 
 if [[ -z "$CHOSEN" ]]; then
   # Addressed to the person, two lines, recommendation first — see the note in cli.js.

package/track-event.mjs

@@ -0,0 +1,40 @@
+#!/usr/bin/env node
+/**
+ * Fire one anon funnel event, then exit.
+ *
+ * Staged next to the onboarding prompt so the agent can run it before opening
+ * the Arize signup/sign-in browser (OAuth) without naming `npx evals`.
+ *
+ * Usage:
+ *   node ~/.arize/onboarding/track-event.mjs auth-opened
+ *
+ * Always exits 0 — tracking must never block auth.
+ */
+
+import { trackEvent } from './analytics.js';
+
+/** @type {Record<string, string>} */
+const EVENTS = {
+  'auth-opened': 'npx_evals_auth_opened',
+};
+
+/**
+ * @param {string[]} argv
+ * @returns {Promise<void>}
+ */
+export async function run(argv) {
+  const key = argv[0] || '';
+  const name = EVENTS[key];
+  if (!name) return;
+
+  await trackEvent(name);
+}
+
+const isDirect =
+  process.argv[1] &&
+  (process.argv[1].endsWith('track-event.mjs') ||
+    process.argv[1].endsWith('track-event'));
+
+if (isDirect) {
+  run(process.argv.slice(2)).finally(() => process.exit(0));
+}

package/vendor/MANIFEST

@@ -1,6 +1,6 @@
 a9cede65f98d3415b13e1c6a437d0fdee27286bfff0186dd72bb65d9869b7864  LICENSE-coding-harness-tracing
 62f22742b58a1a33014a2b6b706588a8d7e2a88ae7bd1a6ebe8c992928483775  certifi-2026.7.22-py3-none-any.whl
-b18aa5a0bd311407aced15656803a05bf4019f49287bb6a0b3683ca02c3c8449  coding_harness_tracing-0.1.0-py3-none-any.whl
+6fa310dbfb392719881f8c9459d9d2a61c958e39c4d0d4e9c4ac29c7cb7403ce  coding_harness_tracing-0.1.0-py3-none-any.whl
 aca14e09ca99c253d2a142e2d5c5c38c8c7cf90840d3cb10b241c29929871fa4  harness-install.bat
 a7ba18a98280e236b20ab4cb9f8a4006068b454586a2af50bf23fb0dbe70583d  harness-install.sh
 b81ee9561e9ca4004139c6cbba3a238c32b03e4894671e181b671e8cb8425d61  python_dotenv-1.2.1-py3-none-any.whl

package/vendor/harness-pin.lock.json

@@ -1,11 +1,11 @@
 {
   "repo": "https://github.com/Arize-ai/coding-harness-tracing",
   "ref": "main",
-  "resolvedSha": "a51baf0a3f45b01b1d23677176c90eb851fb3969",
+  "resolvedSha": "62609f080d6a6b4786114e45dafd90a2945f2233",
   "source": "git",
   "wheel": "coding_harness_tracing-0.1.0-py3-none-any.whl",
   "wheelVersion": "0.1.0",
-  "wheelSha256": "b18aa5a0bd311407aced15656803a05bf4019f49287bb6a0b3683ca02c3c8449",
+  "wheelSha256": "6fa310dbfb392719881f8c9459d9d2a61c958e39c4d0d4e9c4ac29c7cb7403ce",
   "files": [
     "LICENSE-coding-harness-tracing",
     "MANIFEST",