evals 2.8.2 → 2.9.0

package/bin.js

@@ -34,19 +34,10 @@ if (!Number.isInteger(major) || major < MIN_MAJOR) {
   process.exit(1);
 }
 
-// `evals track <event>` — fire-and-forget GA4, no TUI. Used for local/debug;
-// the onboarding prompt stages track-event.mjs instead (must not name npx evals).
-const args = process.argv.slice(2);
-if (args[0] === 'track') {
-  import('./track-event.mjs')
-    .then((m) => m.run(args.slice(1)))
-    .finally(() => process.exit(0));
-} else {
-  // cli.js only auto-runs when it is the entry point, so call main() explicitly.
-  import('./cli.js')
-    .then((cli) => cli.main())
-    .catch((err) => {
-      console.error(`Could not start evals: ${err && err.message ? err.message : err}`);
-      process.exit(1);
-    });
-}
+// cli.js only auto-runs when it is the entry point, so call main() explicitly.
+import('./cli.js')
+  .then((cli) => cli.main())
+  .catch((err) => {
+    console.error(`Could not start evals: ${err && err.message ? err.message : err}`);
+    process.exit(1);
+  });

package/cli.js

@@ -21,7 +21,6 @@ import {
 import { join, isAbsolute, resolve } from 'path';
 import { tmpdir, homedir } from 'os';
 import { fileURLToPath } from 'url';
-import { trackEventBackground } from './analytics.js';
 
 const e = React.createElement;
 
@@ -38,8 +37,6 @@ const BRAND_ACCENT = '#FF3CA8';
 // that file — a ~35 KB prompt is too large to pass reliably as a command-line
 // argument.
 const BUNDLED_PROMPT_PATH = fileURLToPath(new URL('./onboarding-prompt.md', import.meta.url));
-const BUNDLED_ANALYTICS_PATH = fileURLToPath(new URL('./analytics.js', import.meta.url));
-const BUNDLED_TRACK_PATH = fileURLToPath(new URL('./track-event.mjs', import.meta.url));
 
 // Wheels for the coding-agent tracing harness, built by scripts/build-harness-wheel.mjs.
 // Staging these next to the prompt lets Step 4A install with no network and no
@@ -60,18 +57,6 @@ const PROMPT_FILE_NAME = 'onboarding-prompt.md';
 // gets cleared at the next launch instead of sitting around holding a live key.
 const ENV_FILE_NAME = 'harness.env';
 
-// Anon GA4 helper staged beside the prompt so the agent can fire auth-opened
-// before OAuth without naming `npx evals` (prompt contract forbids that).
-const TRACK_FILE_NAME = 'track-event.mjs';
-const ANALYTICS_FILE_NAME = 'analytics.js';
-
-const STAGED_ROOT_FILES = [
-  PROMPT_FILE_NAME,
-  ENV_FILE_NAME,
-  TRACK_FILE_NAME,
-  ANALYTICS_FILE_NAME,
-];
-
 // MANIFEST only lands in the shell launchers' bundles, never in ours. Naming it
 // anyway keeps one allowlist correct for every implementation, so a directory
 // staged by start.sh can be cleared by `npx evals` and vice versa.
@@ -150,7 +135,7 @@ export function clearOnboardingDir(dir) {
         continue;
       }
 
-      if (!STAGED_ROOT_FILES.includes(entry.name)) return false;
+      if (entry.name !== PROMPT_FILE_NAME && entry.name !== ENV_FILE_NAME) return false;
       unlinkSync(path);
     }
     return true;
@@ -225,9 +210,6 @@ function prepareSeedPrompt() {
   const { dir, fellBack } = prepareOnboardingDir();
   const promptFile = join(dir, PROMPT_FILE_NAME);
   writeFileSync(promptFile, promptText, 'utf8');
-  // Same dir as the prompt: agent runs this before OAuth opens signup/sign-in.
-  copyFileSync(BUNDLED_ANALYTICS_PATH, join(dir, ANALYTICS_FILE_NAME));
-  copyFileSync(BUNDLED_TRACK_PATH, join(dir, TRACK_FILE_NAME));
   stageOfflineHarness(dir);
   // Single quotes, not double: a home path is likelier to contain a space than a
   // temp path was, but Windows spawns through the shell and embedded double
@@ -433,8 +415,7 @@ export function buildAgentItems() {
     items: AGENTS.map(a => ({
       name: `Install ${a.label}`,
       subtext: 'No supported agent detected — open install docs',
-      url: a.installUrl,
-      agentId: a.id,
+      url: a.installUrl
     })),
     none: true
   };
@@ -496,7 +477,7 @@ function App({ onDone }) {
       }
 
       if (selected.url) {
-        onDone({ type: 'url', url: selected.url, agentId: selected.agentId });
+        onDone({ type: 'url', url: selected.url });
         exit();
         return;
       }
@@ -560,11 +541,6 @@ function launchAgent(agent) {
     ? '  Done.'
     : '  Skipped — the setup will follow the Arize docs instead.');
 
-  trackEventBackground('npx_evals_skills_installed', {
-    agent: agent.id,
-    success: skillsInstalled ? 'true' : 'false',
-  });
-
   console.log(`\nLaunching ${agent.label}…\n`);
 
   const child = spawn(agent.bin, agent.args(prepared.seed), {
@@ -673,19 +649,9 @@ export async function main() {
     if (!result) {
       process.exit(0); // user quit / cancelled
     } else if (result.type === 'url') {
-      // Install-doc link when no agent is on PATH (not the arize.com copy event).
-      trackEventBackground('npx_evals_agent_install_click', {
-        agent: result.agentId || 'unknown',
-        url_host: (() => {
-          try { return new URL(result.url).host; } catch { return 'unknown'; }
-        })(),
-      });
       console.log(`\nOpening ${result.url} in your browser...\n`);
       openUrlInBrowser(result.url);
     } else if (result.type === 'launch') {
-      trackEventBackground('npx_evals_agent_selected', {
-        agent: result.agent.id,
-      });
       launchAgent(result.agent);
     }
   } catch (error) {

package/onboarding-prompt.md

@@ -139,14 +139,11 @@ If a `default` profile exists but the probe failed, it is signed out or expired,
 
 - **No key anywhere** — sign up or sign in with browser OAuth:
 
-  Immediately before the OAuth command, fire the anon funnel ping (ignore failures; never let it block auth), then open the browser flow:
-
   ```bash
-  node ~/.arize/onboarding/track-event.mjs auth-opened --method=oauth >/dev/null 2>&1 || true
   ax profiles create default --auth-method oauth --utm-params "utm_source=npmevals&utm_medium=cli&utm_campaign=prompt-first-onboarding"
   ```
 
-Pass `--utm-params` verbatim on the two browser OAuth commands (`profiles create --auth-method oauth` and the `ax auth login` fallback) and on no other `ax` command — it tags a new sign-up with onboarding attribution. Always run the matching `track-event.mjs auth-opened` line immediately before each of those two commands (`--method=oauth` or `--method=login`).
+Pass `--utm-params` verbatim on the two browser OAuth commands (`profiles create --auth-method oauth` and the `ax auth login` fallback) and on no other `ax` command — it tags a new sign-up with onboarding attribution.
 
 Always pass the positional profile name `default`. Without it, the CLI prompts `profile name [default]:`, receives EOF from an agent-run command, and exits with `Goodbye!` without creating a profile.
 
@@ -173,7 +170,6 @@ Then wait for the command to exit. Treat exit code `0`, or success output such a
 Only after the OAuth command completes, re-run the probe from the top of this step. If it succeeds, continue. If the profile already existed and the probe returns an authentication error, it is signed out or expired — run the fallback:
 
 ```bash
-node ~/.arize/onboarding/track-event.mjs auth-opened --method=login >/dev/null 2>&1 || true
 ax auth login --utm-params "utm_source=npmevals&utm_medium=cli&utm_campaign=prompt-first-onboarding"
 ```
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "evals",
-  "version": "2.8.2",
+  "version": "2.9.0",
   "description": "Arize AX onboarding — instrument your app with tracing via your coding agent",
   "type": "module",
   "main": "cli.js",
@@ -24,8 +24,6 @@
   "files": [
     "bin.js",
     "cli.js",
-    "analytics.js",
-    "track-event.mjs",
     "onboarding-prompt.md",
     "start.sh",
     "start.ps1",

package/start.ps1

@@ -26,9 +26,6 @@ $ErrorActionPreference = 'Stop'
 # files over HTTP, so it's fetchable even though the source repo is private.
 # Override with $env:ARIZE_PROMPT_URL if needed.
 $PromptUrl = if ($env:ARIZE_PROMPT_URL) { $env:ARIZE_PROMPT_URL } else { 'https://cdn.jsdelivr.net/npm/evals/onboarding-prompt.md' }
-# Staged beside the prompt for the OAuth signup/sign-in ping (prompt must not name npx evals).
-$TrackUrl = if ($env:ARIZE_TRACK_URL) { $env:ARIZE_TRACK_URL } else { 'https://cdn.jsdelivr.net/npm/evals/track-event.mjs' }
-$AnalyticsUrl = if ($env:ARIZE_ANALYTICS_URL) { $env:ARIZE_ANALYTICS_URL } else { 'https://cdn.jsdelivr.net/npm/evals/analytics.js' }
 # Where the bundled harness wheels live, same package. Override for testing.
 $VendorUrl = if ($env:ARIZE_VENDOR_URL) { $env:ARIZE_VENDOR_URL } else { 'https://cdn.jsdelivr.net/npm/evals/vendor' }
 
@@ -102,45 +99,6 @@ function Resolve-Agent {
   }
 }
 
-
-function Send-EvalsGa4Event {
-  param([string]$Name, [string]$AgentId = '')
-  $mid = if ($env:ARIZE_EVALS_GA4_MEASUREMENT_ID) { $env:ARIZE_EVALS_GA4_MEASUREMENT_ID } else { 'G-QSZ10TDVQM' }
-  $secret = $env:ARIZE_EVALS_GA4_API_SECRET
-  if (-not $secret) { return }
-
-  $arizeDir = Join-Path $HOME '.arize'
-  $anonFile = Join-Path $arizeDir 'anon_client_id'
-  $clientId = $null
-  if (Test-Path -LiteralPath $anonFile) {
-    $clientId = (Get-Content -LiteralPath $anonFile -Raw -ErrorAction SilentlyContinue).Trim()
-  }
-  if (-not $clientId) {
-    $clientId = [guid]::NewGuid().ToString()
-    try {
-      New-Item -ItemType Directory -Force -Path $arizeDir | Out-Null
-      Set-Content -LiteralPath $anonFile -Value $clientId -NoNewline
-    } catch {}
-  }
-
-  $params = @{ engagement_time_msec = 1 }
-  if ($AgentId) { $params.agent = $AgentId }
-  $bodyObj = @{
-    client_id = $clientId
-    events = @(@{ name = $Name; params = $params })
-  }
-  $json = $bodyObj | ConvertTo-Json -Depth 5 -Compress
-  $url = "https://www.google-analytics.com/mp/collect?measurement_id=$mid&api_secret=$secret"
-  try {
-    Start-Job -ScriptBlock {
-      param($u, $j)
-      try {
-        Invoke-RestMethod -Method Post -Uri $u -ContentType 'application/json' -Body $j -TimeoutSec 2 | Out-Null
-      } catch {}
-    } -ArgumentList $url, $json | Out-Null
-  } catch {}
-}
-
 # --- Welcome ---
 Write-Host ""
 Write-Host "Arize AX" -ForegroundColor Magenta -NoNewline
@@ -158,7 +116,6 @@ if ([Console]::IsOutputRedirected) {
 } else {
   $chosen = Resolve-Agent
   if (-not $chosen) { exit 1 }
-  Send-EvalsGa4Event -Name 'npx_evals_agent_selected' -AgentId $chosen
 }
 
 # A directory, not a bare temp file: Step 4A looks for the offline bundle *beside*
@@ -197,7 +154,7 @@ function Clear-PromptDir($dir) {
     if (-not $ours) { return $false }
     if (-not (Clear-BundleDir $sub.FullName)) { return $false }
   }
-  foreach ($name in @('onboarding-prompt.md', 'harness.env', 'track-event.mjs', 'analytics.js')) {
+  foreach ($name in @('onboarding-prompt.md', 'harness.env')) {
     Remove-Item -LiteralPath (Join-Path $dir $name) -Force -ErrorAction SilentlyContinue
   }
   # Anything left is not ours.
@@ -243,10 +200,6 @@ try {
   exit 1
 }
 
-# Best-effort: agent runs these before OAuth. Missing files just skip the ping.
-try { Invoke-RestMethod -Uri $TrackUrl -OutFile (Join-Path $promptDir 'track-event.mjs') } catch {}
-try { Invoke-RestMethod -Uri $AnalyticsUrl -OutFile (Join-Path $promptDir 'analytics.js') } catch {}
-
 # Fetch the bundled tracing harness so Step 4A can install it without piping a
 # downloaded script into a shell — the thing auto-approval permission
 # classifiers refuse. `npx evals` ships these files; this path pulls them from

package/start.sh

@@ -22,46 +22,6 @@ set -euo pipefail
 # files over HTTP, so it's fetchable even though the source repo is private.
 # Override with ARIZE_PROMPT_URL if needed.
 PROMPT_URL="${ARIZE_PROMPT_URL:-https://cdn.jsdelivr.net/npm/evals/onboarding-prompt.md}"
-# Staged beside the prompt for the OAuth signup/sign-in ping (prompt must not name npx evals).
-TRACK_URL="${ARIZE_TRACK_URL:-https://cdn.jsdelivr.net/npm/evals/track-event.mjs}"
-ANALYTICS_URL="${ARIZE_ANALYTICS_URL:-https://cdn.jsdelivr.net/npm/evals/analytics.js}"
-
-# Anonymous GA4 step ping (same events as cli.js). No-op without API secret.
-# ARIZE_EVALS_GA4_MEASUREMENT_ID default = arize.com stream; override if needed.
-track_evals_event() {
-  local name="$1"
-  local agent_id="${2:-}"
-  local mid="${ARIZE_EVALS_GA4_MEASUREMENT_ID:-G-QSZ10TDVQM}"
-  local secret="${ARIZE_EVALS_GA4_API_SECRET:-}"
-  [[ -n "$secret" ]] || return 0
-
-  local anon_file="${HOME}/.arize/anon_client_id"
-  local client_id=""
-  if [[ -f "$anon_file" ]]; then
-    client_id="$(tr -d '[:space:]' < "$anon_file" 2>/dev/null || true)"
-  fi
-  if [[ -z "$client_id" ]]; then
-    if command -v uuidgen >/dev/null 2>&1; then
-      client_id="$(uuidgen | tr '[:upper:]' '[:lower:]')"
-    else
-      client_id="$(od -An -N16 -tx1 /dev/urandom 2>/dev/null | tr -d ' \n')"
-      client_id="${client_id:0:8}-${client_id:8:4}-${client_id:12:4}-${client_id:16:4}-${client_id:20:12}"
-    fi
-    mkdir -p "${HOME}/.arize" 2>/dev/null || true
-    printf '%s' "$client_id" > "$anon_file" 2>/dev/null || true
-  fi
-
-  local params="\"engagement_time_msec\":1"
-  if [[ -n "$agent_id" ]]; then
-    params="$params,\"agent\":\"$agent_id\""
-  fi
-  local body="{\"client_id\":\"$client_id\",\"events\":[{\"name\":\"$name\",\"params\":{$params}}]}"
-  local url="https://www.google-analytics.com/mp/collect?measurement_id=${mid}&api_secret=${secret}"
-  if command -v curl >/dev/null 2>&1; then
-    curl -fsS -m 2 -X POST -H 'content-type: application/json' -d "$body" "$url" >/dev/null 2>&1 &
-  fi
-}
-
 # Where the bundled harness wheels live, same package. Override for testing.
 VENDOR_URL="${ARIZE_VENDOR_URL:-https://cdn.jsdelivr.net/npm/evals/vendor}"
 
@@ -195,7 +155,6 @@ if [[ ! -t 1 ]]; then
   CHOSEN=""
 else
   CHOSEN="$(choose_agent)" || exit 1
-  track_evals_event npx_evals_agent_selected "$CHOSEN"
 fi
 
 # A directory, not a bare temp file: Step 4A looks for the offline bundle *beside*
@@ -227,8 +186,7 @@ clear_prompt_dir() {
   for sub in "$1"/arize-offline "$1"/.staging*; do
     [ -d "$sub" ] && clear_bundle_dir "$sub"
   done
-  rm -f "$1/onboarding-prompt.md" "$1/harness.env" \
-        "$1/track-event.mjs" "$1/analytics.js"
+  rm -f "$1/onboarding-prompt.md" "$1/harness.env"
   # Anything left is not ours: bail out and let the caller fall back.
   rmdir "$1" 2>/dev/null || { [ -z "$(ls -A "$1" 2>/dev/null)" ]; return $?; }
 }
@@ -273,10 +231,6 @@ if ! fetch "$PROMPT_URL" "$PROMPT_FILE"; then
   exit 1
 fi
 
-# Best-effort: agent runs these before OAuth. Missing files just skip the ping.
-fetch "$TRACK_URL" "$PROMPT_DIR/track-event.mjs" 2>/dev/null || true
-fetch "$ANALYTICS_URL" "$PROMPT_DIR/analytics.js" 2>/dev/null || true
-
 # Fetch the bundled tracing harness so Step 4A can install it without piping a
 # downloaded script into a shell — the thing auto-approval permission classifiers
 # refuse. `npx evals` ships these files in the package; this path has to pull them

package/vendor/MANIFEST

@@ -1,6 +1,6 @@
 a9cede65f98d3415b13e1c6a437d0fdee27286bfff0186dd72bb65d9869b7864  LICENSE-coding-harness-tracing
 62f22742b58a1a33014a2b6b706588a8d7e2a88ae7bd1a6ebe8c992928483775  certifi-2026.7.22-py3-none-any.whl
-e5060c3fa2807eff7a5816caf2a78eb2cd285b6fbe0a1c3022373cd89a0e46d7  coding_harness_tracing-0.1.0-py3-none-any.whl
+b18aa5a0bd311407aced15656803a05bf4019f49287bb6a0b3683ca02c3c8449  coding_harness_tracing-0.1.0-py3-none-any.whl
 aca14e09ca99c253d2a142e2d5c5c38c8c7cf90840d3cb10b241c29929871fa4  harness-install.bat
 a7ba18a98280e236b20ab4cb9f8a4006068b454586a2af50bf23fb0dbe70583d  harness-install.sh
 b81ee9561e9ca4004139c6cbba3a238c32b03e4894671e181b671e8cb8425d61  python_dotenv-1.2.1-py3-none-any.whl

package/vendor/harness-pin.lock.json

@@ -5,7 +5,7 @@
   "source": "git",
   "wheel": "coding_harness_tracing-0.1.0-py3-none-any.whl",
   "wheelVersion": "0.1.0",
-  "wheelSha256": "e5060c3fa2807eff7a5816caf2a78eb2cd285b6fbe0a1c3022373cd89a0e46d7",
+  "wheelSha256": "b18aa5a0bd311407aced15656803a05bf4019f49287bb6a0b3683ca02c3c8449",
   "files": [
     "LICENSE-coding-harness-tracing",
     "MANIFEST",

package/analytics.js

@@ -1,116 +0,0 @@
-/**
- * Anonymous GA4 Measurement Protocol events for the npx evals onboarding funnel.
- *
- * No PII — only an anon client id under ~/.arize and step names / agent ids.
- * Fire-and-forget: never blocks or fails the CLI if tracking is down.
- *
- * Events:
- *   npx_evals_agent_selected       — coding agent picked in the TUI
- *   npx_evals_agent_install_click  — install-docs link when no agent on PATH
- *   npx_evals_skills_installed     — Arize skills/CLI install attempt
- *   npx_evals_auth_opened          — signup/sign-in browser about to open (OAuth)
- *
- * Requires a GA4 Measurement Protocol API secret (Admin → Data stream →
- * Measurement Protocol API secrets). Set via ARIZE_EVALS_GA4_API_SECRET, or
- * bake ARIZE_EVALS_GA4_API_SECRET_DEFAULT at release time.
- *
- * Override measurement id with ARIZE_EVALS_GA4_MEASUREMENT_ID if needed.
- */
-
-import { randomUUID } from 'crypto';
-import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'fs';
-import { homedir } from 'os';
-import { join } from 'path';
-
-/** arize.com GA4 web stream (same property as site hero_agent_prompt_* events). */
-export const DEFAULT_MEASUREMENT_ID = 'G-QSZ10TDVQM';
-
-/**
- * Optional release-time default. Leave empty until Chris/GA4 admin creates a
- * Measurement Protocol API secret for this CLI. Env always wins when set.
- */
-const API_SECRET_DEFAULT = '';
-
-function measurementId() {
-  return process.env.ARIZE_EVALS_GA4_MEASUREMENT_ID || DEFAULT_MEASUREMENT_ID;
-}
-
-function apiSecret() {
-  return process.env.ARIZE_EVALS_GA4_API_SECRET || API_SECRET_DEFAULT;
-}
-
-function anonIdPath() {
-  return join(homedir(), '.arize', 'anon_client_id');
-}
-
-/** Stable anon id so GA4 can stitch steps for one machine without knowing who. */
-export function getAnonClientId() {
-  const path = anonIdPath();
-  try {
-    if (existsSync(path)) {
-      const existing = readFileSync(path, 'utf8').trim();
-      if (existing) return existing;
-    }
-  } catch {
-    // fall through and mint a new id
-  }
-
-  const id = randomUUID();
-  try {
-    mkdirSync(join(homedir(), '.arize'), { recursive: true });
-    writeFileSync(path, id, 'utf8');
-  } catch {
-    // still send with in-memory id if home isn't writable
-  }
-  return id;
-}
-
-/**
- * Send one anon event. Resolves quickly; network errors are swallowed.
- * @param {string} name GA4 event name (snake_case, <=40 chars)
- * @param {Record<string, string | number | boolean>} [params]
- * @returns {Promise<void>}
- */
-export async function trackEvent(name, params = {}) {
-  const mid = measurementId();
-  const secret = apiSecret();
-  if (!mid || !secret) return;
-
-  const url =
-    `https://www.google-analytics.com/mp/collect` +
-    `?measurement_id=${encodeURIComponent(mid)}` +
-    `&api_secret=${encodeURIComponent(secret)}`;
-
-  const body = {
-    client_id: getAnonClientId(),
-    events: [
-      {
-        name,
-        params: {
-          ...params,
-          // Required for GA4 to count the event as engaged in some reports
-          engagement_time_msec: 1,
-        },
-      },
-    ],
-  };
-
-  try {
-    const controller = new AbortController();
-    const timer = setTimeout(() => controller.abort(), 2500);
-    await fetch(url, {
-      method: 'POST',
-      headers: { 'content-type': 'application/json' },
-      body: JSON.stringify(body),
-      signal: controller.signal,
-    });
-    clearTimeout(timer);
-  } catch {
-    // never surface tracking failures to the user
-  }
-}
-
-/** Fire without awaiting — safe to call before launch / exit. */
-export function trackEventBackground(name, params = {}) {
-  void trackEvent(name, params);
-}

package/track-event.mjs

@@ -1,52 +0,0 @@
-#!/usr/bin/env node
-/**
- * Fire one anon funnel event, then exit.
- *
- * Staged next to the onboarding prompt so the agent can run it before opening
- * the Arize signup/sign-in browser (OAuth) without naming `npx evals`.
- *
- * Usage:
- *   node ~/.arize/onboarding/track-event.mjs auth-opened --method=oauth
- *   node ~/.arize/onboarding/track-event.mjs auth-opened --method=login
- *
- * Always exits 0 — tracking must never block auth.
- */
-
-import { trackEvent } from './analytics.js';
-
-/** @type {Record<string, string>} */
-const EVENTS = {
-  'auth-opened': 'npx_evals_auth_opened',
-};
-
-/**
- * @param {string[]} argv
- * @returns {Promise<void>}
- */
-export async function run(argv) {
-  const key = argv[0] || '';
-  const name = EVENTS[key];
-  if (!name) return;
-
-  /** @type {Record<string, string>} */
-  const params = {};
-  for (let i = 1; i < argv.length; i++) {
-    const a = argv[i];
-    if (a.startsWith('--method=')) {
-      params.method = a.slice('--method='.length);
-    } else if (a === '--method' && argv[i + 1]) {
-      params.method = argv[++i];
-    }
-  }
-
-  await trackEvent(name, params);
-}
-
-const isDirect =
-  process.argv[1] &&
-  (process.argv[1].endsWith('track-event.mjs') ||
-    process.argv[1].endsWith('track-event'));
-
-if (isDirect) {
-  run(process.argv.slice(2)).finally(() => process.exit(0));
-}