evals 2.10.0 → 2.11.0

package/README.md

@@ -48,7 +48,7 @@ The agent runs with its **normal permission model** — `evals` never passes ski
 |----------|-----------|--------|
 | `ARIZE_AGENT=<id>` | `start.sh` / `start.ps1` | Skip the picker and use this agent (`claude`, `codex`, `cursor-agent`, `copilot`, `agy`). |
 | `ARIZE_SKIP_NPX=1` | `start.sh` / `start.ps1` | Force the shell path even when Node/npx is available. |
-| `ARIZE_PROMPT_URL=<url>` | `start.sh` / `start.ps1` | Fetch the onboarding prompt from a custom URL (supports `file://`). |
+| `ARIZE_PROMPT_MANIFEST_URL=<url>` | `start.sh` / `start.ps1` | Fetch the onboarding prompt manifest from a custom URL. |
 | `ARIZE_ONBOARDING_DIR=<dir>` | all three | Stage the prompt somewhere other than `~/.arize/onboarding`. |
 
 The shell launchers also accept `--agent <id>`.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "evals",
-  "version": "2.10.0",
+  "version": "2.11.0",
   "description": "Arize AX onboarding — instrument your app with tracing via your coding agent",
   "type": "module",
   "main": "cli.js",
@@ -10,8 +10,10 @@
   "scripts": {
     "postinstall": "node -e \"console.log('Arize evals installed — run: npx evals')\"",
     "start": "node cli.js",
-    "test": "node --test",
+    "test": "node --test && npm run verify:prompt-manifest",
     "build:wheel": "node scripts/build-harness-wheel.mjs",
+    "build:prompt-manifest": "node scripts/generate-prompt-manifest.mjs",
+    "verify:prompt-manifest": "node scripts/verify-prompt-manifest.mjs",
     "prepack": "npm run build:wheel"
   },
   "dependencies": {
@@ -27,6 +29,7 @@
     "analytics.js",
     "track-event.mjs",
     "onboarding-prompt.md",
+    "prompt-manifest.json",
     "start.sh",
     "start.ps1",
     "vendor/"

package/prompt-manifest.json

@@ -0,0 +1,9 @@
+{
+  "key_id": "2026-09-18",
+  "minimum_ax_cli_version": "0.35.0",
+  "prompt_sha256": "fd0582dd4fb125a2b948d901d76f1e80b775599d1e8b23a886db4a9fbbfdc056",
+  "prompt_url": "https://cdn.jsdelivr.net/npm/evals@2.11.0/onboarding-prompt.md",
+  "revision": "evals@2.11.0",
+  "schema_version": 1,
+  "signature": "H6yB6gs8+8y30/EYWnjQrJy6OV68AM7ktosqpJJqr/udmRwuSNzndcXmVGGHrhDN/nwcCB/Z0C6pZ8f2GJJ6Cw=="
+}

package/start.ps1

@@ -24,8 +24,8 @@ $ErrorActionPreference = 'Stop'
 
 # The prompt ships inside the public `evals` npm package; jsDelivr serves package
 # files over HTTP, so it's fetchable even though the source repo is private.
-# Override with $env:ARIZE_PROMPT_URL if needed.
-$PromptUrl = if ($env:ARIZE_PROMPT_URL) { $env:ARIZE_PROMPT_URL } else { 'https://cdn.jsdelivr.net/npm/evals/onboarding-prompt.md' }
+# Override the manifest URL with $env:ARIZE_PROMPT_MANIFEST_URL if needed.
+$PromptManifestUrl = if ($env:ARIZE_PROMPT_MANIFEST_URL) { $env:ARIZE_PROMPT_MANIFEST_URL } else { 'https://cdn.jsdelivr.net/npm/evals/prompt-manifest.json' }
 $TrackUrl = if ($env:ARIZE_TRACK_URL) { $env:ARIZE_TRACK_URL } else { 'https://cdn.jsdelivr.net/npm/evals/track-event.mjs' }
 $AnalyticsUrl = if ($env:ARIZE_ANALYTICS_URL) { $env:ARIZE_ANALYTICS_URL } else { 'https://cdn.jsdelivr.net/npm/evals/analytics.js' }
 $AnalyticsEndpoint = if ($env:ARIZE_EVALS_ANALYTICS_URL) { $env:ARIZE_EVALS_ANALYTICS_URL } else { 'https://app.arize.com/api/analytics/npx-evals' }
@@ -184,7 +184,7 @@ function Clear-PromptDir($dir) {
     if (-not $ours) { return $false }
     if (-not (Clear-BundleDir $sub.FullName)) { return $false }
   }
-  foreach ($name in @('onboarding-prompt.md', 'harness.env', 'track-event.mjs', 'analytics.js')) {
+  foreach ($name in @('onboarding-prompt.md', 'harness.env', '.prompt-manifest', 'track-event.mjs', 'analytics.js')) {
     Remove-Item -LiteralPath (Join-Path $dir $name) -Force -ErrorAction SilentlyContinue
   }
   # Anything left is not ours.
@@ -222,13 +222,48 @@ if (-not $promptDir) {
 $promptFile = Join-Path $promptDir 'onboarding-prompt.md'
 
 Write-Host "Fetching the onboarding prompt..."
+# This bootstrap verifies the prompt digest but cannot independently verify the
+# manifest signature on every supported PowerShell runtime. Its trust root is TLS to jsDelivr.
 try {
-  Invoke-RestMethod -Uri $PromptUrl -OutFile $promptFile
+  $manifestRaw = (Invoke-WebRequest -Uri $PromptManifestUrl -UseBasicParsing).Content
 } catch {
-  Write-Host "Failed to download the onboarding prompt from $PromptUrl" -ForegroundColor Red
+  Write-Host "Failed to download the onboarding prompt manifest." -ForegroundColor Red
   Write-Host "Check your connection and try again."
   exit 1
 }
+try {
+  if ([regex]::Matches($manifestRaw, '"prompt_url"\s*:').Count -ne 1 -or
+      [regex]::Matches($manifestRaw, '"prompt_sha256"\s*:').Count -ne 1) {
+    throw 'duplicate manifest field'
+  }
+  $promptManifest = $manifestRaw | ConvertFrom-Json
+  $allowLocalPromptTest = $env:ARIZE_ALLOW_LOCAL_PROMPT_TEST -eq '1' -and $env:ARIZE_PROMPT_MANIFEST_URL
+  $validPromptUrl = if ($allowLocalPromptTest) {
+    $promptManifest.prompt_url -match '^http://127\.0\.0\.1:[0-9]+/onboarding-prompt\.md$'
+  } else {
+    $promptManifest.prompt_url -match '^https://cdn\.jsdelivr\.net/npm/evals@[^/]+/onboarding-prompt\.md$'
+  }
+  if ($promptManifest.schema_version -ne 1 -or
+      -not $validPromptUrl -or
+      $promptManifest.prompt_sha256 -notmatch '^[0-9a-fA-F]{64}$') {
+    throw 'invalid onboarding prompt manifest'
+  }
+} catch {
+  Write-Host "The onboarding prompt manifest is invalid." -ForegroundColor Red
+  exit 1
+}
+try {
+  Invoke-RestMethod -Uri $promptManifest.prompt_url -OutFile $promptFile
+} catch {
+  Write-Host "Failed to download the onboarding prompt." -ForegroundColor Red
+  Write-Host "Check your connection and try again."
+  exit 1
+}
+$actualPromptSha256 = (Get-FileHash -Path $promptFile -Algorithm SHA256).Hash.ToLower()
+if ($actualPromptSha256 -ne $promptManifest.prompt_sha256.ToLower()) {
+  Write-Host "The onboarding prompt digest did not match its manifest." -ForegroundColor Red
+  exit 1
+}
 
 # Best-effort: agent runs these before OAuth. Missing files just skip the ping.
 try { Invoke-RestMethod -Uri $TrackUrl -OutFile (Join-Path $promptDir 'track-event.mjs') } catch {}

package/start.sh

@@ -20,8 +20,8 @@ set -euo pipefail
 
 # The prompt ships inside the public `evals` npm package; jsDelivr serves package
 # files over HTTP, so it's fetchable even though the source repo is private.
-# Override with ARIZE_PROMPT_URL if needed.
-PROMPT_URL="${ARIZE_PROMPT_URL:-https://cdn.jsdelivr.net/npm/evals/onboarding-prompt.md}"
+# Override the manifest URL with ARIZE_PROMPT_MANIFEST_URL if needed.
+PROMPT_MANIFEST_URL="${ARIZE_PROMPT_MANIFEST_URL:-https://cdn.jsdelivr.net/npm/evals/prompt-manifest.json}"
 TRACK_URL="${ARIZE_TRACK_URL:-https://cdn.jsdelivr.net/npm/evals/track-event.mjs}"
 ANALYTICS_URL="${ARIZE_ANALYTICS_URL:-https://cdn.jsdelivr.net/npm/evals/analytics.js}"
 ANALYTICS_ENDPOINT="${ARIZE_EVALS_ANALYTICS_URL:-https://app.arize.com/api/analytics/npx-evals}"
@@ -214,7 +214,7 @@ clear_prompt_dir() {
   for sub in "$1"/arize-offline "$1"/.staging*; do
     [ -d "$sub" ] && clear_bundle_dir "$sub"
   done
-  rm -f "$1/onboarding-prompt.md" "$1/harness.env" \
+  rm -f "$1/onboarding-prompt.md" "$1/harness.env" "$1/.prompt-manifest" \
         "$1/track-event.mjs" "$1/analytics.js"
   # Anything left is not ours: bail out and let the caller fall back.
   rmdir "$1" 2>/dev/null || { [ -z "$(ls -A "$1" 2>/dev/null)" ]; return $?; }
@@ -253,12 +253,61 @@ else
   exit 1
 fi
 
+MANIFEST_FILE="$PROMPT_DIR/.prompt-manifest"
+# This bootstrap verifies the prompt digest but cannot independently verify the
+# manifest signature on every supported shell. Its trust root is TLS to jsDelivr.
 echo "Fetching the onboarding prompt…"
+if ! fetch "$PROMPT_MANIFEST_URL" "$MANIFEST_FILE"; then
+  echo "Failed to download the onboarding prompt manifest." >&2
+  echo "Check your connection and try again." >&2
+  exit 1
+fi
+
+manifest_value() {
+  values="$(sed -n "s/^[[:space:]]*\"$1\"[[:space:]]*:[[:space:]]*\"\([^\"]*\)\"[[:space:]]*,\{0,1\}[[:space:]]*$/\1/p" "$MANIFEST_FILE")"
+  [ "$(printf '%s\n' "$values" | sed '/^$/d' | wc -l | tr -d ' ')" -eq 1 ] || return 1
+  printf '%s' "$values"
+}
+
+PROMPT_URL="$(manifest_value prompt_url)" || { echo "The onboarding prompt manifest is invalid." >&2; exit 1; }
+PROMPT_SHA256="$(manifest_value prompt_sha256)" || { echo "The onboarding prompt manifest is invalid." >&2; exit 1; }
+rm -f "$MANIFEST_FILE"
+if [ "${ARIZE_ALLOW_LOCAL_PROMPT_TEST:-}" = "1" ] \
+  && [ -n "${ARIZE_PROMPT_MANIFEST_URL:-}" ]; then
+  case "$PROMPT_URL" in
+    http://127.0.0.1:*/onboarding-prompt.md) ;;
+    *) echo "The local prompt test URL must use 127.0.0.1 and onboarding-prompt.md." >&2; exit 1 ;;
+  esac
+else
+  case "$PROMPT_URL" in
+    https://cdn.jsdelivr.net/npm/evals@*/onboarding-prompt.md) ;;
+    *) echo "The onboarding prompt manifest has an invalid prompt URL." >&2; exit 1 ;;
+  esac
+fi
+case "$PROMPT_SHA256" in
+  *[!0-9a-fA-F]*) echo "The onboarding prompt manifest has an invalid digest." >&2; exit 1 ;;
+esac
+if [ "${#PROMPT_SHA256}" -ne 64 ]; then
+  echo "The onboarding prompt manifest has an invalid digest." >&2
+  exit 1
+fi
 if ! fetch "$PROMPT_URL" "$PROMPT_FILE"; then
-  echo "Failed to download the onboarding prompt from $PROMPT_URL" >&2
+  echo "Failed to download the onboarding prompt." >&2
   echo "Check your connection and try again." >&2
   exit 1
 fi
+if command -v shasum >/dev/null 2>&1; then
+  ACTUAL_SHA256="$(shasum -a 256 "$PROMPT_FILE" | awk '{print $1}')"
+elif command -v sha256sum >/dev/null 2>&1; then
+  ACTUAL_SHA256="$(sha256sum "$PROMPT_FILE" | awk '{print $1}')"
+else
+  echo "Need shasum or sha256sum to verify the onboarding prompt." >&2
+  exit 1
+fi
+if [ "$ACTUAL_SHA256" != "$PROMPT_SHA256" ]; then
+  echo "The onboarding prompt digest did not match its manifest." >&2
+  exit 1
+fi
 
 # Best-effort: agent runs these before OAuth. Missing files just skip the ping.
 fetch "$TRACK_URL" "$PROMPT_DIR/track-event.mjs" 2>/dev/null || true

package/vendor/MANIFEST

@@ -1,6 +1,6 @@
 a9cede65f98d3415b13e1c6a437d0fdee27286bfff0186dd72bb65d9869b7864  LICENSE-coding-harness-tracing
 62f22742b58a1a33014a2b6b706588a8d7e2a88ae7bd1a6ebe8c992928483775  certifi-2026.7.22-py3-none-any.whl
-9f44b6e34618b87ab64f6aa48fd300bbfc6122fd4aca195122f68973747cad61  coding_harness_tracing-0.1.0-py3-none-any.whl
+6fa310dbfb392719881f8c9459d9d2a61c958e39c4d0d4e9c4ac29c7cb7403ce  coding_harness_tracing-0.1.0-py3-none-any.whl
 aca14e09ca99c253d2a142e2d5c5c38c8c7cf90840d3cb10b241c29929871fa4  harness-install.bat
 a7ba18a98280e236b20ab4cb9f8a4006068b454586a2af50bf23fb0dbe70583d  harness-install.sh
 b81ee9561e9ca4004139c6cbba3a238c32b03e4894671e181b671e8cb8425d61  python_dotenv-1.2.1-py3-none-any.whl

package/vendor/harness-pin.lock.json

@@ -1,11 +1,11 @@
 {
   "repo": "https://github.com/Arize-ai/coding-harness-tracing",
   "ref": "main",
-  "resolvedSha": "771ad555aab24ea2fd54a41b22936b6521156b26",
+  "resolvedSha": "62609f080d6a6b4786114e45dafd90a2945f2233",
   "source": "git",
   "wheel": "coding_harness_tracing-0.1.0-py3-none-any.whl",
   "wheelVersion": "0.1.0",
-  "wheelSha256": "9f44b6e34618b87ab64f6aa48fd300bbfc6122fd4aca195122f68973747cad61",
+  "wheelSha256": "6fa310dbfb392719881f8c9459d9d2a61c958e39c4d0d4e9c4ac29c7cb7403ce",
   "files": [
     "LICENSE-coding-harness-tracing",
     "MANIFEST",