evals 2.8.2 → 2.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/analytics.js +25 -48
- package/bin.js +1 -1
- package/cli.js +5 -17
- package/onboarding-prompt.md +39 -9
- package/package.json +1 -1
- package/start.ps1 +9 -22
- package/start.sh +9 -22
- package/track-event.mjs +2 -14
- package/vendor/MANIFEST +1 -1
- package/vendor/coding_harness_tracing-0.1.0-py3-none-any.whl +0 -0
- package/vendor/harness-pin.lock.json +2 -2
package/analytics.js
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* Anonymous
|
|
2
|
+
* Anonymous npx evals TUI funnel events → app-server → Pendo Track.
|
|
3
3
|
*
|
|
4
|
-
* No PII — only
|
|
4
|
+
* No PII — only a stable anon id under ~/.arize and allowlisted event names.
|
|
5
5
|
* Fire-and-forget: never blocks or fails the CLI if tracking is down.
|
|
6
6
|
*
|
|
7
7
|
* Events:
|
|
@@ -9,12 +9,6 @@
|
|
|
9
9
|
* npx_evals_agent_install_click — install-docs link when no agent on PATH
|
|
10
10
|
* npx_evals_skills_installed — Arize skills/CLI install attempt
|
|
11
11
|
* npx_evals_auth_opened — signup/sign-in browser about to open (OAuth)
|
|
12
|
-
*
|
|
13
|
-
* Requires a GA4 Measurement Protocol API secret (Admin → Data stream →
|
|
14
|
-
* Measurement Protocol API secrets). Set via ARIZE_EVALS_GA4_API_SECRET, or
|
|
15
|
-
* bake ARIZE_EVALS_GA4_API_SECRET_DEFAULT at release time.
|
|
16
|
-
*
|
|
17
|
-
* Override measurement id with ARIZE_EVALS_GA4_MEASUREMENT_ID if needed.
|
|
18
12
|
*/
|
|
19
13
|
|
|
20
14
|
import { randomUUID } from 'crypto';
|
|
@@ -22,28 +16,28 @@ import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'fs';
|
|
|
22
16
|
import { homedir } from 'os';
|
|
23
17
|
import { join } from 'path';
|
|
24
18
|
|
|
25
|
-
|
|
26
|
-
|
|
19
|
+
export const DEFAULT_ANALYTICS_URL =
|
|
20
|
+
'https://app.arize.com/api/analytics/npx-evals';
|
|
27
21
|
|
|
28
|
-
/**
|
|
29
|
-
|
|
30
|
-
* Measurement Protocol API secret for this CLI. Env always wins when set.
|
|
31
|
-
*/
|
|
32
|
-
const API_SECRET_DEFAULT = '';
|
|
22
|
+
/** Bound client timeout; server holds the Pendo key. */
|
|
23
|
+
export const TRACK_TIMEOUT_MS = 2500;
|
|
33
24
|
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
25
|
+
export const ALLOWED_EVENTS = new Set([
|
|
26
|
+
'npx_evals_agent_selected',
|
|
27
|
+
'npx_evals_agent_install_click',
|
|
28
|
+
'npx_evals_skills_installed',
|
|
29
|
+
'npx_evals_auth_opened',
|
|
30
|
+
]);
|
|
37
31
|
|
|
38
|
-
function
|
|
39
|
-
return process.env.
|
|
32
|
+
function analyticsUrl() {
|
|
33
|
+
return process.env.ARIZE_EVALS_ANALYTICS_URL || DEFAULT_ANALYTICS_URL;
|
|
40
34
|
}
|
|
41
35
|
|
|
42
36
|
function anonIdPath() {
|
|
43
37
|
return join(homedir(), '.arize', 'anon_client_id');
|
|
44
38
|
}
|
|
45
39
|
|
|
46
|
-
/** Stable anon id so
|
|
40
|
+
/** Stable anon id so Pendo can stitch steps for one machine without knowing who. */
|
|
47
41
|
export function getAnonClientId() {
|
|
48
42
|
const path = anonIdPath();
|
|
49
43
|
try {
|
|
@@ -66,39 +60,22 @@ export function getAnonClientId() {
|
|
|
66
60
|
}
|
|
67
61
|
|
|
68
62
|
/**
|
|
69
|
-
* Send one anon event. Resolves quickly; network errors are swallowed.
|
|
70
|
-
* @param {string} name
|
|
71
|
-
* @param {Record<string, string | number | boolean>} [params]
|
|
63
|
+
* Send one anon funnel event. Resolves quickly; network errors are swallowed.
|
|
64
|
+
* @param {string} name allowlisted event name
|
|
72
65
|
* @returns {Promise<void>}
|
|
73
66
|
*/
|
|
74
|
-
export async function trackEvent(name
|
|
75
|
-
|
|
76
|
-
const secret = apiSecret();
|
|
77
|
-
if (!mid || !secret) return;
|
|
78
|
-
|
|
79
|
-
const url =
|
|
80
|
-
`https://www.google-analytics.com/mp/collect` +
|
|
81
|
-
`?measurement_id=${encodeURIComponent(mid)}` +
|
|
82
|
-
`&api_secret=${encodeURIComponent(secret)}`;
|
|
67
|
+
export async function trackEvent(name) {
|
|
68
|
+
if (!ALLOWED_EVENTS.has(name)) return;
|
|
83
69
|
|
|
84
70
|
const body = {
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
{
|
|
88
|
-
name,
|
|
89
|
-
params: {
|
|
90
|
-
...params,
|
|
91
|
-
// Required for GA4 to count the event as engaged in some reports
|
|
92
|
-
engagement_time_msec: 1,
|
|
93
|
-
},
|
|
94
|
-
},
|
|
95
|
-
],
|
|
71
|
+
event: name,
|
|
72
|
+
anonymousId: getAnonClientId(),
|
|
96
73
|
};
|
|
97
74
|
|
|
98
75
|
try {
|
|
99
76
|
const controller = new AbortController();
|
|
100
|
-
const timer = setTimeout(() => controller.abort(),
|
|
101
|
-
await fetch(
|
|
77
|
+
const timer = setTimeout(() => controller.abort(), TRACK_TIMEOUT_MS);
|
|
78
|
+
await fetch(analyticsUrl(), {
|
|
102
79
|
method: 'POST',
|
|
103
80
|
headers: { 'content-type': 'application/json' },
|
|
104
81
|
body: JSON.stringify(body),
|
|
@@ -111,6 +88,6 @@ export async function trackEvent(name, params = {}) {
|
|
|
111
88
|
}
|
|
112
89
|
|
|
113
90
|
/** Fire without awaiting — safe to call before launch / exit. */
|
|
114
|
-
export function trackEventBackground(name
|
|
115
|
-
void trackEvent(name
|
|
91
|
+
export function trackEventBackground(name) {
|
|
92
|
+
void trackEvent(name);
|
|
116
93
|
}
|
package/bin.js
CHANGED
|
@@ -34,7 +34,7 @@ if (!Number.isInteger(major) || major < MIN_MAJOR) {
|
|
|
34
34
|
process.exit(1);
|
|
35
35
|
}
|
|
36
36
|
|
|
37
|
-
// `evals track <event>` — fire-and-forget
|
|
37
|
+
// `evals track <event>` — fire-and-forget funnel ping, no TUI. Used for local/debug;
|
|
38
38
|
// the onboarding prompt stages track-event.mjs instead (must not name npx evals).
|
|
39
39
|
const args = process.argv.slice(2);
|
|
40
40
|
if (args[0] === 'track') {
|
package/cli.js
CHANGED
|
@@ -60,7 +60,7 @@ const PROMPT_FILE_NAME = 'onboarding-prompt.md';
|
|
|
60
60
|
// gets cleared at the next launch instead of sitting around holding a live key.
|
|
61
61
|
const ENV_FILE_NAME = 'harness.env';
|
|
62
62
|
|
|
63
|
-
// Anon
|
|
63
|
+
// Anon funnel helper staged beside the prompt so the agent can fire auth-opened
|
|
64
64
|
// before OAuth without naming `npx evals` (prompt contract forbids that).
|
|
65
65
|
const TRACK_FILE_NAME = 'track-event.mjs';
|
|
66
66
|
const ANALYTICS_FILE_NAME = 'analytics.js';
|
|
@@ -225,7 +225,6 @@ function prepareSeedPrompt() {
|
|
|
225
225
|
const { dir, fellBack } = prepareOnboardingDir();
|
|
226
226
|
const promptFile = join(dir, PROMPT_FILE_NAME);
|
|
227
227
|
writeFileSync(promptFile, promptText, 'utf8');
|
|
228
|
-
// Same dir as the prompt: agent runs this before OAuth opens signup/sign-in.
|
|
229
228
|
copyFileSync(BUNDLED_ANALYTICS_PATH, join(dir, ANALYTICS_FILE_NAME));
|
|
230
229
|
copyFileSync(BUNDLED_TRACK_PATH, join(dir, TRACK_FILE_NAME));
|
|
231
230
|
stageOfflineHarness(dir);
|
|
@@ -235,7 +234,7 @@ function prepareSeedPrompt() {
|
|
|
235
234
|
// No "in this project": the launcher may well be run from a home directory, and
|
|
236
235
|
// Step 4 is what scopes the work — an app here, an app at another path, a starter
|
|
237
236
|
// app, or the coding agent itself. Keep this wording in step with start.{sh,ps1}.
|
|
238
|
-
const seed = `Read the file '${promptFile}' in full
|
|
237
|
+
const seed = `Read the file '${promptFile}' in full before doing anything else. Prefer your file-reading tool. If none is available, use read-only shell commands with bounded line ranges solely to print this exact file, repeating with successive ranges until EOF. Do not inspect or change anything else, and do not invoke a skill until a step tells you to. Then follow its steps in order from Step 0, walking me through each step and asking me questions as needed.`;
|
|
239
238
|
return { seed, dir, fellBack };
|
|
240
239
|
}
|
|
241
240
|
|
|
@@ -560,10 +559,7 @@ function launchAgent(agent) {
|
|
|
560
559
|
? ' Done.'
|
|
561
560
|
: ' Skipped — the setup will follow the Arize docs instead.');
|
|
562
561
|
|
|
563
|
-
trackEventBackground('npx_evals_skills_installed'
|
|
564
|
-
agent: agent.id,
|
|
565
|
-
success: skillsInstalled ? 'true' : 'false',
|
|
566
|
-
});
|
|
562
|
+
trackEventBackground('npx_evals_skills_installed');
|
|
567
563
|
|
|
568
564
|
console.log(`\nLaunching ${agent.label}…\n`);
|
|
569
565
|
|
|
@@ -673,19 +669,11 @@ export async function main() {
|
|
|
673
669
|
if (!result) {
|
|
674
670
|
process.exit(0); // user quit / cancelled
|
|
675
671
|
} else if (result.type === 'url') {
|
|
676
|
-
|
|
677
|
-
trackEventBackground('npx_evals_agent_install_click', {
|
|
678
|
-
agent: result.agentId || 'unknown',
|
|
679
|
-
url_host: (() => {
|
|
680
|
-
try { return new URL(result.url).host; } catch { return 'unknown'; }
|
|
681
|
-
})(),
|
|
682
|
-
});
|
|
672
|
+
trackEventBackground('npx_evals_agent_install_click');
|
|
683
673
|
console.log(`\nOpening ${result.url} in your browser...\n`);
|
|
684
674
|
openUrlInBrowser(result.url);
|
|
685
675
|
} else if (result.type === 'launch') {
|
|
686
|
-
trackEventBackground('npx_evals_agent_selected'
|
|
687
|
-
agent: result.agent.id,
|
|
688
|
-
});
|
|
676
|
+
trackEventBackground('npx_evals_agent_selected');
|
|
689
677
|
launchAgent(result.agent);
|
|
690
678
|
}
|
|
691
679
|
} catch (error) {
|
package/onboarding-prompt.md
CHANGED
|
@@ -4,7 +4,7 @@ Guide the user from zero to their first traces in Arize AX. Use the AX CLI, the
|
|
|
4
4
|
|
|
5
5
|
**Read all of this file before you act.** If your tools return only part of it, keep reading to the last line. Rules you need are at the end.
|
|
6
6
|
|
|
7
|
-
Work through the flow in order.
|
|
7
|
+
Work through the flow in order. Before approval, install the AX CLI and Arize skills, authenticate, list spaces, and create a space if the user asks in Step 3. Create no other AX resource (like an API key), edit no application file, and install no tracing dependency before the user approves the plan in Step 5.
|
|
8
8
|
|
|
9
9
|
## How to write to the user
|
|
10
10
|
|
|
@@ -94,7 +94,9 @@ python3 -m pip install --upgrade arize-ax-cli
|
|
|
94
94
|
|
|
95
95
|
Try `uv` first, even if `pipx` is installed.
|
|
96
96
|
|
|
97
|
-
**Check your skill list for `arize-instrumentation` and `arize-link` first.** They are usually installed already. If they are listed, do not run the install — invoke them
|
|
97
|
+
**Check your skill list for `arize-instrumentation` and `arize-link` first.** They are usually installed already. If they are listed, do not run the install — go to Step 2, and invoke them only where a step says to: `arize-instrumentation` at Step 6, `arize-link` at Step 8.
|
|
98
|
+
|
|
99
|
+
A skill you loaded before you read this file does not own this flow — this file does. Drop that skill's plan and start at Step 0.
|
|
98
100
|
|
|
99
101
|
Install them only if they are absent, or you cannot see your skill list, and npx is available:
|
|
100
102
|
|
|
@@ -142,11 +144,11 @@ If a `default` profile exists but the probe failed, it is signed out or expired,
|
|
|
142
144
|
Immediately before the OAuth command, fire the anon funnel ping (ignore failures; never let it block auth), then open the browser flow:
|
|
143
145
|
|
|
144
146
|
```bash
|
|
145
|
-
node ~/.arize/onboarding/track-event.mjs auth-opened
|
|
147
|
+
node ~/.arize/onboarding/track-event.mjs auth-opened >/dev/null 2>&1 || true
|
|
146
148
|
ax profiles create default --auth-method oauth --utm-params "utm_source=npmevals&utm_medium=cli&utm_campaign=prompt-first-onboarding"
|
|
147
149
|
```
|
|
148
150
|
|
|
149
|
-
Pass `--utm-params` verbatim on the two browser OAuth commands (`profiles create --auth-method oauth` and the `ax auth login` fallback) and on no other `ax` command — it tags a new sign-up with onboarding attribution. Always run
|
|
151
|
+
Pass `--utm-params` verbatim on the two browser OAuth commands (`profiles create --auth-method oauth` and the `ax auth login` fallback) and on no other `ax` command — it tags a new sign-up with onboarding attribution. Always run `track-event.mjs auth-opened` immediately before each of those two commands.
|
|
150
152
|
|
|
151
153
|
Always pass the positional profile name `default`. Without it, the CLI prompts `profile name [default]:`, receives EOF from an agent-run command, and exits with `Goodbye!` without creating a profile.
|
|
152
154
|
|
|
@@ -173,7 +175,7 @@ Then wait for the command to exit. Treat exit code `0`, or success output such a
|
|
|
173
175
|
Only after the OAuth command completes, re-run the probe from the top of this step. If it succeeds, continue. If the profile already existed and the probe returns an authentication error, it is signed out or expired — run the fallback:
|
|
174
176
|
|
|
175
177
|
```bash
|
|
176
|
-
node ~/.arize/onboarding/track-event.mjs auth-opened
|
|
178
|
+
node ~/.arize/onboarding/track-event.mjs auth-opened >/dev/null 2>&1 || true
|
|
177
179
|
ax auth login --utm-params "utm_source=npmevals&utm_medium=cli&utm_campaign=prompt-first-onboarding"
|
|
178
180
|
```
|
|
179
181
|
|
|
@@ -185,13 +187,41 @@ Run no other remote `ax` command (creating keys, listing all spaces, inspecting
|
|
|
185
187
|
|
|
186
188
|
## Step 3: Select the AX space
|
|
187
189
|
|
|
188
|
-
List
|
|
190
|
+
List every space the authenticated profile can access. The default limit is low, so raise it, and page with `--cursor <cursor>` while the response reports `pagination.has_more` as true:
|
|
191
|
+
|
|
192
|
+
```bash
|
|
193
|
+
ax spaces list --limit 100 --output json
|
|
194
|
+
```
|
|
195
|
+
|
|
196
|
+
**No spaces returned** — the account is new. Create a space (see below). Do not ask the user to choose.
|
|
197
|
+
|
|
198
|
+
**One or more spaces** — ask the user which space to use. Ask this even when the list holds one space. Offer every space by display name, in the order returned, and offer a new space as the last choice:
|
|
199
|
+
|
|
200
|
+
```text
|
|
201
|
+
Which Arize AX space do you want to use?
|
|
202
|
+
|
|
203
|
+
1. <space name>
|
|
204
|
+
2. <space name>
|
|
205
|
+
3. Create a new space
|
|
206
|
+
```
|
|
207
|
+
|
|
208
|
+
If `ARIZE_SPACE_ID` is already set (environment or `.env` / `.env.local`) **and** it appears in that list, mark that space as the one the app uses now and make it the default choice — the list confirms the active profile can reach it, since the app's key and your CLI profile can point at different spaces. If it is set but absent from the list, leave it out of the question.
|
|
209
|
+
|
|
210
|
+
### Create a new space
|
|
211
|
+
|
|
212
|
+
Ask the user for a name, and offer `arize-tracing` as the default. A new space needs an organization ID:
|
|
213
|
+
|
|
214
|
+
```bash
|
|
215
|
+
ax organizations list --output json
|
|
216
|
+
```
|
|
217
|
+
|
|
218
|
+
One organization, use it; several, ask the user which one. Then create the space:
|
|
189
219
|
|
|
190
220
|
```bash
|
|
191
|
-
ax spaces
|
|
221
|
+
ax spaces create --name "<name>" --organization-id "<organization-id>" --output json
|
|
192
222
|
```
|
|
193
223
|
|
|
194
|
-
|
|
224
|
+
A space the user asked for is its own approval — it is the only AX resource you create before Step 5. Take the space **ID** from the command output.
|
|
195
225
|
|
|
196
226
|
Capture the space's **ID**, not its display name, for `ARIZE_SPACE_ID`. `arize-otel` requires the ID; a name silently fails to route traces.
|
|
197
227
|
|
|
@@ -602,7 +632,7 @@ https://arize.com/docs/ax/integrations/platforms/<agent>/<agent>-tracing
|
|
|
602
632
|
## Critical rules
|
|
603
633
|
|
|
604
634
|
- Keep your task list current if you have one (see "Track progress"): one task per milestone, exactly one in progress, complete only when done, never mentioned in chat.
|
|
605
|
-
- Get the user's approval (Step 5) before creating AX resources, editing files, or installing dependencies.
|
|
635
|
+
- Get the user's approval (Step 5) before creating AX resources, editing files, or installing dependencies — except a space the user asked for in Step 3.
|
|
606
636
|
- Authenticate the CLI (Step 2) before any other `ax` call; env vars alone don't. Never interrupt a waiting OAuth command — no stdin close, Ctrl-C, `pkill`, or probe. A slow wait is not a hang.
|
|
607
637
|
- Never print, log, or summarize secrets in chat — API keys, env-file contents/values, or span bodies (prompts, completions, tool args, user data) — and never read env files into chat. Only report traces a command actually returned.
|
|
608
638
|
- One AX API key, ever: reuse a **validated** `ARIZE_API_KEY`, else a **single** `ax api-keys create --env-file <file>` into an already-git-ignored file.
|
package/package.json
CHANGED
package/start.ps1
CHANGED
|
@@ -26,9 +26,9 @@ $ErrorActionPreference = 'Stop'
|
|
|
26
26
|
# files over HTTP, so it's fetchable even though the source repo is private.
|
|
27
27
|
# Override with $env:ARIZE_PROMPT_URL if needed.
|
|
28
28
|
$PromptUrl = if ($env:ARIZE_PROMPT_URL) { $env:ARIZE_PROMPT_URL } else { 'https://cdn.jsdelivr.net/npm/evals/onboarding-prompt.md' }
|
|
29
|
-
# Staged beside the prompt for the OAuth signup/sign-in ping (prompt must not name npx evals).
|
|
30
29
|
$TrackUrl = if ($env:ARIZE_TRACK_URL) { $env:ARIZE_TRACK_URL } else { 'https://cdn.jsdelivr.net/npm/evals/track-event.mjs' }
|
|
31
30
|
$AnalyticsUrl = if ($env:ARIZE_ANALYTICS_URL) { $env:ARIZE_ANALYTICS_URL } else { 'https://cdn.jsdelivr.net/npm/evals/analytics.js' }
|
|
31
|
+
$AnalyticsEndpoint = if ($env:ARIZE_EVALS_ANALYTICS_URL) { $env:ARIZE_EVALS_ANALYTICS_URL } else { 'https://app.arize.com/api/analytics/npx-evals' }
|
|
32
32
|
# Where the bundled harness wheels live, same package. Override for testing.
|
|
33
33
|
$VendorUrl = if ($env:ARIZE_VENDOR_URL) { $env:ARIZE_VENDOR_URL } else { 'https://cdn.jsdelivr.net/npm/evals/vendor' }
|
|
34
34
|
|
|
@@ -103,14 +103,9 @@ function Resolve-Agent {
|
|
|
103
103
|
}
|
|
104
104
|
|
|
105
105
|
|
|
106
|
-
function Send-
|
|
107
|
-
param([string]$Name
|
|
108
|
-
$
|
|
109
|
-
$secret = $env:ARIZE_EVALS_GA4_API_SECRET
|
|
110
|
-
if (-not $secret) { return }
|
|
111
|
-
|
|
112
|
-
$arizeDir = Join-Path $HOME '.arize'
|
|
113
|
-
$anonFile = Join-Path $arizeDir 'anon_client_id'
|
|
106
|
+
function Send-EvalsFunnelEvent {
|
|
107
|
+
param([string]$Name)
|
|
108
|
+
$anonFile = Join-Path $HOME '.arize/anon_client_id'
|
|
114
109
|
$clientId = $null
|
|
115
110
|
if (Test-Path -LiteralPath $anonFile) {
|
|
116
111
|
$clientId = (Get-Content -LiteralPath $anonFile -Raw -ErrorAction SilentlyContinue).Trim()
|
|
@@ -118,26 +113,18 @@ function Send-EvalsGa4Event {
|
|
|
118
113
|
if (-not $clientId) {
|
|
119
114
|
$clientId = [guid]::NewGuid().ToString()
|
|
120
115
|
try {
|
|
121
|
-
New-Item -ItemType Directory -Force -Path $
|
|
116
|
+
New-Item -ItemType Directory -Force -Path (Join-Path $HOME '.arize') | Out-Null
|
|
122
117
|
Set-Content -LiteralPath $anonFile -Value $clientId -NoNewline
|
|
123
118
|
} catch {}
|
|
124
119
|
}
|
|
125
|
-
|
|
126
|
-
$params = @{ engagement_time_msec = 1 }
|
|
127
|
-
if ($AgentId) { $params.agent = $AgentId }
|
|
128
|
-
$bodyObj = @{
|
|
129
|
-
client_id = $clientId
|
|
130
|
-
events = @(@{ name = $Name; params = $params })
|
|
131
|
-
}
|
|
132
|
-
$json = $bodyObj | ConvertTo-Json -Depth 5 -Compress
|
|
133
|
-
$url = "https://www.google-analytics.com/mp/collect?measurement_id=$mid&api_secret=$secret"
|
|
120
|
+
$body = @{ event = $Name; anonymousId = $clientId } | ConvertTo-Json -Compress
|
|
134
121
|
try {
|
|
135
122
|
Start-Job -ScriptBlock {
|
|
136
123
|
param($u, $j)
|
|
137
124
|
try {
|
|
138
125
|
Invoke-RestMethod -Method Post -Uri $u -ContentType 'application/json' -Body $j -TimeoutSec 2 | Out-Null
|
|
139
126
|
} catch {}
|
|
140
|
-
} -ArgumentList $
|
|
127
|
+
} -ArgumentList $AnalyticsEndpoint, $body | Out-Null
|
|
141
128
|
} catch {}
|
|
142
129
|
}
|
|
143
130
|
|
|
@@ -158,7 +145,7 @@ if ([Console]::IsOutputRedirected) {
|
|
|
158
145
|
} else {
|
|
159
146
|
$chosen = Resolve-Agent
|
|
160
147
|
if (-not $chosen) { exit 1 }
|
|
161
|
-
Send-
|
|
148
|
+
Send-EvalsFunnelEvent -Name 'npx_evals_agent_selected'
|
|
162
149
|
}
|
|
163
150
|
|
|
164
151
|
# A directory, not a bare temp file: Step 4A looks for the offline bundle *beside*
|
|
@@ -288,7 +275,7 @@ try {
|
|
|
288
275
|
# where embedded double quotes would not.
|
|
289
276
|
# No "in this project": this may well be run from a home directory, and Step 4 is
|
|
290
277
|
# what scopes the work. Keep this wording in step with cli.js and start.sh.
|
|
291
|
-
$seed = "Read the file '$promptFile' in full
|
|
278
|
+
$seed = "Read the file '$promptFile' in full before doing anything else. Prefer your file-reading tool. If none is available, use read-only shell commands with bounded line ranges solely to print this exact file, repeating with successive ranges until EOF. Do not inspect or change anything else, and do not invoke a skill until a step tells you to. Then follow its steps in order from Step 0, walking me through each step and asking me questions as needed."
|
|
292
279
|
|
|
293
280
|
if (-not $chosen) {
|
|
294
281
|
# Addressed to the person, two lines, recommendation first - see the note in cli.js.
|
package/start.sh
CHANGED
|
@@ -22,20 +22,14 @@ set -euo pipefail
|
|
|
22
22
|
# files over HTTP, so it's fetchable even though the source repo is private.
|
|
23
23
|
# Override with ARIZE_PROMPT_URL if needed.
|
|
24
24
|
PROMPT_URL="${ARIZE_PROMPT_URL:-https://cdn.jsdelivr.net/npm/evals/onboarding-prompt.md}"
|
|
25
|
-
# Staged beside the prompt for the OAuth signup/sign-in ping (prompt must not name npx evals).
|
|
26
25
|
TRACK_URL="${ARIZE_TRACK_URL:-https://cdn.jsdelivr.net/npm/evals/track-event.mjs}"
|
|
27
26
|
ANALYTICS_URL="${ARIZE_ANALYTICS_URL:-https://cdn.jsdelivr.net/npm/evals/analytics.js}"
|
|
27
|
+
ANALYTICS_ENDPOINT="${ARIZE_EVALS_ANALYTICS_URL:-https://app.arize.com/api/analytics/npx-evals}"
|
|
28
28
|
|
|
29
|
-
#
|
|
30
|
-
# ARIZE_EVALS_GA4_MEASUREMENT_ID default = arize.com stream; override if needed.
|
|
29
|
+
# Fire-and-forget funnel ping (same events as cli.js). Never blocks the launcher.
|
|
31
30
|
track_evals_event() {
|
|
32
31
|
local name="$1"
|
|
33
|
-
local
|
|
34
|
-
local mid="${ARIZE_EVALS_GA4_MEASUREMENT_ID:-G-QSZ10TDVQM}"
|
|
35
|
-
local secret="${ARIZE_EVALS_GA4_API_SECRET:-}"
|
|
36
|
-
[[ -n "$secret" ]] || return 0
|
|
37
|
-
|
|
38
|
-
local anon_file="${HOME}/.arize/anon_client_id"
|
|
32
|
+
local anon_file="${HOME:-}/.arize/anon_client_id"
|
|
39
33
|
local client_id=""
|
|
40
34
|
if [[ -f "$anon_file" ]]; then
|
|
41
35
|
client_id="$(tr -d '[:space:]' < "$anon_file" 2>/dev/null || true)"
|
|
@@ -47,21 +41,14 @@ track_evals_event() {
|
|
|
47
41
|
client_id="$(od -An -N16 -tx1 /dev/urandom 2>/dev/null | tr -d ' \n')"
|
|
48
42
|
client_id="${client_id:0:8}-${client_id:8:4}-${client_id:12:4}-${client_id:16:4}-${client_id:20:12}"
|
|
49
43
|
fi
|
|
50
|
-
mkdir -p "${HOME}/.arize" 2>/dev/null || true
|
|
51
|
-
printf '%s' "$client_id" > "$anon_file" 2>/dev/null || true
|
|
52
|
-
fi
|
|
53
|
-
|
|
54
|
-
local params="\"engagement_time_msec\":1"
|
|
55
|
-
if [[ -n "$agent_id" ]]; then
|
|
56
|
-
params="$params,\"agent\":\"$agent_id\""
|
|
44
|
+
[[ -n "${HOME:-}" ]] && mkdir -p "${HOME}/.arize" 2>/dev/null || true
|
|
45
|
+
[[ -n "${HOME:-}" ]] && printf '%s' "$client_id" > "$anon_file" 2>/dev/null || true
|
|
57
46
|
fi
|
|
58
|
-
local body="{\"
|
|
59
|
-
local url="https://www.google-analytics.com/mp/collect?measurement_id=${mid}&api_secret=${secret}"
|
|
47
|
+
local body="{\"event\":\"$name\",\"anonymousId\":\"$client_id\"}"
|
|
60
48
|
if command -v curl >/dev/null 2>&1; then
|
|
61
|
-
curl -fsS -m 2 -X POST -H 'content-type: application/json' -d "$body" "$
|
|
49
|
+
curl -fsS -m 2 -X POST -H 'content-type: application/json' -d "$body" "$ANALYTICS_ENDPOINT" >/dev/null 2>&1 &
|
|
62
50
|
fi
|
|
63
51
|
}
|
|
64
|
-
|
|
65
52
|
# Where the bundled harness wheels live, same package. Override for testing.
|
|
66
53
|
VENDOR_URL="${ARIZE_VENDOR_URL:-https://cdn.jsdelivr.net/npm/evals/vendor}"
|
|
67
54
|
|
|
@@ -195,7 +182,7 @@ if [[ ! -t 1 ]]; then
|
|
|
195
182
|
CHOSEN=""
|
|
196
183
|
else
|
|
197
184
|
CHOSEN="$(choose_agent)" || exit 1
|
|
198
|
-
track_evals_event npx_evals_agent_selected
|
|
185
|
+
track_evals_event npx_evals_agent_selected
|
|
199
186
|
fi
|
|
200
187
|
|
|
201
188
|
# A directory, not a bare temp file: Step 4A looks for the offline bundle *beside*
|
|
@@ -318,7 +305,7 @@ fi
|
|
|
318
305
|
|
|
319
306
|
# No "in this project": this may well be run from a home directory, and Step 4 is
|
|
320
307
|
# what scopes the work. Keep this wording in step with cli.js and start.ps1.
|
|
321
|
-
SEED="Read the file '$PROMPT_FILE' in full
|
|
308
|
+
SEED="Read the file '$PROMPT_FILE' in full before doing anything else. Prefer your file-reading tool. If none is available, use read-only shell commands with bounded line ranges solely to print this exact file, repeating with successive ranges until EOF. Do not inspect or change anything else, and do not invoke a skill until a step tells you to. Then follow its steps in order from Step 0, walking me through each step and asking me questions as needed."
|
|
322
309
|
|
|
323
310
|
if [[ -z "$CHOSEN" ]]; then
|
|
324
311
|
# Addressed to the person, two lines, recommendation first — see the note in cli.js.
|
package/track-event.mjs
CHANGED
|
@@ -6,8 +6,7 @@
|
|
|
6
6
|
* the Arize signup/sign-in browser (OAuth) without naming `npx evals`.
|
|
7
7
|
*
|
|
8
8
|
* Usage:
|
|
9
|
-
* node ~/.arize/onboarding/track-event.mjs auth-opened
|
|
10
|
-
* node ~/.arize/onboarding/track-event.mjs auth-opened --method=login
|
|
9
|
+
* node ~/.arize/onboarding/track-event.mjs auth-opened
|
|
11
10
|
*
|
|
12
11
|
* Always exits 0 — tracking must never block auth.
|
|
13
12
|
*/
|
|
@@ -28,18 +27,7 @@ export async function run(argv) {
|
|
|
28
27
|
const name = EVENTS[key];
|
|
29
28
|
if (!name) return;
|
|
30
29
|
|
|
31
|
-
|
|
32
|
-
const params = {};
|
|
33
|
-
for (let i = 1; i < argv.length; i++) {
|
|
34
|
-
const a = argv[i];
|
|
35
|
-
if (a.startsWith('--method=')) {
|
|
36
|
-
params.method = a.slice('--method='.length);
|
|
37
|
-
} else if (a === '--method' && argv[i + 1]) {
|
|
38
|
-
params.method = argv[++i];
|
|
39
|
-
}
|
|
40
|
-
}
|
|
41
|
-
|
|
42
|
-
await trackEvent(name, params);
|
|
30
|
+
await trackEvent(name);
|
|
43
31
|
}
|
|
44
32
|
|
|
45
33
|
const isDirect =
|
package/vendor/MANIFEST
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
a9cede65f98d3415b13e1c6a437d0fdee27286bfff0186dd72bb65d9869b7864 LICENSE-coding-harness-tracing
|
|
2
2
|
62f22742b58a1a33014a2b6b706588a8d7e2a88ae7bd1a6ebe8c992928483775 certifi-2026.7.22-py3-none-any.whl
|
|
3
|
-
|
|
3
|
+
9f44b6e34618b87ab64f6aa48fd300bbfc6122fd4aca195122f68973747cad61 coding_harness_tracing-0.1.0-py3-none-any.whl
|
|
4
4
|
aca14e09ca99c253d2a142e2d5c5c38c8c7cf90840d3cb10b241c29929871fa4 harness-install.bat
|
|
5
5
|
a7ba18a98280e236b20ab4cb9f8a4006068b454586a2af50bf23fb0dbe70583d harness-install.sh
|
|
6
6
|
b81ee9561e9ca4004139c6cbba3a238c32b03e4894671e181b671e8cb8425d61 python_dotenv-1.2.1-py3-none-any.whl
|
|
Binary file
|
|
@@ -1,11 +1,11 @@
|
|
|
1
1
|
{
|
|
2
2
|
"repo": "https://github.com/Arize-ai/coding-harness-tracing",
|
|
3
3
|
"ref": "main",
|
|
4
|
-
"resolvedSha": "
|
|
4
|
+
"resolvedSha": "771ad555aab24ea2fd54a41b22936b6521156b26",
|
|
5
5
|
"source": "git",
|
|
6
6
|
"wheel": "coding_harness_tracing-0.1.0-py3-none-any.whl",
|
|
7
7
|
"wheelVersion": "0.1.0",
|
|
8
|
-
"wheelSha256": "
|
|
8
|
+
"wheelSha256": "9f44b6e34618b87ab64f6aa48fd300bbfc6122fd4aca195122f68973747cad61",
|
|
9
9
|
"files": [
|
|
10
10
|
"LICENSE-coding-harness-tracing",
|
|
11
11
|
"MANIFEST",
|