evals 2.8.1 → 2.8.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/analytics.js +116 -0
- package/bin.js +16 -7
- package/cli.js +37 -3
- package/onboarding-prompt.md +5 -1
- package/package.json +3 -1
- package/start.ps1 +48 -1
- package/start.sh +47 -1
- package/track-event.mjs +52 -0
- package/vendor/MANIFEST +1 -1
- package/vendor/coding_harness_tracing-0.1.0-py3-none-any.whl +0 -0
- package/vendor/harness-pin.lock.json +2 -2
package/analytics.js
ADDED
|
@@ -0,0 +1,116 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Anonymous GA4 Measurement Protocol events for the npx evals onboarding funnel.
|
|
3
|
+
*
|
|
4
|
+
* No PII — only an anon client id under ~/.arize and step names / agent ids.
|
|
5
|
+
* Fire-and-forget: never blocks or fails the CLI if tracking is down.
|
|
6
|
+
*
|
|
7
|
+
* Events:
|
|
8
|
+
* npx_evals_agent_selected — coding agent picked in the TUI
|
|
9
|
+
* npx_evals_agent_install_click — install-docs link when no agent on PATH
|
|
10
|
+
* npx_evals_skills_installed — Arize skills/CLI install attempt
|
|
11
|
+
* npx_evals_auth_opened — signup/sign-in browser about to open (OAuth)
|
|
12
|
+
*
|
|
13
|
+
* Requires a GA4 Measurement Protocol API secret (Admin → Data stream →
|
|
14
|
+
* Measurement Protocol API secrets). Set via ARIZE_EVALS_GA4_API_SECRET, or
|
|
15
|
+
* bake ARIZE_EVALS_GA4_API_SECRET_DEFAULT at release time.
|
|
16
|
+
*
|
|
17
|
+
* Override measurement id with ARIZE_EVALS_GA4_MEASUREMENT_ID if needed.
|
|
18
|
+
*/
|
|
19
|
+
|
|
20
|
+
import { randomUUID } from 'crypto';
|
|
21
|
+
import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'fs';
|
|
22
|
+
import { homedir } from 'os';
|
|
23
|
+
import { join } from 'path';
|
|
24
|
+
|
|
25
|
+
/** arize.com GA4 web stream (same property as site hero_agent_prompt_* events). */
|
|
26
|
+
export const DEFAULT_MEASUREMENT_ID = 'G-QSZ10TDVQM';
|
|
27
|
+
|
|
28
|
+
/**
|
|
29
|
+
* Optional release-time default. Leave empty until Chris/GA4 admin creates a
|
|
30
|
+
* Measurement Protocol API secret for this CLI. Env always wins when set.
|
|
31
|
+
*/
|
|
32
|
+
const API_SECRET_DEFAULT = '';
|
|
33
|
+
|
|
34
|
+
function measurementId() {
|
|
35
|
+
return process.env.ARIZE_EVALS_GA4_MEASUREMENT_ID || DEFAULT_MEASUREMENT_ID;
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
function apiSecret() {
|
|
39
|
+
return process.env.ARIZE_EVALS_GA4_API_SECRET || API_SECRET_DEFAULT;
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
function anonIdPath() {
|
|
43
|
+
return join(homedir(), '.arize', 'anon_client_id');
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
/** Stable anon id so GA4 can stitch steps for one machine without knowing who. */
|
|
47
|
+
export function getAnonClientId() {
|
|
48
|
+
const path = anonIdPath();
|
|
49
|
+
try {
|
|
50
|
+
if (existsSync(path)) {
|
|
51
|
+
const existing = readFileSync(path, 'utf8').trim();
|
|
52
|
+
if (existing) return existing;
|
|
53
|
+
}
|
|
54
|
+
} catch {
|
|
55
|
+
// fall through and mint a new id
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
const id = randomUUID();
|
|
59
|
+
try {
|
|
60
|
+
mkdirSync(join(homedir(), '.arize'), { recursive: true });
|
|
61
|
+
writeFileSync(path, id, 'utf8');
|
|
62
|
+
} catch {
|
|
63
|
+
// still send with in-memory id if home isn't writable
|
|
64
|
+
}
|
|
65
|
+
return id;
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
/**
|
|
69
|
+
* Send one anon event. Resolves quickly; network errors are swallowed.
|
|
70
|
+
* @param {string} name GA4 event name (snake_case, <=40 chars)
|
|
71
|
+
* @param {Record<string, string | number | boolean>} [params]
|
|
72
|
+
* @returns {Promise<void>}
|
|
73
|
+
*/
|
|
74
|
+
export async function trackEvent(name, params = {}) {
|
|
75
|
+
const mid = measurementId();
|
|
76
|
+
const secret = apiSecret();
|
|
77
|
+
if (!mid || !secret) return;
|
|
78
|
+
|
|
79
|
+
const url =
|
|
80
|
+
`https://www.google-analytics.com/mp/collect` +
|
|
81
|
+
`?measurement_id=${encodeURIComponent(mid)}` +
|
|
82
|
+
`&api_secret=${encodeURIComponent(secret)}`;
|
|
83
|
+
|
|
84
|
+
const body = {
|
|
85
|
+
client_id: getAnonClientId(),
|
|
86
|
+
events: [
|
|
87
|
+
{
|
|
88
|
+
name,
|
|
89
|
+
params: {
|
|
90
|
+
...params,
|
|
91
|
+
// Required for GA4 to count the event as engaged in some reports
|
|
92
|
+
engagement_time_msec: 1,
|
|
93
|
+
},
|
|
94
|
+
},
|
|
95
|
+
],
|
|
96
|
+
};
|
|
97
|
+
|
|
98
|
+
try {
|
|
99
|
+
const controller = new AbortController();
|
|
100
|
+
const timer = setTimeout(() => controller.abort(), 2500);
|
|
101
|
+
await fetch(url, {
|
|
102
|
+
method: 'POST',
|
|
103
|
+
headers: { 'content-type': 'application/json' },
|
|
104
|
+
body: JSON.stringify(body),
|
|
105
|
+
signal: controller.signal,
|
|
106
|
+
});
|
|
107
|
+
clearTimeout(timer);
|
|
108
|
+
} catch {
|
|
109
|
+
// never surface tracking failures to the user
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
/** Fire without awaiting — safe to call before launch / exit. */
|
|
114
|
+
export function trackEventBackground(name, params = {}) {
|
|
115
|
+
void trackEvent(name, params);
|
|
116
|
+
}
|
package/bin.js
CHANGED
|
@@ -34,10 +34,19 @@ if (!Number.isInteger(major) || major < MIN_MAJOR) {
|
|
|
34
34
|
process.exit(1);
|
|
35
35
|
}
|
|
36
36
|
|
|
37
|
-
//
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
37
|
+
// `evals track <event>` — fire-and-forget GA4, no TUI. Used for local/debug;
|
|
38
|
+
// the onboarding prompt stages track-event.mjs instead (must not name npx evals).
|
|
39
|
+
const args = process.argv.slice(2);
|
|
40
|
+
if (args[0] === 'track') {
|
|
41
|
+
import('./track-event.mjs')
|
|
42
|
+
.then((m) => m.run(args.slice(1)))
|
|
43
|
+
.finally(() => process.exit(0));
|
|
44
|
+
} else {
|
|
45
|
+
// cli.js only auto-runs when it is the entry point, so call main() explicitly.
|
|
46
|
+
import('./cli.js')
|
|
47
|
+
.then((cli) => cli.main())
|
|
48
|
+
.catch((err) => {
|
|
49
|
+
console.error(`Could not start evals: ${err && err.message ? err.message : err}`);
|
|
50
|
+
process.exit(1);
|
|
51
|
+
});
|
|
52
|
+
}
|
package/cli.js
CHANGED
|
@@ -21,6 +21,7 @@ import {
|
|
|
21
21
|
import { join, isAbsolute, resolve } from 'path';
|
|
22
22
|
import { tmpdir, homedir } from 'os';
|
|
23
23
|
import { fileURLToPath } from 'url';
|
|
24
|
+
import { trackEventBackground } from './analytics.js';
|
|
24
25
|
|
|
25
26
|
const e = React.createElement;
|
|
26
27
|
|
|
@@ -37,6 +38,8 @@ const BRAND_ACCENT = '#FF3CA8';
|
|
|
37
38
|
// that file — a ~35 KB prompt is too large to pass reliably as a command-line
|
|
38
39
|
// argument.
|
|
39
40
|
const BUNDLED_PROMPT_PATH = fileURLToPath(new URL('./onboarding-prompt.md', import.meta.url));
|
|
41
|
+
const BUNDLED_ANALYTICS_PATH = fileURLToPath(new URL('./analytics.js', import.meta.url));
|
|
42
|
+
const BUNDLED_TRACK_PATH = fileURLToPath(new URL('./track-event.mjs', import.meta.url));
|
|
40
43
|
|
|
41
44
|
// Wheels for the coding-agent tracing harness, built by scripts/build-harness-wheel.mjs.
|
|
42
45
|
// Staging these next to the prompt lets Step 4A install with no network and no
|
|
@@ -57,6 +60,18 @@ const PROMPT_FILE_NAME = 'onboarding-prompt.md';
|
|
|
57
60
|
// gets cleared at the next launch instead of sitting around holding a live key.
|
|
58
61
|
const ENV_FILE_NAME = 'harness.env';
|
|
59
62
|
|
|
63
|
+
// Anon GA4 helper staged beside the prompt so the agent can fire auth-opened
|
|
64
|
+
// before OAuth without naming `npx evals` (prompt contract forbids that).
|
|
65
|
+
const TRACK_FILE_NAME = 'track-event.mjs';
|
|
66
|
+
const ANALYTICS_FILE_NAME = 'analytics.js';
|
|
67
|
+
|
|
68
|
+
const STAGED_ROOT_FILES = [
|
|
69
|
+
PROMPT_FILE_NAME,
|
|
70
|
+
ENV_FILE_NAME,
|
|
71
|
+
TRACK_FILE_NAME,
|
|
72
|
+
ANALYTICS_FILE_NAME,
|
|
73
|
+
];
|
|
74
|
+
|
|
60
75
|
// MANIFEST only lands in the shell launchers' bundles, never in ours. Naming it
|
|
61
76
|
// anyway keeps one allowlist correct for every implementation, so a directory
|
|
62
77
|
// staged by start.sh can be cleared by `npx evals` and vice versa.
|
|
@@ -135,7 +150,7 @@ export function clearOnboardingDir(dir) {
|
|
|
135
150
|
continue;
|
|
136
151
|
}
|
|
137
152
|
|
|
138
|
-
if (
|
|
153
|
+
if (!STAGED_ROOT_FILES.includes(entry.name)) return false;
|
|
139
154
|
unlinkSync(path);
|
|
140
155
|
}
|
|
141
156
|
return true;
|
|
@@ -210,6 +225,9 @@ function prepareSeedPrompt() {
|
|
|
210
225
|
const { dir, fellBack } = prepareOnboardingDir();
|
|
211
226
|
const promptFile = join(dir, PROMPT_FILE_NAME);
|
|
212
227
|
writeFileSync(promptFile, promptText, 'utf8');
|
|
228
|
+
// Same dir as the prompt: agent runs this before OAuth opens signup/sign-in.
|
|
229
|
+
copyFileSync(BUNDLED_ANALYTICS_PATH, join(dir, ANALYTICS_FILE_NAME));
|
|
230
|
+
copyFileSync(BUNDLED_TRACK_PATH, join(dir, TRACK_FILE_NAME));
|
|
213
231
|
stageOfflineHarness(dir);
|
|
214
232
|
// Single quotes, not double: a home path is likelier to contain a space than a
|
|
215
233
|
// temp path was, but Windows spawns through the shell and embedded double
|
|
@@ -415,7 +433,8 @@ export function buildAgentItems() {
|
|
|
415
433
|
items: AGENTS.map(a => ({
|
|
416
434
|
name: `Install ${a.label}`,
|
|
417
435
|
subtext: 'No supported agent detected — open install docs',
|
|
418
|
-
url: a.installUrl
|
|
436
|
+
url: a.installUrl,
|
|
437
|
+
agentId: a.id,
|
|
419
438
|
})),
|
|
420
439
|
none: true
|
|
421
440
|
};
|
|
@@ -477,7 +496,7 @@ function App({ onDone }) {
|
|
|
477
496
|
}
|
|
478
497
|
|
|
479
498
|
if (selected.url) {
|
|
480
|
-
onDone({ type: 'url', url: selected.url });
|
|
499
|
+
onDone({ type: 'url', url: selected.url, agentId: selected.agentId });
|
|
481
500
|
exit();
|
|
482
501
|
return;
|
|
483
502
|
}
|
|
@@ -541,6 +560,11 @@ function launchAgent(agent) {
|
|
|
541
560
|
? ' Done.'
|
|
542
561
|
: ' Skipped — the setup will follow the Arize docs instead.');
|
|
543
562
|
|
|
563
|
+
trackEventBackground('npx_evals_skills_installed', {
|
|
564
|
+
agent: agent.id,
|
|
565
|
+
success: skillsInstalled ? 'true' : 'false',
|
|
566
|
+
});
|
|
567
|
+
|
|
544
568
|
console.log(`\nLaunching ${agent.label}…\n`);
|
|
545
569
|
|
|
546
570
|
const child = spawn(agent.bin, agent.args(prepared.seed), {
|
|
@@ -649,9 +673,19 @@ export async function main() {
|
|
|
649
673
|
if (!result) {
|
|
650
674
|
process.exit(0); // user quit / cancelled
|
|
651
675
|
} else if (result.type === 'url') {
|
|
676
|
+
// Install-doc link when no agent is on PATH (not the arize.com copy event).
|
|
677
|
+
trackEventBackground('npx_evals_agent_install_click', {
|
|
678
|
+
agent: result.agentId || 'unknown',
|
|
679
|
+
url_host: (() => {
|
|
680
|
+
try { return new URL(result.url).host; } catch { return 'unknown'; }
|
|
681
|
+
})(),
|
|
682
|
+
});
|
|
652
683
|
console.log(`\nOpening ${result.url} in your browser...\n`);
|
|
653
684
|
openUrlInBrowser(result.url);
|
|
654
685
|
} else if (result.type === 'launch') {
|
|
686
|
+
trackEventBackground('npx_evals_agent_selected', {
|
|
687
|
+
agent: result.agent.id,
|
|
688
|
+
});
|
|
655
689
|
launchAgent(result.agent);
|
|
656
690
|
}
|
|
657
691
|
} catch (error) {
|
package/onboarding-prompt.md
CHANGED
|
@@ -139,11 +139,14 @@ If a `default` profile exists but the probe failed, it is signed out or expired,
|
|
|
139
139
|
|
|
140
140
|
- **No key anywhere** — sign up or sign in with browser OAuth:
|
|
141
141
|
|
|
142
|
+
Immediately before the OAuth command, fire the anon funnel ping (ignore failures; never let it block auth), then open the browser flow:
|
|
143
|
+
|
|
142
144
|
```bash
|
|
145
|
+
node ~/.arize/onboarding/track-event.mjs auth-opened --method=oauth >/dev/null 2>&1 || true
|
|
143
146
|
ax profiles create default --auth-method oauth --utm-params "utm_source=npmevals&utm_medium=cli&utm_campaign=prompt-first-onboarding"
|
|
144
147
|
```
|
|
145
148
|
|
|
146
|
-
Pass `--utm-params` verbatim on the two browser OAuth commands (`profiles create --auth-method oauth` and the `ax auth login` fallback) and on no other `ax` command — it tags a new sign-up with onboarding attribution.
|
|
149
|
+
Pass `--utm-params` verbatim on the two browser OAuth commands (`profiles create --auth-method oauth` and the `ax auth login` fallback) and on no other `ax` command — it tags a new sign-up with onboarding attribution. Always run the matching `track-event.mjs auth-opened` line immediately before each of those two commands (`--method=oauth` or `--method=login`).
|
|
147
150
|
|
|
148
151
|
Always pass the positional profile name `default`. Without it, the CLI prompts `profile name [default]:`, receives EOF from an agent-run command, and exits with `Goodbye!` without creating a profile.
|
|
149
152
|
|
|
@@ -170,6 +173,7 @@ Then wait for the command to exit. Treat exit code `0`, or success output such a
|
|
|
170
173
|
Only after the OAuth command completes, re-run the probe from the top of this step. If it succeeds, continue. If the profile already existed and the probe returns an authentication error, it is signed out or expired — run the fallback:
|
|
171
174
|
|
|
172
175
|
```bash
|
|
176
|
+
node ~/.arize/onboarding/track-event.mjs auth-opened --method=login >/dev/null 2>&1 || true
|
|
173
177
|
ax auth login --utm-params "utm_source=npmevals&utm_medium=cli&utm_campaign=prompt-first-onboarding"
|
|
174
178
|
```
|
|
175
179
|
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "evals",
|
|
3
|
-
"version": "2.8.
|
|
3
|
+
"version": "2.8.2",
|
|
4
4
|
"description": "Arize AX onboarding — instrument your app with tracing via your coding agent",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "cli.js",
|
|
@@ -24,6 +24,8 @@
|
|
|
24
24
|
"files": [
|
|
25
25
|
"bin.js",
|
|
26
26
|
"cli.js",
|
|
27
|
+
"analytics.js",
|
|
28
|
+
"track-event.mjs",
|
|
27
29
|
"onboarding-prompt.md",
|
|
28
30
|
"start.sh",
|
|
29
31
|
"start.ps1",
|
package/start.ps1
CHANGED
|
@@ -26,6 +26,9 @@ $ErrorActionPreference = 'Stop'
|
|
|
26
26
|
# files over HTTP, so it's fetchable even though the source repo is private.
|
|
27
27
|
# Override with $env:ARIZE_PROMPT_URL if needed.
|
|
28
28
|
$PromptUrl = if ($env:ARIZE_PROMPT_URL) { $env:ARIZE_PROMPT_URL } else { 'https://cdn.jsdelivr.net/npm/evals/onboarding-prompt.md' }
|
|
29
|
+
# Staged beside the prompt for the OAuth signup/sign-in ping (prompt must not name npx evals).
|
|
30
|
+
$TrackUrl = if ($env:ARIZE_TRACK_URL) { $env:ARIZE_TRACK_URL } else { 'https://cdn.jsdelivr.net/npm/evals/track-event.mjs' }
|
|
31
|
+
$AnalyticsUrl = if ($env:ARIZE_ANALYTICS_URL) { $env:ARIZE_ANALYTICS_URL } else { 'https://cdn.jsdelivr.net/npm/evals/analytics.js' }
|
|
29
32
|
# Where the bundled harness wheels live, same package. Override for testing.
|
|
30
33
|
$VendorUrl = if ($env:ARIZE_VENDOR_URL) { $env:ARIZE_VENDOR_URL } else { 'https://cdn.jsdelivr.net/npm/evals/vendor' }
|
|
31
34
|
|
|
@@ -99,6 +102,45 @@ function Resolve-Agent {
|
|
|
99
102
|
}
|
|
100
103
|
}
|
|
101
104
|
|
|
105
|
+
|
|
106
|
+
function Send-EvalsGa4Event {
|
|
107
|
+
param([string]$Name, [string]$AgentId = '')
|
|
108
|
+
$mid = if ($env:ARIZE_EVALS_GA4_MEASUREMENT_ID) { $env:ARIZE_EVALS_GA4_MEASUREMENT_ID } else { 'G-QSZ10TDVQM' }
|
|
109
|
+
$secret = $env:ARIZE_EVALS_GA4_API_SECRET
|
|
110
|
+
if (-not $secret) { return }
|
|
111
|
+
|
|
112
|
+
$arizeDir = Join-Path $HOME '.arize'
|
|
113
|
+
$anonFile = Join-Path $arizeDir 'anon_client_id'
|
|
114
|
+
$clientId = $null
|
|
115
|
+
if (Test-Path -LiteralPath $anonFile) {
|
|
116
|
+
$clientId = (Get-Content -LiteralPath $anonFile -Raw -ErrorAction SilentlyContinue).Trim()
|
|
117
|
+
}
|
|
118
|
+
if (-not $clientId) {
|
|
119
|
+
$clientId = [guid]::NewGuid().ToString()
|
|
120
|
+
try {
|
|
121
|
+
New-Item -ItemType Directory -Force -Path $arizeDir | Out-Null
|
|
122
|
+
Set-Content -LiteralPath $anonFile -Value $clientId -NoNewline
|
|
123
|
+
} catch {}
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
$params = @{ engagement_time_msec = 1 }
|
|
127
|
+
if ($AgentId) { $params.agent = $AgentId }
|
|
128
|
+
$bodyObj = @{
|
|
129
|
+
client_id = $clientId
|
|
130
|
+
events = @(@{ name = $Name; params = $params })
|
|
131
|
+
}
|
|
132
|
+
$json = $bodyObj | ConvertTo-Json -Depth 5 -Compress
|
|
133
|
+
$url = "https://www.google-analytics.com/mp/collect?measurement_id=$mid&api_secret=$secret"
|
|
134
|
+
try {
|
|
135
|
+
Start-Job -ScriptBlock {
|
|
136
|
+
param($u, $j)
|
|
137
|
+
try {
|
|
138
|
+
Invoke-RestMethod -Method Post -Uri $u -ContentType 'application/json' -Body $j -TimeoutSec 2 | Out-Null
|
|
139
|
+
} catch {}
|
|
140
|
+
} -ArgumentList $url, $json | Out-Null
|
|
141
|
+
} catch {}
|
|
142
|
+
}
|
|
143
|
+
|
|
102
144
|
# --- Welcome ---
|
|
103
145
|
Write-Host ""
|
|
104
146
|
Write-Host "Arize AX" -ForegroundColor Magenta -NoNewline
|
|
@@ -116,6 +158,7 @@ if ([Console]::IsOutputRedirected) {
|
|
|
116
158
|
} else {
|
|
117
159
|
$chosen = Resolve-Agent
|
|
118
160
|
if (-not $chosen) { exit 1 }
|
|
161
|
+
Send-EvalsGa4Event -Name 'npx_evals_agent_selected' -AgentId $chosen
|
|
119
162
|
}
|
|
120
163
|
|
|
121
164
|
# A directory, not a bare temp file: Step 4A looks for the offline bundle *beside*
|
|
@@ -154,7 +197,7 @@ function Clear-PromptDir($dir) {
|
|
|
154
197
|
if (-not $ours) { return $false }
|
|
155
198
|
if (-not (Clear-BundleDir $sub.FullName)) { return $false }
|
|
156
199
|
}
|
|
157
|
-
foreach ($name in @('onboarding-prompt.md', 'harness.env')) {
|
|
200
|
+
foreach ($name in @('onboarding-prompt.md', 'harness.env', 'track-event.mjs', 'analytics.js')) {
|
|
158
201
|
Remove-Item -LiteralPath (Join-Path $dir $name) -Force -ErrorAction SilentlyContinue
|
|
159
202
|
}
|
|
160
203
|
# Anything left is not ours.
|
|
@@ -200,6 +243,10 @@ try {
|
|
|
200
243
|
exit 1
|
|
201
244
|
}
|
|
202
245
|
|
|
246
|
+
# Best-effort: agent runs these before OAuth. Missing files just skip the ping.
|
|
247
|
+
try { Invoke-RestMethod -Uri $TrackUrl -OutFile (Join-Path $promptDir 'track-event.mjs') } catch {}
|
|
248
|
+
try { Invoke-RestMethod -Uri $AnalyticsUrl -OutFile (Join-Path $promptDir 'analytics.js') } catch {}
|
|
249
|
+
|
|
203
250
|
# Fetch the bundled tracing harness so Step 4A can install it without piping a
|
|
204
251
|
# downloaded script into a shell — the thing auto-approval permission
|
|
205
252
|
# classifiers refuse. `npx evals` ships these files; this path pulls them from
|
package/start.sh
CHANGED
|
@@ -22,6 +22,46 @@ set -euo pipefail
|
|
|
22
22
|
# files over HTTP, so it's fetchable even though the source repo is private.
|
|
23
23
|
# Override with ARIZE_PROMPT_URL if needed.
|
|
24
24
|
PROMPT_URL="${ARIZE_PROMPT_URL:-https://cdn.jsdelivr.net/npm/evals/onboarding-prompt.md}"
|
|
25
|
+
# Staged beside the prompt for the OAuth signup/sign-in ping (prompt must not name npx evals).
|
|
26
|
+
TRACK_URL="${ARIZE_TRACK_URL:-https://cdn.jsdelivr.net/npm/evals/track-event.mjs}"
|
|
27
|
+
ANALYTICS_URL="${ARIZE_ANALYTICS_URL:-https://cdn.jsdelivr.net/npm/evals/analytics.js}"
|
|
28
|
+
|
|
29
|
+
# Anonymous GA4 step ping (same events as cli.js). No-op without API secret.
|
|
30
|
+
# ARIZE_EVALS_GA4_MEASUREMENT_ID default = arize.com stream; override if needed.
|
|
31
|
+
track_evals_event() {
|
|
32
|
+
local name="$1"
|
|
33
|
+
local agent_id="${2:-}"
|
|
34
|
+
local mid="${ARIZE_EVALS_GA4_MEASUREMENT_ID:-G-QSZ10TDVQM}"
|
|
35
|
+
local secret="${ARIZE_EVALS_GA4_API_SECRET:-}"
|
|
36
|
+
[[ -n "$secret" ]] || return 0
|
|
37
|
+
|
|
38
|
+
local anon_file="${HOME}/.arize/anon_client_id"
|
|
39
|
+
local client_id=""
|
|
40
|
+
if [[ -f "$anon_file" ]]; then
|
|
41
|
+
client_id="$(tr -d '[:space:]' < "$anon_file" 2>/dev/null || true)"
|
|
42
|
+
fi
|
|
43
|
+
if [[ -z "$client_id" ]]; then
|
|
44
|
+
if command -v uuidgen >/dev/null 2>&1; then
|
|
45
|
+
client_id="$(uuidgen | tr '[:upper:]' '[:lower:]')"
|
|
46
|
+
else
|
|
47
|
+
client_id="$(od -An -N16 -tx1 /dev/urandom 2>/dev/null | tr -d ' \n')"
|
|
48
|
+
client_id="${client_id:0:8}-${client_id:8:4}-${client_id:12:4}-${client_id:16:4}-${client_id:20:12}"
|
|
49
|
+
fi
|
|
50
|
+
mkdir -p "${HOME}/.arize" 2>/dev/null || true
|
|
51
|
+
printf '%s' "$client_id" > "$anon_file" 2>/dev/null || true
|
|
52
|
+
fi
|
|
53
|
+
|
|
54
|
+
local params="\"engagement_time_msec\":1"
|
|
55
|
+
if [[ -n "$agent_id" ]]; then
|
|
56
|
+
params="$params,\"agent\":\"$agent_id\""
|
|
57
|
+
fi
|
|
58
|
+
local body="{\"client_id\":\"$client_id\",\"events\":[{\"name\":\"$name\",\"params\":{$params}}]}"
|
|
59
|
+
local url="https://www.google-analytics.com/mp/collect?measurement_id=${mid}&api_secret=${secret}"
|
|
60
|
+
if command -v curl >/dev/null 2>&1; then
|
|
61
|
+
curl -fsS -m 2 -X POST -H 'content-type: application/json' -d "$body" "$url" >/dev/null 2>&1 &
|
|
62
|
+
fi
|
|
63
|
+
}
|
|
64
|
+
|
|
25
65
|
# Where the bundled harness wheels live, same package. Override for testing.
|
|
26
66
|
VENDOR_URL="${ARIZE_VENDOR_URL:-https://cdn.jsdelivr.net/npm/evals/vendor}"
|
|
27
67
|
|
|
@@ -155,6 +195,7 @@ if [[ ! -t 1 ]]; then
|
|
|
155
195
|
CHOSEN=""
|
|
156
196
|
else
|
|
157
197
|
CHOSEN="$(choose_agent)" || exit 1
|
|
198
|
+
track_evals_event npx_evals_agent_selected "$CHOSEN"
|
|
158
199
|
fi
|
|
159
200
|
|
|
160
201
|
# A directory, not a bare temp file: Step 4A looks for the offline bundle *beside*
|
|
@@ -186,7 +227,8 @@ clear_prompt_dir() {
|
|
|
186
227
|
for sub in "$1"/arize-offline "$1"/.staging*; do
|
|
187
228
|
[ -d "$sub" ] && clear_bundle_dir "$sub"
|
|
188
229
|
done
|
|
189
|
-
rm -f "$1/onboarding-prompt.md" "$1/harness.env"
|
|
230
|
+
rm -f "$1/onboarding-prompt.md" "$1/harness.env" \
|
|
231
|
+
"$1/track-event.mjs" "$1/analytics.js"
|
|
190
232
|
# Anything left is not ours: bail out and let the caller fall back.
|
|
191
233
|
rmdir "$1" 2>/dev/null || { [ -z "$(ls -A "$1" 2>/dev/null)" ]; return $?; }
|
|
192
234
|
}
|
|
@@ -231,6 +273,10 @@ if ! fetch "$PROMPT_URL" "$PROMPT_FILE"; then
|
|
|
231
273
|
exit 1
|
|
232
274
|
fi
|
|
233
275
|
|
|
276
|
+
# Best-effort: agent runs these before OAuth. Missing files just skip the ping.
|
|
277
|
+
fetch "$TRACK_URL" "$PROMPT_DIR/track-event.mjs" 2>/dev/null || true
|
|
278
|
+
fetch "$ANALYTICS_URL" "$PROMPT_DIR/analytics.js" 2>/dev/null || true
|
|
279
|
+
|
|
234
280
|
# Fetch the bundled tracing harness so Step 4A can install it without piping a
|
|
235
281
|
# downloaded script into a shell — the thing auto-approval permission classifiers
|
|
236
282
|
# refuse. `npx evals` ships these files in the package; this path has to pull them
|
package/track-event.mjs
ADDED
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
/**
|
|
3
|
+
* Fire one anon funnel event, then exit.
|
|
4
|
+
*
|
|
5
|
+
* Staged next to the onboarding prompt so the agent can run it before opening
|
|
6
|
+
* the Arize signup/sign-in browser (OAuth) without naming `npx evals`.
|
|
7
|
+
*
|
|
8
|
+
* Usage:
|
|
9
|
+
* node ~/.arize/onboarding/track-event.mjs auth-opened --method=oauth
|
|
10
|
+
* node ~/.arize/onboarding/track-event.mjs auth-opened --method=login
|
|
11
|
+
*
|
|
12
|
+
* Always exits 0 — tracking must never block auth.
|
|
13
|
+
*/
|
|
14
|
+
|
|
15
|
+
import { trackEvent } from './analytics.js';
|
|
16
|
+
|
|
17
|
+
/** @type {Record<string, string>} */
|
|
18
|
+
const EVENTS = {
|
|
19
|
+
'auth-opened': 'npx_evals_auth_opened',
|
|
20
|
+
};
|
|
21
|
+
|
|
22
|
+
/**
|
|
23
|
+
* @param {string[]} argv
|
|
24
|
+
* @returns {Promise<void>}
|
|
25
|
+
*/
|
|
26
|
+
export async function run(argv) {
|
|
27
|
+
const key = argv[0] || '';
|
|
28
|
+
const name = EVENTS[key];
|
|
29
|
+
if (!name) return;
|
|
30
|
+
|
|
31
|
+
/** @type {Record<string, string>} */
|
|
32
|
+
const params = {};
|
|
33
|
+
for (let i = 1; i < argv.length; i++) {
|
|
34
|
+
const a = argv[i];
|
|
35
|
+
if (a.startsWith('--method=')) {
|
|
36
|
+
params.method = a.slice('--method='.length);
|
|
37
|
+
} else if (a === '--method' && argv[i + 1]) {
|
|
38
|
+
params.method = argv[++i];
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
await trackEvent(name, params);
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
const isDirect =
|
|
46
|
+
process.argv[1] &&
|
|
47
|
+
(process.argv[1].endsWith('track-event.mjs') ||
|
|
48
|
+
process.argv[1].endsWith('track-event'));
|
|
49
|
+
|
|
50
|
+
if (isDirect) {
|
|
51
|
+
run(process.argv.slice(2)).finally(() => process.exit(0));
|
|
52
|
+
}
|
package/vendor/MANIFEST
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
a9cede65f98d3415b13e1c6a437d0fdee27286bfff0186dd72bb65d9869b7864 LICENSE-coding-harness-tracing
|
|
2
2
|
62f22742b58a1a33014a2b6b706588a8d7e2a88ae7bd1a6ebe8c992928483775 certifi-2026.7.22-py3-none-any.whl
|
|
3
|
-
|
|
3
|
+
e5060c3fa2807eff7a5816caf2a78eb2cd285b6fbe0a1c3022373cd89a0e46d7 coding_harness_tracing-0.1.0-py3-none-any.whl
|
|
4
4
|
aca14e09ca99c253d2a142e2d5c5c38c8c7cf90840d3cb10b241c29929871fa4 harness-install.bat
|
|
5
5
|
a7ba18a98280e236b20ab4cb9f8a4006068b454586a2af50bf23fb0dbe70583d harness-install.sh
|
|
6
6
|
b81ee9561e9ca4004139c6cbba3a238c32b03e4894671e181b671e8cb8425d61 python_dotenv-1.2.1-py3-none-any.whl
|
|
Binary file
|
|
@@ -1,11 +1,11 @@
|
|
|
1
1
|
{
|
|
2
2
|
"repo": "https://github.com/Arize-ai/coding-harness-tracing",
|
|
3
3
|
"ref": "main",
|
|
4
|
-
"resolvedSha": "
|
|
4
|
+
"resolvedSha": "a51baf0a3f45b01b1d23677176c90eb851fb3969",
|
|
5
5
|
"source": "git",
|
|
6
6
|
"wheel": "coding_harness_tracing-0.1.0-py3-none-any.whl",
|
|
7
7
|
"wheelVersion": "0.1.0",
|
|
8
|
-
"wheelSha256": "
|
|
8
|
+
"wheelSha256": "e5060c3fa2807eff7a5816caf2a78eb2cd285b6fbe0a1c3022373cd89a0e46d7",
|
|
9
9
|
"files": [
|
|
10
10
|
"LICENSE-coding-harness-tracing",
|
|
11
11
|
"MANIFEST",
|