evals 2.7.0 → 2.8.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/cli.js +75 -7
- package/onboarding-prompt.md +137 -87
- package/package.json +1 -1
- package/start.ps1 +1 -1
- package/start.sh +1 -1
- package/vendor/MANIFEST +1 -1
- package/vendor/coding_harness_tracing-0.1.0-py3-none-any.whl +0 -0
- package/vendor/harness-pin.lock.json +2 -2
package/cli.js
CHANGED
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
import React, { useState, useEffect } from 'react';
|
|
4
4
|
import { render, Box, Text, useInput, useApp, Static } from 'ink';
|
|
5
|
-
import { exec, spawn } from 'child_process';
|
|
5
|
+
import { exec, spawn, spawnSync } from 'child_process';
|
|
6
6
|
import {
|
|
7
7
|
existsSync,
|
|
8
8
|
readFileSync,
|
|
@@ -62,6 +62,23 @@ const ENV_FILE_NAME = 'harness.env';
|
|
|
62
62
|
// staged by start.sh can be cleared by `npx evals` and vice versa.
|
|
63
63
|
const BUNDLE_EXTRA_FILES = ['MANIFEST'];
|
|
64
64
|
|
|
65
|
+
// The Arize agent skills, installed into the chosen agent's *global* skills
|
|
66
|
+
// directory before we spawn it. Doing it here rather than from the prompt is the
|
|
67
|
+
// whole point: a skill installed mid-session is not in the agent's invocable skill
|
|
68
|
+
// list, so the prompt used to have to hunt for `SKILL.md` by path across four
|
|
69
|
+
// possible agent directories. Installed before launch, `arize-instrumentation` is
|
|
70
|
+
// just a skill the agent can call by name.
|
|
71
|
+
//
|
|
72
|
+
// Deliberately fetched rather than vendored like the harness wheel: the skills
|
|
73
|
+
// change far more often than this package publishes, and a user is better served
|
|
74
|
+
// by the current set than by whichever set was current at our last release.
|
|
75
|
+
const SKILLS_PACKAGE = 'Arize-ai/arize-skills';
|
|
76
|
+
|
|
77
|
+
// Bounded, because this is an enhancement and not a prerequisite — a slow clone
|
|
78
|
+
// must never hold the agent launch hostage. Everything the skills do has a docs
|
|
79
|
+
// fallback in the prompt, so the cost of giving up here is a longer Step 6.
|
|
80
|
+
const SKILLS_TIMEOUT_MS = 120_000;
|
|
81
|
+
|
|
65
82
|
// Where the prompt and the offline bundle get staged: `~/.arize/onboarding`.
|
|
66
83
|
// Deliberately stable rather than a fresh temp directory. The agent is asked to
|
|
67
84
|
// read a file outside its workspace, and that is an approval a human can grant
|
|
@@ -200,18 +217,26 @@ function prepareSeedPrompt() {
|
|
|
200
217
|
// No "in this project": the launcher may well be run from a home directory, and
|
|
201
218
|
// Step 4 is what scopes the work — an app here, an app at another path, a starter
|
|
202
219
|
// app, or the coding agent itself. Keep this wording in step with start.{sh,ps1}.
|
|
203
|
-
const seed = `Read the file '${promptFile}'
|
|
220
|
+
const seed = `Read the file '${promptFile}' in full with your file-reading tool, not a shell command, then follow it to set up Arize AX tracing, walking me through each step and asking me questions as needed.`;
|
|
204
221
|
return { seed, dir, fellBack };
|
|
205
222
|
}
|
|
206
223
|
|
|
207
224
|
// Coding agents we can launch interactively, seeded with the prompt.
|
|
208
225
|
// `args(seed)` returns the argv that starts the agent's REPL pre-loaded with `seed`.
|
|
226
|
+
//
|
|
227
|
+
// `skillsAgent` is the name the `skills` CLI knows the agent by, which is its own
|
|
228
|
+
// registry and not ours: our ids are binary names because the shell launchers
|
|
229
|
+
// `exec` them (see the test that pins bin === id), so `cursor-agent` is `cursor`
|
|
230
|
+
// there and `agy` is `antigravity-cli`. Two catalogs, no overlap in naming rules —
|
|
231
|
+
// hence a field rather than a derivation. All five values were verified against
|
|
232
|
+
// that registry; one it doesn't know exits 1 (`Invalid agents: …`), so drift shows
|
|
233
|
+
// up as a skipped install rather than skills landing somewhere unexpected.
|
|
209
234
|
export const AGENTS = [
|
|
210
|
-
{ id: 'claude', label: 'Claude Code', bin: 'claude', args: (s) => [s], installUrl: 'https://docs.claude.com/en/docs/claude-code' },
|
|
211
|
-
{ id: 'codex', label: 'OpenAI Codex', bin: 'codex', args: (s) => [s], installUrl: 'https://developers.openai.com/codex/cli' },
|
|
212
|
-
{ id: 'cursor-agent', label: 'Cursor', bin: 'cursor-agent', args: (s) => [s], installUrl: 'https://docs.cursor.com/en/cli/overview' },
|
|
213
|
-
{ id: 'copilot', label: 'GitHub Copilot', bin: 'copilot', args: (s) => ['-i', s], installUrl: 'https://github.com/features/copilot/cli' },
|
|
214
|
-
{ id: 'agy', label: 'Antigravity CLI', bin: 'agy', args: (s) => ['-i', s], installUrl: 'https://antigravity.google/docs/cli/getting-started' }
|
|
235
|
+
{ id: 'claude', label: 'Claude Code', bin: 'claude', skillsAgent: 'claude-code', args: (s) => [s], installUrl: 'https://docs.claude.com/en/docs/claude-code' },
|
|
236
|
+
{ id: 'codex', label: 'OpenAI Codex', bin: 'codex', skillsAgent: 'codex', args: (s) => [s], installUrl: 'https://developers.openai.com/codex/cli' },
|
|
237
|
+
{ id: 'cursor-agent', label: 'Cursor', bin: 'cursor-agent', skillsAgent: 'cursor', args: (s) => [s], installUrl: 'https://docs.cursor.com/en/cli/overview' },
|
|
238
|
+
{ id: 'copilot', label: 'GitHub Copilot', bin: 'copilot', skillsAgent: 'github-copilot', args: (s) => ['-i', s], installUrl: 'https://github.com/features/copilot/cli' },
|
|
239
|
+
{ id: 'agy', label: 'Antigravity CLI', bin: 'agy', skillsAgent: 'antigravity-cli', args: (s) => ['-i', s], installUrl: 'https://antigravity.google/docs/cli/getting-started' }
|
|
215
240
|
];
|
|
216
241
|
|
|
217
242
|
// PATH scan — detects an agent without executing it (running it could hang).
|
|
@@ -232,6 +257,40 @@ export function isInstalled(bin) {
|
|
|
232
257
|
return false;
|
|
233
258
|
}
|
|
234
259
|
|
|
260
|
+
// argv for the skills install. Two `--yes` flags and they are not a typo: the first
|
|
261
|
+
// is npx's ("don't prompt before fetching the package"), the last is `skills add`'s
|
|
262
|
+
// ("don't prompt for scope or confirmation"). Without both it blocks on a prompt
|
|
263
|
+
// nobody is watching, because we run between the picker unmounting and the agent
|
|
264
|
+
// taking the terminal.
|
|
265
|
+
//
|
|
266
|
+
// `--global` on purpose. Left to itself the CLI auto-detects scope and picks
|
|
267
|
+
// *project* when run inside one, which would write `.claude/skills/` into whatever
|
|
268
|
+
// repo the user happened to launch us from. Global also outlives this session,
|
|
269
|
+
// which is the behaviour someone onboarding to Arize actually wants.
|
|
270
|
+
export function buildSkillsArgs(skillsAgent, pkg = SKILLS_PACKAGE) {
|
|
271
|
+
return ['--yes', 'skills', 'add', pkg, '--skill', '*', '--agent', skillsAgent, '--global', '--yes'];
|
|
272
|
+
}
|
|
273
|
+
|
|
274
|
+
// Install the Arize skills for the agent we are about to launch. Returns true only
|
|
275
|
+
// on a clean install. Never throws and never blocks the launch: a false here costs
|
|
276
|
+
// the user a docs-path Step 6, while a throw would cost them the whole flow.
|
|
277
|
+
// Output is captured rather than inherited — the CLI prints a box listing all
|
|
278
|
+
// fourteen skills, which is noise on the last screen before the agent takes over.
|
|
279
|
+
export function installArizeSkills(agent, { run = spawnSync, isWin = process.platform === 'win32' } = {}) {
|
|
280
|
+
if (!agent || !agent.skillsAgent) return false;
|
|
281
|
+
try {
|
|
282
|
+
const result = run('npx', buildSkillsArgs(agent.skillsAgent), {
|
|
283
|
+
stdio: ['ignore', 'pipe', 'pipe'],
|
|
284
|
+
timeout: SKILLS_TIMEOUT_MS,
|
|
285
|
+
shell: isWin, // npx is npx.cmd on Windows and needs the shell to resolve
|
|
286
|
+
encoding: 'utf8',
|
|
287
|
+
});
|
|
288
|
+
return result != null && result.status === 0;
|
|
289
|
+
} catch {
|
|
290
|
+
return false;
|
|
291
|
+
}
|
|
292
|
+
}
|
|
293
|
+
|
|
235
294
|
// "ARIZE AX" on one line — 66 columns wide, so Header() drops to the compact
|
|
236
295
|
// two-row art below 68 and to plain text below 30.
|
|
237
296
|
const largeLogo = `
|
|
@@ -473,6 +532,15 @@ function launchAgent(agent) {
|
|
|
473
532
|
console.log('Instrumenting an app still works; tracing this coding agent (Step 4A) needs a writable home directory.');
|
|
474
533
|
}
|
|
475
534
|
|
|
535
|
+
// Before the spawn, so they land in the agent's skill list at session start
|
|
536
|
+
// rather than mid-flow. Announced first: this clones over the network and can
|
|
537
|
+
// take a few seconds, and a launcher that goes quiet looks hung.
|
|
538
|
+
console.log(`\nInstalling the Arize skills for ${agent.label}…`);
|
|
539
|
+
const skillsInstalled = installArizeSkills(agent);
|
|
540
|
+
console.log(skillsInstalled
|
|
541
|
+
? ' Done.'
|
|
542
|
+
: ' Skipped — the setup will follow the Arize docs instead.');
|
|
543
|
+
|
|
476
544
|
console.log(`\nLaunching ${agent.label}…\n`);
|
|
477
545
|
|
|
478
546
|
const child = spawn(agent.bin, agent.args(prepared.seed), {
|
package/onboarding-prompt.md
CHANGED
|
@@ -2,16 +2,28 @@
|
|
|
2
2
|
|
|
3
3
|
Guide the user from zero to their first traces in Arize AX. Use the AX CLI, the Arize agent skills, and the official docs.
|
|
4
4
|
|
|
5
|
-
|
|
5
|
+
**Read all of this file before you act.** If your tools return only part of it, keep reading to the last line. Rules you need are at the end.
|
|
6
|
+
|
|
7
|
+
Work through the flow in order. Installing the AX CLI and Arize skills, authenticating, and listing spaces are fine before approval — that is your own tooling. Do not create AX resources (like API keys), edit application files, or install the app's tracing dependencies before the user approves the plan in Step 5.
|
|
8
|
+
|
|
9
|
+
## How to write to the user
|
|
10
|
+
|
|
11
|
+
**Use ASD-STE100 Simplified Technical English for all responses.** One idea per sentence, active voice, present tense. Use the simplest word that carries the meaning, and keep the same word for the same thing every time. Aim for 20 words or fewer per sentence.
|
|
12
|
+
|
|
13
|
+
**Be succinct, not verbose.** Say what you did and what you need next, then stop. No preamble, no restating the step, no describing work first, and no recap of what the user just read. Do not pad with reassurance or apology. If a sentence does not change what the user knows or does, delete it.
|
|
14
|
+
|
|
15
|
+
Two exceptions. Where a step tells you to send a block as written, send it exactly as written. Never simplify a command, path, or identifier.
|
|
16
|
+
|
|
17
|
+
**Never let a URL wrap** — a wrapped URL is not clickable. Keep the line to 80 characters or fewer. If a label plus the URL is longer, put the URL alone on the next line. Add no punctuation after a URL.
|
|
6
18
|
|
|
7
19
|
## Step 0: Welcome and confirm
|
|
8
20
|
|
|
9
|
-
Greet the user
|
|
21
|
+
Greet the user, show the plan, ask before doing anything. Send this block as written and nothing else: no added steps, no preamble about the approval gate. It comes later.
|
|
10
22
|
|
|
11
23
|
```text
|
|
12
24
|
Welcome to Arize AX. I'll get you set up with tracing. Here's what I'll do:
|
|
13
25
|
|
|
14
|
-
1. Install the AX CLI and Arize skills
|
|
26
|
+
1. Install the AX CLI and Arize skills if needed
|
|
15
27
|
2. Create a free Arize AX account or sign you in
|
|
16
28
|
3. Add tracing to an existing app, a new starter app, or this coding agent itself
|
|
17
29
|
|
|
@@ -20,51 +32,83 @@ Shall I proceed?
|
|
|
20
32
|
|
|
21
33
|
Do not proceed until the user approves.
|
|
22
34
|
|
|
23
|
-
##
|
|
35
|
+
## Track progress
|
|
24
36
|
|
|
25
|
-
**
|
|
37
|
+
**Track this flow in your task list.** It is long. A task list shows the user how far through they are, and keeps your place if your context is compacted.
|
|
26
38
|
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
39
|
+
Find the tool first. Names differ between agents: `TaskCreate` and `TaskUpdate`, `TodoWrite`, or a plan tool. Some agents do not load these tools until you search, so search your tools for "task" or "todo" before you decide you have none.
|
|
40
|
+
|
|
41
|
+
Create the list as soon as the user approves in Step 0. One task per milestone:
|
|
42
|
+
|
|
43
|
+
1. Install the AX CLI and Arize skills
|
|
44
|
+
2. Sign in to Arize AX
|
|
45
|
+
3. Select the AX space
|
|
46
|
+
4. Choose what to trace
|
|
47
|
+
|
|
48
|
+
Add the rest at Step 4, once you know the user's choice. For an existing app or a starter app:
|
|
49
|
+
|
|
50
|
+
5. Agree the plan
|
|
51
|
+
6. Create the API key and add tracing
|
|
52
|
+
7. Run the app and wait for the first traces
|
|
53
|
+
8. Send the link to the traces
|
|
33
54
|
|
|
34
|
-
|
|
55
|
+
For tracing this coding agent:
|
|
35
56
|
|
|
36
|
-
|
|
57
|
+
5. Approve agent tracing
|
|
58
|
+
6. Install harness tracing
|
|
59
|
+
7. Start a new session and wait for traces
|
|
60
|
+
8. Send the link to the traces
|
|
61
|
+
|
|
62
|
+
Rules:
|
|
63
|
+
|
|
64
|
+
- One task per milestone. Never add sub-tasks or split a milestone.
|
|
65
|
+
- Keep exactly one task in progress.
|
|
66
|
+
- Mark a task complete when it is done, not when you start it. A list that lags behind is worse than no list.
|
|
67
|
+
- Do not mention the list in chat. Do not print it or say you updated it.
|
|
68
|
+
- Skip this section only if you searched and found no task list tool. Do not print a checklist instead.
|
|
69
|
+
|
|
70
|
+
## Note
|
|
71
|
+
|
|
72
|
+
**Every step below needs network access.**
|
|
73
|
+
|
|
74
|
+
A connection or DNS error from any command means network, not authentication. Do not retry, do not look for a workaround, and do not re-authenticate or recreate a profile. Agent sandboxes often block the network, and the block can look like a hang.
|
|
75
|
+
|
|
76
|
+
When it happens, ask your harness to escalate: request network or elevated permissions for that command. If it refuses, or offers no escalation, stop and tell the user:
|
|
37
77
|
|
|
38
78
|
```text
|
|
39
79
|
My sandbox is blocking network access, so I can't install the CLI or reach
|
|
40
80
|
Arize. Restart me with network access enabled and I'll pick this up.
|
|
41
81
|
```
|
|
42
82
|
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
The AX CLI must be **arize-ax-cli `0.29.0` or newer** (Step 1 installs the latest) and needs **Python 3.11+** — a hard requirement. If Python is missing, stop and have the user install it from https://www.python.org/downloads/ and re-run. **Node.js 18+ with npx** is optional but installs the Arize skills; without it, don't stop — continue and use the docs paths in Step 6 (the Vercel AI SDK v7 starter needs Node.js 22+).
|
|
83
|
+
## Step 1: Install the AX CLI, check the Arize skills
|
|
46
84
|
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
Check the version with `ax --version`. If it's missing or older than the version above, install or upgrade `arize-ax-cli` — prefer `pipx`, otherwise pip:
|
|
85
|
+
Check the version with `ax --version`. If it's missing or older than 0.29.0, install or upgrade `arize-ax-cli` — prefer `uv` or `pipx`, otherwise pip:
|
|
50
86
|
|
|
51
87
|
```bash
|
|
52
|
-
|
|
53
|
-
# without
|
|
88
|
+
uv tool install arize-ax-cli # already installed: uv tool upgrade arize-ax-cli
|
|
89
|
+
# without uv:
|
|
90
|
+
pipx install arize-ax-cli # already installed: pipx upgrade arize-ax-cli
|
|
91
|
+
# without either:
|
|
54
92
|
python3 -m pip install --upgrade arize-ax-cli
|
|
55
93
|
```
|
|
56
94
|
|
|
57
|
-
|
|
95
|
+
Try `uv` first, even if `pipx` is installed.
|
|
96
|
+
|
|
97
|
+
**Check your skill list for `arize-instrumentation` and `arize-link` first.** They are usually installed already. If they are listed, do not run the install — invoke them by name when needed and go to Step 2.
|
|
98
|
+
|
|
99
|
+
Install them only if they are absent, or you cannot see your skill list, and npx is available:
|
|
58
100
|
|
|
59
101
|
```bash
|
|
60
|
-
npx skills add Arize-ai/arize-skills --skill '*' --yes
|
|
102
|
+
npx skills add Arize-ai/arize-skills --skill '*' --global --yes
|
|
61
103
|
```
|
|
62
104
|
|
|
63
|
-
|
|
105
|
+
Skip this and continue if npx is missing, the install errors, or it does not finish promptly.
|
|
106
|
+
|
|
107
|
+
A skill installed now does **not** join your skill list, so do not call it by name. Read its `SKILL.md` from the path the command prints. Directories are named `arize-<name>`.
|
|
64
108
|
|
|
65
109
|
## Step 2: Authenticate the CLI
|
|
66
110
|
|
|
67
|
-
The CLI authenticates through a profile named `default`, and every later `ax` command
|
|
111
|
+
The CLI authenticates through a profile named `default`, and every later `ax` command uses it. A bare `ARIZE_API_KEY` in the environment does **not** authenticate the CLI; a profile must exist. Get one working before continuing.
|
|
68
112
|
|
|
69
113
|
First check whether the CLI is already authenticated:
|
|
70
114
|
|
|
@@ -74,15 +118,15 @@ ax spaces list --limit 1 --output json
|
|
|
74
118
|
|
|
75
119
|
If this succeeds, a profile is already set up — skip the rest of this step.
|
|
76
120
|
|
|
77
|
-
If it fails,
|
|
121
|
+
If it fails, check whether a `default` profile exists from a prior run:
|
|
78
122
|
|
|
79
123
|
```bash
|
|
80
124
|
ax profiles list
|
|
81
125
|
```
|
|
82
126
|
|
|
83
|
-
If a `default` profile exists but the probe failed, it
|
|
127
|
+
If a `default` profile exists but the probe failed, it is signed out or expired, not missing — for an OAuth profile, refresh it with `ax auth login` (see below) and re-run the probe rather than recreating it. Only create a profile when none exists, or an api-key profile has an invalid key. To create one, look for an existing key first: check `ARIZE_API_KEY` in the environment **or** the project's `.env` / `.env.local`.
|
|
84
128
|
|
|
85
|
-
- **A key is available** — the user
|
|
129
|
+
- **A key is available** — the user has an account and key, so skip the browser flow. Resolve the value, never print it: use `$ARIZE_API_KEY` if exported; if it is only in a dotenv file, load it from there for this one command. Then create an api-key profile and re-run the probe:
|
|
86
130
|
|
|
87
131
|
```bash
|
|
88
132
|
# if the key is only in a dotenv file, load it first (nothing is printed);
|
|
@@ -91,7 +135,7 @@ If a `default` profile exists but the probe failed, it's signed out or expired,
|
|
|
91
135
|
ax profiles create default --auth-method api-key --api-key "$ARIZE_API_KEY"
|
|
92
136
|
```
|
|
93
137
|
|
|
94
|
-
Reuse this key throughout — do **not** create a new one in Step 6. Only if the probe passes
|
|
138
|
+
Reuse this key throughout — do **not** create a new one in Step 6. Only if the probe passes: a stale or wrong-org `ARIZE_API_KEY` still looks present. If it fails, `ax profiles delete default`, take the OAuth path below, and let Step 6 create a key.
|
|
95
139
|
|
|
96
140
|
- **No key anywhere** — sign up or sign in with browser OAuth:
|
|
97
141
|
|
|
@@ -99,15 +143,15 @@ If a `default` profile exists but the probe failed, it's signed out or expired,
|
|
|
99
143
|
ax profiles create default --auth-method oauth --utm-params "utm_source=npmevals&utm_medium=cli&utm_campaign=prompt-first-onboarding"
|
|
100
144
|
```
|
|
101
145
|
|
|
102
|
-
Pass `--utm-params` verbatim on the two browser OAuth commands
|
|
146
|
+
Pass `--utm-params` verbatim on the two browser OAuth commands (`profiles create --auth-method oauth` and the `ax auth login` fallback) and on no other `ax` command — it tags a new sign-up with onboarding attribution.
|
|
103
147
|
|
|
104
148
|
Always pass the positional profile name `default`. Without it, the CLI prompts `profile name [default]:`, receives EOF from an agent-run command, and exits with `Goodbye!` without creating a profile.
|
|
105
149
|
|
|
106
150
|
The rest of this step applies only to the **browser OAuth** branch.
|
|
107
151
|
|
|
108
|
-
Creating an OAuth profile **is** the browser login flow. Treat it as an interactive
|
|
152
|
+
Creating an OAuth profile **is** the browser login flow. Treat it as an interactive handoff: it opens a browser, starts a localhost callback server such as `127.0.0.1:<port>/callback`, and waits for the redirect. The command must stay alive until the redirect lands and the CLI exits.
|
|
109
153
|
|
|
110
|
-
While the OAuth command is waiting, do not close its stdin, send Ctrl-C, `pkill` the AX process, start a second auth command, or run an auth probe. Any of these aborts the
|
|
154
|
+
While the OAuth command is waiting, do not close its stdin, send Ctrl-C, `pkill` the AX process, start a second auth command, or run an auth probe. Any of these aborts the browser flow. There is no exception: every sign-in path — existing account, Google/SSO, existing or new email/password — completes through the same callback, so the command always gets there.
|
|
111
155
|
|
|
112
156
|
After launching the OAuth command, tell the user:
|
|
113
157
|
|
|
@@ -117,21 +161,21 @@ A browser window is opening for Arize AX sign-in.
|
|
|
117
161
|
Sign in — or create a new account — in the browser. I'll continue
|
|
118
162
|
automatically once it completes.
|
|
119
163
|
|
|
120
|
-
Creating a
|
|
164
|
+
Creating a brand new account with email and password? Arize emails you a
|
|
121
165
|
validation link. Click it and finish in the browser; I'll wait.
|
|
122
166
|
```
|
|
123
167
|
|
|
124
|
-
Then wait for the command to exit
|
|
168
|
+
Then wait for the command to exit. Treat exit code `0`, or success output such as `Configuration saved to profile 'default'` or `Active profile set`, as the completion signal. A new email/password sign-up can wait a while — the user has to find the validation email — so a long wait is not a hang. Leave it alone.
|
|
125
169
|
|
|
126
|
-
Only after the OAuth command completes, re-run the probe from the top of this step. If it succeeds, continue. If the
|
|
170
|
+
Only after the OAuth command completes, re-run the probe from the top of this step. If it succeeds, continue. If the profile already existed and the probe returns an authentication error, it is signed out or expired — run the fallback:
|
|
127
171
|
|
|
128
172
|
```bash
|
|
129
173
|
ax auth login --utm-params "utm_source=npmevals&utm_medium=cli&utm_campaign=prompt-first-onboarding"
|
|
130
174
|
```
|
|
131
175
|
|
|
132
|
-
Handle `ax auth login` with the same rules: keep its callback server alive, wait for it to exit or print a success line, then rerun the probe. Do not run
|
|
176
|
+
Handle `ax auth login` with the same rules: keep its callback server alive, wait for it to exit or print a success line, then rerun the probe. Do not run it immediately after creating an OAuth profile; profile creation already logged in.
|
|
133
177
|
|
|
134
|
-
Do not treat `ax profiles show`
|
|
178
|
+
Do not treat `ax profiles show` as proof that OAuth completed; it shows configuration even when the user still needs to authenticate. Use it only to troubleshoot configuration, and never with `--expand`.
|
|
135
179
|
|
|
136
180
|
Run no other remote `ax` command (creating keys, listing all spaces, inspecting resources) until the probe succeeds.
|
|
137
181
|
|
|
@@ -143,9 +187,9 @@ List the spaces the authenticated profile can access:
|
|
|
143
187
|
ax spaces list --output json
|
|
144
188
|
```
|
|
145
189
|
|
|
146
|
-
If `ARIZE_SPACE_ID` is already set (environment or `.env` / `.env.local`), use it **only if it appears in that list** — that confirms the active profile can reach it
|
|
190
|
+
If `ARIZE_SPACE_ID` is already set (environment or `.env` / `.env.local`), use it **only if it appears in that list** — that confirms the active profile can reach it, since the app's key and your CLI profile may point at different spaces. If it is set but absent from the list, select from the list instead. Otherwise: one space returned, use it; several, ask the user which; none, guide the user to create one in the Arize AX UI (or with an organization ID if available), then re-list.
|
|
147
191
|
|
|
148
|
-
Capture the space's **ID
|
|
192
|
+
Capture the space's **ID**, not its display name, for `ARIZE_SPACE_ID`. `arize-otel` requires the ID; a name silently fails to route traces.
|
|
149
193
|
|
|
150
194
|
## Step 4: Choose what to trace
|
|
151
195
|
|
|
@@ -163,25 +207,27 @@ Which one?
|
|
|
163
207
|
|
|
164
208
|
If they pick **3**, go to [Step 4A](#step-4a-trace-this-coding-agent) and skip the folder inspection entirely. If they pick **2**, go straight to "Create a starter app" below — don't inspect anything. For **1**, continue.
|
|
165
209
|
|
|
166
|
-
|
|
210
|
+
You now know the path, so add the remaining tasks to your task list (see "Track progress").
|
|
211
|
+
|
|
212
|
+
For options **1** and **2**, one directory becomes **the app folder**: the app you instrument, which is not always the folder you were launched in. Establish it here and use it for every path in Steps 5 to 7.
|
|
167
213
|
|
|
168
214
|
### Detecting an app
|
|
169
215
|
|
|
170
|
-
Inspect a candidate folder to decide whether an app
|
|
216
|
+
Inspect a candidate folder to decide whether an app exists. Change no files while inspecting. Look for:
|
|
171
217
|
|
|
172
218
|
- Python: `pyproject.toml`, `requirements.txt`, `setup.py`, `Pipfile`, imports.
|
|
173
219
|
- TypeScript/JavaScript: `package.json`, lockfiles, `src`, `app`, `pages`, provider imports.
|
|
174
220
|
- Go: `go.mod`. Java: `pom.xml`, `build.gradle`, `build.gradle.kts`.
|
|
175
221
|
- Existing observability: `opentelemetry`, `TracerProvider`, `ARIZE_*`, `OTEL_*`, `OTLP_*`, Datadog, Honeycomb, Sentry, or other tracing.
|
|
176
|
-
- Agent framework: identify it by its import/package — e.g. `langchain` / `langgraph`, `llama_index`, `crewai`, `autogen`, `semantic_kernel`, `pydantic_ai`, `google.adk`, `dspy`, `agent_framework`, and others. **Route on the framework, not the provider client it wraps** — an `openai` or `anthropic` import inside a framework app is not
|
|
222
|
+
- Agent framework: identify it by its import/package — e.g. `langchain` / `langgraph`, `llama_index`, `crewai`, `autogen`, `semantic_kernel`, `pydantic_ai`, `google.adk`, `dspy`, `agent_framework`, and others. **Route on the framework, not the provider client it wraps** — an `openai` or `anthropic` import inside a framework app is not what to instrument; the framework almost certainly has its own integration (see Step 6).
|
|
177
223
|
|
|
178
|
-
In a monorepo, check the git root to
|
|
224
|
+
In a monorepo, check the git root to orient yourself, but only instrument apps in or below the folder you are scanning. If the project spans several languages, instrument each one (route each through its own integration page in Step 6).
|
|
179
225
|
|
|
180
226
|
Run this detection on the current folder first, then branch on what you found.
|
|
181
227
|
|
|
182
228
|
### If an app exists in the current folder
|
|
183
229
|
|
|
184
|
-
Summarize the detected stack and offer both routes — declining the local app must not be a dead end, because the user may
|
|
230
|
+
Summarize the detected stack and offer both routes — declining the local app must not be a dead end, because the user may have started from a folder above the app they care about:
|
|
185
231
|
|
|
186
232
|
```text
|
|
187
233
|
I found a <language>/<framework> app in this folder. I can trace that, or an
|
|
@@ -197,7 +243,7 @@ On **1**, the app folder is the current folder; continue to Step 5. On **2**, fo
|
|
|
197
243
|
|
|
198
244
|
### If no app exists in the current folder
|
|
199
245
|
|
|
200
|
-
Do not ask whether to instrument the empty folder, and do not assume a starter app
|
|
246
|
+
Do not ask whether to instrument the empty folder, and do not assume they want a starter app. Ask for a path first:
|
|
201
247
|
|
|
202
248
|
```text
|
|
203
249
|
I don't see an app in this folder. Give me the path to the app you want to
|
|
@@ -209,25 +255,25 @@ On a path, follow the next section. On "starter", go to "Create a starter app".
|
|
|
209
255
|
|
|
210
256
|
### If the user gives a path
|
|
211
257
|
|
|
212
|
-
- Expand `~` and resolve a relative path against the current folder, then confirm the
|
|
213
|
-
- Run the detection checklist on that folder. If it holds an app, that folder is the app folder from now on — summarize the stack
|
|
214
|
-
- If the path doesn't exist, or
|
|
215
|
-
- If your
|
|
258
|
+
- Expand `~` and resolve a relative path against the current folder, then confirm the absolute path back to the user before you scan it. A typo caught here beats an API key written into the wrong repo.
|
|
259
|
+
- Run the detection checklist on that folder. If it holds an app, that folder is the app folder from now on — summarize the stack and continue to Step 5.
|
|
260
|
+
- If the path doesn't exist, or has no app in it, say what you found and ask for another path. Don't quietly fall through to a starter app.
|
|
261
|
+
- If your permission or sandbox layer won't let you read and write outside the folder you were launched in, say so instead of half-instrumenting the app. Give the two ways out: the user grants access to that path, or starts you again inside the app folder.
|
|
216
262
|
|
|
217
263
|
### Create a starter app
|
|
218
264
|
|
|
219
|
-
For option 2, or when the user asks for a starter app
|
|
265
|
+
For option 2, or when the user asks for a starter app. Ask which folder to create it in — that becomes the app folder — then offer these choices:
|
|
220
266
|
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
267
|
+
1. OpenAI — Python or TypeScript (with a tool call)
|
|
268
|
+
2. Anthropic — Python (with a tool call; official AX auto-instrumentation)
|
|
269
|
+
3. LangChain with OpenAI — Python or TypeScript (a tool-using agent)
|
|
270
|
+
4. Vercel AI SDK with OpenAI — TypeScript (a tool-using agent)
|
|
225
271
|
|
|
226
|
-
If the user picks an unsupported pairing, explain the supported options and ask again.
|
|
272
|
+
If the user picks an unsupported pairing, explain the supported options and ask again. Use the numbers for the picker.
|
|
227
273
|
|
|
228
274
|
## Step 4A: Trace this coding agent
|
|
229
275
|
|
|
230
|
-
Only for
|
|
276
|
+
Only for tracing this coding agent. This traces the **agent harness**, not an app: every session the user runs, in every project on this machine. It edits files under their home directory, nothing in the current repo. When done, go to Step 8 — Steps 5 to 7 are app-only.
|
|
231
277
|
|
|
232
278
|
Installer harness names: `claude`, `codex`, `cursor`, `copilot`, `antigravity`, `gemini`, `kiro`, `opencode`, `omp` — note Cursor is `cursor`, not `cursor-agent`, and Antigravity is `antigravity`, not `agy`. Default to the agent you are running inside: state which you are and confirm, rather than asking.
|
|
233
279
|
|
|
@@ -235,13 +281,13 @@ Per-agent setup, including the Claude Code and Cursor marketplace-plugin routes,
|
|
|
235
281
|
|
|
236
282
|
### Check the home directory is writable first
|
|
237
283
|
|
|
238
|
-
Everything below writes under `~/.arize` — the credentials file, then the harness
|
|
284
|
+
Everything below writes under `~/.arize` — the credentials file, then the harness. Confirm that works before asking for approval, so a read-only home fails here rather than halfway through an install the user already approved:
|
|
239
285
|
|
|
240
286
|
```bash
|
|
241
287
|
mkdir -p ~/.arize/onboarding && touch ~/.arize/onboarding/.probe && rm -f ~/.arize/onboarding/.probe
|
|
242
288
|
```
|
|
243
289
|
|
|
244
|
-
If that fails, say so and stop this path — agent tracing needs a writable home and there is no workaround from inside the session. Instrumenting an app (options 1 and 2) does not, so offer that instead. If the
|
|
290
|
+
If that fails, say so and stop this path — agent tracing needs a writable home and there is no workaround from inside the session. Instrumenting an app (options 1 and 2) does not, so offer that instead. If the user was already told `~/.arize/onboarding` couldn't be used, this is the same cause; don't re-diagnose it.
|
|
245
291
|
|
|
246
292
|
### Get approval — this needs its own explicit yes
|
|
247
293
|
|
|
@@ -271,7 +317,7 @@ The installer reads credentials from a dotenv file, keeping the API key out of t
|
|
|
271
317
|
: > ~/.arize/onboarding/harness.env && chmod 600 ~/.arize/onboarding/harness.env
|
|
272
318
|
```
|
|
273
319
|
|
|
274
|
-
Add the Step 3 space ID as file contents, plus one `true` line per category the user **accepted**. Unattended installs capture nothing
|
|
320
|
+
Add the Step 3 space ID as file contents, plus one `true` line per category the user **accepted**. Unattended installs capture nothing by default, so omit the line for anything they declined:
|
|
275
321
|
|
|
276
322
|
```dotenv
|
|
277
323
|
ARIZE_SPACE_ID=<space-id-from-step-3>
|
|
@@ -282,15 +328,17 @@ ARIZE_LOG_TOOL_CONTENT=true
|
|
|
282
328
|
|
|
283
329
|
Do **not** set `ARIZE_PROJECT_NAME` — each harness defaults to its own project (`claude-code`, `codex`, …), which keeps two traced agents apart, and you read the real name back after installing. Then create the key into the same file, written atomically and never printed:
|
|
284
330
|
|
|
331
|
+
Replace <date> with the actual date, to avoid a name clash.
|
|
332
|
+
|
|
285
333
|
```bash
|
|
286
|
-
ax api-keys create --name "Coding agent tracing" --env-file ~/.arize/onboarding/harness.env
|
|
334
|
+
ax api-keys create --name "Coding agent tracing <date>" --env-file ~/.arize/onboarding/harness.env
|
|
287
335
|
```
|
|
288
336
|
|
|
289
337
|
**Skip that if `ARIZE_API_KEY` already existed in Step 2** — copy the existing value in without echoing it. At most one key, ever.
|
|
290
338
|
|
|
291
339
|
### Install, then delete the file
|
|
292
340
|
|
|
293
|
-
If a directory named `arize-offline/` sits beside this prompt file, install from it — no download, no remote script, so it is faster and
|
|
341
|
+
If a directory named `arize-offline/` sits beside this prompt file, install from it — no download, no remote script, so it is faster and less likely to be refused:
|
|
294
342
|
|
|
295
343
|
```bash
|
|
296
344
|
# keep `< /dev/null`: an installer too old for --non-interactive then fails
|
|
@@ -308,7 +356,7 @@ ARIZE_ENV_FILE=~/.arize/onboarding/harness.env \
|
|
|
308
356
|
<harness> --non-interactive < /dev/null
|
|
309
357
|
```
|
|
310
358
|
|
|
311
|
-
Delete the env file only
|
|
359
|
+
Delete the env file only after the install has run — a retry needs it, and so does the user if they run the command themselves:
|
|
312
360
|
|
|
313
361
|
```bash
|
|
314
362
|
rm -f ~/.arize/onboarding/harness.env
|
|
@@ -346,7 +394,7 @@ A go-ahead authorizes that one command, nothing wider. **Never** frame this as b
|
|
|
346
394
|
~/.arize/harness/install.sh status --json
|
|
347
395
|
```
|
|
348
396
|
|
|
349
|
-
Exit `0` means configured **and** hooks wired up; `1` nothing configured; `2` configured but hooks missing (the `unregistered` list names which to re-install). Continue only on `0`. Take the project name
|
|
397
|
+
Exit `0` means configured **and** hooks wired up; `1` nothing configured; `2` configured but hooks missing (the `unregistered` list names which to re-install). Continue only on `0`. Take the polling project name from `harnesses[].project_name` rather than guessing. The payload holds no secrets.
|
|
350
398
|
|
|
351
399
|
### Get the first traces
|
|
352
400
|
|
|
@@ -369,9 +417,9 @@ Steps 5 to 7 are for the app paths only; if you took Step 4A, go straight to Ste
|
|
|
369
417
|
|
|
370
418
|
Before creating any remote resource, writing files, or installing dependencies, present one consolidated plan and wait for approval.
|
|
371
419
|
|
|
372
|
-
For an existing app, cover:
|
|
420
|
+
For an existing app, cover: language and framework, package manager, LLM provider or agent framework, any existing tracing to preserve, the env file to update, the instrumentation packages and files that will change, whether a new API key will be created or the existing `ARIZE_API_KEY` reused, and the project name.
|
|
373
421
|
|
|
374
|
-
For a starter app, cover:
|
|
422
|
+
For a starter app, cover: provider and language, target folder, packages to install, and project name.
|
|
375
423
|
|
|
376
424
|
**State the app folder's absolute path in the plan whenever it isn't the folder you were launched in** — every file you touch lands there, and it's the one detail the user cannot verify from context.
|
|
377
425
|
|
|
@@ -397,11 +445,11 @@ The env file lives in the **app folder** from Step 4. Pick its name to match the
|
|
|
397
445
|
|
|
398
446
|
Do not read existing env file contents into chat. Preserve unrelated variables and never reveal their values.
|
|
399
447
|
|
|
400
|
-
Make sure the env file is git-ignored before writing the API key
|
|
448
|
+
Make sure the env file is git-ignored before writing the API key — check the `.gitignore` of the app folder's own repo. If it has one, confirm it covers the file (add `.env` / `.env.local` if not); for a starter app, create one. The API key must never be committed.
|
|
401
449
|
|
|
402
450
|
### Write the non-secret variables
|
|
403
451
|
|
|
404
|
-
Add these two lines to the env file you chose — as file contents, not shell commands. Create the file if needed; if a line
|
|
452
|
+
Add these two lines to the env file you chose — as file contents, not shell commands. Create the file if needed; if a line exists, update it in place rather than duplicating. Neither is secret. Skip `ARIZE_SPACE_ID` if it already holds the value you are using.
|
|
405
453
|
|
|
406
454
|
```dotenv
|
|
407
455
|
ARIZE_SPACE_ID=<space-id>
|
|
@@ -414,10 +462,10 @@ An exported variable beats this file — both `dotenv` implementations leave an
|
|
|
414
462
|
|
|
415
463
|
**Skip this entirely if `ARIZE_API_KEY` was already present and validated in Step 2** — reuse it and leave its env value untouched. Only create a key when you authenticated with browser OAuth and the app has no key yet.
|
|
416
464
|
|
|
417
|
-
Create the key and write it into the env file in one step
|
|
465
|
+
Create the key and write it into the env file in one step. Replace <date> with the actual date, to avoid a name clash.
|
|
418
466
|
|
|
419
467
|
```bash
|
|
420
|
-
ax api-keys create --name "Local Arize AX tracing" --env-file .env
|
|
468
|
+
ax api-keys create --name "Local Arize AX tracing <date>" --env-file .env
|
|
421
469
|
```
|
|
422
470
|
|
|
423
471
|
`--env-file` writes `ARIZE_API_KEY` atomically and **never prints it** — no temp file, no secret in your terminal or chat. Use `.env.local` if that's the app's convention, and always pass the app folder's path (`--env-file /path/to/app/.env`) — a bare `.env` would leave a stray key file in the wrong directory. The file is created if missing, an existing `ARIZE_API_KEY` is replaced in place, and other variables are preserved.
|
|
@@ -429,13 +477,13 @@ If key creation fails, have the user create one in the Arize AX UI and add it to
|
|
|
429
477
|
Handle the LLM provider's own key (e.g. `OPENAI_API_KEY`, `ANTHROPIC_API_KEY`) in the same env file — the app can't make a call or produce traces without it:
|
|
430
478
|
|
|
431
479
|
- **Existing app:** confirm the key is already present. If it's missing, ask the user for it and add it without echoing the value.
|
|
432
|
-
- **New starter app:** add a placeholder line for the provider key
|
|
480
|
+
- **New starter app:** add a placeholder line for the provider key (e.g. `OPENAI_API_KEY=`) to the env file, so the user knows what to fill in. Tell them to set it before running; if they share it now, fill it in without echoing the value.
|
|
433
481
|
|
|
434
482
|
### Add instrumentation
|
|
435
483
|
|
|
436
|
-
**Prefer the
|
|
484
|
+
**Prefer the `arize-instrumentation` skill.** Invoke it by name if it is in your skill list. If you installed the skills yourself in Step 1, read `arize-instrumentation/SKILL.md` from the path that install printed. Either way, follow the skill as your source of truth and **skip the rest of this section**. Use the docs fallback below only if you have no skill.
|
|
437
485
|
|
|
438
|
-
**Route by the framework you detected — search the index for *its* name and follow that page.** Look up the framework
|
|
486
|
+
**Route by the framework you detected — search the index for *its* name and follow that page.** Look up the framework from Step 4, not the provider it wraps, in the index at https://arize.com/docs/ax/integrations (machine-readable: https://arize.com/docs/llms.txt). Most agent frameworks have a page even when they are not in the shortcuts below, so search the index before concluding one is unsupported — it lists every supported provider and framework with the exact setup. Don't force a stack onto a listed framework's setup, and don't wander into unrelated Arize docs once you are on the right page.
|
|
439
487
|
|
|
440
488
|
Common shortcuts:
|
|
441
489
|
|
|
@@ -449,8 +497,8 @@ Common shortcuts:
|
|
|
449
497
|
**Only if the framework genuinely has no page in the index**, fall back in this order — never hand-roll a raw OpenTelemetry `TracerProvider` + OTLP exporter when a helper exists:
|
|
450
498
|
|
|
451
499
|
1. Use a framework-specific OpenInference instrumentor if one exists (`openinference-instrumentation-<name>` / `@arizeai/openinference-instrumentation-<name>`), wired up with `arize-otel` per the manual-instrumentation guide; install it unpinned.
|
|
452
|
-
1. Instrument the underlying provider (OpenAI, Anthropic, Bedrock, …) with its instrumentor **only if the framework calls the provider SDK directly**. Many agent frameworks
|
|
453
|
-
1. Otherwise instrument manually by code — follow https://arize.com/docs/ax/instrument/manual-instrumentation#by-code — or stop and ask the user if you still
|
|
500
|
+
1. Instrument the underlying provider (OpenAI, Anthropic, Bedrock, …) with its instrumentor **only if the framework calls the provider SDK directly**. Many agent frameworks drive the model through their own client layer and emit their own OpenTelemetry spans — a provider instrumentor captures **no traces** for those. Never use it as a blind fallback just because you recognize an `openai`/`anthropic` client.
|
|
501
|
+
1. Otherwise instrument manually by code — follow https://arize.com/docs/ax/instrument/manual-instrumentation#by-code — or stop and ask the user if you still cannot determine a setup. `arize-otel` gives you a tracer, not LLM spans: **set the OpenInference span kind on every span** (`LLM` for a model call, `TOOL` for a tool, `AGENT` for the loop, `CHAIN` for a step), **and set the span status** — `ERROR` with the exception recorded on failure, `OK` otherwise. Without the kind, AX cannot read the spans as LLM calls: Step 7's count looks like success while the trace shows no input, output, model, or token counts. Without the status, failed calls look successful and errors never surface in AX.
|
|
454
502
|
|
|
455
503
|
For existing apps:
|
|
456
504
|
|
|
@@ -462,27 +510,27 @@ For existing apps:
|
|
|
462
510
|
|
|
463
511
|
For starter apps:
|
|
464
512
|
|
|
465
|
-
- Every starter
|
|
513
|
+
- Every starter must make a real LLM call that invokes at least one tool (e.g. a calculator or a canned lookup), with the model deciding to call it and the result fed back for a final answer — so the first traces show a multi-step trajectory, not one flat span.
|
|
466
514
|
- Agent frameworks (LangChain, Vercel AI SDK): define the tool the framework's way; the instrumentor captures the tool and agent spans automatically.
|
|
467
|
-
- Plain providers (OpenAI, Anthropic): use the provider's native tool/function calling, and add tool spans with the OpenInference decorators — `@tracer.tool` on the tool function and `@tracer.agent` on the top-level loop. These decorators
|
|
515
|
+
- Plain providers (OpenAI, Anthropic): use the provider's native tool/function calling, and add tool spans with the OpenInference decorators — `@tracer.tool` on the tool function and `@tracer.agent` on the top-level loop. These decorators need the tracer wrapped in `OITracer` from `openinference-instrumentation` — the raw `arize.otel.register()` tracer does not expose them and `@tracer.tool` raises `AttributeError`; see https://arize.com/docs/ax/instrument/manual-instrumentation.
|
|
468
516
|
- Ensure short-lived scripts flush/shut down the tracer provider before exit, or spans won't export.
|
|
469
517
|
- Give a clear run command and note which provider key env var is needed.
|
|
470
518
|
|
|
471
519
|
### Package guidance
|
|
472
520
|
|
|
473
|
-
Install into the app's existing environment — the one in the app folder, using its virtualenv if it has one — exactly the packages the
|
|
521
|
+
Install into the app's existing environment — the one in the app folder, using its virtualenv if it has one — exactly the packages the framework's integration page lists. Instrumentor names and peer dependencies differ per framework, so follow that page rather than copying from another stack or guessing versions.
|
|
474
522
|
|
|
475
523
|
## Step 7: Run the app and poll for the first trace
|
|
476
524
|
|
|
477
525
|
First confirm the LLM provider's API key is set in the env file (from Step 6); a missing provider key is the most common reason the run produces no traces.
|
|
478
526
|
|
|
479
|
-
If you created a starter app
|
|
527
|
+
If you created a starter app, offer to run it:
|
|
480
528
|
|
|
481
529
|
```text
|
|
482
530
|
Your starter app is ready. Want me to run it for you?
|
|
483
531
|
```
|
|
484
532
|
|
|
485
|
-
If they say yes, run it
|
|
533
|
+
If they say yes, run it and poll. If they say no — or you instrumented their existing app — tell them the exact run command and ask them to run it. When the app folder is not the folder they are sitting in, lead with the `cd` so the command works as pasted:
|
|
486
534
|
|
|
487
535
|
```text
|
|
488
536
|
Run your app with:
|
|
@@ -494,9 +542,9 @@ It should make at least one LLM call. I'll poll Arize AX and let you know as
|
|
|
494
542
|
soon as your first traces arrive.
|
|
495
543
|
```
|
|
496
544
|
|
|
497
|
-
Once the app has run
|
|
545
|
+
Once the app has run, poll for spans. Pass the project name and space ID as literal arguments — do not use `$ARIZE_PROJECT_NAME`/`$ARIZE_SPACE_ID`, which live in the env file and are not exported to your shell. Query every ~15 seconds for up to ~3 minutes.
|
|
498
546
|
|
|
499
|
-
Span bodies can contain prompts, completions, tool arguments, and user data, so don't dump them into chat. Pipe the export through a counter that
|
|
547
|
+
Span bodies can contain prompts, completions, tool arguments, and user data, so don't dump them into chat. Pipe the export through a counter that shows only how many spans arrived:
|
|
500
548
|
|
|
501
549
|
```bash
|
|
502
550
|
ax spans export "<project-name>" --space "<space-id>" --limit 5 --stdout \
|
|
@@ -505,7 +553,7 @@ ax spans export "<project-name>" --space "<space-id>" --limit 5 --stdout \
|
|
|
505
553
|
|
|
506
554
|
A non-zero count confirms traces are arriving. If you need to inspect a span to debug, write the export to a file outside the repo and read only the fields you need — never paste raw span bodies into chat.
|
|
507
555
|
|
|
508
|
-
This uses the CLI profile from Step 2
|
|
556
|
+
This uses the CLI profile from Step 2, OAuth or api-key alike. If the export fails to authenticate, the profile is not valid; re-run the Step 2 probe and re-authenticate, or have the user open the project in the Arize AX UI to confirm traces.
|
|
509
557
|
|
|
510
558
|
- When spans come back, stop polling and continue to Step 8.
|
|
511
559
|
- On timeout, do not fail silently. Tell the user no traces arrived yet, and give likely causes: app didn't make an LLM call, tracing initialized after the client was created, a short-lived script exited before flushing spans, the wrong space/project/env file, or **an exported `ARIZE_*` variable overriding the env file** (`env | grep ARIZE_`). Offer to re-check once they've run it again.
|
|
@@ -514,16 +562,16 @@ Do not fabricate trace results. Only report traces the export command actually r
|
|
|
514
562
|
|
|
515
563
|
## Step 8: Report the first traces with a link
|
|
516
564
|
|
|
517
|
-
Once spans arrive, report the span count and give the user a deep link to the project in Arize AX. Point them at the UI to explore the
|
|
565
|
+
Once spans arrive, report the span count and give the user a deep link to the project in Arize AX. Point them at the UI to explore the traces rather than printing span bodies into chat.
|
|
518
566
|
|
|
519
|
-
Build that link with the
|
|
567
|
+
Build that link with the **`arize-link` skill**: invoke it by name, or read `arize-link/SKILL.md` from the path a Step 1 install printed, and follow it. It owns the URL format and the `ax` commands that find the organization and project IDs; you have the project name and space ID from Step 3.
|
|
520
568
|
|
|
521
569
|
```text
|
|
522
570
|
Your first traces are in Arize AX. Open project `<ARIZE_PROJECT_NAME>` here:
|
|
523
571
|
<project link>
|
|
524
572
|
```
|
|
525
573
|
|
|
526
|
-
If
|
|
574
|
+
If you have no `arize-link` skill at all, don't invent a URL structure: fall back to `https://app.arize.com/` plus instructions to select the project.
|
|
527
575
|
|
|
528
576
|
## Step 9: Point at docs
|
|
529
577
|
|
|
@@ -540,7 +588,8 @@ You're set up. To go further:
|
|
|
540
588
|
If you took Step 4A, add the controls that matter for agent tracing:
|
|
541
589
|
|
|
542
590
|
```text
|
|
543
|
-
- Your agent's tracing page:
|
|
591
|
+
- Your agent's tracing page:
|
|
592
|
+
https://arize.com/docs/ax/integrations/platforms/<agent>/<agent>-tracing
|
|
544
593
|
- Check it's still wired up: ~/.arize/harness/install.sh status
|
|
545
594
|
- Pause it: set ARIZE_TRACE_ENABLED=false in <agent>'s settings
|
|
546
595
|
- Remove it: ~/.arize/harness/install.sh uninstall <installer-name>
|
|
@@ -548,11 +597,12 @@ If you took Step 4A, add the controls that matter for agent tracing:
|
|
|
548
597
|
|
|
549
598
|
## Critical rules
|
|
550
599
|
|
|
600
|
+
- Keep your task list current if you have one (see "Track progress"): one task per milestone, exactly one in progress, complete only when done, never mentioned in chat.
|
|
551
601
|
- Get the user's approval (Step 5) before creating AX resources, editing files, or installing dependencies.
|
|
552
602
|
- Authenticate the CLI (Step 2) before any other `ax` call; env vars alone don't. Never interrupt a waiting OAuth command — no stdin close, Ctrl-C, `pkill`, or probe. A slow wait is not a hang.
|
|
553
603
|
- Never print, log, or summarize secrets in chat — API keys, env-file contents/values, or span bodies (prompts, completions, tool args, user data) — and never read env files into chat. Only report traces a command actually returned.
|
|
554
604
|
- One AX API key, ever: reuse a **validated** `ARIZE_API_KEY`, else a **single** `ax api-keys create --env-file <file>` into an already-git-ignored file.
|
|
555
|
-
- Ambient `ARIZE_*` variables are claims to verify, not facts to adopt: probe a found API key before reusing it, and remember
|
|
605
|
+
- Ambient `ARIZE_*` variables are claims to verify, not facts to adopt: probe a found API key before reusing it, and remember an exported variable overrides the env file you write.
|
|
556
606
|
- Everything you write goes in the **app folder** (Step 4), which may not be the folder you were launched in. Name its absolute path in the plan; pass it explicitly to `--env-file` and the run command.
|
|
557
607
|
- Write the space **ID** (not its name) to `ARIZE_SPACE_ID`, or traces won't route; never create an AX project explicitly (it's made on first ingestion).
|
|
558
608
|
- Initialize tracing before LLM clients exist; flush before short-lived scripts exit. Vercel AI SDK v7 also needs Node.js 22+, `@ai-sdk/otel` registered, and per-call `experimental_telemetry`.
|
package/package.json
CHANGED
package/start.ps1
CHANGED
|
@@ -241,7 +241,7 @@ try {
|
|
|
241
241
|
# where embedded double quotes would not.
|
|
242
242
|
# No "in this project": this may well be run from a home directory, and Step 4 is
|
|
243
243
|
# what scopes the work. Keep this wording in step with cli.js and start.sh.
|
|
244
|
-
$seed = "Read the file '$promptFile'
|
|
244
|
+
$seed = "Read the file '$promptFile' in full with your file-reading tool, not a shell command, then follow it to set up Arize AX tracing, walking me through each step and asking me questions as needed."
|
|
245
245
|
|
|
246
246
|
if (-not $chosen) {
|
|
247
247
|
# Addressed to the person, two lines, recommendation first - see the note in cli.js.
|
package/start.sh
CHANGED
|
@@ -272,7 +272,7 @@ fi
|
|
|
272
272
|
|
|
273
273
|
# No "in this project": this may well be run from a home directory, and Step 4 is
|
|
274
274
|
# what scopes the work. Keep this wording in step with cli.js and start.ps1.
|
|
275
|
-
SEED="Read the file '$PROMPT_FILE'
|
|
275
|
+
SEED="Read the file '$PROMPT_FILE' in full with your file-reading tool, not a shell command, then follow it to set up Arize AX tracing, walking me through each step and asking me questions as needed."
|
|
276
276
|
|
|
277
277
|
if [[ -z "$CHOSEN" ]]; then
|
|
278
278
|
# Addressed to the person, two lines, recommendation first — see the note in cli.js.
|
package/vendor/MANIFEST
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
a9cede65f98d3415b13e1c6a437d0fdee27286bfff0186dd72bb65d9869b7864 LICENSE-coding-harness-tracing
|
|
2
2
|
62f22742b58a1a33014a2b6b706588a8d7e2a88ae7bd1a6ebe8c992928483775 certifi-2026.7.22-py3-none-any.whl
|
|
3
|
-
|
|
3
|
+
e6269ca4232d31c6e3e1cbfc00b4958120378530c26253920a80bcb6ab9c593c coding_harness_tracing-0.1.0-py3-none-any.whl
|
|
4
4
|
aca14e09ca99c253d2a142e2d5c5c38c8c7cf90840d3cb10b241c29929871fa4 harness-install.bat
|
|
5
5
|
a7ba18a98280e236b20ab4cb9f8a4006068b454586a2af50bf23fb0dbe70583d harness-install.sh
|
|
6
6
|
b81ee9561e9ca4004139c6cbba3a238c32b03e4894671e181b671e8cb8425d61 python_dotenv-1.2.1-py3-none-any.whl
|
|
Binary file
|
|
@@ -1,11 +1,11 @@
|
|
|
1
1
|
{
|
|
2
2
|
"repo": "https://github.com/Arize-ai/coding-harness-tracing",
|
|
3
3
|
"ref": "main",
|
|
4
|
-
"resolvedSha": "
|
|
4
|
+
"resolvedSha": "054e4b8ff444ba4ee1ba2fa74c036999b6986829",
|
|
5
5
|
"source": "git",
|
|
6
6
|
"wheel": "coding_harness_tracing-0.1.0-py3-none-any.whl",
|
|
7
7
|
"wheelVersion": "0.1.0",
|
|
8
|
-
"wheelSha256": "
|
|
8
|
+
"wheelSha256": "e6269ca4232d31c6e3e1cbfc00b4958120378530c26253920a80bcb6ab9c593c",
|
|
9
9
|
"files": [
|
|
10
10
|
"LICENSE-coding-harness-tracing",
|
|
11
11
|
"MANIFEST",
|