ax 0.2.2 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +32 -0
- package/LICENSE +21 -0
- package/README.md +403 -0
- package/dist/api/audit.d.ts +47 -0
- package/dist/api/deep-journey.d.ts +90 -0
- package/dist/api/platform.d.ts +139 -0
- package/dist/api/shared.d.ts +14 -0
- package/dist/api/skills.d.ts +25 -0
- package/dist/contract/index.d.ts +22 -0
- package/dist/contract/types.gen.d.ts +3270 -0
- package/dist/contract/version.d.ts +2 -0
- package/dist/index.cjs +600 -0
- package/dist/index.d.ts +4 -0
- package/dist/index.js +563 -0
- package/dist/main.cjs +597 -0
- package/package.json +70 -23
- package/ReadMe.md +0 -15
- package/ax.coffee +0 -38
- package/ax.js +0 -65
package/.env.example
ADDED
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
# ora platform API (agentic journeys / runs). Copy to `.env` and fill in.
|
|
2
|
+
# The public commands (audit, webmcp-audit, deep-journey, skill) use the public
|
|
3
|
+
# ora.ai host and need none of these — `ax audit <url>` works with zero
|
|
4
|
+
# configuration, and so does `ax webmcp-audit http://localhost:3000`.
|
|
5
|
+
|
|
6
|
+
# Secret API key (looks like ora_sk_...). Required by `journey`, which exchanges
|
|
7
|
+
# it for a short-lived bearer token. `webmcp-audit` sends it as a bearer token to
|
|
8
|
+
# lift the anonymous ingest limits (10/min, 20 per 24h per IP); optional there.
|
|
9
|
+
ORA_API_KEY=
|
|
10
|
+
|
|
11
|
+
# Platform API base. Defaults to staging if unset.
|
|
12
|
+
ORA_PLATFORM_URL=https://api.agentfront.sh
|
|
13
|
+
|
|
14
|
+
# Public API base, used by audit, webmcp-audit, deep-journey and skill.
|
|
15
|
+
# Defaults to https://ora.ai if unset. Point it at a local ora to test against
|
|
16
|
+
# one (e.g. http://localhost:3000).
|
|
17
|
+
# ORA_API_URL=https://ora.ai
|
|
18
|
+
|
|
19
|
+
# Tunnel command for auditing a local target (same as --tunnel-cmd):
|
|
20
|
+
# a command that exposes the local server and prints its public https URL.
|
|
21
|
+
# ORA_TUNNEL_CMD='ngrok http 3000 --log stdout'
|
|
22
|
+
|
|
23
|
+
# ora-issued scan API key for `ax audit` (same as --api-key): exempts the
|
|
24
|
+
# caller from the scan rate limits (30/day + 6 force/day + 10/min burst).
|
|
25
|
+
# Issued manually by ora — no self-serve signup. Leave unset for normal use.
|
|
26
|
+
# `ax deep-journey` also accepts it as a partner-key fallback.
|
|
27
|
+
# ORA_SCAN_API_KEY=
|
|
28
|
+
|
|
29
|
+
# ora-issued partner API key for `ax deep-journey` (same as --api-key):
|
|
30
|
+
# unlocks free-text --task runs and the 1000 runs/24h keyed allowance
|
|
31
|
+
# (no per-target cap, no burst guard). Issued manually by ora.
|
|
32
|
+
# ORA_PARTNER_API_KEY=
|
package/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 era labs
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
package/README.md
ADDED
|
@@ -0,0 +1,403 @@
|
|
|
1
|
+
# ax — the ora CLI
|
|
2
|
+
|
|
3
|
+
Score any site's agent readiness — and watch real AI agents navigate it. Powered by [ora](https://ora.ai)'s hosted APIs.
|
|
4
|
+
|
|
5
|
+
```
|
|
6
|
+
npx ax audit https://stripe.com
|
|
7
|
+
```
|
|
8
|
+
|
|
9
|
+
Five commands:
|
|
10
|
+
|
|
11
|
+
- **`audit <url>`** — run ora's hosted agent-readiness audit against a site: live progress, then a layered report of what passed, what's broken, and ora's ranked list of the highest-impact fixes (or `--json` for the raw contract payload). No account or API key needed. Gate CI with `--min-score`.
|
|
12
|
+
- **`webmcp-audit <url>`** — audit a page's `document.modelContext` tools in a real browser. Works on any http(s) URL, localhost included, so you can check a surface before you publish it. Measures the page the way ora's capture worker does and has ora score it; nothing is stored, published, or ranked.
|
|
13
|
+
- **`deep-journey <url>`** — run a real AI agent at a site on one of ora's curated tasks through the public journey API: no key, no workspace. A partner API key unlocks free-text tasks (`--task`) and a larger allowance.
|
|
14
|
+
- **`journey "<intent>"`** — send a real AI agent (claude-code, codex, …) at a site and watch its navigation live as a boxed node-graph, then get the scored insight, tokens, and cost. Requires an `ORA_API_KEY`.
|
|
15
|
+
- **`skill [name]`** — list, print, or install ora's agent skills, digest-verified from the public registry.
|
|
16
|
+
|
|
17
|
+
## audit
|
|
18
|
+
|
|
19
|
+
```
|
|
20
|
+
ax audit <url> [--min-score n] [--max-age s] [--force] [--tunnel-cmd c] [--api-key k] [--json] [--show-passing] [--show-skipped]
|
|
21
|
+
```
|
|
22
|
+
|
|
23
|
+
```
|
|
24
|
+
stripe.com 78/100 B+
|
|
25
|
+
Stripe offers strong developer resource discoverability, but lacks an OpenAPI specification.
|
|
26
|
+
|
|
27
|
+
Discovery ██████░░░░ 6/9 passed · 1 skipped
|
|
28
|
+
✗ Issues (2)
|
|
29
|
+
┌───┬──────────────┬───────────────────────────┬──────────────────────────────┐
|
|
30
|
+
│ │ Check │ What's wrong │ How to fix │
|
|
31
|
+
├───┼──────────────┼───────────────────────────┼──────────────────────────────┤
|
|
32
|
+
│ ✗ │ sitemap.md │ status 404 │ Publish /sitemap.md listing… │
|
|
33
|
+
├───┼──────────────┼───────────────────────────┼──────────────────────────────┤
|
|
34
|
+
│ ⚠ │ robots.txt │ partially restricted │ Allow AI crawlers in robots… │
|
|
35
|
+
└───┴──────────────┴───────────────────────────┴──────────────────────────────┘
|
|
36
|
+
|
|
37
|
+
Top fixes (ranked by ora)
|
|
38
|
+
1. sitemap-md ≈ +4 pts
|
|
39
|
+
2. robots-txt ≈ +1.2 pts
|
|
40
|
+
|
|
41
|
+
cached result (43 min old) · pass --force for a fresh scan
|
|
42
|
+
Full report: https://ora.ai/stripe.com
|
|
43
|
+
```
|
|
44
|
+
|
|
45
|
+
| Flag | Effect |
|
|
46
|
+
|---|---|
|
|
47
|
+
| `--min-score <n>` | Exit `1` when the score is below `n` (0-100) — the CI gate |
|
|
48
|
+
| `--max-age <s>` | Accept a cached result up to `s` seconds old (server default 6h, clamped to 1-24h) |
|
|
49
|
+
| `--force` | Bypass the cache and rescan — spends the stricter 6/day force budget |
|
|
50
|
+
| `--tunnel-cmd <c>` | Expose a local target through your own tunnel command and audit the public URL it prints (also read from `ORA_TUNNEL_CMD`) |
|
|
51
|
+
| `--api-key <k>` | ora-issued scan API key — lifts every scan rate limit (also read from `ORA_SCAN_API_KEY`) |
|
|
52
|
+
| `--json` | The raw ora audit payload on stdout, exactly as the API served it |
|
|
53
|
+
| `--show-passing` | List every passing check, not just the per-layer summary bar |
|
|
54
|
+
| `--show-skipped` | Include not-applicable/pending checks with their reason |
|
|
55
|
+
|
|
56
|
+
The **Top fixes** list is ranked by ora server-side (non-bonus fixes first, then estimated score uplift) — the CLI renders it verbatim. All `≈ +N pts` figures are estimates. Check *tiers* (required / recommended / emerging) are advisory display metadata and never determine the score.
|
|
57
|
+
|
|
58
|
+
### CI gate
|
|
59
|
+
|
|
60
|
+
```yaml
|
|
61
|
+
- name: Agent-readiness gate
|
|
62
|
+
run: npx ax audit https://your-site.com --min-score 70
|
|
63
|
+
```
|
|
64
|
+
|
|
65
|
+
Exit codes are the contract:
|
|
66
|
+
|
|
67
|
+
| Code | Meaning |
|
|
68
|
+
|---|---|
|
|
69
|
+
| `0` | success (and score ≥ `--min-score` when given) |
|
|
70
|
+
| `1` | score below `--min-score` |
|
|
71
|
+
| `2` | usage error — bad flags, malformed URL, local target without a tunnel |
|
|
72
|
+
| `3` | API unreachable, timeout, or rate limit exhausted |
|
|
73
|
+
|
|
74
|
+
Budget notes: ora allows 30 scans + 6 `--force` scans per rolling 24h per IP (plus a 10/min burst limit). Results served from the freshness cache cost nothing, so a CI job that audits on every push stays well inside the budget — tune the window with `--max-age`, and reserve `--force` for verifying a fix you just deployed. High-volume callers can present an ora-issued scan API key (`--api-key` or `ORA_SCAN_API_KEY`), which exempts them from all of these limits; keys are issued manually by ora (no self-serve signup), and an unrecognized key silently falls back to the keyless limits rather than erroring. An auth-gated MCP target reports `mcpAuthRequired` and scores 0 as "could not evaluate"; `--min-score` deliberately skips the gate rather than failing on it.
|
|
75
|
+
|
|
76
|
+
### Auditing localhost (`--tunnel-cmd`)
|
|
77
|
+
|
|
78
|
+
A local dev server only exists on your machine, so ora can't reach it. The simplest option is to audit a publicly reachable deployment of the same code (e.g. a preview URL). To audit localhost itself, bring your own tunnel: pass `--tunnel-cmd` (or set `ORA_TUNNEL_CMD`) with a command that exposes the local server and prints its public `https://` URL. The CLI runs it, audits the URL it prints, and tears the tunnel down when done.
|
|
79
|
+
|
|
80
|
+
```
|
|
81
|
+
ax audit localhost:3000 --tunnel-cmd 'ngrok http 3000 --log stdout'
|
|
82
|
+
```
|
|
83
|
+
|
|
84
|
+
- **The CLI ships no tunnel vendor and never downloads executables at runtime** — any tunnel tool you already have works, as long as it prints its public URL to stdout or stderr.
|
|
85
|
+
- The result is stored as **ephemeral**: excluded from ora's rankings and deleted after a few days.
|
|
86
|
+
- Off-site checks (registry listings, brand search) usually fail for a throwaway tunnel hostname — the report says so. Use tunnel audits to iterate on your on-site surface, not to compare scores.
|
|
87
|
+
- Free tiers of some tunnel vendors serve an interstitial warning page to browser-like requests, which can distort what the scanner sees — prefer a vendor/plan that serves your origin directly.
|
|
88
|
+
|
|
89
|
+
## webmcp-audit
|
|
90
|
+
|
|
91
|
+
```
|
|
92
|
+
ax webmcp-audit <url> [--chrome-endpoint e] [--min-score n] [--api-key k] [--json] [--show-passing]
|
|
93
|
+
```
|
|
94
|
+
|
|
95
|
+
```bash
|
|
96
|
+
ax webmcp-audit http://localhost:3000
|
|
97
|
+
```
|
|
98
|
+
|
|
99
|
+
The dev-loop counterpart to the audit at [webmcp.ora.ai](https://webmcp.ora.ai): check the
|
|
100
|
+
`document.modelContext` tools your page registers *before* you ship it, and get the same answer
|
|
101
|
+
you will get once it is public.
|
|
102
|
+
|
|
103
|
+
Any `http(s)` URL works — a dev server, a staging host, or a site you do not own. A page with no
|
|
104
|
+
WebMCP surface is not an error: it comes back `absent` with the checks that cannot apply marked
|
|
105
|
+
`Not applicable`, which is a perfectly good way to see what the audit looks like before you have
|
|
106
|
+
written any tools.
|
|
107
|
+
|
|
108
|
+
**It uses a Chrome you already have.** WebMCP tools only exist inside a live page, so the audit
|
|
109
|
+
opens yours in a real browser. It finds your installed Chrome, starts a headless one on a throwaway
|
|
110
|
+
profile, and stops it when the audit finishes — nothing to set up, and this package never downloads
|
|
111
|
+
or bundles a browser. Set `CHROME_PATH` if yours lives somewhere unusual.
|
|
112
|
+
|
|
113
|
+
To drive a browser you started yourself, pass `--chrome-endpoint` (a `ws://` URL, an http origin,
|
|
114
|
+
or `host:port`) and start it with `--remote-debugging-port`. The command never attaches to a
|
|
115
|
+
browser you did not name — including anything already on port 9222 — because silently borrowing a
|
|
116
|
+
Chrome with the WebMCP flags on would score your page higher here than it scores on ora.
|
|
117
|
+
|
|
118
|
+
**Do not enable the WebMCP flags on a Chrome you point it at.** The audit asks whether a real
|
|
119
|
+
browser gives an agent WebMCP on your page. A Chrome started with `--enable-features=WebMCPTesting`
|
|
120
|
+
answers yes for every site, so a page can come back `active` here and `testing-only` on ora. The
|
|
121
|
+
report tells you when the browser answered for the page; where that actually moved a check, ora
|
|
122
|
+
annotates that check itself. Keep a flag-enabled Chrome for
|
|
123
|
+
[`@ora-ai/webmcp-verify`](https://www.npmjs.com/package/@ora-ai/webmcp-verify), which needs the API
|
|
124
|
+
present in order to call your tools.
|
|
125
|
+
|
|
126
|
+
**What it measures, and where.** The capture is local: three passes over your entry page, the same
|
|
127
|
+
passes and the same probe code ora's worker runs, plus a viewport screenshot at ora's 1280×720. As
|
|
128
|
+
ora does, it loads the page with images, media and webfonts refused — stylesheets still apply — so
|
|
129
|
+
the screenshot the grader sees is the one ora would have taken. That capture is
|
|
130
|
+
posted to ora, which runs the 16 checks, two model hops and the tool-selection simulation, and
|
|
131
|
+
streams back a score. The CLI scores nothing itself (design decision #1) — grade, findings,
|
|
132
|
+
category scores and the simulation are all rendered as ora sent them.
|
|
133
|
+
|
|
134
|
+
**Availability is a gate, not a score.** Before anything is graded, ora decides whether an
|
|
135
|
+
in-browser agent can use the page at all. A page that cannot comes back `not agent-ready` with the
|
|
136
|
+
reasons why, and no score or grade — the reasons are the answer.
|
|
137
|
+
|
|
138
|
+
Past that gate the score is four pillars — `shared-experience` (30), `task-completion` (25),
|
|
139
|
+
`tool-quality` (25), `trust` (20). It is **not** the 0–100 agent-readiness score from `ax audit`:
|
|
140
|
+
different inputs, different rubric, never comparable.
|
|
141
|
+
|
|
142
|
+
A pillar can read 0 because it earned nothing or because nothing in it could be measured; the
|
|
143
|
+
report marks the second case rather than leaving you to guess.
|
|
144
|
+
|
|
145
|
+
**Two check statuses mean "no score" and are not the same thing.** `Not applicable` means the check
|
|
146
|
+
had nothing to measure and your page is not charged for it. `Not measured` means it applied and
|
|
147
|
+
could not be measured — that counts against evidence coverage, and below the coverage gate ora
|
|
148
|
+
withholds the letter grade while still reporting the score. A withheld grade beside a real score is
|
|
149
|
+
normal, not an error.
|
|
150
|
+
|
|
151
|
+
**What leaves your machine.** Scoring happens on ora's side, so the capture is uploaded: the page
|
|
152
|
+
URL, every tool your page registers (names, descriptions, JSON Schemas, annotations), the WebMCP
|
|
153
|
+
evidence found in your page's own scripts, and **a 1280×720 screenshot of the page**. Point it at a
|
|
154
|
+
logged-in or seeded dev environment and that screenshot shows whatever was on screen. Your source
|
|
155
|
+
code is never read and never sent.
|
|
156
|
+
|
|
157
|
+
**Nothing is stored.** A local audit is scored and returned and then forgotten. It never reaches
|
|
158
|
+
the leaderboard, the badge, or your public scorecard — to appear there, publish and run the audit
|
|
159
|
+
on webmcp.ora.ai. Pass `--json` and pipe it to a file if you want to see exactly what was sent
|
|
160
|
+
back; the capture itself is what the report's header describes.
|
|
161
|
+
|
|
162
|
+
Anonymous use is rate-limited to 10/min and 20 per 24h per IP; an ora API key
|
|
163
|
+
(`--api-key` / `ORA_API_KEY`) lifts that.
|
|
164
|
+
|
|
165
|
+
### CI gate
|
|
166
|
+
|
|
167
|
+
```yaml
|
|
168
|
+
- run: npx ax webmcp-audit http://localhost:3000 --min-score 70
|
|
169
|
+
```
|
|
170
|
+
|
|
171
|
+
Any CI image with Chrome installed works; nothing to launch first.
|
|
172
|
+
|
|
173
|
+
Exit codes match `audit`: `0` ok · `1` below `--min-score` · `2` usage (including no debuggable
|
|
174
|
+
Chrome) · `3` API. The gate reads the score, not the grade — a withheld grade would otherwise let
|
|
175
|
+
everything through.
|
|
176
|
+
|
|
177
|
+
## deep-journey
|
|
178
|
+
|
|
179
|
+
```
|
|
180
|
+
ax deep-journey <url> [--intent id | --task text] [--agent id] [--api-key k] [--no-stream] [--json]
|
|
181
|
+
```
|
|
182
|
+
|
|
183
|
+
```
|
|
184
|
+
ax deep-journey stripe.com --intent pricing
|
|
185
|
+
ax deep-journey zapier.com --task "Find how to build a Slack → Sheets zap" # needs a partner key
|
|
186
|
+
```
|
|
187
|
+
|
|
188
|
+
Runs a real AI agent at a site through ora's **public** journey API — no key, no workspace needed for curated tasks. The agent's trajectory streams live as progress lines; the terminal output is ora's verdict, the billable step count, and the generated insight:
|
|
189
|
+
|
|
190
|
+
```
|
|
191
|
+
✓ task satisfied stripe.com
|
|
192
|
+
14 steps · 6 turns · 41.2s
|
|
193
|
+
|
|
194
|
+
The agent found the pricing page in two hops via llms.txt and correctly
|
|
195
|
+
summarised the standard and custom tiers.
|
|
196
|
+
- llms.txt was load-bearing: it linked /pricing directly
|
|
197
|
+
- no web search was needed; every path came from on-site links
|
|
198
|
+
|
|
199
|
+
For the 4-layer readiness audit, run: ax audit stripe.com
|
|
200
|
+
```
|
|
201
|
+
|
|
202
|
+
Anonymous callers run curated intents (`pricing`, `signup`, `api-docs`, `integrate`, `support`, …) — list them with `GET https://ora.ai/api/journey/intents`, and the accepted agents with `GET https://ora.ai/api/journey/agents`. The public caps are 100 runs per rolling 24h per target and 200 runs per 24h per IP (plus a 20/min burst limit); a target at its cap answers with the most recent stored run (`rate_limited: true`) instead of an error, so repeat CI invocations are cheap. Verdict and step count come from ora's contract as-is — the CLI never re-judges a run.
|
|
203
|
+
|
|
204
|
+
**Keyed tier:** an ora-issued partner API key (`--api-key`, or `ORA_PARTNER_API_KEY` / `ORA_SCAN_API_KEY` in the environment) unlocks `--task` — a free-text task of 4–300 characters that replaces the curated intent — and moves the caller to an allowance of 1000 runs per rolling 24h per key with no per-target cap and no burst guard. Keys are issued manually by ora (no self-serve signup). A wrong or unrecognized key with a *curated* intent silently degrades to the anonymous tier; `--task` without a recognized key is an error.
|
|
205
|
+
|
|
206
|
+
If the live stream goes quiet (no frames for 120s) the CLI does not abandon the run — it switches to polling the run detail until the server reports a terminal state.
|
|
207
|
+
|
|
208
|
+
| Flag | Effect |
|
|
209
|
+
|---|---|
|
|
210
|
+
| `--intent <id>` | Curated task id (server default when omitted) |
|
|
211
|
+
| `--task <text>` | Free-text task — needs a partner API key; mutually exclusive with `--intent` |
|
|
212
|
+
| `--api-key <k>` | ora partner API key; also read from `ORA_PARTNER_API_KEY` (then `ORA_SCAN_API_KEY`) |
|
|
213
|
+
| `--agent <id>` | Agent from the public roster (default: ora's pick) |
|
|
214
|
+
| `--no-stream` | Poll for the result instead of streaming the trajectory |
|
|
215
|
+
| `--json` | Print the terminal run detail (verdict, step_count, result) as JSON |
|
|
216
|
+
|
|
217
|
+
deep-journey exit codes: `0` run succeeded (any verdict) · `1` run failed · `2` usage error · `3` API unreachable / rate-limited.
|
|
218
|
+
|
|
219
|
+
This differs from `ax journey` (below), which drives the authenticated platform with free-text goals and renders the full trajectory graph; `deep-journey` is the zero-setup public path.
|
|
220
|
+
|
|
221
|
+
## skill
|
|
222
|
+
|
|
223
|
+
```
|
|
224
|
+
ax skill # list the registry
|
|
225
|
+
ax skill agent-ready-website # print a SKILL.md
|
|
226
|
+
ax skill agent-ready-website --install # write .claude/skills/<name>/SKILL.md
|
|
227
|
+
ax skill --json # the raw registry index
|
|
228
|
+
```
|
|
229
|
+
|
|
230
|
+
Skills come from ora's public registry (`https://ora.ai/.well-known/agent-skills/`) and every byte is verified against the registry's sha256 digest before it is printed or installed. Skill content is never bundled into this package. `--dir <path>` overrides the install directory, and `--json` prints the registry index as JSON instead of the formatted list.
|
|
231
|
+
|
|
232
|
+
## journey
|
|
233
|
+
|
|
234
|
+
```
|
|
235
|
+
ax journey "<intent>" [--domain d] [--harness h] [--model m] [--json]
|
|
236
|
+
```
|
|
237
|
+
|
|
238
|
+
```
|
|
239
|
+
ax journey "Find the API docs and how to authenticate" --domain stripe.com
|
|
240
|
+
```
|
|
241
|
+
|
|
242
|
+
Triggers a real agent run on ora's platform and renders the journey live — the whole graph repaints in place as the agent moves:
|
|
243
|
+
|
|
244
|
+
```
|
|
245
|
+
▶ stripe.com · claude-code
|
|
246
|
+
|
|
247
|
+
9 steps · 3 reasoning · 1 search
|
|
248
|
+
|
|
249
|
+
╭────────────────────────────────────────────────────────────────────────────────╮
|
|
250
|
+
│ ◆ Find the API docs and how to authenticate. Fetch the actual pages to verify. │
|
|
251
|
+
╰─┬──────────────────────────────────────────────────────────────────────────────╯
|
|
252
|
+
│ (…) The user is asking me to find the API documentation for Stripe…
|
|
253
|
+
│ ╭──────────────────────────────────────────────╮
|
|
254
|
+
╰──┤ 🔍 "Stripe API documentation authentication" │
|
|
255
|
+
╰─┬────────────────────────────────────────────╯
|
|
256
|
+
│ ╭─────────────────────────────────────────╮
|
|
257
|
+
├──┤ 🌐 docs.stripe.com/api/authentication ✓ │
|
|
258
|
+
│ ╰─────────────────────────────────────────╯
|
|
259
|
+
│ ╭───────────────────────────╮
|
|
260
|
+
╰──┤ 🌐 docs.stripe.com/apis ✓ │
|
|
261
|
+
╰───────────────────────────╯
|
|
262
|
+
|
|
263
|
+
────────────────────────────────────────────────────
|
|
264
|
+
✓ completed 9 turns · 30.8s · 119.1k tokens · $0.031
|
|
265
|
+
|
|
266
|
+
Visibility █████████████████████░ 95/100 ✓ intent satisfied
|
|
267
|
+
```
|
|
268
|
+
|
|
269
|
+
Edges come from ora's own per-step attribution (link-follows nest under the page that linked them; searches hang off the intent). `(…)` markers are the agent's reasoning, placed above the action it led to.
|
|
270
|
+
|
|
271
|
+
| Flag | Effect |
|
|
272
|
+
|---|---|
|
|
273
|
+
| `--domain <d>` | Site the agent targets (e.g. `stripe.com`) |
|
|
274
|
+
| `--harness <h>` | Agent harness: `claude-code` (default), `codex`, `openclaw`, `hermess`, `claude-agent`, `eve`, … |
|
|
275
|
+
| `--model <m>` | Model override (e.g. `claude-haiku-4-5-20251001` — cheap and good for demos) |
|
|
276
|
+
| `--json` | Full result as JSON: run metadata, usage, insight, and the raw trajectory |
|
|
277
|
+
|
|
278
|
+
Note: run-to-run variance is real — agents sometimes answer from search snippets without fetching, which scores differently. Run a few journeys before drawing conclusions.
|
|
279
|
+
|
|
280
|
+
Journey exit codes: `0` run outcome `success` · `1` non-success outcome · `2` any error.
|
|
281
|
+
|
|
282
|
+
### Setup
|
|
283
|
+
|
|
284
|
+
`journey` needs an ora platform API key (scopes `runs:read` + `runs:write`):
|
|
285
|
+
|
|
286
|
+
```sh
|
|
287
|
+
export ORA_API_KEY=ora_sk_... # your shell, CI secret store, etc.
|
|
288
|
+
ax journey "Find the pricing page" --domain example.com
|
|
289
|
+
```
|
|
290
|
+
|
|
291
|
+
For local development, copy `.env.example` to `.env` — the CLI reads a `.env` in the working directory on startup (exported variables win over the file).
|
|
292
|
+
|
|
293
|
+
## Library use
|
|
294
|
+
|
|
295
|
+
The same contract-typed client the CLI uses is importable:
|
|
296
|
+
|
|
297
|
+
```ts
|
|
298
|
+
import { audit } from "ax";
|
|
299
|
+
|
|
300
|
+
const { result } = await audit("stripe.com");
|
|
301
|
+
console.log(result.score, result.grade, result.topFixes);
|
|
302
|
+
```
|
|
303
|
+
|
|
304
|
+
`result` is the raw versioned audit payload (`AuditScanResult` / `AuditScoreResult`, generated from ora's OpenAPI spec). `fetchSkill` / `fetchSkillIndex` expose the digest-verified skill registry.
|
|
305
|
+
|
|
306
|
+
The public journey client is exported too — the same no-key path as `ax deep-journey`:
|
|
307
|
+
|
|
308
|
+
```ts
|
|
309
|
+
import { deepJourney, fetchJourneyAgents } from "ax";
|
|
310
|
+
|
|
311
|
+
const { agents, defaultId } = await fetchJourneyAgents();
|
|
312
|
+
const agent = agents.find((a) => a.id === defaultId)!;
|
|
313
|
+
const { detail } = await deepJourney("stripe.com", {
|
|
314
|
+
intentId: "pricing",
|
|
315
|
+
harness: agent.harness,
|
|
316
|
+
model: agent.model,
|
|
317
|
+
});
|
|
318
|
+
console.log(detail.verdict, detail.step_count, detail.result?.insight.summary);
|
|
319
|
+
```
|
|
320
|
+
|
|
321
|
+
`detail` is the contract's `JourneyRunDetail`; `fetchJourneyIntents` lists the curated intents. The same public caps apply — a capped target resolves with the most recent stored run (`cached: true` on the outcome), not an error.
|
|
322
|
+
|
|
323
|
+
## Environment variables
|
|
324
|
+
|
|
325
|
+
All configuration is environment-first: the consumer defines the variables, the CLI only reads them.
|
|
326
|
+
|
|
327
|
+
A `.env` in the working directory is read on startup — copy `.env.example` and fill it in. Anything already exported wins over the file, so a key set for your shell or by CI beats a stale `.env`.
|
|
328
|
+
|
|
329
|
+
| Var | Used by | Default | Purpose |
|
|
330
|
+
|---|---|---|---|
|
|
331
|
+
| `ORA_API_URL` | audit, deep-journey, skill | `https://ora.ai` | Public API base (no auth) |
|
|
332
|
+
| `ORA_PLATFORM_URL` | journey | `https://api.agentfront.sh` | Authenticated platform API base |
|
|
333
|
+
| `ORA_TUNNEL_CMD` | audit | — (optional) | Tunnel command for a local target (same as `--tunnel-cmd`) |
|
|
334
|
+
| `ORA_API_URL` | webmcp-audit | `https://ora.ai` | Also the ingest base for `webmcp-audit` |
|
|
335
|
+
| `ORA_API_KEY` | journey, webmcp-audit | — (required for journey) | Secret key (`ora_sk_…`); journey exchanges it for a bearer token, webmcp-audit sends it to lift the ingest rate limits |
|
|
336
|
+
| `ORA_SCAN_API_KEY` | audit, deep-journey | — (optional) | ora-issued scan API key; lifts every scan rate limit (issued manually by ora). deep-journey accepts it as a partner-key fallback |
|
|
337
|
+
| `ORA_PARTNER_API_KEY` | deep-journey | — (optional) | ora-issued partner API key; unlocks `--task` and the 1000/24h keyed allowance |
|
|
338
|
+
|
|
339
|
+
## Contract versioning
|
|
340
|
+
|
|
341
|
+
The audit types are generated from ora's served OpenAPI spec (`pnpm contract:gen`); the build is pinned to a contract version and CI fails when production drifts from the checked-in types. At runtime, the CLI prints one stderr warning when ora reports a newer contract than the build — upgrade with `npm i -g ax@latest`.
|
|
342
|
+
|
|
343
|
+
## Migrating from 0.1.x
|
|
344
|
+
|
|
345
|
+
- `ax scan` is now `ax audit` (no alias).
|
|
346
|
+
- `--json` prints the raw ora audit payload instead of the old reshaped envelope: stable check ids, layer ids, `grade`, `topFixes`, and `contractVersion` — key your scripts to those fields.
|
|
347
|
+
- Audit API failures now exit `3` (usage errors stay `2`).
|
|
348
|
+
|
|
349
|
+
## Development
|
|
350
|
+
|
|
351
|
+
```sh
|
|
352
|
+
pnpm install
|
|
353
|
+
pnpm test # vitest (watch)
|
|
354
|
+
pnpm test:ci # single run + coverage
|
|
355
|
+
pnpm typecheck
|
|
356
|
+
pnpm lint
|
|
357
|
+
pnpm contract:gen # regenerate src/contract/ from the served OpenAPI spec
|
|
358
|
+
pnpm build # tsup → dist/main.cjs (bin) + dist/index.* (library)
|
|
359
|
+
pnpm smoke # run the built binary the way a user would (needs build)
|
|
360
|
+
pnpm smoke:webmcp # webmcp-audit end to end: real Chrome + mock ingest (needs build)
|
|
361
|
+
pnpm verify:pack # check the tarball npm would publish (needs build)
|
|
362
|
+
node dist/main.cjs audit https://example.com
|
|
363
|
+
```
|
|
364
|
+
|
|
365
|
+
`webmcp-audit` is the one command whose core cannot be unit-tested — it needs a browser. Two
|
|
366
|
+
commands cover it, and CI runs both on every PR:
|
|
367
|
+
|
|
368
|
+
```sh
|
|
369
|
+
AX_WEBMCP_LIVE=1 pnpm test:capture # the three capture passes against a real Chrome
|
|
370
|
+
pnpm build && pnpm smoke:webmcp # the whole chain: browser → capture → ingest → report
|
|
371
|
+
```
|
|
372
|
+
|
|
373
|
+
Both launch their own headless Chrome and stop it again. In a container where Chrome's sandbox
|
|
374
|
+
cannot start, set `AX_WEBMCP_NO_SANDBOX=1` — never on a development machine, since the page being
|
|
375
|
+
audited is untrusted and the sandbox is what contains it.
|
|
376
|
+
|
|
377
|
+
Manual testing without burning the live rate limits:
|
|
378
|
+
|
|
379
|
+
```sh
|
|
380
|
+
node scripts/mock-scan-server.mjs &
|
|
381
|
+
ORA_API_URL=http://localhost:8799 node dist/main.cjs audit https://example.com
|
|
382
|
+
```
|
|
383
|
+
|
|
384
|
+
```sh
|
|
385
|
+
node scripts/mock-webmcp-ingest.mjs &
|
|
386
|
+
ORA_API_URL=http://localhost:8798 node dist/main.cjs webmcp-audit http://localhost:3000
|
|
387
|
+
```
|
|
388
|
+
|
|
389
|
+
The webmcp mock replays the checked-in real ingest fixture, and takes a failure mode as its second
|
|
390
|
+
argument — `shim`, `rate`, `scoring` or `stream` — so every error branch is reachable without the
|
|
391
|
+
live endpoint.
|
|
392
|
+
|
|
393
|
+
To exercise the published-package experience locally: `npm pack`, then `npx ./ora-ai-ax-<version>.tgz audit https://example.com`, or `pnpm link --global` and use `ax` directly.
|
|
394
|
+
|
|
395
|
+
## Releasing
|
|
396
|
+
|
|
397
|
+
Publishing runs from GitHub Actions only — **Actions → Release → Run workflow**,
|
|
398
|
+
pick a `patch`/`minor`/`major` bump. See [RELEASING.md](RELEASING.md) for the
|
|
399
|
+
runbook, required setup, and failure recovery.
|
|
400
|
+
|
|
401
|
+
## License
|
|
402
|
+
|
|
403
|
+
MIT
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
import type { AuditScanResult, AuditScoreResult } from "../contract";
|
|
2
|
+
/** Any failure to obtain a result from ora: network, HTTP, rate limit, timeout. */
|
|
3
|
+
export declare class AuditApiError extends Error {
|
|
4
|
+
}
|
|
5
|
+
/**
|
|
6
|
+
* The raw audit payload: scan-shaped from the stream's terminal event,
|
|
7
|
+
* score-shaped once deep-analysis polling has taken over.
|
|
8
|
+
*/
|
|
9
|
+
export type AuditResult = AuditScanResult | AuditScoreResult;
|
|
10
|
+
export interface AuditOutcome {
|
|
11
|
+
/** The raw `?format=audit` contract payload, untouched. */
|
|
12
|
+
result: AuditResult;
|
|
13
|
+
/**
|
|
14
|
+
* ora's one-line agentic verdict. Rides the stream's `summary_ready` event,
|
|
15
|
+
* not the audit payload, so it is carried alongside rather than injected.
|
|
16
|
+
*/
|
|
17
|
+
verdict?: string;
|
|
18
|
+
}
|
|
19
|
+
export interface AuditOptions {
|
|
20
|
+
/** Receives one-line progress updates while the scan streams and polls. */
|
|
21
|
+
progress?: (line: string) => void;
|
|
22
|
+
/** Base URL override; otherwise $ORA_API_URL, otherwise https://ora.ai. */
|
|
23
|
+
baseUrl?: string;
|
|
24
|
+
/** Abort when the stream is silent for this long (default 60s). */
|
|
25
|
+
idleMs?: number;
|
|
26
|
+
pollEveryMs?: number;
|
|
27
|
+
pollLimit?: number;
|
|
28
|
+
/** Freshness window in seconds (server clamps to [3600, 86400]). */
|
|
29
|
+
maxAgeSeconds?: number;
|
|
30
|
+
/** Bypass the freshness cache (spends the stricter 6/day force budget). */
|
|
31
|
+
force?: boolean;
|
|
32
|
+
/** Store the result as disposable (tunnel hosts are auto-classified). */
|
|
33
|
+
ephemeral?: boolean;
|
|
34
|
+
/**
|
|
35
|
+
* ora-issued scan API key (contract 1.10.0): exempts the caller from every
|
|
36
|
+
* scan-family rate limit. Falls back to $ORA_SCAN_API_KEY. Safe to send
|
|
37
|
+
* blind — the server degrades an unrecognized key to keyless, never a 401.
|
|
38
|
+
*/
|
|
39
|
+
apiKey?: string;
|
|
40
|
+
}
|
|
41
|
+
/**
|
|
42
|
+
* Audit `target` and resolve with ora's raw audit payload. When the stream
|
|
43
|
+
* ends with analysis still marked partial, keeps polling
|
|
44
|
+
* GET /api/score/{domain}?format=audit until it settles (complete or stuck)
|
|
45
|
+
* or the poll budget runs out.
|
|
46
|
+
*/
|
|
47
|
+
export declare function performAudit(target: string, options?: AuditOptions): Promise<AuditOutcome>;
|
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
import type { JourneyRun, JourneyRunDetail } from "../contract";
|
|
2
|
+
/** Any failure to obtain a run from ora: network, HTTP, rate limit, stream error. */
|
|
3
|
+
export declare class DeepJourneyApiError extends Error {
|
|
4
|
+
}
|
|
5
|
+
export interface JourneyIntentOption {
|
|
6
|
+
id: string;
|
|
7
|
+
label: string;
|
|
8
|
+
hint: string;
|
|
9
|
+
template: string;
|
|
10
|
+
}
|
|
11
|
+
export interface JourneyAgentOption {
|
|
12
|
+
id: string;
|
|
13
|
+
label: string;
|
|
14
|
+
variant: string;
|
|
15
|
+
harness: string;
|
|
16
|
+
model: string;
|
|
17
|
+
blurb?: string;
|
|
18
|
+
}
|
|
19
|
+
/** The per-target allowance the POST response advertises via X-RateLimit-*. */
|
|
20
|
+
export interface RunAllowance {
|
|
21
|
+
limit?: number;
|
|
22
|
+
remaining?: number;
|
|
23
|
+
/** Unix ms when the next per-target slot frees; absent when unknown. */
|
|
24
|
+
resetAtMs?: number;
|
|
25
|
+
}
|
|
26
|
+
export interface DeepJourneyOptions {
|
|
27
|
+
/** Curated intent id; the server default when omitted. */
|
|
28
|
+
intentId?: string;
|
|
29
|
+
/**
|
|
30
|
+
* Free-text task (the keyed custom arm, 4-300 chars server-side). Needs a
|
|
31
|
+
* partner API key; mutually exclusive with `intentId` (the command layer
|
|
32
|
+
* enforces the exclusivity, this client just sends whichever arm is set).
|
|
33
|
+
*/
|
|
34
|
+
task?: string;
|
|
35
|
+
/**
|
|
36
|
+
* ora-issued partner API key, sent as `Authorization: Bearer`. Unlocks
|
|
37
|
+
* free-text tasks and moves the caller to the 1000/24h per-key allowance
|
|
38
|
+
* (no per-target cap, no burst guard). Falls back to $ORA_PARTNER_API_KEY,
|
|
39
|
+
* then $ORA_SCAN_API_KEY (the shared partner registry serves both
|
|
40
|
+
* surfaces). Safe to send a wrong key with a curated intent - the server
|
|
41
|
+
* silently degrades to the keyless tier.
|
|
42
|
+
*/
|
|
43
|
+
apiKey?: string;
|
|
44
|
+
harness: string;
|
|
45
|
+
model: string;
|
|
46
|
+
/** Receives one-line progress updates while the run streams/polls. */
|
|
47
|
+
progress?: (line: string) => void;
|
|
48
|
+
/** Base URL override; otherwise $ORA_API_URL, otherwise https://ora.ai. */
|
|
49
|
+
baseUrl?: string;
|
|
50
|
+
/** Abort when the stream is silent for this long (default 120s). */
|
|
51
|
+
idleMs?: number;
|
|
52
|
+
/** Skip the SSE and poll GET /api/journey/runs/{id} instead. */
|
|
53
|
+
noStream?: boolean;
|
|
54
|
+
pollEveryMs?: number;
|
|
55
|
+
pollLimit?: number;
|
|
56
|
+
}
|
|
57
|
+
export interface DeepJourneyOutcome {
|
|
58
|
+
/** The projected run record from the trigger (or the cached latest run). */
|
|
59
|
+
record: JourneyRun;
|
|
60
|
+
/** True when the per-target cap answered with the stored latest run. */
|
|
61
|
+
cached: boolean;
|
|
62
|
+
/** Milliseconds until a per-target slot frees, on a cached answer. */
|
|
63
|
+
retryAfterMs?: number;
|
|
64
|
+
allowance: RunAllowance;
|
|
65
|
+
/** Terminal detail: status, verdict, step_count, and result once succeeded. */
|
|
66
|
+
detail: JourneyRunDetail;
|
|
67
|
+
/** The stream's terminal error message, when the run failed mid-stream. */
|
|
68
|
+
engineError?: string;
|
|
69
|
+
}
|
|
70
|
+
/** The curated intents a public run can execute. */
|
|
71
|
+
export declare function fetchJourneyIntents(baseUrl?: string): Promise<{
|
|
72
|
+
intents: JourneyIntentOption[];
|
|
73
|
+
defaultId: string;
|
|
74
|
+
}>;
|
|
75
|
+
/** The agents an anonymous run request accepts. */
|
|
76
|
+
export declare function fetchJourneyAgents(baseUrl?: string): Promise<{
|
|
77
|
+
agents: JourneyAgentOption[];
|
|
78
|
+
defaultId: string;
|
|
79
|
+
}>;
|
|
80
|
+
export declare function formatWait(ms: number): string;
|
|
81
|
+
/** Non-streaming run detail (verdict, step_count, result once succeeded). */
|
|
82
|
+
export declare function fetchRunDetail(base: string, runId: string): Promise<JourneyRunDetail>;
|
|
83
|
+
/**
|
|
84
|
+
* Run a deep journey against `target` and resolve with the terminal detail.
|
|
85
|
+
* Streams the trajectory by default (progress lines via `options.progress`);
|
|
86
|
+
* with `noStream`, polls the record instead. A per-target-capped trigger is
|
|
87
|
+
* NOT an error: ora answers with the most recent stored run for that target,
|
|
88
|
+
* and the outcome carries `cached: true` plus `retryAfterMs`.
|
|
89
|
+
*/
|
|
90
|
+
export declare function performDeepJourney(target: string, options: DeepJourneyOptions): Promise<DeepJourneyOutcome>;
|