agent-led-growth 0.6.2 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/plugin.json +1 -1
- package/CHANGELOG.md +5 -0
- package/README.md +3 -2
- package/bin/cli.js +68 -34
- package/package.json +1 -1
- package/skills/alg-test/SKILL.md +2 -2
- package/src/verify/index.js +14 -4
- package/src/verify/wizard.js +41 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "agent-led-growth",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.7.0",
|
|
4
4
|
"description": "Score: evals for agents as customers. Runs a real agent through signup, key and first call on your product, before you ship. Track, the analytics half, measures which agents convert after.",
|
|
5
5
|
"author": {
|
|
6
6
|
"name": "Kat Laszlo",
|
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,10 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## 0.7.0 (2026-10-23)
|
|
4
|
+
|
|
5
|
+
- `test` no longer stops when `alg.yml` has no `verify_call` and the product has no OpenAPI document. At a terminal it asks for a GET that needs a key (and the header that carries it, default `Authorization: Bearer {key}`), checks the answer, saves it to `alg.yml` and continues the run; Enter with nothing skips the check for that run. Answering "n" to an inferred call asks the same way instead of cancelling. Before, it printed `verify_call_invalid` and left the person to edit the file and start over.
|
|
6
|
+
- `test --skip-verify`: the signup runs and the agent's key is not checked, for CI and products with no API call to prove a key. The result passes when the agent saved a key, and says the key was not checked; `--json` carries `checker.skipped: true`. Without a terminal, `verify_call_invalid` names the flag.
|
|
7
|
+
|
|
3
8
|
## 0.6.0 (2026-10-22)
|
|
4
9
|
|
|
5
10
|
- One name. The command is `alg` (`npx agent-led-growth` still runs it); the config file is `alg.yml` and the run folder `.alg/`. The skills are `alg` and `alg-test`, the plugin `agent-led-growth`. `agent-ready`, `agent-ready.yml` and `.agent-ready/` are gone.
|
package/README.md
CHANGED
|
@@ -163,6 +163,7 @@ Commit `alg.yml` so CI reuses your answers. `test` needs Claude Code installed a
|
|
|
163
163
|
| --- | --- |
|
|
164
164
|
| `--check` | Show the plan, run nothing. Free. |
|
|
165
165
|
| `--yes` | Skip the confirmation; save an inferred `verify_call` to `alg.yml`. |
|
|
166
|
+
| `--skip-verify` | Test the signup only. The agent's key is not checked, so no `verify_call` is needed. |
|
|
166
167
|
| `--max-budget-usd <n>` | Model-use cap, default 5. Checked after each turn, so a run can end slightly above it. |
|
|
167
168
|
| `--max-turns <n>` | Agent turn budget, default 40. |
|
|
168
169
|
| `--inbox <address>` | An inbox you relay mail from, for products that email a code. Or set `AGENTMAIL_API_KEY` and each run gets its own. |
|
|
@@ -201,7 +202,7 @@ The headline and the seven steps are the same whatever the answers; the answers
|
|
|
201
202
|
|
|
202
203
|
### alg.yml
|
|
203
204
|
|
|
204
|
-
`check` writes it. `test` reads it, and infers a `verify_call` from your OpenAPI document when there is none: a GET that needs a key and no parameters, preferring `/me`, `/account`, `/user`, `/whoami`. A complete file:
|
|
205
|
+
`check` writes it. `test` reads it, and infers a `verify_call` from your OpenAPI document when there is none: a GET that needs a key and no parameters, preferring `/me`, `/account`, `/user`, `/whoami`. With no OpenAPI document, `test` asks you at the prompt for a GET that needs a key (and the header that carries it), saves it to `alg.yml` and goes on; press Enter with nothing to skip the check for that run. Without a terminal, pass `--skip-verify`. A complete file:
|
|
205
206
|
|
|
206
207
|
```yaml
|
|
207
208
|
url: yourproduct.com
|
|
@@ -253,4 +254,4 @@ npx agent-led-growth test --json --yes # creates a real account; ask the dev
|
|
|
253
254
|
|
|
254
255
|
Each fix prompt is written for a coding agent: read `files.prompts[0]`, make the change, run its acceptance tests, deploy, run `check` again.
|
|
255
256
|
|
|
256
|
-
Source and
|
|
257
|
+
Source: https://github.com/katrinalaszlo/score (development) and https://github.com/katrinalaszlo/agent-led-growth-cli (the published build, where issues go). MIT.
|
package/bin/cli.js
CHANGED
|
@@ -32,6 +32,7 @@ import { runStandards } from "../src/audit/standards.js";
|
|
|
32
32
|
import { parseVerifySpec, buildVerifyTask, classify, failedStep, networkFor, describeSpec } from "../src/verify/index.js";
|
|
33
33
|
import { renderVerify, renderPlan, duration } from "../src/verify/render.js";
|
|
34
34
|
import { inferVerifyCall, latestInterface, latestAnswers } from "../src/verify/infer.js";
|
|
35
|
+
import { askVerifyCall } from "../src/verify/wizard.js";
|
|
35
36
|
import { renderPrompt } from "../src/audit/prompts.js";
|
|
36
37
|
import { patternInfo } from "../src/audit/index.js";
|
|
37
38
|
import { createInterface } from "node:readline/promises";
|
|
@@ -276,6 +277,7 @@ program
|
|
|
276
277
|
.alias("verify")
|
|
277
278
|
.description("A real agent (your Claude Code) signs up for and uses your product; a separate checker proves its key. Creates a real account, spends up to $5 of model use, asks first.")
|
|
278
279
|
.option("-y, --yes", "skip the confirmation and save an inferred verify_call to alg.yml")
|
|
280
|
+
.option("--skip-verify", "test the signup only: the agent's key is not checked, so no verify_call is needed")
|
|
279
281
|
.option("--inbox <address>", "an inbox you will relay mail from into the run's work/inbox/, for products that email a code")
|
|
280
282
|
.option("--max-turns <n>", "agent turn budget", (v) => parseInt(v, 10), 40)
|
|
281
283
|
.option("--max-budget-usd <usd>", "stop the agent once model use passes this many dollars (checked after each turn)", parseBudget, 5)
|
|
@@ -289,7 +291,9 @@ program
|
|
|
289
291
|
Examples:
|
|
290
292
|
alg test --check check Claude Code and alg.yml, print the plan, run nothing
|
|
291
293
|
alg test use url, task and verify_call from alg.yml
|
|
292
|
-
(no verify_call: one is taken from the product's OpenAPI document
|
|
294
|
+
(no verify_call: one is taken from the product's OpenAPI document,
|
|
295
|
+
or you are asked for one at the prompt)
|
|
296
|
+
alg test --skip-verify signup only; the key is not checked
|
|
293
297
|
alg test --yes --json no confirmation, JSON result (CI)`)
|
|
294
298
|
.exitOverride(usageExit)
|
|
295
299
|
.action(async (input, opts) => {
|
|
@@ -311,47 +315,77 @@ Examples:
|
|
|
311
315
|
const shown = configPath.startsWith(home + sep) ? `~${configPath.slice(home.length)}` : CONFIG_FILE;
|
|
312
316
|
console.error(` ${style.dim(`Not using ${shown}: it is for ${savedFor || "no product"}.`)}`);
|
|
313
317
|
}
|
|
314
|
-
const addCallHint = `Add the call that proves an agent's key works to ${CONFIG_FILE}, for example: verify_call: GET https://api.example.com/v1/me`;
|
|
318
|
+
const addCallHint = `Add the call that proves an agent's key works to ${CONFIG_FILE}, for example: verify_call: GET https://api.example.com/v1/me, or pass --skip-verify to test the signup without checking the key`;
|
|
319
|
+
const interactive = process.stdin.isTTY && !opts.json;
|
|
320
|
+
const askOnce = async (question) => {
|
|
321
|
+
const rl = createInterface({ input: process.stdin, output: process.stderr });
|
|
322
|
+
const reply = await rl.question(question).catch((err) => {
|
|
323
|
+
if (err.code === "ABORT_ERR") return "";
|
|
324
|
+
throw err;
|
|
325
|
+
});
|
|
326
|
+
rl.close();
|
|
327
|
+
return reply;
|
|
328
|
+
};
|
|
315
329
|
// No verify_call: take one from the product's own OpenAPI document. It is saved only once accepted, by a
|
|
316
|
-
// person at the prompt or by --yes; otherwise it is used for this run alone.
|
|
317
|
-
|
|
318
|
-
|
|
330
|
+
// person at the prompt or by --yes; otherwise it is used for this run alone. When the docs give none, a person
|
|
331
|
+
// at the terminal is asked for the call, or can skip the check; without a terminal, --skip-verify does that.
|
|
332
|
+
let chosen = null;
|
|
333
|
+
let skipped = Boolean(opts.skipVerify);
|
|
334
|
+
if (!skipped && !(config.verify_call || "").trim()) {
|
|
319
335
|
const live = !opts.json && Boolean(process.stderr.isTTY);
|
|
320
336
|
if (live) process.stderr.write(` ${style.dim("Finding a call that proves the agent's key works…")}`);
|
|
321
|
-
inferred = await inferVerifyCall({ url, version: pkg.version, cwd: process.cwd() });
|
|
337
|
+
const inferred = await inferVerifyCall({ url, version: pkg.version, cwd: process.cwd() });
|
|
322
338
|
if (live) process.stderr.write("\r\x1b[2K");
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
const offSiteNote = inferred.offSite ? `\n This API host is on a different site than ${host}; the key the agent gets is sent there.` : "";
|
|
339
|
+
if (inferred.error && !interactive) {
|
|
340
|
+
// For another product's file, the call goes in a file of this product's own, never in that one.
|
|
341
|
+
if (!useSaved) fail(opts, 2, "verify_call_invalid", `No verify_call for ${host}, and none could be taken from the product's docs: ${inferred.error}.`, `Run "alg check ${host}" in a folder of its own, then add the call that proves an agent's key works to the ${CONFIG_FILE} it writes, for example: verify_call: GET https://api.example.com/v1/me, or pass --skip-verify to test the signup without checking the key`);
|
|
342
|
+
fail(opts, 2, "verify_call_invalid", `${CONFIG_FILE} has no verify_call, and none could be taken from the product's docs: ${inferred.error}.`, addCallHint);
|
|
343
|
+
}
|
|
329
344
|
let accepted = Boolean(opts.yes);
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
rl.close();
|
|
339
|
-
if (!/^(y(es)?)?$/i.test(reply.trim())) {
|
|
340
|
-
console.error(` Cancelled. Nothing saved. To use a different call, add it to ${CONFIG_FILE}, for example: verify_call: GET https://api.example.com/v1/me`);
|
|
341
|
-
process.exit(130);
|
|
345
|
+
let typed = false;
|
|
346
|
+
if (inferred.error) {
|
|
347
|
+
const answer = await askVerifyCall({ ask: askOnce, say: (l) => console.error(l), url, reason: inferred.error, style });
|
|
348
|
+
if (answer.skip) skipped = true;
|
|
349
|
+
else {
|
|
350
|
+
chosen = { ...answer, source: "typed" };
|
|
351
|
+
accepted = true;
|
|
352
|
+
typed = true;
|
|
342
353
|
}
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
354
|
+
} else {
|
|
355
|
+
chosen = { ...inferred, source: "inferred" };
|
|
356
|
+
const offSiteNote = inferred.offSite ? `\n This API host is on a different site than ${host}; the key the agent gets is sent there.` : "";
|
|
357
|
+
if (!accepted && interactive) {
|
|
358
|
+
// A person reads the call and where it came from; the "Will check with" line stays for logs and agents.
|
|
359
|
+
const ask = `\n To prove the agent's key works, we'll call your API with it after signup:\n ${style.bold(inferred.call)} ${style.dim(`(from ${inferred.specUrl.replace(/^https?:\/\//, "")})`)}${offSiteNote}\n Use this call? [Y/n] `;
|
|
360
|
+
const reply = await askOnce(ask);
|
|
361
|
+
if (/^(y(es)?)?$/i.test(reply.trim())) accepted = true;
|
|
362
|
+
else {
|
|
363
|
+
// Not that one: the person names the call instead, or skips the check.
|
|
364
|
+
const answer = await askVerifyCall({ ask: askOnce, say: (l) => console.error(l), url, reason: null, style });
|
|
365
|
+
if (answer.skip) {
|
|
366
|
+
chosen = null;
|
|
367
|
+
skipped = true;
|
|
368
|
+
} else {
|
|
369
|
+
chosen = { ...answer, source: "typed" };
|
|
370
|
+
accepted = true;
|
|
371
|
+
typed = true;
|
|
372
|
+
}
|
|
373
|
+
}
|
|
374
|
+
} else if (!opts.json) console.error(`\n Will check with ${inferred.call} on ${inferred.apiHost} (${inferred.headerName}) (from ${inferred.specUrl}${inferred.specVia ? `, linked from ${inferred.specVia}` : ""}).${offSiteNote}`);
|
|
375
|
+
}
|
|
376
|
+
if (skipped) {
|
|
377
|
+
if (!opts.json) console.error(` ${style.dim("Verification skipped: the agent signs up, and its key is not checked.")}`);
|
|
378
|
+
} else if (!useSaved) {
|
|
346
379
|
if (!opts.json) console.error(` ${style.dim(`Used for this run only: ${CONFIG_FILE} here is not for ${host}.`)}`);
|
|
347
380
|
} else if (accepted) {
|
|
348
|
-
writeVerifyCall(configPath,
|
|
349
|
-
if (!opts.json) console.error(` ${style.dim(`Saved to ${CONFIG_FILE}.`)}`);
|
|
381
|
+
writeVerifyCall(configPath, chosen);
|
|
382
|
+
if (!opts.json) console.error(` ${style.dim(`Saved to ${CONFIG_FILE}${typed ? ": the next test uses it without asking" : ""}.`)}`);
|
|
350
383
|
} else if (!opts.json) console.error(` ${style.dim(`Not saved to ${CONFIG_FILE}; pass --yes or run in a terminal to save it.`)}`);
|
|
351
384
|
}
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
385
|
+
if (skipped && !opts.json && opts.skipVerify) console.error(`\n ${style.dim("Verification skipped (--skip-verify): the agent signs up, and its key is not checked.")}`);
|
|
386
|
+
const spec = skipped ? null : parseVerifySpec(chosen ? { ...config, verify_call: chosen.call, verify_header: chosen.header } : config);
|
|
387
|
+
if (spec?.error) fail(opts, 2, "verify_call_invalid", spec.error, addCallHint);
|
|
388
|
+
const checker = { inferred: chosen?.source === "inferred", inferredFrom: chosen?.source === "inferred" ? chosen.specUrl : null, ...(chosen?.specVia ? { inferredVia: chosen.specVia } : {}), ...(chosen ? { apiHost: chosen.apiHost, apiHostOffSite: chosen.offSite } : {}), ...(skipped ? { skipped: true } : {}) };
|
|
355
389
|
if (opts.check) {
|
|
356
390
|
const claude = opts.executor === "claude-print" ? claudeStatus() : { ok: true, detail: `executor ${opts.executor}` };
|
|
357
391
|
const inbox = process.env.AGENTMAIL_API_KEY ? "AgentMail: a fresh inbox for this run, deleted after" : opts.inbox ? `you relay mail from ${opts.inbox} into work/inbox/` : "none (set AGENTMAIL_API_KEY if the product emails a code or link)";
|
|
@@ -361,8 +395,8 @@ Examples:
|
|
|
361
395
|
target: { url, host },
|
|
362
396
|
task,
|
|
363
397
|
claudeCode: claude,
|
|
364
|
-
agent: { hosts: networkFor(url, spec), inbox, maxTurns: opts.maxTurns, maxBudgetUsd: opts.maxBudgetUsd, saves: ["AGENT_READY_KEY", ...spec.fields] },
|
|
365
|
-
checker: { call: describeSpec(spec), calls: ["with the agent's key: must answer " + spec.expect, "with no key: must be refused (400, 401 or 403)", "with a wrong key: must be refused"], ...checker },
|
|
398
|
+
agent: { hosts: networkFor(url, spec), inbox, maxTurns: opts.maxTurns, maxBudgetUsd: opts.maxBudgetUsd, saves: ["AGENT_READY_KEY", ...(spec ? spec.fields : [])] },
|
|
399
|
+
checker: { call: describeSpec(spec), calls: spec ? ["with the agent's key: must answer " + spec.expect, "with no key: must be refused (400, 401 or 403)", "with a wrong key: must be refused"] : ["none: a key the agent saves counts, unchecked"], ...checker },
|
|
366
400
|
};
|
|
367
401
|
// The onboarding pattern chosen in alg.yml, as the last check's interface.json describes it.
|
|
368
402
|
const onboarding = latestInterface(process.cwd(), url)?.doc.onboarding;
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "agent-led-growth",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.7.0",
|
|
4
4
|
"description": "Score: evals for agents as customers. Runs a real agent through signup, key and first call on your product, before you ship. Track, the analytics half, measures which agents convert after.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"bin": {
|
package/skills/alg-test/SKILL.md
CHANGED
|
@@ -14,7 +14,7 @@ A run creates a real account on the product and uses the developer's Claude Code
|
|
|
14
14
|
|
|
15
15
|
`test` reads `alg.yml` in the current directory. If it is missing, run `npx agent-led-growth check <url> --json --yes` first, which writes it.
|
|
16
16
|
|
|
17
|
-
With no `verify_call`, `test` infers one from the product's OpenAPI document: a GET that needs a key (bearer, or an API key header) and has no required parameters, preferring `/me`, `/account`, `/user` or `/whoami`, on the document's first server. An API key in the `Authorization` header gets the prefix the document gives (`Authorization: Token {key}` when it says "Prefix your key with 'Token '"); with no documented prefix that call is skipped. The plan shows the call's API host (`checker.apiHost`) and, when it is on a different site than the product (`checker.apiHostOffSite`), says the key the agent gets is sent there. Tell the developer which host the key goes to. When the product's own OpenAPI document has none, it tries the OpenAPI documents its `llms.txt`, `auth.md` and other agent files link to on the same registrable domain (such as `app.` or `api.` hosts). It reuses the latest `.alg/<host>/*/interface.json` from `check`, or reads the public pages again. `--yes` accepts the call and saves it to `alg.yml`; without `--yes` and without a terminal it is used for that run and not saved. With `--json`, the plan and the result carry `checker.inferred: true` and `checker.inferredFrom`, and `checker.inferredVia` (the page that linked the document) when it came from a link. Show the developer the inferred call before a real run. When nothing fits, `test` exits `2` with `verify_call_invalid`;
|
|
17
|
+
With no `verify_call`, `test` infers one from the product's OpenAPI document: a GET that needs a key (bearer, or an API key header) and has no required parameters, preferring `/me`, `/account`, `/user` or `/whoami`, on the document's first server. An API key in the `Authorization` header gets the prefix the document gives (`Authorization: Token {key}` when it says "Prefix your key with 'Token '"); with no documented prefix that call is skipped. The plan shows the call's API host (`checker.apiHost`) and, when it is on a different site than the product (`checker.apiHostOffSite`), says the key the agent gets is sent there. Tell the developer which host the key goes to. When the product's own OpenAPI document has none, it tries the OpenAPI documents its `llms.txt`, `auth.md` and other agent files link to on the same registrable domain (such as `app.` or `api.` hosts). It reuses the latest `.alg/<host>/*/interface.json` from `check`, or reads the public pages again. `--yes` accepts the call and saves it to `alg.yml`; without `--yes` and without a terminal it is used for that run and not saved. With `--json`, the plan and the result carry `checker.inferred: true` and `checker.inferredFrom`, and `checker.inferredVia` (the page that linked the document) when it came from a link. Show the developer the inferred call before a real run. When nothing fits and a person is at the terminal, `test` asks for the call (a GET that needs a key, then the header that carries it), saves it to `alg.yml` and goes on; Enter with nothing skips the check for that run, and the result says the key was not checked (`checker.skipped: true`). Without a terminal, or with `--json`, it exits `2` with `verify_call_invalid`; pass `--skip-verify` to run the signup without checking the key, or add the call to `alg.yml`.
|
|
18
18
|
|
|
19
19
|
To choose the call, add it. Pick an authenticated read from the product's own API docs that refuses a request without a key:
|
|
20
20
|
|
|
@@ -63,7 +63,7 @@ The JSON is `agent-ready/verify@1`. Report `outcome`, `reason`, each entry in `c
|
|
|
63
63
|
| --- | --- | --- |
|
|
64
64
|
| `0` | `passed` or `handoff` | The key works and no key and a wrong key were refused; or the agent correctly stopped where the onboarding model says a person sets access up first. |
|
|
65
65
|
| `1` | `failed` | The agent got no key, or the checker's calls did not pass. `prompt` names a fix prompt in the run folder. |
|
|
66
|
-
| `2` | | Usage or setup: no `verify_call` and none could be inferred, or Claude Code missing or signed out. |
|
|
66
|
+
| `2` | | Usage or setup: no `verify_call` and none could be inferred (without a terminal; `--skip-verify` runs the signup anyway), or Claude Code missing or signed out. |
|
|
67
67
|
| `3` | `inconclusive` | Not a result about the product: server errors, a call that answers without a key, the agent could not connect, the spending cap, or a run that did not finish. Say why and suggest what to change. |
|
|
68
68
|
| `130` | | Cancelled. |
|
|
69
69
|
|
package/src/verify/index.js
CHANGED
|
@@ -54,7 +54,9 @@ export function parseVerifySpec(config) {
|
|
|
54
54
|
return { method: (match[1] || "GET").toUpperCase(), url: match[2], body: config.verify_body || null, header: { name: header.slice(0, colon).trim(), template: header.slice(colon + 1).trim() }, expect, assert, fields, exchange, hosts, cli: registries };
|
|
55
55
|
}
|
|
56
56
|
|
|
57
|
+
// spec is null when verification was skipped: the agent signs up, and its key is not checked.
|
|
57
58
|
export function describeSpec(spec) {
|
|
59
|
+
if (!spec) return "none: verification skipped, the agent's key is not checked";
|
|
58
60
|
const assertion = spec.assert ? (spec.assert.equals === null ? `, ${spec.assert.path} present` : `, ${spec.assert.path} = ${spec.assert.equals}`) : "";
|
|
59
61
|
const first = spec.exchange ? `${spec.exchange.method} ${spec.exchange.url} for a token (${spec.exchange.tokenPath}), then ` : "";
|
|
60
62
|
return `${first}${spec.method} ${spec.url} with ${spec.header.name}: ${spec.header.template}; expect ${spec.expect}${assertion}`;
|
|
@@ -155,23 +157,30 @@ export async function checkKey(spec, key, { fetchImpl = fetch, fields = {} } = {
|
|
|
155
157
|
export function networkFor(targetUrl, spec) {
|
|
156
158
|
const host = new URL(targetUrl).hostname;
|
|
157
159
|
const apex = registrableDomain(host);
|
|
158
|
-
const hosts = [host, apex, ...PRODUCT_SUBDOMAINS.map((s) => `${s}.${apex}`)
|
|
160
|
+
const hosts = [host, apex, ...PRODUCT_SUBDOMAINS.map((s) => `${s}.${apex}`)];
|
|
161
|
+
if (spec) hosts.push(new URL(spec.url.replace(/\{[^}]+\}/g, "x")).hostname, ...(spec.exchange ? [new URL(spec.exchange.url.replace(/\{[^}]+\}/g, "x")).hostname] : []), ...(spec.hosts || []));
|
|
159
162
|
return [...new Set(hosts)];
|
|
160
163
|
}
|
|
161
164
|
|
|
165
|
+
// With no spec the check is skipped: a key the agent saved counts, and the result says it was not checked.
|
|
166
|
+
async function skippedCheck() {
|
|
167
|
+
const checks = [{ id: "key_works", label: "Key works", pass: null, detail: "not checked: verification skipped (no verify_call)" }];
|
|
168
|
+
return { ok: true, status: 0, detail: checks[0].detail, objects: { checks, checkerInvalid: false, skipped: true } };
|
|
169
|
+
}
|
|
170
|
+
|
|
162
171
|
export function buildVerifyTask({ url, task, spec, fetchImpl = fetch }) {
|
|
163
172
|
const host = new URL(url).hostname;
|
|
164
173
|
return defineSignupTask({
|
|
165
174
|
id: "verify",
|
|
166
175
|
name: host,
|
|
167
176
|
url,
|
|
168
|
-
apiHost: new URL(spec.url.replace(/\{[^}]+\}/g, "x")).hostname,
|
|
177
|
+
apiHost: spec ? new URL(spec.url.replace(/\{[^}]+\}/g, "x")).hostname : host,
|
|
169
178
|
network: networkFor(url, spec),
|
|
170
179
|
credentialEnvName: KEY_ENV,
|
|
171
|
-
extraFields: spec.fields,
|
|
180
|
+
extraFields: spec ? spec.fields : [],
|
|
172
181
|
keyPattern: /^\S{8,}$/,
|
|
173
182
|
taskText: `${task}. The product is ${host}.`,
|
|
174
|
-
verify: { describe: describeSpec(spec), call: (key, runId, fields) => checkKey(spec, key, { fetchImpl, fields }) },
|
|
183
|
+
verify: { describe: describeSpec(spec), call: spec ? (key, runId, fields) => checkKey(spec, key, { fetchImpl, fields }) : skippedCheck },
|
|
175
184
|
});
|
|
176
185
|
}
|
|
177
186
|
|
|
@@ -184,6 +193,7 @@ const REPORTED_5XX = /\b(?:HTTP\s*(?:status)?|status(?:\s*code)?)\W{0,4}5\d\d\b|
|
|
|
184
193
|
export function classify(result, onboarding, resultMd = "") {
|
|
185
194
|
const evaluation = result.evaluation || {};
|
|
186
195
|
const stages = result.reconciled?.stages || [];
|
|
196
|
+
if (evaluation.success && evaluation.objects?.skipped) return { outcome: "passed", exitCode: 0, reason: "A real agent got its own key. The key was not checked: verification was skipped. Add verify_call to alg.yml to prove it works." };
|
|
187
197
|
if (evaluation.success) return { outcome: "passed", exitCode: 0, reason: "A real agent got its own key and the checker confirmed it works." };
|
|
188
198
|
if (evaluation.objects?.checkerInvalid) return { outcome: "inconclusive", exitCode: 3, reason: "The verify call answers without a key, so it cannot prove the agent's key works. Pick a call that needs authentication." };
|
|
189
199
|
if (stages.some((s) => s.agent?.outcome === "product_unavailable")) return { outcome: "inconclusive", exitCode: 3, reason: "The product returned server errors during the run. Try again later." };
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
import { parseVerifySpec } from "./index.js";
|
|
2
|
+
import { sameSite } from "../interface/sources.js";
|
|
3
|
+
|
|
4
|
+
// `test` found no verify_call in alg.yml and none in the product's docs. Instead of stopping, it asks the person
|
|
5
|
+
// at the terminal for the call, or lets them skip the check. ask(question) returns the typed line; say(line)
|
|
6
|
+
// prints one. Returns { call, header, apiHost, offSite } for a call, or { skip: true }.
|
|
7
|
+
|
|
8
|
+
export const EXAMPLE_CALL = "GET https://api.example.com/v1/me";
|
|
9
|
+
export const DEFAULT_HEADER = "Authorization: Bearer {key}";
|
|
10
|
+
|
|
11
|
+
export async function askVerifyCall({ ask, say, url, reason, style }) {
|
|
12
|
+
const dim = style?.dim || ((s) => s);
|
|
13
|
+
const bold = style?.bold || ((s) => s);
|
|
14
|
+
say("");
|
|
15
|
+
say(` We couldn't find an OpenAPI spec to figure out how to test your API key${reason ? ` ${dim(`(${reason})`)}` : ""}.`);
|
|
16
|
+
say(" What's a simple GET endpoint we can use to verify the key works?");
|
|
17
|
+
say(` ${dim(`e.g. ${EXAMPLE_CALL}. Press Enter with nothing to skip verification: the agent signs up, and its key is not checked.`)}`);
|
|
18
|
+
let spec;
|
|
19
|
+
for (;;) {
|
|
20
|
+
const reply = (await ask(" Call: ")).trim();
|
|
21
|
+
if (!reply || /^skip$/i.test(reply)) return { skip: true };
|
|
22
|
+
spec = parseVerifySpec({ verify_call: reply });
|
|
23
|
+
if (!spec.error) break;
|
|
24
|
+
say(` ${spec.error.replace(/^alg\.yml has no verify_call\. /, "")}`);
|
|
25
|
+
}
|
|
26
|
+
const call = `${spec.method} ${spec.url}`;
|
|
27
|
+
let header;
|
|
28
|
+
for (;;) {
|
|
29
|
+
const reply = (await ask(` Header that carries the key ${dim(`[${DEFAULT_HEADER}]`)}: `)).trim() || DEFAULT_HEADER;
|
|
30
|
+
const withHeader = parseVerifySpec({ verify_call: call, verify_header: reply });
|
|
31
|
+
if (!withHeader.error) {
|
|
32
|
+
header = reply;
|
|
33
|
+
break;
|
|
34
|
+
}
|
|
35
|
+
say(` ${withHeader.error}`);
|
|
36
|
+
}
|
|
37
|
+
const apiHost = new URL(spec.url.replace(/\{[^}]+\}/g, "x")).hostname;
|
|
38
|
+
const offSite = !sameSite(spec.url, url);
|
|
39
|
+
say(` ${dim("Will check with")} ${bold(call)} ${dim(`(${header})`)}`);
|
|
40
|
+
return { call, header, apiHost, offSite };
|
|
41
|
+
}
|