@test-lab-ai/cli 0.2.14 → 0.2.16
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +3 -3
- package/README.md +9 -9
- package/bin/testlab.mjs +41 -30
- package/lib/api.mjs +3 -3
- package/lib/config.mjs +4 -7
- package/lib/examples.mjs +10 -10
- package/lib/import.mjs +5 -5
- package/lib/login.mjs +5 -5
- package/lib/skills.mjs +7 -7
- package/lib/toposort.mjs +1 -1
- package/lib/update-check.mjs +1 -1
- package/package.json +1 -1
package/AGENTS.md
CHANGED
|
@@ -4,7 +4,7 @@ This guide is for an AI coding agent (Claude Code, Codex, Cursor, etc.) asked to
|
|
|
4
4
|
move a user's existing tests into test-lab.ai. You do the format translation; the
|
|
5
5
|
`testlab` CLI handles auth and upload.
|
|
6
6
|
|
|
7
|
-
**Quickest reference: run `testlab examples`**
|
|
7
|
+
**Quickest reference: run `testlab examples`** - it prints the exact JSON shape
|
|
8
8
|
for every resource (credentials, labels, data fixtures, plans, pre-steps).
|
|
9
9
|
|
|
10
10
|
## Install (zero-dependency)
|
|
@@ -121,7 +121,7 @@ in prompts. A fixture is `{ key, label?, fields: [...] }`; each field is either
|
|
|
121
121
|
```
|
|
122
122
|
|
|
123
123
|
Generators include `internet.email`, `person.firstName`, `person.fullName`,
|
|
124
|
-
`string.uuid`, `number.int`, `company.name`,
|
|
124
|
+
`string.uuid`, `number.int`, `company.name`, ... - run `testlab examples` for the
|
|
125
125
|
full list. Reference: `{{data.newUser.email}}`. Keys: start with a letter,
|
|
126
126
|
letters/digits/underscores, max 50 chars.
|
|
127
127
|
|
|
@@ -146,4 +146,4 @@ testlab import ./tests
|
|
|
146
146
|
dashboard afterward if the user needs them.
|
|
147
147
|
- Prefer calling the CLI over hand-rolling HTTP. If you must call the API
|
|
148
148
|
directly, it is documented at https://test-lab.ai/docs/api/test-plans and uses
|
|
149
|
-
the same `Authorization: Bearer tl_
|
|
149
|
+
the same `Authorization: Bearer tl_...` key.
|
package/README.md
CHANGED
|
@@ -26,8 +26,8 @@ export TESTLAB_API_KEY=tl_xxxxx
|
|
|
26
26
|
# or: testlab login --key tl_xxxxx
|
|
27
27
|
```
|
|
28
28
|
|
|
29
|
-
Create a key at **Settings
|
|
30
|
-
the import target
|
|
29
|
+
Create a key at **Settings -> API Keys**. A key belongs to one account, so it's
|
|
30
|
+
the import target - to import into an organization account, create the key with
|
|
31
31
|
that org selected.
|
|
32
32
|
|
|
33
33
|
## Commands
|
|
@@ -52,8 +52,8 @@ reference and is written so an AI agent can read it and build a valid import.
|
|
|
52
52
|
## Import format
|
|
53
53
|
|
|
54
54
|
`testlab import` reads a JSON file (or a directory of `*.json`). A file can be a
|
|
55
|
-
single plan, an array of plans, or a **bundle** with any of these sections
|
|
56
|
-
created in order: credentials
|
|
55
|
+
single plan, an array of plans, or a **bundle** with any of these sections -
|
|
56
|
+
created in order: credentials -> labels -> fixtures -> plans:
|
|
57
57
|
|
|
58
58
|
```jsonc
|
|
59
59
|
{
|
|
@@ -98,9 +98,9 @@ in the file doesn't matter. Re-running is safe for credentials/labels/fixtures
|
|
|
98
98
|
|
|
99
99
|
## Reference syntax (inside a plan prompt)
|
|
100
100
|
|
|
101
|
-
- `{{credentials.<key>}}`
|
|
102
|
-
- `{{data.<fixture>.<field>}}`
|
|
103
|
-
- `{{run.shortId}}`
|
|
101
|
+
- `{{credentials.<key>}}` - a stored secret (never shown to the AI model).
|
|
102
|
+
- `{{data.<fixture>.<field>}}` - a value from a data fixture (generated test data).
|
|
103
|
+
- `{{run.shortId}}` - a unique per-run id (for unique emails, names, etc.).
|
|
104
104
|
|
|
105
105
|
## Install the test-lab-plan skill (Claude Code, Codex, Cursor)
|
|
106
106
|
|
|
@@ -148,9 +148,9 @@ Import tests you already have:
|
|
|
148
148
|
## For AI agents
|
|
149
149
|
|
|
150
150
|
Run **`testlab examples`** for the complete JSON reference, or read `AGENTS.md`
|
|
151
|
-
(shipped inside this package). The workflow: read the user's existing tests
|
|
151
|
+
(shipped inside this package). The workflow: read the user's existing tests ->
|
|
152
152
|
convert each into the plan/fixture JSON above (explicit URL in the prompt,
|
|
153
|
-
secrets as `{{credentials.<key>}}`, generated data as `{{data.<fixture>.<field>}}`)
|
|
153
|
+
secrets as `{{credentials.<key>}}`, generated data as `{{data.<fixture>.<field>}}`) ->
|
|
154
154
|
`testlab import`.
|
|
155
155
|
|
|
156
156
|
## Under the hood
|
package/bin/testlab.mjs
CHANGED
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
2
|
/**
|
|
3
|
-
* testlab
|
|
3
|
+
* testlab - import existing test plans into test-lab.ai (driveable by a human,
|
|
4
4
|
* a CI job, or an AI agent).
|
|
5
5
|
*
|
|
6
6
|
* Commands:
|
|
7
|
-
* testlab login [--key tl_
|
|
7
|
+
* testlab login [--key tl_...] Authenticate (browser flow, or paste/flag a key)
|
|
8
8
|
* testlab whoami Show the authenticated account
|
|
9
9
|
* testlab plans list List the account's test plans
|
|
10
10
|
* testlab plans create -f plan.json Create one plan from a JSON file
|
|
@@ -12,8 +12,7 @@
|
|
|
12
12
|
* testlab credentials set <key> --value <value> Upsert an account credential ({{credentials.<key>}})
|
|
13
13
|
* testlab import <path> [--dry-run] Import a plan file or a directory of *.json
|
|
14
14
|
*
|
|
15
|
-
* Auth: --key
|
|
16
|
-
* API: --api-url → $TESTLAB_API_URL → https://www.test-lab.ai
|
|
15
|
+
* Auth: --key -> $TESTLAB_API_KEY -> ~/.test-lab/config.json
|
|
17
16
|
*/
|
|
18
17
|
import { parseArgs } from "node:util"
|
|
19
18
|
import fs from "node:fs"
|
|
@@ -32,10 +31,10 @@ function errExit(msg) {
|
|
|
32
31
|
process.exit(1)
|
|
33
32
|
}
|
|
34
33
|
|
|
35
|
-
const HELP = `testlab
|
|
34
|
+
const HELP = `testlab - import test plans (and their data) into test-lab.ai
|
|
36
35
|
|
|
37
36
|
Usage:
|
|
38
|
-
testlab login [--key tl_
|
|
37
|
+
testlab login [--key tl_...] Authenticate (browser, or paste/flag a key)
|
|
39
38
|
testlab whoami Show the authenticated account
|
|
40
39
|
testlab import <path> [--dry-run] Import a file or directory of *.json
|
|
41
40
|
(credentials, labels, fixtures, plans)
|
|
@@ -45,7 +44,7 @@ Usage:
|
|
|
45
44
|
testlab credentials set <key> --value <value> Set a credential ({{credentials.<key>}})
|
|
46
45
|
testlab credentials list List credential keys (values never shown)
|
|
47
46
|
testlab labels list List your labels
|
|
48
|
-
testlab data list List
|
|
47
|
+
testlab data list List data fixtures + built-in file fixtures
|
|
49
48
|
testlab data create -f fixture.json Create a data fixture ({{data.<fixture>.<field>}})
|
|
50
49
|
testlab scripts upload <file> --plan <id> Upload a local Playwright script for a plan
|
|
51
50
|
(skips paid AI generation; --device optional)
|
|
@@ -59,21 +58,20 @@ Usage:
|
|
|
59
58
|
testlab skills update Refresh installed skills (also auto-runs after a CLI upgrade)
|
|
60
59
|
|
|
61
60
|
Options:
|
|
62
|
-
--key <tl_
|
|
61
|
+
--key <tl_...> API key (else $TESTLAB_API_KEY or ~/.test-lab/config.json)
|
|
63
62
|
--stdin Read the value (key/credential) from stdin
|
|
64
63
|
--force (login) re-authenticate even if a stored key still works
|
|
65
64
|
--dry-run (import) validate + print plan order without writing
|
|
66
65
|
--project <id|name> (import/plans) assign plans to a project, or "none" for account-level
|
|
67
66
|
--version, -v Print the installed CLI version
|
|
68
67
|
|
|
69
|
-
Get a key at https://test-lab.ai/admin/settings/api-keys
|
|
68
|
+
Get a key at https://test-lab.ai/admin/settings/api-keys / run \`testlab examples\` for JSON shapes`
|
|
70
69
|
|
|
71
70
|
function parse() {
|
|
72
71
|
return parseArgs({
|
|
73
72
|
allowPositionals: true,
|
|
74
73
|
options: {
|
|
75
74
|
key: { type: "string" },
|
|
76
|
-
"api-url": { type: "string" },
|
|
77
75
|
value: { type: "string" },
|
|
78
76
|
file: { type: "string", short: "f" },
|
|
79
77
|
name: { type: "string" },
|
|
@@ -154,7 +152,6 @@ async function cmdLogin(flags) {
|
|
|
154
152
|
|
|
155
153
|
const cfg = loadConfig()
|
|
156
154
|
cfg.apiKey = apiKey
|
|
157
|
-
cfg.apiUrl = apiUrl
|
|
158
155
|
const path = saveConfig(cfg)
|
|
159
156
|
log(`✓ Authenticated${who?.email ? ` as ${who.email}` : ""}. Credentials saved to ${path}`)
|
|
160
157
|
}
|
|
@@ -162,7 +159,7 @@ async function cmdLogin(flags) {
|
|
|
162
159
|
async function cmdWhoami(flags) {
|
|
163
160
|
const { apiKey, apiUrl } = requireAuth(flags)
|
|
164
161
|
// The two requests are independent (apiFetch never throws), so fire
|
|
165
|
-
// them together
|
|
162
|
+
// them together - whoami is interactive and the serial form doubled
|
|
166
163
|
// its wall-clock. Identity endpoint shipped after the CLI's first
|
|
167
164
|
// release: older servers 404 on /me and we just skip those lines.
|
|
168
165
|
const [r, me] = await Promise.all([
|
|
@@ -287,7 +284,7 @@ async function resolveProjectId(apiUrl, apiKey, flags) {
|
|
|
287
284
|
}
|
|
288
285
|
if (!process.stdin.isTTY) {
|
|
289
286
|
errExit(
|
|
290
|
-
`You have ${projects.length} projects
|
|
287
|
+
`You have ${projects.length} projects - pick one with --project <id|name> (or --project none):\n` +
|
|
291
288
|
projects.map((p) => ` ${p.id} ${p.name}`).join("\n")
|
|
292
289
|
)
|
|
293
290
|
}
|
|
@@ -340,13 +337,27 @@ async function cmdDataList(flags) {
|
|
|
340
337
|
const fixtures = r.json?.fixtures || []
|
|
341
338
|
if (fixtures.length === 0) {
|
|
342
339
|
log("No data fixtures yet.")
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
340
|
+
} else {
|
|
341
|
+
log("Data fixtures (reference as {{data.<fixture>.<field>}}):")
|
|
342
|
+
for (const fx of fixtures) {
|
|
343
|
+
const fields = (fx.fields || []).map((f) => f.key).join(", ")
|
|
344
|
+
log(` ${fx.key}${fx.label ? ` (${fx.label})` : ""}${fields ? ` - ${fields}` : ""}`)
|
|
345
|
+
}
|
|
346
|
+
log(` ${fixtures.length} data fixture(s)`)
|
|
347
|
+
}
|
|
348
|
+
// Built-in upload-file fixtures (reference in an upload step as {{file.<name>}}).
|
|
349
|
+
const files = r.json?.files || []
|
|
350
|
+
if (files.length > 0) {
|
|
351
|
+
log("\nFile fixtures (reference in an upload step as {{file.<name>}}):")
|
|
352
|
+
const byCat = {}
|
|
353
|
+
for (const f of files) (byCat[f.category] ||= []).push(f)
|
|
354
|
+
for (const cat of Object.keys(byCat).sort()) {
|
|
355
|
+
const defaults = byCat[cat].filter((f) => f.default).map((f) => `{{${f.token}}}`)
|
|
356
|
+
const shown = defaults.length ? defaults : byCat[cat].map((f) => `{{${f.token}}}`)
|
|
357
|
+
log(` ${cat}: ${shown.join(" ")}`)
|
|
358
|
+
}
|
|
359
|
+
log(` ${files.length} file fixture(s) (append a size, e.g. {{file.pdf.25mb}})`)
|
|
348
360
|
}
|
|
349
|
-
log(`\n${fixtures.length} fixture(s)`)
|
|
350
361
|
}
|
|
351
362
|
|
|
352
363
|
async function cmdDataCreate(flags) {
|
|
@@ -401,7 +412,7 @@ async function cmdScriptsUpload(flags, args) {
|
|
|
401
412
|
// on any plan whose device isn't Desktop Chrome (the server validates it).
|
|
402
413
|
const device = flags.device
|
|
403
414
|
// Idempotent on the server (saved_scripts upsert ON CONFLICT), so a transient
|
|
404
|
-
// 5xx is safe to retry
|
|
415
|
+
// 5xx is safe to retry - a successful retry overwrites, it never duplicates.
|
|
405
416
|
const r = await apiFetch(apiUrl, apiKey, "POST", "/api/v1/scripts/upload", { script, planId, device }, { retries: 3 })
|
|
406
417
|
|
|
407
418
|
// 422 = the script was understood but rejected (allow-list or intent review).
|
|
@@ -419,7 +430,7 @@ async function cmdScriptsUpload(flags, args) {
|
|
|
419
430
|
process.exit(1)
|
|
420
431
|
}
|
|
421
432
|
if (!r.ok) {
|
|
422
|
-
// A 5xx (or network error) survived the retries
|
|
433
|
+
// A 5xx (or network error) survived the retries - almost always a transient
|
|
423
434
|
// server blip, not a rejected script. Tell the user it's safe to re-run:
|
|
424
435
|
// the upload is idempotent, so a retry can't create a duplicate. (A genuinely
|
|
425
436
|
// bad script comes back as 422 and is handled above, never here.)
|
|
@@ -427,7 +438,7 @@ async function cmdScriptsUpload(flags, args) {
|
|
|
427
438
|
errExit(
|
|
428
439
|
`${r.status || "network"}: ${r.json?.error || "upload failed"}\n` +
|
|
429
440
|
` This is usually a transient server error, not a rejected script. ` +
|
|
430
|
-
`Re-run the same command
|
|
441
|
+
`Re-run the same command - the upload is idempotent, so it won't create a duplicate.`,
|
|
431
442
|
)
|
|
432
443
|
}
|
|
433
444
|
errExit(`${r.status}: ${r.json?.error || "upload failed"}`)
|
|
@@ -437,7 +448,7 @@ async function cmdScriptsUpload(flags, args) {
|
|
|
437
448
|
// Show the device the SERVER resolved (it defaults an omitted device to the
|
|
438
449
|
// plan's first), not the CLI-local flag which is undefined when --device is omitted.
|
|
439
450
|
const resolvedDevice = r.json?.device || device || "the plan's default device"
|
|
440
|
-
log(`✓ Uploaded ${file}
|
|
451
|
+
log(`✓ Uploaded ${file} -> plan #${planId} (${steps != null ? `${steps} step${steps === 1 ? "" : "s"}, ` : ""}${resolvedDevice})`)
|
|
441
452
|
log(` This plan now runs your script instead of AI generation.`)
|
|
442
453
|
}
|
|
443
454
|
|
|
@@ -469,7 +480,7 @@ async function cmdScriptsGet(flags, args) {
|
|
|
469
480
|
} catch (e) {
|
|
470
481
|
errExit(`could not write ${flags.out}: ${e.message}`)
|
|
471
482
|
}
|
|
472
|
-
log(`✓ Saved plan #${planId} script
|
|
483
|
+
log(`✓ Saved plan #${planId} script -> ${flags.out} (${meta})`)
|
|
473
484
|
log(` Edit it, then re-upload: testlab scripts upload ${flags.out} --plan ${planId}${flags.device ? ` --device "${flags.device}"` : ""}`)
|
|
474
485
|
} else {
|
|
475
486
|
// Status line to stderr so a plain `> file` redirect captures only the
|
|
@@ -510,7 +521,7 @@ async function cmdSkillsInstall(flags, args) {
|
|
|
510
521
|
try {
|
|
511
522
|
const res = await installSkill(name, agent, { global: flags.global })
|
|
512
523
|
installed++
|
|
513
|
-
log(`✓ ${res.name}
|
|
524
|
+
log(`✓ ${res.name} -> ${agent}: ${res.dest}`)
|
|
514
525
|
} catch (e) {
|
|
515
526
|
// When installing for several agents, one failing (e.g. cursor + --global) must not abort the rest.
|
|
516
527
|
if (targets.length > 1) log(` skipped ${agent}: ${e.message}`)
|
|
@@ -528,7 +539,7 @@ function cmdSkillsList() {
|
|
|
528
539
|
}
|
|
529
540
|
|
|
530
541
|
// Install EVERY current skill at every location where any test-lab skill is
|
|
531
|
-
// already present
|
|
542
|
+
// already present - refreshing existing copies AND adding skills shipped since
|
|
532
543
|
// the user last installed, so an update delivers the latest skill SET (not just
|
|
533
544
|
// fresher copies of the old set). `installedSkillTargets()` is the union of
|
|
534
545
|
// (agent, scope) homes; a brand-new skill has no install locations of its own
|
|
@@ -563,7 +574,7 @@ async function cmdSkillsUpdate() {
|
|
|
563
574
|
for (const r of results) {
|
|
564
575
|
const where = `${r.agent}${r.global ? " (global)" : ""}`
|
|
565
576
|
if (!r.ok) log(` ✗ ${r.name} (${where}): ${r.error}`)
|
|
566
|
-
else log(`${r.isNew ? "+ installed" : "✓ refreshed"} ${r.name}
|
|
577
|
+
else log(`${r.isNew ? "+ installed" : "✓ refreshed"} ${r.name} -> ${where}: ${r.dest}`)
|
|
567
578
|
}
|
|
568
579
|
const ok = results.filter((r) => r.ok)
|
|
569
580
|
const added = ok.filter((r) => r.isNew).length
|
|
@@ -574,7 +585,7 @@ async function cmdSkillsUpdate() {
|
|
|
574
585
|
}
|
|
575
586
|
|
|
576
587
|
// After a CLI upgrade, refresh installed skills in place AND add any newly
|
|
577
|
-
// shipped ones
|
|
588
|
+
// shipped ones - runs once per version bump (best effort). Skipped in CI / when
|
|
578
589
|
// NO_UPDATE_NOTIFIER is set.
|
|
579
590
|
async function maybeRefreshSkillsOnUpgrade() {
|
|
580
591
|
if (process.env.CI || process.env.NO_UPDATE_NOTIFIER) return
|
|
@@ -585,7 +596,7 @@ async function maybeRefreshSkillsOnUpgrade() {
|
|
|
585
596
|
try {
|
|
586
597
|
ok = (await refreshInstalledSkills()).filter((r) => r.ok)
|
|
587
598
|
} catch {
|
|
588
|
-
return // best effort
|
|
599
|
+
return // best effort - never break the actual command
|
|
589
600
|
}
|
|
590
601
|
if (ok.length === 0) return
|
|
591
602
|
const added = ok.filter((r) => r.isNew).length
|
|
@@ -593,7 +604,7 @@ async function maybeRefreshSkillsOnUpgrade() {
|
|
|
593
604
|
const bits = []
|
|
594
605
|
if (refreshed) bits.push(`refreshed ${refreshed}`)
|
|
595
606
|
if (added) bits.push(`installed ${added} new`)
|
|
596
|
-
process.stderr.write(`↻ CLI upgraded ${prev}
|
|
607
|
+
process.stderr.write(`↻ CLI upgraded ${prev} -> ${ver}; ${bits.join(", ")} skill location(s).\n`)
|
|
597
608
|
}
|
|
598
609
|
|
|
599
610
|
async function main() {
|
package/lib/api.mjs
CHANGED
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Tiny fetch wrapper for the test-lab public API. Uses global fetch (Node 18+).
|
|
3
|
-
* Returns { ok, status, json }
|
|
3
|
+
* Returns { ok, status, json } - json is the parsed body (or { raw } on non-JSON).
|
|
4
4
|
*
|
|
5
5
|
* Transient resilience: a 5xx or a network error is retried with linear backoff.
|
|
6
6
|
* Retries are SAFE only for idempotent calls, so they default ON for GET and OFF
|
|
7
|
-
* for writes
|
|
7
|
+
* for writes - a write opts in explicitly via `opts.retries` (used by the
|
|
8
8
|
* idempotent script upload, which the server upserts ON CONFLICT). This is what
|
|
9
9
|
* stops a momentary empty-body 500 (a Worker hiccup or an edge rate-limit that
|
|
10
10
|
* short-circuits before the app's JSON error handler) from surfacing as a hard
|
|
@@ -58,7 +58,7 @@ export async function apiFetch(apiUrl, apiKey, method, pathname, body, opts = {}
|
|
|
58
58
|
...(json || {}),
|
|
59
59
|
error:
|
|
60
60
|
res.status >= 500
|
|
61
|
-
? `server error (HTTP ${res.status}) with no response body
|
|
61
|
+
? `server error (HTTP ${res.status}) with no response body - usually transient`
|
|
62
62
|
: `request failed (HTTP ${res.status})`,
|
|
63
63
|
}
|
|
64
64
|
}
|
package/lib/config.mjs
CHANGED
|
@@ -1,9 +1,7 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Config + auth resolution for the testlab CLI.
|
|
3
3
|
*
|
|
4
|
-
*
|
|
5
|
-
* API key: --key flag → $TESTLAB_API_KEY → ~/.test-lab/config.json
|
|
6
|
-
* API base: --api-url → $TESTLAB_API_URL → config → https://www.test-lab.ai
|
|
4
|
+
* API key: --key flag -> $TESTLAB_API_KEY -> ~/.test-lab/config.json
|
|
7
5
|
*/
|
|
8
6
|
import fs from "node:fs"
|
|
9
7
|
import os from "node:os"
|
|
@@ -36,8 +34,7 @@ export function saveConfig(cfg) {
|
|
|
36
34
|
export function resolveAuth(flags) {
|
|
37
35
|
const cfg = loadConfig()
|
|
38
36
|
const apiKey = flags.key || process.env.TESTLAB_API_KEY || cfg.apiKey || null
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
return { apiKey, apiUrl }
|
|
37
|
+
// The base URL used to be overridable (--api-url / TESTLAB_API_URL / config);
|
|
38
|
+
// removed so an API key can't be sent to an arbitrary host.
|
|
39
|
+
return { apiKey, apiUrl: DEFAULT_API_URL }
|
|
43
40
|
}
|
package/lib/examples.mjs
CHANGED
|
@@ -2,9 +2,9 @@
|
|
|
2
2
|
// Written for an AI agent: every resource's exact JSON shape, the command that
|
|
3
3
|
// creates it, the reference syntax, and the combined import-bundle format.
|
|
4
4
|
|
|
5
|
-
export const EXAMPLES_TEXT = `testlab
|
|
5
|
+
export const EXAMPLES_TEXT = `testlab - resource reference (for humans and AI agents)
|
|
6
6
|
|
|
7
|
-
All resources are scoped to the API key's account. Auth: a tl_
|
|
7
|
+
All resources are scoped to the API key's account. Auth: a tl_... key via
|
|
8
8
|
\`testlab login\`, the TESTLAB_API_KEY env var, or --key.
|
|
9
9
|
|
|
10
10
|
═══════════════════════════════════════════════════════════════════════════
|
|
@@ -12,10 +12,10 @@ REFERENCE SYNTAX (use these inside a plan prompt, and in cookie/header values)
|
|
|
12
12
|
═══════════════════════════════════════════════════════════════════════════
|
|
13
13
|
{{credentials.<key>}} a stored secret (never shown to the AI/model)
|
|
14
14
|
{{data.<fixture>.<field>}} a value from a data fixture (generated test data)
|
|
15
|
-
{{run.shortId}} a unique per-run id (for unique emails, names,
|
|
15
|
+
{{run.shortId}} a unique per-run id (for unique emails, names, ...)
|
|
16
16
|
|
|
17
17
|
═══════════════════════════════════════════════════════════════════════════
|
|
18
|
-
1) CREDENTIAL
|
|
18
|
+
1) CREDENTIAL - a secret behind {{credentials.<key>}}
|
|
19
19
|
═══════════════════════════════════════════════════════════════════════════
|
|
20
20
|
Command: testlab credentials set email --value qa@example.com
|
|
21
21
|
JSON (inside an import bundle, under "credentials"):
|
|
@@ -24,14 +24,14 @@ Rules: key starts with a letter; letters/digits/underscores; <=50 chars.
|
|
|
24
24
|
value <= 1000 chars; stored encrypted, never returned.
|
|
25
25
|
|
|
26
26
|
═══════════════════════════════════════════════════════════════════════════
|
|
27
|
-
2) LABEL
|
|
27
|
+
2) LABEL - a tag for grouping plans (auto-created when a plan references it)
|
|
28
28
|
═══════════════════════════════════════════════════════════════════════════
|
|
29
29
|
JSON (under "labels"): "smoke" (just the name)
|
|
30
30
|
Plans can also list labels by name and they're created on the fly:
|
|
31
31
|
"labels": ["smoke", "auth"]
|
|
32
32
|
|
|
33
33
|
═══════════════════════════════════════════════════════════════════════════
|
|
34
|
-
3) DATA FIXTURE
|
|
34
|
+
3) DATA FIXTURE - reusable generated test data behind {{data.<fixture>.<field>}}
|
|
35
35
|
═══════════════════════════════════════════════════════════════════════════
|
|
36
36
|
Command: testlab data create -f fixture.json
|
|
37
37
|
JSON (under "fixtures"):
|
|
@@ -59,7 +59,7 @@ Rules: fixture/field keys start with a letter; letters/digits/underscores;
|
|
|
59
59
|
Reference it in a prompt as {{data.newUser.email}}.
|
|
60
60
|
|
|
61
61
|
═══════════════════════════════════════════════════════════════════════════
|
|
62
|
-
4) TEST PLAN
|
|
62
|
+
4) TEST PLAN - a natural-language test (the URL lives IN the prompt)
|
|
63
63
|
═══════════════════════════════════════════════════════════════════════════
|
|
64
64
|
Command: testlab plans create -f plan.json
|
|
65
65
|
JSON (under "plans"):
|
|
@@ -84,17 +84,17 @@ Rules: name <=200 chars; prompt <=32KB; <=25 labels; <=25 preSteps.
|
|
|
84
84
|
Projects: omit projectId to import account-level. Run "testlab projects list" for ids; "testlab import --project <id|name>" (or --project none) sets it for the whole import, and is auto-picked when you have exactly one.
|
|
85
85
|
|
|
86
86
|
═══════════════════════════════════════════════════════════════════════════
|
|
87
|
-
IMPORT BUNDLE
|
|
87
|
+
IMPORT BUNDLE - create everything at once: testlab import ./bundle.json
|
|
88
88
|
═══════════════════════════════════════════════════════════════════════════
|
|
89
89
|
A file (or a directory of *.json) may contain any of these sections. The CLI
|
|
90
|
-
creates them in order: credentials
|
|
90
|
+
creates them in order: credentials -> labels -> fixtures -> plans, and topo-sorts
|
|
91
91
|
plans so pre-step dependencies (by "ref") are created first.
|
|
92
92
|
{
|
|
93
93
|
"credentials": [ { "key": "password", "value": "hunter2" } ],
|
|
94
94
|
"labels": ["smoke"],
|
|
95
95
|
"fixtures": [ { "key": "newUser", "fields": [ { "key": "email", "mode": "dynamic", "generator": "internet.email" } ] } ],
|
|
96
96
|
"plans": [
|
|
97
|
-
{ "ref": "signup", "name": "Sign up", "prompt": "Go to https://app.example.com/signup, register with {{data.newUser.email}}
|
|
97
|
+
{ "ref": "signup", "name": "Sign up", "prompt": "Go to https://app.example.com/signup, register with {{data.newUser.email}} ..." },
|
|
98
98
|
{ "name": "Onboard", "prompt": "Complete onboarding.", "preSteps": [ { "ref": "signup" } ] }
|
|
99
99
|
]
|
|
100
100
|
}
|
package/lib/import.mjs
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* Import orchestration: load files, then create resources in dependency order
|
|
2
|
+
* Import orchestration: load files, then create resources in dependency order -
|
|
3
3
|
* credentials, labels, data fixtures, then plans (topo-sorted by pre-step deps,
|
|
4
4
|
* wiring intra-batch pre-steps to the ids returned along the way).
|
|
5
5
|
*/
|
|
@@ -67,7 +67,7 @@ export function loadImportFile(target) {
|
|
|
67
67
|
return { credentials, labels, fixtures, plans }
|
|
68
68
|
}
|
|
69
69
|
|
|
70
|
-
/** Resolve a plan's preSteps for the API: intra-batch refs
|
|
70
|
+
/** Resolve a plan's preSteps for the API: intra-batch refs -> concrete ids. */
|
|
71
71
|
export function normalizePreSteps(preSteps, refToId) {
|
|
72
72
|
if (!Array.isArray(preSteps) || preSteps.length === 0) return undefined
|
|
73
73
|
return preSteps.map((ps) => {
|
|
@@ -112,7 +112,7 @@ export async function runImport({
|
|
|
112
112
|
|
|
113
113
|
if (dryRun) {
|
|
114
114
|
log(
|
|
115
|
-
`Dry run
|
|
115
|
+
`Dry run - ${credentials.length} credential(s), ${labels.length} label(s), ${fixtures.length} fixture(s), ${plans.length} plan(s).`
|
|
116
116
|
)
|
|
117
117
|
if (plans.length) {
|
|
118
118
|
log(`Plan creation order:`)
|
|
@@ -137,7 +137,7 @@ export async function runImport({
|
|
|
137
137
|
return r
|
|
138
138
|
}
|
|
139
139
|
|
|
140
|
-
// 1. Credentials (sequential
|
|
140
|
+
// 1. Credentials (sequential - each rewrites the shared encrypted blob).
|
|
141
141
|
for (const c of credentials) {
|
|
142
142
|
if (!c || typeof c.key !== "string") {
|
|
143
143
|
log(` ✗ credential: missing key`)
|
|
@@ -172,7 +172,7 @@ export async function runImport({
|
|
|
172
172
|
})
|
|
173
173
|
}
|
|
174
174
|
|
|
175
|
-
// 4. Plans, in topo order, wiring intra-batch pre-steps by ref
|
|
175
|
+
// 4. Plans, in topo order, wiring intra-batch pre-steps by ref -> created id.
|
|
176
176
|
const refToId = new Map()
|
|
177
177
|
let plansCreated = 0
|
|
178
178
|
let plansFailed = 0
|
package/lib/login.mjs
CHANGED
|
@@ -1,13 +1,13 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* Browser device-login. Reuses test-lab's existing one-time-code
|
|
2
|
+
* Browser device-login. Reuses test-lab's existing one-time-code -> API-key
|
|
3
3
|
* handshake (the same pattern the Chrome extension uses):
|
|
4
4
|
*
|
|
5
5
|
* 1. CLI starts a localhost callback server + opens the browser to
|
|
6
|
-
* <api>/cli/authorize?state
|
|
6
|
+
* <api>/cli/authorize?state=...&port=...
|
|
7
7
|
* 2. The signed-in user approves; the page redirects to
|
|
8
|
-
* http://127.0.0.1:<port>/callback?code
|
|
8
|
+
* http://127.0.0.1:<port>/callback?code=...&state=...
|
|
9
9
|
* 3. CLI verifies `state`, then exchanges the one-time code for a full-scope
|
|
10
|
-
* API key at POST /api/v1/cli/token (direct HTTPS
|
|
10
|
+
* API key at POST /api/v1/cli/token (direct HTTPS - the secret never rides
|
|
11
11
|
* in the browser redirect).
|
|
12
12
|
*
|
|
13
13
|
* If the server doesn't support the flow (older deployment) the caller falls
|
|
@@ -87,7 +87,7 @@ export function browserLogin(apiUrl, { timeoutMs = 600000 } = {}) {
|
|
|
87
87
|
const { port } = server.address()
|
|
88
88
|
const authUrl = `${apiUrl}/cli/authorize?state=${state}&port=${port}`
|
|
89
89
|
console.log(`Opening your browser to authorize the CLI:\n ${authUrl}\n`)
|
|
90
|
-
console.log(`Waiting for you to approve in the browser (up to 10 minutes)
|
|
90
|
+
console.log(`Waiting for you to approve in the browser (up to 10 minutes)...\n`)
|
|
91
91
|
openBrowser(authUrl)
|
|
92
92
|
})
|
|
93
93
|
})
|
package/lib/skills.mjs
CHANGED
|
@@ -10,9 +10,9 @@
|
|
|
10
10
|
* stays as the browsable, open-source view; it is no longer fetched at runtime.
|
|
11
11
|
*
|
|
12
12
|
* Each agent has its own convention (verified against current docs):
|
|
13
|
-
* claude
|
|
14
|
-
* codex
|
|
15
|
-
* cursor
|
|
13
|
+
* claude -> .claude/skills/<name>/SKILL.md (project) | ~/.claude/skills (--global)
|
|
14
|
+
* codex -> .agents/skills/<name>/SKILL.md (project) | ~/.agents/skills (--global)
|
|
15
|
+
* cursor -> .cursor/rules/<name>.mdc (project only - Cursor has no
|
|
16
16
|
* global *file*; user rules live in Settings)
|
|
17
17
|
*
|
|
18
18
|
* claude + codex share the SKILL.md folder format (only the base dir differs);
|
|
@@ -54,8 +54,8 @@ export function detectAgents() {
|
|
|
54
54
|
}
|
|
55
55
|
|
|
56
56
|
/**
|
|
57
|
-
* Where skill `name` is already installed
|
|
58
|
-
* across all agents
|
|
57
|
+
* Where skill `name` is already installed - project (cwd) and global locations
|
|
58
|
+
* across all agents - filtered to the paths that actually exist. Used to
|
|
59
59
|
* refresh installed skills (on `skills update` and after a CLI upgrade).
|
|
60
60
|
*/
|
|
61
61
|
export function installedSkillLocations(name) {
|
|
@@ -74,7 +74,7 @@ export function installedSkillLocations(name) {
|
|
|
74
74
|
* The distinct (agent, scope) locations where AT LEAST ONE test-lab skill is
|
|
75
75
|
* currently installed. `skills update` and the on-upgrade refresh install EVERY
|
|
76
76
|
* current skill at each of these, so a skill added in a later CLI release lands
|
|
77
|
-
* wherever the user already keeps test-lab skills
|
|
77
|
+
* wherever the user already keeps test-lab skills - not just fresher copies of
|
|
78
78
|
* the skills that happened to be installed before.
|
|
79
79
|
*/
|
|
80
80
|
export function installedSkillTargets() {
|
|
@@ -98,7 +98,7 @@ export function agentTarget(agent, { global } = {}) {
|
|
|
98
98
|
return { kind: "skill-dir", base: global ? path.join(home, ".agents", "skills") : path.join(cwd, ".agents", "skills") }
|
|
99
99
|
case "cursor":
|
|
100
100
|
if (global) {
|
|
101
|
-
throw new Error("Cursor has no global rules file
|
|
101
|
+
throw new Error("Cursor has no global rules file - add user rules in Cursor Settings -> Rules, or install per-project (omit --global).")
|
|
102
102
|
}
|
|
103
103
|
return { kind: "cursor-rule", base: path.join(cwd, ".cursor", "rules") }
|
|
104
104
|
default:
|
package/lib/toposort.mjs
CHANGED
|
@@ -55,7 +55,7 @@ export function topoSortPlans(plans) {
|
|
|
55
55
|
}
|
|
56
56
|
}
|
|
57
57
|
|
|
58
|
-
// Kahn's algorithm (iterative
|
|
58
|
+
// Kahn's algorithm (iterative - no recursion depth concerns).
|
|
59
59
|
const queue = []
|
|
60
60
|
for (let i = 0; i < n; i++) if (indegree[i] === 0) queue.push(i)
|
|
61
61
|
const order = []
|
package/lib/update-check.mjs
CHANGED
|
@@ -39,7 +39,7 @@ export function isNewer(a, b) {
|
|
|
39
39
|
/** The notice string for (current, latest), or null if no update / bad input. */
|
|
40
40
|
export function updateNotice(current, latest) {
|
|
41
41
|
if (!current || !latest || !isNewer(latest, current)) return null
|
|
42
|
-
return `\n ⚡ Update available: ${current}
|
|
42
|
+
return `\n ⚡ Update available: ${current} -> ${latest}\n npm i -g ${PKG}@latest\n`
|
|
43
43
|
}
|
|
44
44
|
|
|
45
45
|
function readCache() {
|