@test-lab-ai/cli 0.1.0 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +35 -3
- package/README.md +81 -42
- package/bin/testlab.mjs +63 -10
- package/examples/fixture.json +9 -0
- package/examples/plan.json +7 -0
- package/examples/plans.json +19 -1
- package/lib/examples.mjs +101 -0
- package/lib/import.mjs +104 -40
- package/package.json +1 -1
package/AGENTS.md
CHANGED
|
@@ -4,6 +4,9 @@ This guide is for an AI coding agent (Claude Code, Codex, Cursor, etc.) asked to
|
|
|
4
4
|
move a user's existing tests into test-lab.ai. You do the format translation; the
|
|
5
5
|
`testlab` CLI handles auth and upload.
|
|
6
6
|
|
|
7
|
+
**Quickest reference: run `testlab examples`** — it prints the exact JSON shape
|
|
8
|
+
for every resource (credentials, labels, data fixtures, plans, pre-steps).
|
|
9
|
+
|
|
7
10
|
## Install (zero-dependency)
|
|
8
11
|
|
|
9
12
|
```bash
|
|
@@ -24,6 +27,9 @@ npx @test-lab-ai/cli --help
|
|
|
24
27
|
top-level `credentials` array.
|
|
25
28
|
- Write clear pass/fail expectations into the prompt ("Confirm the dashboard
|
|
26
29
|
loads", "Expect an order-confirmation page with an order number").
|
|
30
|
+
- For generated/randomized data (a unique email per run, a random name),
|
|
31
|
+
define a **data fixture** and reference it as `{{data.FIXTURE.FIELD}}`
|
|
32
|
+
(see below).
|
|
27
33
|
- If the user's repo has a test-lab plan skill/format, prefer it.
|
|
28
34
|
3. **Write** the plans to a JSON file (or a directory of `*.json`).
|
|
29
35
|
4. **Run** `testlab import <path>`. Use `--dry-run` first to show the user the
|
|
@@ -59,10 +65,13 @@ Any of these is valid input to `testlab import`:
|
|
|
59
65
|
// 2. an array of plans
|
|
60
66
|
[ { "name": "...", "prompt": "..." }, { "name": "...", "prompt": "..." } ]
|
|
61
67
|
|
|
62
|
-
// 3.
|
|
68
|
+
// 3. a bundle: any of credentials / labels / fixtures / plans
|
|
69
|
+
// (created in that order; plans are topo-sorted by their pre-step ref)
|
|
63
70
|
{
|
|
64
|
-
"credentials": [ { "key": "
|
|
65
|
-
"
|
|
71
|
+
"credentials": [ { "key": "password", "value": "hunter2" } ],
|
|
72
|
+
"labels": ["smoke"],
|
|
73
|
+
"fixtures": [ { "key": "newUser", "fields": [ { "key": "email", "mode": "dynamic", "generator": "internet.email" } ] } ],
|
|
74
|
+
"plans": [ { "ref": "signup", "name": "...", "prompt": "Register with {{data.newUser.email}} / {{credentials.password}} ..." } ]
|
|
66
75
|
}
|
|
67
76
|
```
|
|
68
77
|
|
|
@@ -91,6 +100,29 @@ prompts (and cookie/header values) as `{{credentials.KEY}}`. Keys start with a
|
|
|
91
100
|
letter and use letters/numbers/underscores only. The CLI upserts credentials
|
|
92
101
|
before creating plans, and values are stored encrypted (never echoed back).
|
|
93
102
|
|
|
103
|
+
## Data fixtures (generated test data)
|
|
104
|
+
|
|
105
|
+
When a test needs fresh/randomized data, define a fixture in the top-level
|
|
106
|
+
`fixtures` array and reference its fields as `{{data.<fixtureKey>.<fieldKey>}}`
|
|
107
|
+
in prompts. A fixture is `{ key, label?, fields: [...] }`; each field is either
|
|
108
|
+
**static** (a literal `value`, may template `{{run.shortId}}`) or **dynamic**
|
|
109
|
+
(`"mode": "dynamic"` + a `generator` rolled fresh every run):
|
|
110
|
+
|
|
111
|
+
```jsonc
|
|
112
|
+
{
|
|
113
|
+
"key": "newUser",
|
|
114
|
+
"fields": [
|
|
115
|
+
{ "key": "email", "mode": "dynamic", "generator": "internet.email" },
|
|
116
|
+
{ "key": "plan", "mode": "static", "value": "pro" }
|
|
117
|
+
]
|
|
118
|
+
}
|
|
119
|
+
```
|
|
120
|
+
|
|
121
|
+
Generators include `internet.email`, `person.firstName`, `person.fullName`,
|
|
122
|
+
`string.uuid`, `number.int`, `company.name`, … — run `testlab examples` for the
|
|
123
|
+
full list. Reference: `{{data.newUser.email}}`. Keys: start with a letter,
|
|
124
|
+
letters/digits/underscores, max 50 chars.
|
|
125
|
+
|
|
94
126
|
## End-to-end example
|
|
95
127
|
|
|
96
128
|
```bash
|
package/README.md
CHANGED
|
@@ -1,78 +1,117 @@
|
|
|
1
1
|
# @test-lab-ai/cli (`testlab`)
|
|
2
2
|
|
|
3
|
-
Import existing test plans
|
|
3
|
+
Import existing test plans (and the test data they need) into
|
|
4
|
+
[test-lab.ai](https://test-lab.ai) from the command line, your CI, or an AI
|
|
5
|
+
agent. **Zero dependencies**, Node 18+.
|
|
4
6
|
|
|
5
7
|
## Install
|
|
6
8
|
|
|
7
|
-
Published to npm. The CLI has **zero dependencies**, so an install pulls nothing
|
|
8
|
-
else and runs no install scripts.
|
|
9
|
-
|
|
10
|
-
One-off, no install:
|
|
11
|
-
|
|
12
|
-
```bash
|
|
13
|
-
npx @test-lab-ai/cli login
|
|
14
|
-
```
|
|
15
|
-
|
|
16
|
-
Or install the `testlab` command globally:
|
|
17
|
-
|
|
18
9
|
```bash
|
|
19
|
-
|
|
20
|
-
testlab
|
|
10
|
+
npx @test-lab-ai/cli login # one-off, no install
|
|
11
|
+
npm i -g @test-lab-ai/cli # or install the `testlab` command globally
|
|
21
12
|
```
|
|
22
13
|
|
|
23
|
-
Requires Node 18+.
|
|
24
|
-
|
|
25
14
|
## Authenticate
|
|
26
15
|
|
|
27
16
|
```bash
|
|
28
17
|
testlab login
|
|
29
18
|
```
|
|
30
19
|
|
|
31
|
-
Opens your browser to approve access, then
|
|
32
|
-
|
|
33
|
-
|
|
20
|
+
Opens your browser to approve access, then saves your key to
|
|
21
|
+
`~/.test-lab/config.json` (readable only by you). For CI / headless / agents,
|
|
22
|
+
skip the browser:
|
|
34
23
|
|
|
35
24
|
```bash
|
|
36
25
|
export TESTLAB_API_KEY=tl_xxxxx
|
|
37
|
-
# or
|
|
38
|
-
testlab login --key tl_xxxxx
|
|
26
|
+
# or: testlab login --key tl_xxxxx
|
|
39
27
|
```
|
|
40
28
|
|
|
41
|
-
|
|
29
|
+
Create a key at **Settings → API Keys**. A key belongs to one account, so it's
|
|
30
|
+
the import target — to import into an organization account, create the key with
|
|
31
|
+
that org selected.
|
|
42
32
|
|
|
43
33
|
## Commands
|
|
44
34
|
|
|
45
35
|
```
|
|
46
36
|
testlab whoami Show the authenticated account
|
|
37
|
+
testlab import <path> [--dry-run] Import a file or directory of *.json
|
|
47
38
|
testlab plans list List your test plans
|
|
48
|
-
testlab plans create -f plan.json
|
|
49
|
-
testlab
|
|
50
|
-
testlab
|
|
51
|
-
testlab
|
|
39
|
+
testlab plans create -f plan.json Create one plan from JSON
|
|
40
|
+
testlab credentials set KEY --value V Set a credential ({{credentials.KEY}})
|
|
41
|
+
testlab data list List your data fixtures
|
|
42
|
+
testlab data create -f fixture.json Create a data fixture ({{data.KEY.FIELD}})
|
|
43
|
+
testlab examples Full JSON reference for every resource
|
|
52
44
|
```
|
|
53
45
|
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
46
|
+
**`testlab examples` prints the exact JSON shape for every resource**
|
|
47
|
+
(credentials, labels, data fixtures, plans, pre-steps). It's the fastest
|
|
48
|
+
reference and is written so an AI agent can read it and build a valid import.
|
|
49
|
+
|
|
50
|
+
## Import format
|
|
51
|
+
|
|
52
|
+
`testlab import` reads a JSON file (or a directory of `*.json`). A file can be a
|
|
53
|
+
single plan, an array of plans, or a **bundle** with any of these sections —
|
|
54
|
+
created in order: credentials → labels → fixtures → plans:
|
|
55
|
+
|
|
56
|
+
```jsonc
|
|
57
|
+
{
|
|
58
|
+
"credentials": [
|
|
59
|
+
{ "key": "password", "value": "hunter2" } // secret behind {{credentials.password}}
|
|
60
|
+
],
|
|
61
|
+
"labels": ["smoke", "auth"],
|
|
62
|
+
"fixtures": [
|
|
63
|
+
{
|
|
64
|
+
"key": "newUser", // referenced as {{data.newUser.email}}
|
|
65
|
+
"fields": [
|
|
66
|
+
{ "key": "email", "mode": "dynamic", "generator": "internet.email" },
|
|
67
|
+
{ "key": "plan", "mode": "static", "value": "pro" }
|
|
68
|
+
]
|
|
69
|
+
}
|
|
70
|
+
],
|
|
71
|
+
"plans": [
|
|
72
|
+
{
|
|
73
|
+
"ref": "signup", // handle for pre-step wiring
|
|
74
|
+
"name": "Sign up",
|
|
75
|
+
"prompt": "Go to https://app.example.com/signup and register with {{data.newUser.email}} / {{credentials.password}}. Confirm the welcome screen.",
|
|
76
|
+
"testType": "quickTest",
|
|
77
|
+
"labels": ["smoke"]
|
|
78
|
+
},
|
|
79
|
+
{
|
|
80
|
+
"name": "Onboarding",
|
|
81
|
+
"prompt": "Complete the onboarding checklist.",
|
|
82
|
+
"preSteps": [ { "ref": "signup" } ] // run "signup" first, share browser state
|
|
83
|
+
}
|
|
84
|
+
]
|
|
85
|
+
}
|
|
86
|
+
```
|
|
59
87
|
|
|
60
88
|
```bash
|
|
61
|
-
testlab import ./
|
|
62
|
-
testlab import ./
|
|
89
|
+
testlab import ./bundle.json --dry-run # validate + print plan order, write nothing
|
|
90
|
+
testlab import ./bundle.json # create everything
|
|
63
91
|
```
|
|
64
92
|
|
|
65
|
-
The CLI topologically sorts plans by their pre-step
|
|
93
|
+
The CLI topologically sorts plans by their pre-step `ref` dependencies, so order
|
|
94
|
+
in the file doesn't matter. Re-running is safe for credentials/labels/fixtures
|
|
95
|
+
(upsert / idempotent); plans are always created fresh.
|
|
66
96
|
|
|
67
|
-
##
|
|
97
|
+
## Reference syntax (inside a plan prompt)
|
|
68
98
|
|
|
69
|
-
|
|
99
|
+
- `{{credentials.KEY}}` — a stored secret (never shown to the AI model).
|
|
100
|
+
- `{{data.FIXTURE.FIELD}}` — a value from a data fixture (generated test data).
|
|
101
|
+
- `{{run.shortId}}` — a unique per-run id (for unique emails, names, etc.).
|
|
70
102
|
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
testlab
|
|
74
|
-
|
|
103
|
+
## For AI agents
|
|
104
|
+
|
|
105
|
+
Run **`testlab examples`** for the complete JSON reference, or read `AGENTS.md`
|
|
106
|
+
(shipped inside this package). The workflow: read the user's existing tests →
|
|
107
|
+
convert each into the plan/fixture JSON above (explicit URL in the prompt,
|
|
108
|
+
secrets as `{{credentials.X}}`, generated data as `{{data.X.Y}}`) →
|
|
109
|
+
`testlab import`.
|
|
75
110
|
|
|
76
|
-
##
|
|
111
|
+
## Advanced
|
|
77
112
|
|
|
78
|
-
|
|
113
|
+
- `--api-url <url>` (or `TESTLAB_API_URL`) targets a non-prod instance (local dev
|
|
114
|
+
/ self-hosted). Normal users never need it.
|
|
115
|
+
- Every command is a thin wrapper over the public
|
|
116
|
+
[Import API](https://test-lab.ai/docs/api/test-plans); an agent can call that
|
|
117
|
+
directly instead of shelling out to the CLI.
|
package/bin/testlab.mjs
CHANGED
|
@@ -22,6 +22,7 @@ import { resolveAuth, loadConfig, saveConfig } from "../lib/config.mjs"
|
|
|
22
22
|
import { apiFetch } from "../lib/api.mjs"
|
|
23
23
|
import { loadImportFile, runImport } from "../lib/import.mjs"
|
|
24
24
|
import { browserLogin } from "../lib/login.mjs"
|
|
25
|
+
import { EXAMPLES_TEXT } from "../lib/examples.mjs"
|
|
25
26
|
|
|
26
27
|
const log = (...a) => console.log(...a)
|
|
27
28
|
function errExit(msg) {
|
|
@@ -29,25 +30,29 @@ function errExit(msg) {
|
|
|
29
30
|
process.exit(1)
|
|
30
31
|
}
|
|
31
32
|
|
|
32
|
-
const HELP = `testlab — import test plans into test-lab.ai
|
|
33
|
+
const HELP = `testlab — import test plans (and their data) into test-lab.ai
|
|
33
34
|
|
|
34
35
|
Usage:
|
|
35
36
|
testlab login [--key tl_…] Authenticate (browser, or paste/flag a key)
|
|
36
37
|
testlab whoami Show the authenticated account
|
|
38
|
+
testlab import <path> [--dry-run] Import a file or directory of *.json
|
|
39
|
+
(credentials, labels, fixtures, plans)
|
|
37
40
|
testlab plans list List your test plans
|
|
38
41
|
testlab plans create -f plan.json Create one plan from JSON
|
|
39
|
-
testlab
|
|
40
|
-
testlab
|
|
41
|
-
testlab
|
|
42
|
+
testlab credentials set KEY --value V Set a credential ({{credentials.KEY}})
|
|
43
|
+
testlab data list List your data fixtures
|
|
44
|
+
testlab data create -f fixture.json Create a data fixture ({{data.KEY.FIELD}})
|
|
45
|
+
testlab examples Print the full JSON reference for every
|
|
46
|
+
resource (designed for AI agents)
|
|
42
47
|
|
|
43
48
|
Options:
|
|
44
49
|
--key <tl_…> API key (else $TESTLAB_API_KEY or ~/.test-lab/config.json)
|
|
45
|
-
--api-url <url> API base (else $TESTLAB_API_URL or https://www.test-lab.ai)
|
|
46
50
|
--stdin Read the value (key/credential) from stdin
|
|
47
51
|
--force (login) re-authenticate even if a stored key still works
|
|
48
|
-
--dry-run (import) validate + print order without writing
|
|
52
|
+
--dry-run (import) validate + print plan order without writing
|
|
53
|
+
--api-url <url> Target a non-prod instance (default https://www.test-lab.ai)
|
|
49
54
|
|
|
50
|
-
Get a key at <api-url>/admin/settings/api-keys`
|
|
55
|
+
Get a key at <api-url>/admin/settings/api-keys · run \`testlab examples\` for JSON shapes`
|
|
51
56
|
|
|
52
57
|
function parse() {
|
|
53
58
|
return parseArgs({
|
|
@@ -188,7 +193,6 @@ async function cmdCredentialsSet(flags, args) {
|
|
|
188
193
|
}
|
|
189
194
|
|
|
190
195
|
async function cmdImport(flags, args) {
|
|
191
|
-
const { apiKey, apiUrl } = requireAuth(flags)
|
|
192
196
|
const target = args[1]
|
|
193
197
|
if (!target) errExit("usage: testlab import <path> [--dry-run]")
|
|
194
198
|
let loaded
|
|
@@ -197,11 +201,54 @@ async function cmdImport(flags, args) {
|
|
|
197
201
|
} catch (e) {
|
|
198
202
|
errExit(e.message)
|
|
199
203
|
}
|
|
200
|
-
|
|
201
|
-
|
|
204
|
+
const dryRun = flags["dry-run"]
|
|
205
|
+
// --dry-run validates locally and never calls the API, so it doesn't need auth.
|
|
206
|
+
const { apiKey, apiUrl } = dryRun ? resolveAuth(flags) : requireAuth(flags)
|
|
207
|
+
log(`Importing from ${target}: ${loaded.plans.length} plan(s), ${loaded.credentials.length} credential(s), ${loaded.labels.length} label(s), ${loaded.fixtures.length} fixture(s)`)
|
|
208
|
+
const res = await runImport({ apiUrl, apiKey, ...loaded, dryRun, log })
|
|
202
209
|
if (!res.ok && !res.dryRun) process.exit(1)
|
|
203
210
|
}
|
|
204
211
|
|
|
212
|
+
async function cmdDataList(flags) {
|
|
213
|
+
const { apiKey, apiUrl } = requireAuth(flags)
|
|
214
|
+
const r = await apiFetch(apiUrl, apiKey, "GET", "/api/v1/data-fixtures")
|
|
215
|
+
if (!r.ok) errExit(`${r.status}: ${r.json?.error || ""}`)
|
|
216
|
+
const fixtures = r.json?.fixtures || []
|
|
217
|
+
if (fixtures.length === 0) {
|
|
218
|
+
log("No data fixtures yet.")
|
|
219
|
+
return
|
|
220
|
+
}
|
|
221
|
+
for (const fx of fixtures) {
|
|
222
|
+
const fields = (fx.fields || []).map((f) => f.key).join(", ")
|
|
223
|
+
log(` ${fx.key}${fx.label ? ` (${fx.label})` : ""}${fields ? ` — ${fields}` : ""}`)
|
|
224
|
+
}
|
|
225
|
+
log(`\n${fixtures.length} fixture(s)`)
|
|
226
|
+
}
|
|
227
|
+
|
|
228
|
+
async function cmdDataCreate(flags) {
|
|
229
|
+
const { apiKey, apiUrl } = requireAuth(flags)
|
|
230
|
+
if (!flags.file) {
|
|
231
|
+
errExit("usage: testlab data create -f <fixture.json> (run `testlab examples` for the shape)")
|
|
232
|
+
}
|
|
233
|
+
let fixture
|
|
234
|
+
try {
|
|
235
|
+
fixture = JSON.parse(fs.readFileSync(flags.file, "utf8"))
|
|
236
|
+
} catch (e) {
|
|
237
|
+
errExit(`could not read ${flags.file}: ${e.message}`)
|
|
238
|
+
}
|
|
239
|
+
const r = await apiFetch(apiUrl, apiKey, "POST", "/api/v1/data-fixtures", {
|
|
240
|
+
key: fixture.key,
|
|
241
|
+
label: fixture.label,
|
|
242
|
+
fields: fixture.fields,
|
|
243
|
+
})
|
|
244
|
+
if (!r.ok) errExit(`${r.status}: ${r.json?.error || ""}`)
|
|
245
|
+
log(`✓ Created data fixture: ${r.json.fixture.key}`)
|
|
246
|
+
}
|
|
247
|
+
|
|
248
|
+
function cmdExamples() {
|
|
249
|
+
log(EXAMPLES_TEXT)
|
|
250
|
+
}
|
|
251
|
+
|
|
205
252
|
async function main() {
|
|
206
253
|
let parsed
|
|
207
254
|
try {
|
|
@@ -229,6 +276,12 @@ async function main() {
|
|
|
229
276
|
case "credentials":
|
|
230
277
|
if (args[1] === "set") return cmdCredentialsSet(flags, args)
|
|
231
278
|
return errExit("usage: testlab credentials set <key> --value <v>")
|
|
279
|
+
case "data":
|
|
280
|
+
if (args[1] === "list") return cmdDataList(flags)
|
|
281
|
+
if (args[1] === "create") return cmdDataCreate(flags)
|
|
282
|
+
return errExit("usage: testlab data <list|create>")
|
|
283
|
+
case "examples":
|
|
284
|
+
return cmdExamples()
|
|
232
285
|
case "import":
|
|
233
286
|
return cmdImport(flags, args)
|
|
234
287
|
default:
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
{
|
|
2
|
+
"key": "newUser",
|
|
3
|
+
"label": "A fresh signup",
|
|
4
|
+
"fields": [
|
|
5
|
+
{ "key": "firstName", "mode": "dynamic", "generator": "person.firstName" },
|
|
6
|
+
{ "key": "email", "mode": "dynamic", "generator": "internet.email" },
|
|
7
|
+
{ "key": "plan", "mode": "static", "value": "pro" }
|
|
8
|
+
]
|
|
9
|
+
}
|
package/examples/plans.json
CHANGED
|
@@ -3,6 +3,19 @@
|
|
|
3
3
|
{ "key": "email", "value": "qa@example.com" },
|
|
4
4
|
{ "key": "password", "value": "replace-me" }
|
|
5
5
|
],
|
|
6
|
+
"labels": ["smoke", "auth", "regression"],
|
|
7
|
+
"fixtures": [
|
|
8
|
+
{
|
|
9
|
+
"key": "newUser",
|
|
10
|
+
"label": "A fresh signup",
|
|
11
|
+
"fields": [
|
|
12
|
+
{ "key": "firstName", "mode": "dynamic", "generator": "person.firstName" },
|
|
13
|
+
{ "key": "lastName", "mode": "dynamic", "generator": "person.lastName" },
|
|
14
|
+
{ "key": "email", "mode": "dynamic", "generator": "internet.email" },
|
|
15
|
+
{ "key": "company", "mode": "static", "value": "Acme Inc" }
|
|
16
|
+
]
|
|
17
|
+
}
|
|
18
|
+
],
|
|
6
19
|
"plans": [
|
|
7
20
|
{
|
|
8
21
|
"ref": "login",
|
|
@@ -13,7 +26,12 @@
|
|
|
13
26
|
"labels": ["smoke", "auth"]
|
|
14
27
|
},
|
|
15
28
|
{
|
|
16
|
-
"
|
|
29
|
+
"name": "Sign up a new user",
|
|
30
|
+
"prompt": "Go to https://app.example.com/signup and register {{data.newUser.firstName}} {{data.newUser.lastName}} with email {{data.newUser.email}} and a strong password. Confirm the welcome screen appears.",
|
|
31
|
+
"testType": "deepTest",
|
|
32
|
+
"labels": ["regression"]
|
|
33
|
+
},
|
|
34
|
+
{
|
|
17
35
|
"name": "Checkout after login",
|
|
18
36
|
"prompt": "Starting on the dashboard, add the first product to the cart and complete checkout. Confirm an order-confirmation page appears with an order number.",
|
|
19
37
|
"testType": "deepTest",
|
package/lib/examples.mjs
ADDED
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
// The full, copy-pasteable resource reference printed by `testlab examples`.
|
|
2
|
+
// Written for an AI agent: every resource's exact JSON shape, the command that
|
|
3
|
+
// creates it, the reference syntax, and the combined import-bundle format.
|
|
4
|
+
|
|
5
|
+
export const EXAMPLES_TEXT = `testlab — resource reference (for humans and AI agents)
|
|
6
|
+
|
|
7
|
+
All resources are scoped to the API key's account. Auth: a tl_… key via
|
|
8
|
+
\`testlab login\`, the TESTLAB_API_KEY env var, or --key.
|
|
9
|
+
|
|
10
|
+
═══════════════════════════════════════════════════════════════════════════
|
|
11
|
+
REFERENCE SYNTAX (use these inside a plan prompt, and in cookie/header values)
|
|
12
|
+
═══════════════════════════════════════════════════════════════════════════
|
|
13
|
+
{{credentials.KEY}} a stored secret (never shown to the AI/model)
|
|
14
|
+
{{data.FIXTURE.FIELD}} a value from a data fixture (generated test data)
|
|
15
|
+
{{run.shortId}} a unique per-run id (for unique emails, names, …)
|
|
16
|
+
|
|
17
|
+
═══════════════════════════════════════════════════════════════════════════
|
|
18
|
+
1) CREDENTIAL — a secret behind {{credentials.KEY}}
|
|
19
|
+
═══════════════════════════════════════════════════════════════════════════
|
|
20
|
+
Command: testlab credentials set email --value qa@example.com
|
|
21
|
+
JSON (inside an import bundle, under "credentials"):
|
|
22
|
+
{ "key": "email", "value": "qa@example.com" }
|
|
23
|
+
Rules: key starts with a letter; letters/digits/underscores; <=50 chars.
|
|
24
|
+
value <= 1000 chars; stored encrypted, never returned.
|
|
25
|
+
|
|
26
|
+
═══════════════════════════════════════════════════════════════════════════
|
|
27
|
+
2) LABEL — a tag for grouping plans (auto-created when a plan references it)
|
|
28
|
+
═══════════════════════════════════════════════════════════════════════════
|
|
29
|
+
JSON (under "labels"): "smoke" (just the name)
|
|
30
|
+
Plans can also list labels by name and they're created on the fly:
|
|
31
|
+
"labels": ["smoke", "auth"]
|
|
32
|
+
|
|
33
|
+
═══════════════════════════════════════════════════════════════════════════
|
|
34
|
+
3) DATA FIXTURE — reusable generated test data behind {{data.KEY.FIELD}}
|
|
35
|
+
═══════════════════════════════════════════════════════════════════════════
|
|
36
|
+
Command: testlab data create -f fixture.json
|
|
37
|
+
JSON (under "fixtures"):
|
|
38
|
+
{
|
|
39
|
+
"key": "newUser",
|
|
40
|
+
"label": "A fresh signup",
|
|
41
|
+
"fields": [
|
|
42
|
+
{ "key": "firstName", "mode": "dynamic", "generator": "person.firstName" },
|
|
43
|
+
{ "key": "email", "mode": "dynamic", "generator": "internet.email" },
|
|
44
|
+
{ "key": "company", "mode": "static", "value": "Acme Inc" }
|
|
45
|
+
]
|
|
46
|
+
}
|
|
47
|
+
Field modes:
|
|
48
|
+
static (default): a literal "value" (may template {{run.shortId}}).
|
|
49
|
+
dynamic: a "generator" rolls a fresh value every run (requires a generator).
|
|
50
|
+
Generators (for dynamic fields):
|
|
51
|
+
person.firstName, person.lastName, person.fullName, person.jobTitle,
|
|
52
|
+
internet.email, internet.username, internet.url, internet.ipv4, phone.number,
|
|
53
|
+
location.streetAddress, location.city, location.state, location.zipCode,
|
|
54
|
+
location.country, company.name, lorem.word, lorem.sentence, lorem.paragraph,
|
|
55
|
+
date.past, date.future, string.uuid, string.alphanumeric, string.numeric,
|
|
56
|
+
number.int, boolean
|
|
57
|
+
Rules: fixture/field keys start with a letter; letters/digits/underscores;
|
|
58
|
+
<=50 chars. <=50 fields/fixture; static value <=4000 chars.
|
|
59
|
+
Reference it in a prompt as {{data.newUser.email}}.
|
|
60
|
+
|
|
61
|
+
═══════════════════════════════════════════════════════════════════════════
|
|
62
|
+
4) TEST PLAN — a natural-language test (the URL lives IN the prompt)
|
|
63
|
+
═══════════════════════════════════════════════════════════════════════════
|
|
64
|
+
Command: testlab plans create -f plan.json
|
|
65
|
+
JSON (under "plans"):
|
|
66
|
+
{
|
|
67
|
+
"ref": "signup", // optional handle for pre-step wiring
|
|
68
|
+
"name": "Sign up a new user",
|
|
69
|
+
"prompt": "Go to https://app.example.com/signup and register with {{data.newUser.email}} / {{credentials.password}}. Confirm the welcome screen appears.",
|
|
70
|
+
"testType": "quickTest", // quickTest | deepTest
|
|
71
|
+
"agentType": "functional", // functional|accessibility|uiux|exploratory|performance|security
|
|
72
|
+
"devices": ["Desktop Chrome"], // or ["iPhone 15 Pro"]
|
|
73
|
+
"labels": ["smoke", "auth"],
|
|
74
|
+
"failOnPreStepFailure": true
|
|
75
|
+
}
|
|
76
|
+
Pre-steps (run another plan first, sharing browser state):
|
|
77
|
+
"preSteps": [
|
|
78
|
+
{ "ref": "signup" }, // a plan IN THIS import, by its ref
|
|
79
|
+
{ "name": "Existing Login" }, // a plan already in the account, by name
|
|
80
|
+
{ "testPlanId": 42 } // an existing plan by id
|
|
81
|
+
]
|
|
82
|
+
Rules: name <=200 chars; prompt <=32KB; <=25 labels; <=25 preSteps.
|
|
83
|
+
|
|
84
|
+
═══════════════════════════════════════════════════════════════════════════
|
|
85
|
+
IMPORT BUNDLE — create everything at once: testlab import ./bundle.json
|
|
86
|
+
═══════════════════════════════════════════════════════════════════════════
|
|
87
|
+
A file (or a directory of *.json) may contain any of these sections. The CLI
|
|
88
|
+
creates them in order: credentials → labels → fixtures → plans, and topo-sorts
|
|
89
|
+
plans so pre-step dependencies (by "ref") are created first.
|
|
90
|
+
{
|
|
91
|
+
"credentials": [ { "key": "password", "value": "hunter2" } ],
|
|
92
|
+
"labels": ["smoke"],
|
|
93
|
+
"fixtures": [ { "key": "newUser", "fields": [ { "key": "email", "mode": "dynamic", "generator": "internet.email" } ] } ],
|
|
94
|
+
"plans": [
|
|
95
|
+
{ "ref": "signup", "name": "Sign up", "prompt": "Go to https://app.example.com/signup, register with {{data.newUser.email}} …" },
|
|
96
|
+
{ "name": "Onboard", "prompt": "Complete onboarding.", "preSteps": [ { "ref": "signup" } ] }
|
|
97
|
+
]
|
|
98
|
+
}
|
|
99
|
+
Preview without writing: testlab import ./bundle.json --dry-run
|
|
100
|
+
A plan file can also be a single plan object, or a bare array of plans.
|
|
101
|
+
`
|
package/lib/import.mjs
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* Import orchestration: load
|
|
3
|
-
* credentials,
|
|
4
|
-
* to the
|
|
2
|
+
* Import orchestration: load files, then create resources in dependency order —
|
|
3
|
+
* credentials, labels, data fixtures, then plans (topo-sorted by pre-step deps,
|
|
4
|
+
* wiring intra-batch pre-steps to the ids returned along the way).
|
|
5
5
|
*/
|
|
6
6
|
import fs from "node:fs"
|
|
7
7
|
import path from "node:path"
|
|
@@ -9,9 +9,11 @@ import { topoSortPlans } from "./toposort.mjs"
|
|
|
9
9
|
import { apiFetch } from "./api.mjs"
|
|
10
10
|
|
|
11
11
|
/**
|
|
12
|
-
* Load a plan file or a directory of *.json files into
|
|
13
|
-
*
|
|
14
|
-
*
|
|
12
|
+
* Load a plan/import file or a directory of *.json files into
|
|
13
|
+
* { credentials, labels, fixtures, plans }. Each file may be:
|
|
14
|
+
* - a single plan object ({ name, prompt, ... }),
|
|
15
|
+
* - an array of plans,
|
|
16
|
+
* - an import bundle: { credentials?, labels?, fixtures?, plans? }.
|
|
15
17
|
*/
|
|
16
18
|
export function loadImportFile(target) {
|
|
17
19
|
const stat = fs.statSync(target)
|
|
@@ -26,6 +28,8 @@ export function loadImportFile(target) {
|
|
|
26
28
|
}
|
|
27
29
|
|
|
28
30
|
const credentials = []
|
|
31
|
+
const labels = []
|
|
32
|
+
const fixtures = []
|
|
29
33
|
const plans = []
|
|
30
34
|
for (const file of files) {
|
|
31
35
|
let data
|
|
@@ -34,18 +38,33 @@ export function loadImportFile(target) {
|
|
|
34
38
|
} catch (e) {
|
|
35
39
|
throw new Error(`${file}: invalid JSON (${e.message})`)
|
|
36
40
|
}
|
|
41
|
+
|
|
37
42
|
if (Array.isArray(data)) {
|
|
38
43
|
plans.push(...data)
|
|
39
|
-
|
|
40
|
-
|
|
44
|
+
continue
|
|
45
|
+
}
|
|
46
|
+
if (!data || typeof data !== "object") {
|
|
47
|
+
throw new Error(`${file}: expected a plan object, an array, or an import bundle`)
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
const isBundle =
|
|
51
|
+
Array.isArray(data.plans) ||
|
|
52
|
+
Array.isArray(data.credentials) ||
|
|
53
|
+
Array.isArray(data.labels) ||
|
|
54
|
+
Array.isArray(data.fixtures)
|
|
55
|
+
|
|
56
|
+
if (isBundle) {
|
|
41
57
|
if (Array.isArray(data.credentials)) credentials.push(...data.credentials)
|
|
42
|
-
|
|
58
|
+
if (Array.isArray(data.labels)) labels.push(...data.labels)
|
|
59
|
+
if (Array.isArray(data.fixtures)) fixtures.push(...data.fixtures)
|
|
60
|
+
if (Array.isArray(data.plans)) plans.push(...data.plans)
|
|
61
|
+
} else if (typeof data.name === "string" && typeof data.prompt === "string") {
|
|
43
62
|
plans.push(data)
|
|
44
63
|
} else {
|
|
45
|
-
throw new Error(`${file}: expected a plan
|
|
64
|
+
throw new Error(`${file}: expected a plan, an array, or { plans, credentials, labels, fixtures }`)
|
|
46
65
|
}
|
|
47
66
|
}
|
|
48
|
-
return { credentials, plans }
|
|
67
|
+
return { credentials, labels, fixtures, plans }
|
|
49
68
|
}
|
|
50
69
|
|
|
51
70
|
/** Resolve a plan's preSteps for the API: intra-batch refs → concrete ids. */
|
|
@@ -64,13 +83,17 @@ export function normalizePreSteps(preSteps, refToId) {
|
|
|
64
83
|
})
|
|
65
84
|
}
|
|
66
85
|
|
|
67
|
-
export async function runImport({
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
86
|
+
export async function runImport({
|
|
87
|
+
apiUrl,
|
|
88
|
+
apiKey,
|
|
89
|
+
credentials = [],
|
|
90
|
+
labels = [],
|
|
91
|
+
fixtures = [],
|
|
92
|
+
plans = [],
|
|
93
|
+
dryRun = false,
|
|
94
|
+
log = console.log,
|
|
95
|
+
}) {
|
|
96
|
+
// Per-plan shape check before any network calls.
|
|
74
97
|
for (let i = 0; i < plans.length; i++) {
|
|
75
98
|
const p = plans[i]
|
|
76
99
|
if (!p || typeof p.name !== "string" || typeof p.prompt !== "string") {
|
|
@@ -80,40 +103,78 @@ export async function runImport({ apiUrl, apiKey, credentials = [], plans = [],
|
|
|
80
103
|
}
|
|
81
104
|
}
|
|
82
105
|
|
|
83
|
-
const sorted = topoSortPlans(plans)
|
|
106
|
+
const sorted = plans.length > 0 ? topoSortPlans(plans) : { ok: true, order: [] }
|
|
84
107
|
if (!sorted.ok) {
|
|
85
108
|
log(`✗ ${sorted.error}`)
|
|
86
109
|
return { ok: false, error: sorted.error }
|
|
87
110
|
}
|
|
88
111
|
|
|
89
112
|
if (dryRun) {
|
|
90
|
-
log(
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
113
|
+
log(
|
|
114
|
+
`Dry run — ${credentials.length} credential(s), ${labels.length} label(s), ${fixtures.length} fixture(s), ${plans.length} plan(s).`
|
|
115
|
+
)
|
|
116
|
+
if (plans.length) {
|
|
117
|
+
log(`Plan creation order:`)
|
|
118
|
+
sorted.order.forEach((idx, n) => {
|
|
119
|
+
const p = plans[idx]
|
|
120
|
+
log(` ${n + 1}. ${p.name}${p.ref ? ` (ref: ${p.ref})` : ""}`)
|
|
121
|
+
})
|
|
122
|
+
}
|
|
95
123
|
return { ok: true, dryRun: true }
|
|
96
124
|
}
|
|
97
125
|
|
|
98
|
-
|
|
99
|
-
|
|
126
|
+
const counts = { created: 0, failed: 0 }
|
|
127
|
+
const post = async (label, pathname, payload) => {
|
|
128
|
+
const r = await apiFetch(apiUrl, apiKey, "POST", pathname, payload)
|
|
129
|
+
if (r.ok) {
|
|
130
|
+
counts.created++
|
|
131
|
+
log(` ✓ ${label}`)
|
|
132
|
+
} else {
|
|
133
|
+
counts.failed++
|
|
134
|
+
log(` ✗ ${label}: ${r.json?.error || r.status}`)
|
|
135
|
+
}
|
|
136
|
+
return r
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
// 1. Credentials (sequential — each rewrites the shared encrypted blob).
|
|
100
140
|
for (const c of credentials) {
|
|
101
141
|
if (!c || typeof c.key !== "string") {
|
|
102
142
|
log(` ✗ credential: missing key`)
|
|
143
|
+
counts.failed++
|
|
103
144
|
continue
|
|
104
145
|
}
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
146
|
+
await post(`credential ${c.key}`, "/api/v1/credentials", { key: c.key, value: c.value })
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
// 2. Labels (idempotent). Accept "name" strings or { name } objects.
|
|
150
|
+
for (const l of labels) {
|
|
151
|
+
const name = typeof l === "string" ? l : l && l.name
|
|
152
|
+
if (!name) {
|
|
153
|
+
log(` ✗ label: missing name`)
|
|
154
|
+
counts.failed++
|
|
155
|
+
continue
|
|
156
|
+
}
|
|
157
|
+
await post(`label ${name}`, "/api/v1/labels", { name })
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
// 3. Data fixtures.
|
|
161
|
+
for (const fx of fixtures) {
|
|
162
|
+
if (!fx || typeof fx.key !== "string") {
|
|
163
|
+
log(` ✗ fixture: missing key`)
|
|
164
|
+
counts.failed++
|
|
165
|
+
continue
|
|
111
166
|
}
|
|
167
|
+
await post(`fixture ${fx.key}`, "/api/v1/data-fixtures", {
|
|
168
|
+
key: fx.key,
|
|
169
|
+
label: fx.label,
|
|
170
|
+
fields: fx.fields,
|
|
171
|
+
})
|
|
112
172
|
}
|
|
113
173
|
|
|
174
|
+
// 4. Plans, in topo order, wiring intra-batch pre-steps by ref → created id.
|
|
114
175
|
const refToId = new Map()
|
|
115
|
-
let
|
|
116
|
-
let
|
|
176
|
+
let plansCreated = 0
|
|
177
|
+
let plansFailed = 0
|
|
117
178
|
for (const idx of sorted.order) {
|
|
118
179
|
const plan = plans[idx]
|
|
119
180
|
const payload = {
|
|
@@ -130,16 +191,19 @@ export async function runImport({ apiUrl, apiKey, credentials = [], plans = [],
|
|
|
130
191
|
}
|
|
131
192
|
const r = await apiFetch(apiUrl, apiKey, "POST", "/api/v1/test-plans", payload)
|
|
132
193
|
if (r.ok) {
|
|
133
|
-
|
|
194
|
+
plansCreated++
|
|
134
195
|
const id = r.json?.testPlan?.id
|
|
135
196
|
if (plan.ref != null && id != null) refToId.set(plan.ref, id)
|
|
136
|
-
log(` ✓ ${plan.name}${id != null ? ` (#${id})` : ""}`)
|
|
197
|
+
log(` ✓ plan ${plan.name}${id != null ? ` (#${id})` : ""}`)
|
|
137
198
|
} else {
|
|
138
|
-
|
|
139
|
-
log(` ✗ ${plan.name}: ${r.json?.error || r.status}`)
|
|
199
|
+
plansFailed++
|
|
200
|
+
log(` ✗ plan ${plan.name}: ${r.json?.error || r.status}`)
|
|
140
201
|
}
|
|
141
202
|
}
|
|
142
203
|
|
|
143
|
-
|
|
144
|
-
|
|
204
|
+
const totalFailed = counts.failed + plansFailed
|
|
205
|
+
log(
|
|
206
|
+
`\n${plansCreated} plan(s) + ${counts.created} other resource(s) created, ${totalFailed} failed`
|
|
207
|
+
)
|
|
208
|
+
return { ok: totalFailed === 0, created: plansCreated + counts.created, failed: totalFailed }
|
|
145
209
|
}
|