@test-lab-ai/cli 0.2.12 → 0.2.14

package/bin/testlab.mjs

@@ -23,7 +23,7 @@ import { apiFetch } from "../lib/api.mjs"
 import { loadImportFile, runImport } from "../lib/import.mjs"
 import { browserLogin } from "../lib/login.mjs"
 import { EXAMPLES_TEXT } from "../lib/examples.mjs"
-import { TESTLAB_SKILLS, AGENTS, detectAgents, installSkill, installedSkillLocations } from "../lib/skills.mjs"
+import { TESTLAB_SKILLS, AGENTS, detectAgents, installSkill, installedSkillLocations, installedSkillTargets } from "../lib/skills.mjs"
 import { checkForUpdate, currentVersion, previousRunVersion } from "../lib/update-check.mjs"
 
 const log = (...a) => console.log(...a)
@@ -49,6 +49,8 @@ Usage:
   testlab data create -f fixture.json      Create a data fixture ({{data.<fixture>.<field>}})
   testlab scripts upload <file> --plan <id>  Upload a local Playwright script for a plan
                                            (skips paid AI generation; --device optional)
+  testlab scripts get <id> [--out <file>]  Download a plan's active script to fix + re-upload
+                                           (prints to stdout, or writes --out <file>; --device optional)
   testlab examples                         Print the full JSON reference for every
                                            resource (designed for AI agents)
   testlab skills install [--agent ...]     Install the test-lab skills (test-lab-plan +
@@ -84,6 +86,7 @@ function parse() {
       project: { type: "string" },
       plan: { type: "string" },
       device: { type: "string" },
+      out: { type: "string", short: "o" },
       version: { type: "boolean", short: "v" },
       help: { type: "boolean", short: "h" },
     },
@@ -370,10 +373,19 @@ async function cmdDataCreate(flags) {
 // paid AI generation. The server validates it (allow-list + intent review),
 // wraps it in the trusted harness, and stores it as the plan's saved script.
 // On rejection the per-line issues are printed so an agent can self-correct.
+// Strict positive-integer plan id from a CLI arg/flag. parseInt is too lenient
+// ("12abc" -> 12, "12.9" -> 12), which would silently fetch/replace the WRONG
+// plan; require all-digits and > 0 so a typo errors instead.
+function parsePlanIdArg(raw) {
+  if (typeof raw !== "string" || !/^\d+$/.test(raw.trim())) return NaN
+  const n = parseInt(raw.trim(), 10)
+  return n > 0 ? n : NaN
+}
+
 async function cmdScriptsUpload(flags, args) {
   const file = args[2]
   if (!file) errExit("usage: testlab scripts upload <file.spec.ts> --plan <planId> [--device <device>]")
-  const planId = flags.plan ? parseInt(flags.plan, 10) : NaN
+  const planId = parsePlanIdArg(flags.plan)
   if (!Number.isInteger(planId)) errExit("--plan <planId> is required (the numeric test-plan id; see `testlab plans list`)")
   const { apiKey, apiUrl } = requireAuth(flags)
 
@@ -388,7 +400,9 @@ async function cmdScriptsUpload(flags, args) {
   // plan's first configured device. Hardcoding "Desktop Chrome" here would 400
   // on any plan whose device isn't Desktop Chrome (the server validates it).
   const device = flags.device
-  const r = await apiFetch(apiUrl, apiKey, "POST", "/api/v1/scripts/upload", { script, planId, device })
+  // Idempotent on the server (saved_scripts upsert ON CONFLICT), so a transient
+  // 5xx is safe to retry — a successful retry overwrites, it never duplicates.
+  const r = await apiFetch(apiUrl, apiKey, "POST", "/api/v1/scripts/upload", { script, planId, device }, { retries: 3 })
 
   // 422 = the script was understood but rejected (allow-list or intent review).
   // Print the structured issues so a human or agent can fix and retry.
@@ -404,7 +418,20 @@ async function cmdScriptsUpload(flags, args) {
     log(`✗ Script rejected by security review: ${r.json.reason}`)
     process.exit(1)
   }
-  if (!r.ok) errExit(`${r.status}: ${r.json?.error || "upload failed"}`)
+  if (!r.ok) {
+    // A 5xx (or network error) survived the retries — almost always a transient
+    // server blip, not a rejected script. Tell the user it's safe to re-run:
+    // the upload is idempotent, so a retry can't create a duplicate. (A genuinely
+    // bad script comes back as 422 and is handled above, never here.)
+    if (r.status === 0 || r.status >= 500) {
+      errExit(
+        `${r.status || "network"}: ${r.json?.error || "upload failed"}\n` +
+          `  This is usually a transient server error, not a rejected script. ` +
+          `Re-run the same command — the upload is idempotent, so it won't create a duplicate.`,
+      )
+    }
+    errExit(`${r.status}: ${r.json?.error || "upload failed"}`)
+  }
 
   const steps = r.json?.stepCount
   // Show the device the SERVER resolved (it defaults an omitted device to the
@@ -414,6 +441,44 @@ async function cmdScriptsUpload(flags, args) {
   log(`  This plan now runs your script instead of AI generation.`)
 }
 
+async function cmdScriptsGet(flags, args) {
+  const planId = parsePlanIdArg(args[2] ?? flags.plan)
+  if (!Number.isInteger(planId)) {
+    errExit("usage: testlab scripts get <planId> [--device <device>] [--out <file>]")
+  }
+  const { apiKey, apiUrl } = requireAuth(flags)
+
+  const qs = new URLSearchParams({ planId: String(planId) })
+  if (flags.device) qs.set("device", String(flags.device))
+  const r = await apiFetch(apiUrl, apiKey, "GET", `/api/v1/scripts?${qs.toString()}`)
+
+  // 404 carries a human message (no script / wrong device); surface it verbatim.
+  if (!r.ok) errExit(`${r.status}: ${r.json?.error || "failed to fetch script"}`)
+  const script = r.json?.script
+  if (typeof script !== "string") errExit("server returned no script")
+
+  const device = r.json?.device || flags.device || "the plan's default device"
+  const steps = r.json?.stepCount
+  const source = r.json?.source
+  const meta = `${steps != null ? `${steps} step${steps === 1 ? "" : "s"}, ` : ""}${device}${source ? `, ${source}` : ""}`
+  const withNl = script.endsWith("\n") ? script : script + "\n"
+
+  if (flags.out) {
+    try {
+      fs.writeFileSync(flags.out, withNl, "utf8")
+    } catch (e) {
+      errExit(`could not write ${flags.out}: ${e.message}`)
+    }
+    log(`✓ Saved plan #${planId} script → ${flags.out} (${meta})`)
+    log(`  Edit it, then re-upload: testlab scripts upload ${flags.out} --plan ${planId}${flags.device ? ` --device "${flags.device}"` : ""}`)
+  } else {
+    // Status line to stderr so a plain `> file` redirect captures only the
+    // script (pipeable), while the human still sees what they fetched.
+    process.stderr.write(`# plan #${planId} script (${meta})\n`)
+    process.stdout.write(withNl)
+  }
+}
+
 function cmdExamples() {
   log(EXAMPLES_TEXT)
 }
@@ -462,44 +527,73 @@ function cmdSkillsList() {
   log(`\nInstall with: testlab skills install [name] [--agent ${AGENTS.join("|")}|all]`)
 }
 
-async function cmdSkillsUpdate() {
-  let refreshed = 0
+// Install EVERY current skill at every location where any test-lab skill is
+// already present — refreshing existing copies AND adding skills shipped since
+// the user last installed, so an update delivers the latest skill SET (not just
+// fresher copies of the old set). `installedSkillTargets()` is the union of
+// (agent, scope) homes; a brand-new skill has no install locations of its own
+// yet, which is exactly why the old per-skill loop never picked it up. Returns
+// one result per (skill, location), with `isNew` flagging a first-time install.
+async function refreshInstalledSkills() {
+  const had = new Set()
   for (const name of TESTLAB_SKILLS) {
-    for (const loc of installedSkillLocations(name)) {
+    for (const loc of installedSkillLocations(name)) had.add(`${name}:${loc.agent}:${loc.global}`)
+  }
+  const results = []
+  for (const { agent, global } of installedSkillTargets()) {
+    for (const name of TESTLAB_SKILLS) {
+      const isNew = !had.has(`${name}:${agent}:${global}`)
       try {
-        const res = await installSkill(name, loc.agent, { global: loc.global })
-        refreshed++
-        log(`✓ ${name} → ${loc.agent}${loc.global ? " (global)" : ""}: ${res.dest}`)
+        const res = await installSkill(name, agent, { global })
+        results.push({ name, agent, global, isNew, ok: true, dest: res.dest })
       } catch (e) {
-        log(`  ✗ ${name} (${loc.agent}): ${e.message}`)
+        results.push({ name, agent, global, isNew, ok: false, error: e.message })
       }
     }
   }
-  if (refreshed === 0) log("No installed skills found here. Run `testlab skills install` first.")
-  else log(`\nRefreshed ${refreshed} location(s). Restart your agent to load the changes.`)
+  return results
+}
+
+async function cmdSkillsUpdate() {
+  const results = await refreshInstalledSkills()
+  if (results.length === 0) {
+    log("No installed skills found here. Run `testlab skills install` first.")
+    return
+  }
+  for (const r of results) {
+    const where = `${r.agent}${r.global ? " (global)" : ""}`
+    if (!r.ok) log(`  ✗ ${r.name} (${where}): ${r.error}`)
+    else log(`${r.isNew ? "+ installed" : "✓ refreshed"} ${r.name} → ${where}: ${r.dest}`)
+  }
+  const ok = results.filter((r) => r.ok)
+  const added = ok.filter((r) => r.isNew).length
+  const bits = []
+  if (ok.length - added) bits.push(`${ok.length - added} refreshed`)
+  if (added) bits.push(`${added} newly installed`)
+  log(`\n${bits.join(", ") || "Nothing to update"}. Restart your agent to load the changes.`)
 }
 
-// After a CLI upgrade, refresh any already-installed skills in place — runs once
-// per version bump (best effort). Skipped in CI / when NO_UPDATE_NOTIFIER is set.
+// After a CLI upgrade, refresh installed skills in place AND add any newly
+// shipped ones — runs once per version bump (best effort). Skipped in CI / when
+// NO_UPDATE_NOTIFIER is set.
 async function maybeRefreshSkillsOnUpgrade() {
   if (process.env.CI || process.env.NO_UPDATE_NOTIFIER) return
   const ver = currentVersion()
   const prev = previousRunVersion(ver)
   if (!prev || prev === ver) return
-  let refreshed = 0
-  for (const name of TESTLAB_SKILLS) {
-    for (const loc of installedSkillLocations(name)) {
-      try {
-        await installSkill(name, loc.agent, { global: loc.global })
-        refreshed++
-      } catch {
-        /* best effort — never break the actual command */
-      }
-    }
-  }
-  if (refreshed > 0) {
-    process.stderr.write(`↻ CLI upgraded ${prev} → ${ver}; refreshed ${refreshed} installed skill location(s).\n`)
+  let ok
+  try {
+    ok = (await refreshInstalledSkills()).filter((r) => r.ok)
+  } catch {
+    return // best effort — never break the actual command
   }
+  if (ok.length === 0) return
+  const added = ok.filter((r) => r.isNew).length
+  const refreshed = ok.length - added
+  const bits = []
+  if (refreshed) bits.push(`refreshed ${refreshed}`)
+  if (added) bits.push(`installed ${added} new`)
+  process.stderr.write(`↻ CLI upgraded ${prev} → ${ver}; ${bits.join(", ")} skill location(s).\n`)
 }
 
 async function main() {
@@ -551,7 +645,8 @@ async function main() {
       return errExit("usage: testlab data <list|create>")
     case "scripts":
       if (args[1] === "upload") return cmdScriptsUpload(flags, args)
-      return errExit("usage: testlab scripts upload <file.spec.ts> --plan <planId> [--device <device>]")
+      if (args[1] === "get") return cmdScriptsGet(flags, args)
+      return errExit("usage: testlab scripts <upload <file> --plan <id> | get <id> [--out <file>]>")
     case "examples":
       return cmdExamples()
     case "skills":

package/lib/api.mjs

@@ -1,31 +1,72 @@
 /**
  * Tiny fetch wrapper for the test-lab public API. Uses global fetch (Node 18+).
  * Returns { ok, status, json } — json is the parsed body (or { raw } on non-JSON).
+ *
+ * Transient resilience: a 5xx or a network error is retried with linear backoff.
+ * Retries are SAFE only for idempotent calls, so they default ON for GET and OFF
+ * for writes — a write opts in explicitly via `opts.retries` (used by the
+ * idempotent script upload, which the server upserts ON CONFLICT). This is what
+ * stops a momentary empty-body 500 (a Worker hiccup or an edge rate-limit that
+ * short-circuits before the app's JSON error handler) from surfacing as a hard
+ * "500: upload failed" when a one-second retry would have gone through.
  */
-export async function apiFetch(apiUrl, apiKey, method, pathname, body) {
+export async function apiFetch(apiUrl, apiKey, method, pathname, body, opts = {}) {
+  const retries = Number.isInteger(opts.retries) ? opts.retries : method === "GET" ? 2 : 0
+  const backoffMs = Number.isInteger(opts.backoffMs) ? opts.backoffMs : 800
+
   const headers = {}
   if (apiKey) headers["Authorization"] = `Bearer ${apiKey}`
   if (body !== undefined) headers["Content-Type"] = "application/json"
 
-  let res
-  try {
-    res = await fetch(`${apiUrl}${pathname}`, {
-      method,
-      headers,
-      body: body !== undefined ? JSON.stringify(body) : undefined,
-    })
-  } catch (e) {
-    return { ok: false, status: 0, json: { error: `network error: ${e.message}` } }
-  }
-
-  const text = await res.text()
-  let json = null
-  if (text) {
+  for (let attempt = 0; ; attempt++) {
+    let res
     try {
-      json = JSON.parse(text)
-    } catch {
-      json = { raw: text }
+      res = await fetch(`${apiUrl}${pathname}`, {
+        method,
+        headers,
+        body: body !== undefined ? JSON.stringify(body) : undefined,
+      })
+    } catch (e) {
+      if (attempt < retries) {
+        await sleep(backoffMs * (attempt + 1))
+        continue
+      }
+      return { ok: false, status: 0, json: { error: `network error: ${e.message}` } }
+    }
+
+    // 5xx = transient (Worker error / edge rate-limit). Retry while budget remains.
+    if (res.status >= 500 && attempt < retries) {
+      await sleep(backoffMs * (attempt + 1))
+      continue
+    }
+
+    const text = await res.text()
+    let json = null
+    if (text) {
+      try {
+        json = JSON.parse(text)
+      } catch {
+        json = { raw: text }
+      }
+    }
+
+    // A failed response with no usable error body (the empty-body 5xx case) would
+    // otherwise reach the caller as `json: null` and print a blank reason.
+    // Synthesize an actionable message keyed off the status instead.
+    if (!res.ok && (!json || (!json.error && !json.issues && !json.reason))) {
+      json = {
+        ...(json || {}),
+        error:
+          res.status >= 500
+            ? `server error (HTTP ${res.status}) with no response body — usually transient`
+            : `request failed (HTTP ${res.status})`,
+      }
     }
+
+    return { ok: res.ok, status: res.status, json }
   }
-  return { ok: res.ok, status: res.status, json }
+}
+
+function sleep(ms) {
+  return new Promise((resolve) => setTimeout(resolve, ms))
 }

package/lib/skills.mjs

@@ -70,6 +70,23 @@ export function installedSkillLocations(name) {
   ].filter((c) => fs.existsSync(c.path))
 }
 
+/**
+ * The distinct (agent, scope) locations where AT LEAST ONE test-lab skill is
+ * currently installed. `skills update` and the on-upgrade refresh install EVERY
+ * current skill at each of these, so a skill added in a later CLI release lands
+ * wherever the user already keeps test-lab skills — not just fresher copies of
+ * the skills that happened to be installed before.
+ */
+export function installedSkillTargets() {
+  const seen = new Map()
+  for (const name of TESTLAB_SKILLS) {
+    for (const loc of installedSkillLocations(name)) {
+      seen.set(`${loc.agent}:${loc.global}`, { agent: loc.agent, global: loc.global })
+    }
+  }
+  return [...seen.values()]
+}
+
 /** Resolve the install target (base dir + format) for one agent. */
 export function agentTarget(agent, { global } = {}) {
   const home = os.homedir()

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@test-lab-ai/cli",
-  "version": "0.2.12",
+  "version": "0.2.14",
   "description": "Import existing test plans into test-lab.ai from the command line (or an AI agent).",
   "type": "module",
   "bin": {

package/skills/test-lab-script/SKILL.md

@@ -32,18 +32,21 @@ This also means the script runs in a **restricted environment**: Playwright and
 
 Follow these in order.
 
-### 1. Confirm the target plan exists and is reachable
+### 1. Design the test as a plan prompt first (with test-lab-plan)
 
-An upload attaches to an **existing** plan by numeric id. Before writing anything:
+The plan's English **prompt is the spec; your script is one implementation of it.** Design the spec before you write a line of Playwright — that is what keeps the script honest and reviewable, and it is the fallback the platform runs if the script is ever cleared. Skipping this is how you end up with a script glued to a throwaway prompt that describes something else.
 
-- `testlab whoami` – confirm the CLI is authenticated. If not, the user runs `testlab login` once or sets `TESTLAB_API_KEY`. If `testlab` is not found, use `npx @test-lab-ai/cli` instead.
-- `testlab plans list` – find the plan id to upload to. The plan must already have a target URL (from its prompt or its project), because the script runs against that origin.
+- **No plan yet?** Use the **test-lab-plan** skill to design it: one user journey, an explicit start URL, and a numbered `Verify that:` block of observable acceptance criteria. Create it (dashboard or `testlab import`), then attach your script.
+- **Plan already exists?** Read its prompt — that *is* your spec. If the prompt is vague, wrong, or describes a different flow, fix it with test-lab-plan **before** writing the script. Never encode behaviour in the script that the prompt doesn't claim.
 
-If there is no suitable plan yet, create one first (in the dashboard, or with the **test-lab-plan** skill + `testlab import`). The plan's English prompt stops driving runs the moment a script is uploaded: your script replaces AI generation for that plan and device.
+Keep the prompt a clean user-journey spec: **no setup, auth, or implementation detail in the prose** — logged-in state, injected cookies, environment config, "no login needed", framework internals: none of it belongs there. Setup lives in the plan's pre-step / project environment, not the prompt body (test-lab-plan enforces this). Your script then *implements* that spec: one `test("Step N: …")` per prompt action, and an `expect` for each numbered acceptance criterion.
 
-### 2. Know the flow and its checks
+### 2. Confirm the plan is reachable and read the real UI
 
-Same discipline as a good test plan: one user journey, explicit start, observable assertions. If you are in the target site's repo, read the relevant component / route so your locators and assertions match real DOM text and real success states, not guesses. If you are not in the repo, pull a snapshot with WebFetch or ask the user for the key screens. Anchor every `expect` to something real.
+- `testlab whoami` – confirm the CLI is authenticated. If not, the user runs `testlab login` once or sets `TESTLAB_API_KEY`. If `testlab` is not found, use `npx @test-lab-ai/cli` instead.
+- `testlab plans list` – find the plan id to upload to. The plan must already have a target URL (from its prompt or its project), because the script runs against that origin. Uploading a script replaces AI generation for that plan + device, so from then on the prompt's job is to be the spec you implemented and the fallback — keep the two in sync.
+- **Read the real UI before asserting.** If you are in the target site's repo, read the relevant component / route so your locators and assertions match real DOM text and real success states, not guesses. If you are not in the repo, pull a snapshot with WebFetch or ask the user for the key screens. Anchor every `expect` to something real **and to a criterion in the prompt.**
+- **Fixing a script the plan already runs?** Download the live one with `testlab scripts get <id> [--device] [--out <file>]` (works for uploaded *and* generated scripts) instead of guessing at it, then edit and re-upload. That is the full fetch → fix → `scripts upload` loop, no dashboard needed.
 
 ### 3. Write the steps against `sharedPage`
 
@@ -140,6 +143,7 @@ The complete lists and the per-rule fixes are in `references/playwright-api.md`.
 7. **No Node / browser / network globals** (`process`, `fs`, `fetch`, `window`, `request`, …) and **no `page.evaluate`/`route`**. If you reached for one, rewrite the step to use the page.
 8. **No computed member keys or prototype access** (`x[expr]`, `.constructor`, `.__proto__`).
 9. **The plan id is right** (`testlab plans list`) and the plan has a target URL.
+10. **The script implements the plan's prompt.** Every prompt action maps to a `test("Step N: …")`, and every numbered acceptance criterion maps to an `expect`. The prompt itself reads as a clean user journey — no setup/auth/implementation detail leaked into the prose (that belongs in the plan's pre-step / environment). If the prompt and the script disagree, fix the prompt with test-lab-plan first.
 
 If any item fails, fix it before uploading – it will be rejected server-side anyway, and fixing first saves a round-trip.
 
@@ -157,7 +161,9 @@ If any item fails, fix it before uploading – it will be rejected server-side a
 
 ## Relationship to test-lab-plan
 
-- **test-lab-plan** – describe a flow in English; the AI generates and maintains the Playwright. Best when you want low effort or don't have a script.
-- **test-lab-script** (this skill) – you own the Playwright; upload it verbatim. Best when you already have a script, need exact control, or want to save generation credits.
+These **compose; they are not either/or.** test-lab-plan designs the spec (the plan's prompt); test-lab-script implements it (the Playwright). Even when you will hand-write the script, start with test-lab-plan so the spec is a clean, reviewable user journey and the script has a concrete contract to satisfy — see Workflow step 1.
+
+- **test-lab-plan** – describe a flow in English; the AI generates and maintains the Playwright. Best when you want low effort, or as the spec step before you write your own script.
+- **test-lab-script** (this skill) – you own the Playwright and upload it verbatim to implement that spec. Best when you need exact control, already have a script, or want to save generation credits.
 
 An uploaded script can still be refined later with AI from the dashboard (it is tagged as uploaded vs generated). `testlab examples` is the canonical, always-current reference for fixture shapes and resource formats.