@devasign/verify 1.0.1 → 1.2.0

package/README.md

@@ -16,3 +16,5 @@ Guarantees:
 - Setup problems (no start command, missing secret names, wrong runtime) are uploaded as a structured diagnosis, and the process exits 0.
 
 Local run against a dev backend: mint a token with `backend/scripts/verify-dev-token.ts`, then `devasign-verify run --api-url http://localhost:8787 --token <jwt> --pr <n> --sha <head-sha>`. Offline: `--plan-file plan.json --results-out results.json`.
+
+Exit code: 0 unless `--fail-on verdict` (a criterion failed) or `--fail-on unverifiable` (a criterion failed or could not be verified) is set; unset, the repository's DevAsign setting applies. An unknown `--fail-on` value exits 2. Criteria the plan could not cover are printed as warnings with their reason and fix link, and the Actions step summary lists every criterion. On Actions the CLI also writes `run-id`, `outcome` and `browsers` to `GITHUB_OUTPUT`.

package/dist/cli.js

@@ -7577,7 +7577,17 @@ function diagnosePlaywrightOutput(output) {
 }
 
 // src/log.ts
+import { appendFileSync } from "node:fs";
 var onActions = () => process.env.GITHUB_ACTIONS === "true";
+function setOutput(name, value) {
+  const file = process.env.GITHUB_OUTPUT;
+  if (!file) return;
+  try {
+    appendFileSync(file, `${name}=${value.replace(/\r?\n/g, " ")}
+`);
+  } catch {
+  }
+}
 var log = {
   info(msg) {
     console.log(`devasign: ${msg}`);
@@ -7627,11 +7637,11 @@ function staticTokenSource(token) {
 }
 
 // src/run.ts
-import { appendFileSync, existsSync as existsSync5, writeFileSync as writeFileSync3 } from "node:fs";
+import { appendFileSync as appendFileSync2, existsSync as existsSync5, writeFileSync as writeFileSync3 } from "node:fs";
 import path7 from "node:path";
 
 // src/types.ts
-var CLI_VERSION = "1.0.1";
+var CLI_VERSION = "1.2.0";
 
 // src/api.ts
 var ApiError = class extends Error {
@@ -7889,6 +7899,7 @@ import { spawn } from "node:child_process";
 import { mkdirSync, writeFileSync } from "node:fs";
 import path3 from "node:path";
 var MAX_CAPTURE = 5 * 1024 * 1024;
+var DRAIN_MS = 2e3;
 async function runCommand(opts) {
   const started = Date.now();
   return new Promise((resolve) => {
@@ -7897,12 +7908,28 @@ async function runCommand(opts) {
     let output = "";
     let timedOut = false;
     let settled = false;
+    let drain;
+    let exited = null;
+    const armDrain = () => {
+      if (!exited) return;
+      clearTimeout(drain);
+      drain = setTimeout(() => finish({ ...exited, timedOut }), DRAIN_MS);
+    };
     const env = { ...process.env, ...opts.env || {}, CI: "true", FORCE_COLOR: "0" };
     for (const k of Object.keys(env)) if (k.startsWith("NODE_TEST_")) delete env[k];
-    const child = spawn(opts.cmd, opts.args, { cwd: opts.cwd, env, stdio: ["ignore", "pipe", "pipe"] });
+    const group = process.platform !== "win32";
+    const child = spawn(opts.cmd, opts.args, { cwd: opts.cwd, env, stdio: ["ignore", "pipe", "pipe"], detached: group });
+    const killAll = () => {
+      try {
+        if (group && child.pid) process.kill(-child.pid, "SIGKILL");
+        else child.kill("SIGKILL");
+      } catch {
+        child.kill("SIGKILL");
+      }
+    };
     const timer = setTimeout(() => {
       timedOut = true;
-      child.kill("SIGKILL");
+      killAll();
     }, opts.timeoutMs);
     const take = (chunk, which) => {
       const s = chunk.toString("utf8");
@@ -7912,6 +7939,7 @@ async function runCommand(opts) {
       if (opts.onLine) {
         for (const line of s.split("\n")) if (line) opts.onLine(line);
       }
+      armDrain();
     };
     child.stdout?.on("data", (c) => take(c, "out"));
     child.stderr?.on("data", (c) => take(c, "err"));
@@ -7919,6 +7947,7 @@ async function runCommand(opts) {
       if (settled) return;
       settled = true;
       clearTimeout(timer);
+      clearTimeout(drain);
       const full = { ...result, stdout, stderr, output, durationMs: Date.now() - started };
       if (opts.logFile) {
         try {
@@ -7935,6 +7964,10 @@ ${output}
       resolve(full);
     };
     child.on("error", (err) => finish({ code: null, signal: null, timedOut, spawnError: err.message }));
+    child.on("exit", (code, signal) => {
+      exited = { code, signal };
+      armDrain();
+    });
     child.on("close", (code, signal) => finish({ code, signal, timedOut }));
   });
 }
@@ -8009,7 +8042,8 @@ async function runFileTests(args) {
 
 // src/runners/playwright.ts
 import { createRequire as createRequire2 } from "node:module";
-import { existsSync as existsSync3, readFileSync as readFileSync4, statSync as statSync3 } from "node:fs";
+import { existsSync as existsSync3, readdirSync as readdirSync2, readFileSync as readFileSync4, statSync as statSync3 } from "node:fs";
+import { homedir } from "node:os";
 import path5 from "node:path";
 var require3 = createRequire2(import.meta.url);
 function playwrightCli(root) {
@@ -8029,9 +8063,17 @@ function generatePlaywrightConfig(a) {
     baseImport,
     "const b: any = (__base as any)?.default ?? __base ?? {};",
     "// One browser project only: recordings once, and no browsers we did not install.",
-    "const projects = Array.isArray(b.projects) && b.projects.length",
-    "  ? [b.projects.find((p: any) => /chrom/i.test(String(p?.name || ''))) ?? b.projects[0]]",
-    "  : undefined;",
+    "// Project fields win over config fields in Playwright, so the customer's own",
+    "// testDir/retries/recording settings are stripped and ours are re-applied here;",
+    "// dependencies/teardown would reference projects we did not keep.",
+    "const __pick = (ps: any[]) => ps.find((p: any) => /chrom/i.test(String(p?.name || ''))) ?? ps[0];",
+    "const __clean = (p: any) => {",
+    "  const q: any = { ...p };",
+    "  for (const k of ['dependencies', 'teardown', 'testDir', 'testMatch', 'testIgnore', 'retries', 'outputDir', 'grep', 'grepInvert', 'snapshotDir']) delete q[k];",
+    "  q.use = { ...(q.use || {}), video: 'on', trace: 'on', screenshot: 'on' };",
+    "  return q;",
+    "};",
+    "const projects = Array.isArray(b.projects) && b.projects.length ? [__clean(__pick(b.projects))] : undefined;",
     "export default defineConfig({",
     "  ...b,",
     `  testDir: ${j(a.testDir)},`,
@@ -8071,6 +8113,20 @@ function stripAnsi(s) {
 }
 var KIND_BY_ATTACHMENT = { video: "video", trace: "trace", screenshot: "screenshot" };
 var TYPE_BY_KIND = { video: "video/webm", trace: "application/zip", screenshot: "image/png" };
+function testOutcome(retries) {
+  if (!retries.length) return "error";
+  const last = retries[retries.length - 1];
+  if (last === "pass") return retries.every((s) => s === "pass") ? "pass" : "flaky";
+  return retries.includes("fail") ? "fail" : "error";
+}
+function fileStatus(perTest) {
+  const outcomes = perTest.map(testOutcome);
+  if (!outcomes.length) return "error";
+  if (outcomes.includes("fail")) return "fail";
+  if (outcomes.includes("flaky")) return "flaky";
+  if (outcomes.includes("error")) return "error";
+  return "pass";
+}
 function mapReport(report, tests, ws, artifacts, overallOutput) {
   const specs = flattenSpecs(report);
   const results = [];
@@ -8079,8 +8135,11 @@ function mapReport(report, tests, ws, artifacts, overallOutput) {
     const attempts = [];
     let n = 0;
     let anyResult = false;
+    const perTest = [];
     for (const spec of mine) {
       for (const pt of spec.tests) {
+        const own = [];
+        perTest.push(own);
         for (const r of [...pt.results || []].sort((x, y) => (x.retry ?? 0) - (y.retry ?? 0))) {
           anyResult = true;
           n += 1;
@@ -8106,6 +8165,7 @@ function mapReport(report, tests, ws, artifacts, overallOutput) {
             artifacts.push({ clientRef: posterRef, kind: "poster", path: screenshotPath, displayPath: ws.relative(screenshotPath), contentType: "image/png", testId: t.id, criterionIds: t.criterionIds, attempt: n, posterFor: videoRef });
             refs.push(posterRef);
           }
+          own.push(status2);
           attempts.push({ n, status: status2, durationMs: Math.round(r.duration ?? 0), error: status2 === "pass" ? void 0 : stripAnsi(message || r.status).split("\n").slice(0, 3).join(" ").slice(0, 500), artifactIds: refs });
         }
       }
@@ -8113,10 +8173,9 @@ function mapReport(report, tests, ws, artifacts, overallOutput) {
     if (!anyResult) {
       const err = report.errors?.[0]?.message || overallOutput.split("\n").find((l) => /Error|error/.test(l)) || "no result for this test";
       attempts.push({ n: 1, status: "error", durationMs: 0, error: err.slice(0, 500), artifactIds: [] });
+      perTest.push(["error"]);
     }
-    const statuses = attempts.map((a) => a.status);
-    const passes = statuses.filter((s) => s === "pass").length;
-    const status = passes === statuses.length ? "pass" : passes > 0 ? "flaky" : statuses.includes("fail") ? "fail" : "error";
+    const status = fileStatus(perTest.filter((t2) => t2.length));
     results.push({
       id: `r-${t.id}`,
       testId: t.id,
@@ -8134,7 +8193,24 @@ function mapReport(report, tests, ws, artifacts, overallOutput) {
   }
   return results;
 }
+function playwrightBrowsersRoot(env = process.env, home = homedir()) {
+  if (env.PLAYWRIGHT_BROWSERS_PATH) return env.PLAYWRIGHT_BROWSERS_PATH;
+  if (process.platform === "darwin") return path5.join(home, "Library", "Caches", "ms-playwright");
+  if (process.platform === "win32") return path5.join(env.LOCALAPPDATA || home, "ms-playwright");
+  return path5.join(home, ".cache", "ms-playwright");
+}
+function hasChromium(root, read = readdirSync2) {
+  try {
+    return read(root).some((d) => /^chromium(_headless_shell)?-\d+$/.test(d));
+  } catch {
+    return false;
+  }
+}
 async function ensureBrowsers(root, ws) {
+  if (hasChromium(playwrightBrowsersRoot())) {
+    log.info("Chromium already installed; skipping download");
+    return { ok: true, log: "cached" };
+  }
   const cli = playwrightCli(root);
   const args = [...cli.args, "install", ...process.platform === "linux" ? ["--with-deps"] : [], "chromium"];
   log.info("installing Chromium for Playwright");
@@ -8235,9 +8311,16 @@ var Workspace = class {
 
 // src/run.ts
 var sleep3 = (ms) => new Promise((r) => setTimeout(r, ms));
+var MIN_SERVER_DEADLINE_MS = 3e4;
+function serverDeadline(current, startedAt, hintMs) {
+  if (typeof hintMs !== "number" || !Number.isFinite(hintMs) || hintMs <= 0) return current;
+  return Math.min(current, startedAt + Math.max(hintMs, MIN_SERVER_DEADLINE_MS));
+}
 async function resolvePlan(api, ctx, setup, timeoutMs) {
-  const deadline = Date.now() + timeoutMs;
+  const startedAt = Date.now();
+  let deadline = startedAt + timeoutMs;
   let polls = 0;
+  let finalPoll = false;
   for (; ; ) {
     const res = await api.resolve({
       sha: ctx.sha,
@@ -8246,14 +8329,19 @@ async function resolvePlan(api, ctx, setup, timeoutMs) {
       attempt: ctx.runAttempt,
       setup: polls === 0 ? setup : void 0,
       actions: { runId: ctx.runId, jobUrl: ctx.jobUrl, runnerOs: ctx.runnerOs },
-      cliVersion: CLI_VERSION
+      cliVersion: CLI_VERSION,
+      // Tells the server this job is leaving, so a plan landing later re-dispatches CI
+      // instead of stranding the run until it times out.
+      ...finalPoll ? { giveUp: true } : {}
     });
     polls += 1;
     if (res.status !== "pending") return res;
-    if (Date.now() > deadline) return res;
+    if (finalPoll) return res;
+    deadline = serverDeadline(deadline, startedAt, res.giveUpAfterMs);
     const wait = Math.min(Math.max(res.retryAfterMs || 5e3, 2e3), 3e4);
     if (polls === 1 || polls % 6 === 0) log.info(`waiting for DevAsign to plan the tests${res.runId ? ` (run ${res.runId})` : ""}\u2026`);
     await sleep3(wait);
+    if (Date.now() >= deadline) finalPoll = true;
   }
 }
 async function executePlan(plan, ws, opts) {
@@ -8284,6 +8372,7 @@ async function executePlan(plan, ws, opts) {
       if (plan.playwright?.installBrowsers || !repoHasPlaywright(ws.root)) {
         const inst = await ensureBrowsers(ws.root, ws);
         if (!inst.ok) log.warn("Chromium install reported a failure; continuing \u2014 the run will tell us if the browser is missing");
+        else setOutput("browsers", "true");
       }
       const generated = pw.filter((t) => t.origin === "generated");
       const existing = pw.filter((t) => t.origin === "existing");
@@ -8304,10 +8393,27 @@ async function executePlan(plan, ws, opts) {
   }
   return { results, artifacts, doctor };
 }
-function summaryTable(results, plan) {
-  const byId = new Map(plan.criteria.map((c) => [c.id, c]));
-  const rows = results.map((r) => `| ${r.criterionIds.map((id) => `${id}. ${byId.get(id)?.text ?? ""}`).join("<br>")} | ${r.status} | ${r.level} ${r.origin} \`${r.test}\` | ${r.attempts.length} |`);
-  return ["| Criterion | Test outcome | Test | Attempts |", "|---|---|---|---|", ...rows].join("\n");
+var cell = (s) => s.replace(/\|/g, "\\|").replace(/\r?\n/g, " ");
+function summaryTable(plan, results) {
+  const unverifiable = new Map((plan.unverifiable ?? []).map((u) => [u.criterionId, u]));
+  const rows = plan.criteria.map((c) => {
+    const mine = results.filter((r) => r.criterionIds.includes(c.id));
+    const u = unverifiable.get(c.id);
+    if (mine.length) {
+      const tests = mine.map((r) => `${r.level} ${r.origin} \`${cell(r.test)}\``).join("<br>");
+      const outcome = mine.map((r) => `${r.status} (${r.attempts.length} attempt${r.attempts.length === 1 ? "" : "s"})`).join("<br>");
+      return `| ${c.id}. ${cell(c.text)} | ${tests} | ${outcome} | |`;
+    }
+    const note = u ? `${cell(u.reason)}${u.fixUrl ? ` \u2014 [configure app start](${u.fixUrl})` : ""}` : "no test planned";
+    return `| ${c.id}. ${cell(c.text)} | \u2014 | unverifiable | ${note} |`;
+  });
+  return ["| Criterion | Test | Outcome | Notes |", "|---|---|---|---|", ...rows].join("\n");
+}
+function announceUnverifiable(plan) {
+  for (const u of plan.unverifiable ?? []) {
+    log.warn(`criterion ${u.criterionId} is unverifiable: ${u.reason}${u.fixUrl ? ` \u2014 fix: ${u.fixUrl}` : ""}`);
+  }
+  if (plan.tests.length === 0) log.warn("no tests could be planned for this PR; every criterion will be reported as unverifiable");
 }
 async function run(opts) {
   const started = Date.now();
@@ -8351,7 +8457,10 @@ async function run(opts) {
     plan = resolved.plan;
     runId = resolved.runId;
     log.info(`plan ${plan.planId}: ${plan.tests.length} test(s) for ${plan.criteria.length} criteria (run ${runId})`);
+    setOutput("run-id", runId);
   }
+  announceUnverifiable(plan);
+  const failOn = opts.failOn ?? plan.failOn ?? "never";
   try {
     const { results, artifacts, doctor } = await executePlan(plan, ws, { yml, testTimeoutMs: opts.testTimeoutMs, setup });
     let finalResults = results;
@@ -8383,31 +8492,33 @@ async function run(opts) {
       log.info("results uploaded \u2014 DevAsign is judging; the PR check run and comment will update");
     }
     const counts = finalResults.reduce((m, r) => (m[r.status] = (m[r.status] || 0) + 1, m), {});
-    log.info(`outcome: ${Object.entries(counts).map(([k, v]) => `${v} ${k}`).join(", ") || "no tests"}${doctor ? ` \xB7 setup needs attention: ${doctor.code}` : ""}`);
+    const unplanned = plan.unverifiable?.length ?? 0;
+    const outcome = `${Object.entries(counts).map(([k, v]) => `${v} ${k}`).join(", ") || "no tests ran"}${unplanned ? `, ${unplanned} unverifiable by plan` : ""}`;
+    log.info(`outcome: ${outcome}${doctor ? ` \xB7 setup needs attention: ${doctor.code}` : ""}`);
+    setOutput("outcome", outcome);
     if (process.env.GITHUB_STEP_SUMMARY) {
       try {
-        appendFileSync(process.env.GITHUB_STEP_SUMMARY, `## DevAsign verification
+        appendFileSync2(process.env.GITHUB_STEP_SUMMARY, `## DevAsign verification
 
 ${doctor ? `> Setup needs attention: ${doctor.message}
 
-` : ""}${summaryTable(finalResults, plan)}
+` : ""}${summaryTable(plan, finalResults)}
 
 Verdicts are judged by DevAsign and posted on the PR (check run "DevAsign \xB7 Verify").
 `);
       } catch {
       }
     }
-    if (opts.failOn === "verdict" && api) {
+    if (failOn !== "never" && api) {
       const deadline = Date.now() + 10 * 6e4;
       while (Date.now() < deadline) {
         const view = await api.getRun(runId);
         if (view.terminal) {
           const fails = view.run.verdicts.filter((v) => v.verdict === "fail");
-          if (fails.length) {
-            log.error(`${fails.length} criteria failed verification: ${fails.map((f) => f.criterionId).join(", ")}`);
-            return 1;
-          }
-          return 0;
+          const unverified = failOn === "unverifiable" ? view.run.verdicts.filter((v) => v.verdict === "unverifiable") : [];
+          for (const v of unverified) log.error(`criterion ${v.criterionId} could not be verified: ${v.reason}${v.fixUrl ? ` \u2014 fix: ${v.fixUrl}` : ""}`);
+          if (fails.length) log.error(`${fails.length} criteria failed verification: ${fails.map((f) => f.criterionId).join(", ")}`);
+          return fails.length || unverified.length ? 1 : 0;
         }
         await sleep3(5e3);
       }
@@ -8431,17 +8542,24 @@ Usage: devasign-verify [run|detect|doctor] [options]
 
 Options:
   --api-url <url>          DevAsign API origin (env DEVASIGN_API_URL)
-  --fail-on never|verdict  Fail the job on a failed criterion (default never)
+  --fail-on <mode>         never (default): always exit 0; verdict: fail the job on a
+                           failed criterion; unverifiable: also fail when a criterion
+                           could not be verified. Unset: the repo's DevAsign setting
   --audience <aud>         OIDC audience (default devasign)
   --token <jwt>            Use this token instead of the Actions OIDC token (local runs)
   --pr <n> --sha <sha>     Override the PR number / head sha (local runs)
-  --resolve-timeout <s>    Max seconds to wait for a plan (default 600)
+  --resolve-timeout <s>    Max seconds to wait for a plan (default 180; DevAsign
+                           may shorten it and re-run this workflow when ready)
   --test-timeout <s>       Per test-file timeout in seconds (default 600)
   --plan-file <path>       Offline: run this plan JSON with no API
   --results-out <path>     Write the results JSON to this path
   --keep                   Keep .devasign/tests and artifacts after the run
   --cwd <dir>              Repository checkout (default cwd)
 `;
+var seconds = (raw, fallback) => {
+  const n = Number(raw);
+  return raw !== void 0 && Number.isFinite(n) && n >= 0 ? n : fallback;
+};
 async function main(argv = process.argv.slice(2)) {
   const { values, positionals } = parseArgs({
     args: argv,
@@ -8487,7 +8605,12 @@ async function main(argv = process.argv.slice(2)) {
     log.error("--api-url (or DEVASIGN_API_URL) is required");
     return 2;
   }
-  const failOn = values["fail-on"] === "verdict" ? "verdict" : "never";
+  const failOnValue = values["fail-on"];
+  if (failOnValue !== void 0 && !["never", "verdict", "unverifiable"].includes(failOnValue)) {
+    log.error(`--fail-on must be never, verdict or unverifiable (got "${failOnValue}")`);
+    return 2;
+  }
+  const failOn = failOnValue;
   const tokenValue = values.token || process.env.DEVASIGN_TOKEN;
   const token = tokenValue ? staticTokenSource(tokenValue) : actionsTokenSource(values.audience || process.env.DEVASIGN_OIDC_AUDIENCE || "devasign");
   try {
@@ -8495,8 +8618,8 @@ async function main(argv = process.argv.slice(2)) {
       apiUrl,
       token,
       failOn,
-      resolveTimeoutMs: (Number(values["resolve-timeout"]) || 600) * 1e3,
-      testTimeoutMs: (Number(values["test-timeout"]) || 600) * 1e3,
+      resolveTimeoutMs: seconds(values["resolve-timeout"], 180) * 1e3,
+      testTimeoutMs: seconds(values["test-timeout"], 600) * 1e3,
       keep: !!values.keep,
       cwd,
       pr: values.pr ? Number(values.pr) : void 0,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@devasign/verify",
-  "version": "1.0.1",
+  "version": "1.2.0",
   "description": "Runs DevAsign's generated acceptance tests inside your CI and reports per-criterion evidence.",
   "license": "GPL-3.0-or-later",
   "type": "module",
@@ -23,7 +23,7 @@
     "prepublishOnly": "npm run typecheck && npm test && npm run build"
   },
   "dependencies": {
-    "@playwright/test": "^1.55.0",
+    "@playwright/test": "1.55.0",
     "tsx": "^4.22.4",
     "yaml": "^2.9.0"
   },