@piwitests/reporter 0.56.0 → 0.57.0

package/dist/cli/index.js

@@ -1137,13 +1137,18 @@ var ASSERTION_MATCHERS = [
   "toBeHidden",
   "toBeEnabled",
   "toBeDisabled",
-  "toHaveURL"
+  "toHaveURL",
+  "toContainText",
+  "toHaveCount",
+  "toBeChecked"
 ];
 var VALUE_MATCHERS = /* @__PURE__ */ new Set([
   "toHaveText",
   "toHaveValue",
   "toHaveAccessibleName",
-  "toHaveURL"
+  "toHaveURL",
+  "toContainText",
+  "toHaveCount"
 ]);
 
 // ../core/src/function-match.ts
@@ -1833,7 +1838,7 @@ function renderAssertStep(step, index, ctx) {
       });
       return [`  // Step ${index + 1}: ${matcher} without an expected value.`];
     }
-    arg = quote(matcher === "toHaveURL" ? urlForCode(assertion.expected, ctx) : assertion.expected);
+    arg = matcher === "toHaveCount" ? String(Number.parseInt(assertion.expected, 10) || 0) : quote(matcher === "toHaveURL" ? urlForCode(assertion.expected, ctx) : assertion.expected);
   }
   const subject = matcher === "toHaveURL" ? ctx.page : locatorForStep(step, index, ctx);
   let line = `  await expect(${subject})${not}.${matcher}(${arg});`;
@@ -1846,10 +1851,25 @@ function renderAssertStep(step, index, ctx) {
 function fileNames(value) {
   return (value ?? "").split("\n").map((name) => name.trim().replace(/^.*[\\/]/, "")).filter(Boolean);
 }
+function renderManualStep(step, index, ctx) {
+  const description = commentText(step.manual?.description ?? "") || "a step to do by hand";
+  ctx.warnings.push({
+    step: index,
+    code: "manual-step",
+    detail: description,
+    message: `Step ${index + 1} cannot be written as code: ${description}`
+  });
+  const lines = [`  // Manual step: ${description}`];
+  const call = step.manual?.call ? commentText(step.manual.call) : "";
+  const location = step.manual?.location ? commentText(step.manual.location) : "";
+  if (call || location) lines.push(`  //   ${[call, location ? `at ${location}` : ""].filter(Boolean).join(" ")}`);
+  return lines;
+}
 function renderRawStep(step, index, ctx) {
   const { page } = ctx;
   if (step.action === "goto") return [`  await ${page}.goto(${quote(urlForCode(step.value ?? step.pageUrl, ctx))});`];
   if (step.action === "assert" || step.action === "assertVisible") return renderAssertStep(step, index, ctx);
+  if (step.action === "manual") return renderManualStep(step, index, ctx);
   if (step.action === "press" && !step.target) {
     return [`  await ${page}.keyboard.press(${quote(step.value ?? "Enter")});`];
   }
@@ -2060,6 +2080,7 @@ function expectFailLine(options) {
   return reason ? `  test.fail(); // ${reason}` : "  test.fail();";
 }
 function samePage(a, b) {
+  if (!a || !b) return true;
   const ka = pageKey(a);
   const kb = pageKey(b);
   return ka === null || kb === null || ka === kb;
@@ -2111,7 +2132,7 @@ function renderSpec(session, options = {}) {
   const pageStarts = [];
   const enterPageOf = (index) => {
     const url = steps[index]?.pageUrl;
-    if (options.testSteps !== "page" || url == null) return;
+    if (options.testSteps !== "page" || !url) return;
     const last = pageStarts[pageStarts.length - 1];
     if (!last || !samePage(last.url, url)) pageStarts.push({ line: bodyLines.length, url });
   };
@@ -2123,7 +2144,7 @@ function renderSpec(session, options = {}) {
     const pattern = pageUrlPattern(next.pageUrl);
     enterPageOf(last + 1);
     if (pattern) bodyLines.push(`  await expect(${page}).toHaveURL(${pattern});`);
-    const acts = next.action !== "assert" && next.action !== "assertVisible";
+    const acts = next.action !== "assert" && next.action !== "assertVisible" && next.action !== "manual";
     const chosen = acts && next.target ? chooseLocator(next.target, options) : null;
     if (chosen) bodyLines.push(`  await expect(${page}.${chosen.text}).toHaveCount(1);`);
   };
@@ -2225,7 +2246,11 @@ function renderSpec(session, options = {}) {
 }
 
 // ../core/src/steps.ts
-var STEPS_VERSION = 1;
+var STEPS_VERSION = 2;
+var VERSION_2_MATCHERS = /* @__PURE__ */ new Set(["toContainText", "toHaveCount", "toBeChecked"]);
+function stepsVersionFor(steps) {
+  return steps.some((s) => s.action === "manual" || !!s.assertion && VERSION_2_MATCHERS.has(s.assertion.matcher)) ? 2 : 1;
+}
 var STEPS_LIMITS = {
   steps: 200,
   alternatives: 10,
@@ -2252,7 +2277,8 @@ var ACTIONS = /* @__PURE__ */ new Set([
   "setInputFiles",
   "dragTo",
   "assertVisible",
-  "assert"
+  "assert",
+  "manual"
 ]);
 function originOf2(url) {
   try {
@@ -2277,7 +2303,7 @@ function toAbsolute(url, origin) {
 function toStepsDocument(session, meta = {}) {
   const origin = originOf2(session.startUrl) ?? originOf2(session.steps[0]?.pageUrl ?? "");
   return {
-    v: STEPS_VERSION,
+    v: stepsVersionFor(session.steps),
     title: meta.title ?? null,
     origin,
     recordedAt: session.startedAt,
@@ -2386,6 +2412,9 @@ function checkAssertion(c, v, path24) {
     return void 0;
   }
   const expected = c.text(v.expected, `${path24}.expected`, STEPS_LIMITS.valueLength, !VALUE_MATCHERS.has(matcher));
+  if (matcher === "toHaveCount" && expected != null && !/^\d{1,6}$/.test(expected)) {
+    c.fail(`${path24}.expected`, "must be a whole number of elements on a toHaveCount assertion");
+  }
   return {
     matcher,
     expected: VALUE_MATCHERS.has(matcher) ? expected : null,
@@ -2394,6 +2423,19 @@ function checkAssertion(c, v, path24) {
     note: c.text(v.note, `${path24}.note`, STEPS_LIMITS.valueLength, true)
   };
 }
+function checkManual(c, v, path24) {
+  if (!isObject(v)) {
+    c.fail(path24, "is required on a manual step");
+    return void 0;
+  }
+  const description = c.text(v.description, `${path24}.description`, STEPS_LIMITS.valueLength, false);
+  if (description != null && !description.trim()) c.fail(`${path24}.description`, "must not be empty");
+  return {
+    description: description ?? "",
+    location: c.text(v.location, `${path24}.location`, STEPS_LIMITS.textLength, true),
+    call: c.text(v.call, `${path24}.call`, STEPS_LIMITS.textLength, true)
+  };
+}
 function checkViewports(c, v, stepCount) {
   if (v == null) return [];
   if (!Array.isArray(v)) {
@@ -2466,6 +2508,9 @@ function checkStep(c, v, path24) {
     step.assertion = { matcher: "toBeVisible", expected: null, actual: null, negated: false, note: null };
   } else if (action === "assert") {
     step.assertion = checkAssertion(c, v.assertion, `${path24}.assertion`);
+  } else if (action === "manual") {
+    step.value = null;
+    step.manual = checkManual(c, v.manual, `${path24}.manual`);
   }
   return step;
 }
@@ -2481,8 +2526,8 @@ function parseSteps(input) {
   if (!isObject(value)) return { ok: false, errors: ["a steps document must be a JSON object"] };
   const c = new Checker();
   const legacy = value.v == null && typeof value.startUrl === "string";
-  if (!legacy && value.v !== STEPS_VERSION) {
-    return { ok: false, errors: [`v: this reads version ${STEPS_VERSION}, not ${JSON.stringify(value.v)}`] };
+  if (!legacy && value.v !== 1 && value.v !== STEPS_VERSION) {
+    return { ok: false, errors: [`v: this reads versions 1 to ${STEPS_VERSION}, not ${JSON.stringify(value.v)}`] };
   }
   if (!Array.isArray(value.steps)) return { ok: false, errors: ["steps: must be a list"] };
   if (value.steps.length > STEPS_LIMITS.steps) {
@@ -2505,7 +2550,7 @@ function parseSteps(input) {
     if (originText && !origin) c.fail("origin", "must be an http(s) origin");
     const viewports = checkViewports(c, value.viewports, steps.length);
     doc = {
-      v: STEPS_VERSION,
+      v: stepsVersionFor(steps),
       title: c.text(value.title, "title", STEPS_LIMITS.textLength, true),
       origin,
       recordedAt: value.recordedAt == null ? 0 : c.number(value.recordedAt, "recordedAt"),

package/dist/index.js

@@ -4549,6 +4549,30 @@ function runFilesOf(answer) {
   }
   return files;
 }
+function instanceOf(answer) {
+  const instance = isRecord2(answer) ? answer.instance : null;
+  if (!isRecord2(instance) || typeof instance.id !== "string" || !REQUEST_ID.test(instance.id)) return null;
+  const startedAt = typeof instance.startedAt === "string" && !Number.isNaN(Date.parse(instance.startedAt)) ? new Date(instance.startedAt).toISOString() : null;
+  return { id: instance.id, startedAt };
+}
+function processKeyOf(id) {
+  const dot = id.lastIndexOf(".");
+  return dot > 0 ? id.slice(0, dot) : null;
+}
+function withTestCoverage(files, tests) {
+  if (!tests || tests.files.length === 0) return files;
+  const byKey = new Map(files.map((file) => [`${file.runtime}\0${file.path}`, { ...file }]));
+  for (const ran of tests.files) {
+    const key = `${ran.runtime}\0${ran.path}`;
+    const file = byKey.get(key) ?? { path: ran.path, runtime: ran.runtime };
+    const functions = normalizeFunctionRanges([...file.functions ?? [], ...ran.functions ?? []]);
+    const lines = normalizeLineRanges([...file.lines ?? [], ...ran.lines ?? []]);
+    if (functions.length > 0) file.functions = functions;
+    if (lines.length > 0) file.lines = lines;
+    byKey.set(key, file);
+  }
+  return [...byKey.values()];
+}
 function isCollectAnswer(answer) {
   return isRecord2(answer) && answer.v === 1 && Array.isArray(answer.groups);
 }
@@ -4600,11 +4624,16 @@ var ServerCoverageCollector = class {
   }
   /**
    * The run-level coverage of an origin, with each server file's executable
-   * lines at `lines` granularity; null when the backend did not answer with it.
+   * lines at `lines` granularity, and the lines and functions the tests'
+   * coverage named added to it. The gap says why part of it is missing: the
+   * backend restarted during the run (its answer names another process than
+   * the run's first request ids), or it did not answer with run-level coverage.
    */
   async fetchRun(origin) {
     const state = this.stateOf(origin);
     const query = `?granularity=${encodeURIComponent(this.options.granularity)}`;
+    let answer = null;
+    let failure = null;
     for (const base of this.pathBases(origin)) {
       const outcome = await this.call(
         `${origin}${base}${RUN_PATH}${query}`,
@@ -4612,21 +4641,47 @@ var ServerCoverageCollector = class {
         this.limits.runTimeoutMs
       );
       if (outcome.kind === "failed") {
-        this.reportFailure(state, `Could not fetch the run's server coverage from ${origin}: ${outcome.reason}.`);
-        return null;
+        failure = outcome.reason;
+        break;
       }
       const answered = outcome.ok ? runFilesOf(outcome.json) : null;
       if (answered) {
         state.base = base;
-        const files = await this.verifyChecksums(
-          answered,
-          fileChecksums(isRecord2(outcome.json) ? outcome.json.files : null)
-        );
-        return this.options.blobs ? this.options.blobs.withBlobs(files.files) : files.files;
+        answer = { files: answered, json: outcome.json };
+        break;
       }
     }
-    this.reportFailure(state, `Could not fetch the run's server coverage: ${this.noEndpoint(origin, RUN_PATH)}.`);
-    return null;
+    const gap = answer ? this.restartGap(origin, state, instanceOf(answer.json)) : { origin, reason: "unreachable" };
+    if (!answer) failure ??= this.noEndpoint(origin, RUN_PATH);
+    if (gap && !this.closer.signal.aborted) this.logger.warn(this.gapMessage(gap, failure));
+    const checksums = fileChecksums(answer?.json.files ?? null, new Map(state.testChecksums));
+    const files = await this.verifyChecksums(withTestCoverage(answer?.files ?? [], state.testCoverage), checksums);
+    let out = files.files;
+    try {
+      if (this.options.blobs) out = await this.options.blobs.withBlobs(out);
+    } catch {
+    }
+    return { files: out, gap };
+  }
+  /**
+   * The gap of an origin whose run-level coverage `instance` answered: a
+   * restart when it, or a collection answer, names another process than the
+   * run's first request ids; null when the backend names no process.
+   */
+  restartGap(origin, state, instance) {
+    if (!instance) return null;
+    const keys = [...state.processKeys];
+    const others = [...state.instances.values()].filter((known) => known.id !== instance.id);
+    if (keys.every((key) => key === instance.id) && others.length === 0) return null;
+    const at = keys[0] === instance.id ? others[others.length - 1]?.startedAt ?? null : instance.startedAt;
+    return { origin, reason: "restarted", ...at ? { at } : {} };
+  }
+  gapMessage(gap, failure) {
+    if (gap.reason === "unreachable") {
+      return `Could not fetch the run's server coverage from ${gap.origin} (${failure}): the run's server coverage is marked incomplete, and keeps the lines the tests' coverage named.`;
+    }
+    const when = gap.at ? ` at about ${gap.at}` : "";
+    return `The backend at ${gap.origin} restarted during the run${when}: the run's server coverage is marked incomplete. It keeps the lines the tests' coverage named; what the earlier process ran outside a request is lost.`;
   }
   /** Abort every call in flight and fail every later one; the tests still waiting resolve incomplete. */
   close() {
@@ -4648,7 +4703,11 @@ var ServerCoverageCollector = class {
         timer: null,
         base: null,
         limit: createLimiter(this.limits.concurrency),
-        warned: false
+        warned: false,
+        processKeys: /* @__PURE__ */ new Set(),
+        instances: /* @__PURE__ */ new Map(),
+        testCoverage: null,
+        testChecksums: /* @__PURE__ */ new Map()
       };
       this.originStates.set(origin, state);
     }
@@ -4718,6 +4777,13 @@ var ServerCoverageCollector = class {
   }
   /** Send one call and hand each group's answer to its test. */
   async send(origin, groups) {
+    const state = this.stateOf(origin);
+    for (const group of groups) {
+      for (const id of group.ids) {
+        const key = processKeyOf(id);
+        if (key !== null) state.processKeys.add(key);
+      }
+    }
     let answer = null;
     try {
       const body = JSON.stringify({
@@ -4728,6 +4794,8 @@ var ServerCoverageCollector = class {
     } catch (error) {
       this.logger.debug(`Server coverage collection from ${origin} failed: ${errorMessage(error)}`);
     }
+    const instance = instanceOf(answer);
+    if (instance && !state.instances.has(instance.id)) state.instances.set(instance.id, instance);
     const answered = /* @__PURE__ */ new Map();
     for (const group of answer?.groups ?? []) {
       if (isRecord2(group) && typeof group.key === "string") answered.set(group.key, group);
@@ -4739,9 +4807,11 @@ var ServerCoverageCollector = class {
       } else {
         if (Array.isArray(result.missing) && result.missing.length > 0) group.entry.incomplete = true;
         fileChecksums(result.files, group.entry.checksums);
+        fileChecksums(result.files, state.testChecksums);
         const capture = sanitizeCoverage({ v: 1, files: result.files });
         if (capture) {
           group.entry.captures.push(capture);
+          state.testCoverage = mergeCoverage(state.testCoverage, capture);
           if (capture.incomplete) group.entry.incomplete = true;
         }
       }
@@ -10496,17 +10566,26 @@ var CoverageCollection = class {
   }
   /**
    * Send the run-level coverage of every origin, and the browser's, to the
-   * dashboard run, in requests within its limits. A failure is logged once,
-   * never thrown.
+   * dashboard run, in requests within its limits; the first names the origins
+   * whose server coverage is incomplete (`incomplete`), and is sent for them
+   * alone when no file is. A failure is logged once, never thrown.
    */
   async sendRunCoverage(httpClient, runId, auth) {
     if (!this.collector && !this.browserBlobs) return;
     this.fetchRunCoverage();
-    const server = (await Promise.all(this.runFiles.values())).flatMap((result) => result ?? []);
+    const runs = await Promise.all(this.runFiles.values());
+    const server = runs.flatMap((result) => result?.files ?? []);
+    const gaps = runs.flatMap((result) => result?.gap ? [result.gap] : []);
     const files = [...server, ...await this.browserRunFiles()];
-    for (const chunk of chunkRunCoverage(files)) {
+    const chunks = chunkRunCoverage(files);
+    if (chunks.length === 0 && gaps.length > 0) chunks.push([]);
+    for (const [i, chunk] of chunks.entries()) {
       try {
-        await httpClient.putJSON(`/api/test-runs/${runId}/coverage`, { v: 1, files: chunk }, auth);
+        await httpClient.putJSON(
+          `/api/test-runs/${runId}/coverage`,
+          { v: 1, files: chunk, ...i === 0 && gaps.length > 0 ? { incomplete: gaps } : {} },
+          auth
+        );
       } catch (error) {
         this.logger.warn(`Could not send the run's coverage to the dashboard: ${errorMessage(error)}`);
         return;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@piwitests/reporter",
-  "version": "0.56.0",
+  "version": "0.57.0",
   "description": "Playwright reporter that streams results, traces and HTML reports to a Piwi Dashboard instance",
   "url": "https://github.com/PiwiTests/platform",
   "homepage": "https://piwitests.dev",