@plune-ai/cli 0.14.3 → 0.16.0

package/dist/reporter-core.d.ts

@@ -44,12 +44,18 @@ interface Execution {
     retry: number;
     worker?: string;
 }
-/** A link, not a file. Uploading artefacts is C3; this carries what someone else already hosts. */
+/** A link, not a file: this carries what someone else already hosts. A file the runner kept is `files`. */
 interface Attachment {
     name: string;
     url: string;
     contentType?: string;
 }
+/** A file the runner kept for an attempt, where it lies on this machine. */
+interface ResultFile {
+    name: string;
+    contentType?: string;
+    path: string;
+}
 /** One assertion's outcome, in the shape the platform already stores for its own runner. */
 interface AssertionRecord {
     type: string;
@@ -108,6 +114,12 @@ interface PendingResult {
     params?: Record<string, unknown>;
     attachments?: Attachment[];
     assertions?: AssertionRecord[];
+    /**
+     * The files the runner kept for this attempt, passed or not — built by `filesOf`. Never part of a
+     * batch: the paths are this machine's. Once the platform has accepted the result, the core uploads
+     * the screenshots among them to it (platform ADR 0040).
+     */
+    files?: ResultFile[];
 }
 /**
  * What a failed attempt says about itself on the run page (#790, platform ADR-0002): every field
@@ -125,7 +137,8 @@ interface FailureDetail {
         line: number;
         column?: number;
     };
-    /** The files the runner kept for the attempt, by name — the files themselves stay in the CI run. */
+    /** The files the runner kept for the attempt, by name. The screenshots among them are uploaded to the
+     * result besides (`files`); the rest stay in the CI run. */
     artifacts?: {
         name: string;
         contentType?: string;
@@ -178,7 +191,7 @@ interface FailedAttempt {
     repoRoot?: string;
 }
 /** What a run submission looks like once a key has resolved. Built by the core, never by a caller. */
-interface ResultSubmission extends Omit<PendingResult, 'keys' | 'testCaseId'> {
+interface ResultSubmission extends Omit<PendingResult, 'keys' | 'testCaseId' | 'files'> {
     testCaseId: string;
 }
 /** One entry of the "what this run intended to execute" list, from which the platform derives
@@ -187,6 +200,12 @@ interface ExpectedEntry {
     testCaseId?: string;
     externalKey?: KeyRef;
 }
+/**
+ * Why the platform would not take the tests offered to it (plune-ai/cli#71): its review queue is at its
+ * ceiling, the project is at its case limit, or anything else — a throttle, an outage, a run that is not
+ * there. What a person does next depends on which, and only the first two have advice of their own.
+ */
+type UnofferedWhy = 'queue' | 'cases' | 'other';
 /** The run as the platform answers with it. Only `id` is load-bearing for the client. */
 interface RunRecord {
     id: string;
@@ -227,12 +246,28 @@ interface RunStats extends SubmitCounts {
      * queue at capacity means "empty it and import again"; nothing else here means that.
      */
     unoffered: number;
+    /**
+     * Why they were not, said by the platform's own words and present only when `unoffered` is not zero:
+     * that is what carries the reason the paragraph above says a subtraction cannot. The platform's line
+     * about it is printed with the failure; this is what a command turns into the advice that fits.
+     */
+    unofferedWhy?: UnofferedWhy;
     /**
      * Results not delivered, written to the fallback file instead of the platform (#790): a batch
      * refused for any reason — its size, the network, the token, a closed run — or a result whose keys
      * the platform could not be asked about. Apart from `rejected` and `unresolved`, which it did answer.
      */
     deferred: number;
+    /**
+     * The screenshots of the accepted results (platform ADR 0040): uploaded to their result; skipped
+     * because the platform would refuse them or the file is not here; refused or unanswered on the way.
+     * None of them costs a result — a result is reported whether its screenshots arrive or not.
+     */
+    screenshots: {
+        uploaded: number;
+        skipped: number;
+        failed: number;
+    };
 }
 /** Everything the core needs to talk to a deployment. */
 interface ReporterConfig {
@@ -352,6 +387,20 @@ interface EnvSettings {
     ignored: string[];
 }
 declare function readEnv(env?: NodeJS.ProcessEnv): EnvSettings;
+/**
+ * The commit and the branch a run belongs to (plune-ai/plune#927), for `meta.sha` and `meta.branch`.
+ *
+ * `reported` is what the runner said about its CI — Playwright's `metadata.ci`, or the copy of it in
+ * its JSON report — and outranks the job's variables: it is a project's own word where the CI is one
+ * Playwright does not know, and it is where the tests ran when another job imports the report. So a
+ * report naming a commit other than this job's gets that commit and no branch of this job's: half of
+ * each would be a commit on a branch it is not on. Playwright gives no branch on GitHub at all, which
+ * is why the job's variables are read there.
+ *
+ * Each field is present only when something said it: outside a CI there is nothing to say, and nothing
+ * is guessed.
+ */
+declare function ciCommit(reported?: unknown, env?: NodeJS.ProcessEnv): Pick<RunMeta, 'sha' | 'branch'>;
 
 /**
  * What ends up in a deferred batch.
@@ -374,10 +423,21 @@ declare const DEFAULT_FALLBACK_PATH: string;
  *
  * `runId: null` is a real state, not a missing value — the platform was unreachable when the run
  * was supposed to be created, so these results have no run yet and replay has to make one.
+ *
+ * `meta` is where that run came from — the commit and branch, the CI run, the runner (plune-ai/plune#927).
+ * Recorded so the run replay makes is the run the reporter would have made; replay reads it and nothing
+ * else about the machine it runs on. A line written before the field existed has none.
+ *
+ * `session` marks the reporter session that wrote the line, made once by that session (plune-ai/cli#45).
+ * A run of 24 tests at a batch of 10 is three lines, and replay opens one run for the lines that share a
+ * marker rather than one for each. A line written before the field existed has none, and is one run's
+ * worth by itself — nothing in it says otherwise.
  */
 interface DeferredBatch {
     runId: string | null;
     externalKey: string | null;
+    session?: string;
+    meta?: RunMeta;
     results: DeferredResult[];
 }
 /**
@@ -415,6 +475,12 @@ declare function repoRootOf(dir: string, stopAt?: string): string | undefined;
  * searched for credentials: the platform does that before its own cut (ADR-0005).
  */
 declare function errorContextOf(attempt: FailedAttempt, home?: string): string;
+/**
+ * The files the runner kept for one attempt, whatever its status: a screenshot of a test that passed
+ * is how a person sees what it checked (plune-ai/plune#913). Which of them the platform takes, the
+ * session decides as it uploads — once, for both roads.
+ */
+declare function filesOf(attempt: FailedAttempt): ResultFile[];
 /**
  * A link only for an address a browser opens as a page — never `file:`, `javascript:` or half a URL.
  * As the platform checks it: `new URL` alone mends `https:/host` and `https:host` into https, and a
@@ -452,6 +518,11 @@ interface ClientOptions {
 }
 interface PlatformClient {
     post<T>(path: string, body: unknown): Promise<ClientOutcome<T>>;
+    /**
+     * A file's bytes as the body, under the file's own type (platform ADR 0040) — not JSON. Through the
+     * same retries as a POST, so a 429 on the platform's upload limit waits out its `Retry-After`.
+     */
+    upload<T>(path: string, bytes: Buffer, contentType: string): Promise<ClientOutcome<T>>;
     /**
      * A DELETE, through the same retries, the same classification and the same token scrubbing as a
      * POST. It is here rather than as a bare `fetch` in the one command that needs it because all
@@ -462,4 +533,4 @@ interface PlatformClient {
 }
 declare function createClient(opts: ClientOptions): PlatformClient;
 
-export { type AssertionRecord, type Attachment, type AttemptError, type ClientFailureKind, type ClientOptions, type ClientOutcome, DEFAULT_FALLBACK_PATH, type DeclaredStep, type DeferredBatch, type DeferredResult, type EnvSettings, type Execution, type ExpectedEntry, type ExternalKeyKind, type FailedAttempt, type FailureDetail, type KeyRef, type PendingResult, type PlatformClient, RESULT_KEY_MAX, type ReporterConfig, type ResultStatus, type ResultSubmission, type RunKind, type RunMeta, type RunRecord, type RunSession, type RunStats, type RunnerLocation, type SubmitCounts, UNSUPPORTED_VARS, appendBatch, createClient, errorContextOf, failureOf, readEnv, repoRootOf, resultKey, startRun, webLink };
+export { type AssertionRecord, type Attachment, type AttemptError, type ClientFailureKind, type ClientOptions, type ClientOutcome, DEFAULT_FALLBACK_PATH, type DeclaredStep, type DeferredBatch, type DeferredResult, type EnvSettings, type Execution, type ExpectedEntry, type ExternalKeyKind, type FailedAttempt, type FailureDetail, type KeyRef, type PendingResult, type PlatformClient, RESULT_KEY_MAX, type ReporterConfig, type ResultFile, type ResultStatus, type ResultSubmission, type RunKind, type RunMeta, type RunRecord, type RunSession, type RunStats, type RunnerLocation, type SubmitCounts, UNSUPPORTED_VARS, type UnofferedWhy, appendBatch, ciCommit, createClient, errorContextOf, failureOf, filesOf, readEnv, repoRootOf, resultKey, startRun, webLink };

package/dist/reporter-core.js

@@ -1,5 +1,7 @@
 // src/reporter-core/session.ts
-import { createHash } from "crypto";
+import { createHash, randomUUID } from "crypto";
+import { promises as disk } from "fs";
+import { basename as basename2, extname } from "path";
 
 // src/cli/api-url.ts
 var DEFAULT_API_URL = "https://beta-api.plune.ai";
@@ -88,7 +90,7 @@ function createClient(opts) {
   const wait = opts.wait ?? sleep;
   const base = opts.apiUrl.replace(/\/+$/, "");
   const scrub = (text) => opts.token === "" ? text : text.split(opts.token).join("***");
-  async function send(method, path3, body) {
+  async function send(method, path3, payload) {
     let last = {
       kind: "unavailable",
       status: null,
@@ -99,11 +101,11 @@ function createClient(opts) {
       try {
         res = await doFetch(`${base}${path3}`, {
           method,
-          headers: { "content-type": "application/json", authorization: `Bearer ${opts.token}` },
+          headers: { "content-type": payload?.type ?? "application/json", authorization: `Bearer ${opts.token}` },
           // A DELETE carries none. Sending `"undefined"` as a body is what a naive
           // `JSON.stringify(body)` would do, and some proxies answer that with a 400 nobody can
           // explain.
-          ...body === void 0 ? {} : { body: JSON.stringify(body) }
+          ...payload === void 0 ? {} : { body: payload.body }
         });
       } catch (err) {
         last = { kind: "unavailable", status: null, detail: scrub(String(err)) };
@@ -129,7 +131,8 @@ function createClient(opts) {
     return { ok: false, ...last };
   }
   return {
-    post: (path3, body) => send("POST", path3, body),
+    post: (path3, body) => send("POST", path3, { body: JSON.stringify(body), type: "application/json" }),
+    upload: (path3, bytes, contentType) => send("POST", path3, { body: bytes, type: contentType }),
     del: (path3) => send("DELETE", path3)
   };
 }
@@ -196,6 +199,28 @@ function defaultRunTitle(env = process.env, now = /* @__PURE__ */ new Date(), cw
   const where = flag(env, "CI") || flag(env, "GITHUB_ACTIONS") ? "ci" : "local";
   return `${path.basename(cwd)} \xB7 ${stamp} \xB7 ${where}`;
 }
+var PROVIDERS = [
+  { is: "GITHUB_ACTIONS", sha: "GITHUB_SHA", branch: ["GITHUB_HEAD_REF", "GITHUB_REF_NAME"] },
+  { is: "GITLAB_CI", sha: "CI_COMMIT_SHA", branch: ["CI_COMMIT_REF_NAME"] },
+  { is: "JENKINS_URL", sha: "GIT_COMMIT", branch: ["GIT_BRANCH"] }
+];
+function ciCommit(reported, env = process.env) {
+  const said = typeof reported === "object" && reported !== null ? reported : {};
+  const text = (raw) => typeof raw === "string" && raw.trim() !== "" ? raw.trim() : void 0;
+  const provider = PROVIDERS.find((p) => value(env, p.is) !== void 0);
+  const job = {
+    sha: provider === void 0 ? void 0 : value(env, provider.sha),
+    branch: provider?.branch.map((name) => value(env, name)).find((v) => v !== void 0)
+  };
+  const sha = text(said["commitHash"]);
+  const sameCommit = sha === void 0 || sha === job.sha;
+  const commit = sha ?? job.sha;
+  const branch = text(said["branch"]) ?? (sameCommit ? job.branch : void 0);
+  return {
+    ...commit !== void 0 ? { sha: commit } : {},
+    ...branch !== void 0 ? { branch } : {}
+  };
+}
 
 // src/reporter-core/fallback.ts
 import * as fs from "fs";
@@ -207,6 +232,8 @@ function appendBatch(file, batch) {
     ts: (/* @__PURE__ */ new Date()).toISOString(),
     runId: batch.runId,
     externalKey: batch.externalKey,
+    ...batch.session !== void 0 ? { session: batch.session } : {},
+    ...batch.meta !== void 0 ? { meta: batch.meta } : {},
     results: batch.results
   };
   fs.appendFileSync(file, `${JSON.stringify(line)}
@@ -222,6 +249,37 @@ var EXPECTED_MAX = 1e4;
 var KEY_MAX = 1024;
 var TITLE_MAX = 300;
 var COUNTED = ["accepted", "duplicate", "conflict", "rejected"];
+var SCREENSHOT_TYPES = /* @__PURE__ */ new Set(["image/png", "image/jpeg", "image/webp"]);
+var SCREENSHOT_BYTES = 2 * 1024 * 1024;
+var SCREENSHOTS_PER_RESULT = 20;
+var TYPE_BY_EXTENSION = {
+  ".png": "image/png",
+  ".jpg": "image/jpeg",
+  ".jpeg": "image/jpeg",
+  ".webp": "image/webp"
+};
+var UPLOADS_AT_ONCE = 4;
+var NAME_MAX = 200;
+function nameOf(file) {
+  const own = file.name.split(/[\\/]/).pop()?.trim() || basename2(file.path);
+  const fileExt = extname(file.path);
+  const ownExt = extname(own);
+  const typed = ownExt !== "" && (ownExt.toLowerCase() === fileExt.toLowerCase() || TYPE_BY_EXTENSION[ownExt.toLowerCase()] !== void 0);
+  const ext = typed ? ownExt : fileExt;
+  let stem = typed ? own.slice(0, -ownExt.length) : own;
+  if (stem.length + ext.length > NAME_MAX) {
+    stem = stem.slice(0, Math.max(0, NAME_MAX - ext.length));
+    if (/[\uD800-\uDBFF]$/.test(stem)) stem = stem.slice(0, -1);
+  }
+  return `${stem}${ext}`;
+}
+var SKIPPED = {
+  hidden: "its name starts with an underscore, which hides an attachment in Playwright's own reports too",
+  type: "Plune takes PNG, JPEG and WebP",
+  limit: `a result holds at most ${SCREENSHOTS_PER_RESULT}`,
+  missing: "the file cannot be read on this machine",
+  size: "Plune takes a file of at most 2 MB"
+};
 function boundKey(key) {
   if (key.value.length <= KEY_MAX) return key;
   const digest = createHash("sha256").update(key.value).digest("hex").slice(0, 16);
@@ -229,6 +287,12 @@ function boundKey(key) {
   if (/[\uD800-\uDBFF]$/.test(head)) head = head.slice(0, -1);
   return { ...key, value: `${head}#${digest}` };
 }
+function whyUnoffered(out) {
+  if (out.ok || out.status !== 429) return "other";
+  if (/review queue quota reached/i.test(out.detail)) return "queue";
+  if (/test case quota reached/i.test(out.detail)) return "cases";
+  return "other";
+}
 function boundTitle(title) {
   return title.length <= TITLE_MAX ? title : `${title.slice(0, TITLE_MAX - 1)}\u2026`;
 }
@@ -272,6 +336,7 @@ async function startRun(passed, declared) {
 `));
   const fallbackPath = cfg.fallbackPath ?? DEFAULT_FALLBACK_PATH;
   const externalKey = cfg.externalKey ?? null;
+  const sessionMarker = randomUUID();
   const batchSize = Math.min(cfg.batchSize ?? BATCH_DEFAULT, BATCH_MAX);
   const offerDiscovered = cfg.offerDiscovered ?? false;
   const token = cfg.token ?? loadToken() ?? "";
@@ -289,7 +354,8 @@ async function startRun(passed, declared) {
     offered: 0,
     created: 0,
     unoffered: 0,
-    deferred: 0
+    deferred: 0,
+    screenshots: { uploaded: 0, skipped: 0, failed: 0 }
   };
   const resolved = /* @__PURE__ */ new Map();
   let buffer = [];
@@ -299,6 +365,7 @@ async function startRun(passed, declared) {
   let offline = token === "";
   let done = false;
   const shortened = /* @__PURE__ */ new Set();
+  const skipsSaid = /* @__PURE__ */ new Set();
   function bound(keys) {
     return keys.map((key) => {
       const fit = boundKey(key);
@@ -321,7 +388,13 @@ async function startRun(passed, declared) {
     log(`plune: ignoring ${env.ignored.join(", ")} \u2014 a run has nowhere to store them yet.`);
   }
   function defer(results) {
-    appendBatch(fallbackPath, { runId, externalKey, results });
+    appendBatch(fallbackPath, {
+      runId,
+      externalKey,
+      session: sessionMarker,
+      ...cfg.meta !== void 0 ? { meta: cfg.meta } : {},
+      results
+    });
     stats.deferred += results.length;
   }
   async function resolveKeys(values) {
@@ -418,6 +491,7 @@ async function startRun(passed, declared) {
       });
       if (!out.ok || !Array.isArray(out.body?.results)) {
         stats.unoffered = batches.slice(index).reduce((n, rest) => n + rest.length, 0);
+        stats.unofferedWhy = whyUnoffered(out);
         log(
           `plune: could not offer ${stats.unoffered} unknown test(s) for review (${out.ok ? "the answer had no results" : out.detail || out.kind}).`
         );
@@ -427,6 +501,60 @@ async function startRun(passed, declared) {
       stats.created += out.body.results.filter((r) => r?.outcome === "created").length;
     }
   }
+  function skip(name, why) {
+    stats.screenshots.skipped += 1;
+    if (skipsSaid.has(why)) return;
+    skipsSaid.add(why);
+    log(`plune: screenshot "${name}" not uploaded \u2014 ${SKIPPED[why]}.`);
+  }
+  async function uploadOne(shot) {
+    let bytes;
+    try {
+      if ((await disk.stat(shot.path)).size > SCREENSHOT_BYTES) return skip(shot.name, "size");
+      bytes = await disk.readFile(shot.path);
+    } catch {
+      return skip(shot.name, "missing");
+    }
+    const out = await client.upload(
+      `/v1/results/${encodeURIComponent(shot.id)}/files?name=${encodeURIComponent(shot.name)}`,
+      bytes,
+      shot.type
+    );
+    if (out.ok) {
+      stats.screenshots.uploaded += 1;
+      return;
+    }
+    stats.screenshots.failed += 1;
+    if (stats.screenshots.failed === 1) {
+      log(`plune: could not upload a screenshot (${out.detail || out.kind}) \u2014 its result is reported without it.`);
+    }
+  }
+  async function upload(items, filesByKey) {
+    if (filesByKey.size === 0 || !Array.isArray(items)) return;
+    const shots = [];
+    for (const item of items) {
+      if (item?.status !== "accepted" || typeof item.id !== "string" || typeof item.resultKey !== "string") continue;
+      let room = SCREENSHOTS_PER_RESULT;
+      for (const file of filesByKey.get(item.resultKey) ?? []) {
+        const type = file.contentType?.split(";")[0]?.trim().toLowerCase() || TYPE_BY_EXTENSION[extname(file.path).toLowerCase()];
+        if (!type?.startsWith("image/")) continue;
+        const name = nameOf(file);
+        if (file.name.startsWith("_")) skip(name, "hidden");
+        else if (!SCREENSHOT_TYPES.has(type)) skip(name, "type");
+        else if (room === 0) skip(name, "limit");
+        else {
+          room -= 1;
+          shots.push({ id: item.id, name, type, path: file.path });
+        }
+      }
+    }
+    let next = 0;
+    await Promise.all(
+      Array.from({ length: Math.min(UPLOADS_AT_ONCE, shots.length) }, async () => {
+        while (next < shots.length) await uploadOne(shots[next++]);
+      })
+    );
+  }
   async function flush() {
     if (buffer.length === 0) return;
     const pending = buffer;
@@ -438,11 +566,13 @@ async function startRun(passed, declared) {
     await resolveKeys(pending.flatMap((p) => p.keys));
     const submissions = [];
     const unanswered = [];
+    const filesByKey = /* @__PURE__ */ new Map();
     for (const result of pending) {
       const match = matchFor(result);
       if (match.kind === "found") {
-        const { keys: _keys, ...rest } = result;
+        const { keys: _keys, files, ...rest } = result;
         submissions.push({ ...rest, testCaseId: match.testCaseId });
+        if (files !== void 0) filesByKey.set(result.resultKey, files);
       } else if (match.kind === "unmatched") {
         stats.unresolved += 1;
         remember(result);
@@ -460,14 +590,16 @@ async function startRun(passed, declared) {
         defer(part);
         continue;
       }
-      const out = await client.post(`/v1/runs/${runId}/results`, {
-        results: part
-      });
+      const out = await client.post(
+        `/v1/runs/${runId}/results`,
+        { results: part }
+      );
       const counts = out.ok ? out.body?.counts : void 0;
       const told = COUNTED.filter((k) => typeof counts?.[k] === "number");
       if (told.length > 0) {
         for (const k of told) stats[k] += counts[k];
         progress();
+        await upload(out.ok ? out.body?.items : void 0, filesByKey);
         continue;
       }
       if (out.ok) {
@@ -497,6 +629,12 @@ async function startRun(passed, declared) {
     if (stats.unresolved > 0) parts.push(`${stats.unresolved} with no matching test case`);
     if (stats.offered > 0) parts.push(`${stats.offered} offered for review`);
     if (stats.created > 0) parts.push(`${stats.created} added as cases (trusted source)`);
+    const shots = stats.screenshots;
+    if (shots.uploaded + shots.skipped + shots.failed > 0) {
+      parts.push(
+        `${shots.uploaded} screenshot(s) uploaded` + (shots.skipped > 0 ? `, ${shots.skipped} skipped` : "") + (shots.failed > 0 ? `, ${shots.failed} failed to upload` : "")
+      );
+    }
     if (stats.deferred > 0) parts.push(`${stats.deferred} not delivered \u2014 written to ${fallbackPath}`);
     if (shortened.size > 0) parts.push(`${shortened.size} test name(s) shortened to fit a key`);
     log(`plune: ${parts.join(" \xB7 ")}`);
@@ -595,6 +733,9 @@ function errorContextOf(attempt, home = homedir2()) {
   const text = attempt.errors.map((e) => e.text).filter((t) => t !== "").join("\n\n");
   return cut(machineless(text, attempt.repoRoot, home));
 }
+function filesOf(attempt) {
+  return attempt.attachments.filter((a) => a.path !== void 0);
+}
 function machineless(raw, repoRoot, home) {
   let text = stripVTControlCharacters(raw).replaceAll("\x1B", "");
   const root = repoRoot === void 0 ? void 0 : pathIn(repoRoot);
@@ -692,9 +833,11 @@ export {
   RESULT_KEY_MAX,
   UNSUPPORTED_VARS,
   appendBatch,
+  ciCommit,
   createClient,
   errorContextOf,
   failureOf,
+  filesOf,
   readEnv,
   repoRootOf,
   resultKey,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@plune-ai/cli",
-  "version": "0.14.3",
+  "version": "0.16.0",
   "description": "AI-powered assertion testing for LLM apps — write assertions, run them against any provider, get a pass/fail report (local, CI, or diff).",
   "keywords": [
     "ai",