@outerlayer/cli 0.33.0-next.20261023003700 → 0.33.0-next.20261023120437

package/skill-pack/maintained/outerlayer-amend/scripts/rerecord-demo.mjs

@@ -0,0 +1,187 @@
+#!/usr/bin/env node
+// Records an item's demo again at the pull request's new head, after an
+// amend pushed. It reads the item's newest demo record, and records again
+// only a demo that was recorded at an earlier commit and names the test that
+// recorded it. Needs only Node.
+//
+//   node rerecord-demo.mjs --item <number> --pr <number> [--repository <owner/name>] [--project <playwright project>]
+//
+// Run it after the amend's last push, with the app running as the
+// repository's instructions say. It prints one line for the amend's report:
+// recorded again, current, or why not. It posts one comment on the item's
+// general thread when the demo is out of date and cannot be recorded again,
+// when a chapter failed its checks, or when the recording failed. It posts
+// nothing when the item has no demo, or the demo is current.
+import { spawnSync } from "node:child_process";
+import { existsSync, mkdtempSync, readdirSync, rmSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { basename, dirname, isAbsolute, join, relative } from "node:path";
+import { pathToFileURL } from "node:url";
+
+const USAGE = "usage: rerecord-demo.mjs --item <number> --pr <number> [--repository <owner/name>] [--project <playwright project>]";
+const CONFIG_NAMES = ["ts", "js", "mjs", "cjs", "mts", "cts"].map((extension) => `playwright.config.${extension}`);
+
+/** The `spawnSync` that runs a program in `cwd`; tests pass a stand-in. */
+function runProgram(command, args, options) {
+  return spawnSync(command, args, { encoding: "utf8", maxBuffer: 256 * 1024 * 1024, ...options });
+}
+
+/** `owner/name` of a repository in any spelling the factory stores, lower-case. */
+function repositoryName(repository) {
+  return repository
+    .replace(/\.git$/, "")
+    .split("/")
+    .filter((part) => part !== "")
+    .slice(-2)
+    .join("/")
+    .toLowerCase();
+}
+
+/** A test path inside the repository, or `null` when the path is absolute or leaves it. */
+function insidePath(test) {
+  if (typeof test !== "string" || test === "" || isAbsolute(test) || test.split("/").includes("..")) return null;
+  return test;
+}
+
+/** The nearest folder from the test's own up to the root that holds a Playwright config, else the root. */
+function configFolder(root, testFile) {
+  let folder = dirname(testFile);
+  while (folder.startsWith(root)) {
+    if (CONFIG_NAMES.some((name) => existsSync(join(folder, name)))) return folder;
+    if (folder === root) break;
+    folder = dirname(folder);
+  }
+  return root;
+}
+
+/** Every `demo/manifest.json` under `folder`. */
+function findManifests(folder) {
+  const found = [];
+  for (const entry of readdirSync(folder, { withFileTypes: true })) {
+    const path = join(folder, entry.name);
+    if (entry.isDirectory()) found.push(...findManifests(path));
+    else if (entry.name === "manifest.json" && basename(folder) === "demo") found.push(path);
+  }
+  return found;
+}
+
+/** The last line of a command's output that parses as a JSON object, or `null`. */
+function lastJson(text) {
+  for (const line of String(text).split("\n").reverse()) {
+    try {
+      const value = JSON.parse(line);
+      if (value !== null && typeof value === "object") return value;
+    } catch {
+      // not the JSON line
+    }
+  }
+  return null;
+}
+
+/**
+ * Records the demo again when it is out of date and names its test.
+ * `spawn(command, args, { cwd })` returns `{ status, stdout, stderr }`.
+ * Returns the outcome and the line for the amend's report.
+ */
+export function rerecordDemo({ item, pr, repository, project, cwd, spawn = runProgram }) {
+  const cli = (args) => {
+    const result = spawn("outerlayer", args, { cwd });
+    // Without `outerlayer` on PATH, the CLI runs through npx.
+    return result.error?.code === "ENOENT" ? spawn("npx", ["-y", "@outerlayer/cli", ...args], { cwd }) : result;
+  };
+  // The line for the amend's report says so when a comment did not post, so nobody waits for it.
+  const comment = (body) => (cli(["work", "comment", "--item", String(item), "--body", body]).status === 0 ? "" : " The comment on the item's thread was not posted.");
+
+  const read = cli(["work", "demo", "--item", String(item), "--json"]);
+  const answer = read.status === 0 ? lastJson(read.stdout) : null;
+  if (answer === null) {
+    const detail = String(read.stderr ?? "").trim();
+    return { outcome: "error", line: `Demo: could not read the item's demo${detail === "" ? "" : ` (${detail})`}.` };
+  }
+  const demo = answer.demo ?? null;
+  if (demo === null) return { outcome: "none", line: "Demo: the item has no demo to record again." };
+  if (repository !== undefined && repositoryName(String(demo.repository ?? "")) !== repositoryName(repository)) {
+    return { outcome: "other-repository", line: `Demo: the item's demo is in ${demo.repository}, not ${repository}.` };
+  }
+
+  const head = String(spawn("git", ["rev-parse", "HEAD"], { cwd }).stdout ?? "").trim();
+  const short = (sha) => (sha === "" ? "an unknown commit" : sha.slice(0, 7));
+  if (demo.commitSha === head) return { outcome: "current", line: `Demo: current, recorded at ${short(head)}.` };
+
+  const byHand = `outerlayer emit demo <manifest> --item ${item} --pr ${pr} --replaces ${demo.artifactId}`;
+  const stale = `The demo is out of date: it was recorded at ${short(String(demo.commitSha ?? ""))}, and the branch is now at ${short(head)}.`;
+  const root = String(spawn("git", ["rev-parse", "--show-toplevel"], { cwd }).stdout ?? "").trim() || cwd;
+  if (demo.record === null) {
+    const why = "Its record cannot be read, so the amend cannot tell which test to run.";
+    const unposted = comment(`${stale} ${why} To record it again, run the demo test, then run \`${byHand}\`.`);
+    return { outcome: "not-recorded", line: `Demo: out of date at ${short(String(demo.commitSha ?? ""))}; ${why.charAt(0).toLowerCase()}${why.slice(1)}${unposted}` };
+  }
+  const test = insidePath(demo.record?.test);
+  const testFile = test === null ? null : join(root, test);
+
+  if (test === null || !existsSync(testFile)) {
+    const why =
+      demo.record?.test === undefined || demo.record?.test === null
+        ? "Its record names no test, so the amend cannot record it again."
+        : `The test ${demo.record.test} is not in the checkout, so the amend cannot record it again.`;
+    const unposted = comment(`${stale} ${why} To record it again, run the demo test, then run \`${byHand}\`.`);
+    return { outcome: "not-recorded", line: `Demo: out of date at ${short(String(demo.commitSha ?? ""))}; ${why.charAt(0).toLowerCase()}${why.slice(1)}${unposted}` };
+  }
+
+  const output = mkdtempSync(join(tmpdir(), "outerlayer-rerecord-"));
+  try {
+    const folder = configFolder(root, testFile);
+    // Playwright reads the filter as a regular expression, so a path such as app/[id]/x.spec.ts needs its characters escaped.
+    const filter = relative(folder, testFile).split("\\").join("/").replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+    const run = spawn("npx", ["playwright", "test", filter, "--output", output, ...(project === undefined ? [] : ["--project", project])], { cwd: folder });
+    // A failed check still writes the manifest, and emit demo reports the failed chapter. Only a missing manifest stops here.
+    const manifests = existsSync(output) ? findManifests(output) : [];
+    if (manifests.length !== 1) {
+      const why =
+        manifests.length === 0
+          ? `Running ${test} left no demo manifest (exit ${run.status}).`
+          : `Running ${test} left ${manifests.length} demo manifests, so the amend cannot tell which to store. Pass --project <name> so the test runs once.`;
+      const unposted = comment(`${stale} ${why} To record it again, run the demo test, then run \`${byHand}\`.`);
+      return { outcome: "error", line: `Demo: out of date at ${short(String(demo.commitSha ?? ""))}; ${why}${unposted}` };
+    }
+
+    const emit = cli(["emit", "demo", manifests[0], "--item", String(item), "--pr", String(pr), "--replaces", demo.artifactId, "--json"]);
+    const result = emit.status === 0 || emit.status === 1 ? lastJson(emit.stdout) : null;
+    if (result === null) {
+      const why = `Storing the new demo failed: ${String(emit.stderr ?? "").trim().split("\n")[0] || `exit ${emit.status}`}.`;
+      const unposted = comment(`${stale} ${why} To record it again, run \`${byHand}\` on the manifest of a new run.`);
+      return { outcome: "error", line: `Demo: out of date at ${short(String(demo.commitSha ?? ""))}; ${why}${unposted}` };
+    }
+    const synced = cli(["sync"]);
+    const unsynced = synced.status === 0 ? "" : ` The sync failed (exit ${synced.status}), so the new demo is stored but the session record is not uploaded. Run \`outerlayer sync\`.`;
+    const failures = Array.isArray(result.failures) ? result.failures : [];
+    if (failures.length > 0) {
+      const unposted = comment(`The demo was recorded again at ${short(head)}, but a chapter failed its checks. A failed chapter is stored but not bound to its criterion:\n${failures.map((line) => `- ${line}`).join("\n")}`);
+      return { outcome: "failed-chapters", line: `Demo: recorded again at ${short(head)}, but ${failures.join("; ")}.${unsynced}${unposted}` };
+    }
+    if (unsynced !== "") return { outcome: "sync-failed", line: `Demo: recorded again at ${short(head)}.${unsynced}` };
+    return { outcome: "recorded", line: `Demo: recorded again at ${short(head)}.` };
+  } finally {
+    rmSync(output, { recursive: true, force: true });
+  }
+}
+
+/** The value after `flag`, or `undefined`. */
+function flagValue(argv, flag) {
+  const at = argv.indexOf(flag);
+  return at === -1 ? undefined : argv[at + 1];
+}
+
+function main(argv) {
+  const item = Number(flagValue(argv, "--item"));
+  const pr = Number(flagValue(argv, "--pr"));
+  if (!Number.isInteger(item) || item <= 0 || !Number.isInteger(pr) || pr <= 0) {
+    process.stderr.write(`${USAGE}\n`);
+    return 2;
+  }
+  const { line } = rerecordDemo({ item, pr, repository: flagValue(argv, "--repository"), project: flagValue(argv, "--project"), cwd: process.cwd() });
+  process.stdout.write(`${line}\n`);
+  return 0;
+}
+
+if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) process.exitCode = main(process.argv.slice(2));

package/skill-pack/maintained/outerlayer-build/SKILL.md

@@ -334,7 +334,8 @@ Run them in order.
 6. **Evidence.** Run `outerlayer emit step "Evidence" --item "$OUTERLAYER_WORK"`,
    then apply the team's changes for this phase. Read [references/evidence.md](references/evidence.md)
    when the pull request opens, and run it. Capture the proof each criterion
-   declares, emit it.
+   declares, emit it. Then check the proofs with `outerlayer work proofs`, as
+   that file says.
 
 Optional steps a team may turn off: the manual test and screenshots in the
 Evidence phase, for a repository with no user interface.
@@ -342,7 +343,11 @@ Evidence phase, for a repository with no user interface.
 ## Ending the run
 
 Run this last, after the pull request is open and the evidence is emitted, and
-run no other tool call after it:
+run no other tool call after it. Before it, run `outerlayer sync` and then
+`outerlayer work proofs --item "$OUTERLAYER_WORK"`. End the run only when it
+exits 0, or when `build-reports/evidence.md` names each proof it still lists
+and the reason. Exit 2 means it could not decide: fix the reason it prints and
+run it again:
 
     outerlayer work release --item "$OUTERLAYER_WORK"
 

package/skill-pack/maintained/outerlayer-build/references/agent-briefs.md

@@ -253,12 +253,22 @@ application's own paths. When a state cannot be reached that way, the criterion
 is not evidenced, and `evidence.md` says why. Before emitting, look at each
 capture: it shows the application's navigation and real rows.
 
-For each criterion, put the capture in a script under
-`build-reports/evidence/` and run the script. A later fix can invalidate what
+Write the demo, when a criterion needs a video or a screenshot, as a Playwright
+test that is part of the change. Put it where the repository keeps its Playwright tests, in the
+folder the instructions name, else in `e2e/demo/`. Name it `*.demo.spec.ts`.
+Commit it in the pull request. The demo record names this test, so an amend
+can run it again at a new head. If the repository's CI cannot start the app,
+mark the file so that CI skips it, as `apps/e2e` does.
+
+Record each criterion that declares `proof: screenshot` as a chapter of the same
+demo, with a `demo.screenshot()` call where the result shows. The frame then
+comes from the committed test, so an amend records it again at the new head. Put
+a capture in a script under `build-reports/evidence/` and run the script only
+when no demo test can reach its state. A later fix can invalidate what
 you captured, and regenerating it must be one command. The scripts go when the
 run's report directory goes. The emitted artifacts outlive the run.
 
-Record the criteria that declare `proof: video` as one demo, as the
+Record the criteria that declare `proof: video` or `proof: screenshot` as one demo, as the
 outerlayer-evidence skill's "Record a demo" says. Write one Playwright test with
 `@outerlayer/playwright` and one chapter for each of those criteria, in the
 criteria's order. Run it, then run `outerlayer emit demo <manifest> --item
@@ -266,7 +276,9 @@ criteria's order. Run it, then run `outerlayer emit demo <manifest> --item
 emit artifact ... --for <criterion id> --pr <number>`. Always pass `--pr`.
 When the tree invalidates a video, record and emit the whole demo again: the
 newer demo replaces the older one. Regenerate any other invalidated evidence
-with `--replaces`. Then run `outerlayer sync`.
+with `--replaces`. Then run `outerlayer sync`, and run `outerlayer work
+proofs --item "$OUTERLAYER_WORK"`. For each proof it lists as out of date or
+missing, capture and emit that criterion again, and run the command once more.
 
 Write `build-reports/evidence.md` naming every criterion with its artifact, or
 with the reason none could be produced. A criterion you could not evidence is

package/skill-pack/maintained/outerlayer-build/references/evidence.md

@@ -15,10 +15,20 @@ the issue, the pull request number, the criteria that declare a proof form, the
 instructions-file pointer and the outerlayer-evidence skill.
 
 A criterion that declares `proof: screenshot` needs a screenshot bound to its
-id. One that declares `proof: video` needs a video: its chapter of the
+id. The evidence agent records it as a chapter of the item's demo, with a
+`demo.screenshot()` call. It emits a screenshot with `emit artifact` only for a
+state no demo test can reach. One that declares `proof: video` needs a video: its chapter of the
 item's one demo. A screenshot never satisfies `proof: video`. A criterion with no proof form is proven by its
 test, and needs no artifact.
 
+After the evidence agent returns, and after every later commit, run
+`outerlayer sync`, then `outerlayer work proofs --item "$OUTERLAYER_WORK"`. It
+lists each declared proof as current, out of date or missing at the checkout's
+head. For each criterion it lists as out of date or missing, run the evidence
+step again for that criterion only, then run the command once more before the
+run ends. A criterion still listed goes in `build-reports/evidence.md` with the
+reason. The command leaves out criteria that declare a test proof.
+
 When a commit lands on the branch after the evidence agent captured, decide for
 each capture from that commit's diff. A capture is stale when the diff touches
 a file the capture's subject runs through: the feature's own files, the shared

package/skill-pack/maintained/outerlayer-build/references/release.md

@@ -90,8 +90,9 @@ report. With several judges, run it on the merged report:
 Then run `outerlayer sync`. List every unsatisfied criterion in the final
 report. A criterion that declares a proof form such as a screenshot or video is
 proven only by an artifact bound to it on the pull request. The judge cannot
-see that, so confirm each has an artifact of the declared kind, and emit any
-that is missing in the evidence phase.
+see that. Run `outerlayer work proofs --item "$OUTERLAYER_WORK"`, as the
+evidence phase says, and emit again any proof it lists as out of date or
+missing.
 
 ## The criteria verdict
 

package/skill-pack/maintained/outerlayer-evidence/SKILL.md

@@ -16,22 +16,20 @@ spec asks for — a reviewer should see the change working, not a gallery.
 
 ## When to capture
 
-Capture AFTER the state exists, never before: run the app, the test, or the
-flow first, and shoot the working result. A screenshot of code is not
-evidence; a screenshot of the rendered page is. If the state takes setup
-(seed data, a logged-in user), finish the setup, verify by eye, then capture.
+Capture AFTER the state exists: run the app, test, or
+flow first, and shoot the working result. A screenshot of code is not evidence. Finish setup (seed data, a login) and check by eye first.
 
 ## Per-kind mechanics
 
 Kind is inferred from the file's media type — name files honestly. Every
-artifact shares one upload cap of 8 MiB, whatever its kind:
+artifact shares one upload cap of 8 MiB:
 
 - **screenshot** (`.png`, `.jpg`) — one focused window or region showing the
   proven state. Crop noise; keep enough chrome (URL bar, test summary line)
   to show it is real.
 - **video** (`.webm`, `.mp4`) — a recording of the flow, start to outcome.
-  Record it as a demo chapter (see "Record a demo"). Video is the kind most
-  likely to hit the 8 MiB cap: trim dead time.
+  Record it as a demo chapter (see "Record a demo"). Video most likely
+  hits the 8 MiB cap: trim dead time.
 - **report** (`.html`, `.pdf`) — generated reports: coverage, benchmark,
   audit output. Emit the file the tool produced, unedited.
 - **log** (`.txt`, `.log`) — command output proving a run happened: test
@@ -82,9 +80,9 @@ the recorded list does not hold gets no row.
 
 Record every video proof as a chapter of the item's one demo. A demo has
 one chapter for each criterion that declares a video, in the criteria's
-order. Each chapter is that criterion's video proof, with a spoken line that
-says what happens and a list of the checks that ran. Emit a video with
-`emit artifact` only when Playwright cannot drive the flow.
+order. Each chapter is that criterion's video proof, with a spoken line and the
+checks that ran. Emit a video with `emit artifact` only when Playwright cannot
+drive the flow.
 
 Write a Playwright test with `@outerlayer/playwright`, one `demo.chapter`
 for each criterion:
@@ -99,11 +97,17 @@ for each criterion:
       });
     });
 
+Commit the test: an amend runs it again.
+
+Record each `proof: screenshot` criterion as a chapter too. Call `await
+demo.screenshot()` inside it, right after the `expect` that shows the result.
+Options: `{ fullPage: true }`, `{ masks: true }`. `emit demo` stores the frame
+when every check passed, or says why not.
+
 Install it with `npm install --save-dev @outerlayer/playwright`. Use the
 recorded criterion id as the first argument, or `null` for an opening scene.
-Keep each narration line short enough to say in the chapter's time. Put the
-`expect` calls that prove the criterion inside the chapter, because each one
-is recorded as a check.
+Keep each narration line short. Put the `expect` calls that prove the
+criterion inside the chapter: each is recorded as a check.
 
 Run the test. It writes `demo/manifest.json` and `demo/video.webm` in the
 test's output folder. Then emit the demo:
@@ -115,11 +119,10 @@ Each chapter is its own video proof with a time range in that video. A
 chapter whose checks all passed is bound to its criterion. A chapter with
 a failed check is stored unbound, and the command exits 1: fix the app and
 record again. The command needs `ffmpeg` and `espeak-ng` on PATH. Pass
-`--no-narration` to store a demo with no audio when you cannot install
-`espeak-ng`. It refuses before it uploads anything when a chapter is out of
-order, a criterion id is not recorded, a line is too long for its chapter,
-or a chapter's clip is over 8 MiB. A video over 8 MiB falls back to one clip
-for each chapter. In a runner build everything it needs is installed.
+`--no-narration` when you cannot install `espeak-ng`. It refuses before it
+uploads anything when a chapter is out of order, a criterion id is not
+recorded, a line is too long, or a chapter's clip is over 8 MiB. A video over 8 MiB falls back to one clip
+for each chapter. A runner build has everything installed.
 
 ## The noise rule