@plainconceptsplatform/workflows 1.1.1 → 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/loops/actions/capture-evidence/action.yml +2 -1
- package/loops/actions/capture-evidence/capture.mjs +88 -12
- package/loops/actions/verify-route-matrix/verify-route-matrix.sh +1 -1
- package/loops/docs/diagrams.md +6 -3
- package/loops/workflows/evidence.yml +152 -44
- package/loops/workflows/work-router.yml +2 -2
- package/package.json +1 -1
|
@@ -3,7 +3,8 @@ name: Capture evidence
|
|
|
3
3
|
description: >
|
|
4
4
|
Reads a pull request's verification plans (written by the harness skill pc-repo-verify) and,
|
|
5
5
|
in capture mode, replays them in headless Chromium against the running app, writing one JPEG
|
|
6
|
-
per waypoint
|
|
6
|
+
per waypoint under files/<plan directory>/evidence/, laid out as the repository is, plus
|
|
7
|
+
manifest.json and comment.md. Inspect mode only reports whether any plan is
|
|
7
8
|
replayable, at which address, and with which start command. No model, no network beyond
|
|
8
9
|
installing its two pinned packages.
|
|
9
10
|
inputs:
|
|
@@ -8,7 +8,10 @@
|
|
|
8
8
|
// prints {"runnable","url","start","plans"} as JSON, so the workflow knows whether to start
|
|
9
9
|
// the app at all and which address to wait for
|
|
10
10
|
// node capture.mjs capture <repo-root> <out-dir> <plan.md>...
|
|
11
|
-
// writes
|
|
11
|
+
// writes one screenshot per waypoint under <out-dir>/files/, laid out as the repository is:
|
|
12
|
+
// <plan's directory>/evidence/NN-<capture>.jpg beside a manifest.json for that plan, so the
|
|
13
|
+
// publish job copies files/ over the default branch as is. Writes the run's manifest.json and
|
|
14
|
+
// comment.md into <out-dir>
|
|
12
15
|
// node capture.mjs unreached <repo-root> <out-dir> <plan.md>...
|
|
13
16
|
// the app never answered: writes manifest.json and comment.md saying so, and nothing else
|
|
14
17
|
//
|
|
@@ -16,7 +19,7 @@
|
|
|
16
19
|
// imports are bare: ESM resolves them from the file's own directory, never from NODE_PATH.
|
|
17
20
|
|
|
18
21
|
import { mkdir, readFile, writeFile } from "node:fs/promises";
|
|
19
|
-
import { basename, dirname, join } from "node:path";
|
|
22
|
+
import { basename, dirname, join, posix } from "node:path";
|
|
20
23
|
import { pathToFileURL } from "node:url";
|
|
21
24
|
|
|
22
25
|
const DEFAULT_URL = "http://localhost:3000";
|
|
@@ -27,6 +30,13 @@ const NAVIGATION_TIMEOUT_MS = Number(process.env.EVIDENCE_NAVIGATION_TIMEOUT_MS
|
|
|
27
30
|
// is the commit that job pushes, so the comment is written here and the publish job fills it in.
|
|
28
31
|
export const IMAGE_BASE = "__EVIDENCE_BASE__";
|
|
29
32
|
export const MARKER = "<!-- agentic-evidence -->";
|
|
33
|
+
// Opens the machine-readable copy of the manifest at the end of the comment, which is what other
|
|
34
|
+
// tools read (Project Health shows the screenshots with each issue) instead of parsing markdown.
|
|
35
|
+
export const EMBED_MARKER = "agentic-evidence-manifest";
|
|
36
|
+
// The folder beside each verification plan that keeps its screenshots.
|
|
37
|
+
export const EVIDENCE_DIR = "evidence";
|
|
38
|
+
// Stays well under GitHub's 65,536 character comment limit with the markdown above it.
|
|
39
|
+
const EMBED_LIMIT = 30000;
|
|
30
40
|
|
|
31
41
|
const ACTIONS = new Set(["open", "click", "dblclick", "fill", "press", "select", "check", "uncheck", "hover", "wait"]);
|
|
32
42
|
const LOCATOR_KEYS = ["role", "text", "label", "placeholder", "testid", "alt", "title", "css"];
|
|
@@ -44,6 +54,17 @@ export function slug(value) {
|
|
|
44
54
|
return text.slice(0, 60) || "waypoint";
|
|
45
55
|
}
|
|
46
56
|
|
|
57
|
+
// The directory a plan lives in, which is where its screenshots are kept. The path comes from the
|
|
58
|
+
// pull request's file list, so it is repository-relative; anything that could climb out of the
|
|
59
|
+
// repository is refused rather than written.
|
|
60
|
+
export function planDir(planPath) {
|
|
61
|
+
const dir = posix.dirname(String(planPath).replace(/\\/g, "/"));
|
|
62
|
+
if (dir === "." || dir.startsWith("/") || dir.split("/").includes("..")) {
|
|
63
|
+
throw new Error(`unsafe plan path ${planPath}`);
|
|
64
|
+
}
|
|
65
|
+
return dir;
|
|
66
|
+
}
|
|
67
|
+
|
|
47
68
|
// The change a plan belongs to, from its directory. Archived changes carry a date prefix.
|
|
48
69
|
export function changeName(planPath) {
|
|
49
70
|
return basename(dirname(planPath)).replace(/^\d{4}-\d{2}-\d{2}-/, "") || "change";
|
|
@@ -422,7 +443,7 @@ async function runStep(page, step, baseUrl) {
|
|
|
422
443
|
}
|
|
423
444
|
}
|
|
424
445
|
|
|
425
|
-
async function replay(browser, plan, outDir,
|
|
446
|
+
async function replay(browser, plan, outDir, dir) {
|
|
426
447
|
const context = await browser.newContext({ viewport: plan.viewport, ignoreHTTPSErrors: true });
|
|
427
448
|
const page = await context.newPage();
|
|
428
449
|
page.setDefaultTimeout(STEP_TIMEOUT_MS);
|
|
@@ -442,10 +463,11 @@ async function replay(browser, plan, outDir, prefix) {
|
|
|
442
463
|
break;
|
|
443
464
|
}
|
|
444
465
|
}
|
|
445
|
-
|
|
466
|
+
// Repository-relative, so the comment, the manifest and the commit all name the same path.
|
|
467
|
+
const file = `${dir}/${EVIDENCE_DIR}/${String(waypoint.n).padStart(2, "0")}-${waypoint.capture}.jpg`;
|
|
446
468
|
let captured = false;
|
|
447
469
|
try {
|
|
448
|
-
await page.screenshot({ path: join(outDir, file), type: "jpeg", quality: 70, fullPage: true });
|
|
470
|
+
await page.screenshot({ path: join(outDir, "files", file), type: "jpeg", quality: 70, fullPage: true });
|
|
449
471
|
captured = true;
|
|
450
472
|
} catch (error) {
|
|
451
473
|
errors.push(`screenshot: ${firstLine(error)}`);
|
|
@@ -459,7 +481,7 @@ async function replay(browser, plan, outDir, prefix) {
|
|
|
459
481
|
errors,
|
|
460
482
|
ignored: waypoint.ignored,
|
|
461
483
|
});
|
|
462
|
-
console.log(`${
|
|
484
|
+
console.log(`${dir} ${waypoint.n} ${waypoint.capture}: ${errors.length === 0 ? "ok" : errors.join("; ")}`);
|
|
463
485
|
}
|
|
464
486
|
} finally {
|
|
465
487
|
await context.close();
|
|
@@ -471,11 +493,58 @@ async function replay(browser, plan, outDir, prefix) {
|
|
|
471
493
|
|
|
472
494
|
const escapeMarkdown = (text) => String(text).replace(/([\\`*_[\]<>|])/g, "\\$1");
|
|
473
495
|
|
|
496
|
+
// The manifest other tools read, as an HTML comment so it never renders. JSON can carry `--` only
|
|
497
|
+
// inside strings, where `\u002d` means the same character, so escaping it keeps the comment closed.
|
|
498
|
+
export function embedManifest(manifest) {
|
|
499
|
+
const compact = (keepExpect) => ({
|
|
500
|
+
version: 2,
|
|
501
|
+
base: IMAGE_BASE,
|
|
502
|
+
sha: manifest.sha ?? "",
|
|
503
|
+
runUrl: manifest.runUrl ?? "",
|
|
504
|
+
...(manifest.app ? { app: manifest.app } : {}),
|
|
505
|
+
plans: (manifest.plans ?? []).map((plan) => ({
|
|
506
|
+
path: plan.path,
|
|
507
|
+
change: plan.change,
|
|
508
|
+
status: plan.status,
|
|
509
|
+
...(plan.reason ? { reason: plan.reason } : {}),
|
|
510
|
+
...(plan.waypoints
|
|
511
|
+
? {
|
|
512
|
+
waypoints: plan.waypoints.map((w) => ({
|
|
513
|
+
n: w.n,
|
|
514
|
+
capture: w.capture,
|
|
515
|
+
...(keepExpect && w.expect ? { expect: w.expect } : {}),
|
|
516
|
+
reached: w.reached,
|
|
517
|
+
file: w.file,
|
|
518
|
+
})),
|
|
519
|
+
}
|
|
520
|
+
: {}),
|
|
521
|
+
})),
|
|
522
|
+
});
|
|
523
|
+
let json = JSON.stringify(compact(true));
|
|
524
|
+
if (json.length > EMBED_LIMIT) json = JSON.stringify(compact(false));
|
|
525
|
+
if (json.length > EMBED_LIMIT) return "";
|
|
526
|
+
return `<!-- ${EMBED_MARKER}\n${json.replace(/-(?=-)/g, "\\u002d")}\n-->`;
|
|
527
|
+
}
|
|
528
|
+
|
|
529
|
+
// Reads the embedded manifest back out of a comment body, or null when there is none.
|
|
530
|
+
export function readEmbeddedManifest(body) {
|
|
531
|
+
const match = String(body ?? "").match(new RegExp(`<!-- ${EMBED_MARKER}\\n([\\s\\S]*?)\\n-->`));
|
|
532
|
+
if (!match) return null;
|
|
533
|
+
try {
|
|
534
|
+
return JSON.parse(match[1]);
|
|
535
|
+
} catch {
|
|
536
|
+
return null;
|
|
537
|
+
}
|
|
538
|
+
}
|
|
539
|
+
|
|
474
540
|
export function renderComment(manifest) {
|
|
475
541
|
const lines = [MARKER, "## Visual evidence", ""];
|
|
476
542
|
const short = manifest.sha ? manifest.sha.slice(0, 7) : "";
|
|
477
543
|
const run = manifest.runUrl ? ` ([run](${manifest.runUrl}))` : "";
|
|
478
544
|
lines.push(`The verification plan replayed against \`${short}\`, the head of this pull request${run}.`, "");
|
|
545
|
+
if ((manifest.plans ?? []).some((plan) => plan.status === "captured")) {
|
|
546
|
+
lines.push(`The screenshots are kept beside each plan, in its \`${EVIDENCE_DIR}/\` folder on the default branch.`, "");
|
|
547
|
+
}
|
|
479
548
|
|
|
480
549
|
if (manifest.app && manifest.app.started === false) {
|
|
481
550
|
lines.push(`Nothing was captured: the app did not answer at ${manifest.app.url} within ${manifest.app.waitedSeconds} seconds of \`${manifest.app.start}\`. The run log has the app's output.`, "");
|
|
@@ -503,6 +572,8 @@ export function renderComment(manifest) {
|
|
|
503
572
|
}
|
|
504
573
|
lines.push("</details>", "");
|
|
505
574
|
}
|
|
575
|
+
const embedded = embedManifest(manifest);
|
|
576
|
+
if (embedded) lines.push(embedded, "");
|
|
506
577
|
return `${lines.join("\n").trimEnd()}\n`;
|
|
507
578
|
}
|
|
508
579
|
|
|
@@ -541,19 +612,18 @@ async function capture(root, outDir, paths) {
|
|
|
541
612
|
await mkdir(outDir, { recursive: true });
|
|
542
613
|
const plans = await loadPlans(root, paths, parse);
|
|
543
614
|
const runnable = plans.filter((plan) => plan.status === "runnable");
|
|
544
|
-
const manifest = { version:
|
|
615
|
+
const manifest = { version: 2, plans: [] };
|
|
545
616
|
|
|
546
617
|
const browser = runnable.length > 0 ? await chromium.launch({ headless: true }) : undefined;
|
|
547
618
|
try {
|
|
548
|
-
let index = 0;
|
|
549
619
|
for (const plan of plans) {
|
|
550
620
|
if (plan.status !== "runnable") {
|
|
551
621
|
manifest.plans.push({ path: plan.path, change: plan.change, status: plan.status, reason: plan.reason ?? "" });
|
|
552
622
|
continue;
|
|
553
623
|
}
|
|
554
|
-
|
|
555
|
-
|
|
556
|
-
const waypoints = await replay(browser, plan, outDir,
|
|
624
|
+
const dir = planDir(plan.path);
|
|
625
|
+
await mkdir(join(outDir, "files", dir, EVIDENCE_DIR), { recursive: true });
|
|
626
|
+
const waypoints = await replay(browser, plan, outDir, dir);
|
|
557
627
|
manifest.plans.push({ path: plan.path, change: plan.change, status: "captured", url: plan.url, waypoints });
|
|
558
628
|
}
|
|
559
629
|
} finally {
|
|
@@ -573,6 +643,12 @@ async function main(argv) {
|
|
|
573
643
|
const manifest = await capture(root, outDir, paths);
|
|
574
644
|
manifest.sha = process.env.EVIDENCE_SHA ?? "";
|
|
575
645
|
manifest.runUrl = process.env.EVIDENCE_RUN_URL ?? "";
|
|
646
|
+
// Each plan keeps its own record beside its screenshots, so the folder explains itself on the
|
|
647
|
+
// default branch without the pull request comment.
|
|
648
|
+
for (const plan of manifest.plans.filter((p) => p.status === "captured")) {
|
|
649
|
+
const record = { version: 2, sha: manifest.sha, runUrl: manifest.runUrl, plan };
|
|
650
|
+
await writeFile(join(outDir, "files", planDir(plan.path), EVIDENCE_DIR, "manifest.json"), `${JSON.stringify(record, null, 2)}\n`);
|
|
651
|
+
}
|
|
576
652
|
await writeFile(join(outDir, "manifest.json"), `${JSON.stringify(manifest, null, 2)}\n`);
|
|
577
653
|
await writeFile(join(outDir, "comment.md"), renderComment(manifest));
|
|
578
654
|
const shots = manifest.plans.flatMap((plan) => plan.waypoints ?? []).filter((waypoint) => waypoint.file).length;
|
|
@@ -586,7 +662,7 @@ async function main(argv) {
|
|
|
586
662
|
await mkdir(outDir, { recursive: true });
|
|
587
663
|
const plans = await loadPlans(root, paths, parse);
|
|
588
664
|
const manifest = {
|
|
589
|
-
version:
|
|
665
|
+
version: 2,
|
|
590
666
|
sha: process.env.EVIDENCE_SHA ?? "",
|
|
591
667
|
runUrl: process.env.EVIDENCE_RUN_URL ?? "",
|
|
592
668
|
app: {
|
|
@@ -1723,7 +1723,7 @@ else
|
|
|
1723
1723
|
echo "FAIL: evidence.yml's publish job holds a write token and must not check out or run repository code" >&2
|
|
1724
1724
|
fi
|
|
1725
1725
|
ev_absent 'secrets\.' "$EVIDENCE_YML" 'evidence.yml must need no secret; GITHUB_TOKEN is enough'
|
|
1726
|
-
ev_absent 'gh (issue|pr) comment.*--attach' "$EVIDENCE_YML" 'gh --attach refuses installation tokens;
|
|
1726
|
+
ev_absent 'gh (issue|pr) comment.*--attach' "$EVIDENCE_YML" 'gh --attach refuses installation tokens; commit the screenshots beside their plans'
|
|
1727
1727
|
fi
|
|
1728
1728
|
|
|
1729
1729
|
# The retired worker stays retired: nothing dispatches it and nothing waits for it.
|
package/loops/docs/diagrams.md
CHANGED
|
@@ -626,7 +626,10 @@ earlier versions of them passed against a deliberately broken guard and were rew
|
|
|
626
626
|
|
|
627
627
|
Archives what a merged pull request looks like. Not an agent: the plan was written by the agent that
|
|
628
628
|
implemented the change (`pc-repo-verify`), and this replays it. The router calls it when a pull
|
|
629
|
-
request merges, and `operation=evidence` runs it by hand. Nothing waits on it
|
|
629
|
+
request merges, and `operation=evidence` runs it by hand. Nothing waits on it. The screenshots are
|
|
630
|
+
committed beside each plan, in an `evidence/` folder on the default branch, so the record stays with
|
|
631
|
+
the change it proves; the comment embeds a manifest that Project Health reads. A pull request that
|
|
632
|
+
is not merged has no plan on the default branch, so its screenshots stay in the run's artifact.
|
|
630
633
|
The capture job runs the pull request's code with a read-only token; only the publish job writes,
|
|
631
634
|
and it runs no repository code.
|
|
632
635
|
|
|
@@ -645,8 +648,8 @@ flowchart TD
|
|
|
645
648
|
evUp -.->|"no, after the timeout"| evUnreached
|
|
646
649
|
evReplay["replay each waypoint<br/>in headless Chromium,<br/>one JPEG per waypoint"] --> evPublish
|
|
647
650
|
evUnreached["record why nothing<br/>was captured"] --> evPublish
|
|
648
|
-
evPublish["publish (GITHUB_TOKEN)<br/>commit
|
|
649
|
-
evDone(("Archived<br/>
|
|
651
|
+
evPublish["publish (GITHUB_TOKEN)<br/>commit beside each plan,<br/>evidence/ on the default branch,<br/>upsert one comment"] --> evDone
|
|
652
|
+
evDone(("Archived<br/>beside the plan"))
|
|
650
653
|
evIdle(("Nothing to capture"))
|
|
651
654
|
|
|
652
655
|
classDef start fill:#ffffff,stroke:#172033,stroke-width:2px,color:#172033
|
|
@@ -3,15 +3,16 @@ name: "Agentics: Evidence"
|
|
|
3
3
|
|
|
4
4
|
# Archives what a merged pull request looks like. The agent that implemented the change wrote a
|
|
5
5
|
# verification plan with the harness skill pc-repo-verify; this replays that plan against the
|
|
6
|
-
# app, with no model, and keeps one screenshot per waypoint
|
|
7
|
-
# it when a pull request merges, and operation=evidence runs it by hand for any pull
|
|
8
|
-
# Nothing waits on it
|
|
6
|
+
# app, with no model, and keeps one screenshot per waypoint beside the plan it came from. The
|
|
7
|
+
# router calls it when a pull request merges, and operation=evidence runs it by hand for any pull
|
|
8
|
+
# request. Nothing waits on it: it is the record, not a gate. Project Health reads the manifest
|
|
9
|
+
# embedded in the comment to show the screenshots with each issue.
|
|
9
10
|
#
|
|
10
11
|
# Two jobs, because one runs the pull request's code and the other writes. `capture` checks out
|
|
11
12
|
# the head, starts the app and drives Chromium with a read-only token and no secret. `publish`
|
|
12
|
-
# checks nothing out: it takes the images as an artifact, commits them
|
|
13
|
-
# branch and writes one comment. Both use GITHUB_TOKEN, so a
|
|
14
|
-
# nothing either job writes raises another workflow event.
|
|
13
|
+
# checks nothing out: it takes the images as an artifact, commits them into an `evidence/` folder
|
|
14
|
+
# beside each plan on the default branch and writes one comment. Both use GITHUB_TOKEN, so a
|
|
15
|
+
# repository configures nothing, and nothing either job writes raises another workflow event.
|
|
15
16
|
#
|
|
16
17
|
# This replaced agent-visual-verify, which never produced a screenshot: it ran only on pull
|
|
17
18
|
# requests nobody would review, raced the merge it was dispatched before, re-planned with a model
|
|
@@ -35,8 +36,10 @@ env:
|
|
|
35
36
|
EVIDENCE_START_TIMEOUT_SECONDS: "900"
|
|
36
37
|
# The SDK when a repository has .NET projects but no global.json to say which.
|
|
37
38
|
EVIDENCE_DOTNET_VERSION: "10.0.x"
|
|
38
|
-
EVIDENCE_BRANCH: evidence
|
|
39
39
|
EVIDENCE_MARKER: "<!-- agentic-evidence -->"
|
|
40
|
+
# How long the run keeps the screenshots it could not archive: a pull request that is not merged
|
|
41
|
+
# has no plan on the default branch to keep them beside.
|
|
42
|
+
EVIDENCE_ARTIFACT_DAYS: "14"
|
|
40
43
|
# The capture runs in mcr.microsoft.com/playwright at this tag, pre-pulled by the runner image
|
|
41
44
|
# (runners/versions.env), so the two move together.
|
|
42
45
|
PLAYWRIGHT_VERSION: "1.62.1"
|
|
@@ -66,8 +69,11 @@ jobs:
|
|
|
66
69
|
const pull_number = Number(process.env.PR_NUMBER);
|
|
67
70
|
const { data: pr } = await github.rest.pulls.get({ ...context.repo, pull_number });
|
|
68
71
|
const files = await github.paginate(github.rest.pulls.listFiles, { ...context.repo, pull_number, per_page: 100 });
|
|
72
|
+
// A plan that was only moved (the archive step, in a later pull request) is skipped: its
|
|
73
|
+
// evidence/ folder moves with it, so replaying it would only repeat the record.
|
|
69
74
|
const plans = files
|
|
70
75
|
.filter((file) => file.status !== 'removed' && /(^|\/)verification-plan\.md$/.test(file.filename))
|
|
76
|
+
.filter((file) => !(file.status === 'renamed' && file.changes === 0))
|
|
71
77
|
.map((file) => file.filename);
|
|
72
78
|
core.setOutput('sha', pr.head.sha);
|
|
73
79
|
core.setOutput('plans', plans.join('\n'));
|
|
@@ -185,6 +191,25 @@ jobs:
|
|
|
185
191
|
echo "::group::Last lines of the app's output"
|
|
186
192
|
tail -n 80 "$log" || true
|
|
187
193
|
echo "::endgroup::"
|
|
194
|
+
# An orchestrator (Aspire, compose) keeps each service's output to itself, so the
|
|
195
|
+
# start command's own log rarely says which service never came up. Show them.
|
|
196
|
+
echo "::group::Listening ports"
|
|
197
|
+
(ss -ltnp 2>/dev/null || netstat -ltnp 2>/dev/null) | head -n 40 || true
|
|
198
|
+
echo "::endgroup::"
|
|
199
|
+
echo "::group::Containers"
|
|
200
|
+
docker ps -a --format '{{.Names}} {{.Status}} {{.Image}}' 2>/dev/null || true
|
|
201
|
+
echo "::endgroup::"
|
|
202
|
+
for id in $(docker ps -aq 2>/dev/null); do
|
|
203
|
+
echo "::group::Container $(docker inspect -f '{{.Name}}' "$id" 2>/dev/null)"
|
|
204
|
+
docker logs --tail 60 "$id" 2>&1 || true
|
|
205
|
+
echo "::endgroup::"
|
|
206
|
+
done
|
|
207
|
+
for f in "$HOME"/.aspire/logs/*.log; do
|
|
208
|
+
[ -f "$f" ] || continue
|
|
209
|
+
echo "::group::$f"
|
|
210
|
+
tail -n 200 "$f" || true
|
|
211
|
+
echo "::endgroup::"
|
|
212
|
+
done
|
|
188
213
|
fi
|
|
189
214
|
|
|
190
215
|
- name: Replay the plans
|
|
@@ -240,7 +265,7 @@ jobs:
|
|
|
240
265
|
with:
|
|
241
266
|
name: evidence-pr-${{ inputs.pr-number }}
|
|
242
267
|
path: ${{ runner.temp }}/evidence
|
|
243
|
-
retention-days:
|
|
268
|
+
retention-days: ${{ env.EVIDENCE_ARTIFACT_DAYS }}
|
|
244
269
|
if-no-files-found: error
|
|
245
270
|
|
|
246
271
|
publish:
|
|
@@ -259,72 +284,113 @@ jobs:
|
|
|
259
284
|
name: evidence-pr-${{ inputs.pr-number }}
|
|
260
285
|
path: ${{ runner.temp }}/evidence
|
|
261
286
|
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
287
|
+
- name: Read the pull request
|
|
288
|
+
id: pr
|
|
289
|
+
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
|
|
290
|
+
env:
|
|
291
|
+
PR_NUMBER: ${{ inputs.pr-number }}
|
|
292
|
+
with:
|
|
293
|
+
script: |
|
|
294
|
+
const { data: pr } = await github.rest.pulls.get({ ...context.repo, pull_number: Number(process.env.PR_NUMBER) });
|
|
295
|
+
core.setOutput('merged', pr.merged ? 'true' : 'false');
|
|
296
|
+
core.setOutput('branch', pr.base.ref);
|
|
297
|
+
|
|
298
|
+
# Beside each plan on the default branch, where the plan itself is archived: the record stays
|
|
299
|
+
# with the change it proves, and moves with it if the change folder ever moves. Only a merged
|
|
300
|
+
# pull request's plans are there to keep it beside. A sparse, blob-less fetch reads only the
|
|
301
|
+
# plan folders, and [skip ci] plus GITHUB_TOKEN keep the commit from starting any workflow.
|
|
302
|
+
- name: Commit the screenshots beside their plans
|
|
303
|
+
id: archive
|
|
304
|
+
if: steps.pr.outputs.merged == 'true'
|
|
268
305
|
env:
|
|
269
306
|
TOKEN: ${{ github.token }}
|
|
270
307
|
REPO: ${{ github.repository }}
|
|
271
308
|
SERVER: ${{ github.server_url }}
|
|
272
|
-
BRANCH: ${{
|
|
309
|
+
BRANCH: ${{ steps.pr.outputs.branch }}
|
|
273
310
|
PR: ${{ inputs.pr-number }}
|
|
274
311
|
run: |
|
|
275
312
|
set -euo pipefail
|
|
276
|
-
src="$RUNNER_TEMP/evidence"
|
|
277
|
-
if !
|
|
313
|
+
src="$RUNNER_TEMP/evidence/files"
|
|
314
|
+
if [ ! -d "$src" ] || [ -z "$(find "$src" -name '*.jpg' -print -quit)" ]; then
|
|
278
315
|
echo "No screenshots to keep; the comment records why."
|
|
279
316
|
echo "base=" >> "$GITHUB_OUTPUT"
|
|
280
317
|
exit 0
|
|
281
318
|
fi
|
|
282
319
|
|
|
283
|
-
|
|
320
|
+
# Each folder holding screenshots is <plan directory>/evidence.
|
|
321
|
+
mapfile -t dirs < <(cd "$src" && find . -type d -name evidence | sed 's|^\./||; s|/evidence$||' | sort)
|
|
322
|
+
|
|
323
|
+
work="$RUNNER_TEMP/evidence-commit"
|
|
284
324
|
rm -rf "$work"
|
|
285
|
-
|
|
325
|
+
auth="AUTHORIZATION: basic $(printf 'x-access-token:%s' "$TOKEN" | base64 -w0)"
|
|
326
|
+
# Blob-less and sparse: only the plan folders are downloaded, however large the repository.
|
|
327
|
+
git -c "http.${SERVER}/.extraheader=${auth}" clone -q --filter=blob:none --no-checkout --depth=1 \
|
|
328
|
+
--branch "$BRANCH" "${SERVER}/${REPO}.git" "$work"
|
|
286
329
|
cd "$work"
|
|
287
|
-
git
|
|
330
|
+
git config "http.${SERVER}/.extraheader" "$auth"
|
|
288
331
|
git config user.name "github-actions[bot]"
|
|
289
332
|
git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
|
|
290
|
-
|
|
291
|
-
|
|
333
|
+
patterns=()
|
|
334
|
+
for dir in "${dirs[@]}"; do patterns+=("/$dir/"); done
|
|
335
|
+
git sparse-checkout set --no-cone "${patterns[@]}"
|
|
292
336
|
|
|
293
337
|
pushed=false
|
|
338
|
+
missing=()
|
|
294
339
|
for attempt in 1 2 3 4 5; do
|
|
295
|
-
if
|
|
296
|
-
git
|
|
297
|
-
git checkout -q -f -B "$BRANCH" FETCH_HEAD
|
|
340
|
+
if [ "$attempt" -eq 1 ]; then
|
|
341
|
+
git checkout -q "$BRANCH"
|
|
298
342
|
else
|
|
299
|
-
git
|
|
300
|
-
git
|
|
301
|
-
printf '%s\n' "# Evidence" "" \
|
|
302
|
-
"Screenshots the evidence workflow replayed from each merged pull request's verification plan, one folder per pull request." \
|
|
303
|
-
"Written by a workflow, never by hand. Nothing here is built, deployed or merged." > README.md
|
|
304
|
-
git add README.md
|
|
343
|
+
git fetch -q --depth=1 origin "$BRANCH"
|
|
344
|
+
git reset -q --hard FETCH_HEAD
|
|
305
345
|
fi
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
346
|
+
missing=()
|
|
347
|
+
for dir in "${dirs[@]}"; do
|
|
348
|
+
# A plan moved or removed since the merge has no folder to keep the record in.
|
|
349
|
+
if [ ! -f "$dir/verification-plan.md" ]; then
|
|
350
|
+
missing+=("$dir")
|
|
351
|
+
continue
|
|
352
|
+
fi
|
|
353
|
+
rm -rf "$dir/evidence"
|
|
354
|
+
mkdir -p "$dir/evidence"
|
|
355
|
+
cp -R "$src/$dir/evidence/." "$dir/evidence/"
|
|
356
|
+
git add -A "$dir/evidence"
|
|
357
|
+
done
|
|
311
358
|
if git diff --cached --quiet; then
|
|
312
359
|
pushed=true
|
|
313
360
|
break
|
|
314
361
|
fi
|
|
315
|
-
git commit -q -m "evidence:
|
|
362
|
+
git commit -q -m "evidence: screenshots for #$PR [skip ci]"
|
|
316
363
|
if git push -q origin "HEAD:refs/heads/$BRANCH"; then
|
|
317
364
|
pushed=true
|
|
318
365
|
break
|
|
319
366
|
fi
|
|
320
|
-
echo "
|
|
367
|
+
echo "$BRANCH moved before the push (attempt $attempt); retrying on its new tip."
|
|
321
368
|
sleep $((attempt * 3))
|
|
322
369
|
done
|
|
323
|
-
[ "$pushed"
|
|
370
|
+
if [ "$pushed" != "true" ]; then
|
|
371
|
+
# A protected default branch refuses the push. The comment says so, and the run's
|
|
372
|
+
# artifact keeps the screenshots, so the merge that already happened stays green.
|
|
373
|
+
echo "::warning::could not push the screenshots to $BRANCH after 5 attempts"
|
|
374
|
+
echo "base=" >> "$GITHUB_OUTPUT"
|
|
375
|
+
echo "reason=the push to $BRANCH was refused" >> "$GITHUB_OUTPUT"
|
|
376
|
+
exit 0
|
|
377
|
+
fi
|
|
378
|
+
|
|
379
|
+
if [ "${#missing[@]}" -eq "${#dirs[@]}" ]; then
|
|
380
|
+
echo "None of the plans is on $BRANCH any more; nothing was kept."
|
|
381
|
+
echo "base=" >> "$GITHUB_OUTPUT"
|
|
382
|
+
echo "reason=none of the plans is on $BRANCH any more" >> "$GITHUB_OUTPUT"
|
|
383
|
+
exit 0
|
|
384
|
+
fi
|
|
324
385
|
|
|
325
386
|
sha="$(git rev-parse HEAD)"
|
|
326
|
-
echo "base=${SERVER}/${REPO}/raw/${sha}
|
|
327
|
-
|
|
387
|
+
echo "base=${SERVER}/${REPO}/raw/${sha}" >> "$GITHUB_OUTPUT"
|
|
388
|
+
{
|
|
389
|
+
echo "missing<<EOF"
|
|
390
|
+
printf '%s\n' "${missing[@]}"
|
|
391
|
+
echo "EOF"
|
|
392
|
+
} >> "$GITHUB_OUTPUT"
|
|
393
|
+
echo "Screenshots kept at ${SERVER}/${REPO}/tree/${sha} beside $(( ${#dirs[@]} - ${#missing[@]} )) plan(s)"
|
|
328
394
|
|
|
329
395
|
# GITHUB_TOKEN on purpose: its comment raises no workflow event, so the router does not wake
|
|
330
396
|
# up to read its own record. One comment per pull request, rewritten on a re-run.
|
|
@@ -333,14 +399,56 @@ jobs:
|
|
|
333
399
|
env:
|
|
334
400
|
PR_NUMBER: ${{ inputs.pr-number }}
|
|
335
401
|
MARKER: ${{ env.EVIDENCE_MARKER }}
|
|
336
|
-
|
|
402
|
+
MERGED: ${{ steps.pr.outputs.merged }}
|
|
403
|
+
IMAGE_BASE: ${{ steps.archive.outputs.base }}
|
|
404
|
+
MISSING: ${{ steps.archive.outputs.missing }}
|
|
405
|
+
REASON: ${{ steps.archive.outputs.reason }}
|
|
406
|
+
RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
|
|
407
|
+
ARTIFACT_DAYS: ${{ env.EVIDENCE_ARTIFACT_DAYS }}
|
|
337
408
|
with:
|
|
338
409
|
script: |
|
|
339
410
|
const fs = require('node:fs');
|
|
340
411
|
const path = require('node:path');
|
|
341
412
|
const issue_number = Number(process.env.PR_NUMBER);
|
|
342
|
-
const
|
|
343
|
-
|
|
413
|
+
const evidence = path.join(process.env.RUNNER_TEMP, 'evidence');
|
|
414
|
+
let source = fs.readFileSync(path.join(evidence, 'comment.md'), 'utf8');
|
|
415
|
+
const base = process.env.IMAGE_BASE ?? '';
|
|
416
|
+
const missing = (process.env.MISSING ?? '').split('\n').map((line) => line.trim()).filter(Boolean);
|
|
417
|
+
const kept = (file) => Boolean(base) && !missing.some((dir) => file.startsWith(`${dir}/evidence/`));
|
|
418
|
+
|
|
419
|
+
// An image that was not archived would render broken: its line goes, and a note says
|
|
420
|
+
// the run's artifact has it instead.
|
|
421
|
+
source = source.replace(/^!\[[^\]]*\]\(__EVIDENCE_BASE__\/([^)]*)\)\n?/gm, (line, file) => (kept(file) ? line : ''));
|
|
422
|
+
const shots = fs.readdirSync(evidence, { recursive: true }).some((name) => String(name).endsWith('.jpg'));
|
|
423
|
+
if (!base) source = source.replace(/^The screenshots are kept beside each plan.*\n\n?/m, '');
|
|
424
|
+
if (shots && (!base || missing.length > 0)) {
|
|
425
|
+
const why = process.env.MERGED !== 'true'
|
|
426
|
+
? 'the pull request is not merged, so its plans are not on the default branch'
|
|
427
|
+
: !base
|
|
428
|
+
? (process.env.REASON || 'the screenshots could not be committed')
|
|
429
|
+
: `${missing.join(', ')} ${missing.length === 1 ? 'is' : 'are'} no longer on the default branch`;
|
|
430
|
+
source = source.replace('## Visual evidence\n', `## Visual evidence\n\n> Not archived: ${why}. The screenshots are in [the run](${process.env.RUN_URL})'s artifact for ${process.env.ARTIFACT_DAYS} days.\n`);
|
|
431
|
+
}
|
|
432
|
+
|
|
433
|
+
// The embedded manifest is what other tools read, so it says exactly what was kept: the
|
|
434
|
+
// base where the files are, and no file for a screenshot that was not archived. It sits
|
|
435
|
+
// in an HTML comment, which a `--` would close; `\u002d` is the same to a JSON reader.
|
|
436
|
+
source = source.replace(/<!-- agentic-evidence-manifest\n([\s\S]*?)\n-->/, (block, json) => {
|
|
437
|
+
let data;
|
|
438
|
+
try {
|
|
439
|
+
data = JSON.parse(json);
|
|
440
|
+
} catch {
|
|
441
|
+
return block;
|
|
442
|
+
}
|
|
443
|
+
data.base = base || null;
|
|
444
|
+
for (const plan of data.plans ?? []) {
|
|
445
|
+
for (const waypoint of plan.waypoints ?? []) {
|
|
446
|
+
if (waypoint.file && !kept(waypoint.file)) waypoint.file = null;
|
|
447
|
+
}
|
|
448
|
+
}
|
|
449
|
+
return `<!-- agentic-evidence-manifest\n${JSON.stringify(data).replace(/-(?=-)/g, '\\u002d')}\n-->`;
|
|
450
|
+
});
|
|
451
|
+
const body = source.split('__EVIDENCE_BASE__').join(base);
|
|
344
452
|
const comments = await github.paginate(github.rest.issues.listComments, { ...context.repo, issue_number, per_page: 100 });
|
|
345
453
|
const existing = comments.find((comment) => comment.user?.type === 'Bot' && (comment.body ?? '').includes(process.env.MARKER));
|
|
346
454
|
if (existing) {
|
|
@@ -630,8 +630,8 @@ jobs:
|
|
|
630
630
|
AGENTMEMORY_SECRET: ${{ secrets.AGENTMEMORY_SECRET }}
|
|
631
631
|
|
|
632
632
|
# Screenshots of a merged pull request, replayed from the verification plan its implementation
|
|
633
|
-
# wrote and archived
|
|
634
|
-
# grant is exactly what the publish job writes:
|
|
633
|
+
# wrote and archived beside that plan. Plain YAML with no model, so no secret is passed and the
|
|
634
|
+
# grant is exactly what the publish job writes: one commit of evidence/ folders and one comment. Its capture
|
|
635
635
|
# job runs the pull request's code with read-only access; only publish holds these writes, and it
|
|
636
636
|
# runs no repository code.
|
|
637
637
|
call-evidence:
|