@plainconceptsplatform/workflows 1.2.0 → 1.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -326,7 +326,11 @@ async function preCommitHookUpdate(repositoryPath) {
|
|
|
326
326
|
if (!await exists(hookPath)) {
|
|
327
327
|
return { target, content: managedLines };
|
|
328
328
|
}
|
|
329
|
+
// A Windows clone with core.autocrlf checks the hook out with CRLF, which the LF patterns below
|
|
330
|
+
// never match: every update from Windows appended one more block (one consumer had ten). The
|
|
331
|
+
// hook is written back with LF, as every managed file is.
|
|
329
332
|
const content = (await readFile(hookPath, "utf8"))
|
|
333
|
+
.replace(/\r\n/g, "\n")
|
|
330
334
|
.replace("pnpm exec if git diff --cached --name-only -- .github | grep -q .; then", "if git diff --cached --name-only -- .github | grep -q .; then");
|
|
331
335
|
// Replace whatever managed block is there, never append beside it. The previous version
|
|
332
336
|
// recognised only two exact shapes -- the current block and the pre-`if` legacy lines -- and
|
|
@@ -21,6 +21,13 @@ inputs:
|
|
|
21
21
|
description: Where capture and unreached write the screenshots, manifest.json and comment.md.
|
|
22
22
|
required: false
|
|
23
23
|
default: ''
|
|
24
|
+
sign-in:
|
|
25
|
+
description: >
|
|
26
|
+
The repository's own sign-in steps for plans that start signed in (`login:`), relative to the
|
|
27
|
+
workspace, which is the default branch's checkout. The repository owns the file; without it
|
|
28
|
+
such a plan replays signed out and its comment says why.
|
|
29
|
+
required: false
|
|
30
|
+
default: '.github/evidence/sign-in.yml'
|
|
24
31
|
playwright-version:
|
|
25
32
|
description: playwright-core version. Must match the mcr.microsoft.com/playwright image tag the capture runs in.
|
|
26
33
|
required: false
|
|
@@ -75,6 +82,7 @@ runs:
|
|
|
75
82
|
ROOT: ${{ inputs.root }}
|
|
76
83
|
PLANS: ${{ inputs.plans }}
|
|
77
84
|
OUT_DIR: ${{ inputs.out-dir }}
|
|
85
|
+
EVIDENCE_SIGN_IN: ${{ inputs.sign-in }}
|
|
78
86
|
PLAYWRIGHT_VERSION: ${{ inputs.playwright-version }}
|
|
79
87
|
# Chromium's own stderr. A browser that dies on launch says why only here.
|
|
80
88
|
DEBUG: pw:browser
|
|
@@ -102,7 +110,7 @@ runs:
|
|
|
102
110
|
if command -v docker > /dev/null 2>&1; then
|
|
103
111
|
# The host network, so the browser reaches the app on localhost where it started;
|
|
104
112
|
# the runner's own user, so what it writes the publish job can read.
|
|
105
|
-
docker run --rm --network host --ipc host --user "$(id -u):$(id -g)" -e HOME=/tmp -e DEBUG -e EVIDENCE_SHA -e EVIDENCE_RUN_URL -e EVIDENCE_STEP_TIMEOUT_MS -e EVIDENCE_NAVIGATION_TIMEOUT_MS -v "$runner:$runner:ro" -v "$GITHUB_WORKSPACE:$GITHUB_WORKSPACE:ro" -v "$OUT_DIR:$OUT_DIR" -w "$GITHUB_WORKSPACE" "mcr.microsoft.com/playwright:v${PLAYWRIGHT_VERSION}-noble" node "$runner/capture.mjs" capture "$ROOT" "$OUT_DIR" "${plans[@]}"
|
|
113
|
+
docker run --rm --network host --ipc host --user "$(id -u):$(id -g)" -e HOME=/tmp -e DEBUG -e EVIDENCE_SHA -e EVIDENCE_RUN_URL -e EVIDENCE_SIGN_IN -e EVIDENCE_STEP_TIMEOUT_MS -e EVIDENCE_NAVIGATION_TIMEOUT_MS -v "$runner:$runner:ro" -v "$GITHUB_WORKSPACE:$GITHUB_WORKSPACE:ro" -v "$OUT_DIR:$OUT_DIR" -w "$GITHUB_WORKSPACE" "mcr.microsoft.com/playwright:v${PLAYWRIGHT_VERSION}-noble" node "$runner/capture.mjs" capture "$ROOT" "$OUT_DIR" "${plans[@]}"
|
|
106
114
|
else
|
|
107
115
|
node "$runner/capture.mjs" capture "$ROOT" "$OUT_DIR" "${plans[@]}"
|
|
108
116
|
fi
|
|
@@ -37,6 +37,10 @@ export const EMBED_MARKER = "agentic-evidence-manifest";
|
|
|
37
37
|
export const EVIDENCE_DIR = "evidence";
|
|
38
38
|
// Stays well under GitHub's 65,536 character comment limit with the markdown above it.
|
|
39
39
|
const EMBED_LIMIT = 30000;
|
|
40
|
+
// The repository's own steps for signing in locally, read from the default branch so a replay of
|
|
41
|
+
// an older pull request finds them too. The repository owns the file: an update never writes it.
|
|
42
|
+
export const SIGN_IN_FILE = ".github/evidence/sign-in.yml";
|
|
43
|
+
const ACCOUNT_PLACEHOLDER = "{account}";
|
|
40
44
|
|
|
41
45
|
const ACTIONS = new Set(["open", "click", "dblclick", "fill", "press", "select", "check", "uncheck", "hover", "wait"]);
|
|
42
46
|
const LOCATOR_KEYS = ["role", "text", "label", "placeholder", "testid", "alt", "title", "css"];
|
|
@@ -129,15 +133,83 @@ export function normalizePlan(doc) {
|
|
|
129
133
|
return { n: index + 1, capture, expect: String(raw.expect ?? "").trim(), steps, ignored };
|
|
130
134
|
});
|
|
131
135
|
|
|
136
|
+
const login = parseLogin(doc.login ?? environment.login, environment);
|
|
132
137
|
return {
|
|
133
138
|
status: "runnable",
|
|
134
139
|
url: String(doc.url ?? environment.url ?? DEFAULT_URL).trim() || DEFAULT_URL,
|
|
135
140
|
start: String(doc.start ?? environment.start ?? "").trim(),
|
|
136
141
|
viewport: parseViewport(doc.viewport ?? environment.viewport),
|
|
142
|
+
...(login ? { login } : {}),
|
|
137
143
|
waypoints,
|
|
138
144
|
};
|
|
139
145
|
}
|
|
140
146
|
|
|
147
|
+
// A plan that starts signed in says so with `login`, rather than signing in in its first
|
|
148
|
+
// waypoint. The plans written before that waypoint signed in name a mechanism (`login: mock-sso`)
|
|
149
|
+
// and say whom in prose ("Sign in as Bob Builder"), so the prose travels along as a hint. A plan
|
|
150
|
+
// can also name the account (`login: { account: Bob Builder }`) or spell out its own steps.
|
|
151
|
+
export function parseLogin(value, environment = {}) {
|
|
152
|
+
if (value === undefined || value === null || value === false) return undefined;
|
|
153
|
+
if (Array.isArray(value)) return { steps: value };
|
|
154
|
+
if (isObject(value) && Array.isArray(value.steps)) return { steps: value.steps };
|
|
155
|
+
if (isObject(value) && value.account !== undefined) return { account: String(value.account) };
|
|
156
|
+
const hint = Object.values(environment).filter((text) => typeof text === "string").join("\n");
|
|
157
|
+
return { hint };
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
// The account a plan's prose asks for: one of the repository's accounts right after "sign in as",
|
|
161
|
+
// the earliest when the prose names several. Only an account the repository lists can match.
|
|
162
|
+
export function accountNamed(text, accounts) {
|
|
163
|
+
let best;
|
|
164
|
+
for (const account of accounts) {
|
|
165
|
+
const escaped = account.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
166
|
+
const at = String(text ?? "").search(new RegExp(`sign(?:ed)?[\\s-]*in\\s+as\\s+(?:the\\s+)?${escaped}\\b`, "i"));
|
|
167
|
+
if (at >= 0 && (best === undefined || at < best.at)) best = { account, at };
|
|
168
|
+
}
|
|
169
|
+
return best?.account;
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
// Reads the repository's sign-in: the steps, in the journey's own step syntax, with `{account}`
|
|
173
|
+
// where the account goes, and the accounts a plan can ask for, the first being the default.
|
|
174
|
+
export function parseSignIn(text, parseYaml) {
|
|
175
|
+
const doc = parseYaml(String(text));
|
|
176
|
+
if (!isObject(doc) || !Array.isArray(doc.steps) || doc.steps.length === 0) {
|
|
177
|
+
throw new Error("the sign-in needs a steps list");
|
|
178
|
+
}
|
|
179
|
+
return { accounts: asArray(doc.accounts).map((account) => String(account)), steps: doc.steps };
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
const fillAccount = (value, account) =>
|
|
183
|
+
typeof value === "string"
|
|
184
|
+
? value.split(ACCOUNT_PLACEHOLDER).join(account)
|
|
185
|
+
: Array.isArray(value)
|
|
186
|
+
? value.map((item) => fillAccount(item, account))
|
|
187
|
+
: isObject(value)
|
|
188
|
+
? Object.fromEntries(Object.entries(value).map(([key, item]) => [key, fillAccount(item, account)]))
|
|
189
|
+
: value;
|
|
190
|
+
|
|
191
|
+
// A first waypoint that clicks one of the repository's accounts signs the plan in itself, as some
|
|
192
|
+
// plans that also say `login:` do. Signing in before it would leave that click nothing to find.
|
|
193
|
+
function signsItselfIn(plan, accounts) {
|
|
194
|
+
const names = new Set(accounts.map((account) => account.toLowerCase()));
|
|
195
|
+
return (plan.waypoints[0]?.steps ?? []).some((step) =>
|
|
196
|
+
step.action === "click" && [step.locator?.text, step.locator?.name].some((name) => name !== undefined && names.has(name.toLowerCase())),
|
|
197
|
+
);
|
|
198
|
+
}
|
|
199
|
+
|
|
200
|
+
// The steps that sign a plan in before its first waypoint, or why there are none to run.
|
|
201
|
+
export function signInFor(plan, signIn) {
|
|
202
|
+
const login = plan.login;
|
|
203
|
+
if (!login) return undefined;
|
|
204
|
+
const parsed = (raw) => raw.map(parseStep).filter((step) => step.ignored === undefined);
|
|
205
|
+
if (login.steps) return { steps: parsed(login.steps) };
|
|
206
|
+
if (!signIn) return { steps: [], error: `the plan starts signed in, but the repository has no ${SIGN_IN_FILE}` };
|
|
207
|
+
if (signIn.error) return { steps: [], error: signIn.error };
|
|
208
|
+
if (signsItselfIn(plan, signIn.accounts)) return undefined;
|
|
209
|
+
const account = login.account ?? accountNamed(login.hint, signIn.accounts) ?? signIn.accounts[0] ?? "";
|
|
210
|
+
return { ...(account ? { account } : {}), steps: parsed(fillAccount(signIn.steps, account)) };
|
|
211
|
+
}
|
|
212
|
+
|
|
141
213
|
function locatorOf(value) {
|
|
142
214
|
if (typeof value === "string") return { text: value };
|
|
143
215
|
if (!isObject(value)) return undefined;
|
|
@@ -450,7 +522,23 @@ async function replay(browser, plan, outDir, dir) {
|
|
|
450
522
|
page.setDefaultNavigationTimeout(NAVIGATION_TIMEOUT_MS);
|
|
451
523
|
|
|
452
524
|
const results = [];
|
|
525
|
+
let signedIn;
|
|
453
526
|
try {
|
|
527
|
+
if (plan.signIn) {
|
|
528
|
+
// Signing in is not a waypoint: nothing is captured, and a failure is recorded while the
|
|
529
|
+
// journey still runs, so its screenshots show where a missing sign-in left it.
|
|
530
|
+
const errors = plan.signIn.error ? [plan.signIn.error] : [];
|
|
531
|
+
for (const step of plan.signIn.steps) {
|
|
532
|
+
try {
|
|
533
|
+
await runStep(page, step, plan.url);
|
|
534
|
+
} catch (error) {
|
|
535
|
+
errors.push(`${describeStep(step)}: ${firstLine(error)}`);
|
|
536
|
+
break;
|
|
537
|
+
}
|
|
538
|
+
}
|
|
539
|
+
signedIn = { ...(plan.signIn.account ? { account: plan.signIn.account } : {}), reached: errors.length === 0, errors };
|
|
540
|
+
console.log(`${dir} sign-in${plan.signIn.account ? ` as ${plan.signIn.account}` : ""}: ${errors.length === 0 ? "ok" : errors.join("; ")}`);
|
|
541
|
+
}
|
|
454
542
|
for (const waypoint of plan.waypoints) {
|
|
455
543
|
const errors = [];
|
|
456
544
|
for (const step of waypoint.steps) {
|
|
@@ -486,7 +574,7 @@ async function replay(browser, plan, outDir, dir) {
|
|
|
486
574
|
} finally {
|
|
487
575
|
await context.close();
|
|
488
576
|
}
|
|
489
|
-
return results;
|
|
577
|
+
return { waypoints: results, signedIn };
|
|
490
578
|
}
|
|
491
579
|
|
|
492
580
|
// ── Report ──────────────────────────────────────────────────────────────────────────────────
|
|
@@ -560,6 +648,10 @@ export function renderComment(manifest) {
|
|
|
560
648
|
lines.push(`Not replayed: ${escapeMarkdown(plan.reason || plan.status)}.`, "");
|
|
561
649
|
continue;
|
|
562
650
|
}
|
|
651
|
+
if (plan.signIn) {
|
|
652
|
+
const as = plan.signIn.account ? ` as ${escapeMarkdown(plan.signIn.account)}` : "";
|
|
653
|
+
lines.push(plan.signIn.reached ? `Signed in${as} first.` : `> Could not sign in${as} first: ${escapeMarkdown(plan.signIn.errors.join("; "))}`, "");
|
|
654
|
+
}
|
|
563
655
|
const missed = plan.waypoints.filter((waypoint) => !waypoint.reached).length;
|
|
564
656
|
const summary = `${plan.waypoints.length} waypoint${plan.waypoints.length === 1 ? "" : "s"}${missed > 0 ? `, ${missed} did not reach their state` : ""}`;
|
|
565
657
|
lines.push("<details>", `<summary>${summary}</summary>`, "");
|
|
@@ -606,6 +698,19 @@ async function inspect(root, paths) {
|
|
|
606
698
|
};
|
|
607
699
|
}
|
|
608
700
|
|
|
701
|
+
// The repository's sign-in, or null when it has none. The path is relative to the working
|
|
702
|
+
// directory, which is the default branch's checkout, not the pull request head.
|
|
703
|
+
async function loadSignIn(path, parseYaml) {
|
|
704
|
+
let text;
|
|
705
|
+
try {
|
|
706
|
+
text = await readFile(path, "utf8");
|
|
707
|
+
} catch (error) {
|
|
708
|
+
if (error?.code === "ENOENT") return null;
|
|
709
|
+
throw error;
|
|
710
|
+
}
|
|
711
|
+
return parseSignIn(text, parseYaml);
|
|
712
|
+
}
|
|
713
|
+
|
|
609
714
|
async function capture(root, outDir, paths) {
|
|
610
715
|
const { parse } = await import("yaml");
|
|
611
716
|
const { chromium } = await import("playwright-core");
|
|
@@ -613,6 +718,11 @@ async function capture(root, outDir, paths) {
|
|
|
613
718
|
const plans = await loadPlans(root, paths, parse);
|
|
614
719
|
const runnable = plans.filter((plan) => plan.status === "runnable");
|
|
615
720
|
const manifest = { version: 2, plans: [] };
|
|
721
|
+
let signIn = null;
|
|
722
|
+
if (runnable.some((plan) => plan.login)) {
|
|
723
|
+
const path = process.env.EVIDENCE_SIGN_IN || SIGN_IN_FILE;
|
|
724
|
+
signIn = await loadSignIn(path, parse).catch((error) => ({ error: `${path}: ${firstLine(error)}` }));
|
|
725
|
+
}
|
|
616
726
|
|
|
617
727
|
const browser = runnable.length > 0 ? await chromium.launch({ headless: true }) : undefined;
|
|
618
728
|
try {
|
|
@@ -623,8 +733,8 @@ async function capture(root, outDir, paths) {
|
|
|
623
733
|
}
|
|
624
734
|
const dir = planDir(plan.path);
|
|
625
735
|
await mkdir(join(outDir, "files", dir, EVIDENCE_DIR), { recursive: true });
|
|
626
|
-
const waypoints = await replay(browser, plan, outDir, dir);
|
|
627
|
-
manifest.plans.push({ path: plan.path, change: plan.change, status: "captured", url: plan.url, waypoints });
|
|
736
|
+
const { waypoints, signedIn } = await replay(browser, { ...plan, signIn: signInFor(plan, signIn) }, outDir, dir);
|
|
737
|
+
manifest.plans.push({ path: plan.path, change: plan.change, status: "captured", url: plan.url, ...(signedIn ? { signIn: signedIn } : {}), waypoints });
|
|
628
738
|
}
|
|
629
739
|
} finally {
|
|
630
740
|
await browser?.close();
|
|
@@ -11,8 +11,9 @@ name: "Agentics: Evidence"
|
|
|
11
11
|
# Two jobs, because one runs the pull request's code and the other writes. `capture` checks out
|
|
12
12
|
# the head, starts the app and drives Chromium with a read-only token and no secret. `publish`
|
|
13
13
|
# checks nothing out: it takes the images as an artifact, commits them into an `evidence/` folder
|
|
14
|
-
# beside each plan on the default branch and writes one comment. Both use GITHUB_TOKEN,
|
|
15
|
-
#
|
|
14
|
+
# beside each plan on the default branch and writes one comment. Both use GITHUB_TOKEN, and nothing
|
|
15
|
+
# either job writes raises another workflow event. The one thing a repository may add is its own
|
|
16
|
+
# .github/evidence/sign-in.yml, for plans that start signed in rather than signing in themselves.
|
|
16
17
|
#
|
|
17
18
|
# This replaced agent-visual-verify, which never produced a screenshot: it ran only on pull
|
|
18
19
|
# requests nobody would review, raced the merge it was dispatched before, re-planned with a model
|