fibcheck 0.0.0-stage → 0.1.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/frozen.js ADDED
@@ -0,0 +1,125 @@
1
+ import { createHash } from "node:crypto";
2
+ import { CLAIM_TYPES } from "./claims/types.js";
3
+ /**
4
+ * sha256 of everything a reviewer approved: the quote, the check, the control and the assertion mode. A claim without an
5
+ * assertion mode hashes exactly as before the mode was covered, so older files without one still load.
6
+ */
7
+ export function contentHash(claim) {
8
+ const fields = [claim.quote, claim.check, claim.control ?? "", ...(claim.assertionMode ? [claim.assertionMode] : [])];
9
+ return createHash("sha256").update(JSON.stringify(fields)).digest("hex");
10
+ }
11
+ /** True when a claim with an assertion mode carries a hash from before the mode was covered. */
12
+ function hashedWithoutMode(claim) {
13
+ return Boolean(claim.assertionMode) && contentHash({ ...claim, assertionMode: undefined }) === claim.contentSha256;
14
+ }
15
+ export function parseFrozenFile(text) {
16
+ let value;
17
+ try {
18
+ value = JSON.parse(text);
19
+ }
20
+ catch {
21
+ throw new Error("The frozen claims file is not valid JSON.");
22
+ }
23
+ if (value?.version !== 1 || typeof value.docsFile !== "string" || !Array.isArray(value.claims)) {
24
+ throw new Error("The frozen claims file must have version 1, a docsFile and a claims list.");
25
+ }
26
+ const incomplete = value.claims.find((claim) => !isComplete(claim));
27
+ if (incomplete !== undefined)
28
+ throw new Error(`Frozen claim ${String(incomplete?.id)} is incomplete.`);
29
+ return value;
30
+ }
31
+ function isComplete(claim) {
32
+ if (!claim)
33
+ return false;
34
+ const text = [claim.id, claim.quote, claim.check, claim.contentSha256].every((field) => typeof field === "string" && field.length > 0);
35
+ const lines = Number.isSafeInteger(claim.startLine) && Number.isSafeInteger(claim.endLine);
36
+ const optional = (claim.control === undefined || typeof claim.control === "string")
37
+ && (claim.assertionMode === undefined || claim.assertionMode === "dom" || claim.assertionMode === "visual");
38
+ return text && lines && optional && CLAIM_TYPES.includes(claim.type);
39
+ }
40
+ /**
41
+ * Splits a frozen file into claims that may run and claims that need a person's review: a missing reviewer,
42
+ * a check edited after its hash was written, or a missing exact-label assertion.
43
+ */
44
+ export function frozenClaims(file) {
45
+ const claims = [];
46
+ const needsReview = [];
47
+ for (const frozen of file.claims) {
48
+ const flag = (why) => needsReview.push({ id: frozen.id, line: frozen.startLine, why });
49
+ if (!frozen.reviewed?.by?.trim() || !frozen.reviewed?.at?.trim())
50
+ flag("No one has reviewed this check yet.");
51
+ else if (hashedWithoutMode(frozen)) {
52
+ flag("This check was frozen before the assertion mode was part of its hash. Re-freeze it with fibcheck freeze and review it again.");
53
+ }
54
+ else if (contentHash(frozen) !== frozen.contentSha256)
55
+ flag("The check changed after review. Freeze and review it again.");
56
+ else {
57
+ const claim = {
58
+ id: frozen.id, type: frozen.type, quote: frozen.quote, check: frozen.check,
59
+ source: { file: file.docsFile, startLine: frozen.startLine, endLine: frozen.endLine },
60
+ ...(frozen.control ? { control: frozen.control } : {}),
61
+ ...(frozen.assertionMode ? { assertionMode: frozen.assertionMode } : {}),
62
+ };
63
+ const missing = missingLabelAssertion(claim);
64
+ if (missing)
65
+ flag(missing);
66
+ else
67
+ claims.push(claim);
68
+ }
69
+ }
70
+ return { claims, needsReview };
71
+ }
72
+ const ELEMENTS = "tabs?|menus?|buttons?|links?|dropdowns?";
73
+ /** Labels the documentation gives a UI element: "*Admin* tab", "the Save button", 'a tab labelled "Admin"'. */
74
+ export function namedLabels(quote) {
75
+ const found = [];
76
+ const before = new RegExp(`(?:\\*\\*?|"|“|\`)([^*"”\`]{1,40})(?:\\*\\*?|"|”|\`)\\s+(${ELEMENTS})\\b`, "gi");
77
+ const after = new RegExp(`\\b(${ELEMENTS})\\s+(?:labell?ed|named|called)\\s+(?:\\*\\*?|"|“|\`|')([^*"”\`']{1,40})`, "gi");
78
+ for (const match of quote.matchAll(before))
79
+ found.push({ label: match[1].trim(), element: match[2].toLowerCase() });
80
+ for (const match of quote.matchAll(after))
81
+ found.push({ label: match[2].trim(), element: match[1].toLowerCase() });
82
+ return found;
83
+ }
84
+ /**
85
+ * Rule for every claim that names a UI element: the check's first assertion is an exact-label assertion for
86
+ * each named element, made before anything is clicked by that label. Returns why the check fails the rule.
87
+ */
88
+ export function missingLabelAssertion(claim) {
89
+ const check = claim.check ?? "";
90
+ const firstAssertion = check.match(/Assert exactly:[^]*?(?:Expected: (?:true|false)\.|$)/i)?.[0] ?? "";
91
+ for (const named of namedLabels(claim.quote)) {
92
+ const problem = labelProblem(check, firstAssertion, named);
93
+ if (problem)
94
+ return problem;
95
+ }
96
+ return undefined;
97
+ }
98
+ function labelProblem(check, firstAssertion, { label, element }) {
99
+ const singular = element.replace(/s$/, "");
100
+ const quoted = new RegExp(`(?:'|"|“)${escape(label)}(?:'|"|”)`, "i");
101
+ if (!quoted.test(firstAssertion) || !new RegExp(`\\b${singular}`, "i").test(firstAssertion)) {
102
+ return `The check's first assertion must name the ${singular} labelled '${label}' exactly, before acting on it.`;
103
+ }
104
+ const firstUse = check.search(new RegExp(`\\b(?:click|select|open|press)\\b[^.]*${escape(label)}`, "i"));
105
+ return firstUse !== -1 && firstUse < check.indexOf(firstAssertion) ? `The check acts on '${label}' before asserting that its ${singular} exists.` : undefined;
106
+ }
107
+ function escape(text) {
108
+ return text.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
109
+ }
110
+ /** Builds a frozen file from extracted claims, for a person to review. Reviewer fields are left empty. */
111
+ export function freeze(docsFile, claims) {
112
+ return {
113
+ version: 1,
114
+ docsFile,
115
+ claims: claims.filter((claim) => claim.type !== "untestable" && claim.check).map((claim) => {
116
+ const entry = { quote: claim.quote, check: claim.check, control: claim.control, assertionMode: claim.assertionMode };
117
+ return {
118
+ id: claim.id, type: claim.type, quote: claim.quote, startLine: claim.source.startLine, endLine: claim.source.endLine,
119
+ check: claim.check, ...(claim.control ? { control: claim.control } : {}),
120
+ ...(claim.assertionMode ? { assertionMode: claim.assertionMode } : {}),
121
+ contentSha256: contentHash(entry), reviewed: { by: "", at: "" },
122
+ };
123
+ }),
124
+ };
125
+ }
@@ -0,0 +1,122 @@
1
+ import { distinctKaneRuns } from "./kane/evidence.js";
2
+ import { KANE_RUN_LIMIT_REASON } from "./loop-types.js";
3
+ /** Rewordings allowed after an agent or check problem, before the claim is left unconfirmed. */
4
+ const MAX_REWORDS = 1;
5
+ const INITIAL_REASON = "Run the browser check extracted from the documentation, then have the model read Kane's result.";
6
+ const NOT_OUTDATED = "fibcheck did not check whether the claim was once true, so it is reported as false rather than outdated.";
7
+ const CAUSE_REASONS = {
8
+ agent_misstep: "Kane's own navigation caused the failure, and no rewording was left.",
9
+ check_problem: "The check did not fit the app, and no rewording was left.",
10
+ setup_problem: "Sign-in, test data or the environment prevented the test.",
11
+ kane_limitation: "Kane cannot observe or perform what this claim needs.",
12
+ claim_contradicted: "The failure looked like a contradiction, but the confirmation steps were not completed.",
13
+ unclear: "The evidence did not show why the run failed.",
14
+ };
15
+ /**
16
+ * Questions one claim without a hand-written check. Kane decides every pass
17
+ * and failure. The model reads each result, says whether the run tested the
18
+ * documented claim, and proposes a reworded check, a reproduction or a
19
+ * control. A false verdict needs two distinct Kane failures that the model
20
+ * reads as contradictions, a recorded reason the test was not at fault, and
21
+ * a passing control. Even then the verdict stays unconfirmed and is marked as a
22
+ * candidate finding: only a person who has read the evidence may call it false.
23
+ */
24
+ export async function questionGenerally(claim, check, general, run, canRunFollowUp) {
25
+ const initial = await run("initial", general.objectiveFor(check), claim.assertionMode);
26
+ const state = { claim, general, run, canRunFollowUp, initial, followUps: [], overrides: [], diagnoses: [], rewordsLeft: general.frozen ? 0 : MAX_REWORDS };
27
+ let attempt = { round: "initial", check, kane: initial };
28
+ for (;;) {
29
+ if (attempt.kane.status === "error" || attempt.kane.status === "cancelled") {
30
+ return outcome(state, "unconfirmed", `Kane did not finish the run (${attempt.kane.status}).`);
31
+ }
32
+ const diagnosis = await diagnoseAttempt(state, attempt);
33
+ if (!diagnosis)
34
+ return outcome(state, "unconfirmed", "The model could not read Kane's result, so it was not interpreted.");
35
+ const next = await decide(state, attempt, diagnosis);
36
+ if ("questioning" in next)
37
+ return next;
38
+ attempt = next;
39
+ }
40
+ }
41
+ async function decide(state, attempt, diagnosis) {
42
+ if (attempt.kane.status === "passed") {
43
+ if (diagnosis.exercisedClaim && diagnosis.visibleWording) {
44
+ state.wording = { documented: state.claim.quote, visible: diagnosis.visibleWording };
45
+ return outcome(state, "worded_differently", `Kane passed a run that exercised the documented behaviour, but the interface words it differently: ${diagnosis.visibleWording}. This is a wording suggestion, not a fib.`);
46
+ }
47
+ if (diagnosis.exercisedClaim)
48
+ return outcome(state, "true", `Kane passed a run that exercised the documented claim. ${diagnosis.reason}`);
49
+ return await reword(state, diagnosis)
50
+ ?? outcome(state, "unconfirmed", `Kane passed, but the run did not exercise the documented claim. ${diagnosis.reason}`);
51
+ }
52
+ if (diagnosis.cause === "claim_contradicted" && diagnosis.exercisedClaim)
53
+ return await reproduceAndControl(state, attempt, diagnosis);
54
+ if (diagnosis.cause === "agent_misstep" || diagnosis.cause === "check_problem") {
55
+ const next = await reword(state, diagnosis);
56
+ if (next)
57
+ return next;
58
+ }
59
+ return outcome(state, "unconfirmed", `${CAUSE_REASONS[diagnosis.cause]} ${diagnosis.reason}`);
60
+ }
61
+ async function reword(state, diagnosis) {
62
+ if (state.rewordsLeft <= 0 || !diagnosis.revisedCheck || !state.canRunFollowUp())
63
+ return undefined;
64
+ state.rewordsLeft--;
65
+ const kane = await state.run("reword", state.general.objectiveFor(diagnosis.revisedCheck), state.claim.assertionMode);
66
+ state.followUps.push({ purpose: "reword", reason: `Reworded after the model's reading: ${diagnosis.reason}`, kane });
67
+ return { round: "reword", check: diagnosis.revisedCheck, kane };
68
+ }
69
+ async function reproduceAndControl(state, failed, diagnosis) {
70
+ if (!diagnosis.ownMistakeRuledOut)
71
+ return outcome(state, "unconfirmed", "The model gave no reason to rule out a test error.");
72
+ if (!state.canRunFollowUp())
73
+ return outcome(state, "unconfirmed", KANE_RUN_LIMIT_REASON);
74
+ // Repeat the exact check that failed, so the reproduction tests the same assertion.
75
+ const check = failed.check;
76
+ const kane = await state.run("retry", state.general.objectiveFor(check), state.claim.assertionMode);
77
+ return await checkReproductionAndControl(state, failed, diagnosis, { round: "retry", check, kane });
78
+ }
79
+ async function checkReproductionAndControl(state, failed, diagnosis, reproduction) {
80
+ state.followUps.push({ purpose: "retry", reason: "Reproduce the same documented assertion in an independent Kane run.", kane: reproduction.kane });
81
+ if (reproduction.kane.status !== "failed" || !distinctKaneRuns(reproduction.kane, failed.kane)) {
82
+ return outcome(state, "unconfirmed", "Kane did not fail the documented assertion again in a separate run.");
83
+ }
84
+ const second = await diagnoseAttempt(state, reproduction);
85
+ if (!second || second.cause !== "claim_contradicted" || !second.exercisedClaim || !second.ownMistakeRuledOut) {
86
+ return outcome(state, "unconfirmed", "The reproduction failed for a different reason, so the result is not consistent.");
87
+ }
88
+ const controlCheck = state.general.frozen ? state.claim.control : diagnosis.controlCheck;
89
+ if (!controlCheck || !state.canRunFollowUp()) {
90
+ const missing = state.general.frozen ? "The frozen claim has no reviewed control, so a setup problem was not ruled out." : "The model proposed no control, so a setup problem was not ruled out.";
91
+ return outcome(state, "unconfirmed", controlCheck ? KANE_RUN_LIMIT_REASON : missing);
92
+ }
93
+ const control = await state.run("control", state.general.objectiveFor(controlCheck), state.claim.assertionMode);
94
+ state.followUps.push({ purpose: "control", reason: `Rule out a setup problem: ${controlCheck}`, kane: control });
95
+ if (control.status !== "passed" || !distinctKaneRuns(control, failed.kane) || !distinctKaneRuns(control, reproduction.kane)) {
96
+ return outcome(state, "unconfirmed", "The control did not pass in its own run, so a setup problem was not ruled out.");
97
+ }
98
+ state.candidateFinding = true;
99
+ return outcome(state, "unconfirmed", `Candidate finding, awaiting a person's review. Two separate Kane runs failed the documented result. ${diagnosis.reason} Test error ruled out: ${diagnosis.ownMistakeRuledOut} The control passed. ${NOT_OUTDATED}`);
100
+ }
101
+ async function diagnoseAttempt(state, attempt) {
102
+ let diagnosis;
103
+ try {
104
+ diagnosis = await state.general.diagnose(state.claim, attempt.check, attempt.kane);
105
+ }
106
+ catch {
107
+ return undefined;
108
+ }
109
+ state.diagnoses.push({ round: attempt.round, cause: diagnosis.cause, exercisedClaim: diagnosis.exercisedClaim, reason: diagnosis.reason });
110
+ const blame = attempt.kane.blame;
111
+ if (blame && diagnosis.kaneBlame === "disagree" && diagnosis.blameEvidence) {
112
+ state.overrides.push({ round: attempt.round, family: blame.family, category: blame.category, hypothesis: "The model read the evidence differently from Kane's own label.", evidence: diagnosis.blameEvidence });
113
+ }
114
+ return diagnosis;
115
+ }
116
+ function outcome(state, verdict, reason) {
117
+ const { initial, followUps, overrides, diagnoses, candidateFinding, wording } = state;
118
+ return {
119
+ kane: initial,
120
+ questioning: { verdict, reason, initialReason: INITIAL_REASON, followUps, overrides, diagnoses, ...(candidateFinding ? { candidateFinding } : {}), ...(wording ? { wording } : {}) },
121
+ };
122
+ }
package/dist/github.js ADDED
@@ -0,0 +1,216 @@
1
+ import { cpSync, existsSync, mkdirSync, writeFileSync } from "node:fs";
2
+ import { join } from "node:path";
3
+ import { NEEDS_REVIEW_PREFIX } from "./check.js";
4
+ import { countResults } from "./report/report.js";
5
+ import { frozenClaims, parseFrozenFile } from "./frozen.js";
6
+ import { loadTarget } from "./target.js";
7
+ const COMMIT = /^[0-9a-f]{40}$/i;
8
+ export async function stageGitHubReadme(input, fetchFile = fetch) {
9
+ const { repository, head, base } = publicPullRequest(input.eventName, input.event);
10
+ const api = githubUrl(input.apiUrl, "api.github.com");
11
+ const server = githubUrl(input.serverUrl, "github.com");
12
+ const targetFile = repositoryPath(input.targetFile, ".json", "The target must be a repository-relative JSON file in the trusted base commit.");
13
+ const docsFile = repositoryPath(input.docsFile ?? "README.md", ".md", "The docs file must be a repository-relative Markdown file.");
14
+ if (!Number.isSafeInteger(input.maxClaims) || input.maxClaims < 0) {
15
+ throw new Error("max-claims must be a nonnegative whole number.");
16
+ }
17
+ const claimsFile = input.claimsFile
18
+ ? repositoryPath(input.claimsFile, ".json", "The claims file must be a repository-relative JSON file in the trusted base commit.")
19
+ : undefined;
20
+ const [readme, profile, frozenText] = await Promise.all([
21
+ publicFile(api, repository, docsFile, head, fetchFile),
22
+ publicFile(api, repository, targetFile, base, fetchFile),
23
+ claimsFile ? publicFile(api, repository, claimsFile, base, fetchFile) : Promise.resolve(undefined),
24
+ ]);
25
+ const frozen = frozenText === undefined ? undefined : parseFrozenFile(frozenText);
26
+ if (frozen && frozen.docsFile !== (input.docsFile ?? "README.md"))
27
+ throw new Error("The frozen claims file belongs to a different docs file.");
28
+ const directory = join(input.directory, "source");
29
+ mkdirSync(directory);
30
+ writeFileSync(join(directory, "README.md"), readme, { flag: "wx" });
31
+ const targetPath = join(input.directory, "target.json");
32
+ writeFileSync(targetPath, profile, { flag: "wx", mode: 0o600 });
33
+ const target = loadTarget(targetPath);
34
+ writeFileSync(targetPath, JSON.stringify({
35
+ ...target,
36
+ repo: new URL(repository, server).href,
37
+ docs: [],
38
+ }));
39
+ const staged = {
40
+ targetPath,
41
+ source: { directory, commit: head },
42
+ runsDir: join(input.directory, "runs"),
43
+ maxClaims: input.maxClaims,
44
+ };
45
+ if (!frozen)
46
+ return staged;
47
+ const { claims, needsReview } = frozenClaims(frozen);
48
+ // The staged copy of the docs file is named README.md.
49
+ const reviewedClaims = claims.map((claim) => ({ ...claim, source: { ...claim.source, file: "README.md" } }));
50
+ return { ...staged, reviewedClaims, frozen: true, needsReview };
51
+ }
52
+ /** The pull request comment: counts, plus the lines a person should review. Never calls a line false on its own. */
53
+ export function githubComment(report, docsFile = "README.md", needsReview = []) {
54
+ if (!report.commit || !COMMIT.test(report.commit))
55
+ throw new Error("A pinned commit is required for a pull request report.");
56
+ const counts = countResults(report);
57
+ const entries = [
58
+ ["Claims tested", counts.tested],
59
+ ["True", counts.verdicts.true],
60
+ ["False", counts.verdicts.false],
61
+ ["Outdated", counts.verdicts.outdated],
62
+ ["Unconfirmed", counts.verdicts.unconfirmed],
63
+ ["Untestable", counts.untestable],
64
+ ["Not run", counts.skipped],
65
+ ];
66
+ if (counts.withoutVerdict > 0)
67
+ entries.push(["Without verdict", counts.withoutVerdict]);
68
+ const review = report.tested.flatMap(({ claim, questioning }) => {
69
+ if (questioning?.candidateFinding)
70
+ return [`- Line ${claim.source.startLine}: Kane failed it twice and a control passed. A person should read the evidence before changing it.`];
71
+ if (questioning?.wording)
72
+ return [`- Line ${claim.source.startLine}: the behaviour held, but the app shows ${questioning.wording.visible}.`];
73
+ return [];
74
+ });
75
+ return [
76
+ "### fibcheck docs check",
77
+ "",
78
+ `Checked \`${docsFile}\` at revision \`${report.commit.slice(0, 12)}\`.`,
79
+ "",
80
+ "| Result | Count |",
81
+ "| --- | ---: |",
82
+ ...entries.map(([label, count]) => `| ${label} | ${count} |`),
83
+ "",
84
+ `Initial Kane runs recorded ${counts.passed} passed, ${counts.failed} failed, ${counts.error} errors, and ${counts.cancelled} cancelled.`,
85
+ ...lineTable(report),
86
+ ...(review.length ? ["", "**For review**", "", ...review] : []),
87
+ ...needsReviewLines(report, needsReview),
88
+ ].join("\n");
89
+ }
90
+ const VERDICT_LABELS = {
91
+ true: "Held",
92
+ worded_differently: "Held, worded differently",
93
+ unconfirmed: "Unconfirmed",
94
+ false: "Did not match the app",
95
+ outdated: "Did not match the app",
96
+ };
97
+ const upperFirst = (text) => `${text.charAt(0).toUpperCase()}${text.slice(1)}`;
98
+ /** Frozen checks that were not run because the line changed or the check was not reviewed. */
99
+ function needsReviewLines(report, needsReview) {
100
+ const lines = [
101
+ ...needsReview.map(({ line, why }) => `- Line ${line}: ${why}`),
102
+ ...report.rejected.filter(({ why }) => why.startsWith(NEEDS_REVIEW_PREFIX)).map(({ why }) => `- ${upperFirst(why.slice(NEEDS_REVIEW_PREFIX.length).trimStart())}`),
103
+ ];
104
+ return lines.length ? ["", "**Needs review (not run)**", "", ...lines] : [];
105
+ }
106
+ /** One row per tested or ranked claim: its line and what happened, without quoting the text. */
107
+ function lineTable(report) {
108
+ const line = (id) => report.claims.find((claim) => claim.id === id)?.source.startLine ?? id.split("#L")[1]?.split("-")[0] ?? id;
109
+ const rows = [
110
+ ...report.tested.map(({ claim, kane, questioning }) => {
111
+ const runs = [kane.status, ...(questioning?.followUps ?? []).map(({ purpose, kane: run }) => `${run.status} (${purpose})`)].join(", ");
112
+ return `| ${claim.source.startLine} | ${questioning ? VERDICT_LABELS[questioning.verdict] : "No verdict"} | ${runs} |`;
113
+ }),
114
+ ...report.notTested.map((id) => `| ${line(id)} | Not run (over the claim limit) | none |`),
115
+ ];
116
+ return rows.length ? ["", "| Line | Result | Kane runs |", "| ---: | --- | --- |", ...rows] : [];
117
+ }
118
+ /** Generated test passwords (see resolveTarget) and local paths are removed before anything reaches a public log. */
119
+ function scrub(text) {
120
+ return text
121
+ .replace(/\s*[\r\n]+\s*/g, " ")
122
+ .replace(/Pk[0-9a-f]{24}/g, "[secret]")
123
+ .replace(/(?:~|\b[A-Za-z]:)?[\\/](?:Users|home|private|tmp|var|opt|root|mnt|Volumes)[\\/]\S+/g, "[path]");
124
+ }
125
+ function claimRuns(report) {
126
+ return report.tested.map(({ claim, kane, questioning }) => ({
127
+ claim,
128
+ verdict: questioning?.verdict ?? "no verdict",
129
+ runs: [{ purpose: "initial", kane }, ...(questioning?.followUps ?? []).map(({ purpose, kane: run }) => ({ purpose, kane: run }))],
130
+ }));
131
+ }
132
+ /**
133
+ * The public job-log transcript: every Kane run for every checked line, with its steps and observations,
134
+ * so a pass or a failure can be read after the runner is gone. Generated test passwords are removed.
135
+ */
136
+ export function kaneTranscript(report) {
137
+ const lines = [];
138
+ for (const { claim, verdict, runs } of claimRuns(report)) {
139
+ lines.push(`::group::Line ${claim.source.startLine} (${verdict}): ${runs.length} Kane run(s)`);
140
+ lines.push(scrub(`Docs: ${claim.quote}`));
141
+ runs.forEach(({ purpose, kane }, index) => {
142
+ lines.push("", scrub(`Run ${index + 1} (${purpose}): ${kane.status}. ${kane.oneLiner}`));
143
+ lines.push(scrub(`Objective: ${kane.objective.replace(/\s+/g, " ")}`));
144
+ for (const step of kane.steps)
145
+ lines.push(scrub(` step ${step.step} ${step.status}: ${step.remark}`));
146
+ for (const o of kane.observations)
147
+ lines.push(scrub(` observed ${o.condition}: expected ${o.expectedValue}, saw ${o.extractedValue}`));
148
+ });
149
+ lines.push("::endgroup::");
150
+ }
151
+ return lines.join("\n");
152
+ }
153
+ /** Copies each run's step screenshots to `destination/line-N/run-K-purpose/`. Returns how many folders were copied. */
154
+ export function copyEvidence(report, destination) {
155
+ let copied = 0;
156
+ for (const { claim, runs } of claimRuns(report)) {
157
+ runs.forEach(({ purpose, kane }, index) => {
158
+ const source = join(kane.outDir, "evidence");
159
+ if (!existsSync(source))
160
+ return;
161
+ cpSync(source, join(destination, `line-${claim.source.startLine}`, `run-${index + 1}-${purpose}`), { recursive: true });
162
+ copied += 1;
163
+ });
164
+ }
165
+ return copied;
166
+ }
167
+ function publicPullRequest(eventName, raw) {
168
+ if (eventName !== "pull_request_target")
169
+ throw new Error("Use the trusted-base pull_request_target workflow.");
170
+ const event = raw;
171
+ const repository = event?.repository?.full_name;
172
+ const pr = event?.pull_request;
173
+ if (typeof repository !== "string" || !/^[\w-]+\/[\w.-]+$/.test(repository) || event?.repository?.private !== false) {
174
+ throw new Error("fibcheck requires a public GitHub repository.");
175
+ }
176
+ if (pr?.head?.repo?.full_name !== repository || pr.base?.repo?.full_name !== repository) {
177
+ throw new Error("fibcheck requires a pull request from a branch in the same repository.");
178
+ }
179
+ const { head, base } = pr;
180
+ if (typeof head.sha !== "string" || typeof base.sha !== "string" || !COMMIT.test(head.sha) || !COMMIT.test(base.sha)) {
181
+ throw new Error("The pull request event must provide pinned head and base commits.");
182
+ }
183
+ return { repository, head: head.sha, base: base.sha };
184
+ }
185
+ function githubUrl(value, host) {
186
+ if (!URL.canParse(value))
187
+ throw new Error("GitHub runner URLs are required.");
188
+ const url = new URL(value);
189
+ if (url.protocol !== "https:" || url.host !== host || url.username || url.password || url.pathname !== "/" || url.search || url.hash) {
190
+ throw new Error("The pull request check requires GitHub.com runner URLs.");
191
+ }
192
+ return url;
193
+ }
194
+ function repositoryPath(value, extension, problem) {
195
+ const parts = value.split("/");
196
+ if (!value.endsWith(extension) || parts.some((part) => !/^[\w.-]+$/.test(part) || part === "." || part === "..")) {
197
+ throw new Error(problem);
198
+ }
199
+ return parts.map(encodeURIComponent).join("/");
200
+ }
201
+ async function publicFile(api, repo, path, revision, request) {
202
+ const url = new URL(`repos/${repo}/contents/${path}`, api);
203
+ url.searchParams.set("ref", revision);
204
+ const response = await request(url, {
205
+ headers: { Accept: "application/vnd.github+json" },
206
+ redirect: "error",
207
+ signal: AbortSignal.timeout(30_000),
208
+ });
209
+ if (!response.ok)
210
+ throw new Error(`GitHub could not read ${path} at the selected revision (HTTP ${response.status}).`);
211
+ const file = await response.json();
212
+ if (file.type !== "file" || file.encoding !== "base64" || typeof file.content !== "string" || !/^[A-Za-z0-9+/=\n\r]+$/.test(file.content)) {
213
+ throw new Error(`GitHub did not return supported file contents for ${path}.`);
214
+ }
215
+ return Buffer.from(file.content, "base64").toString("utf8");
216
+ }
@@ -0,0 +1,66 @@
1
+ import { execFileSync } from "node:child_process";
2
+ import { existsSync, readdirSync, readFileSync } from "node:fs";
3
+ import { join, relative } from "node:path";
4
+ export function distinctKaneRuns(left, right) {
5
+ if (left.sessionDir && right.sessionDir) {
6
+ if (left.sessionDir === right.sessionDir)
7
+ return false;
8
+ return !left.testUrl || !right.testUrl || left.testUrl !== right.testUrl;
9
+ }
10
+ return Boolean(left.testUrl && right.testUrl && left.testUrl !== right.testUrl);
11
+ }
12
+ /** Finds the evidence pack for a run: the path Kane reported, else the newest pack in the session folder. */
13
+ export function findEvidencePack(reported, sessionDir) {
14
+ if (reported && existsSync(reported))
15
+ return reported;
16
+ if (!sessionDir)
17
+ return null;
18
+ const dir = join(sessionDir, "evidence");
19
+ if (!existsSync(dir))
20
+ return null;
21
+ const packs = readdirSync(dir).filter((f) => f.endsWith(".evidence"));
22
+ return packs.length ? join(dir, packs.sort((a, b) => a.localeCompare(b)).at(-1) ?? "") : null;
23
+ }
24
+ /**
25
+ * Unpacks the step screenshots and step records of a Kane evidence pack (a
26
+ * zip file) into `destDir`, and lists them in step order. Uses the system
27
+ * `unzip`, which macOS and GitHub's Linux runners include.
28
+ */
29
+ export function extractStepEvidence(packPath, destDir) {
30
+ execFileSync("unzip", ["-o", "-q", packPath, "tests/*/steps/*", "-d", destDir], { stdio: ["ignore", "ignore", "pipe"] });
31
+ const testsDir = join(destDir, "tests");
32
+ if (!existsSync(testsDir))
33
+ return [];
34
+ const steps = [];
35
+ for (const test of readdirSync(testsDir)) {
36
+ const stepsDir = join(testsDir, test, "steps");
37
+ if (!existsSync(stepsDir))
38
+ continue;
39
+ for (const name of readdirSync(stepsDir))
40
+ steps.push(readStep(join(stepsDir, name), name, destDir));
41
+ }
42
+ return steps.sort((a, b) => a.ordinal - b.ordinal);
43
+ }
44
+ function readStep(dir, name, destDir) {
45
+ const record = readJson(join(dir, "step.json"));
46
+ const file = (f) => (existsSync(join(dir, f)) ? relative(destDir, join(dir, f)) : null);
47
+ const failurePath = join(dir, "failure.yaml");
48
+ return {
49
+ ordinal: typeof record.ordinal === "number" ? record.ordinal : Number.parseInt(name, 10) || 0,
50
+ kind: String(record.kind ?? ""),
51
+ status: String(record.status ?? ""),
52
+ summary: String(record.summary ?? ""),
53
+ url: String(record.url ?? ""),
54
+ screenshot: file("screenshot.jpg") ?? file("screenshot.png"),
55
+ annotated: file("annotated.png"),
56
+ failureYaml: existsSync(failurePath) ? readFileSync(failurePath, "utf8") : null,
57
+ };
58
+ }
59
+ function readJson(path) {
60
+ try {
61
+ return JSON.parse(readFileSync(path, "utf8"));
62
+ }
63
+ catch {
64
+ return {};
65
+ }
66
+ }
@@ -0,0 +1,28 @@
1
+ import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
2
+ import { tmpdir } from "node:os";
3
+ import { join } from "node:path";
4
+ export function escapeKaneBraces(text) {
5
+ return text.replace(/[{}]{2,}/g, (braces) => braces.split("").join(" "));
6
+ }
7
+ export function writeKaneVariables(variables) {
8
+ if (Object.keys(variables).length === 0)
9
+ return undefined;
10
+ const directory = mkdtempSync(join(tmpdir(), "fibcheck-kane-"));
11
+ const path = join(directory, "variables.json");
12
+ const remove = () => rmSync(directory, { recursive: true, force: true });
13
+ try {
14
+ writeFileSync(path, JSON.stringify(variables), { flag: "wx", mode: 0o600 });
15
+ }
16
+ catch (error) {
17
+ remove();
18
+ throw error;
19
+ }
20
+ return { path, remove };
21
+ }
22
+ /**
23
+ * Documentation often names its own default address (for example http://localhost:5230). The test app runs
24
+ * elsewhere and Kane already starts on it, so a local address in a check is replaced by a reference to the app.
25
+ */
26
+ export function localAddressesToApp(check) {
27
+ return check.replace(/\bhttps?:\/\/(?:localhost|127\.0\.0\.1|0\.0\.0\.0)(?::\d+)?(\/[^\s"'<>]*)?/gi, (_match, path) => (path && path !== "/" ? `the app's ${path.replace(/[.,;:]+$/, "")} page` : "the app's start page"));
28
+ }
@@ -0,0 +1,105 @@
1
+ /**
2
+ * Parses the NDJSON that `kane-cli run --agent` prints. Field names come
3
+ * from real Kane CLI 0.10.4 output saved in T1 (see fixtures/kane/).
4
+ * Unknown lines and fields are ignored, as Kane's docs ask.
5
+ */
6
+ function readStream(stdout) {
7
+ const state = { steps: new Map(), pack: null, errorMessage: "", ignoredLines: 0 };
8
+ for (const line of stdout.split("\n")) {
9
+ if (!line.trim())
10
+ continue;
11
+ try {
12
+ applyEvent(state, JSON.parse(line));
13
+ }
14
+ catch {
15
+ state.ignoredLines++;
16
+ }
17
+ }
18
+ return state;
19
+ }
20
+ function applyEvent(state, obj) {
21
+ if (typeof obj.step === "number" && (obj.status === "done" || obj.status === "failed")) {
22
+ state.steps.set(obj.step, { step: obj.step, status: obj.status, remark: str(obj.remark) });
23
+ }
24
+ else if (obj.type === "run_end") {
25
+ state.end = obj;
26
+ }
27
+ else if (obj.type === "coverage" && typeof obj.pack === "string") {
28
+ state.pack = obj.pack;
29
+ }
30
+ else if (obj.type === "error") {
31
+ state.errorMessage = str(obj.message);
32
+ }
33
+ }
34
+ export function parseKaneOutput(stdout, exitCode) {
35
+ const { steps, end, pack, errorMessage, ignoredLines } = readStream(stdout);
36
+ const stepList = [...steps.values()].sort((a, b) => a.step - b.step);
37
+ const failed = stepList.filter((s) => s.status === "failed");
38
+ return {
39
+ status: statusFrom(exitCode, end),
40
+ exitCode,
41
+ oneLiner: str(end?.one_liner) || errorMessage,
42
+ summary: str(end?.summary),
43
+ reason: str(end?.reason) || errorMessage || inconclusiveReason(exitCode, end),
44
+ reasonCode: str(end?.reason_code),
45
+ durationSec: num(end?.duration),
46
+ credits: num(end?.credits_consumed) ?? num(end?.credits),
47
+ testUrl: str(end?.test_url) || null,
48
+ sessionDir: str(end?.session_dir) || null,
49
+ evidencePack: pack,
50
+ steps: stepList,
51
+ failedStep: failed[failed.length - 1] ?? null,
52
+ observations: observationsFrom(end),
53
+ blame: blameFrom(end),
54
+ ignoredLines,
55
+ };
56
+ }
57
+ function statusFrom(exitCode, end) {
58
+ if (exitCode === 2)
59
+ return "error";
60
+ if (exitCode === 3)
61
+ return "cancelled";
62
+ if (end?.status === "passed" && exitCode === 0)
63
+ return "passed";
64
+ if (end?.status === "failed")
65
+ return "failed";
66
+ // A pass with a non-zero exit (Kane's docs: exit 1 can mean the upload failed) is
67
+ // inconclusive. Reporting it as a failure could lead to a false accusation.
68
+ return "error";
69
+ }
70
+ function inconclusiveReason(exitCode, end) {
71
+ if (end?.status === "passed" && exitCode !== 0)
72
+ return `Kane reported a pass but exited with code ${exitCode}; treated as inconclusive.`;
73
+ return end ? "" : `Kane ended without a result (exit code ${exitCode}).`;
74
+ }
75
+ function observationsFrom(end) {
76
+ const memory = end?.context?.memory;
77
+ if (!memory || typeof memory !== "object")
78
+ return [];
79
+ return Object.entries(memory).flatMap(([name, m]) => (m && typeof m === "object" ? [{
80
+ name,
81
+ condition: str(m.condition),
82
+ extractedValue: str(m.extracted_value),
83
+ expectedValue: str(m.expected_value),
84
+ analyzerType: str(m.analyzer_type),
85
+ usedDomCode: Boolean(m.code_js || m.wrapped_js || (Array.isArray(m.locators) ? m.locators.length : m.locators)),
86
+ }] : []));
87
+ }
88
+ function blameFrom(end) {
89
+ const v = end?.verdict;
90
+ if (!v || typeof v !== "object" || !v.family)
91
+ return null;
92
+ return {
93
+ family: str(v.family),
94
+ category: str(v.category),
95
+ rootCause: str(v.root_cause),
96
+ confidence: num(v.confidence),
97
+ agentFaultAssessment: str(v.agent_fault_assessment),
98
+ };
99
+ }
100
+ function str(value) {
101
+ return typeof value === "string" ? value : "";
102
+ }
103
+ function num(value) {
104
+ return typeof value === "number" && Number.isFinite(value) ? value : null;
105
+ }