fibcheck 0.0.0-stage → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +203 -2
- package/dist/app/docker.js +55 -0
- package/dist/attack-run.js +64 -0
- package/dist/attack.js +143 -0
- package/dist/card.js +159 -0
- package/dist/check.js +233 -0
- package/dist/claims/cache.js +45 -0
- package/dist/claims/documents.js +54 -0
- package/dist/claims/extract.js +144 -0
- package/dist/claims/quotes.js +52 -0
- package/dist/claims/rank.js +36 -0
- package/dist/claims/types.js +2 -0
- package/dist/cli.js +235 -0
- package/dist/confirm.js +83 -0
- package/dist/diagnose.js +150 -0
- package/dist/frozen.js +112 -0
- package/dist/general-loop.js +122 -0
- package/dist/github.js +214 -0
- package/dist/kane/evidence.js +66 -0
- package/dist/kane/parse.js +105 -0
- package/dist/kane/runner.js +58 -0
- package/dist/linkding-tags.js +207 -0
- package/dist/loop-types.js +1 -0
- package/dist/loop.js +186 -0
- package/dist/model/budget.js +82 -0
- package/dist/model/client.js +59 -0
- package/dist/model/config.js +124 -0
- package/dist/model/gemini.js +100 -0
- package/dist/model/openai-compatible.js +146 -0
- package/dist/model/types.js +28 -0
- package/dist/report/report.js +297 -0
- package/dist/target.js +70 -0
- package/package.json +46 -4
- package/targets/linkding.json +17 -0
package/dist/card.js
ADDED
|
@@ -0,0 +1,159 @@
|
|
|
1
|
+
import { execFileSync } from "node:child_process";
|
|
2
|
+
import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
|
|
3
|
+
import { tmpdir } from "node:os";
|
|
4
|
+
import { dirname, extname, join, resolve } from "node:path";
|
|
5
|
+
const OWN_STAMPS = {
|
|
6
|
+
false: "FIB",
|
|
7
|
+
outdated: "OUTDATED",
|
|
8
|
+
true: "TRUE",
|
|
9
|
+
worded_differently: "WORDED DIFFERENTLY",
|
|
10
|
+
unconfirmed: "CAN'T TELL",
|
|
11
|
+
};
|
|
12
|
+
/** Third-party projects get no "fib" or "lie": a false line is described as not matching the app. */
|
|
13
|
+
const THIRD_PARTY_STAMPS = {
|
|
14
|
+
...OWN_STAMPS,
|
|
15
|
+
false: "DOESN'T MATCH THE APP",
|
|
16
|
+
};
|
|
17
|
+
/** The words on a card, built only from the recorded verdict and evidence. */
|
|
18
|
+
export function cardText(finding) {
|
|
19
|
+
const stamp = (finding.own ? OWN_STAMPS : THIRD_PARTY_STAMPS)[finding.verdict];
|
|
20
|
+
const { file, line } = finding.source;
|
|
21
|
+
return {
|
|
22
|
+
stamp,
|
|
23
|
+
headline: headline(finding),
|
|
24
|
+
quote: finding.quote,
|
|
25
|
+
source: `${finding.project} · ${file}, line ${line} · ${finding.source.commit.slice(0, 7)}`,
|
|
26
|
+
evidence: evidenceLine(finding),
|
|
27
|
+
score: scoreLine(finding),
|
|
28
|
+
...(finding.own ? {} : {
|
|
29
|
+
thanks: finding.upstream?.status === "merged"
|
|
30
|
+
? `Thank you to the ${finding.project} maintainers for a great project, and for merging the docs fix.`
|
|
31
|
+
: `Thank you to the ${finding.project} maintainers for a great project. A small docs update would help.`,
|
|
32
|
+
}),
|
|
33
|
+
};
|
|
34
|
+
}
|
|
35
|
+
function headline(finding) {
|
|
36
|
+
const isFinding = finding.verdict === "false" || finding.verdict === "outdated";
|
|
37
|
+
if (finding.verdict === "unconfirmed")
|
|
38
|
+
return "Can't tell. We refuse to guess.";
|
|
39
|
+
if (finding.own) {
|
|
40
|
+
if (isFinding)
|
|
41
|
+
return "Your README fibbed. Here's the receipt.";
|
|
42
|
+
if (finding.verdict === "worded_differently")
|
|
43
|
+
return `Your README said it differently. The app says ${finding.visibleWording ?? "something else"}.`;
|
|
44
|
+
return "Your README told the truth. Here's the receipt.";
|
|
45
|
+
}
|
|
46
|
+
if (isFinding)
|
|
47
|
+
return finding.verdict === "outdated" ? "This docs line is outdated." : "This docs line doesn't match the app.";
|
|
48
|
+
if (finding.verdict === "worded_differently")
|
|
49
|
+
return `The app words this differently: ${finding.visibleWording ?? "see the screenshot"}.`;
|
|
50
|
+
return "This docs line holds.";
|
|
51
|
+
}
|
|
52
|
+
/** Only the parts that really happened: two failures, a passing control, a proven fix, or a held behaviour. */
|
|
53
|
+
export function evidenceLine(finding) {
|
|
54
|
+
const { failures, control, fixProof, held } = finding.evidence;
|
|
55
|
+
const parts = [];
|
|
56
|
+
if (failures.length >= 2)
|
|
57
|
+
parts.push("Caught twice by Kane.");
|
|
58
|
+
else if (failures.length === 1)
|
|
59
|
+
parts.push("Caught once by Kane.");
|
|
60
|
+
if (held)
|
|
61
|
+
parts.push("Behaviour held in Kane's run.");
|
|
62
|
+
if (control)
|
|
63
|
+
parts.push("Control passed.");
|
|
64
|
+
if (fixProof)
|
|
65
|
+
parts.push("Fix proven ✓");
|
|
66
|
+
if (finding.upstream?.status === "merged")
|
|
67
|
+
parts.push("Fix merged by the maintainers ✓");
|
|
68
|
+
const lazy = finding.lazyFixes;
|
|
69
|
+
if (lazy && lazy.rejected > 0)
|
|
70
|
+
parts.push(`${lazy.rejected} lazy ${lazy.rejected === 1 ? "fix" : "fixes"} rejected${lazy.byKane ? ` (${lazy.byKane} by Kane)` : ""}.`);
|
|
71
|
+
return parts.join(" ");
|
|
72
|
+
}
|
|
73
|
+
function scoreLine(finding) {
|
|
74
|
+
const { findings, claims } = finding.score;
|
|
75
|
+
const claimWord = claims === 1 ? "claim" : "claims";
|
|
76
|
+
if (finding.own)
|
|
77
|
+
return `${findings} ${findings === 1 ? "fib" : "fibs"} in ${claims} ${claimWord}`;
|
|
78
|
+
return `${findings} outdated or mismatched ${findings === 1 ? "line" : "lines"} in ${claims} ${claimWord} checked`;
|
|
79
|
+
}
|
|
80
|
+
/** Reads and checks the findings file; every referenced screenshot must exist. */
|
|
81
|
+
export function loadFindings(path) {
|
|
82
|
+
const findings = JSON.parse(readFileSync(path, "utf8"));
|
|
83
|
+
for (const finding of findings) {
|
|
84
|
+
for (const file of evidenceFiles(finding)) {
|
|
85
|
+
if (!existsSync(join(dirname(path), file)))
|
|
86
|
+
throw new Error(`Finding ${finding.id} refers to a missing file: ${file}`);
|
|
87
|
+
}
|
|
88
|
+
if ((finding.verdict === "false" || finding.verdict === "outdated") && (finding.evidence.failures.length < 2 || !finding.evidence.control)) {
|
|
89
|
+
throw new Error(`Finding ${finding.id} is ${finding.verdict} without two recorded failures and a control.`);
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
return findings;
|
|
93
|
+
}
|
|
94
|
+
export function evidenceFiles(finding) {
|
|
95
|
+
const { failures, control, fixProof, held } = finding.evidence;
|
|
96
|
+
return [finding.screenshot, ...failures, control, fixProof, held].filter((file) => Boolean(file));
|
|
97
|
+
}
|
|
98
|
+
const escapeHtml = (text) => text.replace(/[&<>"']/g, (c) => ({ "&": "&", "<": "<", ">": ">", '"': """, "'": "'" })[c] ?? c);
|
|
99
|
+
/**
|
|
100
|
+
* The card as HTML, 1200 by 675 pixels. The screenshot sits in its own frame and is only scaled to fit
|
|
101
|
+
* (`object-fit: contain`): never cropped, and nothing is drawn over it.
|
|
102
|
+
*/
|
|
103
|
+
export function renderCardHtml(finding, imageDataUrl) {
|
|
104
|
+
const text = cardText(finding);
|
|
105
|
+
const tone = finding.verdict === "true" || finding.verdict === "worded_differently" ? "#2f7d4f" : finding.verdict === "unconfirmed" ? "#6b6475" : "#b4462f";
|
|
106
|
+
return `<!doctype html><html><head><meta charset="utf-8"><style>
|
|
107
|
+
*{box-sizing:border-box;margin:0}body{width:1200px;height:675px;font-family:-apple-system,system-ui,"Segoe UI",sans-serif;background:#fbf8f1;color:#233f33;display:flex}
|
|
108
|
+
.text{width:520px;padding:40px 36px;display:flex;flex-direction:column;gap:18px}
|
|
109
|
+
.brand{font-weight:800;font-size:20px;letter-spacing:.5px}.stamp{align-self:flex-start;border:4px solid ${tone};color:${tone};font-weight:900;font-size:30px;padding:6px 14px;border-radius:8px;transform:rotate(-3deg)}
|
|
110
|
+
.headline{font-size:24px;font-weight:700;line-height:1.25}.quote{font-size:19px;line-height:1.35;border-left:4px solid #d9cfb8;padding-left:12px;font-style:italic}
|
|
111
|
+
.source{font-size:14px;color:#655c75}.evidence{font-size:17px;font-weight:600}.score{font-size:16px}.thanks{font-size:14px;color:#655c75}
|
|
112
|
+
.shot{flex:1;padding:24px;display:flex;align-items:center;justify-content:center;background:#233f33}
|
|
113
|
+
.shot img{max-width:100%;max-height:100%;object-fit:contain;display:block}
|
|
114
|
+
</style></head><body><div class="text">
|
|
115
|
+
<div class="brand">fibcheck</div><div class="stamp">${escapeHtml(text.stamp)}</div>
|
|
116
|
+
<div class="headline">${escapeHtml(text.headline)}</div>
|
|
117
|
+
<div class="quote">“${escapeHtml(text.quote)}”</div><div class="source">${escapeHtml(text.source)}</div>
|
|
118
|
+
<div class="evidence">${escapeHtml(text.evidence)}</div><div class="score">${escapeHtml(text.score)}</div>
|
|
119
|
+
${text.thanks ? `<div class="thanks">${escapeHtml(text.thanks)}</div>` : ""}
|
|
120
|
+
</div><div class="shot"><img alt="Kane CLI screenshot, scaled only" src="${imageDataUrl}"></div></body></html>`;
|
|
121
|
+
}
|
|
122
|
+
/** Finds a Chrome or Chromium to draw the card; Kane CLI already needs one. */
|
|
123
|
+
export function findChrome(env = process.env) {
|
|
124
|
+
const candidates = [
|
|
125
|
+
env.FIBCHECK_CHROME,
|
|
126
|
+
"/Applications/Google Chrome.app/Contents/MacOS/Google Chrome",
|
|
127
|
+
"/usr/bin/google-chrome",
|
|
128
|
+
"/usr/bin/chromium",
|
|
129
|
+
"/usr/bin/chromium-browser",
|
|
130
|
+
].filter((path) => Boolean(path));
|
|
131
|
+
const found = candidates.find((path) => existsSync(path));
|
|
132
|
+
if (!found)
|
|
133
|
+
throw new Error("Could not find Chrome or Chromium to draw the card. Set FIBCHECK_CHROME to its path.");
|
|
134
|
+
return found;
|
|
135
|
+
}
|
|
136
|
+
/** Writes `<outDir>/<id>.png` for one finding and returns its path. */
|
|
137
|
+
export function writeCard(findingsPath, id, outDir, chrome = findChrome()) {
|
|
138
|
+
const finding = loadFindings(findingsPath).find((entry) => entry.id === id);
|
|
139
|
+
if (!finding)
|
|
140
|
+
throw new Error(`No finding called ${id} in ${findingsPath}.`);
|
|
141
|
+
const image = resolve(dirname(findingsPath), finding.screenshot);
|
|
142
|
+
const mime = extname(image).toLowerCase() === ".png" ? "image/png" : "image/jpeg";
|
|
143
|
+
const html = renderCardHtml(finding, `data:${mime};base64,${readFileSync(image).toString("base64")}`);
|
|
144
|
+
const work = mkdtempSync(join(tmpdir(), "fibcheck-card-"));
|
|
145
|
+
try {
|
|
146
|
+
const page = join(work, "card.html");
|
|
147
|
+
writeFileSync(page, html);
|
|
148
|
+
mkdirSync(outDir, { recursive: true });
|
|
149
|
+
const out = resolve(outDir, `${id}.png`);
|
|
150
|
+
execFileSync(chrome, ["--headless=new", "--disable-gpu", "--hide-scrollbars", "--force-device-scale-factor=1",
|
|
151
|
+
"--window-size=1200,675", `--screenshot=${out}`, `file://${page}`], { stdio: "ignore", timeout: 60_000 });
|
|
152
|
+
if (!existsSync(out))
|
|
153
|
+
throw new Error("Chrome did not write the card.");
|
|
154
|
+
return out;
|
|
155
|
+
}
|
|
156
|
+
finally {
|
|
157
|
+
rmSync(work, { recursive: true, force: true });
|
|
158
|
+
}
|
|
159
|
+
}
|
package/dist/check.js
ADDED
|
@@ -0,0 +1,233 @@
|
|
|
1
|
+
import { execFileSync } from "node:child_process";
|
|
2
|
+
import { existsSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
|
|
3
|
+
import { tmpdir } from "node:os";
|
|
4
|
+
import { join, resolve } from "node:path";
|
|
5
|
+
import { startApp } from "./app/docker.js";
|
|
6
|
+
import { extractClaimsCached } from "./claims/cache.js";
|
|
7
|
+
import { collectDocuments } from "./claims/documents.js";
|
|
8
|
+
import { locateQuote } from "./claims/quotes.js";
|
|
9
|
+
import { rankClaims } from "./claims/rank.js";
|
|
10
|
+
import { runKane } from "./kane/runner.js";
|
|
11
|
+
import { seedLinkdingTags } from "./linkding-tags.js";
|
|
12
|
+
import { modelDiagnoser } from "./diagnose.js";
|
|
13
|
+
import { questionClaim } from "./loop.js";
|
|
14
|
+
import { missingLabelAssertion } from "./frozen.js";
|
|
15
|
+
import { loadTarget, resolveTarget, runningAppVariables } from "./target.js";
|
|
16
|
+
/** Reads a target's docs, extracts claims and runs the selected browser checks. */
|
|
17
|
+
export async function runCheck(client, options) {
|
|
18
|
+
validateOptions(options);
|
|
19
|
+
const log = options.log ?? (() => { });
|
|
20
|
+
const runId = new Date().toISOString().replace(/[:.]/g, "-");
|
|
21
|
+
const workDir = options.workDir ?? ".fibcheck";
|
|
22
|
+
const runDir = resolve(options.runsDir ?? "runs", runId);
|
|
23
|
+
mkdirSync(runDir, { recursive: true });
|
|
24
|
+
const loaded = loadTarget(options.targetPath);
|
|
25
|
+
const target = withAppUrl(resolveTarget(options.login === undefined ? loaded : { ...loaded, login: options.login }), options.appUrl);
|
|
26
|
+
const startedAt = new Date().toISOString();
|
|
27
|
+
const repoDir = options.source?.directory ?? fetchRepo(target.repo, join(workDir, "repos", target.name));
|
|
28
|
+
const commit = options.source?.commit ?? gitHead(repoDir);
|
|
29
|
+
const documents = collectDocuments(repoDir, target.docs ?? []);
|
|
30
|
+
log(`Read ${documents.length} document(s) from ${target.repo}${commit ? ` at ${commit.slice(0, 7)}` : ""}.`);
|
|
31
|
+
const extraction = options.reviewedClaims
|
|
32
|
+
? verifyReviewedClaims(documents, options.reviewedClaims, options.frozen)
|
|
33
|
+
: await extractClaimsCached(client, documents, join(workDir, "cache", "claims"));
|
|
34
|
+
// Every claim that names a UI element must open with an exact-label assertion; a check without one is not run.
|
|
35
|
+
const unlabelled = extraction.claims.flatMap((claim) => {
|
|
36
|
+
const why = claim.type !== "untestable" && claim.check ? missingLabelAssertion(claim) : undefined;
|
|
37
|
+
return why ? [{ claim, why }] : [];
|
|
38
|
+
});
|
|
39
|
+
const testable = extraction.claims.filter((c) => c.type !== "untestable" && c.check && !unlabelled.some(({ claim }) => claim === c));
|
|
40
|
+
for (const { claim, why } of unlabelled) {
|
|
41
|
+
extraction.rejected.push({ quote: claim.quote, file: claim.source.file, why: `Needs review: line ${claim.source.startLine}. ${why}` });
|
|
42
|
+
}
|
|
43
|
+
log(`Found ${extraction.claims.length} claims (${testable.length} testable), ${extraction.rejected.length} rejected quotes, ${extraction.cacheHits} document(s) from cache.`);
|
|
44
|
+
if (extraction.unusable?.length)
|
|
45
|
+
log(`The model's reply could not be used for ${extraction.unusable.length} document part(s); their claims are missing.`);
|
|
46
|
+
// The most specific claims first: named controls, steps, settings and defaults are what go stale.
|
|
47
|
+
const ranked = rankClaims(testable);
|
|
48
|
+
const candidates = ranked.map(({ claim }) => claim);
|
|
49
|
+
const selected = (options.only?.length ? candidates.filter((c) => options.only?.includes(c.id)) : candidates).slice(0, options.maxClaims ?? 5);
|
|
50
|
+
const tested = await testClaims(target, selected, { runDir, runId, options, log, client });
|
|
51
|
+
const testedClaims = new Set(tested.map(({ claim }) => claim.id));
|
|
52
|
+
const notTested = candidates.filter((claim) => !testedClaims.has(claim.id)).map((claim) => claim.id);
|
|
53
|
+
const report = {
|
|
54
|
+
runId,
|
|
55
|
+
target: target.name,
|
|
56
|
+
repo: target.repo,
|
|
57
|
+
commit,
|
|
58
|
+
startedAt,
|
|
59
|
+
finishedAt: new Date().toISOString(),
|
|
60
|
+
documents: documents.map((d) => d.file),
|
|
61
|
+
claims: extraction.claims,
|
|
62
|
+
rejected: extraction.rejected,
|
|
63
|
+
...(extraction.unusable?.length ? { unusableModelOutput: extraction.unusable } : {}),
|
|
64
|
+
tested,
|
|
65
|
+
notTested,
|
|
66
|
+
ranking: ranked.map(({ claim, score, reasons }) => ({ id: claim.id, score, reasons })),
|
|
67
|
+
kaneCredits: totalKaneCredits(tested),
|
|
68
|
+
model: { ...client.usage(), cacheHits: extraction.cacheHits },
|
|
69
|
+
};
|
|
70
|
+
writeFileSync(join(runDir, "check.json"), `${JSON.stringify(report, null, 2)}\n`);
|
|
71
|
+
log(`Saved ${join(runDir, "check.json")}`);
|
|
72
|
+
return report;
|
|
73
|
+
}
|
|
74
|
+
/** The most Kane runs one claim can use: initial check, one rewording, a reproduction and a control. */
|
|
75
|
+
export const DEFAULT_RUNS_PER_CLAIM = 4;
|
|
76
|
+
/** Keeps supplied claims whose quotes appear in their documents, with corrected line numbers. */
|
|
77
|
+
export function verifyReviewedClaims(documents, claims, frozen = false) {
|
|
78
|
+
const verified = [];
|
|
79
|
+
const rejected = [];
|
|
80
|
+
for (const claim of claims) {
|
|
81
|
+
const document = documents.find(({ file }) => file === claim.source.file);
|
|
82
|
+
const source = document && locateQuote(document.content, claim.quote, claim.source);
|
|
83
|
+
if (source)
|
|
84
|
+
verified.push({ ...claim, source });
|
|
85
|
+
else if (frozen && document)
|
|
86
|
+
rejected.push({ quote: claim.quote, file: claim.source.file, why: `Needs review: line ${claim.source.startLine} no longer says what was reviewed, so its frozen check was not run.` });
|
|
87
|
+
else
|
|
88
|
+
rejected.push({ quote: claim.quote, file: claim.source.file, why: document ? "The quote was not found in the document." : "The document was not read." });
|
|
89
|
+
}
|
|
90
|
+
return { claims: verified, rejected, cacheHits: 0 };
|
|
91
|
+
}
|
|
92
|
+
/** Points a target at an app that is already running somewhere else. */
|
|
93
|
+
export function withAppUrl(target, appUrl) {
|
|
94
|
+
if (appUrl === undefined)
|
|
95
|
+
return target;
|
|
96
|
+
if (!URL.canParse(appUrl))
|
|
97
|
+
throw new Error("--url must be a full address such as https://bookmarks.example.org.");
|
|
98
|
+
const url = new URL(appUrl);
|
|
99
|
+
if (url.protocol !== "http:" && url.protocol !== "https:")
|
|
100
|
+
throw new Error("--url must start with http:// or https://.");
|
|
101
|
+
if (url.username || url.password)
|
|
102
|
+
throw new Error("--url must not contain a user name or password. Put login values in .env instead.");
|
|
103
|
+
const baseUrl = `${url.origin}${url.pathname.replace(/\/+$/, "")}`;
|
|
104
|
+
return { ...target, baseUrl };
|
|
105
|
+
}
|
|
106
|
+
function validateOptions({ maxKaneRuns, appRunningSecrets, appRunning, appUrl }) {
|
|
107
|
+
if (maxKaneRuns !== undefined && (!Number.isSafeInteger(maxKaneRuns) || maxKaneRuns < 0)) {
|
|
108
|
+
throw new Error("--max-kane-runs must be a whole number.");
|
|
109
|
+
}
|
|
110
|
+
if (appRunningSecrets !== undefined && !appRunning && !appUrl) {
|
|
111
|
+
throw new Error("appRunningSecrets requires appRunning.");
|
|
112
|
+
}
|
|
113
|
+
}
|
|
114
|
+
function totalKaneCredits(tested) {
|
|
115
|
+
return tested.reduce((sum, { kane, questioning }) => (sum + (kane.credits ?? 0) + (questioning?.followUps.reduce((cost, round) => cost + (round.kane.credits ?? 0), 0) ?? 0)), 0);
|
|
116
|
+
}
|
|
117
|
+
async function testClaims(target, claims, context) {
|
|
118
|
+
const { runDir, runId, options, log, client } = context;
|
|
119
|
+
if (claims.length === 0 || options.maxKaneRuns === 0)
|
|
120
|
+
return [];
|
|
121
|
+
let app;
|
|
122
|
+
const alreadyRunning = Boolean(options.appRunning || options.appUrl);
|
|
123
|
+
const variables = alreadyRunning ? runningAppVariables(target.login, options.appRunningSecrets ?? {}) : target.kaneVariables;
|
|
124
|
+
const variablesFile = writeKaneVariables(variables);
|
|
125
|
+
const managedTarget = alreadyRunning ? undefined : target;
|
|
126
|
+
try {
|
|
127
|
+
if (managedTarget) {
|
|
128
|
+
log(`Starting ${target.app?.image ?? "the app"} on port ${target.app?.port ?? "?"}...`);
|
|
129
|
+
app = await startApp(managedTarget, runId);
|
|
130
|
+
}
|
|
131
|
+
return await runClaims(target, claims, {
|
|
132
|
+
runDir, log, managedTarget, containerName: app?.containerName,
|
|
133
|
+
variablesFile: variablesFile?.path, maxKaneRuns: options.maxKaneRuns, frozen: options.frozen,
|
|
134
|
+
diagnose: options.diagnose === false ? undefined : modelDiagnoser(client, [...Object.values(variables).map(({ value }) => value), new URL(target.baseUrl).host]),
|
|
135
|
+
});
|
|
136
|
+
}
|
|
137
|
+
finally {
|
|
138
|
+
try {
|
|
139
|
+
variablesFile?.remove();
|
|
140
|
+
}
|
|
141
|
+
finally {
|
|
142
|
+
await app?.stop();
|
|
143
|
+
}
|
|
144
|
+
}
|
|
145
|
+
}
|
|
146
|
+
async function runClaims(target, claims, session) {
|
|
147
|
+
const { runDir, log, managedTarget, containerName, variablesFile } = session;
|
|
148
|
+
const tested = [];
|
|
149
|
+
const startUrl = `${target.baseUrl}${target.startPath ?? "/"}`;
|
|
150
|
+
// Without an explicit cap, each claim may use its full questioning budget of 4 Kane runs and no more.
|
|
151
|
+
let runsRemaining = session.maxKaneRuns ?? DEFAULT_RUNS_PER_CLAIM * claims.length;
|
|
152
|
+
const canRun = () => runsRemaining > 0;
|
|
153
|
+
let tagSetup;
|
|
154
|
+
const prepareTags = containerName ? () => (tagSetup ??= seedLinkdingTags(containerName)) : undefined;
|
|
155
|
+
for (const [index, claim] of claims.entries()) {
|
|
156
|
+
if (!canRun()) {
|
|
157
|
+
log("Kane run limit reached. Remaining claims were not tested.");
|
|
158
|
+
break;
|
|
159
|
+
}
|
|
160
|
+
log(`[${index + 1}/${claims.length}] ${claim.id}: "${claim.quote}"`);
|
|
161
|
+
const folder = safeName(claim.id);
|
|
162
|
+
const result = await questionClaim(claim, managedTarget, objectiveFor(claim, target), async (purpose, objective, assertionMode) => {
|
|
163
|
+
runsRemaining--;
|
|
164
|
+
const outDir = join(runDir, "claims", purpose === "initial" ? folder : `${folder}--${purpose}`);
|
|
165
|
+
const kane = await runKane({ objective, startUrl, outDir, assertionMode, variablesFile });
|
|
166
|
+
log(` Kane ${purpose}: ${kane.status} (${kane.credits?.toFixed(1) ?? "?"} credits) ${kane.oneLiner}`);
|
|
167
|
+
return kane;
|
|
168
|
+
}, prepareTags, canRun, session.diagnose && {
|
|
169
|
+
diagnose: session.diagnose,
|
|
170
|
+
objectiveFor: (check) => objectiveFor({ ...claim, check }, target),
|
|
171
|
+
frozen: session.frozen,
|
|
172
|
+
});
|
|
173
|
+
log(` Claim verdict: ${result.questioning.verdict}. ${result.questioning.reason}`);
|
|
174
|
+
tested.push({ claim, ...result });
|
|
175
|
+
}
|
|
176
|
+
return tested;
|
|
177
|
+
}
|
|
178
|
+
export function writeKaneVariables(variables) {
|
|
179
|
+
if (Object.keys(variables).length === 0)
|
|
180
|
+
return undefined;
|
|
181
|
+
const directory = mkdtempSync(join(tmpdir(), "fibcheck-kane-"));
|
|
182
|
+
const path = join(directory, "variables.json");
|
|
183
|
+
const remove = () => rmSync(directory, { recursive: true, force: true });
|
|
184
|
+
try {
|
|
185
|
+
writeFileSync(path, JSON.stringify(variables), { flag: "wx", mode: 0o600 });
|
|
186
|
+
}
|
|
187
|
+
catch (error) {
|
|
188
|
+
remove();
|
|
189
|
+
throw error;
|
|
190
|
+
}
|
|
191
|
+
return { path, remove };
|
|
192
|
+
}
|
|
193
|
+
/**
|
|
194
|
+
* The flow Kane runs for one claim: the login sentence, the documented claim
|
|
195
|
+
* as context, and the check. `{{` in quoted text is broken up so Kane does not
|
|
196
|
+
* read it as a variable.
|
|
197
|
+
*/
|
|
198
|
+
export function objectiveFor(claim, target) {
|
|
199
|
+
const safe = (text) => text.replace(/[{}]{2,}/g, (braces) => braces.split("").join(" "));
|
|
200
|
+
const parts = [
|
|
201
|
+
target.login,
|
|
202
|
+
`The app's documentation (${claim.source.file}, line ${claim.source.startLine}) says: "${safe(claim.quote)}".`,
|
|
203
|
+
safe(localAddressesToApp(claim.check ?? "")),
|
|
204
|
+
];
|
|
205
|
+
return parts.filter(Boolean).join(" ");
|
|
206
|
+
}
|
|
207
|
+
/**
|
|
208
|
+
* Documentation often names its own default address (for example http://localhost:5230). The test app runs
|
|
209
|
+
* elsewhere and Kane already starts on it, so a local address in a check is replaced by a reference to the app.
|
|
210
|
+
*/
|
|
211
|
+
export function localAddressesToApp(check) {
|
|
212
|
+
return check.replace(/\bhttps?:\/\/(?:localhost|127\.0\.0\.1|0\.0\.0\.0)(?::\d+)?(\/[^\s"'<>]*)?/gi, (_match, path) => (path && path !== "/" ? `the app's ${path.replace(/[.,;:]+$/, "")} page` : "the app's start page"));
|
|
213
|
+
}
|
|
214
|
+
function fetchRepo(repo, cloneDir) {
|
|
215
|
+
if (existsSync(repo))
|
|
216
|
+
return repo;
|
|
217
|
+
if (!existsSync(join(cloneDir, ".git"))) {
|
|
218
|
+
mkdirSync(cloneDir, { recursive: true });
|
|
219
|
+
execFileSync("git", ["clone", "--depth", "1", "--quiet", repo, cloneDir], { stdio: ["ignore", "ignore", "pipe"] });
|
|
220
|
+
}
|
|
221
|
+
return cloneDir;
|
|
222
|
+
}
|
|
223
|
+
function gitHead(dir) {
|
|
224
|
+
try {
|
|
225
|
+
return execFileSync("git", ["-C", dir, "rev-parse", "HEAD"], { encoding: "utf8" }).trim();
|
|
226
|
+
}
|
|
227
|
+
catch {
|
|
228
|
+
return null;
|
|
229
|
+
}
|
|
230
|
+
}
|
|
231
|
+
function safeName(id) {
|
|
232
|
+
return id.replace(/[^a-zA-Z0-9_.-]+/g, "_");
|
|
233
|
+
}
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
import { createHash } from "node:crypto";
|
|
2
|
+
import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
|
|
3
|
+
import { join } from "node:path";
|
|
4
|
+
import { EXTRACT_SYSTEM, extractClaims } from "./extract.js";
|
|
5
|
+
/**
|
|
6
|
+
* Extracts claims, reusing an earlier result for an identical document and
|
|
7
|
+
* prompt. On the Gemini free tier (20 requests a day per model) this keeps
|
|
8
|
+
* repeated checks of the same README from spending requests.
|
|
9
|
+
*/
|
|
10
|
+
export async function extractClaimsCached(client, documents, cacheDir) {
|
|
11
|
+
const result = { claims: [], rejected: [], unusable: [], cacheHits: 0 };
|
|
12
|
+
mkdirSync(cacheDir, { recursive: true });
|
|
13
|
+
for (const document of documents) {
|
|
14
|
+
const path = join(cacheDir, `${cacheKey(document)}.json`);
|
|
15
|
+
let part = readCache(path);
|
|
16
|
+
if (part) {
|
|
17
|
+
result.cacheHits++;
|
|
18
|
+
}
|
|
19
|
+
else {
|
|
20
|
+
part = await extractClaims(client, [document]);
|
|
21
|
+
// A document with an unusable model reply is not cached, so the next check tries it again.
|
|
22
|
+
if (!part.unusable?.length)
|
|
23
|
+
writeFileSync(path, `${JSON.stringify(part, null, 2)}\n`);
|
|
24
|
+
}
|
|
25
|
+
result.claims.push(...part.claims);
|
|
26
|
+
result.rejected.push(...part.rejected);
|
|
27
|
+
result.unusable?.push(...(part.unusable ?? []));
|
|
28
|
+
}
|
|
29
|
+
return result;
|
|
30
|
+
}
|
|
31
|
+
/** A damaged cache file counts as a miss, so the document is simply extracted again. */
|
|
32
|
+
function readCache(path) {
|
|
33
|
+
if (!existsSync(path))
|
|
34
|
+
return undefined;
|
|
35
|
+
try {
|
|
36
|
+
return JSON.parse(readFileSync(path, "utf8"));
|
|
37
|
+
}
|
|
38
|
+
catch {
|
|
39
|
+
return undefined;
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
/** Changes when the document, its path or the extraction prompt changes. */
|
|
43
|
+
export function cacheKey(document) {
|
|
44
|
+
return createHash("sha256").update(EXTRACT_SYSTEM).update("\0").update(document.file).update("\0").update(document.content).digest("hex").slice(0, 32);
|
|
45
|
+
}
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
import { existsSync, readdirSync, readFileSync, statSync } from "node:fs";
|
|
2
|
+
import { dirname, join, normalize, relative, sep } from "node:path";
|
|
3
|
+
const README_NAMES = ["README.md", "readme.md", "Readme.md", "README.markdown", "README"];
|
|
4
|
+
/** At most this many linked documents are read, to keep model use bounded. */
|
|
5
|
+
export const MAX_LINKED_DOCS = 10;
|
|
6
|
+
/**
|
|
7
|
+
* Collects the documents to check in a local repository: the README, the
|
|
8
|
+
* Markdown files it links to inside the repository, and any extra files or
|
|
9
|
+
* folders the user names. Paths in the result are relative to the repository.
|
|
10
|
+
*/
|
|
11
|
+
export function collectDocuments(repoDir, extra = []) {
|
|
12
|
+
const readme = README_NAMES.map((name) => join(repoDir, name)).find((path) => existsSync(path));
|
|
13
|
+
if (!readme)
|
|
14
|
+
throw new Error(`No README found in ${repoDir}.`);
|
|
15
|
+
const paths = new Set([readme]);
|
|
16
|
+
for (const linked of linkedMarkdown(readme, repoDir).slice(0, MAX_LINKED_DOCS))
|
|
17
|
+
paths.add(linked);
|
|
18
|
+
for (const item of extra) {
|
|
19
|
+
for (const path of expand(join(repoDir, item)))
|
|
20
|
+
paths.add(path);
|
|
21
|
+
}
|
|
22
|
+
return [...paths].map((path) => ({ file: toPosix(relative(repoDir, path)), content: readFileSync(path, "utf8") }));
|
|
23
|
+
}
|
|
24
|
+
/** Relative links from a Markdown file to other Markdown files inside the repository. */
|
|
25
|
+
export function linkedMarkdown(file, repoDir) {
|
|
26
|
+
const text = readFileSync(file, "utf8");
|
|
27
|
+
const found = [];
|
|
28
|
+
for (const match of text.matchAll(/\]\(([^)\s]+)\)/g)) {
|
|
29
|
+
const target = (match[1] ?? "").split("#")[0] ?? "";
|
|
30
|
+
if (!target || /^[a-z]+:/i.test(target) || target.startsWith("/"))
|
|
31
|
+
continue;
|
|
32
|
+
if (!/\.(md|mdx|markdown)$/i.test(target))
|
|
33
|
+
continue;
|
|
34
|
+
const path = normalize(join(dirname(file), target));
|
|
35
|
+
if (!path.startsWith(normalize(repoDir) + sep))
|
|
36
|
+
continue;
|
|
37
|
+
if (existsSync(path) && !found.includes(path))
|
|
38
|
+
found.push(path);
|
|
39
|
+
}
|
|
40
|
+
return found;
|
|
41
|
+
}
|
|
42
|
+
function expand(path) {
|
|
43
|
+
if (!existsSync(path))
|
|
44
|
+
throw new Error(`Document not found: ${path}`);
|
|
45
|
+
if (!statSync(path).isDirectory())
|
|
46
|
+
return [path];
|
|
47
|
+
return readdirSync(path, { recursive: true, encoding: "utf8" })
|
|
48
|
+
.filter((name) => /\.(md|mdx|markdown)$/i.test(name))
|
|
49
|
+
.sort((a, b) => a.localeCompare(b))
|
|
50
|
+
.map((name) => join(path, name));
|
|
51
|
+
}
|
|
52
|
+
function toPosix(path) {
|
|
53
|
+
return path.split(sep).join("/");
|
|
54
|
+
}
|
|
@@ -0,0 +1,144 @@
|
|
|
1
|
+
import { UnusableModelOutputError } from "../model/types.js";
|
|
2
|
+
import { locateQuote, normalise, splitLines } from "./quotes.js";
|
|
3
|
+
import { CLAIM_TYPES } from "./types.js";
|
|
4
|
+
/** Documents longer than this are sent in parts, so no claim is lost to a long input. */
|
|
5
|
+
export const CHUNK_LINES = 250;
|
|
6
|
+
export const EXTRACT_SYSTEM = `You read the README and usage documentation of an open-source web app and list every claim that a browser test could check against the running app.
|
|
7
|
+
|
|
8
|
+
Claim types:
|
|
9
|
+
- setup: how to install or start the app and reach it (commands, URLs, ports, default logins).
|
|
10
|
+
- ui_flow: a path through the user interface ("go to Settings and click Import", a named page, menu, tab or button).
|
|
11
|
+
- behaviour: what the app does or offers when used (features, keyboard shortcuts, formats it imports or exports, what is shown or hidden, defaults).
|
|
12
|
+
- mobile: claims about phones, tablets or mobile apps.
|
|
13
|
+
- untestable: a concrete statement a browser test cannot check (performance, security, licence, hosting, third-party services, marketing adjectives such as "fast" or "clean"). Give the reason.
|
|
14
|
+
|
|
15
|
+
What counts as a claim (be exhaustive; a borderline claim is better listed than missed):
|
|
16
|
+
- Every item of a feature list, each as its own claim.
|
|
17
|
+
- Limitations and negative statements: what the app "only" allows, cannot do, does not support, or what is disabled.
|
|
18
|
+
- Defaults: what is shown, enabled or selected "by default", default ports, default users and passwords. If a sentence introduces credentials or a command shown in the next lines, quote that sentence.
|
|
19
|
+
- Where something is found in the UI, and what a page or section contains.
|
|
20
|
+
- One sentence can hold several claims. In "While A, B", "A, but B" or "A, so B", A and B are separate claims: list each, quoting the clause that states it. For example, "While the web app only lets you add labels, the admin panel can rename and delete them" gives two claims: "the web app only lets you add labels" (checked in the web app, by looking for rename and delete actions on an existing label) and "the admin panel can rename and delete them" (checked in the admin panel). Never return the whole sentence as one claim. Side remarks about the main UI in a section about something else (for example "the UI currently only allows X") are claims too.
|
|
21
|
+
Go through the document from top to bottom, section by section, and check every sentence and list item.
|
|
22
|
+
|
|
23
|
+
Rules:
|
|
24
|
+
- Quote each claim exactly as written, copied character for character from a single place in the document, without the line-number prefix. Keep each quote to one sentence or one list item.
|
|
25
|
+
- Give the first and last line number of the quote, using the numbers shown before each line.
|
|
26
|
+
- For testable claims, write "check" as a short sequence of browser actions and explicit assertions. When the documentation names a UI element by its label (a tab, menu, button, link or dropdown, such as "the *Admin* tab" or "the Save button"), the check's first assertion is an exact-label assertion for it, made before anything is clicked by that label and placed where the documentation says the element is, in the form "Open the Settings page. Assert exactly: The Settings page content shows a tab labelled 'Admin'. Expected: true." Put the label in single quotes and name the element type. When the documentation names a page, section, menu or tab, assert the exact location and control type before acting. Then say not to reach the result through any other menu, page or route. A control with the same name in a different place or of a different type cannot pass the check: an item in a dropdown or navigation menu is not a tab inside a page, and a page reached from a menu is not a view the documentation says to open first. Assert every stated, observable result separately.
|
|
27
|
+
- Test what the claim actually promises, not a step on the way. "Organize items with labels" needs filtering or grouping by a label, not only adding one. "Share with guests" needs the shared item opened as a guest, by logging out or opening its share link. "Import and export" needs both directions. "Edit" needs the change saved and seen again.
|
|
28
|
+
- Create the test data the check needs first, with unique names, instead of assuming it exists. If the feature may be switched off by default, say to switch it on in the app's settings first and name where.
|
|
29
|
+
- After each navigation, assert the destination by its page title or heading before going on. For saved, edited or deleted results, reload or reopen the page and assert the result again.
|
|
30
|
+
- Never write the app's address, host or port into a check. The test already starts on the running app, which may use a different port from the documentation. Refer to pages by their name instead. When a form needs a web address as data, such as a bookmark's URL, give https://example.com as the value to type.
|
|
31
|
+
- Write each assertion as a positive statement with an expected value, for example "The Tags page shows an Edit action for tag X. Expected: false". Avoid double negatives such as "assert that X is not absent".
|
|
32
|
+
- Split a navigation instruction and a claim about opening a new browser tab or window. Keep the supported navigation and its destination in a testable check. Record the separate tab or window claim as untestable because Kane cannot inspect browser tab or window state.
|
|
33
|
+
- For a limitation, first assert that the relevant page is loaded and the required test data is visible. Then check specifically for actions that would contradict the limitation. For example, if the ordinary interface only supports creating or assigning records, use an existing record and check separately for Edit and Remove there. If a separate administration interface can manage records, quote and test that clause independently in the administration interface.
|
|
34
|
+
- Skip headings, links-only lines, code that is not about using the app, contributor and development instructions, and changelog entries.
|
|
35
|
+
- Do not invent claims or merge separate claims. If there are no claims, return an empty list.`;
|
|
36
|
+
const RESPONSE_SCHEMA = {
|
|
37
|
+
type: "object",
|
|
38
|
+
properties: {
|
|
39
|
+
claims: {
|
|
40
|
+
type: "array",
|
|
41
|
+
items: {
|
|
42
|
+
type: "object",
|
|
43
|
+
properties: {
|
|
44
|
+
type: { type: "string", enum: [...CLAIM_TYPES] },
|
|
45
|
+
quote: { type: "string" },
|
|
46
|
+
startLine: { type: "integer" },
|
|
47
|
+
endLine: { type: "integer" },
|
|
48
|
+
check: { type: "string" },
|
|
49
|
+
reason: { type: "string" },
|
|
50
|
+
},
|
|
51
|
+
required: ["type", "quote", "startLine", "endLine"],
|
|
52
|
+
},
|
|
53
|
+
},
|
|
54
|
+
},
|
|
55
|
+
required: ["claims"],
|
|
56
|
+
};
|
|
57
|
+
/** Lists the claims in each document, keeping only claims whose quote is really in the document. */
|
|
58
|
+
export async function extractClaims(client, documents) {
|
|
59
|
+
const claims = [];
|
|
60
|
+
const rejected = [];
|
|
61
|
+
const unusable = [];
|
|
62
|
+
const seen = new Set();
|
|
63
|
+
for (const document of documents) {
|
|
64
|
+
for (const [index, chunk] of chunkDocument(document).entries()) {
|
|
65
|
+
let value;
|
|
66
|
+
try {
|
|
67
|
+
({ value } = await client.generateJson({
|
|
68
|
+
purpose: "extract-claims",
|
|
69
|
+
system: EXTRACT_SYSTEM,
|
|
70
|
+
prompt: `File: ${document.file}\n\n${chunk}`,
|
|
71
|
+
jsonSchema: RESPONSE_SCHEMA,
|
|
72
|
+
}, parseRawClaims));
|
|
73
|
+
}
|
|
74
|
+
catch (error) {
|
|
75
|
+
if (!(error instanceof UnusableModelOutputError))
|
|
76
|
+
throw error;
|
|
77
|
+
unusable.push({ file: document.file, lines: `${index * CHUNK_LINES + 1}-${(index + 1) * CHUNK_LINES}`, why: error.message });
|
|
78
|
+
continue;
|
|
79
|
+
}
|
|
80
|
+
for (const raw of value) {
|
|
81
|
+
const source = locateQuote(document.content, raw.quote, {
|
|
82
|
+
file: document.file,
|
|
83
|
+
startLine: raw.startLine,
|
|
84
|
+
endLine: raw.endLine,
|
|
85
|
+
});
|
|
86
|
+
if (!source) {
|
|
87
|
+
rejected.push({ quote: raw.quote, file: document.file, why: "quote not found in the document" });
|
|
88
|
+
continue;
|
|
89
|
+
}
|
|
90
|
+
const key = `${source.file}:${source.startLine}:${normalise(raw.quote)}`;
|
|
91
|
+
if (seen.has(key))
|
|
92
|
+
continue;
|
|
93
|
+
seen.add(key);
|
|
94
|
+
claims.push(toClaim(raw, source, claims));
|
|
95
|
+
}
|
|
96
|
+
}
|
|
97
|
+
}
|
|
98
|
+
claims.sort((a, b) => a.source.file.localeCompare(b.source.file) || a.source.startLine - b.source.startLine);
|
|
99
|
+
return unusable.length ? { claims, rejected, unusable } : { claims, rejected };
|
|
100
|
+
}
|
|
101
|
+
/** Splits a document into numbered parts of at most CHUNK_LINES lines, keeping absolute line numbers. */
|
|
102
|
+
export function chunkDocument(document) {
|
|
103
|
+
const lines = splitLines(document.content);
|
|
104
|
+
const chunks = [];
|
|
105
|
+
for (let start = 0; start < lines.length; start += CHUNK_LINES) {
|
|
106
|
+
chunks.push(lines
|
|
107
|
+
.slice(start, start + CHUNK_LINES)
|
|
108
|
+
.map((line, i) => `${String(start + i + 1).padStart(4, " ")}| ${line}`)
|
|
109
|
+
.join("\n"));
|
|
110
|
+
}
|
|
111
|
+
return chunks;
|
|
112
|
+
}
|
|
113
|
+
/** Checks the model's JSON shape; drops malformed items rather than trusting them. */
|
|
114
|
+
export function parseRawClaims(value) {
|
|
115
|
+
const list = value?.claims;
|
|
116
|
+
if (!Array.isArray(list))
|
|
117
|
+
throw new Error("The model reply has no claims list.");
|
|
118
|
+
return list.filter(isRawClaim);
|
|
119
|
+
}
|
|
120
|
+
function isRawClaim(item) {
|
|
121
|
+
if (typeof item !== "object" || item === null)
|
|
122
|
+
return false;
|
|
123
|
+
const c = item;
|
|
124
|
+
return (typeof c.type === "string" &&
|
|
125
|
+
CLAIM_TYPES.includes(c.type) &&
|
|
126
|
+
typeof c.quote === "string" &&
|
|
127
|
+
Number.isInteger(c.startLine) &&
|
|
128
|
+
Number.isInteger(c.endLine) &&
|
|
129
|
+
(c.check === undefined || typeof c.check === "string") &&
|
|
130
|
+
(c.reason === undefined || typeof c.reason === "string"));
|
|
131
|
+
}
|
|
132
|
+
function toClaim(raw, source, existing) {
|
|
133
|
+
const base = `${source.file}#L${source.startLine}`;
|
|
134
|
+
const clashes = existing.filter((c) => c.id === base || c.id.startsWith(`${base}-`)).length;
|
|
135
|
+
const claim = { id: clashes === 0 ? base : `${base}-${clashes + 1}`, type: raw.type, quote: raw.quote.trim(), source };
|
|
136
|
+
if (raw.type === "untestable") {
|
|
137
|
+
if (raw.reason)
|
|
138
|
+
claim.reason = raw.reason;
|
|
139
|
+
}
|
|
140
|
+
else if (raw.check) {
|
|
141
|
+
claim.check = raw.check;
|
|
142
|
+
}
|
|
143
|
+
return claim;
|
|
144
|
+
}
|