fibcheck 0.0.0-stage → 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +13 -0
- package/LICENSE +21 -0
- package/README.md +198 -2
- package/dist/app/docker.js +56 -0
- package/dist/attack-run.js +63 -0
- package/dist/attack.js +147 -0
- package/dist/card.js +204 -0
- package/dist/check.js +210 -0
- package/dist/claims/cache.js +45 -0
- package/dist/claims/documents.js +54 -0
- package/dist/claims/extract.js +144 -0
- package/dist/claims/quotes.js +52 -0
- package/dist/claims/rank.js +36 -0
- package/dist/claims/types.js +5 -0
- package/dist/cli.js +249 -0
- package/dist/confirm.js +92 -0
- package/dist/diagnose.js +150 -0
- package/dist/findings.js +36 -0
- package/dist/frozen.js +125 -0
- package/dist/general-loop.js +122 -0
- package/dist/github.js +216 -0
- package/dist/kane/evidence.js +66 -0
- package/dist/kane/objective.js +28 -0
- package/dist/kane/parse.js +105 -0
- package/dist/kane/runner.js +59 -0
- package/dist/linkding-tags.js +207 -0
- package/dist/loop-types.js +1 -0
- package/dist/loop.js +191 -0
- package/dist/model/budget.js +91 -0
- package/dist/model/client.js +60 -0
- package/dist/model/config.js +125 -0
- package/dist/model/gemini.js +100 -0
- package/dist/model/openai-compatible.js +146 -0
- package/dist/model/types.js +28 -0
- package/dist/report/report.js +298 -0
- package/dist/target.js +85 -0
- package/dist/util.js +6 -0
- package/package.json +47 -4
- package/targets/linkding.json +17 -0
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
/** Prefixes every line with its 1-based number, so the model can cite lines. */
|
|
2
|
+
export function numberLines(content) {
|
|
3
|
+
return splitLines(content)
|
|
4
|
+
.map((line, index) => `${String(index + 1).padStart(4, " ")}| ${line}`)
|
|
5
|
+
.join("\n");
|
|
6
|
+
}
|
|
7
|
+
export function splitLines(content) {
|
|
8
|
+
return content.replace(/\r\n?/g, "\n").split("\n");
|
|
9
|
+
}
|
|
10
|
+
/**
|
|
11
|
+
* Normalises text for quote matching: drops Markdown emphasis, link
|
|
12
|
+
* targets and code marks, unifies quotes and dashes, collapses whitespace
|
|
13
|
+
* and lower-cases. The model often quotes "Admin tab" for "*Admin* tab".
|
|
14
|
+
*/
|
|
15
|
+
export function normalise(text) {
|
|
16
|
+
return text
|
|
17
|
+
.replace(/\[([^\]]*)\]\([^)]*\)/g, "$1")
|
|
18
|
+
.replace(/<[^>]+>/g, " ")
|
|
19
|
+
.replace(/[*_`]/g, "")
|
|
20
|
+
.replace(/[‘’]/g, "'")
|
|
21
|
+
.replace(/[“”]/g, '"')
|
|
22
|
+
.replace(/[–—]/g, "-")
|
|
23
|
+
.replace(/\s+/g, " ")
|
|
24
|
+
.trim()
|
|
25
|
+
.toLowerCase();
|
|
26
|
+
}
|
|
27
|
+
/**
|
|
28
|
+
* Finds where `quote` appears in the document. Tries the cited lines first
|
|
29
|
+
* (with two lines of slack either side), then the whole document. Returns
|
|
30
|
+
* the corrected source, or undefined if the quote is not in the document.
|
|
31
|
+
*/
|
|
32
|
+
export function locateQuote(content, quote, cited) {
|
|
33
|
+
const lines = splitLines(content);
|
|
34
|
+
const wanted = normalise(quote);
|
|
35
|
+
if (wanted.length < 8)
|
|
36
|
+
return undefined;
|
|
37
|
+
// Finds the smallest span (at most 12 lines) inside [from, to] that contains the quote:
|
|
38
|
+
// the earliest end line, then the latest start line for that end.
|
|
39
|
+
const searchWindow = (from, to) => {
|
|
40
|
+
const first = Math.max(1, from);
|
|
41
|
+
const last = Math.min(lines.length, to);
|
|
42
|
+
for (let end = first; end <= last; end++) {
|
|
43
|
+
for (let start = end; start >= Math.max(first, end - 11); start--) {
|
|
44
|
+
if (normalise(lines.slice(start - 1, end).join(" ")).includes(wanted)) {
|
|
45
|
+
return { file: cited.file, startLine: start, endLine: end };
|
|
46
|
+
}
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
return undefined;
|
|
50
|
+
};
|
|
51
|
+
return searchWindow(cited.startLine - 2, cited.endLine + 2) ?? searchWindow(1, lines.length);
|
|
52
|
+
}
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
const NAMED = /`[^`]+`|\*\*[^*]+\*\*|\*[^*\s][^*]*\*|"[^"]+"|“[^”]+”|'[A-Z][^']+'/g;
|
|
2
|
+
const CONTROLS = /\b(tab|menu|button|link|page|settings?|options?|checkbox|toggle|switch|dropdown|drop-down|sidebar|toolbar|icon|dialog|field|panel|section|shortcut|key)s?\b/gi;
|
|
3
|
+
const STEPS = /\b(go to|click|clicking|select|open|navigate|press|tap|choose|enable|disable|deselect|check|uncheck)\b|→|>/gi;
|
|
4
|
+
const DEFAULTS = /\bby default\b|\bdefaults? to\b|\b\d+(\.\d+)?\b/gi;
|
|
5
|
+
const TYPE_WEIGHT = { ui_flow: 3, behaviour: 1, setup: -2, mobile: -5, untestable: -10 };
|
|
6
|
+
/** Scores one claim. Higher means more specific. */
|
|
7
|
+
export function rankClaim(claim) {
|
|
8
|
+
const text = claim.quote;
|
|
9
|
+
const reasons = [];
|
|
10
|
+
let score = TYPE_WEIGHT[claim.type];
|
|
11
|
+
const count = (pattern) => text.match(pattern)?.length ?? 0;
|
|
12
|
+
const named = Math.min(count(NAMED), 3);
|
|
13
|
+
if (named)
|
|
14
|
+
reasons.push(`${named} named item${named === 1 ? "" : "s"}`);
|
|
15
|
+
const controls = Math.min(count(CONTROLS), 3);
|
|
16
|
+
if (controls)
|
|
17
|
+
reasons.push(`${controls} interface word${controls === 1 ? "" : "s"}`);
|
|
18
|
+
const steps = Math.min(count(STEPS), 3);
|
|
19
|
+
if (steps)
|
|
20
|
+
reasons.push(`${steps} step${steps === 1 ? "" : "s"}`);
|
|
21
|
+
const defaults = Math.min(count(DEFAULTS), 2);
|
|
22
|
+
if (defaults)
|
|
23
|
+
reasons.push("a default or value");
|
|
24
|
+
score += named * 3 + controls * 2 + steps * 2 + defaults * 2;
|
|
25
|
+
if (text.trim().split(/\s+/).length < 4) {
|
|
26
|
+
score -= 2;
|
|
27
|
+
reasons.push("very short");
|
|
28
|
+
}
|
|
29
|
+
return { claim, score, reasons };
|
|
30
|
+
}
|
|
31
|
+
/** Orders claims from most to least specific. Ties keep the document order. */
|
|
32
|
+
export function rankClaims(claims) {
|
|
33
|
+
return claims.map((claim, index) => ({ ranked: rankClaim(claim), index }))
|
|
34
|
+
.sort((a, b) => b.ranked.score - a.ranked.score || a.index - b.index)
|
|
35
|
+
.map(({ ranked }) => ranked);
|
|
36
|
+
}
|
package/dist/cli.js
ADDED
|
@@ -0,0 +1,249 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
import { existsSync, readFileSync, realpathSync, writeFileSync } from "node:fs";
|
|
3
|
+
import { join } from "node:path";
|
|
4
|
+
import { parseArgs } from "node:util";
|
|
5
|
+
import { pathToFileURL } from "node:url";
|
|
6
|
+
import { attackFinding } from "./attack-run.js";
|
|
7
|
+
import { attackSummary } from "./attack.js";
|
|
8
|
+
import { writeCard } from "./card.js";
|
|
9
|
+
import { runCheck } from "./check.js";
|
|
10
|
+
import { extractClaims } from "./claims/extract.js";
|
|
11
|
+
import { confirmFinding } from "./confirm.js";
|
|
12
|
+
import { loadFindings } from "./findings.js";
|
|
13
|
+
import { freeze, frozenClaims } from "./frozen.js";
|
|
14
|
+
import { createModelClient, loadDotEnv } from "./model/config.js";
|
|
15
|
+
import { writeReport } from "./report/report.js";
|
|
16
|
+
import { loadTarget, loginSecretsFromEnv, targetPathFor } from "./target.js";
|
|
17
|
+
import { errorMessage, newRunId } from "./util.js";
|
|
18
|
+
export const USAGE = `fibcheck: finds the false claims in a web app's README with Kane CLI.
|
|
19
|
+
|
|
20
|
+
Usage:
|
|
21
|
+
fibcheck check <target.json | repository> [--max-claims N] [--max-kane-runs N] [--only CLAIM_ID ...]
|
|
22
|
+
[--url ADDRESS [--login SENTENCE] | --app-running]
|
|
23
|
+
fibcheck report <check.json>
|
|
24
|
+
fibcheck card <finding-id | --all> [--findings index/findings.json] [--out index/cards]
|
|
25
|
+
fibcheck attack <finding-id> --url ADDRESS [--login SENTENCE] [--findings index/findings.json]
|
|
26
|
+
fibcheck confirm <check.json> <claim-id> --verdict false|outdated [--fix SENTENCE] [--project NAME] [--own]
|
|
27
|
+
[--id ID] [--findings index/findings.json]
|
|
28
|
+
fibcheck freeze <docs.md> --out <claims.json>
|
|
29
|
+
fibcheck --help
|
|
30
|
+
|
|
31
|
+
Use sissbruecker/linkding or its GitHub URL for the bundled Linkding target.
|
|
32
|
+
A target file names the repository, its docs, and how to run the app (see targets/).
|
|
33
|
+
The check command needs Kane CLI (npm install -g @testmuai/kane-cli, then kane-cli login)
|
|
34
|
+
and GEMINI_API_KEY in .env. Without --url it also needs Docker, to start the app from the target.
|
|
35
|
+
--url checks an app that is already running there, with no Docker. --login replaces the target's
|
|
36
|
+
login sentence; write {{NAME}} for a value and put FIBCHECK_LOGIN_NAME=... in .env, for example
|
|
37
|
+
--login "Sign in as {{user}} with password {{pass}}." with FIBCHECK_LOGIN_USER and FIBCHECK_LOGIN_PASS.
|
|
38
|
+
Claims are ranked by how specific they are (named menus, buttons, settings, defaults and steps).
|
|
39
|
+
The top --max-claims (default 5; 5 to 10 is a good range) are tested, and the rest are listed as not tested.
|
|
40
|
+
The report command writes report.html and badge.svg beside a saved check.json.
|
|
41
|
+
The card command draws a PNG share card for a recorded finding, from its verdict and Kane screenshot only.
|
|
42
|
+
The attack command tries lazy fixes for a finding (delete the line, make it vague, hedge it, or guess a
|
|
43
|
+
wrong detail) against the app at --url; each must be rejected by rule or failed by Kane.
|
|
44
|
+
The confirm command records a candidate finding you have reviewed (two failed Kane runs and a passing
|
|
45
|
+
control) in a findings file, with copies of Kane's screenshots, so card and attack can use it. Pass --fix
|
|
46
|
+
with the corrected sentence that a fresh Kane run passed; attack needs it.
|
|
47
|
+
The freeze command writes the browser checks for a docs file (path relative to the repository root) once,
|
|
48
|
+
for a person to review. Fill in "reviewed" for each claim you approve and commit the file; the GitHub Action's
|
|
49
|
+
claims-file input then reruns exactly those checks. Editing a check afterwards needs a fresh freeze and review.
|
|
50
|
+
`;
|
|
51
|
+
export async function main(argv) {
|
|
52
|
+
const [command, ...rest] = argv;
|
|
53
|
+
if (command === undefined || command === "--help" || command === "-h") {
|
|
54
|
+
process.stdout.write(USAGE);
|
|
55
|
+
return 0;
|
|
56
|
+
}
|
|
57
|
+
if (command === "check")
|
|
58
|
+
return await checkCommand(rest);
|
|
59
|
+
if (command === "report")
|
|
60
|
+
return reportCommand(rest);
|
|
61
|
+
if (command === "card")
|
|
62
|
+
return cardCommand(rest);
|
|
63
|
+
if (command === "attack")
|
|
64
|
+
return await attackCommand(rest);
|
|
65
|
+
if (command === "freeze")
|
|
66
|
+
return await freezeCommand(rest);
|
|
67
|
+
if (command === "confirm")
|
|
68
|
+
return confirmCommand(rest);
|
|
69
|
+
process.stderr.write(`Unknown command: ${command}\n\n${USAGE}`);
|
|
70
|
+
return 1;
|
|
71
|
+
}
|
|
72
|
+
function confirmCommand(args) {
|
|
73
|
+
const { values, positionals } = parseArgs({
|
|
74
|
+
args, allowPositionals: true,
|
|
75
|
+
options: {
|
|
76
|
+
verdict: { type: "string" }, fix: { type: "string" }, project: { type: "string" },
|
|
77
|
+
own: { type: "boolean" }, id: { type: "string" }, findings: { type: "string" },
|
|
78
|
+
},
|
|
79
|
+
});
|
|
80
|
+
const [checkPath, claimId] = positionals;
|
|
81
|
+
if (!checkPath || !claimId || positionals.length !== 2 || (values.verdict !== "false" && values.verdict !== "outdated")) {
|
|
82
|
+
process.stderr.write("Usage: fibcheck confirm <check.json> <claim-id> --verdict false|outdated [--fix SENTENCE] [--project NAME] [--own] [--id ID] [--findings index/findings.json]\n");
|
|
83
|
+
return 1;
|
|
84
|
+
}
|
|
85
|
+
const findingsPath = values.findings ?? "index/findings.json";
|
|
86
|
+
const finding = confirmFinding({
|
|
87
|
+
checkPath, claimId, verdict: values.verdict, findingsPath,
|
|
88
|
+
fix: values.fix, project: values.project, own: values.own, id: values.id,
|
|
89
|
+
});
|
|
90
|
+
process.stdout.write(`Added ${finding.id} to ${findingsPath} with ${finding.evidence.failures.length} failed runs and a control.\n`);
|
|
91
|
+
if (!finding.fix)
|
|
92
|
+
process.stdout.write("No --fix was given, so attack cannot run on it yet.\n");
|
|
93
|
+
return 0;
|
|
94
|
+
}
|
|
95
|
+
async function freezeCommand(args) {
|
|
96
|
+
const { values, positionals } = parseArgs({ args, allowPositionals: true, options: { out: { type: "string" } } });
|
|
97
|
+
const [docsFile] = positionals;
|
|
98
|
+
if (!docsFile || !values.out || positionals.length !== 1) {
|
|
99
|
+
process.stderr.write("Usage: fibcheck freeze <docs.md> --out <claims.json>\n");
|
|
100
|
+
return 1;
|
|
101
|
+
}
|
|
102
|
+
loadDotEnv();
|
|
103
|
+
const runId = newRunId();
|
|
104
|
+
const extraction = await extractClaims(createModelClient(runId), [{ file: docsFile, content: readFileSync(docsFile, "utf8") }]);
|
|
105
|
+
const file = freeze(docsFile, extraction.claims);
|
|
106
|
+
writeFileSync(values.out, `${JSON.stringify(file, null, 2)}\n`, { flag: "wx" });
|
|
107
|
+
const { needsReview } = frozenClaims({ ...file, claims: file.claims.map((claim) => ({ ...claim, reviewed: { by: "check", at: "check" } })) });
|
|
108
|
+
process.stdout.write(`Wrote ${file.claims.length} check(s) to ${values.out} for review. None will run until "reviewed" is filled in.\n`);
|
|
109
|
+
for (const { line, why } of needsReview)
|
|
110
|
+
process.stdout.write(` Line ${line}: ${why}\n`);
|
|
111
|
+
return 0;
|
|
112
|
+
}
|
|
113
|
+
async function attackCommand(args) {
|
|
114
|
+
const { values, positionals } = parseArgs({
|
|
115
|
+
args, allowPositionals: true,
|
|
116
|
+
options: { url: { type: "string" }, login: { type: "string" }, findings: { type: "string" } },
|
|
117
|
+
});
|
|
118
|
+
const [id] = positionals;
|
|
119
|
+
if (!id || !values.url) {
|
|
120
|
+
process.stderr.write("Usage: fibcheck attack <finding-id> --url ADDRESS [--login SENTENCE] [--findings index/findings.json]\n");
|
|
121
|
+
return 1;
|
|
122
|
+
}
|
|
123
|
+
const loginError = loginArgError(values.login);
|
|
124
|
+
if (loginError) {
|
|
125
|
+
process.stderr.write(loginError);
|
|
126
|
+
return 1;
|
|
127
|
+
}
|
|
128
|
+
loadDotEnv();
|
|
129
|
+
const runId = newRunId();
|
|
130
|
+
const record = await attackFinding(createModelClient(runId), {
|
|
131
|
+
findingsPath: values.findings ?? "index/findings.json", id, appUrl: values.url, login: values.login,
|
|
132
|
+
secrets: loginSecretsFromEnv(values.login), runsDir: join("runs", `attack-${runId}`),
|
|
133
|
+
log: (line) => process.stdout.write(`${line}\n`),
|
|
134
|
+
});
|
|
135
|
+
for (const result of record.results)
|
|
136
|
+
process.stdout.write(`${result.kind}: ${result.outcome}. ${result.reason}\n`);
|
|
137
|
+
process.stdout.write(`${attackSummary(record.results) || "No lazy fix was rejected."}\n`);
|
|
138
|
+
return 0;
|
|
139
|
+
}
|
|
140
|
+
function cardCommand(args) {
|
|
141
|
+
const { values, positionals } = parseArgs({
|
|
142
|
+
args, allowPositionals: true,
|
|
143
|
+
options: { all: { type: "boolean" }, findings: { type: "string" }, out: { type: "string" } },
|
|
144
|
+
});
|
|
145
|
+
const findingsPath = values.findings ?? "index/findings.json";
|
|
146
|
+
const ids = values.all ? loadFindings(findingsPath).map(({ id }) => id) : positionals;
|
|
147
|
+
if (ids.length === 0) {
|
|
148
|
+
process.stderr.write("Usage: fibcheck card <finding-id | --all> [--findings index/findings.json] [--out index/cards]\n");
|
|
149
|
+
return 1;
|
|
150
|
+
}
|
|
151
|
+
for (const id of ids)
|
|
152
|
+
process.stdout.write(`Saved ${writeCard(findingsPath, id, values.out ?? "index/cards")}\n`);
|
|
153
|
+
return 0;
|
|
154
|
+
}
|
|
155
|
+
function reportCommand(args) {
|
|
156
|
+
const [inputPath] = args;
|
|
157
|
+
if (args.length !== 1 || !inputPath || inputPath.startsWith("-")) {
|
|
158
|
+
process.stderr.write("Usage: fibcheck report <check.json>\n");
|
|
159
|
+
return 1;
|
|
160
|
+
}
|
|
161
|
+
const { htmlPath, badgePath } = writeReport(inputPath);
|
|
162
|
+
process.stdout.write(`Saved ${htmlPath}\nSaved ${badgePath}\n`);
|
|
163
|
+
return 0;
|
|
164
|
+
}
|
|
165
|
+
/** Parses and validates `check` arguments, returning an error message instead of throwing. */
|
|
166
|
+
export function parseCheckArgs(args) {
|
|
167
|
+
const { values, positionals } = parseArgs({
|
|
168
|
+
args,
|
|
169
|
+
allowPositionals: true,
|
|
170
|
+
options: {
|
|
171
|
+
"max-claims": { type: "string" },
|
|
172
|
+
"max-kane-runs": { type: "string" },
|
|
173
|
+
only: { type: "string", multiple: true },
|
|
174
|
+
"app-running": { type: "boolean" },
|
|
175
|
+
url: { type: "string" },
|
|
176
|
+
login: { type: "string" },
|
|
177
|
+
},
|
|
178
|
+
});
|
|
179
|
+
const [target] = positionals;
|
|
180
|
+
if (!target || positionals.length !== 1)
|
|
181
|
+
return `check needs one target file or repository.\n\n${USAGE}`;
|
|
182
|
+
const maxClaims = wholeNumber(values["max-claims"], Number.isInteger);
|
|
183
|
+
if (maxClaims === null)
|
|
184
|
+
return "--max-claims must be a whole number.\n";
|
|
185
|
+
const maxKaneRuns = wholeNumber(values["max-kane-runs"], Number.isSafeInteger);
|
|
186
|
+
if (maxKaneRuns === null)
|
|
187
|
+
return "--max-kane-runs must be a whole number.\n";
|
|
188
|
+
if (values.login !== undefined && values.url === undefined)
|
|
189
|
+
return "--login needs --url.\n";
|
|
190
|
+
const loginError = loginArgError(values.login);
|
|
191
|
+
if (loginError)
|
|
192
|
+
return loginError;
|
|
193
|
+
if (values.url !== undefined && values["app-running"])
|
|
194
|
+
return "Use either --url or --app-running, not both.\n";
|
|
195
|
+
return { target, maxClaims, maxKaneRuns, only: values.only, appRunning: values["app-running"], url: values.url, login: values.login };
|
|
196
|
+
}
|
|
197
|
+
function loginArgError(login) {
|
|
198
|
+
if (login !== undefined && !/\{\{[A-Za-z_][A-Za-z0-9_]*\}\}/.test(login)) {
|
|
199
|
+
return "--login must name its values as {{NAME}} and keep them in .env as FIBCHECK_LOGIN_NAME, never type a password into the command.\n";
|
|
200
|
+
}
|
|
201
|
+
return undefined;
|
|
202
|
+
}
|
|
203
|
+
/** undefined when absent, null when not a non-negative whole number. */
|
|
204
|
+
function wholeNumber(raw, isWhole) {
|
|
205
|
+
if (raw === undefined)
|
|
206
|
+
return undefined;
|
|
207
|
+
const value = Number(raw);
|
|
208
|
+
return isWhole(value) && value >= 0 ? value : null;
|
|
209
|
+
}
|
|
210
|
+
async function checkCommand(args) {
|
|
211
|
+
const parsed = parseCheckArgs(args);
|
|
212
|
+
if (typeof parsed === "string") {
|
|
213
|
+
process.stderr.write(parsed);
|
|
214
|
+
return 1;
|
|
215
|
+
}
|
|
216
|
+
const targetPath = targetPathFor(parsed.target);
|
|
217
|
+
loadDotEnv();
|
|
218
|
+
const alreadyRunning = parsed.url !== undefined || parsed.appRunning === true;
|
|
219
|
+
const login = parsed.login ?? loadTarget(targetPath).login;
|
|
220
|
+
const runId = newRunId();
|
|
221
|
+
const report = await runCheck(createModelClient(runId), {
|
|
222
|
+
targetPath,
|
|
223
|
+
runId,
|
|
224
|
+
maxClaims: parsed.maxClaims,
|
|
225
|
+
maxKaneRuns: parsed.maxKaneRuns,
|
|
226
|
+
only: parsed.only,
|
|
227
|
+
appRunning: parsed.appRunning,
|
|
228
|
+
appUrl: parsed.url,
|
|
229
|
+
login: parsed.login,
|
|
230
|
+
appRunningSecrets: alreadyRunning ? loginSecretsFromEnv(login) : undefined,
|
|
231
|
+
log: (line) => process.stdout.write(`${line}\n`),
|
|
232
|
+
});
|
|
233
|
+
process.stdout.write(summaryLine(report));
|
|
234
|
+
return 0;
|
|
235
|
+
}
|
|
236
|
+
function summaryLine(report) {
|
|
237
|
+
const counts = report.tested.reduce((acc, t) => ((acc[t.kane.status] = (acc[t.kane.status] ?? 0) + 1), acc), {});
|
|
238
|
+
const results = Object.entries(counts).map(([k, v]) => `${v} ${k}`).join(", ") || "none";
|
|
239
|
+
const models = report.model.models?.length ? ` (${report.model.models.join(", ")})` : "";
|
|
240
|
+
return `\nTested ${report.tested.length} claim(s). Initial Kane results: ${results}. ` +
|
|
241
|
+
`Not tested: ${report.notTested.length}. Kane credits: ${report.kaneCredits.toFixed(1)}. Model calls: ${report.model.calls}${models}.\n`;
|
|
242
|
+
}
|
|
243
|
+
const executable = process.argv[1];
|
|
244
|
+
if (executable && existsSync(executable) && import.meta.url === pathToFileURL(realpathSync(executable)).href) {
|
|
245
|
+
main(process.argv.slice(2)).then((code) => (process.exitCode = code), (error) => {
|
|
246
|
+
process.stderr.write(`fibcheck: ${errorMessage(error)}\n`);
|
|
247
|
+
process.exitCode = 1;
|
|
248
|
+
});
|
|
249
|
+
}
|
package/dist/confirm.js
ADDED
|
@@ -0,0 +1,92 @@
|
|
|
1
|
+
import { copyFileSync, existsSync, mkdirSync, readFileSync } from "node:fs";
|
|
2
|
+
import { basename, dirname, extname, join } from "node:path";
|
|
3
|
+
import { readFindingsFile, writeFindingsFile } from "./findings.js";
|
|
4
|
+
import { errorMessage } from "./util.js";
|
|
5
|
+
/**
|
|
6
|
+
* Turns a candidate finding from a saved check into an entry in a findings file, after a person has read the
|
|
7
|
+
* evidence. It copies Kane's screenshots of the two failing runs and the passing control, unchanged, and
|
|
8
|
+
* refuses anything short of that, so `card` and `attack` only ever work from a backed finding.
|
|
9
|
+
*/
|
|
10
|
+
export function confirmFinding(options) {
|
|
11
|
+
if (options.verdict !== "false" && options.verdict !== "outdated")
|
|
12
|
+
throw new Error("A confirmed finding is false or outdated.");
|
|
13
|
+
const report = readJson(options.checkPath, "check");
|
|
14
|
+
const { tested, failures, control } = backedCandidate(report, options);
|
|
15
|
+
const findings = existsSync(options.findingsPath) ? readFindingsFile(options.findingsPath) : [];
|
|
16
|
+
const { claim } = tested;
|
|
17
|
+
const project = options.project ?? report.target;
|
|
18
|
+
const id = options.id ?? `${slug(project)}-${slug(basename(claim.source.file, extname(claim.source.file)))}-l${claim.source.startLine}`;
|
|
19
|
+
if (findings.some((finding) => finding.id === id))
|
|
20
|
+
throw new Error(`${options.findingsPath} already has a finding called ${id}.`);
|
|
21
|
+
const checkDir = dirname(options.checkPath);
|
|
22
|
+
mkdirSync(join(dirname(options.findingsPath), "evidence"), { recursive: true });
|
|
23
|
+
const copy = (kane, name, pick) => {
|
|
24
|
+
const source = screenshotOf(checkDir, kane, pick);
|
|
25
|
+
const file = `evidence/${id}-${name}${extname(source)}`;
|
|
26
|
+
copyFileSync(source, join(dirname(options.findingsPath), file));
|
|
27
|
+
return file;
|
|
28
|
+
};
|
|
29
|
+
const failureFiles = [copy(failures[0], "run1", "failed"), copy(failures[1], "run2", "failed")];
|
|
30
|
+
const finding = {
|
|
31
|
+
id, project, own: options.own ?? false, verdict: options.verdict, quote: claim.quote,
|
|
32
|
+
source: { repo: report.repo, commit: report.commit ?? "", file: claim.source.file, line: claim.source.startLine },
|
|
33
|
+
screenshot: failureFiles[0],
|
|
34
|
+
evidence: { failures: failureFiles, control: copy(control, "control", "last") },
|
|
35
|
+
score: { findings: 1, claims: report.tested.length },
|
|
36
|
+
...(options.fix ? { fix: options.fix } : {}),
|
|
37
|
+
};
|
|
38
|
+
writeFindingsFile(options.findingsPath, [...findings, finding]);
|
|
39
|
+
return finding;
|
|
40
|
+
}
|
|
41
|
+
/** The tested claim, if it is a candidate finding backed by two failed runs and a passing control. */
|
|
42
|
+
function backedCandidate(report, { claimId, checkPath }) {
|
|
43
|
+
const tested = report.tested.find(({ claim }) => claim.id === claimId);
|
|
44
|
+
if (!tested)
|
|
45
|
+
throw new Error(`${claimId} was not tested in ${checkPath}.`);
|
|
46
|
+
const verdict = tested.questioning?.verdict;
|
|
47
|
+
if (!tested.questioning?.candidateFinding && verdict !== "false" && verdict !== "outdated") {
|
|
48
|
+
throw new Error(`${claimId} is not a candidate finding in this check, so there is nothing to confirm.`);
|
|
49
|
+
}
|
|
50
|
+
const { failures, control } = backingRuns(tested);
|
|
51
|
+
if (failures.length < 2 || !control) {
|
|
52
|
+
throw new Error(`${claimId} needs two separate failed Kane runs and a passing control before it can be confirmed.`);
|
|
53
|
+
}
|
|
54
|
+
return { tested, failures, control };
|
|
55
|
+
}
|
|
56
|
+
function readJson(path, what) {
|
|
57
|
+
try {
|
|
58
|
+
return JSON.parse(readFileSync(path, "utf8"));
|
|
59
|
+
}
|
|
60
|
+
catch (error) {
|
|
61
|
+
throw new Error(`Could not read the ${what} file ${path}: ${errorMessage(error)}`);
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
/**
|
|
65
|
+
* The two failed runs behind the candidate and the passing control. The repeat ("retry") reproduces the exact check
|
|
66
|
+
* of the run just before it, which is the run read as a contradiction: the first run, or a reworded run when the
|
|
67
|
+
* first one failed for another reason. Both must have failed.
|
|
68
|
+
*/
|
|
69
|
+
function backingRuns({ kane, questioning }) {
|
|
70
|
+
const followUps = questioning?.followUps ?? [];
|
|
71
|
+
const retryAt = followUps.findIndex(({ purpose }) => purpose === "retry");
|
|
72
|
+
const control = followUps.find(({ purpose, kane: run }) => purpose === "control" && run.status === "passed")?.kane;
|
|
73
|
+
if (retryAt === -1)
|
|
74
|
+
return { failures: [], control };
|
|
75
|
+
const contradicted = retryAt === 0 ? kane : followUps[retryAt - 1].kane;
|
|
76
|
+
const failures = [contradicted, followUps[retryAt].kane].filter((run) => run.status === "failed");
|
|
77
|
+
return { failures, control };
|
|
78
|
+
}
|
|
79
|
+
/** Kane's screenshot for a run, from the claims/ folder saved beside the check: the failed step, or the last step. */
|
|
80
|
+
function screenshotOf(checkDir, kane, pick) {
|
|
81
|
+
const shots = kane.evidence.filter(({ screenshot }) => screenshot);
|
|
82
|
+
const step = (pick === "failed" ? shots.filter(({ status }) => status === "failed").at(-1) : undefined) ?? shots.at(-1);
|
|
83
|
+
if (!step?.screenshot)
|
|
84
|
+
throw new Error(`Kane saved no screenshot for the run in ${basename(kane.outDir)}.`);
|
|
85
|
+
const path = join(checkDir, "claims", basename(kane.outDir), "evidence", step.screenshot);
|
|
86
|
+
if (!existsSync(path))
|
|
87
|
+
throw new Error(`Kane's screenshot is missing: ${path}`);
|
|
88
|
+
return path;
|
|
89
|
+
}
|
|
90
|
+
function slug(text) {
|
|
91
|
+
return text.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-|-$/g, "");
|
|
92
|
+
}
|
package/dist/diagnose.js
ADDED
|
@@ -0,0 +1,150 @@
|
|
|
1
|
+
/** The model's reading of why a Kane run ended as it did. Kane alone decides passed or failed. */
|
|
2
|
+
export const FAILURE_CAUSES = ["agent_misstep", "check_problem", "setup_problem", "kane_limitation", "claim_contradicted", "unclear"];
|
|
3
|
+
export const DIAGNOSE_SYSTEM = `You review one Kane CLI browser test run. The run checked a claim from a web app's public documentation. Kane alone decides whether its run passed or failed, and you never change that. Your job is to read the run and say what it means for the claim.
|
|
4
|
+
|
|
5
|
+
1. Decide whether the run exercised the documented claim: the same page or place, the same kind of control (tab, menu item, button, field), and each stated result. A pass that reached the result through a different place or a different kind of control did not exercise the claim. An item in a dropdown or navigation menu is not a tab inside a page, even when Kane's remark calls it a tab: a pass exercises a claim about a tab or control inside a named page only when a step or observation shows that page loaded and the control found in its content.
|
|
6
|
+
2. For a failed or unfinished run, name the most likely cause:
|
|
7
|
+
- agent_misstep: Kane used the wrong control or page, misread the screen, ran out of steps, or stopped although a valid path was visible.
|
|
8
|
+
- check_problem: the check was wrong or ambiguous, named a control that the documentation sentence does not name and that does not exist, or asserted something the claim does not state.
|
|
9
|
+
If the documentation sentence itself names a control, menu, page or position, and Kane loaded the area the sentence points to and found that named thing absent there, that is claim_contradicted, not check_problem: the sentence is what is wrong. It is only claim_contradicted if Kane looked in the place the sentence names; a search somewhere else is agent_misstep.
|
|
10
|
+
- setup_problem: sign-in did not complete, test data or a fixture was missing, or the environment blocked the test.
|
|
11
|
+
- kane_limitation: Kane cannot observe or perform what the claim needs. Known limits: Kane cannot inspect new browser tabs or windows; Kane's text entry can bypass editors that convert Markdown or other syntax as you type, so literal syntax in a saved result can come from the input method; a word used as a stored variable name can drop out of a condition.
|
|
12
|
+
- claim_contradicted: the right page was loaded, the setup was valid, Kane performed the documented action in the documented place, and the documented result was observably absent.
|
|
13
|
+
- unclear: none of the above is supported by the evidence.
|
|
14
|
+
Observations such as "the login form is shown" or "the welcome wizard opens" are usually Kane checking a conditional instruction at the start of the run. They do not show the app's state later in the run. Judge sign-in and setup from the later steps.
|
|
15
|
+
Choose claim_contradicted only when the evidence shows every part of that definition. Otherwise choose another cause. A wrong accusation against an open-source project is far worse than a missed finding.
|
|
16
|
+
For a passing run, use cause "unclear".
|
|
17
|
+
3. Kane's blame label is a hypothesis, not a fact. Say whether you agree. If you disagree, cite the specific step or observation that contradicts it in blameEvidence.
|
|
18
|
+
4. Propose the next check:
|
|
19
|
+
- revisedCheck: for agent_misstep or check_problem, a corrected sequence of browser actions and explicit assertions that still tests exactly what the claim states, in the same place, with the same control type and each stated result. Never weaken a check so that it passes. For a passing run that did not exercise the claim, a check that does. Otherwise leave it empty.
|
|
20
|
+
- controlCheck: for claim_contradicted only, a nearby flow in the same app with the same login that should pass if the setup is valid and that does not depend on the disputed claim. Otherwise leave it empty.
|
|
21
|
+
- ownMistakeRuledOut: for claim_contradicted, why the failure is not caused by the test itself (sign-in, test data, Kane's input method, the wrong page). Otherwise leave it empty.
|
|
22
|
+
Text shown as [url], [path] or [secret] was removed before you saw it. Never copy these markers into a check: refer to pages by name. Never ask Kane to open, visit or sign in at any web address; the test always starts on the app. If a check needs a web address typed into a form field, use https://example.com and say it is typed into that field.
|
|
23
|
+
For a passing run, seeing that a button, form, menu or option exists does not exercise a claim that the app can do something. The documented action must have been performed and its result observed. A claim that lists several features is exercised only when each listed feature was tried.
|
|
24
|
+
5. For a passing run that exercised the claim, compare the documentation's names for controls, pages and settings with the visible text in Kane's steps and observations. If the behaviour holds but the visible text is different (for example the documentation says "Basic Attack" and the button shows "Attack"), put the visible text in visibleWording, for example: Button labelled "Attack" (documentation: "Basic Attack"). Ignore differences only in capital letters or punctuation. Otherwise leave it empty.
|
|
25
|
+
Write plain sentences. Do not include local addresses, file paths or credentials. Do not ask Kane to sign in again; the login instructions are added separately.`;
|
|
26
|
+
const RESPONSE_SCHEMA = {
|
|
27
|
+
type: "object",
|
|
28
|
+
properties: {
|
|
29
|
+
exercisedClaim: { type: "boolean" },
|
|
30
|
+
cause: { type: "string", enum: [...FAILURE_CAUSES] },
|
|
31
|
+
reason: { type: "string" },
|
|
32
|
+
ownMistakeRuledOut: { type: "string" },
|
|
33
|
+
kaneBlame: { type: "string", enum: ["agree", "disagree", "none"] },
|
|
34
|
+
blameEvidence: { type: "string" },
|
|
35
|
+
revisedCheck: { type: "string" },
|
|
36
|
+
controlCheck: { type: "string" },
|
|
37
|
+
visibleWording: { type: "string" },
|
|
38
|
+
},
|
|
39
|
+
required: ["exercisedClaim", "cause", "reason", "ownMistakeRuledOut", "kaneBlame", "blameEvidence", "revisedCheck", "controlCheck"],
|
|
40
|
+
};
|
|
41
|
+
const MAX_STEPS = 60;
|
|
42
|
+
const MAX_OBSERVATIONS = 30;
|
|
43
|
+
const MAX_TEXT = 600;
|
|
44
|
+
/** Builds a diagnoser that sends only public claim text and a sanitized Kane summary to the model. */
|
|
45
|
+
export function modelDiagnoser(client, secrets = []) {
|
|
46
|
+
return async (claim, check, kane) => {
|
|
47
|
+
const { value } = await client.generateJson({ purpose: "diagnose-run", system: DIAGNOSE_SYSTEM, prompt: diagnosisPrompt(claim, check, kane, secrets), jsonSchema: RESPONSE_SCHEMA }, parseDiagnosis);
|
|
48
|
+
return applyFixedRules(claim, kane, value);
|
|
49
|
+
};
|
|
50
|
+
}
|
|
51
|
+
const TAB = /\btabs?\b/i;
|
|
52
|
+
const MENU_ROUTE = /\bnav(?:igation|bar)?\b|\bmenus?\b|\bdrop-?downs?\b/i;
|
|
53
|
+
/**
|
|
54
|
+
* A fixed rule that the model's reading cannot overrule: when the documentation names a tab, a click that
|
|
55
|
+
* Kane itself describes as being in a navigation bar, menu or dropdown did not test that tab, even when
|
|
56
|
+
* Kane calls the item a tab. The run then does not count as exercising the claim.
|
|
57
|
+
*/
|
|
58
|
+
export function menuRoutedTab(claim, kane) {
|
|
59
|
+
if (!TAB.test(claim.quote))
|
|
60
|
+
return undefined;
|
|
61
|
+
const step = kane.steps.find(({ remark }) => /^click\b/i.test(remark) && TAB.test(remark) && MENU_ROUTE.test(remark));
|
|
62
|
+
return step ? `Step ${step.step} clicked an item Kane described as being in a navigation bar, menu or dropdown ("${step.remark.slice(0, 160)}"), not a tab inside the page the documentation names.` : undefined;
|
|
63
|
+
}
|
|
64
|
+
export function applyFixedRules(claim, kane, diagnosis) {
|
|
65
|
+
const routed = menuRoutedTab(claim, kane);
|
|
66
|
+
if (!routed || !diagnosis.exercisedClaim)
|
|
67
|
+
return diagnosis;
|
|
68
|
+
return {
|
|
69
|
+
...diagnosis,
|
|
70
|
+
exercisedClaim: false,
|
|
71
|
+
cause: diagnosis.cause === "claim_contradicted" ? "agent_misstep" : diagnosis.cause,
|
|
72
|
+
reason: `${routed} ${diagnosis.reason}`,
|
|
73
|
+
visibleWording: "",
|
|
74
|
+
};
|
|
75
|
+
}
|
|
76
|
+
/** The model's input: the public claim, the check and Kane's result without URLs, paths, session links or secrets. */
|
|
77
|
+
export function diagnosisPrompt(claim, check, kane, secrets = []) {
|
|
78
|
+
const clean = (text) => sanitize(text ?? "", secrets);
|
|
79
|
+
const summary = {
|
|
80
|
+
claim: { file: claim.source.file, line: claim.source.startLine, quote: clean(claim.quote) },
|
|
81
|
+
check: clean(check),
|
|
82
|
+
kane: {
|
|
83
|
+
status: kane.status,
|
|
84
|
+
reasonCode: kane.reasonCode,
|
|
85
|
+
oneLiner: clean(kane.oneLiner),
|
|
86
|
+
reason: clean(kane.reason),
|
|
87
|
+
failedStep: kane.failedStep ? clean(kane.failedStep.remark) : null,
|
|
88
|
+
steps: kane.steps.slice(-MAX_STEPS).map((step) => `${step.status}: ${clean(step.remark)}`),
|
|
89
|
+
observations: kane.observations.slice(-MAX_OBSERVATIONS).map((o) => ({
|
|
90
|
+
condition: clean(o.condition), expected: clean(o.expectedValue), observed: clean(o.extractedValue),
|
|
91
|
+
})),
|
|
92
|
+
blame: kane.blame ? {
|
|
93
|
+
family: kane.blame.family, category: kane.blame.category,
|
|
94
|
+
rootCause: clean(kane.blame.rootCause), agentFaultAssessment: clean(kane.blame.agentFaultAssessment),
|
|
95
|
+
} : null,
|
|
96
|
+
},
|
|
97
|
+
};
|
|
98
|
+
return `Review this Kane run.\n\n${JSON.stringify(summary, null, 2)}`;
|
|
99
|
+
}
|
|
100
|
+
/** Public web addresses (such as https://example.com in a check) are kept; local, private and Kane session addresses are not. */
|
|
101
|
+
function isPublicUrl(url) {
|
|
102
|
+
if (!URL.canParse(url))
|
|
103
|
+
return false;
|
|
104
|
+
const host = new URL(url).hostname;
|
|
105
|
+
return !/^(?:localhost|.+\.localhost|.+\.local|\d{1,3}(?:\.\d{1,3}){3}|\[.*\])$/i.test(host) && !/lambdatest|testmu/i.test(host);
|
|
106
|
+
}
|
|
107
|
+
function sanitize(text, secrets) {
|
|
108
|
+
let result = text;
|
|
109
|
+
for (const secret of secrets)
|
|
110
|
+
if (secret.length >= 4)
|
|
111
|
+
result = result.split(secret).join("[secret]");
|
|
112
|
+
result = result
|
|
113
|
+
.replace(/\b[a-z][a-z0-9+.-]*:\/\/[^\s"'<>]+/gi, (url) => (isPublicUrl(url) ? url : "[url]"))
|
|
114
|
+
.replace(/(?:\blocalhost|\b(?:\d{1,3}\.){3}\d{1,3}|\[[0-9a-f:]+\])(?::\d+)?(?:\/\S*)?/gi, "[url]")
|
|
115
|
+
.replace(/(?:~|\b[A-Za-z]:)?[\\/](?:Users|home|private|tmp|var|opt|root|mnt|Volumes)[\\/]\S+/g, "[path]")
|
|
116
|
+
.replace(/~\/\S+/g, "[path]");
|
|
117
|
+
return result.length > MAX_TEXT ? `${result.slice(0, MAX_TEXT)}…` : result;
|
|
118
|
+
}
|
|
119
|
+
export function parseDiagnosis(value) {
|
|
120
|
+
if (typeof value !== "object" || value === null)
|
|
121
|
+
throw new Error("The diagnosis was not an object.");
|
|
122
|
+
const record = value;
|
|
123
|
+
const text = (key) => {
|
|
124
|
+
const field = record[key];
|
|
125
|
+
if (field === undefined || field === null)
|
|
126
|
+
return "";
|
|
127
|
+
if (typeof field !== "string")
|
|
128
|
+
throw new Error(`The diagnosis field ${key} was not text.`);
|
|
129
|
+
return field.trim();
|
|
130
|
+
};
|
|
131
|
+
const cause = record.cause;
|
|
132
|
+
if (typeof cause !== "string" || !FAILURE_CAUSES.includes(cause))
|
|
133
|
+
throw new Error("The diagnosis named an unknown cause.");
|
|
134
|
+
const blame = record.kaneBlame;
|
|
135
|
+
if (blame !== "agree" && blame !== "disagree" && blame !== "none")
|
|
136
|
+
throw new Error("The diagnosis gave an unknown blame opinion.");
|
|
137
|
+
if (typeof record.exercisedClaim !== "boolean")
|
|
138
|
+
throw new Error("The diagnosis did not say whether the run exercised the claim.");
|
|
139
|
+
return {
|
|
140
|
+
exercisedClaim: record.exercisedClaim,
|
|
141
|
+
cause: cause,
|
|
142
|
+
reason: text("reason"),
|
|
143
|
+
ownMistakeRuledOut: text("ownMistakeRuledOut"),
|
|
144
|
+
kaneBlame: blame,
|
|
145
|
+
blameEvidence: text("blameEvidence"),
|
|
146
|
+
revisedCheck: text("revisedCheck"),
|
|
147
|
+
controlCheck: text("controlCheck"),
|
|
148
|
+
visibleWording: text("visibleWording"),
|
|
149
|
+
};
|
|
150
|
+
}
|
package/dist/findings.js
ADDED
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
import { existsSync, readFileSync, writeFileSync } from "node:fs";
|
|
2
|
+
import { dirname, join } from "node:path";
|
|
3
|
+
import { errorMessage } from "./util.js";
|
|
4
|
+
export function readFindingsFile(path) {
|
|
5
|
+
let value;
|
|
6
|
+
try {
|
|
7
|
+
value = JSON.parse(readFileSync(path, "utf8"));
|
|
8
|
+
}
|
|
9
|
+
catch (error) {
|
|
10
|
+
throw new Error(`Could not read the findings file ${path}: ${errorMessage(error)}`);
|
|
11
|
+
}
|
|
12
|
+
if (!Array.isArray(value))
|
|
13
|
+
throw new Error(`Could not read the findings file ${path}: expected a list of findings.`);
|
|
14
|
+
return value;
|
|
15
|
+
}
|
|
16
|
+
export function writeFindingsFile(path, findings) {
|
|
17
|
+
writeFileSync(path, `${JSON.stringify(findings, null, 2)}\n`);
|
|
18
|
+
}
|
|
19
|
+
/** Reads and checks the findings file; every referenced screenshot must exist. */
|
|
20
|
+
export function loadFindings(path) {
|
|
21
|
+
const findings = readFindingsFile(path);
|
|
22
|
+
for (const finding of findings) {
|
|
23
|
+
for (const file of evidenceFiles(finding)) {
|
|
24
|
+
if (!existsSync(join(dirname(path), file)))
|
|
25
|
+
throw new Error(`Finding ${finding.id} refers to a missing file: ${file}`);
|
|
26
|
+
}
|
|
27
|
+
if ((finding.verdict === "false" || finding.verdict === "outdated") && (finding.evidence.failures.length < 2 || !finding.evidence.control)) {
|
|
28
|
+
throw new Error(`Finding ${finding.id} is ${finding.verdict} without two recorded failures and a control.`);
|
|
29
|
+
}
|
|
30
|
+
}
|
|
31
|
+
return findings;
|
|
32
|
+
}
|
|
33
|
+
export function evidenceFiles(finding) {
|
|
34
|
+
const { failures, control, fixProof, held } = finding.evidence;
|
|
35
|
+
return [finding.screenshot, ...failures, control, fixProof, held].filter((file) => Boolean(file));
|
|
36
|
+
}
|