fibcheck 0.1.0 → 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +13 -0
- package/README.md +17 -22
- package/dist/app/docker.js +2 -1
- package/dist/attack-run.js +11 -12
- package/dist/attack.js +8 -4
- package/dist/card.js +96 -51
- package/dist/check.js +24 -47
- package/dist/claims/types.js +3 -0
- package/dist/cli.js +30 -16
- package/dist/confirm.js +16 -7
- package/dist/findings.js +36 -0
- package/dist/frozen.js +16 -3
- package/dist/general-loop.js +8 -8
- package/dist/github.js +4 -2
- package/dist/kane/objective.js +28 -0
- package/dist/kane/parse.js +8 -8
- package/dist/kane/runner.js +2 -1
- package/dist/loop.js +12 -7
- package/dist/model/budget.js +13 -4
- package/dist/model/client.js +2 -1
- package/dist/model/config.js +1 -0
- package/dist/model/gemini.js +3 -3
- package/dist/report/report.js +2 -1
- package/dist/target.js +18 -3
- package/dist/util.js +6 -0
- package/package.json +4 -3
package/dist/cli.js
CHANGED
|
@@ -5,14 +5,16 @@ import { parseArgs } from "node:util";
|
|
|
5
5
|
import { pathToFileURL } from "node:url";
|
|
6
6
|
import { attackFinding } from "./attack-run.js";
|
|
7
7
|
import { attackSummary } from "./attack.js";
|
|
8
|
-
import {
|
|
8
|
+
import { writeCard } from "./card.js";
|
|
9
9
|
import { runCheck } from "./check.js";
|
|
10
10
|
import { extractClaims } from "./claims/extract.js";
|
|
11
11
|
import { confirmFinding } from "./confirm.js";
|
|
12
|
+
import { loadFindings } from "./findings.js";
|
|
12
13
|
import { freeze, frozenClaims } from "./frozen.js";
|
|
13
14
|
import { createModelClient, loadDotEnv } from "./model/config.js";
|
|
14
15
|
import { writeReport } from "./report/report.js";
|
|
15
16
|
import { loadTarget, loginSecretsFromEnv, targetPathFor } from "./target.js";
|
|
17
|
+
import { errorMessage, newRunId } from "./util.js";
|
|
16
18
|
export const USAGE = `fibcheck: finds the false claims in a web app's README with Kane CLI.
|
|
17
19
|
|
|
18
20
|
Usage:
|
|
@@ -53,13 +55,13 @@ export async function main(argv) {
|
|
|
53
55
|
return 0;
|
|
54
56
|
}
|
|
55
57
|
if (command === "check")
|
|
56
|
-
return await
|
|
58
|
+
return await checkCommand(rest);
|
|
57
59
|
if (command === "report")
|
|
58
|
-
return
|
|
60
|
+
return reportCommand(rest);
|
|
59
61
|
if (command === "card")
|
|
60
|
-
return
|
|
62
|
+
return cardCommand(rest);
|
|
61
63
|
if (command === "attack")
|
|
62
|
-
return await
|
|
64
|
+
return await attackCommand(rest);
|
|
63
65
|
if (command === "freeze")
|
|
64
66
|
return await freezeCommand(rest);
|
|
65
67
|
if (command === "confirm")
|
|
@@ -98,7 +100,7 @@ async function freezeCommand(args) {
|
|
|
98
100
|
return 1;
|
|
99
101
|
}
|
|
100
102
|
loadDotEnv();
|
|
101
|
-
const runId =
|
|
103
|
+
const runId = newRunId();
|
|
102
104
|
const extraction = await extractClaims(createModelClient(runId), [{ file: docsFile, content: readFileSync(docsFile, "utf8") }]);
|
|
103
105
|
const file = freeze(docsFile, extraction.claims);
|
|
104
106
|
writeFileSync(values.out, `${JSON.stringify(file, null, 2)}\n`, { flag: "wx" });
|
|
@@ -108,7 +110,7 @@ async function freezeCommand(args) {
|
|
|
108
110
|
process.stdout.write(` Line ${line}: ${why}\n`);
|
|
109
111
|
return 0;
|
|
110
112
|
}
|
|
111
|
-
async function
|
|
113
|
+
async function attackCommand(args) {
|
|
112
114
|
const { values, positionals } = parseArgs({
|
|
113
115
|
args, allowPositionals: true,
|
|
114
116
|
options: { url: { type: "string" }, login: { type: "string" }, findings: { type: "string" } },
|
|
@@ -118,8 +120,13 @@ async function attack(args) {
|
|
|
118
120
|
process.stderr.write("Usage: fibcheck attack <finding-id> --url ADDRESS [--login SENTENCE] [--findings index/findings.json]\n");
|
|
119
121
|
return 1;
|
|
120
122
|
}
|
|
123
|
+
const loginError = loginArgError(values.login);
|
|
124
|
+
if (loginError) {
|
|
125
|
+
process.stderr.write(loginError);
|
|
126
|
+
return 1;
|
|
127
|
+
}
|
|
121
128
|
loadDotEnv();
|
|
122
|
-
const runId =
|
|
129
|
+
const runId = newRunId();
|
|
123
130
|
const record = await attackFinding(createModelClient(runId), {
|
|
124
131
|
findingsPath: values.findings ?? "index/findings.json", id, appUrl: values.url, login: values.login,
|
|
125
132
|
secrets: loginSecretsFromEnv(values.login), runsDir: join("runs", `attack-${runId}`),
|
|
@@ -130,7 +137,7 @@ async function attack(args) {
|
|
|
130
137
|
process.stdout.write(`${attackSummary(record.results) || "No lazy fix was rejected."}\n`);
|
|
131
138
|
return 0;
|
|
132
139
|
}
|
|
133
|
-
function
|
|
140
|
+
function cardCommand(args) {
|
|
134
141
|
const { values, positionals } = parseArgs({
|
|
135
142
|
args, allowPositionals: true,
|
|
136
143
|
options: { all: { type: "boolean" }, findings: { type: "string" }, out: { type: "string" } },
|
|
@@ -145,7 +152,7 @@ function card(args) {
|
|
|
145
152
|
process.stdout.write(`Saved ${writeCard(findingsPath, id, values.out ?? "index/cards")}\n`);
|
|
146
153
|
return 0;
|
|
147
154
|
}
|
|
148
|
-
function
|
|
155
|
+
function reportCommand(args) {
|
|
149
156
|
const [inputPath] = args;
|
|
150
157
|
if (args.length !== 1 || !inputPath || inputPath.startsWith("-")) {
|
|
151
158
|
process.stderr.write("Usage: fibcheck report <check.json>\n");
|
|
@@ -180,13 +187,19 @@ export function parseCheckArgs(args) {
|
|
|
180
187
|
return "--max-kane-runs must be a whole number.\n";
|
|
181
188
|
if (values.login !== undefined && values.url === undefined)
|
|
182
189
|
return "--login needs --url.\n";
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
190
|
+
const loginError = loginArgError(values.login);
|
|
191
|
+
if (loginError)
|
|
192
|
+
return loginError;
|
|
186
193
|
if (values.url !== undefined && values["app-running"])
|
|
187
194
|
return "Use either --url or --app-running, not both.\n";
|
|
188
195
|
return { target, maxClaims, maxKaneRuns, only: values.only, appRunning: values["app-running"], url: values.url, login: values.login };
|
|
189
196
|
}
|
|
197
|
+
function loginArgError(login) {
|
|
198
|
+
if (login !== undefined && !/\{\{[A-Za-z_][A-Za-z0-9_]*\}\}/.test(login)) {
|
|
199
|
+
return "--login must name its values as {{NAME}} and keep them in .env as FIBCHECK_LOGIN_NAME, never type a password into the command.\n";
|
|
200
|
+
}
|
|
201
|
+
return undefined;
|
|
202
|
+
}
|
|
190
203
|
/** undefined when absent, null when not a non-negative whole number. */
|
|
191
204
|
function wholeNumber(raw, isWhole) {
|
|
192
205
|
if (raw === undefined)
|
|
@@ -194,7 +207,7 @@ function wholeNumber(raw, isWhole) {
|
|
|
194
207
|
const value = Number(raw);
|
|
195
208
|
return isWhole(value) && value >= 0 ? value : null;
|
|
196
209
|
}
|
|
197
|
-
async function
|
|
210
|
+
async function checkCommand(args) {
|
|
198
211
|
const parsed = parseCheckArgs(args);
|
|
199
212
|
if (typeof parsed === "string") {
|
|
200
213
|
process.stderr.write(parsed);
|
|
@@ -204,9 +217,10 @@ async function check(args) {
|
|
|
204
217
|
loadDotEnv();
|
|
205
218
|
const alreadyRunning = parsed.url !== undefined || parsed.appRunning === true;
|
|
206
219
|
const login = parsed.login ?? loadTarget(targetPath).login;
|
|
207
|
-
const runId =
|
|
220
|
+
const runId = newRunId();
|
|
208
221
|
const report = await runCheck(createModelClient(runId), {
|
|
209
222
|
targetPath,
|
|
223
|
+
runId,
|
|
210
224
|
maxClaims: parsed.maxClaims,
|
|
211
225
|
maxKaneRuns: parsed.maxKaneRuns,
|
|
212
226
|
only: parsed.only,
|
|
@@ -229,7 +243,7 @@ function summaryLine(report) {
|
|
|
229
243
|
const executable = process.argv[1];
|
|
230
244
|
if (executable && existsSync(executable) && import.meta.url === pathToFileURL(realpathSync(executable)).href) {
|
|
231
245
|
main(process.argv.slice(2)).then((code) => (process.exitCode = code), (error) => {
|
|
232
|
-
process.stderr.write(`fibcheck: ${
|
|
246
|
+
process.stderr.write(`fibcheck: ${errorMessage(error)}\n`);
|
|
233
247
|
process.exitCode = 1;
|
|
234
248
|
});
|
|
235
249
|
}
|
package/dist/confirm.js
CHANGED
|
@@ -1,5 +1,7 @@
|
|
|
1
|
-
import { copyFileSync, existsSync, mkdirSync, readFileSync
|
|
1
|
+
import { copyFileSync, existsSync, mkdirSync, readFileSync } from "node:fs";
|
|
2
2
|
import { basename, dirname, extname, join } from "node:path";
|
|
3
|
+
import { readFindingsFile, writeFindingsFile } from "./findings.js";
|
|
4
|
+
import { errorMessage } from "./util.js";
|
|
3
5
|
/**
|
|
4
6
|
* Turns a candidate finding from a saved check into an entry in a findings file, after a person has read the
|
|
5
7
|
* evidence. It copies Kane's screenshots of the two failing runs and the passing control, unchanged, and
|
|
@@ -10,7 +12,7 @@ export function confirmFinding(options) {
|
|
|
10
12
|
throw new Error("A confirmed finding is false or outdated.");
|
|
11
13
|
const report = readJson(options.checkPath, "check");
|
|
12
14
|
const { tested, failures, control } = backedCandidate(report, options);
|
|
13
|
-
const findings = existsSync(options.findingsPath) ?
|
|
15
|
+
const findings = existsSync(options.findingsPath) ? readFindingsFile(options.findingsPath) : [];
|
|
14
16
|
const { claim } = tested;
|
|
15
17
|
const project = options.project ?? report.target;
|
|
16
18
|
const id = options.id ?? `${slug(project)}-${slug(basename(claim.source.file, extname(claim.source.file)))}-l${claim.source.startLine}`;
|
|
@@ -33,7 +35,7 @@ export function confirmFinding(options) {
|
|
|
33
35
|
score: { findings: 1, claims: report.tested.length },
|
|
34
36
|
...(options.fix ? { fix: options.fix } : {}),
|
|
35
37
|
};
|
|
36
|
-
|
|
38
|
+
writeFindingsFile(options.findingsPath, [...findings, finding]);
|
|
37
39
|
return finding;
|
|
38
40
|
}
|
|
39
41
|
/** The tested claim, if it is a candidate finding backed by two failed runs and a passing control. */
|
|
@@ -56,15 +58,22 @@ function readJson(path, what) {
|
|
|
56
58
|
return JSON.parse(readFileSync(path, "utf8"));
|
|
57
59
|
}
|
|
58
60
|
catch (error) {
|
|
59
|
-
throw new Error(`Could not read the ${what} file ${path}: ${
|
|
61
|
+
throw new Error(`Could not read the ${what} file ${path}: ${errorMessage(error)}`);
|
|
60
62
|
}
|
|
61
63
|
}
|
|
62
|
-
/**
|
|
64
|
+
/**
|
|
65
|
+
* The two failed runs behind the candidate and the passing control. The repeat ("retry") reproduces the exact check
|
|
66
|
+
* of the run just before it, which is the run read as a contradiction: the first run, or a reworded run when the
|
|
67
|
+
* first one failed for another reason. Both must have failed.
|
|
68
|
+
*/
|
|
63
69
|
function backingRuns({ kane, questioning }) {
|
|
64
70
|
const followUps = questioning?.followUps ?? [];
|
|
65
|
-
const
|
|
66
|
-
.filter((run) => run.status === "failed");
|
|
71
|
+
const retryAt = followUps.findIndex(({ purpose }) => purpose === "retry");
|
|
67
72
|
const control = followUps.find(({ purpose, kane: run }) => purpose === "control" && run.status === "passed")?.kane;
|
|
73
|
+
if (retryAt === -1)
|
|
74
|
+
return { failures: [], control };
|
|
75
|
+
const contradicted = retryAt === 0 ? kane : followUps[retryAt - 1].kane;
|
|
76
|
+
const failures = [contradicted, followUps[retryAt].kane].filter((run) => run.status === "failed");
|
|
68
77
|
return { failures, control };
|
|
69
78
|
}
|
|
70
79
|
/** Kane's screenshot for a run, from the claims/ folder saved beside the check: the failed step, or the last step. */
|
package/dist/findings.js
ADDED
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
import { existsSync, readFileSync, writeFileSync } from "node:fs";
|
|
2
|
+
import { dirname, join } from "node:path";
|
|
3
|
+
import { errorMessage } from "./util.js";
|
|
4
|
+
export function readFindingsFile(path) {
|
|
5
|
+
let value;
|
|
6
|
+
try {
|
|
7
|
+
value = JSON.parse(readFileSync(path, "utf8"));
|
|
8
|
+
}
|
|
9
|
+
catch (error) {
|
|
10
|
+
throw new Error(`Could not read the findings file ${path}: ${errorMessage(error)}`);
|
|
11
|
+
}
|
|
12
|
+
if (!Array.isArray(value))
|
|
13
|
+
throw new Error(`Could not read the findings file ${path}: expected a list of findings.`);
|
|
14
|
+
return value;
|
|
15
|
+
}
|
|
16
|
+
export function writeFindingsFile(path, findings) {
|
|
17
|
+
writeFileSync(path, `${JSON.stringify(findings, null, 2)}\n`);
|
|
18
|
+
}
|
|
19
|
+
/** Reads and checks the findings file; every referenced screenshot must exist. */
|
|
20
|
+
export function loadFindings(path) {
|
|
21
|
+
const findings = readFindingsFile(path);
|
|
22
|
+
for (const finding of findings) {
|
|
23
|
+
for (const file of evidenceFiles(finding)) {
|
|
24
|
+
if (!existsSync(join(dirname(path), file)))
|
|
25
|
+
throw new Error(`Finding ${finding.id} refers to a missing file: ${file}`);
|
|
26
|
+
}
|
|
27
|
+
if ((finding.verdict === "false" || finding.verdict === "outdated") && (finding.evidence.failures.length < 2 || !finding.evidence.control)) {
|
|
28
|
+
throw new Error(`Finding ${finding.id} is ${finding.verdict} without two recorded failures and a control.`);
|
|
29
|
+
}
|
|
30
|
+
}
|
|
31
|
+
return findings;
|
|
32
|
+
}
|
|
33
|
+
export function evidenceFiles(finding) {
|
|
34
|
+
const { failures, control, fixProof, held } = finding.evidence;
|
|
35
|
+
return [finding.screenshot, ...failures, control, fixProof, held].filter((file) => Boolean(file));
|
|
36
|
+
}
|
package/dist/frozen.js
CHANGED
|
@@ -1,7 +1,16 @@
|
|
|
1
1
|
import { createHash } from "node:crypto";
|
|
2
2
|
import { CLAIM_TYPES } from "./claims/types.js";
|
|
3
|
+
/**
|
|
4
|
+
* sha256 of everything a reviewer approved: the quote, the check, the control and the assertion mode. A claim without an
|
|
5
|
+
* assertion mode hashes exactly as before the mode was covered, so older files without one still load.
|
|
6
|
+
*/
|
|
3
7
|
export function contentHash(claim) {
|
|
4
|
-
|
|
8
|
+
const fields = [claim.quote, claim.check, claim.control ?? "", ...(claim.assertionMode ? [claim.assertionMode] : [])];
|
|
9
|
+
return createHash("sha256").update(JSON.stringify(fields)).digest("hex");
|
|
10
|
+
}
|
|
11
|
+
/** True when a claim with an assertion mode carries a hash from before the mode was covered. */
|
|
12
|
+
function hashedWithoutMode(claim) {
|
|
13
|
+
return Boolean(claim.assertionMode) && contentHash({ ...claim, assertionMode: undefined }) === claim.contentSha256;
|
|
5
14
|
}
|
|
6
15
|
export function parseFrozenFile(text) {
|
|
7
16
|
let value;
|
|
@@ -39,6 +48,9 @@ export function frozenClaims(file) {
|
|
|
39
48
|
const flag = (why) => needsReview.push({ id: frozen.id, line: frozen.startLine, why });
|
|
40
49
|
if (!frozen.reviewed?.by?.trim() || !frozen.reviewed?.at?.trim())
|
|
41
50
|
flag("No one has reviewed this check yet.");
|
|
51
|
+
else if (hashedWithoutMode(frozen)) {
|
|
52
|
+
flag("This check was frozen before the assertion mode was part of its hash. Re-freeze it with fibcheck freeze and review it again.");
|
|
53
|
+
}
|
|
42
54
|
else if (contentHash(frozen) !== frozen.contentSha256)
|
|
43
55
|
flag("The check changed after review. Freeze and review it again.");
|
|
44
56
|
else {
|
|
@@ -95,16 +107,17 @@ function labelProblem(check, firstAssertion, { label, element }) {
|
|
|
95
107
|
function escape(text) {
|
|
96
108
|
return text.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
97
109
|
}
|
|
98
|
-
/**
|
|
110
|
+
/** Builds a frozen file from extracted claims, for a person to review. Reviewer fields are left empty. */
|
|
99
111
|
export function freeze(docsFile, claims) {
|
|
100
112
|
return {
|
|
101
113
|
version: 1,
|
|
102
114
|
docsFile,
|
|
103
115
|
claims: claims.filter((claim) => claim.type !== "untestable" && claim.check).map((claim) => {
|
|
104
|
-
const entry = { quote: claim.quote, check: claim.check, control: claim.control };
|
|
116
|
+
const entry = { quote: claim.quote, check: claim.check, control: claim.control, assertionMode: claim.assertionMode };
|
|
105
117
|
return {
|
|
106
118
|
id: claim.id, type: claim.type, quote: claim.quote, startLine: claim.source.startLine, endLine: claim.source.endLine,
|
|
107
119
|
check: claim.check, ...(claim.control ? { control: claim.control } : {}),
|
|
120
|
+
...(claim.assertionMode ? { assertionMode: claim.assertionMode } : {}),
|
|
108
121
|
contentSha256: contentHash(entry), reviewed: { by: "", at: "" },
|
|
109
122
|
};
|
|
110
123
|
}),
|
package/dist/general-loop.js
CHANGED
|
@@ -29,7 +29,7 @@ export async function questionGenerally(claim, check, general, run, canRunFollow
|
|
|
29
29
|
if (attempt.kane.status === "error" || attempt.kane.status === "cancelled") {
|
|
30
30
|
return outcome(state, "unconfirmed", `Kane did not finish the run (${attempt.kane.status}).`);
|
|
31
31
|
}
|
|
32
|
-
const diagnosis = await
|
|
32
|
+
const diagnosis = await diagnoseAttempt(state, attempt);
|
|
33
33
|
if (!diagnosis)
|
|
34
34
|
return outcome(state, "unconfirmed", "The model could not read Kane's result, so it was not interpreted.");
|
|
35
35
|
const next = await decide(state, attempt, diagnosis);
|
|
@@ -50,7 +50,7 @@ async function decide(state, attempt, diagnosis) {
|
|
|
50
50
|
?? outcome(state, "unconfirmed", `Kane passed, but the run did not exercise the documented claim. ${diagnosis.reason}`);
|
|
51
51
|
}
|
|
52
52
|
if (diagnosis.cause === "claim_contradicted" && diagnosis.exercisedClaim)
|
|
53
|
-
return await
|
|
53
|
+
return await reproduceAndControl(state, attempt, diagnosis);
|
|
54
54
|
if (diagnosis.cause === "agent_misstep" || diagnosis.cause === "check_problem") {
|
|
55
55
|
const next = await reword(state, diagnosis);
|
|
56
56
|
if (next)
|
|
@@ -62,11 +62,11 @@ async function reword(state, diagnosis) {
|
|
|
62
62
|
if (state.rewordsLeft <= 0 || !diagnosis.revisedCheck || !state.canRunFollowUp())
|
|
63
63
|
return undefined;
|
|
64
64
|
state.rewordsLeft--;
|
|
65
|
-
const kane = await state.run("reword", state.general.objectiveFor(diagnosis.revisedCheck));
|
|
65
|
+
const kane = await state.run("reword", state.general.objectiveFor(diagnosis.revisedCheck), state.claim.assertionMode);
|
|
66
66
|
state.followUps.push({ purpose: "reword", reason: `Reworded after the model's reading: ${diagnosis.reason}`, kane });
|
|
67
67
|
return { round: "reword", check: diagnosis.revisedCheck, kane };
|
|
68
68
|
}
|
|
69
|
-
async function
|
|
69
|
+
async function reproduceAndControl(state, failed, diagnosis) {
|
|
70
70
|
if (!diagnosis.ownMistakeRuledOut)
|
|
71
71
|
return outcome(state, "unconfirmed", "The model gave no reason to rule out a test error.");
|
|
72
72
|
if (!state.canRunFollowUp())
|
|
@@ -74,14 +74,14 @@ async function confirm(state, failed, diagnosis) {
|
|
|
74
74
|
// Repeat the exact check that failed, so the reproduction tests the same assertion.
|
|
75
75
|
const check = failed.check;
|
|
76
76
|
const kane = await state.run("retry", state.general.objectiveFor(check), state.claim.assertionMode);
|
|
77
|
-
return await
|
|
77
|
+
return await checkReproductionAndControl(state, failed, diagnosis, { round: "retry", check, kane });
|
|
78
78
|
}
|
|
79
|
-
async function
|
|
79
|
+
async function checkReproductionAndControl(state, failed, diagnosis, reproduction) {
|
|
80
80
|
state.followUps.push({ purpose: "retry", reason: "Reproduce the same documented assertion in an independent Kane run.", kane: reproduction.kane });
|
|
81
81
|
if (reproduction.kane.status !== "failed" || !distinctKaneRuns(reproduction.kane, failed.kane)) {
|
|
82
82
|
return outcome(state, "unconfirmed", "Kane did not fail the documented assertion again in a separate run.");
|
|
83
83
|
}
|
|
84
|
-
const second = await
|
|
84
|
+
const second = await diagnoseAttempt(state, reproduction);
|
|
85
85
|
if (!second || second.cause !== "claim_contradicted" || !second.exercisedClaim || !second.ownMistakeRuledOut) {
|
|
86
86
|
return outcome(state, "unconfirmed", "The reproduction failed for a different reason, so the result is not consistent.");
|
|
87
87
|
}
|
|
@@ -98,7 +98,7 @@ async function confirmAfterReproduction(state, failed, diagnosis, reproduction)
|
|
|
98
98
|
state.candidateFinding = true;
|
|
99
99
|
return outcome(state, "unconfirmed", `Candidate finding, awaiting a person's review. Two separate Kane runs failed the documented result. ${diagnosis.reason} Test error ruled out: ${diagnosis.ownMistakeRuledOut} The control passed. ${NOT_OUTDATED}`);
|
|
100
100
|
}
|
|
101
|
-
async function
|
|
101
|
+
async function diagnoseAttempt(state, attempt) {
|
|
102
102
|
let diagnosis;
|
|
103
103
|
try {
|
|
104
104
|
diagnosis = await state.general.diagnose(state.claim, attempt.check, attempt.kane);
|
package/dist/github.js
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { cpSync, existsSync, mkdirSync, writeFileSync } from "node:fs";
|
|
2
2
|
import { join } from "node:path";
|
|
3
|
+
import { NEEDS_REVIEW_PREFIX } from "./check.js";
|
|
3
4
|
import { countResults } from "./report/report.js";
|
|
4
5
|
import { frozenClaims, parseFrozenFile } from "./frozen.js";
|
|
5
6
|
import { loadTarget } from "./target.js";
|
|
@@ -93,11 +94,12 @@ const VERDICT_LABELS = {
|
|
|
93
94
|
false: "Did not match the app",
|
|
94
95
|
outdated: "Did not match the app",
|
|
95
96
|
};
|
|
97
|
+
const upperFirst = (text) => `${text.charAt(0).toUpperCase()}${text.slice(1)}`;
|
|
96
98
|
/** Frozen checks that were not run because the line changed or the check was not reviewed. */
|
|
97
99
|
function needsReviewLines(report, needsReview) {
|
|
98
100
|
const lines = [
|
|
99
101
|
...needsReview.map(({ line, why }) => `- Line ${line}: ${why}`),
|
|
100
|
-
...report.rejected.filter(({ why }) => why.startsWith(
|
|
102
|
+
...report.rejected.filter(({ why }) => why.startsWith(NEEDS_REVIEW_PREFIX)).map(({ why }) => `- ${upperFirst(why.slice(NEEDS_REVIEW_PREFIX.length).trimStart())}`),
|
|
101
103
|
];
|
|
102
104
|
return lines.length ? ["", "**Needs review (not run)**", "", ...lines] : [];
|
|
103
105
|
}
|
|
@@ -107,7 +109,7 @@ function lineTable(report) {
|
|
|
107
109
|
const rows = [
|
|
108
110
|
...report.tested.map(({ claim, kane, questioning }) => {
|
|
109
111
|
const runs = [kane.status, ...(questioning?.followUps ?? []).map(({ purpose, kane: run }) => `${run.status} (${purpose})`)].join(", ");
|
|
110
|
-
return `| ${claim.source.startLine} | ${questioning ? VERDICT_LABELS[questioning.verdict]
|
|
112
|
+
return `| ${claim.source.startLine} | ${questioning ? VERDICT_LABELS[questioning.verdict] : "No verdict"} | ${runs} |`;
|
|
111
113
|
}),
|
|
112
114
|
...report.notTested.map((id) => `| ${line(id)} | Not run (over the claim limit) | none |`),
|
|
113
115
|
];
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
|
|
2
|
+
import { tmpdir } from "node:os";
|
|
3
|
+
import { join } from "node:path";
|
|
4
|
+
export function escapeKaneBraces(text) {
|
|
5
|
+
return text.replace(/[{}]{2,}/g, (braces) => braces.split("").join(" "));
|
|
6
|
+
}
|
|
7
|
+
export function writeKaneVariables(variables) {
|
|
8
|
+
if (Object.keys(variables).length === 0)
|
|
9
|
+
return undefined;
|
|
10
|
+
const directory = mkdtempSync(join(tmpdir(), "fibcheck-kane-"));
|
|
11
|
+
const path = join(directory, "variables.json");
|
|
12
|
+
const remove = () => rmSync(directory, { recursive: true, force: true });
|
|
13
|
+
try {
|
|
14
|
+
writeFileSync(path, JSON.stringify(variables), { flag: "wx", mode: 0o600 });
|
|
15
|
+
}
|
|
16
|
+
catch (error) {
|
|
17
|
+
remove();
|
|
18
|
+
throw error;
|
|
19
|
+
}
|
|
20
|
+
return { path, remove };
|
|
21
|
+
}
|
|
22
|
+
/**
|
|
23
|
+
* Documentation often names its own default address (for example http://localhost:5230). The test app runs
|
|
24
|
+
* elsewhere and Kane already starts on it, so a local address in a check is replaced by a reference to the app.
|
|
25
|
+
*/
|
|
26
|
+
export function localAddressesToApp(check) {
|
|
27
|
+
return check.replace(/\bhttps?:\/\/(?:localhost|127\.0\.0\.1|0\.0\.0\.0)(?::\d+)?(\/[^\s"'<>]*)?/gi, (_match, path) => (path && path !== "/" ? `the app's ${path.replace(/[.,;:]+$/, "")} page` : "the app's start page"));
|
|
28
|
+
}
|
package/dist/kane/parse.js
CHANGED
|
@@ -76,14 +76,14 @@ function observationsFrom(end) {
|
|
|
76
76
|
const memory = end?.context?.memory;
|
|
77
77
|
if (!memory || typeof memory !== "object")
|
|
78
78
|
return [];
|
|
79
|
-
return Object.entries(memory).
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
79
|
+
return Object.entries(memory).flatMap(([name, m]) => (m && typeof m === "object" ? [{
|
|
80
|
+
name,
|
|
81
|
+
condition: str(m.condition),
|
|
82
|
+
extractedValue: str(m.extracted_value),
|
|
83
|
+
expectedValue: str(m.expected_value),
|
|
84
|
+
analyzerType: str(m.analyzer_type),
|
|
85
|
+
usedDomCode: Boolean(m.code_js || m.wrapped_js || (Array.isArray(m.locators) ? m.locators.length : m.locators)),
|
|
86
|
+
}] : []));
|
|
87
87
|
}
|
|
88
88
|
function blameFrom(end) {
|
|
89
89
|
const v = end?.verdict;
|
package/dist/kane/runner.js
CHANGED
|
@@ -3,6 +3,7 @@ import { mkdirSync, writeFileSync } from "node:fs";
|
|
|
3
3
|
import { join } from "node:path";
|
|
4
4
|
import { extractStepEvidence, findEvidencePack } from "./evidence.js";
|
|
5
5
|
import { parseKaneOutput } from "./parse.js";
|
|
6
|
+
import { errorMessage } from "../util.js";
|
|
6
7
|
/** Runs one Kane flow in agent mode and returns its structured result. Only Kane decides pass or fail. */
|
|
7
8
|
export async function runKane(options) {
|
|
8
9
|
mkdirSync(options.outDir, { recursive: true });
|
|
@@ -42,7 +43,7 @@ function collectEvidence(result, outDir) {
|
|
|
42
43
|
return { evidence: extractStepEvidence(pack, join(outDir, "evidence")), evidenceError: null };
|
|
43
44
|
}
|
|
44
45
|
catch (error) {
|
|
45
|
-
return { evidence: [], evidenceError: `Could not unpack Kane's evidence: ${
|
|
46
|
+
return { evidence: [], evidenceError: `Could not unpack Kane's evidence: ${errorMessage(error)}` };
|
|
46
47
|
}
|
|
47
48
|
}
|
|
48
49
|
function run(command, args) {
|
package/dist/loop.js
CHANGED
|
@@ -20,15 +20,20 @@ const RETRY_TAB = `Go directly to ${SETTINGS_PATH} after login. First assert exa
|
|
|
20
20
|
const CONTROL = `From the signed-in bookmarks page, open the Settings dropdown in the top navigation. Assert exactly: ${MENU_ASSERTION} Expect true. Click Admin inside that dropdown. Make these as separate assertions: ${TITLE_ASSERTION} ${ACCOUNT_ASSERTION} ${USERS_ASSERTION} Expect true for each.`;
|
|
21
21
|
const CONTROL_PROOF = "The same test login opened Site administration through the top navigation’s Settings dropdown in the passing control run.";
|
|
22
22
|
const TAB_HISTORY = "Linkding’s historical source had an Admin tab in the Settings template at 3ae9cf0. The template was removed at 226eb69.";
|
|
23
|
-
|
|
23
|
+
/** The bundled Linkding target exactly as shipped, which the built-in Linkding checks were written for. */
|
|
24
|
+
function isBundledLinkding(target) {
|
|
25
|
+
return isLinkdingRepo(target.repo)
|
|
26
|
+
&& target.app?.image === "sissbruecker/linkding:1.47.0"
|
|
27
|
+
&& target.app?.env?.LD_SUPERUSER_NAME === "pinotest"
|
|
28
|
+
&& target.app?.env?.LD_SUPERUSER_PASSWORD === "{{secret:app_pass}}"
|
|
29
|
+
&& Boolean(target.login?.includes("pinotest and password {{app_pass}}"));
|
|
30
|
+
}
|
|
31
|
+
export async function questionClaim(claim, run, options) {
|
|
32
|
+
const { managedTarget, originalObjective, prepareTags, canRunFollowUp = () => true, general } = options;
|
|
24
33
|
// A frozen, reviewed check is run exactly as reviewed, never through a built-in special case.
|
|
25
34
|
if (general?.frozen && claim.check)
|
|
26
35
|
return await questionGenerally(claim, claim.check, general, run, canRunFollowUp);
|
|
27
|
-
const linkding = managedTarget &&
|
|
28
|
-
&& managedTarget.app?.image === "sissbruecker/linkding:1.47.0"
|
|
29
|
-
&& managedTarget.app?.env?.LD_SUPERUSER_NAME === "pinotest"
|
|
30
|
-
&& managedTarget.app?.env?.LD_SUPERUSER_PASSWORD === "{{secret:app_pass}}"
|
|
31
|
-
&& managedTarget.login?.includes("pinotest and password {{app_pass}}");
|
|
36
|
+
const linkding = managedTarget && isBundledLinkding(managedTarget);
|
|
32
37
|
if (linkding && managedTarget.login && prepareTags) {
|
|
33
38
|
const tags = linkdingTagClaim(claim);
|
|
34
39
|
if (tags) {
|
|
@@ -39,7 +44,7 @@ export async function questionClaim(claim, managedTarget, originalObjective, run
|
|
|
39
44
|
if (!linkding || !isAdminTabClaim(claim)) {
|
|
40
45
|
if (general && claim.check)
|
|
41
46
|
return await questionGenerally(claim, claim.check, general, run, canRunFollowUp);
|
|
42
|
-
const kane = await run("initial", originalObjective);
|
|
47
|
+
const kane = await run("initial", originalObjective, claim.assertionMode);
|
|
43
48
|
return {
|
|
44
49
|
kane,
|
|
45
50
|
questioning: {
|
package/dist/model/budget.js
CHANGED
|
@@ -56,7 +56,6 @@ export class CallBudget {
|
|
|
56
56
|
ensureDir(this.ledgerPath);
|
|
57
57
|
appendFileSync(this.ledgerPath, `${JSON.stringify(line)}\n`);
|
|
58
58
|
}
|
|
59
|
-
/** Calls and tokens used so far in this run. */
|
|
60
59
|
/** Calls and tokens for this run, and every model that answered. */
|
|
61
60
|
runSummary() {
|
|
62
61
|
return { calls: this.runCalls, ...this.runUsage, models: [...this.runModels].sort((a, b) => a.localeCompare(b)) };
|
|
@@ -64,19 +63,29 @@ export class CallBudget {
|
|
|
64
63
|
readTotals() {
|
|
65
64
|
if (!existsSync(this.totalsPath))
|
|
66
65
|
return { calls: 0, inputTokens: 0, outputTokens: 0 };
|
|
66
|
+
let totals;
|
|
67
67
|
try {
|
|
68
|
-
|
|
68
|
+
totals = JSON.parse(readFileSync(this.totalsPath, "utf8"));
|
|
69
69
|
}
|
|
70
70
|
catch {
|
|
71
|
-
|
|
72
|
-
throw new BudgetExceededError(`The usage totals file ${this.totalsPath} is damaged. Fix or delete it on purpose before making more model calls.`);
|
|
71
|
+
throw this.damaged();
|
|
73
72
|
}
|
|
73
|
+
if (!isCount(totals?.calls) || !isCount(totals?.inputTokens) || !isCount(totals?.outputTokens))
|
|
74
|
+
throw this.damaged();
|
|
75
|
+
return { calls: totals.calls, inputTokens: totals.inputTokens, outputTokens: totals.outputTokens };
|
|
76
|
+
}
|
|
77
|
+
/** Refuse rather than reset: resetting would silently lift the total limit. */
|
|
78
|
+
damaged() {
|
|
79
|
+
return new BudgetExceededError(`The usage totals file ${this.totalsPath} is damaged. Fix or delete it on purpose before making more model calls.`);
|
|
74
80
|
}
|
|
75
81
|
writeTotals(totals) {
|
|
76
82
|
ensureDir(this.totalsPath);
|
|
77
83
|
writeFileSync(this.totalsPath, `${JSON.stringify(totals, null, 2)}\n`);
|
|
78
84
|
}
|
|
79
85
|
}
|
|
86
|
+
function isCount(value) {
|
|
87
|
+
return typeof value === "number" && Number.isFinite(value) && value >= 0;
|
|
88
|
+
}
|
|
80
89
|
function ensureDir(filePath) {
|
|
81
90
|
mkdirSync(dirname(filePath), { recursive: true });
|
|
82
91
|
}
|
package/dist/model/client.js
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { errorMessage } from "../util.js";
|
|
1
2
|
import { UnusableModelOutputError } from "./types.js";
|
|
2
3
|
/**
|
|
3
4
|
* The one entry point for model calls. The provider must announce every HTTP
|
|
@@ -44,7 +45,7 @@ export class ModelClient {
|
|
|
44
45
|
return { value: validate(parsed), response };
|
|
45
46
|
}
|
|
46
47
|
catch (error) {
|
|
47
|
-
throw new UnusableModelOutputError(`The model's reply for "${request.purpose}" had the wrong shape: ${
|
|
48
|
+
throw new UnusableModelOutputError(`The model's reply for "${request.purpose}" had the wrong shape: ${errorMessage(error)}`);
|
|
48
49
|
}
|
|
49
50
|
}
|
|
50
51
|
/** Calls, tokens and answering models for this run, and how spending is limited. */
|
package/dist/model/config.js
CHANGED
|
@@ -107,6 +107,7 @@ export function createModelClient(runId, cwd = process.cwd(), env = process.env)
|
|
|
107
107
|
"or choose any OpenAI-compatible model with FIBCHECK_MODEL_BASE_URL, FIBCHECK_MODEL_API_KEY and FIBCHECK_MODEL.");
|
|
108
108
|
}
|
|
109
109
|
const gemini = new GeminiProvider({
|
|
110
|
+
apiKey: env.GOOGLE_API_KEY || env.GEMINI_API_KEY,
|
|
110
111
|
model: env.FIBCHECK_GEMINI_MODEL || undefined,
|
|
111
112
|
fallbackModel: env.FIBCHECK_GEMINI_FALLBACK_MODEL || undefined,
|
|
112
113
|
});
|
package/dist/model/gemini.js
CHANGED
|
@@ -13,7 +13,7 @@ export const REQUEST_TIMEOUT_MS = 90_000;
|
|
|
13
13
|
/**
|
|
14
14
|
* Retries an overloaded or timed-out Gemini request with exponential backoff.
|
|
15
15
|
* After two capacity failures on the main model, the third attempt uses the
|
|
16
|
-
*
|
|
16
|
+
* fallback model (FIBCHECK_GEMINI_FALLBACK_MODEL), which is the same model unless that is set. When that works, subsequent requests on this client use it
|
|
17
17
|
* for the rest of the run. Each HTTP request reserves its own budget slot.
|
|
18
18
|
*/
|
|
19
19
|
export class GeminiProvider {
|
|
@@ -33,7 +33,7 @@ export class GeminiProvider {
|
|
|
33
33
|
// 60 seconds would hide rate limits and spend free-tier quota our call limits never see.
|
|
34
34
|
this.api =
|
|
35
35
|
options.api ??
|
|
36
|
-
new GoogleGenAI({ httpOptions: { timeout: REQUEST_TIMEOUT_MS, retryOptions: { attempts: 1 } } }).models;
|
|
36
|
+
new GoogleGenAI({ apiKey: options.apiKey, httpOptions: { timeout: REQUEST_TIMEOUT_MS, retryOptions: { attempts: 1 } } }).models;
|
|
37
37
|
}
|
|
38
38
|
async generate(request, beforeAttempt = () => { }) {
|
|
39
39
|
const firstModel = this.activeModel;
|
|
@@ -90,7 +90,7 @@ function toProviderError(error) {
|
|
|
90
90
|
if (isRateLimit(error)) {
|
|
91
91
|
return new RateLimitedError("Gemini returned HTTP 429. Wait for the active rate limit or project quota to reset.");
|
|
92
92
|
}
|
|
93
|
-
if (
|
|
93
|
+
if (isCapacityError(error)) {
|
|
94
94
|
return new ProviderOverloadedError(`Gemini is overloaded or timing out (HTTP ${error.status}). Try again in a few minutes.`);
|
|
95
95
|
}
|
|
96
96
|
if (error instanceof ApiError) {
|
package/dist/report/report.js
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { existsSync, readFileSync, writeFileSync } from "node:fs";
|
|
2
2
|
import { basename, dirname, isAbsolute, join, relative, resolve, sep } from "node:path";
|
|
3
|
+
import { errorMessage } from "../util.js";
|
|
3
4
|
export function writeReport(inputPath) {
|
|
4
5
|
const directory = dirname(resolve(inputPath));
|
|
5
6
|
const htmlPath = join(directory, "report.html");
|
|
@@ -20,7 +21,7 @@ export function writeReport(inputPath) {
|
|
|
20
21
|
badge = artwork.svg;
|
|
21
22
|
}
|
|
22
23
|
catch (error) {
|
|
23
|
-
const message =
|
|
24
|
+
const message = errorMessage(error);
|
|
24
25
|
throw new Error(`Could not read check report at ${inputPath}: ${message}`);
|
|
25
26
|
}
|
|
26
27
|
writeFileSync(badgePath, badge);
|
package/dist/target.js
CHANGED
|
@@ -1,20 +1,35 @@
|
|
|
1
1
|
import { randomBytes } from "node:crypto";
|
|
2
2
|
import { existsSync, readFileSync } from "node:fs";
|
|
3
3
|
import { fileURLToPath } from "node:url";
|
|
4
|
+
import { errorMessage } from "./util.js";
|
|
4
5
|
/** The .env name that holds a login value for an app that is already running, for example FIBCHECK_LOGIN_APP_PASS. */
|
|
5
6
|
export function loginEnvName(name) {
|
|
6
7
|
return `FIBCHECK_LOGIN_${name.toUpperCase()}`;
|
|
7
8
|
}
|
|
9
|
+
function loginPlaceholders(login) {
|
|
10
|
+
return Array.from((login ?? "").matchAll(/\{\{([A-Za-z_][A-Za-z0-9_]*)\}\}/g), (match) => match[1]);
|
|
11
|
+
}
|
|
8
12
|
/** Reads the login values a login sentence refers to ({{NAME}}) from the environment. Missing values are left out. */
|
|
9
13
|
export function loginSecretsFromEnv(login, env = process.env) {
|
|
10
|
-
const names =
|
|
14
|
+
const names = loginPlaceholders(login);
|
|
11
15
|
return Object.fromEntries(names.flatMap((name) => {
|
|
12
16
|
const value = env[loginEnvName(name)];
|
|
13
17
|
return value ? [[name, value]] : [];
|
|
14
18
|
}));
|
|
15
19
|
}
|
|
20
|
+
/** Checks an address given with --url and returns it without a trailing slash. */
|
|
21
|
+
export function appBaseUrl(appUrl) {
|
|
22
|
+
if (!URL.canParse(appUrl))
|
|
23
|
+
throw new Error("--url must be a full address such as https://bookmarks.example.org.");
|
|
24
|
+
const url = new URL(appUrl);
|
|
25
|
+
if (url.protocol !== "http:" && url.protocol !== "https:")
|
|
26
|
+
throw new Error("--url must start with http:// or https://.");
|
|
27
|
+
if (url.username || url.password)
|
|
28
|
+
throw new Error("--url must not contain a user name or password. Put login values in .env instead.");
|
|
29
|
+
return `${url.origin}${url.pathname.replace(/\/+$/, "")}`;
|
|
30
|
+
}
|
|
16
31
|
export function runningAppVariables(login, secrets) {
|
|
17
|
-
const names = new Set(
|
|
32
|
+
const names = new Set(loginPlaceholders(login));
|
|
18
33
|
return Object.fromEntries(Array.from(names, (name) => {
|
|
19
34
|
const value = Object.hasOwn(secrets, name) ? secrets[name] : undefined;
|
|
20
35
|
if (typeof value !== "string" || value.length === 0) {
|
|
@@ -45,7 +60,7 @@ export function loadTarget(path) {
|
|
|
45
60
|
target = JSON.parse(readFileSync(path, "utf8"));
|
|
46
61
|
}
|
|
47
62
|
catch (error) {
|
|
48
|
-
throw new Error(`Could not read target ${path}: ${
|
|
63
|
+
throw new Error(`Could not read target ${path}: ${errorMessage(error)}`);
|
|
49
64
|
}
|
|
50
65
|
for (const key of ["name", "repo"]) {
|
|
51
66
|
if (!target[key])
|