fibcheck 0.1.0 → 0.1.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/cli.js CHANGED
@@ -5,14 +5,16 @@ import { parseArgs } from "node:util";
5
5
  import { pathToFileURL } from "node:url";
6
6
  import { attackFinding } from "./attack-run.js";
7
7
  import { attackSummary } from "./attack.js";
8
- import { loadFindings, writeCard } from "./card.js";
8
+ import { writeCard } from "./card.js";
9
9
  import { runCheck } from "./check.js";
10
10
  import { extractClaims } from "./claims/extract.js";
11
11
  import { confirmFinding } from "./confirm.js";
12
+ import { loadFindings } from "./findings.js";
12
13
  import { freeze, frozenClaims } from "./frozen.js";
13
14
  import { createModelClient, loadDotEnv } from "./model/config.js";
14
15
  import { writeReport } from "./report/report.js";
15
16
  import { loadTarget, loginSecretsFromEnv, targetPathFor } from "./target.js";
17
+ import { errorMessage, newRunId } from "./util.js";
16
18
  export const USAGE = `fibcheck: finds the false claims in a web app's README with Kane CLI.
17
19
 
18
20
  Usage:
@@ -53,13 +55,13 @@ export async function main(argv) {
53
55
  return 0;
54
56
  }
55
57
  if (command === "check")
56
- return await check(rest);
58
+ return await checkCommand(rest);
57
59
  if (command === "report")
58
- return report(rest);
60
+ return reportCommand(rest);
59
61
  if (command === "card")
60
- return card(rest);
62
+ return cardCommand(rest);
61
63
  if (command === "attack")
62
- return await attack(rest);
64
+ return await attackCommand(rest);
63
65
  if (command === "freeze")
64
66
  return await freezeCommand(rest);
65
67
  if (command === "confirm")
@@ -98,7 +100,7 @@ async function freezeCommand(args) {
98
100
  return 1;
99
101
  }
100
102
  loadDotEnv();
101
- const runId = new Date().toISOString().replace(/[:.]/g, "-");
103
+ const runId = newRunId();
102
104
  const extraction = await extractClaims(createModelClient(runId), [{ file: docsFile, content: readFileSync(docsFile, "utf8") }]);
103
105
  const file = freeze(docsFile, extraction.claims);
104
106
  writeFileSync(values.out, `${JSON.stringify(file, null, 2)}\n`, { flag: "wx" });
@@ -108,7 +110,7 @@ async function freezeCommand(args) {
108
110
  process.stdout.write(` Line ${line}: ${why}\n`);
109
111
  return 0;
110
112
  }
111
- async function attack(args) {
113
+ async function attackCommand(args) {
112
114
  const { values, positionals } = parseArgs({
113
115
  args, allowPositionals: true,
114
116
  options: { url: { type: "string" }, login: { type: "string" }, findings: { type: "string" } },
@@ -118,8 +120,13 @@ async function attack(args) {
118
120
  process.stderr.write("Usage: fibcheck attack <finding-id> --url ADDRESS [--login SENTENCE] [--findings index/findings.json]\n");
119
121
  return 1;
120
122
  }
123
+ const loginError = loginArgError(values.login);
124
+ if (loginError) {
125
+ process.stderr.write(loginError);
126
+ return 1;
127
+ }
121
128
  loadDotEnv();
122
- const runId = new Date().toISOString().replace(/[:.]/g, "-");
129
+ const runId = newRunId();
123
130
  const record = await attackFinding(createModelClient(runId), {
124
131
  findingsPath: values.findings ?? "index/findings.json", id, appUrl: values.url, login: values.login,
125
132
  secrets: loginSecretsFromEnv(values.login), runsDir: join("runs", `attack-${runId}`),
@@ -130,7 +137,7 @@ async function attack(args) {
130
137
  process.stdout.write(`${attackSummary(record.results) || "No lazy fix was rejected."}\n`);
131
138
  return 0;
132
139
  }
133
- function card(args) {
140
+ function cardCommand(args) {
134
141
  const { values, positionals } = parseArgs({
135
142
  args, allowPositionals: true,
136
143
  options: { all: { type: "boolean" }, findings: { type: "string" }, out: { type: "string" } },
@@ -145,7 +152,7 @@ function card(args) {
145
152
  process.stdout.write(`Saved ${writeCard(findingsPath, id, values.out ?? "index/cards")}\n`);
146
153
  return 0;
147
154
  }
148
- function report(args) {
155
+ function reportCommand(args) {
149
156
  const [inputPath] = args;
150
157
  if (args.length !== 1 || !inputPath || inputPath.startsWith("-")) {
151
158
  process.stderr.write("Usage: fibcheck report <check.json>\n");
@@ -180,13 +187,19 @@ export function parseCheckArgs(args) {
180
187
  return "--max-kane-runs must be a whole number.\n";
181
188
  if (values.login !== undefined && values.url === undefined)
182
189
  return "--login needs --url.\n";
183
- if (values.login !== undefined && !/\{\{[A-Za-z_][A-Za-z0-9_]*\}\}/.test(values.login)) {
184
- return "--login must name its values as {{NAME}} and keep them in .env as FIBCHECK_LOGIN_NAME, never type a password into the command.\n";
185
- }
190
+ const loginError = loginArgError(values.login);
191
+ if (loginError)
192
+ return loginError;
186
193
  if (values.url !== undefined && values["app-running"])
187
194
  return "Use either --url or --app-running, not both.\n";
188
195
  return { target, maxClaims, maxKaneRuns, only: values.only, appRunning: values["app-running"], url: values.url, login: values.login };
189
196
  }
197
+ function loginArgError(login) {
198
+ if (login !== undefined && !/\{\{[A-Za-z_][A-Za-z0-9_]*\}\}/.test(login)) {
199
+ return "--login must name its values as {{NAME}} and keep them in .env as FIBCHECK_LOGIN_NAME, never type a password into the command.\n";
200
+ }
201
+ return undefined;
202
+ }
190
203
  /** undefined when absent, null when not a non-negative whole number. */
191
204
  function wholeNumber(raw, isWhole) {
192
205
  if (raw === undefined)
@@ -194,7 +207,7 @@ function wholeNumber(raw, isWhole) {
194
207
  const value = Number(raw);
195
208
  return isWhole(value) && value >= 0 ? value : null;
196
209
  }
197
- async function check(args) {
210
+ async function checkCommand(args) {
198
211
  const parsed = parseCheckArgs(args);
199
212
  if (typeof parsed === "string") {
200
213
  process.stderr.write(parsed);
@@ -204,9 +217,10 @@ async function check(args) {
204
217
  loadDotEnv();
205
218
  const alreadyRunning = parsed.url !== undefined || parsed.appRunning === true;
206
219
  const login = parsed.login ?? loadTarget(targetPath).login;
207
- const runId = new Date().toISOString().replace(/[:.]/g, "-");
220
+ const runId = newRunId();
208
221
  const report = await runCheck(createModelClient(runId), {
209
222
  targetPath,
223
+ runId,
210
224
  maxClaims: parsed.maxClaims,
211
225
  maxKaneRuns: parsed.maxKaneRuns,
212
226
  only: parsed.only,
@@ -229,7 +243,7 @@ function summaryLine(report) {
229
243
  const executable = process.argv[1];
230
244
  if (executable && existsSync(executable) && import.meta.url === pathToFileURL(realpathSync(executable)).href) {
231
245
  main(process.argv.slice(2)).then((code) => (process.exitCode = code), (error) => {
232
- process.stderr.write(`fibcheck: ${error instanceof Error ? error.message : String(error)}\n`);
246
+ process.stderr.write(`fibcheck: ${errorMessage(error)}\n`);
233
247
  process.exitCode = 1;
234
248
  });
235
249
  }
package/dist/confirm.js CHANGED
@@ -1,5 +1,7 @@
1
- import { copyFileSync, existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
1
+ import { copyFileSync, existsSync, mkdirSync, readFileSync } from "node:fs";
2
2
  import { basename, dirname, extname, join } from "node:path";
3
+ import { readFindingsFile, writeFindingsFile } from "./findings.js";
4
+ import { errorMessage } from "./util.js";
3
5
  /**
4
6
  * Turns a candidate finding from a saved check into an entry in a findings file, after a person has read the
5
7
  * evidence. It copies Kane's screenshots of the two failing runs and the passing control, unchanged, and
@@ -10,7 +12,7 @@ export function confirmFinding(options) {
10
12
  throw new Error("A confirmed finding is false or outdated.");
11
13
  const report = readJson(options.checkPath, "check");
12
14
  const { tested, failures, control } = backedCandidate(report, options);
13
- const findings = existsSync(options.findingsPath) ? readJson(options.findingsPath, "findings") : [];
15
+ const findings = existsSync(options.findingsPath) ? readFindingsFile(options.findingsPath) : [];
14
16
  const { claim } = tested;
15
17
  const project = options.project ?? report.target;
16
18
  const id = options.id ?? `${slug(project)}-${slug(basename(claim.source.file, extname(claim.source.file)))}-l${claim.source.startLine}`;
@@ -33,7 +35,7 @@ export function confirmFinding(options) {
33
35
  score: { findings: 1, claims: report.tested.length },
34
36
  ...(options.fix ? { fix: options.fix } : {}),
35
37
  };
36
- writeFileSync(options.findingsPath, `${JSON.stringify([...findings, finding], null, 2)}\n`);
38
+ writeFindingsFile(options.findingsPath, [...findings, finding]);
37
39
  return finding;
38
40
  }
39
41
  /** The tested claim, if it is a candidate finding backed by two failed runs and a passing control. */
@@ -56,15 +58,22 @@ function readJson(path, what) {
56
58
  return JSON.parse(readFileSync(path, "utf8"));
57
59
  }
58
60
  catch (error) {
59
- throw new Error(`Could not read the ${what} file ${path}: ${error instanceof Error ? error.message : String(error)}`);
61
+ throw new Error(`Could not read the ${what} file ${path}: ${errorMessage(error)}`);
60
62
  }
61
63
  }
62
- /** The failed runs of the documented check (the first run and its reproductions) and the passing control. */
64
+ /**
65
+ * The two failed runs behind the candidate and the passing control. The repeat ("retry") reproduces the exact check
66
+ * of the run just before it, which is the run read as a contradiction: the first run, or a reworded run when the
67
+ * first one failed for another reason. Both must have failed.
68
+ */
63
69
  function backingRuns({ kane, questioning }) {
64
70
  const followUps = questioning?.followUps ?? [];
65
- const failures = [kane, ...followUps.filter(({ purpose }) => purpose === "retry").map((followUp) => followUp.kane)]
66
- .filter((run) => run.status === "failed");
71
+ const retryAt = followUps.findIndex(({ purpose }) => purpose === "retry");
67
72
  const control = followUps.find(({ purpose, kane: run }) => purpose === "control" && run.status === "passed")?.kane;
73
+ if (retryAt === -1)
74
+ return { failures: [], control };
75
+ const contradicted = retryAt === 0 ? kane : followUps[retryAt - 1].kane;
76
+ const failures = [contradicted, followUps[retryAt].kane].filter((run) => run.status === "failed");
68
77
  return { failures, control };
69
78
  }
70
79
  /** Kane's screenshot for a run, from the claims/ folder saved beside the check: the failed step, or the last step. */
@@ -0,0 +1,36 @@
1
+ import { existsSync, readFileSync, writeFileSync } from "node:fs";
2
+ import { dirname, join } from "node:path";
3
+ import { errorMessage } from "./util.js";
4
+ export function readFindingsFile(path) {
5
+ let value;
6
+ try {
7
+ value = JSON.parse(readFileSync(path, "utf8"));
8
+ }
9
+ catch (error) {
10
+ throw new Error(`Could not read the findings file ${path}: ${errorMessage(error)}`);
11
+ }
12
+ if (!Array.isArray(value))
13
+ throw new Error(`Could not read the findings file ${path}: expected a list of findings.`);
14
+ return value;
15
+ }
16
+ export function writeFindingsFile(path, findings) {
17
+ writeFileSync(path, `${JSON.stringify(findings, null, 2)}\n`);
18
+ }
19
+ /** Reads and checks the findings file; every referenced screenshot must exist. */
20
+ export function loadFindings(path) {
21
+ const findings = readFindingsFile(path);
22
+ for (const finding of findings) {
23
+ for (const file of evidenceFiles(finding)) {
24
+ if (!existsSync(join(dirname(path), file)))
25
+ throw new Error(`Finding ${finding.id} refers to a missing file: ${file}`);
26
+ }
27
+ if ((finding.verdict === "false" || finding.verdict === "outdated") && (finding.evidence.failures.length < 2 || !finding.evidence.control)) {
28
+ throw new Error(`Finding ${finding.id} is ${finding.verdict} without two recorded failures and a control.`);
29
+ }
30
+ }
31
+ return findings;
32
+ }
33
+ export function evidenceFiles(finding) {
34
+ const { failures, control, fixProof, held } = finding.evidence;
35
+ return [finding.screenshot, ...failures, control, fixProof, held].filter((file) => Boolean(file));
36
+ }
package/dist/frozen.js CHANGED
@@ -1,7 +1,16 @@
1
1
  import { createHash } from "node:crypto";
2
2
  import { CLAIM_TYPES } from "./claims/types.js";
3
+ /**
4
+ * sha256 of everything a reviewer approved: the quote, the check, the control and the assertion mode. A claim without an
5
+ * assertion mode hashes exactly as before the mode was covered, so older files without one still load.
6
+ */
3
7
  export function contentHash(claim) {
4
- return createHash("sha256").update(JSON.stringify([claim.quote, claim.check, claim.control ?? ""])).digest("hex");
8
+ const fields = [claim.quote, claim.check, claim.control ?? "", ...(claim.assertionMode ? [claim.assertionMode] : [])];
9
+ return createHash("sha256").update(JSON.stringify(fields)).digest("hex");
10
+ }
11
+ /** True when a claim with an assertion mode carries a hash from before the mode was covered. */
12
+ function hashedWithoutMode(claim) {
13
+ return Boolean(claim.assertionMode) && contentHash({ ...claim, assertionMode: undefined }) === claim.contentSha256;
5
14
  }
6
15
  export function parseFrozenFile(text) {
7
16
  let value;
@@ -39,6 +48,9 @@ export function frozenClaims(file) {
39
48
  const flag = (why) => needsReview.push({ id: frozen.id, line: frozen.startLine, why });
40
49
  if (!frozen.reviewed?.by?.trim() || !frozen.reviewed?.at?.trim())
41
50
  flag("No one has reviewed this check yet.");
51
+ else if (hashedWithoutMode(frozen)) {
52
+ flag("This check was frozen before the assertion mode was part of its hash. Re-freeze it with fibcheck freeze and review it again.");
53
+ }
42
54
  else if (contentHash(frozen) !== frozen.contentSha256)
43
55
  flag("The check changed after review. Freeze and review it again.");
44
56
  else {
@@ -95,16 +107,17 @@ function labelProblem(check, firstAssertion, { label, element }) {
95
107
  function escape(text) {
96
108
  return text.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
97
109
  }
98
- /** Writes a frozen file from extracted claims, for a person to review. Reviewer fields are left empty. */
110
+ /** Builds a frozen file from extracted claims, for a person to review. Reviewer fields are left empty. */
99
111
  export function freeze(docsFile, claims) {
100
112
  return {
101
113
  version: 1,
102
114
  docsFile,
103
115
  claims: claims.filter((claim) => claim.type !== "untestable" && claim.check).map((claim) => {
104
- const entry = { quote: claim.quote, check: claim.check, control: claim.control };
116
+ const entry = { quote: claim.quote, check: claim.check, control: claim.control, assertionMode: claim.assertionMode };
105
117
  return {
106
118
  id: claim.id, type: claim.type, quote: claim.quote, startLine: claim.source.startLine, endLine: claim.source.endLine,
107
119
  check: claim.check, ...(claim.control ? { control: claim.control } : {}),
120
+ ...(claim.assertionMode ? { assertionMode: claim.assertionMode } : {}),
108
121
  contentSha256: contentHash(entry), reviewed: { by: "", at: "" },
109
122
  };
110
123
  }),
@@ -29,7 +29,7 @@ export async function questionGenerally(claim, check, general, run, canRunFollow
29
29
  if (attempt.kane.status === "error" || attempt.kane.status === "cancelled") {
30
30
  return outcome(state, "unconfirmed", `Kane did not finish the run (${attempt.kane.status}).`);
31
31
  }
32
- const diagnosis = await review(state, attempt);
32
+ const diagnosis = await diagnoseAttempt(state, attempt);
33
33
  if (!diagnosis)
34
34
  return outcome(state, "unconfirmed", "The model could not read Kane's result, so it was not interpreted.");
35
35
  const next = await decide(state, attempt, diagnosis);
@@ -50,7 +50,7 @@ async function decide(state, attempt, diagnosis) {
50
50
  ?? outcome(state, "unconfirmed", `Kane passed, but the run did not exercise the documented claim. ${diagnosis.reason}`);
51
51
  }
52
52
  if (diagnosis.cause === "claim_contradicted" && diagnosis.exercisedClaim)
53
- return await confirm(state, attempt, diagnosis);
53
+ return await reproduceAndControl(state, attempt, diagnosis);
54
54
  if (diagnosis.cause === "agent_misstep" || diagnosis.cause === "check_problem") {
55
55
  const next = await reword(state, diagnosis);
56
56
  if (next)
@@ -62,11 +62,11 @@ async function reword(state, diagnosis) {
62
62
  if (state.rewordsLeft <= 0 || !diagnosis.revisedCheck || !state.canRunFollowUp())
63
63
  return undefined;
64
64
  state.rewordsLeft--;
65
- const kane = await state.run("reword", state.general.objectiveFor(diagnosis.revisedCheck));
65
+ const kane = await state.run("reword", state.general.objectiveFor(diagnosis.revisedCheck), state.claim.assertionMode);
66
66
  state.followUps.push({ purpose: "reword", reason: `Reworded after the model's reading: ${diagnosis.reason}`, kane });
67
67
  return { round: "reword", check: diagnosis.revisedCheck, kane };
68
68
  }
69
- async function confirm(state, failed, diagnosis) {
69
+ async function reproduceAndControl(state, failed, diagnosis) {
70
70
  if (!diagnosis.ownMistakeRuledOut)
71
71
  return outcome(state, "unconfirmed", "The model gave no reason to rule out a test error.");
72
72
  if (!state.canRunFollowUp())
@@ -74,14 +74,14 @@ async function confirm(state, failed, diagnosis) {
74
74
  // Repeat the exact check that failed, so the reproduction tests the same assertion.
75
75
  const check = failed.check;
76
76
  const kane = await state.run("retry", state.general.objectiveFor(check), state.claim.assertionMode);
77
- return await confirmAfterReproduction(state, failed, diagnosis, { round: "retry", check, kane });
77
+ return await checkReproductionAndControl(state, failed, diagnosis, { round: "retry", check, kane });
78
78
  }
79
- async function confirmAfterReproduction(state, failed, diagnosis, reproduction) {
79
+ async function checkReproductionAndControl(state, failed, diagnosis, reproduction) {
80
80
  state.followUps.push({ purpose: "retry", reason: "Reproduce the same documented assertion in an independent Kane run.", kane: reproduction.kane });
81
81
  if (reproduction.kane.status !== "failed" || !distinctKaneRuns(reproduction.kane, failed.kane)) {
82
82
  return outcome(state, "unconfirmed", "Kane did not fail the documented assertion again in a separate run.");
83
83
  }
84
- const second = await review(state, reproduction);
84
+ const second = await diagnoseAttempt(state, reproduction);
85
85
  if (!second || second.cause !== "claim_contradicted" || !second.exercisedClaim || !second.ownMistakeRuledOut) {
86
86
  return outcome(state, "unconfirmed", "The reproduction failed for a different reason, so the result is not consistent.");
87
87
  }
@@ -98,7 +98,7 @@ async function confirmAfterReproduction(state, failed, diagnosis, reproduction)
98
98
  state.candidateFinding = true;
99
99
  return outcome(state, "unconfirmed", `Candidate finding, awaiting a person's review. Two separate Kane runs failed the documented result. ${diagnosis.reason} Test error ruled out: ${diagnosis.ownMistakeRuledOut} The control passed. ${NOT_OUTDATED}`);
100
100
  }
101
- async function review(state, attempt) {
101
+ async function diagnoseAttempt(state, attempt) {
102
102
  let diagnosis;
103
103
  try {
104
104
  diagnosis = await state.general.diagnose(state.claim, attempt.check, attempt.kane);
package/dist/github.js CHANGED
@@ -1,5 +1,6 @@
1
1
  import { cpSync, existsSync, mkdirSync, writeFileSync } from "node:fs";
2
2
  import { join } from "node:path";
3
+ import { NEEDS_REVIEW_PREFIX } from "./check.js";
3
4
  import { countResults } from "./report/report.js";
4
5
  import { frozenClaims, parseFrozenFile } from "./frozen.js";
5
6
  import { loadTarget } from "./target.js";
@@ -93,11 +94,12 @@ const VERDICT_LABELS = {
93
94
  false: "Did not match the app",
94
95
  outdated: "Did not match the app",
95
96
  };
97
+ const upperFirst = (text) => `${text.charAt(0).toUpperCase()}${text.slice(1)}`;
96
98
  /** Frozen checks that were not run because the line changed or the check was not reviewed. */
97
99
  function needsReviewLines(report, needsReview) {
98
100
  const lines = [
99
101
  ...needsReview.map(({ line, why }) => `- Line ${line}: ${why}`),
100
- ...report.rejected.filter(({ why }) => why.startsWith("Needs review:")).map(({ why }) => `- ${why.replace(/^Needs review: l/, "L")}`),
102
+ ...report.rejected.filter(({ why }) => why.startsWith(NEEDS_REVIEW_PREFIX)).map(({ why }) => `- ${upperFirst(why.slice(NEEDS_REVIEW_PREFIX.length).trimStart())}`),
101
103
  ];
102
104
  return lines.length ? ["", "**Needs review (not run)**", "", ...lines] : [];
103
105
  }
@@ -107,7 +109,7 @@ function lineTable(report) {
107
109
  const rows = [
108
110
  ...report.tested.map(({ claim, kane, questioning }) => {
109
111
  const runs = [kane.status, ...(questioning?.followUps ?? []).map(({ purpose, kane: run }) => `${run.status} (${purpose})`)].join(", ");
110
- return `| ${claim.source.startLine} | ${questioning ? VERDICT_LABELS[questioning.verdict] ?? questioning.verdict : "No verdict"} | ${runs} |`;
112
+ return `| ${claim.source.startLine} | ${questioning ? VERDICT_LABELS[questioning.verdict] : "No verdict"} | ${runs} |`;
111
113
  }),
112
114
  ...report.notTested.map((id) => `| ${line(id)} | Not run (over the claim limit) | none |`),
113
115
  ];
@@ -0,0 +1,28 @@
1
+ import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
2
+ import { tmpdir } from "node:os";
3
+ import { join } from "node:path";
4
+ export function escapeKaneBraces(text) {
5
+ return text.replace(/[{}]{2,}/g, (braces) => braces.split("").join(" "));
6
+ }
7
+ export function writeKaneVariables(variables) {
8
+ if (Object.keys(variables).length === 0)
9
+ return undefined;
10
+ const directory = mkdtempSync(join(tmpdir(), "fibcheck-kane-"));
11
+ const path = join(directory, "variables.json");
12
+ const remove = () => rmSync(directory, { recursive: true, force: true });
13
+ try {
14
+ writeFileSync(path, JSON.stringify(variables), { flag: "wx", mode: 0o600 });
15
+ }
16
+ catch (error) {
17
+ remove();
18
+ throw error;
19
+ }
20
+ return { path, remove };
21
+ }
22
+ /**
23
+ * Documentation often names its own default address (for example http://localhost:5230). The test app runs
24
+ * elsewhere and Kane already starts on it, so a local address in a check is replaced by a reference to the app.
25
+ */
26
+ export function localAddressesToApp(check) {
27
+ return check.replace(/\bhttps?:\/\/(?:localhost|127\.0\.0\.1|0\.0\.0\.0)(?::\d+)?(\/[^\s"'<>]*)?/gi, (_match, path) => (path && path !== "/" ? `the app's ${path.replace(/[.,;:]+$/, "")} page` : "the app's start page"));
28
+ }
@@ -76,14 +76,14 @@ function observationsFrom(end) {
76
76
  const memory = end?.context?.memory;
77
77
  if (!memory || typeof memory !== "object")
78
78
  return [];
79
- return Object.entries(memory).map(([name, m]) => ({
80
- name,
81
- condition: str(m.condition),
82
- extractedValue: str(m.extracted_value),
83
- expectedValue: str(m.expected_value),
84
- analyzerType: str(m.analyzer_type),
85
- usedDomCode: Boolean(m.code_js || m.wrapped_js || (Array.isArray(m.locators) ? m.locators.length : m.locators)),
86
- }));
79
+ return Object.entries(memory).flatMap(([name, m]) => (m && typeof m === "object" ? [{
80
+ name,
81
+ condition: str(m.condition),
82
+ extractedValue: str(m.extracted_value),
83
+ expectedValue: str(m.expected_value),
84
+ analyzerType: str(m.analyzer_type),
85
+ usedDomCode: Boolean(m.code_js || m.wrapped_js || (Array.isArray(m.locators) ? m.locators.length : m.locators)),
86
+ }] : []));
87
87
  }
88
88
  function blameFrom(end) {
89
89
  const v = end?.verdict;
@@ -3,6 +3,7 @@ import { mkdirSync, writeFileSync } from "node:fs";
3
3
  import { join } from "node:path";
4
4
  import { extractStepEvidence, findEvidencePack } from "./evidence.js";
5
5
  import { parseKaneOutput } from "./parse.js";
6
+ import { errorMessage } from "../util.js";
6
7
  /** Runs one Kane flow in agent mode and returns its structured result. Only Kane decides pass or fail. */
7
8
  export async function runKane(options) {
8
9
  mkdirSync(options.outDir, { recursive: true });
@@ -42,7 +43,7 @@ function collectEvidence(result, outDir) {
42
43
  return { evidence: extractStepEvidence(pack, join(outDir, "evidence")), evidenceError: null };
43
44
  }
44
45
  catch (error) {
45
- return { evidence: [], evidenceError: `Could not unpack Kane's evidence: ${error instanceof Error ? error.message : String(error)}` };
46
+ return { evidence: [], evidenceError: `Could not unpack Kane's evidence: ${errorMessage(error)}` };
46
47
  }
47
48
  }
48
49
  function run(command, args) {
package/dist/loop.js CHANGED
@@ -20,15 +20,20 @@ const RETRY_TAB = `Go directly to ${SETTINGS_PATH} after login. First assert exa
20
20
  const CONTROL = `From the signed-in bookmarks page, open the Settings dropdown in the top navigation. Assert exactly: ${MENU_ASSERTION} Expect true. Click Admin inside that dropdown. Make these as separate assertions: ${TITLE_ASSERTION} ${ACCOUNT_ASSERTION} ${USERS_ASSERTION} Expect true for each.`;
21
21
  const CONTROL_PROOF = "The same test login opened Site administration through the top navigation’s Settings dropdown in the passing control run.";
22
22
  const TAB_HISTORY = "Linkding’s historical source had an Admin tab in the Settings template at 3ae9cf0. The template was removed at 226eb69.";
23
- export async function questionClaim(claim, managedTarget, originalObjective, run, prepareTags, canRunFollowUp = () => true, general) {
23
+ /** The bundled Linkding target exactly as shipped, which the built-in Linkding checks were written for. */
24
+ function isBundledLinkding(target) {
25
+ return isLinkdingRepo(target.repo)
26
+ && target.app?.image === "sissbruecker/linkding:1.47.0"
27
+ && target.app?.env?.LD_SUPERUSER_NAME === "pinotest"
28
+ && target.app?.env?.LD_SUPERUSER_PASSWORD === "{{secret:app_pass}}"
29
+ && Boolean(target.login?.includes("pinotest and password {{app_pass}}"));
30
+ }
31
+ export async function questionClaim(claim, run, options) {
32
+ const { managedTarget, originalObjective, prepareTags, canRunFollowUp = () => true, general } = options;
24
33
  // A frozen, reviewed check is run exactly as reviewed, never through a built-in special case.
25
34
  if (general?.frozen && claim.check)
26
35
  return await questionGenerally(claim, claim.check, general, run, canRunFollowUp);
27
- const linkding = managedTarget && isLinkdingRepo(managedTarget.repo)
28
- && managedTarget.app?.image === "sissbruecker/linkding:1.47.0"
29
- && managedTarget.app?.env?.LD_SUPERUSER_NAME === "pinotest"
30
- && managedTarget.app?.env?.LD_SUPERUSER_PASSWORD === "{{secret:app_pass}}"
31
- && managedTarget.login?.includes("pinotest and password {{app_pass}}");
36
+ const linkding = managedTarget && isBundledLinkding(managedTarget);
32
37
  if (linkding && managedTarget.login && prepareTags) {
33
38
  const tags = linkdingTagClaim(claim);
34
39
  if (tags) {
@@ -39,7 +44,7 @@ export async function questionClaim(claim, managedTarget, originalObjective, run
39
44
  if (!linkding || !isAdminTabClaim(claim)) {
40
45
  if (general && claim.check)
41
46
  return await questionGenerally(claim, claim.check, general, run, canRunFollowUp);
42
- const kane = await run("initial", originalObjective);
47
+ const kane = await run("initial", originalObjective, claim.assertionMode);
43
48
  return {
44
49
  kane,
45
50
  questioning: {
@@ -56,7 +56,6 @@ export class CallBudget {
56
56
  ensureDir(this.ledgerPath);
57
57
  appendFileSync(this.ledgerPath, `${JSON.stringify(line)}\n`);
58
58
  }
59
- /** Calls and tokens used so far in this run. */
60
59
  /** Calls and tokens for this run, and every model that answered. */
61
60
  runSummary() {
62
61
  return { calls: this.runCalls, ...this.runUsage, models: [...this.runModels].sort((a, b) => a.localeCompare(b)) };
@@ -64,19 +63,29 @@ export class CallBudget {
64
63
  readTotals() {
65
64
  if (!existsSync(this.totalsPath))
66
65
  return { calls: 0, inputTokens: 0, outputTokens: 0 };
66
+ let totals;
67
67
  try {
68
- return JSON.parse(readFileSync(this.totalsPath, "utf8"));
68
+ totals = JSON.parse(readFileSync(this.totalsPath, "utf8"));
69
69
  }
70
70
  catch {
71
- // Refuse rather than reset: resetting would silently lift the total limit.
72
- throw new BudgetExceededError(`The usage totals file ${this.totalsPath} is damaged. Fix or delete it on purpose before making more model calls.`);
71
+ throw this.damaged();
73
72
  }
73
+ if (!isCount(totals?.calls) || !isCount(totals?.inputTokens) || !isCount(totals?.outputTokens))
74
+ throw this.damaged();
75
+ return { calls: totals.calls, inputTokens: totals.inputTokens, outputTokens: totals.outputTokens };
76
+ }
77
+ /** Refuse rather than reset: resetting would silently lift the total limit. */
78
+ damaged() {
79
+ return new BudgetExceededError(`The usage totals file ${this.totalsPath} is damaged. Fix or delete it on purpose before making more model calls.`);
74
80
  }
75
81
  writeTotals(totals) {
76
82
  ensureDir(this.totalsPath);
77
83
  writeFileSync(this.totalsPath, `${JSON.stringify(totals, null, 2)}\n`);
78
84
  }
79
85
  }
86
+ function isCount(value) {
87
+ return typeof value === "number" && Number.isFinite(value) && value >= 0;
88
+ }
80
89
  function ensureDir(filePath) {
81
90
  mkdirSync(dirname(filePath), { recursive: true });
82
91
  }
@@ -1,3 +1,4 @@
1
+ import { errorMessage } from "../util.js";
1
2
  import { UnusableModelOutputError } from "./types.js";
2
3
  /**
3
4
  * The one entry point for model calls. The provider must announce every HTTP
@@ -44,7 +45,7 @@ export class ModelClient {
44
45
  return { value: validate(parsed), response };
45
46
  }
46
47
  catch (error) {
47
- throw new UnusableModelOutputError(`The model's reply for "${request.purpose}" had the wrong shape: ${error instanceof Error ? error.message : String(error)}`);
48
+ throw new UnusableModelOutputError(`The model's reply for "${request.purpose}" had the wrong shape: ${errorMessage(error)}`);
48
49
  }
49
50
  }
50
51
  /** Calls, tokens and answering models for this run, and how spending is limited. */
@@ -107,6 +107,7 @@ export function createModelClient(runId, cwd = process.cwd(), env = process.env)
107
107
  "or choose any OpenAI-compatible model with FIBCHECK_MODEL_BASE_URL, FIBCHECK_MODEL_API_KEY and FIBCHECK_MODEL.");
108
108
  }
109
109
  const gemini = new GeminiProvider({
110
+ apiKey: env.GOOGLE_API_KEY || env.GEMINI_API_KEY,
110
111
  model: env.FIBCHECK_GEMINI_MODEL || undefined,
111
112
  fallbackModel: env.FIBCHECK_GEMINI_FALLBACK_MODEL || undefined,
112
113
  });
@@ -13,7 +13,7 @@ export const REQUEST_TIMEOUT_MS = 90_000;
13
13
  /**
14
14
  * Retries an overloaded or timed-out Gemini request with exponential backoff.
15
15
  * After two capacity failures on the main model, the third attempt uses the
16
- * lighter model. When that works, subsequent requests on this client use it
16
+ * fallback model (FIBCHECK_GEMINI_FALLBACK_MODEL), which is the same model unless that is set. When that works, subsequent requests on this client use it
17
17
  * for the rest of the run. Each HTTP request reserves its own budget slot.
18
18
  */
19
19
  export class GeminiProvider {
@@ -33,7 +33,7 @@ export class GeminiProvider {
33
33
  // 60 seconds would hide rate limits and spend free-tier quota our call limits never see.
34
34
  this.api =
35
35
  options.api ??
36
- new GoogleGenAI({ httpOptions: { timeout: REQUEST_TIMEOUT_MS, retryOptions: { attempts: 1 } } }).models;
36
+ new GoogleGenAI({ apiKey: options.apiKey, httpOptions: { timeout: REQUEST_TIMEOUT_MS, retryOptions: { attempts: 1 } } }).models;
37
37
  }
38
38
  async generate(request, beforeAttempt = () => { }) {
39
39
  const firstModel = this.activeModel;
@@ -90,7 +90,7 @@ function toProviderError(error) {
90
90
  if (isRateLimit(error)) {
91
91
  return new RateLimitedError("Gemini returned HTTP 429. Wait for the active rate limit or project quota to reset.");
92
92
  }
93
- if (error instanceof ApiError && (error.status === 503 || error.status === 504)) {
93
+ if (isCapacityError(error)) {
94
94
  return new ProviderOverloadedError(`Gemini is overloaded or timing out (HTTP ${error.status}). Try again in a few minutes.`);
95
95
  }
96
96
  if (error instanceof ApiError) {
@@ -1,5 +1,6 @@
1
1
  import { existsSync, readFileSync, writeFileSync } from "node:fs";
2
2
  import { basename, dirname, isAbsolute, join, relative, resolve, sep } from "node:path";
3
+ import { errorMessage } from "../util.js";
3
4
  export function writeReport(inputPath) {
4
5
  const directory = dirname(resolve(inputPath));
5
6
  const htmlPath = join(directory, "report.html");
@@ -20,7 +21,7 @@ export function writeReport(inputPath) {
20
21
  badge = artwork.svg;
21
22
  }
22
23
  catch (error) {
23
- const message = error instanceof Error ? error.message : String(error);
24
+ const message = errorMessage(error);
24
25
  throw new Error(`Could not read check report at ${inputPath}: ${message}`);
25
26
  }
26
27
  writeFileSync(badgePath, badge);
package/dist/target.js CHANGED
@@ -1,20 +1,35 @@
1
1
  import { randomBytes } from "node:crypto";
2
2
  import { existsSync, readFileSync } from "node:fs";
3
3
  import { fileURLToPath } from "node:url";
4
+ import { errorMessage } from "./util.js";
4
5
  /** The .env name that holds a login value for an app that is already running, for example FIBCHECK_LOGIN_APP_PASS. */
5
6
  export function loginEnvName(name) {
6
7
  return `FIBCHECK_LOGIN_${name.toUpperCase()}`;
7
8
  }
9
+ function loginPlaceholders(login) {
10
+ return Array.from((login ?? "").matchAll(/\{\{([A-Za-z_][A-Za-z0-9_]*)\}\}/g), (match) => match[1]);
11
+ }
8
12
  /** Reads the login values a login sentence refers to ({{NAME}}) from the environment. Missing values are left out. */
9
13
  export function loginSecretsFromEnv(login, env = process.env) {
10
- const names = Array.from((login ?? "").matchAll(/\{\{([A-Za-z_][A-Za-z0-9_]*)\}\}/g), (match) => match[1]);
14
+ const names = loginPlaceholders(login);
11
15
  return Object.fromEntries(names.flatMap((name) => {
12
16
  const value = env[loginEnvName(name)];
13
17
  return value ? [[name, value]] : [];
14
18
  }));
15
19
  }
20
+ /** Checks an address given with --url and returns it without a trailing slash. */
21
+ export function appBaseUrl(appUrl) {
22
+ if (!URL.canParse(appUrl))
23
+ throw new Error("--url must be a full address such as https://bookmarks.example.org.");
24
+ const url = new URL(appUrl);
25
+ if (url.protocol !== "http:" && url.protocol !== "https:")
26
+ throw new Error("--url must start with http:// or https://.");
27
+ if (url.username || url.password)
28
+ throw new Error("--url must not contain a user name or password. Put login values in .env instead.");
29
+ return `${url.origin}${url.pathname.replace(/\/+$/, "")}`;
30
+ }
16
31
  export function runningAppVariables(login, secrets) {
17
- const names = new Set(Array.from((login ?? "").matchAll(/\{\{([A-Za-z_][A-Za-z0-9_]*)\}\}/g), (match) => match[1]));
32
+ const names = new Set(loginPlaceholders(login));
18
33
  return Object.fromEntries(Array.from(names, (name) => {
19
34
  const value = Object.hasOwn(secrets, name) ? secrets[name] : undefined;
20
35
  if (typeof value !== "string" || value.length === 0) {
@@ -45,7 +60,7 @@ export function loadTarget(path) {
45
60
  target = JSON.parse(readFileSync(path, "utf8"));
46
61
  }
47
62
  catch (error) {
48
- throw new Error(`Could not read target ${path}: ${error instanceof Error ? error.message : String(error)}`);
63
+ throw new Error(`Could not read target ${path}: ${errorMessage(error)}`);
49
64
  }
50
65
  for (const key of ["name", "repo"]) {
51
66
  if (!target[key])
package/dist/util.js ADDED
@@ -0,0 +1,6 @@
1
+ export function errorMessage(error) {
2
+ return error instanceof Error ? error.message : String(error);
3
+ }
4
+ export function newRunId() {
5
+ return new Date().toISOString().replace(/[:.]/g, "-");
6
+ }