fibcheck 0.1.0 → 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +13 -0
- package/README.md +17 -22
- package/dist/app/docker.js +2 -1
- package/dist/attack-run.js +11 -12
- package/dist/attack.js +8 -4
- package/dist/card.js +96 -51
- package/dist/check.js +24 -47
- package/dist/claims/types.js +3 -0
- package/dist/cli.js +30 -16
- package/dist/confirm.js +16 -7
- package/dist/findings.js +36 -0
- package/dist/frozen.js +16 -3
- package/dist/general-loop.js +8 -8
- package/dist/github.js +4 -2
- package/dist/kane/objective.js +28 -0
- package/dist/kane/parse.js +8 -8
- package/dist/kane/runner.js +2 -1
- package/dist/loop.js +12 -7
- package/dist/model/budget.js +13 -4
- package/dist/model/client.js +2 -1
- package/dist/model/config.js +1 -0
- package/dist/model/gemini.js +3 -3
- package/dist/report/report.js +2 -1
- package/dist/target.js +18 -3
- package/dist/util.js +6 -0
- package/package.json +4 -3
package/CHANGELOG.md
ADDED
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
## 0.1.1 (2026-11-21)
|
|
4
|
+
|
|
5
|
+
- `attack` checks `--url` and `--login` the same way `check` does, and reads findings files through the same guarded loader as `card` and `confirm`.
|
|
6
|
+
- The model call limit refuses to run when its usage totals file has the wrong shape, not only when it cannot be parsed.
|
|
7
|
+
- `confirm` uses the Kane run that actually showed the contradiction, including a reworded run, and refuses if it did not fail.
|
|
8
|
+
- A claim's assertion mode is kept on every Kane run, saved by `freeze`, and covered by the frozen-claim hash. A frozen file whose hash predates this asks to be re-frozen.
|
|
9
|
+
- Needs Node.js 22 or newer, as `@google/genai` 3.3.0 does.
|
|
10
|
+
|
|
11
|
+
## 0.1.0 (2026-11-20)
|
|
12
|
+
|
|
13
|
+
- First release on npm: `check`, `report`, `freeze`, `confirm`, `attack` and `card`.
|
package/README.md
CHANGED
|
@@ -9,7 +9,7 @@ Built for the Kane CLI Hackathon, with Claude Code.
|
|
|
9
9
|
- **[The fibcheck index](index/README.md):** 8 popular self-hosted web apps checked. 0 of 17 README claims were false. In their docs pages and wikis, 4 wrong or stale instructions were confirmed across 3 projects (Linkding 2, Mealie 1, Uptime Kuma 1), each with two failing Kane runs, a passing control and screenshots. For three of them, Kane also passed a fresh run that followed the corrected wording.
|
|
10
10
|
- **A fix merged upstream:** Mealie's maintainers merged our docs fix ([mealie-recipes/mealie#9212](https://github.com/mealie-recipes/mealie/pull/9212)). A docs PR to Linkding is open, and a note is on the Uptime Kuma issue about the same menu.
|
|
11
11
|
- **Live on pull requests:** in [this demo pull request](https://github.com/Ololadestephen/fibcheck/pull/1), the GitHub Action started Linkding in Docker on GitHub's free runner, Kane failed an outdated "Admin tab" step twice while the control passed, and the comment flagged the line for review ([run log and screenshots](https://github.com/Ololadestephen/fibcheck/actions/runs/44762871602)).
|
|
12
|
-
- **Share cards** for each finding are in [index/cards](index/cards),
|
|
12
|
+
- **Share cards** for each finding are in [index/cards](index/cards): a "docs polygraph" printout with a joke, the quoted line, Kane's own screenshot and the receipts. The jokes only restate recorded facts, and cards about other people's projects never call anyone a liar.
|
|
13
13
|
|
|
14
14
|
## See it
|
|
15
15
|
|
|
@@ -44,32 +44,32 @@ There are three ways to try fibcheck:
|
|
|
44
44
|
|
|
45
45
|
1. **With the judges' Gemini key.** The submission form includes a Gemini API key for judging. It is never in this repository. Put it in `.env` as `GEMINI_API_KEY=...` and follow the quick start. Kane CLI needs your own TestMu AI account (`kane-cli login`).
|
|
46
46
|
2. **With your own model.** Set `FIBCHECK_MODEL_BASE_URL`, `FIBCHECK_MODEL_API_KEY` and `FIBCHECK_MODEL` as shown in [Use your own model](#use-your-own-model), then follow the quick start.
|
|
47
|
-
3. **With no keys at all.**
|
|
47
|
+
3. **With no keys at all.** In a clone of this repository, run `npx fibcheck report examples/sample-check/check.json`. It rebuilds `report.html` and `badge.svg` from a real saved check, without calling Kane or any model: the Linkding check that found the outdated Admin tab step, with both failing Kane runs, the passing control and their 55 step screenshots. The [fibcheck index](index/README.md) links every other claim to its screenshots.
|
|
48
48
|
|
|
49
49
|
## Quick start
|
|
50
50
|
|
|
51
|
-
Use Node.js
|
|
51
|
+
fibcheck is on npm as [`fibcheck`](https://www.npmjs.com/package/fibcheck). Use Node.js 22 or newer with Git, Docker, Kane CLI, and a model for fibcheck's agent. Start Docker and put `GEMINI_API_KEY` in `.env` in the directory where you run fibcheck (use a Gemini project on the free tier), or choose another model as described in [Use your own model](#use-your-own-model).
|
|
52
52
|
|
|
53
53
|
```sh
|
|
54
54
|
npm install -g @testmuai/kane-cli
|
|
55
55
|
kane-cli login --oauth
|
|
56
|
-
|
|
57
|
-
npm run build
|
|
58
|
-
node dist/cli.js check sissbruecker/linkding --max-claims 1
|
|
56
|
+
npx fibcheck check sissbruecker/linkding --max-claims 1
|
|
59
57
|
```
|
|
60
58
|
|
|
61
59
|
The check prints the path to its `check.json`. Pass that path to `report` to write `report.html` and `badge.svg` beside it.
|
|
62
60
|
|
|
63
61
|
```sh
|
|
64
|
-
|
|
62
|
+
npx fibcheck report runs/<run-id>/check.json
|
|
65
63
|
```
|
|
66
64
|
|
|
65
|
+
To work on fibcheck itself, clone this repository and run `npm ci && npm run build`; `node dist/cli.js` then runs the same commands from your copy.
|
|
66
|
+
|
|
67
67
|
The Linkding GitHub clone URL also selects the bundled target. For another public web app, create a target JSON describing its repository, Docker image, port, and any login steps. Use `targets/linkding.json` as an example and pass your file to `check`. `--only` selects a claim by ID. fibcheck ranks the testable claims by how specific they are: named menus, tabs, buttons and settings, default values, shortcuts and step-by-step instructions come first, because those are the lines that go stale when an interface changes. The top `--max-claims` claims are tested (5 by default; 5 to 10 is a good range), and the rest are listed in the report as not tested, in ranked order, with the reasons for each score in `check.json`.
|
|
68
68
|
|
|
69
69
|
To check an app that is already running somewhere, with no Docker, pass its address with `--url`. Use `--login` to describe your own sign-in, writing `{{NAME}}` for each value, and put the values in `.env` as `FIBCHECK_LOGIN_NAME=...`. They are passed to Kane as secret variables and are never sent to Gemini.
|
|
70
70
|
|
|
71
71
|
```sh
|
|
72
|
-
|
|
72
|
+
npx fibcheck check sissbruecker/linkding --url https://bookmarks.example.org \
|
|
73
73
|
--login "If the login form is shown, sign in as {{user}} with password {{pass}}." --max-claims 2
|
|
74
74
|
```
|
|
75
75
|
|
|
@@ -77,7 +77,7 @@ Use `--max-kane-runs 12` to cap Kane invocations per check, including retries an
|
|
|
77
77
|
|
|
78
78
|
## Use it on your own project
|
|
79
79
|
|
|
80
|
-
fibcheck
|
|
80
|
+
Run fibcheck with `npx fibcheck <command>`, or install it with `npm install -g fibcheck` and run `fibcheck <command>`. Run it from the folder that holds your `.env`.
|
|
81
81
|
|
|
82
82
|
1. **Describe your app** in a target file. Point `repo` at your repository (a Git URL or a local folder) and list any docs beyond the README. Either give a Docker image to start, or leave `app` out and pass `--url` for an app that is already running:
|
|
83
83
|
|
|
@@ -96,10 +96,10 @@ fibcheck is not on npm yet, so run it from a clone: `npm ci && npm run build`, t
|
|
|
96
96
|
|
|
97
97
|
`{{secret:app_pass}}` makes fibcheck generate a fresh password for each run, pass it to the container, and give it to Kane as a secret variable. With `--url`, write `{{pass}}` in `--login` instead and put `FIBCHECK_LOGIN_PASS=...` in `.env`. [targets/linkding.json](targets/linkding.json) is a working example.
|
|
98
98
|
|
|
99
|
-
2. **Run a check:** `
|
|
99
|
+
2. **Run a check:** `npx fibcheck check my-app.json --max-claims 5`. fibcheck ranks the claims, Kane tests the top five, and `report` turns the saved `check.json` into an HTML report with every screenshot.
|
|
100
100
|
3. **Read the candidates.** A line that Kane failed twice while a control passed is a candidate finding. Open its screenshots and decide. fibcheck never calls a line false on its own.
|
|
101
|
-
4. **Confirm it, fix the docs, then attack the fix.** Once you have read the screenshots, record the finding with `
|
|
102
|
-
5. **Keep it checked on every pull request.** Freeze the checks with `
|
|
101
|
+
4. **Confirm it, fix the docs, then attack the fix.** Once you have read the screenshots, record the finding with `npx fibcheck confirm runs/<run>/check.json <claim-id> --verdict outdated --fix "<the corrected sentence>" --findings findings.json`. It refuses anything without two failed Kane runs and a passing control, and copies Kane's screenshots beside the findings file. Then run `npx fibcheck attack <finding-id> --url <app address> --findings findings.json`: the attacker must fail to make the old check pass with a shortcut before you send the fix, and `card` draws a share card from the same entry.
|
|
102
|
+
5. **Keep it checked on every pull request.** Freeze the checks with `npx fibcheck freeze docs/getting-started.md --out docs/fibcheck.claims.json`, review each one, and add the [GitHub Action](#check-docs-changes-in-github-actions) with that file as `claims-file`.
|
|
103
103
|
|
|
104
104
|
## Commands
|
|
105
105
|
|
|
@@ -112,7 +112,7 @@ fibcheck is not on npm yet, so run it from a clone: `npm ci && npm run build`, t
|
|
|
112
112
|
| `fibcheck attack <finding-id> --url ADDRESS` | Tries lazy fixes for a recorded finding against the running app |
|
|
113
113
|
| `fibcheck card <finding-id \| --all>` | Draws a share card for a recorded finding from its verdict and Kane screenshot |
|
|
114
114
|
|
|
115
|
-
|
|
115
|
+
Run them with `npx fibcheck <command>`, or as `fibcheck <command>` after `npm install -g fibcheck`.
|
|
116
116
|
|
|
117
117
|
## Use your own model
|
|
118
118
|
|
|
@@ -151,15 +151,6 @@ FIBCHECK_MODEL_MAX_USD=0.5
|
|
|
151
151
|
|
|
152
152
|
Without prices, only the call cap limits use, and the report says that the dollar cost was not tracked. If the model's reply is not usable JSON, that part of a document gives no claims, or the claim stays unconfirmed, and the report lists it. Unusable output never makes a claim false.
|
|
153
153
|
|
|
154
|
-
## Try the local npm package
|
|
155
|
-
|
|
156
|
-
```sh
|
|
157
|
-
npm pack
|
|
158
|
-
npx --package ./fibcheck-0.1.0.tgz -- fibcheck --help
|
|
159
|
-
```
|
|
160
|
-
|
|
161
|
-
The package is not on npm yet. Once it is published, `npx fibcheck check sissbruecker/linkding` will run the same command. The package includes the compiled CLI and the Linkding target. Run `fibcheck check sissbruecker/linkding` from the directory that contains your `.env` when using the installed command.
|
|
162
|
-
|
|
163
154
|
## Check docs changes in GitHub Actions
|
|
164
155
|
|
|
165
156
|
The [example workflow](examples/readme-check.yml) checks a docs file (the root `README.md` by default, or any Markdown file set with `docs-file`) in same-repository pull requests on public GitHub repositories. It runs on GitHub's free `ubuntu-latest` runner, which has Docker and Chrome. It reads the docs file at the PR head commit, as text, and the target JSON at the trusted base commit. The browser checks exercise the Docker image configured in that base target. [GitHub Actions security guidance](https://docs.github.com/en/actions/reference/security/securely-using-pull_request_target)
|
|
@@ -194,6 +185,10 @@ Only public material goes to the model: the documentation text, the claim, and a
|
|
|
194
185
|
|
|
195
186
|
Use `report` on saved checks to render evidence without calling any model or Kane again.
|
|
196
187
|
|
|
188
|
+
## Known limits
|
|
189
|
+
|
|
190
|
+
- Some code is still long and branchy (the check runner, the Kane output parser and the built-in Linkding flows), `src/` is one flat folder of 29 files with mixed file-name styles, one helper name is reused with different shapes, and the NEAR AI backup's address is written into the source instead of being configurable. Desloppify's subjective design reviews have not been run yet.
|
|
191
|
+
|
|
197
192
|
## Development checks
|
|
198
193
|
|
|
199
194
|
```sh
|
package/dist/app/docker.js
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { execFile } from "node:child_process";
|
|
2
2
|
import { promisify } from "node:util";
|
|
3
|
+
import { errorMessage } from "../util.js";
|
|
3
4
|
const exec = promisify(execFile);
|
|
4
5
|
/**
|
|
5
6
|
* Starts the target's Docker image with a fresh container, waits until it
|
|
@@ -51,5 +52,5 @@ export async function waitForHttp(url, timeoutMs, intervalMs = 2_000) {
|
|
|
51
52
|
}
|
|
52
53
|
function firstLine(error) {
|
|
53
54
|
const stderr = error.stderr;
|
|
54
|
-
return (stderr || (
|
|
55
|
+
return (stderr || (errorMessage(error))).split("\n").find((l) => l.trim()) ?? "unknown error";
|
|
55
56
|
}
|
package/dist/attack-run.js
CHANGED
|
@@ -1,21 +1,21 @@
|
|
|
1
|
-
import { mkdirSync,
|
|
1
|
+
import { mkdirSync, writeFileSync } from "node:fs";
|
|
2
2
|
import { dirname, join } from "node:path";
|
|
3
|
-
import { runAttack } from "./attack.js";
|
|
4
|
-
import { loadFindings } from "./card.js";
|
|
5
|
-
import { localAddressesToApp, writeKaneVariables } from "./check.js";
|
|
3
|
+
import { lazyFixTally, runAttack } from "./attack.js";
|
|
6
4
|
import { modelDiagnoser } from "./diagnose.js";
|
|
5
|
+
import { loadFindings, readFindingsFile, writeFindingsFile } from "./findings.js";
|
|
6
|
+
import { escapeKaneBraces, localAddressesToApp, writeKaneVariables } from "./kane/objective.js";
|
|
7
7
|
import { runKane } from "./kane/runner.js";
|
|
8
|
-
import { runningAppVariables } from "./target.js";
|
|
9
|
-
const safe = (text) => text.replace(/[{}]{2,}/g, (braces) => braces.split("").join(" "));
|
|
8
|
+
import { appBaseUrl, runningAppVariables } from "./target.js";
|
|
10
9
|
/** The Kane objective for one lazy fix: sign in, show the proposed docs line, then its check. */
|
|
11
10
|
export function attackObjective(login, attempt) {
|
|
12
|
-
return [login, `Proposed documentation says: "${
|
|
11
|
+
return [login, `Proposed documentation says: "${escapeKaneBraces(attempt.text)}".`, escapeKaneBraces(localAddressesToApp(attempt.check ?? ""))].filter(Boolean).join(" ");
|
|
13
12
|
}
|
|
14
13
|
/**
|
|
15
14
|
* Runs the lazy-fix attack for one recorded finding, saves every attempt to
|
|
16
15
|
* index/attacks/<id>.json, and records the rejected count on the finding.
|
|
17
16
|
*/
|
|
18
17
|
export async function attackFinding(client, options) {
|
|
18
|
+
appBaseUrl(options.appUrl);
|
|
19
19
|
const log = options.log ?? (() => { });
|
|
20
20
|
const finding = loadFindings(options.findingsPath).find(({ id }) => id === options.id);
|
|
21
21
|
if (!finding)
|
|
@@ -56,9 +56,8 @@ function saveAttack(findingsPath, finding, record) {
|
|
|
56
56
|
mkdirSync(dir, { recursive: true });
|
|
57
57
|
const portable = { ...record, results: record.results.map(({ kane, ...rest }) => (kane ? { ...rest, kane: { status: kane.status, oneLiner: kane.oneLiner } } : rest)) };
|
|
58
58
|
writeFileSync(join(dir, `${finding.id}.json`), `${JSON.stringify(portable, null, 2)}\n`);
|
|
59
|
-
const all =
|
|
60
|
-
const
|
|
61
|
-
const
|
|
62
|
-
|
|
63
|
-
writeFileSync(findingsPath, `${JSON.stringify(updated, null, 2)}\n`);
|
|
59
|
+
const all = readFindingsFile(findingsPath);
|
|
60
|
+
const lazyFixes = lazyFixTally(record.results);
|
|
61
|
+
const updated = all.map((entry) => (entry.id === finding.id ? { ...entry, lazyFixes } : entry));
|
|
62
|
+
writeFindingsFile(findingsPath, updated);
|
|
64
63
|
}
|
package/dist/attack.js
CHANGED
|
@@ -73,7 +73,7 @@ export function judgeByRule(attempt, input) {
|
|
|
73
73
|
if (!attempt.text.trim()) {
|
|
74
74
|
return { ...attempt, outcome: "rejected_by_rule", reason: "Deleting the line hides the problem: readers lose the instruction and nothing is left to check." };
|
|
75
75
|
}
|
|
76
|
-
if (
|
|
76
|
+
if (comparableText(attempt.text) === comparableText(input.fix)) {
|
|
77
77
|
return { ...attempt, outcome: "not_rejected", reason: "This is the proven fix itself, not a lazy one." };
|
|
78
78
|
}
|
|
79
79
|
if (HEDGES.test(attempt.text) && !HEDGES.test(input.fix)) {
|
|
@@ -130,14 +130,18 @@ export async function runAttack(client, input, runKaneFor, read) {
|
|
|
130
130
|
}
|
|
131
131
|
return results;
|
|
132
132
|
}
|
|
133
|
-
|
|
134
|
-
export function attackSummary(results) {
|
|
133
|
+
export function lazyFixTally(results) {
|
|
135
134
|
const rejected = results.filter(({ outcome }) => outcome === "rejected_by_rule" || outcome === "rejected_by_kane").length;
|
|
136
135
|
const byKane = results.filter(({ outcome }) => outcome === "rejected_by_kane").length;
|
|
136
|
+
return { rejected, byKane };
|
|
137
|
+
}
|
|
138
|
+
/** The share-card and report line: only counts rejections that happened. */
|
|
139
|
+
export function attackSummary(results) {
|
|
140
|
+
const { rejected, byKane } = lazyFixTally(results);
|
|
137
141
|
if (rejected === 0)
|
|
138
142
|
return "";
|
|
139
143
|
return `${rejected} lazy ${rejected === 1 ? "fix" : "fixes"} rejected${byKane ? ` (${byKane} by Kane)` : ""}.`;
|
|
140
144
|
}
|
|
141
|
-
function
|
|
145
|
+
function comparableText(text) {
|
|
142
146
|
return text.toLowerCase().replace(/[*_`"'.]/g, "").replace(/\s+/g, " ").trim();
|
|
143
147
|
}
|
package/dist/card.js
CHANGED
|
@@ -2,17 +2,18 @@ import { execFileSync } from "node:child_process";
|
|
|
2
2
|
import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
|
|
3
3
|
import { tmpdir } from "node:os";
|
|
4
4
|
import { dirname, extname, join, resolve } from "node:path";
|
|
5
|
+
import { loadFindings } from "./findings.js";
|
|
5
6
|
const OWN_STAMPS = {
|
|
6
|
-
false: "FIB",
|
|
7
|
+
false: "FIB DETECTED",
|
|
7
8
|
outdated: "OUTDATED",
|
|
8
|
-
true: "
|
|
9
|
-
worded_differently: "
|
|
9
|
+
true: "VERIFIED",
|
|
10
|
+
worded_differently: "CLOSE ENOUGH?",
|
|
10
11
|
unconfirmed: "CAN'T TELL",
|
|
11
12
|
};
|
|
12
|
-
/** Third-party projects get no "fib" or "lie": a false line is
|
|
13
|
+
/** Third-party projects get no "fib" or "lie": a false line is a mix-up between the docs and the app. */
|
|
13
14
|
const THIRD_PARTY_STAMPS = {
|
|
14
15
|
...OWN_STAMPS,
|
|
15
|
-
false: "
|
|
16
|
+
false: "MIX-UP",
|
|
16
17
|
};
|
|
17
18
|
/** The words on a card, built only from the recorded verdict and evidence. */
|
|
18
19
|
export function cardText(finding) {
|
|
@@ -21,33 +22,39 @@ export function cardText(finding) {
|
|
|
21
22
|
return {
|
|
22
23
|
stamp,
|
|
23
24
|
headline: headline(finding),
|
|
25
|
+
...(finding.punchline ? { punchline: finding.punchline } : {}),
|
|
24
26
|
quote: finding.quote,
|
|
25
27
|
source: `${finding.project} · ${file}, line ${line} · ${finding.source.commit.slice(0, 7)}`,
|
|
28
|
+
receipts: receipts(finding),
|
|
26
29
|
evidence: evidenceLine(finding),
|
|
27
30
|
score: scoreLine(finding),
|
|
28
31
|
...(finding.own ? {} : {
|
|
29
32
|
thanks: finding.upstream?.status === "merged"
|
|
30
|
-
? `
|
|
31
|
-
: `
|
|
33
|
+
? `Thanks, ${finding.project} maintainers, for merging the fix. The docs and the app are friends again.`
|
|
34
|
+
: `Thanks, ${finding.project} maintainers. Great project, one small docs update.`,
|
|
32
35
|
}),
|
|
33
36
|
};
|
|
34
37
|
}
|
|
35
38
|
function headline(finding) {
|
|
36
|
-
const
|
|
39
|
+
const visible = finding.visibleWording ?? "something else";
|
|
37
40
|
if (finding.verdict === "unconfirmed")
|
|
38
41
|
return "Can't tell. We refuse to guess.";
|
|
39
42
|
if (finding.own) {
|
|
40
|
-
if (
|
|
41
|
-
return "Your README fibbed.
|
|
43
|
+
if (finding.verdict === "false")
|
|
44
|
+
return "Your README fibbed. We have receipts.";
|
|
45
|
+
if (finding.verdict === "outdated")
|
|
46
|
+
return "Your README is living in the past.";
|
|
42
47
|
if (finding.verdict === "worded_differently")
|
|
43
|
-
return `Your README
|
|
44
|
-
return "Your README told the truth.
|
|
48
|
+
return `Your README calls it one thing. The app calls it ${visible}.`;
|
|
49
|
+
return "Your README told the truth. Suspicious, but verified.";
|
|
45
50
|
}
|
|
46
|
-
if (
|
|
47
|
-
return
|
|
51
|
+
if (finding.verdict === "outdated")
|
|
52
|
+
return "The app moved on. This docs line didn't get the memo.";
|
|
53
|
+
if (finding.verdict === "false")
|
|
54
|
+
return "The docs and the app tell different stories.";
|
|
48
55
|
if (finding.verdict === "worded_differently")
|
|
49
|
-
return `The app
|
|
50
|
-
return "
|
|
56
|
+
return `The app calls it ${visible}. The docs call it something else.`;
|
|
57
|
+
return "The docs said it. The app did it. Love to see it.";
|
|
51
58
|
}
|
|
52
59
|
/** Only the parts that really happened: two failures, a passing control, a proven fix, or a held behaviour. */
|
|
53
60
|
export function evidenceLine(finding) {
|
|
@@ -70,54 +77,90 @@ export function evidenceLine(finding) {
|
|
|
70
77
|
parts.push(`${lazy.rejected} lazy ${lazy.rejected === 1 ? "fix" : "fixes"} rejected${lazy.byKane ? ` (${lazy.byKane} by Kane)` : ""}.`);
|
|
71
78
|
return parts.join(" ");
|
|
72
79
|
}
|
|
80
|
+
/** The same evidence as `evidenceLine`, one plain-spoken line each, for the card's receipts. */
|
|
81
|
+
export function receipts(finding) {
|
|
82
|
+
const { failures, control, fixProof, held } = finding.evidence;
|
|
83
|
+
const lines = [];
|
|
84
|
+
if (failures.length >= 2)
|
|
85
|
+
lines.push(`Kane failed it in ${failures.length} separate runs`);
|
|
86
|
+
else if (failures.length === 1)
|
|
87
|
+
lines.push("Kane failed it once");
|
|
88
|
+
if (held)
|
|
89
|
+
lines.push("Kane watched it work");
|
|
90
|
+
if (control)
|
|
91
|
+
lines.push("Control run passed, so the test wasn't broken");
|
|
92
|
+
if (fixProof)
|
|
93
|
+
lines.push("The corrected wording passed Kane");
|
|
94
|
+
const lazy = finding.lazyFixes;
|
|
95
|
+
if (lazy && lazy.rejected > 0)
|
|
96
|
+
lines.push(`${lazy.rejected} lazy ${lazy.rejected === 1 ? "fix" : "fixes"} tried and rejected`);
|
|
97
|
+
if (finding.upstream?.status === "merged")
|
|
98
|
+
lines.push("The maintainers merged the fix");
|
|
99
|
+
return lines;
|
|
100
|
+
}
|
|
73
101
|
function scoreLine(finding) {
|
|
74
102
|
const { findings, claims } = finding.score;
|
|
75
103
|
const claimWord = claims === 1 ? "claim" : "claims";
|
|
76
104
|
if (finding.own)
|
|
77
105
|
return `${findings} ${findings === 1 ? "fib" : "fibs"} in ${claims} ${claimWord}`;
|
|
78
|
-
return `${findings}
|
|
79
|
-
}
|
|
80
|
-
/** Reads and checks the findings file; every referenced screenshot must exist. */
|
|
81
|
-
export function loadFindings(path) {
|
|
82
|
-
const findings = JSON.parse(readFileSync(path, "utf8"));
|
|
83
|
-
for (const finding of findings) {
|
|
84
|
-
for (const file of evidenceFiles(finding)) {
|
|
85
|
-
if (!existsSync(join(dirname(path), file)))
|
|
86
|
-
throw new Error(`Finding ${finding.id} refers to a missing file: ${file}`);
|
|
87
|
-
}
|
|
88
|
-
if ((finding.verdict === "false" || finding.verdict === "outdated") && (finding.evidence.failures.length < 2 || !finding.evidence.control)) {
|
|
89
|
-
throw new Error(`Finding ${finding.id} is ${finding.verdict} without two recorded failures and a control.`);
|
|
90
|
-
}
|
|
91
|
-
}
|
|
92
|
-
return findings;
|
|
93
|
-
}
|
|
94
|
-
export function evidenceFiles(finding) {
|
|
95
|
-
const { failures, control, fixProof, held } = finding.evidence;
|
|
96
|
-
return [finding.screenshot, ...failures, control, fixProof, held].filter((file) => Boolean(file));
|
|
106
|
+
return `${findings} out-of-step ${findings === 1 ? "line" : "lines"} in ${claims} ${claimWord} checked`;
|
|
97
107
|
}
|
|
98
108
|
const escapeHtml = (text) => text.replace(/[&<>"']/g, (c) => ({ "&": "&", "<": "<", ">": ">", '"': """, "'": "'" })[c] ?? c);
|
|
99
109
|
/**
|
|
100
|
-
* The card as HTML, 1200 by 675 pixels. The screenshot sits in its own frame and is
|
|
101
|
-
* (`object-fit: contain`): never cropped, and nothing is drawn over it.
|
|
110
|
+
* The card as HTML, 1200 by 675 pixels: a "docs polygraph" printout. The screenshot sits in its own frame and is
|
|
111
|
+
* only scaled to fit (`object-fit: contain`): never cropped, and nothing is drawn over it.
|
|
102
112
|
*/
|
|
103
113
|
export function renderCardHtml(finding, imageDataUrl) {
|
|
104
114
|
const text = cardText(finding);
|
|
105
|
-
const tone = finding.verdict === "true" || finding.verdict === "worded_differently" ? "#2f7d4f" : finding.verdict === "unconfirmed" ? "#6b6475" : "#
|
|
115
|
+
const tone = finding.verdict === "true" || finding.verdict === "worded_differently" ? "#2f7d4f" : finding.verdict === "unconfirmed" ? "#6b6475" : "#c2412d";
|
|
116
|
+
const calm = finding.verdict === "true" || finding.verdict === "worded_differently";
|
|
117
|
+
const needle = calm
|
|
118
|
+
? "M0 30 C 60 26, 120 34, 180 30 S 300 26, 360 30 S 480 34, 540 30"
|
|
119
|
+
: finding.verdict === "unconfirmed"
|
|
120
|
+
? "M0 30 C 60 30, 120 30, 180 30 L 200 22 L 215 38 L 230 30 C 300 30, 420 30, 540 30"
|
|
121
|
+
: "M0 30 C 40 28, 70 32, 100 30 L 130 6 L 150 54 L 170 2 L 190 58 L 210 10 L 230 48 L 250 30 C 300 32, 340 28, 380 30 L 400 14 L 415 46 L 430 30 C 470 29, 510 31, 540 30";
|
|
122
|
+
const reading = calm ? "needle: calm" : finding.verdict === "unconfirmed" ? "needle: shrugged" : "needle: went wild";
|
|
106
123
|
return `<!doctype html><html><head><meta charset="utf-8"><style>
|
|
107
|
-
*{box-sizing:border-box;margin:0}
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
.
|
|
111
|
-
.
|
|
112
|
-
.
|
|
124
|
+
*{box-sizing:border-box;margin:0}
|
|
125
|
+
body{width:1200px;height:675px;overflow:hidden;font-family:-apple-system,system-ui,"Segoe UI",sans-serif;color:#1f2a24;
|
|
126
|
+
background:#fbf7ec;background-image:linear-gradient(#efe6cf 1px,transparent 1px),linear-gradient(90deg,#efe6cf 1px,transparent 1px);background-size:24px 24px}
|
|
127
|
+
.top{height:64px;display:flex;align-items:center;gap:18px;padding:0 32px;border-bottom:3px solid #1f2a24;background:#fbf7ec}
|
|
128
|
+
.brand{font-weight:900;font-size:24px;letter-spacing:-.5px}.brand span{color:${tone}}
|
|
129
|
+
.sub{font:600 13px ui-monospace,Menlo,monospace;letter-spacing:2px;color:#6d6455;text-transform:uppercase}
|
|
130
|
+
.reading{margin-left:auto;color:${tone}}
|
|
131
|
+
.main{display:flex;height:calc(675px - 64px - 52px)}
|
|
132
|
+
.left{width:540px;padding:24px 28px 16px 32px;display:flex;flex-direction:column;gap:12px}
|
|
133
|
+
.stamp{align-self:flex-start;border:5px double ${tone};color:${tone};font-weight:900;font-size:30px;letter-spacing:1px;padding:4px 14px;border-radius:6px;transform:rotate(-4deg);background:#fffdf6}
|
|
134
|
+
.headline{font-size:30px;font-weight:850;line-height:1.12;letter-spacing:-.5px}
|
|
135
|
+
.punch{font-size:18px;line-height:1.35;color:#3c4a42}
|
|
136
|
+
.label{font:700 12px ui-monospace,Menlo,monospace;letter-spacing:2px;color:#6d6455;text-transform:uppercase;margin-top:4px}
|
|
137
|
+
.quote{font-size:17px;line-height:1.35;font-style:italic;background:#fffdf6;border:2px dashed #d7c9a6;border-radius:8px;padding:8px 12px}
|
|
138
|
+
.source{font-size:12px;color:#6d6455}
|
|
139
|
+
.right{flex:1;padding:22px 32px 16px 8px;display:flex;flex-direction:column;gap:10px}
|
|
140
|
+
.shot{flex:1;min-height:0;background:#1f2a24;border-radius:10px;padding:10px;display:flex;align-items:center;justify-content:center}
|
|
113
141
|
.shot img{max-width:100%;max-height:100%;object-fit:contain;display:block}
|
|
114
|
-
|
|
115
|
-
|
|
142
|
+
.receipts{display:flex;flex-wrap:wrap;gap:6px}
|
|
143
|
+
.receipt{font-size:13px;font-weight:600;background:#fffdf6;border:1.5px solid #1f2a24;border-radius:999px;padding:4px 10px}
|
|
144
|
+
.receipt:before{content:"✓ ";color:${tone};font-weight:900}
|
|
145
|
+
.foot{height:52px;display:flex;align-items:center;justify-content:space-between;gap:24px;padding:0 32px;border-top:3px solid #1f2a24;background:#1f2a24;color:#fbf7ec;font-size:14px}
|
|
146
|
+
.foot b{font-size:15px}
|
|
147
|
+
</style></head><body>
|
|
148
|
+
<div class="top"><div class="brand">fib<span>check</span></div><div class="sub">docs polygraph · printout</div>
|
|
149
|
+
<div class="sub reading">${reading}</div><svg class="needle" width="480" height="60" viewBox="0 0 540 60" preserveAspectRatio="none" aria-hidden="true"><path d="${needle}" fill="none" stroke="${tone}" stroke-width="3" stroke-linejoin="round" stroke-linecap="round"/></svg></div>
|
|
150
|
+
<div class="main"><div class="left">
|
|
151
|
+
<div class="stamp">${escapeHtml(text.stamp)}</div>
|
|
116
152
|
<div class="headline">${escapeHtml(text.headline)}</div>
|
|
117
|
-
|
|
118
|
-
<div class="
|
|
119
|
-
|
|
120
|
-
|
|
153
|
+
${text.punchline ? `<div class="punch">${escapeHtml(text.punchline)}</div>` : ""}
|
|
154
|
+
<div class="label">The docs said</div>
|
|
155
|
+
<div class="quote">“${escapeHtml(text.quote)}”</div>
|
|
156
|
+
<div class="source">${escapeHtml(text.source)}</div>
|
|
157
|
+
</div><div class="right">
|
|
158
|
+
<div class="label">What Kane saw</div>
|
|
159
|
+
<div class="shot"><img alt="Kane CLI screenshot, scaled only" src="${imageDataUrl}"></div>
|
|
160
|
+
<div class="receipts">${text.receipts.map((line) => `<span class="receipt">${escapeHtml(line)}</span>`).join("")}</div>
|
|
161
|
+
</div></div>
|
|
162
|
+
<div class="foot"><b>${escapeHtml(text.score)}</b><span>${escapeHtml(text.thanks ?? "Checked in a real browser by Kane CLI.")}</span></div>
|
|
163
|
+
</body></html>`;
|
|
121
164
|
}
|
|
122
165
|
/** Finds a Chrome or Chromium to draw the card; Kane CLI already needs one. */
|
|
123
166
|
export function findChrome(env = process.env) {
|
|
@@ -133,6 +176,8 @@ export function findChrome(env = process.env) {
|
|
|
133
176
|
throw new Error("Could not find Chrome or Chromium to draw the card. Set FIBCHECK_CHROME to its path.");
|
|
134
177
|
return found;
|
|
135
178
|
}
|
|
179
|
+
/** Cards are laid out at 1200 by 675 and drawn at twice that, 2400 by 1350 pixels, so they stay sharp on phone and retina screens. */
|
|
180
|
+
export const CARD_SCALE = 2;
|
|
136
181
|
/** Writes `<outDir>/<id>.png` for one finding and returns its path. */
|
|
137
182
|
export function writeCard(findingsPath, id, outDir, chrome = findChrome()) {
|
|
138
183
|
const finding = loadFindings(findingsPath).find((entry) => entry.id === id);
|
|
@@ -147,7 +192,7 @@ export function writeCard(findingsPath, id, outDir, chrome = findChrome()) {
|
|
|
147
192
|
writeFileSync(page, html);
|
|
148
193
|
mkdirSync(outDir, { recursive: true });
|
|
149
194
|
const out = resolve(outDir, `${id}.png`);
|
|
150
|
-
execFileSync(chrome, ["--headless=new", "--disable-gpu", "--hide-scrollbars",
|
|
195
|
+
execFileSync(chrome, ["--headless=new", "--disable-gpu", "--hide-scrollbars", `--force-device-scale-factor=${CARD_SCALE}`,
|
|
151
196
|
"--window-size=1200,675", `--screenshot=${out}`, `file://${page}`], { stdio: "ignore", timeout: 60_000 });
|
|
152
197
|
if (!existsSync(out))
|
|
153
198
|
throw new Error("Chrome did not write the card.");
|
package/dist/check.js
CHANGED
|
@@ -1,23 +1,26 @@
|
|
|
1
1
|
import { execFileSync } from "node:child_process";
|
|
2
|
-
import { existsSync, mkdirSync,
|
|
3
|
-
import { tmpdir } from "node:os";
|
|
2
|
+
import { existsSync, mkdirSync, writeFileSync } from "node:fs";
|
|
4
3
|
import { join, resolve } from "node:path";
|
|
5
4
|
import { startApp } from "./app/docker.js";
|
|
6
5
|
import { extractClaimsCached } from "./claims/cache.js";
|
|
7
6
|
import { collectDocuments } from "./claims/documents.js";
|
|
8
7
|
import { locateQuote } from "./claims/quotes.js";
|
|
9
8
|
import { rankClaims } from "./claims/rank.js";
|
|
9
|
+
import { isTestable } from "./claims/types.js";
|
|
10
|
+
import { escapeKaneBraces, localAddressesToApp, writeKaneVariables } from "./kane/objective.js";
|
|
10
11
|
import { runKane } from "./kane/runner.js";
|
|
11
12
|
import { seedLinkdingTags } from "./linkding-tags.js";
|
|
12
13
|
import { modelDiagnoser } from "./diagnose.js";
|
|
13
14
|
import { questionClaim } from "./loop.js";
|
|
15
|
+
import { newRunId } from "./util.js";
|
|
14
16
|
import { missingLabelAssertion } from "./frozen.js";
|
|
15
|
-
import { loadTarget, resolveTarget, runningAppVariables } from "./target.js";
|
|
17
|
+
import { appBaseUrl, loadTarget, resolveTarget, runningAppVariables } from "./target.js";
|
|
18
|
+
export const NEEDS_REVIEW_PREFIX = "Needs review:";
|
|
16
19
|
/** Reads a target's docs, extracts claims and runs the selected browser checks. */
|
|
17
20
|
export async function runCheck(client, options) {
|
|
18
21
|
validateOptions(options);
|
|
19
22
|
const log = options.log ?? (() => { });
|
|
20
|
-
const runId =
|
|
23
|
+
const runId = options.runId ?? newRunId();
|
|
21
24
|
const workDir = options.workDir ?? ".fibcheck";
|
|
22
25
|
const runDir = resolve(options.runsDir ?? "runs", runId);
|
|
23
26
|
mkdirSync(runDir, { recursive: true });
|
|
@@ -33,12 +36,12 @@ export async function runCheck(client, options) {
|
|
|
33
36
|
: await extractClaimsCached(client, documents, join(workDir, "cache", "claims"));
|
|
34
37
|
// Every claim that names a UI element must open with an exact-label assertion; a check without one is not run.
|
|
35
38
|
const unlabelled = extraction.claims.flatMap((claim) => {
|
|
36
|
-
const why = claim
|
|
39
|
+
const why = isTestable(claim) ? missingLabelAssertion(claim) : undefined;
|
|
37
40
|
return why ? [{ claim, why }] : [];
|
|
38
41
|
});
|
|
39
|
-
const testable = extraction.claims.filter((c) => c
|
|
42
|
+
const testable = extraction.claims.filter((c) => isTestable(c) && !unlabelled.some(({ claim }) => claim === c));
|
|
40
43
|
for (const { claim, why } of unlabelled) {
|
|
41
|
-
extraction.rejected.push({ quote: claim.quote, file: claim.source.file, why:
|
|
44
|
+
extraction.rejected.push({ quote: claim.quote, file: claim.source.file, why: `${NEEDS_REVIEW_PREFIX} line ${claim.source.startLine}. ${why}` });
|
|
42
45
|
}
|
|
43
46
|
log(`Found ${extraction.claims.length} claims (${testable.length} testable), ${extraction.rejected.length} rejected quotes, ${extraction.cacheHits} document(s) from cache.`);
|
|
44
47
|
if (extraction.unusable?.length)
|
|
@@ -83,7 +86,7 @@ export function verifyReviewedClaims(documents, claims, frozen = false) {
|
|
|
83
86
|
if (source)
|
|
84
87
|
verified.push({ ...claim, source });
|
|
85
88
|
else if (frozen && document)
|
|
86
|
-
rejected.push({ quote: claim.quote, file: claim.source.file, why:
|
|
89
|
+
rejected.push({ quote: claim.quote, file: claim.source.file, why: `${NEEDS_REVIEW_PREFIX} line ${claim.source.startLine} no longer says what was reviewed, so its frozen check was not run.` });
|
|
87
90
|
else
|
|
88
91
|
rejected.push({ quote: claim.quote, file: claim.source.file, why: document ? "The quote was not found in the document." : "The document was not read." });
|
|
89
92
|
}
|
|
@@ -93,15 +96,7 @@ export function verifyReviewedClaims(documents, claims, frozen = false) {
|
|
|
93
96
|
export function withAppUrl(target, appUrl) {
|
|
94
97
|
if (appUrl === undefined)
|
|
95
98
|
return target;
|
|
96
|
-
|
|
97
|
-
throw new Error("--url must be a full address such as https://bookmarks.example.org.");
|
|
98
|
-
const url = new URL(appUrl);
|
|
99
|
-
if (url.protocol !== "http:" && url.protocol !== "https:")
|
|
100
|
-
throw new Error("--url must start with http:// or https://.");
|
|
101
|
-
if (url.username || url.password)
|
|
102
|
-
throw new Error("--url must not contain a user name or password. Put login values in .env instead.");
|
|
103
|
-
const baseUrl = `${url.origin}${url.pathname.replace(/\/+$/, "")}`;
|
|
104
|
-
return { ...target, baseUrl };
|
|
99
|
+
return { ...target, baseUrl: appBaseUrl(appUrl) };
|
|
105
100
|
}
|
|
106
101
|
function validateOptions({ maxKaneRuns, appRunningSecrets, appRunning, appUrl }) {
|
|
107
102
|
if (maxKaneRuns !== undefined && (!Number.isSafeInteger(maxKaneRuns) || maxKaneRuns < 0)) {
|
|
@@ -114,6 +109,7 @@ function validateOptions({ maxKaneRuns, appRunningSecrets, appRunning, appUrl })
|
|
|
114
109
|
function totalKaneCredits(tested) {
|
|
115
110
|
return tested.reduce((sum, { kane, questioning }) => (sum + (kane.credits ?? 0) + (questioning?.followUps.reduce((cost, round) => cost + (round.kane.credits ?? 0), 0) ?? 0)), 0);
|
|
116
111
|
}
|
|
112
|
+
/** Starts the app, runs each claim and any follow-up checks, then cleans up. */
|
|
117
113
|
async function testClaims(target, claims, context) {
|
|
118
114
|
const { runDir, runId, options, log, client } = context;
|
|
119
115
|
if (claims.length === 0 || options.maxKaneRuns === 0)
|
|
@@ -159,58 +155,39 @@ async function runClaims(target, claims, session) {
|
|
|
159
155
|
}
|
|
160
156
|
log(`[${index + 1}/${claims.length}] ${claim.id}: "${claim.quote}"`);
|
|
161
157
|
const folder = safeName(claim.id);
|
|
162
|
-
const
|
|
158
|
+
const run = async (purpose, objective, assertionMode) => {
|
|
163
159
|
runsRemaining--;
|
|
164
160
|
const outDir = join(runDir, "claims", purpose === "initial" ? folder : `${folder}--${purpose}`);
|
|
165
161
|
const kane = await runKane({ objective, startUrl, outDir, assertionMode, variablesFile });
|
|
166
162
|
log(` Kane ${purpose}: ${kane.status} (${kane.credits?.toFixed(1) ?? "?"} credits) ${kane.oneLiner}`);
|
|
167
163
|
return kane;
|
|
168
|
-
}
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
164
|
+
};
|
|
165
|
+
const result = await questionClaim(claim, run, {
|
|
166
|
+
managedTarget, originalObjective: objectiveFor(claim, target), prepareTags, canRunFollowUp: canRun,
|
|
167
|
+
general: session.diagnose && {
|
|
168
|
+
diagnose: session.diagnose,
|
|
169
|
+
objectiveFor: (check) => objectiveFor({ ...claim, check }, target),
|
|
170
|
+
frozen: session.frozen,
|
|
171
|
+
},
|
|
172
172
|
});
|
|
173
173
|
log(` Claim verdict: ${result.questioning.verdict}. ${result.questioning.reason}`);
|
|
174
174
|
tested.push({ claim, ...result });
|
|
175
175
|
}
|
|
176
176
|
return tested;
|
|
177
177
|
}
|
|
178
|
-
export function writeKaneVariables(variables) {
|
|
179
|
-
if (Object.keys(variables).length === 0)
|
|
180
|
-
return undefined;
|
|
181
|
-
const directory = mkdtempSync(join(tmpdir(), "fibcheck-kane-"));
|
|
182
|
-
const path = join(directory, "variables.json");
|
|
183
|
-
const remove = () => rmSync(directory, { recursive: true, force: true });
|
|
184
|
-
try {
|
|
185
|
-
writeFileSync(path, JSON.stringify(variables), { flag: "wx", mode: 0o600 });
|
|
186
|
-
}
|
|
187
|
-
catch (error) {
|
|
188
|
-
remove();
|
|
189
|
-
throw error;
|
|
190
|
-
}
|
|
191
|
-
return { path, remove };
|
|
192
|
-
}
|
|
193
178
|
/**
|
|
194
179
|
* The flow Kane runs for one claim: the login sentence, the documented claim
|
|
195
180
|
* as context, and the check. `{{` in quoted text is broken up so Kane does not
|
|
196
181
|
* read it as a variable.
|
|
197
182
|
*/
|
|
198
183
|
export function objectiveFor(claim, target) {
|
|
199
|
-
const safe = (text) => text.replace(/[{}]{2,}/g, (braces) => braces.split("").join(" "));
|
|
200
184
|
const parts = [
|
|
201
185
|
target.login,
|
|
202
|
-
`The app's documentation (${claim.source.file}, line ${claim.source.startLine}) says: "${
|
|
203
|
-
|
|
186
|
+
`The app's documentation (${claim.source.file}, line ${claim.source.startLine}) says: "${escapeKaneBraces(claim.quote)}".`,
|
|
187
|
+
escapeKaneBraces(localAddressesToApp(claim.check ?? "")),
|
|
204
188
|
];
|
|
205
189
|
return parts.filter(Boolean).join(" ");
|
|
206
190
|
}
|
|
207
|
-
/**
|
|
208
|
-
* Documentation often names its own default address (for example http://localhost:5230). The test app runs
|
|
209
|
-
* elsewhere and Kane already starts on it, so a local address in a check is replaced by a reference to the app.
|
|
210
|
-
*/
|
|
211
|
-
export function localAddressesToApp(check) {
|
|
212
|
-
return check.replace(/\bhttps?:\/\/(?:localhost|127\.0\.0\.1|0\.0\.0\.0)(?::\d+)?(\/[^\s"'<>]*)?/gi, (_match, path) => (path && path !== "/" ? `the app's ${path.replace(/[.,;:]+$/, "")} page` : "the app's start page"));
|
|
213
|
-
}
|
|
214
191
|
function fetchRepo(repo, cloneDir) {
|
|
215
192
|
if (existsSync(repo))
|
|
216
193
|
return repo;
|
package/dist/claims/types.js
CHANGED