fibcheck 0.0.0-stage → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +203 -2
- package/dist/app/docker.js +55 -0
- package/dist/attack-run.js +64 -0
- package/dist/attack.js +143 -0
- package/dist/card.js +159 -0
- package/dist/check.js +233 -0
- package/dist/claims/cache.js +45 -0
- package/dist/claims/documents.js +54 -0
- package/dist/claims/extract.js +144 -0
- package/dist/claims/quotes.js +52 -0
- package/dist/claims/rank.js +36 -0
- package/dist/claims/types.js +2 -0
- package/dist/cli.js +235 -0
- package/dist/confirm.js +83 -0
- package/dist/diagnose.js +150 -0
- package/dist/frozen.js +112 -0
- package/dist/general-loop.js +122 -0
- package/dist/github.js +214 -0
- package/dist/kane/evidence.js +66 -0
- package/dist/kane/parse.js +105 -0
- package/dist/kane/runner.js +58 -0
- package/dist/linkding-tags.js +207 -0
- package/dist/loop-types.js +1 -0
- package/dist/loop.js +186 -0
- package/dist/model/budget.js +82 -0
- package/dist/model/client.js +59 -0
- package/dist/model/config.js +124 -0
- package/dist/model/gemini.js +100 -0
- package/dist/model/openai-compatible.js +146 -0
- package/dist/model/types.js +28 -0
- package/dist/report/report.js +297 -0
- package/dist/target.js +70 -0
- package/package.json +46 -4
- package/targets/linkding.json +17 -0
package/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Ololade Stephen
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
package/README.md
CHANGED
|
@@ -1,3 +1,204 @@
|
|
|
1
|
-
#
|
|
1
|
+
# fibcheck
|
|
2
2
|
|
|
3
|
-
|
|
3
|
+
fibcheck is a lie detector for READMEs and docs. It reads a web app's documentation, picks the claims a browser can test, and has [Kane CLI](https://www.npmjs.com/package/@testmuai/kane-cli) check each one in a real browser against the running app. Every result is kept with Kane's step screenshots, and nothing is called wrong until Kane has failed it twice, a control has passed, and a person has read the evidence.
|
|
4
|
+
|
|
5
|
+
Built for the Kane CLI Hackathon, with Claude Code.
|
|
6
|
+
|
|
7
|
+
## What it found
|
|
8
|
+
|
|
9
|
+
- **[The fibcheck index](index/README.md):** 8 popular self-hosted web apps checked. 0 of 17 README claims were false. In their docs pages and wikis, 4 wrong or stale instructions were confirmed across 3 projects (Linkding 2, Mealie 1, Uptime Kuma 1), each with two failing Kane runs, a passing control and screenshots. For three of them, Kane also passed a fresh run that followed the corrected wording.
|
|
10
|
+
- **A fix merged upstream:** Mealie's maintainers merged our docs fix ([mealie-recipes/mealie#9212](https://github.com/mealie-recipes/mealie/pull/9212)). A docs PR to Linkding is open, and a note is on the Uptime Kuma issue about the same menu.
|
|
11
|
+
- **Live on pull requests:** in [this demo pull request](https://github.com/Ololadestephen/fibcheck/pull/1), the GitHub Action started Linkding in Docker on GitHub's free runner, Kane failed an outdated "Admin tab" step twice while the control passed, and the comment flagged the line for review ([run log and screenshots](https://github.com/Ololadestephen/fibcheck/actions/runs/44762871602)).
|
|
12
|
+
- **Share cards** for each finding are in [index/cards](index/cards), drawn from Kane's own screenshots.
|
|
13
|
+
|
|
14
|
+
## See it
|
|
15
|
+
|
|
16
|
+
The comment fibcheck posted on the [demo pull request](https://github.com/Ololadestephen/fibcheck/pull/1), after a change put back Linkding's old "click the *Admin* tab" step (rendered from the comment's text):
|
|
17
|
+
|
|
18
|
+
<img src="docs/images/pr-comment.png" alt="fibcheck's pull request comment: line 5 unconfirmed, Kane failed it twice and the control passed, listed for a person's review" width="600">
|
|
19
|
+
|
|
20
|
+
Kane's own screenshot from that run, at the moment the check failed: the Settings page, with no menu open and no Admin tab in its content.
|
|
21
|
+
|
|
22
|
+
<img src="docs/images/kane-settings-no-admin-tab.jpg" alt="Kane screenshot of Linkding's Settings page with no Admin tab" width="600">
|
|
23
|
+
|
|
24
|
+
The HTML report for a saved check (`fibcheck report examples/sample-check/check.json`), and a share card drawn from a finding:
|
|
25
|
+
|
|
26
|
+
<img src="docs/images/report.png" alt="fibcheck HTML report for the Linkding check: one outdated claim, with its Kane runs" width="600">
|
|
27
|
+
|
|
28
|
+
<img src="index/cards/mealie-faq-disable-ingredient-amounts.png" alt="Share card for the Mealie FAQ finding: caught twice by Kane, control passed, fix proven, fix merged by the maintainers" width="600">
|
|
29
|
+
|
|
30
|
+
## How it works
|
|
31
|
+
|
|
32
|
+
1. **Read and rank.** fibcheck's agent reads the README and docs, quotes each claim a browser can test, and ranks the most specific first: named menus, tabs, buttons and settings, defaults, shortcuts and step-by-step instructions, because those go stale when an interface changes.
|
|
33
|
+
2. **Write the check.** For each claim it writes a Kane check: browser steps and explicit assertions. A claim that names a tab, menu, button, link or dropdown must start with an assertion of that exact label, in the place the docs name.
|
|
34
|
+
3. **Run Kane.** `kane-cli run` executes the check in a real browser. Kane alone decides whether the run passed or failed; fibcheck reads its NDJSON result, steps and evidence pack.
|
|
35
|
+
4. **Question the result.** The agent reads Kane's run and says whether it really tested the claim as written. A pass through a different menu does not count. A failure is reproduced in a separate run, and a control that should pass rules out a test mistake. Kane's own blame label is a hypothesis, overruled with evidence when the screenshots disagree.
|
|
36
|
+
5. **A person decides.** Two failures, a reason the test was not at fault, and a passing control make a candidate finding. A person reads the screenshots before anything is called false or outdated.
|
|
37
|
+
6. **Attack the fix.** Before a docs fix is suggested, an attacking agent tries the shortcuts a hurried editor might take, and each one must be rejected (see [below](#the-attacking-agent)).
|
|
38
|
+
|
|
39
|
+
fibcheck's agent runs on Google Gemini by default, or on any OpenAI-compatible model you choose. Kane CLI is the only judge of whether a browser run passed.
|
|
40
|
+
|
|
41
|
+
## For judges
|
|
42
|
+
|
|
43
|
+
There are three ways to try fibcheck:
|
|
44
|
+
|
|
45
|
+
1. **With the judges' Gemini key.** The submission form includes a Gemini API key for judging. It is never in this repository. Put it in `.env` as `GEMINI_API_KEY=...` and follow the quick start. Kane CLI needs your own TestMu AI account (`kane-cli login`).
|
|
46
|
+
2. **With your own model.** Set `FIBCHECK_MODEL_BASE_URL`, `FIBCHECK_MODEL_API_KEY` and `FIBCHECK_MODEL` as shown in [Use your own model](#use-your-own-model), then follow the quick start.
|
|
47
|
+
3. **With no keys at all.** After `npm ci && npm run build`, run `node dist/cli.js report examples/sample-check/check.json`. It rebuilds `report.html` and `badge.svg` from a real saved check, without calling Kane or any model: the Linkding check that found the outdated Admin tab step, with both failing Kane runs, the passing control and their 55 step screenshots. The [fibcheck index](index/README.md) links every other claim to its screenshots.
|
|
48
|
+
|
|
49
|
+
## Quick start
|
|
50
|
+
|
|
51
|
+
Use Node.js 20.12 or newer with Git, Docker, Kane CLI, and a model for fibcheck's agent. Start Docker and put `GEMINI_API_KEY` in `.env` in the directory where you run fibcheck (use a Gemini project on the free tier), or choose another model as described in [Use your own model](#use-your-own-model).
|
|
52
|
+
|
|
53
|
+
```sh
|
|
54
|
+
npm install -g @testmuai/kane-cli
|
|
55
|
+
kane-cli login --oauth
|
|
56
|
+
npm ci
|
|
57
|
+
npm run build
|
|
58
|
+
node dist/cli.js check sissbruecker/linkding --max-claims 1
|
|
59
|
+
```
|
|
60
|
+
|
|
61
|
+
The check prints the path to its `check.json`. Pass that path to `report` to write `report.html` and `badge.svg` beside it.
|
|
62
|
+
|
|
63
|
+
```sh
|
|
64
|
+
node dist/cli.js report runs/REPLACE_WITH_RUN_ID/check.json
|
|
65
|
+
```
|
|
66
|
+
|
|
67
|
+
The Linkding GitHub clone URL also selects the bundled target. For another public web app, create a target JSON describing its repository, Docker image, port, and any login steps. Use `targets/linkding.json` as an example and pass your file to `check`. `--only` selects a claim by ID. fibcheck ranks the testable claims by how specific they are: named menus, tabs, buttons and settings, default values, shortcuts and step-by-step instructions come first, because those are the lines that go stale when an interface changes. The top `--max-claims` claims are tested (5 by default; 5 to 10 is a good range), and the rest are listed in the report as not tested, in ranked order, with the reasons for each score in `check.json`.
|
|
68
|
+
|
|
69
|
+
To check an app that is already running somewhere, with no Docker, pass its address with `--url`. Use `--login` to describe your own sign-in, writing `{{NAME}}` for each value, and put the values in `.env` as `FIBCHECK_LOGIN_NAME=...`. They are passed to Kane as secret variables and are never sent to Gemini.
|
|
70
|
+
|
|
71
|
+
```sh
|
|
72
|
+
node dist/cli.js check sissbruecker/linkding --url https://bookmarks.example.org \
|
|
73
|
+
--login "If the login form is shown, sign in as {{user}} with password {{pass}}." --max-claims 2
|
|
74
|
+
```
|
|
75
|
+
|
|
76
|
+
Use `--max-kane-runs 12` to cap Kane invocations per check, including retries and controls. If the budget runs out before a required follow-up, the claim stays unconfirmed. The report lists claims that were never started. Use `--max-kane-runs 0` to extract claims without starting the app or Kane.
|
|
77
|
+
|
|
78
|
+
## Use it on your own project
|
|
79
|
+
|
|
80
|
+
fibcheck is not on npm yet, so run it from a clone: `npm ci && npm run build`, then `node dist/cli.js <command>`. Once it is published, the same commands run as `npx fibcheck <command>`.
|
|
81
|
+
|
|
82
|
+
1. **Describe your app** in a target file. Point `repo` at your repository (a Git URL or a local folder) and list any docs beyond the README. Either give a Docker image to start, or leave `app` out and pass `--url` for an app that is already running:
|
|
83
|
+
|
|
84
|
+
```json
|
|
85
|
+
{
|
|
86
|
+
"name": "my-app",
|
|
87
|
+
"repo": "https://github.com/me/my-app",
|
|
88
|
+
"docs": ["docs/getting-started.md"],
|
|
89
|
+
"app": {
|
|
90
|
+
"image": "me/my-app:1.4.0", "port": 8080, "containerPort": 80, "readyPath": "/",
|
|
91
|
+
"env": { "ADMIN_USER": "demo", "ADMIN_PASSWORD": "{{secret:app_pass}}" }
|
|
92
|
+
},
|
|
93
|
+
"login": "If a login form is shown, sign in as demo with password {{app_pass}}."
|
|
94
|
+
}
|
|
95
|
+
```
|
|
96
|
+
|
|
97
|
+
`{{secret:app_pass}}` makes fibcheck generate a fresh password for each run, pass it to the container, and give it to Kane as a secret variable. With `--url`, write `{{pass}}` in `--login` instead and put `FIBCHECK_LOGIN_PASS=...` in `.env`. [targets/linkding.json](targets/linkding.json) is a working example.
|
|
98
|
+
|
|
99
|
+
2. **Run a check:** `node dist/cli.js check my-app.json --max-claims 5`. fibcheck ranks the claims, Kane tests the top five, and `report` turns the saved `check.json` into an HTML report with every screenshot.
|
|
100
|
+
3. **Read the candidates.** A line that Kane failed twice while a control passed is a candidate finding. Open its screenshots and decide. fibcheck never calls a line false on its own.
|
|
101
|
+
4. **Confirm it, fix the docs, then attack the fix.** Once you have read the screenshots, record the finding with `node dist/cli.js confirm runs/<run>/check.json <claim-id> --verdict outdated --fix "<the corrected sentence>" --findings findings.json`. It refuses anything without two failed Kane runs and a passing control, and copies Kane's screenshots beside the findings file. Then run `node dist/cli.js attack <finding-id> --url <app address> --findings findings.json`: the attacker must fail to make the old check pass with a shortcut before you send the fix, and `card` draws a share card from the same entry.
|
|
102
|
+
5. **Keep it checked on every pull request.** Freeze the checks with `node dist/cli.js freeze docs/getting-started.md --out docs/fibcheck.claims.json`, review each one, and add the [GitHub Action](#check-docs-changes-in-github-actions) with that file as `claims-file`.
|
|
103
|
+
|
|
104
|
+
## Commands
|
|
105
|
+
|
|
106
|
+
| Command | What it does |
|
|
107
|
+
|---|---|
|
|
108
|
+
| `fibcheck check <target.json \| repository>` | Reads the docs, ranks the claims and runs the top ones with Kane. Options: `--max-claims`, `--max-kane-runs`, `--only`, `--url`, `--login` |
|
|
109
|
+
| `fibcheck report <check.json>` | Writes `report.html` and `badge.svg` from a saved check, without calling Kane or any model |
|
|
110
|
+
| `fibcheck freeze <docs.md> --out <claims.json>` | Writes the checks for a docs file once, for a person to review and commit (see [frozen checks](#check-docs-changes-in-github-actions)) |
|
|
111
|
+
| `fibcheck confirm <check.json> <claim-id> --verdict false\|outdated` | Records a candidate finding you have reviewed in a findings file, with Kane's screenshots, for `card` and `attack` |
|
|
112
|
+
| `fibcheck attack <finding-id> --url ADDRESS` | Tries lazy fixes for a recorded finding against the running app |
|
|
113
|
+
| `fibcheck card <finding-id \| --all>` | Draws a share card for a recorded finding from its verdict and Kane screenshot |
|
|
114
|
+
|
|
115
|
+
Before npm publishing, run them as `node dist/cli.js <command>` after `npm run build`.
|
|
116
|
+
|
|
117
|
+
## Use your own model
|
|
118
|
+
|
|
119
|
+
Gemini's free tier is the default. To use any OpenAI-compatible model instead, set three values in `.env`. fibcheck then uses that model for every step, and no Gemini key is needed.
|
|
120
|
+
|
|
121
|
+
```sh
|
|
122
|
+
# OpenAI
|
|
123
|
+
FIBCHECK_MODEL_BASE_URL=https://api.openai.com/v1
|
|
124
|
+
FIBCHECK_MODEL_API_KEY=sk-...
|
|
125
|
+
FIBCHECK_MODEL=gpt-4.1-mini
|
|
126
|
+
|
|
127
|
+
# NEAR AI Cloud
|
|
128
|
+
FIBCHECK_MODEL_BASE_URL=https://cloud-api.near.ai/v1
|
|
129
|
+
FIBCHECK_MODEL_API_KEY=...
|
|
130
|
+
FIBCHECK_MODEL=google/gemini-2.5-flash-lite
|
|
131
|
+
|
|
132
|
+
# OpenRouter
|
|
133
|
+
FIBCHECK_MODEL_BASE_URL=https://openrouter.ai/api/v1
|
|
134
|
+
FIBCHECK_MODEL_API_KEY=sk-or-...
|
|
135
|
+
FIBCHECK_MODEL=meta-llama/llama-3.3-70b-instruct
|
|
136
|
+
|
|
137
|
+
# A local Ollama model (no key needed on this machine)
|
|
138
|
+
FIBCHECK_MODEL_BASE_URL=http://localhost:11434/v1
|
|
139
|
+
FIBCHECK_MODEL=qwen3:8b
|
|
140
|
+
```
|
|
141
|
+
|
|
142
|
+
Use one block at a time. The model must support JSON output; a model that cannot follow the JSON format leaves claims unconfirmed rather than guessing.
|
|
143
|
+
|
|
144
|
+
The same call limits apply to every model (`FIBCHECK_MAX_CALLS_PER_RUN`, 60 by default, and `FIBCHECK_MAX_CALLS_TOTAL`, 1,500), and every call is recorded in `.fibcheck/usage-ledger.jsonl` without its prompt. If you know the model's prices, add them in dollars per million tokens, and fibcheck enforces a dollar cap across all runs (1 dollar unless `FIBCHECK_MODEL_MAX_USD` says otherwise):
|
|
145
|
+
|
|
146
|
+
```sh
|
|
147
|
+
FIBCHECK_MODEL_PRICE_INPUT=0.4
|
|
148
|
+
FIBCHECK_MODEL_PRICE_OUTPUT=1.6
|
|
149
|
+
FIBCHECK_MODEL_MAX_USD=0.5
|
|
150
|
+
```
|
|
151
|
+
|
|
152
|
+
Without prices, only the call cap limits use, and the report says that the dollar cost was not tracked. If the model's reply is not usable JSON, that part of a document gives no claims, or the claim stays unconfirmed, and the report lists it. Unusable output never makes a claim false.
|
|
153
|
+
|
|
154
|
+
## Try the local npm package
|
|
155
|
+
|
|
156
|
+
```sh
|
|
157
|
+
npm pack
|
|
158
|
+
npx --package ./fibcheck-0.1.0.tgz -- fibcheck --help
|
|
159
|
+
```
|
|
160
|
+
|
|
161
|
+
The package is not on npm yet. Once it is published, `npx fibcheck check sissbruecker/linkding` will run the same command. The package includes the compiled CLI and the Linkding target. Run `fibcheck check sissbruecker/linkding` from the directory that contains your `.env` when using the installed command.
|
|
162
|
+
|
|
163
|
+
## Check docs changes in GitHub Actions
|
|
164
|
+
|
|
165
|
+
The [example workflow](examples/readme-check.yml) checks a docs file (the root `README.md` by default, or any Markdown file set with `docs-file`) in same-repository pull requests on public GitHub repositories. It runs on GitHub's free `ubuntu-latest` runner, which has Docker and Chrome. It reads the docs file at the PR head commit, as text, and the target JSON at the trusted base commit. The browser checks exercise the Docker image configured in that base target. [GitHub Actions security guidance](https://docs.github.com/en/actions/reference/security/securely-using-pull_request_target)
|
|
166
|
+
|
|
167
|
+
1. Add repository secrets `GEMINI_API_KEY`, `LT_USERNAME` and `LT_ACCESS_KEY` (your TestMu AI username and access key), and repository variables `KANE_PROJECT_ID` and `KANE_FOLDER_ID` (from `kane-cli config show`). The Action installs Kane CLI, signs in with them, and writes the model key to a private `.env` outside the workspace.
|
|
168
|
+
2. Add `.github/fibcheck.target.json` to the base branch, using `targets/linkding.json` as a shape example. Set the image and port for the public app you want to check.
|
|
169
|
+
3. Copy the example into `.github/workflows/` in the consumer repository and pin the `uses:` line to the approved fibcheck commit. Set `FIBCHECK_PR_COMMENTS` to `true` to enable the separate comment job.
|
|
170
|
+
|
|
171
|
+
**Frozen checks (recommended).** The model writes a slightly different check each time it reads a sentence, so a pull request check can pass one day and not the next. To avoid that, generate the checks once with `fibcheck freeze <docs.md> --out <claims.json>`, read each one, fill in its `reviewed` field, and commit the file to the base branch. Pass it as `claims-file`: the Action then runs exactly those checks and never writes new ones in CI. If a pull request changes a line that has a frozen check, or a check was edited after review, the comment lists it under "Needs review" and Kane does not run it. Every check for a sentence that names a tab, menu, button, link or dropdown must start with an exact-label assertion (for example "the Settings page content shows a tab labelled 'Admin'"); fibcheck does not run a check without one.
|
|
172
|
+
|
|
173
|
+
This repository runs the same Action on its own [demo docs page](examples/linkding-demo/README.md) with [this workflow](.github/workflows/docs-check.yml): a pull request that changes the page starts Linkding in Docker, checks the page with Kane, and posts the result.
|
|
174
|
+
|
|
175
|
+
The comment lists counts, a row per checked line with its result and every Kane run, and the line numbers a person should review: a line Kane failed twice while a control passed, or a line whose behaviour held under different visible wording. It never calls a line false on its own. The job log prints every Kane run's steps and observations, with test passwords and local paths removed. Set `evidence-artifact: 'true'` to upload Kane's step screenshots as a workflow artifact for 7 days; on a public repository any signed-in GitHub user can download it. Otherwise the raw Kane output stays in the runner's temporary directory and is deleted with the runner. On a dedicated self-hosted runner, pass `env-file` (an absolute path to a private `.env` outside the workspace) with Kane CLI already signed in, instead of the key and sign-in inputs. [GitHub comment permissions](https://docs.github.com/en/rest/issues/comments#create-an-issue-comment)
|
|
176
|
+
|
|
177
|
+
## The attacking agent
|
|
178
|
+
|
|
179
|
+
A failing check can be made to pass without fixing anything. Before fibcheck suggests a docs fix, its attacker plays a hurried editor and proposes four shortcuts: delete the line, make it vague, hedge it ("usually at the top left"), or swap in a confident wrong detail. Each must be rejected. Deletions, vaguer lines and hedges are rejected by fixed rules: they have fewer checkable details than the proven fix. A wrong detail goes to Kane, and it only counts as rejected when Kane fails it in the exact place it names.
|
|
180
|
+
|
|
181
|
+
On the four index findings, all 12 delete, vague and hedge attempts were rejected. The 4 wrong-detail attempts stay "can't tell": Kane failed them, but none of those runs tested the wrong sentence in the place it names, so none is counted. Every attempt is recorded in [index/attacks](index/attacks), and the index explains [what we learned](index/README.md#lazy-fixes-we-tried-and-rejected).
|
|
182
|
+
|
|
183
|
+
The attacker ships with fibcheck as the `attack` command. It works on a finding a person has already confirmed (verdict false or outdated) with a proven fix recorded, against the running app at `--url` (add `--login` if it needs a sign-in). It uses two model calls, one to write the shortcuts and one to read Kane's run, and at most one Kane run, for the wrong-detail attempt. It saves every attempt next to the findings file in `attacks/<finding-id>.json` and records the rejected count on the finding.
|
|
184
|
+
|
|
185
|
+
## What the evidence means
|
|
186
|
+
|
|
187
|
+
fibcheck's agent (Gemini by default, or your chosen model) extracts claims from the selected documentation, so use public source material. Kane executes the browser checks and records step screenshots. fibcheck can then question a failure and run a control. A confirmed outdated verdict requires two distinct Kane failures of the documented assertion, a passing control that rules out a test mistake, and historical evidence of the old behavior. Claims with insufficient evidence stay unconfirmed.
|
|
188
|
+
|
|
189
|
+
The verified Linkding example checks an old instruction to click an Admin tab within Settings. Two independent runs found no in-page tab. A third run used the same account to find the Admin item in the top navigation's Settings dropdown and open Site administration. The control evidence overruled Kane's account diagnosis. After a documentation patch described the dropdown route, a fresh Kane run passed that route.
|
|
190
|
+
|
|
191
|
+
Linkding's Admin-tab and tag-management claims have hand-reviewed flows. Every other claim goes through the general diagnosis. After each Kane run, the model reads the claim and Kane's result, says whether the run tested the claim as written, and names the likely cause of a failure: a Kane misstep, a wrong check, a setup problem, something Kane cannot do, or a real contradiction. It can reword the check once, except for frozen checks, which run exactly as reviewed with their reviewed control. When two separate Kane failures both read as contradictions, with a written reason the test was not at fault and a passing control, the claim becomes a candidate finding. It stays unconfirmed until a person who has read the evidence decides; fibcheck never calls a claim false on its own. The general diagnosis never reports "outdated", because it does not check the project's history. Each claim uses at most 4 Kane runs. Without `--max-kane-runs`, a check is capped at 4 runs per tested claim. If the model is unavailable or its reply is unusable, the claim stays unconfirmed. Where the model disagrees with Kane's own blame label, the report shows the override and its evidence.
|
|
192
|
+
|
|
193
|
+
Only public material goes to the model: the documentation text, the claim, and a summary of Kane's result. Test passwords, local and private addresses, the address of the app under test, file paths and Kane session links are removed before sending. Public example addresses such as https://example.com are kept, because checks type them into forms. On Gemini's free tier, Google may use that content to improve its products. If `NEAR_AI_API_KEY` is set, the same public content goes to NEAR AI Cloud (model `google/gemini-2.5-flash-lite`) only when Gemini refuses or is overloaded, with a hard $0.20 spending cap.
|
|
194
|
+
|
|
195
|
+
Use `report` on saved checks to render evidence without calling any model or Kane again.
|
|
196
|
+
|
|
197
|
+
## Development checks
|
|
198
|
+
|
|
199
|
+
```sh
|
|
200
|
+
npm test
|
|
201
|
+
npm run typecheck
|
|
202
|
+
```
|
|
203
|
+
|
|
204
|
+
The tests run offline with no keys. The suite includes a readiness check on the loopback interface. The live Gemini extraction test runs only when `FIBCHECK_LIVE=1` is set. Code quality is also scanned with Desloppify.
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
import { execFile } from "node:child_process";
|
|
2
|
+
import { promisify } from "node:util";
|
|
3
|
+
const exec = promisify(execFile);
|
|
4
|
+
/**
|
|
5
|
+
* Starts the target's Docker image with a fresh container, waits until it
|
|
6
|
+
* answers, and returns a handle to stop it. Secrets go to Docker through
|
|
7
|
+
* the environment of the `docker` process, never on its command line.
|
|
8
|
+
*/
|
|
9
|
+
export async function startApp(target, runId, timeoutMs = 180_000) {
|
|
10
|
+
const spec = target.app;
|
|
11
|
+
if (!spec)
|
|
12
|
+
throw new Error(`Target ${target.name} has no app section, so it can only be checked with --url.`);
|
|
13
|
+
const containerName = `fibcheck-${target.name}-${runId}`.replace(/[^a-zA-Z0-9_.-]/g, "-");
|
|
14
|
+
const envArgs = Object.keys(target.resolvedEnv).flatMap((key) => ["-e", key]);
|
|
15
|
+
try {
|
|
16
|
+
await exec("docker", ["run", "-d", "--name", containerName, "-p", `${spec.port}:${spec.containerPort}`, ...envArgs, spec.image], { env: { ...process.env, ...target.resolvedEnv } });
|
|
17
|
+
}
|
|
18
|
+
catch (error) {
|
|
19
|
+
throw new Error(`Could not start ${spec.image} with Docker: ${firstLine(error)}`);
|
|
20
|
+
}
|
|
21
|
+
const app = {
|
|
22
|
+
containerName,
|
|
23
|
+
baseUrl: target.baseUrl,
|
|
24
|
+
stop: async () => {
|
|
25
|
+
await exec("docker", ["rm", "-f", containerName]).catch(() => undefined);
|
|
26
|
+
},
|
|
27
|
+
};
|
|
28
|
+
try {
|
|
29
|
+
await waitForHttp(`${target.baseUrl}${spec.readyPath ?? "/"}`, timeoutMs);
|
|
30
|
+
}
|
|
31
|
+
catch (error) {
|
|
32
|
+
await app.stop();
|
|
33
|
+
throw error;
|
|
34
|
+
}
|
|
35
|
+
return app;
|
|
36
|
+
}
|
|
37
|
+
export async function waitForHttp(url, timeoutMs, intervalMs = 2_000) {
|
|
38
|
+
const deadline = Date.now() + timeoutMs;
|
|
39
|
+
while (Date.now() < deadline) {
|
|
40
|
+
try {
|
|
41
|
+
const response = await fetch(url, { redirect: "follow" });
|
|
42
|
+
if (response.ok)
|
|
43
|
+
return;
|
|
44
|
+
}
|
|
45
|
+
catch {
|
|
46
|
+
// Not up yet.
|
|
47
|
+
}
|
|
48
|
+
await new Promise((resolve) => setTimeout(resolve, intervalMs));
|
|
49
|
+
}
|
|
50
|
+
throw new Error(`The app did not answer at ${url} within ${Math.round(timeoutMs / 1000)} seconds.`);
|
|
51
|
+
}
|
|
52
|
+
function firstLine(error) {
|
|
53
|
+
const stderr = error.stderr;
|
|
54
|
+
return (stderr || (error instanceof Error ? error.message : String(error))).split("\n").find((l) => l.trim()) ?? "unknown error";
|
|
55
|
+
}
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
import { mkdirSync, readFileSync, writeFileSync } from "node:fs";
|
|
2
|
+
import { dirname, join } from "node:path";
|
|
3
|
+
import { runAttack } from "./attack.js";
|
|
4
|
+
import { loadFindings } from "./card.js";
|
|
5
|
+
import { localAddressesToApp, writeKaneVariables } from "./check.js";
|
|
6
|
+
import { modelDiagnoser } from "./diagnose.js";
|
|
7
|
+
import { runKane } from "./kane/runner.js";
|
|
8
|
+
import { runningAppVariables } from "./target.js";
|
|
9
|
+
const safe = (text) => text.replace(/[{}]{2,}/g, (braces) => braces.split("").join(" "));
|
|
10
|
+
/** The Kane objective for one lazy fix: sign in, show the proposed docs line, then its check. */
|
|
11
|
+
export function attackObjective(login, attempt) {
|
|
12
|
+
return [login, `Proposed documentation says: "${safe(attempt.text)}".`, safe(localAddressesToApp(attempt.check ?? ""))].filter(Boolean).join(" ");
|
|
13
|
+
}
|
|
14
|
+
/**
|
|
15
|
+
* Runs the lazy-fix attack for one recorded finding, saves every attempt to
|
|
16
|
+
* index/attacks/<id>.json, and records the rejected count on the finding.
|
|
17
|
+
*/
|
|
18
|
+
export async function attackFinding(client, options) {
|
|
19
|
+
const log = options.log ?? (() => { });
|
|
20
|
+
const finding = loadFindings(options.findingsPath).find(({ id }) => id === options.id);
|
|
21
|
+
if (!finding)
|
|
22
|
+
throw new Error(`No finding called ${options.id} in ${options.findingsPath}.`);
|
|
23
|
+
if (finding.verdict !== "false" && finding.verdict !== "outdated")
|
|
24
|
+
throw new Error(`Finding ${finding.id} is ${finding.verdict}; only false or outdated findings get a lazy-fix attack.`);
|
|
25
|
+
if (!finding.fix)
|
|
26
|
+
throw new Error(`Finding ${finding.id} has no proven fix recorded.`);
|
|
27
|
+
const fix = finding.fix;
|
|
28
|
+
const variables = runningAppVariables(options.login, options.secrets ?? {});
|
|
29
|
+
const diagnose = modelDiagnoser(client, [...Object.values(variables).map(({ value }) => value), new URL(options.appUrl).host]);
|
|
30
|
+
const variablesFile = writeKaneVariables(variables);
|
|
31
|
+
try {
|
|
32
|
+
let index = 0;
|
|
33
|
+
const results = await runAttack(client, { original: finding.quote, fix, file: finding.source.file }, async (attempt) => {
|
|
34
|
+
index++;
|
|
35
|
+
log(` Kane checks the ${attempt.kind} attempt: "${attempt.text}"`);
|
|
36
|
+
return await runKane({
|
|
37
|
+
objective: attackObjective(options.login, attempt),
|
|
38
|
+
startUrl: options.appUrl,
|
|
39
|
+
outDir: join(options.runsDir, `${finding.id}-${attempt.kind}-${index}`),
|
|
40
|
+
variablesFile: variablesFile?.path,
|
|
41
|
+
});
|
|
42
|
+
}, async (attempt, kane) => {
|
|
43
|
+
const claim = { id: `${finding.id}-${attempt.kind}`, type: "ui_flow", quote: attempt.text, source: { file: finding.source.file, startLine: finding.source.line, endLine: finding.source.line } };
|
|
44
|
+
return await diagnose(claim, attempt.check ?? "", kane);
|
|
45
|
+
});
|
|
46
|
+
const record = { id: finding.id, at: new Date().toISOString(), original: finding.quote, fix, appUrl: options.appUrl, results };
|
|
47
|
+
saveAttack(options.findingsPath, finding, record);
|
|
48
|
+
return record;
|
|
49
|
+
}
|
|
50
|
+
finally {
|
|
51
|
+
variablesFile?.remove();
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
function saveAttack(findingsPath, finding, record) {
|
|
55
|
+
const dir = join(dirname(findingsPath), "attacks");
|
|
56
|
+
mkdirSync(dir, { recursive: true });
|
|
57
|
+
const portable = { ...record, results: record.results.map(({ kane, ...rest }) => (kane ? { ...rest, kane: { status: kane.status, oneLiner: kane.oneLiner } } : rest)) };
|
|
58
|
+
writeFileSync(join(dir, `${finding.id}.json`), `${JSON.stringify(portable, null, 2)}\n`);
|
|
59
|
+
const all = JSON.parse(readFileSync(findingsPath, "utf8"));
|
|
60
|
+
const rejected = record.results.filter(({ outcome }) => outcome === "rejected_by_rule" || outcome === "rejected_by_kane").length;
|
|
61
|
+
const byKane = record.results.filter(({ outcome }) => outcome === "rejected_by_kane").length;
|
|
62
|
+
const updated = all.map((entry) => (entry.id === finding.id ? { ...entry, lazyFixes: { rejected, byKane } } : entry));
|
|
63
|
+
writeFileSync(findingsPath, `${JSON.stringify(updated, null, 2)}\n`);
|
|
64
|
+
}
|
package/dist/attack.js
ADDED
|
@@ -0,0 +1,143 @@
|
|
|
1
|
+
import { rankClaim } from "./claims/rank.js";
|
|
2
|
+
/**
|
|
3
|
+
* T10, lazy-fix rejection. Before a docs fix is offered, an attacker proposes
|
|
4
|
+
* the shortcuts a hurried editor might take: delete the line, make it vague,
|
|
5
|
+
* hedge it, or swap in a plausible but wrong detail. Each must be rejected.
|
|
6
|
+
* Deletions, vaguer and hedged lines are rejected by fixed rules; a plausible
|
|
7
|
+
* wrong detail is only rejected when Kane fails it in the real app.
|
|
8
|
+
*/
|
|
9
|
+
export const LAZY_KINDS = ["delete", "vague", "hedge", "wrong"];
|
|
10
|
+
export const ATTACK_SYSTEM = `You play a hurried documentation editor who wants a failing docs check to stop failing without doing the real work. You get a documentation sentence that a browser test proved wrong, and the correct replacement that the test proved right.
|
|
11
|
+
Write three lazy replacements for the wrong sentence:
|
|
12
|
+
- vague: keeps the topic but drops the specific place, control or value, so there is nothing precise left to check.
|
|
13
|
+
- hedge: keeps the wrong detail but softens it with words such as "may", "usually" or "in some setups".
|
|
14
|
+
- wrong: a confident, specific and plausible replacement that names a different place, control or value from the correct one, the kind of guess an editor makes without opening the app. It must not match the correct replacement.
|
|
15
|
+
For the wrong one, also write "check": browser actions and explicit assertions that test exactly what the wrong sentence says. Begin with a separate assertion that the exact place and control it names exist (for example "Assert exactly: The header shows a menu at the bottom center. Expected: true."), and say not to use any other menu, page or route to reach the result. Do not tell the browser to open a web address; it starts on the app.
|
|
16
|
+
Reply as JSON.`;
|
|
17
|
+
const ATTACK_SCHEMA = {
|
|
18
|
+
type: "object",
|
|
19
|
+
properties: {
|
|
20
|
+
vague: { type: "string" },
|
|
21
|
+
hedge: { type: "string" },
|
|
22
|
+
wrong: { type: "string" },
|
|
23
|
+
check: { type: "string" },
|
|
24
|
+
},
|
|
25
|
+
required: ["vague", "hedge", "wrong", "check"],
|
|
26
|
+
};
|
|
27
|
+
/** Asks the model for the lazy fixes; a deletion is always added without the model. */
|
|
28
|
+
export async function proposeLazyFixes(client, input) {
|
|
29
|
+
const { value } = await client.generateJson({
|
|
30
|
+
purpose: "attack-lazy-fixes",
|
|
31
|
+
system: ATTACK_SYSTEM,
|
|
32
|
+
prompt: JSON.stringify({ file: input.file, wrong_sentence: input.original, correct_replacement: input.fix }, null, 2),
|
|
33
|
+
jsonSchema: ATTACK_SCHEMA,
|
|
34
|
+
}, parseLazyFixes);
|
|
35
|
+
return [{ kind: "delete", text: "" }, ...value];
|
|
36
|
+
}
|
|
37
|
+
export function parseLazyFixes(value) {
|
|
38
|
+
const record = value;
|
|
39
|
+
const text = (key) => {
|
|
40
|
+
const field = record?.[key];
|
|
41
|
+
if (typeof field !== "string" || !field.trim())
|
|
42
|
+
throw new Error(`The attack reply has no ${key}.`);
|
|
43
|
+
return field.trim();
|
|
44
|
+
};
|
|
45
|
+
return [
|
|
46
|
+
{ kind: "vague", text: text("vague") },
|
|
47
|
+
{ kind: "hedge", text: text("hedge") },
|
|
48
|
+
{ kind: "wrong", text: text("wrong"), check: text("check") },
|
|
49
|
+
];
|
|
50
|
+
}
|
|
51
|
+
const HEDGES = /\b(may|might|usually|typically|sometimes|in some (?:setups|cases|versions)|depending on|possibly|often|should be able to)\b/i;
|
|
52
|
+
/**
|
|
53
|
+
* The concrete details a reader can check: positions, numbers, quoted or emphasised names, and capitalised
|
|
54
|
+
* interface words after the first word.
|
|
55
|
+
*/
|
|
56
|
+
export function detailCount(text) {
|
|
57
|
+
const positions = text.match(/\b(top|bottom|left|right|upper|lower|first|last|sidebar|header|footer)\b/gi)?.length ?? 0;
|
|
58
|
+
const numbers = text.match(/\b\d+(\.\d+)?\b/g)?.length ?? 0;
|
|
59
|
+
const named = text.match(/`[^`]+`|\*[^*]+\*|"[^"]+"|“[^”]+”/g)?.length ?? 0;
|
|
60
|
+
const capitalised = text.split(/\s+/).slice(1).filter((word) => /^[A-Z][a-z]+/.test(word)).length;
|
|
61
|
+
return positions + numbers + named + capitalised;
|
|
62
|
+
}
|
|
63
|
+
/** How specific a sentence is, with the same scoring that ranks claims. */
|
|
64
|
+
export function specificity(text) {
|
|
65
|
+
const claim = { id: "x", type: "ui_flow", quote: text, source: { file: "x", startLine: 1, endLine: 1 } };
|
|
66
|
+
return rankClaim(claim).score;
|
|
67
|
+
}
|
|
68
|
+
/**
|
|
69
|
+
* Applies the fixed rules. Returns a result for attempts the rules reject, or
|
|
70
|
+
* undefined when only Kane can decide (a specific, unhedged replacement).
|
|
71
|
+
*/
|
|
72
|
+
export function judgeByRule(attempt, input) {
|
|
73
|
+
if (!attempt.text.trim()) {
|
|
74
|
+
return { ...attempt, outcome: "rejected_by_rule", reason: "Deleting the line hides the problem: readers lose the instruction and nothing is left to check." };
|
|
75
|
+
}
|
|
76
|
+
if (normalise(attempt.text) === normalise(input.fix)) {
|
|
77
|
+
return { ...attempt, outcome: "not_rejected", reason: "This is the proven fix itself, not a lazy one." };
|
|
78
|
+
}
|
|
79
|
+
if (HEDGES.test(attempt.text) && !HEDGES.test(input.fix)) {
|
|
80
|
+
return { ...attempt, outcome: "rejected_by_rule", reason: `Hedging ("${attempt.text.match(HEDGES)?.[0]}") softens a wrong detail instead of correcting it.` };
|
|
81
|
+
}
|
|
82
|
+
const attemptScore = specificity(attempt.text);
|
|
83
|
+
const fixScore = specificity(input.fix);
|
|
84
|
+
const attemptDetails = detailCount(attempt.text);
|
|
85
|
+
const fixDetails = detailCount(input.fix);
|
|
86
|
+
if (attempt.kind !== "wrong" && (attemptScore < fixScore || attemptDetails < fixDetails)) {
|
|
87
|
+
return {
|
|
88
|
+
...attempt, outcome: "rejected_by_rule",
|
|
89
|
+
reason: `Less specific than the proven fix (${attemptDetails} checkable details against ${fixDetails}; specificity ${attemptScore} against ${fixScore}), so it hides the detail instead of fixing it.`,
|
|
90
|
+
};
|
|
91
|
+
}
|
|
92
|
+
return undefined;
|
|
93
|
+
}
|
|
94
|
+
/**
|
|
95
|
+
* Turns Kane's run of a plausible wrong replacement into a judgement. Kane decides pass or fail; the
|
|
96
|
+
* model's reading only decides whether the run tested the attempt's own place and control. A failure for
|
|
97
|
+
* another reason (sign-in, the wrong page) or a pass through a different route is inconclusive, never a
|
|
98
|
+
* rejection and never an acceptance.
|
|
99
|
+
*/
|
|
100
|
+
export function judgeByKane(attempt, kane, reading) {
|
|
101
|
+
const record = { outDir: kane.outDir, status: kane.status, oneLiner: kane.oneLiner };
|
|
102
|
+
if (kane.status !== "failed" && kane.status !== "passed")
|
|
103
|
+
return { ...attempt, outcome: "inconclusive", reason: `Kane did not finish (${kane.status}).`, kane: record };
|
|
104
|
+
if (!reading)
|
|
105
|
+
return { ...attempt, outcome: "inconclusive", reason: "The run was not read, so it is not known whether Kane tested this attempt's own claim.", kane: record };
|
|
106
|
+
if (kane.status === "failed") {
|
|
107
|
+
if (reading.cause === "claim_contradicted" && reading.exercisedClaim) {
|
|
108
|
+
return { ...attempt, outcome: "rejected_by_kane", reason: `Kane failed it in the app: ${kane.oneLiner}`, kane: record };
|
|
109
|
+
}
|
|
110
|
+
return { ...attempt, outcome: "inconclusive", reason: `Kane failed, but not on this attempt's own claim: ${reading.reason}`, kane: record };
|
|
111
|
+
}
|
|
112
|
+
if (reading.exercisedClaim)
|
|
113
|
+
return { ...attempt, outcome: "not_rejected", reason: "Kane passed it in the place it names, so a person must review whether it is also correct.", kane: record };
|
|
114
|
+
return { ...attempt, outcome: "inconclusive", reason: `Kane passed without testing the place this attempt names: ${reading.reason}`, kane: record };
|
|
115
|
+
}
|
|
116
|
+
/** Runs the attack: rules first, then one Kane run, read by the model, for each attempt the rules cannot settle. */
|
|
117
|
+
export async function runAttack(client, input, runKaneFor, read) {
|
|
118
|
+
const results = [];
|
|
119
|
+
for (const attempt of await proposeLazyFixes(client, input)) {
|
|
120
|
+
const byRule = judgeByRule(attempt, input);
|
|
121
|
+
if (byRule)
|
|
122
|
+
results.push(byRule);
|
|
123
|
+
else if (!attempt.check)
|
|
124
|
+
results.push({ ...attempt, outcome: "inconclusive", reason: "No browser check was written for this attempt." });
|
|
125
|
+
else {
|
|
126
|
+
const kane = await runKaneFor(attempt);
|
|
127
|
+
const reading = read ? await read(attempt, kane).catch(() => undefined) : undefined;
|
|
128
|
+
results.push(judgeByKane(attempt, kane, reading));
|
|
129
|
+
}
|
|
130
|
+
}
|
|
131
|
+
return results;
|
|
132
|
+
}
|
|
133
|
+
/** The share-card and report line: only counts rejections that happened. */
|
|
134
|
+
export function attackSummary(results) {
|
|
135
|
+
const rejected = results.filter(({ outcome }) => outcome === "rejected_by_rule" || outcome === "rejected_by_kane").length;
|
|
136
|
+
const byKane = results.filter(({ outcome }) => outcome === "rejected_by_kane").length;
|
|
137
|
+
if (rejected === 0)
|
|
138
|
+
return "";
|
|
139
|
+
return `${rejected} lazy ${rejected === 1 ? "fix" : "fixes"} rejected${byKane ? ` (${byKane} by Kane)` : ""}.`;
|
|
140
|
+
}
|
|
141
|
+
function normalise(text) {
|
|
142
|
+
return text.toLowerCase().replace(/[*_`"'.]/g, "").replace(/\s+/g, " ").trim();
|
|
143
|
+
}
|