fibcheck 0.0.0-stage → 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +13 -0
- package/LICENSE +21 -0
- package/README.md +198 -2
- package/dist/app/docker.js +56 -0
- package/dist/attack-run.js +63 -0
- package/dist/attack.js +147 -0
- package/dist/card.js +204 -0
- package/dist/check.js +210 -0
- package/dist/claims/cache.js +45 -0
- package/dist/claims/documents.js +54 -0
- package/dist/claims/extract.js +144 -0
- package/dist/claims/quotes.js +52 -0
- package/dist/claims/rank.js +36 -0
- package/dist/claims/types.js +5 -0
- package/dist/cli.js +249 -0
- package/dist/confirm.js +92 -0
- package/dist/diagnose.js +150 -0
- package/dist/findings.js +36 -0
- package/dist/frozen.js +125 -0
- package/dist/general-loop.js +122 -0
- package/dist/github.js +216 -0
- package/dist/kane/evidence.js +66 -0
- package/dist/kane/objective.js +28 -0
- package/dist/kane/parse.js +105 -0
- package/dist/kane/runner.js +59 -0
- package/dist/linkding-tags.js +207 -0
- package/dist/loop-types.js +1 -0
- package/dist/loop.js +191 -0
- package/dist/model/budget.js +91 -0
- package/dist/model/client.js +60 -0
- package/dist/model/config.js +125 -0
- package/dist/model/gemini.js +100 -0
- package/dist/model/openai-compatible.js +146 -0
- package/dist/model/types.js +28 -0
- package/dist/report/report.js +298 -0
- package/dist/target.js +85 -0
- package/dist/util.js +6 -0
- package/package.json +47 -4
- package/targets/linkding.json +17 -0
package/dist/card.js
ADDED
|
@@ -0,0 +1,204 @@
|
|
|
1
|
+
import { execFileSync } from "node:child_process";
|
|
2
|
+
import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
|
|
3
|
+
import { tmpdir } from "node:os";
|
|
4
|
+
import { dirname, extname, join, resolve } from "node:path";
|
|
5
|
+
import { loadFindings } from "./findings.js";
|
|
6
|
+
const OWN_STAMPS = {
|
|
7
|
+
false: "FIB DETECTED",
|
|
8
|
+
outdated: "OUTDATED",
|
|
9
|
+
true: "VERIFIED",
|
|
10
|
+
worded_differently: "CLOSE ENOUGH?",
|
|
11
|
+
unconfirmed: "CAN'T TELL",
|
|
12
|
+
};
|
|
13
|
+
/** Third-party projects get no "fib" or "lie": a false line is a mix-up between the docs and the app. */
|
|
14
|
+
const THIRD_PARTY_STAMPS = {
|
|
15
|
+
...OWN_STAMPS,
|
|
16
|
+
false: "MIX-UP",
|
|
17
|
+
};
|
|
18
|
+
/** The words on a card, built only from the recorded verdict and evidence. */
|
|
19
|
+
export function cardText(finding) {
|
|
20
|
+
const stamp = (finding.own ? OWN_STAMPS : THIRD_PARTY_STAMPS)[finding.verdict];
|
|
21
|
+
const { file, line } = finding.source;
|
|
22
|
+
return {
|
|
23
|
+
stamp,
|
|
24
|
+
headline: headline(finding),
|
|
25
|
+
...(finding.punchline ? { punchline: finding.punchline } : {}),
|
|
26
|
+
quote: finding.quote,
|
|
27
|
+
source: `${finding.project} · ${file}, line ${line} · ${finding.source.commit.slice(0, 7)}`,
|
|
28
|
+
receipts: receipts(finding),
|
|
29
|
+
evidence: evidenceLine(finding),
|
|
30
|
+
score: scoreLine(finding),
|
|
31
|
+
...(finding.own ? {} : {
|
|
32
|
+
thanks: finding.upstream?.status === "merged"
|
|
33
|
+
? `Thanks, ${finding.project} maintainers, for merging the fix. The docs and the app are friends again.`
|
|
34
|
+
: `Thanks, ${finding.project} maintainers. Great project, one small docs update.`,
|
|
35
|
+
}),
|
|
36
|
+
};
|
|
37
|
+
}
|
|
38
|
+
function headline(finding) {
|
|
39
|
+
const visible = finding.visibleWording ?? "something else";
|
|
40
|
+
if (finding.verdict === "unconfirmed")
|
|
41
|
+
return "Can't tell. We refuse to guess.";
|
|
42
|
+
if (finding.own) {
|
|
43
|
+
if (finding.verdict === "false")
|
|
44
|
+
return "Your README fibbed. We have receipts.";
|
|
45
|
+
if (finding.verdict === "outdated")
|
|
46
|
+
return "Your README is living in the past.";
|
|
47
|
+
if (finding.verdict === "worded_differently")
|
|
48
|
+
return `Your README calls it one thing. The app calls it ${visible}.`;
|
|
49
|
+
return "Your README told the truth. Suspicious, but verified.";
|
|
50
|
+
}
|
|
51
|
+
if (finding.verdict === "outdated")
|
|
52
|
+
return "The app moved on. This docs line didn't get the memo.";
|
|
53
|
+
if (finding.verdict === "false")
|
|
54
|
+
return "The docs and the app tell different stories.";
|
|
55
|
+
if (finding.verdict === "worded_differently")
|
|
56
|
+
return `The app calls it ${visible}. The docs call it something else.`;
|
|
57
|
+
return "The docs said it. The app did it. Love to see it.";
|
|
58
|
+
}
|
|
59
|
+
/** Only the parts that really happened: two failures, a passing control, a proven fix, or a held behaviour. */
|
|
60
|
+
export function evidenceLine(finding) {
|
|
61
|
+
const { failures, control, fixProof, held } = finding.evidence;
|
|
62
|
+
const parts = [];
|
|
63
|
+
if (failures.length >= 2)
|
|
64
|
+
parts.push("Caught twice by Kane.");
|
|
65
|
+
else if (failures.length === 1)
|
|
66
|
+
parts.push("Caught once by Kane.");
|
|
67
|
+
if (held)
|
|
68
|
+
parts.push("Behaviour held in Kane's run.");
|
|
69
|
+
if (control)
|
|
70
|
+
parts.push("Control passed.");
|
|
71
|
+
if (fixProof)
|
|
72
|
+
parts.push("Fix proven ✓");
|
|
73
|
+
if (finding.upstream?.status === "merged")
|
|
74
|
+
parts.push("Fix merged by the maintainers ✓");
|
|
75
|
+
const lazy = finding.lazyFixes;
|
|
76
|
+
if (lazy && lazy.rejected > 0)
|
|
77
|
+
parts.push(`${lazy.rejected} lazy ${lazy.rejected === 1 ? "fix" : "fixes"} rejected${lazy.byKane ? ` (${lazy.byKane} by Kane)` : ""}.`);
|
|
78
|
+
return parts.join(" ");
|
|
79
|
+
}
|
|
80
|
+
/** The same evidence as `evidenceLine`, one plain-spoken line each, for the card's receipts. */
|
|
81
|
+
export function receipts(finding) {
|
|
82
|
+
const { failures, control, fixProof, held } = finding.evidence;
|
|
83
|
+
const lines = [];
|
|
84
|
+
if (failures.length >= 2)
|
|
85
|
+
lines.push(`Kane failed it in ${failures.length} separate runs`);
|
|
86
|
+
else if (failures.length === 1)
|
|
87
|
+
lines.push("Kane failed it once");
|
|
88
|
+
if (held)
|
|
89
|
+
lines.push("Kane watched it work");
|
|
90
|
+
if (control)
|
|
91
|
+
lines.push("Control run passed, so the test wasn't broken");
|
|
92
|
+
if (fixProof)
|
|
93
|
+
lines.push("The corrected wording passed Kane");
|
|
94
|
+
const lazy = finding.lazyFixes;
|
|
95
|
+
if (lazy && lazy.rejected > 0)
|
|
96
|
+
lines.push(`${lazy.rejected} lazy ${lazy.rejected === 1 ? "fix" : "fixes"} tried and rejected`);
|
|
97
|
+
if (finding.upstream?.status === "merged")
|
|
98
|
+
lines.push("The maintainers merged the fix");
|
|
99
|
+
return lines;
|
|
100
|
+
}
|
|
101
|
+
function scoreLine(finding) {
|
|
102
|
+
const { findings, claims } = finding.score;
|
|
103
|
+
const claimWord = claims === 1 ? "claim" : "claims";
|
|
104
|
+
if (finding.own)
|
|
105
|
+
return `${findings} ${findings === 1 ? "fib" : "fibs"} in ${claims} ${claimWord}`;
|
|
106
|
+
return `${findings} out-of-step ${findings === 1 ? "line" : "lines"} in ${claims} ${claimWord} checked`;
|
|
107
|
+
}
|
|
108
|
+
const escapeHtml = (text) => text.replace(/[&<>"']/g, (c) => ({ "&": "&", "<": "<", ">": ">", '"': """, "'": "'" })[c] ?? c);
|
|
109
|
+
/**
|
|
110
|
+
* The card as HTML, 1200 by 675 pixels: a "docs polygraph" printout. The screenshot sits in its own frame and is
|
|
111
|
+
* only scaled to fit (`object-fit: contain`): never cropped, and nothing is drawn over it.
|
|
112
|
+
*/
|
|
113
|
+
export function renderCardHtml(finding, imageDataUrl) {
|
|
114
|
+
const text = cardText(finding);
|
|
115
|
+
const tone = finding.verdict === "true" || finding.verdict === "worded_differently" ? "#2f7d4f" : finding.verdict === "unconfirmed" ? "#6b6475" : "#c2412d";
|
|
116
|
+
const calm = finding.verdict === "true" || finding.verdict === "worded_differently";
|
|
117
|
+
const needle = calm
|
|
118
|
+
? "M0 30 C 60 26, 120 34, 180 30 S 300 26, 360 30 S 480 34, 540 30"
|
|
119
|
+
: finding.verdict === "unconfirmed"
|
|
120
|
+
? "M0 30 C 60 30, 120 30, 180 30 L 200 22 L 215 38 L 230 30 C 300 30, 420 30, 540 30"
|
|
121
|
+
: "M0 30 C 40 28, 70 32, 100 30 L 130 6 L 150 54 L 170 2 L 190 58 L 210 10 L 230 48 L 250 30 C 300 32, 340 28, 380 30 L 400 14 L 415 46 L 430 30 C 470 29, 510 31, 540 30";
|
|
122
|
+
const reading = calm ? "needle: calm" : finding.verdict === "unconfirmed" ? "needle: shrugged" : "needle: went wild";
|
|
123
|
+
return `<!doctype html><html><head><meta charset="utf-8"><style>
|
|
124
|
+
*{box-sizing:border-box;margin:0}
|
|
125
|
+
body{width:1200px;height:675px;overflow:hidden;font-family:-apple-system,system-ui,"Segoe UI",sans-serif;color:#1f2a24;
|
|
126
|
+
background:#fbf7ec;background-image:linear-gradient(#efe6cf 1px,transparent 1px),linear-gradient(90deg,#efe6cf 1px,transparent 1px);background-size:24px 24px}
|
|
127
|
+
.top{height:64px;display:flex;align-items:center;gap:18px;padding:0 32px;border-bottom:3px solid #1f2a24;background:#fbf7ec}
|
|
128
|
+
.brand{font-weight:900;font-size:24px;letter-spacing:-.5px}.brand span{color:${tone}}
|
|
129
|
+
.sub{font:600 13px ui-monospace,Menlo,monospace;letter-spacing:2px;color:#6d6455;text-transform:uppercase}
|
|
130
|
+
.reading{margin-left:auto;color:${tone}}
|
|
131
|
+
.main{display:flex;height:calc(675px - 64px - 52px)}
|
|
132
|
+
.left{width:540px;padding:24px 28px 16px 32px;display:flex;flex-direction:column;gap:12px}
|
|
133
|
+
.stamp{align-self:flex-start;border:5px double ${tone};color:${tone};font-weight:900;font-size:30px;letter-spacing:1px;padding:4px 14px;border-radius:6px;transform:rotate(-4deg);background:#fffdf6}
|
|
134
|
+
.headline{font-size:30px;font-weight:850;line-height:1.12;letter-spacing:-.5px}
|
|
135
|
+
.punch{font-size:18px;line-height:1.35;color:#3c4a42}
|
|
136
|
+
.label{font:700 12px ui-monospace,Menlo,monospace;letter-spacing:2px;color:#6d6455;text-transform:uppercase;margin-top:4px}
|
|
137
|
+
.quote{font-size:17px;line-height:1.35;font-style:italic;background:#fffdf6;border:2px dashed #d7c9a6;border-radius:8px;padding:8px 12px}
|
|
138
|
+
.source{font-size:12px;color:#6d6455}
|
|
139
|
+
.right{flex:1;padding:22px 32px 16px 8px;display:flex;flex-direction:column;gap:10px}
|
|
140
|
+
.shot{flex:1;min-height:0;background:#1f2a24;border-radius:10px;padding:10px;display:flex;align-items:center;justify-content:center}
|
|
141
|
+
.shot img{max-width:100%;max-height:100%;object-fit:contain;display:block}
|
|
142
|
+
.receipts{display:flex;flex-wrap:wrap;gap:6px}
|
|
143
|
+
.receipt{font-size:13px;font-weight:600;background:#fffdf6;border:1.5px solid #1f2a24;border-radius:999px;padding:4px 10px}
|
|
144
|
+
.receipt:before{content:"✓ ";color:${tone};font-weight:900}
|
|
145
|
+
.foot{height:52px;display:flex;align-items:center;justify-content:space-between;gap:24px;padding:0 32px;border-top:3px solid #1f2a24;background:#1f2a24;color:#fbf7ec;font-size:14px}
|
|
146
|
+
.foot b{font-size:15px}
|
|
147
|
+
</style></head><body>
|
|
148
|
+
<div class="top"><div class="brand">fib<span>check</span></div><div class="sub">docs polygraph · printout</div>
|
|
149
|
+
<div class="sub reading">${reading}</div><svg class="needle" width="480" height="60" viewBox="0 0 540 60" preserveAspectRatio="none" aria-hidden="true"><path d="${needle}" fill="none" stroke="${tone}" stroke-width="3" stroke-linejoin="round" stroke-linecap="round"/></svg></div>
|
|
150
|
+
<div class="main"><div class="left">
|
|
151
|
+
<div class="stamp">${escapeHtml(text.stamp)}</div>
|
|
152
|
+
<div class="headline">${escapeHtml(text.headline)}</div>
|
|
153
|
+
${text.punchline ? `<div class="punch">${escapeHtml(text.punchline)}</div>` : ""}
|
|
154
|
+
<div class="label">The docs said</div>
|
|
155
|
+
<div class="quote">“${escapeHtml(text.quote)}”</div>
|
|
156
|
+
<div class="source">${escapeHtml(text.source)}</div>
|
|
157
|
+
</div><div class="right">
|
|
158
|
+
<div class="label">What Kane saw</div>
|
|
159
|
+
<div class="shot"><img alt="Kane CLI screenshot, scaled only" src="${imageDataUrl}"></div>
|
|
160
|
+
<div class="receipts">${text.receipts.map((line) => `<span class="receipt">${escapeHtml(line)}</span>`).join("")}</div>
|
|
161
|
+
</div></div>
|
|
162
|
+
<div class="foot"><b>${escapeHtml(text.score)}</b><span>${escapeHtml(text.thanks ?? "Checked in a real browser by Kane CLI.")}</span></div>
|
|
163
|
+
</body></html>`;
|
|
164
|
+
}
|
|
165
|
+
/** Finds a Chrome or Chromium to draw the card; Kane CLI already needs one. */
|
|
166
|
+
export function findChrome(env = process.env) {
|
|
167
|
+
const candidates = [
|
|
168
|
+
env.FIBCHECK_CHROME,
|
|
169
|
+
"/Applications/Google Chrome.app/Contents/MacOS/Google Chrome",
|
|
170
|
+
"/usr/bin/google-chrome",
|
|
171
|
+
"/usr/bin/chromium",
|
|
172
|
+
"/usr/bin/chromium-browser",
|
|
173
|
+
].filter((path) => Boolean(path));
|
|
174
|
+
const found = candidates.find((path) => existsSync(path));
|
|
175
|
+
if (!found)
|
|
176
|
+
throw new Error("Could not find Chrome or Chromium to draw the card. Set FIBCHECK_CHROME to its path.");
|
|
177
|
+
return found;
|
|
178
|
+
}
|
|
179
|
+
/** Cards are laid out at 1200 by 675 and drawn at twice that, 2400 by 1350 pixels, so they stay sharp on phone and retina screens. */
|
|
180
|
+
export const CARD_SCALE = 2;
|
|
181
|
+
/** Writes `<outDir>/<id>.png` for one finding and returns its path. */
|
|
182
|
+
export function writeCard(findingsPath, id, outDir, chrome = findChrome()) {
|
|
183
|
+
const finding = loadFindings(findingsPath).find((entry) => entry.id === id);
|
|
184
|
+
if (!finding)
|
|
185
|
+
throw new Error(`No finding called ${id} in ${findingsPath}.`);
|
|
186
|
+
const image = resolve(dirname(findingsPath), finding.screenshot);
|
|
187
|
+
const mime = extname(image).toLowerCase() === ".png" ? "image/png" : "image/jpeg";
|
|
188
|
+
const html = renderCardHtml(finding, `data:${mime};base64,${readFileSync(image).toString("base64")}`);
|
|
189
|
+
const work = mkdtempSync(join(tmpdir(), "fibcheck-card-"));
|
|
190
|
+
try {
|
|
191
|
+
const page = join(work, "card.html");
|
|
192
|
+
writeFileSync(page, html);
|
|
193
|
+
mkdirSync(outDir, { recursive: true });
|
|
194
|
+
const out = resolve(outDir, `${id}.png`);
|
|
195
|
+
execFileSync(chrome, ["--headless=new", "--disable-gpu", "--hide-scrollbars", `--force-device-scale-factor=${CARD_SCALE}`,
|
|
196
|
+
"--window-size=1200,675", `--screenshot=${out}`, `file://${page}`], { stdio: "ignore", timeout: 60_000 });
|
|
197
|
+
if (!existsSync(out))
|
|
198
|
+
throw new Error("Chrome did not write the card.");
|
|
199
|
+
return out;
|
|
200
|
+
}
|
|
201
|
+
finally {
|
|
202
|
+
rmSync(work, { recursive: true, force: true });
|
|
203
|
+
}
|
|
204
|
+
}
|
package/dist/check.js
ADDED
|
@@ -0,0 +1,210 @@
|
|
|
1
|
+
import { execFileSync } from "node:child_process";
|
|
2
|
+
import { existsSync, mkdirSync, writeFileSync } from "node:fs";
|
|
3
|
+
import { join, resolve } from "node:path";
|
|
4
|
+
import { startApp } from "./app/docker.js";
|
|
5
|
+
import { extractClaimsCached } from "./claims/cache.js";
|
|
6
|
+
import { collectDocuments } from "./claims/documents.js";
|
|
7
|
+
import { locateQuote } from "./claims/quotes.js";
|
|
8
|
+
import { rankClaims } from "./claims/rank.js";
|
|
9
|
+
import { isTestable } from "./claims/types.js";
|
|
10
|
+
import { escapeKaneBraces, localAddressesToApp, writeKaneVariables } from "./kane/objective.js";
|
|
11
|
+
import { runKane } from "./kane/runner.js";
|
|
12
|
+
import { seedLinkdingTags } from "./linkding-tags.js";
|
|
13
|
+
import { modelDiagnoser } from "./diagnose.js";
|
|
14
|
+
import { questionClaim } from "./loop.js";
|
|
15
|
+
import { newRunId } from "./util.js";
|
|
16
|
+
import { missingLabelAssertion } from "./frozen.js";
|
|
17
|
+
import { appBaseUrl, loadTarget, resolveTarget, runningAppVariables } from "./target.js";
|
|
18
|
+
export const NEEDS_REVIEW_PREFIX = "Needs review:";
|
|
19
|
+
/** Reads a target's docs, extracts claims and runs the selected browser checks. */
|
|
20
|
+
export async function runCheck(client, options) {
|
|
21
|
+
validateOptions(options);
|
|
22
|
+
const log = options.log ?? (() => { });
|
|
23
|
+
const runId = options.runId ?? newRunId();
|
|
24
|
+
const workDir = options.workDir ?? ".fibcheck";
|
|
25
|
+
const runDir = resolve(options.runsDir ?? "runs", runId);
|
|
26
|
+
mkdirSync(runDir, { recursive: true });
|
|
27
|
+
const loaded = loadTarget(options.targetPath);
|
|
28
|
+
const target = withAppUrl(resolveTarget(options.login === undefined ? loaded : { ...loaded, login: options.login }), options.appUrl);
|
|
29
|
+
const startedAt = new Date().toISOString();
|
|
30
|
+
const repoDir = options.source?.directory ?? fetchRepo(target.repo, join(workDir, "repos", target.name));
|
|
31
|
+
const commit = options.source?.commit ?? gitHead(repoDir);
|
|
32
|
+
const documents = collectDocuments(repoDir, target.docs ?? []);
|
|
33
|
+
log(`Read ${documents.length} document(s) from ${target.repo}${commit ? ` at ${commit.slice(0, 7)}` : ""}.`);
|
|
34
|
+
const extraction = options.reviewedClaims
|
|
35
|
+
? verifyReviewedClaims(documents, options.reviewedClaims, options.frozen)
|
|
36
|
+
: await extractClaimsCached(client, documents, join(workDir, "cache", "claims"));
|
|
37
|
+
// Every claim that names a UI element must open with an exact-label assertion; a check without one is not run.
|
|
38
|
+
const unlabelled = extraction.claims.flatMap((claim) => {
|
|
39
|
+
const why = isTestable(claim) ? missingLabelAssertion(claim) : undefined;
|
|
40
|
+
return why ? [{ claim, why }] : [];
|
|
41
|
+
});
|
|
42
|
+
const testable = extraction.claims.filter((c) => isTestable(c) && !unlabelled.some(({ claim }) => claim === c));
|
|
43
|
+
for (const { claim, why } of unlabelled) {
|
|
44
|
+
extraction.rejected.push({ quote: claim.quote, file: claim.source.file, why: `${NEEDS_REVIEW_PREFIX} line ${claim.source.startLine}. ${why}` });
|
|
45
|
+
}
|
|
46
|
+
log(`Found ${extraction.claims.length} claims (${testable.length} testable), ${extraction.rejected.length} rejected quotes, ${extraction.cacheHits} document(s) from cache.`);
|
|
47
|
+
if (extraction.unusable?.length)
|
|
48
|
+
log(`The model's reply could not be used for ${extraction.unusable.length} document part(s); their claims are missing.`);
|
|
49
|
+
// The most specific claims first: named controls, steps, settings and defaults are what go stale.
|
|
50
|
+
const ranked = rankClaims(testable);
|
|
51
|
+
const candidates = ranked.map(({ claim }) => claim);
|
|
52
|
+
const selected = (options.only?.length ? candidates.filter((c) => options.only?.includes(c.id)) : candidates).slice(0, options.maxClaims ?? 5);
|
|
53
|
+
const tested = await testClaims(target, selected, { runDir, runId, options, log, client });
|
|
54
|
+
const testedClaims = new Set(tested.map(({ claim }) => claim.id));
|
|
55
|
+
const notTested = candidates.filter((claim) => !testedClaims.has(claim.id)).map((claim) => claim.id);
|
|
56
|
+
const report = {
|
|
57
|
+
runId,
|
|
58
|
+
target: target.name,
|
|
59
|
+
repo: target.repo,
|
|
60
|
+
commit,
|
|
61
|
+
startedAt,
|
|
62
|
+
finishedAt: new Date().toISOString(),
|
|
63
|
+
documents: documents.map((d) => d.file),
|
|
64
|
+
claims: extraction.claims,
|
|
65
|
+
rejected: extraction.rejected,
|
|
66
|
+
...(extraction.unusable?.length ? { unusableModelOutput: extraction.unusable } : {}),
|
|
67
|
+
tested,
|
|
68
|
+
notTested,
|
|
69
|
+
ranking: ranked.map(({ claim, score, reasons }) => ({ id: claim.id, score, reasons })),
|
|
70
|
+
kaneCredits: totalKaneCredits(tested),
|
|
71
|
+
model: { ...client.usage(), cacheHits: extraction.cacheHits },
|
|
72
|
+
};
|
|
73
|
+
writeFileSync(join(runDir, "check.json"), `${JSON.stringify(report, null, 2)}\n`);
|
|
74
|
+
log(`Saved ${join(runDir, "check.json")}`);
|
|
75
|
+
return report;
|
|
76
|
+
}
|
|
77
|
+
/** The most Kane runs one claim can use: initial check, one rewording, a reproduction and a control. */
|
|
78
|
+
export const DEFAULT_RUNS_PER_CLAIM = 4;
|
|
79
|
+
/** Keeps supplied claims whose quotes appear in their documents, with corrected line numbers. */
|
|
80
|
+
export function verifyReviewedClaims(documents, claims, frozen = false) {
|
|
81
|
+
const verified = [];
|
|
82
|
+
const rejected = [];
|
|
83
|
+
for (const claim of claims) {
|
|
84
|
+
const document = documents.find(({ file }) => file === claim.source.file);
|
|
85
|
+
const source = document && locateQuote(document.content, claim.quote, claim.source);
|
|
86
|
+
if (source)
|
|
87
|
+
verified.push({ ...claim, source });
|
|
88
|
+
else if (frozen && document)
|
|
89
|
+
rejected.push({ quote: claim.quote, file: claim.source.file, why: `${NEEDS_REVIEW_PREFIX} line ${claim.source.startLine} no longer says what was reviewed, so its frozen check was not run.` });
|
|
90
|
+
else
|
|
91
|
+
rejected.push({ quote: claim.quote, file: claim.source.file, why: document ? "The quote was not found in the document." : "The document was not read." });
|
|
92
|
+
}
|
|
93
|
+
return { claims: verified, rejected, cacheHits: 0 };
|
|
94
|
+
}
|
|
95
|
+
/** Points a target at an app that is already running somewhere else. */
|
|
96
|
+
export function withAppUrl(target, appUrl) {
|
|
97
|
+
if (appUrl === undefined)
|
|
98
|
+
return target;
|
|
99
|
+
return { ...target, baseUrl: appBaseUrl(appUrl) };
|
|
100
|
+
}
|
|
101
|
+
function validateOptions({ maxKaneRuns, appRunningSecrets, appRunning, appUrl }) {
|
|
102
|
+
if (maxKaneRuns !== undefined && (!Number.isSafeInteger(maxKaneRuns) || maxKaneRuns < 0)) {
|
|
103
|
+
throw new Error("--max-kane-runs must be a whole number.");
|
|
104
|
+
}
|
|
105
|
+
if (appRunningSecrets !== undefined && !appRunning && !appUrl) {
|
|
106
|
+
throw new Error("appRunningSecrets requires appRunning.");
|
|
107
|
+
}
|
|
108
|
+
}
|
|
109
|
+
function totalKaneCredits(tested) {
|
|
110
|
+
return tested.reduce((sum, { kane, questioning }) => (sum + (kane.credits ?? 0) + (questioning?.followUps.reduce((cost, round) => cost + (round.kane.credits ?? 0), 0) ?? 0)), 0);
|
|
111
|
+
}
|
|
112
|
+
/** Starts the app, runs each claim and any follow-up checks, then cleans up. */
|
|
113
|
+
async function testClaims(target, claims, context) {
|
|
114
|
+
const { runDir, runId, options, log, client } = context;
|
|
115
|
+
if (claims.length === 0 || options.maxKaneRuns === 0)
|
|
116
|
+
return [];
|
|
117
|
+
let app;
|
|
118
|
+
const alreadyRunning = Boolean(options.appRunning || options.appUrl);
|
|
119
|
+
const variables = alreadyRunning ? runningAppVariables(target.login, options.appRunningSecrets ?? {}) : target.kaneVariables;
|
|
120
|
+
const variablesFile = writeKaneVariables(variables);
|
|
121
|
+
const managedTarget = alreadyRunning ? undefined : target;
|
|
122
|
+
try {
|
|
123
|
+
if (managedTarget) {
|
|
124
|
+
log(`Starting ${target.app?.image ?? "the app"} on port ${target.app?.port ?? "?"}...`);
|
|
125
|
+
app = await startApp(managedTarget, runId);
|
|
126
|
+
}
|
|
127
|
+
return await runClaims(target, claims, {
|
|
128
|
+
runDir, log, managedTarget, containerName: app?.containerName,
|
|
129
|
+
variablesFile: variablesFile?.path, maxKaneRuns: options.maxKaneRuns, frozen: options.frozen,
|
|
130
|
+
diagnose: options.diagnose === false ? undefined : modelDiagnoser(client, [...Object.values(variables).map(({ value }) => value), new URL(target.baseUrl).host]),
|
|
131
|
+
});
|
|
132
|
+
}
|
|
133
|
+
finally {
|
|
134
|
+
try {
|
|
135
|
+
variablesFile?.remove();
|
|
136
|
+
}
|
|
137
|
+
finally {
|
|
138
|
+
await app?.stop();
|
|
139
|
+
}
|
|
140
|
+
}
|
|
141
|
+
}
|
|
142
|
+
async function runClaims(target, claims, session) {
|
|
143
|
+
const { runDir, log, managedTarget, containerName, variablesFile } = session;
|
|
144
|
+
const tested = [];
|
|
145
|
+
const startUrl = `${target.baseUrl}${target.startPath ?? "/"}`;
|
|
146
|
+
// Without an explicit cap, each claim may use its full questioning budget of 4 Kane runs and no more.
|
|
147
|
+
let runsRemaining = session.maxKaneRuns ?? DEFAULT_RUNS_PER_CLAIM * claims.length;
|
|
148
|
+
const canRun = () => runsRemaining > 0;
|
|
149
|
+
let tagSetup;
|
|
150
|
+
const prepareTags = containerName ? () => (tagSetup ??= seedLinkdingTags(containerName)) : undefined;
|
|
151
|
+
for (const [index, claim] of claims.entries()) {
|
|
152
|
+
if (!canRun()) {
|
|
153
|
+
log("Kane run limit reached. Remaining claims were not tested.");
|
|
154
|
+
break;
|
|
155
|
+
}
|
|
156
|
+
log(`[${index + 1}/${claims.length}] ${claim.id}: "${claim.quote}"`);
|
|
157
|
+
const folder = safeName(claim.id);
|
|
158
|
+
const run = async (purpose, objective, assertionMode) => {
|
|
159
|
+
runsRemaining--;
|
|
160
|
+
const outDir = join(runDir, "claims", purpose === "initial" ? folder : `${folder}--${purpose}`);
|
|
161
|
+
const kane = await runKane({ objective, startUrl, outDir, assertionMode, variablesFile });
|
|
162
|
+
log(` Kane ${purpose}: ${kane.status} (${kane.credits?.toFixed(1) ?? "?"} credits) ${kane.oneLiner}`);
|
|
163
|
+
return kane;
|
|
164
|
+
};
|
|
165
|
+
const result = await questionClaim(claim, run, {
|
|
166
|
+
managedTarget, originalObjective: objectiveFor(claim, target), prepareTags, canRunFollowUp: canRun,
|
|
167
|
+
general: session.diagnose && {
|
|
168
|
+
diagnose: session.diagnose,
|
|
169
|
+
objectiveFor: (check) => objectiveFor({ ...claim, check }, target),
|
|
170
|
+
frozen: session.frozen,
|
|
171
|
+
},
|
|
172
|
+
});
|
|
173
|
+
log(` Claim verdict: ${result.questioning.verdict}. ${result.questioning.reason}`);
|
|
174
|
+
tested.push({ claim, ...result });
|
|
175
|
+
}
|
|
176
|
+
return tested;
|
|
177
|
+
}
|
|
178
|
+
/**
|
|
179
|
+
* The flow Kane runs for one claim: the login sentence, the documented claim
|
|
180
|
+
* as context, and the check. `{{` in quoted text is broken up so Kane does not
|
|
181
|
+
* read it as a variable.
|
|
182
|
+
*/
|
|
183
|
+
export function objectiveFor(claim, target) {
|
|
184
|
+
const parts = [
|
|
185
|
+
target.login,
|
|
186
|
+
`The app's documentation (${claim.source.file}, line ${claim.source.startLine}) says: "${escapeKaneBraces(claim.quote)}".`,
|
|
187
|
+
escapeKaneBraces(localAddressesToApp(claim.check ?? "")),
|
|
188
|
+
];
|
|
189
|
+
return parts.filter(Boolean).join(" ");
|
|
190
|
+
}
|
|
191
|
+
function fetchRepo(repo, cloneDir) {
|
|
192
|
+
if (existsSync(repo))
|
|
193
|
+
return repo;
|
|
194
|
+
if (!existsSync(join(cloneDir, ".git"))) {
|
|
195
|
+
mkdirSync(cloneDir, { recursive: true });
|
|
196
|
+
execFileSync("git", ["clone", "--depth", "1", "--quiet", repo, cloneDir], { stdio: ["ignore", "ignore", "pipe"] });
|
|
197
|
+
}
|
|
198
|
+
return cloneDir;
|
|
199
|
+
}
|
|
200
|
+
function gitHead(dir) {
|
|
201
|
+
try {
|
|
202
|
+
return execFileSync("git", ["-C", dir, "rev-parse", "HEAD"], { encoding: "utf8" }).trim();
|
|
203
|
+
}
|
|
204
|
+
catch {
|
|
205
|
+
return null;
|
|
206
|
+
}
|
|
207
|
+
}
|
|
208
|
+
function safeName(id) {
|
|
209
|
+
return id.replace(/[^a-zA-Z0-9_.-]+/g, "_");
|
|
210
|
+
}
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
import { createHash } from "node:crypto";
|
|
2
|
+
import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
|
|
3
|
+
import { join } from "node:path";
|
|
4
|
+
import { EXTRACT_SYSTEM, extractClaims } from "./extract.js";
|
|
5
|
+
/**
|
|
6
|
+
* Extracts claims, reusing an earlier result for an identical document and
|
|
7
|
+
* prompt. On the Gemini free tier (20 requests a day per model) this keeps
|
|
8
|
+
* repeated checks of the same README from spending requests.
|
|
9
|
+
*/
|
|
10
|
+
export async function extractClaimsCached(client, documents, cacheDir) {
|
|
11
|
+
const result = { claims: [], rejected: [], unusable: [], cacheHits: 0 };
|
|
12
|
+
mkdirSync(cacheDir, { recursive: true });
|
|
13
|
+
for (const document of documents) {
|
|
14
|
+
const path = join(cacheDir, `${cacheKey(document)}.json`);
|
|
15
|
+
let part = readCache(path);
|
|
16
|
+
if (part) {
|
|
17
|
+
result.cacheHits++;
|
|
18
|
+
}
|
|
19
|
+
else {
|
|
20
|
+
part = await extractClaims(client, [document]);
|
|
21
|
+
// A document with an unusable model reply is not cached, so the next check tries it again.
|
|
22
|
+
if (!part.unusable?.length)
|
|
23
|
+
writeFileSync(path, `${JSON.stringify(part, null, 2)}\n`);
|
|
24
|
+
}
|
|
25
|
+
result.claims.push(...part.claims);
|
|
26
|
+
result.rejected.push(...part.rejected);
|
|
27
|
+
result.unusable?.push(...(part.unusable ?? []));
|
|
28
|
+
}
|
|
29
|
+
return result;
|
|
30
|
+
}
|
|
31
|
+
/** A damaged cache file counts as a miss, so the document is simply extracted again. */
|
|
32
|
+
function readCache(path) {
|
|
33
|
+
if (!existsSync(path))
|
|
34
|
+
return undefined;
|
|
35
|
+
try {
|
|
36
|
+
return JSON.parse(readFileSync(path, "utf8"));
|
|
37
|
+
}
|
|
38
|
+
catch {
|
|
39
|
+
return undefined;
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
/** Changes when the document, its path or the extraction prompt changes. */
|
|
43
|
+
export function cacheKey(document) {
|
|
44
|
+
return createHash("sha256").update(EXTRACT_SYSTEM).update("\0").update(document.file).update("\0").update(document.content).digest("hex").slice(0, 32);
|
|
45
|
+
}
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
import { existsSync, readdirSync, readFileSync, statSync } from "node:fs";
|
|
2
|
+
import { dirname, join, normalize, relative, sep } from "node:path";
|
|
3
|
+
const README_NAMES = ["README.md", "readme.md", "Readme.md", "README.markdown", "README"];
|
|
4
|
+
/** At most this many linked documents are read, to keep model use bounded. */
|
|
5
|
+
export const MAX_LINKED_DOCS = 10;
|
|
6
|
+
/**
|
|
7
|
+
* Collects the documents to check in a local repository: the README, the
|
|
8
|
+
* Markdown files it links to inside the repository, and any extra files or
|
|
9
|
+
* folders the user names. Paths in the result are relative to the repository.
|
|
10
|
+
*/
|
|
11
|
+
export function collectDocuments(repoDir, extra = []) {
|
|
12
|
+
const readme = README_NAMES.map((name) => join(repoDir, name)).find((path) => existsSync(path));
|
|
13
|
+
if (!readme)
|
|
14
|
+
throw new Error(`No README found in ${repoDir}.`);
|
|
15
|
+
const paths = new Set([readme]);
|
|
16
|
+
for (const linked of linkedMarkdown(readme, repoDir).slice(0, MAX_LINKED_DOCS))
|
|
17
|
+
paths.add(linked);
|
|
18
|
+
for (const item of extra) {
|
|
19
|
+
for (const path of expand(join(repoDir, item)))
|
|
20
|
+
paths.add(path);
|
|
21
|
+
}
|
|
22
|
+
return [...paths].map((path) => ({ file: toPosix(relative(repoDir, path)), content: readFileSync(path, "utf8") }));
|
|
23
|
+
}
|
|
24
|
+
/** Relative links from a Markdown file to other Markdown files inside the repository. */
|
|
25
|
+
export function linkedMarkdown(file, repoDir) {
|
|
26
|
+
const text = readFileSync(file, "utf8");
|
|
27
|
+
const found = [];
|
|
28
|
+
for (const match of text.matchAll(/\]\(([^)\s]+)\)/g)) {
|
|
29
|
+
const target = (match[1] ?? "").split("#")[0] ?? "";
|
|
30
|
+
if (!target || /^[a-z]+:/i.test(target) || target.startsWith("/"))
|
|
31
|
+
continue;
|
|
32
|
+
if (!/\.(md|mdx|markdown)$/i.test(target))
|
|
33
|
+
continue;
|
|
34
|
+
const path = normalize(join(dirname(file), target));
|
|
35
|
+
if (!path.startsWith(normalize(repoDir) + sep))
|
|
36
|
+
continue;
|
|
37
|
+
if (existsSync(path) && !found.includes(path))
|
|
38
|
+
found.push(path);
|
|
39
|
+
}
|
|
40
|
+
return found;
|
|
41
|
+
}
|
|
42
|
+
function expand(path) {
|
|
43
|
+
if (!existsSync(path))
|
|
44
|
+
throw new Error(`Document not found: ${path}`);
|
|
45
|
+
if (!statSync(path).isDirectory())
|
|
46
|
+
return [path];
|
|
47
|
+
return readdirSync(path, { recursive: true, encoding: "utf8" })
|
|
48
|
+
.filter((name) => /\.(md|mdx|markdown)$/i.test(name))
|
|
49
|
+
.sort((a, b) => a.localeCompare(b))
|
|
50
|
+
.map((name) => join(path, name));
|
|
51
|
+
}
|
|
52
|
+
function toPosix(path) {
|
|
53
|
+
return path.split(sep).join("/");
|
|
54
|
+
}
|
|
@@ -0,0 +1,144 @@
|
|
|
1
|
+
import { UnusableModelOutputError } from "../model/types.js";
|
|
2
|
+
import { locateQuote, normalise, splitLines } from "./quotes.js";
|
|
3
|
+
import { CLAIM_TYPES } from "./types.js";
|
|
4
|
+
/** Documents longer than this are sent in parts, so no claim is lost to a long input. */
|
|
5
|
+
export const CHUNK_LINES = 250;
|
|
6
|
+
export const EXTRACT_SYSTEM = `You read the README and usage documentation of an open-source web app and list every claim that a browser test could check against the running app.
|
|
7
|
+
|
|
8
|
+
Claim types:
|
|
9
|
+
- setup: how to install or start the app and reach it (commands, URLs, ports, default logins).
|
|
10
|
+
- ui_flow: a path through the user interface ("go to Settings and click Import", a named page, menu, tab or button).
|
|
11
|
+
- behaviour: what the app does or offers when used (features, keyboard shortcuts, formats it imports or exports, what is shown or hidden, defaults).
|
|
12
|
+
- mobile: claims about phones, tablets or mobile apps.
|
|
13
|
+
- untestable: a concrete statement a browser test cannot check (performance, security, licence, hosting, third-party services, marketing adjectives such as "fast" or "clean"). Give the reason.
|
|
14
|
+
|
|
15
|
+
What counts as a claim (be exhaustive; a borderline claim is better listed than missed):
|
|
16
|
+
- Every item of a feature list, each as its own claim.
|
|
17
|
+
- Limitations and negative statements: what the app "only" allows, cannot do, does not support, or what is disabled.
|
|
18
|
+
- Defaults: what is shown, enabled or selected "by default", default ports, default users and passwords. If a sentence introduces credentials or a command shown in the next lines, quote that sentence.
|
|
19
|
+
- Where something is found in the UI, and what a page or section contains.
|
|
20
|
+
- One sentence can hold several claims. In "While A, B", "A, but B" or "A, so B", A and B are separate claims: list each, quoting the clause that states it. For example, "While the web app only lets you add labels, the admin panel can rename and delete them" gives two claims: "the web app only lets you add labels" (checked in the web app, by looking for rename and delete actions on an existing label) and "the admin panel can rename and delete them" (checked in the admin panel). Never return the whole sentence as one claim. Side remarks about the main UI in a section about something else (for example "the UI currently only allows X") are claims too.
|
|
21
|
+
Go through the document from top to bottom, section by section, and check every sentence and list item.
|
|
22
|
+
|
|
23
|
+
Rules:
|
|
24
|
+
- Quote each claim exactly as written, copied character for character from a single place in the document, without the line-number prefix. Keep each quote to one sentence or one list item.
|
|
25
|
+
- Give the first and last line number of the quote, using the numbers shown before each line.
|
|
26
|
+
- For testable claims, write "check" as a short sequence of browser actions and explicit assertions. When the documentation names a UI element by its label (a tab, menu, button, link or dropdown, such as "the *Admin* tab" or "the Save button"), the check's first assertion is an exact-label assertion for it, made before anything is clicked by that label and placed where the documentation says the element is, in the form "Open the Settings page. Assert exactly: The Settings page content shows a tab labelled 'Admin'. Expected: true." Put the label in single quotes and name the element type. When the documentation names a page, section, menu or tab, assert the exact location and control type before acting. Then say not to reach the result through any other menu, page or route. A control with the same name in a different place or of a different type cannot pass the check: an item in a dropdown or navigation menu is not a tab inside a page, and a page reached from a menu is not a view the documentation says to open first. Assert every stated, observable result separately.
|
|
27
|
+
- Test what the claim actually promises, not a step on the way. "Organize items with labels" needs filtering or grouping by a label, not only adding one. "Share with guests" needs the shared item opened as a guest, by logging out or opening its share link. "Import and export" needs both directions. "Edit" needs the change saved and seen again.
|
|
28
|
+
- Create the test data the check needs first, with unique names, instead of assuming it exists. If the feature may be switched off by default, say to switch it on in the app's settings first and name where.
|
|
29
|
+
- After each navigation, assert the destination by its page title or heading before going on. For saved, edited or deleted results, reload or reopen the page and assert the result again.
|
|
30
|
+
- Never write the app's address, host or port into a check. The test already starts on the running app, which may use a different port from the documentation. Refer to pages by their name instead. When a form needs a web address as data, such as a bookmark's URL, give https://example.com as the value to type.
|
|
31
|
+
- Write each assertion as a positive statement with an expected value, for example "The Tags page shows an Edit action for tag X. Expected: false". Avoid double negatives such as "assert that X is not absent".
|
|
32
|
+
- Split a navigation instruction and a claim about opening a new browser tab or window. Keep the supported navigation and its destination in a testable check. Record the separate tab or window claim as untestable because Kane cannot inspect browser tab or window state.
|
|
33
|
+
- For a limitation, first assert that the relevant page is loaded and the required test data is visible. Then check specifically for actions that would contradict the limitation. For example, if the ordinary interface only supports creating or assigning records, use an existing record and check separately for Edit and Remove there. If a separate administration interface can manage records, quote and test that clause independently in the administration interface.
|
|
34
|
+
- Skip headings, links-only lines, code that is not about using the app, contributor and development instructions, and changelog entries.
|
|
35
|
+
- Do not invent claims or merge separate claims. If there are no claims, return an empty list.`;
|
|
36
|
+
const RESPONSE_SCHEMA = {
|
|
37
|
+
type: "object",
|
|
38
|
+
properties: {
|
|
39
|
+
claims: {
|
|
40
|
+
type: "array",
|
|
41
|
+
items: {
|
|
42
|
+
type: "object",
|
|
43
|
+
properties: {
|
|
44
|
+
type: { type: "string", enum: [...CLAIM_TYPES] },
|
|
45
|
+
quote: { type: "string" },
|
|
46
|
+
startLine: { type: "integer" },
|
|
47
|
+
endLine: { type: "integer" },
|
|
48
|
+
check: { type: "string" },
|
|
49
|
+
reason: { type: "string" },
|
|
50
|
+
},
|
|
51
|
+
required: ["type", "quote", "startLine", "endLine"],
|
|
52
|
+
},
|
|
53
|
+
},
|
|
54
|
+
},
|
|
55
|
+
required: ["claims"],
|
|
56
|
+
};
|
|
57
|
+
/** Lists the claims in each document, keeping only claims whose quote is really in the document. */
|
|
58
|
+
export async function extractClaims(client, documents) {
|
|
59
|
+
const claims = [];
|
|
60
|
+
const rejected = [];
|
|
61
|
+
const unusable = [];
|
|
62
|
+
const seen = new Set();
|
|
63
|
+
for (const document of documents) {
|
|
64
|
+
for (const [index, chunk] of chunkDocument(document).entries()) {
|
|
65
|
+
let value;
|
|
66
|
+
try {
|
|
67
|
+
({ value } = await client.generateJson({
|
|
68
|
+
purpose: "extract-claims",
|
|
69
|
+
system: EXTRACT_SYSTEM,
|
|
70
|
+
prompt: `File: ${document.file}\n\n${chunk}`,
|
|
71
|
+
jsonSchema: RESPONSE_SCHEMA,
|
|
72
|
+
}, parseRawClaims));
|
|
73
|
+
}
|
|
74
|
+
catch (error) {
|
|
75
|
+
if (!(error instanceof UnusableModelOutputError))
|
|
76
|
+
throw error;
|
|
77
|
+
unusable.push({ file: document.file, lines: `${index * CHUNK_LINES + 1}-${(index + 1) * CHUNK_LINES}`, why: error.message });
|
|
78
|
+
continue;
|
|
79
|
+
}
|
|
80
|
+
for (const raw of value) {
|
|
81
|
+
const source = locateQuote(document.content, raw.quote, {
|
|
82
|
+
file: document.file,
|
|
83
|
+
startLine: raw.startLine,
|
|
84
|
+
endLine: raw.endLine,
|
|
85
|
+
});
|
|
86
|
+
if (!source) {
|
|
87
|
+
rejected.push({ quote: raw.quote, file: document.file, why: "quote not found in the document" });
|
|
88
|
+
continue;
|
|
89
|
+
}
|
|
90
|
+
const key = `${source.file}:${source.startLine}:${normalise(raw.quote)}`;
|
|
91
|
+
if (seen.has(key))
|
|
92
|
+
continue;
|
|
93
|
+
seen.add(key);
|
|
94
|
+
claims.push(toClaim(raw, source, claims));
|
|
95
|
+
}
|
|
96
|
+
}
|
|
97
|
+
}
|
|
98
|
+
claims.sort((a, b) => a.source.file.localeCompare(b.source.file) || a.source.startLine - b.source.startLine);
|
|
99
|
+
return unusable.length ? { claims, rejected, unusable } : { claims, rejected };
|
|
100
|
+
}
|
|
101
|
+
/** Splits a document into numbered parts of at most CHUNK_LINES lines, keeping absolute line numbers. */
|
|
102
|
+
export function chunkDocument(document) {
|
|
103
|
+
const lines = splitLines(document.content);
|
|
104
|
+
const chunks = [];
|
|
105
|
+
for (let start = 0; start < lines.length; start += CHUNK_LINES) {
|
|
106
|
+
chunks.push(lines
|
|
107
|
+
.slice(start, start + CHUNK_LINES)
|
|
108
|
+
.map((line, i) => `${String(start + i + 1).padStart(4, " ")}| ${line}`)
|
|
109
|
+
.join("\n"));
|
|
110
|
+
}
|
|
111
|
+
return chunks;
|
|
112
|
+
}
|
|
113
|
+
/** Checks the model's JSON shape; drops malformed items rather than trusting them. */
|
|
114
|
+
export function parseRawClaims(value) {
|
|
115
|
+
const list = value?.claims;
|
|
116
|
+
if (!Array.isArray(list))
|
|
117
|
+
throw new Error("The model reply has no claims list.");
|
|
118
|
+
return list.filter(isRawClaim);
|
|
119
|
+
}
|
|
120
|
+
function isRawClaim(item) {
|
|
121
|
+
if (typeof item !== "object" || item === null)
|
|
122
|
+
return false;
|
|
123
|
+
const c = item;
|
|
124
|
+
return (typeof c.type === "string" &&
|
|
125
|
+
CLAIM_TYPES.includes(c.type) &&
|
|
126
|
+
typeof c.quote === "string" &&
|
|
127
|
+
Number.isInteger(c.startLine) &&
|
|
128
|
+
Number.isInteger(c.endLine) &&
|
|
129
|
+
(c.check === undefined || typeof c.check === "string") &&
|
|
130
|
+
(c.reason === undefined || typeof c.reason === "string"));
|
|
131
|
+
}
|
|
132
|
+
function toClaim(raw, source, existing) {
|
|
133
|
+
const base = `${source.file}#L${source.startLine}`;
|
|
134
|
+
const clashes = existing.filter((c) => c.id === base || c.id.startsWith(`${base}-`)).length;
|
|
135
|
+
const claim = { id: clashes === 0 ? base : `${base}-${clashes + 1}`, type: raw.type, quote: raw.quote.trim(), source };
|
|
136
|
+
if (raw.type === "untestable") {
|
|
137
|
+
if (raw.reason)
|
|
138
|
+
claim.reason = raw.reason;
|
|
139
|
+
}
|
|
140
|
+
else if (raw.check) {
|
|
141
|
+
claim.check = raw.check;
|
|
142
|
+
}
|
|
143
|
+
return claim;
|
|
144
|
+
}
|