fibcheck 0.0.0-stage → 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,58 @@
1
+ import { spawn } from "node:child_process";
2
+ import { mkdirSync, writeFileSync } from "node:fs";
3
+ import { join } from "node:path";
4
+ import { extractStepEvidence, findEvidencePack } from "./evidence.js";
5
+ import { parseKaneOutput } from "./parse.js";
6
+ /** Runs one Kane flow in agent mode and returns its structured result. Only Kane decides pass or fail. */
7
+ export async function runKane(options) {
8
+ mkdirSync(options.outDir, { recursive: true });
9
+ const { stdout, stderr, exitCode } = await run(options.command ?? "kane-cli", kaneArgs(options));
10
+ writeFileSync(join(options.outDir, "kane.ndjson"), stdout);
11
+ writeFileSync(join(options.outDir, "kane.stderr"), stderr);
12
+ const result = parseKaneOutput(stdout, exitCode);
13
+ const { evidence, evidenceError } = collectEvidence(result, options.outDir);
14
+ const kaneRun = { ...result, objective: options.objective, startUrl: options.startUrl, outDir: options.outDir, evidence, evidenceError };
15
+ writeFileSync(join(options.outDir, "result.json"), `${JSON.stringify(kaneRun, null, 2)}\n`);
16
+ return kaneRun;
17
+ }
18
+ function kaneArgs(options) {
19
+ return [
20
+ ...(options.argsPrefix ?? []),
21
+ "run",
22
+ options.objective,
23
+ "--url",
24
+ options.startUrl,
25
+ "--agent",
26
+ "--headless",
27
+ "--max-steps",
28
+ String(options.maxSteps ?? 25),
29
+ "--timeout",
30
+ String(options.timeoutSec ?? 420),
31
+ ...(options.assertionMode ? ["--assertion-mode", options.assertionMode] : []),
32
+ ...(options.variablesFile ? ["--variables-file", options.variablesFile] : []),
33
+ ];
34
+ }
35
+ function collectEvidence(result, outDir) {
36
+ const pack = findEvidencePack(result.evidencePack, result.sessionDir);
37
+ if (!pack) {
38
+ const expected = result.status === "passed" || result.status === "failed";
39
+ return { evidence: [], evidenceError: expected ? "Kane's evidence pack was not found." : null };
40
+ }
41
+ try {
42
+ return { evidence: extractStepEvidence(pack, join(outDir, "evidence")), evidenceError: null };
43
+ }
44
+ catch (error) {
45
+ return { evidence: [], evidenceError: `Could not unpack Kane's evidence: ${error instanceof Error ? error.message : String(error)}` };
46
+ }
47
+ }
48
+ function run(command, args) {
49
+ return new Promise((resolve, reject) => {
50
+ const child = spawn(command, args, { stdio: ["ignore", "pipe", "pipe"] });
51
+ let stdout = "";
52
+ let stderr = "";
53
+ child.stdout.setEncoding("utf8").on("data", (chunk) => (stdout += chunk));
54
+ child.stderr.setEncoding("utf8").on("data", (chunk) => (stderr += chunk));
55
+ child.on("error", (error) => reject(new Error(`Could not start ${command}: ${error.message}. Install Kane CLI with: npm install -g @testmuai/kane-cli`)));
56
+ child.on("close", (code) => resolve({ stdout, stderr, exitCode: code ?? 1 }));
57
+ });
58
+ }
@@ -0,0 +1,207 @@
1
+ import { execFile } from "node:child_process";
2
+ import { promisify } from "node:util";
3
+ import { normalise } from "./claims/quotes.js";
4
+ import { distinctKaneRuns } from "./kane/evidence.js";
5
+ import { KANE_RUN_LIMIT_REASON } from "./loop-types.js";
6
+ const MAIN_TAG = "fibchecktag";
7
+ const ADMIN_TAG = "fibcheckadmintag";
8
+ const ADMIN_DOC = "docs/src/content/docs/admin.md";
9
+ const MAIN_LIMIT = "linkding UI currently only allows to create or assign tags";
10
+ const ADMIN_CLAIM = "you can use the admin app to manage your tags";
11
+ const ROW_PRESENT = "The ordinary Linkding Tags page lists fibchecktag.";
12
+ const EDIT_OFFERED = "The fibchecktag row has an Edit control on the ordinary Linkding Tags page.";
13
+ const MAIN_EDIT_NAME = "The Edit Tag Name field is fibchecktag.";
14
+ const MAIN_RENAMED = "The ordinary Linkding Tags page lists fibchecktagmain.";
15
+ const ADMIN_RENAMED = "The Site administration Tags list contains fibcheckadmintagproof.";
16
+ const MAIN_CONTROL_PROOF = "Kane renamed the seeded tag from the ordinary Tags page and verified the new name in the table.";
17
+ const TAG_HISTORY = "Linkding added its ordinary tag-management page in commit 82e5b7d.";
18
+ const CHECK_MAIN = `Navigate directly to /tags in the ordinary Linkding app after login. Wait for the Tags table. Assert exactly: ${ROW_PRESENT} Expect true. Look in the Actions column for that specific row. Assert exactly: ${EDIT_OFFERED} Expect false. Do not change the tag or visit the administration app.`;
19
+ const RETRY_MAIN = `After login, open the normal Linkding Tags page at /tags and wait for its table. First make this exact boolean assertion: ${ROW_PRESENT} Expect true. Now inspect only that tag's row and read the actions shown next to its name. Make this exact separate boolean assertion: ${EDIT_OFFERED} Expect false. Leave the tag unchanged.`;
20
+ const CONTROL_MAIN = `After login, open /tags in the ordinary Linkding app. Find fibchecktag in the Tags table and click Edit in that row. Before typing, assert this exact boolean condition: ${MAIN_EDIT_NAME} Expect true. Change the Name to fibchecktagmain and click Save in the Edit Tag form. Verify this exact condition: ${MAIN_RENAMED} Expect true. Do not use the administration app.`;
21
+ const CHECK_ADMIN = `After login, open the Settings dropdown in the top navigation, then choose Admin. On Site administration, click the Tags link in the Bookmarks section. In the admin tag list open the existing fibcheckadmintag record, change its Name to fibcheckadmintagproof, and click Save. Verify this exact condition: ${ADMIN_RENAMED} Expect true.`;
22
+ const exec = promisify(execFile);
23
+ export function linkdingTagClaim(claim) {
24
+ if (claim.source.file !== ADMIN_DOC || (claim.type !== "behaviour" && claim.type !== "ui_flow"))
25
+ return undefined;
26
+ const quote = normalise(claim.quote).replace(/^while /, "").replace(/[.,]$/, "");
27
+ if (quote === normalise(MAIN_LIMIT))
28
+ return "main";
29
+ if (quote === normalise(ADMIN_CLAIM))
30
+ return "admin";
31
+ return undefined;
32
+ }
33
+ /** Only the managed Linkding container uses these disposable fixtures. */
34
+ export async function seedLinkdingTags(containerName) {
35
+ const script = [
36
+ "from django.contrib.auth import get_user_model",
37
+ "from django.utils import timezone",
38
+ "from bookmarks.models import Tag",
39
+ "user = get_user_model().objects.get(username='pinotest')",
40
+ "assert user.is_superuser and user.is_staff",
41
+ `names = ['${MAIN_TAG}', '${ADMIN_TAG}']`,
42
+ "assert not Tag.objects.filter(owner=user, name__in=names + ['fibchecktagmain', 'fibcheckadmintagproof']).exists()",
43
+ "Tag.objects.bulk_create([Tag(name=name, owner=user, date_added=timezone.now()) for name in names])",
44
+ "assert set(Tag.objects.filter(owner=user, name__in=names).values_list('name', flat=True)) == set(names)",
45
+ ].join("\n");
46
+ await exec("docker", ["exec", containerName, "python", "manage.py", "shell", "-c", script]);
47
+ }
48
+ export async function questionLinkdingTags(kind, login, run, canRunFollowUp) {
49
+ if (kind === "admin") {
50
+ const kane = await run("initial", `${login} ${CHECK_ADMIN}`, "dom");
51
+ const verified = renamedTag(kane, "/admin/bookmarks/tag", ADMIN_TAG, "fibcheckadmintagproof", ADMIN_RENAMED);
52
+ return {
53
+ kane,
54
+ questioning: {
55
+ verdict: verified ? "true" : "unconfirmed",
56
+ reason: verified ? "Kane renamed the seeded tag and verified the new name in the administration Tags list."
57
+ : "Kane did not verify a saved rename in the administration Tags list.",
58
+ initialReason: "Check the separate claim by renaming a seeded tag inside the admin app.",
59
+ followUps: [],
60
+ overrides: [],
61
+ },
62
+ };
63
+ }
64
+ const kane = await run("initial", `${login} ${CHECK_MAIN}`, "visual");
65
+ const followUps = [];
66
+ const initialReason = "Verify that the seeded tag exists, then check whether its row offers Edit in the ordinary UI.";
67
+ const outcome = (verdict, reason, overrides = []) => ({
68
+ kane,
69
+ questioning: { verdict, reason, initialReason, followUps, overrides },
70
+ });
71
+ if (!editObservedOnFailedCheck(kane)) {
72
+ return outcome("unconfirmed", "Kane did not establish the documented restriction for the seeded tag.");
73
+ }
74
+ if (!canRunFollowUp())
75
+ return outcome("unconfirmed", KANE_RUN_LIMIT_REASON);
76
+ const retry = await run("retry", `${login} ${RETRY_MAIN}`, "visual");
77
+ followUps.push({ purpose: "retry", reason: "Inspect the same seeded tag in a separate Kane session.", kane: retry });
78
+ if (!distinctKaneRuns(kane, retry) || !editObservedOnFailedCheck(retry)) {
79
+ return outcome("unconfirmed", "Kane did not reproduce the same failed Edit assertion in two distinct runs.");
80
+ }
81
+ if (!canRunFollowUp())
82
+ return outcome("unconfirmed", KANE_RUN_LIMIT_REASON);
83
+ const control = await run("control", `${login} ${CONTROL_MAIN}`, "dom");
84
+ const controlReason = control.objective.includes(MAIN_EDIT_NAME)
85
+ ? "Check the original tag name in the Edit form, then save the rename on the ordinary Tags page."
86
+ : "Use Edit and Save on the ordinary Tags page to test whether the control works.";
87
+ followUps.push({ purpose: "control", reason: controlReason, kane: control });
88
+ if (!distinctKaneRuns(kane, control) || !distinctKaneRuns(retry, control) || !renamedTag(control, "/tags", MAIN_TAG, "fibchecktagmain", MAIN_RENAMED)) {
89
+ return outcome("unconfirmed", "The control did not establish a successful rename in an independent run on the ordinary Tags page.");
90
+ }
91
+ const overrides = [tagBlameOverride("initial", kane), tagBlameOverride("retry", retry)].filter((entry) => entry !== undefined);
92
+ return outcome("outdated", `Two independent Kane failures showed that the ordinary Tags page offers Edit for the seeded tag. ${MAIN_CONTROL_PROOF} ${TAG_HISTORY}`, overrides);
93
+ }
94
+ function editObservedOnFailedCheck(run) {
95
+ const assertionFailures = ["assertion_error.checkpoint_failed", "assertion_error.final_verification", "assertion_error.confirmed_product_bug"];
96
+ if (run.status !== "failed" || !assertionFailures.includes(run.reasonCode) || !run.failedStep)
97
+ return false;
98
+ if (!matchesAssertion(run.failedStep.remark, EDIT_OFFERED, "false"))
99
+ return false;
100
+ if (!observedBoolean(run, ROW_PRESENT, "true", "true") || !observedBoolean(run, EDIT_OFFERED, "false", "true"))
101
+ return false;
102
+ const ready = assertionScreenshot(run, "/tags", ROW_PRESENT, "passed");
103
+ const failed = assertionScreenshot(run, "/tags", EDIT_OFFERED, "failed");
104
+ return Boolean(ready && failed && ready.ordinal < failed.ordinal);
105
+ }
106
+ function renamedTag(run, path, original, renamed, assertion) {
107
+ if (run.status !== "passed" || run.exitCode !== 0 || !observedBoolean(run, assertion, "true", "true"))
108
+ return false;
109
+ const result = assertionScreenshot(run, path, assertion, "passed");
110
+ if (!result)
111
+ return false;
112
+ const originalField = path === "/tags" ? ordinaryTagForm(run) : undefined;
113
+ if (path === "/tags" && run.objective.includes(MAIN_EDIT_NAME) && !originalField)
114
+ return false;
115
+ const actions = captures(run, "passed").filter(({ step }) => step.ordinal < result.ordinal);
116
+ return savedTagRename(actions, path, original, renamed, originalField?.ordinal);
117
+ }
118
+ function ordinaryTagForm(run) {
119
+ const observations = matchingObservations(run, MAIN_EDIT_NAME);
120
+ if (observations.length === 0 || !observations.every((o) => {
121
+ if (statedExpectation(o.condition, "true") !== "true")
122
+ return false;
123
+ return (o.expectedValue === MAIN_TAG && o.extractedValue === MAIN_TAG)
124
+ || (normalise(o.expectedValue) === "true" && normalise(o.extractedValue) === "true");
125
+ }))
126
+ return undefined;
127
+ return assertionScreenshot(run, "/tags", MAIN_EDIT_NAME, "passed");
128
+ }
129
+ function savedTagRename(actions, path, original, renamed, formOrdinal) {
130
+ const selected = selectedTag(actions, path, original, formOrdinal);
131
+ if (!selected)
132
+ return false;
133
+ const ready = formOrdinal ?? selected.step.ordinal;
134
+ const nameInput = new RegExp(`^type:.*["']${renamed}["'].*\\bname\\b`);
135
+ const typed = actions.find((entry) => entry.step.ordinal > ready && entry.step.kind === "type"
136
+ && onTagEditForm(path, entry.path) && nameInput.test(entry.summary));
137
+ if (!typed || changedEditForm(actions, ready, typed.step.ordinal))
138
+ return false;
139
+ const saved = actions.find((entry) => entry.step.ordinal > typed.step.ordinal && entry.path === typed.path
140
+ && entry.step.kind === "click" && /^click:.*\bsave\b/.test(entry.summary));
141
+ return Boolean(saved && !changedEditForm(actions, typed.step.ordinal, saved.step.ordinal));
142
+ }
143
+ function selectedTag(actions, path, original, formOrdinal) {
144
+ const originalName = new RegExp(`\\b${original}\\b`);
145
+ return actions.find((entry) => entry.path === path && entry.step.kind === "click"
146
+ && (path !== "/tags" || /\bedit\b/.test(entry.summary))
147
+ && (formOrdinal === undefined ? originalName.test(entry.summary) : entry.step.ordinal < formOrdinal));
148
+ }
149
+ function changedEditForm(actions, after, before) {
150
+ return actions.some(({ step, summary }) => step.ordinal > after && step.ordinal < before
151
+ && (step.kind === "navigate" || (step.kind === "click" && /\b(edit|cancel|close)\b/.test(summary))));
152
+ }
153
+ function onTagEditForm(listPath, path) {
154
+ // The ordinary app uses an in-page dialog. Django opens a separate edit page.
155
+ return listPath === "/tags" ? path === listPath : /^\/admin\/bookmarks\/tag\/\d+\/change$/.test(path);
156
+ }
157
+ function observedBoolean(run, assertion, expected, actual) {
158
+ const chosen = matchingObservations(run, assertion);
159
+ return chosen.length > 0 && chosen.every((o) => statedExpectation(o.condition, expected) === expected
160
+ && normalise(o.expectedValue) === expected && normalise(o.extractedValue) === actual);
161
+ }
162
+ function matchingObservations(run, assertion) {
163
+ const stripExpectation = (text) => normalise(text).replace(/\.? expect (true|false)\.?$/, "").replace(/\.$/, "");
164
+ return run.observations.filter((o) => stripExpectation(o.condition) === stripExpectation(assertion));
165
+ }
166
+ function statedExpectation(condition, expected) {
167
+ return /\bexpect (true|false)\.?$/.exec(normalise(condition))?.[1] ?? expected;
168
+ }
169
+ function matchesAssertion(remark, assertion, expected) {
170
+ // The step record can truncate trailing instructions. Match the whole
171
+ // assertion sentence and verify the boolean expectation in the observation.
172
+ const summary = normalise(remark);
173
+ const sentence = `assert: ${normalise(assertion)}`;
174
+ if (!summary.startsWith(sentence))
175
+ return false;
176
+ return !expected || `expect ${expected}.`.startsWith(summary.slice(sentence.length).trim());
177
+ }
178
+ function assertionScreenshot(run, path, assertion, status) {
179
+ return captures(run, status).find((entry) => entry.path === path && entry.step.kind === "assert"
180
+ && matchesAssertion(entry.summary, assertion, status === "failed" ? "false" : undefined))?.step;
181
+ }
182
+ function captures(run, status) {
183
+ if (!URL.canParse(run.startUrl))
184
+ return [];
185
+ const origin = new URL(run.startUrl).origin;
186
+ return run.evidence.flatMap((step) => {
187
+ if (!step.screenshot || step.status !== status || !URL.canParse(step.url))
188
+ return [];
189
+ const url = new URL(step.url);
190
+ if (url.origin !== origin)
191
+ return [];
192
+ return [{ step, path: url.pathname.replace(/\/$/, ""), summary: normalise(step.summary) }];
193
+ });
194
+ }
195
+ function tagBlameOverride(round, run) {
196
+ const blame = run.blame;
197
+ if (!blame)
198
+ return undefined;
199
+ const cause = normalise(blame.rootCause);
200
+ if (blame.category === "ui_state_mismatch" && cause.includes("ordinary app user should not be able to edit")) {
201
+ return { round, family: blame.family, category: blame.category, hypothesis: "Kane assumed tag editing in the ordinary app was a product defect.", evidence: `${MAIN_CONTROL_PROOF} ${TAG_HISTORY}` };
202
+ }
203
+ if (blame.category === "config_issue" && cause.includes("test expectation is wrong for the page")) {
204
+ return { round, family: blame.family, category: blame.category, hypothesis: "Kane attributed the failed expectation to the test configuration.", evidence: `The expectation came from the documentation. ${MAIN_CONTROL_PROOF} ${TAG_HISTORY}` };
205
+ }
206
+ return undefined;
207
+ }
@@ -0,0 +1 @@
1
+ export const KANE_RUN_LIMIT_REASON = "The Kane run limit was reached before the required follow-up check.";
package/dist/loop.js ADDED
@@ -0,0 +1,186 @@
1
+ import { normalise } from "./claims/quotes.js";
2
+ import { distinctKaneRuns } from "./kane/evidence.js";
3
+ import { questionGenerally } from "./general-loop.js";
4
+ import { linkdingTagClaim, questionLinkdingTags } from "./linkding-tags.js";
5
+ import { KANE_RUN_LIMIT_REASON } from "./loop-types.js";
6
+ import { isLinkdingRepo } from "./target.js";
7
+ const ADMIN_TAB_QUOTE = "to open the admin app, go the settings view and click on the admin tab";
8
+ const ADMIN_DOC = "docs/src/content/docs/admin.md";
9
+ const SETTINGS_PATH = "/settings/general";
10
+ const ADMIN_PATH = "/admin/";
11
+ const READY_ASSERTION = "The Settings page shows the Profile form.";
12
+ const READY_WITH_SAVE = "The Settings page shows the Profile form and its Save button.";
13
+ const TAB_ASSERTION = "The Settings page contains an in-page Admin tab.";
14
+ const MENU_ASSERTION = "The Settings dropdown contains an Admin menu item.";
15
+ const TITLE_ASSERTION = "The administration page title contains Site administration.";
16
+ const ACCOUNT_ASSERTION = "The Site administration dashboard shows the account pinotest.";
17
+ const USERS_ASSERTION = "The Site administration dashboard shows a Users link.";
18
+ const CHECK_TAB = `Open ${SETTINGS_PATH} after login and wait for it to load. First assert exactly: ${READY_ASSERTION} Expect true. Use visual inspection and check tabs implemented as links as well as native tabs. Assert exactly: ${TAB_ASSERTION} Expect true. Ignore the top navigation dropdown. If the assertion fails, stop there. If it passes, click the in-page Admin tab and assert that the administration page title contains Site administration.`;
19
+ const RETRY_TAB = `Go directly to ${SETTINGS_PATH} after login. First assert exactly: ${READY_ASSERTION} Expect true. Independently inspect the visible navigation inside the Settings page content and look for an Admin tab, including links styled as tabs. Reuse this exact boolean assertion: ${TAB_ASSERTION} Expect true. Capture the failed assertion if there is no tab. If present, click the in-page Admin tab and verify the Site administration page title.`;
20
+ const CONTROL = `From the signed-in bookmarks page, open the Settings dropdown in the top navigation. Assert exactly: ${MENU_ASSERTION} Expect true. Click Admin inside that dropdown. Make these as separate assertions: ${TITLE_ASSERTION} ${ACCOUNT_ASSERTION} ${USERS_ASSERTION} Expect true for each.`;
21
+ const CONTROL_PROOF = "The same test login opened Site administration through the top navigation’s Settings dropdown in the passing control run.";
22
+ const TAB_HISTORY = "Linkding’s historical source had an Admin tab in the Settings template at 3ae9cf0. The template was removed at 226eb69.";
23
+ export async function questionClaim(claim, managedTarget, originalObjective, run, prepareTags, canRunFollowUp = () => true, general) {
24
+ // A frozen, reviewed check is run exactly as reviewed, never through a built-in special case.
25
+ if (general?.frozen && claim.check)
26
+ return await questionGenerally(claim, claim.check, general, run, canRunFollowUp);
27
+ const linkding = managedTarget && isLinkdingRepo(managedTarget.repo)
28
+ && managedTarget.app?.image === "sissbruecker/linkding:1.47.0"
29
+ && managedTarget.app?.env?.LD_SUPERUSER_NAME === "pinotest"
30
+ && managedTarget.app?.env?.LD_SUPERUSER_PASSWORD === "{{secret:app_pass}}"
31
+ && managedTarget.login?.includes("pinotest and password {{app_pass}}");
32
+ if (linkding && managedTarget.login && prepareTags) {
33
+ const tags = linkdingTagClaim(claim);
34
+ if (tags) {
35
+ await prepareTags();
36
+ return questionLinkdingTags(tags, managedTarget.login, run, canRunFollowUp);
37
+ }
38
+ }
39
+ if (!linkding || !isAdminTabClaim(claim)) {
40
+ if (general && claim.check)
41
+ return await questionGenerally(claim, claim.check, general, run, canRunFollowUp);
42
+ const kane = await run("initial", originalObjective);
43
+ return {
44
+ kane,
45
+ questioning: {
46
+ verdict: "unconfirmed",
47
+ reason: "This claim has no reviewed check for verifying that Kane exercised the documented behavior.",
48
+ initialReason: "Run the browser check extracted from the documentation.",
49
+ followUps: [],
50
+ overrides: [],
51
+ },
52
+ };
53
+ }
54
+ const initialReason = "Verify that Settings loaded, then look for the tab inside the page before following it.";
55
+ const followUps = [];
56
+ const kane = await run("initial", withLogin(managedTarget.login, CHECK_TAB), "visual");
57
+ const outcome = (verdict, reason, overrides = []) => ({
58
+ kane,
59
+ questioning: { verdict, reason, initialReason, followUps, overrides },
60
+ });
61
+ if (tabPresent(kane))
62
+ return outcome("true", "Kane observed the in-page Admin tab and verified the administration page.");
63
+ if (!tabAbsent(kane)) {
64
+ return outcome("unconfirmed", "Kane did not establish the result of the in-page Admin tab assertion on the Settings page.");
65
+ }
66
+ if (!canRunFollowUp())
67
+ return outcome("unconfirmed", KANE_RUN_LIMIT_REASON);
68
+ const retry = await run("retry", withLogin(managedTarget.login, RETRY_TAB), "visual");
69
+ followUps.push({ purpose: "retry", reason: "Repeat the in-page tab assertion in a separate run with the same logged-in account.", kane: retry });
70
+ if (tabPresent(retry)) {
71
+ return outcome("unconfirmed", "The two Kane runs disagreed about the in-page Admin tab.");
72
+ }
73
+ if (!tabAbsent(retry) || !distinctKaneRuns(kane, retry)) {
74
+ return outcome("unconfirmed", "Kane did not reproduce the same failed tab assertion in two distinct runs.");
75
+ }
76
+ if (!canRunFollowUp())
77
+ return outcome("unconfirmed", KANE_RUN_LIMIT_REASON);
78
+ const control = await run("control", withLogin(managedTarget.login, CONTROL));
79
+ followUps.push({ purpose: "control", reason: "Check whether this account can reach the admin dashboard through the Settings dropdown.", kane: control });
80
+ if (!menuControlPassed(control) || !distinctKaneRuns(control, kane) || !distinctKaneRuns(control, retry)) {
81
+ return outcome("unconfirmed", "The control did not establish that the same login could access Site administration.");
82
+ }
83
+ const overrides = [testDataOverride("initial", kane), testDataOverride("retry", retry)]
84
+ .filter((entry) => entry !== undefined);
85
+ return outcome("outdated", `Two separate Kane runs found no Admin tab inside the Settings page. ${CONTROL_PROOF} ${TAB_HISTORY}`, overrides);
86
+ }
87
+ function isAdminTabClaim(claim) {
88
+ if (claim.type !== "ui_flow" || claim.source.file !== ADMIN_DOC)
89
+ return false;
90
+ const quote = normalise(claim.quote).replace(/\.$/, "");
91
+ return quote === ADMIN_TAB_QUOTE || quote === "go the settings view and click on the admin tab";
92
+ }
93
+ function withLogin(login, objective) {
94
+ return [login, objective].filter(Boolean).join(" ");
95
+ }
96
+ function tabAbsent(run) {
97
+ const assertionFailure = ["assertion_error.final_verification", "assertion_error.checkpoint_failed"].includes(run.reasonCode);
98
+ return run.status === "failed" && assertionFailure
99
+ && Boolean(run.failedStep && isTabAssertion(run.failedStep.remark))
100
+ && tabAssertionOnReadyPage(run, "failed")
101
+ && tabObservation(run, "false");
102
+ }
103
+ function tabPresent(run) {
104
+ return run.status === "passed"
105
+ && tabAssertionOnReadyPage(run, "passed")
106
+ && tabObservation(run, "true")
107
+ && clickedAdmin(run, /^click:(?!.*\b(?:dropdown|menu)\b).*\badmin\b.*\btab\b/)
108
+ && adminPageVerified(run);
109
+ }
110
+ function tabAssertionOnReadyPage(run, status) {
111
+ const ready = assertionText(READY_ASSERTION);
112
+ if (!observedTrue(run, ready))
113
+ return false;
114
+ const assertions = screenshotsAt(run, SETTINGS_PATH).filter((step) => step.kind === "assert");
115
+ const readinessSummaries = [ready, assertionText(READY_WITH_SAVE)];
116
+ const readyStep = assertions.find((step) => step.status === "passed" && readinessSummaries.includes(assertionText(step.summary)));
117
+ return Boolean(readyStep && assertions.some((step) => step.ordinal > readyStep.ordinal && step.status === status && isTabAssertion(step.summary)));
118
+ }
119
+ function isTabAssertion(text) {
120
+ const condition = assertionText(text);
121
+ return condition === "the settings page contains an in-page admin tab"
122
+ || condition === "the settings page has a tab named admin";
123
+ }
124
+ function tabObservation(run, value) {
125
+ const observations = run.observations.filter((observation) => isTabAssertion(observation.condition));
126
+ return observations.some(({ analyzerType, usedDomCode }) => /^(visual|textual_visual)$/.test(analyzerType ?? "") && usedDomCode === false)
127
+ && observations.every((observation) => normalise(observation.expectedValue) === "true" && normalise(observation.extractedValue) === value);
128
+ }
129
+ function menuControlPassed(run) {
130
+ return run.status === "passed" && adminPageVerified(run) && clickedDropdownAdmin(run)
131
+ && observedTrue(run, assertionText(MENU_ASSERTION))
132
+ && observedTrue(run, assertionText(ACCOUNT_ASSERTION))
133
+ && observedTrue(run, assertionText(USERS_ASSERTION));
134
+ }
135
+ function observedTrue(run, assertion) {
136
+ return run.observations.some((observation) => assertionText(observation.condition) === assertion
137
+ && normalise(observation.expectedValue) === "true" && normalise(observation.extractedValue) === "true");
138
+ }
139
+ function assertionText(text) {
140
+ return normalise(text).replace(/^assert:\s*/, "").replace(/["'.]/g, "").replace(/\s+expect true$/, "");
141
+ }
142
+ function clickedDropdownAdmin(run) {
143
+ const openedDropdown = run.steps.find((step) => step.status === "done" && /^click:.*\bsettings\b.*\bdropdown\b/.test(normalise(step.remark)));
144
+ return Boolean(openedDropdown && clickedAdmin(run, /^click:.*\badmin\b.*\bdropdown\b/, openedDropdown.step));
145
+ }
146
+ function clickedAdmin(run, click, afterStep = 0) {
147
+ const clicked = run.steps.find((step) => step.status === "done" && step.step > afterStep
148
+ && click.test(normalise(step.remark)));
149
+ return Boolean(clicked && !run.steps.some((step) => step.step > clicked.step && /^navigate:/.test(normalise(step.remark))));
150
+ }
151
+ function adminPageVerified(run) {
152
+ return screenshotsAt(run, ADMIN_PATH).length > 0 && run.observations.some((observation) => {
153
+ const expected = normalise(observation.expectedValue);
154
+ const value = normalise(observation.extractedValue);
155
+ const condition = assertionText(observation.condition);
156
+ return (expected === "site administration" && value.startsWith("site administration"))
157
+ || (expected === "true" && value === "true" && /^the (?:administration )?page title (?:contains|includes) site administration$/.test(condition));
158
+ });
159
+ }
160
+ function screenshotsAt(run, path) {
161
+ const start = new URL(run.startUrl);
162
+ return run.evidence.filter((step) => {
163
+ if (!step.screenshot || !URL.canParse(step.url))
164
+ return false;
165
+ const url = new URL(step.url);
166
+ return url.origin === start.origin && url.pathname.replace(/\/$/, "") === path.replace(/\/$/, "");
167
+ });
168
+ }
169
+ function testDataOverride(round, kane) {
170
+ const blame = kane.blame;
171
+ if (blame?.category !== "test_data_issue")
172
+ return undefined;
173
+ const cause = normalise(blame.rootCause);
174
+ const missingAccess = [
175
+ "regular user rather than an administrator",
176
+ "account lacks administrator access",
177
+ "account lacks admin access",
178
+ "account does not have the required administrator access",
179
+ "account or fixture that does not expose administrator settings",
180
+ ].some((phrase) => cause.includes(phrase));
181
+ const suspectedRoleMismatch = (cause.includes("account-role or test-fixture mismatch") && cause.includes("administrator access"))
182
+ || (cause.includes("account or test expectation is mismatched") && cause.includes("pinotest login") && cause.includes("admin tab"));
183
+ if (!missingAccess && !suspectedRoleMismatch)
184
+ return undefined;
185
+ return { round, family: blame.family, category: blame.category, hypothesis: "Kane questioned the test account’s administrator access.", evidence: CONTROL_PROOF };
186
+ }
@@ -0,0 +1,82 @@
1
+ import { appendFileSync, existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
2
+ import { dirname } from "node:path";
3
+ /** Raised before a call is made when it would go over a limit. */
4
+ export class BudgetExceededError extends Error {
5
+ constructor(message) {
6
+ super(message);
7
+ this.name = "BudgetExceededError";
8
+ }
9
+ }
10
+ /**
11
+ * Counts model calls for one run and across all runs, and refuses a call
12
+ * that would exceed either limit. The all-runs total is kept in a small
13
+ * JSON file so the limit holds across separate commands.
14
+ */
15
+ export class CallBudget {
16
+ limits;
17
+ runId;
18
+ totalsPath;
19
+ ledgerPath;
20
+ runCalls = 0;
21
+ runUsage = { inputTokens: 0, outputTokens: 0 };
22
+ runModels = new Set();
23
+ constructor(limits, runId, totalsPath, ledgerPath) {
24
+ this.limits = limits;
25
+ this.runId = runId;
26
+ this.totalsPath = totalsPath;
27
+ this.ledgerPath = ledgerPath;
28
+ }
29
+ /** Reserves one call, or throws if a limit would be exceeded. */
30
+ reserve() {
31
+ if (this.runCalls >= this.limits.maxCallsPerRun) {
32
+ throw new BudgetExceededError(`Model call limit for this run reached (${this.limits.maxCallsPerRun} calls).`);
33
+ }
34
+ const totals = this.readTotals();
35
+ if (totals.calls >= this.limits.maxCallsTotal) {
36
+ throw new BudgetExceededError(`Total model call limit reached (${this.limits.maxCallsTotal} calls). Raise FIBCHECK_MAX_CALLS_TOTAL to continue.`);
37
+ }
38
+ this.runCalls += 1;
39
+ this.writeTotals({ ...totals, calls: totals.calls + 1 });
40
+ }
41
+ /** Records the outcome of a reserved call in the ledger and the totals. */
42
+ record(entry) {
43
+ if (entry.ok)
44
+ this.runModels.add(entry.model);
45
+ this.runUsage = {
46
+ inputTokens: this.runUsage.inputTokens + entry.inputTokens,
47
+ outputTokens: this.runUsage.outputTokens + entry.outputTokens,
48
+ };
49
+ const totals = this.readTotals();
50
+ this.writeTotals({
51
+ calls: totals.calls,
52
+ inputTokens: totals.inputTokens + entry.inputTokens,
53
+ outputTokens: totals.outputTokens + entry.outputTokens,
54
+ });
55
+ const line = { at: new Date().toISOString(), runId: this.runId, ...entry };
56
+ ensureDir(this.ledgerPath);
57
+ appendFileSync(this.ledgerPath, `${JSON.stringify(line)}\n`);
58
+ }
59
+ /** Calls and tokens used so far in this run. */
60
+ /** Calls and tokens for this run, and every model that answered. */
61
+ runSummary() {
62
+ return { calls: this.runCalls, ...this.runUsage, models: [...this.runModels].sort((a, b) => a.localeCompare(b)) };
63
+ }
64
+ readTotals() {
65
+ if (!existsSync(this.totalsPath))
66
+ return { calls: 0, inputTokens: 0, outputTokens: 0 };
67
+ try {
68
+ return JSON.parse(readFileSync(this.totalsPath, "utf8"));
69
+ }
70
+ catch {
71
+ // Refuse rather than reset: resetting would silently lift the total limit.
72
+ throw new BudgetExceededError(`The usage totals file ${this.totalsPath} is damaged. Fix or delete it on purpose before making more model calls.`);
73
+ }
74
+ }
75
+ writeTotals(totals) {
76
+ ensureDir(this.totalsPath);
77
+ writeFileSync(this.totalsPath, `${JSON.stringify(totals, null, 2)}\n`);
78
+ }
79
+ }
80
+ function ensureDir(filePath) {
81
+ mkdirSync(dirname(filePath), { recursive: true });
82
+ }
@@ -0,0 +1,59 @@
1
+ import { UnusableModelOutputError } from "./types.js";
2
+ /**
3
+ * The one entry point for model calls. The provider must announce every HTTP
4
+ * request through `beforeAttempt`, which reserves it in the call budget first;
5
+ * each call is then recorded in the usage ledger.
6
+ */
7
+ export class ModelClient {
8
+ provider;
9
+ budget;
10
+ constructor(provider, budget) {
11
+ this.provider = provider;
12
+ this.budget = budget;
13
+ }
14
+ async generate(request) {
15
+ let attempts = 0;
16
+ try {
17
+ const response = await this.provider.generate(request, () => {
18
+ this.budget.reserve();
19
+ attempts += 1;
20
+ });
21
+ if (attempts === 0)
22
+ throw new Error(`Provider ${this.provider.name} answered without announcing its request.`);
23
+ this.budget.record({ purpose: request.purpose, model: response.model, ...response.usage, ok: true });
24
+ return response;
25
+ }
26
+ catch (error) {
27
+ if (attempts === 0)
28
+ throw error;
29
+ this.budget.record({ purpose: request.purpose, model: this.provider.name, inputTokens: 0, outputTokens: 0, ok: false });
30
+ throw error;
31
+ }
32
+ }
33
+ /** Asks for JSON matching `jsonSchema`, parses it, and checks it with `validate`. */
34
+ async generateJson(request, validate) {
35
+ const response = await this.generate(request);
36
+ let parsed;
37
+ try {
38
+ parsed = JSON.parse(stripCodeFence(response.text));
39
+ }
40
+ catch {
41
+ throw new UnusableModelOutputError(`The model did not return valid JSON for "${request.purpose}".`);
42
+ }
43
+ try {
44
+ return { value: validate(parsed), response };
45
+ }
46
+ catch (error) {
47
+ throw new UnusableModelOutputError(`The model's reply for "${request.purpose}" had the wrong shape: ${error instanceof Error ? error.message : String(error)}`);
48
+ }
49
+ }
50
+ /** Calls, tokens and answering models for this run, and how spending is limited. */
51
+ usage() {
52
+ return { ...this.budget.runSummary(), spendingLimit: this.provider.spendingLimit ?? "call cap only (free tier, no dollar cost)" };
53
+ }
54
+ }
55
+ /** Some OpenAI-compatible models wrap JSON in a Markdown code fence; the fence is removed before parsing. */
56
+ function stripCodeFence(text) {
57
+ const match = text.trim().match(/^```(?:json)?\s*([\s\S]*?)\s*```$/i);
58
+ return match?.[1] ?? text;
59
+ }