@binclusive/a11y 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +285 -0
- package/bin/a11y.mjs +36 -0
- package/bin/diff-scope.mjs +29 -0
- package/data/baseline-rules.json +892 -0
- package/package.json +68 -0
- package/src/agent-lane.ts +138 -0
- package/src/agents-block.ts +157 -0
- package/src/baseline/gen-baseline.ts +166 -0
- package/src/cli.ts +1026 -0
- package/src/collect-dom.ts +119 -0
- package/src/collect-liquid.ts +103 -0
- package/src/collect-swift.ts +227 -0
- package/src/collect-unity.ts +99 -0
- package/src/collect.ts +54 -0
- package/src/commands.ts +314 -0
- package/src/config-scan.ts +177 -0
- package/src/contract.ts +355 -0
- package/src/core.ts +546 -0
- package/src/detect-stack.ts +207 -0
- package/src/diff-scope-cli.ts +12 -0
- package/src/diff-scope.ts +150 -0
- package/src/emit-contract.ts +181 -0
- package/src/enforce.ts +1125 -0
- package/src/eslint-plugin-jsx-a11y.d.ts +11 -0
- package/src/evidence.ts +308 -0
- package/src/github-identity.ts +201 -0
- package/src/hook.ts +242 -0
- package/src/impact-gate.ts +107 -0
- package/src/imports-resolve.ts +248 -0
- package/src/index.ts +183 -0
- package/src/liquid-ast.ts +203 -0
- package/src/liquid-rules.ts +691 -0
- package/src/mcp.ts +363 -0
- package/src/module-scope.ts +137 -0
- package/src/phone-home.ts +470 -0
- package/src/pr-comment.ts +250 -0
- package/src/pr-summary-cli.ts +182 -0
- package/src/pr-summary.ts +291 -0
- package/src/registry.ts +605 -0
- package/src/reporter/contract.ts +154 -0
- package/src/reporter/finding.ts +87 -0
- package/src/reporter/github-adapter.ts +183 -0
- package/src/reporter/null-adapter.ts +51 -0
- package/src/reporter/registry.ts +22 -0
- package/src/reporter-cli.ts +48 -0
- package/src/resolve-components.ts +579 -0
- package/src/runner/budget.ts +90 -0
- package/src/runner/codegraph-lookup.test.ts +93 -0
- package/src/runner/codegraph-lookup.ts +197 -0
- package/src/runner/index.ts +86 -0
- package/src/runner/lookup.ts +69 -0
- package/src/runner/provider.ts +72 -0
- package/src/runner/providers/anthropic.ts +168 -0
- package/src/runner/providers/openai.ts +191 -0
- package/src/runner/reasoner.ts +73 -0
- package/src/runner/reasoning/index.ts +53 -0
- package/src/runner/reasoning/prompt.ts +200 -0
- package/src/runner/reasoning/react.ts +149 -0
- package/src/runner/reasoning/shopify.ts +214 -0
- package/src/runner/reasoning/skills-reasoner.ts +117 -0
- package/src/runner/reasoning/types.ts +99 -0
- package/src/runner/runner.ts +203 -0
- package/src/sarif.ts +148 -0
- package/src/source-identity.ts +0 -0
- package/src/source-trace.ts +814 -0
- package/src/suggest.ts +328 -0
- package/src/suppression-ranges.ts +354 -0
- package/src/suppressor-map.ts +189 -0
- package/src/suppressors.ts +284 -0
- package/src/tsconfig-aliases.ts +155 -0
- package/src/unity-ast.ts +331 -0
- package/src/unity-findings.ts +91 -0
- package/src/unity-guid-registry.ts +82 -0
- package/src/unity-label-resolve.ts +249 -0
- package/src/unity-rule-color-only.ts +127 -0
- package/src/unity-rule-missing-label.ts +156 -0
- package/src/unity-rules-baseline.ts +273 -0
- package/src/wcag-map.ts +35 -0
- package/src/wcag-tags.ts +35 -0
- package/src/workspace-resolve.ts +405 -0
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
import { fileURLToPath } from "node:url";
|
|
2
|
+
import { describe, expect, it } from "vitest";
|
|
3
|
+
import { createCodeGraphLookup } from "./codegraph-lookup";
|
|
4
|
+
import { LookupCounter, meterLookup } from "./lookup";
|
|
5
|
+
|
|
6
|
+
/**
|
|
7
|
+
* These tests drive the REAL published `@binclusive/code-graph` CLI as a subprocess
|
|
8
|
+
* over a real directory (a small `test/fixtures/code-graph-lookup` module), so they
|
|
9
|
+
* exercise the actual stdout contract this seam depends on — not a mock. Each spawns
|
|
10
|
+
* a ts-morph cheap pass (~1-2s), so they carry an explicit per-test timeout.
|
|
11
|
+
*/
|
|
12
|
+
const CODE_GRAPH_SRC = fileURLToPath(
|
|
13
|
+
new URL("../../test/fixtures/code-graph-lookup", import.meta.url),
|
|
14
|
+
);
|
|
15
|
+
const TIMEOUT = 30_000;
|
|
16
|
+
|
|
17
|
+
describe("createCodeGraphLookup — structural lookups over the published CLI", () => {
|
|
18
|
+
it(
|
|
19
|
+
"a `file` query returns the module's structural JSON",
|
|
20
|
+
async () => {
|
|
21
|
+
const tool = createCodeGraphLookup({ root: CODE_GRAPH_SRC });
|
|
22
|
+
const result = await tool.lookup({ kind: "file", target: "schema.ts" });
|
|
23
|
+
|
|
24
|
+
expect(result.status).toBe("ok");
|
|
25
|
+
if (result.status !== "ok") return;
|
|
26
|
+
const data = result.data as { found: boolean; kind?: string; json?: unknown };
|
|
27
|
+
expect(data.found).toBe(true);
|
|
28
|
+
expect(data.kind).toBe("file");
|
|
29
|
+
// The `--file` view is a module record — assert its shape at the boundary.
|
|
30
|
+
const json = data.json as { module?: { file?: string } };
|
|
31
|
+
expect(json.module?.file).toBe("schema.ts");
|
|
32
|
+
},
|
|
33
|
+
TIMEOUT,
|
|
34
|
+
);
|
|
35
|
+
|
|
36
|
+
it(
|
|
37
|
+
"a `summary` query returns the project health JSON",
|
|
38
|
+
async () => {
|
|
39
|
+
const tool = createCodeGraphLookup({ root: CODE_GRAPH_SRC });
|
|
40
|
+
const result = await tool.lookup({ kind: "summary", target: "" });
|
|
41
|
+
|
|
42
|
+
expect(result.status).toBe("ok");
|
|
43
|
+
if (result.status !== "ok") return;
|
|
44
|
+
const data = result.data as { found: boolean; json?: unknown };
|
|
45
|
+
expect(data.found).toBe(true);
|
|
46
|
+
const json = data.json as { fileCount?: number };
|
|
47
|
+
expect(typeof json.fileCount).toBe("number");
|
|
48
|
+
},
|
|
49
|
+
TIMEOUT,
|
|
50
|
+
);
|
|
51
|
+
|
|
52
|
+
it(
|
|
53
|
+
"a missing file is a `found: false` envelope, not a throw",
|
|
54
|
+
async () => {
|
|
55
|
+
const tool = createCodeGraphLookup({ root: CODE_GRAPH_SRC });
|
|
56
|
+
const result = await tool.lookup({ kind: "file", target: "does-not-exist.ts" });
|
|
57
|
+
|
|
58
|
+
expect(result.status).toBe("ok");
|
|
59
|
+
if (result.status !== "ok") return;
|
|
60
|
+
const data = result.data as { found: boolean };
|
|
61
|
+
expect(data.found).toBe(false);
|
|
62
|
+
},
|
|
63
|
+
TIMEOUT,
|
|
64
|
+
);
|
|
65
|
+
|
|
66
|
+
it("an unknown kind is refused without spawning a subprocess", async () => {
|
|
67
|
+
const tool = createCodeGraphLookup({ root: CODE_GRAPH_SRC });
|
|
68
|
+
const result = await tool.lookup({ kind: "not-a-real-kind", target: "x" });
|
|
69
|
+
|
|
70
|
+
expect(result.status).toBe("ok");
|
|
71
|
+
if (result.status !== "ok") return;
|
|
72
|
+
const data = result.data as { found: boolean; reason?: string };
|
|
73
|
+
expect(data.found).toBe(false);
|
|
74
|
+
expect(data.reason).toContain("unsupported lookup kind");
|
|
75
|
+
});
|
|
76
|
+
|
|
77
|
+
it(
|
|
78
|
+
"the per-finding cap still short-circuits the tool via meterLookup",
|
|
79
|
+
async () => {
|
|
80
|
+
const tool = createCodeGraphLookup({ root: CODE_GRAPH_SRC });
|
|
81
|
+
const counter = new LookupCounter(1);
|
|
82
|
+
const metered = meterLookup(tool, counter);
|
|
83
|
+
|
|
84
|
+
const first = await metered.lookup({ kind: "summary", target: "" });
|
|
85
|
+
expect(first.status).toBe("ok");
|
|
86
|
+
const second = await metered.lookup({ kind: "summary", target: "" });
|
|
87
|
+
// Budget spent: the second call is capped and never reaches code-graph.
|
|
88
|
+
expect(second.status).toBe("capped");
|
|
89
|
+
expect(counter.used).toBe(1);
|
|
90
|
+
},
|
|
91
|
+
TIMEOUT,
|
|
92
|
+
);
|
|
93
|
+
});
|
|
@@ -0,0 +1,197 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The code-graph implementation of the structural-lookup seam (#2097).
|
|
3
|
+
*
|
|
4
|
+
* This fills the {@link LookupTool} interface `lookup.ts` left open: it answers a
|
|
5
|
+
* reasoner's structural questions ("which module is this / what does it export",
|
|
6
|
+
* "who calls this function") by shelling out to the published
|
|
7
|
+
* `@binclusive/code-graph` CLI and returning its deterministic JSON. Calling it as
|
|
8
|
+
* a subprocess — rather than importing its internals — keeps the boundary at
|
|
9
|
+
* code-graph's stdout contract: one JSON document per query, nothing else.
|
|
10
|
+
*
|
|
11
|
+
* The tool spends NO model tokens, so it is metered on the count-based per-finding
|
|
12
|
+
* budget by {@link meterLookup} in the runner — this tool never returns `capped`
|
|
13
|
+
* itself (only the meter wrapper does). It is also TOTAL: every failure mode (an
|
|
14
|
+
* unknown query kind, a missing file, no tsconfig for the edge pass, a subprocess
|
|
15
|
+
* crash or timeout) becomes a `{ found: false }` envelope in `data`, never a
|
|
16
|
+
* throw. The AI lane is non-blocking; a structure lookup that can't answer is a
|
|
17
|
+
* gap the reasoner works around, not an error that ends the pass.
|
|
18
|
+
*
|
|
19
|
+
* The CLI is resolved from the installed `@binclusive/code-graph` dependency, so
|
|
20
|
+
* the customer's cwd never matters and the same compiled artifact runs in the
|
|
21
|
+
* Docker image as locally. The published package ships a compiled ESM bin
|
|
22
|
+
* (`dist/index.js`), so it runs under plain `node` — no tsx loader needed.
|
|
23
|
+
*/
|
|
24
|
+
import { spawn } from "node:child_process";
|
|
25
|
+
import { createRequire } from "node:module";
|
|
26
|
+
import type { LookupQuery, LookupResult, LookupTool } from "./lookup";
|
|
27
|
+
|
|
28
|
+
const require = createRequire(import.meta.url);
|
|
29
|
+
/**
|
|
30
|
+
* The published code-graph CLI entry. Resolved by subpath from the installed
|
|
31
|
+
* dependency — the package exposes its bin as `dist/index.js` (no `main`/`exports`
|
|
32
|
+
* map), so the explicit subpath is how a consumer reaches it.
|
|
33
|
+
*/
|
|
34
|
+
const CODE_GRAPH_CLI = require.resolve("@binclusive/code-graph/dist/index.js");
|
|
35
|
+
|
|
36
|
+
/** A slow subprocess must not stall a pass; the pull loop stays responsive. */
|
|
37
|
+
const DEFAULT_TIMEOUT_MS = 30_000;
|
|
38
|
+
|
|
39
|
+
/**
|
|
40
|
+
* The structural questions code-graph can answer for the agent lane. The seam's
|
|
41
|
+
* {@link LookupQuery.kind} is an open `string`; these are the values this tool
|
|
42
|
+
* understands (with a couple of natural aliases). Anything else is a `found:
|
|
43
|
+
* false` envelope, not a throw — the reasoner never crashes on a typo.
|
|
44
|
+
*/
|
|
45
|
+
export type CodeGraphQueryKind =
|
|
46
|
+
/** One module + its functions/imports/importedBy — "what is in this file". */
|
|
47
|
+
| "file"
|
|
48
|
+
/** Callers of one function id (direct + transitive) — "where does this flow / who uses it". */
|
|
49
|
+
| "callers"
|
|
50
|
+
/** Whole-project structural health summary — "orient me in this codebase". */
|
|
51
|
+
| "summary";
|
|
52
|
+
|
|
53
|
+
/**
|
|
54
|
+
* What a code-graph lookup yields, carried in {@link LookupResult}'s `data`. The
|
|
55
|
+
* reasoner narrows on `found`; `json` is the deterministic structural document
|
|
56
|
+
* code-graph emitted for that query kind (shape documented per kind in SPEC.md).
|
|
57
|
+
*/
|
|
58
|
+
export type CodeGraphLookupData =
|
|
59
|
+
| { readonly found: true; readonly kind: CodeGraphQueryKind; readonly json: unknown }
|
|
60
|
+
| { readonly found: false; readonly reason: string };
|
|
61
|
+
|
|
62
|
+
export interface CodeGraphLookupConfig {
|
|
63
|
+
/**
|
|
64
|
+
* Absolute path to the directory code-graph analyzes — the customer's checked
|
|
65
|
+
* -out repo (or the diff-scoped subtree). Every query is relative to this root.
|
|
66
|
+
*/
|
|
67
|
+
readonly root: string;
|
|
68
|
+
/** Per-lookup wall-clock budget. Defaults to {@link DEFAULT_TIMEOUT_MS}. */
|
|
69
|
+
readonly timeoutMs?: number;
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
/**
|
|
73
|
+
* Build a {@link LookupTool} backed by the vendored code-graph CLI. Inject the
|
|
74
|
+
* result into the runner's `RunInput.lookup`; the harness caps how many times the
|
|
75
|
+
* reasoner may call it per finding.
|
|
76
|
+
*/
|
|
77
|
+
export function createCodeGraphLookup(config: CodeGraphLookupConfig): LookupTool {
|
|
78
|
+
const timeoutMs = config.timeoutMs ?? DEFAULT_TIMEOUT_MS;
|
|
79
|
+
|
|
80
|
+
return {
|
|
81
|
+
async lookup(query: LookupQuery): Promise<LookupResult> {
|
|
82
|
+
const plan = planQuery(config.root, query);
|
|
83
|
+
if (plan === null) {
|
|
84
|
+
return ok({ found: false, reason: `unsupported lookup kind: ${query.kind}` });
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
const { code, stdout } = await runCli(plan.argv, timeoutMs);
|
|
88
|
+
if (code !== 0) {
|
|
89
|
+
// code-graph clean-exits 2 with a one-line reason (bad path, no tsconfig
|
|
90
|
+
// for the edge pass, …); 1 is a CI-gate fail we don't drive here.
|
|
91
|
+
return ok({ found: false, reason: `code-graph exited ${code}` });
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
// A missing --file / an empty result writes a stderr warning and NO stdout
|
|
95
|
+
// at exit 0, so empty-or-unparseable stdout is a genuine "not found".
|
|
96
|
+
const parsed = parseJson(stdout);
|
|
97
|
+
if (!parsed.ok) {
|
|
98
|
+
return ok({ found: false, reason: parsed.reason });
|
|
99
|
+
}
|
|
100
|
+
return ok({ found: true, kind: plan.kind, json: parsed.value });
|
|
101
|
+
},
|
|
102
|
+
};
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
/** Wrap code-graph's answer in the seam's `ok` result (never `capped` here). */
|
|
106
|
+
function ok(data: CodeGraphLookupData): LookupResult {
|
|
107
|
+
return { status: "ok", data };
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
/** Normalize the open `kind` string onto a supported query, with aliases. */
|
|
111
|
+
function normalizeKind(kind: string): CodeGraphQueryKind | null {
|
|
112
|
+
switch (kind) {
|
|
113
|
+
case "file":
|
|
114
|
+
case "module":
|
|
115
|
+
return "file";
|
|
116
|
+
case "callers":
|
|
117
|
+
case "blast":
|
|
118
|
+
case "usages":
|
|
119
|
+
return "callers";
|
|
120
|
+
case "summary":
|
|
121
|
+
case "overview":
|
|
122
|
+
return "summary";
|
|
123
|
+
default:
|
|
124
|
+
return null;
|
|
125
|
+
}
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
/** Translate a query into the code-graph argv, or `null` for an unknown kind. */
|
|
129
|
+
function planQuery(
|
|
130
|
+
root: string,
|
|
131
|
+
query: LookupQuery,
|
|
132
|
+
): { readonly kind: CodeGraphQueryKind; readonly argv: readonly string[] } | null {
|
|
133
|
+
const kind = normalizeKind(query.kind);
|
|
134
|
+
if (kind === null) return null;
|
|
135
|
+
|
|
136
|
+
switch (kind) {
|
|
137
|
+
case "file":
|
|
138
|
+
// Module + its functions, as JSON (root-relative file path in target).
|
|
139
|
+
return { kind, argv: [root, "--file", query.target, "--json"] };
|
|
140
|
+
case "callers":
|
|
141
|
+
// Blast radius needs the opt-in edge pass (calls/calledBy) — requires a
|
|
142
|
+
// reachable tsconfig; code-graph clean-exits 2 when there is none.
|
|
143
|
+
return { kind, argv: [root, "--edges", "--blast", query.target, "--json"] };
|
|
144
|
+
case "summary":
|
|
145
|
+
// The bare invocation already emits the JSON summary; target is ignored.
|
|
146
|
+
return { kind, argv: [root] };
|
|
147
|
+
default: {
|
|
148
|
+
const exhaustive: never = kind;
|
|
149
|
+
return exhaustive;
|
|
150
|
+
}
|
|
151
|
+
}
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
/** Run the vendored CLI under tsx, capturing stdout. Total: never rejects. */
|
|
155
|
+
function runCli(
|
|
156
|
+
argv: readonly string[],
|
|
157
|
+
timeoutMs: number,
|
|
158
|
+
): Promise<{ readonly code: number; readonly stdout: string }> {
|
|
159
|
+
return new Promise((resolve) => {
|
|
160
|
+
const child = spawn(process.execPath, [CODE_GRAPH_CLI, ...argv], {
|
|
161
|
+
stdio: ["ignore", "pipe", "pipe"],
|
|
162
|
+
});
|
|
163
|
+
|
|
164
|
+
let stdout = "";
|
|
165
|
+
const timer = setTimeout(() => child.kill("SIGKILL"), timeoutMs);
|
|
166
|
+
|
|
167
|
+
child.stdout.on("data", (chunk: Buffer) => {
|
|
168
|
+
stdout += chunk.toString("utf8");
|
|
169
|
+
});
|
|
170
|
+
// Drain stderr so a chatty warning can't fill the pipe buffer and deadlock.
|
|
171
|
+
child.stderr.resume();
|
|
172
|
+
|
|
173
|
+
child.on("error", () => {
|
|
174
|
+
clearTimeout(timer);
|
|
175
|
+
resolve({ code: -1, stdout: "" });
|
|
176
|
+
});
|
|
177
|
+
child.on("close", (code) => {
|
|
178
|
+
clearTimeout(timer);
|
|
179
|
+
resolve({ code: code ?? -1, stdout });
|
|
180
|
+
});
|
|
181
|
+
});
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
type ParseResult =
|
|
185
|
+
| { readonly ok: true; readonly value: unknown }
|
|
186
|
+
| { readonly ok: false; readonly reason: string };
|
|
187
|
+
|
|
188
|
+
/** Parse code-graph's stdout as JSON. Empty output = a not-found lookup. */
|
|
189
|
+
function parseJson(stdout: string): ParseResult {
|
|
190
|
+
const trimmed = stdout.trim();
|
|
191
|
+
if (trimmed.length === 0) return { ok: false, reason: "no result" };
|
|
192
|
+
try {
|
|
193
|
+
return { ok: true, value: JSON.parse(trimmed) };
|
|
194
|
+
} catch {
|
|
195
|
+
return { ok: false, reason: "code-graph output was not JSON" };
|
|
196
|
+
}
|
|
197
|
+
}
|
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `@binclusive/a11y` runner — the provider-agnostic AI lane (issue #2095).
|
|
3
|
+
*
|
|
4
|
+
* The harness that sits on top of the deterministic engine: a BYO-provider
|
|
5
|
+
* abstraction + a capped, non-blocking pull loop that consumes the engine's
|
|
6
|
+
* deterministic findings and emits agent-provenance findings through the SAME
|
|
7
|
+
* metadata-only contract projection. The code-graph lookups (#2097) live behind
|
|
8
|
+
* the `LookupTool` seam. The reasoning CORE (#2096) — the ported per-framework
|
|
9
|
+
* checklists + pattern catalog and the concrete `AgentReasoner` — fills the
|
|
10
|
+
* reasoning seam (`./reasoning`). The enrich/discover deepening (#2098) is now
|
|
11
|
+
* FILLED: the reasoner returns a `ReasonResult` (in-place enrichment + discovered
|
|
12
|
+
* findings), parsed at a tolerant zod boundary and deduped against the floor.
|
|
13
|
+
*/
|
|
14
|
+
export {
|
|
15
|
+
type BudgetSnapshot,
|
|
16
|
+
meterProvider,
|
|
17
|
+
TokenCeilingExceeded,
|
|
18
|
+
TokenLedger,
|
|
19
|
+
} from "./budget";
|
|
20
|
+
export {
|
|
21
|
+
LookupCounter,
|
|
22
|
+
type LookupQuery,
|
|
23
|
+
type LookupResult,
|
|
24
|
+
type LookupTool,
|
|
25
|
+
meterLookup,
|
|
26
|
+
} from "./lookup";
|
|
27
|
+
export {
|
|
28
|
+
type CodeGraphLookupConfig,
|
|
29
|
+
type CodeGraphLookupData,
|
|
30
|
+
type CodeGraphQueryKind,
|
|
31
|
+
createCodeGraphLookup,
|
|
32
|
+
} from "./codegraph-lookup";
|
|
33
|
+
export {
|
|
34
|
+
isMeterableUsage,
|
|
35
|
+
type Provider,
|
|
36
|
+
type ProviderMessage,
|
|
37
|
+
type ProviderRequest,
|
|
38
|
+
type ProviderResponse,
|
|
39
|
+
type TokenUsage,
|
|
40
|
+
usageTotal,
|
|
41
|
+
} from "./provider";
|
|
42
|
+
export {
|
|
43
|
+
type AnthropicProviderConfig,
|
|
44
|
+
createAnthropicProvider,
|
|
45
|
+
DEFAULT_ANTHROPIC_MODEL,
|
|
46
|
+
} from "./providers/anthropic";
|
|
47
|
+
export {
|
|
48
|
+
type AgentFinding,
|
|
49
|
+
type AgentReasoner,
|
|
50
|
+
EMPTY_RESULT,
|
|
51
|
+
type ReasonContext,
|
|
52
|
+
type ReasonResult,
|
|
53
|
+
} from "./reasoner";
|
|
54
|
+
export {
|
|
55
|
+
DEFAULT_RUNNER_CONFIG,
|
|
56
|
+
type PassOutcome,
|
|
57
|
+
type PassReport,
|
|
58
|
+
type RunInput,
|
|
59
|
+
type RunnerConfig,
|
|
60
|
+
type RunOutcome,
|
|
61
|
+
runAgentLane,
|
|
62
|
+
} from "./runner";
|
|
63
|
+
export {
|
|
64
|
+
type ChecklistArea,
|
|
65
|
+
type Discovery,
|
|
66
|
+
type FixSeverity,
|
|
67
|
+
type FixSuggestion,
|
|
68
|
+
type FixType,
|
|
69
|
+
FIX_TYPES,
|
|
70
|
+
type FrameworkGuidance,
|
|
71
|
+
frameworkGuidanceFor,
|
|
72
|
+
type PatternCatalogEntry,
|
|
73
|
+
REACT_GUIDANCE,
|
|
74
|
+
SHOPIFY_GUIDANCE,
|
|
75
|
+
} from "./reasoning";
|
|
76
|
+
export {
|
|
77
|
+
createSkillsReasoner,
|
|
78
|
+
type SkillsReasonerOptions,
|
|
79
|
+
} from "./reasoning/skills-reasoner";
|
|
80
|
+
export {
|
|
81
|
+
buildSystemPrompt,
|
|
82
|
+
buildUserPrompt,
|
|
83
|
+
type ParsedReasonResponse,
|
|
84
|
+
parseReasonResponse,
|
|
85
|
+
suggestionMessage,
|
|
86
|
+
} from "./reasoning/prompt";
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The structural-lookup seam — how the reasoner asks about code STRUCTURE
|
|
3
|
+
* (which component renders this, where a prop flows) without reading raw source
|
|
4
|
+
* into a model prompt. The concrete implementation is `tools/code-graph` (#2097);
|
|
5
|
+
* the harness only defines the interface and enforces the per-finding cap.
|
|
6
|
+
*
|
|
7
|
+
* A lookup is a SOFT cap, unlike the token ceiling. "one pass per finding + ~3-5
|
|
8
|
+
* lookups" (issue #2095): exceeding the lookup budget yields a `capped` result the
|
|
9
|
+
* reasoner reads as "finalize with what you have" — FEWER lookups, still a
|
|
10
|
+
* finding. Only the token ceiling is hard (it ends the run). Lookups spend no
|
|
11
|
+
* model tokens, so they are metered on their own count-based budget, independent
|
|
12
|
+
* of the token ledger.
|
|
13
|
+
*/
|
|
14
|
+
|
|
15
|
+
/**
|
|
16
|
+
* A structural query. Left deliberately open — `kind`/`target` are the minimal
|
|
17
|
+
* shape the harness needs to pass a query through; #2097 owns the real taxonomy
|
|
18
|
+
* of queries the code-graph answers.
|
|
19
|
+
*/
|
|
20
|
+
export interface LookupQuery {
|
|
21
|
+
readonly kind: string;
|
|
22
|
+
readonly target: string;
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
export type LookupResult =
|
|
26
|
+
| { readonly status: "ok"; readonly data: unknown }
|
|
27
|
+
/** The per-finding lookup budget is spent — finalize the pass with what you have. */
|
|
28
|
+
| { readonly status: "capped" };
|
|
29
|
+
|
|
30
|
+
/** The seam #2097 implements over the code-graph. */
|
|
31
|
+
export interface LookupTool {
|
|
32
|
+
readonly lookup: (query: LookupQuery) => Promise<LookupResult>;
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
/** Counts lookups against a per-finding budget. Fresh per pass. */
|
|
36
|
+
export class LookupCounter {
|
|
37
|
+
#used = 0;
|
|
38
|
+
|
|
39
|
+
constructor(readonly cap: number) {}
|
|
40
|
+
|
|
41
|
+
get used(): number {
|
|
42
|
+
return this.#used;
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
exhausted(): boolean {
|
|
46
|
+
return this.#used >= this.cap;
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
/** Charge one lookup slot. Called by {@link meterLookup} before it delegates. */
|
|
50
|
+
charge(): void {
|
|
51
|
+
this.#used += 1;
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
/**
|
|
56
|
+
* Wrap a lookup tool so it stops delegating once the per-finding cap is reached,
|
|
57
|
+
* returning `{ status: "capped" }` instead. Soft by design: the reasoner keeps
|
|
58
|
+
* going and produces a finding from what it already learned. The slot is charged
|
|
59
|
+
* before delegating, so a throwing tool still consumes its budget.
|
|
60
|
+
*/
|
|
61
|
+
export function meterLookup(tool: LookupTool, counter: LookupCounter): LookupTool {
|
|
62
|
+
return {
|
|
63
|
+
async lookup(query) {
|
|
64
|
+
if (counter.exhausted()) return { status: "capped" };
|
|
65
|
+
counter.charge();
|
|
66
|
+
return tool.lookup(query);
|
|
67
|
+
},
|
|
68
|
+
};
|
|
69
|
+
}
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The provider abstraction — the BYO seam of the AI lane.
|
|
3
|
+
*
|
|
4
|
+
* The runner drives ANY language model through this ONE interface. Nothing here
|
|
5
|
+
* names Anthropic, OpenAI, or any vendor: the customer implements {@link Provider}
|
|
6
|
+
* over their OWN SDK and their OWN key, and injects it. The engine ships no LLM
|
|
7
|
+
* credential and no vendor SDK — provider-agnostic by construction (epic #2083,
|
|
8
|
+
* "BYO provider; Claude Code not required").
|
|
9
|
+
*
|
|
10
|
+
* The one hard requirement the runner places on a provider: every response must
|
|
11
|
+
* report its {@link TokenUsage}. That is what makes the per-PR token ceiling
|
|
12
|
+
* enforceable across an untrusted, vendor-specific model. A provider that cannot
|
|
13
|
+
* report usage cannot be metered — so usage is part of the contract, not optional.
|
|
14
|
+
*/
|
|
15
|
+
|
|
16
|
+
/** One turn in a request. Vendor-neutral: `system` + a flat message list. */
|
|
17
|
+
export interface ProviderMessage {
|
|
18
|
+
readonly role: "user" | "assistant";
|
|
19
|
+
readonly content: string;
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
export interface ProviderRequest {
|
|
23
|
+
/** Optional system framing (the reasoning skills live here — seam for #2096). */
|
|
24
|
+
readonly system?: string;
|
|
25
|
+
readonly messages: readonly ProviderMessage[];
|
|
26
|
+
/** Upper bound on output tokens the caller wants; the provider may honor it. */
|
|
27
|
+
readonly maxOutputTokens?: number;
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
/**
|
|
31
|
+
* Token accounting for one completion. Both counts are required — the ceiling is
|
|
32
|
+
* enforced on `inputTokens + outputTokens`, so a provider that omits either can't
|
|
33
|
+
* be trusted to stay under budget.
|
|
34
|
+
*/
|
|
35
|
+
export interface TokenUsage {
|
|
36
|
+
readonly inputTokens: number;
|
|
37
|
+
readonly outputTokens: number;
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
export interface ProviderResponse {
|
|
41
|
+
readonly text: string;
|
|
42
|
+
readonly usage: TokenUsage;
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
/**
|
|
46
|
+
* The one method the runner calls. Implement it over your model of choice; the
|
|
47
|
+
* runner meters every call against the per-PR token ceiling (see
|
|
48
|
+
* {@link ./budget.meterProvider}). Implementations should surface transport
|
|
49
|
+
* failures by rejecting — the runner treats a rejected pass as a non-fatal,
|
|
50
|
+
* recorded error and keeps going (never fails the run).
|
|
51
|
+
*/
|
|
52
|
+
export interface Provider {
|
|
53
|
+
readonly complete: (request: ProviderRequest) => Promise<ProviderResponse>;
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
/** The total tokens one usage record spends — meaningful only for a {@link isMeterableUsage | meterable} record. */
|
|
57
|
+
export function usageTotal(usage: TokenUsage): number {
|
|
58
|
+
return usage.inputTokens + usage.outputTokens;
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
/**
|
|
62
|
+
* Whether a usage record can be trusted against the ceiling: BOTH counts finite
|
|
63
|
+
* and non-negative. A malformed provider response — `NaN`, `±Infinity`, or a
|
|
64
|
+
* negative count — makes the ceiling comparison silently mis-evaluate
|
|
65
|
+
* (`NaN >= ceiling` is always `false`), disabling the cap and letting the run
|
|
66
|
+
* spend past budget. The ledger gates on this and treats an unmeterable usage as
|
|
67
|
+
* AT-OR-OVER the ceiling, never under it (issue #2169).
|
|
68
|
+
*/
|
|
69
|
+
export function isMeterableUsage(usage: TokenUsage): boolean {
|
|
70
|
+
const ok = (n: number): boolean => Number.isFinite(n) && n >= 0;
|
|
71
|
+
return ok(usage.inputTokens) && ok(usage.outputTokens);
|
|
72
|
+
}
|
|
@@ -0,0 +1,168 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The concrete, ship-in-the-box {@link Provider} — Anthropic's Messages API over
|
|
3
|
+
* global `fetch` (epic #2083, issue #2182).
|
|
4
|
+
*
|
|
5
|
+
* The {@link Provider} interface (`../provider`) is the BYO seam: nothing in the
|
|
6
|
+
* runner names a vendor, and a customer can still inject their own model. This
|
|
7
|
+
* module fills that seam with ONE default implementation so a bare `LLM_API_KEY`
|
|
8
|
+
* drives the AI lane end to end — no vendor SDK, no extra install. The engine
|
|
9
|
+
* ships lean: the whole provider is a single `fetch` POST, mapping the vendor-
|
|
10
|
+
* neutral {@link ProviderRequest} onto the Anthropic request body and the reply
|
|
11
|
+
* back onto {@link ProviderResponse} (crucially, its {@link TokenUsage} — the
|
|
12
|
+
* runner's token ceiling is only enforceable because usage is reported).
|
|
13
|
+
*
|
|
14
|
+
* Per the {@link Provider} contract, this SURFACES transport failures by
|
|
15
|
+
* REJECTING: a non-2xx, a malformed body, or a network error throws. The runner
|
|
16
|
+
* catches a rejected pass and records it as a non-fatal error, so a provider
|
|
17
|
+
* that throws degrades the run to "no agent findings for that pass", never a
|
|
18
|
+
* crash — the AI lane stays non-blocking.
|
|
19
|
+
*
|
|
20
|
+
* A HANG is a failure a `throw` alone does not cover, so the `fetch` is bounded
|
|
21
|
+
* by an {@link AbortController} + timeout (mirroring `src/phone-home.ts`). A
|
|
22
|
+
* stalled provider — network stall, provider outage, no response — is aborted at
|
|
23
|
+
* the bound and surfaced as a rejection like any other transport failure, so the
|
|
24
|
+
* runner records the pass as errored and keeps going. Without this, a single hung
|
|
25
|
+
* request would hang the whole CI job, turning the non-blocking soft-degrade into
|
|
26
|
+
* a hard block on the customer's PR pipeline (issue #2192).
|
|
27
|
+
*/
|
|
28
|
+
import type { Provider, ProviderRequest, ProviderResponse, TokenUsage } from "../provider";
|
|
29
|
+
|
|
30
|
+
/**
|
|
31
|
+
* The default model for the low-cap advisory CI lane: a fast, cost-appropriate
|
|
32
|
+
* model. Overridable via the provider config (wired to `LLM_MODEL`), so dialing
|
|
33
|
+
* up to a stronger model is one env var, not a code change.
|
|
34
|
+
*/
|
|
35
|
+
export const DEFAULT_ANTHROPIC_MODEL = "claude-haiku-4-5-20251001";
|
|
36
|
+
|
|
37
|
+
/** Anthropic's dated API version header — pinned so a server-side default shift can't drift the shape. */
|
|
38
|
+
const ANTHROPIC_VERSION = "2023-06-01";
|
|
39
|
+
const DEFAULT_BASE_URL = "https://api.anthropic.com";
|
|
40
|
+
/** A pass is one short suggestion; cap output when the caller doesn't ask. */
|
|
41
|
+
const DEFAULT_MAX_OUTPUT_TOKENS = 1024;
|
|
42
|
+
|
|
43
|
+
/**
|
|
44
|
+
* The safe default wall-clock bound on one provider request (issue #2192). A
|
|
45
|
+
* low-cap advisory completion (≤ {@link DEFAULT_MAX_OUTPUT_TOKENS} output tokens
|
|
46
|
+
* on a fast model) returns in seconds; 60s leaves generous headroom for a slow
|
|
47
|
+
* healthy response while still bounding a genuine hang, so a stalled request
|
|
48
|
+
* aborts and degrades the AI lane softly instead of stalling CI. Override via
|
|
49
|
+
* {@link AnthropicProviderConfig.timeoutMs} (wired to `LLM_TIMEOUT_MS`).
|
|
50
|
+
*/
|
|
51
|
+
export const DEFAULT_REQUEST_TIMEOUT_MS = 60_000;
|
|
52
|
+
|
|
53
|
+
export interface AnthropicProviderConfig {
|
|
54
|
+
/** The customer's Anthropic key (from `LLM_API_KEY`). Never logged. */
|
|
55
|
+
readonly apiKey: string;
|
|
56
|
+
/** Model id. Defaults to {@link DEFAULT_ANTHROPIC_MODEL}. */
|
|
57
|
+
readonly model?: string;
|
|
58
|
+
/** Injectable for tests / alternate gateways. Defaults to `globalThis.fetch`. */
|
|
59
|
+
readonly fetchImpl?: typeof fetch;
|
|
60
|
+
/** Override the API origin (self-hosted gateway / proxy). Defaults to Anthropic. */
|
|
61
|
+
readonly baseUrl?: string;
|
|
62
|
+
/**
|
|
63
|
+
* Wall-clock bound on one request before it aborts (from `LLM_TIMEOUT_MS`).
|
|
64
|
+
* Defaults to {@link DEFAULT_REQUEST_TIMEOUT_MS}. A non-finite or non-positive
|
|
65
|
+
* value falls back to the default — a bad knob can't disable the bound and
|
|
66
|
+
* re-open the hang hole (issue #2192).
|
|
67
|
+
*/
|
|
68
|
+
readonly timeoutMs?: number;
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
/** Anthropic's `messages` response, narrowed at the boundary — never trusted raw. */
|
|
72
|
+
interface AnthropicResponseBody {
|
|
73
|
+
readonly content?: ReadonlyArray<{ readonly type?: string; readonly text?: string }>;
|
|
74
|
+
readonly usage?: { readonly input_tokens?: number; readonly output_tokens?: number };
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
/** Concatenate the text blocks of a Messages reply into one string. */
|
|
78
|
+
function textOf(body: AnthropicResponseBody): string {
|
|
79
|
+
if (!Array.isArray(body.content)) return "";
|
|
80
|
+
return body.content
|
|
81
|
+
.filter((block) => block.type === "text" && typeof block.text === "string")
|
|
82
|
+
.map((block) => block.text)
|
|
83
|
+
.join("");
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
/**
|
|
87
|
+
* A single reported token count, or `+Infinity` when the body reports it absent,
|
|
88
|
+
* non-numeric, `NaN`, or negative. Anthropic always reports both counts; a body
|
|
89
|
+
* that doesn't is malformed, so we parse-don't-validate at the boundary and map
|
|
90
|
+
* the bad count to a fail-closed sentinel rather than silently charging 0. The
|
|
91
|
+
* ledger meters `+Infinity` as at-or-over the ceiling, stopping the lane instead
|
|
92
|
+
* of letting a malformed response run past the cap (issue #2169).
|
|
93
|
+
*/
|
|
94
|
+
function meterableCount(raw: number | undefined): number {
|
|
95
|
+
return typeof raw === "number" && Number.isFinite(raw) && raw >= 0 ? raw : Number.POSITIVE_INFINITY;
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
/**
|
|
99
|
+
* The token usage the runner meters against the per-PR ceiling — parsed from the
|
|
100
|
+
* untrusted body, never read raw. A missing or malformed count maps to `+Infinity`
|
|
101
|
+
* (see {@link meterableCount}) so the ceiling fails closed on a bad provider
|
|
102
|
+
* response rather than under-counting toward a silent overspend.
|
|
103
|
+
*/
|
|
104
|
+
function usageOf(body: AnthropicResponseBody): TokenUsage {
|
|
105
|
+
return {
|
|
106
|
+
inputTokens: meterableCount(body.usage?.input_tokens),
|
|
107
|
+
outputTokens: meterableCount(body.usage?.output_tokens),
|
|
108
|
+
};
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
/**
|
|
112
|
+
* Build the ship-in-the-box Anthropic {@link Provider}. Inject into the runner's
|
|
113
|
+
* `RunInput.provider`; the harness meters every `complete` call against the token
|
|
114
|
+
* ceiling. Rejects on any transport / shape failure — the runner treats that as a
|
|
115
|
+
* non-fatal, recorded pass error.
|
|
116
|
+
*/
|
|
117
|
+
export function createAnthropicProvider(config: AnthropicProviderConfig): Provider {
|
|
118
|
+
const model = config.model ?? DEFAULT_ANTHROPIC_MODEL;
|
|
119
|
+
const fetchImpl = config.fetchImpl ?? globalThis.fetch;
|
|
120
|
+
const endpoint = `${config.baseUrl ?? DEFAULT_BASE_URL}/v1/messages`;
|
|
121
|
+
const timeoutMs =
|
|
122
|
+
typeof config.timeoutMs === "number" && Number.isFinite(config.timeoutMs) && config.timeoutMs > 0
|
|
123
|
+
? config.timeoutMs
|
|
124
|
+
: DEFAULT_REQUEST_TIMEOUT_MS;
|
|
125
|
+
|
|
126
|
+
return {
|
|
127
|
+
async complete(request: ProviderRequest): Promise<ProviderResponse> {
|
|
128
|
+
// Bound the whole exchange — connect AND body read: a hang at either stage
|
|
129
|
+
// aborts at the timeout and surfaces as a rejection, closing the
|
|
130
|
+
// non-blocking hole a bare `throw` can't (#2192). We own this controller,
|
|
131
|
+
// so any abort here IS our timeout. The signal cancels an in-flight body
|
|
132
|
+
// read too, so a stalled response can't slip past a resolved-headers fetch.
|
|
133
|
+
const controller = new AbortController();
|
|
134
|
+
const timer = setTimeout(() => controller.abort(), timeoutMs);
|
|
135
|
+
try {
|
|
136
|
+
const response = await fetchImpl(endpoint, {
|
|
137
|
+
method: "POST",
|
|
138
|
+
headers: {
|
|
139
|
+
"content-type": "application/json",
|
|
140
|
+
"x-api-key": config.apiKey,
|
|
141
|
+
"anthropic-version": ANTHROPIC_VERSION,
|
|
142
|
+
},
|
|
143
|
+
body: JSON.stringify({
|
|
144
|
+
model,
|
|
145
|
+
max_tokens: request.maxOutputTokens ?? DEFAULT_MAX_OUTPUT_TOKENS,
|
|
146
|
+
...(request.system !== undefined ? { system: request.system } : {}),
|
|
147
|
+
messages: request.messages.map((m) => ({ role: m.role, content: m.content })),
|
|
148
|
+
}),
|
|
149
|
+
signal: controller.signal,
|
|
150
|
+
});
|
|
151
|
+
|
|
152
|
+
if (!response.ok) {
|
|
153
|
+
// Surface as a rejection — the runner records the pass as a non-fatal
|
|
154
|
+
// error and keeps going. The key never reaches the message.
|
|
155
|
+
throw new Error(`Anthropic API returned HTTP ${response.status}`);
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
const body = (await response.json()) as AnthropicResponseBody;
|
|
159
|
+
return { text: textOf(body), usage: usageOf(body) };
|
|
160
|
+
} catch (error) {
|
|
161
|
+
if (controller.signal.aborted) throw new Error(`Anthropic request timed out after ${timeoutMs}ms`);
|
|
162
|
+
throw error;
|
|
163
|
+
} finally {
|
|
164
|
+
clearTimeout(timer);
|
|
165
|
+
}
|
|
166
|
+
},
|
|
167
|
+
};
|
|
168
|
+
}
|