any-doctor 0.0.6 → 0.0.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CONTEXT.md +16 -6
- package/bin/analysis-host.d.ts +6 -3
- package/bin/analysis-host.js +36 -25
- package/bin/analysis.d.ts +10 -2
- package/bin/analysis.js +83 -0
- package/bin/certify.d.ts +15 -0
- package/bin/certify.js +377 -0
- package/bin/cli.js +2 -2
- package/bin/cohort.d.ts +1 -0
- package/bin/cohort.js +1 -3
- package/bin/contract.d.ts +18 -1
- package/bin/doctor-loader.mjs +16 -142
- package/bin/sdk.js +12 -1
- package/docs/HANDOFF.md +64 -0
- package/docs/decisions.md +172 -0
- package/doctors/convex-doctor.mjs +30 -56
- package/doctors/effect-v4-doctor.mjs +1 -46
- package/doctors/openrouter-doctor.mjs +1 -46
- package/doctors/slop-doctor.fixtures.mjs +29 -0
- package/doctors/slop-doctor.mjs +29 -5
- package/fixtures/sensitivity/billing-triple/billing.ts +32 -0
- package/fixtures/sensitivity/billing-triple/expect.json +11 -0
- package/fixtures/sensitivity/dead-prompt-builders/expect.json +10 -0
- package/fixtures/sensitivity/dead-prompt-builders/prompts.ts +13 -0
- package/fixtures/sensitivity/ratelimiter-unbounded-collect/expect.json +9 -0
- package/fixtures/sensitivity/ratelimiter-unbounded-collect/rateLimiter.ts +18 -0
- package/package.json +1 -1
- package/skill/any-doctor.skill.md +25 -0
package/CONTEXT.md
CHANGED
|
@@ -39,9 +39,9 @@ call: the runner's bounded pool, the crash fold (a crash is data — id
|
|
|
39
39
|
plus full error detail, never a throw), the process-wide analysis fold,
|
|
40
40
|
the per-doctor paths, the file-count policy, and the timing. Progress
|
|
41
41
|
events (settle order) are the only side channel — the live line renders
|
|
42
|
-
them, it never joins the fold. Skips are a discovery fact, so
|
|
43
|
-
command layer, which owns discovery,
|
|
44
|
-
outcome.
|
|
42
|
+
them, it never joins the fold. Skips are a discovery fact, so they ride
|
|
43
|
+
in with the CohortSpec: the command layer, which owns discovery, hands
|
|
44
|
+
them to the run and the outcome comes back complete.
|
|
45
45
|
|
|
46
46
|
## Summary
|
|
47
47
|
|
|
@@ -160,9 +160,14 @@ An identity question asked through `ctx.analysis`: which **Binding** a
|
|
|
160
160
|
name resolves to, and every **Reference** to it. `ctx.analysis.bindings(file)`
|
|
161
161
|
returns the file's whole identity model in one answer — every binding
|
|
162
162
|
with its declaration span and its references (positions in ctx.search's
|
|
163
|
-
convention, plus read/write).
|
|
163
|
+
convention, plus read/write). `ctx.analysis.spans(file)` (D26) returns
|
|
164
|
+
every function-like **Span** — functions, methods, arrows, classes with
|
|
165
|
+
name, asyncness, and extent — as an AST fact, replacing the brace-counting
|
|
166
|
+
scans doctors once carried privately (the audit truncation bugs lived in
|
|
167
|
+
those approximations). The engine is optional (oxc-parser +
|
|
164
168
|
eslint-scope behind the Engine seam's second adapter): checks declare
|
|
165
|
-
the analysis they need on their CheckMeta (`needs`
|
|
169
|
+
the analysis they need on their CheckMeta (`needs` — vocabulary:
|
|
170
|
+
"bindings", "spans"), narrow without it,
|
|
166
171
|
and the report renders "narrowed" — a degraded run is visible, never
|
|
167
172
|
silent. References answer by position, so analysis queries compose with
|
|
168
173
|
rule queries: shapes from one engine, identities from the other.
|
|
@@ -242,7 +247,12 @@ workflow's second pass; a doctor ships only after surviving its attack.
|
|
|
242
247
|
|
|
243
248
|
Running a doctor program against its fixtures and diffing the findings.
|
|
244
249
|
The trust gate: a doctor program is not considered working until verify
|
|
245
|
-
passes.
|
|
250
|
+
passes. The policies that compose the gate — the claim contract, the
|
|
251
|
+
per-fixture diff, the shared innocent corpus, the duplicate-location
|
|
252
|
+
probe, the shared sensitivity corpus — live in one module, the
|
|
253
|
+
Certification harness (`src/certify.ts`), behind one interface:
|
|
254
|
+
`certify(mod, fixtures) -> result rows`. The doctor loader calls it once;
|
|
255
|
+
prevention tiers land there, not in loader choreography.
|
|
246
256
|
|
|
247
257
|
## Runner
|
|
248
258
|
|
package/bin/analysis-host.d.ts
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
|
-
import { AnalysisFile, Mode } from "./contract.js";
|
|
2
|
-
import { analysisStatus, analyzeBindings } from "./analysis.js";
|
|
1
|
+
import { AnalysisFile, AnalysisSpans, Mode } from "./contract.js";
|
|
2
|
+
import { analysisStatus, analyzeBindings, analyzeSpans } from "./analysis.js";
|
|
3
3
|
type Analyzer = typeof analyzeBindings;
|
|
4
|
+
type SpansAnalyzer = typeof analyzeSpans;
|
|
4
5
|
type Status = typeof analysisStatus;
|
|
5
6
|
export declare function clearAnalysisCache(): void;
|
|
6
7
|
export interface AnalysisRequestBody {
|
|
@@ -13,8 +14,10 @@ export type AnalysisResponse = {
|
|
|
13
14
|
reason?: string;
|
|
14
15
|
} | {
|
|
15
16
|
file: AnalysisFile;
|
|
17
|
+
} | {
|
|
18
|
+
file: AnalysisSpans;
|
|
16
19
|
} | {
|
|
17
20
|
error: string;
|
|
18
21
|
};
|
|
19
|
-
export declare function handleAnalysisRequest(req: AnalysisRequestBody, mode: Mode, analyzer?: Analyzer, status?: Status): AnalysisResponse;
|
|
22
|
+
export declare function handleAnalysisRequest(req: AnalysisRequestBody, mode: Mode, analyzer?: Analyzer, status?: Status, spansAnalyzer?: SpansAnalyzer): AnalysisResponse;
|
|
20
23
|
export {};
|
package/bin/analysis-host.js
CHANGED
|
@@ -1,18 +1,20 @@
|
|
|
1
1
|
import * as fs from "fs";
|
|
2
2
|
import * as path from "path";
|
|
3
|
-
import { analysisStatus, analyzeBindings } from "./analysis.js";
|
|
3
|
+
import { analysisStatus, analyzeBindings, analyzeSpans } from "./analysis.js";
|
|
4
4
|
import { searchBase, withinBase } from "./search-host.js";
|
|
5
5
|
// One cache per host process. The host lives in the runner process, so
|
|
6
6
|
// the lifetime is the any-doctor invocation; across a cohort's doctors
|
|
7
7
|
// the same unchanged file answers from memory.
|
|
8
8
|
const modelCache = new Map();
|
|
9
|
+
const spansCache = new Map();
|
|
9
10
|
// Test seam: the model cache is keyed by mtime+size for the process
|
|
10
11
|
// lifetime; tests bust it between cases. Invisible to slop-doctor's
|
|
11
12
|
// default run (test-file consumers are the documented narrowing).
|
|
12
13
|
export function clearAnalysisCache() {
|
|
13
14
|
modelCache.clear();
|
|
15
|
+
spansCache.clear();
|
|
14
16
|
}
|
|
15
|
-
export function handleAnalysisRequest(req, mode, analyzer = analyzeBindings, status = analysisStatus) {
|
|
17
|
+
export function handleAnalysisRequest(req, mode, analyzer = analyzeBindings, status = analysisStatus, spansAnalyzer = analyzeSpans) {
|
|
16
18
|
const base = searchBase(mode);
|
|
17
19
|
const root = typeof req.root === "string" ? path.resolve(req.root) : "";
|
|
18
20
|
if (base === "" || !withinBase(root, base)) {
|
|
@@ -22,35 +24,44 @@ export function handleAnalysisRequest(req, mode, analyzer = analyzeBindings, sta
|
|
|
22
24
|
const s = status();
|
|
23
25
|
return s.available ? { available: true } : { available: false, reason: s.reason };
|
|
24
26
|
}
|
|
25
|
-
if (req.kind === "bindings") {
|
|
27
|
+
if (req.kind === "bindings" || req.kind === "spans") {
|
|
26
28
|
if (typeof req.file !== "string" || req.file === "") {
|
|
27
|
-
return { error:
|
|
29
|
+
return { error: `ctx.analysis.${req.kind} needs a "file" path` };
|
|
28
30
|
}
|
|
29
31
|
const abs = path.resolve(root, req.file);
|
|
30
32
|
if (!withinBase(abs, root)) {
|
|
31
33
|
return { error: `ctx.analysis failed: file is outside the search root: ${req.file}` };
|
|
32
34
|
}
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
let size;
|
|
36
|
-
try {
|
|
37
|
-
const stat = fs.statSync(abs);
|
|
38
|
-
mtimeMs = stat.mtimeMs;
|
|
39
|
-
size = stat.size;
|
|
40
|
-
const cached = modelCache.get(abs);
|
|
41
|
-
if (cached && cached.mtimeMs === mtimeMs && cached.size === size)
|
|
42
|
-
return { file: cached.file };
|
|
43
|
-
source = fs.readFileSync(abs, "utf8");
|
|
35
|
+
if (req.kind === "bindings") {
|
|
36
|
+
return cachedModel(abs, root, modelCache, analyzer, req.file);
|
|
44
37
|
}
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
38
|
+
return cachedModel(abs, root, spansCache, spansAnalyzer, req.file);
|
|
39
|
+
}
|
|
40
|
+
return { error: `unknown analysis kind ${JSON.stringify(req.kind)} — known kinds: available, bindings, spans` };
|
|
41
|
+
}
|
|
42
|
+
// The shared per-file model lifecycle: stat (cache hit on mtime+size),
|
|
43
|
+
// read, compute, cache. Bindings and spans are the same policy over two
|
|
44
|
+
// analyzers and two caches.
|
|
45
|
+
function cachedModel(abs, root, cache, compute, relFile) {
|
|
46
|
+
let source;
|
|
47
|
+
let mtimeMs;
|
|
48
|
+
let size;
|
|
49
|
+
try {
|
|
50
|
+
const stat = fs.statSync(abs);
|
|
51
|
+
mtimeMs = stat.mtimeMs;
|
|
52
|
+
size = stat.size;
|
|
53
|
+
const cached = cache.get(abs);
|
|
54
|
+
if (cached && cached.mtimeMs === mtimeMs && cached.size === size)
|
|
55
|
+
return { file: cached.file };
|
|
56
|
+
source = fs.readFileSync(abs, "utf8");
|
|
57
|
+
}
|
|
58
|
+
catch {
|
|
59
|
+
return { error: `ctx.analysis failed: cannot read ${relFile}` };
|
|
54
60
|
}
|
|
55
|
-
|
|
61
|
+
const rel = path.relative(root, abs);
|
|
62
|
+
const r = compute(rel, source);
|
|
63
|
+
if (!r.ok)
|
|
64
|
+
return { error: r.error };
|
|
65
|
+
cache.set(abs, { mtimeMs, size, file: r.file });
|
|
66
|
+
return { file: r.file };
|
|
56
67
|
}
|
package/bin/analysis.d.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { AnalysisFile, BindingInfo, BindingRef } from "./contract.js";
|
|
2
|
-
export type { AnalysisFile, BindingInfo, BindingRef };
|
|
1
|
+
import { AnalysisFile, AnalysisSpans, BindingInfo, BindingRef, SpanInfo } from "./contract.js";
|
|
2
|
+
export type { AnalysisFile, AnalysisSpans, BindingInfo, BindingRef, SpanInfo };
|
|
3
3
|
export interface AnalysisStatus {
|
|
4
4
|
available: true;
|
|
5
5
|
}
|
|
@@ -15,4 +15,12 @@ export type AnalysisResult = {
|
|
|
15
15
|
ok: false;
|
|
16
16
|
error: string;
|
|
17
17
|
};
|
|
18
|
+
export type SpansResult = {
|
|
19
|
+
ok: true;
|
|
20
|
+
file: AnalysisSpans;
|
|
21
|
+
} | {
|
|
22
|
+
ok: false;
|
|
23
|
+
error: string;
|
|
24
|
+
};
|
|
25
|
+
export declare function analyzeSpans(file: string, source: string): SpansResult;
|
|
18
26
|
export declare function analyzeBindings(file: string, source: string): AnalysisResult;
|
package/bin/analysis.js
CHANGED
|
@@ -20,6 +20,89 @@ export function analysisStatus() {
|
|
|
20
20
|
const stack = loadStack();
|
|
21
21
|
return stack.error !== undefined ? { available: false, reason: stack.error } : { available: true };
|
|
22
22
|
}
|
|
23
|
+
// One file in, every function-like span out (D26): an AST fact answering
|
|
24
|
+
// what the doctors' private brace-counting copies could only approximate.
|
|
25
|
+
// Semicolons inside multi-line callbacks, strings containing braces, JSX —
|
|
26
|
+
// none of it can truncate a span that the parser already knows. Named
|
|
27
|
+
// declarations carry their id; arrows and anonymous expressions carry null.
|
|
28
|
+
export function analyzeSpans(file, source) {
|
|
29
|
+
const stack = loadStack();
|
|
30
|
+
if (stack.error !== undefined)
|
|
31
|
+
return { ok: false, error: stack.error };
|
|
32
|
+
let program;
|
|
33
|
+
try {
|
|
34
|
+
program = stack.parseSync(file, source, { sourceType: "module" }).program;
|
|
35
|
+
}
|
|
36
|
+
catch (e) {
|
|
37
|
+
return { ok: false, error: `analysis failed to parse ${file}: ${e instanceof Error ? e.message : String(e)}` };
|
|
38
|
+
}
|
|
39
|
+
const pos = positioner(source);
|
|
40
|
+
const spans = [];
|
|
41
|
+
const visit = (node) => {
|
|
42
|
+
if (!node || typeof node !== "object")
|
|
43
|
+
return;
|
|
44
|
+
const n = node;
|
|
45
|
+
let span = null;
|
|
46
|
+
if (n.type === "FunctionDeclaration" || n.type === "TSDeclareFunction") {
|
|
47
|
+
span = spanOf(n, "function", idName(n.id));
|
|
48
|
+
}
|
|
49
|
+
else if (n.type === "FunctionExpression") {
|
|
50
|
+
span = spanOf(n, "function-expression", idName(n.id));
|
|
51
|
+
}
|
|
52
|
+
else if (n.type === "ArrowFunctionExpression") {
|
|
53
|
+
span = spanOf(n, "arrow", null);
|
|
54
|
+
}
|
|
55
|
+
else if (n.type === "ClassDeclaration" || n.type === "ClassExpression") {
|
|
56
|
+
span = spanOf(n, "class", idName(n.id));
|
|
57
|
+
}
|
|
58
|
+
else if (n.type === "MethodDefinition" || n.type === "TSAbstractMethodDefinition") {
|
|
59
|
+
span = spanOf(n.value, "method", propertyName(n.key));
|
|
60
|
+
}
|
|
61
|
+
if (span !== null)
|
|
62
|
+
spans.push(span);
|
|
63
|
+
for (const key of Object.keys(n)) {
|
|
64
|
+
if (key === "range" || key === "start" || key === "end")
|
|
65
|
+
continue;
|
|
66
|
+
const v = n[key];
|
|
67
|
+
if (Array.isArray(v)) {
|
|
68
|
+
for (const child of v)
|
|
69
|
+
if (isNode(child))
|
|
70
|
+
visit(child);
|
|
71
|
+
}
|
|
72
|
+
else if (isNode(v))
|
|
73
|
+
visit(v);
|
|
74
|
+
}
|
|
75
|
+
};
|
|
76
|
+
visit(program);
|
|
77
|
+
return { ok: true, file: { file, spans } };
|
|
78
|
+
function spanOf(n, kind, name) {
|
|
79
|
+
if (typeof n.start !== "number" || typeof n.end !== "number")
|
|
80
|
+
return null;
|
|
81
|
+
return {
|
|
82
|
+
kind,
|
|
83
|
+
name,
|
|
84
|
+
async: n.async === true,
|
|
85
|
+
line: pos.line(n.start),
|
|
86
|
+
column: pos.column(n.start),
|
|
87
|
+
endLine: pos.line(n.end),
|
|
88
|
+
endColumn: pos.column(n.end),
|
|
89
|
+
};
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
function idName(id) {
|
|
93
|
+
return id && typeof id.name === "string" ? id.name : null;
|
|
94
|
+
}
|
|
95
|
+
function propertyName(key) {
|
|
96
|
+
const k = key;
|
|
97
|
+
if (k && typeof k.name === "string")
|
|
98
|
+
return k.name;
|
|
99
|
+
if (k && typeof k.value === "string")
|
|
100
|
+
return k.value; // computed / literal keys
|
|
101
|
+
return null;
|
|
102
|
+
}
|
|
103
|
+
function isNode(v) {
|
|
104
|
+
return Boolean(v) && typeof v === "object" && typeof v.type === "string";
|
|
105
|
+
}
|
|
23
106
|
// One file in, one identity model out: every binding (declarations,
|
|
24
107
|
// parameters, imports) with the span of its declaring node and every
|
|
25
108
|
// reference to it, read or write. Type-position identifiers never become
|
package/bin/certify.d.ts
ADDED
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
import { buildCtx } from "./sdk.js";
|
|
2
|
+
import type { Fixture, FixtureResult, RunResult } from "./contract.js";
|
|
3
|
+
export interface DoctorModule {
|
|
4
|
+
meta?: unknown;
|
|
5
|
+
doctor?: (ctx: ReturnType<typeof buildCtx>["ctx"]) => unknown;
|
|
6
|
+
}
|
|
7
|
+
export declare function runOnce(root: string, mod: DoctorModule, opts: {
|
|
8
|
+
includeTests: boolean;
|
|
9
|
+
}): Promise<RunResult>;
|
|
10
|
+
export declare class ClaimContractViolation extends Error {
|
|
11
|
+
readonly problems: string[];
|
|
12
|
+
constructor(problems: string[]);
|
|
13
|
+
}
|
|
14
|
+
export declare function validateClaimContract(mod: DoctorModule): void;
|
|
15
|
+
export declare function certify(mod: DoctorModule, fixtures: Fixture[]): Promise<FixtureResult[]>;
|
package/bin/certify.js
ADDED
|
@@ -0,0 +1,377 @@
|
|
|
1
|
+
import * as fs from "fs";
|
|
2
|
+
import * as os from "os";
|
|
3
|
+
import * as path from "path";
|
|
4
|
+
import { buildCtx, setAnalysisDisabled, probeAnalysisAvailable } from "./sdk.js";
|
|
5
|
+
import * as contract from "./contract.js";
|
|
6
|
+
// One execution of a doctor program against a root, framed as a run result.
|
|
7
|
+
// Owned here because both halves need it: the loader's run mode and every
|
|
8
|
+
// certification sandbox. Typed, not Record<string, unknown> — consumers
|
|
9
|
+
// (certify, the loader frame) read .findings and .meta off it directly.
|
|
10
|
+
export async function runOnce(root, mod, opts) {
|
|
11
|
+
const started = Date.now();
|
|
12
|
+
const { ctx, getFindings } = buildCtx(root, opts);
|
|
13
|
+
const fileCount = ctx.files.list().length;
|
|
14
|
+
const result = mod.doctor(ctx);
|
|
15
|
+
if (!result || typeof result.then !== "function") {
|
|
16
|
+
throw new Error("doctor() must be async — declare it `export async function doctor(ctx)`");
|
|
17
|
+
}
|
|
18
|
+
return result.then(() => ({
|
|
19
|
+
protocolVersion: contract.PROTOCOL_VERSION,
|
|
20
|
+
kind: "run",
|
|
21
|
+
root,
|
|
22
|
+
fileCount,
|
|
23
|
+
durationMs: Date.now() - started,
|
|
24
|
+
meta: mod.meta,
|
|
25
|
+
findings: getFindings(),
|
|
26
|
+
}));
|
|
27
|
+
}
|
|
28
|
+
// The claim contract (D23): certification requires each declared check to
|
|
29
|
+
// state the observable condition it establishes, its innocent lookalikes,
|
|
30
|
+
// and - when it needs the identity engine - what happens on unknown. Prose
|
|
31
|
+
// impact is not a testable claim. Thrown before any sandbox runs; the
|
|
32
|
+
// loader renders the problems and exits 3.
|
|
33
|
+
export class ClaimContractViolation extends Error {
|
|
34
|
+
constructor(problems) {
|
|
35
|
+
super("claim contract violations:\n " + problems.join("\n "));
|
|
36
|
+
this.problems = problems;
|
|
37
|
+
this.name = "ClaimContractViolation";
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
export function validateClaimContract(mod) {
|
|
41
|
+
var _a;
|
|
42
|
+
const checks = (_a = mod.meta) === null || _a === void 0 ? void 0 : _a.checks;
|
|
43
|
+
if (!Array.isArray(checks))
|
|
44
|
+
return;
|
|
45
|
+
const problems = [];
|
|
46
|
+
for (const c of checks) {
|
|
47
|
+
if (typeof c.claim !== "string" || c.claim.trim().length === 0) {
|
|
48
|
+
problems.push(`check "${String(c.id)}": claim is required — one sentence, the observable condition detected, not the consequence`);
|
|
49
|
+
}
|
|
50
|
+
if (!Array.isArray(c.lookalikes) || c.lookalikes.length === 0) {
|
|
51
|
+
problems.push(`check "${String(c.id)}": lookalikes is required — at least one innocent shape that must stay silent`);
|
|
52
|
+
}
|
|
53
|
+
if (Array.isArray(c.needs) && c.needs.length > 0
|
|
54
|
+
&& (c.onUnknown !== "narrow" && c.onUnknown !== "skip")) {
|
|
55
|
+
problems.push(`check "${String(c.id)}": onUnknown is required when needs is declared — "narrow" or "skip"`);
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
if (problems.length > 0)
|
|
59
|
+
throw new ClaimContractViolation(problems);
|
|
60
|
+
}
|
|
61
|
+
const SKIP_ANALYSIS = "analysis engine unavailable — pins the analysis-on path";
|
|
62
|
+
// The result-row constructors: every policy speaks in the same row shape,
|
|
63
|
+
// so a verify frame's consumers never see policy-specific spellings.
|
|
64
|
+
const okRow = (name) => ({ name, ok: true, missing: [], unexpected: [] });
|
|
65
|
+
const skipRow = (name) => ({ ...okRow(name), skipped: SKIP_ANALYSIS });
|
|
66
|
+
const errorRow = (name, e) => ({
|
|
67
|
+
name, ok: false, missing: [], unexpected: [],
|
|
68
|
+
error: e instanceof Error ? e.message : String(e),
|
|
69
|
+
});
|
|
70
|
+
function materializeSeed(tmp, rel, content) {
|
|
71
|
+
const abs = path.resolve(tmp, rel);
|
|
72
|
+
if (abs !== tmp && !abs.startsWith(tmp + path.sep)) {
|
|
73
|
+
throw new Error(`fixture seed path escapes the sandbox: ${rel}`);
|
|
74
|
+
}
|
|
75
|
+
fs.mkdirSync(path.dirname(abs), { recursive: true });
|
|
76
|
+
fs.writeFileSync(abs, content);
|
|
77
|
+
}
|
|
78
|
+
// The one sandbox lifecycle: materialize a seed into a fresh temp dir, run,
|
|
79
|
+
// clean up. A crashing sandbox is a named failing result (or the caller's
|
|
80
|
+
// error row) — siblings always run.
|
|
81
|
+
async function inSandbox(seed, run) {
|
|
82
|
+
const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "any-doctor-verify-"));
|
|
83
|
+
try {
|
|
84
|
+
for (const [rel, content] of Object.entries(seed))
|
|
85
|
+
materializeSeed(tmp, rel, content);
|
|
86
|
+
return await run(tmp);
|
|
87
|
+
}
|
|
88
|
+
finally {
|
|
89
|
+
fs.rmSync(tmp, { recursive: true, force: true });
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
// The one corpus walker: a directory tree as a seed map. `skip` carves out
|
|
93
|
+
// manifest files (the sensitivity corpus's expect.json).
|
|
94
|
+
function collectSeed(dir, prefix, seed, skip) {
|
|
95
|
+
for (const entry of fs.readdirSync(dir, { withFileTypes: true })) {
|
|
96
|
+
const abs = path.join(dir, entry.name);
|
|
97
|
+
const rel = prefix ? prefix + "/" + entry.name : entry.name;
|
|
98
|
+
if (entry.isDirectory())
|
|
99
|
+
collectSeed(abs, rel, seed, skip);
|
|
100
|
+
else if (entry.name !== skip)
|
|
101
|
+
seed[rel] = fs.readFileSync(abs, "utf8");
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
// The duplicate-location sensitivity probe (D24, hardened in the review
|
|
105
|
+
// loop): a doctor's own flag-shaped fixture — the one with the most
|
|
106
|
+
// expected findings concentrated in a single seeded file — is re-planted
|
|
107
|
+
// at three locations:
|
|
108
|
+
//
|
|
109
|
+
// 1. the original file, untouched (every fact the fixture proved);
|
|
110
|
+
// 2. a byte-identical twin module (the cross-file location — checks
|
|
111
|
+
// whose violation is inherently cross-module must fire at both);
|
|
112
|
+
// 3. a pair file: the violation twice INSIDE one file, wrapped in two
|
|
113
|
+
// functions (the billing.ts shape — identical chains in separate
|
|
114
|
+
// functions of one module, collapsed to one finding by a dedup keyed
|
|
115
|
+
// on normalized statement text).
|
|
116
|
+
//
|
|
117
|
+
// The pair file's bodies are transformed (exports stripped, imports
|
|
118
|
+
// hoisted) so the two in-file copies stay byte-identical TO EACH OTHER —
|
|
119
|
+
// which is exactly what a text-keyed dedup collapses. Assertions, counting
|
|
120
|
+
// only the rules the fixture expected at that file (unrelated rules cannot
|
|
121
|
+
// inflate a floor):
|
|
122
|
+
//
|
|
123
|
+
// - original and twin must each independently reproduce the fixture's
|
|
124
|
+
// expected count;
|
|
125
|
+
// - when the witness fixture itself seeded TWO OR MORE expected findings
|
|
126
|
+
// in the one file (proof the check reports per-violation, not one
|
|
127
|
+
// verdict per file — openrouter's "no error check anywhere in the
|
|
128
|
+
// file" is a legitimate file-scoped claim), the pair file must yield
|
|
129
|
+
// at least 2x that count — UNLESS it yields zero, which is exempt by
|
|
130
|
+
// design: the pair wrap strips exports and nests bodies in functions,
|
|
131
|
+
// which can remove the very context a check needs (an export-keyed
|
|
132
|
+
// check cannot fire inside the pair file). Less-than-double but
|
|
133
|
+
// nonzero is the collapse signature — N identical violations in one
|
|
134
|
+
// file reduced to a single finding. Doctors whose every fixture
|
|
135
|
+
// seeds at most one violation per file get twin+original policing
|
|
136
|
+
// only and the probe row says so; the skill tells authors to seed a
|
|
137
|
+
// two-violation fixture so the probe can police same-file dedup.
|
|
138
|
+
function buildDuplicateLocationProbe(fixtures) {
|
|
139
|
+
var _a;
|
|
140
|
+
// Witness selection: the flag-shaped fixture with the most expected
|
|
141
|
+
// findings in one seeded file — a 2-in-one-file witness catches per-file
|
|
142
|
+
// collapse directly; any 1-file witness still exercises the twin and
|
|
143
|
+
// pair locations.
|
|
144
|
+
let best = null;
|
|
145
|
+
for (const fixture of fixtures) {
|
|
146
|
+
const perFile = new Map();
|
|
147
|
+
for (const e of fixture.expected)
|
|
148
|
+
perFile.set(e.file, ((_a = perFile.get(e.file)) !== null && _a !== void 0 ? _a : 0) + 1);
|
|
149
|
+
for (const [file, count] of perFile) {
|
|
150
|
+
if (typeof fixture.seed[file] === "string" && (best === null || count > best.count)) {
|
|
151
|
+
best = { fixture, file, count };
|
|
152
|
+
}
|
|
153
|
+
}
|
|
154
|
+
}
|
|
155
|
+
if (best === null)
|
|
156
|
+
return null;
|
|
157
|
+
const { fixture, file: originalFile, count: expectedCount } = best;
|
|
158
|
+
// Rule-less expectations (no `rule` field) can only be matched by
|
|
159
|
+
// rule-less findings — key those as "" so the count filter keeps them.
|
|
160
|
+
const rules = new Set(fixture.expected
|
|
161
|
+
.filter((e) => e.file === originalFile)
|
|
162
|
+
.map((e) => { var _a; return (_a = e.rule) !== null && _a !== void 0 ? _a : ""; }));
|
|
163
|
+
const twinFile = "__probe_twin__/" + originalFile.split("/").pop();
|
|
164
|
+
const pairFile = "__probe_pair__/" + originalFile.split("/").pop();
|
|
165
|
+
if (typeof fixture.seed[twinFile] === "string" || typeof fixture.seed[pairFile] === "string")
|
|
166
|
+
return null;
|
|
167
|
+
const pairContent = pairFileContent(fixture.seed[originalFile]);
|
|
168
|
+
if (pairContent === null)
|
|
169
|
+
return null;
|
|
170
|
+
const seed = { ...fixture.seed, [twinFile]: fixture.seed[originalFile], [pairFile]: pairContent };
|
|
171
|
+
return {
|
|
172
|
+
seed,
|
|
173
|
+
fixtureName: fixture.name,
|
|
174
|
+
analysisOff: fixture.analysis === "off",
|
|
175
|
+
originalFile,
|
|
176
|
+
twinFile,
|
|
177
|
+
pairFile,
|
|
178
|
+
expectedCount,
|
|
179
|
+
rules,
|
|
180
|
+
};
|
|
181
|
+
}
|
|
182
|
+
// The in-file pair: the original content twice, each copy wrapped in a
|
|
183
|
+
// function (module-level declarations cannot repeat, and both copies get
|
|
184
|
+
// the identical transform so they stay byte-equal to each other).
|
|
185
|
+
function pairFileContent(original) {
|
|
186
|
+
const lines = original.split("\n").filter((l) => !/^\s*import\b/.test(l) && !/^\s*export\s*\{/.test(l)
|
|
187
|
+
&& !/^\s*export\s+type\s*\{/.test(l) && !/^\s*export\s+\*\s*from/.test(l));
|
|
188
|
+
const body = lines.map((l) => l
|
|
189
|
+
.replace(/^(\s*)export default (?=(?:async\s+)?(?:function|class)\b)/, "$1")
|
|
190
|
+
.replace(/^(\s*)export default /, "$1const __probeDefault = ")
|
|
191
|
+
.replace(/^(\s*)export (?=(?:async\s+)?(?:function|class|const|let|var|type|interface|enum|abstract|declare)\b)/, "$1")).join("\n");
|
|
192
|
+
if (body.trim().length === 0)
|
|
193
|
+
return null;
|
|
194
|
+
const imports = original.split("\n").filter((l) => /^\s*import\b/.test(l)).join("\n");
|
|
195
|
+
return [
|
|
196
|
+
imports,
|
|
197
|
+
imports ? "" : null,
|
|
198
|
+
"// any-doctor duplicate-location probe — copy 1",
|
|
199
|
+
"function __anyDoctorProbeA() {",
|
|
200
|
+
body,
|
|
201
|
+
"}",
|
|
202
|
+
"// copy 2 — same violation, different location, same file",
|
|
203
|
+
"function __anyDoctorProbeB() {",
|
|
204
|
+
body,
|
|
205
|
+
"}",
|
|
206
|
+
"",
|
|
207
|
+
].filter((l) => l !== null).join("\n");
|
|
208
|
+
}
|
|
209
|
+
// The certification entry point: every policy, in gate order, as result
|
|
210
|
+
// rows a verify frame can carry.
|
|
211
|
+
export async function certify(mod, fixtures) {
|
|
212
|
+
var _a;
|
|
213
|
+
validateClaimContract(mod);
|
|
214
|
+
const results = [];
|
|
215
|
+
// Only a doctor whose checks declare analysis needs can have
|
|
216
|
+
// analysis-on sandboxes — probing anyone else would make a channel-less
|
|
217
|
+
// direct invocation fail fixtures that never touch analysis at all.
|
|
218
|
+
const declaresNeeds = contract.narrowedCheckIds(mod.meta).length > 0;
|
|
219
|
+
let analysisAvailable;
|
|
220
|
+
// The one skip policy (D25): an analysis-on sandbox whose engine is not
|
|
221
|
+
// installed here is an honest skip, not a failure — expectations belong
|
|
222
|
+
// to the full-power path. One decision for every policy that asks.
|
|
223
|
+
const skipFor = async (analysisOn) => {
|
|
224
|
+
if (!declaresNeeds || !analysisOn)
|
|
225
|
+
return false;
|
|
226
|
+
if (analysisAvailable === undefined) {
|
|
227
|
+
analysisAvailable = await inSandbox({}, async (tmp) => probeAnalysisAvailable(tmp));
|
|
228
|
+
}
|
|
229
|
+
return analysisAvailable === false;
|
|
230
|
+
};
|
|
231
|
+
for (const fixture of fixtures) {
|
|
232
|
+
try {
|
|
233
|
+
// The fixture's declared analysis mode: "off" forces the degraded
|
|
234
|
+
// path (pinning the narrowed behavior); the default "on" runs with
|
|
235
|
+
// the engine — and skips honestly when it is not installed here,
|
|
236
|
+
// rather than failing a fixture whose expectations belong to the
|
|
237
|
+
// full-power path.
|
|
238
|
+
setAnalysisDisabled(fixture.analysis === "off");
|
|
239
|
+
if (await skipFor(fixture.analysis !== "off")) {
|
|
240
|
+
results.push(skipRow(fixture.name));
|
|
241
|
+
continue;
|
|
242
|
+
}
|
|
243
|
+
// Verify always lists everything (includeTestsFor): the sandbox is
|
|
244
|
+
// the doctor's own world — a seed named *.test.ts is deliberate
|
|
245
|
+
// test data (effect-v4-doctor's sleep-in-test depends on it).
|
|
246
|
+
const result = await inSandbox(fixture.seed, (tmp) => runOnce(tmp, mod, { includeTests: true }));
|
|
247
|
+
const diff = contract.compareFindings(fixture.expected, result.findings);
|
|
248
|
+
results.push({ name: fixture.name, ok: diff.missing.length === 0 && diff.unexpected.length === 0, ...diff });
|
|
249
|
+
}
|
|
250
|
+
catch (e) {
|
|
251
|
+
results.push(errorRow(fixture.name, e));
|
|
252
|
+
}
|
|
253
|
+
finally {
|
|
254
|
+
setAnalysisDisabled(false);
|
|
255
|
+
}
|
|
256
|
+
}
|
|
257
|
+
// The shared innocent corpus (D23): files that look guilty but aren't —
|
|
258
|
+
// the audit counterexamples as commons. Every doctor runs against them
|
|
259
|
+
// with expected: []; a finding here is a false positive by definition,
|
|
260
|
+
// whoever wrote the check.
|
|
261
|
+
const innocentDir = corpusDir("innocent");
|
|
262
|
+
if (innocentDir !== null) {
|
|
263
|
+
const seed = {};
|
|
264
|
+
collectSeed(innocentDir, "", seed);
|
|
265
|
+
try {
|
|
266
|
+
const diff = await inSandbox(seed, (tmp) => runOnce(tmp, mod, { includeTests: true }))
|
|
267
|
+
.then((r) => contract.compareFindings([], r.findings));
|
|
268
|
+
results.push({
|
|
269
|
+
name: "shared innocent corpus (" + Object.keys(seed).length + " files)",
|
|
270
|
+
ok: diff.missing.length === 0 && diff.unexpected.length === 0,
|
|
271
|
+
...diff,
|
|
272
|
+
});
|
|
273
|
+
}
|
|
274
|
+
catch (e) {
|
|
275
|
+
results.push(errorRow("shared innocent corpus", e));
|
|
276
|
+
}
|
|
277
|
+
}
|
|
278
|
+
// The duplicate-location sensitivity probe (D24).
|
|
279
|
+
const probe = buildDuplicateLocationProbe(fixtures);
|
|
280
|
+
if (probe !== null) {
|
|
281
|
+
const name = `duplicate-location sensitivity (from "${probe.fixtureName}")` + (probe.expectedCount < 2
|
|
282
|
+
? " — no two-in-one-file witness: same-file collapse unpolicied"
|
|
283
|
+
: "");
|
|
284
|
+
if (await skipFor(!probe.analysisOff)) {
|
|
285
|
+
results.push(skipRow(name));
|
|
286
|
+
}
|
|
287
|
+
else {
|
|
288
|
+
try {
|
|
289
|
+
setAnalysisDisabled(probe.analysisOff);
|
|
290
|
+
const findings = await inSandbox(probe.seed, (tmp) => runOnce(tmp, mod, { includeTests: true }))
|
|
291
|
+
.then((r) => r.findings);
|
|
292
|
+
const at = (file) => findings.filter((f) => { var _a; return f.file === file && probe.rules.has((_a = f.rule) !== null && _a !== void 0 ? _a : ""); }).length;
|
|
293
|
+
const problems = [];
|
|
294
|
+
if (at(probe.originalFile) < probe.expectedCount) {
|
|
295
|
+
problems.push(`the untouched original no longer produces its ${probe.expectedCount} finding(s) — got ${at(probe.originalFile)}`);
|
|
296
|
+
}
|
|
297
|
+
if (at(probe.twinFile) < probe.expectedCount) {
|
|
298
|
+
problems.push(`the byte-identical twin module produces ${at(probe.twinFile)} finding(s) where ${probe.expectedCount} expected — a dedup keyed on statement text collapses distinct violations sharing a body`);
|
|
299
|
+
}
|
|
300
|
+
if (probe.expectedCount >= 2 && at(probe.pairFile) > 0 && at(probe.pairFile) < probe.expectedCount * 2) {
|
|
301
|
+
problems.push(`the same violation planted twice in ONE file yields ${at(probe.pairFile)} finding(s) — the collapse signature (N identical violations reduced to one; the billing.ts bug class)`);
|
|
302
|
+
}
|
|
303
|
+
results.push(problems.length === 0
|
|
304
|
+
? okRow(name)
|
|
305
|
+
: { ...errorRow(name, problems.join("; ")), missing: [], unexpected: [] });
|
|
306
|
+
}
|
|
307
|
+
catch (e) {
|
|
308
|
+
results.push(errorRow(name, e));
|
|
309
|
+
}
|
|
310
|
+
finally {
|
|
311
|
+
setAnalysisDisabled(false);
|
|
312
|
+
}
|
|
313
|
+
}
|
|
314
|
+
}
|
|
315
|
+
// The sensitivity corpus (D24): the innocent corpus's complement —
|
|
316
|
+
// confirmed-real patterns from the audits, patterns that MUST produce
|
|
317
|
+
// findings. Each case directory carries its seed files plus an expect.json
|
|
318
|
+
// mapping doctor id -> expected findings; a doctor only runs the cases it
|
|
319
|
+
// has stakes in.
|
|
320
|
+
const sensDir = corpusDir("sensitivity");
|
|
321
|
+
if (sensDir !== null) {
|
|
322
|
+
for (const entry of fs.readdirSync(sensDir, { withFileTypes: true }).sort((a, b) => a.name.localeCompare(b.name))) {
|
|
323
|
+
if (!entry.isDirectory())
|
|
324
|
+
continue;
|
|
325
|
+
const caseDir = path.join(sensDir, entry.name);
|
|
326
|
+
const expectPath = path.join(caseDir, "expect.json");
|
|
327
|
+
if (!fs.existsSync(expectPath))
|
|
328
|
+
continue;
|
|
329
|
+
const name = `sensitivity: ${entry.name}`;
|
|
330
|
+
let manifest;
|
|
331
|
+
try {
|
|
332
|
+
manifest = JSON.parse(fs.readFileSync(expectPath, "utf8"));
|
|
333
|
+
}
|
|
334
|
+
catch (e) {
|
|
335
|
+
results.push({ ...okRow(name), ok: false, error: "unreadable expect.json: " + (e instanceof Error ? e.message : String(e)) });
|
|
336
|
+
continue;
|
|
337
|
+
}
|
|
338
|
+
const expected = (_a = manifest === null || manifest === void 0 ? void 0 : manifest.expect) === null || _a === void 0 ? void 0 : _a[String(mod.meta.id)];
|
|
339
|
+
if (!Array.isArray(expected))
|
|
340
|
+
continue;
|
|
341
|
+
try {
|
|
342
|
+
const seed = {};
|
|
343
|
+
collectSeed(caseDir, "", seed, "expect.json");
|
|
344
|
+
if (await skipFor(true)) {
|
|
345
|
+
results.push(skipRow(name));
|
|
346
|
+
continue;
|
|
347
|
+
}
|
|
348
|
+
const diff = await inSandbox(seed, (tmp) => runOnce(tmp, mod, { includeTests: true }))
|
|
349
|
+
.then((r) => contract.compareFindings(expected, r.findings));
|
|
350
|
+
results.push({ name, ok: diff.missing.length === 0 && diff.unexpected.length === 0, ...diff });
|
|
351
|
+
}
|
|
352
|
+
catch (e) {
|
|
353
|
+
results.push(errorRow(name, e));
|
|
354
|
+
}
|
|
355
|
+
}
|
|
356
|
+
}
|
|
357
|
+
return results;
|
|
358
|
+
}
|
|
359
|
+
// A shipped corpus directory, resolved next to the compiled module (bin/'s
|
|
360
|
+
// sibling fixtures/), or null when absent — an unbundled checkout still
|
|
361
|
+
// certifies, just without the commons. ANY_DOCTOR_CORPUS_ROOT is the test
|
|
362
|
+
// seam: tests point the harness at their own corpus trees instead of
|
|
363
|
+
// planting synthetic stakes in the shipped commons.
|
|
364
|
+
function corpusDir(name) {
|
|
365
|
+
const override = process.env.ANY_DOCTOR_CORPUS_ROOT;
|
|
366
|
+
if (override !== undefined && override !== "") {
|
|
367
|
+
const dir = path.join(path.resolve(override), name);
|
|
368
|
+
return fs.existsSync(dir) ? dir : null;
|
|
369
|
+
}
|
|
370
|
+
try {
|
|
371
|
+
const dir = fs.realpathSync(new URL("../fixtures/" + name, import.meta.url));
|
|
372
|
+
return fs.existsSync(dir) ? dir : null;
|
|
373
|
+
}
|
|
374
|
+
catch {
|
|
375
|
+
return null;
|
|
376
|
+
}
|
|
377
|
+
}
|