@0xcraft/powershot 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +202 -0
- package/README.md +306 -0
- package/dist/agents.js +82 -0
- package/dist/bench.js +179 -0
- package/dist/budget.js +59 -0
- package/dist/bundle.js +173 -0
- package/dist/cache.js +155 -0
- package/dist/cli/agent-command.js +27 -0
- package/dist/cli/app.js +31 -0
- package/dist/cli/args.js +76 -0
- package/dist/cli/bench-command.js +89 -0
- package/dist/cli/dismiss-command.js +42 -0
- package/dist/cli/environment.js +32 -0
- package/dist/cli/reports.js +64 -0
- package/dist/cli/review-command.js +268 -0
- package/dist/cli/session-command.js +62 -0
- package/dist/cli.js +7 -0
- package/dist/config.js +130 -0
- package/dist/delegate.js +84 -0
- package/dist/dismissed.js +130 -0
- package/dist/fspolicy.js +62 -0
- package/dist/git.js +238 -0
- package/dist/ground.js +286 -0
- package/dist/judges/judge.js +85 -0
- package/dist/judges/llm.js +234 -0
- package/dist/judges/prompts.js +86 -0
- package/dist/judges/tools.js +125 -0
- package/dist/lang/packs.js +557 -0
- package/dist/lang/pyright.js +108 -0
- package/dist/lang/python-deps.js +174 -0
- package/dist/lang/ruby-deps.js +77 -0
- package/dist/langtest.js +248 -0
- package/dist/manifest.js +209 -0
- package/dist/otel.js +75 -0
- package/dist/package-meta.js +13 -0
- package/dist/package-smoke.js +110 -0
- package/dist/plan.js +134 -0
- package/dist/position.js +94 -0
- package/dist/report/ansi.js +18 -0
- package/dist/report/codequality.js +19 -0
- package/dist/report/compact.js +15 -0
- package/dist/report/highlight.js +54 -0
- package/dist/report/markdown.js +113 -0
- package/dist/report/sarif.js +66 -0
- package/dist/report/terminal.js +170 -0
- package/dist/report/viewer.js +148 -0
- package/dist/review.js +355 -0
- package/dist/scan.js +67 -0
- package/dist/selftest.js +1928 -0
- package/dist/session.js +140 -0
- package/dist/snapshot.js +101 -0
- package/dist/text.js +50 -0
- package/dist/types.js +2 -0
- package/dist/verifiers/assertion-drift.js +137 -0
- package/dist/verifiers/contract-drift.js +140 -0
- package/dist/verifiers/copy-paste-drift.js +106 -0
- package/dist/verifiers/dead-on-arrival.js +92 -0
- package/dist/verifiers/dropped-guard.js +144 -0
- package/dist/verifiers/foreign-contract-drift.js +114 -0
- package/dist/verifiers/foreign-copy-paste-drift.js +83 -0
- package/dist/verifiers/foreign-dropped-guard.js +78 -0
- package/dist/verifiers/foreign-phantom-api.js +36 -0
- package/dist/verifiers/foreign-phantom-config.js +40 -0
- package/dist/verifiers/foreign-phantom-dep.js +82 -0
- package/dist/verifiers/foreign-reinvented.js +65 -0
- package/dist/verifiers/foreign-scope-creep.js +42 -0
- package/dist/verifiers/foreign-swallowed-error.js +36 -0
- package/dist/verifiers/foreign-tests.js +143 -0
- package/dist/verifiers/foreign-tokens.js +94 -0
- package/dist/verifiers/foreign.js +16 -0
- package/dist/verifiers/index.js +38 -0
- package/dist/verifiers/lying-comment.js +90 -0
- package/dist/verifiers/phantom-api.js +88 -0
- package/dist/verifiers/phantom-config.js +93 -0
- package/dist/verifiers/phantom-dep.js +110 -0
- package/dist/verifiers/reinvented.js +74 -0
- package/dist/verifiers/scope-creep.js +77 -0
- package/dist/verifiers/swallowed-error.js +110 -0
- package/dist/verifiers/vacuous-test.js +138 -0
- package/docs/architecture.md +191 -0
- package/docs/assets/cli-preview.svg +68 -0
- package/docs/assets/powershot-logo.png +0 -0
- package/docs/ci.md +151 -0
- package/examples/github-actions/action.yml +23 -0
- package/examples/github-actions/cli.yml +43 -0
- package/examples/gitlab/.gitlab-ci.yml +21 -0
- package/package.json +65 -0
package/dist/selftest.js
ADDED
|
@@ -0,0 +1,1928 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* One runnable check. Every verifier is asserted in BOTH directions: it must fire
|
|
3
|
+
* on the defect, and stay silent on the clean version. A verifier that only ever
|
|
4
|
+
* fires is a rubber stamp.
|
|
5
|
+
*/
|
|
6
|
+
import assert from 'node:assert/strict';
|
|
7
|
+
import { Project } from 'ts-morph';
|
|
8
|
+
import { phantomApi, phantomDep, reinvented, droppedGuard, swallowedError, vacuousTest, assertionDrift, scopeCreep, contractDrift, copyPasteDrift, deadOnArrival, lyingComment } from './verifiers/index.js';
|
|
9
|
+
import { extractJsonArray, endpoint } from './judges/llm.js';
|
|
10
|
+
import { matchesAny, validateConfig } from './config.js';
|
|
11
|
+
import { titleOverlap } from './review.js';
|
|
12
|
+
import { caretFor, validateSuggestion } from './position.js';
|
|
13
|
+
import { parseFindings } from './judges/judge.js';
|
|
14
|
+
import { JUDGES, COMMON } from './judges/prompts.js';
|
|
15
|
+
import { bundle, groupKey, uncovered } from './bundle.js';
|
|
16
|
+
import { Session } from './session.js';
|
|
17
|
+
import { Dismissals, lastReport, rememberReport } from './dismissed.js';
|
|
18
|
+
import { JudgeCache } from './cache.js';
|
|
19
|
+
import { scanPaths } from './scan.js';
|
|
20
|
+
import { checkRange, collectChanges } from './git.js';
|
|
21
|
+
import { stripControl } from './text.js';
|
|
22
|
+
import { review } from './review.js';
|
|
23
|
+
import { withTargetTree } from './snapshot.js';
|
|
24
|
+
import { loadConfig } from './config.js';
|
|
25
|
+
import { execFileSync } from 'node:child_process';
|
|
26
|
+
import { insideRepo, repoPath } from './fspolicy.js';
|
|
27
|
+
import { Budget, parseLimits } from './budget.js';
|
|
28
|
+
import { SelectionPlan, capabilitiesOf } from './plan.js';
|
|
29
|
+
import { RunManifest, SCHEMA, coverageProblems } from './manifest.js';
|
|
30
|
+
import { ProviderError, redact } from './judges/llm.js';
|
|
31
|
+
import { VERIFIERS } from './verifiers/index.js';
|
|
32
|
+
import { existsSync, mkdtempSync, readFileSync, writeFileSync, mkdirSync, readdirSync, rmSync, symlinkSync, realpathSync } from 'node:fs';
|
|
33
|
+
import { tmpdir } from 'node:os';
|
|
34
|
+
import { join, sep } from 'node:path';
|
|
35
|
+
import { runTool } from './judges/tools.js';
|
|
36
|
+
import { codeQuality } from './report/codequality.js';
|
|
37
|
+
import { viewer } from './report/viewer.js';
|
|
38
|
+
import { absorbDelegated, delegateBrief } from './delegate.js';
|
|
39
|
+
import { TARGETS, findTarget } from './agents.js';
|
|
40
|
+
import { packFor } from './lang/packs.js';
|
|
41
|
+
import { isPhantom, pythonManifest, localModules } from './lang/python-deps.js';
|
|
42
|
+
import { isPhantomGem, rubyManifest } from './lang/ruby-deps.js';
|
|
43
|
+
import { pyrightAvailable } from './lang/pyright.js';
|
|
44
|
+
import { decode, lines as splitLines, stripCR } from './text.js';
|
|
45
|
+
import { parseAddedLines } from './git.js';
|
|
46
|
+
import { terminal } from './report/terminal.js';
|
|
47
|
+
import { compact } from './report/compact.js';
|
|
48
|
+
import { apiKey } from './judges/llm.js';
|
|
49
|
+
import { sarif } from './report/sarif.js';
|
|
50
|
+
import { markdown } from './report/markdown.js';
|
|
51
|
+
import { wrap } from './report/terminal.js';
|
|
52
|
+
import { highlight, isJsx } from './report/highlight.js';
|
|
53
|
+
import { normalizeName, readEnvManifest, relPath } from './ground.js';
|
|
54
|
+
import { incompleteReasons } from './bench.js';
|
|
55
|
+
import { PACKAGE_NAME, PACKAGE_VERSION } from './package-meta.js';
|
|
56
|
+
const root = '/repo';
|
|
57
|
+
/** Build a Ground by hand so verifiers are testable without git or a real repo. */
|
|
58
|
+
function ground(files, deps = []) {
|
|
59
|
+
const project = new Project({ useInMemoryFileSystem: true });
|
|
60
|
+
const beforeProject = new Project({ useInMemoryFileSystem: true });
|
|
61
|
+
const changed = [];
|
|
62
|
+
const entries = [];
|
|
63
|
+
for (const f of files) {
|
|
64
|
+
const sf = project.createSourceFile(root + '/' + f.path, f.after, { overwrite: true });
|
|
65
|
+
const lineCount = f.after.split('\n').length;
|
|
66
|
+
const c = {
|
|
67
|
+
path: f.path,
|
|
68
|
+
added: new Set(Array.from({ length: lineCount }, (_, i) => i + 1)),
|
|
69
|
+
before: f.before,
|
|
70
|
+
};
|
|
71
|
+
changed.push(c);
|
|
72
|
+
entries.push({
|
|
73
|
+
sf,
|
|
74
|
+
changed: c,
|
|
75
|
+
before: f.before === undefined ? undefined : beforeProject.createSourceFile('/before/' + f.path, f.before, { overwrite: true }),
|
|
76
|
+
typed: true,
|
|
77
|
+
});
|
|
78
|
+
}
|
|
79
|
+
const symbolIndex = new Map();
|
|
80
|
+
for (const sf of project.getSourceFiles()) {
|
|
81
|
+
const rel = sf.getFilePath().slice(root.length + 1);
|
|
82
|
+
for (const [name, decls] of sf.getExportedDeclarations()) {
|
|
83
|
+
const decl = decls[0];
|
|
84
|
+
if (!decl)
|
|
85
|
+
continue;
|
|
86
|
+
const key = normalizeName(name);
|
|
87
|
+
const list = symbolIndex.get(key) ?? [];
|
|
88
|
+
list.push({ file: rel, name, line: decl.getStartLineNumber() });
|
|
89
|
+
symbolIndex.set(key, list);
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
return { root, project, beforeProject, changed, files: entries, symbolIndex, deps: new Set(deps), depsFor: () => new Set(deps), typed: false, internalPrefixes: [], foreign: [] };
|
|
93
|
+
}
|
|
94
|
+
let failures = 0;
|
|
95
|
+
function check(name, fn) {
|
|
96
|
+
try {
|
|
97
|
+
fn();
|
|
98
|
+
console.log(' ok ' + name);
|
|
99
|
+
}
|
|
100
|
+
catch (e) {
|
|
101
|
+
failures++;
|
|
102
|
+
console.log(' FAIL ' + name + '\n ' + e.message);
|
|
103
|
+
}
|
|
104
|
+
}
|
|
105
|
+
/** The same, for a check that has to run the real pipeline rather than one verifier. */
|
|
106
|
+
async function checkAsync(name, fn) {
|
|
107
|
+
try {
|
|
108
|
+
await fn();
|
|
109
|
+
console.log(' ok ' + name);
|
|
110
|
+
}
|
|
111
|
+
catch (e) {
|
|
112
|
+
failures++;
|
|
113
|
+
console.log(' FAIL ' + name + '\n ' + e.message);
|
|
114
|
+
}
|
|
115
|
+
}
|
|
116
|
+
function fires(v, g) {
|
|
117
|
+
return v.run(g).length > 0;
|
|
118
|
+
}
|
|
119
|
+
console.log('\nphantom-dep');
|
|
120
|
+
check('fires on an import that is not a declared dependency', () => {
|
|
121
|
+
const g = ground([{ path: 'a.ts', after: "import ky from 'ky'\nexport const x = ky\n" }], ['zod']);
|
|
122
|
+
const found = phantomDep.run(g);
|
|
123
|
+
assert.equal(found.length, 1);
|
|
124
|
+
assert.match(found[0].title, /"ky"/);
|
|
125
|
+
assert.equal(found[0].confidence, 'proven');
|
|
126
|
+
});
|
|
127
|
+
check('silent on a declared dependency', () => {
|
|
128
|
+
const g = ground([{ path: 'a.ts', after: "import { z } from 'zod'\nexport const x = z\n" }], ['zod']);
|
|
129
|
+
assert.equal(fires(phantomDep, g), false);
|
|
130
|
+
});
|
|
131
|
+
check('silent on node builtins and relative imports', () => {
|
|
132
|
+
const g = ground([{ path: 'a.ts', after: "import fs from 'node:fs'\nimport path from 'path'\nimport { y } from './b.js'\nexport const x = [fs, path, y]\n" }], ['zod']);
|
|
133
|
+
assert.equal(fires(phantomDep, g), false);
|
|
134
|
+
});
|
|
135
|
+
check('silent on native package imports', () => {
|
|
136
|
+
const g = ground([{ path: 'a.ts', after: "import '#app/start.js'\nexport const x = import('#app/runtime.js')\n" }], ['zod']);
|
|
137
|
+
assert.equal(fires(phantomDep, g), false);
|
|
138
|
+
});
|
|
139
|
+
check('resolves a scoped subpath to its package', () => {
|
|
140
|
+
const g = ground([{ path: 'a.ts', after: "import x from '@scope/pkg/deep/path'\nexport const y = x\n" }], ['@scope/pkg']);
|
|
141
|
+
assert.equal(fires(phantomDep, g), false);
|
|
142
|
+
});
|
|
143
|
+
console.log('\nreinvented');
|
|
144
|
+
check('fires when a helper already exists elsewhere', () => {
|
|
145
|
+
const g = ground([
|
|
146
|
+
{ path: 'lib/currency.ts', after: 'export function formatMinorUnits(n: number) { return n / 100 }\n' },
|
|
147
|
+
{ path: 'utils/money.ts', after: 'export function formatMinorUnits(n: number) { return n / 100 }\n' },
|
|
148
|
+
]);
|
|
149
|
+
const found = reinvented.run(g);
|
|
150
|
+
assert.ok(found.length >= 1, 'expected a duplication finding');
|
|
151
|
+
assert.equal(found[0].confidence, 'firm'); // heuristic, never claims `proven`
|
|
152
|
+
});
|
|
153
|
+
check('silent on a genuinely new name', () => {
|
|
154
|
+
const g = ground([
|
|
155
|
+
{ path: 'lib/currency.ts', after: 'export function formatMinorUnits(n: number) { return n / 100 }\n' },
|
|
156
|
+
{ path: 'utils/tax.ts', after: 'export function computeVatRate(n: number) { return n * 0.2 }\n' },
|
|
157
|
+
]);
|
|
158
|
+
assert.equal(fires(reinvented, g), false);
|
|
159
|
+
});
|
|
160
|
+
check('silent on short and generic names', () => {
|
|
161
|
+
const g = ground([
|
|
162
|
+
{ path: 'a/render.ts', after: 'export function render() { return 1 }\n' },
|
|
163
|
+
{ path: 'b/render.ts', after: 'export function render() { return 2 }\n' },
|
|
164
|
+
]);
|
|
165
|
+
assert.equal(fires(reinvented, g), false);
|
|
166
|
+
});
|
|
167
|
+
console.log('\ndropped-guard');
|
|
168
|
+
check('fires when an early-return guard disappears', () => {
|
|
169
|
+
const before = 'export function close(inv: any) {\n if (!inv.customer) return null\n return inv.total\n}\n';
|
|
170
|
+
const after = 'export function close(inv: any) {\n return inv.total\n}\n';
|
|
171
|
+
const g = ground([{ path: 'close.ts', after, before }]);
|
|
172
|
+
const found = droppedGuard.run(g);
|
|
173
|
+
assert.equal(found.length, 1);
|
|
174
|
+
assert.match(found[0].title, /!inv\.customer/);
|
|
175
|
+
});
|
|
176
|
+
check('fires when a throwing guard disappears', () => {
|
|
177
|
+
const before = 'export function pay(a: any) {\n if (a <= 0) { throw new Error("bad") }\n return a\n}\n';
|
|
178
|
+
const after = 'export function pay(a: any) {\n return a\n}\n';
|
|
179
|
+
assert.equal(fires(droppedGuard, ground([{ path: 'pay.ts', after, before }])), true);
|
|
180
|
+
});
|
|
181
|
+
check('silent when the guard is kept, even if reformatted', () => {
|
|
182
|
+
const before = 'export function close(inv: any) {\n if (!inv.customer) return null\n return inv.total\n}\n';
|
|
183
|
+
const after = 'export function close(inv: any) {\n if (!inv.customer)\n return null\n return inv.total * 2\n}\n';
|
|
184
|
+
assert.equal(fires(droppedGuard, ground([{ path: 'close.ts', after, before }])), false);
|
|
185
|
+
});
|
|
186
|
+
check('silent when a rewrite respells the same guard for a new type', () => {
|
|
187
|
+
// found by bench on a real repo: `pool` went from array to number, so
|
|
188
|
+
// `pool.length === 0` became `pool === 0` — a rewrite, not a removal
|
|
189
|
+
const before = 'export function check(pool: any[]) {\n if (pool.length === 0) return null\n return pool\n}\n';
|
|
190
|
+
const after = 'export function check(pool: number) {\n if (pool === 0) return null\n return pool\n}\n';
|
|
191
|
+
assert.equal(fires(droppedGuard, ground([{ path: 'c.ts', after, before }])), false);
|
|
192
|
+
});
|
|
193
|
+
check('silent when the guard became obsolete with the code it protected', () => {
|
|
194
|
+
// found by bench: formatWeight moved from ounces to grams, so the old guards
|
|
195
|
+
// referenced locals the rewritten function no longer has
|
|
196
|
+
const before = 'export function fmt(totalOunces: number) {\n const pounds = totalOunces / 16\n if (pounds === 0) return "0"\n return String(pounds)\n}\n';
|
|
197
|
+
const after = 'export function fmt(grams: number) {\n return String(grams / 1000)\n}\n';
|
|
198
|
+
assert.equal(fires(droppedGuard, ground([{ path: 'c.ts', after, before }])), false);
|
|
199
|
+
});
|
|
200
|
+
check('silent for a new file, which cannot have dropped anything', () => {
|
|
201
|
+
const after = 'export function close(inv: any) {\n return inv.total\n}\n';
|
|
202
|
+
assert.equal(fires(droppedGuard, ground([{ path: 'close.ts', after }])), false);
|
|
203
|
+
});
|
|
204
|
+
console.log('\nvacuous-test');
|
|
205
|
+
check('fires on a test that asserts nothing', () => {
|
|
206
|
+
const src = "import { close } from './close.js'\nit('closes the invoice', () => {\n close({})\n})\n";
|
|
207
|
+
const found = vacuousTest.run(ground([{ path: 'close.test.ts', after: src }]));
|
|
208
|
+
assert.equal(found.length, 1);
|
|
209
|
+
assert.match(found[0].title, /asserts nothing/);
|
|
210
|
+
});
|
|
211
|
+
check('silent on a test that does assert', () => {
|
|
212
|
+
const src = "import { close } from './close.js'\nit('closes the invoice', () => {\n expect(close({})).toBe(1)\n})\n";
|
|
213
|
+
assert.equal(fires(vacuousTest, ground([{ path: 'close.test.ts', after: src }])), false);
|
|
214
|
+
});
|
|
215
|
+
check('fires when the test mocks the module under test', () => {
|
|
216
|
+
const src = "import { close } from './close.js'\nvi.mock('./close.js')\nit('works', () => {\n expect(close({})).toBe(1)\n})\n";
|
|
217
|
+
const found = vacuousTest.run(ground([{ path: 'close.test.ts', after: src }]));
|
|
218
|
+
assert.ok(found.some((f) => /mocks/.test(f.title)), 'expected a mocked-unit-under-test finding');
|
|
219
|
+
});
|
|
220
|
+
check('silent when the test mocks a dependency rather than its subject', () => {
|
|
221
|
+
// found by bench: mocking a collaborator you also import is ordinary practice —
|
|
222
|
+
// only the module the test file is named after counts as the unit under test
|
|
223
|
+
const src = "import { toast } from '@lakeside/ui-sdk'\nimport { useThing } from './useThing.js'\n" +
|
|
224
|
+
"vi.mock('@lakeside/ui-sdk')\nit('works', () => {\n expect(useThing()).toBe(1)\n})\n";
|
|
225
|
+
const found = vacuousTest.run(ground([{ path: 'useThing.test.ts', after: src }]));
|
|
226
|
+
assert.equal(found.filter((f) => /mocks/.test(f.title)).length, 0);
|
|
227
|
+
});
|
|
228
|
+
check('ignores files that are not tests', () => {
|
|
229
|
+
const src = "it('not really a test file', () => { doThing() })\n";
|
|
230
|
+
assert.equal(fires(vacuousTest, ground([{ path: 'src/app.ts', after: src }])), false);
|
|
231
|
+
});
|
|
232
|
+
console.log('\nswallowed-error');
|
|
233
|
+
check('fires on an empty catch block', () => {
|
|
234
|
+
const src = 'export function f() {\n try { risky() } catch (e) {}\n}\n';
|
|
235
|
+
const found = swallowedError.run(ground([{ path: 'a.ts', after: src }]));
|
|
236
|
+
assert.equal(found.length, 1);
|
|
237
|
+
assert.equal(found[0].confidence, 'proven');
|
|
238
|
+
assert.match(found[0].title, /Empty catch/);
|
|
239
|
+
});
|
|
240
|
+
check('fires on a catch that only logs', () => {
|
|
241
|
+
const src = 'export function f() {\n try { risky() } catch (e) { console.error(e) }\n}\n';
|
|
242
|
+
const found = swallowedError.run(ground([{ path: 'a.ts', after: src }]));
|
|
243
|
+
assert.equal(found.length, 1);
|
|
244
|
+
assert.equal(found[0].confidence, 'firm'); // a top-level log-only handler can be correct
|
|
245
|
+
assert.match(found[0].title, /only logs/);
|
|
246
|
+
});
|
|
247
|
+
check('fires on an empty .catch() handler', () => {
|
|
248
|
+
const src = 'export function f() {\n fetchThing().catch(() => {})\n}\n';
|
|
249
|
+
const found = swallowedError.run(ground([{ path: 'a.ts', after: src }]));
|
|
250
|
+
assert.equal(found.length, 1);
|
|
251
|
+
assert.match(found[0].title, /Empty \.catch/);
|
|
252
|
+
});
|
|
253
|
+
check('silent when the catch rethrows', () => {
|
|
254
|
+
const src = 'export function f() {\n try { risky() } catch (e) { throw e }\n}\n';
|
|
255
|
+
assert.equal(fires(swallowedError, ground([{ path: 'a.ts', after: src }])), false);
|
|
256
|
+
});
|
|
257
|
+
check('silent when the catch logs and then rethrows', () => {
|
|
258
|
+
const src = 'export function f() {\n try { risky() } catch (e) { console.error(e); throw e }\n}\n';
|
|
259
|
+
assert.equal(fires(swallowedError, ground([{ path: 'a.ts', after: src }])), false);
|
|
260
|
+
});
|
|
261
|
+
check('silent when the catch recovers with real work', () => {
|
|
262
|
+
const src = 'export function f() {\n try { risky() } catch (e) { return fallback() }\n}\n';
|
|
263
|
+
assert.equal(fires(swallowedError, ground([{ path: 'a.ts', after: src }])), false);
|
|
264
|
+
});
|
|
265
|
+
check('respects a comment as a deliberate ignore', () => {
|
|
266
|
+
const src = 'export function f() {\n try { risky() } catch (e) { /* offline is fine here */ }\n}\n';
|
|
267
|
+
assert.equal(fires(swallowedError, ground([{ path: 'a.ts', after: src }])), false);
|
|
268
|
+
});
|
|
269
|
+
check('silent on .catch(handler) that names a real function', () => {
|
|
270
|
+
const src = 'export function f() {\n fetchThing().catch(reportError)\n}\n';
|
|
271
|
+
assert.equal(fires(swallowedError, ground([{ path: 'a.ts', after: src }])), false);
|
|
272
|
+
});
|
|
273
|
+
console.log('\nassertion-drift');
|
|
274
|
+
check('fires when an expectation is edited under a stable subject', () => {
|
|
275
|
+
const before = "it('adds', () => { expect(add(2,2)).toBe(4) })\n";
|
|
276
|
+
const after = "it('adds', () => { expect(add(2,2)).toBe(5) })\n";
|
|
277
|
+
const found = assertionDrift.run(ground([{ path: 'a.test.ts', after, before }]));
|
|
278
|
+
assert.equal(found.length, 1);
|
|
279
|
+
assert.equal(found[0].confidence, 'firm'); // updating an expectation can be legitimate
|
|
280
|
+
assert.match(found[0].title, /from 4 to 5/);
|
|
281
|
+
});
|
|
282
|
+
check('rates it high when no source file changed in the diff', () => {
|
|
283
|
+
const before = "it('adds', () => { expect(add(2,2)).toBe(4) })\n";
|
|
284
|
+
const after = "it('adds', () => { expect(add(2,2)).toBe(5) })\n";
|
|
285
|
+
const found = assertionDrift.run(ground([{ path: 'a.test.ts', after, before }]));
|
|
286
|
+
assert.equal(found[0].severity, 'high');
|
|
287
|
+
});
|
|
288
|
+
check('silent when the test itself was rewritten, not merely bent', () => {
|
|
289
|
+
// narrowed after bench: a "add tests" commit legitimately reshapes existing tests,
|
|
290
|
+
// and only an otherwise-identical test signals an expectation bent to fit the code
|
|
291
|
+
const before = "it('adds', () => { expect(add(2,2)).toBe(4) })\n";
|
|
292
|
+
const after = "it('adds', () => { const r = add(2,2); expect(r).toBe(5) })\n";
|
|
293
|
+
assert.equal(fires(assertionDrift, ground([{ path: 'a.test.ts', after, before }])), false);
|
|
294
|
+
});
|
|
295
|
+
check('silent when the module under test changed too', () => {
|
|
296
|
+
// narrowed after bench: an expectation moving alongside a change to the module it
|
|
297
|
+
// covers is a deliberate behaviour change, not a bent test
|
|
298
|
+
const before = "it('adds', () => { expect(add(2,2)).toBe(4) })\n";
|
|
299
|
+
const after = "it('adds', () => { expect(add(2,2)).toBe(5) })\n";
|
|
300
|
+
const found = assertionDrift.run(ground([
|
|
301
|
+
{ path: 'add.test.ts', after, before },
|
|
302
|
+
{ path: 'add.ts', after: 'export const add = (a: number, b: number) => a + b + 1\n', before: 'export const add = (a: number, b: number) => a + b\n' },
|
|
303
|
+
]));
|
|
304
|
+
assert.equal(found.length, 0);
|
|
305
|
+
});
|
|
306
|
+
check('still fires when an unrelated module changed', () => {
|
|
307
|
+
// a large change may edit one module and bend a test that covers a different one
|
|
308
|
+
const before = "it('adds', () => { expect(add(2,2)).toBe(4) })\n";
|
|
309
|
+
const after = "it('adds', () => { expect(add(2,2)).toBe(5) })\n";
|
|
310
|
+
const found = assertionDrift.run(ground([
|
|
311
|
+
{ path: 'add.test.ts', after, before },
|
|
312
|
+
{ path: 'unrelated.ts', after: 'export const x = 2\n', before: 'export const x = 1\n' },
|
|
313
|
+
]));
|
|
314
|
+
assert.equal(found.length, 1);
|
|
315
|
+
});
|
|
316
|
+
check('silent when the assertion is untouched', () => {
|
|
317
|
+
const src = "it('adds', () => { expect(add(2,2)).toBe(4) })\n";
|
|
318
|
+
assert.equal(fires(assertionDrift, ground([{ path: 'a.test.ts', after: src, before: src }])), false);
|
|
319
|
+
});
|
|
320
|
+
check('silent on a brand new assertion', () => {
|
|
321
|
+
const before = "it('adds', () => { expect(add(2,2)).toBe(4) })\n";
|
|
322
|
+
const after = "it('adds', () => { expect(add(2,2)).toBe(4) })\nit('subs', () => { expect(sub(2,2)).toBe(0) })\n";
|
|
323
|
+
assert.equal(fires(assertionDrift, ground([{ path: 'a.test.ts', after, before }])), false);
|
|
324
|
+
});
|
|
325
|
+
check('tracks node assert.equal too', () => {
|
|
326
|
+
const before = "it('adds', () => { assert.equal(add(2,2), 4) })\n";
|
|
327
|
+
const after = "it('adds', () => { assert.equal(add(2,2), 5) })\n";
|
|
328
|
+
assert.equal(fires(assertionDrift, ground([{ path: 'a.test.ts', after, before }])), true);
|
|
329
|
+
});
|
|
330
|
+
check('ignores non-test files', () => {
|
|
331
|
+
const before = "it('x', () => { expect(a).toBe(1) })\n";
|
|
332
|
+
const after = "it('x', () => { expect(a).toBe(2) })\n";
|
|
333
|
+
assert.equal(fires(assertionDrift, ground([{ path: 'src/app.ts', after, before }])), false);
|
|
334
|
+
});
|
|
335
|
+
console.log('\nbundling');
|
|
336
|
+
check('keeps files that import one another in the same unit', () => {
|
|
337
|
+
const g = ground([
|
|
338
|
+
{ path: 'core.ts', after: 'export function core(n: number) { return n }\n' },
|
|
339
|
+
{ path: 'user.ts', after: "import { core } from './core.js'\nexport const go = () => core(1)\n" },
|
|
340
|
+
]);
|
|
341
|
+
const units = bundle(g, 10_000);
|
|
342
|
+
assert.equal(units.length, 1, 'linked files must share a unit');
|
|
343
|
+
});
|
|
344
|
+
check('splits when one unit would exceed the prompt budget', () => {
|
|
345
|
+
const big = (name) => ({
|
|
346
|
+
path: name,
|
|
347
|
+
after: 'export const ' + name.replace('.ts', '') + ' = [\n' + Array.from({ length: 60 }, (_, i) => ' ' + i + ',').join('\n') + '\n]\n',
|
|
348
|
+
});
|
|
349
|
+
const g = ground([big('a.ts'), big('b.ts'), big('c.ts')]);
|
|
350
|
+
const units = bundle(g, 250);
|
|
351
|
+
assert.ok(units.length > 1, 'expected the change to be split');
|
|
352
|
+
});
|
|
353
|
+
check('a language without an import graph still groups by what its paths encode', () => {
|
|
354
|
+
const same = (a, b) => groupKey(a) === groupKey(b);
|
|
355
|
+
// a header and its implementation are the pair a contract-drift judge needs at once
|
|
356
|
+
assert.ok(same('src/net.h', 'src/net.c'));
|
|
357
|
+
assert.ok(same('a/util.hpp', 'a/util.cpp'));
|
|
358
|
+
// a module and the tests that cover it argue about the same behaviour
|
|
359
|
+
assert.ok(same('pkg/svc.py', 'pkg/test_svc.py'));
|
|
360
|
+
assert.ok(same('pkg/svc.py', 'pkg/svc_test.py'));
|
|
361
|
+
assert.ok(same('a/lib.rs', 'a/lib_test.rs'));
|
|
362
|
+
// a package is the unit in Go and Java, where a file name means less
|
|
363
|
+
assert.ok(same('api/handler.go', 'api/router.go'));
|
|
364
|
+
assert.ok(same('m/A.java', 'm/B.java'));
|
|
365
|
+
// and two unrelated Rust modules are not one conversation
|
|
366
|
+
assert.ok(!same('a/one.rs', 'a/two.rs'));
|
|
367
|
+
});
|
|
368
|
+
check('cuts a file too large for one prompt into several units, losing nothing', () => {
|
|
369
|
+
const g = ground([{ path: 'big.ts', after: Array.from({ length: 900 }, (_, i) => 'export const v' + i + ' = ' + i).join('\n') + '\n' }]);
|
|
370
|
+
const units = bundle(g, 10_000);
|
|
371
|
+
const chunks = units.flatMap((u) => u.files);
|
|
372
|
+
assert.ok(chunks.length > 1, 'a 900-line change must not be one unit');
|
|
373
|
+
const all = g.files[0].changed.added.size;
|
|
374
|
+
const covered = new Set(chunks.flatMap((c) => [...c.added]));
|
|
375
|
+
assert.equal(covered.size, all, 'every added line must reach a judge');
|
|
376
|
+
assert.equal(chunks.reduce((n, c) => n + c.added.size, 0), all, 'no line may be sent twice');
|
|
377
|
+
assert.deepEqual(uncovered(g, units), []);
|
|
378
|
+
});
|
|
379
|
+
check('packs unrelated small components together rather than one unit each', () => {
|
|
380
|
+
const g = ground([
|
|
381
|
+
{ path: 'a.ts', after: 'export const a = 1\n' },
|
|
382
|
+
{ path: 'b.ts', after: 'export const b = 2\n' },
|
|
383
|
+
{ path: 'c.ts', after: 'export const c = 3\n' },
|
|
384
|
+
]);
|
|
385
|
+
// unrelated but tiny: every extra unit is an extra bill
|
|
386
|
+
assert.equal(bundle(g, 10_000).length, 1);
|
|
387
|
+
});
|
|
388
|
+
console.log('\nagent tools');
|
|
389
|
+
const toolGround = ground([{ path: 'money.ts', after: 'export function formatCents(n: number) { return n / 100 }\n' }]);
|
|
390
|
+
check('grep finds code in the repository', () => {
|
|
391
|
+
const out = runTool(toolGround, 'grep', { pattern: 'formatCents' });
|
|
392
|
+
assert.match(out, /money\.ts:1/);
|
|
393
|
+
});
|
|
394
|
+
check('references reports where a symbol is used', () => {
|
|
395
|
+
const out = runTool(toolGround, 'references', { symbol: 'formatCents' });
|
|
396
|
+
assert.match(out, /formatCents/);
|
|
397
|
+
});
|
|
398
|
+
check('read_file refuses to escape the repository', () => {
|
|
399
|
+
const out = runTool(toolGround, 'read_file', { path: '../../../etc/passwd' });
|
|
400
|
+
assert.match(out, /Refused/);
|
|
401
|
+
});
|
|
402
|
+
check('read_file refuses secrets even inside the repository', () => {
|
|
403
|
+
// reviewed code is untrusted input; containment lives in the tool layer
|
|
404
|
+
for (const path of ['.env', '.env.production', 'certs/server.key', 'id_rsa', '.npmrc']) {
|
|
405
|
+
assert.match(runTool(toolGround, 'read_file', { path }), /Refused/, path + ' should be refused');
|
|
406
|
+
}
|
|
407
|
+
});
|
|
408
|
+
check('a symlink cannot carry a read out of the repository', () => {
|
|
409
|
+
// resolve() normalises `..` but follows nothing — only the real path settles this
|
|
410
|
+
const dir = mkdtempSync(join(tmpdir(), 'psh-sym-'));
|
|
411
|
+
const outside = join(tmpdir(), 'psh-outside-' + Date.now() + '.txt');
|
|
412
|
+
writeFileSync(outside, 'secret');
|
|
413
|
+
mkdirSync(join(dir, 'src'), { recursive: true });
|
|
414
|
+
writeFileSync(join(dir, 'src', 'real.ts'), 'export const x = 1\n');
|
|
415
|
+
symlinkSync(outside, join(dir, 'src', 'linked.ts'));
|
|
416
|
+
const g = { ...ground([{ path: 'src/real.ts', after: 'export const x = 1\n' }]), root: dir };
|
|
417
|
+
assert.match(runTool(g, 'read_file', { path: 'src/linked.ts' }), /Refused/);
|
|
418
|
+
rmSync(dir, { recursive: true, force: true });
|
|
419
|
+
rmSync(outside, { force: true });
|
|
420
|
+
});
|
|
421
|
+
check('an unknown tool is reported, not thrown', () => {
|
|
422
|
+
assert.match(runTool(toolGround, 'rm_rf', {}), /Unknown tool/);
|
|
423
|
+
});
|
|
424
|
+
check('a broken regex is reported, not thrown', () => {
|
|
425
|
+
assert.match(runTool(toolGround, 'grep', { pattern: '([' }), /valid regular expression/);
|
|
426
|
+
});
|
|
427
|
+
console.log('\nlanguages beyond TypeScript');
|
|
428
|
+
check('files are routed to the right language pack', () => {
|
|
429
|
+
assert.equal(packFor('src/a.py')?.name, 'python');
|
|
430
|
+
assert.equal(packFor('src/a.pyi')?.name, 'python');
|
|
431
|
+
assert.equal(packFor('cmd/main.go')?.name, 'go');
|
|
432
|
+
assert.equal(packFor('src/a.ts'), undefined); // TypeScript keeps its own oracle
|
|
433
|
+
assert.equal(packFor('README.md'), undefined);
|
|
434
|
+
});
|
|
435
|
+
// Per-language pack checks live in langtest.ts, one process each: eleven wasm
|
|
436
|
+
// grammars cannot share a process without exhausting it. `npm test` runs both.
|
|
437
|
+
console.log('\neditor and provider surface');
|
|
438
|
+
check('compact output is the shape editors already parse', () => {
|
|
439
|
+
const f = { id: 'F1', class: 'verified', check: 'phantom-dep', severity: 'high',
|
|
440
|
+
confidence: 'proven', file: 'src/a.ts', line: 42, title: 'missing dep', span: { column: 9, length: 4 } };
|
|
441
|
+
const line = compact([f]).trim();
|
|
442
|
+
assert.equal(line, 'src/a.ts:42:9: error: missing dep [phantom-dep]');
|
|
443
|
+
// the same regex the shipped VS Code task uses
|
|
444
|
+
assert.ok(/^(.+?):(\d+):(\d+):\s+(error|warning|info):\s+(.*)$/.test(line));
|
|
445
|
+
});
|
|
446
|
+
check('compact maps severity onto the three levels editors understand', () => {
|
|
447
|
+
const at = (severity) => {
|
|
448
|
+
const f = { id: 'F', class: 'verified', check: 'c', severity, confidence: 'proven',
|
|
449
|
+
file: 'a.ts', line: 1, title: 't' };
|
|
450
|
+
return compact([f]).split(': ')[1];
|
|
451
|
+
};
|
|
452
|
+
assert.equal(at('critical'), 'error');
|
|
453
|
+
assert.equal(at('high'), 'error');
|
|
454
|
+
assert.equal(at('medium'), 'warning');
|
|
455
|
+
assert.equal(at('low'), 'info');
|
|
456
|
+
});
|
|
457
|
+
check('compact defaults the column when a finding has no span', () => {
|
|
458
|
+
const f = { id: 'F', class: 'judged', check: 'plausible-logic', severity: 'low',
|
|
459
|
+
confidence: 'tentative', file: 'a.ts', line: 3, title: 'x' };
|
|
460
|
+
assert.match(compact([f]), /^a\.ts:3:1: /);
|
|
461
|
+
});
|
|
462
|
+
check('each provider reads its own key', () => {
|
|
463
|
+
const base = { model: 'm', verifiers: ['*'], judges: ['*'], minSeverity: 'low',
|
|
464
|
+
ignore: [], promptCache: true };
|
|
465
|
+
const saved = { a: process.env.ANTHROPIC_API_KEY, o: process.env.OPENAI_API_KEY,
|
|
466
|
+
g: process.env.GEMINI_API_KEY, gg: process.env.GOOGLE_API_KEY };
|
|
467
|
+
process.env.ANTHROPIC_API_KEY = 'a';
|
|
468
|
+
process.env.OPENAI_API_KEY = 'o';
|
|
469
|
+
delete process.env.GEMINI_API_KEY;
|
|
470
|
+
process.env.GOOGLE_API_KEY = 'g';
|
|
471
|
+
assert.equal(apiKey({ ...base, provider: 'anthropic' }), 'a');
|
|
472
|
+
assert.equal(apiKey({ ...base, provider: 'openai' }), 'o');
|
|
473
|
+
assert.equal(apiKey({ ...base, provider: 'gemini' }), 'g'); // GOOGLE_API_KEY also works
|
|
474
|
+
for (const [k, v] of [['ANTHROPIC_API_KEY', saved.a], ['OPENAI_API_KEY', saved.o],
|
|
475
|
+
['GEMINI_API_KEY', saved.g], ['GOOGLE_API_KEY', saved.gg]]) {
|
|
476
|
+
if (v === undefined)
|
|
477
|
+
delete process.env[k];
|
|
478
|
+
else
|
|
479
|
+
process.env[k] = v;
|
|
480
|
+
}
|
|
481
|
+
});
|
|
482
|
+
console.log('\nreading foreign source');
|
|
483
|
+
check('a CRLF diff is parsed, not silently ignored', () => {
|
|
484
|
+
// Without stripping CR, the captured path contains a character that Git rejects.
|
|
485
|
+
// is "src/a.ts\r" and matches no file, so the whole review comes back empty
|
|
486
|
+
const diff = ['diff --git a/src/a.ts b/src/a.ts', '--- a/src/a.ts', '+++ b/src/a.ts', '@@ -1,0 +2,2 @@', '+const x = 1']
|
|
487
|
+
.join('\r\n');
|
|
488
|
+
const parsed = parseAddedLines(diff);
|
|
489
|
+
assert.deepEqual([...parsed.keys()], ['src/a.ts']);
|
|
490
|
+
assert.deepEqual([...parsed.get('src/a.ts')], [2, 3]);
|
|
491
|
+
});
|
|
492
|
+
check('an LF diff still parses exactly as before', () => {
|
|
493
|
+
const diff = ['+++ b/src/a.ts', '@@ -1 +1 @@', '+const x = 1'].join('\n');
|
|
494
|
+
assert.deepEqual([...parseAddedLines(diff).get('src/a.ts')], [1]);
|
|
495
|
+
});
|
|
496
|
+
check('Git paths are read as NUL-delimited data and passed back literally', () => {
|
|
497
|
+
const dir = realpathSync(mkdtempSync(join(tmpdir(), 'psh-git-path-')));
|
|
498
|
+
const run = (...args) => {
|
|
499
|
+
execFileSync('git', args, { cwd: dir, stdio: ['ignore', 'ignore', 'pipe'] });
|
|
500
|
+
};
|
|
501
|
+
try {
|
|
502
|
+
run('init', '-q', '.');
|
|
503
|
+
run('config', 'user.email', 't@t');
|
|
504
|
+
run('config', 'user.name', 't');
|
|
505
|
+
// Windows forbids control characters, `:` and `*` in filenames. Keep the
|
|
506
|
+
// strongest legal fixture on each platform while exercising the same NUL and
|
|
507
|
+
// literal-path pipeline.
|
|
508
|
+
const tracked = process.platform === 'win32' ? 'line ## heading.ts' : 'line\n## heading.ts';
|
|
509
|
+
const untracked = process.platform === 'win32' ? '[literal].ts' : ':(glob)*.ts';
|
|
510
|
+
const renamed = 'renamed.ts';
|
|
511
|
+
writeFileSync(join(dir, tracked), Array.from({ length: 10 }, (_, i) => 'export const before' + i + ' = ' + i).join('\n') + '\n');
|
|
512
|
+
run('add', '--', tracked);
|
|
513
|
+
run('commit', '-qm', 'seed');
|
|
514
|
+
run('mv', '--', tracked, renamed);
|
|
515
|
+
writeFileSync(join(dir, renamed), Array.from({ length: 10 }, (_, i) => 'export const ' + (i === 9 ? 'after' : 'before' + i) + ' = ' + i).join('\n') + '\n');
|
|
516
|
+
writeFileSync(join(dir, untracked), 'export const literal = 1\n');
|
|
517
|
+
const changes = collectChanges(dir, {});
|
|
518
|
+
const moved = changes.find((change) => change.path === renamed);
|
|
519
|
+
assert.ok(moved?.before?.includes('before9'));
|
|
520
|
+
assert.deepEqual([...moved.added], [10]);
|
|
521
|
+
assert.ok(changes.some((change) => change.path === untracked));
|
|
522
|
+
}
|
|
523
|
+
finally {
|
|
524
|
+
rmSync(dir, { recursive: true, force: true });
|
|
525
|
+
}
|
|
526
|
+
});
|
|
527
|
+
check('lines split on either ending', () => {
|
|
528
|
+
assert.deepEqual(splitLines('a\r\nb\nc'), ['a', 'b', 'c']);
|
|
529
|
+
assert.equal(stripCR('a\r'), 'a');
|
|
530
|
+
assert.equal(stripCR('a'), 'a');
|
|
531
|
+
});
|
|
532
|
+
check('bytes decode without throwing, whatever the encoding', () => {
|
|
533
|
+
assert.equal(decode(Buffer.from('plain ascii')), 'plain ascii');
|
|
534
|
+
assert.equal(decode(Buffer.from('héllo', 'utf8')), 'héllo');
|
|
535
|
+
// a UTF-8 BOM is consumed rather than becoming a stray character
|
|
536
|
+
assert.equal(decode(Buffer.concat([Buffer.from([0xef, 0xbb, 0xbf]), Buffer.from('x = 1')])), 'x = 1');
|
|
537
|
+
// Latin-1: invalid UTF-8, must fall back rather than take the review down
|
|
538
|
+
const latin = Buffer.concat([Buffer.from('caf'), Buffer.from([0xe9])]);
|
|
539
|
+
assert.equal(typeof decode(latin), 'string');
|
|
540
|
+
assert.equal(decode(latin).length, 4);
|
|
541
|
+
});
|
|
542
|
+
console.log('\nincomplete reviews');
|
|
543
|
+
check('a review that did not complete never renders as clean', () => {
|
|
544
|
+
// "Found nothing" and "could not look" must remain different outcomes.
|
|
545
|
+
// are different answers, and only one of them should let a pipeline through
|
|
546
|
+
const clean = terminal([], {
|
|
547
|
+
subtitle: 'workspace', verified: 0, judged: 0, state: 'complete', notLookedAt: [],
|
|
548
|
+
});
|
|
549
|
+
assert.match(clean, /No findings\./);
|
|
550
|
+
assert.equal(/not a verdict/.test(clean), false);
|
|
551
|
+
const partial = terminal([], {
|
|
552
|
+
subtitle: 'workspace', verified: 0, judged: 0, state: 'partial',
|
|
553
|
+
notLookedAt: ['outside.ts (no types)'],
|
|
554
|
+
});
|
|
555
|
+
assert.match(partial, /partial, not a verdict/);
|
|
556
|
+
assert.match(partial, /outside\.ts \(no types\)/);
|
|
557
|
+
});
|
|
558
|
+
check('findings are still shown when a stage failed, with the warning kept', () => {
|
|
559
|
+
const f = { id: 'F1', class: 'verified', check: 'phantom-dep', severity: 'high',
|
|
560
|
+
confidence: 'proven', file: 'a.ts', line: 1, title: 'missing dep' };
|
|
561
|
+
const out = terminal([f], {
|
|
562
|
+
subtitle: 'workspace', verified: 1, judged: 0, state: 'failed', notLookedAt: ['intent: timeout'],
|
|
563
|
+
});
|
|
564
|
+
assert.match(out, /missing dep/);
|
|
565
|
+
assert.match(out, /review is failed/);
|
|
566
|
+
assert.match(out, /intent: timeout/);
|
|
567
|
+
});
|
|
568
|
+
console.log('\nenv manifests');
|
|
569
|
+
check('a commented entry in a template documents an optional variable', () => {
|
|
570
|
+
// found by reviewing our own commit: `# OPENAI_BASE_URL=` is how a template says
|
|
571
|
+
// "this exists and may be left unset", so reading it as undeclared reports the
|
|
572
|
+
// very file that documents it
|
|
573
|
+
const dir = mkdtempSync(join(tmpdir(), 'psh-env-'));
|
|
574
|
+
writeFileSync(join(dir, '.env.example'), 'DATABASE_URL=x\n# OPTIONAL_TUNING=\n#export LEGACY_FLAG=1\n');
|
|
575
|
+
const manifest = readEnvManifest(dir);
|
|
576
|
+
assert.ok(manifest);
|
|
577
|
+
assert.equal(manifest.keys.has('DATABASE_URL'), true);
|
|
578
|
+
assert.equal(manifest.keys.has('OPTIONAL_TUNING'), true);
|
|
579
|
+
assert.equal(manifest.keys.has('LEGACY_FLAG'), true);
|
|
580
|
+
assert.equal(manifest.keys.has('NEVER_WRITTEN'), false); // still absent when truly absent
|
|
581
|
+
rmSync(dir, { recursive: true, force: true });
|
|
582
|
+
});
|
|
583
|
+
console.log('\npython dependencies');
|
|
584
|
+
const pyManifest = { names: new Set(['requests', 'pyyaml', 'scikit-learn', 'python-dateutil']) };
|
|
585
|
+
const pyLocal = new Set(['helpers', 'app']);
|
|
586
|
+
check('the standard library needs no dependency', () => {
|
|
587
|
+
for (const m of ['os', 'json', 'typing', 'asyncio', 'dataclasses', 'zoneinfo', 'tomllib', 'concurrent']) {
|
|
588
|
+
assert.equal(isPhantom(m, pyManifest, pyLocal), false, m + ' is stdlib');
|
|
589
|
+
}
|
|
590
|
+
});
|
|
591
|
+
check('a declared dependency is accounted for', () => {
|
|
592
|
+
assert.equal(isPhantom('requests', pyManifest, pyLocal), false);
|
|
593
|
+
});
|
|
594
|
+
check('an import whose distribution is named differently is accounted for', () => {
|
|
595
|
+
// the trap: depending on PyYAML while importing yaml. Reporting that would be
|
|
596
|
+
// exactly the confidently-wrong answer this tool exists to avoid
|
|
597
|
+
assert.equal(isPhantom('yaml', pyManifest, pyLocal), false);
|
|
598
|
+
assert.equal(isPhantom('sklearn', pyManifest, pyLocal), false);
|
|
599
|
+
assert.equal(isPhantom('dateutil', pyManifest, pyLocal), false);
|
|
600
|
+
});
|
|
601
|
+
check('a module living in this repository is accounted for', () => {
|
|
602
|
+
assert.equal(isPhantom('helpers', pyManifest, pyLocal), false);
|
|
603
|
+
assert.equal(isPhantom('helpers.util', pyManifest, pyLocal), false); // submodule of a local package
|
|
604
|
+
});
|
|
605
|
+
check('an import nothing installs is reported', () => {
|
|
606
|
+
assert.equal(isPhantom('tensorflow', pyManifest, pyLocal), true);
|
|
607
|
+
assert.equal(isPhantom('prefect', pyManifest, pyLocal), true);
|
|
608
|
+
});
|
|
609
|
+
check('manifest names are matched the way PyPI matches them', () => {
|
|
610
|
+
const m = { names: new Set(['python-dateutil']) };
|
|
611
|
+
// PyPI treats - _ . alike and ignores case, so all of these are the same package
|
|
612
|
+
assert.equal(isPhantom('dateutil', m, new Set()), false);
|
|
613
|
+
});
|
|
614
|
+
check('no manifest means nothing to be wrong about', () => {
|
|
615
|
+
assert.equal(pythonManifest('/definitely/not/a/repo'), undefined);
|
|
616
|
+
assert.deepEqual(localModules('/definitely/not/a/repo'), new Set());
|
|
617
|
+
});
|
|
618
|
+
console.log('\nruby gems');
|
|
619
|
+
const gems = { names: new Set(['rails', 'httparty', 'sidekiq']) };
|
|
620
|
+
const rbLocal = new Set(['helpers', 'models']);
|
|
621
|
+
check('the standard library needs no gem', () => {
|
|
622
|
+
for (const r of ['json', 'net/http', 'yaml', 'set', 'openssl', 'digest', 'fileutils']) {
|
|
623
|
+
assert.equal(isPhantomGem(r, gems, rbLocal), false, r + ' is stdlib');
|
|
624
|
+
}
|
|
625
|
+
});
|
|
626
|
+
check('a require provided by a declared gem is accounted for', () => {
|
|
627
|
+
// the Rails trap: the Gemfile says `rails`, the code requires `active_record`
|
|
628
|
+
assert.equal(isPhantomGem('active_record', gems, rbLocal), false);
|
|
629
|
+
assert.equal(isPhantomGem('active_support/core_ext', gems, rbLocal), false);
|
|
630
|
+
assert.equal(isPhantomGem('httparty', gems, rbLocal), false);
|
|
631
|
+
});
|
|
632
|
+
check("a require of this repository's own code is accounted for", () => {
|
|
633
|
+
assert.equal(isPhantomGem('helpers', gems, rbLocal), false);
|
|
634
|
+
assert.equal(isPhantomGem('models/user', gems, rbLocal), false);
|
|
635
|
+
});
|
|
636
|
+
check('a gem nothing declares is reported', () => {
|
|
637
|
+
assert.equal(isPhantomGem('nokogiri', gems, rbLocal), true);
|
|
638
|
+
assert.equal(isPhantomGem('faraday', gems, rbLocal), true);
|
|
639
|
+
});
|
|
640
|
+
check('no Gemfile means nothing to be wrong about', () => {
|
|
641
|
+
assert.equal(rubyManifest('/definitely/not/a/repo'), undefined);
|
|
642
|
+
});
|
|
643
|
+
console.log('\npython semantics');
|
|
644
|
+
check('pyright is optional — its absence is answered, not guessed at', () => {
|
|
645
|
+
// whichever way this machine is set up, the answer must be a boolean rather than
|
|
646
|
+
// a throw: a missing checker means those findings simply do not exist
|
|
647
|
+
assert.equal(typeof pyrightAvailable(process.cwd()), 'boolean');
|
|
648
|
+
});
|
|
649
|
+
console.log('\nintegrations');
|
|
650
|
+
const sample = [
|
|
651
|
+
{ id: 'F1', class: 'verified', check: 'phantom-dep', severity: 'high', confidence: 'proven',
|
|
652
|
+
file: 'src/a.ts', line: 3, title: 'missing dep', evidence: { oracle: 'package.json', detail: 'not declared' } },
|
|
653
|
+
{ id: 'F2', class: 'judged', check: 'plausible-logic', severity: 'low', confidence: 'tentative',
|
|
654
|
+
file: 'src/b.ts', line: 9, title: 'off by one' },
|
|
655
|
+
];
|
|
656
|
+
check('nested modules use the native package import map', () => {
|
|
657
|
+
const manifest = JSON.parse(readFileSync(join(process.cwd(), 'package.json'), 'utf8'));
|
|
658
|
+
assert.equal(manifest.imports?.['#app/*.js'], './dist/*.js');
|
|
659
|
+
const pending = [join(process.cwd(), 'src')];
|
|
660
|
+
while (pending.length > 0) {
|
|
661
|
+
const directory = pending.pop();
|
|
662
|
+
for (const entry of readdirSync(directory, { withFileTypes: true })) {
|
|
663
|
+
const path = join(directory, entry.name);
|
|
664
|
+
if (entry.isDirectory()) {
|
|
665
|
+
pending.push(path);
|
|
666
|
+
}
|
|
667
|
+
else if (entry.name.endsWith('.ts')) {
|
|
668
|
+
const source = readFileSync(path, 'utf8');
|
|
669
|
+
assert.doesNotMatch(source, /(?:\bfrom\s+|\bimport\s*(?:\(\s*)?|\brequire\s*\(\s*)['"]\.\.\//, path);
|
|
670
|
+
}
|
|
671
|
+
}
|
|
672
|
+
}
|
|
673
|
+
});
|
|
674
|
+
check('runtime package metadata follows package.json', () => {
|
|
675
|
+
const manifest = JSON.parse(readFileSync(join(process.cwd(), 'package.json'), 'utf8'));
|
|
676
|
+
assert.equal(PACKAGE_NAME, manifest.name);
|
|
677
|
+
assert.equal(PACKAGE_VERSION, manifest.version);
|
|
678
|
+
});
|
|
679
|
+
check('release smoke and publish consume the one packed artifact', () => {
|
|
680
|
+
const workflow = readFileSync(join(process.cwd(), '.github', 'workflows', 'release.yml'), 'utf8');
|
|
681
|
+
assert.match(workflow, /tags: \['v\*\.\*\.\*'\]/);
|
|
682
|
+
assert.equal(workflow.match(/npm pack --pack-destination/g)?.length, 1);
|
|
683
|
+
assert.match(workflow, /npm run smoke -- "\$\{\{ steps\.artifact\.outputs\.tarball \}\}"/);
|
|
684
|
+
assert.match(workflow, /npm publish "\.\/\$\{\{ steps\.artifact\.outputs\.tarball \}\}"/);
|
|
685
|
+
});
|
|
686
|
+
check('self-review publishes machine findings only for a complete verdict', () => {
|
|
687
|
+
const workflow = readFileSync(join(process.cwd(), '.github', 'workflows', 'review.yml'), 'utf8');
|
|
688
|
+
assert.equal(workflow.match(/node "\$PSH" review/g)?.length, 1);
|
|
689
|
+
assert.match(workflow, /name: Check out the untrusted review target[\s\S]+allow-unsafe-pr-checkout: true/);
|
|
690
|
+
assert.match(workflow, /steps\.review\.outputs\.status == '0' \|\| steps\.review\.outputs\.status == '1'/);
|
|
691
|
+
});
|
|
692
|
+
check('the public action persists judge answers and publishes only a verdict', () => {
|
|
693
|
+
const action = readFileSync(join(process.cwd(), 'action.yml'), 'utf8');
|
|
694
|
+
assert.match(action, /uses: actions\/cache@[a-f0-9]{40}/);
|
|
695
|
+
assert.match(action, /POWERSHOT_CACHE_DIR: \$\{\{ runner\.temp \}\}\/powershot-cache/);
|
|
696
|
+
assert.match(action, /restore-keys:/);
|
|
697
|
+
assert.match(action, /Upload SARIF[\s\S]+steps\.review\.outputs\.complete == 'true'/);
|
|
698
|
+
});
|
|
699
|
+
check('published CI examples preserve one verdict and its exit status', () => {
|
|
700
|
+
const github = readFileSync(join(process.cwd(), 'examples', 'github-actions', 'cli.yml'), 'utf8');
|
|
701
|
+
const gitlab = readFileSync(join(process.cwd(), 'examples', 'gitlab', '.gitlab-ci.yml'), 'utf8');
|
|
702
|
+
assert.match(github, /npm install --global --ignore-scripts @0xcraft\/powershot@1\.0\.0/);
|
|
703
|
+
assert.equal(github.match(/psh review/g)?.length, 1);
|
|
704
|
+
assert.match(github, /--report markdown=powershot\.md[\s\S]+--report sarif=powershot\.sarif/);
|
|
705
|
+
assert.match(github, /\|\| STATUS=\$\?[\s\S]+case "\$STATUS"/);
|
|
706
|
+
assert.match(gitlab, /npm install --global --ignore-scripts @0xcraft\/powershot@1\.0\.0/);
|
|
707
|
+
assert.equal(gitlab.match(/psh review/g)?.length, 1);
|
|
708
|
+
assert.match(gitlab, /--format codequality > gl-code-quality-report\.json \|\| STATUS=\$\?/);
|
|
709
|
+
assert.match(gitlab, /test "\$STATUS" -le 1 \|\| exit "\$STATUS"/);
|
|
710
|
+
});
|
|
711
|
+
check('code quality fingerprints are stable across runs', () => {
|
|
712
|
+
const a = JSON.parse(codeQuality(sample));
|
|
713
|
+
const b = JSON.parse(codeQuality(sample));
|
|
714
|
+
assert.equal(a[0].fingerprint, b[0].fingerprint); // else GitLab calls every finding new
|
|
715
|
+
assert.equal(a[0].severity, 'major');
|
|
716
|
+
assert.equal(a[1].severity, 'info');
|
|
717
|
+
assert.equal(a[0].location.lines.begin, 3);
|
|
718
|
+
});
|
|
719
|
+
check('the viewer is one self-contained page', () => {
|
|
720
|
+
const html = viewer(sample, {
|
|
721
|
+
id: 'abc', target: 'workspace', started: '2026-01-01T10:00:00Z', state: 'complete', notLookedAt: [],
|
|
722
|
+
});
|
|
723
|
+
assert.match(html, /<!doctype html>/);
|
|
724
|
+
assert.equal(/<(script|link|img)[^>]+(src|href)="http/.test(html), false); // no network needed
|
|
725
|
+
assert.equal((html.match(/class="f /g) ?? []).length, 2);
|
|
726
|
+
});
|
|
727
|
+
check('the viewer escapes content rather than rendering it', () => {
|
|
728
|
+
const nasty = [{ ...sample[0], title: '<img src=x onerror=alert(1)>' }];
|
|
729
|
+
const html = viewer(nasty, {
|
|
730
|
+
id: 'x', target: 't', started: '2026-01-01T10:00:00Z', state: 'complete', notLookedAt: [],
|
|
731
|
+
});
|
|
732
|
+
assert.equal(html.includes('<img src=x'), false);
|
|
733
|
+
assert.match(html, /<img/);
|
|
734
|
+
});
|
|
735
|
+
check('delegated findings preserve the agent evidence and suggestion', () => {
|
|
736
|
+
const got = absorbDelegated(JSON.stringify([
|
|
737
|
+
{ file: 'a.ts', line: 2, title: 'real', check: 'plausible-logic', severity: 'high', confidence: 'firm', why: 'w', suggestion: 'return value' },
|
|
738
|
+
]));
|
|
739
|
+
assert.equal(got.length, 1);
|
|
740
|
+
assert.equal(got[0].class, 'judged');
|
|
741
|
+
assert.equal(got[0].check, 'plausible-logic');
|
|
742
|
+
assert.equal(got[0].evidence?.oracle, 'delegated agent');
|
|
743
|
+
assert.equal(got[0].suggestion, 'return value');
|
|
744
|
+
});
|
|
745
|
+
check('delegated output distinguishes an empty verdict from malformed data', () => {
|
|
746
|
+
assert.deepEqual(absorbDelegated('[]'), []);
|
|
747
|
+
assert.throws(() => absorbDelegated('not json'), /not valid JSON/);
|
|
748
|
+
assert.throws(() => absorbDelegated('{"not":"an array"}'), /must be a JSON array/);
|
|
749
|
+
assert.throws(() => absorbDelegated('[{"file":"a.ts","title":"lost line"}]'), /positive integer line/);
|
|
750
|
+
});
|
|
751
|
+
check('delegate --checks selects only the requested judging brief', () => {
|
|
752
|
+
const cfg = {
|
|
753
|
+
provider: 'anthropic', model: 'm', verifiers: ['*'], judges: ['*'],
|
|
754
|
+
minSeverity: 'low', ignore: [], promptCache: true,
|
|
755
|
+
};
|
|
756
|
+
const brief = delegateBrief(ground([{ path: 'a.ts', after: 'export const a = 1\n' }]), cfg, {
|
|
757
|
+
checks: ['intent'], intent: 'add a',
|
|
758
|
+
});
|
|
759
|
+
assert.match(brief, /## Judge: intent/);
|
|
760
|
+
assert.doesNotMatch(brief, /## Judge: plausible-logic/);
|
|
761
|
+
});
|
|
762
|
+
check('every agent target writes a distinct, non-empty file', () => {
|
|
763
|
+
const paths = TARGETS.map((t) => t.path);
|
|
764
|
+
assert.equal(new Set(paths).size, paths.length, 'two targets share a path');
|
|
765
|
+
for (const t of TARGETS) {
|
|
766
|
+
assert.ok(t.render().length > 400, t.name + ' produced no real content');
|
|
767
|
+
assert.match(t.render(), /psh review --verify-only/, t.name + ' omits the command it exists to teach');
|
|
768
|
+
}
|
|
769
|
+
});
|
|
770
|
+
check('the claude target is a skill, with the frontmatter that makes it invocable', () => {
|
|
771
|
+
const skill = findTarget('claude').render();
|
|
772
|
+
assert.match(skill, /^---\nname: powershot-review\n/);
|
|
773
|
+
assert.match(skill, /^description: .{40,}/m);
|
|
774
|
+
});
|
|
775
|
+
check('the cursor target carries cursor rule frontmatter', () => {
|
|
776
|
+
assert.match(findTarget('cursor').render(), /^---\ndescription: /);
|
|
777
|
+
});
|
|
778
|
+
check('every target points at where its tool actually looks', () => {
|
|
779
|
+
assert.equal(findTarget('agents').path, 'AGENTS.md');
|
|
780
|
+
assert.equal(findTarget('copilot').path, '.github/copilot-instructions.md');
|
|
781
|
+
assert.equal(findTarget('cursor').path, '.cursor/rules/powershot.mdc');
|
|
782
|
+
assert.match(findTarget('claude').path, /^\.claude\/skills\//);
|
|
783
|
+
});
|
|
784
|
+
check('an unknown agent name is rejected rather than guessed at', () => {
|
|
785
|
+
assert.equal(findTarget('emacs'), undefined);
|
|
786
|
+
});
|
|
787
|
+
console.log('\njudge cache');
|
|
788
|
+
check('the key covers everything an answer depends on, and nothing else', () => {
|
|
789
|
+
const base = {
|
|
790
|
+
judge: 'plausible-logic', provider: 'anthropic', model: 'glm-4.6',
|
|
791
|
+
prompt: 'find defects', tools: false, content: 'diff A',
|
|
792
|
+
};
|
|
793
|
+
const same = JudgeCache.key(base);
|
|
794
|
+
assert.equal(JudgeCache.key({ ...base }), same); // same question, same key
|
|
795
|
+
// every one of these makes it a different question, and a key that ignored any of
|
|
796
|
+
// them would replay an answer that was given about something else
|
|
797
|
+
assert.notEqual(JudgeCache.key({ ...base, judge: 'security' }), same);
|
|
798
|
+
assert.notEqual(JudgeCache.key({ ...base, provider: 'openai' }), same);
|
|
799
|
+
assert.notEqual(JudgeCache.key({ ...base, model: 'glm-5.3' }), same);
|
|
800
|
+
assert.notEqual(JudgeCache.key({ ...base, prompt: 'find defects, carefully' }), same);
|
|
801
|
+
assert.notEqual(JudgeCache.key({ ...base, tools: true }), same);
|
|
802
|
+
assert.notEqual(JudgeCache.key({ ...base, content: 'diff B' }), same);
|
|
803
|
+
assert.notEqual(JudgeCache.key({ ...base, intent: 'bump the tax rate' }), same);
|
|
804
|
+
});
|
|
805
|
+
check('an answer is reused only for the question it answered', () => {
|
|
806
|
+
const dir = mkdtempSync(join(tmpdir(), 'psh-cache-'));
|
|
807
|
+
const cache = JudgeCache.open(dir);
|
|
808
|
+
const found = [
|
|
809
|
+
{ id: 'F1', class: 'judged', check: 'plausible-logic', severity: 'high', confidence: 'firm',
|
|
810
|
+
file: 'a.ts', line: 2, title: 'inverted condition' },
|
|
811
|
+
];
|
|
812
|
+
const ask = (content) => JudgeCache.key({ judge: 'plausible-logic', provider: 'anthropic', model: 'm', prompt: 'p', tools: false, content });
|
|
813
|
+
const key = ask('the diff');
|
|
814
|
+
assert.equal(cache.get(key), undefined);
|
|
815
|
+
cache.put(key, found, '2026-01-01T00:00:00Z');
|
|
816
|
+
cache.save();
|
|
817
|
+
const reopened = JudgeCache.open(dir);
|
|
818
|
+
assert.equal(reopened.get(key)?.length, 1);
|
|
819
|
+
assert.equal(reopened.get(ask('a different diff')), undefined);
|
|
820
|
+
rmSync(dir, { recursive: true, force: true });
|
|
821
|
+
});
|
|
822
|
+
check('a corrupt cache is ignored rather than fatal', () => {
|
|
823
|
+
const dir = mkdtempSync(join(tmpdir(), 'psh-cache-'));
|
|
824
|
+
mkdirSync(join(dir, '.powershot'), { recursive: true });
|
|
825
|
+
writeFileSync(join(dir, '.powershot', 'judge-cache.json'), 'not json at all');
|
|
826
|
+
assert.equal(JudgeCache.open(dir).size, 0);
|
|
827
|
+
writeFileSync(join(dir, '.powershot', 'judge-cache.json'), 'null');
|
|
828
|
+
assert.equal(JudgeCache.open(dir).size, 0);
|
|
829
|
+
rmSync(dir, { recursive: true, force: true });
|
|
830
|
+
});
|
|
831
|
+
check('a gated cache cannot be redirected into the reviewed tree', () => {
|
|
832
|
+
const dir = realpathSync(mkdtempSync(join(tmpdir(), 'psh-cache-boundary-')));
|
|
833
|
+
const previous = process.env.POWERSHOT_CACHE_DIR;
|
|
834
|
+
process.env.POWERSHOT_CACHE_DIR = dir;
|
|
835
|
+
try {
|
|
836
|
+
assert.throws(() => JudgeCache.open(dir, true), /must resolve outside/);
|
|
837
|
+
assert.equal(existsSync(join(dir, 'powershot')), false);
|
|
838
|
+
}
|
|
839
|
+
finally {
|
|
840
|
+
if (previous === undefined)
|
|
841
|
+
delete process.env.POWERSHOT_CACHE_DIR;
|
|
842
|
+
else
|
|
843
|
+
process.env.POWERSHOT_CACHE_DIR = previous;
|
|
844
|
+
rmSync(dir, { recursive: true, force: true });
|
|
845
|
+
}
|
|
846
|
+
});
|
|
847
|
+
check('gated cache identity survives a second clone of the same remote', () => {
|
|
848
|
+
const parent = realpathSync(mkdtempSync(join(tmpdir(), 'psh-cache-identity-')));
|
|
849
|
+
const cacheRoot = join(parent, 'cache');
|
|
850
|
+
const previous = process.env.POWERSHOT_CACHE_DIR;
|
|
851
|
+
process.env.POWERSHOT_CACHE_DIR = cacheRoot;
|
|
852
|
+
const init = (name, remote) => {
|
|
853
|
+
const dir = join(parent, name);
|
|
854
|
+
mkdirSync(dir);
|
|
855
|
+
execFileSync('git', ['init', '-q', '.'], { cwd: dir });
|
|
856
|
+
execFileSync('git', ['remote', 'add', 'origin', remote], { cwd: dir });
|
|
857
|
+
return dir;
|
|
858
|
+
};
|
|
859
|
+
try {
|
|
860
|
+
const first = init('first', 'https://example.test/team/repo.git');
|
|
861
|
+
const second = init('second', 'https://example.test/team/repo.git');
|
|
862
|
+
const other = init('other', 'https://example.test/team/other.git');
|
|
863
|
+
const found = [{
|
|
864
|
+
id: 'F1', class: 'judged', check: 'intent', severity: 'high', confidence: 'firm',
|
|
865
|
+
file: 'a.ts', line: 1, title: 'same answer',
|
|
866
|
+
}];
|
|
867
|
+
const one = JudgeCache.open(first, true);
|
|
868
|
+
one.put('question', found, '2026-01-01T00:00:00Z');
|
|
869
|
+
one.save();
|
|
870
|
+
assert.equal(JudgeCache.open(second, true).get('question')?.[0]?.title, 'same answer');
|
|
871
|
+
assert.equal(JudgeCache.open(other, true).get('question'), undefined);
|
|
872
|
+
}
|
|
873
|
+
finally {
|
|
874
|
+
if (previous === undefined)
|
|
875
|
+
delete process.env.POWERSHOT_CACHE_DIR;
|
|
876
|
+
else
|
|
877
|
+
process.env.POWERSHOT_CACHE_DIR = previous;
|
|
878
|
+
rmSync(parent, { recursive: true, force: true });
|
|
879
|
+
}
|
|
880
|
+
});
|
|
881
|
+
console.log('\nsession comparison');
|
|
882
|
+
check('a second review says what was fixed, what is new, and what stayed', () => {
|
|
883
|
+
const dir = mkdtempSync(join(tmpdir(), 'psh-diff-'));
|
|
884
|
+
const mk = (titles) => {
|
|
885
|
+
const s = Session.create(dir, 'workspace');
|
|
886
|
+
s.saveReport(titles.map((t, i) => ({
|
|
887
|
+
id: 'F' + i, class: 'judged', check: 'plausible-logic', severity: 'high',
|
|
888
|
+
confidence: 'firm', file: 'a.ts', line: i + 1, title: t,
|
|
889
|
+
})), { state: 'complete', notLookedAt: [] });
|
|
890
|
+
return s;
|
|
891
|
+
};
|
|
892
|
+
const before = mk(['inverted condition', 'off by one']);
|
|
893
|
+
const after = mk(['off by one', 'missing await']);
|
|
894
|
+
const { fixed, introduced, remaining } = Session.compare(before, after);
|
|
895
|
+
assert.deepEqual(fixed.map((f) => f.title), ['inverted condition']);
|
|
896
|
+
assert.deepEqual(introduced.map((f) => f.title), ['missing await']);
|
|
897
|
+
assert.deepEqual(remaining.map((f) => f.title), ['off by one']);
|
|
898
|
+
rmSync(dir, { recursive: true, force: true });
|
|
899
|
+
});
|
|
900
|
+
check('a finding is matched past a line that moved under it', () => {
|
|
901
|
+
const dir = mkdtempSync(join(tmpdir(), 'psh-diff-'));
|
|
902
|
+
const at = (line) => {
|
|
903
|
+
const s = Session.create(dir, 'workspace');
|
|
904
|
+
s.saveReport([{ id: 'F1', class: 'judged', check: 'plausible-logic', severity: 'high',
|
|
905
|
+
confidence: 'firm', file: 'a.ts', line, title: 'off by one' }], { state: 'complete', notLookedAt: [] });
|
|
906
|
+
return s;
|
|
907
|
+
};
|
|
908
|
+
// code above it grew, so the same defect now sits ten lines lower — not a new one
|
|
909
|
+
assert.equal(Session.compare(at(12), at(22)).introduced.length, 0);
|
|
910
|
+
assert.equal(Session.compare(at(12), at(22)).remaining.length, 1);
|
|
911
|
+
rmSync(dir, { recursive: true, force: true });
|
|
912
|
+
});
|
|
913
|
+
check('partial sessions cannot be compared or rendered as clean', () => {
|
|
914
|
+
const dir = mkdtempSync(join(tmpdir(), 'psh-partial-session-'));
|
|
915
|
+
const partial = Session.create(dir, 'workspace');
|
|
916
|
+
partial.saveReport([], { state: 'partial', notLookedAt: ['outside.ts (no types)'] });
|
|
917
|
+
const complete = Session.create(dir, 'workspace');
|
|
918
|
+
complete.saveReport([], { state: 'complete', notLookedAt: [] });
|
|
919
|
+
assert.throws(() => Session.compare(partial, complete), /only complete/);
|
|
920
|
+
const html = viewer([], {
|
|
921
|
+
id: partial.id, target: partial.target, started: partial.started, state: 'partial',
|
|
922
|
+
notLookedAt: partial.report.notLookedAt ?? [],
|
|
923
|
+
});
|
|
924
|
+
assert.match(html, /partial — not a verdict/);
|
|
925
|
+
assert.match(html, /outside\.ts \(no types\)/);
|
|
926
|
+
assert.doesNotMatch(html, />No findings\.<\/p>/);
|
|
927
|
+
rmSync(dir, { recursive: true, force: true });
|
|
928
|
+
});
|
|
929
|
+
console.log('\nsessions and scan');
|
|
930
|
+
const tmp = mkdtempSync(join(tmpdir(), 'psh-'));
|
|
931
|
+
check('a session replays an answer only for the content it answered', () => {
|
|
932
|
+
const s = Session.create(tmp, 'workspace');
|
|
933
|
+
assert.equal(s.get('security', 'a.ts', 'diff v1'), undefined);
|
|
934
|
+
s.record('security', 'a.ts', 'diff v1', [
|
|
935
|
+
{ id: 'F1', class: 'judged', check: 'security', severity: 'high', confidence: 'firm', file: 'a.ts', line: 2, title: 'x' },
|
|
936
|
+
]);
|
|
937
|
+
const reopened = Session.open(tmp, s.id);
|
|
938
|
+
assert.ok(reopened, 'session should reopen from disk');
|
|
939
|
+
assert.equal(reopened.get('security', 'a.ts', 'diff v1')?.length, 1);
|
|
940
|
+
assert.equal(reopened.get('security', 'b.ts', 'diff v1'), undefined); // only what was paid for
|
|
941
|
+
// the same unit name over edited content is a different question, not a hit
|
|
942
|
+
assert.equal(reopened.get('security', 'a.ts', 'diff v2'), undefined);
|
|
943
|
+
});
|
|
944
|
+
check('sessions are listed newest first, with their progress', () => {
|
|
945
|
+
const rows = Session.list(tmp);
|
|
946
|
+
assert.ok(rows.length >= 1);
|
|
947
|
+
assert.ok(rows[0].done >= 1);
|
|
948
|
+
});
|
|
949
|
+
check('opening a session that does not exist returns undefined, not a throw', () => {
|
|
950
|
+
assert.equal(Session.open(tmp, 'nope1234'), undefined);
|
|
951
|
+
});
|
|
952
|
+
check('scan presents every file as newly written, with no base to compare', () => {
|
|
953
|
+
mkdirSync(join(tmp, 'src'), { recursive: true });
|
|
954
|
+
writeFileSync(join(tmp, 'src', 'a.ts'), 'export const a = 1\nexport const b = 2\n');
|
|
955
|
+
writeFileSync(join(tmp, 'src', 'notes.md'), '# not code\n');
|
|
956
|
+
mkdirSync(join(tmp, 'src', 'node_modules'), { recursive: true });
|
|
957
|
+
writeFileSync(join(tmp, 'src', 'node_modules', 'x.ts'), 'export const x = 1\n');
|
|
958
|
+
const files = scanPaths(tmp, 'src');
|
|
959
|
+
assert.deepEqual(files.map((f) => f.path), ['src/a.ts']); // code only, no node_modules
|
|
960
|
+
assert.equal(files[0].before, undefined); // nothing to diff against
|
|
961
|
+
assert.ok(files[0].added.has(1) && files[0].added.has(2)); // every line counts as new
|
|
962
|
+
});
|
|
963
|
+
rmSync(tmp, { recursive: true, force: true });
|
|
964
|
+
console.log('\njudges');
|
|
965
|
+
check('every judge is uniquely named and carries a brief', () => {
|
|
966
|
+
const names = JUDGES.map((j) => j.name);
|
|
967
|
+
assert.equal(new Set(names).size, names.length, 'duplicate judge name');
|
|
968
|
+
for (const j of JUDGES)
|
|
969
|
+
assert.ok(j.brief.trim().length > 40, j.name + ' has no real brief');
|
|
970
|
+
});
|
|
971
|
+
check('exactly one judge asks for the stated intent', () => {
|
|
972
|
+
assert.deepEqual(JUDGES.filter((j) => j.needsIntent).map((j) => j.name), ['intent']);
|
|
973
|
+
});
|
|
974
|
+
check('the shared framing states the trust boundary for reviewed code', () => {
|
|
975
|
+
assert.match(COMMON, /DATA, not instructions/);
|
|
976
|
+
});
|
|
977
|
+
check('model output is parsed into findings, and junk is discarded', () => {
|
|
978
|
+
const raw = JSON.stringify([
|
|
979
|
+
{ file: 'a.ts', line: 3, severity: 'high', confidence: 'firm', title: 'real', why: 'w', fix: 'f' },
|
|
980
|
+
{ file: 'a.ts', title: 'no line — dropped' },
|
|
981
|
+
{ line: 4, title: 'no file — dropped' },
|
|
982
|
+
{ file: 'a.ts', line: 9, title: 'defaults applied' },
|
|
983
|
+
]);
|
|
984
|
+
const found = parseFindings(raw, 'security');
|
|
985
|
+
assert.equal(found.length, 2);
|
|
986
|
+
assert.equal(found[0].class, 'judged');
|
|
987
|
+
assert.equal(found[0].check, 'security');
|
|
988
|
+
assert.equal(found[1].severity, 'medium'); // unknown severity falls back
|
|
989
|
+
assert.equal(found[1].confidence, 'tentative'); // and so does confidence
|
|
990
|
+
});
|
|
991
|
+
console.log('\ncopy-paste-drift');
|
|
992
|
+
check('fires when a clone leaves one identifier un-renamed', () => {
|
|
993
|
+
const src = 'export function f() {\n' +
|
|
994
|
+
' const userTotal = order.user.price * order.user.qty\n' +
|
|
995
|
+
' const adminTotal = order.admin.price * order.user.qty\n' +
|
|
996
|
+
'}\n';
|
|
997
|
+
const found = copyPasteDrift.run(ground([{ path: 'a.ts', after: src }]));
|
|
998
|
+
assert.equal(found.length, 1);
|
|
999
|
+
assert.match(found[0].title, /`user` renamed inconsistently/);
|
|
1000
|
+
});
|
|
1001
|
+
check('silent when the rename is consistent — ordinary parallel code', () => {
|
|
1002
|
+
const src = 'export function f() {\n' +
|
|
1003
|
+
' const userTotal = order.user.price * order.user.qty\n' +
|
|
1004
|
+
' const adminTotal = order.admin.price * order.admin.qty\n' +
|
|
1005
|
+
'}\n';
|
|
1006
|
+
assert.equal(fires(copyPasteDrift, ground([{ path: 'a.ts', after: src }])), false);
|
|
1007
|
+
});
|
|
1008
|
+
check('silent on an exact duplicate, which is a different concern', () => {
|
|
1009
|
+
const src = 'export function f() {\n' +
|
|
1010
|
+
' const a = order.user.price * order.user.qty\n' +
|
|
1011
|
+
' const a = order.user.price * order.user.qty\n' +
|
|
1012
|
+
'}\n';
|
|
1013
|
+
assert.equal(fires(copyPasteDrift, ground([{ path: 'a.ts', after: src }])), false);
|
|
1014
|
+
});
|
|
1015
|
+
check('silent when two things merely share a name', () => {
|
|
1016
|
+
// found by running this check over its own repository: a local `tests` beside a
|
|
1017
|
+
// member `.tests` maps to two NEW names, which is not a rename anyone forgot
|
|
1018
|
+
const src = 'export function f() {\n' +
|
|
1019
|
+
' const tests = async (s: string) => pack.tests!(parse(pack, s).rootNode)\n' +
|
|
1020
|
+
' const docs = async (s: string) => pack.documentedParams!(parse(pack, s).rootNode)\n' +
|
|
1021
|
+
'}\n';
|
|
1022
|
+
assert.equal(fires(copyPasteDrift, ground([{ path: 'a.ts', after: src }])), false);
|
|
1023
|
+
});
|
|
1024
|
+
check('silent when literals differ, since the shapes are not clones', () => {
|
|
1025
|
+
const src = 'export function f() {\n' +
|
|
1026
|
+
' const userTotal = order.user.price * order.user.qty * 2\n' +
|
|
1027
|
+
' const adminTotal = order.admin.price * order.user.qty * 3\n' +
|
|
1028
|
+
'}\n';
|
|
1029
|
+
assert.equal(fires(copyPasteDrift, ground([{ path: 'a.ts', after: src }])), false);
|
|
1030
|
+
});
|
|
1031
|
+
console.log('\ndead-on-arrival');
|
|
1032
|
+
check('fires on a module-private declaration nothing references', () => {
|
|
1033
|
+
const found = deadOnArrival.run(ground([{ path: 'a.ts', after: 'function orphan(n: number) { return n }\n' }]));
|
|
1034
|
+
assert.equal(found.length, 1);
|
|
1035
|
+
assert.match(found[0].title, /orphan/);
|
|
1036
|
+
});
|
|
1037
|
+
check('silent when the declaration is exported — it may be public API', () => {
|
|
1038
|
+
assert.equal(fires(deadOnArrival, ground([{ path: 'a.ts', after: 'export function used(n: number) { return n }\n' }])), false);
|
|
1039
|
+
});
|
|
1040
|
+
check('silent when something references it', () => {
|
|
1041
|
+
const src = 'function helper(n: number) { return n }\nexport const go = () => helper(1)\n';
|
|
1042
|
+
assert.equal(fires(deadOnArrival, ground([{ path: 'a.ts', after: src }])), false);
|
|
1043
|
+
});
|
|
1044
|
+
check('respects the _name convention for a deliberate non-reference', () => {
|
|
1045
|
+
assert.equal(fires(deadOnArrival, ground([{ path: 'a.ts', after: 'function _scratch(n: number) { return n }\n' }])), false);
|
|
1046
|
+
});
|
|
1047
|
+
console.log('\nlying-comment');
|
|
1048
|
+
check('fires when @param names an argument the function does not take', () => {
|
|
1049
|
+
const src = '/**\n * @param currency which currency\n */\nexport function charge(id: string): number { return 1 }\n';
|
|
1050
|
+
const found = lyingComment.run(ground([{ path: 'a.ts', after: src }]));
|
|
1051
|
+
assert.equal(found.length, 1);
|
|
1052
|
+
assert.match(found[0].title, /currency/);
|
|
1053
|
+
});
|
|
1054
|
+
check('fires when @returns promises a value from a void function', () => {
|
|
1055
|
+
const src = '/**\n * @returns a receipt id\n */\nexport function charge(id: string): void {}\n';
|
|
1056
|
+
const found = lyingComment.run(ground([{ path: 'a.ts', after: src }]));
|
|
1057
|
+
assert.ok(found.some((f) => /@returns/.test(f.title)));
|
|
1058
|
+
});
|
|
1059
|
+
check('silent when the documentation matches the signature', () => {
|
|
1060
|
+
const src = '/**\n * @param id who\n * @returns the total\n */\nexport function charge(id: string): number { return 1 }\n';
|
|
1061
|
+
assert.equal(fires(lyingComment, ground([{ path: 'a.ts', after: src }])), false);
|
|
1062
|
+
});
|
|
1063
|
+
check('allows @param documenting a property of a real parameter', () => {
|
|
1064
|
+
const src = '/**\n * @param opts.retries how many\n */\nexport function go(opts: { retries: number }): number { return 1 }\n';
|
|
1065
|
+
assert.equal(fires(lyingComment, ground([{ path: 'a.ts', after: src }])), false);
|
|
1066
|
+
});
|
|
1067
|
+
console.log('\ncontract-drift');
|
|
1068
|
+
// the caller lives in a file the change does not touch, so it is passed as an
|
|
1069
|
+
// unchanged extra file — exactly the blast radius phantom-api cannot see
|
|
1070
|
+
const MAILER_BEFORE = 'export function sendMail(to: string): boolean {\n return to.length > 0\n}\n';
|
|
1071
|
+
const caller = { path: 'signup.ts', after: "import { sendMail } from './mailer.js'\nexport const go = () => sendMail('a')\n" };
|
|
1072
|
+
function driftGround(after) {
|
|
1073
|
+
const g = ground([{ path: 'mailer.ts', after, before: MAILER_BEFORE }, caller]);
|
|
1074
|
+
// the caller file is present for reference resolution but is NOT part of the change
|
|
1075
|
+
g.changed = g.changed.filter((c) => c.path !== 'signup.ts');
|
|
1076
|
+
g.files = g.files.filter((f) => f.changed.path !== 'signup.ts');
|
|
1077
|
+
return g;
|
|
1078
|
+
}
|
|
1079
|
+
check('fires when a required parameter is added and callers are left behind', () => {
|
|
1080
|
+
const found = contractDrift.run(driftGround('export function sendMail(to: string, subject: string): boolean {\n return true\n}\n'));
|
|
1081
|
+
assert.equal(found.length, 1);
|
|
1082
|
+
assert.equal(found[0].confidence, 'proven');
|
|
1083
|
+
assert.match(found[0].title, /requires 2 argument/);
|
|
1084
|
+
assert.match(found[0].evidence.detail, /signup\.ts/);
|
|
1085
|
+
});
|
|
1086
|
+
check('silent when the new parameter is optional — no caller breaks', () => {
|
|
1087
|
+
const found = contractDrift.run(driftGround('export function sendMail(to: string, subject?: string): boolean {\n return true\n}\n'));
|
|
1088
|
+
assert.equal(found.length, 0);
|
|
1089
|
+
});
|
|
1090
|
+
check('silent when a parameter has a default — no caller breaks', () => {
|
|
1091
|
+
const found = contractDrift.run(driftGround("export function sendMail(to: string, subject = 'hi'): boolean {\n return true\n}\n"));
|
|
1092
|
+
assert.equal(found.length, 0);
|
|
1093
|
+
});
|
|
1094
|
+
check('fires when a parameter is removed, since callers now pass too many', () => {
|
|
1095
|
+
const found = contractDrift.run(driftGround('export function sendMail(): boolean {\n return true\n}\n'));
|
|
1096
|
+
assert.equal(found.length, 1);
|
|
1097
|
+
assert.match(found[0].title, /takes 0 parameter/);
|
|
1098
|
+
});
|
|
1099
|
+
check('reports a changed parameter type as firm, not proven', () => {
|
|
1100
|
+
const found = contractDrift.run(driftGround('export function sendMail(to: number): boolean {\n return true\n}\n'));
|
|
1101
|
+
assert.equal(found.length, 1);
|
|
1102
|
+
assert.equal(found[0].confidence, 'firm');
|
|
1103
|
+
});
|
|
1104
|
+
check('silent when the signature did not change', () => {
|
|
1105
|
+
assert.equal(contractDrift.run(driftGround(MAILER_BEFORE)).length, 0);
|
|
1106
|
+
});
|
|
1107
|
+
console.log('\nscope-creep');
|
|
1108
|
+
check('fires when a file is reformatted without changing the program', () => {
|
|
1109
|
+
const before = 'export function double(n: number): number { return n * 2 }\n';
|
|
1110
|
+
const after = 'export function double(n: number): number {\n return n * 2\n}\n';
|
|
1111
|
+
const found = scopeCreep.run(ground([{ path: 'u.ts', after, before }]));
|
|
1112
|
+
assert.equal(found.length, 1);
|
|
1113
|
+
assert.equal(found[0].confidence, 'proven');
|
|
1114
|
+
assert.match(found[0].title, /only formatting/);
|
|
1115
|
+
});
|
|
1116
|
+
check('distinguishes a comment-only edit from pure reformatting', () => {
|
|
1117
|
+
const before = 'export const a = 1\n';
|
|
1118
|
+
const after = '// why a is 1\nexport const a = 1\n';
|
|
1119
|
+
const found = scopeCreep.run(ground([{ path: 'u.ts', after, before }]));
|
|
1120
|
+
assert.equal(found.length, 1);
|
|
1121
|
+
assert.match(found[0].title, /only comments/);
|
|
1122
|
+
});
|
|
1123
|
+
check('silent when the program actually changed', () => {
|
|
1124
|
+
const before = 'export const a = 1\n';
|
|
1125
|
+
const after = 'export const a = 2\n';
|
|
1126
|
+
assert.equal(fires(scopeCreep, ground([{ path: 'u.ts', after, before }])), false);
|
|
1127
|
+
});
|
|
1128
|
+
check('silent for a new file, which always adds something', () => {
|
|
1129
|
+
assert.equal(fires(scopeCreep, ground([{ path: 'u.ts', after: 'export const a = 1\n' }])), false);
|
|
1130
|
+
});
|
|
1131
|
+
check('silent when the file was not touched at all', () => {
|
|
1132
|
+
const same = 'export const a = 1\n';
|
|
1133
|
+
assert.equal(fires(scopeCreep, ground([{ path: 'u.ts', after: same, before: same }])), false);
|
|
1134
|
+
});
|
|
1135
|
+
console.log('\ncaret positioning');
|
|
1136
|
+
check('verifiers pin the exact span, not just the line', () => {
|
|
1137
|
+
const g = ground([{ path: 'a.ts', after: "import ky from 'ky'\nexport const x = ky\n" }], ['zod']);
|
|
1138
|
+
const found = phantomDep.run(g);
|
|
1139
|
+
const span = found[0].span;
|
|
1140
|
+
assert.ok(span, 'expected a span');
|
|
1141
|
+
// the caret must land on the specifier literal 'ky', not the whole import
|
|
1142
|
+
const line = "import ky from 'ky'";
|
|
1143
|
+
assert.equal(line.slice(span.column - 1, span.column - 1 + span.length), "'ky'");
|
|
1144
|
+
});
|
|
1145
|
+
check('the caret shifts with the dedent so it stays under its token', () => {
|
|
1146
|
+
// 0123456789
|
|
1147
|
+
const rendered = 'return inv.total'; // was ' return inv.total', dedent 4
|
|
1148
|
+
const span = { column: 12, length: 3 }; // "inv" in the original line
|
|
1149
|
+
assert.deepEqual(caretFor(span, rendered, 4), { offset: 7, length: 3 });
|
|
1150
|
+
assert.equal(rendered.slice(7, 10), 'inv');
|
|
1151
|
+
});
|
|
1152
|
+
check('an unplaceable span yields no caret rather than a wrong one', () => {
|
|
1153
|
+
assert.equal(caretFor({ column: 99, length: 3 }, 'short', 0), undefined); // past the end
|
|
1154
|
+
assert.equal(caretFor({ column: 2, length: 3 }, 'short', 8), undefined); // before the start
|
|
1155
|
+
assert.equal(caretFor(undefined, 'short', 0), undefined); // no span at all
|
|
1156
|
+
assert.equal(caretFor({ column: 1, length: 3 }, undefined, 0), undefined); // no line
|
|
1157
|
+
});
|
|
1158
|
+
check('a span running past the line end is clamped, not overflowed', () => {
|
|
1159
|
+
assert.deepEqual(caretFor({ column: 3, length: 999 }, 'abcde', 0), { offset: 2, length: 3 });
|
|
1160
|
+
});
|
|
1161
|
+
console.log('\nphantom-api');
|
|
1162
|
+
check('refuses to run without a tsconfig rather than guessing', () => {
|
|
1163
|
+
// g.typed is false in these fixtures: without lib/type resolution every global
|
|
1164
|
+
// would look invented, so the verifier must stay silent instead of inventing findings.
|
|
1165
|
+
const g = ground([{ path: 'a.ts', after: 'export const x = totallyUnknownGlobal\n' }]);
|
|
1166
|
+
assert.equal(phantomApi.run(g).length, 0);
|
|
1167
|
+
});
|
|
1168
|
+
console.log('\nhelpers');
|
|
1169
|
+
check('extractJsonArray survives prose and code fences', () => {
|
|
1170
|
+
assert.deepEqual(extractJsonArray('Sure!\n```json\n[{"a":1}]\n```\n'), [{ a: 1 }]);
|
|
1171
|
+
assert.deepEqual(extractJsonArray('here you go: [1,2] done'), [1, 2]);
|
|
1172
|
+
assert.deepEqual(extractJsonArray('no json at all'), []);
|
|
1173
|
+
});
|
|
1174
|
+
check('a proxy BASE_URL is treated as a base, not a full endpoint', () => {
|
|
1175
|
+
assert.equal(endpoint(undefined, 'https://api.anthropic.com', '/v1/messages'), 'https://api.anthropic.com/v1/messages');
|
|
1176
|
+
assert.equal(endpoint('https://gw.internal', 'https://api.anthropic.com', '/v1/messages'), 'https://gw.internal/v1/messages');
|
|
1177
|
+
assert.equal(endpoint('https://gw.internal/', 'https://api.anthropic.com', '/v1/messages'), 'https://gw.internal/v1/messages');
|
|
1178
|
+
});
|
|
1179
|
+
check('a suggestion takes its indentation from the file, not the model', () => {
|
|
1180
|
+
// measured against a real judge: it got the fix exactly right and returned it at
|
|
1181
|
+
// four spaces where the file used two. Where a line sits is a fact about the file.
|
|
1182
|
+
assert.equal(validateSuggestion(' if (amount > balance) {', ' if (amount <= balance) {'), ' if (amount > balance) {');
|
|
1183
|
+
assert.equal(validateSuggestion('if (amount > balance) {', ' if (amount <= balance) {'), ' if (amount > balance) {');
|
|
1184
|
+
assert.equal(validateSuggestion(' return n', 'return m'), 'return n'); // no indent to restore
|
|
1185
|
+
});
|
|
1186
|
+
check('a suggestion that changes nothing is dropped', () => {
|
|
1187
|
+
assert.equal(validateSuggestion(' return n', ' return n'), undefined);
|
|
1188
|
+
assert.equal(validateSuggestion('return n', ' return n'), undefined); // same code, different layout
|
|
1189
|
+
assert.equal(validateSuggestion(' ', ' return n'), undefined);
|
|
1190
|
+
});
|
|
1191
|
+
check('a suggestion is the whole line, ready to commit', () => {
|
|
1192
|
+
// GitHub and GitLab render a ```suggestion block as one-click apply, so it must
|
|
1193
|
+
// carry the complete replacement line rather than the fragment that changed
|
|
1194
|
+
const f = {
|
|
1195
|
+
id: 'F1', class: 'verified', check: 'phantom-api', severity: 'high', confidence: 'proven',
|
|
1196
|
+
file: 'a.ts', line: 5, title: "Did you mean 'toUpperCase'?",
|
|
1197
|
+
suggestion: ' return inv.customer.toUpperCase()',
|
|
1198
|
+
};
|
|
1199
|
+
const md = markdown([f]);
|
|
1200
|
+
assert.match(md, /```suggestion\n return inv\.customer\.toUpperCase\(\)\n```/);
|
|
1201
|
+
});
|
|
1202
|
+
check('a finding with only advice keeps a plain block, never a suggestion', () => {
|
|
1203
|
+
const f = {
|
|
1204
|
+
id: 'F1', class: 'verified', check: 'swallowed-error', severity: 'high', confidence: 'proven',
|
|
1205
|
+
file: 'a.ts', line: 5, title: 'empty catch', fix: 'Rethrow, or say why ignoring it is safe',
|
|
1206
|
+
};
|
|
1207
|
+
const md = markdown([f]);
|
|
1208
|
+
assert.equal(md.includes('```suggestion'), false); // advice is not an applicable patch
|
|
1209
|
+
assert.match(md, /Rethrow/);
|
|
1210
|
+
});
|
|
1211
|
+
check('one defect reported twice in different words is one finding', () => {
|
|
1212
|
+
// two judges can land on the same place and word it differently
|
|
1213
|
+
assert.ok(titleOverlap('Inverted condition for insufficient funds', 'Condition for insufficient funds is inverted') >= 0.7);
|
|
1214
|
+
assert.ok(titleOverlap('Off-by-one in the loop bound', 'Missing await on the async call') < 0.7);
|
|
1215
|
+
assert.equal(titleOverlap('', 'anything'), 0);
|
|
1216
|
+
});
|
|
1217
|
+
check('sarif output has the shape GitHub code scanning ingests', () => {
|
|
1218
|
+
const doc = JSON.parse(sarif([
|
|
1219
|
+
{ id: 'F1', class: 'verified', check: 'phantom-dep', severity: 'high', confidence: 'proven',
|
|
1220
|
+
file: 'src/a.ts', line: 3, title: 'missing dep' },
|
|
1221
|
+
{ id: 'F2', class: 'judged', check: 'plausible-logic', severity: 'low', confidence: 'tentative',
|
|
1222
|
+
file: 'src/b.ts', line: 9, title: 'off by one' },
|
|
1223
|
+
]));
|
|
1224
|
+
assert.equal(doc.version, '2.1.0');
|
|
1225
|
+
assert.equal(doc.runs[0].tool.driver.rules.length, 2); // one rule per distinct check
|
|
1226
|
+
assert.equal(doc.runs[0].results.length, 2);
|
|
1227
|
+
assert.equal(doc.runs[0].results[0].level, 'error'); // high -> error
|
|
1228
|
+
assert.equal(doc.runs[0].results[1].level, 'note'); // low -> note
|
|
1229
|
+
assert.equal(doc.runs[0].results[0].locations[0].physicalLocation.region.startLine, 3);
|
|
1230
|
+
assert.equal(doc.runs[0].results[0].locations[0].physicalLocation.artifactLocation.uri, 'src/a.ts');
|
|
1231
|
+
});
|
|
1232
|
+
check('titles wrap at word boundaries instead of cutting mid-word', () => {
|
|
1233
|
+
assert.deepEqual(wrap('the quick brown fox', 9), ['the quick', 'brown fox']);
|
|
1234
|
+
assert.deepEqual(wrap('short', 40), ['short']);
|
|
1235
|
+
// a single word longer than the width still gets its own line rather than vanishing
|
|
1236
|
+
assert.deepEqual(wrap('supercalifragilistic', 5), ['supercalifragilistic']);
|
|
1237
|
+
assert.deepEqual(wrap('', 10), ['']);
|
|
1238
|
+
});
|
|
1239
|
+
check('highlighting never loses or mangles the source text', () => {
|
|
1240
|
+
// colour is off in this process (piped), so highlight is identity — the contract
|
|
1241
|
+
// that matters is that the visible characters always survive intact
|
|
1242
|
+
const strip = (s) => s.replace(new RegExp(String.fromCharCode(27) + '\\[[0-9;]*m', 'g'), '');
|
|
1243
|
+
const samples = [
|
|
1244
|
+
'const x = `t ${a} b`',
|
|
1245
|
+
"if (a) { return /re?g/.test('s') } // note",
|
|
1246
|
+
'export default class A extends B {}',
|
|
1247
|
+
'const n = 0xFF + 1000n',
|
|
1248
|
+
'',
|
|
1249
|
+
];
|
|
1250
|
+
for (const src of samples) {
|
|
1251
|
+
assert.equal(strip(highlight(src)), src, 'text changed for: ' + src);
|
|
1252
|
+
}
|
|
1253
|
+
});
|
|
1254
|
+
check('highlighting survives input the scanner cannot parse', () => {
|
|
1255
|
+
assert.equal(typeof highlight('const = = = "unterminated'), 'string'); // must not throw
|
|
1256
|
+
});
|
|
1257
|
+
check('jsx files are detected for the right scanner variant', () => {
|
|
1258
|
+
assert.equal(isJsx('src/App.tsx'), true);
|
|
1259
|
+
assert.equal(isJsx('src/app.ts'), false);
|
|
1260
|
+
});
|
|
1261
|
+
check('ignore globs match the way the config claims', () => {
|
|
1262
|
+
assert.equal(matchesAny('src/a/node_modules/x.ts', ['**/node_modules/**']), true);
|
|
1263
|
+
assert.equal(matchesAny('src/x.generated.ts', ['**/*.generated.*']), true);
|
|
1264
|
+
assert.equal(matchesAny('src/x.ts', ['**/node_modules/**', '**/*.generated.*']), false);
|
|
1265
|
+
// a globstar before a slash matches zero directories too, so a tree at the root
|
|
1266
|
+
// is covered by the same pattern as one nested three deep
|
|
1267
|
+
assert.equal(matchesAny('vendor/dep.ts', ['**/vendor/**']), true);
|
|
1268
|
+
assert.equal(matchesAny('a/b/vendor/c.ts', ['**/vendor/**']), true);
|
|
1269
|
+
assert.equal(matchesAny('x.generated.ts', ['**/*.generated.*']), true);
|
|
1270
|
+
assert.equal(matchesAny('src/vendored.ts', ['**/vendor/**']), false);
|
|
1271
|
+
});
|
|
1272
|
+
console.log('\ndismissals');
|
|
1273
|
+
const dis = (line, code, check = 'swallowed-error', file = 'src/sync.ts') => ({
|
|
1274
|
+
id: 'F1', class: 'verified', check, severity: 'high', confidence: 'proven', file, line, title: 't',
|
|
1275
|
+
frame: { firstLine: line, lines: [code] },
|
|
1276
|
+
});
|
|
1277
|
+
check('a dismissal survives the line moving down the file', () => {
|
|
1278
|
+
// a decision about a line should not expire because something unrelated grew above it
|
|
1279
|
+
assert.equal(Dismissals.fingerprint(dis(5, ' cache.refresh().catch(() => {})')), Dismissals.fingerprint(dis(91, ' cache.refresh().catch(() => {})')));
|
|
1280
|
+
});
|
|
1281
|
+
check('a dismissal lapses when the line it was about changes', () => {
|
|
1282
|
+
// nobody has looked at what the line says now, so the old decision does not cover it
|
|
1283
|
+
assert.notEqual(Dismissals.fingerprint(dis(5, 'cache.refresh().catch(() => {})')), Dismissals.fingerprint(dis(5, 'cache.refresh().catch(log)')));
|
|
1284
|
+
});
|
|
1285
|
+
check('the same code in another file, or under another check, is a separate decision', () => {
|
|
1286
|
+
const base = Dismissals.fingerprint(dis(5, 'x()'));
|
|
1287
|
+
assert.notEqual(base, Dismissals.fingerprint(dis(5, 'x()', 'swallowed-error', 'src/other.ts')));
|
|
1288
|
+
assert.notEqual(base, Dismissals.fingerprint(dis(5, 'x()', 'dropped-guard')));
|
|
1289
|
+
});
|
|
1290
|
+
check('dismissing hides the finding, restoring brings it back', () => {
|
|
1291
|
+
const dir = mkdtempSync(join(tmpdir(), 'psh-dis-'));
|
|
1292
|
+
const f = dis(5, 'cache.refresh().catch(() => {})');
|
|
1293
|
+
const d = Dismissals.open(dir);
|
|
1294
|
+
assert.equal(d.has(f), false);
|
|
1295
|
+
assert.equal(d.add(f, 'best-effort', '2026-01-01T00:00:00Z'), true);
|
|
1296
|
+
assert.equal(d.add(f, 'again', '2026-01-01T00:00:00Z'), false); // already decided
|
|
1297
|
+
assert.equal(Dismissals.open(dir).has(f), true); // and it is on disk, for the team
|
|
1298
|
+
assert.equal(Dismissals.open(dir).remove(Dismissals.fingerprint(f).slice(0, 8)), true);
|
|
1299
|
+
assert.equal(Dismissals.open(dir).has(f), false);
|
|
1300
|
+
rmSync(dir, { recursive: true, force: true });
|
|
1301
|
+
});
|
|
1302
|
+
check('an unreadable dismissal file hides nothing rather than everything', () => {
|
|
1303
|
+
const dir = mkdtempSync(join(tmpdir(), 'psh-dis-'));
|
|
1304
|
+
mkdirSync(join(dir, '.powershot'), { recursive: true });
|
|
1305
|
+
writeFileSync(join(dir, '.powershot', 'dismissed.json'), 'not json at all');
|
|
1306
|
+
assert.equal(Dismissals.open(dir).list().length, 0);
|
|
1307
|
+
assert.equal(Dismissals.open(dir).has(dis(5, 'x()')), false);
|
|
1308
|
+
rmSync(dir, { recursive: true, force: true });
|
|
1309
|
+
});
|
|
1310
|
+
check('the last report is what makes a finding addressable by id', () => {
|
|
1311
|
+
const dir = mkdtempSync(join(tmpdir(), 'psh-dis-'));
|
|
1312
|
+
assert.deepEqual(lastReport(dir), []); // nothing reviewed yet
|
|
1313
|
+
rememberReport(dir, [dis(5, 'x()')]);
|
|
1314
|
+
assert.equal(lastReport(dir)[0]?.file, 'src/sync.ts');
|
|
1315
|
+
rmSync(dir, { recursive: true, force: true });
|
|
1316
|
+
});
|
|
1317
|
+
console.log('\nfile policy');
|
|
1318
|
+
check('repository paths stay relative and slash-separated across path styles', () => {
|
|
1319
|
+
const source = { getFilePath: () => 'C:/repo/src/a.ts' };
|
|
1320
|
+
assert.equal(relPath(source, 'C:\\repo'), 'src/a.ts');
|
|
1321
|
+
assert.equal(repoPath('C:\\repo', 'src\\a.ts'), 'src/a.ts');
|
|
1322
|
+
const dir = realpathSync(mkdtempSync(join(tmpdir(), 'psh-root-form-')));
|
|
1323
|
+
const file = join(dir, 'a.ts');
|
|
1324
|
+
writeFileSync(file, 'export const a = 1\n');
|
|
1325
|
+
try {
|
|
1326
|
+
assert.equal(insideRepo(dir + sep + '.', file), file);
|
|
1327
|
+
}
|
|
1328
|
+
finally {
|
|
1329
|
+
rmSync(dir, { recursive: true, force: true });
|
|
1330
|
+
}
|
|
1331
|
+
});
|
|
1332
|
+
check('the repository boundary blocks what a review never needs to read', () => {
|
|
1333
|
+
const root = process.cwd();
|
|
1334
|
+
for (const path of ['.git/config', '.git/HEAD', '.env', '.env.local', 'src/../.git/config',
|
|
1335
|
+
'deploy.pem', 'sub/.ssh/id_rsa', '.npmrc', '../outside.ts']) {
|
|
1336
|
+
assert.equal(insideRepo(root, path), undefined, path + ' must be refused');
|
|
1337
|
+
}
|
|
1338
|
+
assert.ok(insideRepo(root, 'src/cli.ts'), 'ordinary source must be readable');
|
|
1339
|
+
assert.ok(insideRepo(root, '.env.example'), 'a documented template is reviewable source, not a credential file');
|
|
1340
|
+
});
|
|
1341
|
+
check('a symlink out of the repository is refused, not followed', () => {
|
|
1342
|
+
const dir = mkdtempSync(join(tmpdir(), 'psh-link-'));
|
|
1343
|
+
const real = realpathSync(dir);
|
|
1344
|
+
writeFileSync(join(real, 'ok.ts'), 'export const a = 1\n');
|
|
1345
|
+
const away = realpathSync(mkdtempSync(join(tmpdir(), 'psh-away-')));
|
|
1346
|
+
writeFileSync(join(away, 'secret.ts'), 'export const TOKEN = "sk-secret"\n');
|
|
1347
|
+
symlinkSync(join(away, 'secret.ts'), join(real, 'leak.ts'));
|
|
1348
|
+
// `resolve` normalises `..` but follows nothing, so only the real path settles this
|
|
1349
|
+
assert.equal(insideRepo(real, 'leak.ts'), undefined);
|
|
1350
|
+
assert.ok(insideRepo(real, 'ok.ts'));
|
|
1351
|
+
rmSync(dir, { recursive: true, force: true });
|
|
1352
|
+
rmSync(away, { recursive: true, force: true });
|
|
1353
|
+
});
|
|
1354
|
+
console.log('\nconfig');
|
|
1355
|
+
const KNOWN = { verifiers: ['swallowed-error', 'phantom-dep'], judges: ['plausible-logic', 'security'] };
|
|
1356
|
+
check('a name that matches no check is an error, not a filter selecting nothing', () => {
|
|
1357
|
+
// the quietest way to a clean review: a typo that turns every check off and exits 0
|
|
1358
|
+
const bad = validateConfig({ verifiers: ['swallowed-errors'] }, KNOWN);
|
|
1359
|
+
assert.equal(bad.length, 1);
|
|
1360
|
+
assert.match(bad[0], /did you mean swallowed-error\?/);
|
|
1361
|
+
assert.deepEqual(validateConfig({ verifiers: ['swallowed-error', '*'] }, KNOWN), []);
|
|
1362
|
+
});
|
|
1363
|
+
check('an unknown setting, provider or severity is reported with what was meant', () => {
|
|
1364
|
+
assert.match(validateConfig({ minSeverty: 'high' }, KNOWN)[0], /did you mean minSeverity/);
|
|
1365
|
+
assert.match(validateConfig({ provider: 'antropic' }, KNOWN)[0], /not one of: anthropic/);
|
|
1366
|
+
assert.match(validateConfig({ minSeverity: 'huge' }, KNOWN)[0], /not one of: info/);
|
|
1367
|
+
assert.deepEqual(validateConfig({ provider: 'openai', minSeverity: 'high' }, KNOWN), []);
|
|
1368
|
+
});
|
|
1369
|
+
check('judges accept both the plain list and the { enable } form', () => {
|
|
1370
|
+
assert.deepEqual(validateConfig({ judges: { enable: ['security'] } }, KNOWN), []);
|
|
1371
|
+
assert.equal(validateConfig({ judges: { enable: ['securty'] } }, KNOWN).length, 1);
|
|
1372
|
+
assert.equal(validateConfig({ judges: 'security' }, KNOWN).length, 1); // not a list at all
|
|
1373
|
+
});
|
|
1374
|
+
console.log('\nsession safety');
|
|
1375
|
+
check('a session will not be resumed by a different model than answered it', () => {
|
|
1376
|
+
const dir = mkdtempSync(join(tmpdir(), 'psh-ses-'));
|
|
1377
|
+
const s = Session.create(dir, 'workspace', { provider: 'anthropic', model: 'glm-4.6' });
|
|
1378
|
+
assert.equal(s.askedBy('anthropic', 'glm-4.6'), true);
|
|
1379
|
+
assert.equal(s.askedBy('anthropic', 'glm-5.3'), false);
|
|
1380
|
+
assert.equal(s.askedBy('openai', 'glm-4.6'), false);
|
|
1381
|
+
rmSync(dir, { recursive: true, force: true });
|
|
1382
|
+
});
|
|
1383
|
+
console.log('\ntarget snapshot');
|
|
1384
|
+
await checkAsync('a past commit is reviewed as it was, not as the working tree is now', async () => {
|
|
1385
|
+
const dir = realpathSync(mkdtempSync(join(tmpdir(), 'psh-hist-')));
|
|
1386
|
+
const run = (...args) => execFileSync('git', args, { cwd: dir, stdio: ['ignore', 'ignore', 'pipe'] });
|
|
1387
|
+
run('init', '-q', '.');
|
|
1388
|
+
run('config', 'user.email', 't@t');
|
|
1389
|
+
run('config', 'user.name', 't');
|
|
1390
|
+
writeFileSync(join(dir, 'package.json'), '{"name":"h"}');
|
|
1391
|
+
writeFileSync(join(dir, 'seed.ts'), 'export const seed = 1\n');
|
|
1392
|
+
run('add', '-A');
|
|
1393
|
+
run('commit', '-qm', 'seed');
|
|
1394
|
+
writeFileSync(join(dir, 'a.ts'), 'export function load() {\n try { JSON.parse("{}") } catch {}\n}\n');
|
|
1395
|
+
run('add', '-A');
|
|
1396
|
+
run('commit', '-qm', 'swallow it');
|
|
1397
|
+
const bad = execFileSync('git', ['rev-parse', 'HEAD'], { cwd: dir }).toString().trim();
|
|
1398
|
+
// gone from the working tree: reading the tree instead of the commit reports nothing
|
|
1399
|
+
writeFileSync(join(dir, 'a.ts'), 'export function load() {\n try { JSON.parse("{}") } catch (e) { throw e }\n}\n');
|
|
1400
|
+
run('add', '-A');
|
|
1401
|
+
run('commit', '-qm', 'handle it');
|
|
1402
|
+
const now = await review({ root: dir, range: {}, config: loadConfig(dir), verifyOnly: true });
|
|
1403
|
+
assert.equal(now.findings.length, 0, 'the working tree is clean');
|
|
1404
|
+
const then = await withTargetTree(dir, { commit: bad }, (tree) => review({ root: tree, stateRoot: dir, range: { commit: bad }, config: loadConfig(dir), verifyOnly: true }));
|
|
1405
|
+
assert.equal(then.findings.length, 1, 'the commit that introduced it must still report it');
|
|
1406
|
+
assert.equal(then.findings[0].check, 'swallowed-error');
|
|
1407
|
+
assert.equal(then.findings[0].file, 'a.ts');
|
|
1408
|
+
assert.equal(execFileSync('git', ['worktree', 'list'], { cwd: dir }).toString().trim().split('\n').length, 1);
|
|
1409
|
+
rmSync(dir, { recursive: true, force: true });
|
|
1410
|
+
});
|
|
1411
|
+
await checkAsync('a change cannot dismiss its own findings', async () => {
|
|
1412
|
+
const dir = realpathSync(mkdtempSync(join(tmpdir(), 'psh-gate-')));
|
|
1413
|
+
const run = (...args) => execFileSync('git', args, { cwd: dir, stdio: ['ignore', 'ignore', 'pipe'] });
|
|
1414
|
+
run('init', '-q', '.');
|
|
1415
|
+
run('config', 'user.email', 't@t');
|
|
1416
|
+
run('config', 'user.name', 't');
|
|
1417
|
+
writeFileSync(join(dir, 'package.json'), '{"name":"g"}');
|
|
1418
|
+
run('add', '-A');
|
|
1419
|
+
run('commit', '-qm', 'seed');
|
|
1420
|
+
run('branch', '-M', 'main');
|
|
1421
|
+
run('checkout', '-q', '-b', 'feature');
|
|
1422
|
+
const code = 'export function load() {\n try { JSON.parse("{}") } catch {}\n}\n';
|
|
1423
|
+
writeFileSync(join(dir, 'a.ts'), code);
|
|
1424
|
+
run('add', '-A');
|
|
1425
|
+
run('commit', '-qm', 'add it');
|
|
1426
|
+
const range = { from: 'main', to: 'HEAD' };
|
|
1427
|
+
const found = await review({ root: dir, range, config: loadConfig(dir), verifyOnly: true });
|
|
1428
|
+
assert.equal(found.findings.length, 1);
|
|
1429
|
+
// the change now suppresses its own finding and commits the suppression
|
|
1430
|
+
mkdirSync(join(dir, '.powershot'), { recursive: true });
|
|
1431
|
+
writeFileSync(join(dir, '.powershot', 'dismissed.json'), JSON.stringify([{
|
|
1432
|
+
fingerprint: Dismissals.fingerprint(found.findings[0]),
|
|
1433
|
+
check: 'swallowed-error', file: 'a.ts', code: 'x', title: 't', at: '2026-01-01T00:00:00Z',
|
|
1434
|
+
}]));
|
|
1435
|
+
run('add', '-A');
|
|
1436
|
+
run('commit', '-qm', 'nothing to see here');
|
|
1437
|
+
const gated = await review({ root: dir, range, config: loadConfig(dir), verifyOnly: true });
|
|
1438
|
+
assert.equal(gated.findings.length, 1, 'a suppression the base has not seen must not apply');
|
|
1439
|
+
assert.equal(gated.stats.dismissed, 0);
|
|
1440
|
+
rmSync(dir, { recursive: true, force: true });
|
|
1441
|
+
});
|
|
1442
|
+
console.log('\ntrust boundary');
|
|
1443
|
+
await checkAsync('every ingestion path refuses a link out of the repository', async () => {
|
|
1444
|
+
const dir = realpathSync(mkdtempSync(join(tmpdir(), 'psh-ingress-')));
|
|
1445
|
+
const outside = realpathSync(mkdtempSync(join(tmpdir(), 'psh-outside-')));
|
|
1446
|
+
writeFileSync(join(outside, 'creds.ts'), 'export const TOKEN = "sk-secret"\n');
|
|
1447
|
+
const run = (...a) => execFileSync('git', a, { cwd: dir, stdio: ['ignore', 'ignore', 'pipe'] });
|
|
1448
|
+
run('init', '-q', '.');
|
|
1449
|
+
run('config', 'user.email', 't@t');
|
|
1450
|
+
run('config', 'user.name', 't');
|
|
1451
|
+
writeFileSync(join(dir, 'package.json'), '{"name":"i"}');
|
|
1452
|
+
writeFileSync(join(dir, 'ok.ts'), 'export const ok = 1\n');
|
|
1453
|
+
run('add', '-A');
|
|
1454
|
+
run('commit', '-qm', 'seed');
|
|
1455
|
+
symlinkSync(join(outside, 'creds.ts'), join(dir, 'leak.ts'));
|
|
1456
|
+
const cfg = loadConfig(dir);
|
|
1457
|
+
const paths = (r, files) => files;
|
|
1458
|
+
// untracked diff, whole-directory scan, and single-file scan are three separate
|
|
1459
|
+
// entrances; proving one does not prove the others, which is how this got through
|
|
1460
|
+
const viaDiff = await review({ root: dir, range: {}, config: cfg, verifyOnly: true });
|
|
1461
|
+
assert.equal(viaDiff.stats.files, 0, 'an untracked symlink must not enter the diff');
|
|
1462
|
+
assert.deepEqual(scanPaths(dir, '.').map((f) => f.path), ['ok.ts'], 'walk must skip the link');
|
|
1463
|
+
assert.deepEqual(scanPaths(dir, 'leak.ts'), [], 'naming the link directly must not read it');
|
|
1464
|
+
// and once it is tracked, the compiler project must not pick it up either
|
|
1465
|
+
run('add', '-A');
|
|
1466
|
+
run('commit', '-qm', 'track the link');
|
|
1467
|
+
const tracked = await review({ root: dir, range: { commit: 'HEAD' }, config: cfg, verifyOnly: true });
|
|
1468
|
+
assert.equal(tracked.stats.files, 0, 'a tracked symlink must not enter the project');
|
|
1469
|
+
rmSync(dir, { recursive: true, force: true });
|
|
1470
|
+
rmSync(outside, { recursive: true, force: true });
|
|
1471
|
+
});
|
|
1472
|
+
await checkAsync('a ref that is really an option is refused, not passed to git', async () => {
|
|
1473
|
+
const dir = realpathSync(mkdtempSync(join(tmpdir(), 'psh-ref-')));
|
|
1474
|
+
const run = (...a) => execFileSync('git', a, { cwd: dir, stdio: ['ignore', 'ignore', 'pipe'] });
|
|
1475
|
+
run('init', '-q', '.');
|
|
1476
|
+
run('config', 'user.email', 't@t');
|
|
1477
|
+
run('config', 'user.name', 't');
|
|
1478
|
+
writeFileSync(join(dir, 'a.ts'), 'export const a = 1\n');
|
|
1479
|
+
run('add', '-A');
|
|
1480
|
+
run('commit', '-qm', 'seed');
|
|
1481
|
+
// `git diff --output=x A...B` writes a file: execFileSync stops the shell, not git
|
|
1482
|
+
const written = join(dir, 'proof');
|
|
1483
|
+
assert.throws(() => collectChanges(dir, { from: '--output=' + written, to: 'HEAD' }), /looks like an option/);
|
|
1484
|
+
assert.equal(existsSync(written), false, 'no file may have been written');
|
|
1485
|
+
// and a ref that does not resolve is an error, not an empty diff that reads clean
|
|
1486
|
+
assert.throws(() => collectChanges(dir, { from: 'no-such-ref', to: 'HEAD' }), /not a commit/);
|
|
1487
|
+
rmSync(dir, { recursive: true, force: true });
|
|
1488
|
+
});
|
|
1489
|
+
function markdownBlockEscape(out) {
|
|
1490
|
+
let bar;
|
|
1491
|
+
for (const line of out.split('\n')) {
|
|
1492
|
+
const open = /^(`{3,})/.exec(line);
|
|
1493
|
+
if (bar === undefined && open) {
|
|
1494
|
+
bar = open[1];
|
|
1495
|
+
continue;
|
|
1496
|
+
}
|
|
1497
|
+
if (bar !== undefined) {
|
|
1498
|
+
if (new RegExp('^\\s{0,3}' + bar + '\\s*$').test(line))
|
|
1499
|
+
bar = undefined;
|
|
1500
|
+
continue;
|
|
1501
|
+
}
|
|
1502
|
+
if (/^(?:#{1,6}\s|>\s|[-+*]\s|\d+[.)]\s|(?:[-*_]\s*){3,})/.test(line) &&
|
|
1503
|
+
!/^## PowerShot$|^### `|^> _.*_: /.test(line)) {
|
|
1504
|
+
return line;
|
|
1505
|
+
}
|
|
1506
|
+
}
|
|
1507
|
+
return undefined;
|
|
1508
|
+
}
|
|
1509
|
+
check('a finding cannot break out of the document that carries it', () => {
|
|
1510
|
+
// the markdown goes into a pull-request comment, and the title came from a diff
|
|
1511
|
+
const out = markdown([{
|
|
1512
|
+
id: 'F1', class: 'judged', check: 'x', severity: 'high', confidence: 'firm',
|
|
1513
|
+
file: 'a.ts', line: 1,
|
|
1514
|
+
title: '```\n## Injected heading\n[click](https://evil.example)',
|
|
1515
|
+
evidence: { oracle: 'agent', detail: '`ends the quote`' },
|
|
1516
|
+
frame: { firstLine: 1, lines: ['```', 'malicious'] },
|
|
1517
|
+
fix: '```\nescape',
|
|
1518
|
+
}]);
|
|
1519
|
+
assert.equal(markdownBlockEscape(out), undefined, 'nothing may escape into the document');
|
|
1520
|
+
// CommonMark accepts a closing fence indented by three spaces, so containment must
|
|
1521
|
+
// come from a longer delimiter rather than indentation.
|
|
1522
|
+
const surfaces = [
|
|
1523
|
+
{ file: 'a.ts\n## Injected' },
|
|
1524
|
+
{ file: 'a.ts)](https://evil.example)(' },
|
|
1525
|
+
{ title: '## Injected' },
|
|
1526
|
+
{ title: '> [!WARNING]' },
|
|
1527
|
+
{ title: '- forged result' },
|
|
1528
|
+
{ title: '1. forged result' },
|
|
1529
|
+
{ title: '---' },
|
|
1530
|
+
{ title: '```\n## Injected' },
|
|
1531
|
+
{ frame: { firstLine: 1, lines: ['```', '## Injected'] } },
|
|
1532
|
+
{ frame: { firstLine: 1, lines: [' ```', '## Injected'] } },
|
|
1533
|
+
{ fix: '```\n## Injected' },
|
|
1534
|
+
{ suggestion: '``````\n## Injected' },
|
|
1535
|
+
{ evidence: { oracle: 'a', detail: 'x\n## Injected' } },
|
|
1536
|
+
];
|
|
1537
|
+
for (const extra of surfaces) {
|
|
1538
|
+
const rendered = markdown([{
|
|
1539
|
+
id: 'F1', class: 'judged', check: 'x', severity: 'high', confidence: 'firm',
|
|
1540
|
+
file: 'a.ts', line: 1, title: 'ordinary', ...extra,
|
|
1541
|
+
}]);
|
|
1542
|
+
assert.equal(markdownBlockEscape(rendered), undefined, 'escaped through ' + Object.keys(extra)[0]);
|
|
1543
|
+
}
|
|
1544
|
+
const destinations = markdown([{
|
|
1545
|
+
id: 'F1', class: 'judged', check: 'x', severity: 'high', confidence: 'firm',
|
|
1546
|
+
file: 'javascript:alert(1)#part?query.ts', line: 1, title: 'ordinary',
|
|
1547
|
+
}]);
|
|
1548
|
+
assert.match(destinations, /\.\/javascript%3Aalert%281%29%23part%3Fquery\.ts#L1/);
|
|
1549
|
+
assert.doesNotMatch(destinations, /\]\(javascript:/);
|
|
1550
|
+
const traversal = markdown([{
|
|
1551
|
+
id: 'F1', class: 'judged', check: 'x', severity: 'high', confidence: 'firm',
|
|
1552
|
+
file: '../../issues/1', line: 1, title: 'ordinary',
|
|
1553
|
+
}]);
|
|
1554
|
+
assert.doesNotMatch(traversal, /\]\(\.\/\.\.\//);
|
|
1555
|
+
assert.match(traversal, /%2E%2E\/%2E%2E\/issues\/1#L1/);
|
|
1556
|
+
});
|
|
1557
|
+
check('markdown frames use the reviewed file language', () => {
|
|
1558
|
+
const rendered = markdown([{
|
|
1559
|
+
id: 'F1', class: 'verified', check: 'x', severity: 'low', confidence: 'proven',
|
|
1560
|
+
file: 'worker.py', line: 1, title: 'ordinary',
|
|
1561
|
+
frame: { firstLine: 1, lines: ['def work():'] },
|
|
1562
|
+
}]);
|
|
1563
|
+
assert.match(rendered, /```python\ndef work\(\):\n```/);
|
|
1564
|
+
});
|
|
1565
|
+
check('control characters from a reviewed file never reach a terminal', () => {
|
|
1566
|
+
const E = String.fromCharCode(27);
|
|
1567
|
+
assert.equal(stripControl('a' + E + '[2J' + E + '[Hb'), 'ab');
|
|
1568
|
+
assert.equal(stripControl('t' + E + ']0;pwned' + String.fromCharCode(7) + 'end'), 'tend');
|
|
1569
|
+
assert.equal(stripControl('keep\tthe\ttabs'), 'keep\tthe\ttabs'); // layout is not a control
|
|
1570
|
+
});
|
|
1571
|
+
console.log('\ncontrol plane');
|
|
1572
|
+
check('a budget stops the run and names what it stopped for', () => {
|
|
1573
|
+
const b = new Budget({ requests: 2, elapsedMs: 10_000 }, 1000);
|
|
1574
|
+
assert.equal(b.exhausted(1000), undefined);
|
|
1575
|
+
b.spend({ requests: 2 });
|
|
1576
|
+
assert.match(b.exhausted(1000), /requests budget reached \(2\/2\)/);
|
|
1577
|
+
// time counts even when nothing was spent
|
|
1578
|
+
const t = new Budget({ elapsedMs: 500 }, 1000);
|
|
1579
|
+
assert.equal(t.exhausted(1400), undefined);
|
|
1580
|
+
assert.match(t.exhausted(1600), /elapsedMs budget reached/);
|
|
1581
|
+
});
|
|
1582
|
+
check('a budget spec is validated rather than half-understood', () => {
|
|
1583
|
+
assert.deepEqual(parseLimits('requests=5,elapsedMs=60000'), { requests: 5, elapsedMs: 60000 });
|
|
1584
|
+
assert.match(parseLimits('requessts=5'), /unknown budget/);
|
|
1585
|
+
assert.match(parseLimits('requests=0'), /positive/);
|
|
1586
|
+
assert.deepEqual(parseLimits(''), {});
|
|
1587
|
+
});
|
|
1588
|
+
check('every touched file lands in exactly one disposition', () => {
|
|
1589
|
+
const cfg = { ...loadConfig(process.cwd()), ignore: ['**/vendor/**'] };
|
|
1590
|
+
const changed = [
|
|
1591
|
+
{ path: 'src/a.ts', added: new Set([1]) },
|
|
1592
|
+
{ path: 'vendor/dep.ts', added: new Set([1]) },
|
|
1593
|
+
];
|
|
1594
|
+
const plan = SelectionPlan.build(process.cwd(), changed, cfg);
|
|
1595
|
+
assert.equal(plan.items().length, 2);
|
|
1596
|
+
assert.deepEqual(plan.of('waived').map((i) => i.reason), ['ignored by config']);
|
|
1597
|
+
plan.waive('src/a.ts', 'no parser for this language');
|
|
1598
|
+
assert.equal(plan.keep(changed).length, 0);
|
|
1599
|
+
// a waived file cannot quietly become selected again
|
|
1600
|
+
plan.waive('src/a.ts', 'something else');
|
|
1601
|
+
assert.equal(plan.items().find((i) => i.path === 'src/a.ts')?.reason, 'no parser for this language');
|
|
1602
|
+
});
|
|
1603
|
+
check('the manifest accounts for everything it selected', () => {
|
|
1604
|
+
const m = new RunManifest('abc12345');
|
|
1605
|
+
m.ran('swallowed-error');
|
|
1606
|
+
m.unit({ judge: 'plausible-logic', unit: 'a.ts', outcome: 'completed', findings: 1 });
|
|
1607
|
+
m.unit({ judge: 'plausible-logic', unit: 'b.ts', outcome: 'reused', reason: 'cached', findings: 0 });
|
|
1608
|
+
const base = {
|
|
1609
|
+
operation: 'review', target: { requested: {} },
|
|
1610
|
+
policy: { source: 'base', hash: 'h' },
|
|
1611
|
+
engine: { version: '0', tools: false, verifyOnly: false },
|
|
1612
|
+
files: [{
|
|
1613
|
+
path: 'a.ts', disposition: 'selected', bytes: 1, addedLines: 1,
|
|
1614
|
+
language: 'typescript', checks: ['swallowed-error'],
|
|
1615
|
+
}],
|
|
1616
|
+
skippedChecks: [], findings: { total: 1, verified: 0, judged: 1, dismissed: 0, droppedPosition: 0 },
|
|
1617
|
+
usage: { requests: 1, inputTokens: 2, outputTokens: 3, toolCalls: 0, elapsedMs: 4, units: 1 },
|
|
1618
|
+
failures: [],
|
|
1619
|
+
};
|
|
1620
|
+
const ok = m.build(base);
|
|
1621
|
+
assert.equal(ok.state, 'complete');
|
|
1622
|
+
assert.deepEqual(coverageProblems(ok), []);
|
|
1623
|
+
const sourceFailures = [];
|
|
1624
|
+
const snapshot = new RunManifest('snapshot').build({ ...base, failures: sourceFailures });
|
|
1625
|
+
sourceFailures.push('added after the manifest was built');
|
|
1626
|
+
assert.deepEqual(snapshot.failures, [], 'a finished manifest must not alias mutable pipeline state');
|
|
1627
|
+
// a unit nobody judged must not be reported as a complete run
|
|
1628
|
+
const m2 = new RunManifest('def45678');
|
|
1629
|
+
m2.unit({ judge: 'plausible-logic', unit: 'a.ts', outcome: 'waived', reason: 'requests budget reached', findings: 0 });
|
|
1630
|
+
const partial = m2.build(base);
|
|
1631
|
+
assert.equal(partial.state, 'partial', 'an unjudged unit makes the run partial');
|
|
1632
|
+
// and a failure anywhere means the run is not a verdict
|
|
1633
|
+
assert.equal(m2.build({ ...base, failures: ['judge died'] }).state, 'failed');
|
|
1634
|
+
});
|
|
1635
|
+
check('a manifest that hides an unreached unit is caught as our bug', () => {
|
|
1636
|
+
const broken = {
|
|
1637
|
+
schema: SCHEMA, id: 'x', operation: 'review', started: '', ended: '',
|
|
1638
|
+
repository: {}, target: { requested: {} }, policy: { source: 'base', hash: 'h' },
|
|
1639
|
+
engine: { version: '0', tools: false, verifyOnly: false },
|
|
1640
|
+
files: [{
|
|
1641
|
+
path: 'a.ts', disposition: 'failed', bytes: 0, addedLines: 0,
|
|
1642
|
+
language: 'typescript', checks: [],
|
|
1643
|
+
}],
|
|
1644
|
+
units: [{ judge: 'j', unit: 'u', outcome: 'waived', findings: 0 }],
|
|
1645
|
+
checks: { ran: [], skipped: [] },
|
|
1646
|
+
findings: { total: 0, verified: 0, judged: 0, dismissed: 0, droppedPosition: 0 },
|
|
1647
|
+
usage: { requests: 0, inputTokens: 0, outputTokens: 0, toolCalls: 0, elapsedMs: 0, units: 0 },
|
|
1648
|
+
state: 'complete', failures: [], notLookedAt: [],
|
|
1649
|
+
};
|
|
1650
|
+
const problems = coverageProblems(broken);
|
|
1651
|
+
assert.ok(problems.some((p) => /never judged/.test(p)));
|
|
1652
|
+
assert.ok(problems.some((p) => /waived without a reason/.test(p)));
|
|
1653
|
+
assert.ok(problems.some((p) => /failed selection/.test(p)));
|
|
1654
|
+
});
|
|
1655
|
+
check('a check cannot be both run and skipped under one identity', () => {
|
|
1656
|
+
const broken = {
|
|
1657
|
+
schema: SCHEMA, id: 'x', operation: 'review', started: '', ended: '',
|
|
1658
|
+
repository: {}, target: { requested: {} }, policy: { source: 'base', hash: 'h' },
|
|
1659
|
+
engine: { version: '0', tools: false, verifyOnly: true }, files: [], units: [],
|
|
1660
|
+
checks: { ran: ['phantom-api'], skipped: [{ check: 'phantom-api', missing: 'types' }] },
|
|
1661
|
+
findings: { total: 0, verified: 0, judged: 0, dismissed: 0, droppedPosition: 0 },
|
|
1662
|
+
usage: { requests: 0, inputTokens: 0, outputTokens: 0, toolCalls: 0, elapsedMs: 0, units: 0 },
|
|
1663
|
+
state: 'partial', failures: [], notLookedAt: ['phantom-api had no oracle'],
|
|
1664
|
+
};
|
|
1665
|
+
assert.ok(coverageProblems(broken).some((p) => /both ran and skipped/.test(p)));
|
|
1666
|
+
});
|
|
1667
|
+
check('global and per-file check coverage must describe the same work', () => {
|
|
1668
|
+
const base = {
|
|
1669
|
+
schema: SCHEMA, id: 'x', operation: 'review', started: '', ended: '',
|
|
1670
|
+
repository: {}, target: { requested: {} }, policy: { source: 'base', hash: 'h' },
|
|
1671
|
+
engine: { version: '0', tools: false, verifyOnly: true }, units: [],
|
|
1672
|
+
findings: { total: 0, verified: 0, judged: 0, dismissed: 0, droppedPosition: 0 },
|
|
1673
|
+
usage: { requests: 0, inputTokens: 0, outputTokens: 0, toolCalls: 0, elapsedMs: 0, units: 0 },
|
|
1674
|
+
state: 'failed', failures: ['broken coverage'], notLookedAt: ['broken coverage'],
|
|
1675
|
+
};
|
|
1676
|
+
const onlyGlobal = {
|
|
1677
|
+
...base,
|
|
1678
|
+
files: [{
|
|
1679
|
+
path: 'a.ts', disposition: 'selected', bytes: 1, addedLines: 1,
|
|
1680
|
+
language: 'typescript', checks: [],
|
|
1681
|
+
}],
|
|
1682
|
+
checks: { ran: ['phantom-api'], skipped: [] },
|
|
1683
|
+
};
|
|
1684
|
+
assert.ok(coverageProblems(onlyGlobal).some((p) => /received no selected file/.test(p)));
|
|
1685
|
+
const onlyFile = {
|
|
1686
|
+
...base,
|
|
1687
|
+
files: [{
|
|
1688
|
+
path: 'a.ts', disposition: 'selected', bytes: 1, addedLines: 1,
|
|
1689
|
+
language: 'typescript', checks: ['phantom-api', 'phantom-api'],
|
|
1690
|
+
}],
|
|
1691
|
+
checks: { ran: [], skipped: [] },
|
|
1692
|
+
};
|
|
1693
|
+
const problems = coverageProblems(onlyFile);
|
|
1694
|
+
assert.ok(problems.some((p) => /not recorded as ran/.test(p)));
|
|
1695
|
+
assert.ok(problems.some((p) => /received check twice/.test(p)));
|
|
1696
|
+
});
|
|
1697
|
+
check('bench refuses to score a result with per-file or oracle gaps', () => {
|
|
1698
|
+
const result = {
|
|
1699
|
+
findings: [], stats: { files: 1, verified: 0, judged: 0, dismissed: 0 }, failures: [],
|
|
1700
|
+
plan: { items: () => [{ path: 'outside.ts', disposition: 'selected', missing: ['types'] }] },
|
|
1701
|
+
skippedChecks: [{ check: 'foreign-phantom-api', missing: 'python-types' }],
|
|
1702
|
+
};
|
|
1703
|
+
assert.deepEqual(incompleteReasons(result), [
|
|
1704
|
+
'1 file(s) reviewed with fewer checks than the rest: outside.ts (no types)',
|
|
1705
|
+
'1 check(s) had no oracle to run against: foreign-phantom-api (no python-types)',
|
|
1706
|
+
]);
|
|
1707
|
+
});
|
|
1708
|
+
check('every check declares what it needs, and is skipped without it', () => {
|
|
1709
|
+
for (const v of VERIFIERS) {
|
|
1710
|
+
assert.ok(Array.isArray(v.needs) && v.needs.length > 0, v.name + ' must declare what it needs');
|
|
1711
|
+
}
|
|
1712
|
+
// a scan has no base version, so the before/after checks must not silently pass
|
|
1713
|
+
const noBase = capabilitiesOf({ typed: false, foreign: [], changed: [{ path: 'a.ts', added: new Set([1]) }] });
|
|
1714
|
+
assert.equal(noBase.has('base'), false);
|
|
1715
|
+
assert.equal(noBase.has('references'), false);
|
|
1716
|
+
assert.equal(noBase.has('syntax'), true);
|
|
1717
|
+
const full = capabilitiesOf({ typed: true, foreign: [], changed: [{ path: 'a.ts', added: new Set([1]), before: 'x' }] });
|
|
1718
|
+
assert.equal(full.has('references'), true);
|
|
1719
|
+
assert.equal(full.has('base'), true);
|
|
1720
|
+
});
|
|
1721
|
+
check('a check with no oracle behind it is skipped, not recorded as satisfied', () => {
|
|
1722
|
+
// a repository with a tsconfig and some Python in it must not satisfy the Python
|
|
1723
|
+
// check through the TypeScript checker — they are different oracles
|
|
1724
|
+
const py = [{ pack: { name: 'python' } }];
|
|
1725
|
+
const withTs = capabilitiesOf({ typed: true, foreign: py, changed: [], root: process.cwd() });
|
|
1726
|
+
assert.equal(withTs.has('types'), true);
|
|
1727
|
+
assert.equal(withTs.has('python-types'), pyrightAvailable(process.cwd()));
|
|
1728
|
+
// and phantom-api for Python asks for exactly that one
|
|
1729
|
+
const check = VERIFIERS.find((v) => v.name === 'phantom-api' && v.needs.includes('python-types'));
|
|
1730
|
+
assert.ok(check, 'the python member check must declare python-types');
|
|
1731
|
+
});
|
|
1732
|
+
check('a provider failure says what kind it is, with no key in the message', () => {
|
|
1733
|
+
const e = new ProviderError('auth', 'Anthropic', 'bad key sk-abcdefghijklmnop in header');
|
|
1734
|
+
assert.equal(e.kind, 'auth');
|
|
1735
|
+
assert.equal(e.retryable, false);
|
|
1736
|
+
assert.ok(!e.message.includes('sk-abcdefghijklmnop'), 'a key must never reach a message');
|
|
1737
|
+
assert.equal(redact('authorization: Bearer abcdefghijklmnop'), 'authorization: Bearer <redacted>');
|
|
1738
|
+
assert.equal(redact('{"api_key":"AIzaSyABCDEFGHIJKLMNOPQRSTUVWXYZ01"}'), '{"api_key":"<redacted>"}');
|
|
1739
|
+
});
|
|
1740
|
+
console.log('\nthe repository must not steer its reviewer');
|
|
1741
|
+
await checkAsync('a ref is validated before anything reads one', async () => {
|
|
1742
|
+
const dir = realpathSync(mkdtempSync(join(tmpdir(), 'psh-ref2-')));
|
|
1743
|
+
const run = (...a) => execFileSync('git', a, { cwd: dir, stdio: ['ignore', 'ignore', 'pipe'] });
|
|
1744
|
+
run('init', '-q', '.');
|
|
1745
|
+
run('config', 'user.email', 't@t');
|
|
1746
|
+
run('config', 'user.name', 't');
|
|
1747
|
+
writeFileSync(join(dir, 'a.ts'), 'export const a = 1\n');
|
|
1748
|
+
run('add', '-A');
|
|
1749
|
+
run('commit', '-qm', 'seed');
|
|
1750
|
+
// every ref reaches git through a different call — policy, snapshot, diff, intent —
|
|
1751
|
+
// so the check has to be at the entrance, not inside the one that happened to be
|
|
1752
|
+
// audited. Reading the policy resolved `--commit` to `<ref>^` and wrote a file.
|
|
1753
|
+
const written = join(dir, 'proof');
|
|
1754
|
+
for (const range of [
|
|
1755
|
+
{ commit: '--output=' + written },
|
|
1756
|
+
{ from: '--output=' + written, to: 'HEAD' },
|
|
1757
|
+
{ from: 'HEAD', to: '--output=' + written },
|
|
1758
|
+
]) {
|
|
1759
|
+
assert.throws(() => checkRange(dir, range), /looks like an option/);
|
|
1760
|
+
}
|
|
1761
|
+
assert.equal(readdirSync(dir).some((n) => n.startsWith('proof')), false, 'no file may have been written');
|
|
1762
|
+
rmSync(dir, { recursive: true, force: true });
|
|
1763
|
+
});
|
|
1764
|
+
check('a file the tsconfig does not own is read, not type-checked', () => {
|
|
1765
|
+
// asking the checker for diagnostics on a file it has no program for throws from
|
|
1766
|
+
// inside TypeScript, which took down the whole review — including our own
|
|
1767
|
+
const g = ground([{ path: 'a.ts', after: 'export const a = 1\n' }]);
|
|
1768
|
+
g.files[0].typed = false;
|
|
1769
|
+
assert.deepEqual(phantomApi.run({ ...g, typed: true }), []);
|
|
1770
|
+
});
|
|
1771
|
+
await checkAsync('per-file limits follow the checks the caller actually selected', async () => {
|
|
1772
|
+
const dir = realpathSync(mkdtempSync(join(tmpdir(), 'psh-capabilities-')));
|
|
1773
|
+
try {
|
|
1774
|
+
writeFileSync(join(dir, 'tsconfig.json'), JSON.stringify({ include: ['inside.ts'] }));
|
|
1775
|
+
writeFileSync(join(dir, 'inside.ts'), 'export const inside = 1\n');
|
|
1776
|
+
writeFileSync(join(dir, 'outside.ts'), 'export const outside = 1\n');
|
|
1777
|
+
const changes = [{ path: 'outside.ts', added: new Set([1]) }];
|
|
1778
|
+
const common = { root: dir, range: {}, changes, config: loadConfig(dir), verifyOnly: true };
|
|
1779
|
+
const stateOf = (result) => new RunManifest('cap-test').build({
|
|
1780
|
+
operation: 'scan', target: { requested: {} }, policy: { source: 'default', hash: 'h' },
|
|
1781
|
+
engine: { version: '0', tools: false, verifyOnly: true }, files: result.plan?.items() ?? [],
|
|
1782
|
+
skippedChecks: result.skippedChecks ?? [],
|
|
1783
|
+
findings: { total: result.findings.length, verified: result.stats.verified, judged: result.stats.judged,
|
|
1784
|
+
dismissed: result.stats.dismissed, droppedPosition: 0 },
|
|
1785
|
+
usage: result.usage, failures: result.failures,
|
|
1786
|
+
}).state;
|
|
1787
|
+
const syntaxOnly = await review({ ...common, checks: ['swallowed-error'] });
|
|
1788
|
+
assert.deepEqual(syntaxOnly.plan?.items()[0]?.missing, undefined);
|
|
1789
|
+
assert.deepEqual(syntaxOnly.skippedChecks, []);
|
|
1790
|
+
assert.equal(stateOf(syntaxOnly), 'complete');
|
|
1791
|
+
const typed = await review({ ...common, checks: ['phantom-api'] });
|
|
1792
|
+
assert.deepEqual(typed.plan?.items()[0]?.missing, ['types']);
|
|
1793
|
+
assert.deepEqual(typed.skippedChecks, [{ check: 'phantom-api', missing: 'types' }]);
|
|
1794
|
+
assert.equal(stateOf(typed), 'partial');
|
|
1795
|
+
}
|
|
1796
|
+
finally {
|
|
1797
|
+
rmSync(dir, { recursive: true, force: true });
|
|
1798
|
+
}
|
|
1799
|
+
});
|
|
1800
|
+
await checkAsync('foreign checks advertise only files their language pack can inspect', async () => {
|
|
1801
|
+
const dir = realpathSync(mkdtempSync(join(tmpdir(), 'psh-pack-coverage-')));
|
|
1802
|
+
try {
|
|
1803
|
+
writeFileSync(join(dir, 'existing.rb'), 'def existing\n true\nend\n');
|
|
1804
|
+
const changes = [{
|
|
1805
|
+
path: 'existing.rb',
|
|
1806
|
+
added: new Set([2]),
|
|
1807
|
+
before: 'def existing\n false\nend\n',
|
|
1808
|
+
}];
|
|
1809
|
+
const manifest = new RunManifest('ruby-coverage');
|
|
1810
|
+
const result = await review({
|
|
1811
|
+
root: dir,
|
|
1812
|
+
range: {},
|
|
1813
|
+
changes,
|
|
1814
|
+
config: loadConfig(dir),
|
|
1815
|
+
verifyOnly: true,
|
|
1816
|
+
checks: ['foreign-phantom-dep', 'foreign-vacuous-test', 'foreign-lying-comment'],
|
|
1817
|
+
manifest,
|
|
1818
|
+
});
|
|
1819
|
+
const record = manifest.build({
|
|
1820
|
+
operation: 'scan', target: { requested: {} }, policy: { source: 'default', hash: 'h' },
|
|
1821
|
+
engine: { version: '0', tools: false, verifyOnly: true }, files: result.plan.items(),
|
|
1822
|
+
skippedChecks: result.skippedChecks ?? [],
|
|
1823
|
+
findings: { total: result.findings.length, verified: result.stats.verified, judged: result.stats.judged,
|
|
1824
|
+
dismissed: result.stats.dismissed, droppedPosition: 0 },
|
|
1825
|
+
usage: result.usage, failures: result.failures,
|
|
1826
|
+
});
|
|
1827
|
+
assert.deepEqual(record.checks.ran, ['foreign-phantom-dep']);
|
|
1828
|
+
assert.deepEqual(record.files[0].checks, ['foreign-phantom-dep']);
|
|
1829
|
+
assert.equal(record.state, 'complete');
|
|
1830
|
+
}
|
|
1831
|
+
finally {
|
|
1832
|
+
rmSync(dir, { recursive: true, force: true });
|
|
1833
|
+
}
|
|
1834
|
+
});
|
|
1835
|
+
await checkAsync('base applicability is scoped to the verifier language and file', async () => {
|
|
1836
|
+
const dir = realpathSync(mkdtempSync(join(tmpdir(), 'psh-pack-base-')));
|
|
1837
|
+
try {
|
|
1838
|
+
writeFileSync(join(dir, 'existing.rb'), 'def existing\n true\nend\n');
|
|
1839
|
+
writeFileSync(join(dir, 'new.py'), 'def added(value: str) -> str:\n return value\n');
|
|
1840
|
+
const changes = [
|
|
1841
|
+
{ path: 'existing.rb', added: new Set([2]), before: 'def existing\n false\nend\n' },
|
|
1842
|
+
{ path: 'new.py', added: new Set([1, 2]) },
|
|
1843
|
+
];
|
|
1844
|
+
const manifest = new RunManifest('mixed-base');
|
|
1845
|
+
const result = await review({
|
|
1846
|
+
root: dir, range: {}, changes, config: loadConfig(dir), verifyOnly: true,
|
|
1847
|
+
checks: ['foreign-contract-drift'], manifest,
|
|
1848
|
+
});
|
|
1849
|
+
const record = manifest.build({
|
|
1850
|
+
operation: 'scan', target: { requested: {} }, policy: { source: 'default', hash: 'h' },
|
|
1851
|
+
engine: { version: '0', tools: false, verifyOnly: true }, files: result.plan.items(),
|
|
1852
|
+
skippedChecks: result.skippedChecks ?? [],
|
|
1853
|
+
findings: { total: result.findings.length, verified: result.stats.verified, judged: result.stats.judged,
|
|
1854
|
+
dismissed: result.stats.dismissed, droppedPosition: 0 },
|
|
1855
|
+
usage: result.usage, failures: result.failures,
|
|
1856
|
+
});
|
|
1857
|
+
assert.deepEqual(record.checks, { ran: [], skipped: [] });
|
|
1858
|
+
assert.deepEqual(record.files.map((file) => file.checks), [[], []]);
|
|
1859
|
+
assert.equal(record.files.some((file) => file.missing?.includes('python-types')), false);
|
|
1860
|
+
assert.equal(record.state, 'complete');
|
|
1861
|
+
}
|
|
1862
|
+
finally {
|
|
1863
|
+
rmSync(dir, { recursive: true, force: true });
|
|
1864
|
+
}
|
|
1865
|
+
});
|
|
1866
|
+
check('CLI rejects selections that would otherwise run nothing and report clean', () => {
|
|
1867
|
+
const dir = realpathSync(mkdtempSync(join(tmpdir(), 'psh-cli-checks-')));
|
|
1868
|
+
const cli = join(process.cwd(), 'dist', 'cli.js');
|
|
1869
|
+
const status = (args) => {
|
|
1870
|
+
try {
|
|
1871
|
+
execFileSync(process.execPath, [cli, ...args], {
|
|
1872
|
+
cwd: dir,
|
|
1873
|
+
env: { ...process.env, CI: 'true' },
|
|
1874
|
+
stdio: ['ignore', 'pipe', 'pipe'],
|
|
1875
|
+
});
|
|
1876
|
+
return 0;
|
|
1877
|
+
}
|
|
1878
|
+
catch (error) {
|
|
1879
|
+
return error.status ?? -1;
|
|
1880
|
+
}
|
|
1881
|
+
};
|
|
1882
|
+
try {
|
|
1883
|
+
execFileSync('git', ['init', '-q', '.'], { cwd: dir });
|
|
1884
|
+
writeFileSync(join(dir, 'example.rb'), 'def example\n true\nend\n');
|
|
1885
|
+
writeFileSync(join(dir, 'bad.json'), '{not json');
|
|
1886
|
+
writeFileSync(join(dir, 'empty.json'), '[]');
|
|
1887
|
+
assert.equal(status(['scan', 'example.rb', '--verify-only', '--checks', ',', '--format', 'manifest']), 2);
|
|
1888
|
+
assert.equal(status(['scan', 'example.rb', '--verify-only', '--checks', 'plausible-logic', '--format', 'manifest']), 2);
|
|
1889
|
+
assert.equal(status(['delegate', '--checks', 'phantom-api']), 2);
|
|
1890
|
+
assert.equal(status(['delegate']), 0);
|
|
1891
|
+
assert.equal(existsSync(join(dir, '.powershot', 'sessions')), false, 'delegate must not create an unused session');
|
|
1892
|
+
assert.equal(status(['scan', 'example.rb', '--format', 'unknown']), 2);
|
|
1893
|
+
assert.equal(status(['scan', 'example.rb', '--report', 'unknown=report.txt']), 2);
|
|
1894
|
+
assert.equal(status(['scan', 'example.rb', '--budget', 'requests=zero']), 2);
|
|
1895
|
+
assert.equal(existsSync(join(dir, '.powershot', 'sessions')), false, 'bad usage must fail before session creation');
|
|
1896
|
+
assert.equal(status(['scan', 'example.rb', '--absorb', 'bad.json', '--format', 'manifest']), 2);
|
|
1897
|
+
assert.equal(status(['scan', 'example.rb', '--absorb=', '--format', 'manifest']), 2);
|
|
1898
|
+
assert.equal(existsSync(join(dir, '.powershot', 'sessions')), false, 'invalid absorb must fail before session creation');
|
|
1899
|
+
assert.equal(status(['scan', 'example.rb', '--verify-only', '--absorb', 'empty.json', '--format', 'manifest']), 0);
|
|
1900
|
+
}
|
|
1901
|
+
finally {
|
|
1902
|
+
rmSync(dir, { recursive: true, force: true });
|
|
1903
|
+
}
|
|
1904
|
+
});
|
|
1905
|
+
check('a budget stop is partial and the manifest names what was not reviewed', () => {
|
|
1906
|
+
const b = new Budget({ requests: 1 }, 0);
|
|
1907
|
+
b.spend({ requests: 1 });
|
|
1908
|
+
assert.ok(b.exhausted(0), 'the budget is spent');
|
|
1909
|
+
const m = new RunManifest('bbb11111');
|
|
1910
|
+
m.unit({ judge: 'plausible-logic', unit: 'a.ts', outcome: 'waived', reason: 'requests budget reached (1/1)', findings: 0 });
|
|
1911
|
+
const built = m.build({
|
|
1912
|
+
operation: 'review', target: { requested: {} }, policy: { source: 'base', hash: 'h' },
|
|
1913
|
+
engine: { version: '0', tools: false, verifyOnly: false }, files: [], skippedChecks: [],
|
|
1914
|
+
findings: { total: 0, verified: 0, judged: 0, dismissed: 0, droppedPosition: 0 },
|
|
1915
|
+
usage: b.used, failures: [],
|
|
1916
|
+
budgetStop: 'requests budget reached (1/1)',
|
|
1917
|
+
});
|
|
1918
|
+
assert.equal(built.state, 'partial', 'a planned limit is not an execution failure');
|
|
1919
|
+
assert.deepEqual(built.failures, []);
|
|
1920
|
+
assert.ok(built.notLookedAt.some((reason) => /stopped early: requests budget reached/.test(reason)));
|
|
1921
|
+
});
|
|
1922
|
+
console.log('');
|
|
1923
|
+
if (failures > 0) {
|
|
1924
|
+
console.error(failures + ' check(s) failed');
|
|
1925
|
+
process.exit(1);
|
|
1926
|
+
}
|
|
1927
|
+
console.log('all checks passed');
|
|
1928
|
+
//# sourceMappingURL=selftest.js.map
|