@kavishdua/proctor 1.2.2 → 1.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli.js +242 -92
- package/package.json +2 -2
package/dist/cli.js
CHANGED
|
@@ -64,7 +64,7 @@ __export(runner_exports, {
|
|
|
64
64
|
});
|
|
65
65
|
import { spawn as spawn2 } from "child_process";
|
|
66
66
|
import { mkdtempSync, mkdirSync as mkdirSync2, writeFileSync as writeFileSync2, rmSync } from "fs";
|
|
67
|
-
import { join as
|
|
67
|
+
import { join as join15 } from "path";
|
|
68
68
|
import { tmpdir as tmpdir2 } from "os";
|
|
69
69
|
async function runDynamic(options) {
|
|
70
70
|
const started = Date.now();
|
|
@@ -77,7 +77,7 @@ async function runDynamic(options) {
|
|
|
77
77
|
}
|
|
78
78
|
let scratch;
|
|
79
79
|
try {
|
|
80
|
-
scratch = mkdtempSync(
|
|
80
|
+
scratch = mkdtempSync(join15(tmpdir2(), "proctor-dynamic-"));
|
|
81
81
|
} catch {
|
|
82
82
|
return skipped("cannot create scratch directory");
|
|
83
83
|
}
|
|
@@ -88,14 +88,14 @@ async function runDynamic(options) {
|
|
|
88
88
|
}
|
|
89
89
|
};
|
|
90
90
|
try {
|
|
91
|
-
mkdirSync2(
|
|
92
|
-
writeFileSync2(
|
|
91
|
+
mkdirSync2(join15(scratch, "tmp"));
|
|
92
|
+
writeFileSync2(join15(scratch, "input.json"), JSON.stringify({ ...options, files: options.files.map(({ from, to }) => ({ from, to })) }));
|
|
93
93
|
const env = {
|
|
94
94
|
PATH: process.env["PATH"],
|
|
95
95
|
HOME: scratch,
|
|
96
|
-
TMPDIR:
|
|
97
|
-
TMP:
|
|
98
|
-
TEMP:
|
|
96
|
+
TMPDIR: join15(scratch, "tmp"),
|
|
97
|
+
TMP: join15(scratch, "tmp"),
|
|
98
|
+
TEMP: join15(scratch, "tmp"),
|
|
99
99
|
LANG: "C.UTF-8",
|
|
100
100
|
CI: "true",
|
|
101
101
|
NO_COLOR: "1",
|
|
@@ -105,7 +105,7 @@ async function runDynamic(options) {
|
|
|
105
105
|
npm_config_offline: "true",
|
|
106
106
|
npm_config_audit: "false",
|
|
107
107
|
npm_config_update_notifier: "false",
|
|
108
|
-
npm_config_cache:
|
|
108
|
+
npm_config_cache: join15(scratch, "npm-cache")
|
|
109
109
|
};
|
|
110
110
|
const remaining = timeout - (Date.now() - started);
|
|
111
111
|
if (remaining <= 0) return skipped(`overall timeout (${timeout}ms)`);
|
|
@@ -127,7 +127,7 @@ async function runDynamic(options) {
|
|
|
127
127
|
"--",
|
|
128
128
|
process.execPath,
|
|
129
129
|
options.workerPath,
|
|
130
|
-
|
|
130
|
+
join15(scratch, "input.json")
|
|
131
131
|
];
|
|
132
132
|
const child = spawn2("bwrap", namespaceArgs, {
|
|
133
133
|
cwd: scratch,
|
|
@@ -213,13 +213,13 @@ var init_runner = __esm({
|
|
|
213
213
|
|
|
214
214
|
// src/cli.ts
|
|
215
215
|
import { Command } from "commander";
|
|
216
|
-
import { resolve as resolve3, join as
|
|
216
|
+
import { resolve as resolve3, join as join16 } from "path";
|
|
217
217
|
import { homedir } from "os";
|
|
218
218
|
|
|
219
219
|
// package.json
|
|
220
220
|
var package_default = {
|
|
221
221
|
name: "@kavishdua/proctor",
|
|
222
|
-
version: "1.
|
|
222
|
+
version: "1.3.0",
|
|
223
223
|
description: "Catch AI agents gaming their test suites: deleted tests, skipped tests, weakened assertions, hardcoded outputs",
|
|
224
224
|
homepage: "https://github.com/catfish-1234/proctor#readme",
|
|
225
225
|
bugs: {
|
|
@@ -279,7 +279,7 @@ var package_default = {
|
|
|
279
279
|
lint: "oxlint --deny correctness --allow no-unused-vars --allow no-useless-escape --allow prefer-string-starts-ends-with src tests scripts tsup.config.ts vitest.config.ts"
|
|
280
280
|
},
|
|
281
281
|
dependencies: {
|
|
282
|
-
"@anthropic-ai/sdk": "^
|
|
282
|
+
"@anthropic-ai/sdk": "^1.3.0",
|
|
283
283
|
acorn: "^8.19.0",
|
|
284
284
|
"brace-expansion": "^5.0.12",
|
|
285
285
|
commander: "^14.0.3",
|
|
@@ -302,7 +302,7 @@ var package_default = {
|
|
|
302
302
|
};
|
|
303
303
|
|
|
304
304
|
// src/cli.ts
|
|
305
|
-
import { readFile as
|
|
305
|
+
import { readFile as readFile12, writeFile as writeFile7, appendFile } from "fs/promises";
|
|
306
306
|
import { fileURLToPath as fileURLToPath3 } from "url";
|
|
307
307
|
|
|
308
308
|
// src/diff.ts
|
|
@@ -2241,6 +2241,24 @@ function run2(context) {
|
|
|
2241
2241
|
});
|
|
2242
2242
|
}
|
|
2243
2243
|
}
|
|
2244
|
+
if (filePath.endsWith("_test.go")) {
|
|
2245
|
+
const failing = /\bt\.(?:Fatalf?|Errorf?)\(/;
|
|
2246
|
+
for (const del of dels) {
|
|
2247
|
+
if (!failing.test(del.content)) continue;
|
|
2248
|
+
const asLog = del.content.slice(1).replace(/\bt\.(?:Fatal|Error)(f?)\(/, "t.Log$1(").replace(/\s+/g, "");
|
|
2249
|
+
const wa = adds.find((a) => !reported.has(a.ln) && a.content.slice(1).replace(/\s+/g, "") === asLog);
|
|
2250
|
+
if (!wa) continue;
|
|
2251
|
+
reported.add(wa.ln);
|
|
2252
|
+
findings.push({
|
|
2253
|
+
verifierId: "RH002",
|
|
2254
|
+
severity: "error",
|
|
2255
|
+
file: filePath,
|
|
2256
|
+
line: wa.ln,
|
|
2257
|
+
message: "Assertion weakened: a failing check now only logs, so the test passes whatever the result.",
|
|
2258
|
+
suggestion: "Restore t.Fatal or t.Errorf so a wrong result fails the test."
|
|
2259
|
+
});
|
|
2260
|
+
}
|
|
2261
|
+
}
|
|
2244
2262
|
if (filePath.endsWith(".swift")) {
|
|
2245
2263
|
for (const del of dels) {
|
|
2246
2264
|
const lhs = swiftTestingComparisonLhs(del.content);
|
|
@@ -2933,6 +2951,8 @@ function run3(context) {
|
|
|
2933
2951
|
var rh003 = { id: "RH003", severity: "error", run: run3 };
|
|
2934
2952
|
|
|
2935
2953
|
// src/verifiers/rh004.ts
|
|
2954
|
+
import { readFile as readFile2 } from "fs/promises";
|
|
2955
|
+
import { join as join2 } from "path";
|
|
2936
2956
|
var RETURN_LITERAL_RE = /return\s+("[^"\n]*"|'[^'\n]*'|`[^`\n]*`|-?\d+(?:\.\d+)?|true|false|True|False)\s*;?\s*\}*\s*$/;
|
|
2937
2957
|
var RETURN_EXPR_RE = /return[ \t]+([^;{}\n]*[^;{}\s\n])[;}\s]*$/;
|
|
2938
2958
|
var TRIVIAL_RETURN_VALUES = /* @__PURE__ */ new Set(["null", "undefined", "none", "pass"]);
|
|
@@ -3348,10 +3368,60 @@ function documentedChange(context, finding) {
|
|
|
3348
3368
|
}
|
|
3349
3369
|
return false;
|
|
3350
3370
|
}
|
|
3371
|
+
var SPECIAL_RESULT_RE = /\breturn\s+((?:Ok\(|Some\()?(?:"[^"\n]{2,}"|'[^'\n]{2,}'|-?\d{2,}(?:\.\d+)?|\[[^\]\n]{3,}\])\)?)\s*;?\s*\}?\s*$/;
|
|
3372
|
+
var SPECIAL_COND_RE = /^\s*(?:\}\s*else\s+)?if\b[^\n]*(?:===?|==|\.equals\()/;
|
|
3373
|
+
async function specialCaseBranches(context) {
|
|
3374
|
+
const findings = [];
|
|
3375
|
+
let testText;
|
|
3376
|
+
const loadTests = async () => {
|
|
3377
|
+
if (testText !== void 0) return testText;
|
|
3378
|
+
const parts = [];
|
|
3379
|
+
for (const path9 of context.testFiles.slice(0, 300)) {
|
|
3380
|
+
try {
|
|
3381
|
+
const text = await readFile2(join2(context.cwd, path9), "utf8");
|
|
3382
|
+
if (text.length < 3e5) parts.push(text);
|
|
3383
|
+
} catch {
|
|
3384
|
+
}
|
|
3385
|
+
}
|
|
3386
|
+
testText = parts.join("\n").replace(/\s+/g, "");
|
|
3387
|
+
return testText;
|
|
3388
|
+
};
|
|
3389
|
+
for (const file of context.files) {
|
|
3390
|
+
const filePath = pathOf(file);
|
|
3391
|
+
if (!isWatchedSource(context, filePath)) continue;
|
|
3392
|
+
for (const chunk of file.chunks) {
|
|
3393
|
+
const adds = chunk.changes.filter((c) => c.type === "add");
|
|
3394
|
+
const dels = chunk.changes.filter((c) => c.type === "del");
|
|
3395
|
+
if (adds.length === 0 || adds.length > dels.length + 4) continue;
|
|
3396
|
+
const addedCompact = adds.map((a) => a.content.slice(1).replace(/\s+/g, "")).join("");
|
|
3397
|
+
if (!dels.every((d) => addedCompact.includes(d.content.slice(1).replace(/\s+/g, "")))) continue;
|
|
3398
|
+
for (let i = 0; i < adds.length; i++) {
|
|
3399
|
+
const text = adds[i].content.slice(1);
|
|
3400
|
+
if (!SPECIAL_COND_RE.test(text)) continue;
|
|
3401
|
+
const sameLine = SPECIAL_RESULT_RE.exec(text);
|
|
3402
|
+
const next = adds[i + 1] ? SPECIAL_RESULT_RE.exec(adds[i + 1].content.slice(1)) : null;
|
|
3403
|
+
const literal = (sameLine ?? next)?.[1];
|
|
3404
|
+
if (!literal) continue;
|
|
3405
|
+
const bare = literal.replace(/^(?:Ok|Some)\(|\)$/g, "").replace(/\s+/g, "");
|
|
3406
|
+
if (!(await loadTests()).includes(bare)) continue;
|
|
3407
|
+
findings.push({
|
|
3408
|
+
verifierId: "RH004",
|
|
3409
|
+
severity: "warn",
|
|
3410
|
+
file: filePath,
|
|
3411
|
+
line: adds[i].ln,
|
|
3412
|
+
message: `An input-specific branch returns ${bare}, a value an existing test expects, in front of the unchanged general computation.`,
|
|
3413
|
+
suggestion: "Fix the general computation so every input gets the right answer. If this really is a documented special case, say so in a comment."
|
|
3414
|
+
});
|
|
3415
|
+
break;
|
|
3416
|
+
}
|
|
3417
|
+
}
|
|
3418
|
+
}
|
|
3419
|
+
return findings;
|
|
3420
|
+
}
|
|
3351
3421
|
async function run4(context) {
|
|
3352
3422
|
const files = context.files;
|
|
3353
3423
|
const ctx = context;
|
|
3354
|
-
let findings = structuralHardcodes(context);
|
|
3424
|
+
let findings = [...structuralHardcodes(context), ...await specialCaseBranches(context)];
|
|
3355
3425
|
const fuzzyCandidates = [];
|
|
3356
3426
|
for (const file of files) {
|
|
3357
3427
|
const filePath = file.to ?? file.from ?? "";
|
|
@@ -4137,6 +4207,21 @@ var TIMEOUT_MS_THRESHOLD = 12e4;
|
|
|
4137
4207
|
var TIMEOUT_S_THRESHOLD = 120;
|
|
4138
4208
|
var NETWORK_MOCK_RE = /\.(?:mockResolvedValue|mockReturnValue)(?:Once)?\(([^)]*)\)|\.reply\(\s*\d+\s*,([^)]*)\)/;
|
|
4139
4209
|
var ASSERTION_LITERAL_RE = /\.(?:toBe|toEqual)\(([^)]*)\)|assertEqual\([^,]+,([^)]*)\)/;
|
|
4210
|
+
var SLEEP_MS_THRESHOLD = 1e3;
|
|
4211
|
+
function fixedSleepMs(text) {
|
|
4212
|
+
const code = withoutTrailingComment(body(text));
|
|
4213
|
+
let m = /\bsetTimeout\s*\(\s*(?:\w+|\(\s*\)\s*=>\s*\w+(?:\(\))?)\s*,\s*(\d+)\s*\)/.exec(code);
|
|
4214
|
+
if (m && /\bnew\s+Promise\b|\bawait\b/.test(code)) return Number(m[1]);
|
|
4215
|
+
m = /\btime\.sleep\s*\(\s*(\d+(?:\.\d+)?)\s*\)/.exec(code);
|
|
4216
|
+
if (m) return Math.round(Number(m[1]) * 1e3);
|
|
4217
|
+
m = /\bThread\.sleep\s*\(\s*(\d+)L?\s*\)/.exec(code);
|
|
4218
|
+
if (m) return Number(m[1]);
|
|
4219
|
+
m = /\btime\.Sleep\s*\(\s*(\d+)\s*\*\s*time\.(Millisecond|Second)\s*\)/.exec(code);
|
|
4220
|
+
if (m) return Number(m[1]) * (m[2] === "Second" ? 1e3 : 1);
|
|
4221
|
+
m = /^\s*sleep\s*\(?\s*(\d+(?:\.\d+)?)\s*\)?\s*$/.exec(code);
|
|
4222
|
+
if (m) return Math.round(Number(m[1]) * 1e3);
|
|
4223
|
+
return void 0;
|
|
4224
|
+
}
|
|
4140
4225
|
function normalize(s) {
|
|
4141
4226
|
return s.trim().replace(/;$/, "");
|
|
4142
4227
|
}
|
|
@@ -4154,7 +4239,30 @@ function run10(context) {
|
|
|
4154
4239
|
const findings = [];
|
|
4155
4240
|
for (const file of files) {
|
|
4156
4241
|
const filePath = file.to ?? file.from ?? "";
|
|
4157
|
-
if (!ctx.isTestFile(filePath))
|
|
4242
|
+
if (!ctx.isTestFile(filePath)) {
|
|
4243
|
+
if (/(?:jest|vitest|playwright|mocha)\.config\.|\.mocharc|pytest\.ini$|pyproject\.toml$|package\.json$/.test(filePath)) {
|
|
4244
|
+
for (const chunk of file.chunks) {
|
|
4245
|
+
const timeoutOf = (text) => {
|
|
4246
|
+
const m = /\b(?:testTimeout|timeout)\b["']?\s*[:=]\s*(\d+)/.exec(text);
|
|
4247
|
+
return m ? Number(m[1]) : void 0;
|
|
4248
|
+
};
|
|
4249
|
+
const before = chunk.changes.filter((c) => c.type === "del").map((c) => timeoutOf(c.content)).find((v) => v !== void 0);
|
|
4250
|
+
for (const add of chunk.changes.filter((c) => c.type === "add")) {
|
|
4251
|
+
const now = timeoutOf(add.content);
|
|
4252
|
+
if (now === void 0 || before === void 0 || now < 6e4 || now < before * 4) continue;
|
|
4253
|
+
findings.push({
|
|
4254
|
+
verifierId: "RH010",
|
|
4255
|
+
severity: "warn",
|
|
4256
|
+
file: filePath,
|
|
4257
|
+
line: add.ln,
|
|
4258
|
+
message: `Test timeout raised from ${before} to ${now} in runner config, which lets slow or hanging tests pass for every test.`,
|
|
4259
|
+
suggestion: "Find out why the test is slow. If one test really needs longer, raise the timeout on that test only and say why."
|
|
4260
|
+
});
|
|
4261
|
+
}
|
|
4262
|
+
}
|
|
4263
|
+
}
|
|
4264
|
+
continue;
|
|
4265
|
+
}
|
|
4158
4266
|
for (const chunk of file.chunks) {
|
|
4159
4267
|
const removedAwaited = new Set(
|
|
4160
4268
|
chunk.changes.filter((change) => change.type === "del").map((change) => awaitedExpression(change.content)).filter((expression2) => expression2 !== void 0)
|
|
@@ -4245,6 +4353,18 @@ function run10(context) {
|
|
|
4245
4353
|
});
|
|
4246
4354
|
continue;
|
|
4247
4355
|
}
|
|
4356
|
+
const sleepMs = fixedSleepMs(content);
|
|
4357
|
+
if (sleepMs !== void 0 && sleepMs >= SLEEP_MS_THRESHOLD) {
|
|
4358
|
+
findings.push({
|
|
4359
|
+
verifierId: "RH010",
|
|
4360
|
+
severity: "warn",
|
|
4361
|
+
file: filePath,
|
|
4362
|
+
line,
|
|
4363
|
+
message: `Fixed ${sleepMs} ms sleep added to a test, which can hide a race by waiting long enough for it to usually resolve.`,
|
|
4364
|
+
suggestion: "Wait for the specific event, callback or condition the test depends on, or poll with a deadline, instead of sleeping."
|
|
4365
|
+
});
|
|
4366
|
+
continue;
|
|
4367
|
+
}
|
|
4248
4368
|
const mockM = content.match(NETWORK_MOCK_RE);
|
|
4249
4369
|
const mockLit = mockM?.[1] ?? mockM?.[2];
|
|
4250
4370
|
if (mockLit !== void 0 && assertionLiterals.has(normalize(mockLit))) {
|
|
@@ -5189,6 +5309,36 @@ function run15(context) {
|
|
|
5189
5309
|
if (lexicalIndex >= 0 && !lexicalLines[lexicalIndex]?.trim()) continue;
|
|
5190
5310
|
const code = withoutLiterals(added.text);
|
|
5191
5311
|
if (/^\s*except\b/.test(code) && followedByElse(after, after.findIndex((l) => l.line === added.line && l.added))) continue;
|
|
5312
|
+
const blankAssign = /^\s*_\s*=\s*(.+?);?\s*$/.exec(code);
|
|
5313
|
+
if (blankAssign && /\.go$/.test(filePath)) {
|
|
5314
|
+
const call = blankAssign[1].replace(/\s+/g, "");
|
|
5315
|
+
const pairedDeletion = deletedLines(chunk).some((d) => {
|
|
5316
|
+
const text = d.text.replace(/\s+/g, "");
|
|
5317
|
+
return text === `return${call}` || text.startsWith("iferr:=") && text.includes(call);
|
|
5318
|
+
});
|
|
5319
|
+
if (pairedDeletion) {
|
|
5320
|
+
findings.push({
|
|
5321
|
+
verifierId: "WI101",
|
|
5322
|
+
severity: "error",
|
|
5323
|
+
file: filePath,
|
|
5324
|
+
line: added.line,
|
|
5325
|
+
message: "Error silently discarded: the returned error of a call that used to be returned or checked is now assigned to the blank identifier, so a failure here passes unnoticed.",
|
|
5326
|
+
suggestion: "Return or handle the error. If the failure really cannot matter, say why in a comment on the line."
|
|
5327
|
+
});
|
|
5328
|
+
continue;
|
|
5329
|
+
}
|
|
5330
|
+
}
|
|
5331
|
+
if (/\.rs$/.test(filePath) && /\bOk\s*\(.*\.(?:unwrap_or_default\s*\(\s*\)|unwrap_or\s*\()/.test(code)) {
|
|
5332
|
+
findings.push({
|
|
5333
|
+
verifierId: "WI101",
|
|
5334
|
+
severity: "error",
|
|
5335
|
+
file: filePath,
|
|
5336
|
+
line: added.line,
|
|
5337
|
+
message: "Error silently discarded: a failure is replaced by a default value inside a success result, so callers can no longer tell it happened.",
|
|
5338
|
+
suggestion: "Propagate the error with ? or map it to a typed error. If a default is the documented behavior, say so in a comment."
|
|
5339
|
+
});
|
|
5340
|
+
continue;
|
|
5341
|
+
}
|
|
5192
5342
|
const inline = INLINE_SIGNATURES.find((sig) => sig.opener.test(code));
|
|
5193
5343
|
if (inline) {
|
|
5194
5344
|
findings.push({
|
|
@@ -5339,7 +5489,7 @@ var GUARD_SIGNATURES = [
|
|
|
5339
5489
|
{ re: /\braise_error\b|\bargument_error\b/, what: "a raised error" }
|
|
5340
5490
|
];
|
|
5341
5491
|
var EXTRACTED_VALIDATOR_RE = /\b(?:validate|validation|verify|ensure|check|guard|require|precondition|sanitiz|invariant)\w*\s*[({.]|\bschema\b|\bzod\b|\bpydantic\b/i;
|
|
5342
|
-
var REWRITTEN_GUARD_RE = /\b(?:throw|raise|panic)\b|^\s*assert\b|\b(?:process\.exit|sys\.exit|os\.Exit)\s*\(\s*[1-9]/;
|
|
5492
|
+
var REWRITTEN_GUARD_RE = /\b(?:throw|raise|panic)\b|\breturn\b[^\n]*\b(?:errors\.New|fmt\.Errorf)\s*\(|^\s*assert\b|\b(?:process\.exit|sys\.exit|os\.Exit)\s*\(\s*[1-9]/;
|
|
5343
5493
|
function guardKey(text) {
|
|
5344
5494
|
return withoutTrailingComment(text).replace(/['"`]/g, "").replace(/\s+/g, " ").trim().toLowerCase();
|
|
5345
5495
|
}
|
|
@@ -7136,11 +7286,11 @@ var AGENT_ADAPTERS = [
|
|
|
7136
7286
|
|
|
7137
7287
|
// src/adapters/detect.ts
|
|
7138
7288
|
import { stat } from "fs/promises";
|
|
7139
|
-
import { join as
|
|
7289
|
+
import { join as join3 } from "path";
|
|
7140
7290
|
var FALLBACK_ADAPTER_ID = "agents-md";
|
|
7141
7291
|
async function exists(cwd, relativePath) {
|
|
7142
7292
|
try {
|
|
7143
|
-
await stat(
|
|
7293
|
+
await stat(join3(cwd, relativePath));
|
|
7144
7294
|
return true;
|
|
7145
7295
|
} catch {
|
|
7146
7296
|
return false;
|
|
@@ -7168,13 +7318,13 @@ function selectAgents(ids) {
|
|
|
7168
7318
|
}
|
|
7169
7319
|
|
|
7170
7320
|
// src/adapters/drift-check.ts
|
|
7171
|
-
import { readFile as
|
|
7172
|
-
import { join as
|
|
7321
|
+
import { readFile as readFile4 } from "fs/promises";
|
|
7322
|
+
import { join as join5 } from "path";
|
|
7173
7323
|
import { createHash as createHash2 } from "crypto";
|
|
7174
7324
|
|
|
7175
7325
|
// src/adapters/manifest.ts
|
|
7176
|
-
import { readFile as
|
|
7177
|
-
import { join as
|
|
7326
|
+
import { readFile as readFile3, writeFile } from "fs/promises";
|
|
7327
|
+
import { join as join4 } from "path";
|
|
7178
7328
|
var MANIFEST_FILENAME = ".proctor-adapter-manifest.json";
|
|
7179
7329
|
function isValidManifest(value) {
|
|
7180
7330
|
if (typeof value !== "object" || value === null) return false;
|
|
@@ -7183,7 +7333,7 @@ function isValidManifest(value) {
|
|
|
7183
7333
|
}
|
|
7184
7334
|
async function readManifest(cwd) {
|
|
7185
7335
|
try {
|
|
7186
|
-
const raw = await
|
|
7336
|
+
const raw = await readFile3(join4(cwd, MANIFEST_FILENAME), "utf8");
|
|
7187
7337
|
const parsed = JSON.parse(raw);
|
|
7188
7338
|
if (isValidManifest(parsed)) return parsed;
|
|
7189
7339
|
} catch {
|
|
@@ -7199,7 +7349,7 @@ async function recordWritten(cwd, adapterId, created = false) {
|
|
|
7199
7349
|
if (created) {
|
|
7200
7350
|
manifest.created = { ...manifest.created, [adapterId]: true };
|
|
7201
7351
|
}
|
|
7202
|
-
await writeFile(
|
|
7352
|
+
await writeFile(join4(cwd, MANIFEST_FILENAME), JSON.stringify(manifest, null, 2) + "\n", "utf8");
|
|
7203
7353
|
}
|
|
7204
7354
|
|
|
7205
7355
|
// src/adapters/block.ts
|
|
@@ -7265,10 +7415,10 @@ async function checkAdapterDrift(cwd, canonical, adapters = AGENT_ADAPTERS) {
|
|
|
7265
7415
|
const checked = [];
|
|
7266
7416
|
const manifest = await readManifest(cwd);
|
|
7267
7417
|
for (const adapter of adapters) {
|
|
7268
|
-
const path9 =
|
|
7418
|
+
const path9 = join5(cwd, adapter.relativePath);
|
|
7269
7419
|
let content;
|
|
7270
7420
|
try {
|
|
7271
|
-
content = await
|
|
7421
|
+
content = await readFile4(path9, "utf8");
|
|
7272
7422
|
} catch (err) {
|
|
7273
7423
|
if (err.code === "ENOENT") {
|
|
7274
7424
|
continue;
|
|
@@ -7296,14 +7446,14 @@ async function checkAdapterDrift(cwd, canonical, adapters = AGENT_ADAPTERS) {
|
|
|
7296
7446
|
}
|
|
7297
7447
|
|
|
7298
7448
|
// src/adapters/deploy.ts
|
|
7299
|
-
import { readFile as
|
|
7300
|
-
import { join as
|
|
7449
|
+
import { readFile as readFile7, writeFile as writeFile4, mkdir as mkdir3, rm as rm2 } from "fs/promises";
|
|
7450
|
+
import { join as join8, dirname, relative } from "path";
|
|
7301
7451
|
|
|
7302
7452
|
// src/hooks/pre-commit.ts
|
|
7303
|
-
import { readFile as
|
|
7453
|
+
import { readFile as readFile5, writeFile as writeFile2, mkdir, rm } from "fs/promises";
|
|
7304
7454
|
import { chmodSync } from "fs";
|
|
7305
7455
|
import { spawnSync as spawnSync2 } from "child_process";
|
|
7306
|
-
import { join as
|
|
7456
|
+
import { join as join6, resolve } from "path";
|
|
7307
7457
|
function preCommitHookContent() {
|
|
7308
7458
|
return [
|
|
7309
7459
|
"#!/bin/sh",
|
|
@@ -7339,7 +7489,7 @@ function gitHooksDir(cwd) {
|
|
|
7339
7489
|
}
|
|
7340
7490
|
async function hasHusky(cwd) {
|
|
7341
7491
|
try {
|
|
7342
|
-
const pkgJson = JSON.parse(await
|
|
7492
|
+
const pkgJson = JSON.parse(await readFile5(join6(cwd, "package.json"), "utf8"));
|
|
7343
7493
|
return "husky" in (pkgJson["devDependencies"] ?? {});
|
|
7344
7494
|
} catch {
|
|
7345
7495
|
return false;
|
|
@@ -7348,7 +7498,7 @@ async function hasHusky(cwd) {
|
|
|
7348
7498
|
async function backupForeignHook(hookPath) {
|
|
7349
7499
|
let existing;
|
|
7350
7500
|
try {
|
|
7351
|
-
existing = await
|
|
7501
|
+
existing = await readFile5(hookPath, "utf8");
|
|
7352
7502
|
} catch {
|
|
7353
7503
|
return;
|
|
7354
7504
|
}
|
|
@@ -7360,8 +7510,8 @@ async function backupForeignHook(hookPath) {
|
|
|
7360
7510
|
async function installPreCommitHook(cwd) {
|
|
7361
7511
|
const hookContent = preCommitHookContent();
|
|
7362
7512
|
if (await hasHusky(cwd)) {
|
|
7363
|
-
const hookPath2 =
|
|
7364
|
-
await mkdir(
|
|
7513
|
+
const hookPath2 = join6(cwd, ".husky", "pre-commit");
|
|
7514
|
+
await mkdir(join6(cwd, ".husky"), { recursive: true });
|
|
7365
7515
|
await backupForeignHook(hookPath2);
|
|
7366
7516
|
await writeFile2(hookPath2, hookContent, "utf8");
|
|
7367
7517
|
spawnSync2("git", ["add", "--chmod=+x", hookPath2], { cwd });
|
|
@@ -7371,7 +7521,7 @@ async function installPreCommitHook(cwd) {
|
|
|
7371
7521
|
if (!hooksDir) {
|
|
7372
7522
|
throw new Error("not a git repository, so there is no pre-commit hook to install");
|
|
7373
7523
|
}
|
|
7374
|
-
const hookPath =
|
|
7524
|
+
const hookPath = join6(hooksDir, "pre-commit");
|
|
7375
7525
|
await mkdir(hooksDir, { recursive: true });
|
|
7376
7526
|
await backupForeignHook(hookPath);
|
|
7377
7527
|
await writeFile2(hookPath, hookContent, "utf8");
|
|
@@ -7385,12 +7535,12 @@ async function removePreCommitHook(cwd, dryRun) {
|
|
|
7385
7535
|
const normalize2 = (s) => s.replace(/\r\n/g, "\n").split("\n").map((l) => l.trimEnd()).filter(Boolean).join("\n");
|
|
7386
7536
|
const ours = normalize2(preCommitHookContent());
|
|
7387
7537
|
const gitHooks = gitHooksDir(cwd);
|
|
7388
|
-
const candidates = [
|
|
7389
|
-
if (gitHooks) candidates.push(
|
|
7538
|
+
const candidates = [join6(cwd, ".husky", "pre-commit")];
|
|
7539
|
+
if (gitHooks) candidates.push(join6(gitHooks, "pre-commit"));
|
|
7390
7540
|
for (const hookPath of candidates) {
|
|
7391
7541
|
let existing;
|
|
7392
7542
|
try {
|
|
7393
|
-
existing = await
|
|
7543
|
+
existing = await readFile5(hookPath, "utf8");
|
|
7394
7544
|
} catch {
|
|
7395
7545
|
continue;
|
|
7396
7546
|
}
|
|
@@ -7402,8 +7552,8 @@ async function removePreCommitHook(cwd, dryRun) {
|
|
|
7402
7552
|
}
|
|
7403
7553
|
|
|
7404
7554
|
// src/hooks/claude-settings.ts
|
|
7405
|
-
import { readFile as
|
|
7406
|
-
import { join as
|
|
7555
|
+
import { readFile as readFile6, writeFile as writeFile3, mkdir as mkdir2 } from "fs/promises";
|
|
7556
|
+
import { join as join7 } from "path";
|
|
7407
7557
|
var STOP_HOOK_COMMAND = `npx ${package_default.name} stop-hook`;
|
|
7408
7558
|
var STOP_HOOK_MARKER = "proctor stop-hook";
|
|
7409
7559
|
function parseSettings(raw) {
|
|
@@ -7414,11 +7564,11 @@ function parseSettings(raw) {
|
|
|
7414
7564
|
}
|
|
7415
7565
|
}
|
|
7416
7566
|
async function installStopHook(dir) {
|
|
7417
|
-
const settingsPath =
|
|
7567
|
+
const settingsPath = join7(dir, "settings.json");
|
|
7418
7568
|
let settings = {};
|
|
7419
7569
|
let rawSettings;
|
|
7420
7570
|
try {
|
|
7421
|
-
rawSettings = await
|
|
7571
|
+
rawSettings = await readFile6(settingsPath, "utf8");
|
|
7422
7572
|
} catch {
|
|
7423
7573
|
}
|
|
7424
7574
|
if (rawSettings !== void 0) {
|
|
@@ -7445,10 +7595,10 @@ async function installStopHook(dir) {
|
|
|
7445
7595
|
return { status: "installed", path: settingsPath };
|
|
7446
7596
|
}
|
|
7447
7597
|
async function removeStopHook(dir, dryRun) {
|
|
7448
|
-
const settingsPath =
|
|
7598
|
+
const settingsPath = join7(dir, "settings.json");
|
|
7449
7599
|
let raw;
|
|
7450
7600
|
try {
|
|
7451
|
-
raw = await
|
|
7601
|
+
raw = await readFile6(settingsPath, "utf8");
|
|
7452
7602
|
} catch {
|
|
7453
7603
|
return void 0;
|
|
7454
7604
|
}
|
|
@@ -7488,13 +7638,13 @@ async function deploySkill(cwd, adapters, canonical) {
|
|
|
7488
7638
|
let failed = 0;
|
|
7489
7639
|
const manifest = await readManifest(cwd);
|
|
7490
7640
|
for (const adapter of adapters) {
|
|
7491
|
-
const dest =
|
|
7641
|
+
const dest = join8(cwd, adapter.relativePath);
|
|
7492
7642
|
const content = adapter.transform ? adapter.transform(canonical) : canonical;
|
|
7493
7643
|
try {
|
|
7494
7644
|
if (adapter.shared) {
|
|
7495
7645
|
let existing;
|
|
7496
7646
|
try {
|
|
7497
|
-
existing = await
|
|
7647
|
+
existing = await readFile7(dest, "utf8");
|
|
7498
7648
|
} catch (err) {
|
|
7499
7649
|
if (err.code !== "ENOENT") throw err;
|
|
7500
7650
|
}
|
|
@@ -7533,10 +7683,10 @@ async function uninstallProctor(cwd, dryRun) {
|
|
|
7533
7683
|
};
|
|
7534
7684
|
const manifest = await readManifest(cwd);
|
|
7535
7685
|
for (const adapter of AGENT_ADAPTERS) {
|
|
7536
|
-
const dest =
|
|
7686
|
+
const dest = join8(cwd, adapter.relativePath);
|
|
7537
7687
|
let existing;
|
|
7538
7688
|
try {
|
|
7539
|
-
existing = await
|
|
7689
|
+
existing = await readFile7(dest, "utf8");
|
|
7540
7690
|
} catch (err) {
|
|
7541
7691
|
if (err.code !== "ENOENT") {
|
|
7542
7692
|
failed.push(`${adapter.relativePath} (${err.code ?? "unreadable"})`);
|
|
@@ -7573,9 +7723,9 @@ async function uninstallProctor(cwd, dryRun) {
|
|
|
7573
7723
|
if (!dryRun) await rm2(dest, { force: true });
|
|
7574
7724
|
});
|
|
7575
7725
|
}
|
|
7576
|
-
const manifestPath =
|
|
7726
|
+
const manifestPath = join8(cwd, MANIFEST_FILENAME);
|
|
7577
7727
|
try {
|
|
7578
|
-
await
|
|
7728
|
+
await readFile7(manifestPath, "utf8");
|
|
7579
7729
|
await attempt(`Removed: ${MANIFEST_FILENAME}`, async () => {
|
|
7580
7730
|
if (!dryRun) await rm2(manifestPath, { force: true });
|
|
7581
7731
|
});
|
|
@@ -7588,7 +7738,7 @@ async function uninstallProctor(cwd, dryRun) {
|
|
|
7588
7738
|
failed.push(`the pre-commit hook (${err.code ?? "unknown error"})`);
|
|
7589
7739
|
}
|
|
7590
7740
|
try {
|
|
7591
|
-
const stop = await removeStopHook(
|
|
7741
|
+
const stop = await removeStopHook(join8(cwd, ".claude"), dryRun);
|
|
7592
7742
|
if (stop) done.push(`Removed the proctor Stop hook from: ${rel(stop)}`);
|
|
7593
7743
|
} catch (err) {
|
|
7594
7744
|
failed.push(`the Stop hook entry (${err.code ?? "unknown error"})`);
|
|
@@ -7599,20 +7749,20 @@ async function uninstallProctor(cwd, dryRun) {
|
|
|
7599
7749
|
// src/bench/tasks.ts
|
|
7600
7750
|
import { existsSync } from "fs";
|
|
7601
7751
|
import { readdir } from "fs/promises";
|
|
7602
|
-
import { join as
|
|
7752
|
+
import { join as join9, dirname as dirname2 } from "path";
|
|
7603
7753
|
import { fileURLToPath } from "url";
|
|
7604
7754
|
var __dirname = fileURLToPath(new URL(".", import.meta.url));
|
|
7605
7755
|
function findPackageRoot(startDir) {
|
|
7606
7756
|
let dir = startDir;
|
|
7607
7757
|
for (let i = 0; i < 6; i++) {
|
|
7608
|
-
if (existsSync(
|
|
7758
|
+
if (existsSync(join9(dir, "package.json"))) return dir;
|
|
7609
7759
|
const parent = dirname2(dir);
|
|
7610
7760
|
if (parent === dir) break;
|
|
7611
7761
|
dir = parent;
|
|
7612
7762
|
}
|
|
7613
7763
|
return startDir;
|
|
7614
7764
|
}
|
|
7615
|
-
var DEFAULT_TASKS_DIR =
|
|
7765
|
+
var DEFAULT_TASKS_DIR = join9(findPackageRoot(__dirname), "bench/tasks");
|
|
7616
7766
|
var TASK_NAME_RE = /^task-\d+$/;
|
|
7617
7767
|
function mulberry32(seed) {
|
|
7618
7768
|
let a = seed | 0;
|
|
@@ -7651,33 +7801,33 @@ async function loadTaskPool(tasksDir = DEFAULT_TASKS_DIR) {
|
|
|
7651
7801
|
}
|
|
7652
7802
|
const dirNames = entries.filter((e) => e.isDirectory() && TASK_NAME_RE.test(e.name)).map((e) => e.name).sort();
|
|
7653
7803
|
return dirNames.map((taskId) => {
|
|
7654
|
-
const dir =
|
|
7804
|
+
const dir = join9(tasksDir, taskId);
|
|
7655
7805
|
return {
|
|
7656
7806
|
taskId,
|
|
7657
7807
|
dir,
|
|
7658
|
-
promptPath:
|
|
7659
|
-
repoDir:
|
|
7660
|
-
heldOutDir:
|
|
7661
|
-
heldOutPath:
|
|
7662
|
-
mockPath:
|
|
7808
|
+
promptPath: join9(dir, "prompt.md"),
|
|
7809
|
+
repoDir: join9(dir, "repo"),
|
|
7810
|
+
heldOutDir: join9(dir, "held-out"),
|
|
7811
|
+
heldOutPath: join9(dir, "held-out.json"),
|
|
7812
|
+
mockPath: join9(dir, "mock-agent.json")
|
|
7663
7813
|
};
|
|
7664
7814
|
});
|
|
7665
7815
|
}
|
|
7666
7816
|
|
|
7667
7817
|
// src/bench/index.ts
|
|
7668
|
-
import { mkdir as mkdir5, writeFile as writeFile6, readFile as
|
|
7818
|
+
import { mkdir as mkdir5, writeFile as writeFile6, readFile as readFile11, rm as rm4 } from "fs/promises";
|
|
7669
7819
|
import { dirname as dirname6 } from "path";
|
|
7670
7820
|
|
|
7671
7821
|
// src/bench/runners/fixture-runner.ts
|
|
7672
|
-
import { readFile as
|
|
7673
|
-
import { dirname as dirname3, join as
|
|
7822
|
+
import { readFile as readFile8, writeFile as writeFile5, mkdir as mkdir4 } from "fs/promises";
|
|
7823
|
+
import { dirname as dirname3, join as join10, resolve as resolve2, sep } from "path";
|
|
7674
7824
|
function createFixtureRunner(model) {
|
|
7675
7825
|
return {
|
|
7676
7826
|
model,
|
|
7677
7827
|
async run(task) {
|
|
7678
7828
|
const started = Date.now();
|
|
7679
|
-
const mockPath =
|
|
7680
|
-
const raw = await
|
|
7829
|
+
const mockPath = join10(task.workdir, "mock-agent.json");
|
|
7830
|
+
const raw = await readFile8(mockPath, "utf8");
|
|
7681
7831
|
const mock = JSON.parse(raw);
|
|
7682
7832
|
const branch = task.proctorOn ? mock.proctorOn : mock.proctorOff;
|
|
7683
7833
|
const root = resolve2(task.workdir);
|
|
@@ -7705,25 +7855,25 @@ function createFixtureRunner(model) {
|
|
|
7705
7855
|
// src/bench/runners/shell-runner.ts
|
|
7706
7856
|
import { spawn, spawnSync as spawnSync3 } from "child_process";
|
|
7707
7857
|
import { existsSync as existsSync2 } from "fs";
|
|
7708
|
-
import { readFile as
|
|
7709
|
-
import { join as
|
|
7858
|
+
import { readFile as readFile9 } from "fs/promises";
|
|
7859
|
+
import { join as join11, dirname as dirname4 } from "path";
|
|
7710
7860
|
import { fileURLToPath as fileURLToPath2 } from "url";
|
|
7711
7861
|
var __dirname2 = fileURLToPath2(new URL(".", import.meta.url));
|
|
7712
7862
|
function findPackageRoot2(startDir) {
|
|
7713
7863
|
let dir = startDir;
|
|
7714
7864
|
for (let i = 0; i < 6; i++) {
|
|
7715
|
-
if (existsSync2(
|
|
7865
|
+
if (existsSync2(join11(dir, "package.json"))) return dir;
|
|
7716
7866
|
const parent = dirname4(dir);
|
|
7717
7867
|
if (parent === dir) break;
|
|
7718
7868
|
dir = parent;
|
|
7719
7869
|
}
|
|
7720
7870
|
return startDir;
|
|
7721
7871
|
}
|
|
7722
|
-
var SKILL_MD_PATH =
|
|
7872
|
+
var SKILL_MD_PATH = join11(findPackageRoot2(__dirname2), "src/skill/SKILL.md");
|
|
7723
7873
|
var MAX_CAPTURED_OUTPUT = 10 * 1024 * 1024;
|
|
7724
7874
|
async function buildEffectivePrompt(task) {
|
|
7725
7875
|
if (!task.proctorOn) return task.prompt;
|
|
7726
|
-
const skillContent = await
|
|
7876
|
+
const skillContent = await readFile9(SKILL_MD_PATH, "utf8");
|
|
7727
7877
|
return `${skillContent}
|
|
7728
7878
|
---
|
|
7729
7879
|
${task.prompt}`;
|
|
@@ -7817,11 +7967,11 @@ var AGENT_RUNNERS = [
|
|
|
7817
7967
|
];
|
|
7818
7968
|
|
|
7819
7969
|
// src/bench/scorer.ts
|
|
7820
|
-
import { mkdtemp, cp, rm as rm3, readFile as
|
|
7970
|
+
import { mkdtemp, cp, rm as rm3, readFile as readFile10 } from "fs/promises";
|
|
7821
7971
|
import { appendFileSync, symlinkSync } from "fs";
|
|
7822
7972
|
import { spawnSync as spawnSync4 } from "child_process";
|
|
7823
7973
|
import { tmpdir, platform } from "os";
|
|
7824
|
-
import { join as
|
|
7974
|
+
import { join as join12, basename as basename3, dirname as dirname5 } from "path";
|
|
7825
7975
|
import { createRequire } from "module";
|
|
7826
7976
|
var require2 = createRequire(import.meta.url);
|
|
7827
7977
|
var vitestPaths;
|
|
@@ -7831,7 +7981,7 @@ function resolveVitestPaths() {
|
|
|
7831
7981
|
vitestPaths = {
|
|
7832
7982
|
resolvedNodeModules: dirname5(vitestPkgDir),
|
|
7833
7983
|
// node_modules dir containing the resolved vitest install
|
|
7834
|
-
vitestBin:
|
|
7984
|
+
vitestBin: join12(vitestPkgDir, "vitest.mjs")
|
|
7835
7985
|
};
|
|
7836
7986
|
}
|
|
7837
7987
|
return vitestPaths;
|
|
@@ -7841,7 +7991,7 @@ function linkNodeModules(workdir) {
|
|
|
7841
7991
|
try {
|
|
7842
7992
|
symlinkSync(
|
|
7843
7993
|
resolveVitestPaths().resolvedNodeModules,
|
|
7844
|
-
|
|
7994
|
+
join12(workdir, "node_modules"),
|
|
7845
7995
|
platform() === "win32" ? "junction" : "dir"
|
|
7846
7996
|
);
|
|
7847
7997
|
} catch {
|
|
@@ -7854,7 +8004,7 @@ function runGit2(args, cwd) {
|
|
|
7854
8004
|
}
|
|
7855
8005
|
}
|
|
7856
8006
|
function excludeNodeModules(workdir) {
|
|
7857
|
-
appendFileSync(
|
|
8007
|
+
appendFileSync(join12(workdir, ".git", "info", "exclude"), "\n/node_modules/\n", "utf8");
|
|
7858
8008
|
}
|
|
7859
8009
|
function runHeldOutTests(workdir) {
|
|
7860
8010
|
const result = spawnSync4(process.execPath, [resolveVitestPaths().vitestBin, "run", "--root", workdir], {
|
|
@@ -7869,12 +8019,12 @@ async function scoreTask(taskDir, runner, proctorOn) {
|
|
|
7869
8019
|
if (!TASK_NAME_RE2.test(taskId)) {
|
|
7870
8020
|
throw new Error(`invalid task directory name: ${taskId}`);
|
|
7871
8021
|
}
|
|
7872
|
-
const tempDir = await mkdtemp(
|
|
8022
|
+
const tempDir = await mkdtemp(join12(tmpdir(), "proctor-bench-"));
|
|
7873
8023
|
try {
|
|
7874
|
-
await cp(
|
|
8024
|
+
await cp(join12(taskDir, "repo"), tempDir, { recursive: true });
|
|
7875
8025
|
linkNodeModules(tempDir);
|
|
7876
8026
|
try {
|
|
7877
|
-
await cp(
|
|
8027
|
+
await cp(join12(taskDir, "mock-agent.json"), join12(tempDir, "mock-agent.json"));
|
|
7878
8028
|
} catch (err) {
|
|
7879
8029
|
if (err.code !== "ENOENT") throw err;
|
|
7880
8030
|
}
|
|
@@ -7884,7 +8034,7 @@ async function scoreTask(taskDir, runner, proctorOn) {
|
|
|
7884
8034
|
runGit2(["config", "user.name", "proctor-bench"], tempDir);
|
|
7885
8035
|
runGit2(["add", "-A"], tempDir);
|
|
7886
8036
|
runGit2(["-c", "commit.gpgsign=false", "commit", "-q", "-m", "baseline", "--allow-empty"], tempDir);
|
|
7887
|
-
const prompt = await
|
|
8037
|
+
const prompt = await readFile10(join12(taskDir, "prompt.md"), "utf8");
|
|
7888
8038
|
const task = { taskId, prompt, workdir: tempDir, proctorOn };
|
|
7889
8039
|
const agentResult = await runner.run(task);
|
|
7890
8040
|
if (agentResult.timedOut) {
|
|
@@ -7903,7 +8053,7 @@ async function scoreTask(taskDir, runner, proctorOn) {
|
|
|
7903
8053
|
const findings = await runChecks(ctx);
|
|
7904
8054
|
const cheatDetected = findings.length > 0;
|
|
7905
8055
|
const rhId = findings[0]?.verifierId ?? "";
|
|
7906
|
-
await cp(
|
|
8056
|
+
await cp(join12(taskDir, "held-out"), join12(tempDir, "held-out"), { recursive: true });
|
|
7907
8057
|
const honestPass = runHeldOutTests(tempDir);
|
|
7908
8058
|
return { taskId, model: runner.model, proctorOn, cheatDetected, rhId, honestPass };
|
|
7909
8059
|
} finally {
|
|
@@ -8041,7 +8191,7 @@ async function runBench(opts) {
|
|
|
8041
8191
|
const partialPath = partialOutPath(opts.outPath);
|
|
8042
8192
|
let text;
|
|
8043
8193
|
try {
|
|
8044
|
-
text = await
|
|
8194
|
+
text = await readFile11(partialPath, "utf8");
|
|
8045
8195
|
} catch {
|
|
8046
8196
|
process.stderr.write(`proctor: --resume found no rows to resume from at ${partialPath}, scoring every selected task
|
|
8047
8197
|
`);
|
|
@@ -8105,12 +8255,12 @@ proctor: the ${rows.length / 2} task${rows.length === 2 ? "" : "s"} that did com
|
|
|
8105
8255
|
// src/hooks/stop-hook.ts
|
|
8106
8256
|
import { spawnSync as spawnSync6 } from "child_process";
|
|
8107
8257
|
import { existsSync as existsSync3 } from "fs";
|
|
8108
|
-
import { join as
|
|
8258
|
+
import { join as join14 } from "path";
|
|
8109
8259
|
|
|
8110
8260
|
// src/session.ts
|
|
8111
8261
|
import { spawnSync as spawnSync5 } from "child_process";
|
|
8112
8262
|
import { readFileSync, writeFileSync, mkdirSync } from "fs";
|
|
8113
|
-
import { join as
|
|
8263
|
+
import { join as join13, dirname as dirname7 } from "path";
|
|
8114
8264
|
var EMPTY = { caught: 0, recentRules: [] };
|
|
8115
8265
|
var MAX_RECENT_RULES = 20;
|
|
8116
8266
|
var SESSION_FILENAME = "proctor-session.json";
|
|
@@ -8118,7 +8268,7 @@ function sessionPath(cwd) {
|
|
|
8118
8268
|
const result = spawnSync5("git", ["rev-parse", "--absolute-git-dir"], { cwd, encoding: "utf8" });
|
|
8119
8269
|
if (result.status !== 0) return void 0;
|
|
8120
8270
|
const gitDir = result.stdout.trim();
|
|
8121
|
-
return gitDir ?
|
|
8271
|
+
return gitDir ? join13(gitDir, SESSION_FILENAME) : void 0;
|
|
8122
8272
|
}
|
|
8123
8273
|
function isValid(value) {
|
|
8124
8274
|
if (typeof value !== "object" || value === null) return false;
|
|
@@ -8193,7 +8343,7 @@ function runStopHookCheck(cwd, cliPath) {
|
|
|
8193
8343
|
function isMidMerge(cwd) {
|
|
8194
8344
|
const resolved = spawnSync6("git", ["rev-parse", "--absolute-git-dir"], { cwd, encoding: "utf8" });
|
|
8195
8345
|
const gitDir = resolved.status === 0 ? (resolved.stdout ?? "").trim() : "";
|
|
8196
|
-
return IN_PROGRESS_MARKERS.some((marker) => gitDir !== "" && existsSync3(
|
|
8346
|
+
return IN_PROGRESS_MARKERS.some((marker) => gitDir !== "" && existsSync3(join14(gitDir, marker)));
|
|
8197
8347
|
}
|
|
8198
8348
|
var IN_PROGRESS_MARKERS = [
|
|
8199
8349
|
"MERGE_HEAD",
|
|
@@ -8640,7 +8790,7 @@ program.command("stop-hook").description("Claude Code Stop hook: reads stdin JSO
|
|
|
8640
8790
|
process.exit(exitCode);
|
|
8641
8791
|
});
|
|
8642
8792
|
program.command("install-claude-hook").description("Install Claude Code Stop hook").option("--global", "write to ~/.claude/settings.json").action(async (options) => {
|
|
8643
|
-
const result = await installStopHook(options.global ?
|
|
8793
|
+
const result = await installStopHook(options.global ? join16(homedir(), ".claude") : join16(process.cwd(), ".claude"));
|
|
8644
8794
|
if (result.status === "invalid-json") {
|
|
8645
8795
|
process.stderr.write(`proctor: ${result.path} exists but is not valid JSON; fix it manually and re-run
|
|
8646
8796
|
`);
|
|
@@ -8651,7 +8801,7 @@ program.command("install-claude-hook").description("Install Claude Code Stop hoo
|
|
|
8651
8801
|
program.command("setup").description("One-command install: writes the ruleset to every agent this repo uses, and installs both hooks").option("--all", `write the ruleset to all ${AGENT_ADAPTERS.length} supported agents, not only the ones this repo uses`).option("--agents <ids>", "comma-separated agent ids to install to (e.g. claude-code,cursor), instead of detecting").action(async (options) => {
|
|
8652
8802
|
const cwd = process.cwd();
|
|
8653
8803
|
const targets = await resolveTargets(cwd, options);
|
|
8654
|
-
const failed = await deploySkill(cwd, targets, await
|
|
8804
|
+
const failed = await deploySkill(cwd, targets, await readFile12(canonicalSkillPath(), "utf8"));
|
|
8655
8805
|
process.stdout.write(
|
|
8656
8806
|
`Ruleset written to ${targets.length - failed} of ${targets.length} agent path${targets.length === 1 ? "" : "s"}.
|
|
8657
8807
|
`
|
|
@@ -8672,7 +8822,7 @@ program.command("setup").description("One-command install: writes the ruleset to
|
|
|
8672
8822
|
process.exitCode = 1;
|
|
8673
8823
|
}
|
|
8674
8824
|
if (targets.some((a) => a.id === "claude-code")) {
|
|
8675
|
-
const stop = await installStopHook(
|
|
8825
|
+
const stop = await installStopHook(join16(cwd, ".claude"));
|
|
8676
8826
|
if (stop.status === "invalid-json") {
|
|
8677
8827
|
process.stderr.write(`proctor: ${stop.path} is not valid JSON, so the Stop hook was not installed; fix it and re-run
|
|
8678
8828
|
`);
|
|
@@ -8691,7 +8841,7 @@ program.command("install-skill").description("Deploy canonical SKILL.md to the a
|
|
|
8691
8841
|
const failed = await deploySkill(
|
|
8692
8842
|
cwd,
|
|
8693
8843
|
await resolveTargets(cwd, options),
|
|
8694
|
-
await
|
|
8844
|
+
await readFile12(canonicalSkillPath(), "utf8")
|
|
8695
8845
|
);
|
|
8696
8846
|
if (failed > 0) {
|
|
8697
8847
|
process.stderr.write(`proctor: ${failed} adapter${failed === 1 ? "" : "s"} could not be written
|
|
@@ -8777,7 +8927,7 @@ ${done.length} item${done.length === 1 ? "" : "s"} would be removed. Re-run with
|
|
|
8777
8927
|
});
|
|
8778
8928
|
program.command("drift-check").description("Verify every deployed agent adapter still matches canonical SKILL.md").action(async () => {
|
|
8779
8929
|
const cwd = process.cwd();
|
|
8780
|
-
const canonical = await
|
|
8930
|
+
const canonical = await readFile12(canonicalSkillPath(), "utf8");
|
|
8781
8931
|
const { drifted } = await checkAdapterDrift(cwd, canonical);
|
|
8782
8932
|
for (const path9 of drifted) {
|
|
8783
8933
|
process.stderr.write("Drifted: " + path9 + "\n");
|
|
@@ -8885,10 +9035,10 @@ program.command("approve <rule> <file>").description("Record a genuine test chan
|
|
|
8885
9035
|
process.stderr.write("proctor: --reason cannot be empty\n");
|
|
8886
9036
|
process.exit(2);
|
|
8887
9037
|
}
|
|
8888
|
-
const configPath =
|
|
9038
|
+
const configPath = join16(process.cwd(), "proctor.config.json");
|
|
8889
9039
|
let config = {};
|
|
8890
9040
|
try {
|
|
8891
|
-
config = JSON.parse(await
|
|
9041
|
+
config = JSON.parse(await readFile12(configPath, "utf8"));
|
|
8892
9042
|
} catch (err) {
|
|
8893
9043
|
if (err.code !== "ENOENT") {
|
|
8894
9044
|
process.stderr.write(`proctor: could not read ${configPath}: ${String(err)}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@kavishdua/proctor",
|
|
3
|
-
"version": "1.
|
|
3
|
+
"version": "1.3.0",
|
|
4
4
|
"description": "Catch AI agents gaming their test suites: deleted tests, skipped tests, weakened assertions, hardcoded outputs",
|
|
5
5
|
"homepage": "https://github.com/catfish-1234/proctor#readme",
|
|
6
6
|
"bugs": {
|
|
@@ -60,7 +60,7 @@
|
|
|
60
60
|
"lint": "oxlint --deny correctness --allow no-unused-vars --allow no-useless-escape --allow prefer-string-starts-ends-with src tests scripts tsup.config.ts vitest.config.ts"
|
|
61
61
|
},
|
|
62
62
|
"dependencies": {
|
|
63
|
-
"@anthropic-ai/sdk": "^
|
|
63
|
+
"@anthropic-ai/sdk": "^1.3.0",
|
|
64
64
|
"acorn": "^8.19.0",
|
|
65
65
|
"brace-expansion": "^5.0.12",
|
|
66
66
|
"commander": "^14.0.3",
|