@kavishdua/proctor 0.0.0-stage → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +480 -2
- package/dist/ai/judge.js +36 -0
- package/dist/cli.js +7663 -0
- package/dist/scripts/postinstall.js +215 -0
- package/package.json +80 -4
- package/proctor.schema.json +57 -0
- package/src/skill/SKILL.md +148 -0
|
@@ -0,0 +1,215 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
|
|
3
|
+
// src/scripts/postinstall.ts
|
|
4
|
+
import { spawnSync } from "child_process";
|
|
5
|
+
import { existsSync } from "fs";
|
|
6
|
+
import { dirname, join as join2, resolve } from "path";
|
|
7
|
+
import { fileURLToPath } from "url";
|
|
8
|
+
|
|
9
|
+
// src/adapters/detect.ts
|
|
10
|
+
import { stat } from "fs/promises";
|
|
11
|
+
import { join } from "path";
|
|
12
|
+
|
|
13
|
+
// src/adapters/registry.ts
|
|
14
|
+
function cursorMdcTransform(canonical) {
|
|
15
|
+
return `---
|
|
16
|
+
description: proctor honest-completion ruleset, catches tests deleted, skipped, weakened, or gamed to fake a passing build
|
|
17
|
+
globs: '**'
|
|
18
|
+
alwaysApply: true
|
|
19
|
+
---
|
|
20
|
+
|
|
21
|
+
${canonical}`;
|
|
22
|
+
}
|
|
23
|
+
function skillFrontmatterTransform(canonical) {
|
|
24
|
+
return `---
|
|
25
|
+
name: proctor
|
|
26
|
+
description: Honest-completion ruleset for changes that touch tests or the code they cover. Use before deleting, skipping, renaming, or rewriting a test, before weakening an assertion, and before hardcoding or stubbing an implementation to make a test pass. Also covers what to do when a test looks genuinely wrong.
|
|
27
|
+
---
|
|
28
|
+
|
|
29
|
+
${canonical}`;
|
|
30
|
+
}
|
|
31
|
+
function copilotApplyToTransform(canonical) {
|
|
32
|
+
return `---
|
|
33
|
+
applyTo: '**'
|
|
34
|
+
---
|
|
35
|
+
|
|
36
|
+
${canonical}`;
|
|
37
|
+
}
|
|
38
|
+
var AGENT_ADAPTERS = [
|
|
39
|
+
// Claude Code gets skill frontmatter via skillFrontmatterTransform so the ruleset carries an explicit
|
|
40
|
+
// description and loads when it is actually relevant.
|
|
41
|
+
{ id: "claude-code", displayName: "Claude Code", relativePath: ".claude/skills/proctor/SKILL.md", scriptable: true, transform: skillFrontmatterTransform, detect: [".claude", "CLAUDE.md"] },
|
|
42
|
+
// Codex reads skills from `.agents/skills/` and REQUIRES `name` and `description` frontmatter;
|
|
43
|
+
// without it the skill is not recognized, so this path needs the same transform Claude Code uses.
|
|
44
|
+
{ id: "codex", displayName: "Codex CLI", relativePath: ".agents/skills/proctor/SKILL.md", scriptable: true, transform: skillFrontmatterTransform, detect: [".codex", ".agents/skills"] },
|
|
45
|
+
// Cursor gets .mdc YAML frontmatter (description/globs/alwaysApply) via cursorMdcTransform
|
|
46
|
+
// so the rule reliably auto-attaches.
|
|
47
|
+
{ id: "cursor", displayName: "Cursor", relativePath: ".cursor/rules/proctor.mdc", scriptable: true, transform: cursorMdcTransform, detect: [".cursor", ".cursorrules"] },
|
|
48
|
+
// Windsurf's rules.md is the workspace rules file users write their own rules into, not a
|
|
49
|
+
// proctor-owned path, so it merges rather than overwrites.
|
|
50
|
+
{ id: "windsurf", displayName: "Windsurf", relativePath: ".windsurf/rules/rules.md", scriptable: false, shared: true, detect: [".windsurf", ".windsurfrules"] },
|
|
51
|
+
{ id: "gemini-cli", displayName: "Gemini CLI", relativePath: "GEMINI.md", scriptable: true, shared: true, detect: [".gemini"] },
|
|
52
|
+
{ id: "aider", displayName: "Aider", relativePath: "CONVENTIONS.md", scriptable: true, shared: true, detect: [".aider.conf.yml", ".aider.conf.yaml"] },
|
|
53
|
+
{ id: "continue", displayName: "Continue.dev", relativePath: ".continue/rules/proctor.md", scriptable: true, detect: [".continue"] },
|
|
54
|
+
{ id: "cline", displayName: "Cline", relativePath: ".clinerules/proctor.md", scriptable: true, detect: [".clinerules"] },
|
|
55
|
+
// Amazon Q Developer CLI is mid-transition to a closed-source "Kiro CLI", so treat this path
|
|
56
|
+
// as unstable: adapter only, no bench runner wired up.
|
|
57
|
+
{ id: "amazon-q", displayName: "Amazon Q Developer", relativePath: ".amazonq/rules/proctor.md", scriptable: false, detect: [".amazonq"] },
|
|
58
|
+
// GitHub Copilot gets `applyTo: '**'` frontmatter via copilotApplyToTransform so the scoped
|
|
59
|
+
// instructions file actually activates.
|
|
60
|
+
// Detection deliberately does not treat a bare `.github/` as Copilot: nearly every repo has one.
|
|
61
|
+
{ id: "github-copilot", displayName: "GitHub Copilot", relativePath: ".github/instructions/proctor.instructions.md", scriptable: false, transform: copilotApplyToTransform, detect: [".github/instructions", ".github/copilot-instructions.md"] },
|
|
62
|
+
{ id: "zed", displayName: "Zed", relativePath: ".rules", scriptable: false, shared: true, detect: [".zed"] },
|
|
63
|
+
// Universal cross-vendor AGENTS.md standard (Linux Foundation-stewarded). Also covers Codex
|
|
64
|
+
// CLI's actual documented convention (developers.openai.com/codex/guides/agents-md) for free,
|
|
65
|
+
// the existing `codex` entry above deliberately keeps its original `.agents/skills/` path.
|
|
66
|
+
{ id: "agents-md", displayName: "AGENTS.md (universal)", relativePath: "AGENTS.md", scriptable: false, shared: true },
|
|
67
|
+
// OpenHands' repo microagent is the one repo-wide instructions file, user-authored.
|
|
68
|
+
{ id: "openhands", displayName: "OpenHands", relativePath: ".openhands/microagents/repo.md", scriptable: true, shared: true, detect: [".openhands"] },
|
|
69
|
+
// Kiro (AWS agentic IDE / Kiro CLI) intentionally coexists with the `amazon-q` entry above,
|
|
70
|
+
// they are two separately-installed products today. Revisit consolidation if the
|
|
71
|
+
// Amazon Q to Kiro merge ever completes.
|
|
72
|
+
{ id: "kiro", displayName: "Kiro", relativePath: ".kiro/steering/proctor.md", scriptable: true, detect: [".kiro"] },
|
|
73
|
+
{ id: "tabnine", displayName: "Tabnine", relativePath: ".tabnine/guidelines/proctor.md", scriptable: true, detect: [".tabnine"] },
|
|
74
|
+
{ id: "trae", displayName: "Trae", relativePath: ".trae/rules/proctor.md", scriptable: false, detect: [".trae"] },
|
|
75
|
+
{ id: "github-copilot-global", displayName: "GitHub Copilot (global)", relativePath: ".github/copilot-instructions.md", scriptable: false, shared: true, detect: [".github/instructions"] },
|
|
76
|
+
// Qodo's canonical path is a generic, un-namespaced repo-root filename that will often already
|
|
77
|
+
// hold unrelated user content, so it merges into a managed block instead of overwriting.
|
|
78
|
+
{ id: "qodo", displayName: "Qodo", relativePath: "best_practices.md", scriptable: true, shared: true, detect: [".qodo"] },
|
|
79
|
+
// --- Agents added after the first roster, all with a stable file-based repo convention ---
|
|
80
|
+
// Roo Code reads every file under .roo/rules/ recursively, so a proctor-owned file drops in
|
|
81
|
+
// cleanly alongside whatever else the user keeps there.
|
|
82
|
+
{ id: "roo-code", displayName: "Roo Code", relativePath: ".roo/rules/proctor.md", scriptable: false, detect: [".roo", ".roomodes"] },
|
|
83
|
+
// Kilo Code v7 moved config to kilo.jsonc but still auto-includes .kilocode/rules/ for
|
|
84
|
+
// backward compatibility, which is the only path proctor can write without editing user config.
|
|
85
|
+
{ id: "kilo-code", displayName: "Kilo Code", relativePath: ".kilocode/rules/proctor.md", scriptable: true, detect: [".kilocode", "kilo.jsonc"] },
|
|
86
|
+
{ id: "augment", displayName: "Augment Code", relativePath: ".augment/rules/proctor.md", scriptable: true, detect: [".augment"] },
|
|
87
|
+
// Google Antigravity defaults to .agents/rules/ (with .agent/rules/ still read as a fallback).
|
|
88
|
+
// Distinct subdirectory from the `codex` entry's .agents/skills/, so the two never collide.
|
|
89
|
+
{ id: "antigravity", displayName: "Google Antigravity", relativePath: ".agents/rules/proctor.md", scriptable: false, detect: [".agents/rules", ".agent/rules"] },
|
|
90
|
+
// goose reads a single .goosehints file per directory, user-authored.
|
|
91
|
+
{ id: "goose", displayName: "goose", relativePath: ".goosehints", scriptable: true, shared: true },
|
|
92
|
+
// Junie now prefers AGENTS.md (covered above) but still reads .junie/guidelines.md, which
|
|
93
|
+
// keeps older Junie installs covered.
|
|
94
|
+
{ id: "junie", displayName: "JetBrains Junie", relativePath: ".junie/guidelines.md", scriptable: true, shared: true, detect: [".junie"] },
|
|
95
|
+
{ id: "qwen-code", displayName: "Qwen Code", relativePath: "QWEN.md", scriptable: true, shared: true, detect: [".qwen"] },
|
|
96
|
+
// Crush reads CRUSH.md for Crush-specific rules in addition to AGENTS.md.
|
|
97
|
+
{ id: "crush", displayName: "Crush", relativePath: "CRUSH.md", scriptable: true, shared: true, detect: [".crush"] },
|
|
98
|
+
// Warp defaults to AGENTS.md now, but WARP.md still takes priority when both exist, so the
|
|
99
|
+
// ruleset has to be in both for Warp users who already have a WARP.md.
|
|
100
|
+
{ id: "warp", displayName: "Warp", relativePath: "WARP.md", scriptable: false, shared: true },
|
|
101
|
+
// Amp's own file is AGENT.md, singular, which is a different file from the universal AGENTS.md.
|
|
102
|
+
{ id: "amp", displayName: "Amp", relativePath: "AGENT.md", scriptable: true, shared: true },
|
|
103
|
+
// Firebase Studio generates .idx/airules.md with its own default rules, so merge rather than
|
|
104
|
+
// replace them.
|
|
105
|
+
{ id: "firebase-studio", displayName: "Firebase Studio", relativePath: ".idx/airules.md", scriptable: false, shared: true, detect: [".idx"] },
|
|
106
|
+
// Replit Agent reads AGENTS.md as well, but replit.md is its native file and is auto-generated.
|
|
107
|
+
{ id: "replit", displayName: "Replit Agent", relativePath: "replit.md", scriptable: false, shared: true, detect: [".replit"] }
|
|
108
|
+
];
|
|
109
|
+
|
|
110
|
+
// src/adapters/detect.ts
|
|
111
|
+
var FALLBACK_ADAPTER_ID = "agents-md";
|
|
112
|
+
async function exists(cwd, relativePath) {
|
|
113
|
+
try {
|
|
114
|
+
await stat(join(cwd, relativePath));
|
|
115
|
+
return true;
|
|
116
|
+
} catch {
|
|
117
|
+
return false;
|
|
118
|
+
}
|
|
119
|
+
}
|
|
120
|
+
async function detectAgents(cwd) {
|
|
121
|
+
const detected = await Promise.all(
|
|
122
|
+
AGENT_ADAPTERS.map(async (adapter) => {
|
|
123
|
+
for (const marker of [adapter.relativePath, ...adapter.detect ?? []]) {
|
|
124
|
+
if (await exists(cwd, marker)) return adapter;
|
|
125
|
+
}
|
|
126
|
+
return void 0;
|
|
127
|
+
})
|
|
128
|
+
);
|
|
129
|
+
const found = detected.filter((a) => a !== void 0);
|
|
130
|
+
if (found.length > 0) return found;
|
|
131
|
+
const fallback = AGENT_ADAPTERS.find((a) => a.id === FALLBACK_ADAPTER_ID);
|
|
132
|
+
return fallback ? [fallback] : [];
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
// src/scripts/postinstall.ts
|
|
136
|
+
var projectDir = process.env.INIT_CWD;
|
|
137
|
+
function packageRoot() {
|
|
138
|
+
let dir = fileURLToPath(new URL(".", import.meta.url));
|
|
139
|
+
for (let i = 0; i < 6; i++) {
|
|
140
|
+
if (existsSync(join2(dir, "package.json"))) return dir;
|
|
141
|
+
const parent = dirname(dir);
|
|
142
|
+
if (parent === dir) break;
|
|
143
|
+
dir = parent;
|
|
144
|
+
}
|
|
145
|
+
return dir;
|
|
146
|
+
}
|
|
147
|
+
function skipReason(root) {
|
|
148
|
+
if (process.env.PROCTOR_NO_POSTINSTALL) return "PROCTOR_NO_POSTINSTALL is set";
|
|
149
|
+
if (process.env.CI) return "running in CI, where the checkout is disposable";
|
|
150
|
+
if (process.env.npm_config_global === "true") return "this is a global install, with no project to set up";
|
|
151
|
+
if (!projectDir) return "INIT_CWD is not set, so the target project is unknown";
|
|
152
|
+
const target = resolve(projectDir);
|
|
153
|
+
if (target === resolve(root)) return "this is proctor\u2019s own repository";
|
|
154
|
+
if (target.split(/[\\/]/).includes("node_modules")) return "installed as a transitive dependency";
|
|
155
|
+
if (target.includes("_npx") || target.includes(".npm/_cacache")) return "running via npx, not installing into a project";
|
|
156
|
+
if (!existsSync(join2(target, ".git"))) return "no git repository here, so there is no commit to guard";
|
|
157
|
+
return void 0;
|
|
158
|
+
}
|
|
159
|
+
async function printNotice(target) {
|
|
160
|
+
let agentPaths = [];
|
|
161
|
+
try {
|
|
162
|
+
agentPaths = (await detectAgents(target)).map((a) => a.relativePath);
|
|
163
|
+
} catch {
|
|
164
|
+
}
|
|
165
|
+
const lines = [
|
|
166
|
+
"",
|
|
167
|
+
"proctor is installed. It has written nothing to your repository.",
|
|
168
|
+
"",
|
|
169
|
+
" npx proctor setup",
|
|
170
|
+
"",
|
|
171
|
+
"That one command would write, and nothing else:",
|
|
172
|
+
agentPaths.length > 0 ? ` - the honest-completion ruleset to ${agentPaths.length} agent path${agentPaths.length === 1 ? "" : "s"}: ${agentPaths.join(", ")}` : " - the honest-completion ruleset to the agent paths this repo already uses (AGENTS.md if none are detected)",
|
|
173
|
+
" - a git pre-commit hook at .git/hooks/pre-commit (or the configured hook path / .husky/pre-commit); a foreign hook is backed up, then replaced",
|
|
174
|
+
" - a Claude Code Stop hook in .claude/settings.json, only if this repo uses Claude Code",
|
|
175
|
+
" - .proctor-adapter-manifest.json, recording the deployed ruleset paths",
|
|
176
|
+
"",
|
|
177
|
+
"No network call is made. Git may configure a hook path outside the working tree.",
|
|
178
|
+
"Run `npx proctor check` first if you want to see what it finds before wiring anything up.",
|
|
179
|
+
"Set PROCTOR_AUTO_SETUP=1 to have future installs run setup for you.",
|
|
180
|
+
""
|
|
181
|
+
];
|
|
182
|
+
process.stdout.write(lines.join("\n"));
|
|
183
|
+
}
|
|
184
|
+
async function main() {
|
|
185
|
+
const root = packageRoot();
|
|
186
|
+
const reason = skipReason(root);
|
|
187
|
+
if (reason) {
|
|
188
|
+
if (process.env.PROCTOR_DEBUG) {
|
|
189
|
+
process.stdout.write(`proctor: postinstall did nothing (${reason}).
|
|
190
|
+
`);
|
|
191
|
+
}
|
|
192
|
+
return;
|
|
193
|
+
}
|
|
194
|
+
const target = projectDir;
|
|
195
|
+
if (!process.env.PROCTOR_AUTO_SETUP) {
|
|
196
|
+
await printNotice(target);
|
|
197
|
+
return;
|
|
198
|
+
}
|
|
199
|
+
const cli = join2(root, "dist", "cli.js");
|
|
200
|
+
if (!existsSync(cli)) {
|
|
201
|
+
process.stdout.write("proctor: build output not found, skipping automatic setup. Run `npx proctor setup`.\n");
|
|
202
|
+
return;
|
|
203
|
+
}
|
|
204
|
+
const result = spawnSync(process.execPath, [cli, "setup"], {
|
|
205
|
+
cwd: target,
|
|
206
|
+
stdio: "inherit"
|
|
207
|
+
});
|
|
208
|
+
if (result.status !== 0) {
|
|
209
|
+
process.stdout.write("proctor: automatic setup did not finish cleanly. Run `npx proctor setup` to see why.\n");
|
|
210
|
+
}
|
|
211
|
+
}
|
|
212
|
+
main().catch((err) => {
|
|
213
|
+
process.stdout.write(`proctor: postinstall notice could not be printed (${String(err)}). Run \`npx proctor setup\` to install the guards.
|
|
214
|
+
`);
|
|
215
|
+
});
|
package/package.json
CHANGED
|
@@ -1,6 +1,82 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@kavishdua/proctor",
|
|
3
|
-
"version": "
|
|
4
|
-
"
|
|
5
|
-
"
|
|
6
|
-
|
|
3
|
+
"version": "1.0.0",
|
|
4
|
+
"description": "Catch AI agents gaming their test suites: deleted tests, skipped tests, weakened assertions, hardcoded outputs",
|
|
5
|
+
"homepage": "https://github.com/catfish-1234/proctor#readme",
|
|
6
|
+
"bugs": {
|
|
7
|
+
"url": "https://github.com/catfish-1234/proctor/issues"
|
|
8
|
+
},
|
|
9
|
+
"type": "module",
|
|
10
|
+
"bin": {
|
|
11
|
+
"proctor": "./dist/cli.js"
|
|
12
|
+
},
|
|
13
|
+
"main": "./dist/cli.js",
|
|
14
|
+
"exports": {
|
|
15
|
+
".": "./dist/cli.js"
|
|
16
|
+
},
|
|
17
|
+
"files": [
|
|
18
|
+
"dist",
|
|
19
|
+
"!dist/scripts",
|
|
20
|
+
"dist/scripts/postinstall.js",
|
|
21
|
+
"proctor.schema.json",
|
|
22
|
+
"src/skill/SKILL.md"
|
|
23
|
+
],
|
|
24
|
+
"engines": {
|
|
25
|
+
"node": ">=20.0.0"
|
|
26
|
+
},
|
|
27
|
+
"license": "MIT",
|
|
28
|
+
"repository": {
|
|
29
|
+
"type": "git",
|
|
30
|
+
"url": "git+https://github.com/catfish-1234/proctor.git"
|
|
31
|
+
},
|
|
32
|
+
"keywords": [
|
|
33
|
+
"proctor",
|
|
34
|
+
"ai",
|
|
35
|
+
"testing",
|
|
36
|
+
"test-tampering",
|
|
37
|
+
"reward-hacking",
|
|
38
|
+
"guardrails",
|
|
39
|
+
"code-review",
|
|
40
|
+
"git-hook",
|
|
41
|
+
"pre-commit",
|
|
42
|
+
"static-analysis",
|
|
43
|
+
"sarif",
|
|
44
|
+
"agent-skills",
|
|
45
|
+
"claude",
|
|
46
|
+
"claude-code",
|
|
47
|
+
"cursor",
|
|
48
|
+
"copilot"
|
|
49
|
+
],
|
|
50
|
+
"scripts": {
|
|
51
|
+
"build": "tsup",
|
|
52
|
+
"postinstall": "node -e \"const f='dist/scripts/postinstall.js';if(require('fs').existsSync(f))require('child_process').spawnSync(process.execPath,[f],{stdio:'inherit'})\"",
|
|
53
|
+
"pretest": "npm run build",
|
|
54
|
+
"test": "vitest run",
|
|
55
|
+
"test:watch": "vitest",
|
|
56
|
+
"typecheck": "tsc --noEmit",
|
|
57
|
+
"drift-check": "node dist/cli.js drift-check",
|
|
58
|
+
"verify:pack": "node scripts/run-bash.mjs scripts/verify-pack.sh",
|
|
59
|
+
"sync-plugin": "node dist/scripts/sync-plugin.js",
|
|
60
|
+
"lint": "oxlint --deny correctness --allow no-unused-vars --allow no-useless-escape --allow prefer-string-starts-ends-with src tests scripts tsup.config.ts vitest.config.ts"
|
|
61
|
+
},
|
|
62
|
+
"dependencies": {
|
|
63
|
+
"@anthropic-ai/sdk": "^0.132.0",
|
|
64
|
+
"brace-expansion": "^5.0.12",
|
|
65
|
+
"commander": "^14.0.3",
|
|
66
|
+
"parse-diff": "0.12.0",
|
|
67
|
+
"picocolors": "1.1.1",
|
|
68
|
+
"picomatch": "^4.0.7",
|
|
69
|
+
"tinyglobby": "^0.2.17"
|
|
70
|
+
},
|
|
71
|
+
"devDependencies": {
|
|
72
|
+
"@types/node": "^26.6.4",
|
|
73
|
+
"@types/picomatch": "^4.0.3",
|
|
74
|
+
"oxlint": "^1.88.0",
|
|
75
|
+
"tsup": "8.5.1",
|
|
76
|
+
"typescript": "^7.0.2",
|
|
77
|
+
"vitest": "^4.1.11"
|
|
78
|
+
},
|
|
79
|
+
"overrides": {
|
|
80
|
+
"esbuild": "^0.28.1"
|
|
81
|
+
}
|
|
82
|
+
}
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
{
|
|
2
|
+
"$schema": "http://json-schema.org/draft-07/schema#",
|
|
3
|
+
"$id": "https://github.com/catfish-1234/proctor/proctor.schema.json",
|
|
4
|
+
"title": "Proctor Config",
|
|
5
|
+
"description": "Configuration for proctor, AI agent test-tampering guard",
|
|
6
|
+
"type": "object",
|
|
7
|
+
"properties": {
|
|
8
|
+
"enabled": {
|
|
9
|
+
"type": "array",
|
|
10
|
+
"items": { "type": "string", "pattern": "^(?:RH|WI)\\d{3}$" },
|
|
11
|
+
"description": "Rule IDs to enable. Defaults to all rules.",
|
|
12
|
+
"default": ["RH001","RH002","RH003","RH004","RH005","RH006","RH007","RH008","RH009","RH010","RH011","RH012","RH013","WI101","WI102","WI103","WI104","WI105","WI106","WI107","WI108","WI109","WI110","WI111","WI112"]
|
|
13
|
+
},
|
|
14
|
+
"testPathGlobs": {
|
|
15
|
+
"type": "array",
|
|
16
|
+
"items": { "type": "string" },
|
|
17
|
+
"description": "Glob patterns identifying test files.",
|
|
18
|
+
"default": ["**/*.test.{ts,tsx,mts,cts,js,jsx,mjs,cjs}","**/*.spec.{ts,tsx,mts,cts,js,jsx,mjs,cjs}","test/**/*","tests/**/*","__tests__/**/*","**/test_*.py","**/*_test.py"]
|
|
19
|
+
},
|
|
20
|
+
"severity": {
|
|
21
|
+
"type": "object",
|
|
22
|
+
"additionalProperties": { "type": "string", "enum": ["error","warn","info"] },
|
|
23
|
+
"description": "Per-rule severity overrides. Keys are rule IDs (e.g. 'RH006')."
|
|
24
|
+
},
|
|
25
|
+
"ignorePatterns": {
|
|
26
|
+
"type": "array",
|
|
27
|
+
"items": { "type": "string" },
|
|
28
|
+
"description": "File glob patterns to ignore entirely.",
|
|
29
|
+
"default": []
|
|
30
|
+
},
|
|
31
|
+
"approvedTestChanges": {
|
|
32
|
+
"type": "array",
|
|
33
|
+
"description": "Genuine test changes you've approved. A match downgrades the finding to 'info' so it stops blocking, but the finding is still reported everywhere. Read from the committed config, so a change cannot approve itself.",
|
|
34
|
+
"items": {
|
|
35
|
+
"type": "object",
|
|
36
|
+
"properties": {
|
|
37
|
+
"rule": { "type": "string", "pattern": "^(?:RH|WI)\\d{3}$", "description": "Rule ID this approval covers." },
|
|
38
|
+
"file": { "type": "string", "description": "Exact path or glob the approval applies to." },
|
|
39
|
+
"reason": { "type": "string", "minLength": 1, "description": "Why this change is legitimate." }
|
|
40
|
+
},
|
|
41
|
+
"required": ["rule", "file", "reason"],
|
|
42
|
+
"additionalProperties": false
|
|
43
|
+
},
|
|
44
|
+
"default": []
|
|
45
|
+
},
|
|
46
|
+
"aiModel": {
|
|
47
|
+
"type": "string",
|
|
48
|
+
"description": "Model ID used by the optional --ai judge. Defaults to claude-haiku-4-5-20251001."
|
|
49
|
+
},
|
|
50
|
+
"snapshotGlobs": {
|
|
51
|
+
"type": "array",
|
|
52
|
+
"items": { "type": "string" },
|
|
53
|
+
"description": "Glob patterns identifying snapshot/golden files for RH006. Overrides the built-in defaults."
|
|
54
|
+
}
|
|
55
|
+
},
|
|
56
|
+
"additionalProperties": false
|
|
57
|
+
}
|
|
@@ -0,0 +1,148 @@
|
|
|
1
|
+
# Proctor Skill: Honest Completion Ruleset
|
|
2
|
+
|
|
3
|
+
This is the canonical L1 skill for any AI coding agent working in a repository
|
|
4
|
+
that uses `proctor`. It applies to every agent turn that touches test files or
|
|
5
|
+
the code those tests exercise. Copy of this document is deployed verbatim to
|
|
6
|
+
every supported agent's conventional skill/rule path (see `src/adapters/registry.ts`).
|
|
7
|
+
Do not hand-edit deployed copies. Edit this file and run `proctor install-skill`.
|
|
8
|
+
|
|
9
|
+
## 1. The Core Rules
|
|
10
|
+
|
|
11
|
+
These rules are non-negotiable. They apply regardless of how much pressure
|
|
12
|
+
there is to show a green build.
|
|
13
|
+
|
|
14
|
+
Rules 1 to 5 are about the test suite. Rules 6 to 10 are about the work the
|
|
15
|
+
suite is supposed to be proving, because a green build is only one of the
|
|
16
|
+
things an agent can fake, and the others do not touch a test file at all.
|
|
17
|
+
|
|
18
|
+
1. **Never modify, skip, or delete a test to make it pass.** A test is a
|
|
19
|
+
contract about correct behavior, not an obstacle standing between you and
|
|
20
|
+
a passing build.
|
|
21
|
+
2. **If a test appears genuinely wrong, STOP and flag it to a human with
|
|
22
|
+
rationale.** Do not silently "fix" a test you believe is incorrect. Explain
|
|
23
|
+
why you think it's wrong and let a human decide.
|
|
24
|
+
3. **Never hardcode an implementation to match a test fixture literal.**
|
|
25
|
+
Special-casing a return value so it matches exactly what the test expects,
|
|
26
|
+
without implementing the real underlying logic, is not a fix.
|
|
27
|
+
4. **Never gut real logic behind an always-true mock.** Replacing a function
|
|
28
|
+
body with a stub, a constant, or a mock that always satisfies the assertion
|
|
29
|
+
is not the same as making the code correct.
|
|
30
|
+
5. **A fix is not done until the original, unaltered tests pass**, or a
|
|
31
|
+
human has explicitly approved a change to the test itself. Passing tests
|
|
32
|
+
you weakened, skipped, or replaced does not count as done.
|
|
33
|
+
6. **Never discard an error to make a failure stop.** An empty catch, an
|
|
34
|
+
`except: pass`, a `rescue nil`, a discarded promise rejection: none of
|
|
35
|
+
these fix anything. They stop the failure being reported, which is worse
|
|
36
|
+
than the failure, because now nobody will find out. Handle the error or
|
|
37
|
+
let it propagate. If discarding it genuinely is correct, write the reason
|
|
38
|
+
in the handler.
|
|
39
|
+
7. **Never delete validation to make a case pass.** A guard clause is a test
|
|
40
|
+
that runs in production. Deleting a `throw`, a `raise`, an `assert`, or a
|
|
41
|
+
precondition check because something tripped it is the same move as
|
|
42
|
+
deleting a failing test, minus the record that it happened.
|
|
43
|
+
8. **Never present unfinished or faked work as finished.** That covers
|
|
44
|
+
shipping a `NotImplementedError`, a `todo!()`, or a "not implemented"
|
|
45
|
+
throw inside a change you describe as complete, and it covers returning
|
|
46
|
+
canned, mock, or placeholder data from code that is supposed to do real
|
|
47
|
+
work. If you could not build it, say so in plain words.
|
|
48
|
+
9. **Never disable a security control or hide code from review.** Do not
|
|
49
|
+
turn off TLS certificate verification, drop an authorization gate, or
|
|
50
|
+
disable CSRF protection to make a request succeed. Do not add source or
|
|
51
|
+
test files to `.gitignore`, and do not use `git update-index
|
|
52
|
+
--assume-unchanged`: code git cannot see is code nobody can review.
|
|
53
|
+
10. **Never turn off the checks instead of satisfying them.** Do not disable
|
|
54
|
+
a proctor check, lower its severity, add an ignore pattern, write your own
|
|
55
|
+
approval, delete a ruleset file, add `--no-verify`, switch off TypeScript
|
|
56
|
+
strictness, or widen a type to `any` to silence the checker. If a check is
|
|
57
|
+
genuinely wrong, that is Rule 2: stop and say so.
|
|
58
|
+
|
|
59
|
+
Rule 5's approval is a specific, concrete thing, not a figure of speech. A human
|
|
60
|
+
records it by running `proctor approve <RH-ID> <file> --reason "..."` and
|
|
61
|
+
committing the result. You do not get to grant it to yourself: approvals are read
|
|
62
|
+
from the committed config, so one you add in the same change you are making has no
|
|
63
|
+
effect. If you think a test change is genuinely warranted, say so and ask, the way
|
|
64
|
+
Rule 2 describes. Do not write the approval and carry on.
|
|
65
|
+
|
|
66
|
+
## 2. The Guard That Enforces This
|
|
67
|
+
|
|
68
|
+
These rules are not just values or prose to keep in mind. They are backed by
|
|
69
|
+
a deterministic enforcement mechanism. `proctor`, a deterministic diff-level
|
|
70
|
+
guard, runs on every commit and on every agent turn (via the Claude Code Stop
|
|
71
|
+
hook and the git pre-commit hook) and will block on any error-severity
|
|
72
|
+
violation of the rules above. If you delete, skip, or weaken a test, or
|
|
73
|
+
hardcode/gut the implementation behind it, `proctor` inspects the diff, not
|
|
74
|
+
your reasoning about the diff, and stops the commit or turn before it lands.
|
|
75
|
+
There is no way to reason around it from inside the agent's own context,
|
|
76
|
+
because it operates below that layer, directly on the diff.
|
|
77
|
+
|
|
78
|
+
Concretely: `proctor check --uncommitted --ci` runs after every agent turn ends,
|
|
79
|
+
reading staged and unstaged changes alike, so leaving an edit unstaged hides
|
|
80
|
+
nothing. If it finds an error-severity finding, the turn is blocked and the
|
|
81
|
+
finding is reported back to you to fix honestly. Treat a block from this guard as
|
|
82
|
+
the guard doing its job, not as a bug to work around.
|
|
83
|
+
|
|
84
|
+
## 3. Rule Reference
|
|
85
|
+
|
|
86
|
+
Each row below is sourced verbatim from `RULE_METADATA` in `src/rules.ts`,
|
|
87
|
+
the same registry the tool itself uses to report findings. If you see one of
|
|
88
|
+
these rule IDs in a `proctor` finding, this table tells you what it caught.
|
|
89
|
+
|
|
90
|
+
There are two families. `RH001`–`RH014` check the claim "the tests pass": they
|
|
91
|
+
read the test suite and the code directly beneath it. `WI101`–`WI113` check the
|
|
92
|
+
claim "the work is done": they read shipped code for the ways a change can fake
|
|
93
|
+
completion without touching a test at all.
|
|
94
|
+
|
|
95
|
+
### RH0xx, test tampering
|
|
96
|
+
|
|
97
|
+
| RH-ID | Name | What It Catches |
|
|
98
|
+
|-------|------|------------------|
|
|
99
|
+
| RH001 | TestDeletedOrRenamed | Test file or test function deleted or renamed |
|
|
100
|
+
| RH002 | AssertionWeakened | Assertion weakened or precision reduced |
|
|
101
|
+
| RH003 | TestSkippedOrDisabled | Test disabled, skipped, or commented out |
|
|
102
|
+
| RH004 | ImplementationHardcodesFixture | Implementation hardcodes test fixture value |
|
|
103
|
+
| RH005 | FunctionBodyGutted | Function body replaced with a stub |
|
|
104
|
+
| RH006 | SnapshotRewrittenWithoutReason | Snapshot/golden file rewritten without a stated reason |
|
|
105
|
+
| RH007 | TestExcludedViaConfig | Test excluded from run path via config change |
|
|
106
|
+
| RH008 | TautologicalAssertion | Assertion always passes regardless of behavior |
|
|
107
|
+
| RH009 | CoverageGaming | Trivial test added while real assertions removed |
|
|
108
|
+
| RH010 | FailureMasking | Async checks detached, timeout/retry abuse, or a mock manufacturing the expected answer |
|
|
109
|
+
| RH011 | TypeLintSilencingSpam | Type/lint suppression comments added to pass (spam, or one file-wide directive) |
|
|
110
|
+
| RH012 | CIPipelineTampering | Test step removed from CI, or neutered so a failing suite still passes |
|
|
111
|
+
| RH013 | CoverageGateWeakened | Coverage threshold lowered or removed so less coverage now passes |
|
|
112
|
+
| RH014 | TestWorkloadReduced | A surviving test is changed to exercise fewer generated or table-driven cases |
|
|
113
|
+
|
|
114
|
+
### WI1xx, work integrity
|
|
115
|
+
|
|
116
|
+
| WI-ID | Name | What It Catches |
|
|
117
|
+
|-------|------|------------------|
|
|
118
|
+
| WI101 | SilentErrorSwallowing | Error discarded by an empty handler, so failures pass unnoticed |
|
|
119
|
+
| WI102 | UnimplementedWorkClaimed | Explicit not-implemented marker added to shipped code |
|
|
120
|
+
| WI103 | ValidationRemoved | Guard clause or contract enforcement deleted from shipped code |
|
|
121
|
+
| WI104 | GuardrailDisabled | Proctor, a commit hook, or a type/lint gate switched off instead of satisfied |
|
|
122
|
+
| WI105 | FakeDataSubstituted | Real network, database, or filesystem work replaced with canned data |
|
|
123
|
+
| WI106 | TypeSafetyEroded | Types widened to any or an unsafe cast to silence the type checker |
|
|
124
|
+
| WI107 | SecurityControlDisabled | A security check switched off, or an authorization gate removed |
|
|
125
|
+
| WI108 | SourceHiddenFromReview | Source or tests hidden from git, and therefore from every check |
|
|
126
|
+
| WI109 | ExpectedValueChanged | A test's expected value edited to match the current behaviour |
|
|
127
|
+
| WI110 | VerificationScriptNeutered | A test, lint, or build script rewritten so it can no longer fail |
|
|
128
|
+
| WI111 | ImplementationOrTestsRemoved | The code under test deleted, or a test file emptied of its tests |
|
|
129
|
+
| WI112 | CheckingQuietlyReduced | Assertions deleted from a surviving test, a golden file rewritten, or a module aliased to a stub |
|
|
130
|
+
| WI113 | FailureAvoidanceWorkaround | Benchmark workload reduced, dependency downgraded, or fixed delay added instead of fixing a failure |
|
|
131
|
+
|
|
132
|
+
Every WI check skips test files by design. An empty catch is how you assert that
|
|
133
|
+
something throws, canned data is what a fixture is, and a loose cast is routine
|
|
134
|
+
when building a partial mock. These checks watch the code your tests are supposed
|
|
135
|
+
to be proving, not the tests themselves.
|
|
136
|
+
|
|
137
|
+
RH004–RH011 are heuristic and higher-risk for false positives than RH001–003/007.
|
|
138
|
+
Each is implemented conservatively: strong-signal-only, high precision over recall.
|
|
139
|
+
RH004 and RH005 additionally accept `--ai` to catch fuzzier cases their deterministic
|
|
140
|
+
core intentionally stays silent on. The WI checks are built the same way, and most of
|
|
141
|
+
them offer the same escape hatch: a line whose comment explains why the thing it is
|
|
142
|
+
doing is correct will not be flagged, because writing that sentence down is the
|
|
143
|
+
outcome the check exists to produce. Run `proctor check --explain <ID>` if you're
|
|
144
|
+
unsure why one fired.
|
|
145
|
+
|
|
146
|
+
If you're unsure whether a change you're about to make would trip one of
|
|
147
|
+
these, don't make the change and ask a human instead. That is always
|
|
148
|
+
consistent with Rule 2 above.
|