@olegkoval/agent-skills 1.40.0 → 1.40.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/adapters/claude/olko-github-pr/skills/lekker-review/scripts/selftest.mjs +174 -0
- package/package.json +1 -1
- package/plugins/olko-apple-kit/.claude-plugin/plugin.json +1 -1
- package/plugins/olko-creative/.claude-plugin/plugin.json +1 -1
- package/plugins/olko-garmin-kit/.claude-plugin/plugin.json +1 -1
- package/plugins/olko-git-tools/.claude-plugin/plugin.json +1 -1
- package/plugins/olko-github-pr/.claude-plugin/plugin.json +1 -1
- package/plugins/olko-github-pr/skills/lekker-review/scripts/selftest.mjs +174 -0
- package/plugins/olko-github-pr/skills/lekker-review/workflow.js +86 -3
- package/plugins/olko-obsidian/.claude-plugin/plugin.json +1 -1
- package/plugins/olko-product/.claude-plugin/plugin.json +1 -1
- package/plugins/olko-reflection/.claude-plugin/plugin.json +1 -1
- package/plugins/olko-release/.claude-plugin/plugin.json +1 -1
- package/plugins/olko-skill-meta/.claude-plugin/plugin.json +1 -1
- package/plugins/olko-web-ops/.claude-plugin/plugin.json +1 -1
|
@@ -0,0 +1,174 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
// Zero-agent regression test for lekker-review's PURE logic.
|
|
3
|
+
//
|
|
4
|
+
// Why this exists: every defect found in the 2026-08-31 hardening pass was in
|
|
5
|
+
// pure, synchronous code - the dedup bucket key, the hard-rule exemption gate,
|
|
6
|
+
// the model/effort routing - yet the only way to exercise any of it was a live
|
|
7
|
+
// workflow run costing ~7 agents and 70+ seconds. This runs the same logic in
|
|
8
|
+
// milliseconds with no agents at all. Run it after ANY edit to workflow.js:
|
|
9
|
+
//
|
|
10
|
+
// node ~/.claude/skills/lekker-review/scripts/selftest.mjs
|
|
11
|
+
//
|
|
12
|
+
// It lifts the real functions out of workflow.js by source extraction rather
|
|
13
|
+
// than importing, because workflow.js is written for the Workflow harness (top
|
|
14
|
+
// level `return`, an injected `args` global) and is not importable as a module.
|
|
15
|
+
import { readFileSync } from 'node:fs'
|
|
16
|
+
import { fileURLToPath } from 'node:url'
|
|
17
|
+
import { dirname, join } from 'node:path'
|
|
18
|
+
|
|
19
|
+
const SKILL = dirname(dirname(fileURLToPath(import.meta.url)))
|
|
20
|
+
// Optional arg: a different workflow.js to test. Used to prove this suite
|
|
21
|
+
// actually discriminates - point it at a pre-fix backup and it must FAIL.
|
|
22
|
+
const target = process.argv[2] || join(SKILL, 'workflow.js')
|
|
23
|
+
const src = readFileSync(target, 'utf8')
|
|
24
|
+
console.log(`selftest target: ${target}`)
|
|
25
|
+
|
|
26
|
+
function lift(name) {
|
|
27
|
+
const i = src.indexOf(`function ${name}`)
|
|
28
|
+
if (i === -1) throw new Error(`selftest: function ${name} not found in workflow.js - was it renamed?`)
|
|
29
|
+
let d = 0, j = i
|
|
30
|
+
for (;; j++) {
|
|
31
|
+
if (src[j] === '{') d++
|
|
32
|
+
else if (src[j] === '}') { d--; if (d === 0) break }
|
|
33
|
+
}
|
|
34
|
+
return src.slice(i, j + 1)
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
function liftConst(name) {
|
|
38
|
+
const m = new RegExp(`^const ${name} = .*$`, 'm').exec(src)
|
|
39
|
+
if (!m) throw new Error(`selftest: const ${name} not found in workflow.js`)
|
|
40
|
+
return m[0]
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
const preamble = [
|
|
44
|
+
liftConst('HARD_RULES'),
|
|
45
|
+
(() => { try { return liftConst('SAME_ISSUE_LINE_WINDOW') } catch { return 'const SAME_ISSUE_LINE_WINDOW = 30' } })(),
|
|
46
|
+
"const SEVERITY_RANK = { observation: 0, idiomatic: 1, important: 2, critical: 3 }",
|
|
47
|
+
...['titleTokens', 'sameIssue', 'nearbyLines', 'spanWithinWindow', 'hardRuleCorroborated',
|
|
48
|
+
'isHardRule', 'longest', 'mergeFindings', 'dedup', 'shouldVerify'].map(n => {
|
|
49
|
+
try { return lift(n) } catch { return `function ${n}() { throw new Error('${n} absent from this workflow.js') }` }
|
|
50
|
+
}),
|
|
51
|
+
].join('\n')
|
|
52
|
+
|
|
53
|
+
const { dedup, isHardRule, shouldVerify, sameIssue } =
|
|
54
|
+
new Function(preamble + '\nreturn { dedup, isHardRule, shouldVerify, sameIssue }')()
|
|
55
|
+
|
|
56
|
+
let failed = 0
|
|
57
|
+
function check(name, actual, expected) {
|
|
58
|
+
const a = JSON.stringify(actual), e = JSON.stringify(expected)
|
|
59
|
+
if (a === e) { console.log(` ok ${name}`) }
|
|
60
|
+
else { console.log(` FAIL ${name}\n expected ${e}\n actual ${a}`); failed++ }
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
console.log('\ndedup: the same defect anchored at different lines must merge')
|
|
64
|
+
// Regression: bucketing on `file:line` meant these two were never compared,
|
|
65
|
+
// despite a title similarity of 0.64 against a 0.4 threshold. Observed live.
|
|
66
|
+
const dupes = [
|
|
67
|
+
{ file: 'src/total.ts', line: 14, severity: 'critical', title: 'Off-by-one loop skips the first cart line', badCode: 'for (let i = 1;', description: 'aaa' },
|
|
68
|
+
{ file: 'src/total.ts', line: 9, severity: 'critical', title: 'cartTotal skips the first line item (off-by-one loop start)', badCode: 'for (let i = 1;', description: 'bb' },
|
|
69
|
+
]
|
|
70
|
+
check('two anchors, one issue -> 1 finding', dedup(dupes).length, 1)
|
|
71
|
+
check('merge keeps the highest severity', dedup([
|
|
72
|
+
{ file: 'a.ts', line: 3, severity: 'observation', title: 'Off-by-one loop skips first line', badCode: '', description: '' },
|
|
73
|
+
{ file: 'a.ts', line: 5, severity: 'critical', title: 'Off-by-one loop skips the first line', badCode: '', description: '' },
|
|
74
|
+
])[0].severity, 'critical')
|
|
75
|
+
|
|
76
|
+
console.log('\ndedup: distinct issues must NOT be merged')
|
|
77
|
+
check('similar titles 390 lines apart stay separate', dedup([
|
|
78
|
+
{ file: 'big.ts', line: 10, severity: 'important', title: 'Missing pagination on the products query', badCode: '', description: '' },
|
|
79
|
+
{ file: 'big.ts', line: 400, severity: 'important', title: 'Missing pagination on the orders query', badCode: '', description: '' },
|
|
80
|
+
]).length, 2)
|
|
81
|
+
check('same line, unrelated titles stay separate', dedup([
|
|
82
|
+
{ file: 'a.ts', line: 7, severity: 'important', title: 'Unbounded retry loop hides throttling', badCode: '', description: '' },
|
|
83
|
+
{ file: 'a.ts', line: 7, severity: 'important', title: 'Metafield namespace hardcoded in the query', badCode: '', description: '' },
|
|
84
|
+
]).length, 2)
|
|
85
|
+
// Grouping must not depend on arrival order. With only g[0] compared, findings
|
|
86
|
+
// at 25, 50 and 1 all joined when 25 arrived first, spanning 49 lines.
|
|
87
|
+
const spanCase = [
|
|
88
|
+
{ file: 'a.ts', line: 25, severity: 'important', title: 'Missing pagination on the query', badCode: '', description: '' },
|
|
89
|
+
{ file: 'a.ts', line: 50, severity: 'important', title: 'Missing pagination on the query', badCode: '', description: '' },
|
|
90
|
+
{ file: 'a.ts', line: 1, severity: 'important', title: 'Missing pagination on the query', badCode: '', description: '' },
|
|
91
|
+
]
|
|
92
|
+
check('a group never spans more than the window (25, 50, 1)', dedup(spanCase).length, 2)
|
|
93
|
+
check('the same set in a different order gives the same answer',
|
|
94
|
+
dedup([spanCase[2], spanCase[0], spanCase[1]]).length, dedup(spanCase).length)
|
|
95
|
+
|
|
96
|
+
check('different files never merge', dedup([
|
|
97
|
+
{ file: 'a.ts', line: 7, severity: 'critical', title: 'Off-by-one loop skips the first line', badCode: '', description: '' },
|
|
98
|
+
{ file: 'b.ts', line: 7, severity: 'critical', title: 'Off-by-one loop skips the first line', badCode: '', description: '' },
|
|
99
|
+
]).length, 2)
|
|
100
|
+
|
|
101
|
+
console.log('\nhard rules: a tag must be corroborated to skip verification')
|
|
102
|
+
// Regression: any agent could bypass the verifier by writing rule: "TS-1".
|
|
103
|
+
// Observed live - a test-coverage finding and a comment-policy finding both did.
|
|
104
|
+
check('TS-1 on a comment-policy finding is NOT exempt',
|
|
105
|
+
isHardRule({ rule: 'TS-1', file: 'a.ts', line: 6, title: 'Comment restates the function', badCode: '/** Sum a cart. */', description: 'a comment earns its place' }), false)
|
|
106
|
+
check('TS-1 on a test-coverage finding is NOT exempt',
|
|
107
|
+
isHardRule({ rule: 'TS-1', file: 'a.ts', line: 8, title: 'No test coverage', badCode: 'for (let i = 1;', description: 'zero test files added' }), false)
|
|
108
|
+
check('an unknown rule string is NOT exempt',
|
|
109
|
+
isHardRule({ rule: 'MADE-UP', file: 'a.ts', line: 1, title: 't', badCode: 'x as Foo', description: '' }), false)
|
|
110
|
+
// TS-1 is judged on quoted code only: prose is full of `as` and `any`.
|
|
111
|
+
check('the word "any" in PROSE alone is NOT exempt',
|
|
112
|
+
isHardRule({ rule: 'TS-1', file: 'a.ts', line: 1, title: 'fails on any cart with items', badCode: 'total += lines[i].price', description: 'any agent could trip this' }), false)
|
|
113
|
+
check('the phrase "such as" in prose alone is NOT exempt',
|
|
114
|
+
isHardRule({ rule: 'TS-1', file: 'a.ts', line: 1, title: 'issue', badCode: 'const n = 1', description: 'a primitive such as String is used' }), false)
|
|
115
|
+
|
|
116
|
+
console.log('\nhard rules: genuine violations must STILL be exempt')
|
|
117
|
+
check('TS-1 with a real cast', isHardRule({ rule: 'TS-1', file: 'a.ts', line: 1, title: 'cast', badCode: 'const x = y as Foo;', description: '' }), true)
|
|
118
|
+
check('TS-1 with a real any', isHardRule({ rule: 'TS-1', file: 'a.ts', line: 1, title: 'any', badCode: 'function f(x: any) {}', description: '' }), true)
|
|
119
|
+
check('TS-2 with a .js path', isHardRule({ rule: 'TS-2', file: 'web/thing.js', line: 1, title: 'js added', badCode: '', description: '' }), true)
|
|
120
|
+
check('GQL-1 with a nodes query',isHardRule({ rule: 'GQL-1', file: 'q.graphql', line: 1, title: 'no pageInfo', badCode: 'products { nodes { id } }', description: '' }), true)
|
|
121
|
+
check('PR-1 anchored on the PR title', isHardRule({ rule: 'PR-1', file: 'PR title', line: 1, title: 'missing prefix', badCode: '', description: '' }), true)
|
|
122
|
+
// A cast to a lowercase built-in is as much a TS-1 violation as a cast to a
|
|
123
|
+
// named type. Missing it sent a genuine hard rule to a verifier that cannot
|
|
124
|
+
// answer a policy claim, where it could be dropped.
|
|
125
|
+
for (const cast of ['x as string', 'x as number', 'x as unknown as Foo', 'x as const', 'x as boolean']) {
|
|
126
|
+
check(`TS-1 corroborated by \`${cast}\``,
|
|
127
|
+
isHardRule({ rule: 'TS-1', file: 'a.ts', line: 1, title: 'cast', badCode: cast, description: '' }), true)
|
|
128
|
+
}
|
|
129
|
+
check('TS-1 corroborated by an any annotation',
|
|
130
|
+
isHardRule({ rule: 'TS-1', file: 'a.ts', line: 1, title: 'any', badCode: 'function f(x: any) {}', description: '' }), true)
|
|
131
|
+
check('TS-1 corroborated by an any[] ',
|
|
132
|
+
isHardRule({ rule: 'TS-1', file: 'a.ts', line: 1, title: 'any', badCode: 'const xs: any[] = []', description: '' }), true)
|
|
133
|
+
|
|
134
|
+
console.log('\nverification scope by depth')
|
|
135
|
+
const crit = { severity: 'critical' }, imp = { severity: 'important' }, obs = { severity: 'observation' }
|
|
136
|
+
check('scan verifies nothing', [crit, imp, obs].map(f => shouldVerify(f, 'scan')), [false, false, false])
|
|
137
|
+
check('medium verifies criticals only', [crit, imp, obs].map(f => shouldVerify(f, 'medium')), [true, false, false])
|
|
138
|
+
check('deep verifies crit + important', [crit, imp, obs].map(f => shouldVerify(f, 'deep')), [true, true, false])
|
|
139
|
+
check('a corroborated hard rule is never verified',
|
|
140
|
+
shouldVerify({ severity: 'critical', rule: 'TS-2', file: 'x.js', badCode: '', description: '', title: '' }, 'deep'), false)
|
|
141
|
+
|
|
142
|
+
// Per-host model routing is optional: a deployment may pin models per host via
|
|
143
|
+
// a MODEL_TABLE, or leave every agent() call to name its own model. Test it
|
|
144
|
+
// only when it is present, so this suite runs against either shape.
|
|
145
|
+
const routingAssign = /const\s+MODEL\s*=\s*MODEL_TABLE\s*\[\s*HOST\s*\]/.exec(src)
|
|
146
|
+
const hasRouting = Boolean(routingAssign)
|
|
147
|
+
// A guard that can silently disable itself is worse than no guard. If the file
|
|
148
|
+
// clearly HAS a MODEL_TABLE but the assignment did not parse, that is a failure,
|
|
149
|
+
// not a reason to skip.
|
|
150
|
+
if (!hasRouting && /MODEL_TABLE/.test(src)) {
|
|
151
|
+
check('MODEL_TABLE is present but its assignment was not recognised', false, true)
|
|
152
|
+
}
|
|
153
|
+
if (!hasRouting) {
|
|
154
|
+
console.log('\nmodel + effort routing: not configured in this workflow.js, skipped')
|
|
155
|
+
} else {
|
|
156
|
+
console.log('\nmodel + effort routing is pinned per host, never inherited')
|
|
157
|
+
const routing = new Function('input', [
|
|
158
|
+
src.slice(src.indexOf('const HOST = '), routingAssign.index + routingAssign[0].length),
|
|
159
|
+
'return { HOST, MODEL }',
|
|
160
|
+
].join('\n'))
|
|
161
|
+
check('an absent host arg falls back to the default table', routing({}).MODEL, routing({ host: 'claude' }).MODEL)
|
|
162
|
+
check('an unknown host falls back to the default, not an invalid model', routing({ host: 'nonsense' }).HOST, 'claude')
|
|
163
|
+
check('host matching is case-insensitive', routing({ host: 'CODEX' }).HOST, 'codex')
|
|
164
|
+
// Every role must resolve to a non-empty model, and effort must be set.
|
|
165
|
+
for (const host of ['claude', 'codex']) {
|
|
166
|
+
const m = routing({ host }).MODEL
|
|
167
|
+
const roles = Object.keys(m).filter(k => k !== 'effort')
|
|
168
|
+
check(`${host}: every role resolves to a model`, roles.every(r => typeof m[r] === 'string' && m[r].length > 0), true)
|
|
169
|
+
check(`${host}: effort is set`, typeof m.effort === 'string' && m.effort.length > 0, true)
|
|
170
|
+
}
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
console.log(failed === 0 ? '\nall checks passed\n' : `\n${failed} check(s) FAILED\n`)
|
|
174
|
+
process.exit(failed === 0 ? 0 : 1)
|
package/package.json
CHANGED
|
@@ -0,0 +1,174 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
// Zero-agent regression test for lekker-review's PURE logic.
|
|
3
|
+
//
|
|
4
|
+
// Why this exists: every defect found in the 2026-08-31 hardening pass was in
|
|
5
|
+
// pure, synchronous code - the dedup bucket key, the hard-rule exemption gate,
|
|
6
|
+
// the model/effort routing - yet the only way to exercise any of it was a live
|
|
7
|
+
// workflow run costing ~7 agents and 70+ seconds. This runs the same logic in
|
|
8
|
+
// milliseconds with no agents at all. Run it after ANY edit to workflow.js:
|
|
9
|
+
//
|
|
10
|
+
// node ~/.claude/skills/lekker-review/scripts/selftest.mjs
|
|
11
|
+
//
|
|
12
|
+
// It lifts the real functions out of workflow.js by source extraction rather
|
|
13
|
+
// than importing, because workflow.js is written for the Workflow harness (top
|
|
14
|
+
// level `return`, an injected `args` global) and is not importable as a module.
|
|
15
|
+
import { readFileSync } from 'node:fs'
|
|
16
|
+
import { fileURLToPath } from 'node:url'
|
|
17
|
+
import { dirname, join } from 'node:path'
|
|
18
|
+
|
|
19
|
+
const SKILL = dirname(dirname(fileURLToPath(import.meta.url)))
|
|
20
|
+
// Optional arg: a different workflow.js to test. Used to prove this suite
|
|
21
|
+
// actually discriminates - point it at a pre-fix backup and it must FAIL.
|
|
22
|
+
const target = process.argv[2] || join(SKILL, 'workflow.js')
|
|
23
|
+
const src = readFileSync(target, 'utf8')
|
|
24
|
+
console.log(`selftest target: ${target}`)
|
|
25
|
+
|
|
26
|
+
function lift(name) {
|
|
27
|
+
const i = src.indexOf(`function ${name}`)
|
|
28
|
+
if (i === -1) throw new Error(`selftest: function ${name} not found in workflow.js - was it renamed?`)
|
|
29
|
+
let d = 0, j = i
|
|
30
|
+
for (;; j++) {
|
|
31
|
+
if (src[j] === '{') d++
|
|
32
|
+
else if (src[j] === '}') { d--; if (d === 0) break }
|
|
33
|
+
}
|
|
34
|
+
return src.slice(i, j + 1)
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
function liftConst(name) {
|
|
38
|
+
const m = new RegExp(`^const ${name} = .*$`, 'm').exec(src)
|
|
39
|
+
if (!m) throw new Error(`selftest: const ${name} not found in workflow.js`)
|
|
40
|
+
return m[0]
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
const preamble = [
|
|
44
|
+
liftConst('HARD_RULES'),
|
|
45
|
+
(() => { try { return liftConst('SAME_ISSUE_LINE_WINDOW') } catch { return 'const SAME_ISSUE_LINE_WINDOW = 30' } })(),
|
|
46
|
+
"const SEVERITY_RANK = { observation: 0, idiomatic: 1, important: 2, critical: 3 }",
|
|
47
|
+
...['titleTokens', 'sameIssue', 'nearbyLines', 'spanWithinWindow', 'hardRuleCorroborated',
|
|
48
|
+
'isHardRule', 'longest', 'mergeFindings', 'dedup', 'shouldVerify'].map(n => {
|
|
49
|
+
try { return lift(n) } catch { return `function ${n}() { throw new Error('${n} absent from this workflow.js') }` }
|
|
50
|
+
}),
|
|
51
|
+
].join('\n')
|
|
52
|
+
|
|
53
|
+
const { dedup, isHardRule, shouldVerify, sameIssue } =
|
|
54
|
+
new Function(preamble + '\nreturn { dedup, isHardRule, shouldVerify, sameIssue }')()
|
|
55
|
+
|
|
56
|
+
let failed = 0
|
|
57
|
+
function check(name, actual, expected) {
|
|
58
|
+
const a = JSON.stringify(actual), e = JSON.stringify(expected)
|
|
59
|
+
if (a === e) { console.log(` ok ${name}`) }
|
|
60
|
+
else { console.log(` FAIL ${name}\n expected ${e}\n actual ${a}`); failed++ }
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
console.log('\ndedup: the same defect anchored at different lines must merge')
|
|
64
|
+
// Regression: bucketing on `file:line` meant these two were never compared,
|
|
65
|
+
// despite a title similarity of 0.64 against a 0.4 threshold. Observed live.
|
|
66
|
+
const dupes = [
|
|
67
|
+
{ file: 'src/total.ts', line: 14, severity: 'critical', title: 'Off-by-one loop skips the first cart line', badCode: 'for (let i = 1;', description: 'aaa' },
|
|
68
|
+
{ file: 'src/total.ts', line: 9, severity: 'critical', title: 'cartTotal skips the first line item (off-by-one loop start)', badCode: 'for (let i = 1;', description: 'bb' },
|
|
69
|
+
]
|
|
70
|
+
check('two anchors, one issue -> 1 finding', dedup(dupes).length, 1)
|
|
71
|
+
check('merge keeps the highest severity', dedup([
|
|
72
|
+
{ file: 'a.ts', line: 3, severity: 'observation', title: 'Off-by-one loop skips first line', badCode: '', description: '' },
|
|
73
|
+
{ file: 'a.ts', line: 5, severity: 'critical', title: 'Off-by-one loop skips the first line', badCode: '', description: '' },
|
|
74
|
+
])[0].severity, 'critical')
|
|
75
|
+
|
|
76
|
+
console.log('\ndedup: distinct issues must NOT be merged')
|
|
77
|
+
check('similar titles 390 lines apart stay separate', dedup([
|
|
78
|
+
{ file: 'big.ts', line: 10, severity: 'important', title: 'Missing pagination on the products query', badCode: '', description: '' },
|
|
79
|
+
{ file: 'big.ts', line: 400, severity: 'important', title: 'Missing pagination on the orders query', badCode: '', description: '' },
|
|
80
|
+
]).length, 2)
|
|
81
|
+
check('same line, unrelated titles stay separate', dedup([
|
|
82
|
+
{ file: 'a.ts', line: 7, severity: 'important', title: 'Unbounded retry loop hides throttling', badCode: '', description: '' },
|
|
83
|
+
{ file: 'a.ts', line: 7, severity: 'important', title: 'Metafield namespace hardcoded in the query', badCode: '', description: '' },
|
|
84
|
+
]).length, 2)
|
|
85
|
+
// Grouping must not depend on arrival order. With only g[0] compared, findings
|
|
86
|
+
// at 25, 50 and 1 all joined when 25 arrived first, spanning 49 lines.
|
|
87
|
+
const spanCase = [
|
|
88
|
+
{ file: 'a.ts', line: 25, severity: 'important', title: 'Missing pagination on the query', badCode: '', description: '' },
|
|
89
|
+
{ file: 'a.ts', line: 50, severity: 'important', title: 'Missing pagination on the query', badCode: '', description: '' },
|
|
90
|
+
{ file: 'a.ts', line: 1, severity: 'important', title: 'Missing pagination on the query', badCode: '', description: '' },
|
|
91
|
+
]
|
|
92
|
+
check('a group never spans more than the window (25, 50, 1)', dedup(spanCase).length, 2)
|
|
93
|
+
check('the same set in a different order gives the same answer',
|
|
94
|
+
dedup([spanCase[2], spanCase[0], spanCase[1]]).length, dedup(spanCase).length)
|
|
95
|
+
|
|
96
|
+
check('different files never merge', dedup([
|
|
97
|
+
{ file: 'a.ts', line: 7, severity: 'critical', title: 'Off-by-one loop skips the first line', badCode: '', description: '' },
|
|
98
|
+
{ file: 'b.ts', line: 7, severity: 'critical', title: 'Off-by-one loop skips the first line', badCode: '', description: '' },
|
|
99
|
+
]).length, 2)
|
|
100
|
+
|
|
101
|
+
console.log('\nhard rules: a tag must be corroborated to skip verification')
|
|
102
|
+
// Regression: any agent could bypass the verifier by writing rule: "TS-1".
|
|
103
|
+
// Observed live - a test-coverage finding and a comment-policy finding both did.
|
|
104
|
+
check('TS-1 on a comment-policy finding is NOT exempt',
|
|
105
|
+
isHardRule({ rule: 'TS-1', file: 'a.ts', line: 6, title: 'Comment restates the function', badCode: '/** Sum a cart. */', description: 'a comment earns its place' }), false)
|
|
106
|
+
check('TS-1 on a test-coverage finding is NOT exempt',
|
|
107
|
+
isHardRule({ rule: 'TS-1', file: 'a.ts', line: 8, title: 'No test coverage', badCode: 'for (let i = 1;', description: 'zero test files added' }), false)
|
|
108
|
+
check('an unknown rule string is NOT exempt',
|
|
109
|
+
isHardRule({ rule: 'MADE-UP', file: 'a.ts', line: 1, title: 't', badCode: 'x as Foo', description: '' }), false)
|
|
110
|
+
// TS-1 is judged on quoted code only: prose is full of `as` and `any`.
|
|
111
|
+
check('the word "any" in PROSE alone is NOT exempt',
|
|
112
|
+
isHardRule({ rule: 'TS-1', file: 'a.ts', line: 1, title: 'fails on any cart with items', badCode: 'total += lines[i].price', description: 'any agent could trip this' }), false)
|
|
113
|
+
check('the phrase "such as" in prose alone is NOT exempt',
|
|
114
|
+
isHardRule({ rule: 'TS-1', file: 'a.ts', line: 1, title: 'issue', badCode: 'const n = 1', description: 'a primitive such as String is used' }), false)
|
|
115
|
+
|
|
116
|
+
console.log('\nhard rules: genuine violations must STILL be exempt')
|
|
117
|
+
check('TS-1 with a real cast', isHardRule({ rule: 'TS-1', file: 'a.ts', line: 1, title: 'cast', badCode: 'const x = y as Foo;', description: '' }), true)
|
|
118
|
+
check('TS-1 with a real any', isHardRule({ rule: 'TS-1', file: 'a.ts', line: 1, title: 'any', badCode: 'function f(x: any) {}', description: '' }), true)
|
|
119
|
+
check('TS-2 with a .js path', isHardRule({ rule: 'TS-2', file: 'web/thing.js', line: 1, title: 'js added', badCode: '', description: '' }), true)
|
|
120
|
+
check('GQL-1 with a nodes query',isHardRule({ rule: 'GQL-1', file: 'q.graphql', line: 1, title: 'no pageInfo', badCode: 'products { nodes { id } }', description: '' }), true)
|
|
121
|
+
check('PR-1 anchored on the PR title', isHardRule({ rule: 'PR-1', file: 'PR title', line: 1, title: 'missing prefix', badCode: '', description: '' }), true)
|
|
122
|
+
// A cast to a lowercase built-in is as much a TS-1 violation as a cast to a
|
|
123
|
+
// named type. Missing it sent a genuine hard rule to a verifier that cannot
|
|
124
|
+
// answer a policy claim, where it could be dropped.
|
|
125
|
+
for (const cast of ['x as string', 'x as number', 'x as unknown as Foo', 'x as const', 'x as boolean']) {
|
|
126
|
+
check(`TS-1 corroborated by \`${cast}\``,
|
|
127
|
+
isHardRule({ rule: 'TS-1', file: 'a.ts', line: 1, title: 'cast', badCode: cast, description: '' }), true)
|
|
128
|
+
}
|
|
129
|
+
check('TS-1 corroborated by an any annotation',
|
|
130
|
+
isHardRule({ rule: 'TS-1', file: 'a.ts', line: 1, title: 'any', badCode: 'function f(x: any) {}', description: '' }), true)
|
|
131
|
+
check('TS-1 corroborated by an any[] ',
|
|
132
|
+
isHardRule({ rule: 'TS-1', file: 'a.ts', line: 1, title: 'any', badCode: 'const xs: any[] = []', description: '' }), true)
|
|
133
|
+
|
|
134
|
+
console.log('\nverification scope by depth')
|
|
135
|
+
const crit = { severity: 'critical' }, imp = { severity: 'important' }, obs = { severity: 'observation' }
|
|
136
|
+
check('scan verifies nothing', [crit, imp, obs].map(f => shouldVerify(f, 'scan')), [false, false, false])
|
|
137
|
+
check('medium verifies criticals only', [crit, imp, obs].map(f => shouldVerify(f, 'medium')), [true, false, false])
|
|
138
|
+
check('deep verifies crit + important', [crit, imp, obs].map(f => shouldVerify(f, 'deep')), [true, true, false])
|
|
139
|
+
check('a corroborated hard rule is never verified',
|
|
140
|
+
shouldVerify({ severity: 'critical', rule: 'TS-2', file: 'x.js', badCode: '', description: '', title: '' }, 'deep'), false)
|
|
141
|
+
|
|
142
|
+
// Per-host model routing is optional: a deployment may pin models per host via
|
|
143
|
+
// a MODEL_TABLE, or leave every agent() call to name its own model. Test it
|
|
144
|
+
// only when it is present, so this suite runs against either shape.
|
|
145
|
+
const routingAssign = /const\s+MODEL\s*=\s*MODEL_TABLE\s*\[\s*HOST\s*\]/.exec(src)
|
|
146
|
+
const hasRouting = Boolean(routingAssign)
|
|
147
|
+
// A guard that can silently disable itself is worse than no guard. If the file
|
|
148
|
+
// clearly HAS a MODEL_TABLE but the assignment did not parse, that is a failure,
|
|
149
|
+
// not a reason to skip.
|
|
150
|
+
if (!hasRouting && /MODEL_TABLE/.test(src)) {
|
|
151
|
+
check('MODEL_TABLE is present but its assignment was not recognised', false, true)
|
|
152
|
+
}
|
|
153
|
+
if (!hasRouting) {
|
|
154
|
+
console.log('\nmodel + effort routing: not configured in this workflow.js, skipped')
|
|
155
|
+
} else {
|
|
156
|
+
console.log('\nmodel + effort routing is pinned per host, never inherited')
|
|
157
|
+
const routing = new Function('input', [
|
|
158
|
+
src.slice(src.indexOf('const HOST = '), routingAssign.index + routingAssign[0].length),
|
|
159
|
+
'return { HOST, MODEL }',
|
|
160
|
+
].join('\n'))
|
|
161
|
+
check('an absent host arg falls back to the default table', routing({}).MODEL, routing({ host: 'claude' }).MODEL)
|
|
162
|
+
check('an unknown host falls back to the default, not an invalid model', routing({ host: 'nonsense' }).HOST, 'claude')
|
|
163
|
+
check('host matching is case-insensitive', routing({ host: 'CODEX' }).HOST, 'codex')
|
|
164
|
+
// Every role must resolve to a non-empty model, and effort must be set.
|
|
165
|
+
for (const host of ['claude', 'codex']) {
|
|
166
|
+
const m = routing({ host }).MODEL
|
|
167
|
+
const roles = Object.keys(m).filter(k => k !== 'effort')
|
|
168
|
+
check(`${host}: every role resolves to a model`, roles.every(r => typeof m[r] === 'string' && m[r].length > 0), true)
|
|
169
|
+
check(`${host}: effort is set`, typeof m.effort === 'string' && m.effort.length > 0, true)
|
|
170
|
+
}
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
console.log(failed === 0 ? '\nall checks passed\n' : `\n${failed} check(s) FAILED\n`)
|
|
174
|
+
process.exit(failed === 0 ? 0 : 1)
|
|
@@ -97,8 +97,52 @@ const PROOF_SCHEMA = {
|
|
|
97
97
|
|
|
98
98
|
const HARD_RULES = ['TS-1', 'TS-2', 'GQL-1', 'PR-1']
|
|
99
99
|
|
|
100
|
+
// Does the finding's OWN evidence corroborate the hard rule it claims?
|
|
101
|
+
//
|
|
102
|
+
// A `rule` tag is worth a lot: it preserves Critical severity AND skips
|
|
103
|
+
// adversarial verification. Nothing used to check that the tag matched the
|
|
104
|
+
// finding, so any reviewer agent could bypass the verifier by writing
|
|
105
|
+
// `rule: "TS-1"`. Seen in practice: a missing-test-coverage finding and a
|
|
106
|
+
// comment-policy finding were both tagged TS-1, which is about `as X` casts
|
|
107
|
+
// and `any`, and both shipped Critical and unverified.
|
|
108
|
+
//
|
|
109
|
+
// An uncorroborated tag does NOT lose its severity here; it simply stops being
|
|
110
|
+
// exempt, so the verifier weighs it like any other finding. That is the
|
|
111
|
+
// conservative direction: unproven claims get scrutiny, not a free pass.
|
|
112
|
+
function hardRuleCorroborated(finding) {
|
|
113
|
+
const evidence = [finding.badCode, finding.description, finding.title]
|
|
114
|
+
.map(function(x) { return String(x || '') }).join('\n')
|
|
115
|
+
const file = String(finding.file || '')
|
|
116
|
+
// TS-1 is judged on the QUOTED CODE only. Prose is full of the words `as` and
|
|
117
|
+
// `any` ("any cart with items"), and matching those re-opened the very bypass
|
|
118
|
+
// this function exists to close.
|
|
119
|
+
const code = String(finding.badCode || '')
|
|
120
|
+
|
|
121
|
+
switch (finding.rule) {
|
|
122
|
+
case 'TS-1':
|
|
123
|
+
// A cast to a capitalised type OR to a lowercase built-in: `as string`
|
|
124
|
+
// is every bit as much a TS-1 violation as `as Foo`, and missing it sent
|
|
125
|
+
// a genuine hard-rule finding to a verifier whose five challenges cannot
|
|
126
|
+
// answer a standards claim, where it could be dropped outright.
|
|
127
|
+
return /\bas\s+(?:[A-Z_$][\w$]*|string|number|boolean|bigint|symbol|object|unknown|never|any|const)\b/.test(code)
|
|
128
|
+
|| /:\s*any\b|<\s*any[\s,>]|\bany\[\]/.test(code)
|
|
129
|
+
case 'TS-2':
|
|
130
|
+
return /\.js$/.test(file)
|
|
131
|
+
case 'GQL-1':
|
|
132
|
+
return /\bnodes\b|\bpageInfo\b|\bedges\b/.test(evidence)
|
|
133
|
+
case 'PR-1':
|
|
134
|
+
// PR-1 is about the PR title, so it has no source file to anchor to.
|
|
135
|
+
return /pr\s*(title|description)/i.test(file + '\n' + evidence)
|
|
136
|
+
default:
|
|
137
|
+
return false
|
|
138
|
+
}
|
|
139
|
+
}
|
|
140
|
+
|
|
100
141
|
function isHardRule(finding) {
|
|
101
|
-
|
|
142
|
+
if (typeof finding.rule !== 'string' || HARD_RULES.indexOf(finding.rule) === -1) {
|
|
143
|
+
return false
|
|
144
|
+
}
|
|
145
|
+
return hardRuleCorroborated(finding)
|
|
102
146
|
}
|
|
103
147
|
|
|
104
148
|
const SEVERITY_RANK = { critical: 3, important: 2, observation: 1, idiomatic: 0 }
|
|
@@ -162,10 +206,43 @@ function mergeFindings(a, b) {
|
|
|
162
206
|
return merged
|
|
163
207
|
}
|
|
164
208
|
|
|
209
|
+
// How far apart two anchor lines may be and still count as the same issue.
|
|
210
|
+
// Reviewers routinely anchor one defect at different lines: the loop header,
|
|
211
|
+
// the body, the function signature. Kept tight enough that two genuinely
|
|
212
|
+
// distinct findings in one file are not merged because their titles rhyme.
|
|
213
|
+
const SAME_ISSUE_LINE_WINDOW = 30
|
|
214
|
+
|
|
215
|
+
function nearbyLines(a, b) {
|
|
216
|
+
const la = Number(a.line)
|
|
217
|
+
const lb = Number(b.line)
|
|
218
|
+
if (!Number.isFinite(la) || !Number.isFinite(lb)) {
|
|
219
|
+
// A finding with no usable line (a PR-level note) only merges with another
|
|
220
|
+
// one at the same missing line, which is what strict equality gives.
|
|
221
|
+
return a.line === b.line
|
|
222
|
+
}
|
|
223
|
+
return Math.abs(la - lb) <= SAME_ISSUE_LINE_WINDOW
|
|
224
|
+
}
|
|
225
|
+
|
|
226
|
+
// Would adding `candidate` keep the whole group inside the window? Uses the
|
|
227
|
+
// group's min and max so the answer never depends on arrival order.
|
|
228
|
+
function spanWithinWindow(group, candidate) {
|
|
229
|
+
const lines = group.concat([candidate]).map(function(f) { return Number(f.line) })
|
|
230
|
+
if (!lines.every(function(n) { return Number.isFinite(n) })) {
|
|
231
|
+
// Any unusable line falls back to the strict pairwise rule.
|
|
232
|
+
return group.every(function(m) { return nearbyLines(m, candidate) })
|
|
233
|
+
}
|
|
234
|
+
return Math.max.apply(null, lines) - Math.min.apply(null, lines) <= SAME_ISSUE_LINE_WINDOW
|
|
235
|
+
}
|
|
236
|
+
|
|
165
237
|
function dedup(allFindings) {
|
|
238
|
+
// Bucket by FILE, not by `file:line`. Bucketing on the exact line meant two
|
|
239
|
+
// reviewers describing one defect at lines 9 and 14 landed in different
|
|
240
|
+
// buckets, so `sameIssue` was never consulted: the issue was verified twice,
|
|
241
|
+
// burning two verifier agents and emitting two inline comments for one
|
|
242
|
+
// problem, which is exactly what this pass exists to prevent.
|
|
166
243
|
const byLocation = new Map()
|
|
167
244
|
for (const f of allFindings) {
|
|
168
|
-
const key =
|
|
245
|
+
const key = String(f.file)
|
|
169
246
|
if (!byLocation.has(key)) {
|
|
170
247
|
byLocation.set(key, [])
|
|
171
248
|
}
|
|
@@ -176,7 +253,13 @@ function dedup(allFindings) {
|
|
|
176
253
|
for (const candidates of byLocation.values()) {
|
|
177
254
|
const groups = []
|
|
178
255
|
for (const candidate of candidates) {
|
|
179
|
-
|
|
256
|
+
// Compare against the whole group's SPAN, not just its first member.
|
|
257
|
+
// Checking only g[0] made grouping order-dependent: findings at 25, 50
|
|
258
|
+
// and 1 all joined when 25 arrived first, leaving a group spanning 49
|
|
259
|
+
// lines despite a 30-line window.
|
|
260
|
+
const match = groups.find(function(g) {
|
|
261
|
+
return sameIssue(g[0], candidate) && spanWithinWindow(g, candidate)
|
|
262
|
+
})
|
|
180
263
|
if (match) {
|
|
181
264
|
match.push(candidate)
|
|
182
265
|
} else {
|