dsh-vibe-math 2.3.11 → 2.3.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +17 -40
- package/{AUDIT-CHECKLIST.md → docs/AUDIT-CHECKLIST.md} +323 -304
- package/docs/COMPAT-AUDIT-ROUND2.md +325 -0
- package/docs/generate_framework_diagram_v2.py +114 -0
- package/docs/generate_framework_diagram_v3.py +127 -0
- package/{RELEASE-NOTES-2.1.0.md → docs/release-notes/RELEASE-NOTES-2.1.0.md} +143 -143
- package/{RELEASE-NOTES-2.2.0.md → docs/release-notes/RELEASE-NOTES-2.2.0.md} +266 -266
- package/{RELEASE-NOTES-2.2.1.md → docs/release-notes/RELEASE-NOTES-2.2.1.md} +43 -43
- package/{RELEASE-NOTES-2.2.2.md → docs/release-notes/RELEASE-NOTES-2.2.2.md} +88 -88
- package/{RELEASE-NOTES-2.3.0.md → docs/release-notes/RELEASE-NOTES-2.3.0.md} +207 -207
- package/{RELEASE-NOTES-2.3.1.md → docs/release-notes/RELEASE-NOTES-2.3.1.md} +134 -134
- package/{RELEASE-NOTES-2.3.10.md → docs/release-notes/RELEASE-NOTES-2.3.10.md} +105 -105
- package/{RELEASE-NOTES-2.3.11.md → docs/release-notes/RELEASE-NOTES-2.3.11.md} +57 -57
- package/docs/release-notes/RELEASE-NOTES-2.3.12.md +80 -0
- package/docs/release-notes/RELEASE-NOTES-2.3.13.md +137 -0
- package/{RELEASE-NOTES-2.3.2.md → docs/release-notes/RELEASE-NOTES-2.3.2.md} +145 -145
- package/{RELEASE-NOTES-2.3.3.md → docs/release-notes/RELEASE-NOTES-2.3.3.md} +115 -115
- package/{RELEASE-NOTES-2.3.4.md → docs/release-notes/RELEASE-NOTES-2.3.4.md} +69 -69
- package/{RELEASE-NOTES-2.3.5.md → docs/release-notes/RELEASE-NOTES-2.3.5.md} +63 -63
- package/{RELEASE-NOTES-2.3.6.md → docs/release-notes/RELEASE-NOTES-2.3.6.md} +66 -66
- package/{RELEASE-NOTES-2.3.7.md → docs/release-notes/RELEASE-NOTES-2.3.7.md} +59 -59
- package/{RELEASE-NOTES-2.3.8.md → docs/release-notes/RELEASE-NOTES-2.3.8.md} +45 -45
- package/{RELEASE-NOTES-2.3.9.md → docs/release-notes/RELEASE-NOTES-2.3.9.md} +70 -70
- package/docs/test-timing.md +18 -17
- package/installer.js +258 -53
- package/package.json +50 -40
- package/{audit-formal-sensitivity.mjs → tests/audit-formal-sensitivity.mjs} +342 -342
- package/tests/audit-installer-compat.test.mjs +136 -0
- package/tests/audit-installer-policy.test.mjs +245 -0
- package/{audit-persona-sensitivity.mjs → tests/audit-persona-sensitivity.mjs} +249 -249
- package/{audit-persona-surface.test.mjs → tests/audit-persona-surface.test.mjs} +349 -349
- package/{audit-prompt-invariants.mjs → tests/audit-prompt-invariants.mjs} +508 -508
- package/{audit-spec-traceability.mjs → tests/audit-spec-traceability.mjs} +193 -193
- package/{audit-v5-integrity.mjs → tests/audit-v5-integrity.mjs} +448 -448
- package/{audit-v5-sensitivity.mjs → tests/audit-v5-sensitivity.mjs} +384 -384
- package/{e2e-v5-round2.test.mjs → tests/e2e-v5-round2.test.mjs} +521 -521
- package/{formal-verify-v2.test.mjs → tests/formal-verify-v2.test.mjs} +1315 -1315
- package/{formal-verify-v3.test.mjs → tests/formal-verify-v3.test.mjs} +1257 -1257
- package/{formal-verify-v4.test.mjs → tests/formal-verify-v4.test.mjs} +1082 -1082
- package/{formal-verify-v5.test.mjs → tests/formal-verify-v5.test.mjs} +708 -708
- package/{prompt-v5-integrity.test.mjs → tests/prompt-v5-integrity.test.mjs} +3 -3
- package/{run-tests.mjs → tests/run-tests.mjs} +121 -118
- package/{selfdrive-v5.mjs → tests/selfdrive-v5.mjs} +470 -470
- package/vibe-math-v4//345/256/236/347/216/260/346/226/271/346/241/210.md +1 -1
- package/vibe-math-v5//345/256/236/347/216/260/346/226/271/346/241/210.md +1 -1
- package/vibe-math-v5//346/236/266/346/236/204/345/233/276.md +1 -1
- package//347/244/272/344/276/213/345/233/276//345/256/236/351/231/205/344/275/277/347/224/250/347/244/272/344/276/213-/351/225/277/346/210/252/345/233/276.png +0 -0
- package//347/244/272/344/276/213/345/233/276//346/241/206/346/236/266/345/233/276-v2.png +0 -0
- package//347/244/272/344/276/213/345/233/276//346/241/206/346/236/266/345/233/276-v3.png +0 -0
- /package/{RELEASE-NOTES-2.0.22.md → docs/release-notes/RELEASE-NOTES-2.0.22.md} +0 -0
|
@@ -1,508 +1,508 @@
|
|
|
1
|
-
#!/usr/bin/env node
|
|
2
|
-
/**
|
|
3
|
-
* PROMPT/INTERACTION INVARIANTS — the specific defect CLASSES this project has actually shipped,
|
|
4
|
-
* encoded as static invariants over all four presets. This is the mechanical answer to
|
|
5
|
-
* "are the prompt defects really fixed, and can they come back silently?".
|
|
6
|
-
*
|
|
7
|
-
* Every check below exists because the class was found in a real audit round:
|
|
8
|
-
* I1 abbreviated tool names in agent-facing text (lean_archive is not a registered tool)
|
|
9
|
-
* I2 a fidelity defect expressed as a 0 vote ("偏离 → 0" records 命题为假)
|
|
10
|
-
* I3 the defect rule missing from the injected text (不要投 0 + decision:'defect')
|
|
11
|
-
* I4 `defect` advertised but not handled by code (the v2 dead-channel class)
|
|
12
|
-
* I5 the reply contract not offering `defect` (a defect would be unrecordable)
|
|
13
|
-
* I6 a defect accepted without a reason (silent, unauditable decisions)
|
|
14
|
-
* I7 the wrong reply field name in the fidelity text (`verdict` where the parser reads `Result`)
|
|
15
|
-
* I8 the `formal` reply channel live in `off` mode (off must be a TRUE no-op)
|
|
16
|
-
* I9 corpus non-determinism / machine-path leaks / missing mode coverage
|
|
17
|
-
* I10 the prompt-rule sensitivity probes going missing (a guard that is not proven to go red)
|
|
18
|
-
* I11 the "no Lean toolchain" guidance naming only one of the two failure codes
|
|
19
|
-
* I12 the fidelity branch promising a hold that `encourage` cannot enforce
|
|
20
|
-
* I13 the Lean switch unreachable THROUGH the closed tool schema (v3 2.3.1: all four params
|
|
21
|
-
* missing from vibe_math_set_params, invisible to every suite because suites call handlers)
|
|
22
|
-
* I14 a parameter the tool schema advertises but the parameter layer silently drops
|
|
23
|
-
*
|
|
24
|
-
* Run: node audit-prompt-invariants.mjs (add --json for a machine-readable report)
|
|
25
|
-
* node audit-prompt-invariants.mjs --self-probe
|
|
26
|
-
* prove the guard is a guard: re-run itself on mutated sources and require the matching
|
|
27
|
-
* invariant to go RED (control run must stay green)
|
|
28
|
-
*/
|
|
29
|
-
import { readFileSync, existsSync, mkdtempSync, writeFileSync, rmSync } from 'node:fs'
|
|
30
|
-
import { spawnSync } from 'node:child_process'
|
|
31
|
-
import { tmpdir } from 'node:os'
|
|
32
|
-
import { fileURLToPath } from 'node:url'
|
|
33
|
-
import { join } from 'node:path'
|
|
34
|
-
|
|
35
|
-
const HERE = fileURLToPath(new URL('
|
|
36
|
-
/**
|
|
37
|
-
* `PROMPT_INVARIANTS_MUTATE` carries a JSON `[rel, from, to]` triple: a single file is mutated IN
|
|
38
|
-
* MEMORY for one child run, so `--self-probe` can demonstrate that the invariant keyed to it really
|
|
39
|
-
* turns red. (Not a NUL-separated string: env values may not contain NUL bytes.)
|
|
40
|
-
*/
|
|
41
|
-
function readRaw(rel) {
|
|
42
|
-
const p = join(HERE, rel)
|
|
43
|
-
if (!existsSync(p)) return null
|
|
44
|
-
let text = readFileSync(p, 'utf8')
|
|
45
|
-
const mut = process.env.PROMPT_INVARIANTS_MUTATE
|
|
46
|
-
if (mut) {
|
|
47
|
-
try {
|
|
48
|
-
const [rel2, from, to] = JSON.parse(mut)
|
|
49
|
-
if (rel2 === rel && from) text = text.replace(from, to)
|
|
50
|
-
} catch (e) { /* a malformed mutation is a harness error, not an invariant failure */ }
|
|
51
|
-
}
|
|
52
|
-
return text
|
|
53
|
-
}
|
|
54
|
-
const read = readRaw
|
|
55
|
-
|
|
56
|
-
/**
|
|
57
|
-
* Each mutation is a real defect shape from this project's history. `expect` is a substring that
|
|
58
|
-
* MUST appear in the failing run; `control: true` marks the unmutated run, which must stay green.
|
|
59
|
-
*/
|
|
60
|
-
const SELF_PROBE_MUTATIONS = [
|
|
61
|
-
{ name: 'control (no mutation)', rel: '', from: '', to: '', expect: '', control: true },
|
|
62
|
-
{
|
|
63
|
-
name: 'v3: one set_params registration loses the Lean params (I13 — the v2.3.1 shipped defect)',
|
|
64
|
-
rel: 'vibe-math-v3/vibe-math-v3.js',
|
|
65
|
-
from: 'formalVerify: { type: \'string\'',
|
|
66
|
-
to: 'formalVerifyDISABLED: { type: \'string\'',
|
|
67
|
-
expect: 'v3 I13: every vibe_math_set_params schema advertises',
|
|
68
|
-
},
|
|
69
|
-
{
|
|
70
|
-
name: 'v3: objParams stops closing the schema (I13 premise)',
|
|
71
|
-
rel: 'vibe-math-v3/vibe-math-v3.js',
|
|
72
|
-
from: "additionalProperties: false, required: required || [] }",
|
|
73
|
-
to: 'required: required || [] }',
|
|
74
|
-
expect: 'v3 I13: every objParams definition closes tool schemas',
|
|
75
|
-
},
|
|
76
|
-
{
|
|
77
|
-
name: 'v4: the schema advertises a knob the parameter layer drops (I14)',
|
|
78
|
-
rel: 'vibe-math-v4/vibe-math-v4.js',
|
|
79
|
-
from: "leanTimeoutMs:{type:'integer'}",
|
|
80
|
-
to: "leanTimeoutMs:{type:'integer'},bogusKnob:{type:'string'}",
|
|
81
|
-
expect: 'v4 I14: every key vibe_v4_set advertises is actually accepted',
|
|
82
|
-
},
|
|
83
|
-
{
|
|
84
|
-
name: 'v5: normalizeParams stops accepting an advertised key (I14)',
|
|
85
|
-
rel: 'vibe-math-v5/vibe-math-v5.js',
|
|
86
|
-
from: "'meetingKeepEvery', 'leanTimeoutMs']",
|
|
87
|
-
to: "'meetingKeepEvery']",
|
|
88
|
-
expect: 'advertised but dropped: [leanTimeoutMs]',
|
|
89
|
-
},
|
|
90
|
-
]
|
|
91
|
-
|
|
92
|
-
if (process.argv.includes('--self-probe')) {
|
|
93
|
-
const bad = []
|
|
94
|
-
for (const mut of SELF_PROBE_MUTATIONS) {
|
|
95
|
-
const r = spawnSync(process.execPath, [fileURLToPath(import.meta.url), '--json'], {
|
|
96
|
-
cwd: HERE,
|
|
97
|
-
env: Object.assign({}, process.env, mut.control ? {} : { PROMPT_INVARIANTS_MUTATE: JSON.stringify([mut.rel, mut.from, mut.to]) }),
|
|
98
|
-
encoding: 'utf8',
|
|
99
|
-
})
|
|
100
|
-
let parsed = null
|
|
101
|
-
try { parsed = JSON.parse(r.stdout) } catch (e) { /* fall through to the diagnostic below */ }
|
|
102
|
-
const text = (r.stdout || '') + (r.stderr || '')
|
|
103
|
-
if (mut.control) {
|
|
104
|
-
const ok = r.status === 0 && parsed && parsed.failed === 0
|
|
105
|
-
console.log((ok ? 'PASS ' : 'FAIL ') + 'control: the unmutated run stays green (' + (parsed ? parsed.failed + ' failures' : 'unparsable output') + ')')
|
|
106
|
-
if (!ok) bad.push('control run was not green')
|
|
107
|
-
continue
|
|
108
|
-
}
|
|
109
|
-
const hit = text.includes(mut.expect)
|
|
110
|
-
const red = r.status === 1 && parsed && parsed.failed > 0
|
|
111
|
-
const ok = red && hit
|
|
112
|
-
console.log((ok ? 'PASS ' : 'FAIL ') + mut.name + (ok ? '' : ' → exit=' + r.status + ' hit=' + hit))
|
|
113
|
-
if (!ok) bad.push(mut.name + ' (exit ' + r.status + ', expected ' + JSON.stringify(mut.expect) + ')')
|
|
114
|
-
}
|
|
115
|
-
console.log('')
|
|
116
|
-
console.log('PROMPT INVARIANT SELF-PROBE: ' + (SELF_PROBE_MUTATIONS.length - bad.length) + '/' + SELF_PROBE_MUTATIONS.length + ' as required')
|
|
117
|
-
for (const b of bad) console.error(' FAIL ' + b)
|
|
118
|
-
process.exit(bad.length === 0 ? 0 : 1)
|
|
119
|
-
}
|
|
120
|
-
|
|
121
|
-
/**
|
|
122
|
-
* Blank out comments (line + block) while preserving string/template REGEX literals, so invariants
|
|
123
|
-
* about "text shown to an agent" do not fire on a COMMENT that quotes an anti-pattern. A naive regex
|
|
124
|
-
* would both miss block comments and mangle strings containing `//` (URLs), so this walks the
|
|
125
|
-
* source as a tiny scanner. Newlines are preserved to keep any line-based diagnostics aligned.
|
|
126
|
-
*
|
|
127
|
-
* Regex literals matter: all four presets contain `/[\\/:*?"<>|\u0000-\u001f]+/` — a regex whose
|
|
128
|
-
* character class contains a DOUBLE QUOTE. Without regex handling the scanner treats that quote as
|
|
129
|
-
* the start of a string, keeps the state machine wrong for the rest of the file, and then either
|
|
130
|
-
* leaves a comment in the "code" stream (a spurious invariant failure) or blanks real code (a
|
|
131
|
-
* missed defect). X5/X6 below are self-checks for exactly that.
|
|
132
|
-
*/
|
|
133
|
-
function stripComments(src) {
|
|
134
|
-
const out = []
|
|
135
|
-
let i = 0
|
|
136
|
-
let state = 'code' // code | line | block | sq | dq | tpl | regex
|
|
137
|
-
let prev = '' // last significant code token (a single char, or a whole word such as `return`)
|
|
138
|
-
let word = '' // identifier/keyword accumulator, so `return /re/` is not read as division
|
|
139
|
-
// A `/` starts a REGEX LITERAL after an operator/keyword, and DIVISION after a value. Tracking only
|
|
140
|
-
// the previous CHARACTER is not enough: `return /^\s*import/.test(l)` (v3 really contains it) puts
|
|
141
|
-
// an identifier before the slash. Getting this wrong is exactly the bug X5–X8 guard against.
|
|
142
|
-
const REGEX_AFTER_KEYWORD = /^(?:return|typeof|case|delete|void|instanceof|in|of|yield|await|new|do|else)$/
|
|
143
|
-
while (i < src.length) {
|
|
144
|
-
const c = src[i]
|
|
145
|
-
const c2 = src[i + 1]
|
|
146
|
-
if (state === 'code') {
|
|
147
|
-
if (c === '/' && c2 === '/') { state = 'line'; out.push(' '); i += 2; continue }
|
|
148
|
-
if (c === '/' && c2 === '*') { state = 'block'; out.push(' '); i += 2; continue }
|
|
149
|
-
if (c === '/' && (prev === '' || REGEX_AFTER_KEYWORD.test(prev) || /[(,=:[!&|?{};+\-*%~^<>]/.test(prev))) {
|
|
150
|
-
state = 'regex'; out.push(c); i++; continue
|
|
151
|
-
}
|
|
152
|
-
if (c === "'") state = 'sq'
|
|
153
|
-
else if (c === '"') state = 'dq'
|
|
154
|
-
else if (c === '`') state = 'tpl'
|
|
155
|
-
if (/[A-Za-z0-9_$]/.test(c)) word += c
|
|
156
|
-
else { if (word) { prev = word; word = '' } if (!/\s/.test(c)) prev = c }
|
|
157
|
-
out.push(c); i++; continue
|
|
158
|
-
}
|
|
159
|
-
if (state === 'line') {
|
|
160
|
-
if (c === '\n') { state = 'code'; out.push(c) } else out.push(' ')
|
|
161
|
-
i++; continue
|
|
162
|
-
}
|
|
163
|
-
if (state === 'block') {
|
|
164
|
-
if (c === '*' && c2 === '/') { state = 'code'; out.push(' '); i += 2; continue }
|
|
165
|
-
out.push(c === '\n' ? c : ' '); i++; continue
|
|
166
|
-
}
|
|
167
|
-
if (state === 'regex') {
|
|
168
|
-
if (c === '\\') { out.push(c, c2 === undefined ? '' : c2); i += 2; continue }
|
|
169
|
-
if (c === '[') { state = 'regexClass'; out.push(c); i++; continue }
|
|
170
|
-
if (c === '/') { // end of the literal: copy it plus any flags
|
|
171
|
-
out.push(c); i++
|
|
172
|
-
while (i < src.length && /[a-z]/i.test(src[i])) { out.push(src[i]); i++ }
|
|
173
|
-
state = 'code'; prev = ')'
|
|
174
|
-
continue
|
|
175
|
-
}
|
|
176
|
-
out.push(c); i++; continue
|
|
177
|
-
}
|
|
178
|
-
if (state === 'regexClass') {
|
|
179
|
-
if (c === '\\') { out.push(c, c2 === undefined ? '' : c2); i += 2; continue }
|
|
180
|
-
if (c === ']') state = 'regex'
|
|
181
|
-
out.push(c); i++; continue
|
|
182
|
-
}
|
|
183
|
-
// inside a string/template: copy verbatim, honouring escapes and the closing quote
|
|
184
|
-
if (c === '\\') { out.push(c, c2 === undefined ? '' : c2); i += 2; continue }
|
|
185
|
-
if ((state === 'sq' && c === "'") || (state === 'dq' && c === '"') || (state === 'tpl' && c === '`')) state = 'code'
|
|
186
|
-
out.push(c); i++
|
|
187
|
-
}
|
|
188
|
-
return out.join('')
|
|
189
|
-
}
|
|
190
|
-
|
|
191
|
-
/**
|
|
192
|
-
* Balanced-bracket slice starting at src[openIdx] (one of ( [ {), string/comment aware.
|
|
193
|
-
* Used by the TOOL-SURFACE invariants (I13/I14) to read a real parameter schema out of the source.
|
|
194
|
-
*/
|
|
195
|
-
function balanced(src, openIdx) {
|
|
196
|
-
let depth = 0
|
|
197
|
-
let i = openIdx
|
|
198
|
-
let state = 'code'
|
|
199
|
-
while (i < src.length) {
|
|
200
|
-
const c = src[i]
|
|
201
|
-
const c2 = src[i + 1]
|
|
202
|
-
if (state === 'code') {
|
|
203
|
-
if (c === "'" || c === '"' || c === '`') { state = c; i++; continue }
|
|
204
|
-
if (c === '(' || c === '[' || c === '{') depth++
|
|
205
|
-
else if (c === ')' || c === ']' || c === '}') { depth--; if (depth === 0) return src.slice(openIdx, i + 1) }
|
|
206
|
-
i++; continue
|
|
207
|
-
}
|
|
208
|
-
if (c === '\\') { i += 2; continue }
|
|
209
|
-
if (c === state) state = 'code'
|
|
210
|
-
i++
|
|
211
|
-
}
|
|
212
|
-
return src.slice(openIdx)
|
|
213
|
-
}
|
|
214
|
-
/** Split an object-body on TOP-LEVEL commas only (brackets and strings respected). */
|
|
215
|
-
function splitTopLevel(body) {
|
|
216
|
-
const parts = []
|
|
217
|
-
let cur = ''
|
|
218
|
-
let depth = 0
|
|
219
|
-
let state = 'code'
|
|
220
|
-
for (let i = 0; i < body.length; i++) {
|
|
221
|
-
const c = body[i]
|
|
222
|
-
const c2 = body[i + 1]
|
|
223
|
-
if (state === 'code') {
|
|
224
|
-
if (c === "'" || c === '"' || c === '`') { state = c; cur += c; continue }
|
|
225
|
-
if (c === '{' || c === '[' || c === '(') depth++
|
|
226
|
-
else if (c === '}' || c === ']' || c === ')') depth--
|
|
227
|
-
if (c === ',' && depth === 0) { parts.push(cur); cur = ''; continue }
|
|
228
|
-
cur += c; continue
|
|
229
|
-
}
|
|
230
|
-
if (c === '\\') { cur += c + (c2 || ''); i++; continue }
|
|
231
|
-
if (c === state) state = 'code'
|
|
232
|
-
cur += c
|
|
233
|
-
}
|
|
234
|
-
parts.push(cur)
|
|
235
|
-
return parts.map((p) => p.trim()).filter(Boolean)
|
|
236
|
-
}
|
|
237
|
-
/** Property keys of the object literal at/after `at` (null when there is no object literal there). */
|
|
238
|
-
function objectKeys(src, at) {
|
|
239
|
-
const open = src.indexOf('{', at)
|
|
240
|
-
if (open < 0) return null
|
|
241
|
-
const region = balanced(src, open)
|
|
242
|
-
return splitTopLevel(region.slice(1, -1))
|
|
243
|
-
.map((p) => { const m = p.match(/^['"]?([A-Za-z_$][\w$]*)['"]?\s*:/); return m ? m[1] : null })
|
|
244
|
-
.filter(Boolean)
|
|
245
|
-
}
|
|
246
|
-
|
|
247
|
-
const PRESETS = [
|
|
248
|
-
{ tag: 'v2', js: 'vibe-math-v2/vibe-math-v2.js', suite: 'formal-verify-v2.test.mjs', corpus: 'prompt-corpus-v2/formal-verify-v2.md', valueField: 'Result', prefix: 'vibe_math_', setTool: 'vibe_math_set_params' },
|
|
249
|
-
{ tag: 'v3', js: 'vibe-math-v3/vibe-math-v3.js', suite: 'formal-verify-v3.test.mjs', corpus: 'prompt-corpus-v3/formal-verify-v3.md', valueField: 'Result', prefix: 'vibe_math_', setTool: 'vibe_math_set_params' },
|
|
250
|
-
{ tag: 'v4', js: 'vibe-math-v4/vibe-math-v4.js', suite: 'formal-verify-v4.test.mjs', corpus: 'prompt-corpus-v4/formal-verify-v4.md', valueField: 'verdict', prefix: 'vibe_v4_', setTool: 'vibe_v4_set' },
|
|
251
|
-
{ tag: 'v5', js: 'vibe-math-v5/vibe-math-v5.js', suite: 'formal-verify-v5.test.mjs', corpus: 'prompt-corpus-v5/prompt-corpus-v5.md', valueField: 'verdict', prefix: 'vibe_v5_', setTool: 'vibe_v5_set' },
|
|
252
|
-
]
|
|
253
|
-
const LEAN_PARAMS = ['formalVerify', 'leanCommand', 'leanArgs', 'leanTimeoutMs']
|
|
254
|
-
|
|
255
|
-
let passed = 0
|
|
256
|
-
const failures = []
|
|
257
|
-
const notes = []
|
|
258
|
-
function check(cond, label, detail) {
|
|
259
|
-
if (cond) { passed++; return true }
|
|
260
|
-
failures.push(label + (detail ? ' — ' + detail : ''))
|
|
261
|
-
return false
|
|
262
|
-
}
|
|
263
|
-
|
|
264
|
-
for (const P of PRESETS) {
|
|
265
|
-
const js = read(P.js)
|
|
266
|
-
const suite = read(P.suite)
|
|
267
|
-
const corpus = read(P.corpus)
|
|
268
|
-
if (!check(js !== null, P.tag + ': plugin source readable', P.js)) continue
|
|
269
|
-
if (!check(suite !== null, P.tag + ': suite readable', P.suite)) continue
|
|
270
|
-
if (!check(corpus !== null, P.tag + ': prompt corpus shipped', P.corpus)) continue
|
|
271
|
-
|
|
272
|
-
// I1/I2 run on the source with comments blanked: a comment may legitimately quote an
|
|
273
|
-
// anti-pattern as documentation, but any STRING can reach an agent.
|
|
274
|
-
const code = stripComments(js)
|
|
275
|
-
|
|
276
|
-
// I1 — no abbreviated tool name in code/strings (code identifiers are leanArchive / leanRunTool,
|
|
277
|
-
// so a bare lean_* token is always a string that can be shown to an agent).
|
|
278
|
-
const bare = code.match(/(^|[^A-Za-z0-9_])lean_(run|archive|lib)\b/g) || []
|
|
279
|
-
check(bare.length === 0, P.tag + ' I1: no abbreviated Lean tool name in agent-facing text', 'found ' + JSON.stringify(bare.slice(0, 3)))
|
|
280
|
-
|
|
281
|
-
// I2 — never tell a voter to answer 0 for a faithfulness defect (comments excluded).
|
|
282
|
-
check(!/偏离\s*(?:→|->|=>)\s*0/.test(code), P.tag + ' I2: no "偏离 → 0" instruction')
|
|
283
|
-
|
|
284
|
-
// I3 — the defect rule is in the injected text.
|
|
285
|
-
check(js.includes('不要投 0'), P.tag + ' I3: injected text forbids a 0 vote on a defect')
|
|
286
|
-
check(/'defect'/.test(js) || /"defect"/.test(js), P.tag + ' I3: injected text names decision=\'defect\'')
|
|
287
|
-
|
|
288
|
-
// I4 — the defect decision is actually HANDLED (comparison + a downgrade to `attempted`).
|
|
289
|
-
const comparesDefect = /(?:===|==)\s*'defect'/.test(js) || /'defect'\s*(?:===|==)/.test(js)
|
|
290
|
-
check(comparesDefect, P.tag + ' I4: the code compares decision against \'defect\'')
|
|
291
|
-
const downgrades = /status:\s*'attempted'/.test(js) || /status='attempted'/.test(js) || /status:\s*"attempted"/.test(js)
|
|
292
|
-
check(downgrades, P.tag + " I4: a defect downgrades the record to 'attempted'")
|
|
293
|
-
|
|
294
|
-
// I5 — the reply contract offers the defect decision.
|
|
295
|
-
const contractCount = js.split('"decision":"used|blocked|defect"').length - 1
|
|
296
|
-
check(contractCount >= 1, P.tag + ' I5: the reply contract offers used|blocked|defect', 'occurrences=' + contractCount)
|
|
297
|
-
|
|
298
|
-
// I6 — a defect without a reason is refused.
|
|
299
|
-
check(/!note/.test(js), P.tag + ' I6: a defect/blocker without a note is rejected')
|
|
300
|
-
|
|
301
|
-
// I7 — the fidelity instruction names the field the parser really reads.
|
|
302
|
-
{
|
|
303
|
-
const lines = js.split(/\r?\n/)
|
|
304
|
-
const i = lines.findIndex((l) => l.includes('不要投 0'))
|
|
305
|
-
const window = i >= 0 ? lines.slice(Math.max(0, i - 3), i + 8).join('\n') : ''
|
|
306
|
-
check(i >= 0 && window.includes(P.valueField),
|
|
307
|
-
P.tag + ' I7: the fidelity instruction names ' + P.valueField + ' (the real reply field)',
|
|
308
|
-
i < 0 ? 'the 不要投 0 rule was not found' : 'window: ' + window.slice(0, 120).replace(/\n/g, ' | '))
|
|
309
|
-
if (P.tag === 'v2' || P.tag === 'v3') {
|
|
310
|
-
check(!/给出\s*verdict/.test(js), P.tag + ' I7b: no "给出 verdict" in a Result-based preset')
|
|
311
|
-
}
|
|
312
|
-
}
|
|
313
|
-
|
|
314
|
-
// I8 — the `formal` reply channel is inert in off mode (tools stay usable on purpose).
|
|
315
|
-
const offGuards = {
|
|
316
|
-
v2: /absorbFormal(?:From)?Reply[\s\S]{0,900}!formalOn\(\)/,
|
|
317
|
-
v3: /absorbFormal(?:From)?Reply[\s\S]{0,900}!formalOn\(\)/,
|
|
318
|
-
v4: /applyFormalReply[\s\S]{0,900}!formalOn\(\)/,
|
|
319
|
-
v5: /formalOn\(\)\s*&&\s*p\.formal/,
|
|
320
|
-
}
|
|
321
|
-
check(offGuards[P.tag].test(js), P.tag + ' I8: the reply channel is gated on formalOn() (off stays a no-op)')
|
|
322
|
-
|
|
323
|
-
// I9 — the corpus covers every mode and is deterministic / machine-path free.
|
|
324
|
-
check(corpus.includes('【Lean 形式化验证(鼓励模式)】'), P.tag + ' I9: corpus covers encourage mode')
|
|
325
|
-
check(corpus.includes('【Lean 形式化验证(强制模式)】'), P.tag + ' I9: corpus covers REQUIRE mode')
|
|
326
|
-
check(corpus.includes('不要投 0'), P.tag + ' I9: corpus contains the fidelity/defect rule')
|
|
327
|
-
check(/used\|blocked\|defect/.test(corpus), P.tag + ' I9: corpus contains the reply contract line')
|
|
328
|
-
check(/【顺手形式化/.test(corpus), P.tag + ' I9: corpus contains the work-round line')
|
|
329
|
-
check(!/[A-Za-z]:[\\/]/.test(corpus), P.tag + ' I9: corpus leaks no absolute path')
|
|
330
|
-
check(!/\d{4}-\d{2}-\d{2}[ T]\d{2}:\d{2}/.test(corpus), P.tag + ' I9: corpus carries no timestamp (deterministic)')
|
|
331
|
-
check(!/vibe-v[0-9]-[a-z]+-[A-Za-z0-9]{4,}/.test(corpus), P.tag + ' I9: corpus leaks no temp-dir name')
|
|
332
|
-
|
|
333
|
-
// I10 — the prompt-rule probes exist for this preset (a guard nobody can prove is a guard nobody has).
|
|
334
|
-
const probeSrc = read('audit-formal-sensitivity.mjs') || ''
|
|
335
|
-
for (const kind of ['fidelity-defect-rule-removed', 'abbreviated-tool-name-injected', 'require-wording-removed', 'defect-decision-not-offered']) {
|
|
336
|
-
check(probeSrc.includes("tag + '-" + kind + "'") || probeSrc.includes("'" + P.tag + '-' + kind + "'"),
|
|
337
|
-
P.tag + ' I10: probe exists for ' + kind)
|
|
338
|
-
}
|
|
339
|
-
|
|
340
|
-
// I10b — the suite asserts the defect path (behaviourally, not just wording).
|
|
341
|
-
check(/\bdefect\b/.test(suite), P.tag + ' I10b: the suite exercises the defect path')
|
|
342
|
-
|
|
343
|
-
// I11 — the "no toolchain" guidance must name BOTH failure codes. An agent that only knows
|
|
344
|
-
// LEAN_NOT_FOUND treats NO_SUBPROCESS as an unknown failure and retries instead of recording the
|
|
345
|
-
// blocker (contract §6 hard rule 4).
|
|
346
|
-
{
|
|
347
|
-
const lines = js.split(/\r?\n/)
|
|
348
|
-
const gi = lines.findIndex((l) => l.includes('宿主无 Lean 工具链'))
|
|
349
|
-
const win = gi >= 0 ? lines.slice(Math.max(0, gi - 3), gi + 1).join('\n') : ''
|
|
350
|
-
check(gi >= 0 && /LEAN_NOT_FOUND/.test(win) && /NO_SUBPROCESS/.test(win),
|
|
351
|
-
P.tag + ' I11: the no-toolchain guidance names LEAN_NOT_FOUND AND NO_SUBPROCESS',
|
|
352
|
-
gi < 0 ? 'the guidance line was not found' : 'window: ' + win.slice(0, 140).replace(/\n/g, ' | '))
|
|
353
|
-
}
|
|
354
|
-
|
|
355
|
-
// I12 — the fidelity branch must qualify its promise BY MODE. Only `require` has a gate, so the
|
|
356
|
-
// text must say "本次裁定不定论" for require AND explicitly tell the voter, for encourage, that
|
|
357
|
-
// this mode has no gate and their abstention is what prevents a conclusion. Shipping the
|
|
358
|
-
// unconditional claim was a real defect that survived in three presets after v5 was fixed.
|
|
359
|
-
check(js.includes('本档没有门禁'), P.tag + ' I12: the fidelity text states the encourage branch has no gate')
|
|
360
|
-
check(js.includes('本次裁定**不定论**') || js.includes('本次裁定不定论'), P.tag + ' I12: the fidelity text states the require hold')
|
|
361
|
-
|
|
362
|
-
// I13 — the Lean switch must be REACHABLE THROUGH THE TOOL SCHEMA. Every tool schema here is
|
|
363
|
-
// built by `objParams`, which closes it with `additionalProperties:false`: a key the schema does
|
|
364
|
-
// not list is REJECTED by any schema-validating provider. v3 shipped 2.3.0/2.3.1 with all four
|
|
365
|
-
// Lean params missing from `vibe_math_set_params` and every suite stayed green, because the
|
|
366
|
-
// suites call the handler directly and never look at the schema — the feature could not be turned
|
|
367
|
-
// on at all. So: EVERY registration of the parameter tool (v2/v3 register it twice, for two agent
|
|
368
|
-
// scopes) must advertise all four, and the closed-schema premise must still hold.
|
|
369
|
-
{
|
|
370
|
-
// EVERY objParams definition must close its schema: one open definition (v2/v3 define it twice,
|
|
371
|
-
// for two agent scopes) would let tools registered through it accept arbitrary keys.
|
|
372
|
-
const defs = [...code.matchAll(/function objParams\(/g)].map((m) => m.index)
|
|
373
|
-
const closedDefs = defs.filter((i) => /additionalProperties:\s*false/.test(code.slice(i, i + 220)))
|
|
374
|
-
check(defs.length > 0 && closedDefs.length === defs.length,
|
|
375
|
-
P.tag + ' I13: every objParams definition closes tool schemas (additionalProperties:false)',
|
|
376
|
-
'definitions=' + defs.length + ' closed=' + closedDefs.length +
|
|
377
|
-
(defs.length ? '; if this is gone the schema no longer rejects unlisted keys and I13 loses its premise' : ''))
|
|
378
|
-
const regRe = new RegExp("registerTool\\(\\s*['\"]" + P.setTool + "['\"]", 'g')
|
|
379
|
-
const schemas = []
|
|
380
|
-
let m
|
|
381
|
-
while ((m = regRe.exec(code))) {
|
|
382
|
-
const rest = code.slice(m.index)
|
|
383
|
-
const oi = rest.indexOf('objParams(')
|
|
384
|
-
schemas.push(oi < 0 ? null : objectKeys(rest, oi + 'objParams'.length))
|
|
385
|
-
}
|
|
386
|
-
check(schemas.length >= 1, P.tag + ' I13: the parameter tool ' + P.setTool + ' is registered',
|
|
387
|
-
'no registerTool(\'' + P.setTool + '\') call found')
|
|
388
|
-
const bad = schemas.map((s, i) => (s === null ? 'reg#' + i + ': no objParams schema'
|
|
389
|
-
: LEAN_PARAMS.filter((k) => !s.includes(k)))).filter((x) => (Array.isArray(x) ? x.length : true))
|
|
390
|
-
check(schemas.length >= 1 && bad.length === 0,
|
|
391
|
-
P.tag + ' I13: every ' + P.setTool + ' schema advertises ' + LEAN_PARAMS.join('/'),
|
|
392
|
-
bad.length ? JSON.stringify(bad) : 'no registration found')
|
|
393
|
-
|
|
394
|
-
// I14 — a key the schema ACCEPTS must really be accepted by the parameter layer. Otherwise the
|
|
395
|
-
// tool advertises a knob that is silently dropped: the caller sees {ok:true} and nothing changes.
|
|
396
|
-
// The accept gate is DEFAULT_PARAMS for v2/v3/v4 (`if (k in params)`) and the typed lists inside
|
|
397
|
-
// normalizeParams for v5 (only those keys are copied through).
|
|
398
|
-
let accepts = null
|
|
399
|
-
if (P.tag === 'v5') {
|
|
400
|
-
const fn = code.search(/function\s+normalizeParams\s*\(/)
|
|
401
|
-
const region = fn < 0 ? '' : balanced(code, code.indexOf('{', fn))
|
|
402
|
-
const names = []
|
|
403
|
-
for (const kind of ['ints', 'bools', 'strs', 'arrs']) {
|
|
404
|
-
const ai = region.search(new RegExp('const\\s+' + kind + '\\s*=\\s*\\['))
|
|
405
|
-
if (ai < 0) continue
|
|
406
|
-
const arr = balanced(region, region.indexOf('[', ai))
|
|
407
|
-
for (const lit of arr.match(/'[A-Za-z_$][\w$]*'/g) || []) names.push(lit.slice(1, -1))
|
|
408
|
-
}
|
|
409
|
-
accepts = names.length ? [...new Set(names)] : null
|
|
410
|
-
} else {
|
|
411
|
-
const di = code.search(/DEFAULT_PARAMS\s*=\s*\{/)
|
|
412
|
-
accepts = di < 0 ? null : objectKeys(code, di)
|
|
413
|
-
}
|
|
414
|
-
check(accepts !== null && accepts.length > 0, P.tag + ' I14: the parameter accept-set is readable',
|
|
415
|
-
'extractor found no parameter set — the invariant cannot be checked')
|
|
416
|
-
const union = [...new Set(schemas.filter(Boolean).flat())]
|
|
417
|
-
const dropped = accepts ? union.filter((k) => !accepts.includes(k)) : []
|
|
418
|
-
check(accepts !== null && dropped.length === 0,
|
|
419
|
-
P.tag + ' I14: every key ' + P.setTool + ' advertises is actually accepted (no silently-dropped knob)',
|
|
420
|
-
'advertised but dropped: [' + dropped.join(',') + ']')
|
|
421
|
-
check(union.length >= LEAN_PARAMS.length, P.tag + ' I14: the parameter schema was parsed (' + union.length + ' keys)',
|
|
422
|
-
'schema extraction returned ' + union.length + ' keys')
|
|
423
|
-
}
|
|
424
|
-
|
|
425
|
-
notes.push(P.tag + ': plugin ' + js.length + 'B · suite ' + suite.length + 'B · corpus ' + corpus.length + 'B')
|
|
426
|
-
}
|
|
427
|
-
|
|
428
|
-
// Cross-preset: the probe script must refuse to report success on an empty selection (false green).
|
|
429
|
-
{
|
|
430
|
-
const probeSrc = read('audit-formal-sensitivity.mjs') || ''
|
|
431
|
-
check(/selected\.length === 0/.test(probeSrc), 'X1: the probe runner fails on an empty selection instead of reporting success')
|
|
432
|
-
const runner = read('run-tests.mjs') || ''
|
|
433
|
-
check(/no suites matched/.test(runner), 'X2: the suite runner fails when no suite matches')
|
|
434
|
-
check(/argv\[i \+ 1\]/.test(runner) && /argv\[\+\+i\]/.test(runner), 'X3: the suite runner accepts both --flag=x and --flag x')
|
|
435
|
-
}
|
|
436
|
-
|
|
437
|
-
// X5–X7: guard the guard. I1/I13/I14 trust stripComments(), and all four presets contain a regex
|
|
438
|
-
// literal whose character class holds a double quote (`/[\\/:*?"<>|…]+/`). If the scanner mistook
|
|
439
|
-
// that quote for a string start, the rest of the file would be misread — a comment quoting an
|
|
440
|
-
// anti-pattern would look like code (spurious failure) or real code would be blanked (missed bug).
|
|
441
|
-
{
|
|
442
|
-
const fixtureRegex = 'const t = s.replace(/[\\\\/:*?"<>|]+/g, "-")\n// lean_run is NOT a tool name\nconst b = 1'
|
|
443
|
-
check(!stripComments(fixtureRegex).includes('lean_run'),
|
|
444
|
-
'X5: the comment scanner survives a regex literal containing a quote (a following comment stays blanked)')
|
|
445
|
-
const fixtureUrl = "L.push('see https://example.com/a')\nconst c = 2"
|
|
446
|
-
check(stripComments(fixtureUrl).includes('https://example.com/a'),
|
|
447
|
-
'X6: a // inside a STRING is not treated as a comment start (URLs must survive)')
|
|
448
|
-
const fixtureBlock = "const d = /a\\/b/g\n/* lean_lib quoted in a block comment */\nconst e = 3"
|
|
449
|
-
check(!stripComments(fixtureBlock).includes('lean_lib'),
|
|
450
|
-
'X7: an escaped slash inside a regex does not end it early (a block comment after it stays blanked)')
|
|
451
|
-
for (const [label, src] of [['X5', fixtureRegex], ['X6', fixtureUrl], ['X7', fixtureBlock]]) {
|
|
452
|
-
check(stripComments(src).split('\n').length === src.split('\n').length,
|
|
453
|
-
label + ': comment blanking preserves line structure (diagnostics stay aligned)')
|
|
454
|
-
}
|
|
455
|
-
}
|
|
456
|
-
|
|
457
|
-
// X8 — the strongest scanner oracle there is: the comment-stripped source of EVERY preset must still
|
|
458
|
-
// PARSE. A scanner that mis-lexes (regex read as division, or a quote inside a regex read as a string
|
|
459
|
-
// start) corrupts real code, and `node --check` sees it. Sensitivity is MEASURED, not assumed:
|
|
460
|
-
// · the 2.3.2 scanner (no regex support at all) → SyntaxError on all four presets → X8 red;
|
|
461
|
-
// · the 2.3.3 scanner (regex-aware but no keyword rule) → X8 still green, which is exactly why
|
|
462
|
-
// X8b below exists (it covers that narrower gap).
|
|
463
|
-
{
|
|
464
|
-
const dir = mkdtempSync(join(tmpdir(), 'prompt-invariants-strip-'))
|
|
465
|
-
try {
|
|
466
|
-
for (const P of PRESETS) {
|
|
467
|
-
const src = read(P.js)
|
|
468
|
-
if (!src) continue
|
|
469
|
-
const f = join(dir, P.tag + '.mjs')
|
|
470
|
-
writeFileSync(f, stripComments(src))
|
|
471
|
-
const r = spawnSync(process.execPath, ['--check', f], { encoding: 'utf8' })
|
|
472
|
-
check(r.status === 0,
|
|
473
|
-
'X8: the comment-stripped source of ' + P.tag + ' is still valid JS (the scanner must not corrupt code)',
|
|
474
|
-
String(r.stderr || '').split('\n').filter((l) => l.trim()).slice(-2).join(' ').slice(0, 160))
|
|
475
|
-
}
|
|
476
|
-
// X8b: a fixture that needs the KEYWORD rule — a regex AFTER `return` whose character class holds
|
|
477
|
-
// a quote (v3 really has `return /^\s*…/.test()`, so the rule is not hypothetical). A scanner that
|
|
478
|
-
// reads that `/` as division enters "string" state at the quote and then leaves the NEXT comment
|
|
479
|
-
// unblanked. Sensitivity MEASURED: the 2.3.3 scanner fails the second assertion below
|
|
480
|
-
// (comment-blanked=false) while passing the first; the current scanner passes both.
|
|
481
|
-
const fixtureKeyword = [
|
|
482
|
-
'function f(l) { return /["\']/.test(l) }',
|
|
483
|
-
'// lean_run must stay blanked',
|
|
484
|
-
'const h = 4 / 2',
|
|
485
|
-
].join('\n')
|
|
486
|
-
const f2 = join(dir, 'fixture.mjs')
|
|
487
|
-
const stripped = stripComments(fixtureKeyword)
|
|
488
|
-
writeFileSync(f2, stripped)
|
|
489
|
-
const r2 = spawnSync(process.execPath, ['--check', f2], { encoding: 'utf8' })
|
|
490
|
-
check(r2.status === 0, 'X8b: a regex after a KEYWORD (return /…/) does not corrupt the scan', String(r2.stderr || '').slice(0, 120))
|
|
491
|
-
check(!stripped.includes('lean_run'), 'X8b: and the comment after it is still blanked (the keyword rule is what decides this)')
|
|
492
|
-
} finally {
|
|
493
|
-
rmSync(dir, { recursive: true, force: true })
|
|
494
|
-
}
|
|
495
|
-
}
|
|
496
|
-
|
|
497
|
-
const out = { passed, failed: failures.length, failures, notes }
|
|
498
|
-
if (process.argv.includes('--json')) {
|
|
499
|
-
console.log(JSON.stringify(out, null, 2))
|
|
500
|
-
} else {
|
|
501
|
-
console.log('-- prompt/interaction invariants (all four presets) --')
|
|
502
|
-
for (const n of notes) console.log(' note ' + n)
|
|
503
|
-
console.log('')
|
|
504
|
-
for (const f of failures) console.error(' FAIL ' + f)
|
|
505
|
-
console.log('')
|
|
506
|
-
console.log('PROMPT INVARIANTS: ' + passed + ' passed, ' + failures.length + ' failed')
|
|
507
|
-
}
|
|
508
|
-
process.exit(failures.length === 0 ? 0 : 1)
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
/**
|
|
3
|
+
* PROMPT/INTERACTION INVARIANTS — the specific defect CLASSES this project has actually shipped,
|
|
4
|
+
* encoded as static invariants over all four presets. This is the mechanical answer to
|
|
5
|
+
* "are the prompt defects really fixed, and can they come back silently?".
|
|
6
|
+
*
|
|
7
|
+
* Every check below exists because the class was found in a real audit round:
|
|
8
|
+
* I1 abbreviated tool names in agent-facing text (lean_archive is not a registered tool)
|
|
9
|
+
* I2 a fidelity defect expressed as a 0 vote ("偏离 → 0" records 命题为假)
|
|
10
|
+
* I3 the defect rule missing from the injected text (不要投 0 + decision:'defect')
|
|
11
|
+
* I4 `defect` advertised but not handled by code (the v2 dead-channel class)
|
|
12
|
+
* I5 the reply contract not offering `defect` (a defect would be unrecordable)
|
|
13
|
+
* I6 a defect accepted without a reason (silent, unauditable decisions)
|
|
14
|
+
* I7 the wrong reply field name in the fidelity text (`verdict` where the parser reads `Result`)
|
|
15
|
+
* I8 the `formal` reply channel live in `off` mode (off must be a TRUE no-op)
|
|
16
|
+
* I9 corpus non-determinism / machine-path leaks / missing mode coverage
|
|
17
|
+
* I10 the prompt-rule sensitivity probes going missing (a guard that is not proven to go red)
|
|
18
|
+
* I11 the "no Lean toolchain" guidance naming only one of the two failure codes
|
|
19
|
+
* I12 the fidelity branch promising a hold that `encourage` cannot enforce
|
|
20
|
+
* I13 the Lean switch unreachable THROUGH the closed tool schema (v3 2.3.1: all four params
|
|
21
|
+
* missing from vibe_math_set_params, invisible to every suite because suites call handlers)
|
|
22
|
+
* I14 a parameter the tool schema advertises but the parameter layer silently drops
|
|
23
|
+
*
|
|
24
|
+
* Run: node tests/audit-prompt-invariants.mjs (add --json for a machine-readable report)
|
|
25
|
+
* node tests/audit-prompt-invariants.mjs --self-probe
|
|
26
|
+
* prove the guard is a guard: re-run itself on mutated sources and require the matching
|
|
27
|
+
* invariant to go RED (control run must stay green)
|
|
28
|
+
*/
|
|
29
|
+
import { readFileSync, existsSync, mkdtempSync, writeFileSync, rmSync } from 'node:fs'
|
|
30
|
+
import { spawnSync } from 'node:child_process'
|
|
31
|
+
import { tmpdir } from 'node:os'
|
|
32
|
+
import { fileURLToPath } from 'node:url'
|
|
33
|
+
import { join } from 'node:path'
|
|
34
|
+
|
|
35
|
+
const HERE = fileURLToPath(new URL('../', import.meta.url))
|
|
36
|
+
/**
|
|
37
|
+
* `PROMPT_INVARIANTS_MUTATE` carries a JSON `[rel, from, to]` triple: a single file is mutated IN
|
|
38
|
+
* MEMORY for one child run, so `--self-probe` can demonstrate that the invariant keyed to it really
|
|
39
|
+
* turns red. (Not a NUL-separated string: env values may not contain NUL bytes.)
|
|
40
|
+
*/
|
|
41
|
+
function readRaw(rel) {
|
|
42
|
+
const p = join(HERE, rel)
|
|
43
|
+
if (!existsSync(p)) return null
|
|
44
|
+
let text = readFileSync(p, 'utf8')
|
|
45
|
+
const mut = process.env.PROMPT_INVARIANTS_MUTATE
|
|
46
|
+
if (mut) {
|
|
47
|
+
try {
|
|
48
|
+
const [rel2, from, to] = JSON.parse(mut)
|
|
49
|
+
if (rel2 === rel && from) text = text.replace(from, to)
|
|
50
|
+
} catch (e) { /* a malformed mutation is a harness error, not an invariant failure */ }
|
|
51
|
+
}
|
|
52
|
+
return text
|
|
53
|
+
}
|
|
54
|
+
const read = readRaw
|
|
55
|
+
|
|
56
|
+
/**
|
|
57
|
+
* Each mutation is a real defect shape from this project's history. `expect` is a substring that
|
|
58
|
+
* MUST appear in the failing run; `control: true` marks the unmutated run, which must stay green.
|
|
59
|
+
*/
|
|
60
|
+
const SELF_PROBE_MUTATIONS = [
|
|
61
|
+
{ name: 'control (no mutation)', rel: '', from: '', to: '', expect: '', control: true },
|
|
62
|
+
{
|
|
63
|
+
name: 'v3: one set_params registration loses the Lean params (I13 — the v2.3.1 shipped defect)',
|
|
64
|
+
rel: 'vibe-math-v3/vibe-math-v3.js',
|
|
65
|
+
from: 'formalVerify: { type: \'string\'',
|
|
66
|
+
to: 'formalVerifyDISABLED: { type: \'string\'',
|
|
67
|
+
expect: 'v3 I13: every vibe_math_set_params schema advertises',
|
|
68
|
+
},
|
|
69
|
+
{
|
|
70
|
+
name: 'v3: objParams stops closing the schema (I13 premise)',
|
|
71
|
+
rel: 'vibe-math-v3/vibe-math-v3.js',
|
|
72
|
+
from: "additionalProperties: false, required: required || [] }",
|
|
73
|
+
to: 'required: required || [] }',
|
|
74
|
+
expect: 'v3 I13: every objParams definition closes tool schemas',
|
|
75
|
+
},
|
|
76
|
+
{
|
|
77
|
+
name: 'v4: the schema advertises a knob the parameter layer drops (I14)',
|
|
78
|
+
rel: 'vibe-math-v4/vibe-math-v4.js',
|
|
79
|
+
from: "leanTimeoutMs:{type:'integer'}",
|
|
80
|
+
to: "leanTimeoutMs:{type:'integer'},bogusKnob:{type:'string'}",
|
|
81
|
+
expect: 'v4 I14: every key vibe_v4_set advertises is actually accepted',
|
|
82
|
+
},
|
|
83
|
+
{
|
|
84
|
+
name: 'v5: normalizeParams stops accepting an advertised key (I14)',
|
|
85
|
+
rel: 'vibe-math-v5/vibe-math-v5.js',
|
|
86
|
+
from: "'meetingKeepEvery', 'leanTimeoutMs']",
|
|
87
|
+
to: "'meetingKeepEvery']",
|
|
88
|
+
expect: 'advertised but dropped: [leanTimeoutMs]',
|
|
89
|
+
},
|
|
90
|
+
]
|
|
91
|
+
|
|
92
|
+
if (process.argv.includes('--self-probe')) {
|
|
93
|
+
const bad = []
|
|
94
|
+
for (const mut of SELF_PROBE_MUTATIONS) {
|
|
95
|
+
const r = spawnSync(process.execPath, [fileURLToPath(import.meta.url), '--json'], {
|
|
96
|
+
cwd: HERE,
|
|
97
|
+
env: Object.assign({}, process.env, mut.control ? {} : { PROMPT_INVARIANTS_MUTATE: JSON.stringify([mut.rel, mut.from, mut.to]) }),
|
|
98
|
+
encoding: 'utf8',
|
|
99
|
+
})
|
|
100
|
+
let parsed = null
|
|
101
|
+
try { parsed = JSON.parse(r.stdout) } catch (e) { /* fall through to the diagnostic below */ }
|
|
102
|
+
const text = (r.stdout || '') + (r.stderr || '')
|
|
103
|
+
if (mut.control) {
|
|
104
|
+
const ok = r.status === 0 && parsed && parsed.failed === 0
|
|
105
|
+
console.log((ok ? 'PASS ' : 'FAIL ') + 'control: the unmutated run stays green (' + (parsed ? parsed.failed + ' failures' : 'unparsable output') + ')')
|
|
106
|
+
if (!ok) bad.push('control run was not green')
|
|
107
|
+
continue
|
|
108
|
+
}
|
|
109
|
+
const hit = text.includes(mut.expect)
|
|
110
|
+
const red = r.status === 1 && parsed && parsed.failed > 0
|
|
111
|
+
const ok = red && hit
|
|
112
|
+
console.log((ok ? 'PASS ' : 'FAIL ') + mut.name + (ok ? '' : ' → exit=' + r.status + ' hit=' + hit))
|
|
113
|
+
if (!ok) bad.push(mut.name + ' (exit ' + r.status + ', expected ' + JSON.stringify(mut.expect) + ')')
|
|
114
|
+
}
|
|
115
|
+
console.log('')
|
|
116
|
+
console.log('PROMPT INVARIANT SELF-PROBE: ' + (SELF_PROBE_MUTATIONS.length - bad.length) + '/' + SELF_PROBE_MUTATIONS.length + ' as required')
|
|
117
|
+
for (const b of bad) console.error(' FAIL ' + b)
|
|
118
|
+
process.exit(bad.length === 0 ? 0 : 1)
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
/**
|
|
122
|
+
* Blank out comments (line + block) while preserving string/template REGEX literals, so invariants
|
|
123
|
+
* about "text shown to an agent" do not fire on a COMMENT that quotes an anti-pattern. A naive regex
|
|
124
|
+
* would both miss block comments and mangle strings containing `//` (URLs), so this walks the
|
|
125
|
+
* source as a tiny scanner. Newlines are preserved to keep any line-based diagnostics aligned.
|
|
126
|
+
*
|
|
127
|
+
* Regex literals matter: all four presets contain `/[\\/:*?"<>|\u0000-\u001f]+/` — a regex whose
|
|
128
|
+
* character class contains a DOUBLE QUOTE. Without regex handling the scanner treats that quote as
|
|
129
|
+
* the start of a string, keeps the state machine wrong for the rest of the file, and then either
|
|
130
|
+
* leaves a comment in the "code" stream (a spurious invariant failure) or blanks real code (a
|
|
131
|
+
* missed defect). X5/X6 below are self-checks for exactly that.
|
|
132
|
+
*/
|
|
133
|
+
function stripComments(src) {
|
|
134
|
+
const out = []
|
|
135
|
+
let i = 0
|
|
136
|
+
let state = 'code' // code | line | block | sq | dq | tpl | regex
|
|
137
|
+
let prev = '' // last significant code token (a single char, or a whole word such as `return`)
|
|
138
|
+
let word = '' // identifier/keyword accumulator, so `return /re/` is not read as division
|
|
139
|
+
// A `/` starts a REGEX LITERAL after an operator/keyword, and DIVISION after a value. Tracking only
|
|
140
|
+
// the previous CHARACTER is not enough: `return /^\s*import/.test(l)` (v3 really contains it) puts
|
|
141
|
+
// an identifier before the slash. Getting this wrong is exactly the bug X5–X8 guard against.
|
|
142
|
+
const REGEX_AFTER_KEYWORD = /^(?:return|typeof|case|delete|void|instanceof|in|of|yield|await|new|do|else)$/
|
|
143
|
+
while (i < src.length) {
|
|
144
|
+
const c = src[i]
|
|
145
|
+
const c2 = src[i + 1]
|
|
146
|
+
if (state === 'code') {
|
|
147
|
+
if (c === '/' && c2 === '/') { state = 'line'; out.push(' '); i += 2; continue }
|
|
148
|
+
if (c === '/' && c2 === '*') { state = 'block'; out.push(' '); i += 2; continue }
|
|
149
|
+
if (c === '/' && (prev === '' || REGEX_AFTER_KEYWORD.test(prev) || /[(,=:[!&|?{};+\-*%~^<>]/.test(prev))) {
|
|
150
|
+
state = 'regex'; out.push(c); i++; continue
|
|
151
|
+
}
|
|
152
|
+
if (c === "'") state = 'sq'
|
|
153
|
+
else if (c === '"') state = 'dq'
|
|
154
|
+
else if (c === '`') state = 'tpl'
|
|
155
|
+
if (/[A-Za-z0-9_$]/.test(c)) word += c
|
|
156
|
+
else { if (word) { prev = word; word = '' } if (!/\s/.test(c)) prev = c }
|
|
157
|
+
out.push(c); i++; continue
|
|
158
|
+
}
|
|
159
|
+
if (state === 'line') {
|
|
160
|
+
if (c === '\n') { state = 'code'; out.push(c) } else out.push(' ')
|
|
161
|
+
i++; continue
|
|
162
|
+
}
|
|
163
|
+
if (state === 'block') {
|
|
164
|
+
if (c === '*' && c2 === '/') { state = 'code'; out.push(' '); i += 2; continue }
|
|
165
|
+
out.push(c === '\n' ? c : ' '); i++; continue
|
|
166
|
+
}
|
|
167
|
+
if (state === 'regex') {
|
|
168
|
+
if (c === '\\') { out.push(c, c2 === undefined ? '' : c2); i += 2; continue }
|
|
169
|
+
if (c === '[') { state = 'regexClass'; out.push(c); i++; continue }
|
|
170
|
+
if (c === '/') { // end of the literal: copy it plus any flags
|
|
171
|
+
out.push(c); i++
|
|
172
|
+
while (i < src.length && /[a-z]/i.test(src[i])) { out.push(src[i]); i++ }
|
|
173
|
+
state = 'code'; prev = ')'
|
|
174
|
+
continue
|
|
175
|
+
}
|
|
176
|
+
out.push(c); i++; continue
|
|
177
|
+
}
|
|
178
|
+
if (state === 'regexClass') {
|
|
179
|
+
if (c === '\\') { out.push(c, c2 === undefined ? '' : c2); i += 2; continue }
|
|
180
|
+
if (c === ']') state = 'regex'
|
|
181
|
+
out.push(c); i++; continue
|
|
182
|
+
}
|
|
183
|
+
// inside a string/template: copy verbatim, honouring escapes and the closing quote
|
|
184
|
+
if (c === '\\') { out.push(c, c2 === undefined ? '' : c2); i += 2; continue }
|
|
185
|
+
if ((state === 'sq' && c === "'") || (state === 'dq' && c === '"') || (state === 'tpl' && c === '`')) state = 'code'
|
|
186
|
+
out.push(c); i++
|
|
187
|
+
}
|
|
188
|
+
return out.join('')
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
/**
|
|
192
|
+
* Balanced-bracket slice starting at src[openIdx] (one of ( [ {), string/comment aware.
|
|
193
|
+
* Used by the TOOL-SURFACE invariants (I13/I14) to read a real parameter schema out of the source.
|
|
194
|
+
*/
|
|
195
|
+
function balanced(src, openIdx) {
|
|
196
|
+
let depth = 0
|
|
197
|
+
let i = openIdx
|
|
198
|
+
let state = 'code'
|
|
199
|
+
while (i < src.length) {
|
|
200
|
+
const c = src[i]
|
|
201
|
+
const c2 = src[i + 1]
|
|
202
|
+
if (state === 'code') {
|
|
203
|
+
if (c === "'" || c === '"' || c === '`') { state = c; i++; continue }
|
|
204
|
+
if (c === '(' || c === '[' || c === '{') depth++
|
|
205
|
+
else if (c === ')' || c === ']' || c === '}') { depth--; if (depth === 0) return src.slice(openIdx, i + 1) }
|
|
206
|
+
i++; continue
|
|
207
|
+
}
|
|
208
|
+
if (c === '\\') { i += 2; continue }
|
|
209
|
+
if (c === state) state = 'code'
|
|
210
|
+
i++
|
|
211
|
+
}
|
|
212
|
+
return src.slice(openIdx)
|
|
213
|
+
}
|
|
214
|
+
/** Split an object-body on TOP-LEVEL commas only (brackets and strings respected). */
|
|
215
|
+
function splitTopLevel(body) {
|
|
216
|
+
const parts = []
|
|
217
|
+
let cur = ''
|
|
218
|
+
let depth = 0
|
|
219
|
+
let state = 'code'
|
|
220
|
+
for (let i = 0; i < body.length; i++) {
|
|
221
|
+
const c = body[i]
|
|
222
|
+
const c2 = body[i + 1]
|
|
223
|
+
if (state === 'code') {
|
|
224
|
+
if (c === "'" || c === '"' || c === '`') { state = c; cur += c; continue }
|
|
225
|
+
if (c === '{' || c === '[' || c === '(') depth++
|
|
226
|
+
else if (c === '}' || c === ']' || c === ')') depth--
|
|
227
|
+
if (c === ',' && depth === 0) { parts.push(cur); cur = ''; continue }
|
|
228
|
+
cur += c; continue
|
|
229
|
+
}
|
|
230
|
+
if (c === '\\') { cur += c + (c2 || ''); i++; continue }
|
|
231
|
+
if (c === state) state = 'code'
|
|
232
|
+
cur += c
|
|
233
|
+
}
|
|
234
|
+
parts.push(cur)
|
|
235
|
+
return parts.map((p) => p.trim()).filter(Boolean)
|
|
236
|
+
}
|
|
237
|
+
/** Property keys of the object literal at/after `at` (null when there is no object literal there). */
|
|
238
|
+
function objectKeys(src, at) {
|
|
239
|
+
const open = src.indexOf('{', at)
|
|
240
|
+
if (open < 0) return null
|
|
241
|
+
const region = balanced(src, open)
|
|
242
|
+
return splitTopLevel(region.slice(1, -1))
|
|
243
|
+
.map((p) => { const m = p.match(/^['"]?([A-Za-z_$][\w$]*)['"]?\s*:/); return m ? m[1] : null })
|
|
244
|
+
.filter(Boolean)
|
|
245
|
+
}
|
|
246
|
+
|
|
247
|
+
const PRESETS = [
|
|
248
|
+
{ tag: 'v2', js: 'vibe-math-v2/vibe-math-v2.js', suite: 'tests/formal-verify-v2.test.mjs', corpus: 'prompt-corpus-v2/formal-verify-v2.md', valueField: 'Result', prefix: 'vibe_math_', setTool: 'vibe_math_set_params' },
|
|
249
|
+
{ tag: 'v3', js: 'vibe-math-v3/vibe-math-v3.js', suite: 'tests/formal-verify-v3.test.mjs', corpus: 'prompt-corpus-v3/formal-verify-v3.md', valueField: 'Result', prefix: 'vibe_math_', setTool: 'vibe_math_set_params' },
|
|
250
|
+
{ tag: 'v4', js: 'vibe-math-v4/vibe-math-v4.js', suite: 'tests/formal-verify-v4.test.mjs', corpus: 'prompt-corpus-v4/formal-verify-v4.md', valueField: 'verdict', prefix: 'vibe_v4_', setTool: 'vibe_v4_set' },
|
|
251
|
+
{ tag: 'v5', js: 'vibe-math-v5/vibe-math-v5.js', suite: 'tests/formal-verify-v5.test.mjs', corpus: 'prompt-corpus-v5/prompt-corpus-v5.md', valueField: 'verdict', prefix: 'vibe_v5_', setTool: 'vibe_v5_set' },
|
|
252
|
+
]
|
|
253
|
+
const LEAN_PARAMS = ['formalVerify', 'leanCommand', 'leanArgs', 'leanTimeoutMs']
|
|
254
|
+
|
|
255
|
+
let passed = 0
|
|
256
|
+
const failures = []
|
|
257
|
+
const notes = []
|
|
258
|
+
function check(cond, label, detail) {
|
|
259
|
+
if (cond) { passed++; return true }
|
|
260
|
+
failures.push(label + (detail ? ' — ' + detail : ''))
|
|
261
|
+
return false
|
|
262
|
+
}
|
|
263
|
+
|
|
264
|
+
for (const P of PRESETS) {
|
|
265
|
+
const js = read(P.js)
|
|
266
|
+
const suite = read(P.suite)
|
|
267
|
+
const corpus = read(P.corpus)
|
|
268
|
+
if (!check(js !== null, P.tag + ': plugin source readable', P.js)) continue
|
|
269
|
+
if (!check(suite !== null, P.tag + ': suite readable', P.suite)) continue
|
|
270
|
+
if (!check(corpus !== null, P.tag + ': prompt corpus shipped', P.corpus)) continue
|
|
271
|
+
|
|
272
|
+
// I1/I2 run on the source with comments blanked: a comment may legitimately quote an
|
|
273
|
+
// anti-pattern as documentation, but any STRING can reach an agent.
|
|
274
|
+
const code = stripComments(js)
|
|
275
|
+
|
|
276
|
+
// I1 — no abbreviated tool name in code/strings (code identifiers are leanArchive / leanRunTool,
|
|
277
|
+
// so a bare lean_* token is always a string that can be shown to an agent).
|
|
278
|
+
const bare = code.match(/(^|[^A-Za-z0-9_])lean_(run|archive|lib)\b/g) || []
|
|
279
|
+
check(bare.length === 0, P.tag + ' I1: no abbreviated Lean tool name in agent-facing text', 'found ' + JSON.stringify(bare.slice(0, 3)))
|
|
280
|
+
|
|
281
|
+
// I2 — never tell a voter to answer 0 for a faithfulness defect (comments excluded).
|
|
282
|
+
check(!/偏离\s*(?:→|->|=>)\s*0/.test(code), P.tag + ' I2: no "偏离 → 0" instruction')
|
|
283
|
+
|
|
284
|
+
// I3 — the defect rule is in the injected text.
|
|
285
|
+
check(js.includes('不要投 0'), P.tag + ' I3: injected text forbids a 0 vote on a defect')
|
|
286
|
+
check(/'defect'/.test(js) || /"defect"/.test(js), P.tag + ' I3: injected text names decision=\'defect\'')
|
|
287
|
+
|
|
288
|
+
// I4 — the defect decision is actually HANDLED (comparison + a downgrade to `attempted`).
|
|
289
|
+
const comparesDefect = /(?:===|==)\s*'defect'/.test(js) || /'defect'\s*(?:===|==)/.test(js)
|
|
290
|
+
check(comparesDefect, P.tag + ' I4: the code compares decision against \'defect\'')
|
|
291
|
+
const downgrades = /status:\s*'attempted'/.test(js) || /status='attempted'/.test(js) || /status:\s*"attempted"/.test(js)
|
|
292
|
+
check(downgrades, P.tag + " I4: a defect downgrades the record to 'attempted'")
|
|
293
|
+
|
|
294
|
+
// I5 — the reply contract offers the defect decision.
|
|
295
|
+
const contractCount = js.split('"decision":"used|blocked|defect"').length - 1
|
|
296
|
+
check(contractCount >= 1, P.tag + ' I5: the reply contract offers used|blocked|defect', 'occurrences=' + contractCount)
|
|
297
|
+
|
|
298
|
+
// I6 — a defect without a reason is refused.
|
|
299
|
+
check(/!note/.test(js), P.tag + ' I6: a defect/blocker without a note is rejected')
|
|
300
|
+
|
|
301
|
+
// I7 — the fidelity instruction names the field the parser really reads.
|
|
302
|
+
{
|
|
303
|
+
const lines = js.split(/\r?\n/)
|
|
304
|
+
const i = lines.findIndex((l) => l.includes('不要投 0'))
|
|
305
|
+
const window = i >= 0 ? lines.slice(Math.max(0, i - 3), i + 8).join('\n') : ''
|
|
306
|
+
check(i >= 0 && window.includes(P.valueField),
|
|
307
|
+
P.tag + ' I7: the fidelity instruction names ' + P.valueField + ' (the real reply field)',
|
|
308
|
+
i < 0 ? 'the 不要投 0 rule was not found' : 'window: ' + window.slice(0, 120).replace(/\n/g, ' | '))
|
|
309
|
+
if (P.tag === 'v2' || P.tag === 'v3') {
|
|
310
|
+
check(!/给出\s*verdict/.test(js), P.tag + ' I7b: no "给出 verdict" in a Result-based preset')
|
|
311
|
+
}
|
|
312
|
+
}
|
|
313
|
+
|
|
314
|
+
// I8 — the `formal` reply channel is inert in off mode (tools stay usable on purpose).
|
|
315
|
+
const offGuards = {
|
|
316
|
+
v2: /absorbFormal(?:From)?Reply[\s\S]{0,900}!formalOn\(\)/,
|
|
317
|
+
v3: /absorbFormal(?:From)?Reply[\s\S]{0,900}!formalOn\(\)/,
|
|
318
|
+
v4: /applyFormalReply[\s\S]{0,900}!formalOn\(\)/,
|
|
319
|
+
v5: /formalOn\(\)\s*&&\s*p\.formal/,
|
|
320
|
+
}
|
|
321
|
+
check(offGuards[P.tag].test(js), P.tag + ' I8: the reply channel is gated on formalOn() (off stays a no-op)')
|
|
322
|
+
|
|
323
|
+
// I9 — the corpus covers every mode and is deterministic / machine-path free.
|
|
324
|
+
check(corpus.includes('【Lean 形式化验证(鼓励模式)】'), P.tag + ' I9: corpus covers encourage mode')
|
|
325
|
+
check(corpus.includes('【Lean 形式化验证(强制模式)】'), P.tag + ' I9: corpus covers REQUIRE mode')
|
|
326
|
+
check(corpus.includes('不要投 0'), P.tag + ' I9: corpus contains the fidelity/defect rule')
|
|
327
|
+
check(/used\|blocked\|defect/.test(corpus), P.tag + ' I9: corpus contains the reply contract line')
|
|
328
|
+
check(/【顺手形式化/.test(corpus), P.tag + ' I9: corpus contains the work-round line')
|
|
329
|
+
check(!/[A-Za-z]:[\\/]/.test(corpus), P.tag + ' I9: corpus leaks no absolute path')
|
|
330
|
+
check(!/\d{4}-\d{2}-\d{2}[ T]\d{2}:\d{2}/.test(corpus), P.tag + ' I9: corpus carries no timestamp (deterministic)')
|
|
331
|
+
check(!/vibe-v[0-9]-[a-z]+-[A-Za-z0-9]{4,}/.test(corpus), P.tag + ' I9: corpus leaks no temp-dir name')
|
|
332
|
+
|
|
333
|
+
// I10 — the prompt-rule probes exist for this preset (a guard nobody can prove is a guard nobody has).
|
|
334
|
+
const probeSrc = read('tests/audit-formal-sensitivity.mjs') || ''
|
|
335
|
+
for (const kind of ['fidelity-defect-rule-removed', 'abbreviated-tool-name-injected', 'require-wording-removed', 'defect-decision-not-offered']) {
|
|
336
|
+
check(probeSrc.includes("tag + '-" + kind + "'") || probeSrc.includes("'" + P.tag + '-' + kind + "'"),
|
|
337
|
+
P.tag + ' I10: probe exists for ' + kind)
|
|
338
|
+
}
|
|
339
|
+
|
|
340
|
+
// I10b — the suite asserts the defect path (behaviourally, not just wording).
|
|
341
|
+
check(/\bdefect\b/.test(suite), P.tag + ' I10b: the suite exercises the defect path')
|
|
342
|
+
|
|
343
|
+
// I11 — the "no toolchain" guidance must name BOTH failure codes. An agent that only knows
|
|
344
|
+
// LEAN_NOT_FOUND treats NO_SUBPROCESS as an unknown failure and retries instead of recording the
|
|
345
|
+
// blocker (contract §6 hard rule 4).
|
|
346
|
+
{
|
|
347
|
+
const lines = js.split(/\r?\n/)
|
|
348
|
+
const gi = lines.findIndex((l) => l.includes('宿主无 Lean 工具链'))
|
|
349
|
+
const win = gi >= 0 ? lines.slice(Math.max(0, gi - 3), gi + 1).join('\n') : ''
|
|
350
|
+
check(gi >= 0 && /LEAN_NOT_FOUND/.test(win) && /NO_SUBPROCESS/.test(win),
|
|
351
|
+
P.tag + ' I11: the no-toolchain guidance names LEAN_NOT_FOUND AND NO_SUBPROCESS',
|
|
352
|
+
gi < 0 ? 'the guidance line was not found' : 'window: ' + win.slice(0, 140).replace(/\n/g, ' | '))
|
|
353
|
+
}
|
|
354
|
+
|
|
355
|
+
// I12 — the fidelity branch must qualify its promise BY MODE. Only `require` has a gate, so the
|
|
356
|
+
// text must say "本次裁定不定论" for require AND explicitly tell the voter, for encourage, that
|
|
357
|
+
// this mode has no gate and their abstention is what prevents a conclusion. Shipping the
|
|
358
|
+
// unconditional claim was a real defect that survived in three presets after v5 was fixed.
|
|
359
|
+
check(js.includes('本档没有门禁'), P.tag + ' I12: the fidelity text states the encourage branch has no gate')
|
|
360
|
+
check(js.includes('本次裁定**不定论**') || js.includes('本次裁定不定论'), P.tag + ' I12: the fidelity text states the require hold')
|
|
361
|
+
|
|
362
|
+
// I13 — the Lean switch must be REACHABLE THROUGH THE TOOL SCHEMA. Every tool schema here is
|
|
363
|
+
// built by `objParams`, which closes it with `additionalProperties:false`: a key the schema does
|
|
364
|
+
// not list is REJECTED by any schema-validating provider. v3 shipped 2.3.0/2.3.1 with all four
|
|
365
|
+
// Lean params missing from `vibe_math_set_params` and every suite stayed green, because the
|
|
366
|
+
// suites call the handler directly and never look at the schema — the feature could not be turned
|
|
367
|
+
// on at all. So: EVERY registration of the parameter tool (v2/v3 register it twice, for two agent
|
|
368
|
+
// scopes) must advertise all four, and the closed-schema premise must still hold.
|
|
369
|
+
{
|
|
370
|
+
// EVERY objParams definition must close its schema: one open definition (v2/v3 define it twice,
|
|
371
|
+
// for two agent scopes) would let tools registered through it accept arbitrary keys.
|
|
372
|
+
const defs = [...code.matchAll(/function objParams\(/g)].map((m) => m.index)
|
|
373
|
+
const closedDefs = defs.filter((i) => /additionalProperties:\s*false/.test(code.slice(i, i + 220)))
|
|
374
|
+
check(defs.length > 0 && closedDefs.length === defs.length,
|
|
375
|
+
P.tag + ' I13: every objParams definition closes tool schemas (additionalProperties:false)',
|
|
376
|
+
'definitions=' + defs.length + ' closed=' + closedDefs.length +
|
|
377
|
+
(defs.length ? '; if this is gone the schema no longer rejects unlisted keys and I13 loses its premise' : ''))
|
|
378
|
+
const regRe = new RegExp("registerTool\\(\\s*['\"]" + P.setTool + "['\"]", 'g')
|
|
379
|
+
const schemas = []
|
|
380
|
+
let m
|
|
381
|
+
while ((m = regRe.exec(code))) {
|
|
382
|
+
const rest = code.slice(m.index)
|
|
383
|
+
const oi = rest.indexOf('objParams(')
|
|
384
|
+
schemas.push(oi < 0 ? null : objectKeys(rest, oi + 'objParams'.length))
|
|
385
|
+
}
|
|
386
|
+
check(schemas.length >= 1, P.tag + ' I13: the parameter tool ' + P.setTool + ' is registered',
|
|
387
|
+
'no registerTool(\'' + P.setTool + '\') call found')
|
|
388
|
+
const bad = schemas.map((s, i) => (s === null ? 'reg#' + i + ': no objParams schema'
|
|
389
|
+
: LEAN_PARAMS.filter((k) => !s.includes(k)))).filter((x) => (Array.isArray(x) ? x.length : true))
|
|
390
|
+
check(schemas.length >= 1 && bad.length === 0,
|
|
391
|
+
P.tag + ' I13: every ' + P.setTool + ' schema advertises ' + LEAN_PARAMS.join('/'),
|
|
392
|
+
bad.length ? JSON.stringify(bad) : 'no registration found')
|
|
393
|
+
|
|
394
|
+
// I14 — a key the schema ACCEPTS must really be accepted by the parameter layer. Otherwise the
|
|
395
|
+
// tool advertises a knob that is silently dropped: the caller sees {ok:true} and nothing changes.
|
|
396
|
+
// The accept gate is DEFAULT_PARAMS for v2/v3/v4 (`if (k in params)`) and the typed lists inside
|
|
397
|
+
// normalizeParams for v5 (only those keys are copied through).
|
|
398
|
+
let accepts = null
|
|
399
|
+
if (P.tag === 'v5') {
|
|
400
|
+
const fn = code.search(/function\s+normalizeParams\s*\(/)
|
|
401
|
+
const region = fn < 0 ? '' : balanced(code, code.indexOf('{', fn))
|
|
402
|
+
const names = []
|
|
403
|
+
for (const kind of ['ints', 'bools', 'strs', 'arrs']) {
|
|
404
|
+
const ai = region.search(new RegExp('const\\s+' + kind + '\\s*=\\s*\\['))
|
|
405
|
+
if (ai < 0) continue
|
|
406
|
+
const arr = balanced(region, region.indexOf('[', ai))
|
|
407
|
+
for (const lit of arr.match(/'[A-Za-z_$][\w$]*'/g) || []) names.push(lit.slice(1, -1))
|
|
408
|
+
}
|
|
409
|
+
accepts = names.length ? [...new Set(names)] : null
|
|
410
|
+
} else {
|
|
411
|
+
const di = code.search(/DEFAULT_PARAMS\s*=\s*\{/)
|
|
412
|
+
accepts = di < 0 ? null : objectKeys(code, di)
|
|
413
|
+
}
|
|
414
|
+
check(accepts !== null && accepts.length > 0, P.tag + ' I14: the parameter accept-set is readable',
|
|
415
|
+
'extractor found no parameter set — the invariant cannot be checked')
|
|
416
|
+
const union = [...new Set(schemas.filter(Boolean).flat())]
|
|
417
|
+
const dropped = accepts ? union.filter((k) => !accepts.includes(k)) : []
|
|
418
|
+
check(accepts !== null && dropped.length === 0,
|
|
419
|
+
P.tag + ' I14: every key ' + P.setTool + ' advertises is actually accepted (no silently-dropped knob)',
|
|
420
|
+
'advertised but dropped: [' + dropped.join(',') + ']')
|
|
421
|
+
check(union.length >= LEAN_PARAMS.length, P.tag + ' I14: the parameter schema was parsed (' + union.length + ' keys)',
|
|
422
|
+
'schema extraction returned ' + union.length + ' keys')
|
|
423
|
+
}
|
|
424
|
+
|
|
425
|
+
notes.push(P.tag + ': plugin ' + js.length + 'B · suite ' + suite.length + 'B · corpus ' + corpus.length + 'B')
|
|
426
|
+
}
|
|
427
|
+
|
|
428
|
+
// Cross-preset: the probe script must refuse to report success on an empty selection (false green).
|
|
429
|
+
{
|
|
430
|
+
const probeSrc = read('tests/audit-formal-sensitivity.mjs') || ''
|
|
431
|
+
check(/selected\.length === 0/.test(probeSrc), 'X1: the probe runner fails on an empty selection instead of reporting success')
|
|
432
|
+
const runner = read('tests/run-tests.mjs') || ''
|
|
433
|
+
check(/no suites matched/.test(runner), 'X2: the suite runner fails when no suite matches')
|
|
434
|
+
check(/argv\[i \+ 1\]/.test(runner) && /argv\[\+\+i\]/.test(runner), 'X3: the suite runner accepts both --flag=x and --flag x')
|
|
435
|
+
}
|
|
436
|
+
|
|
437
|
+
// X5–X7: guard the guard. I1/I13/I14 trust stripComments(), and all four presets contain a regex
|
|
438
|
+
// literal whose character class holds a double quote (`/[\\/:*?"<>|…]+/`). If the scanner mistook
|
|
439
|
+
// that quote for a string start, the rest of the file would be misread — a comment quoting an
|
|
440
|
+
// anti-pattern would look like code (spurious failure) or real code would be blanked (missed bug).
|
|
441
|
+
{
|
|
442
|
+
const fixtureRegex = 'const t = s.replace(/[\\\\/:*?"<>|]+/g, "-")\n// lean_run is NOT a tool name\nconst b = 1'
|
|
443
|
+
check(!stripComments(fixtureRegex).includes('lean_run'),
|
|
444
|
+
'X5: the comment scanner survives a regex literal containing a quote (a following comment stays blanked)')
|
|
445
|
+
const fixtureUrl = "L.push('see https://example.com/a')\nconst c = 2"
|
|
446
|
+
check(stripComments(fixtureUrl).includes('https://example.com/a'),
|
|
447
|
+
'X6: a // inside a STRING is not treated as a comment start (URLs must survive)')
|
|
448
|
+
const fixtureBlock = "const d = /a\\/b/g\n/* lean_lib quoted in a block comment */\nconst e = 3"
|
|
449
|
+
check(!stripComments(fixtureBlock).includes('lean_lib'),
|
|
450
|
+
'X7: an escaped slash inside a regex does not end it early (a block comment after it stays blanked)')
|
|
451
|
+
for (const [label, src] of [['X5', fixtureRegex], ['X6', fixtureUrl], ['X7', fixtureBlock]]) {
|
|
452
|
+
check(stripComments(src).split('\n').length === src.split('\n').length,
|
|
453
|
+
label + ': comment blanking preserves line structure (diagnostics stay aligned)')
|
|
454
|
+
}
|
|
455
|
+
}
|
|
456
|
+
|
|
457
|
+
// X8 — the strongest scanner oracle there is: the comment-stripped source of EVERY preset must still
|
|
458
|
+
// PARSE. A scanner that mis-lexes (regex read as division, or a quote inside a regex read as a string
|
|
459
|
+
// start) corrupts real code, and `node --check` sees it. Sensitivity is MEASURED, not assumed:
|
|
460
|
+
// · the 2.3.2 scanner (no regex support at all) → SyntaxError on all four presets → X8 red;
|
|
461
|
+
// · the 2.3.3 scanner (regex-aware but no keyword rule) → X8 still green, which is exactly why
|
|
462
|
+
// X8b below exists (it covers that narrower gap).
|
|
463
|
+
{
|
|
464
|
+
const dir = mkdtempSync(join(tmpdir(), 'prompt-invariants-strip-'))
|
|
465
|
+
try {
|
|
466
|
+
for (const P of PRESETS) {
|
|
467
|
+
const src = read(P.js)
|
|
468
|
+
if (!src) continue
|
|
469
|
+
const f = join(dir, P.tag + '.mjs')
|
|
470
|
+
writeFileSync(f, stripComments(src))
|
|
471
|
+
const r = spawnSync(process.execPath, ['--check', f], { encoding: 'utf8' })
|
|
472
|
+
check(r.status === 0,
|
|
473
|
+
'X8: the comment-stripped source of ' + P.tag + ' is still valid JS (the scanner must not corrupt code)',
|
|
474
|
+
String(r.stderr || '').split('\n').filter((l) => l.trim()).slice(-2).join(' ').slice(0, 160))
|
|
475
|
+
}
|
|
476
|
+
// X8b: a fixture that needs the KEYWORD rule — a regex AFTER `return` whose character class holds
|
|
477
|
+
// a quote (v3 really has `return /^\s*…/.test()`, so the rule is not hypothetical). A scanner that
|
|
478
|
+
// reads that `/` as division enters "string" state at the quote and then leaves the NEXT comment
|
|
479
|
+
// unblanked. Sensitivity MEASURED: the 2.3.3 scanner fails the second assertion below
|
|
480
|
+
// (comment-blanked=false) while passing the first; the current scanner passes both.
|
|
481
|
+
const fixtureKeyword = [
|
|
482
|
+
'function f(l) { return /["\']/.test(l) }',
|
|
483
|
+
'// lean_run must stay blanked',
|
|
484
|
+
'const h = 4 / 2',
|
|
485
|
+
].join('\n')
|
|
486
|
+
const f2 = join(dir, 'fixture.mjs')
|
|
487
|
+
const stripped = stripComments(fixtureKeyword)
|
|
488
|
+
writeFileSync(f2, stripped)
|
|
489
|
+
const r2 = spawnSync(process.execPath, ['--check', f2], { encoding: 'utf8' })
|
|
490
|
+
check(r2.status === 0, 'X8b: a regex after a KEYWORD (return /…/) does not corrupt the scan', String(r2.stderr || '').slice(0, 120))
|
|
491
|
+
check(!stripped.includes('lean_run'), 'X8b: and the comment after it is still blanked (the keyword rule is what decides this)')
|
|
492
|
+
} finally {
|
|
493
|
+
rmSync(dir, { recursive: true, force: true })
|
|
494
|
+
}
|
|
495
|
+
}
|
|
496
|
+
|
|
497
|
+
const out = { passed, failed: failures.length, failures, notes }
|
|
498
|
+
if (process.argv.includes('--json')) {
|
|
499
|
+
console.log(JSON.stringify(out, null, 2))
|
|
500
|
+
} else {
|
|
501
|
+
console.log('-- prompt/interaction invariants (all four presets) --')
|
|
502
|
+
for (const n of notes) console.log(' note ' + n)
|
|
503
|
+
console.log('')
|
|
504
|
+
for (const f of failures) console.error(' FAIL ' + f)
|
|
505
|
+
console.log('')
|
|
506
|
+
console.log('PROMPT INVARIANTS: ' + passed + ' passed, ' + failures.length + ' failed')
|
|
507
|
+
}
|
|
508
|
+
process.exit(failures.length === 0 ? 0 : 1)
|