dsh-vibe-math 2.7.1 → 2.7.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.en.md +682 -690
- package/README.md +19 -27
- package/cordis.patch.yml +112 -16
- package/docs/AUDIT-CHECKLIST.md +744 -518
- package/docs/COMPAT-AUDIT-ROUND2.md +35 -9
- package/docs/COMPAT-AUDIT-ROUND3.md +154 -154
- package/docs/final-paper.md +172 -172
- package/docs/formal-verification.md +588 -588
- package/docs/math-computation.md +194 -163
- package/docs/parameter-schema.md +37 -0
- package/docs/release-notes/README.en.md +49 -0
- package/docs/release-notes/README.md +48 -0
- package/docs/release-notes/RELEASE-NOTES-2.7.2.en.md +72 -0
- package/docs/release-notes/RELEASE-NOTES-2.7.2.md +72 -0
- package/docs/release-notes/RELEASE-NOTES-2.7.3.en.md +67 -0
- package/docs/release-notes/RELEASE-NOTES-2.7.3.md +67 -0
- package/docs/release-notes/TEMPLATE.md +126 -0
- package/docs/status-report-fields.md +167 -0
- package/docs/test-timing.md +157 -154
- package/installer.js +27 -17
- package/package.json +55 -3
- package/prompt-corpus-persona/persona-corpus.json +9 -9
- package/prompt-corpus-persona/persona-corpus.md +225 -33
- package/prompt-corpus-v2/formal-verify-v2.json +305 -113
- package/prompt-corpus-v2/formal-verify-v2.md +2399 -279
- package/prompt-corpus-v3/formal-verify-v3.json +134 -110
- package/prompt-corpus-v3/formal-verify-v3.md +409 -141
- package/prompt-corpus-v4/formal-verify-v4.json +18 -18
- package/prompt-corpus-v4/formal-verify-v4.md +49 -44
- package/prompt-corpus-v5/prompt-corpus-v5.json +82 -82
- package/prompt-corpus-v5/prompt-corpus-v5.md +513 -259
- package/scripts/update-doc-counts.mjs +71 -0
- package/tests/audit-artifact-docs.mjs +72 -0
- package/tests/audit-artifact-docs.mutants.mjs +58 -0
- package/tests/audit-installer-assertions.mutants.mjs +63 -0
- package/tests/audit-installer-compat.test.mjs +52 -4
- package/tests/audit-installer-policy.test.mjs +41 -8
- package/tests/audit-math-computation-contract.mjs +107 -9
- package/tests/audit-math-computation-contract.mutants.mjs +106 -0
- package/tests/audit-math-computation-parity.mjs +164 -3
- package/tests/audit-math-computation-sensitivity.mjs +6 -4
- package/tests/audit-package-membership.mjs +57 -0
- package/tests/audit-package-membership.mutants.mjs +28 -0
- package/tests/audit-participant-set-parity.mjs +160 -0
- package/tests/audit-path-discipline.mjs +83 -0
- package/tests/audit-path-discipline.mutants.mjs +51 -0
- package/tests/audit-persona-sensitivity.mjs +14 -0
- package/tests/audit-persona-surface.mutants.mjs +69 -0
- package/tests/audit-persona-surface.test.mjs +63 -22
- package/tests/audit-prompt-duplication.mjs +143 -0
- package/tests/audit-prompt-duplication.mutants.mjs +50 -0
- package/tests/audit-prompt-invariants.mjs +43 -5
- package/tests/audit-readme-bilingual.test.mjs +152 -0
- package/tests/audit-readme-counts.mjs +95 -0
- package/tests/audit-readme-counts.mutants.mjs +52 -0
- package/tests/audit-registration.mjs +78 -0
- package/tests/audit-status-report-fields.mjs +172 -0
- package/tests/audit-status-report-fields.mutants.mjs +64 -0
- package/tests/audit-v3-registration-parity.mjs +156 -0
- package/tests/audit-v5-integrity.mjs +300 -13
- package/tests/audit-v5-lean-abstention.mjs +392 -0
- package/tests/audit-v5-lean-abstention.mutants.mjs +106 -0
- package/tests/audit-v5-prompt-duplication.mjs +88 -0
- package/tests/audit-v5-prompt-duplication.mutants.mjs +34 -0
- package/tests/audit-v5-sensitivity.mjs +15 -8
- package/tests/e2e-identity-a6.mutants.mjs +65 -0
- package/tests/e2e-v5-round2.test.mjs +566 -12
- package/tests/formal-verify-v2.test.mjs +91 -5
- package/tests/formal-verify-v3.mutants.mjs +117 -0
- package/tests/formal-verify-v3.test.mjs +85 -6
- package/tests/formal-verify-v4.mutants.mjs +152 -0
- package/tests/formal-verify-v4.test.mjs +628 -9
- package/tests/formal-verify-v5.test.mjs +68 -9
- package/tests/helpers/doc-counts-format.mjs +41 -0
- package/tests/helpers/math-computation-fake-seam.mjs +4 -0
- package/tests/math-computation-a1.mutants.mjs +72 -0
- package/tests/math-computation-archive-rerun.mutants.mjs +65 -0
- package/tests/math-computation-discovery.mutants.mjs +76 -0
- package/tests/math-computation-shared.test.mjs +396 -2
- package/tests/math-computation-v2.test.mjs +767 -0
- package/tests/math-computation-v3.test.mjs +759 -0
- package/tests/math-computation-v4.mutants.mjs +133 -0
- package/tests/math-computation-v4.test.mjs +739 -0
- package/tests/prompt-v5-integrity.test.mjs +13 -2
- package/tests/run-tests.mjs +139 -9
- package/tests/run-tests.mutants.mjs +70 -0
- package/tests/selfdrive-v5.mjs +53 -1
- package/tests/selfdrive-v5.mutants.mjs +132 -0
- package/tests/v2-fix-probes.mutants.mjs +100 -0
- package/tests/v2-fix-probes.test.mjs +1039 -0
- package/tests/v2-list-agents-and-next-step.mutants.mjs +106 -0
- package/tests/v3-fix-probes.mutants.mjs +95 -0
- package/tests/v3-fix-probes.test.mjs +891 -0
- package/tests/v5-institute-fixes.mutants.mjs +190 -0
- package/vibe-math-v2/agent.cordis.yml +62 -6
- package/vibe-math-v2/math-computation.js +229 -29
- package/vibe-math-v2/math-engines.js +12 -12
- package/vibe-math-v2/vibe-math-v2.js +458 -119
- package/vibe-math-v2//345/256/236/347/216/260/346/226/271/346/241/210.md +5 -1
- package/vibe-math-v3/agent.cordis.yml +72 -6
- package/vibe-math-v3/math-computation.js +229 -29
- package/vibe-math-v3/math-engines.js +12 -12
- package/vibe-math-v3/vibe-math-v3.js +482 -139
- package/vibe-math-v4/agent.cordis.yml +44 -8
- package/vibe-math-v4/math-computation.js +229 -29
- package/vibe-math-v4/math-engines.js +12 -12
- package/vibe-math-v4/vibe-math-v4.js +245 -52
- package/vibe-math-v4//345/256/236/347/216/260/346/226/271/346/241/210.md +2 -0
- package/vibe-math-v5/agent.cordis.yml +46 -12
- package/vibe-math-v5/math-computation.js +229 -29
- package/vibe-math-v5/math-engines.js +12 -12
- package/vibe-math-v5/vibe-math-v5.js +1307 -243
- package/vibe-math-v5//345/256/236/347/216/260/346/226/271/346/241/210.md +1728 -1655
- package/vibe-math-v5//346/236/266/346/236/204/345/233/276.md +693 -693
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
// Dedicated cross-preset path-discipline audit (migrated out of formal-verify-v4 so that running ONE
|
|
2
|
+
// preset's suite cannot miss this class).
|
|
3
|
+
//
|
|
4
|
+
// Covers:
|
|
5
|
+
// * v4 / v5 : every member-visible library declaration carries the member root (Members/<x>/...).
|
|
6
|
+
// * v3 : the writer `applyAgentWrites` accepts ONLY Problems|Progress|Propos|Methods|Notes at the
|
|
7
|
+
// PROJECT ROOT, so v3's declarations must stay root-relative - a Members/<x>/... declaration
|
|
8
|
+
// is a write the writer SILENTLY DISCARDS.
|
|
9
|
+
// * v2 / v3 : PAPER_PATH_NOTE structure, by DELIVERED MATERIAL (not by function site):
|
|
10
|
+
// - the evidence-index DATA LIST must not embed the prose rule;
|
|
11
|
+
// - every delivered member-facing material that carries copyable paths must append it -
|
|
12
|
+
// for v2 that is the digest, which INLINES the evidence index; for v3 also the journal.
|
|
13
|
+
// Function-body windows are STRICTLY bounded (no fixed-length fallback): a window that can
|
|
14
|
+
// swallow the next function produces false passes.
|
|
15
|
+
import { readFileSync } from 'node:fs'
|
|
16
|
+
|
|
17
|
+
const R = new URL('../', import.meta.url)
|
|
18
|
+
const ENV_OF = {
|
|
19
|
+
'vibe-math-v2/vibe-math-v2.js': 'V2_PLUGIN',
|
|
20
|
+
'vibe-math-v3/vibe-math-v3.js': 'V3_PLUGIN',
|
|
21
|
+
'vibe-math-v4/vibe-math-v4.js': 'V4_PLUGIN',
|
|
22
|
+
'vibe-math-v5/vibe-math-v5.js': 'V5_PLUGIN',
|
|
23
|
+
}
|
|
24
|
+
// a mutant harness (tests/audit-path-discipline.mutants.mjs) points these at scratch copies
|
|
25
|
+
const rd = (p) => {
|
|
26
|
+
const e = ENV_OF[p] && process.env[ENV_OF[p]]
|
|
27
|
+
return e ? readFileSync(e, 'utf8') : readFileSync(new URL(p, R), 'utf8')
|
|
28
|
+
}
|
|
29
|
+
let passed = 0
|
|
30
|
+
let failed = 0
|
|
31
|
+
const fails = []
|
|
32
|
+
const assert = (c, m) => { if (c) { passed++; console.log(' ok - ' + m) } else { failed++; fails.push(m); console.error(' FAIL - ' + m) } }
|
|
33
|
+
|
|
34
|
+
const v2 = rd('vibe-math-v2/vibe-math-v2.js')
|
|
35
|
+
const v3 = rd('vibe-math-v3/vibe-math-v3.js')
|
|
36
|
+
const v4 = rd('vibe-math-v4/vibe-math-v4.js')
|
|
37
|
+
const v5 = rd('vibe-math-v5/vibe-math-v5.js')
|
|
38
|
+
|
|
39
|
+
const ROOT = /Members\/<[^>]+>\/(?:Progress|Propos|Methods|Subproblems)\//g
|
|
40
|
+
const LEGACY = /(?:^|[^/>])(?:Progress|Propos|Methods|Subproblems)\/<[^>]+>\//g
|
|
41
|
+
|
|
42
|
+
for (const [k, src] of [['v5', v5]]) { // v5 pending vm-v5's re-verification of its bases
|
|
43
|
+
assert((src.match(ROOT) || []).length >= 4, k + ': member-visible declarations carry the member root')
|
|
44
|
+
assert((src.match(LEGACY) || []).length === 0, k + ': no legacy project-root-relative member-library declaration')
|
|
45
|
+
}
|
|
46
|
+
// v4 (critical correction): writer/reader live at the PROJECT ROOT - a Members/<x>/... path is never scanned
|
|
47
|
+
assert(!/Members\//.test(v4), 'v4: no Members/ occurrence at all (widened predicate: literal ids escape the placeholder form) (the framework never scans it)')
|
|
48
|
+
assert(/(?:Progress|Propos|Methods|Subproblems)\/<[^>]+>\//.test(v4), 'v4: declarations use the project-root bases the writer/reader use')
|
|
49
|
+
assert(/progress\.md/.test(v4), 'class guard: v4 documents/writes the Progress base it reads')
|
|
50
|
+
|
|
51
|
+
assert(!/Members\//.test(v3), 'v3: no Members/ occurrence at all (widened predicate: literal ids escape the placeholder form) (applyAgentWrites would DISCARD such a write)')
|
|
52
|
+
assert(/applyAgentWrites[\s\S]{0,600}?(Problems|Progress|Propos|Methods|Notes)/.test(v3), 'class guard: v3 writer whitelists the sections its prompts name')
|
|
53
|
+
|
|
54
|
+
// strictly bounded function-body window: never fall back to a fixed length
|
|
55
|
+
const bodyOf = (src, fn) => {
|
|
56
|
+
const a = src.indexOf('function ' + fn)
|
|
57
|
+
if (a < 0) return null
|
|
58
|
+
const b = src.slice(a + 10).search(/\n {2}(?:async )?function /)
|
|
59
|
+
if (b < 0) return null
|
|
60
|
+
return src.slice(a, a + 10 + b)
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
for (const [k, src] of [['v2', v2], ['v3', v3]]) {
|
|
64
|
+
const def = (src.match(/const PAPER_PATH_NOTE/g) || []).length
|
|
65
|
+
assert(def === 1, k + ': PAPER_PATH_NOTE defined exactly once (found ' + def + ')')
|
|
66
|
+
const idxBody = bodyOf(src, 'paperEvidenceIndex')
|
|
67
|
+
const digBody = bodyOf(src, 'buildPaperDigest')
|
|
68
|
+
assert(idxBody !== null && digBody !== null, k + ': both function windows are bounded strictly (no fixed-length fallback)')
|
|
69
|
+
// the data list is a pure path list: no prose, no note
|
|
70
|
+
assert(!/PAPER_PATH_NOTE/.test(idxBody || ''), k + ': the evidence-index DATA LIST embeds no prose rule')
|
|
71
|
+
// the delivered material inlines that list and therefore carries the paths AND must carry the note
|
|
72
|
+
assert(/paperEvidenceIndex\(\)/.test(digBody || ''), k + ': the delivered digest INLINES the evidence index (so it carries the copyable paths)')
|
|
73
|
+
assert(/PAPER_PATH_NOTE/.test(digBody || ''), k + ': the delivered digest material appends PAPER_PATH_NOTE')
|
|
74
|
+
if (k === 'v3') {
|
|
75
|
+
const jr = bodyOf(src, 'writeJournal')
|
|
76
|
+
assert(jr !== null && /PAPER_PATH_NOTE/.test(jr), 'v3: the member-facing journal (writeJournal) also appends PAPER_PATH_NOTE')
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
console.log('')
|
|
81
|
+
console.log('passed=' + passed + ' failed=' + failed)
|
|
82
|
+
if (failed) { console.error('FAILURES:'); for (const f of fails) console.error(' - ' + f); process.exit(1) }
|
|
83
|
+
console.log('ALL GREEN')
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
// Registered mutant harness for tests/audit-path-discipline.mjs - the audit carries its own proof.
|
|
2
|
+
// One mutant per assertion family; each MUST make the audit fail, otherwise the guard does not bite.
|
|
3
|
+
// Usage: node tests/audit-path-discipline.mutants.mjs (exit 0 = every family reddened)
|
|
4
|
+
import { readFileSync, writeFileSync, mkdirSync, copyFileSync, rmSync, mkdtempSync } from 'node:fs'
|
|
5
|
+
import { execFileSync } from 'node:child_process'
|
|
6
|
+
import { fileURLToPath } from 'node:url'
|
|
7
|
+
import { dirname, join } from 'node:path'
|
|
8
|
+
import { tmpdir } from 'node:os'
|
|
9
|
+
const here = dirname(fileURLToPath(import.meta.url))
|
|
10
|
+
const repo = join(here, '..')
|
|
11
|
+
const audit = join(here, 'audit-path-discipline.mjs')
|
|
12
|
+
// Unique per run (audit C: a fixed repo-local dir could collide with a concurrent run and dirtied the tree)
|
|
13
|
+
const tmp = mkdtempSync(join(tmpdir(), 'audit-mutants-'))
|
|
14
|
+
const families = [
|
|
15
|
+
{ name: 'v4-text-shape (delivered declaration re-prefixed)', preset: 'vibe-math-v4', env: 'V4_PLUGIN',
|
|
16
|
+
from: "+' Propos/<" + String.fromCharCode(20320) + ">/<id>.md",
|
|
17
|
+
to: "+' Members/<" + String.fromCharCode(20320) + ">/Propos/<" + String.fromCharCode(20320) + ">/<id>.md",
|
|
18
|
+
expect: /no Members\/ occurrence at all/ },
|
|
19
|
+
{ name: 'v4-literal-members (widened Members/ predicate)', preset: 'vibe-math-v4', env: 'V4_PLUGIN',
|
|
20
|
+
from: "/Propos/r-1/p-1.md", to: "/Members/r-1/Propos/p-1.md",
|
|
21
|
+
expect: /no Members\/ occurrence at all/ },
|
|
22
|
+
{ name: 'v3-writer-whitelist class guard (Members/ reintroduced)', preset: 'vibe-math-v3', env: 'V3_PLUGIN',
|
|
23
|
+
from: '(^|[^/>])(Propos)\\/<([^>]+)>\\/', to: '$1Members/<$3>/$2/<$3>/', regex: true,
|
|
24
|
+
expect: /v3: no Members\// },
|
|
25
|
+
{ name: 'v2 PAPER_PATH_NOTE structure (digest append dropped)', preset: 'vibe-math-v2', env: 'V2_PLUGIN',
|
|
26
|
+
from: ' L.push(PAPER_PATH_NOTE)', to: '', expect: /delivered digest material appends PAPER_PATH_NOTE/ },
|
|
27
|
+
]
|
|
28
|
+
mkdirSync(tmp, { recursive: true })
|
|
29
|
+
let ok = 0, bad = 0
|
|
30
|
+
try {
|
|
31
|
+
for (const f of families) {
|
|
32
|
+
const src = join(repo, f.preset, f.preset + '.js')
|
|
33
|
+
let t = readFileSync(src, 'utf8')
|
|
34
|
+
const hits = f.regex ? (t.match(new RegExp(f.from, 'm')) || []).length : t.split(f.from).length - 1
|
|
35
|
+
if (hits < 1) { console.error(' NO-OP - ' + f.name + ' (predicate did not match)'); bad++; continue }
|
|
36
|
+
t = f.regex ? t.replace(new RegExp(f.from, 'm'), f.to) : t.split(f.from).join(f.to)
|
|
37
|
+
const out = join(tmp, f.preset + '.js')
|
|
38
|
+
writeFileSync(out, t, 'utf8')
|
|
39
|
+
let red = false, msg = ''
|
|
40
|
+
try {
|
|
41
|
+
execFileSync(process.execPath, [audit], { env: Object.assign({}, process.env, { [f.env]: out }), encoding: 'utf8' })
|
|
42
|
+
} catch (e) { red = true; msg = String((e && (e.stdout || '')) + (e && (e.stderr || ''))) }
|
|
43
|
+
const named = f.expect.test(msg)
|
|
44
|
+
if (red && named) { ok++; console.log(' ok - ' + f.name + ' reddened the audit by name') }
|
|
45
|
+
else { bad++; console.error(' FAIL - ' + f.name + ' red=' + red + ' named=' + named + ' :: ' + msg.split('\n').filter((l) => l.includes('FAIL')).slice(0, 1).join('')) }
|
|
46
|
+
}
|
|
47
|
+
} finally { rmSync(tmp, { recursive: true, force: true }) }
|
|
48
|
+
console.log('')
|
|
49
|
+
console.log('mutant families reddening the audit: ' + ok + '/' + families.length)
|
|
50
|
+
if (bad) process.exit(1)
|
|
51
|
+
console.log('ALL FAMILIES BITE')
|
|
@@ -65,6 +65,20 @@ function runSuite(root) {
|
|
|
65
65
|
|
|
66
66
|
// A sanity gate first: the UNMUTATED copy must be GREEN through the override, otherwise
|
|
67
67
|
// every probe below would "detect" the override itself rather than the mutation (trap 2).
|
|
68
|
+
//
|
|
69
|
+
// CONCURRENCY FIX: under run-tests this suite shares the machine with a plain
|
|
70
|
+
// `audit-persona-surface` job, which WRITES the shipped corpus (`prompt-corpus-persona/*`) while the
|
|
71
|
+
// children below READ it - and under PERSONA_ROOT the surface suite asserts those files EXIST. In a
|
|
72
|
+
// tree where the corpus has not been generated yet (or is being regenerated), a child could observe the
|
|
73
|
+
// directory without the files and fail, which showed up as "sensitivity red only under concurrency 4".
|
|
74
|
+
// So: generate the corpus ONCE here first (parent run, no PERSONA_ROOT), then run the control. The
|
|
75
|
+
// surface suite's own write is atomic (temp file + rename), so concurrent writers cannot tear the files.
|
|
76
|
+
{
|
|
77
|
+
const warm = spawnSync(process.execPath, [SUITE], { env: process.env, encoding: 'utf8', cwd: REPO })
|
|
78
|
+
const warmOut = String(warm.stdout || '') + String(warm.stderr || '')
|
|
79
|
+
console.log(' ok - corpus warmed by a plain parent run (exit=' + warm.status + ')' + (warm.status === 0 ? '' : ' [warning: plain run not green]'))
|
|
80
|
+
if (warm.status !== 0) console.error(warmOut.split('\n').filter((l) => l.includes('FAIL')).slice(0, 4).join('\n'))
|
|
81
|
+
}
|
|
68
82
|
const sanityRoot = prepareRoot()
|
|
69
83
|
{
|
|
70
84
|
const r = runSuite(sanityRoot)
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
/**
|
|
3
|
+
* Mutant harness for tests/audit-persona-surface.test.mjs (ships: `tests/*.mutants.mjs`).
|
|
4
|
+
*
|
|
5
|
+
* Converts the dev-only scratch proof (`_oneoff/auditR2/persona-mutant.mjs`) into package-reproducible
|
|
6
|
+
* form: a COPY of the four preset dirs is driven through the guard's `PERSONA_ROOT` seam, so the shipped
|
|
7
|
+
* files are never touched, and one single-site mutation must redden the guard BY NAME.
|
|
8
|
+
*
|
|
9
|
+
* The invariant: "every registered tool is mentioned in the persona, or explicitly allow-listed" — a newly
|
|
10
|
+
* registered tool that the persona never mentions (and the allow-list does not cover) must FAIL.
|
|
11
|
+
*
|
|
12
|
+
* Run: node tests/audit-persona-surface.mutants.mjs
|
|
13
|
+
*/
|
|
14
|
+
import { mkdtempSync, cpSync, readFileSync, writeFileSync, rmSync } from 'node:fs'
|
|
15
|
+
import { tmpdir } from 'node:os'
|
|
16
|
+
import { join } from 'node:path'
|
|
17
|
+
import { spawnSync } from 'node:child_process'
|
|
18
|
+
import { fileURLToPath } from 'node:url'
|
|
19
|
+
|
|
20
|
+
const HERE = fileURLToPath(new URL('./', import.meta.url))
|
|
21
|
+
const REPO = join(HERE, '..')
|
|
22
|
+
const GUARD = join(HERE, 'audit-persona-surface.test.mjs')
|
|
23
|
+
const EXPECT = 'every registered tool is mentioned in the persona, or explicitly allow-listed'
|
|
24
|
+
let passed = 0, failed = 0
|
|
25
|
+
const ok = (c, l, d) => { if (c) { passed++; console.log(' ok ' + l) } else { failed++; console.log(' FAIL ' + l + (d ? ' — ' + d : '')) } }
|
|
26
|
+
|
|
27
|
+
const runGuard = (env) => {
|
|
28
|
+
const r = spawnSync(process.execPath, [GUARD], { cwd: REPO, encoding: 'utf8', maxBuffer: 1 << 26, env: Object.assign({}, process.env, env || {}) })
|
|
29
|
+
const o = String(r.stdout || '') + String(r.stderr || '')
|
|
30
|
+
return { status: r.status, out: o, sum: (o.split('\n').filter((l) => /RESULT:/.test(l)).slice(-1)[0] || '').trim(), failLines: o.split('\n').filter((l) => /^\s*FAIL\b|^ - /.test(l)) }
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
// baseline 1: the real tree
|
|
34
|
+
const base = runGuard(null)
|
|
35
|
+
ok(base.status === 0, 'baseline: the shipped guard is green on the real tree', base.sum.slice(0, 70))
|
|
36
|
+
|
|
37
|
+
// baseline 2: the COPIED tree the mutation will use (proves the copy is faithful, not the mutation)
|
|
38
|
+
const root = mkdtempSync(join(tmpdir(), 'persona-surface-mut-'))
|
|
39
|
+
for (const dir of ['vibe-math-v2', 'vibe-math-v3', 'vibe-math-v4', 'vibe-math-v5']) cpSync(join(REPO, dir), join(root, dir), { recursive: true })
|
|
40
|
+
const copyBase = runGuard({ PERSONA_ROOT: root })
|
|
41
|
+
ok(copyBase.status === 0, 'baseline: the same guard is green on the copied tree (PERSONA_ROOT seam is faithful)', copyBase.sum.slice(0, 70))
|
|
42
|
+
|
|
43
|
+
// mutant: register a tool the persona never mentions and the allow-list does not cover
|
|
44
|
+
const v2 = join(root, 'vibe-math-v2', 'vibe-math-v2.js')
|
|
45
|
+
const src = readFileSync(v2, 'utf8')
|
|
46
|
+
const anchor = "registerTool('vibe_math_status'"
|
|
47
|
+
if (src.indexOf(anchor) === -1) ok(false, 'the single-site anchor applies (registerTool call site)', 'ANCHOR MISS')
|
|
48
|
+
else {
|
|
49
|
+
writeFileSync(v2, src.replace(anchor, "registerTool('vibe_math_brand_new_unmentioned', objParams({}), () => ({ok:true}))\n " + anchor))
|
|
50
|
+
const mut = runGuard({ PERSONA_ROOT: root })
|
|
51
|
+
const named = mut.out.split('\n').some((l) => l.indexOf(EXPECT) !== -1)
|
|
52
|
+
ok(mut.status !== 0 && named, '★ a newly registered, unmentioned tool reddens the guard by name (unit: registered tool names)', mut.sum.slice(0, 80))
|
|
53
|
+
if (!named) for (const l of mut.failLines.slice(0, 4)) console.log(' ' + l.trim().slice(0, 140))
|
|
54
|
+
}
|
|
55
|
+
// Stronger seam evidence than "green on the copy" (R17): DELETE a preset dir from a FRESH copy and the
|
|
56
|
+
// guard must fail with ENOENT naming the COPIED path — that proves the seam is actually READ, rather than
|
|
57
|
+
// the guard silently falling back to the real tree (which would make the mutation case above meaningless).
|
|
58
|
+
const root2 = mkdtempSync(join(tmpdir(), 'persona-surface-seam-'))
|
|
59
|
+
for (const dir of ['vibe-math-v2', 'vibe-math-v3', 'vibe-math-v5']) cpSync(join(REPO, dir), join(root2, dir), { recursive: true })
|
|
60
|
+
const missing = runGuard({ PERSONA_ROOT: root2 })
|
|
61
|
+
const namesCopiedPath = missing.out.indexOf(root2) !== -1 && /ENOENT/.test(missing.out)
|
|
62
|
+
ok(missing.status !== 0 && namesCopiedPath,
|
|
63
|
+
'★ seam is READ: removing a preset from the copy makes the guard fail with ENOENT on the COPIED path (unit: ENOENT paths)',
|
|
64
|
+
'exit=' + missing.status + ' namesCopied=' + namesCopiedPath)
|
|
65
|
+
rmSync(root2, { recursive: true, force: true })
|
|
66
|
+
rmSync(root, { recursive: true, force: true })
|
|
67
|
+
console.log('')
|
|
68
|
+
console.log('=== PERSONA SURFACE MUTANTS: ' + passed + ' passed, ' + failed + ' failed ===')
|
|
69
|
+
process.exit(failed === 0 ? 0 : 1)
|
|
@@ -2,6 +2,14 @@
|
|
|
2
2
|
/**
|
|
3
3
|
* PERSONA SURFACE — prompt/allocation consistency for the four presets.
|
|
4
4
|
*
|
|
5
|
+
* DEFECT CLASS (why this guard exists; it belongs to the dual-persona-block parity family of §3)
|
|
6
|
+
* "capability advertised but never granted, or granted but never advertised": the persona block is
|
|
7
|
+
* the main agent view of the tool surface, so a registered-but-unnamed tool is invisible and a
|
|
8
|
+
* named-but-unregistered tool is a trap. Every other suite calls apply(ctx) directly and never
|
|
9
|
+
* parses the YAML, so this class cannot fail anywhere else.
|
|
10
|
+
*
|
|
11
|
+
* Run: node tests/audit-persona-surface.test.mjs
|
|
12
|
+
*
|
|
5
13
|
* WHY THIS EXISTS
|
|
6
14
|
* The persona row of each `agent.cordis.yml` is the prompt the MAIN agent actually
|
|
7
15
|
* receives, and it is the only place the main agent learns which `vibe_*` tools exist
|
|
@@ -26,7 +34,10 @@
|
|
|
26
34
|
* `text` must not receive a different tool surface from a new host that uses `prefix`;
|
|
27
35
|
* C. mentions ⊆ registered: every `vibe_*` token in the persona is a registered tool
|
|
28
36
|
* (a `name*` mention is allowed when some registered tool carries that prefix);
|
|
29
|
-
* D. registered
|
|
37
|
+
* D. every registered tool is either MENTIONED in the persona or consciously ALLOW-LISTED, and the
|
|
38
|
+
* allow-list is itself validated (real tools only, empty by default) — so adding a tool forcing a
|
|
39
|
+
* decision: name it, or add it to the list with a reason. (Earlier this was a snapshot equality,
|
|
40
|
+
* which went red whenever a legitimately newly-named tool emptied the "undocumented" set.)
|
|
30
41
|
* without documenting it (or deleting a tool the persona still advertises) fails
|
|
31
42
|
* loudly and forces the author to make a decision;
|
|
32
43
|
* E. the parameter surface of the Lean feature (the four names) and its semantics
|
|
@@ -36,7 +47,7 @@
|
|
|
36
47
|
* the coordinator's persona?" — virtually all of them are `(member)` / `(academician)` /
|
|
37
48
|
* `(resident)` tools that only subagents call.
|
|
38
49
|
*/
|
|
39
|
-
import { readFileSync, writeFileSync, mkdirSync, existsSync } from 'node:fs'
|
|
50
|
+
import { readFileSync, writeFileSync, mkdirSync, existsSync, renameSync } from 'node:fs'
|
|
40
51
|
import { join, resolve } from 'node:path'
|
|
41
52
|
import { fileURLToPath } from 'node:url'
|
|
42
53
|
|
|
@@ -133,9 +144,12 @@ const PRESETS = [
|
|
|
133
144
|
js: 'vibe-math-v2.js',
|
|
134
145
|
prefix: 'vibe_math_',
|
|
135
146
|
tools: 28,
|
|
136
|
-
//
|
|
137
|
-
//
|
|
138
|
-
|
|
147
|
+
// D. registered \ mentioned ⊆ allow-list: EVERY registered tool must be mentioned in the persona,
|
|
148
|
+
// unless its name is listed here as a conscious exception (the allow-list is empty by default, so a
|
|
149
|
+
// newly registered tool cannot slip in unnamed). An entry must be a REAL registered tool, and a
|
|
150
|
+
// preset may also pin names that must be mentioned (never allow-listed).
|
|
151
|
+
allowList: [],
|
|
152
|
+
required: [],
|
|
139
153
|
lean: { tools: ['vibe_math_lean_run', 'vibe_math_lean_archive', 'vibe_math_lean_lib', 'vibe_math_lean_read', 'vibe_math_lean_job'], extra: [] },
|
|
140
154
|
// the shared math-computation tool (docs/math-computation.md) is prefix-less by
|
|
141
155
|
// design (INTERFACE-FREEZE.md §3), so it is pinned explicitly here.
|
|
@@ -146,11 +160,11 @@ const PRESETS = [
|
|
|
146
160
|
js: 'vibe-math-v3.js',
|
|
147
161
|
prefix: 'vibe_math_',
|
|
148
162
|
tools: 36,
|
|
149
|
-
// `vibe_math_sync_meta` is
|
|
150
|
-
//
|
|
151
|
-
//
|
|
152
|
-
|
|
153
|
-
|
|
163
|
+
// `vibe_math_sync_meta` is named by the promotion-contract line, and the write-lock pair is now
|
|
164
|
+
// named too (the lock is described in the persona, so leaving the two tools unmentioned was the
|
|
165
|
+
// last hole). Both stay REQUIRED: they must be mentioned, never allow-listed.
|
|
166
|
+
allowList: [],
|
|
167
|
+
required: ['vibe_math_claim_write', 'vibe_math_release_write'],
|
|
154
168
|
lean: { tools: ['vibe_math_lean_run', 'vibe_math_lean_archive', 'vibe_math_lean_lib', 'vibe_math_lean_read', 'vibe_math_lean_job'], extra: [] },
|
|
155
169
|
// the shared math-computation tool (docs/math-computation.md) is prefix-less by
|
|
156
170
|
// design (INTERFACE-FREEZE.md §3), so it is pinned explicitly here.
|
|
@@ -161,15 +175,18 @@ const PRESETS = [
|
|
|
161
175
|
js: 'vibe-math-v4.js',
|
|
162
176
|
prefix: 'vibe_v4_',
|
|
163
177
|
tools: 35,
|
|
164
|
-
//
|
|
165
|
-
//
|
|
166
|
-
|
|
178
|
+
// Member-facing tools (mail, library cards, task board, write lock) that the v4 persona describes
|
|
179
|
+
// GENERICALLY rather than naming one by one; the coordinator drives residents through
|
|
180
|
+
// vibe_v4_message / _meeting / _add_member. Allow-listed ON PURPOSE, so a NEWLY registered tool
|
|
181
|
+
// still has to be named (or added here consciously).
|
|
182
|
+
allowList: [
|
|
167
183
|
'vibe_v4_send_message', 'vibe_v4_publish_progress', 'vibe_v4_record_proposition',
|
|
168
184
|
'vibe_v4_record_method', 'vibe_v4_record_subproblem', 'vibe_v4_read_progress',
|
|
169
185
|
'vibe_v4_list_residents', 'vibe_v4_propose_task', 'vibe_v4_claim_task',
|
|
170
186
|
'vibe_v4_task_done', 'vibe_v4_list_tasks', 'vibe_v4_report_context',
|
|
171
187
|
'vibe_v4_claim_write', 'vibe_v4_release_write',
|
|
172
188
|
],
|
|
189
|
+
required: [],
|
|
173
190
|
lean: { tools: ['vibe_v4_lean_run', 'vibe_v4_lean_archive', 'vibe_v4_lean_lib', 'vibe_v4_lean_read', 'vibe_v4_lean_job'], extra: ['vibe_v4_formal_report'] },
|
|
174
191
|
// the shared math-computation tool (docs/math-computation.md) is prefix-less by
|
|
175
192
|
// design (INTERFACE-FREEZE.md §3), so it is pinned explicitly here.
|
|
@@ -180,17 +197,18 @@ const PRESETS = [
|
|
|
180
197
|
js: 'vibe-math-v5.js',
|
|
181
198
|
prefix: 'vibe_v5_',
|
|
182
199
|
tools: 40,
|
|
183
|
-
//
|
|
184
|
-
// institute-level controls plus the hiring authority. The final-paper pair is
|
|
185
|
-
// OFFICE control and IS named in the persona (vibe_v5_paper / vibe_v5_finalize_paper),
|
|
186
|
-
//
|
|
187
|
-
|
|
200
|
+
// Member- and academician-facing tools that the v5 persona describes GENERICALLY; the office (main
|
|
201
|
+
// agent) holds only the institute-level controls plus the hiring authority. The final-paper pair is
|
|
202
|
+
// an OFFICE control and IS named in the persona (vibe_v5_paper / vibe_v5_finalize_paper), so it must
|
|
203
|
+
// NOT be allow-listed here. Allow-listed ON PURPOSE: a NEWLY registered tool still has to be named.
|
|
204
|
+
allowList: [
|
|
188
205
|
'vibe_v5_wait', 'vibe_v5_record_progress', 'vibe_v5_record_proposition',
|
|
189
206
|
'vibe_v5_record_method', 'vibe_v5_record_subproblem', 'vibe_v5_read_library',
|
|
190
207
|
'vibe_v5_propose_verify', 'vibe_v5_verdict', 'vibe_v5_task_create',
|
|
191
208
|
'vibe_v5_task_list', 'vibe_v5_task_get', 'vibe_v5_task_update',
|
|
192
209
|
'vibe_v5_overview', 'vibe_v5_assign', 'vibe_v5_prioritize', 'vibe_v5_nudge',
|
|
193
210
|
],
|
|
211
|
+
required: [],
|
|
194
212
|
lean: { tools: ['vibe_v5_lean_run', 'vibe_v5_lean_archive', 'vibe_v5_lean_lib', 'vibe_v5_lean_read', 'vibe_v5_lean_job'], extra: [] },
|
|
195
213
|
// the shared math-computation tool (docs/math-computation.md) is prefix-less by
|
|
196
214
|
// design (INTERFACE-FREEZE.md §3), so it is pinned explicitly here.
|
|
@@ -264,8 +282,21 @@ for (const P of PRESETS) {
|
|
|
264
282
|
for (const w of wildcards) {
|
|
265
283
|
ok([...registered].some((n) => n.startsWith(w)), `${P.dir} [${key}]: wildcard mention ${w}* matches no registered tool`)
|
|
266
284
|
}
|
|
267
|
-
const
|
|
268
|
-
|
|
285
|
+
const unmentioned = [...registered].filter((n) => !mentioned.has(n)).sort()
|
|
286
|
+
// ALLOW-LIST design (replaces the old "equals a reviewed snapshot" equality): a registered tool may
|
|
287
|
+
// be absent from the persona ONLY if its name is consciously allow-listed for this preset. The
|
|
288
|
+
// allow-list is empty by default (v2/v3), so a newly registered tool turns this red until it is
|
|
289
|
+
// either mentioned in the persona or deliberately added to the list with a reason.
|
|
290
|
+
const allowed = new Set(P.allowList || [])
|
|
291
|
+
const illegal = unmentioned.filter((n) => !allowed.has(n))
|
|
292
|
+
ok(illegal.length === 0, `${P.dir} [${key}]: every registered tool is mentioned in the persona, or explicitly allow-listed`,
|
|
293
|
+
'unmentioned and not allow-listed: ' + JSON.stringify(illegal))
|
|
294
|
+
const staleAllowed = [...allowed].filter((n) => !registered.has(n)).sort()
|
|
295
|
+
ok(staleAllowed.length === 0, `${P.dir} [${key}]: every allow-list entry is a real registered tool`,
|
|
296
|
+
'allow-listed but not registered: ' + JSON.stringify(staleAllowed))
|
|
297
|
+
for (const must of (P.required || [])) {
|
|
298
|
+
ok(mentioned.has(must) && !allowed.has(must), `${P.dir} [${key}]: ${must} must be MENTIONED in the persona (never allow-listed)`)
|
|
299
|
+
}
|
|
269
300
|
}
|
|
270
301
|
|
|
271
302
|
// ---- F. the slash-command surface -------------------------------------
|
|
@@ -360,8 +391,18 @@ try {
|
|
|
360
391
|
'### config.prefix', '', '```text', r.prefix, '```', '', '### config.text', '', '```text', r.text, '```', '')
|
|
361
392
|
}
|
|
362
393
|
mkdirSync(corpusDir, { recursive: true })
|
|
363
|
-
|
|
364
|
-
|
|
394
|
+
// CONCURRENCY FIX: `audit-persona-sensitivity` spawns this suite with PERSONA_ROOT, and those child
|
|
395
|
+
// runs READ these two shipped files while a parallel `run-tests` job may be WRITING them. A plain
|
|
396
|
+
// writeFileSync let a reader observe a half-written JSON (the flake: sensitivity red only under
|
|
397
|
+
// concurrency 4, SETUP-FAIL in its control run). Write a unique temp file in the same directory and
|
|
398
|
+
// rename it into place, so a reader sees either the old or the new file, never a partial one.
|
|
399
|
+
const writeAtomic = (target, data) => {
|
|
400
|
+
const tmp = target + '.tmp-' + process.pid + '-' + Math.random().toString(36).slice(2)
|
|
401
|
+
writeFileSync(tmp, data, 'utf8')
|
|
402
|
+
renameSync(tmp, target)
|
|
403
|
+
}
|
|
404
|
+
writeAtomic(join(corpusDir, 'persona-corpus.md'), md.join('\n'))
|
|
405
|
+
writeAtomic(join(corpusDir, 'persona-corpus.json'), JSON.stringify({ presets: rows }, null, 2) + '\n')
|
|
365
406
|
}
|
|
366
407
|
// Round-trip check: the shipped corpus must name all four presets and carry line 0 of each block.
|
|
367
408
|
// Skipped under PERSONA_ROOT: there `rows` describe the mutated copy while the corpus on disk
|
|
@@ -0,0 +1,143 @@
|
|
|
1
|
+
// ============================================================================
|
|
2
|
+
// audit-prompt-duplication.mjs — shipped invariant for v2/v3/v4 (v5 has its own,
|
|
3
|
+
// tests/audit-v5-prompt-duplication.mjs): every preset's injected toolkit prompt
|
|
4
|
+
// text exists TWICE in its `agent.cordis.yml` (two agent-instructions entries:
|
|
5
|
+
// office + members). In these three presets the SECOND entry continues with more
|
|
6
|
+
// YAML after the shared toolkit text, so the invariant is stated as:
|
|
7
|
+
// (1) the two copies share a substantial COMMON PREFIX of prompt lines (≥ 60% of
|
|
8
|
+
// the shorter copy, and every line the first copy adds beyond the prefix is
|
|
9
|
+
// pure YAML scaffolding — never prompt prose), and
|
|
10
|
+
// (2) the review-10 clauses (V2-1 disclosures, V2-3 path bases) are present inside
|
|
11
|
+
// that shared prefix, i.e. in BOTH copies.
|
|
12
|
+
// No YAML anchor is used on purpose (it would change what the loader expands).
|
|
13
|
+
// Falsifiability: `tests/audit-prompt-duplication.mutants.mjs` (single-copy edit ⇒ named red).
|
|
14
|
+
// Run: node tests/audit-prompt-duplication.mjs [--json]
|
|
15
|
+
// ============================================================================
|
|
16
|
+
import { readFileSync, existsSync } from 'node:fs'
|
|
17
|
+
import { fileURLToPath } from 'node:url'
|
|
18
|
+
import { dirname, join } from 'node:path'
|
|
19
|
+
|
|
20
|
+
const HERE = dirname(fileURLToPath(import.meta.url))
|
|
21
|
+
const REPO = join(HERE, '..')
|
|
22
|
+
const PRESETS = ['vibe-math-v2', 'vibe-math-v3', 'vibe-math-v4']
|
|
23
|
+
// In-memory single-site mutation for the mutants driver: [preset, from, to] (first occurrence only).
|
|
24
|
+
const MUT = process.env.PROMPT_DUP_MUTATE ? JSON.parse(process.env.PROMPT_DUP_MUTATE) : null
|
|
25
|
+
const SKIP_CLAUSES = process.env.PROMPT_DUP_SKIP_CLAUSES === '1'
|
|
26
|
+
|
|
27
|
+
const PATH_CLAUSES = [
|
|
28
|
+
['V2-3 base statement', /every path here is resolved against the SESSION WORKING DIRECTORY/],
|
|
29
|
+
['V2-3 short-form equivalence', /names the SAME file/],
|
|
30
|
+
]
|
|
31
|
+
// Clause expectations are CODE-TRUTH-DRIVEN, not global: a DISCLOSURE is required only in presets
|
|
32
|
+
// whose code actually implements that behaviour, and the guard checks BOTH directions (prompt
|
|
33
|
+
// declares it ⇔ the JS carries the behaviour marker, re-measured every run). Each entry is
|
|
34
|
+
// [name, prompt regex, code marker regex, which presets the code justifies]. A preset whose JS lacks
|
|
35
|
+
// the marker must NOT declare the clause (`must NOT be declared here`) — declaring a behaviour that
|
|
36
|
+
// does not exist is worse than the undisclosed-behaviour gap it was meant to close.
|
|
37
|
+
const DISCLOSURES = [
|
|
38
|
+
// window 1 (V2-1 ①–③)
|
|
39
|
+
['V2-1 paper force-finalise / reaping disclosure', /DISCLOSURE \(existing behaviour\): the FINAL PAPER/, /reapedThisRun|forcedAfterCap/, ['v2', 'v3']],
|
|
40
|
+
['V2-1 auto-resolve disclosure', /DISCLOSURE \(existing behaviour\): switching the mode to `auto`/, /auto-resolved/, ['v2', 'v3']],
|
|
41
|
+
['V2-1 progress-push disclosure', /DISCLOSURE \(existing behaviour\): the framework itself pushes a progress/, /\u8fdb\u5ea6\u66f4\u65b0\uff1a/, ['v2', 'v3']],
|
|
42
|
+
// window 2 (V2-1 ④–⑦)
|
|
43
|
+
['V2-1 project-lock takeover disclosure', /DISCLOSURE \(existing behaviour\): a second session can TAKE OVER/, /lock taken over/, ['v3']],
|
|
44
|
+
['V2-1 auto-reprioritisation disclosure', /DISCLOSURE \(existing behaviour\): the framework can adjust problem\/proposition priorities/, /priorities auto-adjusted/, ['v2', 'v3']],
|
|
45
|
+
['V2-1 math-policy refusal disclosure', /DISCLOSURE \(existing behaviour\): when mathMode is `typed`/, /mathMode|typed\+shell/, ['v2', 'v3', 'v4']],
|
|
46
|
+
['V2-3e math-policy line', /math_computation policy: mathComputation/, /mathComputation/, ['v2', 'v3', 'v4']],
|
|
47
|
+
['V2-1 post-compaction recap disclosure', /DISCLOSURE \(existing behaviour\): after a real context compaction the framework injects/, /\u6838\u5fc3\u89c4\u5219\u91cd\u7533|CONTEXT COMPACT/, ['v4']],
|
|
48
|
+
]
|
|
49
|
+
const PRESET_KEY = { 'vibe-math-v2': 'v2', 'vibe-math-v3': 'v3', 'vibe-math-v4': 'v4' }
|
|
50
|
+
// Declaration-syntax scanner (§9.7⑨): anchored on the SHAPE of a declaration, not on a substring.
|
|
51
|
+
const NAME_RE = '[a-zA-Z][A-Za-z0-9]*'
|
|
52
|
+
const DECL_BULLET = new RegExp('^\\s*-\\s+`?(' + NAME_RE + ')`?\\s*=\\s*\\S')
|
|
53
|
+
const DECL_LIST = new RegExp('^\\s*(?:' + NAME_RE + '\\s*=\\s*[^,\\n]{1,40})(?:\\s*[,,]\\s*' + NAME_RE + '\\s*=\\s*[^,\\n]{1,40})+\\s*$')
|
|
54
|
+
function declarations(rawLines) {
|
|
55
|
+
const out = []
|
|
56
|
+
rawLines.forEach((raw, i) => {
|
|
57
|
+
const line = raw.replace(/\r$/, '')
|
|
58
|
+
const m1 = DECL_BULLET.exec(line)
|
|
59
|
+
if (m1) { out.push({ name: m1[1], line: i + 1 }); return }
|
|
60
|
+
if (DECL_LIST.test(line)) for (const m of line.matchAll(new RegExp('\\b(' + NAME_RE + ')\\s*=\\s*', 'g'))) out.push({ name: m[1], line: i + 1 })
|
|
61
|
+
})
|
|
62
|
+
return out
|
|
63
|
+
}
|
|
64
|
+
const normLine = (l) => l.replace(/\s+/g, ' ').replace(/\{\{cwd\}\}|\{\{model\}\}/g, '<T>').trim()
|
|
65
|
+
const SCAFFOLD = /^(-|#|\s*$)|^(suffix|text|priority|config|name|id|content|role):|^You are a coding agent powered by/
|
|
66
|
+
|
|
67
|
+
const failures = []
|
|
68
|
+
let passed = 0
|
|
69
|
+
const check = (cond, msg) => { if (cond) passed += 1; else failures.push(msg) }
|
|
70
|
+
const report = []
|
|
71
|
+
|
|
72
|
+
for (const preset of PRESETS) {
|
|
73
|
+
const rel = preset + '/agent.cordis.yml'
|
|
74
|
+
const abs = join(REPO, rel)
|
|
75
|
+
if (!existsSync(abs)) { check(false, rel + ' is missing'); continue }
|
|
76
|
+
let text = readFileSync(abs, 'utf8')
|
|
77
|
+
if (MUT && MUT[0] === preset) {
|
|
78
|
+
const i = text.indexOf(MUT[1])
|
|
79
|
+
if (i === -1) { check(false, preset + ': mutant anchor not found (harness problem)'); continue }
|
|
80
|
+
text = text.slice(0, i) + MUT[2] + text.slice(i + MUT[1].length)
|
|
81
|
+
}
|
|
82
|
+
const lines = text.split('\n')
|
|
83
|
+
const starts = []
|
|
84
|
+
lines.forEach((l, i) => { if (/## Vibe Math V\d+ toolkit/.test(l) || /This session includes "Vibe Math V/.test(l)) starts.push(i) })
|
|
85
|
+
const heads = []
|
|
86
|
+
for (const s of starts) if (!heads.length || s - heads[heads.length - 1] > 3) heads.push(s)
|
|
87
|
+
check(heads.length === 2, preset + ': exactly TWO toolkit prompt copies (got ' + heads.length + ' at ' + JSON.stringify(heads.map((h) => h + 1)) + ')')
|
|
88
|
+
if (heads.length !== 2) { report.push({ preset, copies: heads.length }); continue }
|
|
89
|
+
const A = lines.slice(heads[0], heads[1]).map(normLine).filter((x) => x)
|
|
90
|
+
const B = lines.slice(heads[1]).map(normLine).filter((x) => x)
|
|
91
|
+
let common = 0
|
|
92
|
+
while (common < A.length && common < B.length && A[common] === B[common]) common += 1
|
|
93
|
+
const shared = A.slice(0, common).join(' ')
|
|
94
|
+
const tailA = A.slice(common)
|
|
95
|
+
const shorter = Math.min(A.length, B.length)
|
|
96
|
+
const ratio = common / Math.max(1, shorter)
|
|
97
|
+
check(ratio >= 0.6, preset + ': the two copies share a substantial common prefix (' + common + '/' + shorter + ' = ' + ratio.toFixed(2) + ' ≥ 0.60)')
|
|
98
|
+
const proseTail = tailA.filter((l) => !SCAFFOLD.test(l))
|
|
99
|
+
check(proseTail.length === 0, preset + ': everything the first copy adds beyond the shared prefix is YAML scaffolding, not prompt prose (' + proseTail.length + ' prose line(s) left'
|
|
100
|
+
+ (proseTail.length ? ': ' + JSON.stringify(proseTail.slice(0, 2).map((x) => x.slice(0, 60))) : '') + ')')
|
|
101
|
+
if (!SKIP_CLAUSES) {
|
|
102
|
+
for (const [name, re] of PATH_CLAUSES) check(re.test(shared), preset + ': ' + name + ' present in BOTH copies (inside the shared prefix)')
|
|
103
|
+
// bidirectional: the prompt declares the behaviour ⇔ the JS carries its marker
|
|
104
|
+
const jsAbs = join(REPO, preset, preset + '.js')
|
|
105
|
+
const js = existsSync(jsAbs) ? readFileSync(jsAbs, 'utf8') : ''
|
|
106
|
+
check(!!js, preset + ': the JS source is readable for the code-truth check')
|
|
107
|
+
DISCLOSURES.forEach(([name, promptRe, codeRe, applies]) => {
|
|
108
|
+
const expected = applies.indexOf(PRESET_KEY[preset]) !== -1
|
|
109
|
+
const inCode = codeRe.test(js)
|
|
110
|
+
check(inCode === expected, preset + ': code-truth for "' + name + '" (expected=' + expected + ', code marker present=' + inCode + ')')
|
|
111
|
+
if (expected) check(promptRe.test(shared), preset + ': ' + name + ' present in BOTH copies (inside the shared prefix)')
|
|
112
|
+
else check(!promptRe.test(shared), preset + ': ' + name + ' must NOT be declared here (the code does not implement it)')
|
|
113
|
+
})
|
|
114
|
+
// ── declaration-syntax scanner (§9.7⑨) ────────────────────────────────────────────────
|
|
115
|
+
// A parameter the prompt PRESENTS AS A DECLARATION must be readable as `params.<name>` in that
|
|
116
|
+
// preset's JS, and must be declared in BOTH copies. The scan anchors on the declaration's
|
|
117
|
+
// SYNTAX (a bullet starting with `- <name> = …`, or a pure `name = value, name2 = value2`
|
|
118
|
+
// list), never on a substring: a mention inside a disclosure sentence must not count as a
|
|
119
|
+
// declaration. On today's prompts this scan is EMPTY by construction (vacuous-by-construction);
|
|
120
|
+
// the mutant driver injects a declaration line into ONE copy to prove the scan bites.
|
|
121
|
+
const declA = declarations(lines.slice(heads[0], heads[1]))
|
|
122
|
+
const declB = declarations(lines.slice(heads[1]))
|
|
123
|
+
const namesA = [...new Set(declA.map((d) => d.name))]
|
|
124
|
+
const namesB = new Set(declB.map((d) => d.name))
|
|
125
|
+
for (const name of namesA) {
|
|
126
|
+
check(namesB.has(name), preset + ': parameter "' + name + '" declared in copy 1 (L' + declA.find((d) => d.name === name).line + ' of its slice) must also be declared in copy 2')
|
|
127
|
+
check(new RegExp('params\\.' + name + '\\b').test(js), preset + ': parameter "' + name + '" is presented as a declaration but the JS never reads params.' + name)
|
|
128
|
+
}
|
|
129
|
+
for (const d of declB) {
|
|
130
|
+
if (!namesA.includes(d.name)) check(false, preset + ': parameter "' + d.name + '" is declared in copy 2 only (L' + d.line + ' of its slice) — both copies must agree')
|
|
131
|
+
}
|
|
132
|
+
report.push({ preset, copies: 2, sharedLines: common, ratio: Number(ratio.toFixed(3)), declarations: namesA.length })
|
|
133
|
+
}
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
if (process.argv.includes('--json')) console.log(JSON.stringify({ passed, failed: failures.length, failures, report }))
|
|
137
|
+
else {
|
|
138
|
+
console.log('v2/v3/v4 prompt duplication audit')
|
|
139
|
+
for (const r of report) console.log(' ' + r.preset + ': copies=' + (r.copies || 2) + ' sharedLines=' + r.sharedLines + ' ratio=' + r.ratio + ' declaredParams=' + (r.declarations === undefined ? '-' : r.declarations))
|
|
140
|
+
for (const f of failures) console.log(' FAIL - ' + f)
|
|
141
|
+
console.log((failures.length ? 'FAILED' : 'ALL GREEN') + ' — passed=' + passed + ' failed=' + failures.length)
|
|
142
|
+
}
|
|
143
|
+
process.exit(failures.length ? 1 : 0)
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
// Falsifiability driver for tests/audit-prompt-duplication.mjs (v2/v3/v4): a SINGLE-SITE edit in ONE
|
|
2
|
+
// prompt copy of ONE preset must turn both the equality check and the clause check NAMED-red, while
|
|
3
|
+
// the unmutated control stays green.
|
|
4
|
+
import { spawnSync } from 'node:child_process'
|
|
5
|
+
const SUITE = 'D:/wd/vibemath开发/Vibe-Mathematics/tests/audit-prompt-duplication.mjs'
|
|
6
|
+
const run = (env) => {
|
|
7
|
+
const r = spawnSync(process.execPath, [SUITE, '--json'], { encoding: 'utf8', timeout: 120000, env: Object.assign({}, process.env, env || {}) })
|
|
8
|
+
let parsed = null
|
|
9
|
+
try { parsed = JSON.parse(r.stdout) } catch (e) { /* reported */ }
|
|
10
|
+
return { code: r.status, parsed, out: String(r.stdout || '') + String(r.stderr || '') }
|
|
11
|
+
}
|
|
12
|
+
const control = run({ PROMPT_DUP_SKIP_CLAUSES: '1' }) // duplication-only control (clauses land with the prompt edits)
|
|
13
|
+
const MUTS = [
|
|
14
|
+
['v2 single-copy drift', ['vibe-math-v2', 'Data lives under {{cwd}}/VibeMath/Projects/<project>/', 'Data lives under {{cwd}}/VibeMath/Projects/<project>/ (mutated in ONE copy only)']],
|
|
15
|
+
['v3 single-copy drift', ['vibe-math-v3', 'TRUST RULE: only Verified/', 'TRUST RULE (mutated in ONE copy only): only Verified/']],
|
|
16
|
+
['v4 single-copy drift', ['vibe-math-v4', 'TRUST RULE: only Verified/', 'TRUST RULE (mutated in ONE copy only): only Verified/']],
|
|
17
|
+
]
|
|
18
|
+
// Declaration-syntax scanner self-probe (§9.7⑨): inject ONE declaration-syntax line into ONE copy at
|
|
19
|
+
// that preset's own anchor. v2/v3 read params.activityLogCap ⇒ the single-copy rule must fire; v4
|
|
20
|
+
// never reads it ⇒ the code-truth rule must fire.
|
|
21
|
+
const DECL_MUTS = [
|
|
22
|
+
['v2 declaration line (one copy)', ['vibe-math-v2', 'When the user asks about progress, call vibe_math_report and summarize in plain language.', 'When the user asks about progress, call vibe_math_report and summarize in plain language.\r\n - activityLogCap = 100']],
|
|
23
|
+
['v3 declaration line (one copy)', ['vibe-math-v3', 'When the user asks about progress, call vibe_math_report and summarize in plain language.', 'When the user asks about progress, call vibe_math_report and summarize in plain language.\r\n - activityLogCap = 100']],
|
|
24
|
+
['v4 declaration line (unread field)', ['vibe-math-v4', 'When the user asks about progress, call vibe_v4_report and summarize in plain language.', 'When the user asks about progress, call vibe_v4_report and summarize in plain language.\r\n - activityLogCap = 100']],
|
|
25
|
+
]
|
|
26
|
+
let problems = 0
|
|
27
|
+
console.log('control (duplication-only, clauses skipped): exit=' + control.code + ' | ' + JSON.stringify(control.parsed && { passed: control.parsed.passed, failed: control.parsed.failed })
|
|
28
|
+
+ (control.parsed && control.parsed.report ? ' | ' + control.parsed.report.map((r) => r.preset + ':ratio=' + r.ratio).join(' ') : ''))
|
|
29
|
+
if (control.code !== 0) problems += 1
|
|
30
|
+
const NAMED = /share a substantial common prefix|present in BOTH copies|scaffolding, not prompt prose/
|
|
31
|
+
for (const [name, mut] of MUTS) {
|
|
32
|
+
const r = run({ PROMPT_DUP_MUTATE: JSON.stringify(mut), PROMPT_DUP_SKIP_CLAUSES: '1' })
|
|
33
|
+
const named = !!(r.parsed && (r.parsed.failures || []).some((f) => NAMED.test(f)))
|
|
34
|
+
const ok = r.code !== 0 && named
|
|
35
|
+
if (!ok) problems += 1
|
|
36
|
+
console.log(name.padEnd(24) + ': exit=' + r.code + (ok ? ' RED (as required)' : ' NOT RED'))
|
|
37
|
+
for (const f of ((r.parsed && r.parsed.failures) || []).slice(0, 1)) console.log(' → ' + String(f).slice(0, 150))
|
|
38
|
+
}
|
|
39
|
+
// declaration-syntax scanner probes run in DEFAULT mode (the scan lives in the clause block)
|
|
40
|
+
const DECL_NAMED = /must also be declared in copy 2|never reads params\.|declared in copy 2 only/
|
|
41
|
+
for (const [name, mut] of DECL_MUTS) {
|
|
42
|
+
const r = run({ PROMPT_DUP_MUTATE: JSON.stringify(mut) })
|
|
43
|
+
const named = !!(r.parsed && (r.parsed.failures || []).some((f) => DECL_NAMED.test(f)))
|
|
44
|
+
const ok = r.code !== 0 && named
|
|
45
|
+
if (!ok) problems += 1
|
|
46
|
+
console.log(name.padEnd(32) + ': exit=' + r.code + (ok ? ' RED (as required)' : ' NOT RED'))
|
|
47
|
+
for (const f of ((r.parsed && r.parsed.failures) || []).filter((f) => DECL_NAMED.test(f)).slice(0, 2)) console.log(' → ' + String(f).slice(0, 160))
|
|
48
|
+
}
|
|
49
|
+
console.log(problems === 0 ? '\nPROMPT-DUPLICATION (v2/v3/v4) MUTANTS: every single-copy edit turns the NAMED check red' : '\nPROMPT-DUPLICATION MUTANTS: ' + problems + ' problem(s)')
|
|
50
|
+
process.exit(problems === 0 ? 0 : 1)
|