dsh-vibe-math 2.7.2 → 2.8.0

package/tests/audit-prompt-duplication.mjs

@@ -0,0 +1,143 @@
+// ============================================================================
+// audit-prompt-duplication.mjs — shipped invariant for v2/v3/v4 (v5 has its own,
+// tests/audit-v5-prompt-duplication.mjs): every preset's injected toolkit prompt
+// text exists TWICE in its `agent.cordis.yml` (two agent-instructions entries:
+// office + members). In these three presets the SECOND entry continues with more
+// YAML after the shared toolkit text, so the invariant is stated as:
+//   (1) the two copies share a substantial COMMON PREFIX of prompt lines (≥ 60% of
+//       the shorter copy, and every line the first copy adds beyond the prefix is
+//       pure YAML scaffolding — never prompt prose), and
+//   (2) the review-10 clauses (V2-1 disclosures, V2-3 path bases) are present inside
+//       that shared prefix, i.e. in BOTH copies.
+// No YAML anchor is used on purpose (it would change what the loader expands).
+// Falsifiability: `tests/audit-prompt-duplication.mutants.mjs` (single-copy edit ⇒ named red).
+// Run: node tests/audit-prompt-duplication.mjs [--json]
+// ============================================================================
+import { readFileSync, existsSync } from 'node:fs'
+import { fileURLToPath } from 'node:url'
+import { dirname, join } from 'node:path'
+
+const HERE = dirname(fileURLToPath(import.meta.url))
+const REPO = join(HERE, '..')
+const PRESETS = ['vibe-math-v2', 'vibe-math-v3', 'vibe-math-v4']
+// In-memory single-site mutation for the mutants driver: [preset, from, to] (first occurrence only).
+const MUT = process.env.PROMPT_DUP_MUTATE ? JSON.parse(process.env.PROMPT_DUP_MUTATE) : null
+const SKIP_CLAUSES = process.env.PROMPT_DUP_SKIP_CLAUSES === '1'
+
+const PATH_CLAUSES = [
+  ['V2-3 base statement', /every path here is resolved against the SESSION WORKING DIRECTORY/],
+  ['V2-3 short-form equivalence', /names the SAME file/],
+]
+// Clause expectations are CODE-TRUTH-DRIVEN, not global: a DISCLOSURE is required only in presets
+// whose code actually implements that behaviour, and the guard checks BOTH directions (prompt
+// declares it ⇔ the JS carries the behaviour marker, re-measured every run). Each entry is
+// [name, prompt regex, code marker regex, which presets the code justifies]. A preset whose JS lacks
+// the marker must NOT declare the clause (`must NOT be declared here`) — declaring a behaviour that
+// does not exist is worse than the undisclosed-behaviour gap it was meant to close.
+const DISCLOSURES = [
+  // window 1 (V2-1 ①–③)
+  ['V2-1 paper force-finalise / reaping disclosure', /DISCLOSURE \(existing behaviour\): the FINAL PAPER/, /reapedThisRun|forcedAfterCap/, ['v2', 'v3']],
+  ['V2-1 auto-resolve disclosure', /DISCLOSURE \(existing behaviour\): switching the mode to `auto`/, /auto-resolved/, ['v2', 'v3']],
+  ['V2-1 progress-push disclosure', /DISCLOSURE \(existing behaviour\): the framework itself pushes a progress/, /\u8fdb\u5ea6\u66f4\u65b0\uff1a/, ['v2', 'v3']],
+  // window 2 (V2-1 ④–⑦)
+  ['V2-1 project-lock takeover disclosure', /DISCLOSURE \(existing behaviour\): a second session can TAKE OVER/, /lock taken over/, ['v3']],
+  ['V2-1 auto-reprioritisation disclosure', /DISCLOSURE \(existing behaviour\): the framework can adjust problem\/proposition priorities/, /priorities auto-adjusted/, ['v2', 'v3']],
+  ['V2-1 math-policy refusal disclosure', /DISCLOSURE \(existing behaviour\): when mathMode is `typed`/, /mathMode|typed\+shell/, ['v2', 'v3', 'v4']],
+  ['V2-3e math-policy line', /math_computation policy: mathComputation/, /mathComputation/, ['v2', 'v3', 'v4']],
+  ['V2-1 post-compaction recap disclosure', /DISCLOSURE \(existing behaviour\): after a real context compaction the framework injects/, /\u6838\u5fc3\u89c4\u5219\u91cd\u7533|CONTEXT COMPACT/, ['v4']],
+]
+const PRESET_KEY = { 'vibe-math-v2': 'v2', 'vibe-math-v3': 'v3', 'vibe-math-v4': 'v4' }
+// Declaration-syntax scanner (§9.7⑨): anchored on the SHAPE of a declaration, not on a substring.
+const NAME_RE = '[a-zA-Z][A-Za-z0-9]*'
+const DECL_BULLET = new RegExp('^\\s*-\\s+`?(' + NAME_RE + ')`?\\s*=\\s*\\S')
+const DECL_LIST = new RegExp('^\\s*(?:' + NAME_RE + '\\s*=\\s*[^,\\n]{1,40})(?:\\s*[,，]\\s*' + NAME_RE + '\\s*=\\s*[^,\\n]{1,40})+\\s*$')
+function declarations(rawLines) {
+  const out = []
+  rawLines.forEach((raw, i) => {
+    const line = raw.replace(/\r$/, '')
+    const m1 = DECL_BULLET.exec(line)
+    if (m1) { out.push({ name: m1[1], line: i + 1 }); return }
+    if (DECL_LIST.test(line)) for (const m of line.matchAll(new RegExp('\\b(' + NAME_RE + ')\\s*=\\s*', 'g'))) out.push({ name: m[1], line: i + 1 })
+  })
+  return out
+}
+const normLine = (l) => l.replace(/\s+/g, ' ').replace(/\{\{cwd\}\}|\{\{model\}\}/g, '<T>').trim()
+const SCAFFOLD = /^(-|#|\s*$)|^(suffix|text|priority|config|name|id|content|role):|^You are a coding agent powered by/
+
+const failures = []
+let passed = 0
+const check = (cond, msg) => { if (cond) passed += 1; else failures.push(msg) }
+const report = []
+
+for (const preset of PRESETS) {
+  const rel = preset + '/agent.cordis.yml'
+  const abs = join(REPO, rel)
+  if (!existsSync(abs)) { check(false, rel + ' is missing'); continue }
+  let text = readFileSync(abs, 'utf8')
+  if (MUT && MUT[0] === preset) {
+    const i = text.indexOf(MUT[1])
+    if (i === -1) { check(false, preset + ': mutant anchor not found (harness problem)'); continue }
+    text = text.slice(0, i) + MUT[2] + text.slice(i + MUT[1].length)
+  }
+  const lines = text.split('\n')
+  const starts = []
+  lines.forEach((l, i) => { if (/## Vibe Math V\d+ toolkit/.test(l) || /This session includes "Vibe Math V/.test(l)) starts.push(i) })
+  const heads = []
+  for (const s of starts) if (!heads.length || s - heads[heads.length - 1] > 3) heads.push(s)
+  check(heads.length === 2, preset + ': exactly TWO toolkit prompt copies (got ' + heads.length + ' at ' + JSON.stringify(heads.map((h) => h + 1)) + ')')
+  if (heads.length !== 2) { report.push({ preset, copies: heads.length }); continue }
+  const A = lines.slice(heads[0], heads[1]).map(normLine).filter((x) => x)
+  const B = lines.slice(heads[1]).map(normLine).filter((x) => x)
+  let common = 0
+  while (common < A.length && common < B.length && A[common] === B[common]) common += 1
+  const shared = A.slice(0, common).join(' ')
+  const tailA = A.slice(common)
+  const shorter = Math.min(A.length, B.length)
+  const ratio = common / Math.max(1, shorter)
+  check(ratio >= 0.6, preset + ': the two copies share a substantial common prefix (' + common + '/' + shorter + ' = ' + ratio.toFixed(2) + ' ≥ 0.60)')
+  const proseTail = tailA.filter((l) => !SCAFFOLD.test(l))
+  check(proseTail.length === 0, preset + ': everything the first copy adds beyond the shared prefix is YAML scaffolding, not prompt prose (' + proseTail.length + ' prose line(s) left'
+    + (proseTail.length ? ': ' + JSON.stringify(proseTail.slice(0, 2).map((x) => x.slice(0, 60))) : '') + ')')
+  if (!SKIP_CLAUSES) {
+    for (const [name, re] of PATH_CLAUSES) check(re.test(shared), preset + ': ' + name + ' present in BOTH copies (inside the shared prefix)')
+    // bidirectional: the prompt declares the behaviour ⇔ the JS carries its marker
+    const jsAbs = join(REPO, preset, preset + '.js')
+    const js = existsSync(jsAbs) ? readFileSync(jsAbs, 'utf8') : ''
+    check(!!js, preset + ': the JS source is readable for the code-truth check')
+    DISCLOSURES.forEach(([name, promptRe, codeRe, applies]) => {
+      const expected = applies.indexOf(PRESET_KEY[preset]) !== -1
+      const inCode = codeRe.test(js)
+      check(inCode === expected, preset + ': code-truth for "' + name + '" (expected=' + expected + ', code marker present=' + inCode + ')')
+      if (expected) check(promptRe.test(shared), preset + ': ' + name + ' present in BOTH copies (inside the shared prefix)')
+      else check(!promptRe.test(shared), preset + ': ' + name + ' must NOT be declared here (the code does not implement it)')
+    })
+    // ── declaration-syntax scanner (§9.7⑨) ────────────────────────────────────────────────
+    // A parameter the prompt PRESENTS AS A DECLARATION must be readable as `params.<name>` in that
+    // preset's JS, and must be declared in BOTH copies. The scan anchors on the declaration's
+    // SYNTAX (a bullet starting with `- <name> = …`, or a pure `name = value, name2 = value2`
+    // list), never on a substring: a mention inside a disclosure sentence must not count as a
+    // declaration. On today's prompts this scan is EMPTY by construction (vacuous-by-construction);
+    // the mutant driver injects a declaration line into ONE copy to prove the scan bites.
+    const declA = declarations(lines.slice(heads[0], heads[1]))
+    const declB = declarations(lines.slice(heads[1]))
+    const namesA = [...new Set(declA.map((d) => d.name))]
+    const namesB = new Set(declB.map((d) => d.name))
+    for (const name of namesA) {
+      check(namesB.has(name), preset + ': parameter "' + name + '" declared in copy 1 (L' + declA.find((d) => d.name === name).line + ' of its slice) must also be declared in copy 2')
+      check(new RegExp('params\\.' + name + '\\b').test(js), preset + ': parameter "' + name + '" is presented as a declaration but the JS never reads params.' + name)
+    }
+    for (const d of declB) {
+      if (!namesA.includes(d.name)) check(false, preset + ': parameter "' + d.name + '" is declared in copy 2 only (L' + d.line + ' of its slice) — both copies must agree')
+    }
+    report.push({ preset, copies: 2, sharedLines: common, ratio: Number(ratio.toFixed(3)), declarations: namesA.length })
+  }
+}
+
+if (process.argv.includes('--json')) console.log(JSON.stringify({ passed, failed: failures.length, failures, report }))
+else {
+  console.log('v2/v3/v4 prompt duplication audit')
+  for (const r of report) console.log('  ' + r.preset + ': copies=' + (r.copies || 2) + ' sharedLines=' + r.sharedLines + ' ratio=' + r.ratio + ' declaredParams=' + (r.declarations === undefined ? '-' : r.declarations))
+  for (const f of failures) console.log('  FAIL - ' + f)
+  console.log((failures.length ? 'FAILED' : 'ALL GREEN') + ' — passed=' + passed + ' failed=' + failures.length)
+}
+process.exit(failures.length ? 1 : 0)

package/tests/audit-prompt-duplication.mutants.mjs

@@ -0,0 +1,51 @@
+// Falsifiability driver for tests/audit-prompt-duplication.mjs (v2/v3/v4): a SINGLE-SITE edit in ONE
+// prompt copy of ONE preset must turn both the equality check and the clause check NAMED-red, while
+// the unmutated control stays green.
+import { spawnSync } from 'node:child_process'
+const SUITE = 'D:/wd/vibemath开发/Vibe-Mathematics/tests/audit-prompt-duplication.mjs'
+const run = (env) => {
+  const r = spawnSync(process.execPath, [SUITE, '--json'], { encoding: 'utf8', timeout: 120000, env: Object.assign({}, process.env, env || {}) })
+  let parsed = null
+  try { parsed = JSON.parse(r.stdout) } catch (e) { /* reported */ }
+  return { code: r.status, parsed, out: String(r.stdout || '') + String(r.stderr || '') }
+}
+const control = run({})   // task-9: with all four presets carrying the shared LaTeX-missing block, the clause
+                          // block is ACTIVE in the control too — a skipped clause check is a vacuous guard.
+const MUTS = [
+  ['v2 single-copy drift', ['vibe-math-v2', 'Data lives under {{cwd}}/VibeMath/Projects/<project>/', 'Data lives under {{cwd}}/VibeMath/Projects/<project>/ (mutated in ONE copy only)']],
+  ['v3 single-copy drift', ['vibe-math-v3', 'TRUST RULE: only Verified/', 'TRUST RULE (mutated in ONE copy only): only Verified/']],
+  ['v4 single-copy drift', ['vibe-math-v4', 'TRUST RULE: only Verified/', 'TRUST RULE (mutated in ONE copy only): only Verified/']],
+]
+// Declaration-syntax scanner self-probe (§9.7⑨): inject ONE declaration-syntax line into ONE copy at
+// that preset's own anchor. v2/v3 read params.activityLogCap ⇒ the single-copy rule must fire; v4
+// never reads it ⇒ the code-truth rule must fire.
+const DECL_MUTS = [
+  ['v2 declaration line (one copy)', ['vibe-math-v2', 'When the user asks about progress, call vibe_math_report and summarize in plain language.', 'When the user asks about progress, call vibe_math_report and summarize in plain language.\r\n        - activityLogCap = 100']],
+  ['v3 declaration line (one copy)', ['vibe-math-v3', 'When the user asks about progress, call vibe_math_report and summarize in plain language.', 'When the user asks about progress, call vibe_math_report and summarize in plain language.\r\n        - activityLogCap = 100']],
+  ['v4 declaration line (unread field)', ['vibe-math-v4', 'When the user asks about progress, call vibe_v4_report and summarize in plain language.', 'When the user asks about progress, call vibe_v4_report and summarize in plain language.\r\n        - activityLogCap = 100']],
+]
+let problems = 0
+console.log('control (duplication + clauses, default mode): exit=' + control.code + ' | ' + JSON.stringify(control.parsed && { passed: control.parsed.passed, failed: control.parsed.failed })
+  + (control.parsed && control.parsed.report ? ' | ' + control.parsed.report.map((r) => r.preset + ':ratio=' + r.ratio).join(' ') : ''))
+if (control.code !== 0) problems += 1
+const NAMED = /share a substantial common prefix|present in BOTH copies|scaffolding, not prompt prose/
+for (const [name, mut] of MUTS) {
+  const r = run({ PROMPT_DUP_MUTATE: JSON.stringify(mut), PROMPT_DUP_SKIP_CLAUSES: '1' })
+  const named = !!(r.parsed && (r.parsed.failures || []).some((f) => NAMED.test(f)))
+  const ok = r.code !== 0 && named
+  if (!ok) problems += 1
+  console.log(name.padEnd(24) + ': exit=' + r.code + (ok ? '  RED (as required)' : '  NOT RED'))
+  for (const f of ((r.parsed && r.parsed.failures) || []).slice(0, 1)) console.log('    → ' + String(f).slice(0, 150))
+}
+// declaration-syntax scanner probes run in DEFAULT mode (the scan lives in the clause block)
+const DECL_NAMED = /must also be declared in copy 2|never reads params\.|declared in copy 2 only/
+for (const [name, mut] of DECL_MUTS) {
+  const r = run({ PROMPT_DUP_MUTATE: JSON.stringify(mut) })
+  const named = !!(r.parsed && (r.parsed.failures || []).some((f) => DECL_NAMED.test(f)))
+  const ok = r.code !== 0 && named
+  if (!ok) problems += 1
+  console.log(name.padEnd(32) + ': exit=' + r.code + (ok ? '  RED (as required)' : '  NOT RED'))
+  for (const f of ((r.parsed && r.parsed.failures) || []).filter((f) => DECL_NAMED.test(f)).slice(0, 2)) console.log('    → ' + String(f).slice(0, 160))
+}
+console.log(problems === 0 ? '\nPROMPT-DUPLICATION (v2/v3/v4) MUTANTS: every single-copy edit turns the NAMED check red' : '\nPROMPT-DUPLICATION MUTANTS: ' + problems + ' problem(s)')
+process.exit(problems === 0 ? 0 : 1)