mindforge-cc 11.9.1 → 11.9.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (121) hide show
  1. package/.agent/CLAUDE.md +37 -13
  2. package/.agent/hooks/mindforge-block-no-verify.js +61 -13
  3. package/.agent/hooks/mindforge-config-protection.js +82 -3
  4. package/.agent/hooks/mindforge-context-monitor.js +1 -1
  5. package/.agent/hooks/mindforge-workflow-guard.js +2 -2
  6. package/.agent/hooks/run-with-flags.js +190 -20
  7. package/.agent/mindforge/browse.md +2 -2
  8. package/.agent/mindforge/checkpoint.md +1 -1
  9. package/.agent/mindforge/consult.md +1 -1
  10. package/.agent/mindforge/cost-report.md +1 -1
  11. package/.agent/mindforge/harness-audit.md +1 -1
  12. package/.agent/mindforge/orch-add-feature.md +1 -1
  13. package/.agent/mindforge/orch-build-mvp.md +1 -1
  14. package/.agent/mindforge/orch-change-feature.md +1 -1
  15. package/.agent/mindforge/orch-fix-defect.md +1 -1
  16. package/.agent/mindforge/orch-refine-code.md +1 -1
  17. package/.agent/mindforge/qa.md +2 -2
  18. package/.claude/CLAUDE.md +37 -13
  19. package/.claude/commands/mindforge/browse.md +2 -2
  20. package/.claude/commands/mindforge/checkpoint.md +1 -1
  21. package/.claude/commands/mindforge/consult.md +1 -1
  22. package/.claude/commands/mindforge/cost-report.md +1 -1
  23. package/.claude/commands/mindforge/harness-audit.md +1 -1
  24. package/.claude/commands/mindforge/orch-add-feature.md +1 -1
  25. package/.claude/commands/mindforge/orch-build-mvp.md +1 -1
  26. package/.claude/commands/mindforge/orch-change-feature.md +1 -1
  27. package/.claude/commands/mindforge/orch-fix-defect.md +1 -1
  28. package/.claude/commands/mindforge/orch-refine-code.md +1 -1
  29. package/.claude/commands/mindforge/qa.md +2 -2
  30. package/.mindforge/MINDFORGE-SCHEMA.json +126 -13
  31. package/.mindforge/config.json +4 -4
  32. package/.mindforge/engine/autonomous/headless-adapter.md +2 -2
  33. package/.mindforge/engine/cost-tracking/router.md +1 -1
  34. package/.mindforge/engine/cost-tracking/token-ledger.md +21 -24
  35. package/.mindforge/engine/temporal-protocol.md +2 -2
  36. package/.mindforge/governance/change-classifier.md +20 -4
  37. package/.mindforge/memory/sync-manifest.json +1 -1
  38. package/.mindforge/metrics/METRICS-SCHEMA.md +13 -4
  39. package/.mindforge/personas/cost-optimizer.md +2 -2
  40. package/.mindforge/personas/multi-model-bridge.md +1 -1
  41. package/.mindforge/skills/agent-architecture-audit/SKILL.md +2 -2
  42. package/.mindforge/skills/cost-aware-routing/SKILL.md +3 -3
  43. package/.mindforge/skills/multi-llm-consult/SKILL.md +2 -2
  44. package/.mindforge/skills/orch-pipeline/SKILL.md +4 -4
  45. package/CHANGELOG.md +402 -0
  46. package/MINDFORGE.md +13 -6
  47. package/README.md +51 -2
  48. package/RELEASENOTES.md +55 -2
  49. package/SECURITY.md +22 -3
  50. package/bin/autonomous/audit-writer.js +48 -33
  51. package/bin/autonomous/auto-runner.js +65 -2
  52. package/bin/change-classifier.js +151 -16
  53. package/bin/dashboard/api-router.js +28 -47
  54. package/bin/dashboard/error-response.js +44 -0
  55. package/bin/dashboard/frontend/app.js +429 -0
  56. package/bin/dashboard/frontend/index.html +14 -390
  57. package/bin/dashboard/metrics-aggregator.js +75 -30
  58. package/bin/dashboard/revops-api.js +12 -2
  59. package/bin/dashboard/server.js +245 -6
  60. package/bin/dashboard/sse-bridge.js +11 -8
  61. package/bin/dashboard/temporal-api.js +11 -5
  62. package/bin/engine/remediation-engine.js +12 -1
  63. package/bin/engine/sre-manager.js +1 -1
  64. package/bin/engine/temporal-cli.js +56 -6
  65. package/bin/engine/temporal-hub.js +41 -9
  66. package/bin/engine/verification-runner.js +134 -17
  67. package/bin/engine/verify-cli.js +25 -7
  68. package/bin/eval/eval-harness.js +212 -1
  69. package/bin/eval/golden-set-retrieval.json +9 -0
  70. package/bin/governance/approval-record.js +147 -0
  71. package/bin/governance/approve.js +12 -7
  72. package/bin/governance/policy-engine.js +41 -3
  73. package/bin/governance/policy-gate-hardened.js +36 -1
  74. package/bin/governance/verify-approvals.js +163 -0
  75. package/bin/harness-audit.js +224 -10
  76. package/bin/hindsight-injector.js +8 -2
  77. package/bin/hooks/instinct-capture-hook.js +19 -5
  78. package/bin/install.js +63 -3
  79. package/bin/installer/harness-adapter-compliance.js +339 -28
  80. package/bin/installer/hook-registration.js +504 -0
  81. package/bin/installer-core.js +451 -63
  82. package/bin/learning/instinct-cli.js +14 -24
  83. package/bin/memory/knowledge-capture.js +23 -3
  84. package/bin/memory/knowledge-graph.js +70 -31
  85. package/bin/memory/vector-hub.js +500 -44
  86. package/bin/migrations/0.6.0-to-1.0.0.js +30 -25
  87. package/bin/migrations/1.0.0-to-2.0.0.js +22 -23
  88. package/bin/mindforge-cli.js +110 -17
  89. package/bin/models/cost-tracker.js +126 -29
  90. package/bin/models/model-client.js +6 -1
  91. package/bin/models/model-router.js +28 -7
  92. package/bin/models/usage-record.js +71 -0
  93. package/bin/revops/debt-monitor.js +57 -13
  94. package/bin/security/trust-gate-hook.js +50 -6
  95. package/bin/skill-validator.js +6 -1
  96. package/bin/skills-builder/skill-scorer.js +46 -6
  97. package/bin/updater/self-update.js +6 -1
  98. package/bin/updater/version-comparator.js +21 -1
  99. package/bin/utils/file-lock.js +106 -0
  100. package/bin/utils/mindforge-params.js +124 -0
  101. package/bin/utils/mindforge-version.js +99 -0
  102. package/bin/utils/redact-secrets.js +106 -0
  103. package/bin/validate-config.js +75 -17
  104. package/bin/wizard/setup-wizard.js +4 -1
  105. package/bin/wizard/theme.js +9 -1
  106. package/changelogs/index.json +11 -9
  107. package/changelogs/v11.9.2.md +209 -0
  108. package/changelogs/v11.9.3.md +195 -0
  109. package/docs/References/config-reference.md +76 -14
  110. package/docs/References/sdk-api.md +1 -1
  111. package/docs/Templates/Codebase/architecture.md +1 -1
  112. package/docs/commands-reference.md +4 -5
  113. package/docs/faq.md +25 -5
  114. package/docs/getting-started.md +3 -3
  115. package/docs/sdk-reference.md +15 -7
  116. package/docs/troubleshooting.md +10 -6
  117. package/docs/user-guide.md +14 -14
  118. package/examples/sdk-integration/README.md +1 -1
  119. package/package.json +10 -4
  120. package/subagents/.claude-plugin/marketplace.json +1 -1
  121. package/bin/dashboard/approval-handler.js +0 -136
@@ -7,28 +7,114 @@ const fs = require('fs');
7
7
  const MAX_OUTPUT_LENGTH = 2000;
8
8
 
9
9
  /**
10
- * Stage definitions each maps a stage name to its command and optional skip condition.
11
- * The tests stage guards against recursion: if NODE_ENV=test (set by run-all.js) or
12
- * MINDFORGE_VERIFICATION_ACTIVE=1 (set by this runner), we skip to prevent infinite nesting.
10
+ * Detect an ESLint configuration. Flat config (eslint 9+) and the legacy .eslintrc family both
11
+ * count, as does an `eslintConfig` key in package.json, because `npx eslint .` succeeds with any
12
+ * of them and fails with none.
13
+ */
14
+ function hasEslintConfig(cwd) {
15
+ const named = [
16
+ 'eslint.config.js', 'eslint.config.mjs', 'eslint.config.cjs',
17
+ 'eslint.config.ts', 'eslint.config.mts', 'eslint.config.cts',
18
+ '.eslintrc', '.eslintrc.js', '.eslintrc.cjs', '.eslintrc.json',
19
+ '.eslintrc.yml', '.eslintrc.yaml',
20
+ ];
21
+ if (named.some((f) => fs.existsSync(path.join(cwd, f)))) return true;
22
+ try {
23
+ const pkg = JSON.parse(fs.readFileSync(path.join(cwd, 'package.json'), 'utf8'));
24
+ return Boolean(pkg.eslintConfig);
25
+ } catch { return false; }
26
+ }
27
+
28
+ /**
29
+ * Stage definitions — each maps a stage name to its command and an optional availability check.
30
+ *
31
+ * `skipIf(cwd)` returns a REASON STRING to skip, or false to run. A bare `true` still skips, for
32
+ * compatibility, but a reason is strongly preferred: an unexplained skip in the report is nearly as
33
+ * unhelpful as an unexplained failure.
34
+ *
35
+ * WHY EVERY STAGE NEEDS ONE. Only `typecheck` had an availability check, so `mindforge verify` on a
36
+ * freshly installed, entirely healthy consumer project reported:
37
+ *
38
+ * | tests | ❌ fail |
39
+ * | lint | ❌ fail |
40
+ * | audit | ❌ fail |
41
+ * | typecheck | ⏭️ skip |
42
+ * **Summary:** 0 passed, 3 failed, 1 skipped exit 1
43
+ *
44
+ * Measured on `node bin/install.js --claude --local` into an empty project: 1,836 files installed,
45
+ * and no `tests/`, no ESLint config and no `bin/verify-audit.js` — because the installer ships none
46
+ * of them, which is correct. So all three stages were reporting the ABSENCE of a tool as the
47
+ * FAILURE of the thing it would have checked. The audit stage's "result" was a raw Node module
48
+ * loader stack trace.
49
+ *
50
+ * "I cannot check this here" and "this is broken" are opposite statements, and a verifier that
51
+ * conflates them is worse than one that runs nothing: it trains its user to ignore red.
52
+ *
53
+ * The recursion guard on `tests` is unchanged and separate — NODE_ENV=test is set by run-all.js and
54
+ * MINDFORGE_VERIFICATION_ACTIVE=1 by this runner, and skipping prevents infinite nesting.
13
55
  */
14
56
  const STAGE_DEFS = {
15
57
  tests: {
16
58
  command: 'node tests/run-all.js',
17
- skipIf: () =>
18
- process.env.MINDFORGE_VERIFICATION_ACTIVE === '1' ||
19
- process.env.NODE_ENV === 'test',
59
+ skipIf: (cwd) => {
60
+ if (process.env.MINDFORGE_VERIFICATION_ACTIVE === '1' || process.env.NODE_ENV === 'test') {
61
+ return 'already inside a verification run (recursion guard)';
62
+ }
63
+ if (!fs.existsSync(path.join(cwd, 'tests', 'run-all.js'))) {
64
+ return 'no tests/run-all.js here — the package does not install a test suite';
65
+ }
66
+ return false;
67
+ },
20
68
  },
21
69
  lint: {
22
- command: 'npx eslint . --max-warnings=0',
23
- skipIf: null,
70
+ // `--max-warnings=0` used to be here, which made this stage IMPOSSIBLE to pass in the repo it
71
+ // ships from: measured, `npx eslint .` reports 199 problems / 0 errors / 199 warnings. So
72
+ // `mindforge verify` reported a lint FAILURE on a tree whose lint is green by the project's own
73
+ // definition — a confidently wrong answer, which is worse than no answer.
74
+ //
75
+ // Aligned with the project's own contract rather than a stricter threshold invented here:
76
+ // package.json "lint": "eslint ." (warnings tolerated)
77
+ // CI eslint . --max-warnings=9999 (warnings tolerated)
78
+ // sdk CI eslint src/ --max-warnings 0 (strict, but only over sdk/src)
79
+ // Two of the three tolerate warnings and the third is scoped to a different tree, so a
80
+ // repo-wide zero-warning gate was this file's opinion alone. Errors still fail the stage,
81
+ // because eslint exits non-zero on an error regardless of the warning threshold.
82
+ command: 'npx eslint .',
83
+ skipIf: (cwd) => (hasEslintConfig(cwd)
84
+ ? false
85
+ : 'no ESLint configuration here — nothing defines what lint means for this project'),
24
86
  },
25
87
  audit: {
26
88
  command: 'node bin/verify-audit.js',
27
- skipIf: null,
89
+ // BOTH the verifier AND the log — and the comment that used to sit here asserted the opposite of
90
+ // measured behaviour, which is how the remaining gap survived review.
91
+ //
92
+ // It claimed "an absent AUDIT.jsonl is a legitimate empty chain that verify-audit.js reports on
93
+ // correctly". Measured in an empty project: `node bin/verify-audit.js` prints
94
+ // "❌ audit chain BROKEN at entry 0: unreadable: ENOENT: no such file or directory" and exits 1.
95
+ // It does not report an empty chain, it reports corruption. So wherever the script IS present and
96
+ // the log is not — a --with-utils install, a fresh checkout — this stage still produced a red for
97
+ // a project that has simply never written an audit entry. That is the same
98
+ // absence-reported-as-failure defect the rest of this change exists to remove.
99
+ //
100
+ // The script being missing is the state of every ordinary consumer install (the installer does not
101
+ // copy bin/verify-audit.js), and there the stage died in Node's module loader and printed the
102
+ // stack trace as its verification result.
103
+ skipIf: (cwd) => {
104
+ if (!fs.existsSync(path.join(cwd, 'bin', 'verify-audit.js'))) {
105
+ return 'bin/verify-audit.js is not installed here, so the audit chain cannot be verified';
106
+ }
107
+ if (!fs.existsSync(path.join(cwd, '.planning', 'AUDIT.jsonl'))) {
108
+ return 'no .planning/AUDIT.jsonl yet — nothing has been audited, which is not a broken chain';
109
+ }
110
+ return false;
111
+ },
28
112
  },
29
113
  typecheck: {
30
114
  command: 'npx tsc --noEmit',
31
- skipIf: (cwd) => !fs.existsSync(path.join(cwd, 'tsconfig.json')),
115
+ skipIf: (cwd) => (fs.existsSync(path.join(cwd, 'tsconfig.json'))
116
+ ? false
117
+ : 'no tsconfig.json here — nothing to typecheck'),
32
118
  },
33
119
  };
34
120
 
@@ -41,9 +127,18 @@ function executeStage(name, cwd) {
41
127
  return { name, status: 'skip', durationMs: 0, output: `Unknown stage: ${name}` };
42
128
  }
43
129
 
44
- // Check skip condition
45
- if (def.skipIf && def.skipIf(cwd)) {
46
- return { name, status: 'skip', durationMs: 0, output: '' };
130
+ // Check availability / skip condition. A reason string is carried through to the report, because
131
+ // an unexplained skip leaves the reader unable to tell "not applicable here" from "silently
132
+ // broken" the same ambiguity that conflating absence with failure created.
133
+ const skip = def.skipIf && def.skipIf(cwd);
134
+ if (skip) {
135
+ return {
136
+ name,
137
+ status: 'skip',
138
+ durationMs: 0,
139
+ output: typeof skip === 'string' ? skip : '',
140
+ reason: typeof skip === 'string' ? skip : undefined,
141
+ };
47
142
  }
48
143
 
49
144
  const start = Date.now();
@@ -112,20 +207,42 @@ function formatReport(result) {
112
207
  lines.push('');
113
208
  lines.push(`**Timestamp:** ${result.timestamp}`);
114
209
  lines.push('');
115
- lines.push('| Stage | Status | Duration |');
116
- lines.push('|-------|--------|----------|');
210
+ lines.push('| Stage | Status | Duration | Note |');
211
+ lines.push('|-------|--------|----------|------|');
117
212
 
118
213
  for (const stage of result.stages) {
119
214
  const emoji = statusEmoji[stage.status] || '?';
120
215
  const duration = stage.durationMs > 0 ? `${stage.durationMs}ms` : '-';
121
- lines.push(`| ${stage.name} | ${emoji} ${stage.status} | ${duration} |`);
216
+ // A skip without its reason is unreadable: this table used to show a bare `⏭️ skip` and left the
217
+ // reader to guess whether the stage was inapplicable or quietly broken. Pipes are escaped so a
218
+ // reason can never break the markdown table.
219
+ const note = stage.status === 'skip' && stage.reason
220
+ ? String(stage.reason).replace(/\|/g, '\\|')
221
+ : '';
222
+ lines.push(`| ${stage.name} | ${emoji} ${stage.status} | ${duration} | ${note} |`);
122
223
  }
123
224
 
124
225
  lines.push('');
125
226
  lines.push(`**Summary:** ${result.summary.passed} passed, ${result.summary.failed} failed, ${result.summary.skipped} skipped (${result.summary.totalDurationMs}ms total)`);
227
+
228
+ // Say it outright when NOTHING ran. Adding availability checks fixed the false failures, but it
229
+ // moved a consumer install from "3 failed, exit 1" to "0 failed, exit 0" — and bin/engine/
230
+ // verify-cli.js exits on `failed > 0`, so a script reading $? now sees success for a run that
231
+ // verified nothing. Whether an all-skipped run should exit non-zero is an open maintainer
232
+ // decision and is deliberately NOT pre-empted here; what is not optional is that the report
233
+ // refuses to look like a pass.
234
+ if (result.summary.passed === 0 && result.summary.failed === 0 && result.summary.skipped > 0) {
235
+ lines.push('');
236
+ lines.push('> **NOTHING WAS VERIFIED.** Every stage was skipped, so this report is not '
237
+ + 'evidence that anything works — see the Note column for what was unavailable. A verifier '
238
+ + 'that ran no checks must not be read as a passing verifier.');
239
+ }
126
240
  lines.push('');
127
241
 
128
242
  return lines.join('\n');
129
243
  }
130
244
 
131
- module.exports = { runVerification, formatReport };
245
+ // STAGE_DEFS is exported so a test can compare the lint stage's threshold against the project's own
246
+ // lint script rather than hardcoding what it expects to find. Read-only by contract: mutating it
247
+ // would change every subsequent run in the same process.
248
+ module.exports = { runVerification, formatReport, STAGE_DEFS };
@@ -3,8 +3,19 @@
3
3
 
4
4
  /**
5
5
  * verify-cli.js — Entrypoint for the `verify` CLI command.
6
- * Calls the unified verification runner across all stages and writes
7
- * the formatted report to .planning/VERIFICATION.md.
6
+ * Calls the unified verification runner across all stages, prints the formatted report, and writes it
7
+ * to .planning/VERIFICATION.md only when asked with --write.
8
+ *
9
+ * WHY --write RATHER THAN ALWAYS. `.planning/VERIFICATION.md` is TRACKED (`git ls-files` returns it),
10
+ * and this file used to overwrite it on every run — so a contributor who ran the project's own
11
+ * verification dirtied a tracked file and had to remember to discard it. Nothing reads the root
12
+ * report: the only other reference is tests/e2e.test.js:399, and that reads
13
+ * `.planning/phases/1/VERIFICATION-1.md`, a per-phase fixture in a generated project, not this file.
14
+ * The report was already printed to stdout, so gating the file loses no information.
15
+ *
16
+ * Untracking the file is the other half and belongs to the operator (`git rm --cached` plus a
17
+ * .gitignore entry). This change is correct either way: with the file untracked, --write becomes a
18
+ * convenience; while it stays tracked, --write is what stops the dirtying.
8
19
  */
9
20
 
10
21
  const path = require('path');
@@ -13,17 +24,24 @@ const { runVerification, formatReport } = require('./verification-runner');
13
24
 
14
25
  const STAGES = ['tests', 'lint', 'audit', 'typecheck'];
15
26
  const CWD = process.env.MINDFORGE_ROOT || path.resolve(__dirname, '../..');
27
+ const REPORT_NAME = 'VERIFICATION.md';
16
28
 
17
29
  async function main() {
18
- const planningDir = path.join(CWD, '.planning');
19
- if (!fs.existsSync(planningDir)) {
20
- fs.mkdirSync(planningDir, { recursive: true });
21
- }
30
+ const shouldWrite = process.argv.slice(2).includes('--write');
22
31
 
23
32
  const result = await runVerification({ cwd: CWD, stages: STAGES });
24
33
  const report = formatReport(result);
25
34
 
26
- fs.writeFileSync(path.join(planningDir, 'VERIFICATION.md'), report);
35
+ if (shouldWrite) {
36
+ const planningDir = path.join(CWD, '.planning');
37
+ if (!fs.existsSync(planningDir)) {
38
+ fs.mkdirSync(planningDir, { recursive: true });
39
+ }
40
+ const reportPath = path.join(planningDir, REPORT_NAME);
41
+ fs.writeFileSync(reportPath, report);
42
+ process.stdout.write(`Report written to ${reportPath}\n`);
43
+ }
44
+
27
45
  process.stdout.write(report + '\n');
28
46
  process.exit(result.summary.failed > 0 ? 1 : 0);
29
47
  }
@@ -1,5 +1,16 @@
1
1
  'use strict';
2
2
 
3
+ const fs = require('fs');
4
+ const os = require('os');
5
+ const path = require('path');
6
+
7
+ // Repo root derived from THIS file's location, never from cwd.
8
+ // bin/utils/paths.js resolves PROJECT_ROOT by walking up from process.cwd(),
9
+ // which on a consumer install lands inside node_modules — the wrong root for a
10
+ // corpus scan, and it would silently yield an empty corpus. __dirname is always
11
+ // <root>/bin/eval.
12
+ const REPO_ROOT = path.resolve(__dirname, '..', '..');
13
+
3
14
  /**
4
15
  * Recall@K — fraction of relevant items found in the top-k retrieved results.
5
16
  * @param {string[]} retrieved - IDs in ranked order
@@ -79,4 +90,204 @@ async function runEval({ goldenSet, retriever, k }) {
79
90
  return { meanRecallAtK, meanNDCG, perQuery };
80
91
  }
81
92
 
82
- module.exports = { recallAtK, ndcg, runEval };
93
+ // ── Corpus + runnable golden-set gate (FTS-01) ───────────────────────────────
94
+ // Before this block the file had NO require.main guard, so the documented
95
+ // command `node bin/eval/eval-harness.js --set golden-set-retrieval.json`
96
+ // printed nothing and exited 0 — a gate that could not fail, and therefore not
97
+ // a gate. golden-set-retrieval.json and this harness had zero callers.
98
+ //
99
+ // The shipped golden set names documents by BASENAME (`audit-hash`,
100
+ // `model-router`, `stuck-detector`, …). Those ids exist nowhere in
101
+ // .mindforge/celestial.db, so recall measured against the live trace DB is 0.00
102
+ // by construction whatever the query builder does. The corpus the golden set
103
+ // actually describes is the repo's own module/skill documentation — so build it,
104
+ // index it into a THROWAWAY database under os.tmpdir(), and measure that.
105
+
106
+ const CORPUS_ROOTS = [
107
+ { dir: 'bin', exts: ['.js'] },
108
+ { dir: '.mindforge/skills', exts: ['.md'] },
109
+ { dir: '.mindforge/engine', exts: ['.md'] },
110
+ { dir: '.agent/hooks', exts: ['.js'] },
111
+ ];
112
+ const CORPUS_MAX_BYTES = 20000;
113
+ const CORPUS_SKIP_DIRS = new Set(['node_modules', 'dist', 'coverage']);
114
+
115
+ function _walk(dir, exts, out) {
116
+ let entries;
117
+ try {
118
+ entries = fs.readdirSync(dir, { withFileTypes: true });
119
+ } catch {
120
+ return out; // a missing corpus root is reported via corpusSize, not a throw
121
+ }
122
+ for (const entry of entries) {
123
+ const p = path.join(dir, entry.name);
124
+ if (entry.isDirectory()) {
125
+ if (!CORPUS_SKIP_DIRS.has(entry.name) && !entry.name.startsWith('.')) {
126
+ _walk(p, exts, out);
127
+ }
128
+ } else if (exts.includes(path.extname(entry.name))) {
129
+ out.push(p);
130
+ }
131
+ }
132
+ return out;
133
+ }
134
+
135
+ /**
136
+ * Doc id for a corpus file: its basename, except SKILL.md, which is keyed by its
137
+ * containing skill directory (that is the name the golden set uses).
138
+ * @param {string} file - absolute or relative file path
139
+ * @returns {string} document id
140
+ */
141
+ function docIdForFile(file) {
142
+ const base = path.basename(file, path.extname(file));
143
+ return base === 'SKILL' ? path.basename(path.dirname(file)) : base;
144
+ }
145
+
146
+ /**
147
+ * Enumerate the document corpus the golden set's `relevant` ids refer to.
148
+ * @param {string} [root] - project root; defaults to REPO_ROOT
149
+ * @returns {Map<string, {id: string, file: string, content: string}>}
150
+ */
151
+ function buildDocCorpus(root = REPO_ROOT) {
152
+ const corpus = new Map();
153
+ for (const { dir, exts } of CORPUS_ROOTS) {
154
+ for (const file of _walk(path.join(root, dir), exts, [])) {
155
+ const id = docIdForFile(file);
156
+ if (corpus.has(id)) continue; // first wins; basename collisions are rare
157
+ let text = '';
158
+ try {
159
+ text = fs.readFileSync(file, 'utf8').slice(0, CORPUS_MAX_BYTES);
160
+ } catch {
161
+ continue;
162
+ }
163
+ const rel = path.relative(root, file);
164
+ corpus.set(id, { id, file: rel, content: `${id} ${rel}\n${text}` });
165
+ }
166
+ }
167
+ return corpus;
168
+ }
169
+
170
+ /**
171
+ * Index the doc corpus into a throwaway VectorHub and score the golden set.
172
+ * NEVER touches .mindforge/celestial.db — the database lives in os.tmpdir() and
173
+ * is deleted again before this resolves.
174
+ * @param {Object} [opts]
175
+ * @param {string} [opts.goldenSetPath] - defaults to ./golden-set-retrieval.json
176
+ * @param {string} [opts.root] - corpus root; defaults to REPO_ROOT
177
+ * @param {number} [opts.k] - cutoff; defaults to 10
178
+ * @returns {Promise<Object>} runEval metrics plus corpus/coverage diagnostics
179
+ */
180
+ async function runGoldenSetEval(opts = {}) {
181
+ const goldenSetPath = opts.goldenSetPath || path.join(__dirname, 'golden-set-retrieval.json');
182
+ const root = opts.root || REPO_ROOT;
183
+ const k = opts.k || 10;
184
+
185
+ const golden = JSON.parse(fs.readFileSync(goldenSetPath, 'utf8'));
186
+ const goldenSet = golden.queries || [];
187
+ const corpus = buildDocCorpus(root);
188
+
189
+ const relevantIds = [...new Set(goldenSet.flatMap(q => q.relevant || []))];
190
+ const unresolved = relevantIds.filter(id => !corpus.has(id));
191
+
192
+ const { VectorHub } = require('../memory/vector-hub');
193
+ const dbDir = fs.mkdtempSync(path.join(os.tmpdir(), 'mindforge-eval-'));
194
+ const hub = new VectorHub(path.join(dbDir, 'eval-corpus.db'));
195
+ await hub.init();
196
+ try {
197
+ // The hub is throwaway, so raise the autosave batch size: the default of 10
198
+ // exports the whole growing database to disk every 10 documents, which turns
199
+ // a 0.5 s index build into a quadratic one for no durability benefit here.
200
+ hub._batchSize = Number.MAX_SAFE_INTEGER;
201
+ for (const doc of corpus.values()) {
202
+ await hub.saveKnowledge({ id: doc.id, type: 'doc', content: doc.content, source: doc.file });
203
+ }
204
+ const retriever = async (query) =>
205
+ (await hub.searchKnowledge(query, { limit: k })).map(r => r.id);
206
+ const metrics = await runEval({ goldenSet, retriever, k });
207
+ return {
208
+ ...metrics,
209
+ k,
210
+ corpusSize: corpus.size,
211
+ relevantIdCount: relevantIds.length,
212
+ unresolvedRelevantIds: unresolved,
213
+ baseline: golden.baseline || null,
214
+ };
215
+ } finally {
216
+ await hub.close();
217
+ fs.rmSync(dbDir, { recursive: true, force: true });
218
+ }
219
+ }
220
+
221
+ module.exports = {
222
+ recallAtK,
223
+ ndcg,
224
+ runEval,
225
+ buildDocCorpus,
226
+ docIdForFile,
227
+ runGoldenSetEval,
228
+ };
229
+
230
+ // ── CLI ──────────────────────────────────────────────────────────────────────
231
+ // node bin/eval/eval-harness.js [--set golden-set-retrieval.json] [--k 10]
232
+ // [--min-recall 0.55] [--json]
233
+ // Exits 1 when mean recall@k is below --min-recall, or when any golden
234
+ // `relevant` id no longer resolves to a real document (golden-set drift).
235
+ // `npm run eval:retrieval` runs this with the committed baseline floor.
236
+ if (require.main === module) {
237
+ const argv = process.argv.slice(2);
238
+ const flag = (name, fallback) => {
239
+ const i = argv.indexOf(name);
240
+ return i >= 0 && argv[i + 1] !== undefined ? argv[i + 1] : fallback;
241
+ };
242
+ const setArg = flag('--set', 'golden-set-retrieval.json');
243
+ const goldenSetPath = path.isAbsolute(setArg)
244
+ ? setArg
245
+ : path.join(__dirname, path.basename(setArg));
246
+ const num = (name, fallback) => {
247
+ const raw = flag(name, String(fallback));
248
+ const n = Number(raw);
249
+ // Fail closed: an unparseable threshold must never silently become 0.
250
+ if (!Number.isFinite(n) || n < 0) {
251
+ console.error(`[eval] ERROR: ${name} must be a non-negative number, got ${JSON.stringify(raw)}`);
252
+ process.exit(1);
253
+ }
254
+ return n;
255
+ };
256
+ const k = Math.max(1, Math.trunc(num('--k', 10)));
257
+ const minRecall = num('--min-recall', 0);
258
+
259
+ runGoldenSetEval({ goldenSetPath, k })
260
+ .then((res) => {
261
+ if (argv.includes('--json')) {
262
+ console.log(JSON.stringify(res, null, 2));
263
+ } else {
264
+ console.log(`[eval] corpus: ${res.corpusSize} docs · golden queries: ${res.perQuery.length} · k=${res.k}`);
265
+ for (const q of res.perQuery) {
266
+ console.log(` recall=${q.recall.toFixed(3)} nDCG=${q.ndcg.toFixed(3)} hits=${q.retrieved.length} ${q.query}`);
267
+ }
268
+ console.log(`[eval] mean recall@${res.k} = ${res.meanRecallAtK.toFixed(4)}`);
269
+ console.log(`[eval] mean nDCG@${res.k} = ${res.meanNDCG.toFixed(4)}`);
270
+ if (res.baseline && typeof res.baseline.meanRecallAtK === 'number') {
271
+ // Round BEFORE choosing the sign, so a difference smaller than the
272
+ // printed precision reads as +0.0000 rather than a bogus -0.0000.
273
+ const delta = Number((res.meanRecallAtK - res.baseline.meanRecallAtK).toFixed(4));
274
+ console.log(`[eval] committed baseline recall@${res.baseline.k || res.k} = ${res.baseline.meanRecallAtK.toFixed(4)} (delta ${delta >= 0 ? '+' : ''}${delta.toFixed(4)})`);
275
+ }
276
+ }
277
+ if (res.unresolvedRelevantIds.length > 0) {
278
+ console.error(`[eval] FAIL: ${res.unresolvedRelevantIds.length} golden id(s) no longer resolve to a document: ${res.unresolvedRelevantIds.join(', ')}`);
279
+ process.exit(1);
280
+ }
281
+ if (res.meanRecallAtK < minRecall) {
282
+ console.error(`[eval] FAIL: mean recall@${res.k} ${res.meanRecallAtK.toFixed(4)} < --min-recall ${minRecall}`);
283
+ process.exit(1);
284
+ }
285
+ })
286
+ .catch((err) => {
287
+ // sql.js throws bare strings for some binding errors, so an Error-shaped
288
+ // formatter alone would print "undefined" and hide the real failure.
289
+ const detail = (err && (err.stack || err.message)) || String(err);
290
+ console.error(`[eval] ERROR: ${detail}`);
291
+ process.exit(1);
292
+ });
293
+ }
@@ -1,6 +1,15 @@
1
1
  {
2
2
  "description": "Golden set for retrieval quality evaluation. Each entry has a natural-language query and the IDs of documents that SHOULD be retrieved.",
3
3
  "version": "1.0.0",
4
+ "baseline": {
5
+ "note": "Committed measurement so a retrieval regression is detectable. Reproduce with `npm run eval:retrieval`. Corpus = repo docs enumerated by buildDocCorpus() in bin/eval/eval-harness.js; retriever = VectorHub.searchKnowledge (OR-joined terms, ranked by tf-idf over FTS4 matchinfo('pcnx')). Before FTS-01 both figures were 0.0000 because the whole query was one FTS phrase.",
6
+ "k": 10,
7
+ "meanRecallAtK": 0.6417,
8
+ "meanNDCG": 0.5698,
9
+ "corpusSize": 517,
10
+ "gateMinRecall": 0.55,
11
+ "measuredOn": "v11.9.2 (fix/v11.9.2-ship-blockers), sql.js 1.14.1 / SQLite 3.49.1"
12
+ },
4
13
  "queries": [
5
14
  {
6
15
  "query": "how does the audit hash chain work",
@@ -0,0 +1,147 @@
1
+ 'use strict';
2
+ /**
3
+ * Canonical approval-record integrity, shared by the writer and every verifier.
4
+ *
5
+ * WHY THIS EXISTS — what the old record actually proved: nothing.
6
+ *
7
+ * bin/governance/approve.js stamped
8
+ * signature = sha256(`${id}:${reason}:${timestamp}:${os.hostname()}`)
9
+ * and called it a signature. Three problems, all measured:
10
+ *
11
+ * 1. It is NOT a signature. Nothing signs it; there is no key and no asymmetry. Anyone who
12
+ * can write the file can compute the value.
13
+ * 2. It is UNVERIFIABLE by design. `os.hostname()` is part of the preimage but is NOT a field
14
+ * of the record, so a verifier holding the record cannot reproduce the digest. CI never
15
+ * could have checked it.
16
+ * 3. Nothing checked it anyway. control-plane.yml asserted `!rec.signature` — presence only.
17
+ *
18
+ * So the field was simultaneously misnamed, uncheckable and unchecked. It is replaced by
19
+ * `record_checksum`: a plain SHA-256 over the record's OWN canonical fields, which any holder
20
+ * can recompute. That makes it a real tamper-check on the record's contents — and nothing more,
21
+ * which is why it is not called a signature.
22
+ *
23
+ * WHAT THIS DOES NOT PROVIDE. A checksum over a git-tracked file is an INTEGRITY control, never
24
+ * an AUTHORIZATION one: anyone with push access can mint a fresh record with a valid checksum.
25
+ * Authorization on this repo comes from branch protection (a required PR plus required status
26
+ * checks on main and develop), not from a file. Verified live before this was written:
27
+ * `branches/{main,develop}/protection` returned 404 and `rulesets` was empty, so for the whole
28
+ * period this record format existed there was no enforcement layer of any kind behind it.
29
+ *
30
+ * The construction deliberately mirrors bin/governance/audit-hash.js — twelve lines, one
31
+ * function, used by writer and verifier alike. That module's discipline ("MUST be the single
32
+ * source of truth for both") is exactly what kept the audit chain honest while this record rotted,
33
+ * so it is copied rather than reinvented.
34
+ */
35
+
36
+ const crypto = require('crypto');
37
+
38
+ /** Field written last and excluded from its own digest. */
39
+ const CHECKSUM_FIELD = 'record_checksum';
40
+
41
+ /** Current record schema. A record without it is a pre-v2 record and is not accepted. */
42
+ const SCHEMA = 'mindforge.approval/v2';
43
+
44
+ /** How long a recorded acknowledgement stays current. */
45
+ const TTL_HOURS = 72;
46
+
47
+ /**
48
+ * SHA-256 over the record's own fields, with the checksum field removed.
49
+ *
50
+ * Determinism note: like audit-hash.js this relies on JSON.stringify emitting keys in insertion
51
+ * order, which is why writer and verifier MUST both go through this function. A verifier that
52
+ * rebuilt the object literal in a different order would compute a different digest and reject
53
+ * every valid record — so do not inline this construction anywhere, including in YAML.
54
+ *
55
+ * @param {object} record
56
+ * @returns {string} `sha256:<hex>`
57
+ */
58
+ function checksumRecord(record) {
59
+ const material = { ...record };
60
+ delete material[CHECKSUM_FIELD];
61
+ return `sha256:${crypto.createHash('sha256').update(JSON.stringify(material)).digest('hex')}`;
62
+ }
63
+
64
+ /** ISO timestamp `hours` after `fromIso`. */
65
+ function expiryFrom(fromIso, hours = TTL_HOURS) {
66
+ return new Date(new Date(fromIso).getTime() + hours * 3600 * 1000).toISOString();
67
+ }
68
+
69
+ const REQUIRED_FIELDS = ['schema', 'id', 'project', 'version', 'tier', 'approved_by', 'timestamp', 'expires_at', 'reason', 'identity_verification'];
70
+
71
+ /**
72
+ * Check one approval record's integrity and currency.
73
+ *
74
+ * This answers "is this record intact, and is it about the release being built?" It does NOT
75
+ * answer "was this change authorized" — see the header.
76
+ *
77
+ * The currency checks are the ones that matter in practice. A record with no expiry and no
78
+ * version binding satisfied the old gate in perpetuity: the committed approval was minted
79
+ * 2026-06-11 for version 11.5.1 and was still accepted 67 days and 286 commits later, against
80
+ * 11.9.2, because the gate read only the directory's contents.
81
+ *
82
+ * @param {object} record
83
+ * @param {{currentVersion:string, now?:Date}} ctx
84
+ * @returns {{ok:boolean, problems:string[], stale:boolean}}
85
+ */
86
+ function verifyRecord(record, ctx) {
87
+ const problems = [];
88
+ let stale = false;
89
+
90
+ if (!record || typeof record !== 'object') {
91
+ return { ok: false, problems: ['record is not a JSON object'], stale: false };
92
+ }
93
+
94
+ if (record.schema !== SCHEMA) {
95
+ problems.push(
96
+ record.signature && !record.schema
97
+ ? 'pre-v2 record (has the removed `signature` field, no `schema`). Its digest included ' +
98
+ 'os.hostname(), which is not a field of the record, so it can never be verified. Re-mint it.'
99
+ : `unknown schema ${JSON.stringify(record.schema)} — expected ${SCHEMA}`);
100
+ }
101
+
102
+ for (const f of REQUIRED_FIELDS) {
103
+ if (record[f] === undefined || record[f] === null || record[f] === '') {
104
+ problems.push(`missing required field \`${f}\``);
105
+ }
106
+ }
107
+
108
+ if (record[CHECKSUM_FIELD]) {
109
+ const expected = checksumRecord(record);
110
+ if (record[CHECKSUM_FIELD] !== expected) {
111
+ problems.push(`${CHECKSUM_FIELD} does not match the record contents — it has been edited since minting`);
112
+ }
113
+ } else {
114
+ problems.push(`missing \`${CHECKSUM_FIELD}\``);
115
+ }
116
+
117
+ const iv = record.identity_verification;
118
+ const isVerified = iv && iv.verified === true;
119
+ const isAckedUnverified = iv && iv.verified === false
120
+ && iv.method === 'git_identity_unverified' && iv.unverified_ack === true;
121
+ if (!isVerified && !isAckedUnverified) {
122
+ problems.push('identity is neither GPG-verified nor an explicitly acknowledged unverified ' +
123
+ 'approval (verified:false + method git_identity_unverified + unverified_ack:true)');
124
+ }
125
+
126
+ // Currency. Reported separately from malformedness because a stale record is a correctly-made
127
+ // record that has simply stopped applying — a different thing from a corrupt one.
128
+ if (record.version && ctx.currentVersion && record.version !== ctx.currentVersion) {
129
+ stale = true;
130
+ problems.push(`records version ${record.version} but the build is ${ctx.currentVersion} — ` +
131
+ 'an approval does not carry forward to a later release');
132
+ }
133
+ const now = ctx.now || new Date();
134
+ if (record.expires_at) {
135
+ const exp = new Date(record.expires_at);
136
+ if (Number.isNaN(exp.getTime())) problems.push(`expires_at is not a valid date: ${record.expires_at}`);
137
+ else if (exp <= now) {
138
+ stale = true;
139
+ const ageH = Math.round((now - exp) / 3600000);
140
+ problems.push(`expired ${ageH}h ago (expires_at ${record.expires_at})`);
141
+ }
142
+ }
143
+
144
+ return { ok: problems.length === 0, problems, stale };
145
+ }
146
+
147
+ module.exports = { checksumRecord, verifyRecord, expiryFrom, CHECKSUM_FIELD, SCHEMA, TTL_HOURS, REQUIRED_FIELDS };