mindforge-cc 11.9.1 → 11.9.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.agent/CLAUDE.md +37 -13
- package/.agent/hooks/mindforge-block-no-verify.js +61 -13
- package/.agent/hooks/mindforge-config-protection.js +82 -3
- package/.agent/hooks/mindforge-context-monitor.js +1 -1
- package/.agent/hooks/mindforge-workflow-guard.js +2 -2
- package/.agent/hooks/run-with-flags.js +190 -20
- package/.agent/mindforge/browse.md +2 -2
- package/.agent/mindforge/checkpoint.md +1 -1
- package/.agent/mindforge/consult.md +1 -1
- package/.agent/mindforge/cost-report.md +1 -1
- package/.agent/mindforge/harness-audit.md +1 -1
- package/.agent/mindforge/orch-add-feature.md +1 -1
- package/.agent/mindforge/orch-build-mvp.md +1 -1
- package/.agent/mindforge/orch-change-feature.md +1 -1
- package/.agent/mindforge/orch-fix-defect.md +1 -1
- package/.agent/mindforge/orch-refine-code.md +1 -1
- package/.agent/mindforge/qa.md +2 -2
- package/.claude/CLAUDE.md +37 -13
- package/.claude/commands/mindforge/browse.md +2 -2
- package/.claude/commands/mindforge/checkpoint.md +1 -1
- package/.claude/commands/mindforge/consult.md +1 -1
- package/.claude/commands/mindforge/cost-report.md +1 -1
- package/.claude/commands/mindforge/harness-audit.md +1 -1
- package/.claude/commands/mindforge/orch-add-feature.md +1 -1
- package/.claude/commands/mindforge/orch-build-mvp.md +1 -1
- package/.claude/commands/mindforge/orch-change-feature.md +1 -1
- package/.claude/commands/mindforge/orch-fix-defect.md +1 -1
- package/.claude/commands/mindforge/orch-refine-code.md +1 -1
- package/.claude/commands/mindforge/qa.md +2 -2
- package/.mindforge/MINDFORGE-SCHEMA.json +126 -13
- package/.mindforge/config.json +4 -4
- package/.mindforge/engine/autonomous/headless-adapter.md +2 -2
- package/.mindforge/engine/cost-tracking/router.md +1 -1
- package/.mindforge/engine/cost-tracking/token-ledger.md +21 -24
- package/.mindforge/engine/temporal-protocol.md +2 -2
- package/.mindforge/governance/change-classifier.md +20 -4
- package/.mindforge/memory/sync-manifest.json +1 -1
- package/.mindforge/metrics/METRICS-SCHEMA.md +13 -4
- package/.mindforge/personas/cost-optimizer.md +2 -2
- package/.mindforge/personas/multi-model-bridge.md +1 -1
- package/.mindforge/skills/agent-architecture-audit/SKILL.md +2 -2
- package/.mindforge/skills/cost-aware-routing/SKILL.md +3 -3
- package/.mindforge/skills/multi-llm-consult/SKILL.md +2 -2
- package/.mindforge/skills/orch-pipeline/SKILL.md +4 -4
- package/CHANGELOG.md +402 -0
- package/MINDFORGE.md +13 -6
- package/README.md +51 -2
- package/RELEASENOTES.md +55 -2
- package/SECURITY.md +22 -3
- package/bin/autonomous/audit-writer.js +48 -33
- package/bin/autonomous/auto-runner.js +65 -2
- package/bin/change-classifier.js +151 -16
- package/bin/dashboard/api-router.js +28 -47
- package/bin/dashboard/error-response.js +44 -0
- package/bin/dashboard/frontend/app.js +429 -0
- package/bin/dashboard/frontend/index.html +14 -390
- package/bin/dashboard/metrics-aggregator.js +75 -30
- package/bin/dashboard/revops-api.js +12 -2
- package/bin/dashboard/server.js +245 -6
- package/bin/dashboard/sse-bridge.js +11 -8
- package/bin/dashboard/temporal-api.js +11 -5
- package/bin/engine/remediation-engine.js +12 -1
- package/bin/engine/sre-manager.js +1 -1
- package/bin/engine/temporal-cli.js +56 -6
- package/bin/engine/temporal-hub.js +41 -9
- package/bin/engine/verification-runner.js +134 -17
- package/bin/engine/verify-cli.js +25 -7
- package/bin/eval/eval-harness.js +212 -1
- package/bin/eval/golden-set-retrieval.json +9 -0
- package/bin/governance/approval-record.js +147 -0
- package/bin/governance/approve.js +12 -7
- package/bin/governance/policy-engine.js +41 -3
- package/bin/governance/policy-gate-hardened.js +36 -1
- package/bin/governance/verify-approvals.js +163 -0
- package/bin/harness-audit.js +224 -10
- package/bin/hindsight-injector.js +8 -2
- package/bin/hooks/instinct-capture-hook.js +19 -5
- package/bin/install.js +63 -3
- package/bin/installer/harness-adapter-compliance.js +339 -28
- package/bin/installer/hook-registration.js +504 -0
- package/bin/installer-core.js +451 -63
- package/bin/learning/instinct-cli.js +14 -24
- package/bin/memory/knowledge-capture.js +23 -3
- package/bin/memory/knowledge-graph.js +70 -31
- package/bin/memory/vector-hub.js +500 -44
- package/bin/migrations/0.6.0-to-1.0.0.js +30 -25
- package/bin/migrations/1.0.0-to-2.0.0.js +22 -23
- package/bin/mindforge-cli.js +110 -17
- package/bin/models/cost-tracker.js +126 -29
- package/bin/models/model-client.js +6 -1
- package/bin/models/model-router.js +28 -7
- package/bin/models/usage-record.js +71 -0
- package/bin/revops/debt-monitor.js +57 -13
- package/bin/security/trust-gate-hook.js +50 -6
- package/bin/skill-validator.js +6 -1
- package/bin/skills-builder/skill-scorer.js +46 -6
- package/bin/updater/self-update.js +6 -1
- package/bin/updater/version-comparator.js +21 -1
- package/bin/utils/file-lock.js +106 -0
- package/bin/utils/mindforge-params.js +124 -0
- package/bin/utils/mindforge-version.js +99 -0
- package/bin/utils/redact-secrets.js +106 -0
- package/bin/validate-config.js +75 -17
- package/bin/wizard/setup-wizard.js +4 -1
- package/bin/wizard/theme.js +9 -1
- package/changelogs/index.json +11 -9
- package/changelogs/v11.9.2.md +209 -0
- package/changelogs/v11.9.3.md +195 -0
- package/docs/References/config-reference.md +76 -14
- package/docs/References/sdk-api.md +1 -1
- package/docs/Templates/Codebase/architecture.md +1 -1
- package/docs/commands-reference.md +4 -5
- package/docs/faq.md +25 -5
- package/docs/getting-started.md +3 -3
- package/docs/sdk-reference.md +15 -7
- package/docs/troubleshooting.md +10 -6
- package/docs/user-guide.md +14 -14
- package/examples/sdk-integration/README.md +1 -1
- package/package.json +10 -4
- package/subagents/.claude-plugin/marketplace.json +1 -1
- package/bin/dashboard/approval-handler.js +0 -136
|
@@ -7,28 +7,114 @@ const fs = require('fs');
|
|
|
7
7
|
const MAX_OUTPUT_LENGTH = 2000;
|
|
8
8
|
|
|
9
9
|
/**
|
|
10
|
-
*
|
|
11
|
-
*
|
|
12
|
-
*
|
|
10
|
+
* Detect an ESLint configuration. Flat config (eslint 9+) and the legacy .eslintrc family both
|
|
11
|
+
* count, as does an `eslintConfig` key in package.json, because `npx eslint .` succeeds with any
|
|
12
|
+
* of them and fails with none.
|
|
13
|
+
*/
|
|
14
|
+
function hasEslintConfig(cwd) {
|
|
15
|
+
const named = [
|
|
16
|
+
'eslint.config.js', 'eslint.config.mjs', 'eslint.config.cjs',
|
|
17
|
+
'eslint.config.ts', 'eslint.config.mts', 'eslint.config.cts',
|
|
18
|
+
'.eslintrc', '.eslintrc.js', '.eslintrc.cjs', '.eslintrc.json',
|
|
19
|
+
'.eslintrc.yml', '.eslintrc.yaml',
|
|
20
|
+
];
|
|
21
|
+
if (named.some((f) => fs.existsSync(path.join(cwd, f)))) return true;
|
|
22
|
+
try {
|
|
23
|
+
const pkg = JSON.parse(fs.readFileSync(path.join(cwd, 'package.json'), 'utf8'));
|
|
24
|
+
return Boolean(pkg.eslintConfig);
|
|
25
|
+
} catch { return false; }
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
/**
|
|
29
|
+
* Stage definitions — each maps a stage name to its command and an optional availability check.
|
|
30
|
+
*
|
|
31
|
+
* `skipIf(cwd)` returns a REASON STRING to skip, or false to run. A bare `true` still skips, for
|
|
32
|
+
* compatibility, but a reason is strongly preferred: an unexplained skip in the report is nearly as
|
|
33
|
+
* unhelpful as an unexplained failure.
|
|
34
|
+
*
|
|
35
|
+
* WHY EVERY STAGE NEEDS ONE. Only `typecheck` had an availability check, so `mindforge verify` on a
|
|
36
|
+
* freshly installed, entirely healthy consumer project reported:
|
|
37
|
+
*
|
|
38
|
+
* | tests | ❌ fail |
|
|
39
|
+
* | lint | ❌ fail |
|
|
40
|
+
* | audit | ❌ fail |
|
|
41
|
+
* | typecheck | ⏭️ skip |
|
|
42
|
+
* **Summary:** 0 passed, 3 failed, 1 skipped exit 1
|
|
43
|
+
*
|
|
44
|
+
* Measured on `node bin/install.js --claude --local` into an empty project: 1,836 files installed,
|
|
45
|
+
* and no `tests/`, no ESLint config and no `bin/verify-audit.js` — because the installer ships none
|
|
46
|
+
* of them, which is correct. So all three stages were reporting the ABSENCE of a tool as the
|
|
47
|
+
* FAILURE of the thing it would have checked. The audit stage's "result" was a raw Node module
|
|
48
|
+
* loader stack trace.
|
|
49
|
+
*
|
|
50
|
+
* "I cannot check this here" and "this is broken" are opposite statements, and a verifier that
|
|
51
|
+
* conflates them is worse than one that runs nothing: it trains its user to ignore red.
|
|
52
|
+
*
|
|
53
|
+
* The recursion guard on `tests` is unchanged and separate — NODE_ENV=test is set by run-all.js and
|
|
54
|
+
* MINDFORGE_VERIFICATION_ACTIVE=1 by this runner, and skipping prevents infinite nesting.
|
|
13
55
|
*/
|
|
14
56
|
const STAGE_DEFS = {
|
|
15
57
|
tests: {
|
|
16
58
|
command: 'node tests/run-all.js',
|
|
17
|
-
skipIf: () =>
|
|
18
|
-
process.env.MINDFORGE_VERIFICATION_ACTIVE === '1' ||
|
|
19
|
-
|
|
59
|
+
skipIf: (cwd) => {
|
|
60
|
+
if (process.env.MINDFORGE_VERIFICATION_ACTIVE === '1' || process.env.NODE_ENV === 'test') {
|
|
61
|
+
return 'already inside a verification run (recursion guard)';
|
|
62
|
+
}
|
|
63
|
+
if (!fs.existsSync(path.join(cwd, 'tests', 'run-all.js'))) {
|
|
64
|
+
return 'no tests/run-all.js here — the package does not install a test suite';
|
|
65
|
+
}
|
|
66
|
+
return false;
|
|
67
|
+
},
|
|
20
68
|
},
|
|
21
69
|
lint: {
|
|
22
|
-
|
|
23
|
-
|
|
70
|
+
// `--max-warnings=0` used to be here, which made this stage IMPOSSIBLE to pass in the repo it
|
|
71
|
+
// ships from: measured, `npx eslint .` reports 199 problems / 0 errors / 199 warnings. So
|
|
72
|
+
// `mindforge verify` reported a lint FAILURE on a tree whose lint is green by the project's own
|
|
73
|
+
// definition — a confidently wrong answer, which is worse than no answer.
|
|
74
|
+
//
|
|
75
|
+
// Aligned with the project's own contract rather than a stricter threshold invented here:
|
|
76
|
+
// package.json "lint": "eslint ." (warnings tolerated)
|
|
77
|
+
// CI eslint . --max-warnings=9999 (warnings tolerated)
|
|
78
|
+
// sdk CI eslint src/ --max-warnings 0 (strict, but only over sdk/src)
|
|
79
|
+
// Two of the three tolerate warnings and the third is scoped to a different tree, so a
|
|
80
|
+
// repo-wide zero-warning gate was this file's opinion alone. Errors still fail the stage,
|
|
81
|
+
// because eslint exits non-zero on an error regardless of the warning threshold.
|
|
82
|
+
command: 'npx eslint .',
|
|
83
|
+
skipIf: (cwd) => (hasEslintConfig(cwd)
|
|
84
|
+
? false
|
|
85
|
+
: 'no ESLint configuration here — nothing defines what lint means for this project'),
|
|
24
86
|
},
|
|
25
87
|
audit: {
|
|
26
88
|
command: 'node bin/verify-audit.js',
|
|
27
|
-
|
|
89
|
+
// BOTH the verifier AND the log — and the comment that used to sit here asserted the opposite of
|
|
90
|
+
// measured behaviour, which is how the remaining gap survived review.
|
|
91
|
+
//
|
|
92
|
+
// It claimed "an absent AUDIT.jsonl is a legitimate empty chain that verify-audit.js reports on
|
|
93
|
+
// correctly". Measured in an empty project: `node bin/verify-audit.js` prints
|
|
94
|
+
// "❌ audit chain BROKEN at entry 0: unreadable: ENOENT: no such file or directory" and exits 1.
|
|
95
|
+
// It does not report an empty chain, it reports corruption. So wherever the script IS present and
|
|
96
|
+
// the log is not — a --with-utils install, a fresh checkout — this stage still produced a red for
|
|
97
|
+
// a project that has simply never written an audit entry. That is the same
|
|
98
|
+
// absence-reported-as-failure defect the rest of this change exists to remove.
|
|
99
|
+
//
|
|
100
|
+
// The script being missing is the state of every ordinary consumer install (the installer does not
|
|
101
|
+
// copy bin/verify-audit.js), and there the stage died in Node's module loader and printed the
|
|
102
|
+
// stack trace as its verification result.
|
|
103
|
+
skipIf: (cwd) => {
|
|
104
|
+
if (!fs.existsSync(path.join(cwd, 'bin', 'verify-audit.js'))) {
|
|
105
|
+
return 'bin/verify-audit.js is not installed here, so the audit chain cannot be verified';
|
|
106
|
+
}
|
|
107
|
+
if (!fs.existsSync(path.join(cwd, '.planning', 'AUDIT.jsonl'))) {
|
|
108
|
+
return 'no .planning/AUDIT.jsonl yet — nothing has been audited, which is not a broken chain';
|
|
109
|
+
}
|
|
110
|
+
return false;
|
|
111
|
+
},
|
|
28
112
|
},
|
|
29
113
|
typecheck: {
|
|
30
114
|
command: 'npx tsc --noEmit',
|
|
31
|
-
skipIf: (cwd) =>
|
|
115
|
+
skipIf: (cwd) => (fs.existsSync(path.join(cwd, 'tsconfig.json'))
|
|
116
|
+
? false
|
|
117
|
+
: 'no tsconfig.json here — nothing to typecheck'),
|
|
32
118
|
},
|
|
33
119
|
};
|
|
34
120
|
|
|
@@ -41,9 +127,18 @@ function executeStage(name, cwd) {
|
|
|
41
127
|
return { name, status: 'skip', durationMs: 0, output: `Unknown stage: ${name}` };
|
|
42
128
|
}
|
|
43
129
|
|
|
44
|
-
// Check skip condition
|
|
45
|
-
|
|
46
|
-
|
|
130
|
+
// Check availability / skip condition. A reason string is carried through to the report, because
|
|
131
|
+
// an unexplained skip leaves the reader unable to tell "not applicable here" from "silently
|
|
132
|
+
// broken" — the same ambiguity that conflating absence with failure created.
|
|
133
|
+
const skip = def.skipIf && def.skipIf(cwd);
|
|
134
|
+
if (skip) {
|
|
135
|
+
return {
|
|
136
|
+
name,
|
|
137
|
+
status: 'skip',
|
|
138
|
+
durationMs: 0,
|
|
139
|
+
output: typeof skip === 'string' ? skip : '',
|
|
140
|
+
reason: typeof skip === 'string' ? skip : undefined,
|
|
141
|
+
};
|
|
47
142
|
}
|
|
48
143
|
|
|
49
144
|
const start = Date.now();
|
|
@@ -112,20 +207,42 @@ function formatReport(result) {
|
|
|
112
207
|
lines.push('');
|
|
113
208
|
lines.push(`**Timestamp:** ${result.timestamp}`);
|
|
114
209
|
lines.push('');
|
|
115
|
-
lines.push('| Stage | Status | Duration |');
|
|
116
|
-
lines.push('
|
|
210
|
+
lines.push('| Stage | Status | Duration | Note |');
|
|
211
|
+
lines.push('|-------|--------|----------|------|');
|
|
117
212
|
|
|
118
213
|
for (const stage of result.stages) {
|
|
119
214
|
const emoji = statusEmoji[stage.status] || '?';
|
|
120
215
|
const duration = stage.durationMs > 0 ? `${stage.durationMs}ms` : '-';
|
|
121
|
-
|
|
216
|
+
// A skip without its reason is unreadable: this table used to show a bare `⏭️ skip` and left the
|
|
217
|
+
// reader to guess whether the stage was inapplicable or quietly broken. Pipes are escaped so a
|
|
218
|
+
// reason can never break the markdown table.
|
|
219
|
+
const note = stage.status === 'skip' && stage.reason
|
|
220
|
+
? String(stage.reason).replace(/\|/g, '\\|')
|
|
221
|
+
: '';
|
|
222
|
+
lines.push(`| ${stage.name} | ${emoji} ${stage.status} | ${duration} | ${note} |`);
|
|
122
223
|
}
|
|
123
224
|
|
|
124
225
|
lines.push('');
|
|
125
226
|
lines.push(`**Summary:** ${result.summary.passed} passed, ${result.summary.failed} failed, ${result.summary.skipped} skipped (${result.summary.totalDurationMs}ms total)`);
|
|
227
|
+
|
|
228
|
+
// Say it outright when NOTHING ran. Adding availability checks fixed the false failures, but it
|
|
229
|
+
// moved a consumer install from "3 failed, exit 1" to "0 failed, exit 0" — and bin/engine/
|
|
230
|
+
// verify-cli.js exits on `failed > 0`, so a script reading $? now sees success for a run that
|
|
231
|
+
// verified nothing. Whether an all-skipped run should exit non-zero is an open maintainer
|
|
232
|
+
// decision and is deliberately NOT pre-empted here; what is not optional is that the report
|
|
233
|
+
// refuses to look like a pass.
|
|
234
|
+
if (result.summary.passed === 0 && result.summary.failed === 0 && result.summary.skipped > 0) {
|
|
235
|
+
lines.push('');
|
|
236
|
+
lines.push('> **NOTHING WAS VERIFIED.** Every stage was skipped, so this report is not '
|
|
237
|
+
+ 'evidence that anything works — see the Note column for what was unavailable. A verifier '
|
|
238
|
+
+ 'that ran no checks must not be read as a passing verifier.');
|
|
239
|
+
}
|
|
126
240
|
lines.push('');
|
|
127
241
|
|
|
128
242
|
return lines.join('\n');
|
|
129
243
|
}
|
|
130
244
|
|
|
131
|
-
|
|
245
|
+
// STAGE_DEFS is exported so a test can compare the lint stage's threshold against the project's own
|
|
246
|
+
// lint script rather than hardcoding what it expects to find. Read-only by contract: mutating it
|
|
247
|
+
// would change every subsequent run in the same process.
|
|
248
|
+
module.exports = { runVerification, formatReport, STAGE_DEFS };
|
package/bin/engine/verify-cli.js
CHANGED
|
@@ -3,8 +3,19 @@
|
|
|
3
3
|
|
|
4
4
|
/**
|
|
5
5
|
* verify-cli.js — Entrypoint for the `verify` CLI command.
|
|
6
|
-
* Calls the unified verification runner across all stages and writes
|
|
7
|
-
*
|
|
6
|
+
* Calls the unified verification runner across all stages, prints the formatted report, and writes it
|
|
7
|
+
* to .planning/VERIFICATION.md only when asked with --write.
|
|
8
|
+
*
|
|
9
|
+
* WHY --write RATHER THAN ALWAYS. `.planning/VERIFICATION.md` is TRACKED (`git ls-files` returns it),
|
|
10
|
+
* and this file used to overwrite it on every run — so a contributor who ran the project's own
|
|
11
|
+
* verification dirtied a tracked file and had to remember to discard it. Nothing reads the root
|
|
12
|
+
* report: the only other reference is tests/e2e.test.js:399, and that reads
|
|
13
|
+
* `.planning/phases/1/VERIFICATION-1.md`, a per-phase fixture in a generated project, not this file.
|
|
14
|
+
* The report was already printed to stdout, so gating the file loses no information.
|
|
15
|
+
*
|
|
16
|
+
* Untracking the file is the other half and belongs to the operator (`git rm --cached` plus a
|
|
17
|
+
* .gitignore entry). This change is correct either way: with the file untracked, --write becomes a
|
|
18
|
+
* convenience; while it stays tracked, --write is what stops the dirtying.
|
|
8
19
|
*/
|
|
9
20
|
|
|
10
21
|
const path = require('path');
|
|
@@ -13,17 +24,24 @@ const { runVerification, formatReport } = require('./verification-runner');
|
|
|
13
24
|
|
|
14
25
|
const STAGES = ['tests', 'lint', 'audit', 'typecheck'];
|
|
15
26
|
const CWD = process.env.MINDFORGE_ROOT || path.resolve(__dirname, '../..');
|
|
27
|
+
const REPORT_NAME = 'VERIFICATION.md';
|
|
16
28
|
|
|
17
29
|
async function main() {
|
|
18
|
-
const
|
|
19
|
-
if (!fs.existsSync(planningDir)) {
|
|
20
|
-
fs.mkdirSync(planningDir, { recursive: true });
|
|
21
|
-
}
|
|
30
|
+
const shouldWrite = process.argv.slice(2).includes('--write');
|
|
22
31
|
|
|
23
32
|
const result = await runVerification({ cwd: CWD, stages: STAGES });
|
|
24
33
|
const report = formatReport(result);
|
|
25
34
|
|
|
26
|
-
|
|
35
|
+
if (shouldWrite) {
|
|
36
|
+
const planningDir = path.join(CWD, '.planning');
|
|
37
|
+
if (!fs.existsSync(planningDir)) {
|
|
38
|
+
fs.mkdirSync(planningDir, { recursive: true });
|
|
39
|
+
}
|
|
40
|
+
const reportPath = path.join(planningDir, REPORT_NAME);
|
|
41
|
+
fs.writeFileSync(reportPath, report);
|
|
42
|
+
process.stdout.write(`Report written to ${reportPath}\n`);
|
|
43
|
+
}
|
|
44
|
+
|
|
27
45
|
process.stdout.write(report + '\n');
|
|
28
46
|
process.exit(result.summary.failed > 0 ? 1 : 0);
|
|
29
47
|
}
|
package/bin/eval/eval-harness.js
CHANGED
|
@@ -1,5 +1,16 @@
|
|
|
1
1
|
'use strict';
|
|
2
2
|
|
|
3
|
+
const fs = require('fs');
|
|
4
|
+
const os = require('os');
|
|
5
|
+
const path = require('path');
|
|
6
|
+
|
|
7
|
+
// Repo root derived from THIS file's location, never from cwd.
|
|
8
|
+
// bin/utils/paths.js resolves PROJECT_ROOT by walking up from process.cwd(),
|
|
9
|
+
// which on a consumer install lands inside node_modules — the wrong root for a
|
|
10
|
+
// corpus scan, and it would silently yield an empty corpus. __dirname is always
|
|
11
|
+
// <root>/bin/eval.
|
|
12
|
+
const REPO_ROOT = path.resolve(__dirname, '..', '..');
|
|
13
|
+
|
|
3
14
|
/**
|
|
4
15
|
* Recall@K — fraction of relevant items found in the top-k retrieved results.
|
|
5
16
|
* @param {string[]} retrieved - IDs in ranked order
|
|
@@ -79,4 +90,204 @@ async function runEval({ goldenSet, retriever, k }) {
|
|
|
79
90
|
return { meanRecallAtK, meanNDCG, perQuery };
|
|
80
91
|
}
|
|
81
92
|
|
|
82
|
-
|
|
93
|
+
// ── Corpus + runnable golden-set gate (FTS-01) ───────────────────────────────
|
|
94
|
+
// Before this block the file had NO require.main guard, so the documented
|
|
95
|
+
// command `node bin/eval/eval-harness.js --set golden-set-retrieval.json`
|
|
96
|
+
// printed nothing and exited 0 — a gate that could not fail, and therefore not
|
|
97
|
+
// a gate. golden-set-retrieval.json and this harness had zero callers.
|
|
98
|
+
//
|
|
99
|
+
// The shipped golden set names documents by BASENAME (`audit-hash`,
|
|
100
|
+
// `model-router`, `stuck-detector`, …). Those ids exist nowhere in
|
|
101
|
+
// .mindforge/celestial.db, so recall measured against the live trace DB is 0.00
|
|
102
|
+
// by construction whatever the query builder does. The corpus the golden set
|
|
103
|
+
// actually describes is the repo's own module/skill documentation — so build it,
|
|
104
|
+
// index it into a THROWAWAY database under os.tmpdir(), and measure that.
|
|
105
|
+
|
|
106
|
+
const CORPUS_ROOTS = [
|
|
107
|
+
{ dir: 'bin', exts: ['.js'] },
|
|
108
|
+
{ dir: '.mindforge/skills', exts: ['.md'] },
|
|
109
|
+
{ dir: '.mindforge/engine', exts: ['.md'] },
|
|
110
|
+
{ dir: '.agent/hooks', exts: ['.js'] },
|
|
111
|
+
];
|
|
112
|
+
const CORPUS_MAX_BYTES = 20000;
|
|
113
|
+
const CORPUS_SKIP_DIRS = new Set(['node_modules', 'dist', 'coverage']);
|
|
114
|
+
|
|
115
|
+
function _walk(dir, exts, out) {
|
|
116
|
+
let entries;
|
|
117
|
+
try {
|
|
118
|
+
entries = fs.readdirSync(dir, { withFileTypes: true });
|
|
119
|
+
} catch {
|
|
120
|
+
return out; // a missing corpus root is reported via corpusSize, not a throw
|
|
121
|
+
}
|
|
122
|
+
for (const entry of entries) {
|
|
123
|
+
const p = path.join(dir, entry.name);
|
|
124
|
+
if (entry.isDirectory()) {
|
|
125
|
+
if (!CORPUS_SKIP_DIRS.has(entry.name) && !entry.name.startsWith('.')) {
|
|
126
|
+
_walk(p, exts, out);
|
|
127
|
+
}
|
|
128
|
+
} else if (exts.includes(path.extname(entry.name))) {
|
|
129
|
+
out.push(p);
|
|
130
|
+
}
|
|
131
|
+
}
|
|
132
|
+
return out;
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
/**
|
|
136
|
+
* Doc id for a corpus file: its basename, except SKILL.md, which is keyed by its
|
|
137
|
+
* containing skill directory (that is the name the golden set uses).
|
|
138
|
+
* @param {string} file - absolute or relative file path
|
|
139
|
+
* @returns {string} document id
|
|
140
|
+
*/
|
|
141
|
+
function docIdForFile(file) {
|
|
142
|
+
const base = path.basename(file, path.extname(file));
|
|
143
|
+
return base === 'SKILL' ? path.basename(path.dirname(file)) : base;
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
/**
|
|
147
|
+
* Enumerate the document corpus the golden set's `relevant` ids refer to.
|
|
148
|
+
* @param {string} [root] - project root; defaults to REPO_ROOT
|
|
149
|
+
* @returns {Map<string, {id: string, file: string, content: string}>}
|
|
150
|
+
*/
|
|
151
|
+
function buildDocCorpus(root = REPO_ROOT) {
|
|
152
|
+
const corpus = new Map();
|
|
153
|
+
for (const { dir, exts } of CORPUS_ROOTS) {
|
|
154
|
+
for (const file of _walk(path.join(root, dir), exts, [])) {
|
|
155
|
+
const id = docIdForFile(file);
|
|
156
|
+
if (corpus.has(id)) continue; // first wins; basename collisions are rare
|
|
157
|
+
let text = '';
|
|
158
|
+
try {
|
|
159
|
+
text = fs.readFileSync(file, 'utf8').slice(0, CORPUS_MAX_BYTES);
|
|
160
|
+
} catch {
|
|
161
|
+
continue;
|
|
162
|
+
}
|
|
163
|
+
const rel = path.relative(root, file);
|
|
164
|
+
corpus.set(id, { id, file: rel, content: `${id} ${rel}\n${text}` });
|
|
165
|
+
}
|
|
166
|
+
}
|
|
167
|
+
return corpus;
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
/**
|
|
171
|
+
* Index the doc corpus into a throwaway VectorHub and score the golden set.
|
|
172
|
+
* NEVER touches .mindforge/celestial.db — the database lives in os.tmpdir() and
|
|
173
|
+
* is deleted again before this resolves.
|
|
174
|
+
* @param {Object} [opts]
|
|
175
|
+
* @param {string} [opts.goldenSetPath] - defaults to ./golden-set-retrieval.json
|
|
176
|
+
* @param {string} [opts.root] - corpus root; defaults to REPO_ROOT
|
|
177
|
+
* @param {number} [opts.k] - cutoff; defaults to 10
|
|
178
|
+
* @returns {Promise<Object>} runEval metrics plus corpus/coverage diagnostics
|
|
179
|
+
*/
|
|
180
|
+
async function runGoldenSetEval(opts = {}) {
|
|
181
|
+
const goldenSetPath = opts.goldenSetPath || path.join(__dirname, 'golden-set-retrieval.json');
|
|
182
|
+
const root = opts.root || REPO_ROOT;
|
|
183
|
+
const k = opts.k || 10;
|
|
184
|
+
|
|
185
|
+
const golden = JSON.parse(fs.readFileSync(goldenSetPath, 'utf8'));
|
|
186
|
+
const goldenSet = golden.queries || [];
|
|
187
|
+
const corpus = buildDocCorpus(root);
|
|
188
|
+
|
|
189
|
+
const relevantIds = [...new Set(goldenSet.flatMap(q => q.relevant || []))];
|
|
190
|
+
const unresolved = relevantIds.filter(id => !corpus.has(id));
|
|
191
|
+
|
|
192
|
+
const { VectorHub } = require('../memory/vector-hub');
|
|
193
|
+
const dbDir = fs.mkdtempSync(path.join(os.tmpdir(), 'mindforge-eval-'));
|
|
194
|
+
const hub = new VectorHub(path.join(dbDir, 'eval-corpus.db'));
|
|
195
|
+
await hub.init();
|
|
196
|
+
try {
|
|
197
|
+
// The hub is throwaway, so raise the autosave batch size: the default of 10
|
|
198
|
+
// exports the whole growing database to disk every 10 documents, which turns
|
|
199
|
+
// a 0.5 s index build into a quadratic one for no durability benefit here.
|
|
200
|
+
hub._batchSize = Number.MAX_SAFE_INTEGER;
|
|
201
|
+
for (const doc of corpus.values()) {
|
|
202
|
+
await hub.saveKnowledge({ id: doc.id, type: 'doc', content: doc.content, source: doc.file });
|
|
203
|
+
}
|
|
204
|
+
const retriever = async (query) =>
|
|
205
|
+
(await hub.searchKnowledge(query, { limit: k })).map(r => r.id);
|
|
206
|
+
const metrics = await runEval({ goldenSet, retriever, k });
|
|
207
|
+
return {
|
|
208
|
+
...metrics,
|
|
209
|
+
k,
|
|
210
|
+
corpusSize: corpus.size,
|
|
211
|
+
relevantIdCount: relevantIds.length,
|
|
212
|
+
unresolvedRelevantIds: unresolved,
|
|
213
|
+
baseline: golden.baseline || null,
|
|
214
|
+
};
|
|
215
|
+
} finally {
|
|
216
|
+
await hub.close();
|
|
217
|
+
fs.rmSync(dbDir, { recursive: true, force: true });
|
|
218
|
+
}
|
|
219
|
+
}
|
|
220
|
+
|
|
221
|
+
module.exports = {
|
|
222
|
+
recallAtK,
|
|
223
|
+
ndcg,
|
|
224
|
+
runEval,
|
|
225
|
+
buildDocCorpus,
|
|
226
|
+
docIdForFile,
|
|
227
|
+
runGoldenSetEval,
|
|
228
|
+
};
|
|
229
|
+
|
|
230
|
+
// ── CLI ──────────────────────────────────────────────────────────────────────
|
|
231
|
+
// node bin/eval/eval-harness.js [--set golden-set-retrieval.json] [--k 10]
|
|
232
|
+
// [--min-recall 0.55] [--json]
|
|
233
|
+
// Exits 1 when mean recall@k is below --min-recall, or when any golden
|
|
234
|
+
// `relevant` id no longer resolves to a real document (golden-set drift).
|
|
235
|
+
// `npm run eval:retrieval` runs this with the committed baseline floor.
|
|
236
|
+
if (require.main === module) {
|
|
237
|
+
const argv = process.argv.slice(2);
|
|
238
|
+
const flag = (name, fallback) => {
|
|
239
|
+
const i = argv.indexOf(name);
|
|
240
|
+
return i >= 0 && argv[i + 1] !== undefined ? argv[i + 1] : fallback;
|
|
241
|
+
};
|
|
242
|
+
const setArg = flag('--set', 'golden-set-retrieval.json');
|
|
243
|
+
const goldenSetPath = path.isAbsolute(setArg)
|
|
244
|
+
? setArg
|
|
245
|
+
: path.join(__dirname, path.basename(setArg));
|
|
246
|
+
const num = (name, fallback) => {
|
|
247
|
+
const raw = flag(name, String(fallback));
|
|
248
|
+
const n = Number(raw);
|
|
249
|
+
// Fail closed: an unparseable threshold must never silently become 0.
|
|
250
|
+
if (!Number.isFinite(n) || n < 0) {
|
|
251
|
+
console.error(`[eval] ERROR: ${name} must be a non-negative number, got ${JSON.stringify(raw)}`);
|
|
252
|
+
process.exit(1);
|
|
253
|
+
}
|
|
254
|
+
return n;
|
|
255
|
+
};
|
|
256
|
+
const k = Math.max(1, Math.trunc(num('--k', 10)));
|
|
257
|
+
const minRecall = num('--min-recall', 0);
|
|
258
|
+
|
|
259
|
+
runGoldenSetEval({ goldenSetPath, k })
|
|
260
|
+
.then((res) => {
|
|
261
|
+
if (argv.includes('--json')) {
|
|
262
|
+
console.log(JSON.stringify(res, null, 2));
|
|
263
|
+
} else {
|
|
264
|
+
console.log(`[eval] corpus: ${res.corpusSize} docs · golden queries: ${res.perQuery.length} · k=${res.k}`);
|
|
265
|
+
for (const q of res.perQuery) {
|
|
266
|
+
console.log(` recall=${q.recall.toFixed(3)} nDCG=${q.ndcg.toFixed(3)} hits=${q.retrieved.length} ${q.query}`);
|
|
267
|
+
}
|
|
268
|
+
console.log(`[eval] mean recall@${res.k} = ${res.meanRecallAtK.toFixed(4)}`);
|
|
269
|
+
console.log(`[eval] mean nDCG@${res.k} = ${res.meanNDCG.toFixed(4)}`);
|
|
270
|
+
if (res.baseline && typeof res.baseline.meanRecallAtK === 'number') {
|
|
271
|
+
// Round BEFORE choosing the sign, so a difference smaller than the
|
|
272
|
+
// printed precision reads as +0.0000 rather than a bogus -0.0000.
|
|
273
|
+
const delta = Number((res.meanRecallAtK - res.baseline.meanRecallAtK).toFixed(4));
|
|
274
|
+
console.log(`[eval] committed baseline recall@${res.baseline.k || res.k} = ${res.baseline.meanRecallAtK.toFixed(4)} (delta ${delta >= 0 ? '+' : ''}${delta.toFixed(4)})`);
|
|
275
|
+
}
|
|
276
|
+
}
|
|
277
|
+
if (res.unresolvedRelevantIds.length > 0) {
|
|
278
|
+
console.error(`[eval] FAIL: ${res.unresolvedRelevantIds.length} golden id(s) no longer resolve to a document: ${res.unresolvedRelevantIds.join(', ')}`);
|
|
279
|
+
process.exit(1);
|
|
280
|
+
}
|
|
281
|
+
if (res.meanRecallAtK < minRecall) {
|
|
282
|
+
console.error(`[eval] FAIL: mean recall@${res.k} ${res.meanRecallAtK.toFixed(4)} < --min-recall ${minRecall}`);
|
|
283
|
+
process.exit(1);
|
|
284
|
+
}
|
|
285
|
+
})
|
|
286
|
+
.catch((err) => {
|
|
287
|
+
// sql.js throws bare strings for some binding errors, so an Error-shaped
|
|
288
|
+
// formatter alone would print "undefined" and hide the real failure.
|
|
289
|
+
const detail = (err && (err.stack || err.message)) || String(err);
|
|
290
|
+
console.error(`[eval] ERROR: ${detail}`);
|
|
291
|
+
process.exit(1);
|
|
292
|
+
});
|
|
293
|
+
}
|
|
@@ -1,6 +1,15 @@
|
|
|
1
1
|
{
|
|
2
2
|
"description": "Golden set for retrieval quality evaluation. Each entry has a natural-language query and the IDs of documents that SHOULD be retrieved.",
|
|
3
3
|
"version": "1.0.0",
|
|
4
|
+
"baseline": {
|
|
5
|
+
"note": "Committed measurement so a retrieval regression is detectable. Reproduce with `npm run eval:retrieval`. Corpus = repo docs enumerated by buildDocCorpus() in bin/eval/eval-harness.js; retriever = VectorHub.searchKnowledge (OR-joined terms, ranked by tf-idf over FTS4 matchinfo('pcnx')). Before FTS-01 both figures were 0.0000 because the whole query was one FTS phrase.",
|
|
6
|
+
"k": 10,
|
|
7
|
+
"meanRecallAtK": 0.6417,
|
|
8
|
+
"meanNDCG": 0.5698,
|
|
9
|
+
"corpusSize": 517,
|
|
10
|
+
"gateMinRecall": 0.55,
|
|
11
|
+
"measuredOn": "v11.9.2 (fix/v11.9.2-ship-blockers), sql.js 1.14.1 / SQLite 3.49.1"
|
|
12
|
+
},
|
|
4
13
|
"queries": [
|
|
5
14
|
{
|
|
6
15
|
"query": "how does the audit hash chain work",
|
|
@@ -0,0 +1,147 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
/**
|
|
3
|
+
* Canonical approval-record integrity, shared by the writer and every verifier.
|
|
4
|
+
*
|
|
5
|
+
* WHY THIS EXISTS — what the old record actually proved: nothing.
|
|
6
|
+
*
|
|
7
|
+
* bin/governance/approve.js stamped
|
|
8
|
+
* signature = sha256(`${id}:${reason}:${timestamp}:${os.hostname()}`)
|
|
9
|
+
* and called it a signature. Three problems, all measured:
|
|
10
|
+
*
|
|
11
|
+
* 1. It is NOT a signature. Nothing signs it; there is no key and no asymmetry. Anyone who
|
|
12
|
+
* can write the file can compute the value.
|
|
13
|
+
* 2. It is UNVERIFIABLE by design. `os.hostname()` is part of the preimage but is NOT a field
|
|
14
|
+
* of the record, so a verifier holding the record cannot reproduce the digest. CI never
|
|
15
|
+
* could have checked it.
|
|
16
|
+
* 3. Nothing checked it anyway. control-plane.yml asserted `!rec.signature` — presence only.
|
|
17
|
+
*
|
|
18
|
+
* So the field was simultaneously misnamed, uncheckable and unchecked. It is replaced by
|
|
19
|
+
* `record_checksum`: a plain SHA-256 over the record's OWN canonical fields, which any holder
|
|
20
|
+
* can recompute. That makes it a real tamper-check on the record's contents — and nothing more,
|
|
21
|
+
* which is why it is not called a signature.
|
|
22
|
+
*
|
|
23
|
+
* WHAT THIS DOES NOT PROVIDE. A checksum over a git-tracked file is an INTEGRITY control, never
|
|
24
|
+
* an AUTHORIZATION one: anyone with push access can mint a fresh record with a valid checksum.
|
|
25
|
+
* Authorization on this repo comes from branch protection (a required PR plus required status
|
|
26
|
+
* checks on main and develop), not from a file. Verified live before this was written:
|
|
27
|
+
* `branches/{main,develop}/protection` returned 404 and `rulesets` was empty, so for the whole
|
|
28
|
+
* period this record format existed there was no enforcement layer of any kind behind it.
|
|
29
|
+
*
|
|
30
|
+
* The construction deliberately mirrors bin/governance/audit-hash.js — twelve lines, one
|
|
31
|
+
* function, used by writer and verifier alike. That module's discipline ("MUST be the single
|
|
32
|
+
* source of truth for both") is exactly what kept the audit chain honest while this record rotted,
|
|
33
|
+
* so it is copied rather than reinvented.
|
|
34
|
+
*/
|
|
35
|
+
|
|
36
|
+
const crypto = require('crypto');
|
|
37
|
+
|
|
38
|
+
/** Field written last and excluded from its own digest. */
|
|
39
|
+
const CHECKSUM_FIELD = 'record_checksum';
|
|
40
|
+
|
|
41
|
+
/** Current record schema. A record without it is a pre-v2 record and is not accepted. */
|
|
42
|
+
const SCHEMA = 'mindforge.approval/v2';
|
|
43
|
+
|
|
44
|
+
/** How long a recorded acknowledgement stays current. */
|
|
45
|
+
const TTL_HOURS = 72;
|
|
46
|
+
|
|
47
|
+
/**
|
|
48
|
+
* SHA-256 over the record's own fields, with the checksum field removed.
|
|
49
|
+
*
|
|
50
|
+
* Determinism note: like audit-hash.js this relies on JSON.stringify emitting keys in insertion
|
|
51
|
+
* order, which is why writer and verifier MUST both go through this function. A verifier that
|
|
52
|
+
* rebuilt the object literal in a different order would compute a different digest and reject
|
|
53
|
+
* every valid record — so do not inline this construction anywhere, including in YAML.
|
|
54
|
+
*
|
|
55
|
+
* @param {object} record
|
|
56
|
+
* @returns {string} `sha256:<hex>`
|
|
57
|
+
*/
|
|
58
|
+
function checksumRecord(record) {
|
|
59
|
+
const material = { ...record };
|
|
60
|
+
delete material[CHECKSUM_FIELD];
|
|
61
|
+
return `sha256:${crypto.createHash('sha256').update(JSON.stringify(material)).digest('hex')}`;
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
/** ISO timestamp `hours` after `fromIso`. */
|
|
65
|
+
function expiryFrom(fromIso, hours = TTL_HOURS) {
|
|
66
|
+
return new Date(new Date(fromIso).getTime() + hours * 3600 * 1000).toISOString();
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
const REQUIRED_FIELDS = ['schema', 'id', 'project', 'version', 'tier', 'approved_by', 'timestamp', 'expires_at', 'reason', 'identity_verification'];
|
|
70
|
+
|
|
71
|
+
/**
|
|
72
|
+
* Check one approval record's integrity and currency.
|
|
73
|
+
*
|
|
74
|
+
* This answers "is this record intact, and is it about the release being built?" It does NOT
|
|
75
|
+
* answer "was this change authorized" — see the header.
|
|
76
|
+
*
|
|
77
|
+
* The currency checks are the ones that matter in practice. A record with no expiry and no
|
|
78
|
+
* version binding satisfied the old gate in perpetuity: the committed approval was minted
|
|
79
|
+
* 2026-06-11 for version 11.5.1 and was still accepted 67 days and 286 commits later, against
|
|
80
|
+
* 11.9.2, because the gate read only the directory's contents.
|
|
81
|
+
*
|
|
82
|
+
* @param {object} record
|
|
83
|
+
* @param {{currentVersion:string, now?:Date}} ctx
|
|
84
|
+
* @returns {{ok:boolean, problems:string[], stale:boolean}}
|
|
85
|
+
*/
|
|
86
|
+
function verifyRecord(record, ctx) {
|
|
87
|
+
const problems = [];
|
|
88
|
+
let stale = false;
|
|
89
|
+
|
|
90
|
+
if (!record || typeof record !== 'object') {
|
|
91
|
+
return { ok: false, problems: ['record is not a JSON object'], stale: false };
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
if (record.schema !== SCHEMA) {
|
|
95
|
+
problems.push(
|
|
96
|
+
record.signature && !record.schema
|
|
97
|
+
? 'pre-v2 record (has the removed `signature` field, no `schema`). Its digest included ' +
|
|
98
|
+
'os.hostname(), which is not a field of the record, so it can never be verified. Re-mint it.'
|
|
99
|
+
: `unknown schema ${JSON.stringify(record.schema)} — expected ${SCHEMA}`);
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
for (const f of REQUIRED_FIELDS) {
|
|
103
|
+
if (record[f] === undefined || record[f] === null || record[f] === '') {
|
|
104
|
+
problems.push(`missing required field \`${f}\``);
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
if (record[CHECKSUM_FIELD]) {
|
|
109
|
+
const expected = checksumRecord(record);
|
|
110
|
+
if (record[CHECKSUM_FIELD] !== expected) {
|
|
111
|
+
problems.push(`${CHECKSUM_FIELD} does not match the record contents — it has been edited since minting`);
|
|
112
|
+
}
|
|
113
|
+
} else {
|
|
114
|
+
problems.push(`missing \`${CHECKSUM_FIELD}\``);
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
const iv = record.identity_verification;
|
|
118
|
+
const isVerified = iv && iv.verified === true;
|
|
119
|
+
const isAckedUnverified = iv && iv.verified === false
|
|
120
|
+
&& iv.method === 'git_identity_unverified' && iv.unverified_ack === true;
|
|
121
|
+
if (!isVerified && !isAckedUnverified) {
|
|
122
|
+
problems.push('identity is neither GPG-verified nor an explicitly acknowledged unverified ' +
|
|
123
|
+
'approval (verified:false + method git_identity_unverified + unverified_ack:true)');
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
// Currency. Reported separately from malformedness because a stale record is a correctly-made
|
|
127
|
+
// record that has simply stopped applying — a different thing from a corrupt one.
|
|
128
|
+
if (record.version && ctx.currentVersion && record.version !== ctx.currentVersion) {
|
|
129
|
+
stale = true;
|
|
130
|
+
problems.push(`records version ${record.version} but the build is ${ctx.currentVersion} — ` +
|
|
131
|
+
'an approval does not carry forward to a later release');
|
|
132
|
+
}
|
|
133
|
+
const now = ctx.now || new Date();
|
|
134
|
+
if (record.expires_at) {
|
|
135
|
+
const exp = new Date(record.expires_at);
|
|
136
|
+
if (Number.isNaN(exp.getTime())) problems.push(`expires_at is not a valid date: ${record.expires_at}`);
|
|
137
|
+
else if (exp <= now) {
|
|
138
|
+
stale = true;
|
|
139
|
+
const ageH = Math.round((now - exp) / 3600000);
|
|
140
|
+
problems.push(`expired ${ageH}h ago (expires_at ${record.expires_at})`);
|
|
141
|
+
}
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
return { ok: problems.length === 0, problems, stale };
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
module.exports = { checksumRecord, verifyRecord, expiryFrom, CHECKSUM_FIELD, SCHEMA, TTL_HOURS, REQUIRED_FIELDS };
|