mindforge-cc 11.9.0 → 11.9.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.agent/mindforge/consult.md +1 -1
- package/.agent/mindforge/cost-report.md +1 -1
- package/.agent/mindforge/ship.md +2 -2
- package/.agent/mindforge/update.md +1 -1
- package/.agent/mindforge/wf-catalog.md +5 -2
- package/.claude/commands/mindforge/consult.md +1 -1
- package/.claude/commands/mindforge/cost-report.md +1 -1
- package/.claude/commands/mindforge/ship.md +2 -2
- package/.claude/commands/mindforge/skill-tdd.md +53 -0
- package/.claude/commands/mindforge/skills-index.md +118 -0
- package/.claude/commands/mindforge/systematic-debug.md +60 -0
- package/.claude/commands/mindforge/update.md +1 -1
- package/.claude/commands/mindforge/wf-accessibility-audit.md +31 -0
- package/.claude/commands/mindforge/wf-ai-model-eval.md +31 -0
- package/.claude/commands/mindforge/wf-api-migration.md +31 -0
- package/.claude/commands/mindforge/wf-architecture-modernization.md +32 -0
- package/.claude/commands/mindforge/wf-catalog.md +62 -0
- package/.claude/commands/mindforge/wf-code-audit.md +31 -0
- package/.claude/commands/mindforge/wf-code-explainer.md +31 -0
- package/.claude/commands/mindforge/wf-competitive-analysis.md +31 -0
- package/.claude/commands/mindforge/wf-competitive-teardown.md +31 -0
- package/.claude/commands/mindforge/wf-cost-analysis.md +31 -0
- package/.claude/commands/mindforge/wf-data-pipeline-validate.md +31 -0
- package/.claude/commands/mindforge/wf-database-migration.md +31 -0
- package/.claude/commands/mindforge/wf-debug-detective.md +32 -0
- package/.claude/commands/mindforge/wf-design-system-audit.md +31 -0
- package/.claude/commands/mindforge/wf-documentation-gen.md +31 -0
- package/.claude/commands/mindforge/wf-feature-planner.md +31 -0
- package/.claude/commands/mindforge/wf-incident-response.md +31 -0
- package/.claude/commands/mindforge/wf-multi-repo-sync.md +31 -0
- package/.claude/commands/mindforge/wf-mutation-testing.md +31 -0
- package/.claude/commands/mindforge/wf-onboard-codebase.md +31 -0
- package/.claude/commands/mindforge/wf-orchestrate-review.md +35 -0
- package/.claude/commands/mindforge/wf-perf-optimize.md +31 -0
- package/.claude/commands/mindforge/wf-pr-review.md +31 -0
- package/.claude/commands/mindforge/wf-refactor-plan.md +31 -0
- package/.claude/commands/mindforge/wf-release-prep.md +31 -0
- package/.claude/commands/mindforge/wf-security-threat-model.md +31 -0
- package/.claude/commands/mindforge/wf-tdd-sprint.md +31 -0
- package/.claude/commands/mindforge/wf-tech-evaluation.md +31 -0
- package/.claude/commands/mindforge/wf-ux-heuristic-audit.md +31 -0
- package/.claude/commands/mindforge/wf-verification-loop.md +45 -0
- package/.claude/commands/mindforge/wf-workflow-optimizer.md +51 -0
- package/.claude/commands/mindforge/wf-writer-reviewer.md +30 -0
- package/.mindforge/MINDFORGE-SCHEMA.json +126 -13
- package/.mindforge/config.json +3 -3
- package/.mindforge/engine/cost-tracking/router.md +1 -1
- package/.mindforge/engine/cost-tracking/token-ledger.md +21 -24
- package/.mindforge/memory/sync-manifest.json +1 -1
- package/.mindforge/metrics/METRICS-SCHEMA.md +13 -4
- package/.mindforge/personas/cost-optimizer.md +2 -2
- package/.mindforge/personas/doc-auditor.md +1 -1
- package/.mindforge/personas/monorepo-architect.md +2 -1
- package/.mindforge/personas/multi-model-bridge.md +1 -1
- package/.mindforge/personas/release-manager.md +5 -3
- package/.mindforge/skills/cost-aware-routing/SKILL.md +3 -3
- package/.mindforge/skills/documentation/SKILL.md +4 -1
- package/.mindforge/skills/multi-llm-consult/SKILL.md +2 -2
- package/CHANGELOG.md +216 -1821
- package/MINDFORGE.md +4 -4
- package/README.md +103 -509
- package/RELEASENOTES.md +75 -0
- package/SECURITY.md +1 -1
- package/bin/autonomous/audit-writer.js +48 -33
- package/bin/dashboard/api-router.js +11 -10
- package/bin/dashboard/error-response.js +44 -0
- package/bin/dashboard/frontend/index.html +20 -3
- package/bin/dashboard/metrics-aggregator.js +29 -8
- package/bin/dashboard/revops-api.js +12 -2
- package/bin/dashboard/server.js +85 -5
- package/bin/dashboard/temporal-api.js +11 -5
- package/bin/engine/remediation-engine.js +12 -1
- package/bin/engine/temporal-hub.js +41 -9
- package/bin/eval/eval-harness.js +212 -1
- package/bin/eval/golden-set-retrieval.json +9 -0
- package/bin/governance/policy-engine.js +8 -0
- package/bin/hindsight-injector.js +8 -2
- package/bin/hooks/instinct-capture-hook.js +7 -1
- package/bin/install.js +1 -1
- package/bin/learning/instinct-cli.js +7 -24
- package/bin/memory/knowledge-capture.js +23 -3
- package/bin/memory/knowledge-graph.js +70 -31
- package/bin/memory/vector-hub.js +304 -31
- package/bin/mindforge-cli.js +43 -11
- package/bin/models/cost-tracker.js +22 -23
- package/bin/models/model-router.js +28 -7
- package/bin/models/usage-record.js +71 -0
- package/bin/updater/changelog-fetcher.js +95 -5
- package/bin/utils/file-lock.js +106 -0
- package/bin/utils/mindforge-params.js +124 -0
- package/bin/utils/readiness-gate.js +3 -3
- package/bin/validate-config.js +34 -16
- package/changelogs/README.md +109 -0
- package/changelogs/index.json +103 -0
- package/changelogs/v0.1.0.md +21 -0
- package/changelogs/v0.2.0.md +15 -0
- package/changelogs/v0.3.0.md +28 -0
- package/changelogs/v0.4.0.md +29 -0
- package/changelogs/v0.5.0.md +34 -0
- package/changelogs/v0.6.0.md +34 -0
- package/changelogs/v1.0.0.md +60 -0
- package/changelogs/v1.0.1.md +15 -0
- package/changelogs/v1.0.2.md +10 -0
- package/changelogs/v1.0.3.md +10 -0
- package/changelogs/v1.0.4.md +10 -0
- package/changelogs/v1.0.5.md +10 -0
- package/changelogs/v10.0.0.md +71 -0
- package/changelogs/v10.0.2.md +15 -0
- package/changelogs/v10.0.3.md +21 -0
- package/changelogs/v10.0.4.md +16 -0
- package/changelogs/v10.0.5.md +16 -0
- package/changelogs/v10.0.6.md +15 -0
- package/changelogs/v10.0.7.md +15 -0
- package/changelogs/v10.0.8.md +15 -0
- package/changelogs/v10.0.9.md +15 -0
- package/changelogs/v10.1.0.md +16 -0
- package/changelogs/v10.1.1.md +15 -0
- package/changelogs/v10.2.0.md +14 -0
- package/changelogs/v10.3.0.md +13 -0
- package/changelogs/v10.4.0.md +13 -0
- package/changelogs/v10.5.0.md +13 -0
- package/changelogs/v10.6.0.md +13 -0
- package/changelogs/v10.7.0.md +15 -0
- package/changelogs/v11.0.0.md +66 -0
- package/changelogs/v11.0.1.md +10 -0
- package/changelogs/v11.1.0.md +56 -0
- package/changelogs/v11.2.0.md +41 -0
- package/changelogs/v11.2.1.md +38 -0
- package/changelogs/v11.3.0.md +45 -0
- package/changelogs/v11.3.1.md +39 -0
- package/changelogs/v11.4.0.md +38 -0
- package/changelogs/v11.5.0.md +77 -0
- package/changelogs/v11.5.1.md +47 -0
- package/changelogs/v11.6.0.md +45 -0
- package/changelogs/v11.7.0.md +30 -0
- package/changelogs/v11.7.1.md +8 -0
- package/changelogs/v11.8.0.md +19 -0
- package/changelogs/v11.8.1.md +28 -0
- package/changelogs/v11.8.2.md +22 -0
- package/changelogs/v11.8.3.md +22 -0
- package/changelogs/v11.9.0.md +17 -0
- package/changelogs/v11.9.1.md +8 -0
- package/changelogs/v11.9.2.md +209 -0
- package/changelogs/v2.0.0-alpha.1.md +42 -0
- package/changelogs/v2.0.0-alpha.10.md +13 -0
- package/changelogs/v2.0.0-alpha.11.md +14 -0
- package/changelogs/v2.0.0-alpha.12.md +16 -0
- package/changelogs/v2.0.0-alpha.2.md +14 -0
- package/changelogs/v2.0.0-alpha.3.md +17 -0
- package/changelogs/v2.0.0-alpha.4.md +17 -0
- package/changelogs/v2.0.0-alpha.7.md +18 -0
- package/changelogs/v2.0.0-alpha.8.md +11 -0
- package/changelogs/v2.0.0-alpha.9.md +12 -0
- package/changelogs/v2.0.0.md +35 -0
- package/changelogs/v2.1.0.md +22 -0
- package/changelogs/v2.1.1.md +25 -0
- package/changelogs/v2.1.2.md +21 -0
- package/changelogs/v2.3.2.md +19 -0
- package/changelogs/v2.3.3.md +16 -0
- package/changelogs/v2.3.4.md +16 -0
- package/changelogs/v2.3.5.md +21 -0
- package/changelogs/v2.4.0.md +29 -0
- package/changelogs/v2.5.0.md +26 -0
- package/changelogs/v2.6.0.md +26 -0
- package/changelogs/v3.0.0.md +17 -0
- package/changelogs/v4.0.0.md +19 -0
- package/changelogs/v4.1.0.md +19 -0
- package/changelogs/v4.3.0.md +31 -0
- package/changelogs/v5.0.0-alpha.1.md +35 -0
- package/changelogs/v5.0.0-alpha.2.md +22 -0
- package/changelogs/v5.1.0.md +42 -0
- package/changelogs/v5.2.0.md +18 -0
- package/changelogs/v5.3.0.md +14 -0
- package/changelogs/v5.4.0.md +28 -0
- package/changelogs/v5.5.0.md +11 -0
- package/changelogs/v5.6.0.md +11 -0
- package/changelogs/v5.7.0.md +12 -0
- package/changelogs/v5.8.0.md +12 -0
- package/changelogs/v5.9.0.md +11 -0
- package/changelogs/v6.0.0-alpha.md +45 -0
- package/changelogs/v6.1.0-alpha.md +29 -0
- package/changelogs/v6.2.0-alpha.md +30 -0
- package/changelogs/v6.2.0.md +21 -0
- package/changelogs/v6.3.0.md +14 -0
- package/changelogs/v6.4.0.md +13 -0
- package/changelogs/v6.5.0.md +13 -0
- package/changelogs/v6.6.0.md +12 -0
- package/changelogs/v6.7.0.md +12 -0
- package/changelogs/v7.0.0.md +13 -0
- package/changelogs/v8.0.0.md +14 -0
- package/changelogs/v8.1.0.md +13 -0
- package/changelogs/v8.2.0.md +12 -0
- package/changelogs/v8.2.1.md +11 -0
- package/changelogs/v9.0.0.md +20 -0
- package/docs/References/config-reference.md +73 -14
- package/docs/commands-reference.md +12 -6
- package/docs/faq.md +3 -3
- package/docs/getting-started.md +28 -3
- package/docs/sdk-reference.md +1 -1
- package/docs/user-guide.md +2 -2
- package/package.json +5 -2
|
@@ -8,6 +8,7 @@ const express = require('express');
|
|
|
8
8
|
const router = express.Router();
|
|
9
9
|
const TemporalHub = require('../engine/temporal-hub');
|
|
10
10
|
const HindsightInjector = require('../hindsight-injector');
|
|
11
|
+
const { sendServerError } = require('./error-response');
|
|
11
12
|
|
|
12
13
|
/**
|
|
13
14
|
* GET /api/temporal/history
|
|
@@ -18,7 +19,7 @@ router.get('/history', (req, res) => {
|
|
|
18
19
|
const history = TemporalHub.getHistory();
|
|
19
20
|
res.json(history);
|
|
20
21
|
} catch (err) {
|
|
21
|
-
res
|
|
22
|
+
sendServerError(res, 'GET /api/temporal/history', err, 'Failed to retrieve temporal history');
|
|
22
23
|
}
|
|
23
24
|
});
|
|
24
25
|
|
|
@@ -37,7 +38,7 @@ router.get('/snapshot/:auditId/:file', (req, res) => {
|
|
|
37
38
|
|
|
38
39
|
res.send(content);
|
|
39
40
|
} catch (err) {
|
|
40
|
-
res
|
|
41
|
+
sendServerError(res, 'GET /api/temporal/snapshot/:auditId/:file', err, 'Failed to retrieve snapshot file');
|
|
41
42
|
}
|
|
42
43
|
});
|
|
43
44
|
|
|
@@ -52,7 +53,9 @@ router.get('/snapshot/:auditId/meta', (req, res) => {
|
|
|
52
53
|
if (!snap) return res.status(404).json({ error: 'Snapshot not found' });
|
|
53
54
|
res.json(snap);
|
|
54
55
|
} catch (err) {
|
|
55
|
-
|
|
56
|
+
// Previously swallowed the error entirely — no client detail, but no server log
|
|
57
|
+
// either, so a broken history dir was undiagnosable.
|
|
58
|
+
sendServerError(res, 'GET /api/temporal/snapshot/:auditId/meta', err, 'Failed to retrieve snapshot metadata');
|
|
56
59
|
}
|
|
57
60
|
});
|
|
58
61
|
|
|
@@ -72,10 +75,13 @@ router.post('/inject', async (req, res) => {
|
|
|
72
75
|
if (result.success) {
|
|
73
76
|
res.json(result);
|
|
74
77
|
} else {
|
|
75
|
-
|
|
78
|
+
// result.error is HindsightInjector's own `err.message` (hindsight-injector.js:59)
|
|
79
|
+
// and can be an fs error carrying an absolute path — never forward it verbatim.
|
|
80
|
+
sendServerError(res, 'POST /api/temporal/inject', result.error, 'Hindsight injection failed',
|
|
81
|
+
{ success: false });
|
|
76
82
|
}
|
|
77
83
|
} catch (err) {
|
|
78
|
-
res
|
|
84
|
+
sendServerError(res, 'POST /api/temporal/inject', err, 'Hindsight injection failed');
|
|
79
85
|
}
|
|
80
86
|
});
|
|
81
87
|
|
|
@@ -109,7 +109,18 @@ class RemediationEngine {
|
|
|
109
109
|
}
|
|
110
110
|
|
|
111
111
|
await SemanticHub.ensureInit();
|
|
112
|
-
|
|
112
|
+
// SemanticHub.getGoldenTraces(skillFilter = null) takes a STRING filter,
|
|
113
|
+
// not an options object: `{ limit: 3 }` flowed straight through to
|
|
114
|
+
// vectorHub.searchTraces() as the query text. That used to stringify to
|
|
115
|
+
// "[object Object]" and silently search for that literal; FTS-01 makes a
|
|
116
|
+
// non-string query throw a TypeError, so the call shape is corrected here
|
|
117
|
+
// in the same change. The cap the old `{ limit: 3 }` intended is applied
|
|
118
|
+
// locally, because getGoldenTraces() has no limit parameter.
|
|
119
|
+
const GOLDEN_TRACE_LIMIT = 3;
|
|
120
|
+
const allGoldenTraces = await SemanticHub.getGoldenTraces();
|
|
121
|
+
const goldenTraces = Array.isArray(allGoldenTraces)
|
|
122
|
+
? allGoldenTraces.slice(0, GOLDEN_TRACE_LIMIT)
|
|
123
|
+
: [];
|
|
113
124
|
|
|
114
125
|
if (!goldenTraces || goldenTraces.length === 0) {
|
|
115
126
|
return { strategy: 'GOLDEN_TRACE_INJECTION', result: 'no_traces_found' };
|
|
@@ -32,12 +32,23 @@ class TemporalHub {
|
|
|
32
32
|
}
|
|
33
33
|
|
|
34
34
|
static _verifyMetadata(metadata) {
|
|
35
|
-
if (!metadata.integrity) return false;
|
|
35
|
+
if (!metadata || typeof metadata.integrity !== 'string') return false;
|
|
36
36
|
const { integrity, ...rest } = metadata;
|
|
37
37
|
const expected = crypto.createHmac('sha256', HMAC_KEY)
|
|
38
38
|
.update(JSON.stringify(rest))
|
|
39
39
|
.digest('hex');
|
|
40
|
-
|
|
40
|
+
// timingSafeEqual throws RangeError on unequal BYTE lengths. A String's `.length`
|
|
41
|
+
// counts UTF-16 code units, NOT bytes, so a 64-unit `integrity` containing any
|
|
42
|
+
// non-ASCII character is 65+ bytes and still produced unequal Buffers and still
|
|
43
|
+
// threw. Materialise both Buffers and compare their real byte lengths.
|
|
44
|
+
// NOTE: this is a CORRECTNESS fix, not a security guarantee — HMAC_KEY is a
|
|
45
|
+
// literal in shipped source, the HMAC covers only metadata and not snapshot file
|
|
46
|
+
// CONTENTS, and an absent SNAPSHOT-META.json still bypasses verification. This
|
|
47
|
+
// was never an authenticity control.
|
|
48
|
+
const actual = Buffer.from(integrity, 'utf8');
|
|
49
|
+
const expectedBuf = Buffer.from(expected, 'utf8');
|
|
50
|
+
if (actual.length !== expectedBuf.length) return false;
|
|
51
|
+
return crypto.timingSafeEqual(actual, expectedBuf);
|
|
41
52
|
}
|
|
42
53
|
|
|
43
54
|
/**
|
|
@@ -124,20 +135,41 @@ class TemporalHub {
|
|
|
124
135
|
}
|
|
125
136
|
|
|
126
137
|
const metaPath = path.join(snapshotDir, 'SNAPSHOT-META.json');
|
|
138
|
+
// Read and verify as two SEPARATE stages so only one specific, expected condition
|
|
139
|
+
// (the metadata file genuinely not existing) can reach the tolerant legacy path.
|
|
140
|
+
// The previous single try/catch sniffed err.message for 'integrity verification',
|
|
141
|
+
// so ANY other throw — a JSON.parse SyntaxError, or the crypto RangeError from a
|
|
142
|
+
// non-ASCII `integrity` — fell through to 'proceeding without integrity check' and
|
|
143
|
+
// restored the snapshot anyway. Unexpected throws must fail CLOSED.
|
|
144
|
+
//
|
|
145
|
+
// Error messages deliberately carry no err.message: JSON.parse embeds file content
|
|
146
|
+
// and fs errors embed absolute paths, and these strings reach an HTTP response via
|
|
147
|
+
// hindsight-injector -> temporal-api. Detail goes to the server log only.
|
|
148
|
+
let metaRaw = null;
|
|
127
149
|
try {
|
|
128
|
-
|
|
129
|
-
const metaData = JSON.parse(metaRaw);
|
|
130
|
-
if (!TemporalHub._verifyMetadata(metaData)) {
|
|
131
|
-
throw new Error(`Snapshot ${auditId} failed integrity verification — metadata may be tampered.`);
|
|
132
|
-
}
|
|
150
|
+
metaRaw = await fsPromises.readFile(metaPath, 'utf8');
|
|
133
151
|
} catch (err) {
|
|
134
|
-
if (err.
|
|
135
|
-
|
|
152
|
+
if (err.code !== 'ENOENT') {
|
|
153
|
+
console.error(`[temporal-hub] metadata read failed for ${auditId}:`, err);
|
|
154
|
+
throw new Error(`Snapshot ${auditId} metadata could not be read (${err.code || err.name}) — refusing to restore.`);
|
|
136
155
|
}
|
|
137
156
|
// Missing metadata file on legacy snapshots — allow rollback with warning
|
|
138
157
|
console.warn(`[temporal-hub] No verifiable metadata for ${auditId}, proceeding without integrity check.`);
|
|
139
158
|
}
|
|
140
159
|
|
|
160
|
+
if (metaRaw !== null) {
|
|
161
|
+
let verified = false;
|
|
162
|
+
try {
|
|
163
|
+
verified = TemporalHub._verifyMetadata(JSON.parse(metaRaw));
|
|
164
|
+
} catch (err) {
|
|
165
|
+
console.error(`[temporal-hub] metadata parse/verify failed for ${auditId}:`, err);
|
|
166
|
+
throw new Error(`Snapshot ${auditId} failed integrity verification — metadata unreadable or malformed (${err.name}).`);
|
|
167
|
+
}
|
|
168
|
+
if (!verified) {
|
|
169
|
+
throw new Error(`Snapshot ${auditId} failed integrity verification — metadata may be tampered.`);
|
|
170
|
+
}
|
|
171
|
+
}
|
|
172
|
+
|
|
141
173
|
try {
|
|
142
174
|
const allEntries = await fsPromises.readdir(snapshotDir);
|
|
143
175
|
const files = allEntries.filter(f => f !== 'SNAPSHOT-META.json');
|
package/bin/eval/eval-harness.js
CHANGED
|
@@ -1,5 +1,16 @@
|
|
|
1
1
|
'use strict';
|
|
2
2
|
|
|
3
|
+
const fs = require('fs');
|
|
4
|
+
const os = require('os');
|
|
5
|
+
const path = require('path');
|
|
6
|
+
|
|
7
|
+
// Repo root derived from THIS file's location, never from cwd.
|
|
8
|
+
// bin/utils/paths.js resolves PROJECT_ROOT by walking up from process.cwd(),
|
|
9
|
+
// which on a consumer install lands inside node_modules — the wrong root for a
|
|
10
|
+
// corpus scan, and it would silently yield an empty corpus. __dirname is always
|
|
11
|
+
// <root>/bin/eval.
|
|
12
|
+
const REPO_ROOT = path.resolve(__dirname, '..', '..');
|
|
13
|
+
|
|
3
14
|
/**
|
|
4
15
|
* Recall@K — fraction of relevant items found in the top-k retrieved results.
|
|
5
16
|
* @param {string[]} retrieved - IDs in ranked order
|
|
@@ -79,4 +90,204 @@ async function runEval({ goldenSet, retriever, k }) {
|
|
|
79
90
|
return { meanRecallAtK, meanNDCG, perQuery };
|
|
80
91
|
}
|
|
81
92
|
|
|
82
|
-
|
|
93
|
+
// ── Corpus + runnable golden-set gate (FTS-01) ───────────────────────────────
|
|
94
|
+
// Before this block the file had NO require.main guard, so the documented
|
|
95
|
+
// command `node bin/eval/eval-harness.js --set golden-set-retrieval.json`
|
|
96
|
+
// printed nothing and exited 0 — a gate that could not fail, and therefore not
|
|
97
|
+
// a gate. golden-set-retrieval.json and this harness had zero callers.
|
|
98
|
+
//
|
|
99
|
+
// The shipped golden set names documents by BASENAME (`audit-hash`,
|
|
100
|
+
// `model-router`, `stuck-detector`, …). Those ids exist nowhere in
|
|
101
|
+
// .mindforge/celestial.db, so recall measured against the live trace DB is 0.00
|
|
102
|
+
// by construction whatever the query builder does. The corpus the golden set
|
|
103
|
+
// actually describes is the repo's own module/skill documentation — so build it,
|
|
104
|
+
// index it into a THROWAWAY database under os.tmpdir(), and measure that.
|
|
105
|
+
|
|
106
|
+
const CORPUS_ROOTS = [
|
|
107
|
+
{ dir: 'bin', exts: ['.js'] },
|
|
108
|
+
{ dir: '.mindforge/skills', exts: ['.md'] },
|
|
109
|
+
{ dir: '.mindforge/engine', exts: ['.md'] },
|
|
110
|
+
{ dir: '.agent/hooks', exts: ['.js'] },
|
|
111
|
+
];
|
|
112
|
+
const CORPUS_MAX_BYTES = 20000;
|
|
113
|
+
const CORPUS_SKIP_DIRS = new Set(['node_modules', 'dist', 'coverage']);
|
|
114
|
+
|
|
115
|
+
function _walk(dir, exts, out) {
|
|
116
|
+
let entries;
|
|
117
|
+
try {
|
|
118
|
+
entries = fs.readdirSync(dir, { withFileTypes: true });
|
|
119
|
+
} catch {
|
|
120
|
+
return out; // a missing corpus root is reported via corpusSize, not a throw
|
|
121
|
+
}
|
|
122
|
+
for (const entry of entries) {
|
|
123
|
+
const p = path.join(dir, entry.name);
|
|
124
|
+
if (entry.isDirectory()) {
|
|
125
|
+
if (!CORPUS_SKIP_DIRS.has(entry.name) && !entry.name.startsWith('.')) {
|
|
126
|
+
_walk(p, exts, out);
|
|
127
|
+
}
|
|
128
|
+
} else if (exts.includes(path.extname(entry.name))) {
|
|
129
|
+
out.push(p);
|
|
130
|
+
}
|
|
131
|
+
}
|
|
132
|
+
return out;
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
/**
|
|
136
|
+
* Doc id for a corpus file: its basename, except SKILL.md, which is keyed by its
|
|
137
|
+
* containing skill directory (that is the name the golden set uses).
|
|
138
|
+
* @param {string} file - absolute or relative file path
|
|
139
|
+
* @returns {string} document id
|
|
140
|
+
*/
|
|
141
|
+
function docIdForFile(file) {
|
|
142
|
+
const base = path.basename(file, path.extname(file));
|
|
143
|
+
return base === 'SKILL' ? path.basename(path.dirname(file)) : base;
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
/**
|
|
147
|
+
* Enumerate the document corpus the golden set's `relevant` ids refer to.
|
|
148
|
+
* @param {string} [root] - project root; defaults to REPO_ROOT
|
|
149
|
+
* @returns {Map<string, {id: string, file: string, content: string}>}
|
|
150
|
+
*/
|
|
151
|
+
function buildDocCorpus(root = REPO_ROOT) {
|
|
152
|
+
const corpus = new Map();
|
|
153
|
+
for (const { dir, exts } of CORPUS_ROOTS) {
|
|
154
|
+
for (const file of _walk(path.join(root, dir), exts, [])) {
|
|
155
|
+
const id = docIdForFile(file);
|
|
156
|
+
if (corpus.has(id)) continue; // first wins; basename collisions are rare
|
|
157
|
+
let text = '';
|
|
158
|
+
try {
|
|
159
|
+
text = fs.readFileSync(file, 'utf8').slice(0, CORPUS_MAX_BYTES);
|
|
160
|
+
} catch {
|
|
161
|
+
continue;
|
|
162
|
+
}
|
|
163
|
+
const rel = path.relative(root, file);
|
|
164
|
+
corpus.set(id, { id, file: rel, content: `${id} ${rel}\n${text}` });
|
|
165
|
+
}
|
|
166
|
+
}
|
|
167
|
+
return corpus;
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
/**
|
|
171
|
+
* Index the doc corpus into a throwaway VectorHub and score the golden set.
|
|
172
|
+
* NEVER touches .mindforge/celestial.db — the database lives in os.tmpdir() and
|
|
173
|
+
* is deleted again before this resolves.
|
|
174
|
+
* @param {Object} [opts]
|
|
175
|
+
* @param {string} [opts.goldenSetPath] - defaults to ./golden-set-retrieval.json
|
|
176
|
+
* @param {string} [opts.root] - corpus root; defaults to REPO_ROOT
|
|
177
|
+
* @param {number} [opts.k] - cutoff; defaults to 10
|
|
178
|
+
* @returns {Promise<Object>} runEval metrics plus corpus/coverage diagnostics
|
|
179
|
+
*/
|
|
180
|
+
async function runGoldenSetEval(opts = {}) {
|
|
181
|
+
const goldenSetPath = opts.goldenSetPath || path.join(__dirname, 'golden-set-retrieval.json');
|
|
182
|
+
const root = opts.root || REPO_ROOT;
|
|
183
|
+
const k = opts.k || 10;
|
|
184
|
+
|
|
185
|
+
const golden = JSON.parse(fs.readFileSync(goldenSetPath, 'utf8'));
|
|
186
|
+
const goldenSet = golden.queries || [];
|
|
187
|
+
const corpus = buildDocCorpus(root);
|
|
188
|
+
|
|
189
|
+
const relevantIds = [...new Set(goldenSet.flatMap(q => q.relevant || []))];
|
|
190
|
+
const unresolved = relevantIds.filter(id => !corpus.has(id));
|
|
191
|
+
|
|
192
|
+
const { VectorHub } = require('../memory/vector-hub');
|
|
193
|
+
const dbDir = fs.mkdtempSync(path.join(os.tmpdir(), 'mindforge-eval-'));
|
|
194
|
+
const hub = new VectorHub(path.join(dbDir, 'eval-corpus.db'));
|
|
195
|
+
await hub.init();
|
|
196
|
+
try {
|
|
197
|
+
// The hub is throwaway, so raise the autosave batch size: the default of 10
|
|
198
|
+
// exports the whole growing database to disk every 10 documents, which turns
|
|
199
|
+
// a 0.5 s index build into a quadratic one for no durability benefit here.
|
|
200
|
+
hub._batchSize = Number.MAX_SAFE_INTEGER;
|
|
201
|
+
for (const doc of corpus.values()) {
|
|
202
|
+
await hub.saveKnowledge({ id: doc.id, type: 'doc', content: doc.content, source: doc.file });
|
|
203
|
+
}
|
|
204
|
+
const retriever = async (query) =>
|
|
205
|
+
(await hub.searchKnowledge(query, { limit: k })).map(r => r.id);
|
|
206
|
+
const metrics = await runEval({ goldenSet, retriever, k });
|
|
207
|
+
return {
|
|
208
|
+
...metrics,
|
|
209
|
+
k,
|
|
210
|
+
corpusSize: corpus.size,
|
|
211
|
+
relevantIdCount: relevantIds.length,
|
|
212
|
+
unresolvedRelevantIds: unresolved,
|
|
213
|
+
baseline: golden.baseline || null,
|
|
214
|
+
};
|
|
215
|
+
} finally {
|
|
216
|
+
await hub.close();
|
|
217
|
+
fs.rmSync(dbDir, { recursive: true, force: true });
|
|
218
|
+
}
|
|
219
|
+
}
|
|
220
|
+
|
|
221
|
+
module.exports = {
|
|
222
|
+
recallAtK,
|
|
223
|
+
ndcg,
|
|
224
|
+
runEval,
|
|
225
|
+
buildDocCorpus,
|
|
226
|
+
docIdForFile,
|
|
227
|
+
runGoldenSetEval,
|
|
228
|
+
};
|
|
229
|
+
|
|
230
|
+
// ── CLI ──────────────────────────────────────────────────────────────────────
|
|
231
|
+
// node bin/eval/eval-harness.js [--set golden-set-retrieval.json] [--k 10]
|
|
232
|
+
// [--min-recall 0.55] [--json]
|
|
233
|
+
// Exits 1 when mean recall@k is below --min-recall, or when any golden
|
|
234
|
+
// `relevant` id no longer resolves to a real document (golden-set drift).
|
|
235
|
+
// `npm run eval:retrieval` runs this with the committed baseline floor.
|
|
236
|
+
if (require.main === module) {
|
|
237
|
+
const argv = process.argv.slice(2);
|
|
238
|
+
const flag = (name, fallback) => {
|
|
239
|
+
const i = argv.indexOf(name);
|
|
240
|
+
return i >= 0 && argv[i + 1] !== undefined ? argv[i + 1] : fallback;
|
|
241
|
+
};
|
|
242
|
+
const setArg = flag('--set', 'golden-set-retrieval.json');
|
|
243
|
+
const goldenSetPath = path.isAbsolute(setArg)
|
|
244
|
+
? setArg
|
|
245
|
+
: path.join(__dirname, path.basename(setArg));
|
|
246
|
+
const num = (name, fallback) => {
|
|
247
|
+
const raw = flag(name, String(fallback));
|
|
248
|
+
const n = Number(raw);
|
|
249
|
+
// Fail closed: an unparseable threshold must never silently become 0.
|
|
250
|
+
if (!Number.isFinite(n) || n < 0) {
|
|
251
|
+
console.error(`[eval] ERROR: ${name} must be a non-negative number, got ${JSON.stringify(raw)}`);
|
|
252
|
+
process.exit(1);
|
|
253
|
+
}
|
|
254
|
+
return n;
|
|
255
|
+
};
|
|
256
|
+
const k = Math.max(1, Math.trunc(num('--k', 10)));
|
|
257
|
+
const minRecall = num('--min-recall', 0);
|
|
258
|
+
|
|
259
|
+
runGoldenSetEval({ goldenSetPath, k })
|
|
260
|
+
.then((res) => {
|
|
261
|
+
if (argv.includes('--json')) {
|
|
262
|
+
console.log(JSON.stringify(res, null, 2));
|
|
263
|
+
} else {
|
|
264
|
+
console.log(`[eval] corpus: ${res.corpusSize} docs · golden queries: ${res.perQuery.length} · k=${res.k}`);
|
|
265
|
+
for (const q of res.perQuery) {
|
|
266
|
+
console.log(` recall=${q.recall.toFixed(3)} nDCG=${q.ndcg.toFixed(3)} hits=${q.retrieved.length} ${q.query}`);
|
|
267
|
+
}
|
|
268
|
+
console.log(`[eval] mean recall@${res.k} = ${res.meanRecallAtK.toFixed(4)}`);
|
|
269
|
+
console.log(`[eval] mean nDCG@${res.k} = ${res.meanNDCG.toFixed(4)}`);
|
|
270
|
+
if (res.baseline && typeof res.baseline.meanRecallAtK === 'number') {
|
|
271
|
+
// Round BEFORE choosing the sign, so a difference smaller than the
|
|
272
|
+
// printed precision reads as +0.0000 rather than a bogus -0.0000.
|
|
273
|
+
const delta = Number((res.meanRecallAtK - res.baseline.meanRecallAtK).toFixed(4));
|
|
274
|
+
console.log(`[eval] committed baseline recall@${res.baseline.k || res.k} = ${res.baseline.meanRecallAtK.toFixed(4)} (delta ${delta >= 0 ? '+' : ''}${delta.toFixed(4)})`);
|
|
275
|
+
}
|
|
276
|
+
}
|
|
277
|
+
if (res.unresolvedRelevantIds.length > 0) {
|
|
278
|
+
console.error(`[eval] FAIL: ${res.unresolvedRelevantIds.length} golden id(s) no longer resolve to a document: ${res.unresolvedRelevantIds.join(', ')}`);
|
|
279
|
+
process.exit(1);
|
|
280
|
+
}
|
|
281
|
+
if (res.meanRecallAtK < minRecall) {
|
|
282
|
+
console.error(`[eval] FAIL: mean recall@${res.k} ${res.meanRecallAtK.toFixed(4)} < --min-recall ${minRecall}`);
|
|
283
|
+
process.exit(1);
|
|
284
|
+
}
|
|
285
|
+
})
|
|
286
|
+
.catch((err) => {
|
|
287
|
+
// sql.js throws bare strings for some binding errors, so an Error-shaped
|
|
288
|
+
// formatter alone would print "undefined" and hide the real failure.
|
|
289
|
+
const detail = (err && (err.stack || err.message)) || String(err);
|
|
290
|
+
console.error(`[eval] ERROR: ${detail}`);
|
|
291
|
+
process.exit(1);
|
|
292
|
+
});
|
|
293
|
+
}
|
|
@@ -1,6 +1,15 @@
|
|
|
1
1
|
{
|
|
2
2
|
"description": "Golden set for retrieval quality evaluation. Each entry has a natural-language query and the IDs of documents that SHOULD be retrieved.",
|
|
3
3
|
"version": "1.0.0",
|
|
4
|
+
"baseline": {
|
|
5
|
+
"note": "Committed measurement so a retrieval regression is detectable. Reproduce with `npm run eval:retrieval`. Corpus = repo docs enumerated by buildDocCorpus() in bin/eval/eval-harness.js; retriever = VectorHub.searchKnowledge (OR-joined terms, ranked by tf-idf over FTS4 matchinfo('pcnx')). Before FTS-01 both figures were 0.0000 because the whole query was one FTS phrase.",
|
|
6
|
+
"k": 10,
|
|
7
|
+
"meanRecallAtK": 0.6417,
|
|
8
|
+
"meanNDCG": 0.5698,
|
|
9
|
+
"corpusSize": 517,
|
|
10
|
+
"gateMinRecall": 0.55,
|
|
11
|
+
"measuredOn": "v11.9.2 (fix/v11.9.2-ship-blockers), sql.js 1.14.1 / SQLite 3.49.1"
|
|
12
|
+
},
|
|
4
13
|
"queries": [
|
|
5
14
|
{
|
|
6
15
|
"query": "how does the audit hash chain work",
|
|
@@ -165,6 +165,12 @@ class PolicyEngine {
|
|
|
165
165
|
}
|
|
166
166
|
|
|
167
167
|
logAudit(intent, impactScore, verdict) {
|
|
168
|
+
// LOCK-01: AuditWriter.write -> appendAuditEntrySync now takes a FAIL-CLOSED lock,
|
|
169
|
+
// so this can reject under contention. It is intentionally not awaited (the verdict
|
|
170
|
+
// path is synchronous), so catch here — otherwise the rejection escapes to a global
|
|
171
|
+
// unhandledRejection handler and the lost audit record is invisible at this site.
|
|
172
|
+
// The verdict is still returned: promoting an audit-write failure to an implicit
|
|
173
|
+
// DENY is a behaviour change for v12, not a patch.
|
|
168
174
|
this._auditWriter.write({
|
|
169
175
|
timestamp: new Date().toISOString(),
|
|
170
176
|
requestId: verdict.requestId,
|
|
@@ -175,6 +181,8 @@ class PolicyEngine {
|
|
|
175
181
|
impactScore,
|
|
176
182
|
verdict: verdict.verdict,
|
|
177
183
|
reason: verdict.reason
|
|
184
|
+
}).catch(err => {
|
|
185
|
+
console.error(`[APO-AUDIT-FAIL] [${verdict.requestId}] RISK-AUDIT append failed — decision NOT recorded: ${err.message}`);
|
|
178
186
|
});
|
|
179
187
|
}
|
|
180
188
|
|
|
@@ -18,8 +18,14 @@ class HindsightInjector {
|
|
|
18
18
|
console.log(`[hindsight] Injecting fix at ${auditId}: "${fixDescription}"`);
|
|
19
19
|
|
|
20
20
|
try {
|
|
21
|
-
// 1. Rollback .planning directory
|
|
22
|
-
|
|
21
|
+
// 1. Rollback .planning directory.
|
|
22
|
+
// MUST be awaited: rollbackTo is async (engine/temporal-hub.js), so without
|
|
23
|
+
// await its rejection escapes this try/catch entirely, the process dies on an
|
|
24
|
+
// unhandled rejection, AND execution still falls through to steps 2-3 — which
|
|
25
|
+
// fsync a hash-chained `hindsight_injected` entry and flip auto-state.json for
|
|
26
|
+
// a rollback that never happened. The chain then verifies as valid but records
|
|
27
|
+
// an event that did not occur.
|
|
28
|
+
await TemporalHub.rollbackTo(auditId);
|
|
23
29
|
|
|
24
30
|
// 2. Append the "Hindsight" event to AUDIT.jsonl via the unified, hash-chained,
|
|
25
31
|
// durable append (UC-04b) so this entry links into the single verifiable chain.
|
|
@@ -189,7 +189,13 @@ function main() {
|
|
|
189
189
|
if (!fs.existsSync(storeDir)) {
|
|
190
190
|
fs.mkdirSync(storeDir, { recursive: true });
|
|
191
191
|
}
|
|
192
|
-
|
|
192
|
+
// LOCK-01: instinct-cli's prune/import does read-modify-atomic-rename under this
|
|
193
|
+
// same lock, so an unlocked append here can be clobbered by that rename. Low
|
|
194
|
+
// maxTries because this is a hook fast path — the catch below keeps it non-fatal.
|
|
195
|
+
const { withFileLock } = require('../utils/file-lock');
|
|
196
|
+
withFileLock(storePath, () => {
|
|
197
|
+
fs.appendFileSync(storePath, JSON.stringify(entry) + '\n');
|
|
198
|
+
}, { maxTries: 10, label: 'instinct-store' });
|
|
193
199
|
incrementSessionCount();
|
|
194
200
|
} catch {
|
|
195
201
|
// Non-fatal — hooks must not block
|
package/bin/install.js
CHANGED
|
@@ -123,7 +123,7 @@ function printHelp() {
|
|
|
123
123
|
npx mindforge-cc@latest --all --global Global install for all runtimes
|
|
124
124
|
|
|
125
125
|
${c.bold('DOCUMENTATION')}
|
|
126
|
-
${c.dim('https://github.com/
|
|
126
|
+
${c.dim('https://github.com/sairam0424/MindForge')}
|
|
127
127
|
${c.dim('docs/enterprise-setup.md (after install)')}
|
|
128
128
|
\n`);
|
|
129
129
|
}
|
|
@@ -24,6 +24,7 @@ const path = require('path');
|
|
|
24
24
|
|
|
25
25
|
const guard = require('./lib/ssrf-guard');
|
|
26
26
|
const { detectProject } = require('../hooks/lib/detect-project');
|
|
27
|
+
const { withFileLock } = require('../utils/file-lock');
|
|
27
28
|
|
|
28
29
|
const CONFIG_PATH = path.join(process.cwd(), '.mindforge', 'config.json');
|
|
29
30
|
|
|
@@ -71,33 +72,15 @@ function writeStoreAtomic(p, entries) {
|
|
|
71
72
|
}
|
|
72
73
|
|
|
73
74
|
/**
|
|
74
|
-
* Advisory lock
|
|
75
|
-
*
|
|
75
|
+
* Advisory lock — now a thin delegate to the shared fail-closed lock (LOCK-01).
|
|
76
|
+
* The implementation moved to bin/utils/file-lock.js so the audit chain and the
|
|
77
|
+
* knowledge graph use the SAME lock semantics as the instinct store. Behaviour is
|
|
78
|
+
* unchanged: O_EXCL create, 50 tries, ~20ms waits, 10s stale reclaim, unlink in
|
|
79
|
+
* finally, THROW (never write anyway) when unacquirable — same error text.
|
|
76
80
|
* Read the store INSIDE this so a prune/import rewrite can't race a hook append.
|
|
77
81
|
*/
|
|
78
82
|
function withStoreLock(p, fn) {
|
|
79
|
-
|
|
80
|
-
const maxTries = 50, waitMs = 20, staleMs = 10000;
|
|
81
|
-
let held = false;
|
|
82
|
-
for (let i = 0; i < maxTries && !held; i++) {
|
|
83
|
-
try {
|
|
84
|
-
const fd = fs.openSync(lock, 'wx');
|
|
85
|
-
fs.closeSync(fd);
|
|
86
|
-
held = true;
|
|
87
|
-
} catch (err) {
|
|
88
|
-
if (err.code !== 'EEXIST') throw err;
|
|
89
|
-
// stale-break: if the lockfile is older than staleMs, remove it.
|
|
90
|
-
try {
|
|
91
|
-
const age = Date.now() - fs.statSync(lock).mtimeMs;
|
|
92
|
-
if (age > staleMs) { fs.unlinkSync(lock); continue; }
|
|
93
|
-
} catch { /* lock vanished — retry */ }
|
|
94
|
-
const until = Date.now() + waitMs;
|
|
95
|
-
while (Date.now() < until) { /* busy-wait (short) */ }
|
|
96
|
-
}
|
|
97
|
-
}
|
|
98
|
-
if (!held) throw new Error(`could not acquire instinct-store lock: ${lock}`);
|
|
99
|
-
try { return fn(); }
|
|
100
|
-
finally { try { fs.unlinkSync(lock); } catch { /* already gone */ } }
|
|
83
|
+
return withFileLock(p, fn, { label: 'instinct-store' });
|
|
101
84
|
}
|
|
102
85
|
|
|
103
86
|
function currentProjectId() {
|
|
@@ -133,7 +133,14 @@ function createCausalEdges(bugId, allEntries, vectors) {
|
|
|
133
133
|
weight: sim,
|
|
134
134
|
reason: `Bug pattern potentially caused by code pattern (sim: ${sim.toFixed(3)})`,
|
|
135
135
|
});
|
|
136
|
-
} catch {
|
|
136
|
+
} catch (err) {
|
|
137
|
+
// LOCK-01: this catch predates the graph-edges lock. Without the guard below,
|
|
138
|
+
// an edge LOST to lock contention is reported as a skipped duplicate — a
|
|
139
|
+
// silent write loss wearing the label of a no-op.
|
|
140
|
+
if (/could not acquire/.test(err.message)) {
|
|
141
|
+
console.warn(`[KnowledgeCapture] edge NOT written — graph-edges lock unavailable: ${err.message}`);
|
|
142
|
+
}
|
|
143
|
+
}
|
|
137
144
|
}
|
|
138
145
|
}
|
|
139
146
|
}
|
|
@@ -165,7 +172,14 @@ function createInformsEdges(decisionId, allEntries, vectors) {
|
|
|
165
172
|
weight: sim,
|
|
166
173
|
reason: `Decision informs domain knowledge (sim: ${sim.toFixed(3)})`,
|
|
167
174
|
});
|
|
168
|
-
} catch {
|
|
175
|
+
} catch (err) {
|
|
176
|
+
// LOCK-01: this catch predates the graph-edges lock. Without the guard below,
|
|
177
|
+
// an edge LOST to lock contention is reported as a skipped duplicate — a
|
|
178
|
+
// silent write loss wearing the label of a no-op.
|
|
179
|
+
if (/could not acquire/.test(err.message)) {
|
|
180
|
+
console.warn(`[KnowledgeCapture] edge NOT written — graph-edges lock unavailable: ${err.message}`);
|
|
181
|
+
}
|
|
182
|
+
}
|
|
169
183
|
}
|
|
170
184
|
}
|
|
171
185
|
}
|
|
@@ -180,7 +194,13 @@ function reinforceRelatedEdges(nodeId) {
|
|
|
180
194
|
for (const edge of edges.slice(0, 3)) { // Top 3 edges only
|
|
181
195
|
Graph.reinforceEdge(edge.id);
|
|
182
196
|
}
|
|
183
|
-
} catch {
|
|
197
|
+
} catch (err) {
|
|
198
|
+
// LOCK-01: reinforceEdge now takes the graph-edges lock, so "non-critical" would
|
|
199
|
+
// also absorb a contention failure. Surface that one; keep the rest quiet.
|
|
200
|
+
if (/could not acquire/.test(err.message)) {
|
|
201
|
+
console.warn(`[KnowledgeCapture] reinforcement NOT recorded — graph-edges lock unavailable: ${err.message}`);
|
|
202
|
+
}
|
|
203
|
+
}
|
|
184
204
|
}
|
|
185
205
|
|
|
186
206
|
// ── Event-specific capture functions ─────────────────────────────────────────
|