mindforge-cc 11.9.2 → 11.9.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.agent/CLAUDE.md +37 -13
- package/.agent/hooks/mindforge-block-no-verify.js +61 -13
- package/.agent/hooks/mindforge-config-protection.js +82 -3
- package/.agent/hooks/mindforge-context-monitor.js +1 -1
- package/.agent/hooks/mindforge-workflow-guard.js +2 -2
- package/.agent/hooks/run-with-flags.js +190 -20
- package/.agent/mindforge/browse.md +2 -2
- package/.agent/mindforge/checkpoint.md +1 -1
- package/.agent/mindforge/harness-audit.md +1 -1
- package/.agent/mindforge/orch-add-feature.md +1 -1
- package/.agent/mindforge/orch-build-mvp.md +1 -1
- package/.agent/mindforge/orch-change-feature.md +1 -1
- package/.agent/mindforge/orch-fix-defect.md +1 -1
- package/.agent/mindforge/orch-refine-code.md +1 -1
- package/.agent/mindforge/qa.md +2 -2
- package/.claude/CLAUDE.md +37 -13
- package/.claude/commands/mindforge/browse.md +2 -2
- package/.claude/commands/mindforge/checkpoint.md +1 -1
- package/.claude/commands/mindforge/harness-audit.md +1 -1
- package/.claude/commands/mindforge/orch-add-feature.md +1 -1
- package/.claude/commands/mindforge/orch-build-mvp.md +1 -1
- package/.claude/commands/mindforge/orch-change-feature.md +1 -1
- package/.claude/commands/mindforge/orch-fix-defect.md +1 -1
- package/.claude/commands/mindforge/orch-refine-code.md +1 -1
- package/.claude/commands/mindforge/qa.md +2 -2
- package/.mindforge/MINDFORGE-SCHEMA.json +1 -1
- package/.mindforge/config.json +3 -3
- package/.mindforge/engine/autonomous/headless-adapter.md +2 -2
- package/.mindforge/engine/temporal-protocol.md +2 -2
- package/.mindforge/governance/change-classifier.md +20 -4
- package/.mindforge/memory/sync-manifest.json +1 -1
- package/.mindforge/skills/agent-architecture-audit/SKILL.md +2 -2
- package/.mindforge/skills/orch-pipeline/SKILL.md +4 -4
- package/CHANGELOG.md +194 -0
- package/MINDFORGE.md +13 -6
- package/README.md +4 -3
- package/RELEASENOTES.md +2 -2
- package/SECURITY.md +22 -3
- package/bin/autonomous/auto-runner.js +65 -2
- package/bin/change-classifier.js +151 -16
- package/bin/dashboard/api-router.js +18 -38
- package/bin/dashboard/frontend/app.js +429 -0
- package/bin/dashboard/frontend/index.html +13 -406
- package/bin/dashboard/metrics-aggregator.js +46 -22
- package/bin/dashboard/server.js +160 -1
- package/bin/dashboard/sse-bridge.js +11 -8
- package/bin/engine/sre-manager.js +1 -1
- package/bin/engine/temporal-cli.js +56 -6
- package/bin/engine/verification-runner.js +134 -17
- package/bin/engine/verify-cli.js +25 -7
- package/bin/governance/approval-record.js +147 -0
- package/bin/governance/approve.js +12 -7
- package/bin/governance/policy-engine.js +33 -3
- package/bin/governance/policy-gate-hardened.js +36 -1
- package/bin/governance/verify-approvals.js +163 -0
- package/bin/harness-audit.js +224 -10
- package/bin/hooks/instinct-capture-hook.js +12 -4
- package/bin/install.js +63 -3
- package/bin/installer/harness-adapter-compliance.js +339 -28
- package/bin/installer/hook-registration.js +504 -0
- package/bin/installer-core.js +451 -63
- package/bin/learning/instinct-cli.js +7 -0
- package/bin/memory/vector-hub.js +196 -13
- package/bin/migrations/0.6.0-to-1.0.0.js +30 -25
- package/bin/migrations/1.0.0-to-2.0.0.js +22 -23
- package/bin/mindforge-cli.js +67 -6
- package/bin/models/cost-tracker.js +104 -6
- package/bin/models/model-client.js +6 -1
- package/bin/revops/debt-monitor.js +57 -13
- package/bin/security/trust-gate-hook.js +50 -6
- package/bin/skill-validator.js +6 -1
- package/bin/skills-builder/skill-scorer.js +46 -6
- package/bin/updater/self-update.js +6 -1
- package/bin/updater/version-comparator.js +21 -1
- package/bin/utils/mindforge-version.js +99 -0
- package/bin/utils/redact-secrets.js +106 -0
- package/bin/validate-config.js +42 -2
- package/bin/wizard/setup-wizard.js +4 -1
- package/bin/wizard/theme.js +9 -1
- package/changelogs/index.json +11 -9
- package/changelogs/v11.9.3.md +195 -0
- package/docs/References/config-reference.md +5 -2
- package/docs/References/sdk-api.md +1 -1
- package/docs/Templates/Codebase/architecture.md +1 -1
- package/docs/commands-reference.md +4 -5
- package/docs/faq.md +25 -5
- package/docs/getting-started.md +3 -3
- package/docs/sdk-reference.md +15 -7
- package/docs/troubleshooting.md +10 -6
- package/docs/user-guide.md +14 -14
- package/examples/sdk-integration/README.md +1 -1
- package/package.json +7 -3
- package/subagents/.claude-plugin/marketplace.json +1 -1
- package/bin/dashboard/approval-handler.js +0 -136
|
@@ -23,6 +23,7 @@ const fs = require('fs');
|
|
|
23
23
|
const path = require('path');
|
|
24
24
|
|
|
25
25
|
const guard = require('./lib/ssrf-guard');
|
|
26
|
+
const { redactSecrets } = require('../utils/redact-secrets');
|
|
26
27
|
const { detectProject } = require('../hooks/lib/detect-project');
|
|
27
28
|
const { withFileLock } = require('../utils/file-lock');
|
|
28
29
|
|
|
@@ -197,7 +198,13 @@ async function cmdImport(args, cfg) {
|
|
|
197
198
|
for (const e of incoming) {
|
|
198
199
|
if (!e || !guard.validateInstinctId(e.id || '')) { process.stderr.write(`skipping entry with invalid id: ${e && e.id}\n`); continue; }
|
|
199
200
|
if ((Number(e.confidence) || 0) < minC) continue;
|
|
201
|
+
// Import is the store's second ingress and it accepts an https:// source, so incoming text is
|
|
202
|
+
// untrusted twice over: it may carry a credential from whoever exported it. Redacting here
|
|
203
|
+
// rather than at every egress means export, promote, list and the LLM promotion path into a
|
|
204
|
+
// published SKILL.md all inherit already-clean data — one choke point instead of five.
|
|
200
205
|
valid.push(Object.assign({}, e, {
|
|
206
|
+
observation: redactSecrets(e.observation),
|
|
207
|
+
behavior: redactSecrets(e.behavior),
|
|
201
208
|
project: projectName, project_id: scopeId, source: 'imported',
|
|
202
209
|
imported_from: source, created_at: now, updated_at: now,
|
|
203
210
|
}));
|
package/bin/memory/vector-hub.js
CHANGED
|
@@ -8,6 +8,7 @@
|
|
|
8
8
|
const crypto = require('crypto');
|
|
9
9
|
const path = require('path');
|
|
10
10
|
const fs = require('fs');
|
|
11
|
+
const { withFileLock } = require('../utils/file-lock');
|
|
11
12
|
|
|
12
13
|
// ── FTS retrieval (FTS-01) ───────────────────────────────────────────────────
|
|
13
14
|
// A MATCH argument is an FTS *query expression*, not a literal. Wrapping the
|
|
@@ -167,6 +168,9 @@ class VectorHub {
|
|
|
167
168
|
// deliberately bias toward flushing.
|
|
168
169
|
this._pendingSaves = 0;
|
|
169
170
|
this._exitGuardInstalled = false;
|
|
171
|
+
// null until a file is loaded or written. null means "no expectation", so a first write into a
|
|
172
|
+
// fresh directory is never treated as a conflict.
|
|
173
|
+
this._diskFingerprint = null;
|
|
170
174
|
}
|
|
171
175
|
|
|
172
176
|
_installExitGuard() {
|
|
@@ -174,10 +178,59 @@ class VectorHub {
|
|
|
174
178
|
this._exitGuardInstalled = true;
|
|
175
179
|
// 'exit' handlers can only run synchronous code — saveSync() fits exactly.
|
|
176
180
|
process.once('exit', () => {
|
|
177
|
-
if (this._db
|
|
181
|
+
if (!this._db) return;
|
|
182
|
+
// _pendingSaves === 0 means every scheduled save already ran commitDb, which consumes
|
|
183
|
+
// its tmp file by renaming it. So there is nothing outstanding and the in-memory DB is
|
|
184
|
+
// durable — the reap below is safe without a further write.
|
|
185
|
+
const durable = this._pendingSaves > 0 ? this.saveSync() : true;
|
|
186
|
+
if (durable) this._reapAbandonedExport();
|
|
178
187
|
});
|
|
179
188
|
}
|
|
180
189
|
|
|
190
|
+
/**
|
|
191
|
+
* Delete THIS process's abandoned tmp export.
|
|
192
|
+
*
|
|
193
|
+
* THE LEAK. save() is a two-step chain: writeTmpDurable() writes and fsyncs
|
|
194
|
+
* `<db>.tmp.<pid>.async`, then commitDb() renames it into place. A 'exit' handler can only
|
|
195
|
+
* run synchronous code, so a process that exits with a save in flight abandons the pending
|
|
196
|
+
* .then() — the microtask never runs, commitDb never renames, and the tmp file stays on disk
|
|
197
|
+
* forever. Nothing anywhere deleted it. Measured in this repository: 176 orphaned
|
|
198
|
+
* `celestial.db.tmp.<pid>.async` files totalling 1.8 GB, against a live database of 10.6 MB.
|
|
199
|
+
* Each orphan is a complete copy of the database, so the directory grew by ~11 MB per exit.
|
|
200
|
+
*
|
|
201
|
+
* Reproduced with a control arm: three runs that call process.exit() without close() leave
|
|
202
|
+
* three orphans, one per pid; three runs that await close() leave zero. Orphan size records
|
|
203
|
+
* how far the chain got — 0 bytes if the process died inside fs.open, a full export if the
|
|
204
|
+
* write landed and only the rename was lost.
|
|
205
|
+
*
|
|
206
|
+
* The trigger is ordinary, not exotic. nexus-tracer.js is the framework-wide tracing
|
|
207
|
+
* singleton, so any command that traces and then exits hits this window;
|
|
208
|
+
* bin/migrations/v9-unified-memory.js calls process.exit() and never calls close() at all.
|
|
209
|
+
*
|
|
210
|
+
* WHY THIS IS SAFE, AND WHY IT IS GATED. An earlier snapshot from the SAME process is a
|
|
211
|
+
* subset of what saveSync() just exported: saveSync() serialises the entire current
|
|
212
|
+
* in-memory database, which already contains every row the abandoned export held. So once
|
|
213
|
+
* persistence is confirmed the orphan is provably redundant.
|
|
214
|
+
*
|
|
215
|
+
* It is confirmed, not assumed. When saveSync() fails — commitDb throws, or the conflict
|
|
216
|
+
* sidecar could not be written — the abandoned export may be the only copy of those rows on
|
|
217
|
+
* disk, and deleting it would be exactly the silent data destruction commitDb refuses to
|
|
218
|
+
* commit. In that case the caller does NOT reap, and the file is deliberately left behind for
|
|
219
|
+
* a human. Accumulating a file on a failed write is the correct trade against destroying the
|
|
220
|
+
* last copy of it. Measured: zero `.sync` orphans exist in this repository, so that path does
|
|
221
|
+
* not fire in practice — the accumulation was entirely the async one.
|
|
222
|
+
*
|
|
223
|
+
* Both suffixes are swept because saveSync()'s own tmp leaks by the same argument if
|
|
224
|
+
* commitDb throws after the write; that arm is currently unobserved but not impossible.
|
|
225
|
+
*/
|
|
226
|
+
_reapAbandonedExport() {
|
|
227
|
+
for (const suffix of ['async', 'sync']) {
|
|
228
|
+
try {
|
|
229
|
+
fs.unlinkSync(`${this.dbPath}.tmp.${process.pid}.${suffix}`);
|
|
230
|
+
} catch { /* not present — the normal case, since commitDb usually consumes it */ }
|
|
231
|
+
}
|
|
232
|
+
}
|
|
233
|
+
|
|
181
234
|
_ensureDir() {
|
|
182
235
|
const dir = path.dirname(this.dbPath);
|
|
183
236
|
if (!fs.existsSync(dir)) {
|
|
@@ -220,6 +273,9 @@ class VectorHub {
|
|
|
220
273
|
if (fs.existsSync(this.dbPath)) {
|
|
221
274
|
const buffer = fs.readFileSync(this.dbPath);
|
|
222
275
|
this._db = new SQL.Database(buffer);
|
|
276
|
+
// What the file looked like when this process took its copy. Every later write compares
|
|
277
|
+
// against it, so a rewrite by another process is detected instead of silently clobbered.
|
|
278
|
+
this._diskFingerprint = dbFingerprint(this.dbPath);
|
|
223
279
|
} else {
|
|
224
280
|
this._db = new SQL.Database();
|
|
225
281
|
}
|
|
@@ -513,11 +569,24 @@ class VectorHub {
|
|
|
513
569
|
// COMPLETED (success or failure). The exit guard fires saveSync() while any
|
|
514
570
|
// scheduled save is still outstanding — see _installExitGuard().
|
|
515
571
|
this._pendingSaves++;
|
|
516
|
-
this._saveChain = this._saveChain
|
|
572
|
+
this._saveChain = this._saveChain
|
|
573
|
+
// Write the tmp asynchronously, then commit it under the lock. The commit MUST be the last
|
|
574
|
+
// step — see commitDb's note on the check-then-write gap that made this guard bypassable.
|
|
575
|
+
.then(() => writeTmpDurable(dbPath, buffer))
|
|
576
|
+
.then((tmpPath) => {
|
|
577
|
+
const res = commitDb(dbPath, this._diskFingerprint, buffer, tmpPath);
|
|
578
|
+
if (res.ok) {
|
|
579
|
+
this._diskFingerprint = res.fingerprint;
|
|
580
|
+
this._pendingSaves--; // decrement ONLY on a durable write; see below
|
|
581
|
+
}
|
|
582
|
+
})
|
|
517
583
|
.catch((err) => {
|
|
584
|
+
// Left deliberately outstanding. _pendingSaves gates the exit guard, so decrementing after a
|
|
585
|
+
// FAILED save would make the guard skip its last-resort saveSync() and drop this batch — the
|
|
586
|
+
// exact loss the counter exists to prevent. withFileLock throws when it cannot acquire, so
|
|
587
|
+
// lock contention lands here, and leaving the count high is what makes the retry happen.
|
|
518
588
|
console.warn(`[VectorHub] Failed to save database: ${err.message}`);
|
|
519
|
-
})
|
|
520
|
-
.then(() => { this._pendingSaves--; });
|
|
589
|
+
});
|
|
521
590
|
return this._saveChain;
|
|
522
591
|
}
|
|
523
592
|
|
|
@@ -525,13 +594,18 @@ class VectorHub {
|
|
|
525
594
|
* Synchronous, crash-safe persistence — used only on shutdown to GUARANTEE
|
|
526
595
|
* no acknowledged write is lost if the process exits before the async save
|
|
527
596
|
* chain drains. Correctness over non-blocking here.
|
|
597
|
+
*
|
|
598
|
+
* @returns {boolean} true when the in-memory database is durably on disk — either committed
|
|
599
|
+
* over the live file or preserved in a conflict sidecar. false when it is NOT, which is the
|
|
600
|
+
* signal _reapAbandonedExport() needs: on false, an abandoned tmp export may hold the only
|
|
601
|
+
* copy of those rows and must be left alone.
|
|
528
602
|
*/
|
|
529
603
|
saveSync() {
|
|
530
|
-
if (!this._db) return;
|
|
604
|
+
if (!this._db) return false;
|
|
531
605
|
try {
|
|
532
606
|
this._ensureDir();
|
|
533
607
|
const buffer = Buffer.from(this._db.export());
|
|
534
|
-
const tmpPath = `${this.dbPath}.tmp.${process.pid}`;
|
|
608
|
+
const tmpPath = `${this.dbPath}.tmp.${process.pid}.sync`;
|
|
535
609
|
const fd = fs.openSync(tmpPath, 'w');
|
|
536
610
|
try {
|
|
537
611
|
fs.writeSync(fd, buffer);
|
|
@@ -539,13 +613,26 @@ class VectorHub {
|
|
|
539
613
|
} finally {
|
|
540
614
|
fs.closeSync(fd);
|
|
541
615
|
}
|
|
542
|
-
|
|
616
|
+
// Commit under the lock. Worst case this blocks the exit handler for withFileLock's bounded
|
|
617
|
+
// retry ceiling (~1-2s) and then throws into the catch below — bounded, logged, and far better
|
|
618
|
+
// than writing over a file another process just changed.
|
|
619
|
+
const res = commitDb(this.dbPath, this._diskFingerprint, buffer, tmpPath);
|
|
620
|
+
if (!res.ok) {
|
|
621
|
+
this._pendingSaves = 0; // the data is in the sidecar; retrying would clobber again
|
|
622
|
+
// A written sidecar IS durable persistence — the bytes are on disk under a name nothing
|
|
623
|
+
// else will touch. A NULL sidecar is not: commitDb unlinked the tmp and wrote nothing,
|
|
624
|
+
// so this export exists only in memory and is about to be lost with the process.
|
|
625
|
+
return res.sidecar !== null && res.sidecar !== undefined;
|
|
626
|
+
}
|
|
627
|
+
this._diskFingerprint = res.fingerprint;
|
|
543
628
|
// A sync export captures the full in-memory DB — a superset of anything the
|
|
544
629
|
// outstanding async saves would have written — so the pending work is now
|
|
545
630
|
// durably satisfied. Clearing the counter prevents a redundant second flush.
|
|
546
631
|
this._pendingSaves = 0;
|
|
632
|
+
return true;
|
|
547
633
|
} catch (err) {
|
|
548
634
|
console.warn(`[VectorHub] Failed to save database (sync): ${err.message}`);
|
|
635
|
+
return false;
|
|
549
636
|
}
|
|
550
637
|
}
|
|
551
638
|
|
|
@@ -840,9 +927,108 @@ class VectorHub {
|
|
|
840
927
|
// ── Durable async DB file write (UC-09) ───────────────────────────────────────
|
|
841
928
|
// Crash-safe: write to a tmp file, fsync, then atomically rename over the target.
|
|
842
929
|
// A crash mid-write leaves the previous good .db intact (rename is atomic on POSIX).
|
|
843
|
-
|
|
930
|
+
/**
|
|
931
|
+
* A cheap identity for the on-disk database: size plus mtime in ms.
|
|
932
|
+
*
|
|
933
|
+
* Not a digest — this runs before every save, and hashing a 10MB+ file on each write would be a real
|
|
934
|
+
* cost for a check whose only job is "did somebody else touch this?". size+mtimeMs changes on any
|
|
935
|
+
* rewrite by another process, which is exactly the event being detected.
|
|
936
|
+
*/
|
|
937
|
+
function dbFingerprint(dbPath) {
|
|
938
|
+
try {
|
|
939
|
+
const st = fs.statSync(dbPath);
|
|
940
|
+
return `${st.size}:${st.mtimeMs}`;
|
|
941
|
+
} catch {
|
|
942
|
+
return null; // absent is a legitimate state, not an error
|
|
943
|
+
}
|
|
944
|
+
}
|
|
945
|
+
|
|
946
|
+
/**
|
|
947
|
+
* Refuse to silently overwrite another process's writes.
|
|
948
|
+
*
|
|
949
|
+
* sql.js holds the whole database in memory and persists by exporting the ENTIRE file and renaming it
|
|
950
|
+
* over the path. Two processes each hold their own copy and each rewrite the whole file, so the last
|
|
951
|
+
* writer wins and the other's rows are gone. Measured: two concurrent writers, 15 acknowledged
|
|
952
|
+
* recordTrace() calls each, 30 expected — 15 on disk, ALL from writer A. Writer B's 15 acknowledged
|
|
953
|
+
* writes vanished, with no error on either side.
|
|
954
|
+
*
|
|
955
|
+
* A caveat on the corroborating evidence, because it was overstated. `.mindforge/` does carry
|
|
956
|
+
* orphaned `celestial.db.tmp.<pid>` files, and one IS a valid database with skills rows absent from
|
|
957
|
+
* the live file — but those rows are not lost user data. Audited all 171 non-empty orphans against
|
|
958
|
+
* the live database: 161 are strict subsets, 9 exceed it only on `traces_search_segdir` (an FTS5
|
|
959
|
+
* segment-directory count, an index-merge artifact rather than rows), and exactly one
|
|
960
|
+
* (`celestial.db.tmp.4027`, 63.5 MB, pre-dating the `.async`/`.sync` suffixes) holds 1,373 skill
|
|
961
|
+
* names the live file lacks — every one of them a `Synthesized Skill (mf-*) - ev_<hash>` row, the
|
|
962
|
+
* generated filler already slated for deletion. So the orphans corroborate that the clobber window
|
|
963
|
+
* was ENTERED; they are not evidence that anything worth keeping was destroyed.
|
|
964
|
+
*
|
|
965
|
+
* The cross-process loss itself is proven by the two-writer measurement above, which does not
|
|
966
|
+
* depend on this. Recorded because "there is a database on disk holding rows the live file lacks"
|
|
967
|
+
* reads as recoverable data loss, and here it is not.
|
|
968
|
+
*
|
|
969
|
+
* Making sql.js genuinely multi-process safe means replacing the driver — the whole-file export IS the
|
|
970
|
+
* problem, and locking alone does not fix it, because both processes loaded a stale copy before either
|
|
971
|
+
* saved. What is fixable now is the SILENCE. On a detected clobber the export goes to a
|
|
972
|
+
* `.conflict.<pid>` sidecar and the caller is told, so the data still exists and somebody knows.
|
|
973
|
+
* Refusing loudly beats destroying quietly — the same choice as auto-runner refusing to record a
|
|
974
|
+
* completion it cannot substantiate.
|
|
975
|
+
*
|
|
976
|
+
* WHY CHECK AND RENAME ARE ONE CRITICAL SECTION. A first version checked the fingerprint and then
|
|
977
|
+
* handed the buffer to an ASYNC writer. Measured with a deterministic stage — parent writes 3, spawns a
|
|
978
|
+
* child that writes 5 and exits, parent then closes:
|
|
979
|
+
*
|
|
980
|
+
* [parent] GUARD ...898 == ...898 <- check passes
|
|
981
|
+
* === child writes and saves === <- file is now ...249
|
|
982
|
+
* [parent] ASYNC-WROTE new=...309 <- writes its PRE-CHILD snapshot anyway
|
|
983
|
+
* live rows: P0,P1,P2,seed <- the child's 5 rows destroyed, silently
|
|
984
|
+
*
|
|
985
|
+
* spawnSync blocked the event loop, so the child fit entirely inside the gap between the check and the
|
|
986
|
+
* write. A check followed by a later write is not a guard — it is the same clobber with extra steps.
|
|
987
|
+
* So the fingerprint comparison and the atomic rename now happen together, synchronously, under the
|
|
988
|
+
* shared fail-closed lock. withFileLock rejects a thenable outright, which makes reintroducing that gap
|
|
989
|
+
* a loud TypeError rather than silent data loss.
|
|
990
|
+
*
|
|
991
|
+
* The expensive part (writing and fsyncing the tmp file) stays OUTSIDE the lock and may still be async:
|
|
992
|
+
* the tmp path is pid-scoped, so no other process can observe or touch it.
|
|
993
|
+
*
|
|
994
|
+
* @param {string} tmpPath a fully-written, fsynced tmp file to rename into place
|
|
995
|
+
* @returns {{ok: boolean, fingerprint?: string, sidecar?: string}}
|
|
996
|
+
*/
|
|
997
|
+
function commitDb(dbPath, expected, buffer, tmpPath) {
|
|
998
|
+
return withFileLock(dbPath, () => {
|
|
999
|
+
const actual = dbFingerprint(dbPath);
|
|
1000
|
+
if (expected === null || actual === null || actual === expected) {
|
|
1001
|
+
fs.renameSync(tmpPath, dbPath);
|
|
1002
|
+
return { ok: true, fingerprint: dbFingerprint(dbPath) };
|
|
1003
|
+
}
|
|
1004
|
+
|
|
1005
|
+
// Conflict. Keep this process's bytes and leave the other process's file untouched.
|
|
1006
|
+
const sidecar = `${dbPath}.conflict.${process.pid}.${buffer.length}`;
|
|
1007
|
+
try {
|
|
1008
|
+
fs.renameSync(tmpPath, sidecar); // already fsynced; a rename beats a second full write
|
|
1009
|
+
} catch (err) {
|
|
1010
|
+
console.error(`[VectorHub] CONFLICT and the sidecar could not be written: ${err.message}`);
|
|
1011
|
+
try { fs.unlinkSync(tmpPath); } catch { /* nothing further to do */ }
|
|
1012
|
+
return { ok: false, sidecar: null };
|
|
1013
|
+
}
|
|
1014
|
+
console.error(
|
|
1015
|
+
`[VectorHub] REFUSING TO OVERWRITE ${dbPath}: another process changed it since this one loaded it `
|
|
1016
|
+
+ `(expected ${expected}, found ${actual}). sql.js rewrites the whole file, so continuing would `
|
|
1017
|
+
+ `discard that process's rows. This process's data went to ${sidecar} instead — nothing is lost, `
|
|
1018
|
+
+ 'but the two databases must be reconciled by hand.');
|
|
1019
|
+
return { ok: false, sidecar };
|
|
1020
|
+
}, { label: 'vector-hub-db' });
|
|
1021
|
+
}
|
|
1022
|
+
|
|
1023
|
+
/**
|
|
1024
|
+
* Write and fsync the pid-scoped tmp file, resolving to its path. Does NOT rename it into place —
|
|
1025
|
+
* that is commitDb's job, because the rename has to share a critical section with the staleness check.
|
|
1026
|
+
* Staying async here is the point: fsync of a multi-megabyte export is the expensive part, and the tmp
|
|
1027
|
+
* path cannot be observed by another process.
|
|
1028
|
+
*/
|
|
1029
|
+
function writeTmpDurable(dbPath, buffer) {
|
|
844
1030
|
return new Promise((resolve, reject) => {
|
|
845
|
-
const tmpPath = `${dbPath}.tmp.${process.pid}`;
|
|
1031
|
+
const tmpPath = `${dbPath}.tmp.${process.pid}.async`;
|
|
846
1032
|
const fail = (err) => { fs.unlink(tmpPath, () => reject(err)); };
|
|
847
1033
|
fs.open(tmpPath, 'w', (openErr, fd) => {
|
|
848
1034
|
if (openErr) return reject(openErr);
|
|
@@ -852,10 +1038,7 @@ function writeDbDurable(dbPath, buffer) {
|
|
|
852
1038
|
fs.close(fd, (closeErr) => {
|
|
853
1039
|
if (syncErr) return fail(syncErr);
|
|
854
1040
|
if (closeErr) return fail(closeErr);
|
|
855
|
-
|
|
856
|
-
if (renameErr) return fail(renameErr);
|
|
857
|
-
resolve();
|
|
858
|
-
});
|
|
1041
|
+
resolve(tmpPath);
|
|
859
1042
|
});
|
|
860
1043
|
});
|
|
861
1044
|
});
|
|
@@ -3,18 +3,19 @@
|
|
|
3
3
|
*
|
|
4
4
|
* Changes:
|
|
5
5
|
* 1. HANDOFF.json: add `plugin_api_version` field
|
|
6
|
-
* 2. AUDIT.jsonl:
|
|
6
|
+
* 2. AUDIT.jsonl: append a migration record — existing entries are NEVER rewritten (see step 2)
|
|
7
7
|
* 3. MINDFORGE.md: convert VERIFY_PASS_RATE_WARNING_THRESHOLD if in old 0-100 format
|
|
8
8
|
* 4. STATE.md: add v1.0.0 compatibility note if it doesn't already have one
|
|
9
9
|
*/
|
|
10
10
|
'use strict';
|
|
11
11
|
|
|
12
12
|
const fs = require('fs');
|
|
13
|
+
const { appendAuditEntrySync } = require('../autonomous/audit-writer');
|
|
13
14
|
|
|
14
15
|
module.exports = {
|
|
15
16
|
fromVersion: '0.6.0',
|
|
16
17
|
toVersion: '1.0.0',
|
|
17
|
-
description: 'Add plugin_api_version;
|
|
18
|
+
description: 'Add plugin_api_version; record the migration in the audit log; normalise MINDFORGE.md thresholds',
|
|
18
19
|
|
|
19
20
|
async run(paths) {
|
|
20
21
|
// ── 1. HANDOFF.json ───────────────────────────────────────────────────────
|
|
@@ -36,31 +37,35 @@ module.exports = {
|
|
|
36
37
|
}
|
|
37
38
|
|
|
38
39
|
// ── 2. AUDIT.jsonl ────────────────────────────────────────────────────────
|
|
40
|
+
//
|
|
41
|
+
// APPEND-ONLY. This step used to rewrite every entry to backfill `session_id`, which BROKE the
|
|
42
|
+
// hash chain and then reported success. Measured on a 50-entry chain written by the real writer:
|
|
43
|
+
//
|
|
44
|
+
// before -> audit chain valid: 50 entries exit 0
|
|
45
|
+
// after -> audit chain BROKEN at entry 0: hash mismatch (entry mutated) exit 1
|
|
46
|
+
//
|
|
47
|
+
// 50 of 50 entries mutated, integrity destroyed at the very first entry, and the migration printed
|
|
48
|
+
// "backfilled session_id in 50 of 50 entries" and carried on to report "All migrations complete".
|
|
49
|
+
// bin/governance/audit-hash.js hashes {...entry, previous_hash} with JSON.stringify, so ANY added
|
|
50
|
+
// key changes the material — a back-linked log cannot be edited in place, only appended to. No file
|
|
51
|
+
// in this directory referenced the canonical hasher.
|
|
52
|
+
//
|
|
53
|
+
// AND THE BACKFILL BOUGHT NOTHING. The only consumer of `session_id` on an audit entry is
|
|
54
|
+
// bin/dashboard/metrics-aggregator.js:253,286, which reads
|
|
55
|
+
// `entry.authored_by || entry.session_id || 'unknown'`. So the rewrite swapped the placeholder
|
|
56
|
+
// 'unknown' for the placeholder 'migrated-from-pre-1.0' — no consumer distinguishes them — at the
|
|
57
|
+
// cost of every integrity guarantee in the file. Deleting it loses nothing.
|
|
58
|
+
//
|
|
59
|
+
// What IS worth recording is that a migration touched this project, so one entry is APPENDED
|
|
60
|
+
// through the canonical writer. An append extends the chain instead of invalidating it.
|
|
39
61
|
if (fs.existsSync(paths.audit)) {
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
try {
|
|
46
|
-
const entry = JSON.parse(line);
|
|
47
|
-
if (!entry.session_id) {
|
|
48
|
-
entry.session_id = 'migrated-from-pre-1.0';
|
|
49
|
-
modified++;
|
|
50
|
-
return JSON.stringify(entry);
|
|
51
|
-
}
|
|
52
|
-
return line;
|
|
53
|
-
} catch {
|
|
54
|
-
return line; // Preserve unparseable lines exactly as-is (quarantine pattern)
|
|
55
|
-
}
|
|
62
|
+
appendAuditEntrySync(paths.audit, {
|
|
63
|
+
event: 'schema_migrated',
|
|
64
|
+
target_id: 'AUDIT.jsonl',
|
|
65
|
+
description: 'schema 0.6.0 -> 1.0.0; existing entries left byte-identical (append-only log)',
|
|
66
|
+
agent: 'migrate',
|
|
56
67
|
});
|
|
57
|
-
|
|
58
|
-
if (modified > 0) {
|
|
59
|
-
fs.writeFileSync(paths.audit, updated.join('\n') + '\n');
|
|
60
|
-
console.log(` • AUDIT.jsonl: backfilled session_id in ${modified} of ${lines.length} entries`);
|
|
61
|
-
} else {
|
|
62
|
-
console.log(' • AUDIT.jsonl: all entries already have session_id');
|
|
63
|
-
}
|
|
68
|
+
console.log(' • AUDIT.jsonl: recorded the migration as a new entry; existing entries untouched');
|
|
64
69
|
}
|
|
65
70
|
|
|
66
71
|
// ── 3. MINDFORGE.md ───────────────────────────────────────────────────────
|
|
@@ -11,11 +11,12 @@
|
|
|
11
11
|
|
|
12
12
|
const fs = require('fs');
|
|
13
13
|
const path = require('path');
|
|
14
|
+
const { appendAuditEntrySync } = require('../autonomous/audit-writer');
|
|
14
15
|
|
|
15
16
|
module.exports = {
|
|
16
17
|
fromVersion: '1.0.0',
|
|
17
18
|
toVersion: '2.0.0',
|
|
18
|
-
description: 'Additive schema upgrade:
|
|
19
|
+
description: 'Additive schema upgrade: record the migration in the audit log; model_group in tokens',
|
|
19
20
|
|
|
20
21
|
async run(paths) {
|
|
21
22
|
const backupDir = path.join(path.dirname(paths.handoff), '.backups', `v1-to-v2-${Date.now()}`);
|
|
@@ -65,29 +66,27 @@ module.exports = {
|
|
|
65
66
|
});
|
|
66
67
|
|
|
67
68
|
// ── 2. AUDIT.jsonl ────────────────────────────────────────────────────────
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
}
|
|
69
|
+
//
|
|
70
|
+
// APPEND-ONLY, for the same reason as 0.6.0-to-1.0.0.js step 2: this rewrote every entry to add
|
|
71
|
+
// `runtime` and `agent_id`, and bin/governance/audit-hash.js hashes {...entry, previous_hash}, so
|
|
72
|
+
// any added key changes the hash material and the chain breaks at the first entry while the
|
|
73
|
+
// migration reports success.
|
|
74
|
+
//
|
|
75
|
+
// Worse than its sibling, because here the backfill had NO consumer at all: `git grep agent_id`
|
|
76
|
+
// and `git grep model_group` outside bin/migrations/ return zero readers in bin/. The chain was
|
|
77
|
+
// being destroyed to populate fields nothing reads.
|
|
78
|
+
//
|
|
79
|
+
// Note safeMigrate() cannot express this — it takes raw content and returns replacement content,
|
|
80
|
+
// which is a rewrite by construction. An append goes through the canonical writer instead.
|
|
81
|
+
if (fs.existsSync(paths.audit)) {
|
|
82
|
+
appendAuditEntrySync(paths.audit, {
|
|
83
|
+
event: 'schema_migrated',
|
|
84
|
+
target_id: 'AUDIT.jsonl',
|
|
85
|
+
description: 'schema 1.0.0 -> 2.0.0; existing entries left byte-identical (append-only log)',
|
|
86
|
+
agent: 'migrate',
|
|
87
87
|
});
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
});
|
|
88
|
+
console.log(' • AUDIT.jsonl: recorded the migration as a new entry; existing entries untouched');
|
|
89
|
+
}
|
|
91
90
|
|
|
92
91
|
// ── 3. token-usage.jsonl ──────────────────────────────────────────────────
|
|
93
92
|
const tokensFile = path.join(path.dirname(paths.handoff), 'token-usage.jsonl');
|
package/bin/mindforge-cli.js
CHANGED
|
@@ -167,6 +167,17 @@ const COMMANDS = {
|
|
|
167
167
|
};
|
|
168
168
|
|
|
169
169
|
// ── Workflow subcommand (non-script, handled inline) ─────────────────────────
|
|
170
|
+
//
|
|
171
|
+
// Routable but NOT a COMMANDS key, because it dispatches to a module rather than spawning a script.
|
|
172
|
+
// That made the CLI lie about itself: `workflow` is the most-documented verb in the project (126
|
|
173
|
+
// references across the docs) and it works, yet it appeared in neither `--help` nor the
|
|
174
|
+
// "Available commands" list, so a user who mistyped it was told it does not exist. Declared here, next
|
|
175
|
+
// to the handler that makes it real, and consumed by both self-report sites below — one source, so the
|
|
176
|
+
// two cannot drift.
|
|
177
|
+
const INLINE_COMMANDS = ['workflow'];
|
|
178
|
+
/** Every verb the router will actually dispatch: table-driven plus inline. */
|
|
179
|
+
const ROUTABLE = [...Object.keys(COMMANDS), ...INLINE_COMMANDS].sort();
|
|
180
|
+
|
|
170
181
|
if (COMMAND === 'workflow') {
|
|
171
182
|
const workflowRunner = require('./workflows/workflow-runner');
|
|
172
183
|
workflowRunner.run(COMMAND_ARGS[0], COMMAND_ARGS.slice(1));
|
|
@@ -174,8 +185,18 @@ if (COMMAND === 'workflow') {
|
|
|
174
185
|
}
|
|
175
186
|
|
|
176
187
|
if (ARGS.includes('--version') || ARGS.includes('-V')) {
|
|
177
|
-
|
|
178
|
-
|
|
188
|
+
// Resolve by package NAME. In an install this file lands at <project>/bin/mindforge-cli.js, so
|
|
189
|
+
// '../package.json' is the CONSUMER's manifest and `--version` confidently printed THEIR app's
|
|
190
|
+
// version as MindForge's — measured: 1.0.0 for a host app at 1.0.0, while MindForge was 11.9.2.
|
|
191
|
+
// Exit non-zero on an unresolvable version rather than guessing: the whole point of this command
|
|
192
|
+
// is to be trusted, and a plausible wrong answer is worse than an honest failure.
|
|
193
|
+
try {
|
|
194
|
+
console.log(require('./utils/mindforge-version').resolveMindforgeVersion().version);
|
|
195
|
+
process.exit(0);
|
|
196
|
+
} catch (err) {
|
|
197
|
+
console.error(err.message);
|
|
198
|
+
process.exit(1);
|
|
199
|
+
}
|
|
179
200
|
}
|
|
180
201
|
|
|
181
202
|
if (!COMMAND || ARGS.includes('--help') || ARGS.includes('-h')) {
|
|
@@ -187,8 +208,10 @@ const target = COMMANDS[COMMAND];
|
|
|
187
208
|
if (!target) {
|
|
188
209
|
console.error(`Unknown command: ${COMMAND}`);
|
|
189
210
|
|
|
190
|
-
// Suggest similar commands using Levenshtein distance
|
|
191
|
-
|
|
211
|
+
// Suggest similar commands using Levenshtein distance. ROUTABLE, not Object.keys(COMMANDS): a
|
|
212
|
+
// near-miss on `workflow` (`worklow`, `wokflow`) previously produced no suggestion at all, because
|
|
213
|
+
// the only verb it resembles was absent from the pool.
|
|
214
|
+
const suggestions = ROUTABLE
|
|
192
215
|
.map(cmd => ({ cmd, dist: levenshtein(COMMAND, cmd) }))
|
|
193
216
|
.filter(s => s.dist <= 3)
|
|
194
217
|
.sort((a, b) => a.dist - b.dist)
|
|
@@ -197,7 +220,7 @@ if (!target) {
|
|
|
197
220
|
if (suggestions.length > 0) {
|
|
198
221
|
console.error(`\nDid you mean: ${suggestions.map(s => s.cmd).join(', ')}?`);
|
|
199
222
|
} else {
|
|
200
|
-
console.error('Available commands: ' +
|
|
223
|
+
console.error('Available commands: ' + ROUTABLE.join(', '));
|
|
201
224
|
}
|
|
202
225
|
process.exit(1);
|
|
203
226
|
}
|
|
@@ -213,8 +236,42 @@ const finalArgs = [...(target.defaultArgs || []), ...COMMAND_ARGS];
|
|
|
213
236
|
|
|
214
237
|
console.log(`🚀 Executing: ${COMMAND} (${target.description})`);
|
|
215
238
|
|
|
239
|
+
// The child runs in the USER'S project, not in MindForge's install directory.
|
|
240
|
+
//
|
|
241
|
+
// This was `cwd: ROOT`, which made every routed command operate on the framework's own tree
|
|
242
|
+
// instead of the caller's. The worst case was `security-scan`, the command the protocol mandates
|
|
243
|
+
// pre-commit for any Auth/Payment/PII change: measured against a fixture project whose
|
|
244
|
+
// MINDFORGE.md declared 4 settings including `[MIN_SOUL_SCORE] = 99` (schema maximum is 10) and
|
|
245
|
+
// `[COST_HARD_LIMIT_USD] = not-a-number`, it printed
|
|
246
|
+
//
|
|
247
|
+
// ✅ MINDFORGE.md valid — 43 settings configured
|
|
248
|
+
//
|
|
249
|
+
// and exited 0. 43 is MindForge's OWN setting count. Two different fixture configs produced
|
|
250
|
+
// byte-identical output, which is the proof it read neither: a security gate that cannot fail,
|
|
251
|
+
// because it never sees the input it claims to check.
|
|
252
|
+
//
|
|
253
|
+
// MEASURED BLAST RADIUS before changing it — all 27 routed commands run under both values against
|
|
254
|
+
// the same fixture. 7 differ, every one of them moving from the vendor's tree to the caller's, and
|
|
255
|
+
// NONE regressed from success to failure:
|
|
256
|
+
//
|
|
257
|
+
// security-scan validated MindForge's config -> reads the caller's
|
|
258
|
+
// classify TIER=2 from MindForge's git diff -> diffs the caller's repo
|
|
259
|
+
// pr-review, cross-review loaded MindForge's ConfigManager -> looks in the caller's project
|
|
260
|
+
// learning, record-learning reported MindForge's state -> reports the caller's
|
|
261
|
+
// test-memory same ConfigManager shift
|
|
262
|
+
//
|
|
263
|
+
// `classify` deserves its own note: for every consumer it was classifying MindForge's changes.
|
|
264
|
+
//
|
|
265
|
+
// CI is unaffected, checked rather than assumed. control-plane.yml:80 runs `security-scan` as the
|
|
266
|
+
// required ⚖️ Governance Enforcement check and mindforge-ci.yml:46 runs validate-config.js
|
|
267
|
+
// directly; both execute with the repo root as cwd, so ROOT and process.cwd() are the same path
|
|
268
|
+
// there and behaviour is byte-identical.
|
|
269
|
+
//
|
|
270
|
+
// A script that needs the FRAMEWORK's own assets must resolve them from __dirname, which is
|
|
271
|
+
// independent of cwd — see the SCHEMA_PATH note in bin/validate-config.js. Anchoring vendor assets
|
|
272
|
+
// to the process's working directory is what coupled these two unrelated things in the first place.
|
|
216
273
|
const result = spawnSync('node', [scriptPath, ...finalArgs], {
|
|
217
|
-
cwd:
|
|
274
|
+
cwd: process.cwd(),
|
|
218
275
|
stdio: 'inherit',
|
|
219
276
|
env: { ...process.env, MINDFORGE_CLI: 'true' }
|
|
220
277
|
});
|
|
@@ -252,8 +309,12 @@ function printUsage() {
|
|
|
252
309
|
for (const [name, cfg] of Object.entries(COMMANDS)) {
|
|
253
310
|
console.log(` ${name.padEnd(15)} ${cfg.description}`);
|
|
254
311
|
}
|
|
312
|
+
// Listed separately because it dispatches to a module rather than spawning a script, so it has no
|
|
313
|
+
// COMMANDS entry to carry a description. Omitting it made --help contradict the router.
|
|
314
|
+
console.log(` ${'workflow'.padEnd(15)} Run a registered dynamic workflow (see \`workflow list\`)`);
|
|
255
315
|
console.log('\nExamples:');
|
|
256
316
|
console.log(' node bin/mindforge-cli.js security-scan');
|
|
257
317
|
console.log(' node bin/mindforge-cli.js headless --phase 1');
|
|
318
|
+
console.log(' node bin/mindforge-cli.js workflow list');
|
|
258
319
|
console.log('\n');
|
|
259
320
|
}
|