mindforge-cc 10.7.0 → 11.2.0

package/bin/memory/knowledge-store.js

@@ -15,6 +15,30 @@ const path = require('path');
 const os   = require('os');
 const crypto = require('crypto');
 
+// ── Durable append (UC-09) ──────────────────────────────────────────────────
+// The knowledge-store public API (add/deprecate/reinforce) is SYNCHRONOUS and
+// callers read-after-write synchronously (e.g. `const id = Store.add(...)`
+// immediately followed by `Store.readAll()`). Routing through the async
+// append-queue would make those reads observe stale data and would require an
+// API change across 9+ consumers — out of scope for UC-09.
+//
+// Instead we centralize every append through one durable, fsync'd, synchronous
+// writer. This delivers UC-09's durability guarantee (acknowledged writes are on
+// disk before the call returns) and a single serialized append path per file,
+// while preserving the synchronous read-after-write contract. appendFileSync's
+// per-call append is atomic on POSIX, so concurrent in-process appends do not
+// interleave at the byte level.
+function appendDurableSync(filePath, line) {
+  const record = line.endsWith('\n') ? line : line + '\n';
+  const fd = fs.openSync(filePath, 'a');
+  try {
+    fs.writeSync(fd, record);
+    fs.fsyncSync(fd);
+  } finally {
+    fs.closeSync(fd);
+  }
+}
+
 // ── ID Index for fast lookups (built lazily, invalidated on writes) ───────────
 let _idIndex = null;       // Map<id, entry> — latest version per ID
 let _indexDirty = true;    // Invalidated whenever entries are appended
@@ -126,6 +150,22 @@ function getFilePath(type) {
   }
 }
 
+// ── File Integrity ────────────────────────────────────────────────────────────
+
+/**
+ * Ensures a JSONL file doesn't end with a partial/truncated line.
+ * Appends a trailing newline if missing — prevents corruption from propagating.
+ */
+function verifyFileIntegrity(filePath) {
+  if (!fs.existsSync(filePath)) return true;
+  const content = fs.readFileSync(filePath, 'utf8');
+  if (content.length === 0) return true;
+  if (!content.endsWith('\n')) {
+    fs.appendFileSync(filePath, '\n');
+  }
+  return true;
+}
+
 // ── Write operations ──────────────────────────────────────────────────────────
 
 /**
@@ -184,11 +224,13 @@ function add(entry) {
   };
 
   const filePath = getFilePath(entry.type);
-  fs.appendFileSync(filePath, JSON.stringify(full) + '\n');
+  verifyFileIntegrity(filePath);
+  appendDurableSync(filePath, JSON.stringify(full));
 
   // Also append to unified knowledge-base.jsonl for cross-type queries
   if (filePath !== paths.KB_PATH) {
-    fs.appendFileSync(paths.KB_PATH, JSON.stringify(full) + '\n');
+    verifyFileIntegrity(paths.KB_PATH);
+    appendDurableSync(paths.KB_PATH, JSON.stringify(full));
   }
 
   _invalidateIndex();
@@ -217,9 +259,11 @@ function deprecate(id, reason, supersededBy = null) {
     deprecated_at:     new Date().toISOString(),
   };
 
-  fs.appendFileSync(filePath, JSON.stringify(deprecated) + '\n');
+  verifyFileIntegrity(filePath);
+  appendDurableSync(filePath, JSON.stringify(deprecated));
   if (filePath !== paths.KB_PATH) {
-    fs.appendFileSync(paths.KB_PATH, JSON.stringify(deprecated) + '\n');
+    verifyFileIntegrity(paths.KB_PATH);
+    appendDurableSync(paths.KB_PATH, JSON.stringify(deprecated));
   }
 
   _invalidateIndex();
@@ -246,9 +290,11 @@ function reinforce(id) {
   };
 
   const filePath = getFilePath(entry.type);
-  fs.appendFileSync(filePath, JSON.stringify(reinforced) + '\n');
+  verifyFileIntegrity(filePath);
+  appendDurableSync(filePath, JSON.stringify(reinforced));
   if (filePath !== paths.KB_PATH) {
-    fs.appendFileSync(paths.KB_PATH, JSON.stringify(reinforced) + '\n');
+    verifyFileIntegrity(paths.KB_PATH);
+    appendDurableSync(paths.KB_PATH, JSON.stringify(reinforced));
   }
 
   _invalidateIndex();

package/bin/memory/retrieval-fusion.js

@@ -0,0 +1,58 @@
+'use strict';
+/**
+ * MindForge — Reciprocal Rank Fusion (UC-20).
+ * Merges multiple ranked lists using scale-free RRF scoring.
+ *
+ * RRF eliminates the need for score normalization across retrieval paths
+ * with incomparable scoring functions (embedding similarity, BM25, graph
+ * traversal, FTS rank). Only ordinal rank matters, not score magnitude.
+ *
+ * Formula:
+ *   rrfScore(item) = SUM( 1 / (K + rank_i) ) for all lists containing the item
+ *
+ * Where K=60 is the standard constant from the original RRF paper
+ * (Cormack, Clarke, Butt — 2009).
+ */
+
+const K = 60; // Standard RRF constant — dampens the influence of high ranks
+
+/**
+ * Fuse multiple ranked result lists using Reciprocal Rank Fusion.
+ *
+ * Each list is an array of objects with at least an `id` field.
+ * Items appearing in multiple lists accumulate RRF score and rank higher.
+ *
+ * @param {Array<Array<{id: string, [key: string]: any}>>} rankedLists
+ *   Array of ranked lists. Each list is ordered by relevance (index 0 = most relevant).
+ * @returns {Array<{id: string, rrfScore: number, [key: string]: any}>}
+ *   Fused results sorted by RRF score descending. Each item retains its
+ *   original properties from the first list it appeared in.
+ */
+function fuseResults(rankedLists) {
+  if (!rankedLists || rankedLists.length === 0) return [];
+
+  const scores = new Map(); // id -> merged item with rrfScore
+
+  for (const list of rankedLists) {
+    if (!Array.isArray(list)) continue;
+
+    for (let rank = 0; rank < list.length; rank++) {
+      const item = list[rank];
+      if (!item || !item.id) continue;
+
+      const id = item.id;
+      const rrfContribution = 1 / (K + rank + 1); // rank is 0-based, +1 makes it 1-based
+
+      if (scores.has(id)) {
+        const existing = scores.get(id);
+        existing.rrfScore += rrfContribution;
+      } else {
+        scores.set(id, { ...item, rrfScore: rrfContribution });
+      }
+    }
+  }
+
+  return [...scores.values()].sort((a, b) => b.rrfScore - a.rrfScore);
+}
+
+module.exports = { fuseResults, K };

package/bin/memory/semantic-hub.js

@@ -83,7 +83,7 @@ class SemanticHub {
     try {
       const data = await fs.readFile(this.syncManifest, 'utf8');
       manifest = JSON.parse(data);
-    } catch (e) {}
+    } catch (e) { /* intentionally empty */ }
 
     manifest[libraryName] = {
       lastSync: new Date().toISOString(),
@@ -106,7 +106,7 @@ class SemanticHub {
         sqliteTraces = await vectorHub.searchTraces(skillFilter);
       } else {
         sqliteTraces = vectorHub.query(
-          "SELECT * FROM traces WHERE event = ? LIMIT 20",
+          'SELECT * FROM traces WHERE event = ? LIMIT 20',
           ['reasoning_trace']
         );
       }

package/bin/memory/vector-hub.js

@@ -23,6 +23,30 @@ class VectorHub {
     this.initialized = false;
     this._writeCount = 0;
     this._batchSize = 10;
+    // UC-09: serialized async persistence chain. Successive save() calls queue
+    // behind one another so two exports never write the .db file concurrently
+    // (a corrupted half-written database would otherwise be possible).
+    this._saveChain = Promise.resolve();
+    // Count of async save()s that have been SCHEDULED but not yet COMPLETED their
+    // durable disk write. A boolean here is unsafe: with two rapid saves the chain
+    // is [writeA → clear → writeB → clear], leaving a window where the flag reads
+    // "clean" while writeB is still pending — a hard process.exit() in that window
+    // would make the exit guard skip saveSync() and lose the last batch (the exact
+    // data loss this guard exists to prevent). A counter has no such gap: it only
+    // returns to 0 once EVERY scheduled save has completed. saveSync() always
+    // exports the current in-memory DB, so over-flushing on exit is harmless — we
+    // deliberately bias toward flushing.
+    this._pendingSaves = 0;
+    this._exitGuardInstalled = false;
+  }
+
+  _installExitGuard() {
+    if (this._exitGuardInstalled) return;
+    this._exitGuardInstalled = true;
+    // 'exit' handlers can only run synchronous code — saveSync() fits exactly.
+    process.once('exit', () => {
+      if (this._db && this._pendingSaves > 0) this.saveSync();
+    });
   }
 
   _ensureDir() {
@@ -167,22 +191,74 @@ class VectorHub {
     this._db.run('CREATE UNIQUE INDEX IF NOT EXISTS idx_migrations_name ON _migrations(name)');
 
     this.initialized = true;
+    this._installExitGuard();
     this.save();
     console.log(`[VectorHub] Initialized WASM SQLite persistence at ${this.dbPath}`);
   }
 
   /**
-   * Persist the in-memory database to disk.
+   * Persist the in-memory database to disk (UC-09).
+   *
+   * sql.js export() is intrinsically synchronous, but the (potentially large)
+   * FILE WRITE no longer blocks the event loop: we snapshot the bytes
+   * synchronously, then write+fsync them asynchronously. Successive saves are
+   * serialized on a single chain so two exports never write the .db file
+   * concurrently. The write is crash-safe (tmp file + atomic rename + fsync),
+   * so a partial write can never leave a corrupted database on disk.
+   *
+   * @returns {Promise<void>} Resolves once the snapshot is durably on disk.
    */
   save() {
+    if (!this._db) return Promise.resolve();
+
+    let buffer;
+    try {
+      this._ensureDir();
+      // Snapshot the DB synchronously so the bytes reflect this exact moment.
+      buffer = Buffer.from(this._db.export());
+    } catch (err) {
+      console.warn(`[VectorHub] Failed to export database: ${err.message}`);
+      return Promise.resolve();
+    }
+
+    const dbPath = this.dbPath;
+    // Increment when SCHEDULED; decrement only once this specific save has
+    // COMPLETED (success or failure). The exit guard fires saveSync() while any
+    // scheduled save is still outstanding — see _installExitGuard().
+    this._pendingSaves++;
+    this._saveChain = this._saveChain.then(() => writeDbDurable(dbPath, buffer))
+      .catch((err) => {
+        console.warn(`[VectorHub] Failed to save database: ${err.message}`);
+      })
+      .then(() => { this._pendingSaves--; });
+    return this._saveChain;
+  }
+
+  /**
+   * Synchronous, crash-safe persistence — used only on shutdown to GUARANTEE
+   * no acknowledged write is lost if the process exits before the async save
+   * chain drains. Correctness over non-blocking here.
+   */
+  saveSync() {
     if (!this._db) return;
     try {
       this._ensureDir();
-      const data = this._db.export();
-      const buffer = Buffer.from(data);
-      fs.writeFileSync(this.dbPath, buffer);
+      const buffer = Buffer.from(this._db.export());
+      const tmpPath = `${this.dbPath}.tmp.${process.pid}`;
+      const fd = fs.openSync(tmpPath, 'w');
+      try {
+        fs.writeSync(fd, buffer);
+        fs.fsyncSync(fd);
+      } finally {
+        fs.closeSync(fd);
+      }
+      fs.renameSync(tmpPath, this.dbPath);
+      // A sync export captures the full in-memory DB — a superset of anything the
+      // outstanding async saves would have written — so the pending work is now
+      // durably satisfied. Clearing the counter prevents a redundant second flush.
+      this._pendingSaves = 0;
     } catch (err) {
-      console.warn(`[VectorHub] Failed to save database: ${err.message}`);
+      console.warn(`[VectorHub] Failed to save database (sync): ${err.message}`);
     }
   }
 
@@ -199,10 +275,13 @@ class VectorHub {
 
   /**
    * Close the database and save final state to disk.
+   * Drains any pending async saves, then performs a guaranteed synchronous
+   * durable write so no acknowledged data is lost on shutdown (UC-09).
    */
   async close() {
     if (this._db) {
-      this.save();
+      try { await this._saveChain; } catch { /* logged in save() */ }
+      this.saveSync();
       this._db.close();
       this._db = null;
       this.initialized = false;
@@ -455,6 +534,32 @@ class VectorHub {
   }
 }
 
+// ── Durable async DB file write (UC-09) ───────────────────────────────────────
+// Crash-safe: write to a tmp file, fsync, then atomically rename over the target.
+// A crash mid-write leaves the previous good .db intact (rename is atomic on POSIX).
+function writeDbDurable(dbPath, buffer) {
+  return new Promise((resolve, reject) => {
+    const tmpPath = `${dbPath}.tmp.${process.pid}`;
+    const fail = (err) => { fs.unlink(tmpPath, () => reject(err)); };
+    fs.open(tmpPath, 'w', (openErr, fd) => {
+      if (openErr) return reject(openErr);
+      fs.write(fd, buffer, 0, buffer.length, 0, (writeErr) => {
+        if (writeErr) { fs.close(fd, () => fail(writeErr)); return; }
+        fs.fsync(fd, (syncErr) => {
+          fs.close(fd, (closeErr) => {
+            if (syncErr) return fail(syncErr);
+            if (closeErr) return fail(closeErr);
+            fs.rename(tmpPath, dbPath, (renameErr) => {
+              if (renameErr) return fail(renameErr);
+              resolve();
+            });
+          });
+        });
+      });
+    });
+  });
+}
+
 // ── Factory Function ──────────────────────────────────────────────────────────
 
 /**

package/bin/migrations/10.7.0-to-11.0.0.js

@@ -0,0 +1,110 @@
+'use strict';
+
+const fs = require('fs');
+const path = require('path');
+
+const MIGRATION_ID = '10.7.0-to-11.0.0';
+const TARGET_VERSION = '11.0.0';
+
+async function migrate(projectRoot) {
+  const results = { steps: [], success: true };
+
+  // Step 1: Backup config.json
+  const configPath = path.join(projectRoot, '.mindforge', 'config.json');
+  if (fs.existsSync(configPath)) {
+    const backupPath = configPath + '.v10-backup';
+    fs.copyFileSync(configPath, backupPath);
+    results.steps.push({ step: 'backup_config', status: 'done', path: backupPath });
+
+    // Step 2: Add new config sections
+    try {
+      const config = JSON.parse(fs.readFileSync(configPath, 'utf8'));
+
+      if (!config.temporal) {
+        config.temporal = { max_snapshots: 50, max_age_days: 7 };
+      }
+      if (!config.rate_limiting) {
+        config.rate_limiting = { dashboard_rpm: 100, model_rpm: {} };
+      }
+      if (!config.session) {
+        config.session = { token_expiry_hours: 24 };
+      }
+      if (!config.wave_execution) {
+        config.wave_execution = { max_concurrency: 3 };
+      }
+
+      config.version = TARGET_VERSION;
+
+      fs.writeFileSync(configPath, JSON.stringify(config, null, 2) + '\n');
+      results.steps.push({ step: 'update_config', status: 'done' });
+    } catch (e) {
+      results.steps.push({ step: 'update_config', status: 'warning', error: e.message });
+    }
+  }
+
+  // Step 3: Archive old AUDIT lines if > 5000
+  const auditPath = path.join(projectRoot, '.planning', 'AUDIT.jsonl');
+  if (fs.existsSync(auditPath)) {
+    try {
+      const content = fs.readFileSync(auditPath, 'utf8');
+      const lines = content.split('\n').filter(l => l.trim());
+      if (lines.length > 5000) {
+        const archiveDir = path.join(projectRoot, '.planning', 'audit-archive');
+        if (!fs.existsSync(archiveDir)) fs.mkdirSync(archiveDir, { recursive: true });
+
+        const zlib = require('zlib');
+        const timestamp = new Date().toISOString().replace(/[:.]/g, '-');
+        const archivePath = path.join(archiveDir, `AUDIT-pre-v11-${timestamp}.jsonl.gz`);
+        const toArchive = lines.slice(0, -500).join('\n') + '\n';
+        fs.writeFileSync(archivePath, zlib.gzipSync(toArchive));
+
+        const remaining = lines.slice(-500).join('\n') + '\n';
+        fs.writeFileSync(auditPath, remaining);
+        results.steps.push({ step: 'archive_audit', status: 'done', archived: lines.length - 500 });
+      } else {
+        results.steps.push({ step: 'archive_audit', status: 'skipped', reason: 'under_threshold' });
+      }
+    } catch (e) {
+      results.steps.push({ step: 'archive_audit', status: 'warning', error: e.message });
+    }
+  }
+
+  // Step 4: GC old snapshots
+  try {
+    const TemporalHub = require('../engine/temporal-hub');
+    const gcResult = await TemporalHub.gc({ maxSnapshots: 50, maxAgeDays: 30 });
+    results.steps.push({ step: 'snapshot_gc', status: 'done', deleted: gcResult.deleted });
+  } catch (e) {
+    results.steps.push({ step: 'snapshot_gc', status: 'warning', error: e.message });
+  }
+
+  // Step 5: Bump schema_version in HANDOFF.json
+  const handoffPath = path.join(projectRoot, '.planning', 'HANDOFF.json');
+  if (fs.existsSync(handoffPath)) {
+    try {
+      const handoff = JSON.parse(fs.readFileSync(handoffPath, 'utf8'));
+      handoff.schema_version = TARGET_VERSION;
+      fs.writeFileSync(handoffPath, JSON.stringify(handoff, null, 2) + '\n');
+      results.steps.push({ step: 'bump_handoff_version', status: 'done' });
+    } catch (e) {
+      results.steps.push({ step: 'bump_handoff_version', status: 'warning', error: e.message });
+    }
+  }
+
+  // Step 6: Update MINDFORGE.md VERSION
+  const mindforgeFile = path.join(projectRoot, 'MINDFORGE.md');
+  if (fs.existsSync(mindforgeFile)) {
+    try {
+      let content = fs.readFileSync(mindforgeFile, 'utf8');
+      content = content.replace(/VERSION\s*=\s*[\d.]+/, `VERSION = ${TARGET_VERSION}`);
+      fs.writeFileSync(mindforgeFile, content);
+      results.steps.push({ step: 'bump_mindforge_version', status: 'done' });
+    } catch (e) {
+      results.steps.push({ step: 'bump_mindforge_version', status: 'warning', error: e.message });
+    }
+  }
+
+  return results;
+}
+
+module.exports = { MIGRATION_ID, TARGET_VERSION, migrate };

package/bin/migrations/schema-versions.js

@@ -71,6 +71,19 @@ const SCHEMA_HISTORY = [
       'Plugin API version upgraded to 2.0.0',
     ],
   },
+  {
+    version: '11.0.0',
+    date: '2026-05-28',
+    description: 'v11.0.0 - Persona Expansion: temporal config, rate limiting, wave execution, audit archival',
+    handoff_fields_added: [],
+    handoff_fields_removed: [],
+    audit_fields_added: [],
+    breaking: [
+      'config.json gains temporal, rate_limiting, session, wave_execution sections',
+      'AUDIT.jsonl auto-archived if exceeding 5000 lines',
+      'MINDFORGE.md VERSION format drops suffix (was X.Y.Z-SUFFIX, now X.Y.Z)',
+    ],
+  },
 ];
 
 module.exports = { SCHEMA_HISTORY };

package/bin/mindforge-cli.js

@@ -115,11 +115,6 @@ const COMMANDS = {
     script: 'bin/autonomous/mesh-self-healer.js',
     description: 'Auto-detect and repair reasoning drifts in the active swarm'
   },
-  'quantum-verify': {
-    script: 'bin/governance/quantum-crypto.js',
-    description: 'Verify framework integrity using post-quantum signatures',
-    defaultArgs: ['--verify', '.mindforge/engine/']
-  },
   // Planned: jira-sync, confluence-sync (not yet implemented)
   'metrics': {
     script: 'bin/dashboard/metrics-aggregator.js',
@@ -138,6 +133,10 @@ const COMMANDS = {
     script: 'bin/engine/learning-manager.js',
     description: 'Append a new Learning Entry to the Evolution Log',
     defaultArgs: ['record']
+  },
+  'verify': {
+    script: 'bin/engine/verify-cli.js',
+    description: 'Run unified verification (tests, lint, audit, typecheck) and write report'
   }
 };
 

package/bin/models/anthropic-provider.js

@@ -15,7 +15,7 @@ class AnthropicProvider {
 
     const data = JSON.stringify({
       model,
-      system: systemPrompt,
+      system: [{ type: 'text', text: systemPrompt, cache_control: { type: 'ephemeral' } }],
       messages: [{ role: 'user', content: userMessage }],
       max_tokens: maxTokens,
       temperature,
@@ -45,15 +45,24 @@ class AnthropicProvider {
 
             const inputTokens = json.usage.input_tokens;
             const outputTokens = json.usage.output_tokens;
-            
-            // Basic cost calculation (Sonnet 3.5 prices)
-            const cost = (inputTokens * 0.000003) + (outputTokens * 0.000015);
+            const cacheRead = json.usage.cache_read_input_tokens || 0;
+            const cacheCreate = json.usage.cache_creation_input_tokens || 0;
+
+            const { priceCall } = require('./pricing-registry');
+            const cost = priceCall(json.model, {
+              input_tokens: inputTokens,
+              output_tokens: outputTokens,
+              cache_read_input_tokens: cacheRead,
+              cache_creation_input_tokens: cacheCreate,
+            });
 
             resolve({
               model: json.model,
               content: json.content[0].text,
               input_tokens: inputTokens,
               output_tokens: outputTokens,
+              cache_read_input_tokens: cacheRead,
+              cache_creation_input_tokens: cacheCreate,
               cost_usd: cost,
               provider: 'anthropic'
             });
@@ -72,6 +81,51 @@ class AnthropicProvider {
       req.end();
     });
   }
+
+  async streamComplete(messages, options = {}) {
+    const model = options.model || 'claude-sonnet-4-6';
+    const maxTokens = options.maxTokens || 4096;
+
+    const data = JSON.stringify({
+      model,
+      messages,
+      max_tokens: maxTokens,
+      stream: true,
+    });
+
+    return new Promise((resolve, reject) => {
+      const req = https.request({
+        hostname: 'api.anthropic.com',
+        path: '/v1/messages',
+        method: 'POST',
+        headers: {
+          'Content-Type': 'application/json',
+          'x-api-key': this.apiKey,
+          'anthropic-version': '2023-06-01',
+          'Content-Length': Buffer.byteLength(data),
+        },
+        timeout: 300_000,
+      }, res => {
+        if (res.statusCode !== 200) {
+          let body = '';
+          res.on('data', chunk => body += chunk);
+          res.on('end', () => {
+            reject(new Error(`Anthropic streaming failed: ${res.statusCode}`));
+          });
+          return;
+        }
+        resolve(res);
+      });
+
+      req.on('error', reject);
+      req.on('timeout', () => {
+        req.destroy();
+        reject(new Error('Anthropic stream timeout'));
+      });
+      req.write(data);
+      req.end();
+    });
+  }
 }
 
 module.exports = AnthropicProvider;