mindforge-cc 10.7.0 → 11.2.0

package/bin/models/cloud-broker.js

@@ -7,17 +7,37 @@
 const fs = require('fs');
 const path = require('path');
 
+// Per-provider latency ring buffer (last 10 calls)
+const latencyHistory = new Map();
+
+function recordLatency(provider, durationMs) {
+  if (!latencyHistory.has(provider)) {
+    latencyHistory.set(provider, []);
+  }
+  const history = latencyHistory.get(provider);
+  history.push(durationMs);
+  if (history.length > 10) history.shift();
+}
+
+function getP95Latency(provider) {
+  const history = latencyHistory.get(provider);
+  if (!history || history.length === 0) return 500;
+  const sorted = [...history].sort((a, b) => a - b);
+  const idx = Math.ceil(sorted.length * 0.95) - 1;
+  return sorted[Math.min(idx, sorted.length - 1)];
+}
+
 class CloudBroker {
   constructor(config = {}) {
     this.providers = config.providers || ['anthropic', 'google', 'aws', 'azure'];
     this.statsPath = config.statsPath || path.join(__dirname, 'performance-stats.json');
-    this.blacklist = new Map(); // provider -> expiry (Date)
-    this.failureWindow = new Map(); // provider:taskType -> count
+    this.blacklist = new Map();
+    this.failureWindow = new Map();
     this.state = {
-      'anthropic': { latency: 450, costMultiplier: 1.0, healthy: true },
-      'google': { latency: 600, costMultiplier: 0.85, healthy: true },
-      'aws': { latency: 550, costMultiplier: 0.95, healthy: true },
-      'azure': { latency: 650, costMultiplier: 1.1, healthy: true }
+      'anthropic': { costMultiplier: 1.0, healthy: true },
+      'google': { costMultiplier: 0.85, healthy: true },
+      'aws': { costMultiplier: 0.95, healthy: true },
+      'azure': { costMultiplier: 1.1, healthy: true }
     };
     this.reloadStats();
   }
@@ -71,21 +91,20 @@ class CloudBroker {
         return true;
       })
       .map(([id, data]) => {
-        // Calculate Success Probability for this task
         const stats = this.performanceStats[id]?.[taskType] || { success: 1, failure: 0 };
         const totalTasks = stats.success + stats.failure;
         const successProb = totalTasks > 0 ? (stats.success / totalTasks) : 0.5;
 
-        // Score Calculation (The "Affinity" Algorithm)
         const latencyWeight = 0.2;
         const costWeight = 0.3;
-        const affinityWeight = 0.5; 
+        const affinityWeight = 0.5;
 
-        const score = (data.latency * latencyWeight) + 
-                      (data.costMultiplier * 1000 * costWeight) + 
+        const providerLatency = getP95Latency(id);
+        const score = (providerLatency * latencyWeight) +
+                      (data.costMultiplier * 1000 * costWeight) +
                       ((1.0 - successProb) * 2000 * affinityWeight);
 
-        return { id, score, successProb: successProb.toFixed(2) };
+        return { id, score, successProb: successProb.toFixed(2), p95: providerLatency };
       });
 
     scored.sort((a, b) => a.score - b.score);
@@ -110,7 +129,7 @@ class CloudBroker {
 
     const fallback = Object.entries(this.state)
       .filter(([id, data]) => id !== failedProvider && data.healthy)
-      .sort((a, b) => a[1].latency - b[1].latency)[0];
+      .sort((a, b) => getP95Latency(a[0]) - getP95Latency(b[0]))[0];
 
     return fallback ? fallback[0] : 'google'; // Default fallback
   }
@@ -130,33 +149,62 @@ class CloudBroker {
     return mappings[provider]?.[modelGroup] || mappings[provider]?.['sonnet'];
   }
 
+  /**
+   * Removes failure entries whose sliding window (5 min) has expired.
+   */
+  _pruneStaleFailures() {
+    const now = Date.now();
+    const WINDOW_MS = 5 * 60 * 1000;
+
+    for (const [key, entry] of this.failureWindow.entries()) {
+      if (now - entry.firstFailureAt > WINDOW_MS) {
+        this.failureWindow.delete(key);
+      }
+    }
+  }
+
   /**
    * Records a task failure and manages the circuit breaker.
    */
   recordFailure(provider, taskType = 'default') {
-    const key = `${provider}:${taskType}`;
-    const failures = (this.failureWindow.get(key) || 0) + 1;
-    this.failureWindow.set(key, failures);
+    this._pruneStaleFailures();
 
-    if (failures >= 3) {
-      const expiry = new Date(Date.now() + 10 * 60 * 1000); // 10 min blacklist
+    const key = `${provider}:${taskType}`;
+    const existing = this.failureWindow.get(key);
+    const entry = existing
+      ? { count: existing.count + 1, firstFailureAt: existing.firstFailureAt }
+      : { count: 1, firstFailureAt: Date.now() };
+    this.failureWindow.set(key, entry);
+
+    if (entry.count >= 3) {
+      const expiry = new Date(Date.now() + 10 * 60 * 1000);
       this.blacklist.set(provider, expiry);
       console.warn(`[MCA-CIRCUIT-OPEN] Provider '${provider}' blacklisted for 10 min due to consecutive failures on '${taskType}'.`);
-      this.failureWindow.set(key, 0); // Reset window upon blacklisting
+      this.failureWindow.delete(key);
     }
   }
 
   /**
    * Hardening: Simulate provider failures to verify Fallback Protocol.
    */
+  recordLatency(provider, durationMs) {
+    recordLatency(provider, durationMs);
+  }
+
+  getP95Latency(provider) {
+    return getP95Latency(provider);
+  }
+
   startChaosMode() {
     console.log('[ENTERPRISE-RESILIENCE] CloudBroker Chaos Mode ACTIVE. Simulating jitter and provider dropouts...');
     setInterval(() => {
       const providers = Object.keys(this.state);
       const randomProvider = providers[Math.floor(Math.random() * providers.length)];
-      this.state[randomProvider].latency = Math.random() > 0.7 ? 5000 : 100;
+      recordLatency(randomProvider, Math.random() > 0.7 ? 5000 : 100);
     }, 10000);
   }
 }
 
 module.exports = CloudBroker;
+module.exports.recordLatency = recordLatency;
+module.exports.getP95Latency = getP95Latency;

package/bin/models/cost-tracker.js

@@ -101,10 +101,12 @@ function getSummary(params = { days: 7 }) {
         result.calls++;
         
         const model = entry.model || 'unknown';
-        if (!result.by_model[model]) result.by_model[model] = { cost: 0, calls: 0, tokens: 0 };
+        if (!result.by_model[model]) result.by_model[model] = { cost: 0, calls: 0, tokens: 0, cache_read_tokens: 0, cache_creation_tokens: 0 };
         result.by_model[model].cost += cost;
         result.by_model[model].calls++;
         result.by_model[model].tokens += (entry.input_tokens || 0) + (entry.output_tokens || 0);
+        result.by_model[model].cache_read_tokens += (entry.cache_read_input_tokens || 0);
+        result.by_model[model].cache_creation_tokens += (entry.cache_creation_input_tokens || 0);
 
         const phase = entry.phase || 'unknown';
         if (!result.by_phase[phase]) result.by_phase[phase] = 0;

package/bin/models/difficulty-scorer.js

@@ -0,0 +1,54 @@
+'use strict';
+/**
+ * MindForge — Difficulty Scorer (UC-06). Pure heuristic 1-10.
+ * Used by model-router in SHADOW MODE to log intended routing
+ * without altering actual model selection.
+ */
+
+const HIGH_KW = /auth|jwt|oauth|crypto|security|payment|pii|gdpr|hipaa|encrypt|secret|credential/i;
+const MED_KW = /refactor|migrate|architect|design|performance|concurrency|async/i;
+
+/**
+ * Score a task for difficulty on a 1-10 scale.
+ * @param {object} task
+ * @param {string} [task.description] — free-text task description
+ * @param {string[]} [task.files] — files involved
+ * @param {number} [task.tier] — security tier (1-3)
+ * @returns {number} integer difficulty score in [1, 10]
+ */
+function score(task = {}) {
+  const desc = task.description || '';
+  const files = task.files || [];
+  const tier = task.tier || 0;
+
+  let s = 3; // baseline
+
+  // Keyword analysis (description + file paths)
+  if (HIGH_KW.test(desc) || files.some(f => HIGH_KW.test(f))) {
+    s += 4;
+  } else if (MED_KW.test(desc)) {
+    s += 2;
+  }
+
+  // File count complexity
+  if (files.length > 10) {
+    s += 2;
+  } else if (files.length > 5) {
+    s += 1;
+  }
+
+  // Long description signals complexity
+  if (desc.length > 500) {
+    s += 1;
+  }
+
+  // Tier-3 floor: security/privacy tasks never score below 7
+  if (tier >= 3) {
+    s = Math.max(s, 7);
+  }
+
+  // Clamp to [1, 10]
+  return Math.min(Math.max(s, 1), 10);
+}
+
+module.exports = { score };

package/bin/models/gemini-provider.js

@@ -46,10 +46,14 @@ class GeminiProvider {
               return reject(Object.assign(new Error(json.error?.message || 'Gemini API error'), { status: res.statusCode }));
             }
 
-            // Gemini 1.5 Pro billing is complex; using $1.25 / 1M input as baseline
             const inputTokens = json.usageMetadata.promptTokenCount;
             const outputTokens = json.usageMetadata.candidatesTokenCount;
-            const cost = (inputTokens * 0.00000125) + (outputTokens * 0.00000375);
+
+            const { priceCall } = require('./pricing-registry');
+            const cost = priceCall(modelId, {
+              input_tokens: inputTokens,
+              output_tokens: outputTokens,
+            });
 
             resolve({
               model: modelId,
@@ -74,6 +78,57 @@ class GeminiProvider {
       req.end();
     });
   }
+
+  async streamComplete(messages, options = {}) {
+    const model = options.model || 'gemini-2.5-pro';
+    const maxTokens = options.maxTokens || 8192;
+
+    const contents = messages.map(msg => ({
+      role: msg.role === 'assistant' ? 'model' : 'user',
+      parts: [{ text: msg.content }],
+    }));
+
+    const data = JSON.stringify({
+      contents,
+      generationConfig: {
+        maxOutputTokens: maxTokens,
+      },
+    });
+
+    const modelId = model.startsWith('models/') ? model : `models/${model}`;
+
+    return new Promise((resolve, reject) => {
+      const req = https.request({
+        hostname: 'generativelanguage.googleapis.com',
+        path: `/v1beta/${modelId}:streamGenerateContent?alt=sse`,
+        method: 'POST',
+        headers: {
+          'Content-Type': 'application/json',
+          'x-goog-api-key': this.apiKey,
+          'Content-Length': Buffer.byteLength(data),
+        },
+        timeout: 300_000,
+      }, res => {
+        if (res.statusCode !== 200) {
+          let body = '';
+          res.on('data', chunk => body += chunk);
+          res.on('end', () => {
+            reject(new Error(`Gemini streaming failed: ${res.statusCode}`));
+          });
+          return;
+        }
+        resolve(res);
+      });
+
+      req.on('error', reject);
+      req.on('timeout', () => {
+        req.destroy();
+        reject(new Error('Gemini stream timeout'));
+      });
+      req.write(data);
+      req.end();
+    });
+  }
 }
 
 module.exports = GeminiProvider;

package/bin/models/model-client.js

@@ -82,6 +82,26 @@ class ModelClient {
     }
   }
 
+  static async streamComplete(params) {
+    const {
+      persona = 'developer',
+      tier = 1,
+      messages,
+      maxTokens,
+      taskName = 'unknown',
+    } = params;
+
+    const routing = Router.route(persona, tier);
+    const modelId = routing.model;
+    const provider = this._getProvider(modelId);
+
+    if (!provider || !provider.streamComplete) {
+      throw new Error(`Streaming not supported for model: ${modelId}`);
+    }
+
+    return provider.streamComplete(messages, { ...params, model: modelId });
+  }
+
   static _getProvider(modelId) {
     if (modelId.startsWith('claude') || modelId.startsWith('anthropic.claude')) {
       if (!process.env.ANTHROPIC_API_KEY) return null;

package/bin/models/model-router.js

@@ -36,15 +36,13 @@ const PERSONA_MAP = {
 };
 
 let _settingsCache = null;
+let _settingsMtime = 0;
+const CACHE_CHECK_INTERVAL_MS = 60000;
+let _lastCacheCheck = 0;
 
-function readMindforgeSettings() {
-  if (_settingsCache) return _settingsCache;
-  const configPath = path.join(process.cwd(), 'MINDFORGE.md');
-  if (!fs.existsSync(configPath)) return DEFAULTS;
-
-  const content = fs.readFileSync(configPath, 'utf8');
+function parseSettings(filePath) {
+  const content = fs.readFileSync(filePath, 'utf8');
   const settings = { ...DEFAULTS };
-  
   const lines = content.split('\n');
   for (const line of lines) {
     const match = line.match(/^([A-Z0-9_]+)=(.*)$/);
@@ -52,50 +50,83 @@ function readMindforgeSettings() {
       settings[match[1]] = match[2].trim();
     }
   }
-  _settingsCache = settings;
   return settings;
 }
 
-function route(persona = 'developer', tier = 1) {
+function readMindforgeSettings() {
+  const now = Date.now();
+  if (now - _lastCacheCheck < CACHE_CHECK_INTERVAL_MS && _settingsCache) {
+    return _settingsCache;
+  }
+  _lastCacheCheck = now;
+
+  const configPath = path.join(process.cwd(), 'MINDFORGE.md');
+  try {
+    const stat = fs.statSync(configPath);
+    if (stat.mtimeMs !== _settingsMtime) {
+      _settingsMtime = stat.mtimeMs;
+      _settingsCache = parseSettings(configPath);
+    }
+  } catch {
+    if (!_settingsCache) _settingsCache = { ...DEFAULTS };
+  }
+
+  return _settingsCache;
+}
+
+function route(persona = 'developer', tier = 1, taskContext) {
   const settings = readMindforgeSettings();
-  
+  let result;
+
   // 1. Tier 3 override (Security/Privacy always uses SECURITY_MODEL)
   if (tier === 3) {
-    return {
+    result = {
       model: settings.SECURITY_MODEL,
       setting: 'SECURITY_MODEL',
       reason: 'Tier 3 (Security/Privacy) override'
     };
   }
-
   // 2. Persona mapping (Specific personas like research, debug, qa)
-  if (persona !== 'developer' && PERSONA_MAP[persona]) {
+  else if (persona !== 'developer' && PERSONA_MAP[persona]) {
     const settingKey = PERSONA_MAP[persona];
-    return {
+    result = {
       model: settings[settingKey],
       setting: settingKey,
       reason: `Mapped from specific persona "${persona}"`
     };
   }
-
   // 3. Budget Bias (Tier 1 uses QUICK_MODEL for default developer tasks)
-  if (tier === 1) {
-    return {
+  else if (tier === 1) {
+    result = {
       model: settings.QUICK_MODEL,
       setting: 'QUICK_MODEL',
       reason: 'Tier 1 Budget Bias (efficiency mode)'
     };
   }
-
   // 4. Default mapping
-  const settingKey = 'EXECUTOR_MODEL';
-  const model = settings[settingKey];
-
-  return {
-    model,
-    setting: settingKey,
-    reason: `Default EXECUTOR_MODEL for tier ${tier}`
-  };
+  else {
+    const settingKey = 'EXECUTOR_MODEL';
+    result = {
+      model: settings[settingKey],
+      setting: settingKey,
+      reason: `Default EXECUTOR_MODEL for tier ${tier}`
+    };
+  }
+
+  // Shadow-mode: difficulty-aware routing (UC-06)
+  // Logs what model the difficulty scorer WOULD select, without changing the result.
+  if (taskContext) {
+    const { score: scoreDifficulty } = require('./difficulty-scorer');
+    const difficulty = scoreDifficulty(taskContext);
+    const shadowModel = difficulty <= 3 ? settings.QUICK_MODEL
+                      : difficulty >= 8 ? settings.PLANNER_MODEL
+                      : settings.EXECUTOR_MODEL;
+    if (shadowModel !== result.model) {
+      process.stderr.write(`[model-router:shadow] difficulty=${difficulty} would route to ${shadowModel} (actual: ${result.model})\n`);
+    }
+  }
+
+  return result;
 }
 
 function getModel(settingKey) {
@@ -105,6 +136,8 @@ function getModel(settingKey) {
 
 function clearCache() {
   _settingsCache = null;
+  _settingsMtime = 0;
+  _lastCacheCheck = 0;
 }
 
 function getAllSettings() {

package/bin/models/openai-provider.js

@@ -46,9 +46,12 @@ class OpenAIProvider {
 
             const inputTokens = json.usage.prompt_tokens;
             const outputTokens = json.usage.completion_tokens;
-            
-            // Basic cost calculation (GPT-4o prices)
-            const cost = (inputTokens * 0.000005) + (outputTokens * 0.000015);
+
+            const { priceCall } = require('./pricing-registry');
+            const cost = priceCall(json.model, {
+              input_tokens: inputTokens,
+              output_tokens: outputTokens,
+            });
 
             resolve({
               model: json.model,
@@ -73,6 +76,50 @@ class OpenAIProvider {
       req.end();
     });
   }
+
+  async streamComplete(messages, options = {}) {
+    const model = options.model || 'gpt-4o';
+    const maxTokens = options.maxTokens || 4096;
+
+    const data = JSON.stringify({
+      model,
+      messages,
+      max_tokens: maxTokens,
+      stream: true,
+    });
+
+    return new Promise((resolve, reject) => {
+      const req = https.request({
+        hostname: 'api.openai.com',
+        path: '/v1/chat/completions',
+        method: 'POST',
+        headers: {
+          'Content-Type': 'application/json',
+          'Authorization': `Bearer ${this.apiKey}`,
+          'Content-Length': Buffer.byteLength(data),
+        },
+        timeout: 300_000,
+      }, res => {
+        if (res.statusCode !== 200) {
+          let body = '';
+          res.on('data', chunk => body += chunk);
+          res.on('end', () => {
+            reject(new Error(`OpenAI streaming failed: ${res.statusCode}`));
+          });
+          return;
+        }
+        resolve(res);
+      });
+
+      req.on('error', reject);
+      req.on('timeout', () => {
+        req.destroy();
+        reject(new Error('OpenAI stream timeout'));
+      });
+      req.write(data);
+      req.end();
+    });
+  }
 }
 
 module.exports = OpenAIProvider;

package/bin/models/pricing-registry.js

@@ -0,0 +1,128 @@
+/**
+ * MindForge v2 — Pricing Registry (UC-05)
+ *
+ * Single source of truth for all model pricing. Loads from
+ * .mindforge/config.json `revops.market_registry` and normalizes
+ * to per-1M-token units. All providers and cost-tracker MUST
+ * query this module instead of hardcoding rates.
+ *
+ * Buckets: input, output, cache_read, cache_creation
+ */
+'use strict';
+
+const fs = require('fs');
+const path = require('path');
+
+const CONFIG_PATH = path.join(__dirname, '..', '..', '.mindforge', 'config.json');
+
+// Fallback per-1M rates when model is unknown (generous estimate to avoid under-billing)
+const FALLBACK_RATES = {
+  input: 5.0,
+  output: 15.0,
+  cache_read: 0.5,
+  cache_creation: 6.25,
+};
+
+let _priceTable = null;
+
+/**
+ * Load and normalize the market_registry from config.json.
+ * Config values are in per-1K-token units. We multiply by 1000 to get per-1M.
+ * Cache buckets: cache_read = 10% of input, cache_creation = 125% of input
+ * (unless explicitly provided in config).
+ */
+function loadPriceTable() {
+  const raw = fs.readFileSync(CONFIG_PATH, 'utf8');
+  const config = JSON.parse(raw);
+  const registry = config.revops && config.revops.market_registry;
+
+  if (!registry || typeof registry !== 'object') {
+    process.stderr.write('[pricing-registry] WARN: market_registry missing from config.json, using fallbacks\n');
+    return {};
+  }
+
+  const table = {};
+  for (const [modelId, entry] of Object.entries(registry)) {
+    const inputPer1M = (entry.cost_input || 0) * 1000;
+    const outputPer1M = (entry.cost_output || 0) * 1000;
+
+    // Cache bucket derivation: use explicit config fields if present,
+    // otherwise derive from Anthropic-standard ratios
+    const cacheReadPer1M = entry.cost_cache_read != null
+      ? entry.cost_cache_read * 1000
+      : inputPer1M * 0.1;
+    const cacheCreationPer1M = entry.cost_cache_creation != null
+      ? entry.cost_cache_creation * 1000
+      : inputPer1M * 1.25;
+
+    table[modelId] = {
+      input: inputPer1M,
+      output: outputPer1M,
+      cache_read: cacheReadPer1M,
+      cache_creation: cacheCreationPer1M,
+    };
+  }
+  return table;
+}
+
+function ensureLoaded() {
+  if (_priceTable === null) {
+    _priceTable = loadPriceTable();
+  }
+  return _priceTable;
+}
+
+/**
+ * Get the per-1M-token price for a model+bucket.
+ * @param {string} modelId - e.g. 'claude-sonnet-4-6'
+ * @param {'input'|'output'|'cache_read'|'cache_creation'} bucket
+ * @returns {number} USD per 1M tokens
+ */
+function getPrice(modelId, bucket) {
+  const table = ensureLoaded();
+  const entry = table[modelId];
+  if (!entry) {
+    process.stderr.write(`[pricing-registry] WARN: unknown model "${modelId}", using fallback rates\n`);
+    return FALLBACK_RATES[bucket] || FALLBACK_RATES.input;
+  }
+  return entry[bucket] != null ? entry[bucket] : (FALLBACK_RATES[bucket] || 0);
+}
+
+/**
+ * Calculate total cost for a single API call.
+ * @param {string} modelId
+ * @param {object} usage
+ * @param {number} usage.input_tokens
+ * @param {number} usage.output_tokens
+ * @param {number} [usage.cache_read_input_tokens=0]
+ * @param {number} [usage.cache_creation_input_tokens=0]
+ * @returns {number} Total USD cost
+ */
+function priceCall(modelId, usage) {
+  const inputTokens = usage.input_tokens || 0;
+  const outputTokens = usage.output_tokens || 0;
+  const cacheReadTokens = usage.cache_read_input_tokens || 0;
+  const cacheCreationTokens = usage.cache_creation_input_tokens || 0;
+
+  const inputRate = getPrice(modelId, 'input');
+  const outputRate = getPrice(modelId, 'output');
+  const cacheReadRate = getPrice(modelId, 'cache_read');
+  const cacheCreationRate = getPrice(modelId, 'cache_creation');
+
+  const cost =
+    (inputTokens / 1_000_000) * inputRate +
+    (outputTokens / 1_000_000) * outputRate +
+    (cacheReadTokens / 1_000_000) * cacheReadRate +
+    (cacheCreationTokens / 1_000_000) * cacheCreationRate;
+
+  return cost;
+}
+
+/**
+ * Clear the cached price table (for testing or config reload).
+ */
+function clearCache() {
+  _priceTable = null;
+}
+
+module.exports = { getPrice, priceCall, clearCache };

package/bin/review/ads-engine.js

@@ -49,7 +49,7 @@ Include a [ADS_METRICS] block for your counter-proposal or critique logic.`,
     sessionId,
     phaseNum
   });
-  const redCritique = redResponse.content;
+  let redCritique = redResponse.content;
   process.stdout.write('done.\n');
 
   // Red-Team Jailbreak: Force higher-fidelity critiques if Auditor is too lenient