mindforge-cc 1.0.5 → 2.0.0-alpha.6

package/bin/models/anthropic-provider.js

@@ -0,0 +1,77 @@
+/**
+ * MindForge v2 — Anthropic Provider
+ */
+'use strict';
+
+const https = require('https');
+
+class AnthropicProvider {
+  constructor(apiKey) {
+    this.apiKey = apiKey;
+  }
+
+  async complete(params) {
+    const { model, systemPrompt, userMessage, maxTokens = 4096, temperature = 0.7 } = params;
+
+    const data = JSON.stringify({
+      model,
+      system: systemPrompt,
+      messages: [{ role: 'user', content: userMessage }],
+      max_tokens: maxTokens,
+      temperature,
+    });
+
+    return new Promise((resolve, reject) => {
+      const req = https.request({
+        hostname: 'api.anthropic.com',
+        path: '/v1/messages',
+        method: 'POST',
+        headers: {
+          'Content-Type': 'application/json',
+          'x-api-key': this.apiKey,
+          'anthropic-version': '2023-06-01',
+          'Content-Length': Buffer.byteLength(data),
+        },
+        timeout: 120_000,
+      }, res => {
+        let body = '';
+        res.on('data', chunk => body += chunk);
+        res.on('end', () => {
+          try {
+            const json = JSON.parse(body);
+            if (res.statusCode !== 200) {
+              return reject(Object.assign(new Error(json.error?.message || 'Anthropic API error'), { status: res.statusCode }));
+            }
+
+            const inputTokens = json.usage.input_tokens;
+            const outputTokens = json.usage.output_tokens;
+            
+            // Basic cost calculation (Sonnet 3.5 prices)
+            const cost = (inputTokens * 0.000003) + (outputTokens * 0.000015);
+
+            resolve({
+              model: json.model,
+              content: json.content[0].text,
+              input_tokens: inputTokens,
+              output_tokens: outputTokens,
+              cost_usd: cost,
+              provider: 'anthropic'
+            });
+          } catch (e) {
+            reject(new Error('Failed to parse Anthropic response: ' + e.message));
+          }
+        });
+      });
+
+      req.on('error', reject);
+      req.on('timeout', () => {
+        req.destroy();
+        reject(Object.assign(new Error('Anthropic timeout'), { status: 408 }));
+      });
+      req.write(data);
+      req.end();
+    });
+  }
+}
+
+module.exports = AnthropicProvider;

package/bin/models/cost-tracker.js

@@ -0,0 +1,118 @@
+/**
+ * MindForge v2 — Cost Tracker
+ */
+'use strict';
+
+const fs = require('fs');
+const path = require('path');
+
+const METRICS_DIR = path.join(process.cwd(), '.mindforge', 'metrics');
+const USAGE_LOG = path.join(METRICS_DIR, 'token-usage.jsonl');
+
+function ensureDir() {
+  if (!fs.existsSync(METRICS_DIR)) {
+    fs.mkdirSync(METRICS_DIR, { recursive: true });
+  }
+}
+
+let _dailyCache = { value: 0, computed_at: 0 };
+
+function getTodaySpend() {
+  if (!fs.existsSync(USAGE_LOG)) return 0;
+  
+  const today = new Date().toISOString().slice(0, 10);
+  const content = fs.readFileSync(USAGE_LOG, 'utf8');
+  const lines = content.trim().split('\n');
+  
+  let total = 0;
+  for (const line of lines) {
+    try {
+      const entry = JSON.parse(line);
+      if (entry.date === today) {
+        total += entry.cost_usd || 0;
+      }
+    } catch (e) {
+      process.stderr.write('[cost-tracker] Skipped malformed entry\n');
+    }
+  }
+  return total;
+}
+
+function getTodaySpendCached() {
+  const AGE_MS = Date.now() - _dailyCache.computed_at;
+  if (AGE_MS > 60_000) {
+    _dailyCache.value = getTodaySpend();
+    _dailyCache.computed_at = Date.now();
+  }
+  return _dailyCache.value;
+}
+
+async function preflight(estimatedCost = 0) {
+  const settings = require('./model-router').getAllSettings();
+  const hardLimit = parseFloat(settings.MODEL_COST_HARD_LIMIT_USD || '0.0');
+  
+  if (hardLimit <= 0) return;
+
+  const todaySpend = getTodaySpendCached();
+  const projected = todaySpend + estimatedCost;
+
+  if (projected >= hardLimit) {
+    throw Object.assign(
+      new Error(`Daily cost limit $${hardLimit} reached (Today: $${todaySpend.toFixed(4)})`),
+      { code: 'COST_LIMIT_REACHED', spend: todaySpend, limit: hardLimit }
+    );
+  }
+}
+
+async function record(entry) {
+  ensureDir();
+  const enriched = {
+    ...entry,
+    date: new Date().toISOString().slice(0, 10),
+    timestamp: new Date().toISOString()
+  };
+  fs.appendFileSync(USAGE_LOG, JSON.stringify(enriched) + '\n');
+  _dailyCache.computed_at = 0; // Invalidate cache
+}
+
+function getSummary(params = { days: 7 }) {
+  if (!fs.existsSync(USAGE_LOG)) return { total_usd: 0, by_model: {} };
+  
+  const cutoffDate = new Date();
+  cutoffDate.setDate(cutoffDate.getDate() - params.days);
+  const cutoffStr = cutoffDate.toISOString().slice(0, 10);
+
+  const content = fs.readFileSync(USAGE_LOG, 'utf8');
+  const lines = content.trim().split('\n');
+  
+  const result = {
+    total_usd: 0,
+    by_model: {},
+    by_phase: {},
+    calls: 0
+  };
+
+  for (const line of lines) {
+    try {
+      const entry = JSON.parse(line);
+      if (entry.date >= cutoffStr) {
+        const cost = entry.cost_usd || 0;
+        result.total_usd += cost;
+        result.calls++;
+        
+        const model = entry.model || 'unknown';
+        if (!result.by_model[model]) result.by_model[model] = { cost: 0, calls: 0, tokens: 0 };
+        result.by_model[model].cost += cost;
+        result.by_model[model].calls++;
+        result.by_model[model].tokens += (entry.input_tokens || 0) + (entry.output_tokens || 0);
+
+        const phase = entry.phase || 'unknown';
+        if (!result.by_phase[phase]) result.by_phase[phase] = 0;
+        result.by_phase[phase] += cost;
+      }
+    } catch (e) { /* ignore parse errors for summary */ }
+  }
+  return result;
+}
+
+module.exports = { record, preflight, getTodaySpend, getTodaySpendCached, getSummary };

package/bin/models/gemini-provider.js

@@ -0,0 +1,79 @@
+/**
+ * MindForge v2 — Gemini Provider
+ * Using header-based auth for security.
+ */
+'use strict';
+
+const https = require('https');
+
+class GeminiProvider {
+  constructor(apiKey) {
+    this.apiKey = apiKey;
+  }
+
+  async complete(params) {
+    const { model, systemPrompt, userMessage, maxTokens = 8192, temperature = 0.2 } = params;
+
+    const data = JSON.stringify({
+      system_instruction: { parts: [{ text: systemPrompt }] },
+      contents: [{ parts: [{ text: userMessage }] }],
+      generationConfig: {
+        maxOutputTokens: maxTokens,
+        temperature,
+      },
+    });
+
+    const modelId = model.startsWith('models/') ? model : `models/${model}`;
+
+    return new Promise((resolve, reject) => {
+      const req = https.request({
+        hostname: 'generativelanguage.googleapis.com',
+        path: `/v1beta/${modelId}:generateContent`,
+        method: 'POST',
+        headers: {
+          'Content-Type': 'application/json',
+          'x-goog-api-key': this.apiKey, // Header auth
+          'Content-Length': Buffer.byteLength(data),
+        },
+        timeout: 180_000,
+      }, res => {
+        let body = '';
+        res.on('data', chunk => body += chunk);
+        res.on('end', () => {
+          try {
+            const json = JSON.parse(body);
+            if (res.statusCode !== 200) {
+              return reject(Object.assign(new Error(json.error?.message || 'Gemini API error'), { status: res.statusCode }));
+            }
+
+            // Gemini 1.5 Pro billing is complex; using $1.25 / 1M input as baseline
+            const inputTokens = json.usageMetadata.promptTokenCount;
+            const outputTokens = json.usageMetadata.candidatesTokenCount;
+            const cost = (inputTokens * 0.00000125) + (outputTokens * 0.00000375);
+
+            resolve({
+              model: modelId,
+              content: json.candidates[0].content.parts[0].text,
+              input_tokens: inputTokens,
+              output_tokens: outputTokens,
+              cost_usd: cost,
+              provider: 'google'
+            });
+          } catch (e) {
+            reject(new Error('Failed to parse Gemini response: ' + e.message));
+          }
+        });
+      });
+
+      req.on('error', reject);
+      req.on('timeout', () => {
+        req.destroy();
+        reject(Object.assign(new Error('Gemini timeout'), { status: 408 }));
+      });
+      req.write(data);
+      req.end();
+    });
+  }
+}
+
+module.exports = GeminiProvider;

package/bin/models/model-client.js

@@ -0,0 +1,98 @@
+/**
+ * MindForge v2 — Model Client
+ * Unified client with routing, fallbacks, and cost tracking.
+ */
+'use strict';
+
+const Router = require('./model-router');
+const CostTracker = require('./cost-tracker');
+const AnthropicProvider = require('./anthropic-provider');
+const OpenAIProvider = require('./openai-provider');
+const GeminiProvider = require('./gemini-provider');
+
+const FALLBACK_CHAINS = {
+  'claude-3-opus-20240229': ['gpt-4o', 'claude-3-5-sonnet-20240620'],
+  'gpt-4o': ['claude-3-5-sonnet-20240620'],
+  'gemini-1.5-pro': ['claude-3-5-sonnet-20240620'],
+};
+
+class ModelClient {
+  static async complete(params) {
+    const {
+      persona = 'developer',
+      tier = 1,
+      maxTokens,
+      temperature,
+      taskName = 'unknown',
+      sessionId = 'unknown',
+      phaseNum = 0
+    } = params;
+
+    // 1. Route to model
+    const routing = Router.route(persona, tier);
+    let modelId = routing.model;
+
+    // 2. Pre-flight cost check
+    try {
+      await CostTracker.preflight(0.05); // Conservative estimate
+    } catch (e) {
+      if (e.code === 'COST_LIMIT_REACHED') throw e;
+    }
+
+    // 3. Execute with fallbacks
+    let result = null;
+    let attempts = [modelId, ...(FALLBACK_CHAINS[modelId] || [])];
+
+    for (const currentModel of attempts) {
+      try {
+        const provider = this._getProvider(currentModel);
+        if (!provider) continue;
+
+        result = await provider.complete({
+          model: currentModel,
+          systemPrompt: params.systemPrompt,
+          userMessage: params.userMessage,
+          maxTokens,
+          temperature
+        });
+
+        // Add metadata
+        result.task_name = taskName;
+        result.session_id = sessionId;
+        result.phase = phaseNum;
+
+        if (currentModel !== modelId) {
+          result.content = `[FALLBACK NOTICE: ${modelId} unavailable — used ${currentModel} instead.]\n\n${result.content}`;
+        }
+
+        // 4. Record cost
+        await CostTracker.record(result);
+        return result;
+
+      } catch (err) {
+        process.stderr.write(`[model-client] ${currentModel} failed: ${err.message}\n`);
+        if (attempts.indexOf(currentModel) === attempts.length - 1) {
+          throw err;
+        }
+      }
+    }
+  }
+
+  static _getProvider(modelId) {
+    if (modelId.includes('claude')) {
+      if (!process.env.ANTHROPIC_API_KEY) return null;
+      return new AnthropicProvider(process.env.ANTHROPIC_API_KEY);
+    }
+    if (modelId.includes('gpt')) {
+      if (!process.env.OPENAI_API_KEY) return null;
+      return new OpenAIProvider(process.env.OPENAI_API_KEY);
+    }
+    if (modelId.includes('gemini')) {
+      if (!process.env.GOOGLE_API_KEY) return null;
+      return new GeminiProvider(process.env.GOOGLE_API_KEY);
+    }
+    return null;
+  }
+}
+
+module.exports = ModelClient;

package/bin/models/model-router.js

@@ -0,0 +1,111 @@
+/**
+ * MindForge v2 — Model Router
+ * Resolves persona and tier to a specific model ID based on settings and context.
+ */
+'use strict';
+
+const fs = require('fs');
+const path = require('path');
+
+// Default model assignments
+const DEFAULTS = {
+  PLANNER_MODEL:          'claude-3-opus-20240229',
+  EXECUTOR_MODEL:         'claude-3-5-sonnet-20240620',
+  REVIEWER_MODEL:         'gpt-4o',
+  SECURITY_MODEL:         'claude-3-opus-20240229',
+  RESEARCH_MODEL:         'gemini-1.5-pro',
+  QA_MODEL:               'claude-3-5-sonnet-20240620',
+  DEBUG_MODEL:            'claude-3-opus-20240229',
+  QUICK_MODEL:            'claude-3-5-haiku-20241022',
+  CROSS_REVIEW_SECONDARY: 'gpt-4o',
+  CROSS_REVIEW_TERTIARY:  'gemini-1.5-pro',
+};
+
+// Persona to setting key mapping
+const PERSONA_MAP = {
+  'developer':         'EXECUTOR_MODEL',
+  'architect':         'PLANNER_MODEL',
+  'planner':           'PLANNER_MODEL',
+  'security-reviewer': 'SECURITY_MODEL',
+  'qa-engineer':       'QA_MODEL',
+  'research-agent':    'RESEARCH_MODEL',
+  'debug-specialist':  'DEBUG_MODEL',
+};
+
+let _settingsCache = null;
+
+function readMindforgeSettings() {
+  if (_settingsCache) return _settingsCache;
+  const configPath = path.join(process.cwd(), 'MINDFORGE.md');
+  if (!fs.existsSync(configPath)) return DEFAULTS;
+
+  const content = fs.readFileSync(configPath, 'utf8');
+  const settings = { ...DEFAULTS };
+  
+  const lines = content.split('\n');
+  for (const line of lines) {
+    const match = line.match(/^([A-Z0-9_]+)=(.*)$/);
+    if (match) {
+      settings[match[1]] = match[2].trim();
+    }
+  }
+  _settingsCache = settings;
+  return settings;
+}
+
+function route(persona = 'developer', tier = 1) {
+  const settings = readMindforgeSettings();
+  
+  // 1. Tier 3 override (Security/Privacy always uses SECURITY_MODEL)
+  if (tier === 3) {
+    return {
+      model: settings.SECURITY_MODEL,
+      setting: 'SECURITY_MODEL',
+      reason: 'Tier 3 (Security/Privacy) override'
+    };
+  }
+
+  // 2. Persona mapping (Specific personas like research, debug, qa)
+  if (persona !== 'developer' && PERSONA_MAP[persona]) {
+    const settingKey = PERSONA_MAP[persona];
+    return {
+      model: settings[settingKey],
+      setting: settingKey,
+      reason: `Mapped from specific persona "${persona}"`
+    };
+  }
+
+  // 3. Budget Bias (Tier 1 uses QUICK_MODEL for default developer tasks)
+  if (tier === 1) {
+    return {
+      model: settings.QUICK_MODEL,
+      setting: 'QUICK_MODEL',
+      reason: 'Tier 1 Budget Bias (efficiency mode)'
+    };
+  }
+
+  // 4. Default mapping
+  const settingKey = 'EXECUTOR_MODEL';
+  const model = settings[settingKey];
+
+  return {
+    model,
+    setting: settingKey,
+    reason: `Default EXECUTOR_MODEL for tier ${tier}`
+  };
+}
+
+function getModel(settingKey) {
+  const settings = readMindforgeSettings();
+  return settings[settingKey] || DEFAULTS[settingKey];
+}
+
+function clearCache() {
+  _settingsCache = null;
+}
+
+function getAllSettings() {
+  return readMindforgeSettings();
+}
+
+module.exports = { route, getModel, clearCache, getAllSettings };

package/bin/models/openai-provider.js

@@ -0,0 +1,78 @@
+/**
+ * MindForge v2 — OpenAI Provider
+ */
+'use strict';
+
+const https = require('https');
+
+class OpenAIProvider {
+  constructor(apiKey) {
+    this.apiKey = apiKey;
+  }
+
+  async complete(params) {
+    const { model, systemPrompt, userMessage, maxTokens = 4096, temperature = 0.7 } = params;
+
+    const data = JSON.stringify({
+      model,
+      messages: [
+        { role: 'system', content: systemPrompt },
+        { role: 'user', content: userMessage }
+      ],
+      max_tokens: maxTokens,
+      temperature,
+    });
+
+    return new Promise((resolve, reject) => {
+      const req = https.request({
+        hostname: 'api.openai.com',
+        path: '/v1/chat/completions',
+        method: 'POST',
+        headers: {
+          'Content-Type': 'application/json',
+          'Authorization': `Bearer ${this.apiKey}`,
+          'Content-Length': Buffer.byteLength(data),
+        },
+        timeout: 120_000,
+      }, res => {
+        let body = '';
+        res.on('data', chunk => body += chunk);
+        res.on('end', () => {
+          try {
+            const json = JSON.parse(body);
+            if (res.statusCode !== 200) {
+              return reject(Object.assign(new Error(json.error?.message || 'OpenAI API error'), { status: res.statusCode }));
+            }
+
+            const inputTokens = json.usage.prompt_tokens;
+            const outputTokens = json.usage.completion_tokens;
+            
+            // Basic cost calculation (GPT-4o prices)
+            const cost = (inputTokens * 0.000005) + (outputTokens * 0.000015);
+
+            resolve({
+              model: json.model,
+              content: json.choices[0].message.content,
+              input_tokens: inputTokens,
+              output_tokens: outputTokens,
+              cost_usd: cost,
+              provider: 'openai'
+            });
+          } catch (e) {
+            reject(new Error('Failed to parse OpenAI response: ' + e.message));
+          }
+        });
+      });
+
+      req.on('error', reject);
+      req.on('timeout', () => {
+        req.destroy();
+        reject(Object.assign(new Error('OpenAI timeout'), { status: 408 }));
+      });
+      req.write(data);
+      req.end();
+    });
+  }
+}
+
+module.exports = OpenAIProvider;

package/bin/research/research-engine.js

@@ -0,0 +1,115 @@
+/**
+ * MindForge v2 — Research Engine
+ * Leverages Gemini 1M context for deep research.
+ */
+'use strict';
+
+const fs = require('fs');
+const path = require('path');
+const https = require('https');
+const http = require('http');
+const { URL } = require('url');
+const dns = require('dns').promises;
+const ModelClient = require('../models/model-client');
+const Router = require('../models/model-router');
+
+const PRIVATE_RANGES = [
+  /^127\./,
+  /^10\./,
+  /^172\.(1[6-9]|2\d|3[01])\./,
+  /^192\.168\./,
+  /^169\.254\./,
+  /^::1$/,
+  /^fc00:/,
+  /^fe80:/,
+];
+
+const SYSTEM_PROMPTS = {
+  general: 'You are a thorough technical researcher. Analyze the provided context deeply and answer the question comprehensively. Cite specific evidence.',
+  library_comparison: 'You are a senior engineer evaluating libraries. Analyze: API design, maintenance, performance, security, and community.',
+  codebase_analysis: 'You are a senior architect auditing a codebase. Identify patterns, debt, security issues, and bottlenecks.',
+  compliance: 'You are a compliance engineer reviewing implementation against regulations. Identify gaps and remediation steps.',
+};
+
+async function isSafeUrl(url) {
+  try {
+    const parsed = new URL(url);
+    if (!['http:', 'https:'].includes(parsed.protocol)) return false;
+    const { address } = await dns.lookup(parsed.hostname);
+    return !PRIVATE_RANGES.some(r => r.test(address));
+  } catch {
+    return false;
+  }
+}
+
+async function fetchUrl(url) {
+  if (!await isSafeUrl(url)) throw new Error(`URL blocked: ${url}`);
+  
+  return new Promise((resolve, reject) => {
+    const protocol = url.startsWith('https') ? https : http;
+    const timeout = setTimeout(() => { req.destroy(); reject(new Error('Timeout')); }, 30_000);
+
+    const req = protocol.get(url, res => {
+      let body = '';
+      res.on('data', c => body += c);
+      res.on('end', () => { clearTimeout(timeout); resolve(body.slice(0, 500_000)); });
+    });
+    req.on('error', e => { clearTimeout(timeout); reject(e); });
+  });
+}
+
+function packageLocalContext(paths) {
+  let content = '';
+  for (const p of paths) {
+    const fullPath = path.resolve(process.cwd(), p);
+    if (!fs.existsSync(fullPath)) continue;
+    
+    if (fs.statSync(fullPath).isDirectory()) {
+      content += walkDir(fullPath);
+    } else {
+      content += `\n\n### File: ${p}\n${fs.readFileSync(fullPath, 'utf8')}\n`;
+    }
+  }
+  return content;
+}
+
+function walkDir(dir) {
+  let content = '';
+  const files = fs.readdirSync(dir);
+  for (const f of files) {
+    const full = path.join(dir, f);
+    if (f.startsWith('.') || f === 'node_modules') continue;
+    if (fs.statSync(full).isDirectory()) content += walkDir(full);
+    else if (['.js', '.ts', '.md', '.json'].some(ext => f.endsWith(ext))) {
+      content += `\n\n### File: ${full}\n${fs.readFileSync(full, 'utf8')}\n`;
+    }
+  }
+  return content;
+}
+
+async function research(params) {
+  const { topic, question, type = 'general', contextPaths = [], urls = [] } = params;
+  
+  let context = packageLocalContext(contextPaths);
+  for (const url of urls) {
+    try {
+      const html = await fetchUrl(url);
+      const text = html.replace(/<[^>]+>/g, ' ').replace(/\s+/g, ' ').trim();
+      context += `\n\n### URL: ${url}\n${text.slice(0, 100_000)}`;
+    } catch (e) {
+      context += `\n\n### URL: ${url}\n[Failed to fetch: ${e.message}]`;
+    }
+  }
+
+  const result = await ModelClient.complete({
+    persona: 'research-agent',
+    systemPrompt: SYSTEM_PROMPTS[type] || SYSTEM_PROMPTS.general,
+    userMessage: `Topic: ${topic}\n\nContext:\n${context}\n\nQuestion: ${question}`,
+    taskName: `research-${topic}`,
+    maxTokens: 8192
+  });
+
+  return { ...result, context_length: context.length };
+}
+
+module.exports = { research, packageLocalContext, fetchUrl };