@polymorphism-tech/morph-spec 4.2.0 → 4.3.0

package/src/lib/context/context-tracker.js

@@ -0,0 +1,273 @@
+/**
+ * Context Tracker — Real-time token monitoring
+ *
+ * Tracks token usage per thread/session, detects peak usage moments,
+ * calculates averages, and identifies optimization opportunities.
+ * Integrates with analytics-engine to record context events.
+ */
+
+import { recordContextEvent } from '../analytics/analytics-engine.js';
+import { addThreadEvent } from '../threads/thread-manager.js';
+
+// ============================================================================
+// Constants
+// ============================================================================
+
+const WARNING_THRESHOLD = 0.75;   // 75% of context window
+const CRITICAL_THRESHOLD = 0.90;  // 90% of context window
+
+export const OPTIMIZATION_STRATEGIES = {
+  MCP_HYGIENE: 'mcp_hygiene',
+  MICRO_AGENT: 'micro_agent_switch',
+  CONTEXT_BUNDLE: 'context_bundle',
+  PRIMING_FILE: 'priming_file',
+  STANDARDS_SUBSET: 'standards_subset'
+};
+
+// In-memory session state (not persisted — resets each session)
+const sessions = new Map();
+
+// ============================================================================
+// Session Management
+// ============================================================================
+
+/**
+ * Start tracking a session
+ * @param {Object} opts
+ * @param {string} opts.feature - Feature name
+ * @param {string} [opts.threadId] - Thread ID
+ * @param {number} [opts.contextWindow=200000] - Model context window size
+ * @param {string} [opts.model] - Model name
+ * @returns {string} Session ID
+ */
+export function startSession({ feature, threadId = null, contextWindow = 200000, model = 'claude-sonnet-4-6' }) {
+  const sessionId = threadId || `session-${Date.now()}`;
+
+  sessions.set(sessionId, {
+    sessionId,
+    feature,
+    threadId,
+    model,
+    contextWindow,
+    startedAt: new Date().toISOString(),
+    peakUsage: 0,
+    currentUsage: 0,
+    samples: [],
+    events: []
+  });
+
+  recordContextEvent({
+    type: 'context_session_started',
+    feature,
+    threadId,
+    data: { model, contextWindow }
+  });
+
+  return sessionId;
+}
+
+/**
+ * Record a token usage sample for a session
+ * @param {string} sessionId - Session ID
+ * @param {Object} usage
+ * @param {number} usage.tokensUsed - Total tokens in context
+ * @param {Object} [usage.breakdown] - Token breakdown by source
+ * @returns {Object} Current session state with any optimization recommendations
+ */
+export function recordTokenUsage(sessionId, { tokensUsed, breakdown = {} }) {
+  const session = sessions.get(sessionId);
+  if (!session) {
+    throw new Error(`Session not found: ${sessionId}`);
+  }
+
+  const utilization = tokensUsed / session.contextWindow;
+  session.currentUsage = tokensUsed;
+
+  if (tokensUsed > session.peakUsage) {
+    session.peakUsage = tokensUsed;
+  }
+
+  const sample = {
+    timestamp: new Date().toISOString(),
+    tokensUsed,
+    utilization: Math.round(utilization * 100) / 100,
+    breakdown
+  };
+  session.samples.push(sample);
+
+  // Record to analytics
+  recordContextEvent({
+    type: 'token_usage',
+    feature: session.feature,
+    threadId: session.threadId,
+    data: { tokensUsed, utilization, breakdown }
+  });
+
+  // Check thresholds and identify opportunities
+  const recommendations = identifyOptimizations(session, utilization);
+
+  if (utilization >= CRITICAL_THRESHOLD) {
+    const event = { type: 'context_critical', utilization, recommendations };
+    session.events.push(event);
+
+    if (session.threadId) {
+      try {
+        addThreadEvent(session.threadId, 'context_critical', { utilization });
+      } catch {
+        // Thread may not exist in state yet
+      }
+    }
+
+    recordContextEvent({
+      type: 'context_critical',
+      feature: session.feature,
+      threadId: session.threadId,
+      data: { utilization, tokensUsed, recommendations }
+    });
+  } else if (utilization >= WARNING_THRESHOLD) {
+    recordContextEvent({
+      type: 'context_warning',
+      feature: session.feature,
+      threadId: session.threadId,
+      data: { utilization, tokensUsed }
+    });
+  }
+
+  return { session, recommendations };
+}
+
+/**
+ * End a session and record summary
+ * @param {string} sessionId - Session ID
+ * @returns {Object} Session summary
+ */
+export function endSession(sessionId) {
+  const session = sessions.get(sessionId);
+  if (!session) return null;
+
+  const avgUsage = session.samples.length > 0
+    ? Math.round(session.samples.reduce((sum, s) => sum + s.tokensUsed, 0) / session.samples.length)
+    : 0;
+
+  const summary = {
+    sessionId,
+    feature: session.feature,
+    model: session.model,
+    startedAt: session.startedAt,
+    endedAt: new Date().toISOString(),
+    peakUsage: session.peakUsage,
+    avgUsage,
+    samples: session.samples.length,
+    criticalMoments: session.events.filter(e => e.type === 'context_critical').length
+  };
+
+  recordContextEvent({
+    type: 'context_session_ended',
+    feature: session.feature,
+    threadId: session.threadId,
+    data: summary
+  });
+
+  sessions.delete(sessionId);
+  return summary;
+}
+
+// ============================================================================
+// Optimization Analysis
+// ============================================================================
+
+/**
+ * Identify context optimization opportunities based on current utilization
+ * @param {Object} session - Session object
+ * @param {number} utilization - Current utilization (0-1)
+ * @returns {Array} Array of optimization recommendations
+ */
+export function identifyOptimizations(session, utilization) {
+  const recommendations = [];
+
+  if (utilization >= 0.5) {
+    recommendations.push({
+      strategy: OPTIMIZATION_STRATEGIES.MCP_HYGIENE,
+      description: 'Disable unused MCP servers to free ~15-30K tokens',
+      estimatedSavings: '15,000-30,000 tokens',
+      priority: utilization >= 0.75 ? 'high' : 'medium'
+    });
+  }
+
+  if (utilization >= 0.65) {
+    recommendations.push({
+      strategy: OPTIMIZATION_STRATEGIES.STANDARDS_SUBSET,
+      description: 'Load only task-relevant standards instead of all standards',
+      estimatedSavings: '10,000-20,000 tokens',
+      priority: 'medium'
+    });
+  }
+
+  if (utilization >= 0.80) {
+    recommendations.push({
+      strategy: OPTIMIZATION_STRATEGIES.MICRO_AGENT,
+      description: 'Switch to a micro-agent with minimal context for focused tasks',
+      estimatedSavings: '50,000-100,000 tokens',
+      priority: 'high'
+    });
+    recommendations.push({
+      strategy: OPTIMIZATION_STRATEGIES.PRIMING_FILE,
+      description: 'Use a 500-token priming file instead of full CLAUDE.md',
+      estimatedSavings: '22,000 tokens',
+      priority: 'high'
+    });
+  }
+
+  if (utilization >= 0.90) {
+    recommendations.push({
+      strategy: OPTIMIZATION_STRATEGIES.CONTEXT_BUNDLE,
+      description: 'Create a context bundle checkpoint (180K → 15K token compression)',
+      estimatedSavings: '165,000 tokens',
+      priority: 'critical'
+    });
+  }
+
+  return recommendations;
+}
+
+/**
+ * Get statistics for a session
+ * @param {string} sessionId - Session ID
+ * @returns {Object|null} Session statistics
+ */
+export function getSessionStats(sessionId) {
+  const session = sessions.get(sessionId);
+  if (!session) return null;
+
+  const avgUsage = session.samples.length > 0
+    ? Math.round(session.samples.reduce((sum, s) => sum + s.tokensUsed, 0) / session.samples.length)
+    : 0;
+
+  return {
+    sessionId,
+    feature: session.feature,
+    model: session.model,
+    contextWindow: session.contextWindow,
+    currentUsage: session.currentUsage,
+    peakUsage: session.peakUsage,
+    avgUsage,
+    peakUtilization: Math.round(session.peakUsage / session.contextWindow * 100),
+    currentUtilization: Math.round(session.currentUsage / session.contextWindow * 100),
+    samples: session.samples.length
+  };
+}
+
+/**
+ * Get all active sessions
+ * @returns {Array} Active session summaries
+ */
+export function getActiveSessions() {
+  return Array.from(sessions.values()).map(s => ({
+    sessionId: s.sessionId,
+    feature: s.feature,
+    model: s.model,
+    currentUsage: s.currentUsage,
+    peakUsage: s.peakUsage,
+    utilization: Math.round(s.currentUsage / s.contextWindow * 100)
+  }));
+}

package/src/lib/context/core-four-tracker.js

@@ -0,0 +1,201 @@
+/**
+ * Core Four Tracker — Context/Model/Prompt/Tools tracking
+ *
+ * Tracks the "Core Four" metrics that determine LLM behavior:
+ *   Context  — Token count breakdown by source
+ *   Model    — Model name and context window
+ *   Prompt   — Current request summary
+ *   Tools    — Tool usage counts (Read/Write/Edit/Bash/etc.)
+ *
+ * Reads from analytics-engine for historical data.
+ * Formats output for ASCII terminal display.
+ */
+
+import { readFileSync, existsSync } from 'fs';
+import { join } from 'path';
+import { recordContextEvent } from '../analytics/analytics-engine.js';
+
+// In-memory state for current session
+let currentSession = {
+  model: null,
+  contextWindow: 200000,
+  contextBreakdown: {},
+  promptSummary: '',
+  toolCalls: {}
+};
+
+// ============================================================================
+// Session Initialization
+// ============================================================================
+
+/**
+ * Initialize Core Four tracking for a session
+ * @param {Object} opts
+ * @param {string} opts.model - Model name
+ * @param {number} [opts.contextWindow] - Context window size
+ * @param {string} [opts.feature] - Feature being worked on
+ */
+export function initCoreFour({ model, contextWindow = 200000, feature = '' }) {
+  currentSession = {
+    model,
+    contextWindow,
+    feature,
+    contextBreakdown: {
+      'CLAUDE.md': 0,
+      standards: 0,
+      templates: 0,
+      spec: 0,
+      conversation: 0,
+      other: 0
+    },
+    promptSummary: '',
+    toolCalls: {
+      Read: 0,
+      Write: 0,
+      Edit: 0,
+      Bash: 0,
+      Glob: 0,
+      Grep: 0,
+      Task: 0,
+      other: 0
+    },
+    startedAt: new Date().toISOString()
+  };
+}
+
+// ============================================================================
+// Tracking Updates
+// ============================================================================
+
+/**
+ * Update context breakdown by source
+ * @param {Object} breakdown - { sourceName: tokenCount }
+ */
+export function updateContextBreakdown(breakdown) {
+  for (const [source, tokens] of Object.entries(breakdown)) {
+    currentSession.contextBreakdown[source] = tokens;
+  }
+}
+
+/**
+ * Update current prompt summary
+ * @param {string} summary - Brief summary of current request
+ */
+export function updatePromptSummary(summary) {
+  currentSession.promptSummary = summary;
+}
+
+/**
+ * Record a tool call
+ * @param {string} toolName - Tool name (Read|Write|Edit|Bash|etc.)
+ */
+export function recordToolCall(toolName) {
+  const key = currentSession.toolCalls.hasOwnProperty(toolName) ? toolName : 'other';
+  currentSession.toolCalls[key] = (currentSession.toolCalls[key] || 0) + 1;
+
+  if (currentSession.feature) {
+    recordContextEvent({
+      type: 'tool_call',
+      feature: currentSession.feature,
+      data: { tool: toolName }
+    });
+  }
+}
+
+// ============================================================================
+// Display Formatting
+// ============================================================================
+
+/**
+ * Get the current Core Four state
+ * @returns {Object} Core Four metrics
+ */
+export function getCoreFour() {
+  const totalContextTokens = Object.values(currentSession.contextBreakdown)
+    .reduce((sum, v) => sum + v, 0);
+
+  const contextUtilization = currentSession.contextWindow > 0
+    ? Math.round(totalContextTokens / currentSession.contextWindow * 100)
+    : 0;
+
+  return {
+    context: {
+      total: totalContextTokens,
+      window: currentSession.contextWindow,
+      utilization: contextUtilization,
+      breakdown: currentSession.contextBreakdown
+    },
+    model: {
+      name: currentSession.model || 'unknown',
+      contextWindow: currentSession.contextWindow,
+      provider: 'Anthropic'
+    },
+    prompt: {
+      summary: currentSession.promptSummary || '(no active prompt)',
+      length: currentSession.promptSummary?.length || 0
+    },
+    tools: {
+      calls: { ...currentSession.toolCalls },
+      total: Object.values(currentSession.toolCalls).reduce((sum, v) => sum + v, 0)
+    }
+  };
+}
+
+/**
+ * Format Core Four as ASCII display string
+ * @returns {string} ASCII formatted Core Four display
+ */
+export function formatCoreFour() {
+  const { context, model, prompt, tools } = getCoreFour();
+  const lines = [];
+
+  lines.push('╔══════════════════════════════════════════╗');
+  lines.push('║          CORE FOUR DASHBOARD             ║');
+  lines.push('╠══════════════════════════════════════════╣');
+
+  // Context
+  lines.push('║ CONTEXT                                  ║');
+  const bar = buildProgressBar(context.utilization, 30);
+  lines.push(`║  ${bar} ${String(context.utilization).padStart(3)}%          ║`);
+  lines.push(`║  Total: ${String(context.total.toLocaleString()).padEnd(10)} Window: ${String(context.window.toLocaleString()).padEnd(8)}║`);
+
+  // Breakdown
+  for (const [src, tokens] of Object.entries(context.breakdown)) {
+    if (tokens > 0) {
+      const pct = Math.round(tokens / Math.max(context.total, 1) * 100);
+      lines.push(`║    ${src.padEnd(14)} ${String(tokens.toLocaleString()).padStart(8)} (${String(pct).padStart(2)}%)     ║`);
+    }
+  }
+  lines.push('║                                          ║');
+
+  // Model
+  lines.push('║ MODEL                                    ║');
+  lines.push(`║  ${model.name.padEnd(40)}║`);
+  lines.push(`║  Window: ${String(model.contextWindow.toLocaleString()).padEnd(32)}║`);
+  lines.push('║                                          ║');
+
+  // Prompt
+  lines.push('║ PROMPT                                   ║');
+  const pStr = (prompt.summary || '—').substring(0, 38);
+  lines.push(`║  ${pStr.padEnd(40)}║`);
+  lines.push('║                                          ║');
+
+  // Tools
+  lines.push('║ TOOLS                                    ║');
+  lines.push(`║  Total calls: ${String(tools.total).padEnd(27)}║`);
+  for (const [tool, count] of Object.entries(tools.calls)) {
+    if (count > 0) {
+      lines.push(`║    ${tool.padEnd(10)} ${String(count).padStart(4)} calls${' '.repeat(20)}║`);
+    }
+  }
+  lines.push('╚══════════════════════════════════════════╝');
+
+  return lines.join('\n');
+}
+
+function buildProgressBar(pct, width) {
+  const filled = Math.round(pct / 100 * width);
+  const empty = width - filled;
+  const color = pct >= 90 ? '█' : pct >= 75 ? '▓' : '░';
+  return '[' + color.repeat(filled) + '░'.repeat(empty) + ']';
+}

package/src/lib/context/mcp-optimizer.js

@@ -0,0 +1,200 @@
+/**
+ * MCP Optimizer — Analyze and optimize MCP server configurations
+ *
+ * Reads .claude/settings.json (or .claude/settings.local.json),
+ * detects unused MCP servers based on analytics,
+ * estimates token savings per server, and generates recommendations.
+ */
+
+import { readFileSync, existsSync } from 'fs';
+import { join } from 'path';
+
+const SETTINGS_PATHS = [
+  join(process.cwd(), '.claude/settings.local.json'),
+  join(process.cwd(), '.claude/settings.json'),
+  join(process.env.USERPROFILE || process.env.HOME || '', '.claude/settings.json')
+];
+
+// Estimated token overhead per MCP server in context window
+const MCP_TOKEN_OVERHEAD = {
+  default: 3000,    // average per server
+  heavy: 8000,      // known heavy servers (filesystem, database)
+  light: 1500       // known light servers (simple tools)
+};
+
+const HEAVY_SERVERS = ['filesystem', 'postgres', 'sqlite', 'mysql', 'mongodb'];
+const LIGHT_SERVERS = ['time', 'echo', 'fetch'];
+
+// ============================================================================
+// Settings Loading
+// ============================================================================
+
+/**
+ * Load Claude settings file
+ * @returns {Object|null} Settings object or null
+ */
+export function loadClaudeSettings() {
+  for (const path of SETTINGS_PATHS) {
+    if (existsSync(path)) {
+      try {
+        return { settings: JSON.parse(readFileSync(path, 'utf8')), path };
+      } catch {
+        continue;
+      }
+    }
+  }
+  return null;
+}
+
+/**
+ * Extract MCP server list from settings
+ * @param {Object} settings - Claude settings object
+ * @returns {Array} Array of { name, config } objects
+ */
+export function extractMCPServers(settings) {
+  const servers = [];
+
+  // Claude settings format: mcpServers: { name: { command, args, env } }
+  if (settings.mcpServers) {
+    for (const [name, config] of Object.entries(settings.mcpServers)) {
+      servers.push({ name, config });
+    }
+  }
+
+  return servers;
+}
+
+// ============================================================================
+// Usage Analysis
+// ============================================================================
+
+/**
+ * Detect unused MCP servers by checking analytics logs
+ * @param {Array} servers - Array of MCP server descriptors
+ * @param {number} [lookbackDays=30] - Days to look back in analytics
+ * @returns {Array} servers with usage stats added
+ */
+export function detectUnusedServers(servers, lookbackDays = 30) {
+  // Read analytics for MCP tool usage patterns
+  const analyticsPath = join(process.cwd(), '.morph/analytics/threads-log.jsonl');
+  const usedServerNames = new Set();
+
+  if (existsSync(analyticsPath)) {
+    try {
+      const cutoff = new Date();
+      cutoff.setDate(cutoff.getDate() - lookbackDays);
+
+      const lines = readFileSync(analyticsPath, 'utf8').trim().split('\n').filter(Boolean);
+      for (const line of lines) {
+        try {
+          const event = JSON.parse(line);
+          if (event.timestamp && new Date(event.timestamp) > cutoff) {
+            // Check for MCP tool references in event data
+            const eventStr = JSON.stringify(event);
+            for (const server of servers) {
+              if (eventStr.includes(server.name)) {
+                usedServerNames.add(server.name);
+              }
+            }
+          }
+        } catch {
+          // Skip malformed lines
+        }
+      }
+    } catch {
+      // Analytics not available — mark all as unknown usage
+    }
+  }
+
+  return servers.map(server => {
+    const isHeavy = HEAVY_SERVERS.some(h => server.name.toLowerCase().includes(h));
+    const isLight = LIGHT_SERVERS.some(l => server.name.toLowerCase().includes(l));
+    const overhead = isHeavy ? MCP_TOKEN_OVERHEAD.heavy : isLight ? MCP_TOKEN_OVERHEAD.light : MCP_TOKEN_OVERHEAD.default;
+
+    const usageSeen = usedServerNames.has(server.name);
+
+    return {
+      ...server,
+      overhead,
+      usageSeen,
+      usageStatus: usedServerNames.size === 0 ? 'unknown' : (usageSeen ? 'used' : 'unused')
+    };
+  });
+}
+
+// ============================================================================
+// Recommendations
+// ============================================================================
+
+/**
+ * Analyze MCP configuration and generate optimization recommendations
+ * @returns {Object} Analysis results with recommendations
+ */
+export function analyzeMCP() {
+  const result = loadClaudeSettings();
+
+  if (!result) {
+    return {
+      found: false,
+      message: 'No Claude settings file found. Checked: .claude/settings.json and .claude/settings.local.json',
+      servers: [],
+      recommendations: []
+    };
+  }
+
+  const { settings, path: settingsPath } = result;
+  const servers = extractMCPServers(settings);
+
+  if (servers.length === 0) {
+    return {
+      found: true,
+      settingsPath,
+      servers: [],
+      totalOverhead: 0,
+      recommendations: [],
+      message: 'No MCP servers configured.'
+    };
+  }
+
+  const analyzedServers = detectUnusedServers(servers);
+  const totalOverhead = analyzedServers.reduce((sum, s) => sum + s.overhead, 0);
+
+  const unusedServers = analyzedServers.filter(s => s.usageStatus === 'unused');
+  const unusedOverhead = unusedServers.reduce((sum, s) => sum + s.overhead, 0);
+
+  const recommendations = [];
+
+  if (unusedServers.length > 0) {
+    recommendations.push({
+      type: 'disable_unused',
+      title: `Disable ${unusedServers.length} unused MCP server(s)`,
+      description: `${unusedServers.map(s => s.name).join(', ')} have not been used recently.`,
+      estimatedSavings: unusedOverhead,
+      servers: unusedServers.map(s => s.name),
+      action: 'Remove from mcpServers in .claude/settings.json'
+    });
+  }
+
+  const heavyServers = analyzedServers.filter(s => s.overhead >= MCP_TOKEN_OVERHEAD.heavy && s.usageStatus !== 'unused');
+  if (heavyServers.length > 0) {
+    recommendations.push({
+      type: 'defer_heavy',
+      title: `Consider deferring ${heavyServers.length} heavy MCP server(s)`,
+      description: `${heavyServers.map(s => s.name).join(', ')} use significant context window space.`,
+      estimatedSavings: heavyServers.reduce((sum, s) => sum + s.overhead, 0) / 2,
+      servers: heavyServers.map(s => s.name),
+      action: 'Only enable when needed for that specific task'
+    });
+  }
+
+  return {
+    found: true,
+    settingsPath,
+    servers: analyzedServers,
+    totalServers: servers.length,
+    totalOverhead,
+    unusedCount: unusedServers.length,
+    potentialSavings: unusedOverhead,
+    recommendations: recommendations.sort((a, b) => b.estimatedSavings - a.estimatedSavings)
+  };
+}