@polymorphism-tech/morph-spec 3.2.0 → 4.3.0

package/src/lib/analysis/index.js

@@ -0,0 +1,7 @@
+/**
+ * Analysis Utilities
+ *
+ * Tools for analyzing code complexity, feature scope, and project metrics.
+ */
+
+export * from './complexity-analyzer.js';

package/src/lib/analytics/analytics-engine.js

@@ -0,0 +1,345 @@
+/**
+ * Analytics Engine — Metrics collection and JSONL storage
+ *
+ * Records events to append-only JSONL files:
+ * - .morph/analytics/threads-log.jsonl
+ * - .morph/analytics/context-log.jsonl
+ * - .morph/analytics/trust-log.jsonl
+ *
+ * Provides aggregation, ASCII chart generation, and 30-day dashboards.
+ * Auto-prunes entries older than 90 days.
+ */
+
+import { readFileSync, writeFileSync, appendFileSync, existsSync, mkdirSync } from 'fs';
+import { join } from 'path';
+
+const ANALYTICS_DIR = join(process.cwd(), '.morph/analytics');
+const THREADS_LOG = join(ANALYTICS_DIR, 'threads-log.jsonl');
+const CONTEXT_LOG = join(ANALYTICS_DIR, 'context-log.jsonl');
+const TRUST_LOG = join(ANALYTICS_DIR, 'trust-log.jsonl');
+const PRUNE_DAYS = 90;
+const DASHBOARD_DAYS = 30;
+
+// ============================================================================
+// Internal Helpers
+// ============================================================================
+
+function ensureAnalyticsDir() {
+  if (!existsSync(ANALYTICS_DIR)) {
+    mkdirSync(ANALYTICS_DIR, { recursive: true });
+  }
+}
+
+function appendJSONL(filePath, record) {
+  ensureAnalyticsDir();
+  appendFileSync(filePath, JSON.stringify(record) + '\n', 'utf8');
+}
+
+function readJSONL(filePath) {
+  if (!existsSync(filePath)) return [];
+
+  const lines = readFileSync(filePath, 'utf8').trim().split('\n').filter(Boolean);
+  const records = [];
+
+  for (const line of lines) {
+    try {
+      records.push(JSON.parse(line));
+    } catch {
+      // Skip malformed lines
+    }
+  }
+  return records;
+}
+
+function pruneJSONL(filePath) {
+  if (!existsSync(filePath)) return;
+
+  const cutoff = new Date();
+  cutoff.setDate(cutoff.getDate() - PRUNE_DAYS);
+
+  const records = readJSONL(filePath);
+  const fresh = records.filter(r => {
+    const ts = new Date(r.timestamp || r.createdAt || 0);
+    return ts > cutoff;
+  });
+
+  if (fresh.length < records.length) {
+    writeFileSync(filePath, fresh.map(r => JSON.stringify(r)).join('\n') + '\n', 'utf8');
+  }
+}
+
+function recentRecords(records, days = DASHBOARD_DAYS) {
+  const cutoff = new Date();
+  cutoff.setDate(cutoff.getDate() - days);
+  return records.filter(r => {
+    const ts = new Date(r.timestamp || r.createdAt || 0);
+    return ts > cutoff;
+  });
+}
+
+// ============================================================================
+// Event Recording
+// ============================================================================
+
+/**
+ * Record a thread analytics event
+ * @param {Object} event
+ * @param {string} event.type - Event type (thread_created|thread_completed|thread_failed|checkpoint_passed|etc.)
+ * @param {string} event.feature - Feature name
+ * @param {string} [event.threadId] - Thread ID
+ * @param {string} [event.agent] - Agent name
+ * @param {Object} [event.data] - Additional event data
+ */
+export function recordEvent(event) {
+  const record = {
+    ...event,
+    timestamp: new Date().toISOString()
+  };
+
+  // Route to appropriate log
+  if (event.type?.startsWith('context_') || event.type === 'token_usage') {
+    appendJSONL(CONTEXT_LOG, record);
+  } else if (event.type?.startsWith('trust_')) {
+    appendJSONL(TRUST_LOG, record);
+  } else {
+    appendJSONL(THREADS_LOG, record);
+  }
+}
+
+/**
+ * Record a context event (token usage, optimization, etc.)
+ * @param {Object} event
+ */
+export function recordContextEvent(event) {
+  recordEvent({ ...event, type: event.type || 'context_usage' });
+}
+
+/**
+ * Record a trust level change event
+ * @param {string} feature
+ * @param {string} level - new trust level
+ * @param {string} reason
+ */
+export function recordTrustEvent(feature, level, reason = '') {
+  appendJSONL(TRUST_LOG, {
+    timestamp: new Date().toISOString(),
+    type: 'trust_level_changed',
+    feature,
+    level,
+    reason
+  });
+}
+
+/**
+ * Record auto-approval event (gate approved by trust)
+ * @param {string} feature
+ * @param {string} gate - 'design' | 'tasks' | 'proposal'
+ * @param {string} trustLevel
+ * @param {number} passRate
+ */
+export function recordAutoApproval(feature, gate, trustLevel, passRate) {
+  appendJSONL(TRUST_LOG, {
+    timestamp: new Date().toISOString(),
+    type: 'trust_auto_approved',
+    feature,
+    gate,
+    trustLevel,
+    passRate,
+    timeSavedMinutes: 5 // estimated minutes saved per auto-approval
+  });
+}
+
+/**
+ * Get trust progression for a feature
+ * @param {string} featureName
+ * @returns {Array} Trust level history
+ */
+export function getTrustProgression(featureName) {
+  const events = readJSONL(TRUST_LOG)
+    .filter(e => e.feature === featureName && e.type === 'trust_level_changed')
+    .sort((a, b) => new Date(a.timestamp) - new Date(b.timestamp));
+  return events;
+}
+
+// ============================================================================
+// Feature Analytics
+// ============================================================================
+
+/**
+ * Get analytics for a specific feature
+ * @param {string} feature - Feature name
+ * @returns {Object} Feature analytics
+ */
+export function getFeatureAnalytics(feature) {
+  const allThreadEvents = readJSONL(THREADS_LOG).filter(r => r.feature === feature);
+  const allContextEvents = readJSONL(CONTEXT_LOG).filter(r => r.feature === feature);
+
+  const threadsByStatus = allThreadEvents
+    .filter(e => e.type === 'thread_completed' || e.type === 'thread_failed')
+    .reduce((acc, e) => {
+      const status = e.type === 'thread_completed' ? 'completed' : 'failed';
+      acc[status] = (acc[status] || 0) + 1;
+      return acc;
+    }, {});
+
+  const checkpoints = allThreadEvents.filter(e =>
+    e.type === 'checkpoint_passed' || e.type === 'checkpoint_failed'
+  );
+  const checkpointPassRate = checkpoints.length > 0
+    ? Math.round(checkpoints.filter(e => e.type === 'checkpoint_passed').length / checkpoints.length * 100)
+    : 100;
+
+  const totalDuration = allThreadEvents
+    .filter(e => e.type === 'thread_completed' && e.data?.duration)
+    .reduce((sum, e) => sum + e.data.duration, 0);
+
+  const toolCallEvents = allThreadEvents.filter(e => e.type === 'tool_call');
+  const tokenEvents = allContextEvents.filter(e => e.data?.tokensUsed);
+  const totalTokens = tokenEvents.reduce((sum, e) => sum + (e.data.tokensUsed || 0), 0);
+
+  return {
+    feature,
+    threads: threadsByStatus,
+    checkpointPassRate,
+    totalDuration,
+    totalToolCalls: toolCallEvents.length,
+    totalTokensUsed: totalTokens,
+    optimizationEvents: allContextEvents.filter(e => e.type === 'context_optimized').length
+  };
+}
+
+/**
+ * Get project-wide analytics dashboard (30-day summary)
+ * @returns {Object} Project analytics
+ */
+export function getProjectAnalytics() {
+  const allThreadEvents = recentRecords(readJSONL(THREADS_LOG));
+  const allContextEvents = recentRecords(readJSONL(CONTEXT_LOG));
+  const allTrustEvents = recentRecords(readJSONL(TRUST_LOG));
+
+  const features = [...new Set(allThreadEvents.map(e => e.feature).filter(Boolean))];
+
+  const completedThreads = allThreadEvents.filter(e => e.type === 'thread_completed');
+  const failedThreads = allThreadEvents.filter(e => e.type === 'thread_failed');
+  const checkpoints = allThreadEvents.filter(e =>
+    e.type === 'checkpoint_passed' || e.type === 'checkpoint_failed'
+  );
+
+  const parallelEvents = allThreadEvents.filter(e => e.data?.type === 'parallel');
+  const avgParallel = parallelEvents.length > 0
+    ? parallelEvents.reduce((sum, e) => sum + (e.data?.concurrency || 1), 0) / parallelEvents.length
+    : 1;
+
+  const autoApprovalEvents = allTrustEvents.filter(e => e.type === 'trust_auto_approved');
+  const avgTokens = allContextEvents.filter(e => e.data?.tokensUsed).length > 0
+    ? Math.round(allContextEvents.reduce((sum, e) => sum + (e.data?.tokensUsed || 0), 0) / allContextEvents.filter(e => e.data?.tokensUsed).length)
+    : 0;
+
+  return {
+    period: `${DASHBOARD_DAYS} days`,
+    features: features.length,
+    threads: {
+      total: completedThreads.length + failedThreads.length,
+      completed: completedThreads.length,
+      failed: failedThreads.length
+    },
+    checkpoints: {
+      total: checkpoints.length,
+      passed: checkpoints.filter(e => e.type === 'checkpoint_passed').length,
+      passRate: checkpoints.length > 0
+        ? Math.round(checkpoints.filter(e => e.type === 'checkpoint_passed').length / checkpoints.length * 100)
+        : 100
+    },
+    parallelization: {
+      avgConcurrency: Math.round(avgParallel * 10) / 10,
+      parallelThreads: parallelEvents.length
+    },
+    trust: {
+      autoApprovals: autoApprovalEvents.length,
+      trustChanges: allTrustEvents.filter(e => e.type === 'trust_level_changed').length
+    },
+    context: {
+      avgTokensPerSession: avgTokens,
+      optimizations: allContextEvents.filter(e => e.type === 'context_optimized').length
+    }
+  };
+}
+
+// ============================================================================
+// ASCII Charts
+// ============================================================================
+
+/**
+ * Generate an ASCII bar chart
+ * @param {Object} data - { label: value }
+ * @param {Object} [opts]
+ * @param {number} [opts.width=40] - Chart width in chars
+ * @param {string} [opts.title] - Chart title
+ * @returns {string} ASCII chart string
+ */
+export function generateAsciiChart(data, { width = 40, title = '' } = {}) {
+  const entries = Object.entries(data);
+  if (entries.length === 0) return '  (no data)';
+
+  const maxValue = Math.max(...entries.map(([, v]) => v), 1);
+  const maxLabelLen = Math.max(...entries.map(([k]) => k.length));
+  const lines = [];
+
+  if (title) {
+    lines.push(`  ${title}`);
+    lines.push('  ' + '─'.repeat(width + maxLabelLen + 5));
+  }
+
+  for (const [label, value] of entries) {
+    const barLen = Math.round((value / maxValue) * width);
+    const bar = '█'.repeat(barLen);
+    const paddedLabel = label.padStart(maxLabelLen);
+    lines.push(`  ${paddedLabel} │${bar} ${value}`);
+  }
+
+  return lines.join('\n');
+}
+
+/**
+ * Generate thread timeline ASCII chart
+ * @param {Array} events - Array of thread events with timestamps
+ * @param {string} label - Chart label
+ * @returns {string} ASCII timeline
+ */
+export function generateTimelineChart(events, label = 'Timeline') {
+  if (events.length === 0) return '  (no data)';
+
+  const now = new Date();
+  const days = Array.from({ length: 7 }, (_, i) => {
+    const d = new Date(now);
+    d.setDate(d.getDate() - (6 - i));
+    return d.toLocaleDateString('en-US', { weekday: 'short' });
+  });
+
+  const counts = new Array(7).fill(0);
+  for (const event of events) {
+    const ts = new Date(event.timestamp);
+    const daysAgo = Math.floor((now - ts) / (1000 * 60 * 60 * 24));
+    if (daysAgo < 7) {
+      counts[6 - daysAgo]++;
+    }
+  }
+
+  const data = {};
+  days.forEach((d, i) => { data[d] = counts[i]; });
+
+  return generateAsciiChart(data, { title: label });
+}
+
+// ============================================================================
+// Maintenance
+// ============================================================================
+
+/**
+ * Prune all analytics logs (remove entries older than 90 days)
+ */
+export function pruneAnalytics() {
+  pruneJSONL(THREADS_LOG);
+  pruneJSONL(CONTEXT_LOG);
+  pruneJSONL(TRUST_LOG);
+}

package/src/lib/{checkpoint-hooks.js → checkpoints/checkpoint-hooks.js}

@@ -1,6 +1,8 @@
 import { readFileSync, existsSync } from 'fs';
 import { join } from 'path';
 import { execSync } from 'child_process';
+import { executeStopHooks, isMaxRetriesExceeded, incrementRetryCount } from '../hooks/stop-hook-executor.js';
+import { recordEvent } from '../analytics/analytics-engine.js';
 
 /**
  * Checkpoint Hooks - Automated validation orchestration
@@ -224,11 +226,44 @@ export async function runCheckpointHooks(featureName, checkpointNum) {
 
   console.log('\n' + '━'.repeat(60) + '\n');
 
+  // Run stop hooks for L-Thread support
+  let stopHookResult = null;
+  if (featureName) {
+    try {
+      // Use feature name as pseudo thread ID for stop hook feedback
+      const pseudoThreadId = `checkpoint-${featureName}-${checkpointNum}`;
+      stopHookResult = await executeStopHooks(pseudoThreadId, 'on-stop', { feature: featureName });
+
+      if (!stopHookResult.passed) {
+        // Check retry count
+        if (isMaxRetriesExceeded(pseudoThreadId)) {
+          console.log(chalk.red('\n❌ Stop hooks failed 5+ times. Escalating to user.'));
+        } else {
+          const retryCount = incrementRetryCount(pseudoThreadId);
+          console.log(chalk.yellow(`\n⚠ Stop hooks failed (attempt ${retryCount}/5). Feedback saved.`));
+          if (stopHookResult.feedback?.suggestions?.length > 0) {
+            stopHookResult.feedback.suggestions.forEach(s => console.log(`  → ${s}`));
+          }
+        }
+
+        // Record to analytics
+        recordEvent({
+          type: 'stop_hook_failed_at_checkpoint',
+          feature: featureName,
+          data: { checkpointNum, retryCount: getRetryCount ? undefined : undefined }
+        });
+      }
+    } catch {
+      // Stop hooks are non-critical — don't block checkpoint if they error
+    }
+  }
+
   return {
     passed,
     checkpointNum,
     timestamp: new Date().toISOString(),
     results,
+    stopHookResult,
     summary: {
       errors: errorCount,
       warnings: warningCount,

package/src/lib/checkpoints/index.js

@@ -0,0 +1,7 @@
+/**
+ * Checkpoint System
+ *
+ * Automatic validation hooks triggered at checkpoints during feature implementation.
+ */
+
+export * from './checkpoint-hooks.js';

package/src/lib/context/context-bundler.js

@@ -0,0 +1,240 @@
+/**
+ * Context Bundler — Checkpoint compression (180K → 15K)
+ *
+ * Compresses checkpoint state into a compact bundle for session resumption.
+ * Saves bundles to .morph/bundles/{bundleId}.json
+ *
+ * A bundle contains:
+ *   - decisions[]: key architectural decisions made
+ *   - artifacts[]: paths to important generated files
+ *   - nextSteps[]: what to implement next
+ *   - metadata: original vs bundled token counts
+ */
+
+import { readFileSync, writeFileSync, existsSync, mkdirSync, readdirSync } from 'fs';
+import { join } from 'path';
+import { randomUUID } from 'crypto';
+
+const BUNDLES_DIR = join(process.cwd(), '.morph/bundles');
+
+// ============================================================================
+// Bundle Creation
+// ============================================================================
+
+/**
+ * Create a context bundle from checkpoint data
+ * @param {Object} opts
+ * @param {string} opts.feature - Feature name
+ * @param {number} opts.checkpointNum - Checkpoint number this bundle captures
+ * @param {Array} opts.decisions - Key decisions made: [{ id, title, summary }]
+ * @param {Array} opts.artifacts - Important file paths: [{ path, description }]
+ * @param {Array} opts.nextSteps - Next tasks to implement: [string]
+ * @param {number} [opts.originalTokens=180000] - Estimated original context size
+ * @param {Object} [opts.meta] - Additional metadata
+ * @returns {Object} Bundle object with ID
+ */
+export function createBundle({
+  feature,
+  checkpointNum,
+  decisions = [],
+  artifacts = [],
+  nextSteps = [],
+  originalTokens = 180000,
+  meta = {}
+}) {
+  const bundleId = `${feature}-cp${checkpointNum}-${randomUUID().substring(0, 8)}`;
+  const now = new Date().toISOString();
+
+  // Estimate bundle size (decisions + artifacts + nextSteps summary)
+  const bundledContent = JSON.stringify({ decisions, artifacts, nextSteps, meta });
+  const bundledTokens = Math.round(bundledContent.length / 4); // ~4 chars/token
+
+  const bundle = {
+    bundleId,
+    feature,
+    checkpointNum,
+    createdAt: now,
+    decisions,
+    artifacts,
+    nextSteps,
+    meta,
+    tokenMetrics: {
+      originalTokens,
+      bundledTokens,
+      compressionRatio: Math.round(originalTokens / Math.max(bundledTokens, 1)),
+      savingsTokens: originalTokens - bundledTokens
+    },
+    resumeInstructions: buildResumeInstructions(feature, checkpointNum, decisions, nextSteps)
+  };
+
+  return bundle;
+}
+
+/**
+ * Save a bundle to disk
+ * @param {Object} bundle - Bundle object (from createBundle)
+ * @returns {string} Path to saved bundle file
+ */
+export function saveBundle(bundle) {
+  if (!existsSync(BUNDLES_DIR)) {
+    mkdirSync(BUNDLES_DIR, { recursive: true });
+  }
+
+  const filePath = join(BUNDLES_DIR, `${bundle.bundleId}.json`);
+  writeFileSync(filePath, JSON.stringify(bundle, null, 2), 'utf8');
+
+  // Also update feature state with bundle reference
+  try {
+    const statePath = join(process.cwd(), '.morph/state.json');
+    if (existsSync(statePath)) {
+      const state = JSON.parse(readFileSync(statePath, 'utf8'));
+      if (state.features?.[bundle.feature]) {
+        state.features[bundle.feature].contextBundles = state.features[bundle.feature].contextBundles || [];
+        state.features[bundle.feature].contextBundles.push({
+          bundleId: bundle.bundleId,
+          checkpointNum: bundle.checkpointNum,
+          createdAt: bundle.createdAt,
+          originalTokens: bundle.tokenMetrics.originalTokens,
+          bundledTokens: bundle.tokenMetrics.bundledTokens,
+          path: filePath
+        });
+        state.metadata = state.metadata || {};
+        state.metadata.lastUpdated = new Date().toISOString();
+        writeFileSync(statePath, JSON.stringify(state, null, 2), 'utf8');
+      }
+    }
+  } catch {
+    // Non-fatal if state update fails
+  }
+
+  return filePath;
+}
+
+/**
+ * Load a bundle from disk
+ * @param {string} bundleId - Bundle ID
+ * @returns {Object|null} Bundle object or null if not found
+ */
+export function loadBundle(bundleId) {
+  const filePath = join(BUNDLES_DIR, `${bundleId}.json`);
+
+  if (!existsSync(filePath)) {
+    return null;
+  }
+
+  try {
+    return JSON.parse(readFileSync(filePath, 'utf8'));
+  } catch {
+    return null;
+  }
+}
+
+/**
+ * List all bundles, optionally filtered by feature
+ * @param {string} [feature] - Filter by feature name
+ * @returns {Array} Array of bundle summaries
+ */
+export function listBundles(feature = null) {
+  if (!existsSync(BUNDLES_DIR)) return [];
+
+  const files = readdirSync(BUNDLES_DIR).filter(f => f.endsWith('.json'));
+  const bundles = [];
+
+  for (const file of files) {
+    try {
+      const bundle = JSON.parse(readFileSync(join(BUNDLES_DIR, file), 'utf8'));
+      if (!feature || bundle.feature === feature) {
+        bundles.push({
+          bundleId: bundle.bundleId,
+          feature: bundle.feature,
+          checkpointNum: bundle.checkpointNum,
+          createdAt: bundle.createdAt,
+          originalTokens: bundle.tokenMetrics?.originalTokens,
+          bundledTokens: bundle.tokenMetrics?.bundledTokens,
+          compressionRatio: bundle.tokenMetrics?.compressionRatio,
+          nextStepsCount: bundle.nextSteps?.length || 0
+        });
+      }
+    } catch {
+      // Skip malformed files
+    }
+  }
+
+  return bundles.sort((a, b) => new Date(b.createdAt) - new Date(a.createdAt));
+}
+
+/**
+ * Get the latest bundle for a feature at or before a checkpoint number
+ * @param {string} feature - Feature name
+ * @param {number} [checkpointNum] - Target checkpoint (latest if not specified)
+ * @returns {Object|null} Bundle or null
+ */
+export function getLatestBundle(feature, checkpointNum = Infinity) {
+  const bundles = listBundles(feature)
+    .filter(b => b.checkpointNum <= checkpointNum)
+    .sort((a, b) => b.checkpointNum - a.checkpointNum);
+
+  if (bundles.length === 0) return null;
+  return loadBundle(bundles[0].bundleId);
+}
+
+// ============================================================================
+// Session Replay
+// ============================================================================
+
+/**
+ * Generate minimal context for session replay from a bundle
+ * @param {Object} bundle - Bundle object
+ * @returns {string} Compact context string (~15K tokens) for session resumption
+ */
+export function generateResumeContext(bundle) {
+  const lines = [];
+  lines.push(`# Session Resume — ${bundle.feature} @ Checkpoint ${bundle.checkpointNum}`);
+  lines.push(`> Bundle ID: ${bundle.bundleId} | Created: ${bundle.createdAt}`);
+  lines.push(`> Context compression: ${bundle.tokenMetrics?.originalTokens?.toLocaleString()} → ${bundle.tokenMetrics?.bundledTokens?.toLocaleString()} tokens`);
+  lines.push('');
+
+  if (bundle.decisions?.length > 0) {
+    lines.push('## Key Decisions Made');
+    for (const d of bundle.decisions) {
+      lines.push(`- **${d.id || ''}${d.id ? ':' : ''} ${d.title}** — ${d.summary}`);
+      if (d.fullPath) lines.push(`  *(Full: ${d.fullPath})*`);
+    }
+    lines.push('');
+  }
+
+  if (bundle.artifacts?.length > 0) {
+    lines.push('## Created Artifacts');
+    for (const a of bundle.artifacts) {
+      lines.push(`- \`${a.path}\` — ${a.description || ''}`);
+    }
+    lines.push('');
+  }
+
+  if (bundle.nextSteps?.length > 0) {
+    lines.push('## Next Steps (continue from here)');
+    bundle.nextSteps.forEach((step, i) => {
+      lines.push(`${i + 1}. ${step}`);
+    });
+    lines.push('');
+  }
+
+  lines.push(bundle.resumeInstructions || '');
+
+  return lines.join('\n');
+}
+
+// ============================================================================
+// Helpers
+// ============================================================================
+
+function buildResumeInstructions(feature, checkpointNum, decisions, nextSteps) {
+  return `## Resume Instructions
+
+1. Load minimal standards: run \`morph-spec prime feature\`
+2. Read spec: \`.morph/project/outputs/${feature}/spec.md\` (key sections only)
+3. Check tasks: \`morph-spec state get ${feature}\`
+4. Continue from: ${nextSteps[0] || `task after checkpoint ${checkpointNum}`}
+
+> This bundle was created at checkpoint ${checkpointNum}. All decisions above are final.`;
+}