@stackmemoryai/stackmemory 0.8.0 → 1.0.0

package/scripts/testing/collect-metrics.ts

@@ -1,457 +0,0 @@
-#!/usr/bin/env node
-import { Database } from '../../src/core/database/database.js';
-import { FrameManager } from '../../src/core/frame/frame-manager.js';
-import { SessionManager } from '../../src/core/context/session-manager.js';
-import { ContextRetriever } from '../../src/core/retrieval/context-retriever.js';
-import { performance } from 'perf_hooks';
-import * as fs from 'fs/promises';
-import * as path from 'path';
-
-interface SessionMetrics {
-  sessionId: string;
-  variant: 'with_stackmemory' | 'without_stackmemory';
-  startTime: Date;
-  endTime?: Date;
-  contextReestablishmentTime: number;
-  toolCalls: number;
-  framesCreated: number;
-  framesClosedProperly: number;
-  decisionsAnchored: number;
-  errorsEncountered: number;
-  completionTime: number;
-  reworkInstances: number;
-  contextRetrievals: number;
-  contextRelevanceScores: number[];
-  memoryUsage: number;
-  tokenUsage: number;
-}
-
-interface ComparisonReport {
-  improvement: {
-    contextSpeed: number;
-    taskCompletion: number;
-    errorRecovery: number;
-    consistency: number;
-  };
-  statistics: {
-    sampleSize: number;
-    confidence: number;
-    pValue: number;
-  };
-  recommendations: string[];
-}
-
-export class MetricsCollector {
-  private db: Database;
-  private frameManager: FrameManager;
-  private sessionManager: SessionManager;
-  private retriever: ContextRetriever;
-  private metrics: Map<string, SessionMetrics> = new Map();
-
-  constructor() {
-    this.db = Database.getInstance();
-    this.frameManager = FrameManager.getInstance();
-    this.sessionManager = SessionManager.getInstance();
-    this.retriever = new ContextRetriever();
-  }
-
-  async initialize(): Promise<void> {
-    await this.db.initialize();
-  }
-
-  async startSession(
-    variant: 'with_stackmemory' | 'without_stackmemory'
-  ): Promise<string> {
-    const sessionId = `test-${variant}-${Date.now()}`;
-
-    if (variant === 'with_stackmemory') {
-      await this.sessionManager.createSession(sessionId);
-    }
-
-    this.metrics.set(sessionId, {
-      sessionId,
-      variant,
-      startTime: new Date(),
-      contextReestablishmentTime: 0,
-      toolCalls: 0,
-      framesCreated: 0,
-      framesClosedProperly: 0,
-      decisionsAnchored: 0,
-      errorsEncountered: 0,
-      completionTime: 0,
-      reworkInstances: 0,
-      contextRetrievals: 0,
-      contextRelevanceScores: [],
-      memoryUsage: process.memoryUsage().heapUsed,
-      tokenUsage: 0,
-    });
-
-    return sessionId;
-  }
-
-  async measureContextReestablishment(sessionId: string): Promise<number> {
-    const start = performance.now();
-    const metrics = this.metrics.get(sessionId);
-
-    if (!metrics) throw new Error(`Session ${sessionId} not found`);
-
-    if (metrics.variant === 'with_stackmemory') {
-      // Measure time to retrieve context
-      const context = await this.retriever.getRelevantContext(
-        'continue previous work',
-        10000
-      );
-      const duration = performance.now() - start;
-      metrics.contextReestablishmentTime = duration;
-      metrics.contextRetrievals++;
-      metrics.tokenUsage += context.totalTokens || 0;
-      return duration;
-    } else {
-      // Simulate manual context reestablishment
-      const simulatedTime = 300000; // 5 minutes
-      metrics.contextReestablishmentTime = simulatedTime;
-      return simulatedTime;
-    }
-  }
-
-  trackToolCall(sessionId: string, _toolName: string): void {
-    const metrics = this.metrics.get(sessionId);
-    if (metrics) {
-      metrics.toolCalls++;
-    }
-  }
-
-  trackFrameCreation(sessionId: string, _frameId: string): void {
-    const metrics = this.metrics.get(sessionId);
-    if (metrics) {
-      metrics.framesCreated++;
-    }
-  }
-
-  trackFrameClosure(
-    sessionId: string,
-    _frameId: string,
-    properClosure: boolean
-  ): void {
-    const metrics = this.metrics.get(sessionId);
-    if (metrics && properClosure) {
-      metrics.framesClosedProperly++;
-    }
-  }
-
-  trackDecision(sessionId: string, _decision: string): void {
-    const metrics = this.metrics.get(sessionId);
-    if (metrics) {
-      metrics.decisionsAnchored++;
-    }
-  }
-
-  trackError(sessionId: string, _error: Error): void {
-    const metrics = this.metrics.get(sessionId);
-    if (metrics) {
-      metrics.errorsEncountered++;
-    }
-  }
-
-  trackRework(sessionId: string): void {
-    const metrics = this.metrics.get(sessionId);
-    if (metrics) {
-      metrics.reworkInstances++;
-    }
-  }
-
-  async scoreContextRelevance(
-    sessionId: string,
-    _query: string,
-    _retrievedContext: unknown
-  ): Promise<number> {
-    // In real implementation, this would use LLM to score relevance
-    const score = Math.random() * 0.3 + 0.7; // Mock: 0.7-1.0 range
-
-    const metrics = this.metrics.get(sessionId);
-    if (metrics) {
-      metrics.contextRelevanceScores.push(score);
-    }
-
-    return score;
-  }
-
-  async endSession(sessionId: string): Promise<SessionMetrics> {
-    const metrics = this.metrics.get(sessionId);
-    if (!metrics) throw new Error(`Session ${sessionId} not found`);
-
-    metrics.endTime = new Date();
-    metrics.completionTime =
-      metrics.endTime.getTime() - metrics.startTime.getTime();
-    metrics.memoryUsage = process.memoryUsage().heapUsed - metrics.memoryUsage;
-
-    return metrics;
-  }
-
-  async collectSessionMetrics(sessionId: string): Promise<SessionMetrics> {
-    const metrics = this.metrics.get(sessionId);
-    if (!metrics) throw new Error(`Session ${sessionId} not found`);
-
-    // Collect additional metrics from database
-    if (metrics.variant === 'with_stackmemory') {
-      const frames = await this.db.query(
-        'SELECT * FROM frames WHERE session_id = ?',
-        [sessionId]
-      );
-
-      const events = await this.db.query(
-        'SELECT * FROM events WHERE session_id = ?',
-        [sessionId]
-      );
-
-      const anchors = await this.db.query(
-        'SELECT * FROM anchors WHERE session_id = ?',
-        [sessionId]
-      );
-
-      metrics.framesCreated = frames.length;
-      metrics.framesClosedProperly = frames.filter(
-        (f: { state: string }) => f.state === 'closed'
-      ).length;
-      metrics.decisionsAnchored = anchors.length;
-      metrics.toolCalls = events.filter(
-        (e: { type: string }) => e.type === 'tool_call'
-      ).length;
-      metrics.errorsEncountered = events.filter(
-        (e: { type: string }) => e.type === 'error'
-      ).length;
-    }
-
-    return metrics;
-  }
-
-  async compareVariants(
-    withStackMemory: SessionMetrics[],
-    withoutStackMemory: SessionMetrics[]
-  ): Promise<ComparisonReport> {
-    // Calculate improvements
-    const avgWith = this.calculateAverages(withStackMemory);
-    const avgWithout = this.calculateAverages(withoutStackMemory);
-
-    const contextSpeedImprovement =
-      ((avgWithout.contextReestablishmentTime -
-        avgWith.contextReestablishmentTime) /
-        avgWithout.contextReestablishmentTime) *
-      100;
-
-    const taskCompletionImprovement =
-      ((avgWithout.completionTime - avgWith.completionTime) /
-        avgWithout.completionTime) *
-      100;
-
-    const errorRecoveryImprovement =
-      ((avgWithout.errorsEncountered - avgWith.errorsEncountered) /
-        Math.max(avgWithout.errorsEncountered, 1)) *
-      100;
-
-    const consistencyImprovement =
-      ((avgWith.decisionsAnchored - avgWithout.decisionsAnchored) /
-        Math.max(avgWithout.decisionsAnchored, 1)) *
-      100;
-
-    // Calculate statistical significance (simplified)
-    const pValue = this.calculatePValue(withStackMemory, withoutStackMemory);
-    const confidence = (1 - pValue) * 100;
-
-    return {
-      improvement: {
-        contextSpeed: contextSpeedImprovement,
-        taskCompletion: taskCompletionImprovement,
-        errorRecovery: errorRecoveryImprovement,
-        consistency: consistencyImprovement,
-      },
-      statistics: {
-        sampleSize: withStackMemory.length + withoutStackMemory.length,
-        confidence,
-        pValue,
-      },
-      recommendations: this.generateRecommendations({
-        contextSpeedImprovement,
-        taskCompletionImprovement,
-        errorRecoveryImprovement,
-        consistencyImprovement,
-      }),
-    };
-  }
-
-  private calculateAverages(
-    metrics: SessionMetrics[]
-  ): Partial<SessionMetrics> {
-    const sum = metrics.reduce(
-      (acc, m) => ({
-        contextReestablishmentTime:
-          acc.contextReestablishmentTime + m.contextReestablishmentTime,
-        completionTime: acc.completionTime + m.completionTime,
-        errorsEncountered: acc.errorsEncountered + m.errorsEncountered,
-        decisionsAnchored: acc.decisionsAnchored + m.decisionsAnchored,
-        reworkInstances: acc.reworkInstances + m.reworkInstances,
-      }),
-      {
-        contextReestablishmentTime: 0,
-        completionTime: 0,
-        errorsEncountered: 0,
-        decisionsAnchored: 0,
-        reworkInstances: 0,
-      }
-    );
-
-    return {
-      contextReestablishmentTime:
-        sum.contextReestablishmentTime / metrics.length,
-      completionTime: sum.completionTime / metrics.length,
-      errorsEncountered: sum.errorsEncountered / metrics.length,
-      decisionsAnchored: sum.decisionsAnchored / metrics.length,
-      reworkInstances: sum.reworkInstances / metrics.length,
-    };
-  }
-
-  private calculatePValue(
-    _group1: SessionMetrics[],
-    _group2: SessionMetrics[]
-  ): number {
-    // Simplified t-test calculation
-    // In production, use proper statistical library
-    return 0.02; // Mock significant result
-  }
-
-  private generateRecommendations(
-    improvements: Record<string, number>
-  ): string[] {
-    const recommendations: string[] = [];
-
-    if (improvements.contextSpeedImprovement > 50) {
-      recommendations.push(
-        'StackMemory significantly reduces context reestablishment time'
-      );
-    }
-
-    if (improvements.taskCompletionImprovement > 30) {
-      recommendations.push('Tasks complete faster with StackMemory enabled');
-    }
-
-    if (improvements.errorRecoveryImprovement > 20) {
-      recommendations.push(
-        'Error recovery is more efficient with saved context'
-      );
-    }
-
-    if (improvements.consistencyImprovement > 40) {
-      recommendations.push(
-        'Decision consistency greatly improved with anchored context'
-      );
-    }
-
-    return recommendations;
-  }
-
-  async saveMetrics(
-    sessionId: string,
-    outputDir: string = './scripts/testing/results'
-  ): Promise<void> {
-    const metrics = this.metrics.get(sessionId);
-    if (!metrics) return;
-
-    await fs.mkdir(outputDir, { recursive: true });
-    const filename = path.join(outputDir, `${sessionId}.json`);
-    await fs.writeFile(filename, JSON.stringify(metrics, null, 2));
-  }
-
-  async generateReport(
-    outputPath: string = './scripts/testing/results/report.md'
-  ): Promise<void> {
-    const withStackMemory = Array.from(this.metrics.values()).filter(
-      (m) => m.variant === 'with_stackmemory'
-    );
-    const withoutStackMemory = Array.from(this.metrics.values()).filter(
-      (m) => m.variant === 'without_stackmemory'
-    );
-
-    const comparison = await this.compareVariants(
-      withStackMemory,
-      withoutStackMemory
-    );
-
-    const report = `# StackMemory Effectiveness Report
-
-## Executive Summary
-- Sample Size: ${comparison.statistics.sampleSize} sessions
-- Statistical Confidence: ${comparison.statistics.confidence.toFixed(1)}%
-- P-Value: ${comparison.statistics.pValue}
-
-## Performance Improvements
-- Context Reestablishment: ${comparison.improvement.contextSpeed.toFixed(1)}% faster
-- Task Completion: ${comparison.improvement.taskCompletion.toFixed(1)}% faster
-- Error Recovery: ${comparison.improvement.errorRecovery.toFixed(1)}% better
-- Decision Consistency: ${comparison.improvement.consistency.toFixed(1)}% improved
-
-## Recommendations
-${comparison.recommendations.map((r) => `- ${r}`).join('\n')}
-
-## Detailed Metrics
-
-### With StackMemory
-${this.formatMetricsTable(withStackMemory)}
-
-### Without StackMemory
-${this.formatMetricsTable(withoutStackMemory)}
-
-Generated: ${new Date().toISOString()}
-`;
-
-    await fs.writeFile(outputPath, report);
-    console.log(`Report generated: ${outputPath}`);
-  }
-
-  private formatMetricsTable(metrics: SessionMetrics[]): string {
-    if (metrics.length === 0) return 'No data available';
-
-    const avg = this.calculateAverages(metrics);
-
-    return `
-| Metric | Average |
-|--------|---------|
-| Context Reestablishment | ${(avg.contextReestablishmentTime / 1000).toFixed(2)}s |
-| Task Completion | ${(avg.completionTime / 1000 / 60).toFixed(2)} min |
-| Errors Encountered | ${avg.errorsEncountered.toFixed(1)} |
-| Decisions Anchored | ${avg.decisionsAnchored.toFixed(1)} |
-| Rework Instances | ${avg.reworkInstances.toFixed(1)} |
-`;
-  }
-}
-
-// CLI interface
-if (import.meta.url === `file://${process.argv[1]}`) {
-  const collector = new MetricsCollector();
-
-  const command = process.argv[2];
-
-  async function main() {
-    await collector.initialize();
-
-    switch (command) {
-      case 'start':
-        const variant = process.argv[3] as
-          | 'with_stackmemory'
-          | 'without_stackmemory';
-        const sessionId = await collector.startSession(variant);
-        console.log(`Session started: ${sessionId}`);
-        break;
-
-      case 'report':
-        await collector.generateReport();
-        break;
-
-      default:
-        console.log(
-          'Usage: collect-metrics.ts [start|report] [with_stackmemory|without_stackmemory]'
-        );
-    }
-  }
-
-  main().catch(console.error);
-}

package/scripts/testing/quick-effectiveness-demo.js

@@ -1,187 +0,0 @@
-#!/usr/bin/env node
-
-import { performance } from 'perf_hooks';
-
-// Simple demonstration of StackMemory effectiveness
-class QuickEffectivenessDemo {
-  constructor() {
-    this.scenarios = {
-      withoutStackMemory: {
-        contextReestablishment: 300000, // 5 minutes in ms
-        taskCompletion: 1800000, // 30 minutes
-        reworkRate: 0.25, // 25% of work needs to be redone
-        errorRecovery: 600000, // 10 minutes
-        contextAccuracy: 0.60 // 60% accuracy
-      },
-      withStackMemory: {
-        contextReestablishment: 15000, // 15 seconds
-        taskCompletion: 1080000, // 18 minutes (40% faster)
-        reworkRate: 0.05, // 5% rework
-        errorRecovery: 180000, // 3 minutes (70% faster)
-        contextAccuracy: 0.95 // 95% accuracy
-      }
-    };
-  }
-
-  formatTime(ms) {
-    if (ms < 60000) {
-      return `${(ms / 1000).toFixed(1)}s`;
-    }
-    return `${(ms / 60000).toFixed(1)} min`;
-  }
-
-  calculateImprovement(without, with_) {
-    return ((without - with_) / without * 100).toFixed(1);
-  }
-
-  runDemo() {
-    console.log('\n' + '='.repeat(60));
-    console.log('STACKMEMORY EFFECTIVENESS DEMONSTRATION');
-    console.log('='.repeat(60));
-    console.log('\nBased on real-world testing patterns and expected performance:\n');
-
-    const { withoutStackMemory: without, withStackMemory: with_ } = this.scenarios;
-
-    // Context Reestablishment
-    console.log('📊 CONTEXT REESTABLISHMENT TIME');
-    console.log('─'.repeat(40));
-    console.log(`Without StackMemory: ${this.formatTime(without.contextReestablishment)}`);
-    console.log(`With StackMemory: ${this.formatTime(with_.contextReestablishment)}`);
-    console.log(`\x1b[32m✓ Improvement: ${this.calculateImprovement(without.contextReestablishment, with_.contextReestablishment)}%\x1b[0m\n`);
-
-    // Task Completion
-    console.log('⏱️  TASK COMPLETION TIME');
-    console.log('─'.repeat(40));
-    console.log(`Without StackMemory: ${this.formatTime(without.taskCompletion)}`);
-    console.log(`With StackMemory: ${this.formatTime(with_.taskCompletion)}`);
-    console.log(`\x1b[32m✓ Improvement: ${this.calculateImprovement(without.taskCompletion, with_.taskCompletion)}%\x1b[0m\n`);
-
-    // Rework Rate
-    console.log('🔄 REWORK RATE');
-    console.log('─'.repeat(40));
-    console.log(`Without StackMemory: ${(without.reworkRate * 100).toFixed(0)}% of work needs redoing`);
-    console.log(`With StackMemory: ${(with_.reworkRate * 100).toFixed(0)}% of work needs redoing`);
-    console.log(`\x1b[32m✓ Improvement: ${this.calculateImprovement(without.reworkRate, with_.reworkRate)}%\x1b[0m\n`);
-
-    // Error Recovery
-    console.log('🔧 ERROR RECOVERY TIME');
-    console.log('─'.repeat(40));
-    console.log(`Without StackMemory: ${this.formatTime(without.errorRecovery)}`);
-    console.log(`With StackMemory: ${this.formatTime(with_.errorRecovery)}`);
-    console.log(`\x1b[32m✓ Improvement: ${this.calculateImprovement(without.errorRecovery, with_.errorRecovery)}%\x1b[0m\n`);
-
-    // Context Accuracy
-    console.log('🎯 CONTEXT ACCURACY');
-    console.log('─'.repeat(40));
-    console.log(`Without StackMemory: ${(without.contextAccuracy * 100).toFixed(0)}% accuracy`);
-    console.log(`With StackMemory: ${(with_.contextAccuracy * 100).toFixed(0)}% accuracy`);
-    console.log(`\x1b[32m✓ Improvement: ${((with_.contextAccuracy - without.contextAccuracy) / without.contextAccuracy * 100).toFixed(1)}%\x1b[0m\n`);
-
-    // Real-World Scenarios
-    console.log('=' .repeat(60));
-    console.log('REAL-WORLD SCENARIO IMPACTS');
-    console.log('='.repeat(60));
-
-    console.log('\n1️⃣  MULTI-SESSION FEATURE DEVELOPMENT');
-    console.log('   Scenario: Building e-commerce checkout over 3 sessions');
-    console.log(`   Without StackMemory: ${this.formatTime(without.contextReestablishment * 3)} lost to context`);
-    console.log(`   With StackMemory: ${this.formatTime(with_.contextReestablishment * 3)} for all sessions`);
-    console.log(`   \x1b[32mTime Saved: ${this.formatTime((without.contextReestablishment - with_.contextReestablishment) * 3)}\x1b[0m`);
-
-    console.log('\n2️⃣  COMPLEX DEBUGGING');
-    console.log('   Scenario: Debugging production issue with team handoff');
-    console.log(`   Without StackMemory: ${this.formatTime(without.errorRecovery + without.contextReestablishment)}`);
-    console.log(`   With StackMemory: ${this.formatTime(with_.errorRecovery + with_.contextReestablishment)}`);
-    console.log(`   \x1b[32mTime Saved: ${this.formatTime((without.errorRecovery + without.contextReestablishment) - (with_.errorRecovery + with_.contextReestablishment))}\x1b[0m`);
-
-    console.log('\n3️⃣  LARGE REFACTORING');
-    console.log('   Scenario: Auth system migration over 5 sessions');
-    console.log(`   Without StackMemory: ${(without.reworkRate * 100).toFixed(0)}% of changes need rework`);
-    console.log(`   With StackMemory: ${(with_.reworkRate * 100).toFixed(0)}% of changes need rework`);
-    console.log(`   \x1b[32mRework Reduced: ${((without.reworkRate - with_.reworkRate) * 100).toFixed(0)}% of total work\x1b[0m`);
-
-    // Success Criteria
-    console.log('\n' + '='.repeat(60));
-    console.log('SUCCESS CRITERIA EVALUATION');
-    console.log('='.repeat(60));
-
-    const criteria = [
-      {
-        name: 'Context Reestablishment <30s',
-        met: with_.contextReestablishment < 30000,
-        actual: this.formatTime(with_.contextReestablishment),
-        target: '<30s'
-      },
-      {
-        name: 'Context Speed Improvement >90%',
-        met: this.calculateImprovement(without.contextReestablishment, with_.contextReestablishment) > 90,
-        actual: `${this.calculateImprovement(without.contextReestablishment, with_.contextReestablishment)}%`,
-        target: '>90%'
-      },
-      {
-        name: 'Task Completion Improvement >30%',
-        met: this.calculateImprovement(without.taskCompletion, with_.taskCompletion) > 30,
-        actual: `${this.calculateImprovement(without.taskCompletion, with_.taskCompletion)}%`,
-        target: '>30%'
-      },
-      {
-        name: 'Context Accuracy >90%',
-        met: with_.contextAccuracy > 0.90,
-        actual: `${(with_.contextAccuracy * 100).toFixed(0)}%`,
-        target: '>90%'
-      }
-    ];
-
-    console.log();
-    criteria.forEach(c => {
-      const status = c.met ? '\x1b[32m✅ PASSED\x1b[0m' : '\x1b[31m❌ FAILED\x1b[0m';
-      console.log(`${status} ${c.name}`);
-      console.log(`     Target: ${c.target} | Actual: ${c.actual}`);
-    });
-
-    // Overall Productivity Impact
-    console.log('\n' + '='.repeat(60));
-    console.log('OVERALL PRODUCTIVITY IMPACT');
-    console.log('='.repeat(60));
-
-    const weeklyHours = 40;
-    const contextSwitchesPerWeek = 20; // Average developer context switches
-    const weeklyTimeSaved = (without.contextReestablishment - with_.contextReestablishment) * contextSwitchesPerWeek;
-    const productivityGain = (weeklyTimeSaved / (weeklyHours * 60 * 60 * 1000)) * 100;
-
-    console.log(`\n📈 Weekly Impact for Average Developer:`);
-    console.log(`   Context Switches: ${contextSwitchesPerWeek} per week`);
-    console.log(`   Time Saved: ${this.formatTime(weeklyTimeSaved)} per week`);
-    console.log(`   Productivity Gain: ${productivityGain.toFixed(1)}% more productive time`);
-    
-    console.log(`\n📊 Annual Impact:`);
-    console.log(`   Time Saved: ${this.formatTime(weeklyTimeSaved * 52)} per year`);
-    console.log(`   Equivalent to: ${(weeklyTimeSaved * 52 / (8 * 60 * 60 * 1000)).toFixed(1)} full workdays`);
-
-    // Final Verdict
-    console.log('\n' + '='.repeat(60));
-    const allPassed = criteria.every(c => c.met);
-    if (allPassed) {
-      console.log('\x1b[32m🎉 STACKMEMORY DELIVERS SIGNIFICANT IMPROVEMENTS\x1b[0m');
-      console.log('\x1b[32m   All success criteria met!\x1b[0m');
-      console.log('\x1b[32m   Expected productivity gain: 20-40%\x1b[0m');
-    } else {
-      console.log('\x1b[33m⚠️  PARTIAL SUCCESS\x1b[0m');
-      console.log('\x1b[33m   Some criteria not met, optimization needed\x1b[0m');
-    }
-    console.log('='.repeat(60));
-
-    // Key Takeaways
-    console.log('\n📝 KEY TAKEAWAYS:');
-    console.log('─'.repeat(40));
-    console.log('1. Context reestablishment reduced from minutes to seconds');
-    console.log('2. Significant reduction in rework and errors');
-    console.log('3. Perfect for multi-session and team collaboration');
-    console.log('4. Measurable productivity improvement of 20-40%');
-    console.log('5. ROI: Saves ~1 hour per day for active developers');
-  }
-}
-
-// Run the demo
-const demo = new QuickEffectivenessDemo();
-demo.runDemo();