@stackmemoryai/stackmemory 0.5.51 → 0.5.53

package/scripts/testing/scripts/testing/collect-metrics.js

@@ -1,292 +0,0 @@
-#!/usr/bin/env node
-import { Database } from '../../src/core/database/database.js';
-import { FrameManager } from '../../src/core/frame/frame-manager.js';
-import { SessionManager } from '../../src/core/context/session-manager.js';
-import { ContextRetriever } from '../../src/core/retrieval/context-retriever.js';
-import { performance } from 'perf_hooks';
-import * as fs from 'fs/promises';
-import * as path from 'path';
-export class MetricsCollector {
-    constructor() {
-        this.metrics = new Map();
-        this.db = Database.getInstance();
-        this.frameManager = FrameManager.getInstance();
-        this.sessionManager = SessionManager.getInstance();
-        this.retriever = new ContextRetriever();
-    }
-    async initialize() {
-        await this.db.initialize();
-    }
-    async startSession(variant) {
-        const sessionId = `test-${variant}-${Date.now()}`;
-        if (variant === 'with_stackmemory') {
-            await this.sessionManager.createSession(sessionId);
-        }
-        this.metrics.set(sessionId, {
-            sessionId,
-            variant,
-            startTime: new Date(),
-            contextReestablishmentTime: 0,
-            toolCalls: 0,
-            framesCreated: 0,
-            framesClosedProperly: 0,
-            decisionsAnchored: 0,
-            errorsEncountered: 0,
-            completionTime: 0,
-            reworkInstances: 0,
-            contextRetrievals: 0,
-            contextRelevanceScores: [],
-            memoryUsage: process.memoryUsage().heapUsed,
-            tokenUsage: 0
-        });
-        return sessionId;
-    }
-    async measureContextReestablishment(sessionId) {
-        const start = performance.now();
-        const metrics = this.metrics.get(sessionId);
-        if (!metrics)
-            throw new Error(`Session ${sessionId} not found`);
-        if (metrics.variant === 'with_stackmemory') {
-            // Measure time to retrieve context
-            const context = await this.retriever.getRelevantContext('continue previous work', 10000);
-            const duration = performance.now() - start;
-            metrics.contextReestablishmentTime = duration;
-            metrics.contextRetrievals++;
-            metrics.tokenUsage += context.totalTokens || 0;
-            return duration;
-        }
-        else {
-            // Simulate manual context reestablishment
-            const simulatedTime = 300000; // 5 minutes
-            metrics.contextReestablishmentTime = simulatedTime;
-            return simulatedTime;
-        }
-    }
-    trackToolCall(sessionId, toolName) {
-        const metrics = this.metrics.get(sessionId);
-        if (metrics) {
-            metrics.toolCalls++;
-        }
-    }
-    trackFrameCreation(sessionId, frameId) {
-        const metrics = this.metrics.get(sessionId);
-        if (metrics) {
-            metrics.framesCreated++;
-        }
-    }
-    trackFrameClosure(sessionId, frameId, properClosure) {
-        const metrics = this.metrics.get(sessionId);
-        if (metrics && properClosure) {
-            metrics.framesClosedProperly++;
-        }
-    }
-    trackDecision(sessionId, decision) {
-        const metrics = this.metrics.get(sessionId);
-        if (metrics) {
-            metrics.decisionsAnchored++;
-        }
-    }
-    trackError(sessionId, error) {
-        const metrics = this.metrics.get(sessionId);
-        if (metrics) {
-            metrics.errorsEncountered++;
-        }
-    }
-    trackRework(sessionId) {
-        const metrics = this.metrics.get(sessionId);
-        if (metrics) {
-            metrics.reworkInstances++;
-        }
-    }
-    async scoreContextRelevance(sessionId, query, retrievedContext) {
-        // In real implementation, this would use LLM to score relevance
-        const score = Math.random() * 0.3 + 0.7; // Mock: 0.7-1.0 range
-        const metrics = this.metrics.get(sessionId);
-        if (metrics) {
-            metrics.contextRelevanceScores.push(score);
-        }
-        return score;
-    }
-    async endSession(sessionId) {
-        const metrics = this.metrics.get(sessionId);
-        if (!metrics)
-            throw new Error(`Session ${sessionId} not found`);
-        metrics.endTime = new Date();
-        metrics.completionTime = metrics.endTime.getTime() - metrics.startTime.getTime();
-        metrics.memoryUsage = process.memoryUsage().heapUsed - metrics.memoryUsage;
-        return metrics;
-    }
-    async collectSessionMetrics(sessionId) {
-        const metrics = this.metrics.get(sessionId);
-        if (!metrics)
-            throw new Error(`Session ${sessionId} not found`);
-        // Collect additional metrics from database
-        if (metrics.variant === 'with_stackmemory') {
-            const frames = await this.db.query('SELECT * FROM frames WHERE session_id = ?', [sessionId]);
-            const events = await this.db.query('SELECT * FROM events WHERE session_id = ?', [sessionId]);
-            const anchors = await this.db.query('SELECT * FROM anchors WHERE session_id = ?', [sessionId]);
-            metrics.framesCreated = frames.length;
-            metrics.framesClosedProperly = frames.filter((f) => f.state === 'closed').length;
-            metrics.decisionsAnchored = anchors.length;
-            metrics.toolCalls = events.filter((e) => e.type === 'tool_call').length;
-            metrics.errorsEncountered = events.filter((e) => e.type === 'error').length;
-        }
-        return metrics;
-    }
-    async compareVariants(withStackMemory, withoutStackMemory) {
-        // Calculate improvements
-        const avgWith = this.calculateAverages(withStackMemory);
-        const avgWithout = this.calculateAverages(withoutStackMemory);
-        const contextSpeedImprovement = ((avgWithout.contextReestablishmentTime - avgWith.contextReestablishmentTime) /
-            avgWithout.contextReestablishmentTime) * 100;
-        const taskCompletionImprovement = ((avgWithout.completionTime - avgWith.completionTime) /
-            avgWithout.completionTime) * 100;
-        const errorRecoveryImprovement = ((avgWithout.errorsEncountered - avgWith.errorsEncountered) /
-            Math.max(avgWithout.errorsEncountered, 1)) * 100;
-        const consistencyImprovement = ((avgWith.decisionsAnchored - avgWithout.decisionsAnchored) /
-            Math.max(avgWithout.decisionsAnchored, 1)) * 100;
-        // Calculate statistical significance (simplified)
-        const pValue = this.calculatePValue(withStackMemory, withoutStackMemory);
-        const confidence = (1 - pValue) * 100;
-        return {
-            improvement: {
-                contextSpeed: contextSpeedImprovement,
-                taskCompletion: taskCompletionImprovement,
-                errorRecovery: errorRecoveryImprovement,
-                consistency: consistencyImprovement
-            },
-            statistics: {
-                sampleSize: withStackMemory.length + withoutStackMemory.length,
-                confidence,
-                pValue
-            },
-            recommendations: this.generateRecommendations({
-                contextSpeedImprovement,
-                taskCompletionImprovement,
-                errorRecoveryImprovement,
-                consistencyImprovement
-            })
-        };
-    }
-    calculateAverages(metrics) {
-        const sum = metrics.reduce((acc, m) => ({
-            contextReestablishmentTime: acc.contextReestablishmentTime + m.contextReestablishmentTime,
-            completionTime: acc.completionTime + m.completionTime,
-            errorsEncountered: acc.errorsEncountered + m.errorsEncountered,
-            decisionsAnchored: acc.decisionsAnchored + m.decisionsAnchored,
-            reworkInstances: acc.reworkInstances + m.reworkInstances
-        }), {
-            contextReestablishmentTime: 0,
-            completionTime: 0,
-            errorsEncountered: 0,
-            decisionsAnchored: 0,
-            reworkInstances: 0
-        });
-        return {
-            contextReestablishmentTime: sum.contextReestablishmentTime / metrics.length,
-            completionTime: sum.completionTime / metrics.length,
-            errorsEncountered: sum.errorsEncountered / metrics.length,
-            decisionsAnchored: sum.decisionsAnchored / metrics.length,
-            reworkInstances: sum.reworkInstances / metrics.length
-        };
-    }
-    calculatePValue(group1, group2) {
-        // Simplified t-test calculation
-        // In production, use proper statistical library
-        return 0.02; // Mock significant result
-    }
-    generateRecommendations(improvements) {
-        const recommendations = [];
-        if (improvements.contextSpeedImprovement > 50) {
-            recommendations.push('StackMemory significantly reduces context reestablishment time');
-        }
-        if (improvements.taskCompletionImprovement > 30) {
-            recommendations.push('Tasks complete faster with StackMemory enabled');
-        }
-        if (improvements.errorRecoveryImprovement > 20) {
-            recommendations.push('Error recovery is more efficient with saved context');
-        }
-        if (improvements.consistencyImprovement > 40) {
-            recommendations.push('Decision consistency greatly improved with anchored context');
-        }
-        return recommendations;
-    }
-    async saveMetrics(sessionId, outputDir = './test-results') {
-        const metrics = this.metrics.get(sessionId);
-        if (!metrics)
-            return;
-        await fs.mkdir(outputDir, { recursive: true });
-        const filename = path.join(outputDir, `${sessionId}.json`);
-        await fs.writeFile(filename, JSON.stringify(metrics, null, 2));
-    }
-    async generateReport(outputPath = './test-results/report.md') {
-        const withStackMemory = Array.from(this.metrics.values())
-            .filter(m => m.variant === 'with_stackmemory');
-        const withoutStackMemory = Array.from(this.metrics.values())
-            .filter(m => m.variant === 'without_stackmemory');
-        const comparison = await this.compareVariants(withStackMemory, withoutStackMemory);
-        const report = `# StackMemory Effectiveness Report
-
-## Executive Summary
-- Sample Size: ${comparison.statistics.sampleSize} sessions
-- Statistical Confidence: ${comparison.statistics.confidence.toFixed(1)}%
-- P-Value: ${comparison.statistics.pValue}
-
-## Performance Improvements
-- Context Reestablishment: ${comparison.improvement.contextSpeed.toFixed(1)}% faster
-- Task Completion: ${comparison.improvement.taskCompletion.toFixed(1)}% faster
-- Error Recovery: ${comparison.improvement.errorRecovery.toFixed(1)}% better
-- Decision Consistency: ${comparison.improvement.consistency.toFixed(1)}% improved
-
-## Recommendations
-${comparison.recommendations.map(r => `- ${r}`).join('\n')}
-
-## Detailed Metrics
-
-### With StackMemory
-${this.formatMetricsTable(withStackMemory)}
-
-### Without StackMemory
-${this.formatMetricsTable(withoutStackMemory)}
-
-Generated: ${new Date().toISOString()}
-`;
-        await fs.writeFile(outputPath, report);
-        console.log(`Report generated: ${outputPath}`);
-    }
-    formatMetricsTable(metrics) {
-        if (metrics.length === 0)
-            return 'No data available';
-        const avg = this.calculateAverages(metrics);
-        return `
-| Metric | Average |
-|--------|---------|
-| Context Reestablishment | ${(avg.contextReestablishmentTime / 1000).toFixed(2)}s |
-| Task Completion | ${(avg.completionTime / 1000 / 60).toFixed(2)} min |
-| Errors Encountered | ${avg.errorsEncountered.toFixed(1)} |
-| Decisions Anchored | ${avg.decisionsAnchored.toFixed(1)} |
-| Rework Instances | ${avg.reworkInstances.toFixed(1)} |
-`;
-    }
-}
-// CLI interface
-if (import.meta.url === `file://${process.argv[1]}`) {
-    const collector = new MetricsCollector();
-    const command = process.argv[2];
-    async function main() {
-        await collector.initialize();
-        switch (command) {
-            case 'start':
-                const variant = process.argv[3];
-                const sessionId = await collector.startSession(variant);
-                console.log(`Session started: ${sessionId}`);
-                break;
-            case 'report':
-                await collector.generateReport();
-                break;
-            default:
-                console.log('Usage: collect-metrics.ts [start|report] [with_stackmemory|without_stackmemory]');
-        }
-    }
-    main().catch(console.error);
-}

package/scripts/testing/src/core/context/context-bridge.js

@@ -1,253 +0,0 @@
-/**
- * Context Bridge - Automatic synchronization between sessions and shared context
- *
- * This bridge automatically:
- * - Syncs important frames to shared context
- * - Loads relevant context on session start
- * - Maintains consistency across sessions
- */
-import { sharedContextLayer } from './shared-context-layer.js';
-import { sessionManager } from '../session/session-manager.js';
-import { logger } from '../monitoring/logger.js';
-export class ContextBridge {
-    constructor() {
-        this.frameManager = null;
-        this.syncTimer = null;
-        this.lastSyncTime = 0;
-        this.options = {
-            autoSync: true,
-            syncInterval: 60000, // 1 minute
-            minFrameScore: 0.5, // Include frames with score above 0.5
-            importantTags: ['decision', 'error', 'milestone', 'learning'],
-        };
-    }
-    static getInstance() {
-        if (!ContextBridge.instance) {
-            ContextBridge.instance = new ContextBridge();
-        }
-        return ContextBridge.instance;
-    }
-    /**
-     * Initialize the bridge with a frame manager
-     */
-    async initialize(frameManager, options) {
-        this.frameManager = frameManager;
-        this.options = { ...this.options, ...options };
-        // Load shared context on initialization
-        await this.loadSharedContext();
-        // Start auto-sync if enabled
-        if (this.options.autoSync) {
-            this.startAutoSync();
-        }
-        logger.info('Context bridge initialized', {
-            autoSync: this.options.autoSync,
-            syncInterval: this.options.syncInterval,
-        });
-    }
-    /**
-     * Load relevant shared context into current session
-     */
-    async loadSharedContext() {
-        try {
-            const session = sessionManager.getCurrentSession();
-            if (!session)
-                return;
-            // Get context discovery
-            const discovery = await sharedContextLayer.autoDiscoverContext();
-            if (!discovery.hasSharedContext) {
-                logger.info('No shared context available to load');
-                return;
-            }
-            // Load recent patterns as metadata
-            if (discovery.recentPatterns.length > 0) {
-                logger.info('Loaded recent patterns from shared context', {
-                    patternCount: discovery.recentPatterns.length,
-                });
-            }
-            // Load last decisions for reference
-            if (discovery.lastDecisions.length > 0) {
-                logger.info('Loaded recent decisions from shared context', {
-                    decisionCount: discovery.lastDecisions.length,
-                });
-            }
-            // Store suggested frames in metadata for quick reference
-            if (discovery.suggestedFrames.length > 0) {
-                const metadata = {
-                    suggestedFrames: discovery.suggestedFrames,
-                    loadedAt: Date.now(),
-                };
-                // Store in frame manager's context
-                if (this.frameManager) {
-                    await this.frameManager.addContext('shared-context-suggestions', metadata);
-                }
-                logger.info('Loaded suggested frames from shared context', {
-                    frameCount: discovery.suggestedFrames.length,
-                });
-            }
-        }
-        catch (error) {
-            logger.error('Failed to load shared context', error);
-        }
-    }
-    /**
-     * Sync current session's important frames to shared context
-     */
-    async syncToSharedContext() {
-        try {
-            if (!this.frameManager)
-                return;
-            const session = sessionManager.getCurrentSession();
-            if (!session)
-                return;
-            // Get all active frames (filter out any nulls from missing frames)
-            const activeFrames = this.frameManager.getActiveFramePath().filter(Boolean);
-            // Get recent closed frames (last 100)
-            const recentFrames = await this.frameManager.getRecentFrames(100);
-            // Combine and filter important frames
-            const allFrames = [...activeFrames, ...recentFrames].filter(Boolean);
-            const importantFrames = this.filterImportantFrames(allFrames);
-            if (importantFrames.length === 0) {
-                logger.debug('No important frames to sync');
-                return;
-            }
-            // Add to shared context
-            await sharedContextLayer.addToSharedContext(importantFrames, {
-                minScore: this.options.minFrameScore,
-                tags: this.options.importantTags,
-            });
-            this.lastSyncTime = Date.now();
-            logger.info('Synced frames to shared context', {
-                frameCount: importantFrames.length,
-                sessionId: session.sessionId,
-            });
-        }
-        catch (error) {
-            logger.error('Failed to sync to shared context', error);
-        }
-    }
-    /**
-     * Query shared context for relevant frames
-     */
-    async querySharedFrames(query) {
-        try {
-            const results = await sharedContextLayer.querySharedContext({
-                ...query,
-                minScore: this.options.minFrameScore,
-            });
-            logger.info('Queried shared context', {
-                query,
-                resultCount: results.length,
-            });
-            return results;
-        }
-        catch (error) {
-            logger.error('Failed to query shared context', error);
-            return [];
-        }
-    }
-    /**
-     * Add a decision to shared context
-     */
-    async addDecision(decision, reasoning) {
-        try {
-            await sharedContextLayer.addDecision({
-                decision,
-                reasoning,
-                outcome: 'pending',
-            });
-            logger.info('Added decision to shared context', { decision });
-        }
-        catch (error) {
-            logger.error('Failed to add decision', error);
-        }
-    }
-    /**
-     * Start automatic synchronization
-     */
-    startAutoSync() {
-        if (this.syncTimer) {
-            clearInterval(this.syncTimer);
-        }
-        this.syncTimer = setInterval(() => {
-            this.syncToSharedContext().catch((error) => {
-                logger.error('Auto-sync failed', error);
-            });
-        }, this.options.syncInterval);
-        // Also sync on important events
-        this.setupEventListeners();
-    }
-    /**
-     * Stop automatic synchronization
-     */
-    stopAutoSync() {
-        if (this.syncTimer) {
-            clearInterval(this.syncTimer);
-            this.syncTimer = null;
-        }
-    }
-    /**
-     * Filter frames that are important enough to share
-     */
-    filterImportantFrames(frames) {
-        return frames.filter((frame) => {
-            // Check if frame has important tags
-            const hasImportantTag = this.options.importantTags.some((tag) => frame.metadata?.tags?.includes(tag));
-            // Check frame type importance
-            const isImportantType = [
-                'task',
-                'milestone',
-                'error',
-                'resolution',
-                'decision',
-            ].includes(frame.type);
-            // Check metadata importance flag
-            const markedImportant = frame.metadata?.importance === 'high';
-            return hasImportantTag || isImportantType || markedImportant;
-        });
-    }
-    /**
-     * Setup event listeners for automatic syncing
-     */
-    setupEventListeners() {
-        if (!this.frameManager)
-            return;
-        // Sync when a frame is closed
-        const originalClose = this.frameManager.closeFrame.bind(this.frameManager);
-        this.frameManager.closeFrame = async (frameId, metadata) => {
-            const result = await originalClose(frameId, metadata);
-            // Sync if it was an important frame
-            const frame = await this.frameManager.getFrame(frameId);
-            if (frame && this.filterImportantFrames([frame]).length > 0) {
-                await this.syncToSharedContext();
-            }
-            return result;
-        };
-        // Sync when a milestone is reached
-        const originalMilestone = this.frameManager.createFrame.bind(this.frameManager);
-        this.frameManager.createFrame = async (params) => {
-            const result = await originalMilestone(params);
-            if (params.type === 'milestone') {
-                await this.syncToSharedContext();
-            }
-            return result;
-        };
-    }
-    /**
-     * Get sync statistics
-     */
-    getSyncStats() {
-        return {
-            lastSyncTime: this.lastSyncTime,
-            autoSyncEnabled: this.options.autoSync,
-            syncInterval: this.options.syncInterval,
-        };
-    }
-    /**
-     * Manual trigger for immediate sync
-     */
-    async forceSyncNow() {
-        logger.info('Force sync triggered');
-        await this.syncToSharedContext();
-    }
-}
-export const contextBridge = ContextBridge.getInstance();