@stackmemoryai/stackmemory 0.3.17 → 0.3.19

package/scripts/testing/src/core/trace/db-trace-wrapper.js

@@ -0,0 +1,251 @@
+/**
+ * Database Operations Trace Wrapper
+ * Wraps SQLite operations with comprehensive tracing for debugging
+ */
+import Database from 'better-sqlite3';
+import { trace } from './debug-trace.js';
+import { logger } from '../monitoring/logger.js';
+/**
+ * Create a traced database instance
+ */
+export function createTracedDatabase(filename, options) {
+    const db = new Database(filename, options);
+    if (options?.traceEnabled !== false) {
+        return wrapDatabase(db, options?.slowQueryThreshold);
+    }
+    return db;
+}
+/**
+ * Wrap an existing database with tracing
+ */
+export function wrapDatabase(db, slowQueryThreshold = 100) {
+    // Wrap prepare method to trace all queries
+    const originalPrepare = db.prepare.bind(db);
+    db.prepare = function (source) {
+        const statement = originalPrepare(source);
+        return wrapStatement(statement, source, slowQueryThreshold);
+    };
+    // Wrap exec for direct SQL execution
+    const originalExec = db.exec.bind(db);
+    db.exec = function (source) {
+        return trace.traceSync('query', `EXEC: ${source.substring(0, 50)}...`, {}, () => {
+            const startTime = performance.now();
+            const result = originalExec(source);
+            const duration = performance.now() - startTime;
+            if (duration > slowQueryThreshold) {
+                logger.warn(`Slow query detected: ${duration.toFixed(0)}ms`, {
+                    query: source.substring(0, 200),
+                    duration,
+                });
+            }
+            return result;
+        });
+    };
+    // Wrap transaction for transaction tracking
+    const originalTransaction = db.transaction.bind(db);
+    db.transaction = function (fn) {
+        return originalTransaction(function (...args) {
+            return trace.traceSync('query', 'TRANSACTION', { args: args.length }, () => {
+                return fn.apply(this, args);
+            });
+        });
+    };
+    // Add query statistics tracking
+    db.__queryStats = {
+        totalQueries: 0,
+        slowQueries: 0,
+        totalDuration: 0,
+        queryTypes: {},
+    };
+    return db;
+}
+/**
+ * Wrap a statement with tracing
+ */
+function wrapStatement(statement, source, slowQueryThreshold) {
+    const queryType = source.trim().split(/\s+/)[0].toUpperCase();
+    const shortQuery = source.substring(0, 100).replace(/\s+/g, ' ');
+    // Wrap run method
+    const originalRun = statement.run.bind(statement);
+    statement.run = function (...params) {
+        return trace.traceSync('query', `${queryType}: ${shortQuery}`, params, () => {
+            const startTime = performance.now();
+            const result = originalRun(...params);
+            const duration = performance.now() - startTime;
+            // Track statistics
+            updateQueryStats(statement, queryType, duration, slowQueryThreshold);
+            // Log slow queries
+            if (duration > slowQueryThreshold) {
+                logger.warn(`Slow ${queryType} query: ${duration.toFixed(0)}ms`, {
+                    query: shortQuery,
+                    params,
+                    duration,
+                    changes: result.changes,
+                });
+            }
+            return result;
+        });
+    };
+    // Wrap get method
+    const originalGet = statement.get.bind(statement);
+    statement.get = function (...params) {
+        return trace.traceSync('query', `${queryType} (get): ${shortQuery}`, params, () => {
+            const startTime = performance.now();
+            const result = originalGet(...params);
+            const duration = performance.now() - startTime;
+            updateQueryStats(statement, queryType, duration, slowQueryThreshold);
+            if (duration > slowQueryThreshold) {
+                logger.warn(`Slow ${queryType} query: ${duration.toFixed(0)}ms`, {
+                    query: shortQuery,
+                    params,
+                    duration,
+                    found: result != null,
+                });
+            }
+            return result;
+        });
+    };
+    // Wrap all method
+    const originalAll = statement.all.bind(statement);
+    statement.all = function (...params) {
+        return trace.traceSync('query', `${queryType} (all): ${shortQuery}`, params, () => {
+            const startTime = performance.now();
+            const result = originalAll(...params);
+            const duration = performance.now() - startTime;
+            updateQueryStats(statement, queryType, duration, slowQueryThreshold);
+            if (duration > slowQueryThreshold) {
+                logger.warn(`Slow ${queryType} query: ${duration.toFixed(0)}ms`, {
+                    query: shortQuery,
+                    params,
+                    duration,
+                    rows: result.length,
+                });
+            }
+            // Warn about potential N+1 queries
+            if (result.length > 100 && queryType === 'SELECT') {
+                logger.warn(`Large result set: ${result.length} rows`, {
+                    query: shortQuery,
+                    suggestion: 'Consider pagination or more specific queries',
+                });
+            }
+            return result;
+        });
+    };
+    // Wrap iterate method for cursor operations
+    const originalIterate = statement.iterate.bind(statement);
+    statement.iterate = function (...params) {
+        const startTime = performance.now();
+        let rowCount = 0;
+        const iterator = originalIterate(...params);
+        const wrappedIterator = {
+            [Symbol.iterator]() {
+                return this;
+            },
+            next() {
+                const result = iterator.next();
+                if (!result.done) {
+                    rowCount++;
+                }
+                else {
+                    const duration = performance.now() - startTime;
+                    updateQueryStats(statement, queryType, duration, slowQueryThreshold);
+                    if (duration > slowQueryThreshold) {
+                        logger.warn(`Slow ${queryType} iteration: ${duration.toFixed(0)}ms`, {
+                            query: shortQuery,
+                            params,
+                            duration,
+                            rows: rowCount,
+                        });
+                    }
+                }
+                return result;
+            },
+        };
+        return wrappedIterator;
+    };
+    return statement;
+}
+/**
+ * Update query statistics
+ */
+function updateQueryStats(statement, queryType, duration, slowQueryThreshold) {
+    const db = statement.database;
+    if (db.__queryStats) {
+        db.__queryStats.totalQueries++;
+        db.__queryStats.totalDuration += duration;
+        if (duration > slowQueryThreshold) {
+            db.__queryStats.slowQueries++;
+        }
+        if (!db.__queryStats.queryTypes[queryType]) {
+            db.__queryStats.queryTypes[queryType] = 0;
+        }
+        db.__queryStats.queryTypes[queryType]++;
+    }
+}
+/**
+ * Get query statistics from a traced database
+ */
+export function getQueryStatistics(db) {
+    const stats = db.__queryStats;
+    if (!stats)
+        return null;
+    return {
+        ...stats,
+        averageDuration: stats.totalQueries > 0
+            ? stats.totalDuration / stats.totalQueries
+            : 0,
+    };
+}
+/**
+ * Helper to trace a specific query with context
+ */
+export async function traceQuery(db, queryName, query, params, fn) {
+    return trace.traceAsync('query', queryName, { query, params }, async () => {
+        try {
+            const result = fn();
+            // Log successful complex queries for debugging
+            if (query.includes('JOIN') || query.includes('GROUP BY')) {
+                logger.debug(`Complex query executed: ${queryName}`, {
+                    query: query.substring(0, 200),
+                    params,
+                });
+            }
+            return result;
+        }
+        catch (error) {
+            // Enhanced error logging for database errors
+            logger.error(`Database query failed: ${queryName}`, error, {
+                query,
+                params,
+                errorCode: error.code,
+            });
+            throw error;
+        }
+    });
+}
+/**
+ * Create a traced transaction with automatic rollback on error
+ */
+export function createTracedTransaction(db, name, fn) {
+    return trace.traceSync('query', `TRANSACTION: ${name}`, {}, () => {
+        const startTime = performance.now();
+        try {
+            const tx = db.transaction(fn);
+            const result = tx.deferred();
+            const duration = performance.now() - startTime;
+            logger.info(`Transaction completed: ${name}`, {
+                duration,
+                success: true,
+            });
+            return result;
+        }
+        catch (error) {
+            const duration = performance.now() - startTime;
+            logger.error(`Transaction failed: ${name}`, error, {
+                duration,
+                success: false,
+            });
+            throw error;
+        }
+    });
+}

package/scripts/testing/src/core/trace/debug-trace.js

@@ -0,0 +1,398 @@
+/**
+ * Debug Trace Module - Comprehensive execution tracing for LLM debugging
+ *
+ * This module provides detailed execution tracing to help LLMs understand
+ * exactly what happened during code execution, making debugging much easier.
+ */
+import { performance } from 'perf_hooks';
+import { logger } from '../monitoring/logger.js';
+import * as fs from 'fs';
+import * as path from 'path';
+import { v4 as uuidv4 } from 'uuid';
+export class TraceContext {
+    constructor() {
+        this.currentTrace = null;
+        this.traceStack = [];
+        this.allTraces = [];
+        this.startTime = Date.now();
+        this.sensitivePatterns = [
+            /api[_-]?key/i,
+            /token/i,
+            /secret/i,
+            /password/i,
+            /bearer/i,
+            /authorization/i,
+            /client[_-]?id/i,
+            /client[_-]?secret/i,
+        ];
+        this.config = this.loadConfig();
+        if (this.config.output === 'file' || this.config.output === 'both') {
+            this.initializeOutputFile();
+        }
+    }
+    static getInstance() {
+        if (!TraceContext.instance) {
+            TraceContext.instance = new TraceContext();
+        }
+        return TraceContext.instance;
+    }
+    loadConfig() {
+        return {
+            enabled: process.env.DEBUG_TRACE === 'true' || process.env.STACKMEMORY_DEBUG === 'true',
+            verbosity: process.env.TRACE_VERBOSITY || 'full',
+            output: process.env.TRACE_OUTPUT || 'console',
+            includeParams: process.env.TRACE_PARAMS !== 'false',
+            includeResults: process.env.TRACE_RESULTS !== 'false',
+            maskSensitive: process.env.TRACE_MASK_SENSITIVE !== 'false',
+            performanceThreshold: parseInt(process.env.TRACE_PERF_THRESHOLD || '100'),
+            maxDepth: parseInt(process.env.TRACE_MAX_DEPTH || '20'),
+            captureMemory: process.env.TRACE_MEMORY === 'true',
+        };
+    }
+    initializeOutputFile() {
+        const traceDir = path.join(process.env.HOME || '.', '.stackmemory', 'traces');
+        if (!fs.existsSync(traceDir)) {
+            fs.mkdirSync(traceDir, { recursive: true });
+        }
+        const timestamp = new Date().toISOString().replace(/[:.]/g, '-');
+        this.outputFile = path.join(traceDir, `trace-${timestamp}.jsonl`);
+    }
+    maskSensitiveData(obj) {
+        if (!this.config.maskSensitive)
+            return obj;
+        if (typeof obj !== 'object' || obj === null)
+            return obj;
+        const masked = Array.isArray(obj) ? [...obj] : { ...obj };
+        for (const key in masked) {
+            if (typeof key === 'string') {
+                // Check if key matches sensitive patterns
+                const isSensitive = this.sensitivePatterns.some(pattern => pattern.test(key));
+                if (isSensitive) {
+                    masked[key] = '[MASKED]';
+                }
+                else if (typeof masked[key] === 'object') {
+                    masked[key] = this.maskSensitiveData(masked[key]);
+                }
+                else if (typeof masked[key] === 'string' && masked[key].length > 20) {
+                    // Check if value looks like a token/key
+                    if (/^[a-zA-Z0-9_-]{20,}$/.test(masked[key])) {
+                        masked[key] = masked[key].substring(0, 8) + '...[MASKED]';
+                    }
+                }
+            }
+        }
+        return masked;
+    }
+    captureMemory() {
+        if (!this.config.captureMemory)
+            return undefined;
+        return process.memoryUsage();
+    }
+    formatDuration(ms) {
+        if (ms < 1000)
+            return `${ms.toFixed(0)}ms`;
+        if (ms < 60000)
+            return `${(ms / 1000).toFixed(2)}s`;
+        return `${(ms / 60000).toFixed(2)}m`;
+    }
+    formatMemory(bytes) {
+        const mb = bytes / 1024 / 1024;
+        return `${mb.toFixed(2)}MB`;
+    }
+    getIndent(depth) {
+        return '  '.repeat(depth);
+    }
+    formatTraceEntry(entry, includeChildren = true) {
+        const indent = this.getIndent(entry.depth);
+        const duration = entry.duration ? ` [${this.formatDuration(entry.duration)}]` : '';
+        const memory = entry.memory?.delta
+            ? ` (Δmem: ${this.formatMemory(entry.memory.delta.heapUsed)})`
+            : '';
+        let output = `${indent}→ [${entry.type.toUpperCase()}:${entry.id.substring(0, 8)}] ${entry.name}${duration}${memory}`;
+        if (entry.error) {
+            output += `\n${indent}  ✗ ERROR: ${entry.error.message || entry.error}`;
+            if (entry.error.stack && this.config.verbosity === 'full') {
+                output += `\n${indent}    Stack: ${entry.error.stack.split('\n')[1]?.trim()}`;
+            }
+        }
+        if (this.config.includeParams && entry.params && Object.keys(entry.params).length > 0) {
+            const maskedParams = this.maskSensitiveData(entry.params);
+            output += `\n${indent}  ▸ Params: ${JSON.stringify(maskedParams, null, 2).replace(/\n/g, '\n' + indent + '    ')}`;
+        }
+        if (this.config.includeResults && entry.result !== undefined && !entry.error) {
+            const maskedResult = this.maskSensitiveData(entry.result);
+            const resultStr = JSON.stringify(maskedResult, null, 2);
+            if (resultStr.length < 200) {
+                output += `\n${indent}  ◂ Result: ${resultStr.replace(/\n/g, '\n' + indent + '    ')}`;
+            }
+            else {
+                output += `\n${indent}  ◂ Result: [${typeof maskedResult}] ${resultStr.substring(0, 100)}...`;
+            }
+        }
+        if (entry.duration && entry.duration > this.config.performanceThreshold) {
+            output += `\n${indent}  ⚠ SLOW: Exceeded ${this.config.performanceThreshold}ms threshold`;
+        }
+        if (includeChildren && entry.children.length > 0) {
+            for (const child of entry.children) {
+                output += '\n' + this.formatTraceEntry(child, true);
+            }
+        }
+        if (entry.endTime && entry.depth > 0) {
+            output += `\n${indent}← [${entry.type.toUpperCase()}:${entry.id.substring(0, 8)}] completed`;
+        }
+        return output;
+    }
+    outputTrace(entry) {
+        if (!this.config.enabled)
+            return;
+        const formatted = this.formatTraceEntry(entry, false);
+        if (this.config.output === 'console' || this.config.output === 'both') {
+            console.log(formatted);
+        }
+        if ((this.config.output === 'file' || this.config.output === 'both') && this.outputFile) {
+            const jsonLine = JSON.stringify({
+                ...entry,
+                formatted,
+                timestamp: new Date().toISOString(),
+            }) + '\n';
+            fs.appendFileSync(this.outputFile, jsonLine);
+        }
+    }
+    startTrace(type, name, params, metadata) {
+        if (!this.config.enabled)
+            return '';
+        const id = uuidv4();
+        const parentId = this.currentTrace?.id;
+        const depth = this.traceStack.length;
+        if (depth > this.config.maxDepth) {
+            return id; // Prevent infinite recursion
+        }
+        const entry = {
+            id,
+            parentId,
+            type,
+            name,
+            startTime: performance.now(),
+            depth,
+            params: this.config.includeParams ? params : undefined,
+            metadata,
+            children: [],
+            memory: this.captureMemory() ? { before: this.captureMemory() } : undefined,
+        };
+        if (this.currentTrace) {
+            this.currentTrace.children.push(entry);
+        }
+        else {
+            this.allTraces.push(entry);
+        }
+        this.traceStack.push(entry);
+        this.currentTrace = entry;
+        this.outputTrace(entry);
+        return id;
+    }
+    endTrace(id, result, error) {
+        if (!this.config.enabled)
+            return;
+        const index = this.traceStack.findIndex(t => t.id === id);
+        if (index === -1)
+            return;
+        const entry = this.traceStack[index];
+        entry.endTime = performance.now();
+        entry.duration = entry.endTime - entry.startTime;
+        entry.result = this.config.includeResults && !error ? result : undefined;
+        entry.error = error;
+        if (entry.memory?.before) {
+            entry.memory.after = this.captureMemory();
+            if (entry.memory.after) {
+                entry.memory.delta = {
+                    rss: entry.memory.after.rss - entry.memory.before.rss,
+                    heapUsed: entry.memory.after.heapUsed - entry.memory.before.heapUsed,
+                };
+            }
+        }
+        this.outputTrace(entry);
+        // Remove from stack and update current
+        this.traceStack.splice(index);
+        this.currentTrace = this.traceStack[this.traceStack.length - 1] || null;
+    }
+    async traceAsync(type, name, params, fn) {
+        const id = this.startTrace(type, name, params);
+        try {
+            const result = await fn();
+            this.endTrace(id, result);
+            return result;
+        }
+        catch (error) {
+            this.endTrace(id, undefined, error);
+            throw error;
+        }
+    }
+    traceSync(type, name, params, fn) {
+        const id = this.startTrace(type, name, params);
+        try {
+            const result = fn();
+            this.endTrace(id, result);
+            return result;
+        }
+        catch (error) {
+            this.endTrace(id, undefined, error);
+            throw error;
+        }
+    }
+    async command(name, options, fn) {
+        return this.traceAsync('command', name, options, fn);
+    }
+    async step(name, fn) {
+        return this.traceAsync('step', name, undefined, fn);
+    }
+    async query(sql, params, fn) {
+        return this.traceAsync('query', sql.substring(0, 50), params, fn);
+    }
+    async api(method, url, body, fn) {
+        return this.traceAsync('api', `${method} ${url}`, { body }, fn);
+    }
+    getExecutionSummary() {
+        if (!this.config.enabled)
+            return 'Tracing disabled';
+        const totalDuration = Date.now() - this.startTime;
+        const errorCount = this.countErrors(this.allTraces);
+        const slowCount = this.countSlowOperations(this.allTraces);
+        let summary = `\n${'='.repeat(80)}\n`;
+        summary += `EXECUTION SUMMARY\n`;
+        summary += `${'='.repeat(80)}\n`;
+        summary += `Total Duration: ${this.formatDuration(totalDuration)}\n`;
+        summary += `Total Operations: ${this.countOperations(this.allTraces)}\n`;
+        summary += `Errors: ${errorCount}\n`;
+        summary += `Slow Operations (>${this.config.performanceThreshold}ms): ${slowCount}\n`;
+        if (this.config.captureMemory) {
+            const memUsage = process.memoryUsage();
+            summary += `Final Memory: RSS=${this.formatMemory(memUsage.rss)}, Heap=${this.formatMemory(memUsage.heapUsed)}\n`;
+        }
+        if (this.outputFile) {
+            summary += `Trace Log: ${this.outputFile}\n`;
+        }
+        summary += `${'='.repeat(80)}`;
+        return summary;
+    }
+    countOperations(traces) {
+        let count = traces.length;
+        for (const trace of traces) {
+            count += this.countOperations(trace.children);
+        }
+        return count;
+    }
+    countErrors(traces) {
+        let count = 0;
+        for (const trace of traces) {
+            if (trace.error)
+                count++;
+            count += this.countErrors(trace.children);
+        }
+        return count;
+    }
+    countSlowOperations(traces) {
+        let count = 0;
+        for (const trace of traces) {
+            if (trace.duration && trace.duration > this.config.performanceThreshold)
+                count++;
+            count += this.countSlowOperations(trace.children);
+        }
+        return count;
+    }
+    getLastError() {
+        const findLastError = (traces) => {
+            for (let i = traces.length - 1; i >= 0; i--) {
+                const trace = traces[i];
+                if (trace.error)
+                    return trace;
+                const childError = findLastError(trace.children);
+                if (childError)
+                    return childError;
+            }
+            return null;
+        };
+        return findLastError(this.allTraces);
+    }
+    exportTraces() {
+        return this.allTraces;
+    }
+    reset() {
+        this.currentTrace = null;
+        this.traceStack = [];
+        this.allTraces = [];
+        this.startTime = Date.now();
+    }
+}
+// Singleton instance
+export const trace = TraceContext.getInstance();
+// Decorator for tracing class methods
+export function Trace(type = 'function') {
+    return function (target, propertyKey, descriptor) {
+        const originalMethod = descriptor.value;
+        const isAsync = originalMethod.constructor.name === 'AsyncFunction';
+        if (isAsync) {
+            descriptor.value = async function (...args) {
+                const className = target.constructor.name;
+                const methodName = `${className}.${propertyKey}`;
+                return trace.traceAsync(type, methodName, args, async () => {
+                    return originalMethod.apply(this, args);
+                });
+            };
+        }
+        else {
+            descriptor.value = function (...args) {
+                const className = target.constructor.name;
+                const methodName = `${className}.${propertyKey}`;
+                return trace.traceSync(type, methodName, args, () => {
+                    return originalMethod.apply(this, args);
+                });
+            };
+        }
+        return descriptor;
+    };
+}
+// Decorator for tracing entire classes
+export function TraceClass(type = 'function') {
+    return function (constructor) {
+        const prototype = constructor.prototype;
+        const propertyNames = Object.getOwnPropertyNames(prototype);
+        for (const propertyName of propertyNames) {
+            if (propertyName === 'constructor')
+                continue;
+            const descriptor = Object.getOwnPropertyDescriptor(prototype, propertyName);
+            if (!descriptor || typeof descriptor.value !== 'function')
+                continue;
+            Trace(type)(prototype, propertyName, descriptor);
+            Object.defineProperty(prototype, propertyName, descriptor);
+        }
+        return constructor;
+    };
+}
+// Helper for critical operations
+export function TraceCritical(target, propertyKey, descriptor) {
+    const originalMethod = descriptor.value;
+    descriptor.value = async function (...args) {
+        const className = target.constructor.name;
+        const methodName = `${className}.${propertyKey} [CRITICAL]`;
+        // Store context before execution
+        const contextBefore = {
+            memory: process.memoryUsage(),
+            timestamp: new Date().toISOString(),
+            args: trace['maskSensitiveData'](args),
+        };
+        try {
+            return await trace.traceAsync('function', methodName, contextBefore, async () => {
+                return originalMethod.apply(this, args);
+            });
+        }
+        catch (error) {
+            // Enhanced error logging for critical operations
+            logger.error(`Critical operation failed: ${methodName}`, error, {
+                context: contextBefore,
+                stack: error.stack,
+            });
+            throw error;
+        }
+    };
+    return descriptor;
+}