claude-mem 3.2.0 → 3.2.1

package/dist/core/compression/TranscriptCompressor.js

@@ -1,602 +0,0 @@
-import { query } from '@anthropic-ai/claude-code';
-import fs, { createWriteStream } from 'fs';
-import path, { join } from 'path';
-import os from 'os';
-import { PathResolver } from '../../shared/paths.js';
-import { PromptOrchestrator, createAnalysisContext } from '../orchestration/PromptOrchestrator.js';
-import { DEBUG_MESSAGES } from '../../constants.js';
-import { log } from '../../shared/logger.js';
-/**
- * TranscriptCompressor handles the analysis and compression of Claude Code conversation transcripts
- * into a searchable memory database format using the Model Context Protocol.
- */
-export class TranscriptCompressor {
-    paths;
-    logStream = null;
-    logFile = null;
-    promptOrchestrator;
-    // <Block> 1.1 ====================================
-    // Constructor Initialization - Natural flow (8/10)
-    constructor(options = {}) {
-        this.paths = new PathResolver();
-        this.promptOrchestrator = new PromptOrchestrator();
-        this.ensureClaudeMemStructure();
-        this.initializeLogging();
-        log.debug('🤖 TranscriptCompressor initialized');
-    }
-    // </Block> =======================================
-    // <Block> 1.2 ====================================
-    // Directory Structure Validation - Natural flow (8/10)
-    /**
-     * Ensures that the required directory structure exists
-     */
-    ensureClaudeMemStructure() {
-        const configDir = this.paths.getConfigDir();
-        const indexDir = this.paths.getIndexDir();
-        const archiveDir = this.paths.getArchiveDir();
-        const logsDir = this.paths.getLogsDir();
-        PathResolver.ensureDirectories([configDir, indexDir, archiveDir, logsDir]);
-    }
-    initializeLogging() {
-        const logsDir = this.paths.getLogsDir();
-        const timestamp = new Date().toISOString().replace(/[:.]/g, '-');
-        this.logFile = join(logsDir, `claude-mem-${timestamp}.log`);
-        this.logStream = createWriteStream(this.logFile, { flags: 'a' });
-        this.debugLog('🚀 DEBUG LOG STARTED');
-        this.debugLog(`📁 Log file: ${this.logFile}`);
-        this.debugLog('═'.repeat(60));
-    }
-    debugLog(message) {
-        if (!this.logStream)
-            return;
-        const timestamp = new Date().toISOString();
-        const logLine = `[${timestamp}] ${message}\n`;
-        this.logStream.write(logLine);
-    }
-    closeLogging() {
-        if (this.logStream) {
-            this.debugLog('✅ DEBUG LOG ENDED');
-            this.logStream.end();
-        }
-    }
-    // </Block> =======================================
-    // <Block> 1.3 ====================================
-    // </Block> =======================================
-    // <Block> 1.4 ====================================
-    // Main Compression Flow - DEBUG GUARDS INTERRUPT FLOW (5/10)
-    /**
-     * Main compression method that processes a transcript and creates compressed memories
-     */
-    async compress(transcriptPath, sessionId) {
-        this.debugLog(`🚀 Starting compression for: ${transcriptPath}`);
-        this.debugLog(`📋 Session ID: ${sessionId || 'auto-generated'}`);
-        try {
-            const projectPrefix = PathResolver.getCurrentProjectPrefix();
-            log.debug(DEBUG_MESSAGES.PROJECT_NAME(projectPrefix));
-            this.debugLog(`📝 PROJECT PREFIX: ${projectPrefix}`);
-            // Read and parse transcript
-            const content = fs.readFileSync(transcriptPath, 'utf-8');
-            this.debugLog(`📖 Reading transcript: ${content.length} bytes`);
-            const lines = content.trim().split('\n').filter(line => line.trim());
-            const messages = [];
-            let parseErrors = 0;
-            for (let i = 0; i < lines.length; i++) {
-                try {
-                    const parsed = JSON.parse(lines[i]);
-                    messages.push(parsed);
-                }
-                catch (e) {
-                    parseErrors++;
-                    log.debug(`Parse error on line ${i + 1}: ${e.message}`);
-                }
-            }
-            log.debug(DEBUG_MESSAGES.TRANSCRIPT_STATS(content.length, messages.length));
-            if (parseErrors > 0) {
-                log.debug(`Parse errors: ${parseErrors}`);
-            }
-            this.debugLog(`📊 Transcript loaded: ${lines.length} lines, ${messages.length} messages, ${parseErrors} parse errors`);
-            // Generate final session ID
-            const finalSessionId = sessionId || path.basename(transcriptPath, '.jsonl');
-            // Get timestamp from first message or use current time
-            const firstMessage = messages.find(m => m.timestamp);
-            let timestamp = new Date().toISOString();
-            if (firstMessage?.timestamp) {
-                const ts = Number(firstMessage.timestamp);
-                // Check if timestamp is in seconds (Unix) or milliseconds
-                // Unix timestamps are typically 10 digits, JS timestamps are 13
-                const dateValue = ts < 10000000000 ? ts * 1000 : ts;
-                try {
-                    timestamp = new Date(dateValue).toISOString();
-                }
-                catch (e) {
-                    // Fall back to current time if timestamp is invalid
-                    this.debugLog(`⚠️ Invalid timestamp: ${firstMessage.timestamp}, using current time`);
-                }
-            }
-            // Archive filename for reference
-            const archiveFilename = `${finalSessionId}.jsonl.archive`;
-            // Format conversation for analysis
-            const conversationText = this.formatConversationForPrompt(messages);
-            // Create analysis prompt using PromptOrchestrator
-            const analysisContext = createAnalysisContext(conversationText, finalSessionId, {
-                projectName: projectPrefix,
-                trigger: 'manual'
-            });
-            const analysisPrompt = this.promptOrchestrator.createAnalysisPrompt(analysisContext);
-            log.debug('📤 Analysis prompt created');
-            log.debug(`📊 Prompt length: ${analysisPrompt.prompt.length} characters`);
-            // LOG THE FULL PROMPT TO DEBUG FILE
-            const promptDebugPath = path.join(this.paths.getLogsDir(), `claude-prompt-${Date.now()}.txt`);
-            fs.writeFileSync(promptDebugPath, `=== CLAUDE ANALYSIS PROMPT ===\n${analysisPrompt.prompt}\n`);
-            this.debugLog(`📝 Full prompt saved to: ${promptDebugPath}`);
-            // Find Claude Code executable and MCP config
-            const claudePath = this.findClaudeExecutable();
-            const mcpConfigPath = this.findMCPConfig();
-            if (claudePath) {
-                log.debug(DEBUG_MESSAGES.CLAUDE_PATH_FOUND(claudePath));
-            }
-            if (mcpConfigPath) {
-                log.debug(DEBUG_MESSAGES.MCP_CONFIG_USED(mcpConfigPath));
-            }
-            // Call Claude SDK for analysis  
-            this.debugLog('🤖 Calling Claude SDK with MCP tools...');
-            const response = await query({
-                prompt: analysisPrompt.prompt,
-                options: {
-                    allowedTools: [
-                        'mcp__claude-mem__chroma_list_collections',
-                        'mcp__claude-mem__chroma_create_collection',
-                        'mcp__claude-mem__chroma_peek_collection',
-                        'mcp__claude-mem__chroma_get_collection_info',
-                        'mcp__claude-mem__chroma_get_collection_count',
-                        'mcp__claude-mem__chroma_modify_collection',
-                        'mcp__claude-mem__chroma_fork_collection',
-                        'mcp__claude-mem__chroma_delete_collection',
-                        'mcp__claude-mem__chroma_add_documents',
-                        'mcp__claude-mem__chroma_query_documents',
-                        'mcp__claude-mem__chroma_get_documents',
-                        'mcp__claude-mem__chroma_update_documents',
-                        'mcp__claude-mem__chroma_delete_documents',
-                    ],
-                    pathToClaudeCodeExecutable: '/Users/alexnewman/.nvm/versions/node/v24.5.0/bin/claude',
-                },
-            });
-            this.debugLog('✅ Claude SDK response received');
-            // Process response and extract summaries from JSON
-            this.debugLog('🔄 Processing Claude JSON response...');
-            const extractResult = await this.processClaudeResponse(response);
-            this.debugLog(`📋 Extracted ${extractResult.summaries.length} summaries from JSON`);
-            if (extractResult.overview) {
-                this.debugLog(`📝 Overview: ${extractResult.overview}`);
-            }
-            log.debug(DEBUG_MESSAGES.COMPRESSION_COMPLETE(extractResult.summaries.length));
-            // Continue processing even with zero summaries - let the natural flow handle empty results
-            // Create archive and update index
-            const archivePath = this.createArchive(transcriptPath, projectPrefix, finalSessionId, content);
-            this.debugLog(`📦 Archive created: ${archivePath}`);
-            this.appendToIndex(extractResult.summaries, extractResult.overview, projectPrefix, finalSessionId, messages, archivePath);
-            this.debugLog(`📥 Written ${extractResult.summaries.length} summaries to index`);
-            log.debug(`✅ SUCCESS`);
-            log.debug(`Archive created: ${archivePath}`);
-            log.debug(`Summaries created: ${extractResult.summaries.length}`);
-            this.debugLog('✅ Compression completed successfully');
-            this.closeLogging();
-            return archivePath;
-        }
-        catch (error) {
-            log.error('COMPRESSION FAILED', error, {
-                transcriptPath,
-                sessionId
-            });
-            this.debugLog(`❌ ERROR: ${error instanceof Error ? error.message : String(error)}`);
-            this.closeLogging();
-            throw error;
-        }
-    }
-    // </Block> =======================================
-    /**
-     * Finds Claude Code executable in common locations
-     */
-    findClaudeExecutable() {
-        const possiblePaths = [
-            '/opt/homebrew/bin/claude',
-            '/usr/local/bin/claude',
-            process.env.CLAUDE_CODE_PATH,
-        ].filter(Boolean);
-        for (const path of possiblePaths) {
-            if (path && fs.existsSync(path)) {
-                return path;
-            }
-        }
-        return undefined;
-    }
-    /**
-     * Finds MCP configuration file
-     */
-    findMCPConfig() {
-        const possibleConfigs = [
-            join(process.cwd(), '.mcp.json'),
-            join(os.homedir(), '.claude.json'),
-            join(os.homedir(), '.claude', '.mcp.json'),
-        ];
-        const mcpConfigPath = possibleConfigs.find(fs.existsSync);
-        return mcpConfigPath || join(os.homedir(), '.claude.json');
-    }
-    // <Block> 1.5 ====================================
-    // Claude Response Processing - JSON extraction with pipe-separated output (9/10)
-    /**
-     * Processes Claude response to extract summaries from JSON
-     */
-    async processClaudeResponse(response) {
-        let fullContent = '';
-        // Extract content using polymorphic handlers
-        fullContent = await this.extractResponseContent(response, []);
-        // DEBUG: Log the full content to see what Claude is returning
-        this.debugLog(`🔍 Claude response content length: ${fullContent.length}`);
-        // Write raw response to debug file for troubleshooting
-        const debugPath = path.join(this.paths.getLogsDir(), `claude-response-${Date.now()}.txt`);
-        fs.writeFileSync(debugPath, `=== CLAUDE RAW RESPONSE ===\n${fullContent}\n`);
-        this.debugLog(`📝 Raw response saved to: ${debugPath}`);
-        // Extract JSON from response tags
-        const extractResult = this.extractJSONResponse(fullContent);
-        this.debugLog(`📊 Extracted ${extractResult.summaries.length} summaries from JSON`);
-        if (extractResult.summaries.length === 0) {
-            this.debugLog(`⚠️ No summaries found in JSON response`);
-        }
-        return extractResult;
-    }
-    /**
-     * Extracts content from response
-     */
-    async extractResponseContent(response, summaries) {
-        // Handle streaming response
-        if (response && typeof response === 'object' && Symbol.asyncIterator in response) {
-            let content = '';
-            for await (const message of response) {
-                content += this.extractMessageContent(message);
-                if (message?.type === 'result' && message?.result) {
-                    content = message.result;
-                }
-            }
-            return content;
-        }
-        // Handle string response
-        if (typeof response === 'string') {
-            return response;
-        }
-        // Handle array response
-        if (Array.isArray(response)) {
-            return response.map(item => {
-                if (typeof item === 'string')
-                    return item;
-                if (item?.text)
-                    return item.text;
-                if (item?.content)
-                    return item.content;
-                return '';
-            }).filter(Boolean).join('\n');
-        }
-        // Handle object response
-        if (typeof response === 'object' && response !== null) {
-            if (response?.text)
-                return response.text;
-            if (response?.content)
-                return response.content;
-            if (response?.message)
-                return response.message;
-            return '';
-        }
-        return '';
-    }
-    /**
-     * Extracts content from a single message
-     */
-    extractMessageContent(message) {
-        let content = '';
-        if (message?.content)
-            content += message.content;
-        if (message?.text)
-            content += message.text;
-        if (message?.data)
-            content += message.data;
-        if (message?.message?.content && Array.isArray(message.message.content)) {
-            message.message.content.forEach((item) => {
-                if (item.type === 'text' && item.text) {
-                    content += item.text;
-                }
-            });
-        }
-        return content;
-    }
-    /**
-     * Extracts JSON response and returns raw JSON objects
-     */
-    extractJSONResponse(content) {
-        try {
-            // Extract JSON from response tags
-            const jsonMatch = content.match(/<JSONResponse>([\s\S]*?)<\/JSONResponse>/);
-            if (!jsonMatch) {
-                this.debugLog(`⚠️ No <JSONResponse> tags found in response`);
-                return { overview: null, summaries: [] };
-            }
-            const jsonContent = jsonMatch[1].trim();
-            this.debugLog(`✅ Found JSON response: ${jsonContent.length} chars`);
-            // Parse the JSON
-            const parsed = JSON.parse(jsonContent);
-            if (!parsed.summaries || !Array.isArray(parsed.summaries)) {
-                this.debugLog(`⚠️ Invalid JSON structure: missing summaries array`);
-                return { overview: null, summaries: [] };
-            }
-            // Return raw JSON objects instead of converting to pipe-separated format
-            const validSummaries = [];
-            parsed.summaries.forEach((summary, index) => {
-                if (!summary.text || !summary.document_id) {
-                    this.debugLog(`⚠️ Skipping invalid summary at index ${index}`);
-                    return;
-                }
-                // Ensure required fields are present
-                const validSummary = {
-                    text: summary.text,
-                    document_id: summary.document_id,
-                    keywords: summary.keywords || '',
-                    timestamp: summary.timestamp || new Date().toISOString(),
-                    archive: summary.archive || `${summary.document_id}.jsonl.archive`
-                };
-                validSummaries.push(validSummary);
-                this.debugLog(`✅ Valid summary ${index + 1}: ${summary.document_id}`);
-            });
-            // Store overview if present
-            if (parsed.overview) {
-                this.debugLog(`📝 Session overview: ${parsed.overview}`);
-            }
-            return { overview: parsed.overview || null, summaries: validSummaries };
-        }
-        catch (error) {
-            this.debugLog(`❌ Failed to parse JSON response: ${error}`);
-            // Fallback: try to extract any pipe-separated lines that might exist
-            this.debugLog(`🔄 Attempting fallback to pipe-separated format...`);
-            const legacyLines = this.extractLegacyPipeSeparatedLines(content);
-            // Convert legacy lines to JSON format for consistency
-            const legacySummaries = legacyLines.map((line, index) => {
-                const parts = line.split(' | ');
-                return {
-                    text: parts[0] || '',
-                    document_id: parts[1] || `legacy_${Date.now()}_${index}`,
-                    keywords: parts[2] || '',
-                    timestamp: parts[3] || new Date().toISOString(),
-                    archive: parts[4] || `legacy_${Date.now()}_${index}.jsonl.archive`
-                };
-            });
-            return { overview: null, summaries: legacySummaries };
-        }
-    }
-    /**
-     * Legacy fallback for pipe-separated format
-     */
-    extractLegacyPipeSeparatedLines(content) {
-        const lines = content.split('\n');
-        const pipeLines = [];
-        lines.forEach((line) => {
-            const trimmed = line.trim();
-            if (trimmed && trimmed.includes(' | ') && trimmed.split(' | ').length >= 3) {
-                pipeLines.push(trimmed);
-            }
-        });
-        this.debugLog(`📊 Fallback extracted ${pipeLines.length} pipe-separated lines`);
-        return pipeLines;
-    }
-    // </Block> =======================================
-    // <Block> 1.7 ====================================
-    // Conversation Formatting - LONG BUT MOSTLY NATURAL (6/10)
-    /**
-     * Formats conversation messages for analysis prompt
-     */
-    formatConversationForPrompt(messages) {
-        const pipeLines = [];
-        messages.forEach((m, index) => {
-            const role = m.type === 'assistant' ? 'assistant'
-                : m.type === 'user' ? 'user'
-                    : (m.type === 'result' || m.type === 'system' || m.type === 'summary') ? 'system'
-                        : m.message?.role || m.role;
-            const content = this.extractContent(m);
-            const sessionId = m.session_id || '';
-            const timestamp = this.normalizeTimestamp(m);
-            const messageUuid = m.uuid || '';
-            // Escape pipe characters in content to prevent format corruption
-            const escapedContent = content.replace(/\|/g, '\\|');
-            // Format: content | session_id | role | timestamp | message_uuid
-            const pipeLine = `${escapedContent} | ${sessionId} | ${role} | ${timestamp} | ${messageUuid}`;
-            pipeLines.push(pipeLine);
-        });
-        log.debug(`Field filtering complete: ${pipeLines.length} messages processed`);
-        return `<!-- TRANSCRIPT -->\n${pipeLines.join('\n')}\n<!-- /TRANSCRIPT -->`;
-    }
-    // </Block> =======================================
-    // <Block> 1.6 ====================================
-    // Message Content Extraction - Simplified (8/10)
-    /**
-     * Extracts content from message object
-     */
-    extractContent(m) {
-        let content = '';
-        // Extract by type
-        if (m.type === 'assistant' || m.type === 'user') {
-            const messageContent = m.message?.content;
-            if (Array.isArray(messageContent)) {
-                content = messageContent
-                    .map((item) => item.text || item.content || '')
-                    .filter(Boolean)
-                    .join(' ');
-            }
-            else if (messageContent) {
-                content = String(messageContent).trim();
-            }
-        }
-        else if (m.type === 'summary') {
-            // Handle summary messages that have a different structure
-            content = m.summary || '';
-        }
-        else if (m.type === 'result') {
-            if (m.subtype === 'success' && m.result) {
-                content = `[Result: ${m.result}]`;
-            }
-            else if (m.subtype === 'error_max_turns') {
-                content = '[Error: Maximum turns reached]';
-            }
-            else if (m.subtype === 'error_during_execution') {
-                content = '[Error during execution]';
-            }
-        }
-        else if (m.type === 'system') {
-            if (m.subtype === 'init') {
-                content = `[System initialized: ${m.model}, tools: ${m.tools?.length || 0}, MCP servers: ${m.mcp_servers?.length || 0}]`;
-            }
-            else {
-                // Handle other system messages
-                content = String(m.content || '').trim();
-            }
-        }
-        // Fallback to generic content extraction
-        if (!content) {
-            content = String(m.message?.content || m.content || '');
-            if (Array.isArray(content)) {
-                content = content
-                    .map((item) => item.text || item.content || '')
-                    .filter(Boolean)
-                    .join(' ');
-            }
-        }
-        // Append tool use result if present
-        if (m.toolUseResult) {
-            const toolSummary = this.summarizeToolResult(m.toolUseResult, content);
-            if (toolSummary) {
-                content = content ? `${content}\n\n${toolSummary}` : toolSummary;
-            }
-        }
-        return String(content).trim();
-    }
-    // </Block> =======================================
-    /**
-     * Summarizes tool use results
-     */
-    summarizeToolResult(toolResult, existingContent) {
-        const summaryParts = [];
-        if (toolResult.stdout) {
-            const stdout = String(toolResult.stdout);
-            if (stdout.length > 200) {
-                const lineCount = stdout.split('\n').length;
-                const charCount = stdout.length;
-                const lines = stdout.split('\n');
-                const preview = lines.slice(0, 3).join('\n');
-                const suffix = lines.length > 6 ? `\n...\n${lines.slice(-2).join('\n')}` : '';
-                summaryParts.push(`Result: ${preview}${suffix} (${lineCount} lines, ${charCount} chars)`);
-            }
-            else {
-                summaryParts.push(`Result: ${stdout}`);
-            }
-        }
-        if (toolResult.stderr && toolResult.stderr.trim()) {
-            summaryParts.push(`Error: ${toolResult.stderr}`);
-        }
-        if (toolResult.interrupted) {
-            summaryParts.push('(interrupted)');
-        }
-        if (toolResult.isImage) {
-            summaryParts.push('(image output)');
-        }
-        return summaryParts.join('\n');
-    }
-    /**
-     * Normalizes timestamp formats
-     */
-    normalizeTimestamp(m) {
-        const ts = m.timestamp || m.message?.timestamp || m.created_at || m.message?.created_at;
-        if (!ts)
-            return '';
-        try {
-            const date = new Date(ts);
-            if (isNaN(date.getTime()))
-                return '';
-            return date.toISOString().replace('T', ' ');
-        }
-        catch (e) {
-            log.debug(`Invalid timestamp format: ${ts}`);
-            return '';
-        }
-    }
-    // <Block> 1.8 ====================================
-    // Archive Creation - Natural flow (9/10)
-    /**
-     * Creates an archive file of the original transcript
-     */
-    createArchive(transcriptPath, projectPrefix, sessionId, content) {
-        const projectArchiveDir = this.paths.getProjectArchiveDir(projectPrefix);
-        PathResolver.ensureDirectory(projectArchiveDir);
-        const archivePath = join(projectArchiveDir, `${sessionId}.jsonl.archive`);
-        fs.writeFileSync(archivePath, content);
-        log.debug(`📦 Created archive: ${archivePath}`);
-        return archivePath;
-    }
-    // </Block> =======================================
-    /**
-     * Appends summaries in JSONL format to the index file
-     * Each line is a JSON object with type field for easy parsing
-     */
-    appendToIndex(summaries, overview, projectPrefix, sessionId, messages, archivePath) {
-        // Use PathResolver's getIndexPath() for consistency
-        const indexPath = this.paths.getIndexPath();
-        const indexDir = this.paths.getConfigDir();
-        PathResolver.ensureDirectory(indexDir);
-        const timestamp = new Date().toISOString();
-        // Write session header as JSON object
-        const sessionHeader = {
-            type: "session",
-            session_id: sessionId,
-            timestamp: timestamp,
-            project: projectPrefix
-        };
-        fs.appendFileSync(indexPath, JSON.stringify(sessionHeader) + '\n');
-        // Add overview as JSON object if present
-        if (overview) {
-            const overviewObj = {
-                type: "overview",
-                content: overview,
-                session_id: sessionId,
-                project: projectPrefix,
-                timestamp: timestamp
-            };
-            fs.appendFileSync(indexPath, JSON.stringify(overviewObj) + '\n');
-        }
-        // If no summaries from Claude, write diagnostic info
-        if (!summaries || summaries.length === 0) {
-            log.debug('📝 No summaries extracted from JSON response');
-            const diagnosticObj = {
-                type: "diagnostic",
-                message: "NO SUMMARIES EXTRACTED - Check logs for valid JSON response",
-                session_id: sessionId,
-                project: projectPrefix,
-                timestamp: timestamp
-            };
-            fs.appendFileSync(indexPath, JSON.stringify(diagnosticObj) + '\n');
-            this.debugLog(`⚠️ No summaries for session ${sessionId} - Check if Claude returned valid JSON in <JSONResponse> tags`);
-        }
-        else {
-            // Write each summary as JSONL memory object
-            summaries.forEach((summary) => {
-                const memoryObj = {
-                    type: "memory",
-                    text: summary.text,
-                    document_id: summary.document_id,
-                    keywords: summary.keywords,
-                    session_id: sessionId,
-                    project: projectPrefix,
-                    timestamp: summary.timestamp || timestamp,
-                    archive: path.basename(archivePath)
-                };
-                fs.appendFileSync(indexPath, JSON.stringify(memoryObj) + '\n');
-            });
-            log.debug(`📝 Appended ${summaries.length} summaries to index as JSONL`);
-        }
-        log.debug(`Index path: ${indexPath}`);
-    }
-}