@smallironman/mcp-memory-keeper 0.12.2-fork1

package/dist/__tests__/integration/issue24-fix-validation.test.js

@@ -0,0 +1,158 @@
+"use strict";
+// Test to validate the fix for issue #24
+// Tests the actual flow through index.ts handler
+Object.defineProperty(exports, "__esModule", { value: true });
+const globals_1 = require("@jest/globals");
+// Mock the functions used in index.ts for context_get
+function estimateTokens(text) {
+    return Math.ceil(text.length / 4);
+}
+function calculateSize(value) {
+    return Buffer.byteLength(value, 'utf8');
+}
+function calculateResponseMetrics(items) {
+    let totalSize = 0;
+    for (const item of items) {
+        const itemSize = item.size || calculateSize(item.value);
+        totalSize += itemSize;
+    }
+    // Convert to JSON string to get actual response size
+    const jsonString = JSON.stringify(items);
+    const estimatedTokens = estimateTokens(jsonString);
+    const averageSize = items.length > 0 ? Math.round(totalSize / items.length) : 0;
+    return { totalSize, estimatedTokens, averageSize };
+}
+(0, globals_1.describe)('Issue #24 Fix Validation', () => {
+    (0, globals_1.it)('should correctly calculate tokens for response with metadata', () => {
+        // Create sample items as they would come from the database
+        const dbItems = [];
+        for (let i = 0; i < 100; i++) {
+            dbItems.push({
+                id: `id-${i}`,
+                session_id: 'test-session',
+                key: `test_item_${i}`,
+                value: `This is test content that is moderately long to simulate real data. `.repeat(3),
+                category: 'task',
+                priority: 'high',
+                channel: 'test',
+                metadata: JSON.stringify({ index: i }),
+                size: 200,
+                created_at: '2025-01-20T10:00:00Z',
+                updated_at: '2025-01-20T10:00:00Z',
+            });
+        }
+        // Test without metadata (original calculation)
+        const metricsWithoutMetadata = calculateResponseMetrics(dbItems);
+        console.log('Without metadata - Items:', dbItems.length, 'Tokens:', metricsWithoutMetadata.estimatedTokens);
+        // Test with metadata (as per the fix)
+        const itemsWithMetadata = dbItems.map(item => ({
+            key: item.key,
+            value: item.value,
+            category: item.category,
+            priority: item.priority,
+            channel: item.channel,
+            metadata: item.metadata ? JSON.parse(item.metadata) : null,
+            size: item.size || calculateSize(item.value),
+            created_at: item.created_at,
+            updated_at: item.updated_at,
+        }));
+        const metricsWithMetadata = calculateResponseMetrics(itemsWithMetadata);
+        console.log('With metadata - Items:', itemsWithMetadata.length, 'Tokens:', metricsWithMetadata.estimatedTokens);
+        // The metadata version might have fewer tokens due to JSON parsing
+        // but the overall response structure adds overhead
+        // Build full response structure as the handler does
+        const fullResponse = {
+            items: itemsWithMetadata,
+            pagination: {
+                total: 100,
+                returned: 100,
+                offset: 0,
+                hasMore: false,
+                nextOffset: null,
+                totalCount: 100,
+                page: 1,
+                pageSize: 100,
+                totalPages: 1,
+                hasNextPage: false,
+                hasPreviousPage: false,
+                previousOffset: null,
+                totalSize: metricsWithMetadata.totalSize,
+                averageSize: metricsWithMetadata.averageSize,
+                defaultsApplied: { limit: true, sort: true },
+                truncated: false,
+                truncatedCount: 0,
+            },
+        };
+        const finalResponseJson = JSON.stringify(fullResponse, null, 2);
+        const finalTokens = estimateTokens(finalResponseJson);
+        console.log('Final response - Size:', finalResponseJson.length, 'Tokens:', finalTokens);
+        // This demonstrates the issue: the final response can be much larger
+        // than what calculateResponseMetrics estimates
+        console.log('Token difference:', finalTokens - metricsWithMetadata.estimatedTokens);
+        // With 100 items and metadata, we should be approaching or exceeding limits
+        if (finalTokens > 18000) {
+            console.log('WARNING: Response exceeds safe token limit!');
+        }
+    });
+    (0, globals_1.it)('should demonstrate that 50 items with metadata stays under limit', () => {
+        // Create sample items
+        const dbItems = [];
+        for (let i = 0; i < 50; i++) {
+            dbItems.push({
+                id: `id-${i}`,
+                session_id: 'test-session',
+                key: `test_item_${i}`,
+                value: `This is test content that is moderately long to simulate real data. `.repeat(3),
+                category: 'task',
+                priority: 'high',
+                channel: 'test',
+                metadata: JSON.stringify({ index: i }),
+                size: 200,
+                created_at: '2025-01-20T10:00:00Z',
+                updated_at: '2025-01-20T10:00:00Z',
+            });
+        }
+        // Transform with metadata
+        const itemsWithMetadata = dbItems.map(item => ({
+            key: item.key,
+            value: item.value,
+            category: item.category,
+            priority: item.priority,
+            channel: item.channel,
+            metadata: item.metadata ? JSON.parse(item.metadata) : null,
+            size: item.size || calculateSize(item.value),
+            created_at: item.created_at,
+            updated_at: item.updated_at,
+        }));
+        const metrics = calculateResponseMetrics(itemsWithMetadata);
+        // Build full response
+        const fullResponse = {
+            items: itemsWithMetadata,
+            pagination: {
+                total: 50,
+                returned: 50,
+                offset: 0,
+                hasMore: false,
+                nextOffset: null,
+                totalCount: 50,
+                page: 1,
+                pageSize: 50,
+                totalPages: 1,
+                hasNextPage: false,
+                hasPreviousPage: false,
+                previousOffset: null,
+                totalSize: metrics.totalSize,
+                averageSize: metrics.averageSize,
+                defaultsApplied: { limit: true, sort: true },
+                truncated: false,
+                truncatedCount: 0,
+            },
+        };
+        const finalResponseJson = JSON.stringify(fullResponse, null, 2);
+        const finalTokens = estimateTokens(finalResponseJson);
+        console.log('50 items with metadata - Tokens:', finalTokens);
+        // 50 items should be safe
+        (0, globals_1.expect)(finalTokens).toBeLessThan(18000);
+        (0, globals_1.expect)(finalTokens).toBeLessThan(25000); // Well under MCP limit
+    });
+});

package/dist/__tests__/integration/issue24-reproduce.test.js

@@ -0,0 +1,225 @@
+"use strict";
+// Attempt to reproduce the exact issue from #24
+// User reported: "Error: MCP tool "context_get" response (26866 tokens) exceeds maximum allowed tokens (25000)"
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+const database_1 = require("../../utils/database");
+const RepositoryManager_1 = require("../../repositories/RepositoryManager");
+const os = __importStar(require("os"));
+const path = __importStar(require("path"));
+const fs = __importStar(require("fs"));
+function estimateTokens(text) {
+    return Math.ceil(text.length / 4);
+}
+function calculateSize(value) {
+    return Buffer.byteLength(value, 'utf8');
+}
+describe('Issue #24 - Reproduce exact error', () => {
+    let dbManager;
+    let repositories;
+    let tempDbPath;
+    let testSessionId;
+    beforeEach(() => {
+        tempDbPath = path.join(os.tmpdir(), `test-issue24-reproduce-${Date.now()}.db`);
+        dbManager = new database_1.DatabaseManager({
+            filename: tempDbPath,
+            maxSize: 10 * 1024 * 1024,
+            walMode: true,
+        });
+        repositories = new RepositoryManager_1.RepositoryManager(dbManager);
+        // Create test session
+        const session = repositories.sessions.create({
+            name: 'Issue #24 Reproduce',
+            description: 'Attempting to reproduce exact token overflow',
+        });
+        testSessionId = session.id;
+    });
+    afterEach(() => {
+        dbManager.close();
+        try {
+            fs.unlinkSync(tempDbPath);
+            fs.unlinkSync(`${tempDbPath}-wal`);
+            fs.unlinkSync(`${tempDbPath}-shm`);
+        }
+        catch (_e) {
+            // Ignore
+        }
+    });
+    it('should reproduce token overflow with realistic data', () => {
+        // To get 26866 tokens, we need roughly 107,464 characters
+        // Let's create items with realistic content that would cause this
+        const itemCount = 150; // More items than default limit
+        const largeContent = `
+This is a realistic context item that might be saved during a coding session.
+It contains multiple lines of information, including:
+- Task descriptions and implementation details
+- Code snippets and examples
+- Decision rationale and architectural notes
+- Progress updates and status information
+- Error messages and debugging information
+- References to files and functions
+- Links and external resources
+
+The content is substantial enough to represent real-world usage where developers
+save important context about their work. This might include detailed explanations
+of complex logic, API documentation, configuration settings, or troubleshooting notes.
+
+Additional metadata might include timestamps, categories, priorities, and custom
+tags that help organize and retrieve the information later. All of this contributes
+to the overall size of the response when multiple items are returned together.
+`.trim();
+        // Create many items with substantial content
+        for (let i = 0; i < itemCount; i++) {
+            repositories.contexts.save(testSessionId, {
+                key: `context_item_${String(i).padStart(3, '0')}`,
+                value: `${largeContent}\n\nItem ${i} specific notes: ${String(i).repeat(20)}`,
+                category: ['task', 'decision', 'progress', 'note'][i % 4],
+                priority: ['high', 'normal', 'low'][i % 3],
+                channel: `channel-${i % 5}`,
+                metadata: JSON.stringify({
+                    index: i,
+                    timestamp: new Date().toISOString(),
+                    tags: ['important', 'review', 'todo', 'done'][i % 4],
+                    additionalData: {
+                        relatedFiles: [`file${i}.ts`, `test${i}.spec.ts`],
+                        relatedIssues: [`#${i * 10}`, `#${i * 10 + 1}`],
+                    },
+                }),
+            });
+        }
+        // Query without specifying limit (should use default)
+        const result = repositories.contexts.queryEnhanced({
+            sessionId: testSessionId,
+            includeMetadata: true,
+            // No limit specified - will use repository default of 100
+        });
+        console.log(`Created ${itemCount} items, retrieved ${result.items.length} items`);
+        // Transform items with metadata as the handler does
+        const itemsWithMetadata = result.items.map(item => ({
+            key: item.key,
+            value: item.value,
+            category: item.category,
+            priority: item.priority,
+            channel: item.channel,
+            metadata: item.metadata ? JSON.parse(item.metadata) : null,
+            size: item.size || calculateSize(item.value),
+            created_at: item.created_at,
+            updated_at: item.updated_at,
+        }));
+        // Create the full response structure
+        const fullResponse = {
+            items: itemsWithMetadata,
+            pagination: {
+                total: result.totalCount,
+                returned: result.items.length,
+                offset: 0,
+                hasMore: result.totalCount > result.items.length,
+                nextOffset: result.items.length < result.totalCount ? result.items.length : null,
+                totalCount: result.totalCount,
+                page: 1,
+                pageSize: result.items.length,
+                totalPages: Math.ceil(result.totalCount / result.items.length),
+                hasNextPage: result.totalCount > result.items.length,
+                hasPreviousPage: false,
+                previousOffset: null,
+                totalSize: itemsWithMetadata.reduce((sum, item) => sum + (item.size || 0), 0),
+                averageSize: Math.round(itemsWithMetadata.reduce((sum, item) => sum + (item.size || 0), 0) / result.items.length),
+                defaultsApplied: { limit: true, sort: true },
+                truncated: false,
+                truncatedCount: 0,
+            },
+        };
+        const responseJson = JSON.stringify(fullResponse, null, 2);
+        const tokens = estimateTokens(responseJson);
+        console.log(`Response size: ${responseJson.length} bytes`);
+        console.log(`Estimated tokens: ${tokens}`);
+        console.log(`Average item size: ${Math.round(responseJson.length / result.items.length)} bytes`);
+        // Check if we're reproducing the issue
+        if (tokens > 25000) {
+            console.log('✅ Successfully reproduced token overflow!');
+            console.log(`Tokens: ${tokens} > 25000 limit`);
+        }
+        else if (tokens > 20000) {
+            console.log('⚠️ Approaching token limit but not exceeding');
+        }
+        else {
+            console.log('❌ Did not reproduce token overflow');
+            console.log('Need larger items or more items to reproduce');
+        }
+        // The fix should prevent this from happening
+        // With our changes:
+        // 1. Default limit is 50 when includeMetadata is true (at handler level)
+        // 2. Token limit is 18000 (more conservative)
+        // 3. Response is truncated if it exceeds limits
+    });
+    it('should show how the fix prevents overflow', () => {
+        // Same setup but showing how limit of 50 prevents the issue
+        const itemCount = 150;
+        const largeContent = `Large content...`.repeat(50); // Smaller for this test
+        for (let i = 0; i < itemCount; i++) {
+            repositories.contexts.save(testSessionId, {
+                key: `item_${i}`,
+                value: largeContent,
+                category: 'task',
+                priority: 'high',
+            });
+        }
+        // With the fix, when includeMetadata is true, default limit should be 50
+        // But this is handled at the handler level, not repository level
+        // Repository still defaults to 100
+        const resultWith50 = repositories.contexts.queryEnhanced({
+            sessionId: testSessionId,
+            includeMetadata: true,
+            limit: 50, // Explicitly set to what our fix would use
+        });
+        const itemsWithMetadata = resultWith50.items.map(item => ({
+            key: item.key,
+            value: item.value,
+            category: item.category,
+            priority: item.priority,
+            channel: item.channel,
+            metadata: item.metadata ? JSON.parse(item.metadata) : null,
+            size: item.size || calculateSize(item.value),
+            created_at: item.created_at,
+            updated_at: item.updated_at,
+        }));
+        const responseJson = JSON.stringify({ items: itemsWithMetadata }, null, 2);
+        const tokens = estimateTokens(responseJson);
+        console.log(`With limit=50: ${resultWith50.items.length} items, ${tokens} tokens`);
+        // Should be well under limit
+        expect(tokens).toBeLessThan(18000);
+        expect(tokens).toBeLessThan(25000);
+    });
+});

package/dist/__tests__/integration/issue24-token-limit.test.js

@@ -0,0 +1,199 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+const database_1 = require("../../utils/database");
+const RepositoryManager_1 = require("../../repositories/RepositoryManager");
+const os = __importStar(require("os"));
+const path = __importStar(require("path"));
+const fs = __importStar(require("fs"));
+describe('Issue #24 - Token Limit with includeMetadata', () => {
+    let dbManager;
+    let repositories;
+    let tempDbPath;
+    let testSessionId;
+    beforeEach(() => {
+        tempDbPath = path.join(os.tmpdir(), `test-issue24-${Date.now()}.db`);
+        dbManager = new database_1.DatabaseManager({
+            filename: tempDbPath,
+            maxSize: 10 * 1024 * 1024,
+            walMode: true,
+        });
+        repositories = new RepositoryManager_1.RepositoryManager(dbManager);
+        // Create test session
+        const session = repositories.sessions.create({
+            name: 'Issue #24 Test Session',
+            description: 'Testing token limit with metadata',
+        });
+        testSessionId = session.id;
+    });
+    afterEach(() => {
+        dbManager.close();
+        try {
+            fs.unlinkSync(tempDbPath);
+            fs.unlinkSync(`${tempDbPath}-wal`);
+            fs.unlinkSync(`${tempDbPath}-shm`);
+        }
+        catch (_e) {
+            // Ignore
+        }
+    });
+    it('should not exceed token limit when includeMetadata is true with many items', () => {
+        // Create many items with substantial content to simulate real-world scenario
+        const itemCount = 200; // Create enough items to potentially exceed token limits
+        for (let i = 0; i < itemCount; i++) {
+            repositories.contexts.save(testSessionId, {
+                key: `test_item_${i}`,
+                value: `This is a test item with a moderately long value that simulates real-world context data.
+                It includes multiple lines and enough content to make the response substantial when
+                many items are returned together. Each item contributes to the total token count.
+                Item number: ${i}. Additional metadata and timestamps will be included when
+                includeMetadata is set to true, further increasing the response size.`.repeat(2),
+                category: i % 2 === 0 ? 'task' : 'decision',
+                priority: i % 3 === 0 ? 'high' : 'normal',
+                channel: 'test-channel',
+                metadata: JSON.stringify({
+                    index: i,
+                    timestamp: new Date().toISOString(),
+                    tags: ['test', 'issue24', 'metadata'],
+                }),
+            });
+        }
+        // Query with includeMetadata: true (this was causing the issue)
+        const result = repositories.contexts.queryEnhanced({
+            sessionId: testSessionId,
+            includeMetadata: true,
+            // No limit specified - repository uses default of 100
+            // The handler level would apply dynamic limits, but repository doesn't know about includeMetadata
+        });
+        // Repository default is 100 items (it doesn't know about metadata)
+        // The dynamic limiting happens at the handler level in index.ts
+        expect(result.items.length).toBeLessThanOrEqual(100);
+        expect(result.totalCount).toBe(itemCount);
+        // Simulate the response construction with metadata
+        const itemsWithMetadata = result.items.map(item => ({
+            key: item.key,
+            value: item.value,
+            category: item.category,
+            priority: item.priority,
+            channel: item.channel,
+            metadata: item.metadata ? JSON.parse(item.metadata) : null,
+            size: item.size || Buffer.byteLength(item.value, 'utf8'),
+            created_at: item.created_at,
+            updated_at: item.updated_at,
+        }));
+        // Create the full response structure
+        const response = {
+            items: itemsWithMetadata,
+            pagination: {
+                total: result.totalCount,
+                returned: result.items.length,
+                offset: 0,
+                hasMore: result.totalCount > result.items.length,
+                nextOffset: result.items.length < result.totalCount ? result.items.length : null,
+                totalCount: result.totalCount,
+                page: 1,
+                pageSize: 50,
+                totalPages: Math.ceil(result.totalCount / 50),
+                hasNextPage: result.totalCount > result.items.length,
+                hasPreviousPage: false,
+                previousOffset: null,
+                totalSize: 0,
+                averageSize: 0,
+                defaultsApplied: { limit: true, sort: true },
+                truncated: false,
+                truncatedCount: 0,
+            },
+        };
+        // Calculate token estimate for the response
+        const responseJson = JSON.stringify(response, null, 2);
+        const estimatedTokens = Math.ceil(responseJson.length / 4);
+        console.log(`Response size: ${responseJson.length} bytes`);
+        console.log(`Estimated tokens: ${estimatedTokens}`);
+        console.log(`Items returned: ${result.items.length} of ${result.totalCount}`);
+        // Note: Repository returns 100 items by default, which may exceed token limits
+        // This demonstrates why the handler level needs to apply dynamic limiting
+        // The handler would truncate this response to prevent token overflow
+        console.log(`Repository returned ${result.items.length} items with ${estimatedTokens} tokens`);
+        // This shows the problem: repository default of 100 can exceed limits
+        if (estimatedTokens > 25000) {
+            console.log('This demonstrates why dynamic limiting at handler level is needed!');
+        }
+    });
+    it('should handle explicit high limit with metadata by truncating', () => {
+        // Create many items
+        const itemCount = 500;
+        for (let i = 0; i < itemCount; i++) {
+            repositories.contexts.save(testSessionId, {
+                key: `large_item_${i}`,
+                value: `Large content ${i}`.repeat(50), // Make items larger
+                category: 'task',
+                priority: 'high',
+                channel: 'large-channel',
+            });
+        }
+        // Query with a high explicit limit and metadata
+        const result = repositories.contexts.queryEnhanced({
+            sessionId: testSessionId,
+            includeMetadata: true,
+            limit: 1000, // Explicitly request many items
+        });
+        // Repository will cap at the actual number of items available
+        // Since we created 500 items, we should get all 500
+        // The token limiting would happen at the handler level, not repository level
+        expect(result.items.length).toBe(500); // Gets all items since limit > itemCount
+        // Calculate response size
+        const itemsWithMetadata = result.items.map(item => ({
+            key: item.key,
+            value: item.value,
+            category: item.category,
+            priority: item.priority,
+            channel: item.channel,
+            metadata: item.metadata ? JSON.parse(item.metadata) : null,
+            size: item.size || Buffer.byteLength(item.value, 'utf8'),
+            created_at: item.created_at,
+            updated_at: item.updated_at,
+        }));
+        const responseJson = JSON.stringify({ items: itemsWithMetadata }, null, 2);
+        const estimatedTokens = Math.ceil(responseJson.length / 4);
+        console.log(`Large query - Items: ${result.items.length}, Tokens: ${estimatedTokens}`);
+        // This test shows that without handler-level limiting, large queries would overflow
+        // The handler's dynamic limiting would truncate this to safe levels
+        if (estimatedTokens > 25000) {
+            console.log(`Would need truncation: ${estimatedTokens} tokens exceeds 25000 limit`);
+        }
+        // The test demonstrates the need for handler-level dynamic limiting
+        expect(result.items.length).toBe(500); // Repository returns what was requested
+    });
+});