@anyshift/mcp-proxy 0.2.0 → 0.2.2

package/README.md

@@ -133,39 +133,82 @@ npx @anyshift/mcp-proxy
 
 ## How It Works
 
+```mermaid
+graph TB
+    AI[🤖 AI Agent<br/>Claude]
+
+    AI -->|MCP Protocol| Proxy
+
+    subgraph Proxy["@anyshift/mcp-proxy"]
+        Start[Receive tool call]
+        CheckJQ{JQ tool?}
+        ExecuteJQ[Execute JQ locally]
+        Forward[Forward to child MCP]
+        GetResponse[Get response from child]
+        CheckSize{Size ≥ 1000 chars?}
+        WriteFile[📄 Write FULL data to file]
+        ReturnFile[Return file reference]
+        CheckTrunc{Size > 40K chars?}
+        Truncate[Truncate with notice]
+        ReturnDirect[Return response]
+
+        Start --> CheckJQ
+        CheckJQ -->|Yes| ExecuteJQ
+        CheckJQ -->|No| Forward
+        Forward --> GetResponse
+        ExecuteJQ --> CheckTrunc
+        GetResponse --> CheckSize
+        CheckSize -->|Yes| WriteFile
+        WriteFile --> ReturnFile
+        CheckSize -->|No| CheckTrunc
+        CheckTrunc -->|Yes| Truncate
+        CheckTrunc -->|No| ReturnDirect
+
+        ReturnFile -.->|📄 Small reference| AI
+        Truncate -.->|Truncated text| AI
+        ReturnDirect -.->|Full response| AI
+    end
+
+    Proxy -->|stdio + env vars| Child[Child MCP<br/>anyshift/datadog/grafana]
+    Child -.->|Response| Proxy
+
+    style AI fill:#e1f5ff
+    style Proxy fill:#fff4e1
+    style Child fill:#e8f5e9
+    style WriteFile fill:#c8e6c9
+    style ReturnFile fill:#c8e6c9
 ```
-┌─────────────┐
-│  AI Agent   │
-└──────┬──────┘
-       │ MCP Protocol (stdio)
-       ▼
-┌─────────────────────────────────┐
-│    @anyshift/mcp-proxy          │
-│                                 │
-│  1. Spawns child MCP            │
-│     with pass-through env vars  │
-│                                 │
-│  2. Discovers child tools       │
-│     + adds JQ tool              │
-│                                 │
-│  3. Forwards tool calls         │
-│                                 │
-│  4. Applies truncation          │
-│     if response > MAX_TOKENS    │
-│                                 │
-│  5. Writes to file              │
-│     if response > MIN_CHARS     │
-│                                 │
-│  6. Returns modified response   │
-│     to AI agent                 │
-└────────────┬────────────────────┘
-             │ Child process (stdio)
-             ▼
-      ┌──────────────┐
-      │  Child MCP   │  (mcp-grafana, custom-mcp, etc.)
-      │    Server    │  Gets env vars WITHOUT MCP_PROXY_ prefix
-      └──────────────┘
+
+### Response Handling Examples
+
+**Small responses (< 1,000 chars):**
+```
+Child: 500 chars → Proxy: Return directly → AI: 500 chars ✓
+```
+
+**Medium responses (1,000 - 40,000 chars):**
+```
+Child: 5,000 chars → Proxy: Write to file → AI: "📄 File: path/to/file.json" ✓
+```
+
+**Large responses (> 40,000 chars):**
+```
+Child: 100,000 chars → Proxy: Write FULL 100K to file → AI: "📄 File: ..." ✓
+Note: File contains complete data, not truncated!
+```
+
+**JQ tool queries:**
 ```
+AI: JQ query → Proxy: Execute locally → AI: Result (truncated if > 40K) ✓
+```
+
+### Key Design Principle
+
+**File writing happens BEFORE truncation.** This ensures:
+- Files always contain complete, untruncated data
+- Large responses are accessible via file references
+- AI receives small, manageable responses
+- No data loss due to context limits
 
 ## Integration Examples
 

package/dist/__tests__/helpers/testUtils.d.ts

@@ -0,0 +1,127 @@
+/**
+ * Test utilities and helpers for MCP Proxy tests
+ */
+/**
+ * Create an MCP response with specified character count
+ * @param charCount - Number of characters in the response
+ * @param jsonData - Optional JSON data to include (overrides charCount)
+ * @returns MCP-formatted response
+ */
+export declare function createMCPResponse(charCount: number, jsonData?: object): {
+    content: {
+        type: string;
+        text: string;
+    }[];
+};
+/**
+ * Create an MCP response with specific JSON data
+ */
+export declare function createMCPResponseWithJSON(jsonData: object): {
+    content: {
+        type: string;
+        text: string;
+    }[];
+};
+/**
+ * Create an error response
+ */
+export declare function createErrorResponse(errorMessage: string): {
+    content: {
+        type: string;
+        text: string;
+    }[];
+    isError: boolean;
+};
+/**
+ * Extract file path from file reference response
+ */
+export declare function extractFilePath(result: any): string;
+/**
+ * Extract reported file size from file reference
+ */
+export declare function extractReportedSize(result: any): number;
+/**
+ * Check if response is a file reference
+ */
+export declare function isFileReference(result: any): boolean;
+/**
+ * Check if response is truncated
+ */
+export declare function isTruncated(result: any): boolean;
+/**
+ * Create a mock file system for testing
+ */
+export declare class MockFileSystem {
+    private files;
+    writeFile(path: string, content: string): Promise<void>;
+    readFile(path: string): Promise<string>;
+    existsSync(path: string): boolean;
+    clear(): void;
+    getFiles(): string[];
+}
+/**
+ * Test fixtures for common scenarios
+ */
+export declare const fixtures: {
+    smallResponse: {
+        content: {
+            type: string;
+            text: string;
+        }[];
+    };
+    mediumResponse: {
+        content: {
+            type: string;
+            text: string;
+        }[];
+    };
+    largeResponse: {
+        content: {
+            type: string;
+            text: string;
+        }[];
+    };
+    atThreshold: {
+        content: {
+            type: string;
+            text: string;
+        }[];
+    };
+    justBelowThreshold: {
+        content: {
+            type: string;
+            text: string;
+        }[];
+    };
+    atTruncationLimit: {
+        content: {
+            type: string;
+            text: string;
+        }[];
+    };
+    overTruncationLimit: {
+        content: {
+            type: string;
+            text: string;
+        }[];
+    };
+    errorResponse: {
+        content: {
+            type: string;
+            text: string;
+        }[];
+        isError: boolean;
+    };
+    jsonResponse: {
+        content: {
+            type: string;
+            text: string;
+        }[];
+    };
+    withPagination: {
+        content: {
+            type: string;
+            text: string;
+        }[];
+    };
+};

package/dist/__tests__/helpers/testUtils.js

@@ -0,0 +1,122 @@
+/**
+ * Test utilities and helpers for MCP Proxy tests
+ */
+/**
+ * Create an MCP response with specified character count
+ * @param charCount - Number of characters in the response
+ * @param jsonData - Optional JSON data to include (overrides charCount)
+ * @returns MCP-formatted response
+ */
+export function createMCPResponse(charCount, jsonData) {
+    const content = jsonData
+        ? JSON.stringify(jsonData)
+        : 'x'.repeat(Math.max(0, charCount - 50)) + JSON.stringify({ padding: true });
+    return {
+        content: [{ type: 'text', text: content }]
+    };
+}
+/**
+ * Create an MCP response with specific JSON data
+ */
+export function createMCPResponseWithJSON(jsonData) {
+    return {
+        content: [{ type: 'text', text: JSON.stringify(jsonData) }]
+    };
+}
+/**
+ * Create an error response
+ */
+export function createErrorResponse(errorMessage) {
+    return {
+        content: [{
+                type: 'text',
+                text: JSON.stringify({ status: 'error', message: errorMessage })
+            }],
+        isError: true
+    };
+}
+/**
+ * Extract file path from file reference response
+ */
+export function extractFilePath(result) {
+    const text = result.content[0].text;
+    const match = text.match(/📄 File: (.+)/);
+    if (!match) {
+        throw new Error('No file path found in response');
+    }
+    return match[1].split('\n')[0].trim();
+}
+/**
+ * Extract reported file size from file reference
+ */
+export function extractReportedSize(result) {
+    const text = result.content[0].text;
+    const match = text.match(/Size: (\d+) characters/);
+    if (!match) {
+        throw new Error('No size information found in response');
+    }
+    return parseInt(match[1], 10);
+}
+/**
+ * Check if response is a file reference
+ */
+export function isFileReference(result) {
+    return result.content &&
+        result.content[0] &&
+        result.content[0].text &&
+        result.content[0].text.includes('📄 File:');
+}
+/**
+ * Check if response is truncated
+ */
+export function isTruncated(result) {
+    return result.content &&
+        result.content[0] &&
+        result.content[0].text &&
+        result.content[0].text.includes('=== RESPONSE TRUNCATED ===');
+}
+/**
+ * Create a mock file system for testing
+ */
+export class MockFileSystem {
+    files = new Map();
+    async writeFile(path, content) {
+        this.files.set(path, content);
+    }
+    async readFile(path) {
+        const content = this.files.get(path);
+        if (content === undefined) {
+            throw new Error(`File not found: ${path}`);
+        }
+        return content;
+    }
+    existsSync(path) {
+        return this.files.has(path);
+    }
+    clear() {
+        this.files.clear();
+    }
+    getFiles() {
+        return Array.from(this.files.keys());
+    }
+}
+/**
+ * Test fixtures for common scenarios
+ */
+export const fixtures = {
+    smallResponse: createMCPResponse(500),
+    mediumResponse: createMCPResponse(5000),
+    largeResponse: createMCPResponse(100000),
+    atThreshold: createMCPResponse(1000),
+    justBelowThreshold: createMCPResponse(999),
+    atTruncationLimit: createMCPResponse(40000),
+    overTruncationLimit: createMCPResponse(50000),
+    errorResponse: createErrorResponse('Test error'),
+    jsonResponse: createMCPResponseWithJSON({ data: { nested: 'value' } }),
+    withPagination: createMCPResponseWithJSON({
+        data: [{ item: 1 }, { item: 2 }],
+        pagination: { page: 1, total: 10 },
+        has_more: true,
+        next_page: '/api/page/2'
+    })
+};

package/dist/__tests__/unit/queryAssistSchema.test.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/__tests__/unit/queryAssistSchema.test.js

@@ -0,0 +1,267 @@
+import { describe, it, expect } from '@jest/globals';
+import { generateQueryAssistSchema } from '../../fileWriter/schema.js';
+describe('Query-Assist Schema Generator', () => {
+    describe('Basic Structure Detection', () => {
+        it('should detect simple object structure', () => {
+            const data = {
+                id: '123',
+                name: 'Test',
+                count: 42
+            };
+            const schema = generateQueryAssistSchema(data, { maxDepth: 2, maxPaths: 20 });
+            expect(schema).toContain('📊 STRUCTURE GUIDE');
+            expect(schema).toContain('.id');
+            expect(schema).toContain('.name');
+            expect(schema).toContain('.count');
+            expect(schema).toContain('string');
+            expect(schema).toContain('number');
+        });
+        it('should detect array structure', () => {
+            const data = {
+                items: [
+                    { id: '1', price: 100 },
+                    { id: '2', price: 200 }
+                ]
+            };
+            // With maxDepth=3, we can see array item fields
+            const schema = generateQueryAssistSchema(data, { maxDepth: 3, maxPaths: 20 });
+            expect(schema).toContain('.items');
+            expect(schema).toContain('array[2]');
+            expect(schema).toContain('.items[].id');
+            expect(schema).toContain('.items[].price');
+        });
+        it('should detect nested object structure', () => {
+            const data = {
+                user: {
+                    profile: {
+                        name: 'Alice',
+                        age: 30
+                    }
+                }
+            };
+            const schema = generateQueryAssistSchema(data, { maxDepth: 2, maxPaths: 20 });
+            expect(schema).toContain('.user');
+            expect(schema).toContain('.user.profile');
+            expect(schema).toContain('object');
+        });
+    });
+    describe('Depth Limiting', () => {
+        it('should stop at max depth and show warning', () => {
+            const data = {
+                level1: {
+                    level2: {
+                        level3: {
+                            level4: 'too deep'
+                        }
+                    }
+                }
+            };
+            const schema = generateQueryAssistSchema(data, { maxDepth: 2, maxPaths: 20 });
+            // Should show paths up to depth 2
+            expect(schema).toContain('.level1.level2');
+            // Should show depth limit warning in consolidated format
+            expect(schema).toContain('⚠️  Limits: DEPTH (max: 2)');
+            // level3 can appear in exploration prompts, but not as a path entry
+            // Check that level3 is not shown as a separate path line
+            const lines = schema.split('\n');
+            const pathLines = lines.filter(l => l.includes('→') && l.includes('.level'));
+            expect(pathLines.some(l => l.includes('.level1.level2.level3') && l.includes(' → '))).toBe(false);
+        });
+        it('should show exploration prompts when depth limit hit', () => {
+            const data = {
+                deep: {
+                    nested: {
+                        value: 'hidden'
+                    }
+                }
+            };
+            const schema = generateQueryAssistSchema(data, { maxDepth: 1, maxPaths: 20 });
+            expect(schema).toContain('💡 EXPLORATION GUIDE');
+            expect(schema).toContain('⚠️  Limits: DEPTH (max: 1)');
+            expect(schema).toContain('View keys:');
+            expect(schema).toContain('Check type:');
+        });
+    });
+    describe('Key Limiting', () => {
+        it('should limit keys per object and show warning', () => {
+            const data = {};
+            for (let i = 0; i < 100; i++) {
+                data[`field_${i}`] = i;
+            }
+            const schema = generateQueryAssistSchema(data, { maxDepth: 2, maxPaths: 100, maxKeys: 20 });
+            expect(schema).toContain('💡 EXPLORATION GUIDE');
+            expect(schema).toContain('⚠️  Limits: KEYS (20 shown, 80 more)');
+        });
+        it('should show key exploration prompts when limit hit', () => {
+            const data = {};
+            for (let i = 0; i < 60; i++) {
+                data[`key${i}`] = `value${i}`;
+            }
+            const schema = generateQueryAssistSchema(data, { maxDepth: 2, maxPaths: 100, maxKeys: 30 });
+            expect(schema).toContain('💡 EXPLORATION GUIDE');
+            expect(schema).toContain('⚠️  Limits: KEYS (30 shown, 30 more)');
+            expect(schema).toContain('View keys:');
+            expect(schema).toContain('Count items:');
+        });
+    });
+    describe('Path Limiting', () => {
+        it('should limit total paths shown and prioritize important ones', () => {
+            // Create data with many fields to exceed path limit
+            const data = {
+                id: '123',
+                name: 'Test'
+            };
+            // Add many top-level fields
+            for (let i = 0; i < 20; i++) {
+                data[`field${i}`] = { nested: `value${i}` };
+            }
+            const schema = generateQueryAssistSchema(data, { maxDepth: 2, maxPaths: 10, maxKeys: 50 });
+            // Should show path limit warning in consolidated format
+            expect(schema).toContain('💡 EXPLORATION GUIDE');
+            expect(schema).toContain('PATHS (10 of');
+            // Should show some paths (prioritizes objects over primitives)
+            expect(schema).toContain('.field');
+            // Count number of path lines shown (should be exactly 10)
+            const pathLines = schema.split('\n').filter(l => l.includes(' → '));
+            expect(pathLines.length).toBeLessThanOrEqual(11); // 10 paths + root = 11
+        });
+        it('should show path exploration prompts when limit hit', () => {
+            const largeData = {};
+            for (let i = 0; i < 30; i++) {
+                largeData[`field${i}`] = { nested: 'value' };
+            }
+            const schema = generateQueryAssistSchema(largeData, { maxDepth: 2, maxPaths: 10, maxKeys: 50 });
+            expect(schema).toContain('💡 EXPLORATION GUIDE');
+            expect(schema).toContain('PATHS (10 of');
+            expect(schema).toContain('View keys:');
+            expect(schema).toContain('List all paths:');
+        });
+    });
+    describe('Mixed Schema Detection', () => {
+        it('should detect heterogeneous arrays', () => {
+            const data = {
+                items: [
+                    { type: 'book', pages: 200 },
+                    { type: 'video', duration: 120 },
+                    { type: 'audio', length: 180 }
+                ]
+            };
+            const schema = generateQueryAssistSchema(data, { maxDepth: 2, maxPaths: 20 });
+            expect(schema).toContain('💡 EXPLORATION GUIDE');
+            expect(schema).toContain('MIXED SCHEMAS');
+        });
+        it('should show mixed schema exploration prompts', () => {
+            const data = {
+                data: [
+                    { a: 1 },
+                    { b: 2 },
+                    { c: 3 }
+                ]
+            };
+            const schema = generateQueryAssistSchema(data, { maxDepth: 2, maxPaths: 20 });
+            expect(schema).toContain('💡 EXPLORATION GUIDE');
+            expect(schema).toContain('MIXED SCHEMAS');
+            expect(schema).toContain('Check variance:');
+        });
+    });
+    describe('Numeric Keys Detection', () => {
+        it('should detect numeric string keys and show representative structure', () => {
+            const data = {
+                '0': { name: 'Alice', age: 30 },
+                '1': { name: 'Bob', age: 25 },
+                '2': { name: 'Charlie', age: 35 }
+            };
+            const schema = generateQueryAssistSchema(data, { maxDepth: 3, maxPaths: 20 });
+            // Should detect numeric keys
+            expect(schema).toContain('object (numeric keys)');
+            expect(schema).toContain('(3 keys)');
+            // Should show representative structure with .[<idx>] notation
+            expect(schema).toContain('.[<idx>]');
+            // Should show nested structure of representative item
+            expect(schema).toContain('.[<idx>].name');
+            expect(schema).toContain('.[<idx>].age');
+            // Should show explanation note in exploration guide
+            expect(schema).toContain('💡 EXPLORATION GUIDE');
+            expect(schema).toContain('NUMERIC KEYS:');
+            expect(schema).toContain('.["0"], .["1"]');
+            expect(schema).toContain('.[0], .[1]');
+            // Should NOT enumerate individual keys
+            expect(schema).not.toContain('.0 ');
+            expect(schema).not.toContain('.1 ');
+            expect(schema).not.toContain('.2 ');
+        });
+    });
+    describe('Nullable Fields', () => {
+        it('should detect null values', () => {
+            const data = {
+                present: 'value',
+                missing: null,
+                empty: ''
+            };
+            const schema = generateQueryAssistSchema(data, { maxDepth: 2, maxPaths: 20 });
+            expect(schema).toContain('.missing');
+            expect(schema).toContain('null');
+            expect(schema).toContain('(nullable)');
+        });
+    });
+    describe('Schema Compactness', () => {
+        it('should not include common JQ patterns (moved to tool description)', () => {
+            const data = { simple: 'data' };
+            const schema = generateQueryAssistSchema(data, { maxDepth: 2, maxPaths: 20 });
+            // Common JQ patterns are now in the JQ tool description, not in every file reference
+            expect(schema).not.toContain('COMMON JQ PATTERNS:');
+            expect(schema).not.toContain('List all keys:');
+            // Schema should still contain the structure guide header
+            expect(schema).toContain('📊 STRUCTURE GUIDE');
+        });
+    });
+    describe('Size Constraints', () => {
+        it('should generate compact output for large structures', () => {
+            // Create a large nested structure
+            const data = {};
+            for (let i = 0; i < 100; i++) {
+                data[`key${i}`] = {
+                    nested: {
+                        deep: {
+                            value: i
+                        }
+                    }
+                };
+            }
+            const schema = generateQueryAssistSchema(data, {
+                maxDepth: 2,
+                maxPaths: 20,
+                maxKeys: 50
+            });
+            // Schema should be compact (under 5KB as designed)
+            expect(schema.length).toBeLessThan(5000);
+            // Should contain warnings about limits
+            expect(schema).toContain('⚠️');
+        });
+    });
+    describe('Empty Data Handling', () => {
+        it('should handle empty object', () => {
+            const data = {};
+            const schema = generateQueryAssistSchema(data, { maxDepth: 2, maxPaths: 20 });
+            expect(schema).toContain('📊 STRUCTURE GUIDE');
+            expect(schema).toContain('(root)');
+            expect(schema).toContain('object');
+            expect(schema).toContain('(0 keys)');
+        });
+        it('should handle empty array', () => {
+            const data = { items: [] };
+            const schema = generateQueryAssistSchema(data, { maxDepth: 2, maxPaths: 20 });
+            expect(schema).toContain('.items');
+            expect(schema).toContain('array[0]');
+        });
+    });
+    describe('Data Size Reporting', () => {
+        it('should report data size in characters', () => {
+            const data = { test: 'value' };
+            const schema = generateQueryAssistSchema(data, { maxDepth: 2, maxPaths: 20 });
+            expect(schema).toContain('Size:');
+            expect(schema).toContain('characters');
+            expect(schema).toMatch(/Size: \d+(,\d{3})* characters/);
+        });
+    });
+});

package/dist/__tests__/unit/truncation.test.d.ts

@@ -0,0 +1,5 @@
+/**
+ * Unit tests for truncation module
+ * Tests token estimation and truncation logic
+ */
+export {};