@anyshift/mcp-proxy 0.3.2 → 0.3.4

package/dist/fileWriter/schema.d.ts

@@ -32,6 +32,10 @@ export interface QueryAssistOptions {
     maxKeys?: number;
     dataSize?: number;
 }
+export interface JsonlSchemaOptions extends QueryAssistOptions {
+    totalLines: number;
+    sampleSize?: number;
+}
 /**
  * Generate query-assist schema for JSON data
  * Main entry point for schema generation
@@ -40,3 +44,10 @@ export interface QueryAssistOptions {
  * @returns Compact text schema optimized for JQ queries
  */
 export declare function generateQueryAssistSchema(data: unknown, options?: QueryAssistOptions): string;
+/**
+ * Generate schema for JSONL data by sampling multiple records
+ * @param lines - Array of raw JSON lines
+ * @param options - Configuration options including totalLines and sampleSize
+ * @returns Formatted text schema optimized for JQ queries on JSONL files
+ */
+export declare function generateJsonlQueryAssistSchema(lines: string[], options: JsonlSchemaOptions): string;

package/dist/fileWriter/schema.js

@@ -268,3 +268,117 @@ export function generateQueryAssistSchema(data, options = {}) {
     // Format as text
     return formatQueryAssist(selectedPaths, limits, dataSize);
 }
+/**
+ * Select sample indices evenly distributed through the data
+ */
+function selectSampleIndices(totalLines, sampleSize) {
+    if (totalLines <= sampleSize) {
+        return Array.from({ length: totalLines }, (_, i) => i);
+    }
+    const indices = [0]; // Always include first
+    const step = Math.floor(totalLines / (sampleSize - 1));
+    for (let i = 1; i < sampleSize - 1; i++) {
+        indices.push(i * step);
+    }
+    indices.push(totalLines - 1); // Always include last
+    return indices;
+}
+/**
+ * Analyze schema consistency across sampled records
+ */
+function analyzeSchemaConsistency(records) {
+    if (records.length < 2) {
+        return { isConsistent: true, optionalFields: [] };
+    }
+    // Collect all keys from all records
+    const allKeys = new Set();
+    const keyPresenceCount = new Map();
+    for (const record of records) {
+        if (record && typeof record === 'object' && !Array.isArray(record)) {
+            const keys = Object.keys(record);
+            for (const key of keys) {
+                allKeys.add(key);
+                keyPresenceCount.set(key, (keyPresenceCount.get(key) || 0) + 1);
+            }
+        }
+    }
+    // Find optional fields (not present in all records)
+    const optionalFields = [];
+    for (const key of allKeys) {
+        if (keyPresenceCount.get(key) !== records.length) {
+            optionalFields.push(key);
+        }
+    }
+    return {
+        isConsistent: optionalFields.length === 0,
+        optionalFields
+    };
+}
+/**
+ * Generate schema for JSONL data by sampling multiple records
+ * @param lines - Array of raw JSON lines
+ * @param options - Configuration options including totalLines and sampleSize
+ * @returns Formatted text schema optimized for JQ queries on JSONL files
+ */
+export function generateJsonlQueryAssistSchema(lines, options) {
+    const { totalLines, sampleSize = 5 } = options;
+    const parts = [];
+    parts.push('📊 JSONL STRUCTURE GUIDE (for JQ queries)');
+    parts.push('');
+    parts.push(`Format: JSON Lines (${totalLines.toLocaleString()} records)`);
+    if (options.dataSize) {
+        parts.push(`Size: ${options.dataSize.toLocaleString()} characters`);
+    }
+    parts.push('');
+    // JQ usage guide for JSONL
+    parts.push('JQ USAGE (JSONL files):');
+    parts.push('  • Stream records:     jq -c "." file.jsonl');
+    parts.push('  • Filter:             jq -c "select(.field == value)" file.jsonl');
+    parts.push('  • Extract field:      jq -r ".fieldName" file.jsonl');
+    parts.push('  • Count records:      jq -s "length" file.jsonl');
+    parts.push('  • First N records:    jq -s ".[:N]" file.jsonl');
+    parts.push('  • Unique values:      jq -s "[.[].field] | unique" file.jsonl');
+    parts.push('');
+    // Sample records evenly distributed through the file
+    const indicesToSample = selectSampleIndices(totalLines, sampleSize);
+    const sampledRecords = [];
+    for (const idx of indicesToSample) {
+        if (lines[idx]) {
+            try {
+                sampledRecords.push(JSON.parse(lines[idx].trim()));
+            }
+            catch {
+                // Skip unparseable lines
+            }
+        }
+    }
+    if (sampledRecords.length === 0) {
+        parts.push('⚠️ Could not parse any records');
+        return parts.join('\n');
+    }
+    // Analyze schema consistency across samples
+    const schemaAnalysis = analyzeSchemaConsistency(sampledRecords);
+    if (schemaAnalysis.isConsistent) {
+        parts.push(`RECORD STRUCTURE (consistent across ${sampledRecords.length} sampled records):`);
+    }
+    else {
+        parts.push(`RECORD STRUCTURE (⚠️ variations detected in ${sampledRecords.length} samples):`);
+        if (schemaAnalysis.optionalFields.length > 0) {
+            parts.push(`  Optional fields: ${schemaAnalysis.optionalFields.join(', ')}`);
+        }
+    }
+    parts.push('');
+    // Generate schema from first record (representative)
+    const recordSchema = generateQueryAssistSchema(sampledRecords[0], {
+        maxDepth: options.maxDepth,
+        maxPaths: options.maxPaths,
+        maxKeys: options.maxKeys,
+    });
+    // Indent the record schema
+    const indentedSchema = recordSchema
+        .split('\n')
+        .map(l => '  ' + l)
+        .join('\n');
+    parts.push(indentedSchema);
+    return parts.join('\n');
+}

package/dist/fileWriter/writer.js

@@ -1,10 +1,58 @@
 import fs from 'fs/promises';
 import path from 'path';
 import { generateToolId } from '../utils/filename.js';
-import { generateQueryAssistSchema } from './schema.js';
+import { generateQueryAssistSchema, generateJsonlQueryAssistSchema } from './schema.js';
 import { parseDynatraceDqlResponse, isDynatraceDqlTool, } from './dynatrace.js';
 // Default minimum character count to trigger file writing
 const DEFAULT_MIN_CHARS = 1000;
+/**
+ * Detect whether content is JSON, JSONL, or plain text
+ */
+function detectContentFormat(content) {
+    const trimmed = content.trim();
+    // Empty content
+    if (!trimmed)
+        return 'text';
+    // Try parsing as single JSON object/array
+    if (trimmed.startsWith('{') || trimmed.startsWith('[')) {
+        try {
+            JSON.parse(trimmed);
+            return 'json';
+        }
+        catch {
+            // Could be JSONL starting with {
+        }
+    }
+    // Check for JSONL (multiple JSON objects, one per line)
+    const lines = trimmed.split('\n').filter(l => l.trim());
+    if (lines.length > 1) {
+        // Sample first few lines to avoid parsing huge files
+        const sampleSize = Math.min(lines.length, 10);
+        const allSampledLinesAreJson = lines.slice(0, sampleSize).every(line => {
+            try {
+                JSON.parse(line.trim());
+                return true;
+            }
+            catch {
+                return false;
+            }
+        });
+        if (allSampledLinesAreJson) {
+            return 'jsonl';
+        }
+    }
+    return 'text';
+}
+/**
+ * Get file extension based on content format
+ */
+function getFileExtension(format) {
+    switch (format) {
+        case 'jsonl': return '.jsonl';
+        case 'text': return '.txt';
+        default: return '.json';
+    }
+}
 /**
  * Helper function to detect if a response contains an error
  */
@@ -299,15 +347,31 @@ export async function handleToolResponse(config, toolName, args, responseData) {
     }
     // Success case: write to file
     try {
-        const filename = `${tool_id}.json`;
+        // Detect content format for appropriate extension and schema
+        const format = detectContentFormat(contentToWrite);
+        const extension = getFileExtension(format);
+        const filename = `${tool_id}${extension}`;
         const filepath = path.join(config.outputPath, filename);
         // Ensure output directory exists
         await fs.mkdir(config.outputPath, { recursive: true });
         // Write the exact content we counted
         await fs.writeFile(filepath, contentToWrite);
-        // Generate query-assist schema if we have valid JSON
+        // Generate query-assist schema based on format
         let fileSchema;
-        if (parsedForSchema) {
+        if (format === 'jsonl') {
+            // JSONL format: sample multiple records for schema
+            const lines = contentToWrite.trim().split('\n').filter(l => l.trim());
+            fileSchema = generateJsonlQueryAssistSchema(lines, {
+                maxDepth: config.schemaMaxDepth ?? 2,
+                maxPaths: config.schemaMaxPaths ?? 20,
+                maxKeys: config.schemaMaxKeys ?? 50,
+                dataSize: contentLength,
+                totalLines: lines.length,
+                sampleSize: 5
+            });
+        }
+        else if (parsedForSchema) {
+            // Standard JSON format: use existing schema generator
             // Use the clean data (without pagination) for schema analysis
             const { pagination, has_more, next_page, previous_page, page, page_size, total_pages, ...cleanData } = parsedForSchema;
             // Generate compact query-assist schema using config values

package/dist/jq/handler.js

@@ -37,9 +37,10 @@ export async function executeJqQuery(config, jqQuery, filePath) {
     if (!existsSync(filePath)) {
         throw new Error(`File not found: ${filePath}`);
     }
-    // Validate file extension
-    if (!filePath.toLowerCase().endsWith('.json')) {
-        throw new Error(`Only JSON files (.json) are supported for jq processing: ${filePath}`);
+    // Validate file extension (support both .json and .jsonl)
+    const lowerPath = filePath.toLowerCase();
+    if (!lowerPath.endsWith('.json') && !lowerPath.endsWith('.jsonl')) {
+        throw new Error(`Only JSON (.json) and JSONL (.jsonl) files are supported for jq processing: ${filePath}`);
     }
     // Validate path is within allowed directories
     validatePathWithinAllowedDirs(filePath, config.allowedPaths);

package/dist/jq/tool.js

@@ -8,7 +8,7 @@ export const ExecuteJqQuerySchema = z.object({
         .describe('The jq query to execute on the JSON file. Query will be sanitized to prevent environment variable access.'),
     file_path: z
         .string()
-        .describe('Absolute path starting with "/" pointing to the JSON file to process. Must be a valid, existing file with .json extension. The file will be validated for existence and readability before processing.'),
+        .describe('Absolute path starting with "/" pointing to the JSON or JSONL file to process. Must be a valid, existing file with .json or .jsonl extension. The file will be validated for existence and readability before processing.'),
     description: z
         .string()
         .optional()
@@ -109,7 +109,7 @@ export const JQ_TOOL_DEFINITION = {
             },
             file_path: {
                 type: 'string',
-                description: 'Absolute path starting with "/" pointing to the JSON file to process. Must be a valid, existing file with .json extension. The file will be validated for existence and readability before processing.',
+                description: 'Absolute path starting with "/" pointing to the JSON or JSONL file to process. Must be a valid, existing file with .json or .jsonl extension. The file will be validated for existence and readability before processing.',
             },
             description: {
                 type: 'string',

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@anyshift/mcp-proxy",
-  "version": "0.3.2",
+  "version": "0.3.4",
   "description": "Generic MCP proxy that adds truncation, file writing, and JQ capabilities to any MCP server",
   "type": "module",
   "main": "dist/index.js",