@goonnguyen/human-mcp 1.0.2

package/src/tools/eyes/index.ts

@@ -0,0 +1,225 @@
+import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
+import { z } from "zod";
+import { processImage } from "./processors/image.js";
+import { processVideo } from "./processors/video.js";
+import { processGif } from "./processors/gif.js";
+import { GeminiClient } from "./utils/gemini-client.js";
+import { 
+  EyesInputSchema, 
+  CompareInputSchema,
+  type EyesInput,
+  type CompareInput
+} from "./schemas.js";
+import { logger } from "@/utils/logger.js";
+import { handleError } from "@/utils/errors.js";
+import type { Config } from "@/utils/config.js";
+
+export async function registerEyesTool(server: McpServer, config: Config) {
+  const geminiClient = new GeminiClient(config);
+  
+  // Register eyes.analyze tool
+  server.registerTool(
+    "eyes.analyze",
+    {
+      title: "Vision Analysis Tool",
+      description: "Analyze images, videos, and GIFs using AI vision capabilities",
+      inputSchema: {
+        source: z.string().describe("Path, URL, or base64 data URI of the media to analyze"),
+        type: z.enum(["image", "video", "gif"]).describe("Type of media to analyze"),
+        detail_level: z.enum(["quick", "detailed"]).optional().default("detailed").describe("Level of detail in analysis"),
+        prompt: z.string().optional().describe("Custom prompt for analysis"),
+        max_frames: z.number().optional().describe("Maximum number of frames to analyze for videos/GIFs")
+      }
+    },
+    async (args) => {
+      try {
+        return await handleAnalyze(geminiClient, args, config);
+      } catch (error) {
+        const mcpError = handleError(error);
+        logger.error(`Tool eyes.analyze error:`, mcpError);
+        
+        return {
+          content: [{
+            type: "text" as const,
+            text: `Error: ${mcpError.message}`
+          }],
+          isError: true
+        };
+      }
+    }
+  );
+
+  // Register eyes.compare tool
+  server.registerTool(
+    "eyes.compare",
+    {
+      title: "Image Comparison Tool", 
+      description: "Compare two images and identify differences",
+      inputSchema: {
+        source1: z.string().describe("Path, URL, or base64 data URI of the first image"),
+        source2: z.string().describe("Path, URL, or base64 data URI of the second image"),
+        comparison_type: z.enum(["pixel", "structural", "semantic"]).optional().default("semantic").describe("Type of comparison to perform")
+      }
+    },
+    async (args) => {
+      try {
+        return await handleCompare(geminiClient, args);
+      } catch (error) {
+        const mcpError = handleError(error);
+        logger.error(`Tool eyes.compare error:`, mcpError);
+        
+        return {
+          content: [{
+            type: "text" as const, 
+            text: `Error: ${mcpError.message}`
+          }],
+          isError: true
+        };
+      }
+    }
+  );
+}
+
+async function handleAnalyze(
+  geminiClient: GeminiClient, 
+  args: unknown,
+  config: Config
+) {
+  const input = EyesInputSchema.parse(args) as EyesInput;
+  const { source, type, detail_level } = input;
+  
+  logger.info(`Analyzing ${type} with detail level: ${detail_level}`);
+  
+  const model = geminiClient.getModel(detail_level || "detailed");
+  const options = {
+    ...input,
+    fetchTimeout: config.server.fetchTimeout
+  };
+  let result;
+  
+  switch (type) {
+    case "image":
+      result = await processImage(model, source, options);
+      break;
+    case "video":
+      result = await processVideo(model, source, options);
+      break;
+    case "gif":
+      result = await processGif(model, source, options);
+      break;
+    default:
+      throw new Error(`Unsupported media type: ${type}`);
+  }
+  
+  return {
+    content: [
+      {
+        type: "text" as const,
+        text: result.analysis
+      }
+    ],
+    isError: false
+  };
+}
+
+async function handleCompare(
+  geminiClient: GeminiClient,
+  args: unknown
+) {
+  const input = CompareInputSchema.parse(args) as CompareInput;
+  const { source1, source2, comparison_type } = input;
+  
+  logger.info(`Comparing images with type: ${comparison_type}`);
+  
+  const model = geminiClient.getModel("detailed");
+  
+  const prompt = `Compare these two images and identify the differences. Focus on:
+  
+${comparison_type === "pixel" ? 
+  "- Exact pixel-level differences\n- Color value changes\n- Any visual artifacts or rendering differences" :
+  comparison_type === "structural" ?
+  "- Layout changes\n- Element positioning differences\n- Size and proportion changes\n- Structural modifications" :
+  "- Semantic meaning differences\n- Content changes\n- Functional differences\n- User experience impact"
+}
+
+Please provide:
+1. SUMMARY: Brief overview of main differences
+2. SPECIFIC DIFFERENCES: Detailed list of changes found
+3. IMPACT ASSESSMENT: How these differences might affect users
+4. RECOMMENDATIONS: Suggested actions based on the differences
+
+Be precise with locations and measurements where possible.`;
+  
+  try {
+    const [image1Data, image2Data] = await Promise.all([
+      loadImageForComparison(source1),
+      loadImageForComparison(source2)
+    ]);
+    
+    const response = await model.generateContent([
+      { text: prompt },
+      {
+        inlineData: {
+          mimeType: image1Data.mimeType,
+          data: image1Data.data
+        }
+      },
+      { text: "Image 1 (above) vs Image 2 (below):" },
+      {
+        inlineData: {
+          mimeType: image2Data.mimeType,
+          data: image2Data.data
+        }
+      }
+    ]);
+    
+    const result = await response.response;
+    const comparisonText = result.text();
+    
+    return {
+      content: [
+        {
+          type: "text" as const,
+          text: comparisonText || "No differences detected or analysis failed"
+        }
+      ],
+      isError: false
+    };
+    
+  } catch (error) {
+    throw new Error(`Failed to compare images: ${error instanceof Error ? error.message : 'Unknown error'}`);
+  }
+}
+
+async function loadImageForComparison(source: string): Promise<{ data: string; mimeType: string }> {
+  if (source.startsWith('data:image/')) {
+    const [header, data] = source.split(',');
+    if (!header || !data) {
+      throw new Error("Invalid base64 image format");
+    }
+    const mimeMatch = header.match(/data:(image\/[^;]+)/);
+    if (!mimeMatch || !mimeMatch[1]) {
+      throw new Error("Invalid base64 image format");
+    }
+    return { data, mimeType: mimeMatch[1] };
+  }
+  
+  if (source.startsWith('http://') || source.startsWith('https://')) {
+    const response = await fetch(source);
+    if (!response.ok) {
+      throw new Error(`Failed to fetch image: ${response.statusText}`);
+    }
+    const buffer = await response.arrayBuffer();
+    return {
+      data: Buffer.from(buffer).toString('base64'),
+      mimeType: response.headers.get('content-type') || 'image/jpeg'
+    };
+  }
+  
+  const fs = await import('fs/promises');
+  const buffer = await fs.readFile(source);
+  return {
+    data: buffer.toString('base64'),
+    mimeType: 'image/jpeg'
+  };
+}

package/src/tools/eyes/processors/gif.ts

@@ -0,0 +1,137 @@
+import { GenerativeModel } from "@google/generative-ai";
+import sharp from "sharp";
+import fs from "fs/promises";
+import type { AnalysisOptions, ProcessingResult } from "@/types";
+import { createPrompt, parseAnalysisResponse } from "../utils/formatters.js";
+import { logger } from "@/utils/logger.js";
+import { ProcessingError } from "@/utils/errors.js";
+
+export async function processGif(
+  model: GenerativeModel,
+  source: string,
+  options: AnalysisOptions
+): Promise<ProcessingResult> {
+  const startTime = Date.now();
+  
+  try {
+    logger.debug(`Processing GIF: ${source.substring(0, 50)}...`);
+    
+    const gifData = await loadGif(source);
+    const frames = await extractGifFrames(gifData);
+    
+    if (frames.length === 0) {
+      throw new ProcessingError("No frames could be extracted from GIF");
+    }
+    
+    const prompt = createPrompt(options) + `
+
+This is an animated GIF analysis with ${frames.length} frames. Pay attention to:
+- Animation timing and smoothness
+- UI state transitions
+- Loading states or progress indicators
+- Error animations or feedback
+- Interactive element hover states
+- Any visual glitches in the animation`;
+    
+    const mediaData = frames.map(frame => ({
+      mimeType: 'image/png',
+      data: frame
+    }));
+    
+    const response = await model.generateContent([
+      { text: prompt },
+      ...mediaData.map(data => ({
+        inlineData: {
+          mimeType: data.mimeType,
+          data: data.data
+        }
+      }))
+    ]);
+    
+    const result = await response.response;
+    const analysisText = result.text();
+    
+    if (!analysisText) {
+      throw new ProcessingError("No analysis result from Gemini");
+    }
+    
+    const parsed = parseAnalysisResponse(analysisText);
+    const processingTime = Date.now() - startTime;
+    
+    return {
+      description: parsed.description || "GIF analysis completed",
+      analysis: parsed.analysis || analysisText,
+      elements: parsed.elements || [],
+      insights: parsed.insights || [],
+      recommendations: parsed.recommendations || [],
+      metadata: {
+        processing_time_ms: processingTime,
+        model_used: model.model,
+        frames_analyzed: frames.length
+      }
+    };
+    
+  } catch (error) {
+    logger.error("GIF processing error:", error);
+    throw new ProcessingError(`Failed to process GIF: ${error instanceof Error ? error.message : 'Unknown error'}`);
+  }
+}
+
+async function loadGif(source: string): Promise<Buffer> {
+  if (source.startsWith('data:image/gif')) {
+    const [, data] = source.split(',');
+    if (!data) {
+      throw new ProcessingError("Invalid base64 GIF format");
+    }
+    return Buffer.from(data, 'base64');
+  }
+  
+  if (source.startsWith('http://') || source.startsWith('https://')) {
+    const response = await fetch(source);
+    if (!response.ok) {
+      throw new ProcessingError(`Failed to fetch GIF: ${response.statusText}`);
+    }
+    return Buffer.from(await response.arrayBuffer());
+  }
+  
+  try {
+    return await fs.readFile(source);
+  } catch (error) {
+    throw new ProcessingError(`Failed to load GIF file: ${error instanceof Error ? error.message : 'Unknown error'}`);
+  }
+}
+
+async function extractGifFrames(gifBuffer: Buffer): Promise<string[]> {
+  try {
+    const image = sharp(gifBuffer, { animated: true });
+    const { pages } = await image.metadata();
+    
+    if (!pages || pages <= 1) {
+      const singleFrame = await image
+        .resize(512, 512, { fit: 'inside', withoutEnlargement: true })
+        .png()
+        .toBuffer();
+      return [singleFrame.toString('base64')];
+    }
+    
+    const frames: string[] = [];
+    const maxFrames = Math.min(pages, 16);
+    
+    for (let i = 0; i < maxFrames; i++) {
+      const frame = await sharp(gifBuffer, { 
+        animated: true,
+        page: i 
+      })
+        .resize(512, 512, { fit: 'inside', withoutEnlargement: true })
+        .png()
+        .toBuffer();
+      
+      frames.push(frame.toString('base64'));
+    }
+    
+    return frames;
+    
+  } catch (error) {
+    throw new ProcessingError(`Failed to extract GIF frames: ${error instanceof Error ? error.message : 'Unknown error'}`);
+  }
+}

package/src/tools/eyes/processors/image.ts

@@ -0,0 +1,123 @@
+import { GenerativeModel } from "@google/generative-ai";
+import sharp from "sharp";
+import fs from "fs/promises";
+import type { AnalysisOptions, ProcessingResult } from "@/types";
+import { createPrompt, parseAnalysisResponse } from "../utils/formatters.js";
+import { logger } from "@/utils/logger.js";
+import { ProcessingError } from "@/utils/errors.js";
+
+export async function processImage(
+  model: GenerativeModel,
+  source: string,
+  options: AnalysisOptions
+): Promise<ProcessingResult> {
+  const startTime = Date.now();
+  
+  try {
+    logger.debug(`Processing image: ${source.substring(0, 50)}...`);
+    
+    const { imageData, mimeType } = await loadImage(source, options.fetchTimeout);
+    const prompt = createPrompt(options);
+    
+    const response = await model.generateContent([
+      { text: prompt },
+      {
+        inlineData: {
+          mimeType,
+          data: imageData
+        }
+      }
+    ]);
+    
+    const result = await response.response;
+    const analysisText = result.text();
+    
+    if (!analysisText) {
+      throw new ProcessingError("No analysis result from Gemini");
+    }
+    
+    const parsed = parseAnalysisResponse(analysisText);
+    const processingTime = Date.now() - startTime;
+    
+    return {
+      description: parsed.description || "Image analysis completed",
+      analysis: parsed.analysis || analysisText,
+      elements: parsed.elements || [],
+      insights: parsed.insights || [],
+      recommendations: parsed.recommendations || [],
+      metadata: {
+        processing_time_ms: processingTime,
+        model_used: model.model,
+      }
+    };
+    
+  } catch (error) {
+    logger.error("Image processing error:", error);
+    throw new ProcessingError(`Failed to process image: ${error instanceof Error ? error.message : 'Unknown error'}`);
+  }
+}
+
+async function loadImage(source: string, fetchTimeout?: number): Promise<{ imageData: string; mimeType: string }> {
+  if (source.startsWith('data:image/')) {
+    const [header, data] = source.split(',');
+    if (!header || !data) {
+      throw new ProcessingError("Invalid base64 image format");
+    }
+    const mimeMatch = header.match(/data:(image\/[^;]+)/);
+    if (!mimeMatch || !mimeMatch[1]) {
+      throw new ProcessingError("Invalid base64 image format");
+    }
+    return {
+      imageData: data,
+      mimeType: mimeMatch[1]
+    };
+  }
+  
+  if (source.startsWith('http://') || source.startsWith('https://')) {
+    const controller = new AbortController();
+    const timeoutId = setTimeout(() => controller.abort(), fetchTimeout || 30000);
+    
+    try {
+      const response = await fetch(source, { signal: controller.signal });
+      clearTimeout(timeoutId);
+      
+      if (!response.ok) {
+        throw new ProcessingError(`Failed to fetch image: ${response.statusText}`);
+      }
+      
+      const buffer = await response.arrayBuffer();
+      const uint8Array = new Uint8Array(buffer);
+    
+      const processedImage = await sharp(uint8Array)
+        .resize(1024, 1024, { fit: 'inside', withoutEnlargement: true })
+        .jpeg({ quality: 85 })
+        .toBuffer();
+      
+      return {
+        imageData: processedImage.toString('base64'),
+        mimeType: 'image/jpeg'
+      };
+    } catch (error) {
+      clearTimeout(timeoutId);
+      if (error instanceof Error && error.name === 'AbortError') {
+        throw new ProcessingError(`Fetch timeout: Failed to download image from ${source}`);
+      }
+      throw new ProcessingError(`Failed to fetch image: ${error instanceof Error ? error.message : 'Unknown error'}`);
+    }
+  }
+  
+  try {
+    const buffer = await fs.readFile(source);
+    const processedImage = await sharp(buffer)
+      .resize(1024, 1024, { fit: 'inside', withoutEnlargement: true })
+      .jpeg({ quality: 85 })
+      .toBuffer();
+    
+    return {
+      imageData: processedImage.toString('base64'),
+      mimeType: 'image/jpeg'
+    };
+  } catch (error) {
+    throw new ProcessingError(`Failed to load image file: ${error instanceof Error ? error.message : 'Unknown error'}`);
+  }
+}

package/src/tools/eyes/processors/video.ts

@@ -0,0 +1,135 @@
+import { GenerativeModel } from "@google/generative-ai";
+import ffmpeg from "fluent-ffmpeg";
+import fs from "fs/promises";
+import path from "path";
+import os from "os";
+import sharp from "sharp";
+import type { VideoOptions, ProcessingResult } from "@/types";
+import { createPrompt, parseAnalysisResponse } from "../utils/formatters.js";
+import { logger } from "@/utils/logger.js";
+import { ProcessingError } from "@/utils/errors.js";
+
+export async function processVideo(
+  model: GenerativeModel,
+  source: string,
+  options: VideoOptions
+): Promise<ProcessingResult> {
+  const startTime = Date.now();
+  const maxFrames = options.max_frames || 32;
+  const sampleRate = options.sample_rate || 1;
+  
+  let tempDir: string | null = null;
+  
+  try {
+    logger.debug(`Processing video: ${source.substring(0, 50)}... (max ${maxFrames} frames)`);
+    
+    tempDir = await fs.mkdtemp(path.join(os.tmpdir(), 'human-mcp-video-'));
+    const frames = await extractFrames(source, tempDir, maxFrames, sampleRate);
+    
+    if (frames.length === 0) {
+      throw new ProcessingError("No frames could be extracted from video");
+    }
+    
+    const prompt = createPrompt(options) + `
+
+This is a video analysis with ${frames.length} frames extracted. Focus on:
+- Temporal changes between frames
+- Animation or transition issues
+- Error states that appear over time
+- UI state changes and interactions
+- Any progressive degradation or improvement`;
+    
+    const mediaData = await Promise.all(
+      frames.map(async (framePath) => {
+        const buffer = await fs.readFile(framePath);
+        const processedFrame = await sharp(buffer)
+          .resize(512, 512, { fit: 'inside', withoutEnlargement: true })
+          .jpeg({ quality: 80 })
+          .toBuffer();
+        
+        return {
+          mimeType: 'image/jpeg',
+          data: processedFrame.toString('base64')
+        };
+      })
+    );
+    
+    const response = await model.generateContent([
+      { text: prompt },
+      ...mediaData.map(data => ({
+        inlineData: {
+          mimeType: data.mimeType,
+          data: data.data
+        }
+      }))
+    ]);
+    
+    const result = await response.response;
+    const analysisText = result.text();
+    
+    if (!analysisText) {
+      throw new ProcessingError("No analysis result from Gemini");
+    }
+    
+    const parsed = parseAnalysisResponse(analysisText);
+    const processingTime = Date.now() - startTime;
+    
+    return {
+      description: parsed.description || "Video analysis completed",
+      analysis: parsed.analysis || analysisText,
+      elements: parsed.elements || [],
+      insights: parsed.insights || [],
+      recommendations: parsed.recommendations || [],
+      metadata: {
+        processing_time_ms: processingTime,
+        model_used: model.model,
+        frames_analyzed: frames.length
+      }
+    };
+    
+  } catch (error) {
+    logger.error("Video processing error:", error);
+    throw new ProcessingError(`Failed to process video: ${error instanceof Error ? error.message : 'Unknown error'}`);
+  } finally {
+    if (tempDir) {
+      await fs.rm(tempDir, { recursive: true, force: true }).catch(() => {});
+    }
+  }
+}
+
+async function extractFrames(
+  videoSource: string,
+  outputDir: string,
+  maxFrames: number,
+  sampleRate: number
+): Promise<string[]> {
+  return new Promise((resolve, reject) => {
+    const framePattern = path.join(outputDir, 'frame_%04d.jpg');
+    const frames: string[] = [];
+    
+    ffmpeg(videoSource)
+      .outputOptions([
+        '-vf', `fps=1/${sampleRate}`,
+        '-vframes', maxFrames.toString(),
+        '-q:v', '2'
+      ])
+      .output(framePattern)
+      .on('end', async () => {
+        try {
+          const files = await fs.readdir(outputDir);
+          const frameFiles = files
+            .filter(file => file.startsWith('frame_') && file.endsWith('.jpg'))
+            .sort()
+            .map(file => path.join(outputDir, file));
+          
+          resolve(frameFiles);
+        } catch (error) {
+          reject(error);
+        }
+      })
+      .on('error', (error) => {
+        reject(new ProcessingError(`FFmpeg error: ${error.message}`));
+      })
+      .run();
+  });
+}

package/src/tools/eyes/schemas.ts

@@ -0,0 +1,51 @@
+import { z } from "zod";
+
+export const EyesInputSchema = z.object({
+  source: z.string().describe("URL, file path, or base64 encoded content"),
+  type: z.enum(["image", "video", "gif"]).describe("Type of visual content"),
+  analysis_type: z.enum([
+    "general",
+    "ui_debug", 
+    "error_detection",
+    "accessibility",
+    "performance",
+    "layout"
+  ]).default("general"),
+  detail_level: z.enum(["quick", "detailed"]).default("detailed"),
+  specific_focus: z.string().optional().describe("Specific areas or elements to focus on"),
+  extract_text: z.boolean().default(true),
+  detect_ui_elements: z.boolean().default(true),
+  analyze_colors: z.boolean().default(false),
+  check_accessibility: z.boolean().default(false)
+});
+
+export const EyesOutputSchema = z.object({
+  analysis: z.string(),
+  detected_elements: z.array(z.object({
+    type: z.string(),
+    location: z.object({
+      x: z.number(),
+      y: z.number(),
+      width: z.number(),
+      height: z.number()
+    }),
+    properties: z.record(z.any())
+  })),
+  debugging_insights: z.array(z.string()),
+  recommendations: z.array(z.string()),
+  metadata: z.object({
+    processing_time_ms: z.number(),
+    model_used: z.string(),
+    frames_analyzed: z.number().optional()
+  })
+});
+
+export const CompareInputSchema = z.object({
+  source1: z.string(),
+  source2: z.string(),
+  comparison_type: z.enum(["pixel", "structural", "semantic"]).default("semantic")
+});
+
+export type EyesInput = z.infer<typeof EyesInputSchema>;
+export type EyesOutput = z.infer<typeof EyesOutputSchema>;
+export type CompareInput = z.infer<typeof CompareInputSchema>;