@goonnguyen/human-mcp 1.3.0 → 2.0.0

package/src/tools/eyes/processors/gif.ts

@@ -1,137 +0,0 @@
-import { GenerativeModel } from "@google/generative-ai";
-import sharp from "sharp";
-import fs from "fs/promises";
-import type { AnalysisOptions, ProcessingResult } from "@/types";
-import { createPrompt, parseAnalysisResponse } from "../utils/formatters.js";
-import { logger } from "@/utils/logger.js";
-import { ProcessingError } from "@/utils/errors.js";
-
-export async function processGif(
-  model: GenerativeModel,
-  source: string,
-  options: AnalysisOptions
-): Promise<ProcessingResult> {
-  const startTime = Date.now();
-  
-  try {
-    logger.debug(`Processing GIF: ${source.substring(0, 50)}...`);
-    
-    const gifData = await loadGif(source);
-    const frames = await extractGifFrames(gifData);
-    
-    if (frames.length === 0) {
-      throw new ProcessingError("No frames could be extracted from GIF");
-    }
-    
-    const prompt = createPrompt(options) + `
-
-This is an animated GIF analysis with ${frames.length} frames. Pay attention to:
-- Animation timing and smoothness
-- UI state transitions
-- Loading states or progress indicators
-- Error animations or feedback
-- Interactive element hover states
-- Any visual glitches in the animation`;
-    
-    const mediaData = frames.map(frame => ({
-      mimeType: 'image/png',
-      data: frame
-    }));
-    
-    const response = await model.generateContent([
-      { text: prompt },
-      ...mediaData.map(data => ({
-        inlineData: {
-          mimeType: data.mimeType,
-          data: data.data
-        }
-      }))
-    ]);
-    
-    const result = await response.response;
-    const analysisText = result.text();
-    
-    if (!analysisText) {
-      throw new ProcessingError("No analysis result from Gemini");
-    }
-    
-    const parsed = parseAnalysisResponse(analysisText);
-    const processingTime = Date.now() - startTime;
-    
-    return {
-      description: parsed.description || "GIF analysis completed",
-      analysis: parsed.analysis || analysisText,
-      elements: parsed.elements || [],
-      insights: parsed.insights || [],
-      recommendations: parsed.recommendations || [],
-      metadata: {
-        processing_time_ms: processingTime,
-        model_used: model.model,
-        frames_analyzed: frames.length
-      }
-    };
-    
-  } catch (error) {
-    logger.error("GIF processing error:", error);
-    throw new ProcessingError(`Failed to process GIF: ${error instanceof Error ? error.message : 'Unknown error'}`);
-  }
-}
-
-async function loadGif(source: string): Promise<Buffer> {
-  if (source.startsWith('data:image/gif')) {
-    const [, data] = source.split(',');
-    if (!data) {
-      throw new ProcessingError("Invalid base64 GIF format");
-    }
-    return Buffer.from(data, 'base64');
-  }
-  
-  if (source.startsWith('http://') || source.startsWith('https://')) {
-    const response = await fetch(source);
-    if (!response.ok) {
-      throw new ProcessingError(`Failed to fetch GIF: ${response.statusText}`);
-    }
-    return Buffer.from(await response.arrayBuffer());
-  }
-  
-  try {
-    return await fs.readFile(source);
-  } catch (error) {
-    throw new ProcessingError(`Failed to load GIF file: ${error instanceof Error ? error.message : 'Unknown error'}`);
-  }
-}
-
-async function extractGifFrames(gifBuffer: Buffer): Promise<string[]> {
-  try {
-    const image = sharp(gifBuffer, { animated: true });
-    const { pages } = await image.metadata();
-    
-    if (!pages || pages <= 1) {
-      const singleFrame = await image
-        .resize(512, 512, { fit: 'inside', withoutEnlargement: true })
-        .png()
-        .toBuffer();
-      return [singleFrame.toString('base64')];
-    }
-    
-    const frames: string[] = [];
-    const maxFrames = Math.min(pages, 16);
-    
-    for (let i = 0; i < maxFrames; i++) {
-      const frame = await sharp(gifBuffer, { 
-        animated: true,
-        page: i 
-      })
-        .resize(512, 512, { fit: 'inside', withoutEnlargement: true })
-        .png()
-        .toBuffer();
-      
-      frames.push(frame.toString('base64'));
-    }
-    
-    return frames;
-    
-  } catch (error) {
-    throw new ProcessingError(`Failed to extract GIF frames: ${error instanceof Error ? error.message : 'Unknown error'}`);
-  }
-}

package/src/tools/eyes/processors/image.ts

@@ -1,213 +0,0 @@
-import { GenerativeModel } from "@google/generative-ai";
-import sharp from "sharp";
-import fs from "fs/promises";
-import type { AnalysisOptions, ProcessingResult } from "@/types";
-import { createPrompt, parseAnalysisResponse } from "../utils/formatters.js";
-import { logger } from "@/utils/logger.js";
-import { ProcessingError } from "@/utils/errors.js";
-import { getCloudflareR2 } from "@/utils/cloudflare-r2.js";
-
-export async function processImage(
-  model: GenerativeModel,
-  source: string,
-  options: AnalysisOptions
-): Promise<ProcessingResult> {
-  const startTime = Date.now();
-  
-  try {
-    logger.debug(`Processing image: ${source.substring(0, 50)}...`);
-    
-    const { imageData, mimeType } = await loadImage(source, options.fetchTimeout);
-    const prompt = createPrompt(options);
-    
-    const response = await model.generateContent([
-      { text: prompt },
-      {
-        inlineData: {
-          mimeType,
-          data: imageData
-        }
-      }
-    ]);
-    
-    const result = await response.response;
-    const analysisText = result.text();
-    
-    if (!analysisText) {
-      throw new ProcessingError("No analysis result from Gemini");
-    }
-    
-    const parsed = parseAnalysisResponse(analysisText);
-    const processingTime = Date.now() - startTime;
-    
-    return {
-      description: parsed.description || "Image analysis completed",
-      analysis: parsed.analysis || analysisText,
-      elements: parsed.elements || [],
-      insights: parsed.insights || [],
-      recommendations: parsed.recommendations || [],
-      metadata: {
-        processing_time_ms: processingTime,
-        model_used: model.model,
-      }
-    };
-    
-  } catch (error) {
-    logger.error("Image processing error:", error);
-    throw new ProcessingError(`Failed to process image: ${error instanceof Error ? error.message : 'Unknown error'}`);
-  }
-}
-
-async function loadImage(source: string, fetchTimeout?: number): Promise<{ imageData: string; mimeType: string }> {
-  // Detect Claude Desktop virtual paths and auto-upload to Cloudflare
-  if (source.startsWith('/mnt/user-data/') || source.startsWith('/mnt/')) {
-    logger.info(`Detected Claude Desktop virtual path: ${source}`);
-    
-    // Extract filename from path
-    const filename = source.split('/').pop() || 'upload.jpg';
-    
-    // Try to read from a temporary upload directory (if middleware saved it)
-    const tempPath = `/tmp/mcp-uploads/${filename}`;
-    
-    try {
-      // Check if file was temporarily saved by middleware
-      if (await fs.access(tempPath).then(() => true).catch(() => false)) {
-        const buffer = await fs.readFile(tempPath);
-        
-        // Upload to Cloudflare R2 if configured
-        const cloudflare = getCloudflareR2();
-        if (cloudflare) {
-          const publicUrl = await cloudflare.uploadFile(buffer, filename);
-          
-          // Clean up temp file
-          await fs.unlink(tempPath).catch(() => {});
-          
-          // Now fetch from the CDN URL
-          return loadImage(publicUrl, fetchTimeout);
-        }
-      }
-    } catch (error) {
-      logger.warn(`Could not process temp file: ${error instanceof Error ? error.message : 'Unknown error'}`);
-    }
-    
-    // If no temp file or Cloudflare not configured, provide helpful error
-    throw new ProcessingError(
-      `Local file access not supported via HTTP transport.\n` +
-      `The file path "${source}" is not accessible.\n\n` +
-      `Solutions:\n` +
-      `1. Upload your file to Cloudflare R2 first using the /mcp/upload endpoint\n` +
-      `2. Use a public URL instead of a local file path\n` +
-      `3. Convert the image to a base64 data URI\n` +
-      `4. Use the stdio transport for local file access`
-    );
-  }
-  
-  // Existing base64 handling
-  if (source.startsWith('data:image/')) {
-    const [header, data] = source.split(',');
-    if (!header || !data) {
-      throw new ProcessingError("Invalid base64 image format");
-    }
-    const mimeMatch = header.match(/data:(image\/[^;]+)/);
-    if (!mimeMatch || !mimeMatch[1]) {
-      throw new ProcessingError("Invalid base64 image format");
-    }
-    
-    // Optional: For large base64 images, upload to Cloudflare R2 if configured
-    const cloudflare = getCloudflareR2();
-    if (cloudflare && data.length > 1024 * 1024) { // > 1MB base64
-      logger.info('Large base64 image detected, uploading to Cloudflare R2');
-      try {
-        const publicUrl = await cloudflare.uploadBase64(data, mimeMatch[1]);
-        return loadImage(publicUrl, fetchTimeout);
-      } catch (error) {
-        logger.warn('Failed to upload large base64 to Cloudflare R2:', error);
-        // Continue with base64 processing
-      }
-    }
-    
-    return {
-      imageData: data,
-      mimeType: mimeMatch[1]
-    };
-  }
-  
-  // Existing URL handling
-  if (source.startsWith('http://') || source.startsWith('https://')) {
-    const controller = new AbortController();
-    const timeoutId = setTimeout(() => controller.abort(), fetchTimeout || 30000);
-    
-    try {
-      const response = await fetch(source, { signal: controller.signal });
-      clearTimeout(timeoutId);
-      
-      if (!response.ok) {
-        throw new ProcessingError(`Failed to fetch image: ${response.statusText}`);
-      }
-      
-      const buffer = await response.arrayBuffer();
-      const uint8Array = new Uint8Array(buffer);
-    
-      const processedImage = await sharp(uint8Array)
-        .resize(1024, 1024, { fit: 'inside', withoutEnlargement: true })
-        .jpeg({ quality: 85 })
-        .toBuffer();
-      
-      return {
-        imageData: processedImage.toString('base64'),
-        mimeType: 'image/jpeg'
-      };
-    } catch (error) {
-      clearTimeout(timeoutId);
-      if (error instanceof Error && error.name === 'AbortError') {
-        throw new ProcessingError(`Fetch timeout: Failed to download image from ${source}`);
-      }
-      throw new ProcessingError(`Failed to fetch image: ${error instanceof Error ? error.message : 'Unknown error'}`);
-    }
-  }
-  
-  // Local file handling - auto-upload to Cloudflare for HTTP transport
-  try {
-    const stats = await fs.stat(source);
-    if (!stats.isFile()) {
-      throw new ProcessingError(`Path is not a file: ${source}`);
-    }
-    
-    // If using HTTP transport, upload to Cloudflare R2 if configured
-    const cloudflare = getCloudflareR2();
-    if (process.env.TRANSPORT_TYPE === 'http' && cloudflare) {
-      logger.info(`HTTP transport detected, uploading local file to Cloudflare R2: ${source}`);
-      try {
-        const buffer = await fs.readFile(source);
-        const filename = source.split('/').pop() || 'upload.jpg';
-        const publicUrl = await cloudflare.uploadFile(buffer, filename);
-        
-        // Fetch from CDN
-        return loadImage(publicUrl, fetchTimeout);
-      } catch (error) {
-        logger.warn(`Failed to upload to Cloudflare R2: ${error instanceof Error ? error.message : 'Unknown error'}`);
-        // Continue with local file processing
-      }
-    }
-    
-    // For stdio transport or when Cloudflare is not configured, process locally
-    const buffer = await fs.readFile(source);
-    const processedImage = await sharp(buffer)
-      .resize(1024, 1024, { fit: 'inside', withoutEnlargement: true })
-      .jpeg({ quality: 85 })
-      .toBuffer();
-    
-    return {
-      imageData: processedImage.toString('base64'),
-      mimeType: 'image/jpeg'
-    };
-  } catch (error) {
-    if (error instanceof Error && error.message.includes('ENOENT')) {
-      throw new ProcessingError(
-        `File not found: ${source}\n` +
-        `When using HTTP transport, files are automatically uploaded to Cloudflare R2 if configured.`
-      );
-    }
-    throw new ProcessingError(`Failed to load image file: ${error instanceof Error ? error.message : 'Unknown error'}`);
-  }
-}

package/src/tools/eyes/processors/video.ts

@@ -1,135 +0,0 @@
-import { GenerativeModel } from "@google/generative-ai";
-import ffmpeg from "fluent-ffmpeg";
-import fs from "fs/promises";
-import path from "path";
-import os from "os";
-import sharp from "sharp";
-import type { VideoOptions, ProcessingResult } from "@/types";
-import { createPrompt, parseAnalysisResponse } from "../utils/formatters.js";
-import { logger } from "@/utils/logger.js";
-import { ProcessingError } from "@/utils/errors.js";
-
-export async function processVideo(
-  model: GenerativeModel,
-  source: string,
-  options: VideoOptions
-): Promise<ProcessingResult> {
-  const startTime = Date.now();
-  const maxFrames = options.max_frames || 32;
-  const sampleRate = options.sample_rate || 1;
-  
-  let tempDir: string | null = null;
-  
-  try {
-    logger.debug(`Processing video: ${source.substring(0, 50)}... (max ${maxFrames} frames)`);
-    
-    tempDir = await fs.mkdtemp(path.join(os.tmpdir(), 'human-mcp-video-'));
-    const frames = await extractFrames(source, tempDir, maxFrames, sampleRate);
-    
-    if (frames.length === 0) {
-      throw new ProcessingError("No frames could be extracted from video");
-    }
-    
-    const prompt = createPrompt(options) + `
-
-This is a video analysis with ${frames.length} frames extracted. Focus on:
-- Temporal changes between frames
-- Animation or transition issues
-- Error states that appear over time
-- UI state changes and interactions
-- Any progressive degradation or improvement`;
-    
-    const mediaData = await Promise.all(
-      frames.map(async (framePath) => {
-        const buffer = await fs.readFile(framePath);
-        const processedFrame = await sharp(buffer)
-          .resize(512, 512, { fit: 'inside', withoutEnlargement: true })
-          .jpeg({ quality: 80 })
-          .toBuffer();
-        
-        return {
-          mimeType: 'image/jpeg',
-          data: processedFrame.toString('base64')
-        };
-      })
-    );
-    
-    const response = await model.generateContent([
-      { text: prompt },
-      ...mediaData.map(data => ({
-        inlineData: {
-          mimeType: data.mimeType,
-          data: data.data
-        }
-      }))
-    ]);
-    
-    const result = await response.response;
-    const analysisText = result.text();
-    
-    if (!analysisText) {
-      throw new ProcessingError("No analysis result from Gemini");
-    }
-    
-    const parsed = parseAnalysisResponse(analysisText);
-    const processingTime = Date.now() - startTime;
-    
-    return {
-      description: parsed.description || "Video analysis completed",
-      analysis: parsed.analysis || analysisText,
-      elements: parsed.elements || [],
-      insights: parsed.insights || [],
-      recommendations: parsed.recommendations || [],
-      metadata: {
-        processing_time_ms: processingTime,
-        model_used: model.model,
-        frames_analyzed: frames.length
-      }
-    };
-    
-  } catch (error) {
-    logger.error("Video processing error:", error);
-    throw new ProcessingError(`Failed to process video: ${error instanceof Error ? error.message : 'Unknown error'}`);
-  } finally {
-    if (tempDir) {
-      await fs.rm(tempDir, { recursive: true, force: true }).catch(() => {});
-    }
-  }
-}
-
-async function extractFrames(
-  videoSource: string,
-  outputDir: string,
-  maxFrames: number,
-  sampleRate: number
-): Promise<string[]> {
-  return new Promise((resolve, reject) => {
-    const framePattern = path.join(outputDir, 'frame_%04d.jpg');
-    const frames: string[] = [];
-    
-    ffmpeg(videoSource)
-      .outputOptions([
-        '-vf', `fps=1/${sampleRate}`,
-        '-vframes', maxFrames.toString(),
-        '-q:v', '2'
-      ])
-      .output(framePattern)
-      .on('end', async () => {
-        try {
-          const files = await fs.readdir(outputDir);
-          const frameFiles = files
-            .filter(file => file.startsWith('frame_') && file.endsWith('.jpg'))
-            .sort()
-            .map(file => path.join(outputDir, file));
-          
-          resolve(frameFiles);
-        } catch (error) {
-          reject(error);
-        }
-      })
-      .on('error', (error) => {
-        reject(new ProcessingError(`FFmpeg error: ${error.message}`));
-      })
-      .run();
-  });
-}

package/src/tools/eyes/schemas.ts

@@ -1,51 +0,0 @@
-import { z } from "zod";
-
-export const EyesInputSchema = z.object({
-  source: z.string().describe("URL, file path, or base64 encoded content"),
-  type: z.enum(["image", "video", "gif"]).describe("Type of visual content"),
-  analysis_type: z.enum([
-    "general",
-    "ui_debug", 
-    "error_detection",
-    "accessibility",
-    "performance",
-    "layout"
-  ]).default("general"),
-  detail_level: z.enum(["quick", "detailed"]).default("detailed"),
-  specific_focus: z.string().optional().describe("Specific areas or elements to focus on"),
-  extract_text: z.boolean().default(true),
-  detect_ui_elements: z.boolean().default(true),
-  analyze_colors: z.boolean().default(false),
-  check_accessibility: z.boolean().default(false)
-});
-
-export const EyesOutputSchema = z.object({
-  analysis: z.string(),
-  detected_elements: z.array(z.object({
-    type: z.string(),
-    location: z.object({
-      x: z.number(),
-      y: z.number(),
-      width: z.number(),
-      height: z.number()
-    }),
-    properties: z.record(z.any())
-  })),
-  debugging_insights: z.array(z.string()),
-  recommendations: z.array(z.string()),
-  metadata: z.object({
-    processing_time_ms: z.number(),
-    model_used: z.string(),
-    frames_analyzed: z.number().optional()
-  })
-});
-
-export const CompareInputSchema = z.object({
-  source1: z.string(),
-  source2: z.string(),
-  comparison_type: z.enum(["pixel", "structural", "semantic"]).default("semantic")
-});
-
-export type EyesInput = z.infer<typeof EyesInputSchema>;
-export type EyesOutput = z.infer<typeof EyesOutputSchema>;
-export type CompareInput = z.infer<typeof CompareInputSchema>;

package/src/tools/eyes/utils/formatters.ts

@@ -1,126 +0,0 @@
-import type { AnalysisOptions, ProcessingResult, DetectedElement } from "@/types";
-
-export function createPrompt(options: AnalysisOptions): string {
-  const { analysis_type, detail_level, specific_focus } = options;
-  
-  let basePrompt = "";
-  
-  switch (analysis_type) {
-    case "ui_debug":
-      basePrompt = `You are a UI debugging expert. Analyze this visual content for layout issues, rendering problems, misalignments, broken elements, and visual bugs. Focus on identifying what's wrong with the user interface.`;
-      break;
-    case "error_detection":
-      basePrompt = `You are an error detection specialist. Look for visible error messages, error states, broken functionality, missing content, and any signs of system failures or exceptions.`;
-      break;
-    case "accessibility":
-      basePrompt = `You are an accessibility expert. Analyze this content for accessibility issues including color contrast, text readability, missing alt text, poor focus indicators, and compliance with WCAG guidelines.`;
-      break;
-    case "performance":
-      basePrompt = `You are a performance analysis expert. Look for signs of slow loading, layout shifts, render blocking, large images, and other performance-related visual indicators.`;
-      break;
-    case "layout":
-      basePrompt = `You are a layout analysis expert. Focus on responsive design issues, element positioning, spacing, alignment, overflow problems, and overall visual hierarchy.`;
-      break;
-    default:
-      basePrompt = `You are a visual analysis expert. Provide a comprehensive analysis of this visual content.`;
-  }
-  
-  const detailInstructions = {
-    quick: "Provide a concise analysis focusing on the most important findings.",
-    detailed: "Provide a thorough analysis with specific details about each finding."
-  };
-  
-  const focusInstruction = specific_focus 
-    ? `\n\nPay special attention to: ${specific_focus}`
-    : "";
-  
-  return `${basePrompt}
-
-${detailInstructions[detail_level]}
-
-Please structure your response as follows:
-1. OVERVIEW: Brief summary of what you see
-2. KEY FINDINGS: Main issues or points of interest
-3. DETAILED ANALYSIS: Comprehensive breakdown
-4. UI ELEMENTS: List detected interactive elements with approximate positions
-5. RECOMMENDATIONS: Specific actionable suggestions
-6. DEBUGGING INSIGHTS: Technical insights for developers
-
-${focusInstruction}
-
-Be specific, technical, and provide exact details where possible. Include coordinates, colors, sizes, and any measurable properties you can identify.`;
-}
-
-export function parseAnalysisResponse(response: string): Partial<ProcessingResult> {
-  const sections = {
-    overview: extractSection(response, "OVERVIEW"),
-    findings: extractSection(response, "KEY FINDINGS"),
-    analysis: extractSection(response, "DETAILED ANALYSIS"),
-    elements: extractSection(response, "UI ELEMENTS"),
-    recommendations: extractSection(response, "RECOMMENDATIONS"),
-    insights: extractSection(response, "DEBUGGING INSIGHTS")
-  };
-  
-  return {
-    description: sections.overview || response.substring(0, 500),
-    analysis: sections.analysis || response,
-    elements: parseUIElements(sections.elements),
-    insights: parseList(sections.insights),
-    recommendations: parseList(sections.recommendations)
-  };
-}
-
-function extractSection(text: string, sectionName: string): string {
-  const regex = new RegExp(`${sectionName}:?\\s*([\\s\\S]*?)(?=\\n\\n[A-Z]+:|$)`, 'i');
-  const match = text.match(regex);
-  return match?.[1]?.trim() || "";
-}
-
-function parseList(text: string): string[] {
-  if (!text) return [];
-  return text
-    .split('\n')
-    .map(line => line.replace(/^[-*•]\s*/, '').trim())
-    .filter(line => line.length > 0);
-}
-
-function parseUIElements(text: string): DetectedElement[] {
-  if (!text) return [];
-  
-  const elements: DetectedElement[] = [];
-  const lines = text.split('\n').filter(line => line.trim());
-  
-  for (const line of lines) {
-    const coordMatch = line.match(/(\d+),\s*(\d+).*?(\d+)x(\d+)|x:\s*(\d+).*?y:\s*(\d+).*?w:\s*(\d+).*?h:\s*(\d+)/i);
-    if (coordMatch) {
-      const [, x1, y1, w1, h1, x2, y2, w2, h2] = coordMatch;
-      const x = parseInt(x1 || x2 || "0");
-      const y = parseInt(y1 || y2 || "0");
-      const width = parseInt(w1 || w2 || "0");
-      const height = parseInt(h1 || h2 || "0");
-      
-      if (!isNaN(x) && !isNaN(y) && !isNaN(width) && !isNaN(height)) {
-        elements.push({
-          type: extractElementType(line),
-          location: { x, y, width, height },
-          properties: { description: line.trim() }
-        });
-      }
-    }
-  }
-  
-  return elements;
-}
-
-function extractElementType(line: string): string {
-  const types = ["button", "input", "link", "image", "text", "menu", "modal", "form", "icon"];
-  const lowerLine = line.toLowerCase();
-  
-  for (const type of types) {
-    if (lowerLine.includes(type)) {
-      return type;
-    }
-  }
-  
-  return "element";
-}