nvidia-vision-mcp 0.1.0 → 0.2.0

package/README.md

@@ -9,6 +9,13 @@ This is useful when the AI model you are using cannot see images directly. A com
 - Describes local images and screenshots
 - Extracts visible text from images
 - Answers specific questions about an image
+- Turns a UI screenshot into code, a prompt, a spec, or a description
+- OCRs screenshots optimized for code, terminal output, documents, or general text
+- Diagnoses error screenshots and proposes fixes
+- Interprets technical diagrams (architecture, flow, UML, ER, sequence, system)
+- Reads charts and dashboards to surface insights and trends
+- Compares two UI screenshots to flag visual drift
+- General-purpose image understanding as a fallback
 - Deletes temporary screenshot files after use
 
 ## Setup
@@ -71,6 +78,39 @@ Extracts text from an image or screenshot. Useful for UI errors, terminal output
 
 Answers a custom question about an image. For example, you can ask where a button is, what color an element uses, or whether an error message is visible.
 
+`ui_to_artifact`
+
+Turns a UI screenshot into a reusable artifact. Choose `artifact_type`:
+
+- `code` — production-ready code recreating the UI (optionally set `target`, e.g. `react + tailwind`).
+- `prompt` — a text-to-UI prompt that reproduces the screenshot.
+- `spec` — a structured UI specification.
+- `description` — a written description for documentation.
+
+`extract_text_from_screenshot`
+
+OCR tuned for a specific `kind` of content: `code`, `terminal`, `document`, or `general` (default). Reproduces text verbatim with structure preserved.
+
+`diagnose_error_screenshot`
+
+Analyzes an error snapshot (stack trace, crash dialog, failed build, browser console). Extracts the error, explains it, finds the likely root cause, and lists ordered fix steps. Pass optional `context` for what was being attempted.
+
+`understand_technical_diagram`
+
+Interprets a technical diagram. Set `diagram_type` to `architecture`, `flow`, `uml`, `er`, `sequence`, `system`, or `auto` (default). Optionally ask a follow-up `question`.
+
+`analyze_data_visualization`
+
+Reads a chart, graph, or dashboard. Reports visualization type, axes/units, key values, trends, and insights. Optionally answer a specific `question`. Will not fabricate unreadable numbers.
+
+`ui_diff_check`
+
+Compares two UI screenshots (`image_path_a` / `image_path_b`) and flags visual or implementation drift, with per-difference severity and recommendations. Optionally `focus` on an aspect like `spacing`, `colors`, `layout`, or `typography`.
+
+`image_analysis`
+
+General-purpose image understanding when a more specific tool does not fit. Pass any freeform `task` instruction.
+
 `delete_file`
 
 Deletes a local file. This is mostly for cleaning up temporary screenshots.
@@ -98,6 +138,41 @@ Describe a screenshot and remove it afterwards:
 describe_image(image_path="/tmp/screenshot.png", cleanup=true)
 ```
 
+Turn a UI screenshot into React + Tailwind code:
+
+```text
+ui_to_artifact(
+  image_path="/tmp/screenshot.png",
+  artifact_type="code",
+  target="react + tailwind"
+)
+```
+
+OCR terminal output from a screenshot:
+
+```text
+extract_text_from_screenshot(image_path="/tmp/terminal.png", kind="terminal")
+```
+
+Diagnose a build error screenshot with context:
+
+```text
+diagnose_error_screenshot(
+  image_path="/tmp/build-error.png",
+  context="Running vite build on a React + TypeScript project"
+)
+```
+
+Compare two versions of a UI:
+
+```text
+ui_diff_check(
+  image_path_a="/tmp/before.png",
+  image_path_b="/tmp/after.png",
+  focus="spacing"
+)
+```
+
 ## Notes
 
 This server intentionally stays narrow. It exists to help models inspect local screenshots when another tool can produce the image file but cannot explain what is inside it.

package/package.json

@@ -1,11 +1,11 @@
 {
   "name": "nvidia-vision-mcp",
-  "version": "0.1.0",
+  "version": "0.2.0",
   "description": "MCP server for reading local images with NVIDIA vision models.",
   "license": "MIT",
   "type": "module",
   "bin": {
-    "nvidia-vision-mcp": "./src/server.js"
+    "nvidia-vision-mcp": "src/server.js"
   },
   "files": [
     "src",

package/src/server.js

@@ -41,17 +41,44 @@ function getMimeType(filePath) {
   return mimeTypes[extname(filePath).toLowerCase()] ?? "image/png";
 }
 
-async function callVision(prompt, imagePath) {
+async function loadImageContext(imagePath) {
   const fullPath = resolve(imagePath);
 
   if (!existsSync(fullPath)) {
-    return `Error: file not found: ${imagePath}`;
+    const error = new Error(`file not found: ${imagePath}`);
+    error.code = "ENOENT";
+    error.imagePath = imagePath;
+    throw error;
   }
 
   const image = await readFile(fullPath);
   const mimeType = getMimeType(fullPath);
   const base64Image = image.toString("base64");
 
+  return {
+    type: "image_url",
+    image_url: { url: `data:${mimeType};base64,${base64Image}` },
+  };
+}
+
+async function callVision(prompt, imagePaths, options = {}) {
+  const paths = Array.isArray(imagePaths) ? imagePaths : [imagePaths];
+  const maxTokens = options.maxTokens ?? 1024;
+  const temperature = options.temperature ?? 0.2;
+
+  const content = [{ type: "text", text: prompt }];
+
+  for (const imagePath of paths) {
+    try {
+      content.push(await loadImageContext(imagePath));
+    } catch (error) {
+      if (error.code === "ENOENT") {
+        return `Error: ${error.message}`;
+      }
+      throw error;
+    }
+  }
+
   const response = await fetch(`${NVIDIA_BASE_URL}/chat/completions`, {
     method: "POST",
     headers: {
@@ -60,20 +87,9 @@ async function callVision(prompt, imagePath) {
     },
     body: JSON.stringify({
       model: getModel(),
-      messages: [
-        {
-          role: "user",
-          content: [
-            { type: "text", text: prompt },
-            {
-              type: "image_url",
-              image_url: { url: `data:${mimeType};base64,${base64Image}` },
-            },
-          ],
-        },
-      ],
-      max_tokens: 1024,
-      temperature: 0.2,
+      messages: [{ role: "user", content }],
+      max_tokens: maxTokens,
+      temperature,
     }),
   });
 
@@ -168,6 +184,249 @@ server.tool(
   },
 );
 
+server.tool(
+  "ui_to_artifact",
+  "Turn a UI screenshot into a reusable artifact: code, a prompt, a spec, or a written description.",
+  {
+    image_path: z.string().describe("Absolute or relative path to the UI screenshot."),
+    artifact_type: z
+      .enum(["code", "prompt", "spec", "description"])
+      .describe("Kind of artifact to produce from the screenshot."),
+    target: z
+      .string()
+      .optional()
+      .describe("Optional target for the artifact, e.g. 'react + tailwind', 'html/css', or a component name."),
+    cleanup: z.boolean().default(false).describe("Delete the image file after reading it."),
+  },
+  async ({ image_path, artifact_type, target, cleanup }) => {
+    const targetHint = target ? ` Target stack/component: ${target}.` : "";
+
+    const prompts = {
+      code:
+        `Recreate this UI as production-ready code.${targetHint} Match layout, spacing, colors, typography, and component structure as closely as possible. Return only the code in a single fenced block, with a one-line summary before it.`,
+      prompt:
+        `Write a text-to-UI prompt that would reproduce this screenshot in a generative UI tool.${targetHint} Describe layout, sections, components, colors, typography, spacing, and interactions in enough detail to recreate it.`,
+      spec:
+        `Produce a concise UI specification for this screenshot.${targetHint} Cover layout, sections, components, copy, states, colors, typography, spacing, and accessibility notes as a structured list.`,
+      description:
+        `Describe this UI in detail for documentation purposes. Cover layout, visible components, content, colors, typography, and notable interaction cues.`,
+    };
+
+    let result = await callVision(prompts[artifact_type], image_path, { maxTokens: 2048 });
+
+    if (cleanup) {
+      result += `\n\n${await removeFile(image_path)}`;
+    }
+
+    return textResponse(result);
+  },
+);
+
+server.tool(
+  "extract_text_from_screenshot",
+  "OCR a screenshot optimized for a specific kind of content: code, terminal output, documents, or general text.",
+  {
+    image_path: z.string().describe("Absolute or relative path to the screenshot."),
+    kind: z
+      .enum(["code", "terminal", "document", "general"])
+      .default("general")
+      .describe("Type of content to optimize OCR for."),
+    cleanup: z.boolean().default(false).describe("Delete the image file after reading it."),
+  },
+  async ({ image_path, kind, cleanup }) => {
+    const prompts = {
+      code:
+        "Extract source code visible in this screenshot. Reproduce it verbatim in a single fenced code block, preserving indentation, language syntax, and line breaks. Add a language hint to the fence if you can infer it. Do not add commentary.",
+      terminal:
+        "Extract the terminal/command-line output visible in this screenshot verbatim. Preserve the original line breaks, prompts, paths, and ANSI-style structure. Do not add commentary.",
+      document:
+        "Extract the document text visible in this screenshot. Preserve headings, paragraphs, lists, and line breaks as faithfully as possible in markdown. Do not add commentary.",
+      general:
+        "Extract all visible text from this screenshot. Preserve line breaks and structure where possible. If there is no text, say that no text was found.",
+    };
+
+    let result = await callVision(prompts[kind], image_path, { maxTokens: 2048 });
+
+    if (cleanup) {
+      result += `\n\n${await removeFile(image_path)}`;
+    }
+
+    return textResponse(result);
+  },
+);
+
+server.tool(
+  "diagnose_error_screenshot",
+  "Analyze an error screenshot (stack trace, crash dialog, failed build, browser console) and propose actionable fixes.",
+  {
+    image_path: z.string().describe("Absolute or relative path to the error screenshot."),
+    context: z
+      .string()
+      .optional()
+      .describe("Optional context: what was being attempted, language, framework, or relevant logs."),
+    cleanup: z.boolean().default(false).describe("Delete the image file after reading it."),
+  },
+  async ({ image_path, context, cleanup }) => {
+    const prompt = `You are a senior engineer diagnosing an error from a screenshot.
+
+Steps:
+1. Extract the exact error message, file paths, line numbers, and stack trace verbatim.
+2. Explain what the error means in plain language.
+3. Identify the most likely root cause.
+4. List concrete, ordered steps to fix it, including code or command examples where helpful.
+
+${context ? `Additional context from the user:\n${context}\n` : ""}
+If the screenshot does not actually show an error, say so and describe what is visible instead.`;
+
+    let result = await callVision(prompt, image_path, { maxTokens: 2048 });
+
+    if (cleanup) {
+      result += `\n\n${await removeFile(image_path)}`;
+    }
+
+    return textResponse(result);
+  },
+);
+
+server.tool(
+  "understand_technical_diagram",
+  "Interpret a technical diagram (architecture, flow, UML, ER, sequence, system) and explain its structure.",
+  {
+    image_path: z.string().describe("Absolute or relative path to the diagram image."),
+    diagram_type: z
+      .enum(["architecture", "flow", "uml", "er", "sequence", "system", "auto"])
+      .default("auto")
+      .describe("Diagram type, or 'auto' to infer it."),
+    question: z
+      .string()
+      .optional()
+      .describe("Optional specific question about the diagram."),
+    cleanup: z.boolean().default(false).describe("Delete the image file after reading it."),
+  },
+  async ({ image_path, diagram_type, question, cleanup }) => {
+    const typeHint =
+      diagram_type === "auto"
+        ? "First identify the diagram type, then describe it."
+        : `Treat this as a ${diagram_type} diagram.`;
+
+    const questionHint = question ? `\n\nThen answer this question:\n${question}` : "";
+
+    const prompt = `Interpret this technical diagram.
+
+${typeHint}
+
+Provide:
+- A short summary of what the diagram represents.
+- The key entities, components, or nodes and their roles.
+- The relationships, connections, or flows between them (as a list).
+- Any labels, annotations, or notable details.
+- Inferred purpose or system being modeled.${questionHint}`;
+
+    let result = await callVision(prompt, image_path, { maxTokens: 2048 });
+
+    if (cleanup) {
+      result += `\n\n${await removeFile(image_path)}`;
+    }
+
+    return textResponse(result);
+  },
+);
+
+server.tool(
+  "analyze_data_visualization",
+  "Read a chart, graph, or dashboard from an image to surface insights, trends, and notable values.",
+  {
+    image_path: z.string().describe("Absolute or relative path to the chart/dashboard image."),
+    question: z
+      .string()
+      .optional()
+      .describe("Optional specific question about the data."),
+    cleanup: z.boolean().default(false).describe("Delete the image file after reading it."),
+  },
+  async ({ image_path, question, cleanup }) => {
+    const questionHint = question ? `\n\nThen answer this question:\n${question}` : "";
+
+    const prompt = `Analyze this data visualization carefully and accurately.
+
+Provide:
+- The type of visualization (bar, line, pie, scatter, dashboard, etc.) and its title if present.
+- The axes, units, legend, and categories shown.
+- The key values, ranges, highs, lows, and any outliers.
+- The main trends, comparisons, and notable patterns.
+- A concise set of insights a reader should take away.${questionHint}
+
+If values are not readable, say so instead of guessing. Do not fabricate numbers.`;
+
+    let result = await callVision(prompt, image_path, { maxTokens: 2048 });
+
+    if (cleanup) {
+      result += `\n\n${await removeFile(image_path)}`;
+    }
+
+    return textResponse(result);
+  },
+);
+
+server.tool(
+  "ui_diff_check",
+  "Compare two UI screenshots and flag visual or implementation drift between them.",
+  {
+    image_path_a: z.string().describe("Path to the first (e.g. before / baseline) screenshot."),
+    image_path_b: z.string().describe("Path to the second (e.g. after / candidate) screenshot."),
+    label_a: z.string().default("A").describe("Label for the first screenshot."),
+    label_b: z.string().default("B").describe("Label for the second screenshot."),
+    focus: z
+      .string()
+      .optional()
+      .describe("Optional aspect to focus on, e.g. 'spacing', 'colors', 'layout', 'typography'."),
+    cleanup: z.boolean().default(false).describe("Delete both image files after reading them."),
+  },
+  async ({ image_path_a, image_path_b, label_a, label_b, focus, cleanup }) => {
+    const focusHint = focus ? `\nPay special attention to: ${focus}.` : "";
+
+    const prompt = `Compare these two UI screenshots. The first is "${label_a}", the second is "${label_b}".
+
+Provide:
+- A verdict: are they visually equivalent, or is there drift?
+- A bulleted list of specific differences (layout, spacing, colors, typography, components, copy, states).
+- For each difference, note which screenshot (${label_a} or ${label_b}) it applies to and the likely cause.
+- A severity for each difference (blocking / minor / cosmetic).
+- A short recommendation on what (if anything) should change.${focusHint}
+
+If the two images are not comparable (different screens), say so and summarize each separately.`;
+
+    let result = await callVision(prompt, [image_path_a, image_path_b], { maxTokens: 2048 });
+
+    if (cleanup) {
+      result += `\n\n${await removeFile(image_path_a)}`;
+      result += `\n${await removeFile(image_path_b)}`;
+    }
+
+    return textResponse(result);
+  },
+);
+
+server.tool(
+  "image_analysis",
+  "General-purpose image understanding when a more specific tool does not fit.",
+  {
+    image_path: z.string().describe("Absolute or relative path to the image file."),
+    task: z
+      .string()
+      .describe("Freeform instruction or question describing what to do with the image."),
+    cleanup: z.boolean().default(false).describe("Delete the image file after reading it."),
+  },
+  async ({ image_path, task, cleanup }) => {
+    let result = await callVision(task, image_path, { maxTokens: 2048 });
+
+    if (cleanup) {
+      result += `\n\n${await removeFile(image_path)}`;
+    }
+
+    return textResponse(result);
+  },
+);
+
 server.tool(
   "delete_file",
   "Delete a local file, usually a temporary screenshot that is no longer needed.",