mcp-vision-web-bridge 0.2.0

package/src/model-client.mjs

@@ -0,0 +1,757 @@
+import { execFile as execFileCallback } from 'node:child_process';
+import { readFile, mkdtemp, readdir, rm, stat } from 'node:fs/promises';
+import { isIP } from 'node:net';
+import { homedir, platform, tmpdir } from 'node:os';
+import { extname, join, resolve } from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { promisify } from 'node:util';
+
+const execFile = promisify(execFileCallback);
+
+const DEFAULT_SYSTEM_PROMPT = [
+  'You are a helpful assistant.',
+  'Answer clearly and avoid exposing private data unless the user explicitly included it.'
+].join('\n');
+
+const DEFAULT_IMAGE_PROMPT = 'Please describe this image. If it contains text, extract the text.';
+const DEFAULT_MODEL = process.env.MODEL_NAME || 'replace-with-your-vision-model';
+const DEFAULT_BASE_URL = process.env.MODEL_BASE_URL || process.env.OPENAI_BASE_URL || 'https://api.example.com/v1';
+const DEFAULT_MAX_IMAGE_BYTES = Number.parseInt(process.env.MAX_IMAGE_BYTES || '10485760', 10);
+
+const MIME_BY_EXTENSION = new Map([
+  ['.png', 'image/png'],
+  ['.jpg', 'image/jpeg'],
+  ['.jpeg', 'image/jpeg'],
+  ['.gif', 'image/gif'],
+  ['.webp', 'image/webp'],
+  ['.tif', 'image/tiff'],
+  ['.tiff', 'image/tiff']
+]);
+
+const MACOS_CLIPBOARD_IMAGE_SCRIPT = `
+ObjC.import("AppKit");
+ObjC.import("Foundation");
+
+const out = ObjC.unwrap($.NSProcessInfo.processInfo.environment.objectForKey("OUT"));
+const pb = $.NSPasteboard.generalPasteboard;
+const types = ObjC.deepUnwrap(pb.types) || [];
+const candidates = ["public.png", "public.jpeg", "public.tiff", "com.compuserve.gif"];
+const chosen = candidates.find((type) => types.includes(type));
+
+if (!chosen) {
+  throw new Error("No image found in clipboard");
+}
+
+const data = pb.dataForType(chosen);
+if (!data) {
+  throw new Error("Clipboard image data is empty");
+}
+
+if (!data.writeToFileAtomically(out, true)) {
+  throw new Error("Failed to write clipboard image");
+}
+
+console.log(chosen);
+`;
+
+const WINDOWS_CLIPBOARD_IMAGE_SCRIPT = `
+Add-Type -AssemblyName System.Windows.Forms
+Add-Type -AssemblyName System.Drawing
+$image = [System.Windows.Forms.Clipboard]::GetImage()
+if ($null -eq $image) {
+  throw "No image found in clipboard"
+}
+$image.Save($env:OUT, [System.Drawing.Imaging.ImageFormat]::Png)
+`;
+
+export function buildChatMessages({ prompt, system = DEFAULT_SYSTEM_PROMPT }) {
+  if (!prompt || typeof prompt !== 'string') {
+    throw new Error('prompt is required');
+  }
+
+  const messages = [];
+  if (system) {
+    messages.push({
+      role: 'system',
+      content: system
+    });
+  }
+  messages.push({
+    role: 'user',
+    content: prompt
+  });
+
+  return messages;
+}
+
+export function buildChatRequest({
+  messages,
+  model = DEFAULT_MODEL,
+  maxTokens = 8192,
+  temperature
+}) {
+  const request = {
+    model,
+    messages,
+    max_tokens: maxTokens,
+    stream: false
+  };
+
+  if (typeof temperature === 'number') {
+    request.temperature = temperature;
+  }
+
+  return request;
+}
+
+export async function askModel({
+  prompt,
+  system,
+  model = DEFAULT_MODEL,
+  baseUrl = DEFAULT_BASE_URL,
+  apiKey = process.env.MODEL_API_KEY,
+  maxTokens = 8192,
+  temperature,
+  fetchImpl = fetch
+}) {
+  if (!apiKey) {
+    throw new Error('Missing MODEL_API_KEY');
+  }
+
+  const url = new URL('chat/completions', normalizeBaseUrl(baseUrl));
+  const request = buildChatRequest({
+    messages: buildChatMessages({
+      prompt,
+      system
+    }),
+    model,
+    maxTokens,
+    temperature
+  });
+
+  const response = await fetchImpl(url, {
+    method: 'POST',
+    headers: {
+      authorization: `Bearer ${apiKey}`,
+      'content-type': 'application/json'
+    },
+    body: JSON.stringify(request)
+  });
+
+  const payload = await safeJson(response);
+  if (!response.ok) {
+    throw new Error(formatModelError(response, payload, 'Model request'));
+  }
+
+  return extractChatText(payload);
+}
+
+export async function askModelWithImage({
+  prompt = DEFAULT_IMAGE_PROMPT,
+  system,
+  imagePath,
+  imageUrl,
+  imageBase64,
+  mimeType = 'image/png',
+  fromClipboard = false,
+  latestClaudeUpload = false,
+  maxAgeMinutes = 240,
+  model = DEFAULT_MODEL,
+  baseUrl = DEFAULT_BASE_URL,
+  apiKey = process.env.MODEL_API_KEY,
+  maxTokens = 8192,
+  temperature,
+  fetchImpl = fetch
+}) {
+  if (!apiKey) {
+    throw new Error('Missing MODEL_API_KEY');
+  }
+
+  const imageInput = await resolveImageInput({
+    imagePath,
+    imageUrl,
+    imageBase64,
+    mimeType,
+    fromClipboard,
+    latestClaudeUpload,
+    maxAgeMinutes
+  });
+  const messages = [];
+
+  if (system) {
+    messages.push({
+      role: 'system',
+      content: system
+    });
+  }
+
+  messages.push({
+    role: 'user',
+    content: [
+      {
+        type: 'text',
+        text: prompt
+      },
+      {
+        type: 'image_url',
+        image_url: {
+          url: imageInput.url
+        }
+      }
+    ]
+  });
+
+  const url = new URL('chat/completions', normalizeBaseUrl(baseUrl));
+  const response = await fetchImpl(url, {
+    method: 'POST',
+    headers: {
+      authorization: `Bearer ${apiKey}`,
+      'content-type': 'application/json'
+    },
+    body: JSON.stringify(buildChatRequest({
+      messages,
+      model,
+      maxTokens,
+      temperature
+    }))
+  });
+
+  const payload = await safeJson(response);
+  if (!response.ok) {
+    throw new Error(formatModelError(response, payload, 'Vision request'));
+  }
+
+  return {
+    text: extractChatText(payload),
+    source: imageInput.source
+  };
+}
+
+export async function buildImageUrl({
+  imagePath,
+  imageUrl,
+  imageBase64,
+  mimeType = 'image/png',
+  fromClipboard = false,
+  latestClaudeUpload = false,
+  maxAgeMinutes = 240
+}) {
+  const imageInput = await resolveImageInput({
+    imagePath,
+    imageUrl,
+    imageBase64,
+    mimeType,
+    fromClipboard,
+    latestClaudeUpload,
+    maxAgeMinutes
+  });
+
+  return imageInput.url;
+}
+
+export async function resolveImageInput({
+  imagePath,
+  imageUrl,
+  imageBase64,
+  mimeType = 'image/png',
+  fromClipboard = false,
+  latestClaudeUpload = false,
+  maxAgeMinutes = 240,
+  allowPrivateNetworkUrls = process.env.ALLOW_PRIVATE_NETWORK_URLS === 'true'
+}) {
+  const provided = [
+    imagePath,
+    imageUrl,
+    imageBase64,
+    fromClipboard ? 'clipboard' : '',
+    latestClaudeUpload ? 'latestClaudeUpload' : ''
+  ].filter(Boolean).length;
+
+  if (provided !== 1) {
+    throw new Error('Provide exactly one image source');
+  }
+
+  if (imageUrl) {
+    validateImageUrl(imageUrl, {
+      allowPrivateNetworkUrls
+    });
+    return {
+      url: imageUrl,
+      source: {
+        type: imageUrl.startsWith('data:') ? 'data_url' : 'image_url'
+      }
+    };
+  }
+
+  if (imageBase64) {
+    return {
+      url: normalizeBase64Image(imageBase64, mimeType),
+      source: {
+        type: 'base64',
+        mimeType
+      }
+    };
+  }
+
+  if (fromClipboard) {
+    assertEnabled('ALLOW_CLIPBOARD_IMAGES', 'Clipboard image reading is disabled by default');
+    return {
+      url: await readClipboardImageAsDataUrl(),
+      source: {
+        type: 'clipboard'
+      }
+    };
+  }
+
+  if (latestClaudeUpload) {
+    const latest = await findLatestClaudeUpload({
+      maxAgeMinutes
+    });
+    return {
+      url: await readLocalImageAsDataUrl(latest.path),
+      source: {
+        type: 'latest_upload',
+        mimeType: latest.mimeType,
+        size: latest.size
+      }
+    };
+  }
+
+  assertEnabled('ALLOW_LOCAL_IMAGE_PATHS', 'Local image path reading is disabled by default');
+  return {
+    url: await readLocalImageAsDataUrl(imagePath),
+    source: {
+      type: 'local_path'
+    }
+  };
+}
+
+export async function findLatestClaudeUpload({
+  uploadDirs = parseClaudeUploadDirs(),
+  maxAgeMinutes = 240,
+  now = Date.now()
+} = {}) {
+  const maxAgeMs = maxAgeMinutes * 60 * 1000;
+  const candidates = [];
+
+  for (const dir of uploadDirs) {
+    const resolvedDir = resolveLocalPath(dir);
+    let entries;
+
+    try {
+      entries = await readdir(resolvedDir, {
+        withFileTypes: true
+      });
+    } catch (error) {
+      if (error?.code === 'ENOENT') continue;
+      throw error;
+    }
+
+    for (const entry of entries) {
+      if (!entry.isFile()) continue;
+
+      const path = join(resolvedDir, entry.name);
+      const info = await stat(path);
+      const ageMs = now - info.mtimeMs;
+
+      if (ageMs < 0 || ageMs > maxAgeMs) continue;
+      if (info.size > getMaxImageBytes()) continue;
+
+      const mimeType = await detectImageMimeTypeFromFile(path);
+      if (!mimeType?.startsWith('image/')) continue;
+
+      candidates.push({
+        path,
+        mimeType,
+        mtimeMs: info.mtimeMs,
+        size: info.size
+      });
+    }
+  }
+
+  candidates.sort((a, b) => b.mtimeMs - a.mtimeMs);
+
+  if (!candidates.length) {
+    throw new Error('No recent uploaded image found');
+  }
+
+  return candidates[0];
+}
+
+export function getDefaultClaudeUploadDirs(osPlatform = platform()) {
+  if (osPlatform === 'darwin') {
+    return [
+      '~/Library/Application Support/Claude-3p/pending-uploads',
+      '~/Library/Application Support/Claude/pending-uploads'
+    ];
+  }
+
+  if (osPlatform === 'win32') {
+    return [
+      '%APPDATA%\\Claude-3p\\pending-uploads',
+      '%APPDATA%\\Claude\\pending-uploads',
+      '%LOCALAPPDATA%\\Claude-3p\\pending-uploads',
+      '%LOCALAPPDATA%\\Claude\\pending-uploads'
+    ];
+  }
+
+  return [
+    '~/.config/Claude-3p/pending-uploads',
+    '~/.config/Claude/pending-uploads',
+    '~/.local/share/Claude-3p/pending-uploads',
+    '~/.local/share/Claude/pending-uploads'
+  ];
+}
+
+export function extractChatText(payload) {
+  const content = payload?.choices?.[0]?.message?.content;
+
+  if (typeof content === 'string') {
+    return content;
+  }
+
+  if (Array.isArray(content)) {
+    return content
+      .map((part) => {
+        if (typeof part === 'string') return part;
+        if (part?.type === 'text' && typeof part.text === 'string') return part.text;
+        return '';
+      })
+      .filter(Boolean)
+      .join('\n');
+  }
+
+  const errorMessage = payload?.error?.message;
+  if (errorMessage) {
+    throw new Error(errorMessage);
+  }
+
+  throw new Error('No text returned from model');
+}
+
+async function safeJson(response) {
+  try {
+    return await response.json();
+  } catch {
+    return {};
+  }
+}
+
+function formatModelError(response, payload, label) {
+  const providerMessage = sanitizeErrorMessage(payload?.error?.message || payload?.message || '');
+  const statusHint = getStatusHint(response.status);
+  const details = [statusHint, providerMessage].filter(Boolean).join(' ');
+
+  return details ? `${label} failed with ${response.status}. ${details}` : `${label} failed with ${response.status}`;
+}
+
+function getStatusHint(status) {
+  if (status === 400) return 'Check whether the selected model supports vision and whether the image is a valid supported format.';
+  if (status === 401 || status === 403) return 'Check MODEL_API_KEY and provider permissions.';
+  if (status === 404) return 'Check MODEL_BASE_URL and MODEL_NAME.';
+  if (status === 413) return 'The image or request is too large.';
+  if (status === 429) return 'The provider rate limit or quota was reached.';
+  if (status >= 500) return 'The provider endpoint returned a server error.';
+  return '';
+}
+
+function sanitizeErrorMessage(value) {
+  if (!value || typeof value !== 'string') {
+    return '';
+  }
+
+  let sanitized = value;
+  for (const secret of [process.env.MODEL_API_KEY, process.env.OPENAI_API_KEY]) {
+    if (secret) {
+      sanitized = sanitized.split(secret).join('[redacted]');
+    }
+  }
+  return sanitized.replace(/Bearer\s+[A-Za-z0-9._~+/=-]+/gi, 'Bearer [redacted]');
+}
+
+function normalizeBase64Image(value, mimeType) {
+  const trimmed = value.trim();
+  if (trimmed.startsWith('data:')) {
+    return trimmed;
+  }
+
+  return `data:${mimeType};base64,${trimmed.replace(/\s/g, '')}`;
+}
+
+async function readLocalImageAsDataUrl(imagePath) {
+  const resolvedPath = resolveLocalPath(imagePath);
+  const info = await stat(resolvedPath);
+  if (!info.isFile()) {
+    throw new Error('Image path must be a file');
+  }
+  if (info.size > getMaxImageBytes()) {
+    throw new Error(`Image exceeds MAX_IMAGE_BYTES (${getMaxImageBytes()})`);
+  }
+
+  const data = await readFile(resolvedPath);
+  const mimeType = detectImageMimeType(data) || MIME_BY_EXTENSION.get(extname(resolvedPath).toLowerCase());
+
+  if (!mimeType) {
+    throw new Error('Unsupported image type. Use PNG, JPEG, GIF, WebP, or TIFF.');
+  }
+
+  return `data:${mimeType};base64,${data.toString('base64')}`;
+}
+
+async function detectImageMimeTypeFromFile(path) {
+  const data = await readFile(path);
+  return detectImageMimeType(data) || MIME_BY_EXTENSION.get(extname(path).toLowerCase());
+}
+
+function detectImageMimeType(data) {
+  if (data.length >= 8 && data.subarray(0, 8).equals(Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]))) {
+    return 'image/png';
+  }
+
+  if (data.length >= 3 && data[0] === 0xff && data[1] === 0xd8 && data[2] === 0xff) {
+    return 'image/jpeg';
+  }
+
+  if (data.length >= 6) {
+    const signature = data.subarray(0, 6).toString('ascii');
+    if (signature === 'GIF87a' || signature === 'GIF89a') {
+      return 'image/gif';
+    }
+  }
+
+  if (
+    data.length >= 12 &&
+    data.subarray(0, 4).toString('ascii') === 'RIFF' &&
+    data.subarray(8, 12).toString('ascii') === 'WEBP'
+  ) {
+    return 'image/webp';
+  }
+
+  if (data.length >= 4) {
+    const header = data.subarray(0, 4).toString('hex');
+    if (header === '49492a00' || header === '4d4d002a') {
+      return 'image/tiff';
+    }
+  }
+
+  return undefined;
+}
+
+async function readClipboardImageAsDataUrl() {
+  const osPlatform = platform();
+
+  if (osPlatform === 'darwin') {
+    return readMacosClipboardImageAsDataUrl();
+  }
+
+  if (osPlatform === 'win32') {
+    return readWindowsClipboardImageAsDataUrl();
+  }
+
+  return readLinuxClipboardImageAsDataUrl();
+}
+
+async function readMacosClipboardImageAsDataUrl() {
+  const dir = await mkdtemp(join(tmpdir(), 'mcp-clipboard-image-'));
+  const rawPath = join(dir, 'clipboard-image');
+  const pngPath = join(dir, 'clipboard-image.png');
+
+  try {
+    const { stdout } = await execFile(
+      'osascript',
+      ['-l', 'JavaScript', '-e', MACOS_CLIPBOARD_IMAGE_SCRIPT],
+      {
+        env: {
+          ...process.env,
+          OUT: rawPath
+        },
+        timeout: 10000
+      }
+    );
+    const pasteboardType = stdout.trim().split(/\s+/).at(-1);
+
+    if (pasteboardType === 'public.png') {
+      const data = await readFile(rawPath);
+      return `data:image/png;base64,${data.toString('base64')}`;
+    }
+
+    if (pasteboardType === 'public.jpeg') {
+      const data = await readFile(rawPath);
+      return `data:image/jpeg;base64,${data.toString('base64')}`;
+    }
+
+    if (pasteboardType === 'com.compuserve.gif') {
+      const data = await readFile(rawPath);
+      return `data:image/gif;base64,${data.toString('base64')}`;
+    }
+
+    await execFile('sips', ['-s', 'format', 'png', rawPath, '--out', pngPath], {
+      timeout: 10000
+    });
+    const data = await readFile(pngPath);
+    return `data:image/png;base64,${data.toString('base64')}`;
+  } finally {
+    await rm(dir, {
+      recursive: true,
+      force: true
+    });
+  }
+}
+
+async function readWindowsClipboardImageAsDataUrl() {
+  const dir = await mkdtemp(join(tmpdir(), 'mcp-clipboard-image-'));
+  const pngPath = join(dir, 'clipboard-image.png');
+
+  try {
+    await execFile(
+      'powershell.exe',
+      ['-NoProfile', '-STA', '-Command', WINDOWS_CLIPBOARD_IMAGE_SCRIPT],
+      {
+        env: {
+          ...process.env,
+          OUT: pngPath
+        },
+        timeout: 10000
+      }
+    );
+    const data = await readFile(pngPath);
+    return `data:image/png;base64,${data.toString('base64')}`;
+  } finally {
+    await rm(dir, {
+      recursive: true,
+      force: true
+    });
+  }
+}
+
+async function readLinuxClipboardImageAsDataUrl() {
+  const wayland = await tryReadLinuxClipboardCommand('wl-paste', ['--type', 'image/png']);
+  if (wayland) {
+    return wayland;
+  }
+
+  const x11 = await tryReadLinuxClipboardCommand('xclip', ['-selection', 'clipboard', '-t', 'image/png', '-o']);
+  if (x11) {
+    return x11;
+  }
+
+  throw new Error('No clipboard image found. On Linux, install wl-clipboard or xclip and copy a PNG-compatible image.');
+}
+
+async function tryReadLinuxClipboardCommand(command, args) {
+  try {
+    const { stdout } = await execFile(command, args, {
+      encoding: 'buffer',
+      maxBuffer: getMaxImageBytes(),
+      timeout: 10000
+    });
+
+    if (!Buffer.isBuffer(stdout) || stdout.length === 0) {
+      return undefined;
+    }
+
+    if (stdout.length > getMaxImageBytes()) {
+      throw new Error(`Clipboard image exceeds MAX_IMAGE_BYTES (${getMaxImageBytes()})`);
+    }
+
+    return `data:image/png;base64,${stdout.toString('base64')}`;
+  } catch {
+    return undefined;
+  }
+}
+
+function normalizeBaseUrl(value) {
+  const url = new URL(value);
+  if (!url.pathname.endsWith('/')) {
+    url.pathname += '/';
+  }
+  return url.href;
+}
+
+function resolveLocalPath(value) {
+  if (!value || typeof value !== 'string') {
+    throw new Error('imagePath is required');
+  }
+
+  const expandedValue = expandEnvironmentVariables(value);
+
+  if (expandedValue.startsWith('file://')) {
+    return fileURLToPath(expandedValue);
+  }
+
+  if (expandedValue === '~') {
+    return homedir();
+  }
+
+  if (expandedValue.startsWith('~/')) {
+    return resolve(join(homedir(), expandedValue.slice(2)));
+  }
+
+  return resolve(expandedValue);
+}
+
+function parseClaudeUploadDirs() {
+  const configured = process.env.CLAUDE_UPLOAD_DIRS;
+  if (!configured) {
+    return getDefaultClaudeUploadDirs();
+  }
+
+  const delimiter = process.env.CLAUDE_UPLOAD_DIRS_DELIMITER || (platform() === 'win32' ? ';' : ':');
+
+  return configured
+    .split(delimiter)
+    .map((value) => value.trim())
+    .filter(Boolean);
+}
+
+function expandEnvironmentVariables(value) {
+  return value
+    .replace(/%([^%]+)%/g, (match, name) => process.env[name] || match)
+    .replace(/\$([A-Z_][A-Z0-9_]*)/gi, (match, name) => process.env[name] || match);
+}
+
+function assertEnabled(name, message) {
+  if (process.env[name] !== 'true') {
+    throw new Error(`${message}. Set ${name}=true to enable it.`);
+  }
+}
+
+function validateImageUrl(value, { allowPrivateNetworkUrls = false } = {}) {
+  const url = new URL(value);
+  if (!['http:', 'https:', 'data:'].includes(url.protocol)) {
+    throw new Error('image_url must be http, https, or data URL');
+  }
+  if (url.protocol !== 'data:' && !allowPrivateNetworkUrls && isPrivateHostname(url.hostname)) {
+    throw new Error('Private network image URLs are disabled by default');
+  }
+}
+
+function getMaxImageBytes() {
+  if (!Number.isInteger(DEFAULT_MAX_IMAGE_BYTES) || DEFAULT_MAX_IMAGE_BYTES <= 0) {
+    return 10 * 1024 * 1024;
+  }
+  return DEFAULT_MAX_IMAGE_BYTES;
+}
+
+function isPrivateHostname(hostname) {
+  const normalized = hostname.toLowerCase().replace(/^\[|\]$/g, '');
+  if (normalized === 'localhost' || normalized.endsWith('.localhost')) return true;
+
+  const ipVersion = isIP(normalized);
+  if (!ipVersion) return false;
+
+  if (ipVersion === 6) {
+    return normalized === '::1' || normalized.startsWith('fc') || normalized.startsWith('fd') || normalized.startsWith('fe80:');
+  }
+
+  const parts = normalized.split('.').map((part) => Number.parseInt(part, 10));
+  const [a, b] = parts;
+  return (
+    a === 10 ||
+    a === 127 ||
+    (a === 169 && b === 254) ||
+    (a === 172 && b >= 16 && b <= 31) ||
+    (a === 192 && b === 168) ||
+    a === 0
+  );
+}