agent-reader 1.0.0

package/src/core/assets.js

@@ -0,0 +1,429 @@
+import { createHash } from 'node:crypto';
+import dns from 'node:dns/promises';
+import { promises as fs } from 'node:fs';
+import net from 'node:net';
+import path from 'node:path';
+
+const LOCAL_IMAGE_EXTENSIONS = new Set(['.png', '.jpg', '.jpeg', '.gif', '.svg', '.webp']);
+const DEFAULT_MAX_INLINE_BYTES = 200 * 1024;
+const DEFAULT_MAX_REMOTE_BYTES = 10 * 1024 * 1024;
+const DEFAULT_REMOTE_TIMEOUT_MS = 10_000;
+const DEFAULT_ASSET_DIR_NAME = 'assets';
+
+function isRemoteUrl(value) {
+  return /^https?:\/\//i.test(value);
+}
+
+function normalizeSrc(src) {
+  return src.trim();
+}
+
+function extFromPath(value) {
+  const parsed = value.split('?')[0].split('#')[0];
+  return path.extname(parsed).toLowerCase();
+}
+
+function guessMimeTypeFromExt(ext) {
+  switch (ext) {
+    case '.jpg':
+    case '.jpeg':
+      return 'image/jpeg';
+    case '.png':
+      return 'image/png';
+    case '.gif':
+      return 'image/gif';
+    case '.svg':
+      return 'image/svg+xml';
+    case '.webp':
+      return 'image/webp';
+    default:
+      return 'application/octet-stream';
+  }
+}
+
+function toDataUrl(mimeType, buffer) {
+  return `data:${mimeType};base64,${buffer.toString('base64')}`;
+}
+
+function toSafeAssetName(originalName, contentBuffer) {
+  const extension = path.extname(originalName).toLowerCase();
+  const baseName = path.basename(originalName, extension).replace(/[^a-zA-Z0-9_-]/g, '_') || 'image';
+  const hash = createHash('sha256').update(contentBuffer).digest('hex').slice(0, 8);
+  return `${baseName}-${hash}${extension || '.bin'}`;
+}
+
+async function ensureAssetDir(outDir, assetDirName) {
+  const assetsDir = path.join(outDir, assetDirName);
+  await fs.mkdir(assetsDir, { recursive: true });
+  return assetsDir;
+}
+
+function isPrivateOrReservedIPv4(ip) {
+  const parts = ip.split('.').map((part) => Number(part));
+  if (parts.length !== 4 || parts.some((part) => Number.isNaN(part) || part < 0 || part > 255)) {
+    return true;
+  }
+
+  const [a, b] = parts;
+  if (a === 10 || a === 127 || a === 0) return true;
+  if (a === 169 && b === 254) return true;
+  if (a === 172 && b >= 16 && b <= 31) return true;
+  if (a === 192 && b === 168) return true;
+  if (a === 100 && b >= 64 && b <= 127) return true;
+  if (a === 198 && (b === 18 || b === 19)) return true;
+  if (a >= 224) return true;
+  return false;
+}
+
+function isPrivateOrReservedIPv6(ip) {
+  const normalized = ip.toLowerCase();
+  return (
+    normalized === '::1' ||
+    normalized.startsWith('fc') ||
+    normalized.startsWith('fd') ||
+    normalized.startsWith('fe8') ||
+    normalized.startsWith('fe9') ||
+    normalized.startsWith('fea') ||
+    normalized.startsWith('feb') ||
+    normalized.startsWith('::ffff:127.')
+  );
+}
+
+function isUnsafeIp(ip) {
+  if (net.isIPv4(ip)) {
+    return isPrivateOrReservedIPv4(ip);
+  }
+  if (net.isIPv6(ip)) {
+    return isPrivateOrReservedIPv6(ip);
+  }
+  return true;
+}
+
+async function resolveAndValidateHost(hostname) {
+  const lowerHost = hostname.toLowerCase();
+  if (lowerHost === 'localhost') {
+    return false;
+  }
+
+  if (net.isIP(lowerHost)) {
+    return !isUnsafeIp(lowerHost);
+  }
+
+  const records = await dns.lookup(hostname, { all: true, verbatim: true });
+  if (!records.length) {
+    return false;
+  }
+
+  return records.every((record) => !isUnsafeIp(record.address));
+}
+
+async function readResponseWithLimit(response, maxBytes) {
+  const contentLength = response.headers.get('content-length');
+  if (contentLength && Number(contentLength) > maxBytes) {
+    throw new Error(`remote image too large: ${contentLength} bytes`);
+  }
+
+  if (!response.body) {
+    throw new Error('response body is empty');
+  }
+
+  const reader = response.body.getReader();
+  const chunks = [];
+  let total = 0;
+
+  while (true) {
+    const { done, value } = await reader.read();
+    if (done) {
+      break;
+    }
+
+    total += value.byteLength;
+    if (total > maxBytes) {
+      throw new Error(`remote image exceeded ${maxBytes} bytes`);
+    }
+
+    chunks.push(Buffer.from(value));
+  }
+
+  return Buffer.concat(chunks);
+}
+
+async function fetchImageWithRedirects(initialUrl, {
+  maxBytes,
+  timeoutMs,
+  maxRedirects = 5,
+}) {
+  let current = initialUrl;
+
+  for (let redirectCount = 0; redirectCount <= maxRedirects; redirectCount += 1) {
+    const url = new URL(current);
+    if (!['http:', 'https:'].includes(url.protocol)) {
+      throw new Error(`unsupported protocol: ${url.protocol}`);
+    }
+
+    const publicHost = await resolveAndValidateHost(url.hostname);
+    if (!publicHost) {
+      throw new Error(`blocked host: ${url.hostname}`);
+    }
+
+    const controller = new AbortController();
+    const timeout = setTimeout(() => controller.abort(), timeoutMs);
+
+    let response;
+    try {
+      response = await fetch(current, {
+        method: 'GET',
+        redirect: 'manual',
+        signal: controller.signal,
+      });
+    } finally {
+      clearTimeout(timeout);
+    }
+
+    if (response.status >= 300 && response.status < 400) {
+      const location = response.headers.get('location');
+      if (!location) {
+        throw new Error(`redirect without location from ${current}`);
+      }
+      current = new URL(location, current).toString();
+      continue;
+    }
+
+    if (!response.ok) {
+      throw new Error(`download failed with status ${response.status}`);
+    }
+
+    const contentType = response.headers.get('content-type') || '';
+    const mimeType = contentType.split(';')[0].trim().toLowerCase();
+    if (!mimeType.startsWith('image/')) {
+      throw new Error(`unexpected content-type: ${contentType || 'unknown'}`);
+    }
+
+    const buffer = await readResponseWithLimit(response, maxBytes);
+    return {
+      mimeType,
+      buffer,
+      finalUrl: current,
+    };
+  }
+
+  throw new Error(`too many redirects for ${initialUrl}`);
+}
+
+function replaceImgSrc(tag, newSrc) {
+  return tag.replace(/(\bsrc\s*=\s*['"])([^'"]*)(['"])/i, (match, prefix, _value, suffix) => {
+    void match;
+    return `${prefix}${newSrc}${suffix}`;
+  });
+}
+
+async function processLocalImage(src, {
+  baseDir,
+  outDir,
+  inlineAll,
+  maxInlineBytes,
+  assetDirName,
+  assets,
+  warnings,
+}) {
+  const cleanSrc = normalizeSrc(src);
+
+  const extension = extFromPath(cleanSrc);
+  if (!LOCAL_IMAGE_EXTENSIONS.has(extension)) {
+    warnings.push(`skip non-image local asset: ${cleanSrc}`);
+    return cleanSrc;
+  }
+
+  const isAbsoluteFile = path.isAbsolute(cleanSrc);
+  if (!isAbsoluteFile && !baseDir) {
+    warnings.push(`skip relative image without baseDir: ${cleanSrc}`);
+    return cleanSrc;
+  }
+
+  const resolvedPath = isAbsoluteFile ? cleanSrc : path.resolve(baseDir, cleanSrc);
+
+  let fileBuffer;
+  try {
+    fileBuffer = await fs.readFile(resolvedPath);
+  } catch {
+    warnings.push(`local image not found: ${cleanSrc}`);
+    return cleanSrc;
+  }
+
+  const mimeType = guessMimeTypeFromExt(extension);
+  if (inlineAll || fileBuffer.byteLength <= maxInlineBytes) {
+    return toDataUrl(mimeType, fileBuffer);
+  }
+
+  if (!outDir) {
+    warnings.push(`skip copying large image without outDir: ${cleanSrc}`);
+    return cleanSrc;
+  }
+
+  const assetName = toSafeAssetName(path.basename(cleanSrc), fileBuffer);
+  const assetsDir = await ensureAssetDir(outDir, assetDirName);
+  const targetPath = path.join(assetsDir, assetName);
+  await fs.writeFile(targetPath, fileBuffer);
+  assets.push(targetPath);
+  return `${assetDirName}/${assetName}`;
+}
+
+async function processRemoteImage(src, {
+  outDir,
+  inlineAll,
+  fetchRemote,
+  maxInlineBytes,
+  maxRemoteBytes,
+  remoteTimeoutMs,
+  assetDirName,
+  assets,
+  warnings,
+}) {
+  if (!fetchRemote) {
+    return {
+      newSrc: src,
+      retainedRemote: true,
+    };
+  }
+
+  try {
+    const { mimeType, buffer, finalUrl } = await fetchImageWithRedirects(src, {
+      maxBytes: maxRemoteBytes,
+      timeoutMs: remoteTimeoutMs,
+    });
+
+    if (inlineAll || buffer.byteLength <= maxInlineBytes) {
+      return {
+        newSrc: toDataUrl(mimeType, buffer),
+        retainedRemote: false,
+      };
+    }
+
+    if (!outDir) {
+      warnings.push(`skip copying remote image without outDir: ${src}`);
+      return {
+        newSrc: src,
+        retainedRemote: true,
+      };
+    }
+
+    const url = new URL(finalUrl);
+    const fallbackExt = mimeType.includes('/') ? `.${mimeType.split('/')[1].split('+')[0]}` : '.bin';
+    const baseName = path.basename(url.pathname) || `remote${fallbackExt}`;
+    const ext = path.extname(baseName) || fallbackExt;
+    const assetName = toSafeAssetName(`${path.basename(baseName, path.extname(baseName))}${ext}`, buffer);
+
+    const assetsDir = await ensureAssetDir(outDir, assetDirName);
+    const targetPath = path.join(assetsDir, assetName);
+    await fs.writeFile(targetPath, buffer);
+    assets.push(targetPath);
+
+    return {
+      newSrc: `${assetDirName}/${assetName}`,
+      retainedRemote: false,
+    };
+  } catch (error) {
+    warnings.push(`remote image fetch failed (${src}): ${error.message}`);
+    return {
+      newSrc: src,
+      retainedRemote: true,
+    };
+  }
+}
+
+export async function processAssets(html, options = {}) {
+  const {
+    baseDir,
+    outDir,
+    inlineAll = false,
+    fetchRemote = true,
+    maxInlineBytes = DEFAULT_MAX_INLINE_BYTES,
+    maxRemoteBytes = DEFAULT_MAX_REMOTE_BYTES,
+    remoteTimeoutMs = DEFAULT_REMOTE_TIMEOUT_MS,
+    assetDirName = DEFAULT_ASSET_DIR_NAME,
+  } = options;
+
+  const warnings = [];
+  const assets = [];
+  const cache = new Map();
+  let retainedRemote = false;
+
+  const imgTagRegex = /<img\b[^>]*\bsrc\s*=\s*['"][^'"]*['"][^>]*>/gi;
+  let result = '';
+  let cursor = 0;
+
+  for (const match of html.matchAll(imgTagRegex)) {
+    const index = match.index ?? 0;
+    const wholeTag = match[0];
+    const srcMatch = wholeTag.match(/\bsrc\s*=\s*['"]([^'"]*)['"]/i);
+
+    result += html.slice(cursor, index);
+    cursor = index + wholeTag.length;
+
+    if (!srcMatch) {
+      result += wholeTag;
+      continue;
+    }
+
+    const src = normalizeSrc(srcMatch[1]);
+
+    if (!src || src.startsWith('data:')) {
+      result += wholeTag;
+      continue;
+    }
+
+    if (cache.has(src)) {
+      const cached = cache.get(src);
+      if (cached.retainedRemote) {
+        retainedRemote = true;
+      }
+      result += replaceImgSrc(wholeTag, cached.newSrc);
+      continue;
+    }
+
+    let payload;
+    if (isRemoteUrl(src)) {
+      payload = await processRemoteImage(src, {
+        outDir,
+        inlineAll,
+        fetchRemote,
+        maxInlineBytes,
+        maxRemoteBytes,
+        remoteTimeoutMs,
+        assetDirName,
+        assets,
+        warnings,
+      });
+    } else {
+      const newSrc = await processLocalImage(src, {
+        baseDir,
+        outDir,
+        inlineAll,
+        maxInlineBytes,
+        assetDirName,
+        assets,
+        warnings,
+      });
+      payload = {
+        newSrc,
+        retainedRemote: false,
+      };
+    }
+
+    cache.set(src, payload);
+    if (payload.retainedRemote) {
+      retainedRemote = true;
+    }
+
+    result += replaceImgSrc(wholeTag, payload.newSrc);
+  }
+
+  result += html.slice(cursor);
+
+  return {
+    html: result,
+    assets,
+    warnings,
+    retainedRemote,
+  };
+}