@link-assistant/web-capture 1.10.4 → 1.10.6

package/.prettierignore

@@ -5,3 +5,4 @@ dist
 package-lock.json
 .eslintcache
 CLAUDE.md
+tests/fixtures/deepwiki/*.md

package/CHANGELOG.md

@@ -1,5 +1,17 @@
 # @link-assistant/web-capture
 
+## 1.10.6
+
+### Patch Changes
+
+- Render JavaScript-heavy pages before Markdown conversion so DeepWiki search pages capture their full content.
+
+## 1.10.5
+
+### Patch Changes
+
+- 05132b4: Support public Google Drive image file URLs across capture formats.
+
 ## 1.10.4
 
 ### Patch Changes

package/bin/web-capture.js

@@ -497,6 +497,17 @@ function archiveExtension(archiveFormat) {
   return 'zip';
 }
 
+function sanitizeImageExtension(extension) {
+  if (!extension) {
+    return null;
+  }
+
+  const normalized = extension.toLowerCase().replace(/^\./, '');
+  return ['jpg', 'jpeg', 'png', 'gif', 'webp', 'svg'].includes(normalized)
+    ? normalized
+    : null;
+}
+
 function writeGoogleDocsTextOutput({
   content,
   absoluteUrl,
@@ -601,6 +612,10 @@ async function captureUrl(url, options) {
   // Import required modules
   const {
     fetchHtml,
+    fetchGoogleDriveImage,
+    fetchGoogleDriveImageHtml,
+    googleDriveImageHtmlForUrl,
+    googleDriveImageTextForUrl,
     convertToUtf8,
     convertRelativeUrls,
     normalizeUrlForTextContent,
@@ -926,22 +941,42 @@ async function captureUrl(url, options) {
   }
 
   try {
+    if (['image', 'png', 'jpeg', 'screenshot'].includes(normalizedFormat)) {
+      const googleDriveImage = await fetchGoogleDriveImage(absoluteUrl);
+      if (googleDriveImage) {
+        if (explicitOutput === '-') {
+          process.stdout.write(googleDriveImage.buffer);
+        } else {
+          const outPath = explicitOutput || googleDriveImage.filename;
+          fs.writeFileSync(outPath, googleDriveImage.buffer);
+          console.error(`Google Drive image saved to: ${outPath}`);
+        }
+        return;
+      }
+    }
+
     if (normalizedFormat === 'txt' || normalizedFormat === 'text') {
       let text;
       if (isGithubRepositoryUrl(absoluteUrl)) {
         const snapshot = await fetchGithubRepositorySnapshot(absoluteUrl);
         text = formatGithubRepositoryText(snapshot);
       } else {
-        const response = await fetch(normalizeUrlForTextContent(absoluteUrl));
-        if (!response.ok) {
-          throw new Error(`HTTP ${response.status}: ${response.statusText}`);
-        }
-        const contentType =
-          response.headers.get('content-type') || 'text/plain';
-        if (!contentType.includes('text/')) {
-          throw new Error(`Expected text content, got ${contentType}`);
+        const googleDriveImageHtml =
+          await fetchGoogleDriveImageHtml(absoluteUrl);
+        if (googleDriveImageHtml) {
+          text = googleDriveImageTextForUrl(absoluteUrl);
+        } else {
+          const response = await fetch(normalizeUrlForTextContent(absoluteUrl));
+          if (!response.ok) {
+            throw new Error(`HTTP ${response.status}: ${response.statusText}`);
+          }
+          const contentType =
+            response.headers.get('content-type') || 'text/plain';
+          if (!contentType.includes('text/')) {
+            throw new Error(`Expected text content, got ${contentType}`);
+          }
+          text = await response.text();
         }
-        text = await response.text();
       }
       const output =
         explicitOutput === '-'
@@ -992,17 +1027,30 @@ async function captureUrl(url, options) {
       const { dismissPopups, scrollToLoadContent } =
         await import('../src/popups.js');
       const browserOpts = theme ? { colorScheme: theme } : {};
+      const googleDriveImage = await fetchGoogleDriveImage(absoluteUrl);
+      const googleDriveHtml = googleDriveImage
+        ? googleDriveImageHtmlForUrl(absoluteUrl, {
+            imageSrc: `data:${googleDriveImage.contentType};base64,${googleDriveImage.buffer.toString('base64')}`,
+          })
+        : null;
       const browser = await createBrowser(engine, browserOpts);
       try {
         const page = await browser.newPage();
         await page.setViewport({ width, height });
-        await page.goto(absoluteUrl, {
-          waitUntil: 'networkidle0',
-          timeout: 30000,
-        });
-        await new Promise((resolve) => setTimeout(resolve, 5000));
-        await scrollToLoadContent(page);
-        await dismissPopups(page);
+        if (googleDriveHtml) {
+          await page.setContent(googleDriveHtml, {
+            waitUntil: 'networkidle0',
+            timeout: 30000,
+          });
+        } else {
+          await page.goto(absoluteUrl, {
+            waitUntil: 'networkidle0',
+            timeout: 30000,
+          });
+          await new Promise((resolve) => setTimeout(resolve, 5000));
+          await scrollToLoadContent(page);
+          await dismissPopups(page);
+        }
         const rawPage = page.rawPage || page;
         const pdfBuffer = await rawPage.pdf({
           format: 'A4',
@@ -1079,11 +1127,19 @@ async function captureUrl(url, options) {
       // Collect images
       const $ = cheerio.load(html);
       const images = [];
+      const imageExtensionByUrl = new Map();
       $('img').each(function () {
         const src = $(this).attr('src');
         if (src && !src.startsWith('data:') && !src.startsWith('blob:')) {
           try {
-            images.push(new URL(src, absoluteUrl).href);
+            const imgUrl = new URL(src, absoluteUrl).href;
+            images.push(imgUrl);
+            const hintedExtension = sanitizeImageExtension(
+              $(this).attr('data-web-capture-extension')
+            );
+            if (hintedExtension && !imageExtensionByUrl.has(imgUrl)) {
+              imageExtensionByUrl.set(imgUrl, hintedExtension);
+            }
           } catch {
             /* skip */
           }
@@ -1100,7 +1156,10 @@ async function captureUrl(url, options) {
       ) {
         let idx = 1;
         for (const imgUrl of uniqueImages) {
-          const ext = imgUrl.match(/\.(jpe?g|gif|webp|svg|png)/i)?.[1] || 'png';
+          const ext =
+            imageExtensionByUrl.get(imgUrl) ||
+            imgUrl.match(/\.(jpe?g|gif|webp|svg|png)/i)?.[1] ||
+            'png';
           imageMap.set(imgUrl, `images/image-${idx}.${ext}`);
           idx++;
         }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@link-assistant/web-capture",
-  "version": "1.10.4",
+  "version": "1.10.6",
   "description": "CLI and microservice to render web pages as HTML, Markdown, or PNG",
   "main": "src/index.js",
   "bin": {

package/src/archive.js

@@ -58,12 +58,19 @@ export async function archiveHandler(req, res) {
     // Collect images from the HTML
     const $ = cheerio.load(html);
     const images = [];
+    const imageExtensionByUrl = new Map();
     $('img').each(function () {
       const src = $(this).attr('src');
       if (src && !src.startsWith('data:') && !src.startsWith('blob:')) {
         try {
           const imgUrl = new URL(src, absoluteUrl).href;
           images.push(imgUrl);
+          const hintedExtension = sanitizeImageExtension(
+            $(this).attr('data-web-capture-extension')
+          );
+          if (hintedExtension && !imageExtensionByUrl.has(imgUrl)) {
+            imageExtensionByUrl.set(imgUrl, hintedExtension);
+          }
         } catch {
           /* skip invalid URLs */
         }
@@ -89,7 +96,8 @@ export async function archiveHandler(req, res) {
     if (localImages && uniqueImages.length > 0) {
       let idx = 1;
       for (const imgUrl of uniqueImages) {
-        const ext = guessImageExtension(imgUrl);
+        const ext =
+          imageExtensionByUrl.get(imgUrl) || guessImageExtension(imgUrl);
         const filename = `image-${idx}.${ext}`;
         imageMap.set(imgUrl, `images/${filename}`);
         idx++;
@@ -277,3 +285,14 @@ function guessImageExtension(url) {
   }
   return 'png';
 }
+
+function sanitizeImageExtension(extension) {
+  if (!extension) {
+    return null;
+  }
+
+  const normalized = extension.toLowerCase().replace(/^\./, '');
+  return ['jpg', 'jpeg', 'png', 'gif', 'webp', 'svg'].includes(normalized)
+    ? normalized
+    : null;
+}

package/src/browser.js

@@ -23,7 +23,7 @@ const SERVER_CHROME_ARGS = [
  * Unified page interface wrapping browser-commander's makeBrowserCommander.
  *
  * Exposes:
- * - setExtraHTTPHeaders / setUserAgent / setViewport / goto / content / screenshot / close
+ * - setExtraHTTPHeaders / setUserAgent / setViewport / goto / setContent / content / screenshot / close
  *   (via the underlying raw page for cross-engine compatibility)
  * - commander.pdf()           – PDF generation (browser-commander v0.8.0)
  * - commander.emulateMedia()  – Color scheme emulation (browser-commander v0.7.0)
@@ -37,6 +37,7 @@ const SERVER_CHROME_ARGS = [
  * @property {Function} setUserAgent - Set user agent
  * @property {Function} setViewport - Set viewport size
  * @property {Function} goto - Navigate to URL
+ * @property {Function} setContent - Set page HTML content
  * @property {Function} content - Get page HTML content
  * @property {Function} screenshot - Take screenshot
  * @property {Function} close - Close the page
@@ -64,7 +65,7 @@ export async function createBrowser(engine = 'puppeteer', options = {}) {
       ? 'playwright'
       : 'puppeteer';
 
-  const { colorScheme, ...launchOpts } = options;
+  const { colorScheme, args = [], ...launchOpts } = options;
 
   // Generate unique userDataDir for this session to avoid conflicts
   const userDataDir = path.join(
@@ -75,7 +76,7 @@ export async function createBrowser(engine = 'puppeteer', options = {}) {
   // Use browser-commander's launchBrowser with server-specific args
   const { browser, page: initialPage } = await launchBrowser({
     engine: engineType,
-    args: SERVER_CHROME_ARGS,
+    args: [...SERVER_CHROME_ARGS, ...args],
     headless: true,
     userDataDir,
     slowMo: 0,
@@ -162,6 +163,16 @@ function createPageAdapter(rawPage, commander, engineType) {
         await rawPage.goto(url, options);
       }
     },
+    async setContent(html, options = {}) {
+      const contentOptions = { ...options };
+      if (
+        engineType === 'playwright' &&
+        contentOptions.waitUntil === 'networkidle0'
+      ) {
+        contentOptions.waitUntil = 'networkidle';
+      }
+      await rawPage.setContent(html, contentOptions);
+    },
     async content() {
       return await rawPage.content();
     },

package/src/docx.js

@@ -76,13 +76,14 @@ export async function docxHandler(req, res) {
             const imgResp = await fetch(imgUrl);
             if (imgResp.ok) {
               const buffer = await imgResp.buffer();
+              const contentType = imgResp.headers.get('content-type') || '';
               children.push(
                 new Paragraph({
                   children: [
                     new ImageRun({
                       data: buffer,
                       transformation: { width: 600, height: 400 },
-                      type: guessImageType(imgUrl),
+                      type: guessImageType(imgUrl, contentType),
                     }),
                   ],
                 })
@@ -170,7 +171,24 @@ export async function docxHandler(req, res) {
   }
 }
 
-function guessImageType(url) {
+function guessImageType(url, contentType = '') {
+  const normalizedContentType = contentType.toLowerCase().split(';')[0].trim();
+  if (normalizedContentType === 'image/jpeg') {
+    return 'jpg';
+  }
+  if (normalizedContentType === 'image/png') {
+    return 'png';
+  }
+  if (normalizedContentType === 'image/gif') {
+    return 'gif';
+  }
+  if (normalizedContentType === 'image/bmp') {
+    return 'bmp';
+  }
+  if (normalizedContentType === 'image/svg+xml') {
+    return 'svg';
+  }
+
   const pathname = new URL(url).pathname.toLowerCase();
   if (pathname.endsWith('.jpg') || pathname.endsWith('.jpeg')) {
     return 'jpg';

package/src/fetch.js

@@ -1,4 +1,5 @@
 import fetch from 'node-fetch';
+import { convertGoogleDriveUrl } from './lib.js';
 
 export async function fetchHandler(req, res) {
   const url = req.query.url;
@@ -6,7 +7,7 @@ export async function fetchHandler(req, res) {
     return res.status(400).send('Missing `url` parameter');
   }
   try {
-    const response = await fetch(url);
+    const response = await fetch(convertGoogleDriveUrl(url));
     // Copy status and headers
     res.status(response.status);
 

package/src/image.js

@@ -1,5 +1,5 @@
 import { createBrowser, getBrowserEngine } from './browser.js';
-import { isStackOverflowQuestionUrl } from './lib.js';
+import { fetchGoogleDriveImage, isStackOverflowQuestionUrl } from './lib.js';
 import { dismissPopups, scrollToLoadContent } from './popups.js';
 
 /**
@@ -39,6 +39,17 @@ export async function imageHandler(req, res) {
 
   try {
     const absoluteUrl = url.startsWith('http') ? url : `https://${url}`;
+    const googleDriveImage = await fetchGoogleDriveImage(absoluteUrl);
+    if (googleDriveImage) {
+      res.set('Content-Type', googleDriveImage.contentType);
+      res.set(
+        'Content-Disposition',
+        `inline; filename="${googleDriveImage.filename}"`
+      );
+      res.end(googleDriveImage.buffer);
+      return;
+    }
+
     const engine = getBrowserEngine(req);
     const waitUntil =
       engine === 'playwright' && isStackOverflowQuestionUrl(absoluteUrl)

package/src/lib.js

@@ -14,11 +14,17 @@ import { retry } from './retry.js';
 
 const STACKPRINTER_RETRIES = 3;
 const STACKPRINTER_RETRY_BASE_DELAY_MS = 1000;
+const GOOGLE_DRIVE_FILE_ID_RE = /^[A-Za-z0-9_-]+$/;
 
 export async function fetchHtml(url) {
   if (!url) {
     throw new Error('Missing URL parameter');
   }
+  const googleDriveImageHtml = await fetchGoogleDriveImageHtml(url);
+  if (googleDriveImageHtml) {
+    return googleDriveImageHtml;
+  }
+
   const stackPrinterUrl = stackPrinterUrlForQuestion(url);
   if (stackPrinterUrl) {
     return await fetchStackPrinterHtml(stackPrinterUrl);
@@ -28,6 +34,163 @@ export async function fetchHtml(url) {
   return response.text();
 }
 
+export function getGoogleDriveFileId(url) {
+  if (!url) {
+    return null;
+  }
+
+  try {
+    const parsed = new URL(url);
+    const hostname = parsed.hostname.toLowerCase();
+    const pathnameParts = parsed.pathname.split('/').filter(Boolean);
+    let fileId = null;
+
+    if (
+      hostname === 'drive.google.com' ||
+      hostname === 'www.drive.google.com'
+    ) {
+      if (pathnameParts[0] === 'file' && pathnameParts[1] === 'd') {
+        fileId = pathnameParts[2] || null;
+      } else if (pathnameParts[0] === 'open' || pathnameParts[0] === 'uc') {
+        fileId = parsed.searchParams.get('id');
+      }
+    } else if (
+      hostname === 'drive.usercontent.google.com' &&
+      pathnameParts[0] === 'download'
+    ) {
+      fileId = parsed.searchParams.get('id');
+    }
+
+    return fileId && GOOGLE_DRIVE_FILE_ID_RE.test(fileId) ? fileId : null;
+  } catch {
+    return null;
+  }
+}
+
+export function isGoogleDriveFileUrl(url) {
+  return Boolean(getGoogleDriveFileId(url));
+}
+
+export function convertGoogleDriveUrl(url) {
+  const fileId = getGoogleDriveFileId(url);
+  if (!fileId) {
+    return url;
+  }
+
+  return `https://drive.google.com/uc?export=download&id=${fileId}`;
+}
+
+export function googleDriveImageHtmlForUrl(url, options = {}) {
+  const fileId = getGoogleDriveFileId(url);
+  if (!fileId) {
+    return null;
+  }
+
+  const imageUrl = options.imageSrc || convertGoogleDriveUrl(url);
+  const escapedImageUrl = he.escape(imageUrl);
+  const escapedSourceUrl = he.escape(url);
+  const extensionAttribute = options.imageExtension
+    ? ` data-web-capture-extension="${he.escape(options.imageExtension)}"`
+    : '';
+
+  return `<!DOCTYPE html>
+<html>
+<head>
+  <meta charset="utf-8">
+  <title>Google Drive image</title>
+</head>
+<body>
+  <h1>Google Drive image</h1>
+  <p><a href="${escapedSourceUrl}">Original Google Drive file</a></p>
+  <img src="${escapedImageUrl}" alt="Google Drive image"${extensionAttribute}>
+</body>
+</html>`;
+}
+
+export function googleDriveImageTextForUrl(url) {
+  if (!isGoogleDriveFileUrl(url)) {
+    return null;
+  }
+
+  return [
+    'Google Drive image',
+    '',
+    `Original Google Drive file: ${url}`,
+    `Image: ${convertGoogleDriveUrl(url)}`,
+    '',
+  ].join('\n');
+}
+
+export async function fetchGoogleDriveImage(url) {
+  const fileId = getGoogleDriveFileId(url);
+  if (!fileId) {
+    return null;
+  }
+
+  const downloadUrl = convertGoogleDriveUrl(url);
+  const response = await fetch(downloadUrl);
+  if (!response.ok) {
+    throw new Error(`Google Drive returned HTTP ${response.status}`);
+  }
+
+  const contentType = response.headers.get('content-type') || '';
+  if (!isImageContentType(contentType)) {
+    throw new Error(`Google Drive file is not an image (${contentType})`);
+  }
+
+  return {
+    buffer: await response.buffer(),
+    contentType,
+    downloadUrl,
+    filename: `google-drive-${fileId}.${imageExtensionForContentType(contentType)}`,
+  };
+}
+
+export async function fetchGoogleDriveImageHtml(url) {
+  if (!isGoogleDriveFileUrl(url)) {
+    return null;
+  }
+
+  const downloadUrl = convertGoogleDriveUrl(url);
+  let imageExtension = null;
+  try {
+    const response = await fetch(downloadUrl, { method: 'HEAD' });
+    if (response.ok) {
+      const contentType = response.headers.get('content-type');
+      if (!isImageContentType(contentType)) {
+        return null;
+      }
+      imageExtension = imageExtensionForContentType(contentType);
+    }
+  } catch {
+    // Some public Drive responses reject HEAD even though GET works. In that
+    // case still render the direct image document and let consumers fetch it.
+  }
+
+  return googleDriveImageHtmlForUrl(url, { imageExtension });
+}
+
+function isImageContentType(contentType) {
+  return (contentType || '').toLowerCase().startsWith('image/');
+}
+
+function imageExtensionForContentType(contentType) {
+  const normalized = (contentType || '').toLowerCase().split(';')[0].trim();
+  if (normalized === 'image/jpeg' || normalized === 'image/jpg') {
+    return 'jpg';
+  }
+  if (normalized === 'image/gif') {
+    return 'gif';
+  }
+  if (normalized === 'image/webp') {
+    return 'webp';
+  }
+  if (normalized === 'image/svg+xml') {
+    return 'svg';
+  }
+  return 'png';
+}
+
 async function fetchStackPrinterHtml(url) {
   return await retry(
     async () => {