@link-assistant/web-capture 1.7.10 → 1.7.11

package/CHANGELOG.md

@@ -1,5 +1,27 @@
 # @link-assistant/web-capture
 
+## 1.7.11
+
+### Patch Changes
+
+- 71e7eb9: Fall back to the Google Docs export pipeline when browser capture cannot read editor model chunks, preserving archive content and embedded images.
+- 7b4d2d2: Fix Rust CLI `--capture browser` silently routing through direct HTTP
+  fetches for non-Google-Docs URLs (issue #80):
+  - `rust/src/main.rs` now passes `--capture browser` through
+    `render_html` for markdown, archive, and html output formats so the
+    flag actually launches headless Chrome instead of calling
+    `fetch_html`.
+  - `rust/src/browser.rs` `capture_screenshot` is no longer a stub — it
+    launches headless Chrome with `--screenshot` and returns real PNG
+    bytes.
+  - Adds Rust integration tests that run a local HTTP fixture whose DOM
+    is mutated by JavaScript after load and verify the rendered HTML,
+    the CLI `--capture browser` markdown output, and the PNG signature
+    on the screenshot bytes.
+
+  No JS package behavior changes in this patch; the Rust CLI is the only
+  affected surface.
+
 ## 1.7.10
 
 ### Patch Changes

package/bin/web-capture.js

@@ -459,7 +459,7 @@ async function captureUrl(url, options) {
     fetchGoogleDoc,
     fetchGoogleDocAsMarkdown,
     fetchGoogleDocFromDocsApi,
-    captureGoogleDocWithBrowser,
+    captureGoogleDocWithBrowserOrFallback,
     selectGoogleDocsCaptureMethod,
   } = await import('../src/gdocs.js');
   const { stripBase64Images } = await import('../src/extract-images.js');
@@ -511,40 +511,60 @@ async function captureUrl(url, options) {
       // Screenshot/PDF/DOCX formats should use the regular browser pipeline below.
     } else if (gdocsMethod === 'browser-model') {
       try {
-        const result = await captureGoogleDocWithBrowser(absoluteUrl, {
-          engine,
-          apiToken,
-          log,
-        });
+        const result = await captureGoogleDocWithBrowserOrFallback(
+          absoluteUrl,
+          {
+            engine,
+            apiToken,
+            format: normalizedFormat,
+            log,
+            onFallback: (err) => {
+              console.error(
+                `Warning: Google Docs browser capture could not read editor content (${err.message}); falling back to Google Docs export API.`
+              );
+            },
+          }
+        );
+        const resultMethod = result.fallback ? result.method : gdocsMethod;
+        const resultCapture = result.capture || {};
         log.debug(() => ({
-          event: 'gdocs.capture.browser-model.rendered',
+          event: result.fallback
+            ? 'gdocs.capture.browser-model.fallback-rendered'
+            : 'gdocs.capture.browser-model.rendered',
           documentId: result.documentId,
-          blocks: result.capture?.blocks?.length || 0,
-          tables: result.capture?.tables?.length || 0,
-          images: result.capture?.images?.length || 0,
+          method: resultMethod,
+          blocks: resultCapture.blocks?.length || 0,
+          tables: resultCapture.tables?.length || 0,
+          images: resultCapture.images?.length || result.images?.length || 0,
           markdownBytes: Buffer.byteLength(result.markdown || ''),
           htmlBytes: Buffer.byteLength(result.html || ''),
           textBytes: Buffer.byteLength(result.text || ''),
         }));
         if (normalizedFormat === 'archive' || normalizedFormat === 'zip') {
-          const { localizeGoogleDocsModelImages } =
-            await import('../src/gdocs.js');
-          const localized = await localizeGoogleDocsModelImages(result, {
-            log,
-          });
-          log.debug(() => ({
-            event: 'gdocs.capture.browser-model.archive.localized',
-            images: localized.images.length,
-            markdownBytes: Buffer.byteLength(localized.markdown),
-            htmlBytes: Buffer.byteLength(localized.html),
-          }));
-          await writeGoogleDocsArchive({
-            archiveResult: {
+          let archiveResult;
+          if (result.fallback) {
+            archiveResult = result;
+          } else {
+            const { localizeGoogleDocsModelImages } =
+              await import('../src/gdocs.js');
+            const localized = await localizeGoogleDocsModelImages(result, {
+              log,
+            });
+            log.debug(() => ({
+              event: 'gdocs.capture.browser-model.archive.localized',
+              images: localized.images.length,
+              markdownBytes: Buffer.byteLength(localized.markdown),
+              htmlBytes: Buffer.byteLength(localized.html),
+            }));
+            archiveResult = {
               ...result,
               markdown: localized.markdown,
               html: localized.html,
               images: localized.images,
-            },
+            };
+          }
+          await writeGoogleDocsArchive({
+            archiveResult,
             absoluteUrl,
             explicitOutput,
             dataDir,
@@ -554,10 +574,10 @@ async function captureUrl(url, options) {
         } else {
           const rendered =
             normalizedFormat === 'html'
-              ? result.html
+              ? result.html || result.content
               : normalizedFormat === 'txt' || normalizedFormat === 'text'
-                ? result.text
-                : result.markdown;
+                ? result.text || result.content
+                : result.markdown || result.content;
           const ext =
             normalizedFormat === 'html'
               ? 'html'
@@ -570,7 +590,7 @@ async function captureUrl(url, options) {
             ext,
             explicitOutput,
             dataDir,
-            label: `Google Doc (${gdocsMethod})`,
+            label: `Google Doc (${resultMethod})`,
           });
         }
         return;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@link-assistant/web-capture",
-  "version": "1.7.10",
+  "version": "1.7.11",
   "description": "CLI and microservice to render web pages as HTML, Markdown, or PNG",
   "main": "src/index.js",
   "bin": {

package/src/gdocs-fallback.js

@@ -0,0 +1,152 @@
+// Google Docs browser-model fallback helpers.
+//
+// When browser-model capture cannot read DOCS_modelChunk data from the editor
+// page, fall back to the public export pipeline so that content and images
+// are preserved (instead of returning an empty or lossy capture).
+
+const GDOCS_BROWSER_MODEL_UNAVAILABLE = 'GDOCS_BROWSER_MODEL_UNAVAILABLE';
+
+/**
+ * Build the sentinel error thrown when browser-model capture finds no chunks.
+ *
+ * @param {string} message - Human-readable error message
+ * @returns {Error} Error carrying the sentinel code
+ */
+export function googleDocsBrowserModelUnavailableError(message) {
+  const err = new Error(message);
+  err.code = GDOCS_BROWSER_MODEL_UNAVAILABLE;
+  return err;
+}
+
+/**
+ * Check whether an error means Google Docs browser-model data was unavailable.
+ *
+ * @param {Error} err - Error to classify
+ * @returns {boolean} True if export fallback is appropriate
+ */
+export function isGoogleDocsBrowserModelUnavailableError(err) {
+  return (
+    err?.code === GDOCS_BROWSER_MODEL_UNAVAILABLE ||
+    String(err?.message || '').includes('did not expose DOCS_modelChunk data')
+  );
+}
+
+/**
+ * Fetch a Google Doc through the public export pipeline for a requested output
+ * format. This is used as the lossless fallback when browser-model capture
+ * cannot read DOCS_modelChunk data from the editor page.
+ *
+ * @param {Object} deps - Injected dependencies from gdocs.js
+ * @param {string} url - Google Docs URL
+ * @param {Object} [options] - Options
+ * @returns {Promise<Object>} Export result normalized for CLI rendering
+ */
+export async function fetchGoogleDocByExportFormat(deps, url, options = {}) {
+  const { fetchGoogleDoc, fetchGoogleDocAsMarkdown, fetchGoogleDocAsArchive } =
+    deps;
+  const { format = 'markdown', apiToken, log } = options;
+  const normalized = (format || 'markdown').toLowerCase();
+
+  if (normalized === 'archive' || normalized === 'zip') {
+    const archiveResult = await fetchGoogleDocAsArchive(url, { apiToken, log });
+    return {
+      ...archiveResult,
+      content: archiveResult.markdown,
+      sourceFormat: 'archive',
+    };
+  }
+
+  if (normalized === 'markdown' || normalized === 'md') {
+    const result = await fetchGoogleDocAsMarkdown(url, { apiToken, log });
+    return {
+      ...result,
+      content: result.markdown,
+      sourceFormat: 'markdown',
+    };
+  }
+
+  if (normalized === 'html') {
+    const result = await fetchGoogleDoc(url, {
+      format: 'html',
+      apiToken,
+      log,
+    });
+    return {
+      ...result,
+      html: result.content,
+      sourceFormat: 'html',
+    };
+  }
+
+  if (normalized === 'txt' || normalized === 'text') {
+    const result = await fetchGoogleDoc(url, {
+      format: 'txt',
+      apiToken,
+      log,
+    });
+    return {
+      ...result,
+      text: result.content,
+      sourceFormat: 'txt',
+    };
+  }
+
+  throw new Error(
+    `Unsupported Google Docs export fallback format "${format}".`
+  );
+}
+
+/**
+ * Capture a Google Doc through the browser model, falling back to public export
+ * when the editor does not expose model chunks.
+ *
+ * @param {Object} deps - Injected dependencies from gdocs.js
+ * @param {string} url - Google Docs URL
+ * @param {Object} [options] - Capture options
+ * @returns {Promise<Object>} Browser-model or public-export result
+ */
+export async function captureGoogleDocWithBrowserOrFallback(
+  deps,
+  url,
+  options = {}
+) {
+  const { captureGoogleDocWithBrowser } = deps;
+  const {
+    format = 'markdown',
+    fallback = true,
+    onFallback,
+    ...browserOptions
+  } = options;
+
+  try {
+    const result = await captureGoogleDocWithBrowser(url, browserOptions);
+    return {
+      ...result,
+      method: 'browser-model',
+      fallback: false,
+    };
+  } catch (err) {
+    if (!fallback || !isGoogleDocsBrowserModelUnavailableError(err)) {
+      throw err;
+    }
+
+    onFallback?.(err);
+    browserOptions.log?.warn?.(() => ({
+      event: 'gdocs.browser-model.fallback-public-export',
+      reason: err.message,
+      format,
+    }));
+
+    const result = await fetchGoogleDocByExportFormat(deps, url, {
+      format,
+      apiToken: browserOptions.apiToken,
+      log: browserOptions.log,
+    });
+    return {
+      ...result,
+      method: 'public-export',
+      fallback: true,
+      browserError: err.message,
+    };
+  }
+}

package/src/gdocs.js

@@ -13,8 +13,18 @@ import { convertHtmlToMarkdown } from './lib.js';
 import { createBrowser as defaultCreateBrowser } from './browser.js';
 import { preprocessGoogleDocsExportHtml } from './gdocs-preprocess.js';
 import { localizeGoogleDocsModelImages } from './gdocs-images.js';
-
-export { preprocessGoogleDocsExportHtml, localizeGoogleDocsModelImages };
+import {
+  googleDocsBrowserModelUnavailableError,
+  isGoogleDocsBrowserModelUnavailableError,
+  fetchGoogleDocByExportFormat as fetchGoogleDocByExportFormatImpl,
+  captureGoogleDocWithBrowserOrFallback as captureGoogleDocWithBrowserOrFallbackImpl,
+} from './gdocs-fallback.js';
+
+export {
+  preprocessGoogleDocsExportHtml,
+  localizeGoogleDocsModelImages,
+  isGoogleDocsBrowserModelUnavailableError,
+};
 
 const GDOCS_URL_PATTERN = /docs\.google\.com\/document\/d\/([a-zA-Z0-9_-]+)/;
 
@@ -308,6 +318,43 @@ export async function fetchGoogleDocAsMarkdown(url, options = {}) {
   };
 }
 
+/**
+ * Fetch a Google Doc through the public export pipeline for a requested output
+ * format. Thin wrapper over the fallback module that injects sibling fetchers.
+ *
+ * @param {string} url - Google Docs URL
+ * @param {Object} [options] - Options
+ * @returns {Promise<Object>} Export result normalized for CLI rendering
+ */
+export function fetchGoogleDocByExportFormat(url, options = {}) {
+  return fetchGoogleDocByExportFormatImpl(
+    { fetchGoogleDoc, fetchGoogleDocAsMarkdown, fetchGoogleDocAsArchive },
+    url,
+    options
+  );
+}
+
+/**
+ * Capture a Google Doc through the browser model, falling back to public export
+ * when the editor does not expose model chunks. Thin wrapper.
+ *
+ * @param {string} url - Google Docs URL
+ * @param {Object} [options] - Capture options
+ * @returns {Promise<Object>} Browser-model or public-export result
+ */
+export function captureGoogleDocWithBrowserOrFallback(url, options = {}) {
+  return captureGoogleDocWithBrowserOrFallbackImpl(
+    {
+      captureGoogleDocWithBrowser,
+      fetchGoogleDoc,
+      fetchGoogleDocAsMarkdown,
+      fetchGoogleDocAsArchive,
+    },
+    url,
+    options
+  );
+}
+
 /**
  * Capture a Google Doc from the editor page model (`DOCS_modelChunk`).
  *
@@ -409,7 +456,7 @@ export async function captureGoogleDocWithBrowser(url, options = {}) {
       textBytes: Buffer.byteLength(capture.text || ''),
     }));
     if (capture.blocks.length === 0) {
-      throw new Error(
+      throw googleDocsBrowserModelUnavailableError(
         'Google Docs editor page did not expose DOCS_modelChunk data'
       );
     }

package/tests/unit/gdocs.test.js

@@ -6,6 +6,8 @@ import {
   buildEditUrl,
   buildDocsApiUrl,
   extractBase64Images,
+  captureGoogleDocWithBrowserOrFallback,
+  isGoogleDocsBrowserModelUnavailableError,
   parseGoogleDocsModelChunks,
   renderGoogleDocsCapture,
   renderDocsApiDocument,
@@ -22,6 +24,7 @@ import {
 import fs from 'fs';
 import os from 'os';
 import path from 'path';
+import nock from 'nock';
 
 jest.setTimeout(30000);
 
@@ -539,6 +542,49 @@ describe('gdocs', () => {
     });
   });
 
+  describe('captureGoogleDocWithBrowserOrFallback (issue #81)', () => {
+    afterEach(() => {
+      nock.cleanAll();
+    });
+
+    it('falls back to public export archive when the editor exposes no model chunks', async () => {
+      const tinyPng =
+        'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8/5+hHgAHggJ/PchI7wAAAABJRU5ErkJggg==';
+      nock('https://docs.google.com')
+        .get('/document/d/fallback-doc/export')
+        .query({ format: 'html' })
+        .reply(
+          200,
+          `<html><body><h1>Fallback Doc</h1><p>Complete text</p><img src="data:image/png;base64,${tinyPng}" alt="diagram"></body></html>`,
+          { 'content-type': 'text/html; charset=utf-8' }
+        );
+
+      const fallbackErrors = [];
+      const result = await captureGoogleDocWithBrowserOrFallback(
+        'https://docs.google.com/document/d/fallback-doc/edit',
+        {
+          format: 'archive',
+          waitMs: 0,
+          createBrowser: async () =>
+            fakeBrowserReturningModel({ chunks: [], cidUrlMap: {} }),
+          onFallback: (err) => fallbackErrors.push(err),
+        }
+      );
+
+      expect(result.method).toBe('public-export');
+      expect(result.fallback).toBe(true);
+      expect(result.markdown).toContain('Fallback Doc');
+      expect(result.markdown).toContain('Complete text');
+      expect(result.images).toHaveLength(1);
+      expect(result.html).toContain('images/image-01.png');
+      expect(fallbackErrors).toHaveLength(1);
+      expect(isGoogleDocsBrowserModelUnavailableError(fallbackErrors[0])).toBe(
+        true
+      );
+      expect(nock.isDone()).toBe(true);
+    });
+  });
+
   describe('GDOCS_EXPORT_FORMATS', () => {
     it('contains all expected formats', () => {
       expect(GDOCS_EXPORT_FORMATS).toEqual({
@@ -806,3 +852,23 @@ describe('gdocs', () => {
     });
   });
 });
+
+function fakeBrowserReturningModel(modelData) {
+  return {
+    async newPage() {
+      return {
+        async addInitScript() {},
+        async setUserAgent() {},
+        async setExtraHTTPHeaders() {},
+        async setViewport() {},
+        async goto() {},
+        async waitForTimeout() {},
+        async evaluate() {
+          return modelData;
+        },
+        async close() {},
+      };
+    },
+    async close() {},
+  };
+}