@link-assistant/web-capture 1.7.15 → 1.7.17

package/CHANGELOG.md

@@ -1,5 +1,17 @@
 # @link-assistant/web-capture
 
+## 1.7.17
+
+### Patch Changes
+
+- 86b6fc5: Wait for Google Docs browser-model chunks to quiesce before parsing captures.
+
+## 1.7.16
+
+### Patch Changes
+
+- bf7eaec: Keep Google Docs browser-model soft line breaks outside inline mark tags and preserve image dimensions in HTML output.
+
 ## 1.7.15
 
 ### Patch Changes

package/README.md

@@ -102,6 +102,11 @@ web-capture https://docs.google.com/document/d/DOC_ID/edit --capture api
 web-capture https://docs.google.com/document/d/DOC_ID/edit --capture api --apiToken YOUR_TOKEN
 ```
 
+Browser-model capture waits until `DOCS_modelChunk` data stops changing before
+parsing. For very large or slow documents, tune the wait with
+`WEB_CAPTURE_GDOCS_STABILITY_MS` (default `1500`) and
+`WEB_CAPTURE_GDOCS_MAX_WAIT_MS` (default `30000`).
+
 ## API Endpoints (Server Mode)
 
 Start the server with `web-capture --serve` and use the endpoints below.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@link-assistant/web-capture",
-  "version": "1.7.15",
+  "version": "1.7.17",
   "description": "CLI and microservice to render web pages as HTML, Markdown, or PNG",
   "main": "src/index.js",
   "bin": {

package/src/gdocs-model-wait.js

@@ -0,0 +1,186 @@
+/* global document, window */
+import { googleDocsBrowserModelUnavailableError } from './gdocs-fallback.js';
+
+const DEFAULT_GDOCS_MODEL_STABILITY_MS = 1500;
+const DEFAULT_GDOCS_MODEL_MAX_WAIT_MS = 30000;
+const DEFAULT_GDOCS_MODEL_POLL_MS = 250;
+
+export async function waitForGoogleDocsModelQuiescence(
+  page,
+  { documentId, maxWaitMs, stabilityMs, pollMs, log }
+) {
+  const started = Date.now();
+  let lastFingerprint = null;
+  let stableSince = null;
+  let lastModelData = null;
+  let lastStableForMs = 0;
+  let pollCount = 0;
+
+  while (true) {
+    const modelData = await readGoogleDocsModelData(page);
+    pollCount += 1;
+    const fingerprint = googleDocsModelFingerprint(modelData);
+    lastModelData = modelData;
+
+    if (
+      fingerprint.chunks > 0 &&
+      sameModelFingerprint(fingerprint, lastFingerprint)
+    ) {
+      if (stableSince === null) {
+        stableSince = Date.now();
+      }
+      lastStableForMs = Date.now() - stableSince;
+      if (lastStableForMs >= stabilityMs) {
+        log?.debug?.(() => ({
+          event: 'gdocs.browser-model.quiesced',
+          documentId,
+          chunks: fingerprint.chunks,
+          chunkPayloadBytes: fingerprint.payloadBytes,
+          cidUrls: Object.keys(modelData.cidUrlMap || {}).length,
+          pollCount,
+          stableForMs: lastStableForMs,
+          elapsedMs: Date.now() - started,
+        }));
+        return {
+          ...modelData,
+          chunkPayloadBytes: fingerprint.payloadBytes,
+          pollCount,
+          stableForMs: lastStableForMs,
+        };
+      }
+    } else {
+      lastFingerprint = fingerprint;
+      stableSince = null;
+      lastStableForMs = 0;
+    }
+
+    const elapsedMs = Date.now() - started;
+    if (elapsedMs >= maxWaitMs) {
+      const lastChunks = lastModelData?.chunks?.length || 0;
+      const lastCidUrls = Object.keys(lastModelData?.cidUrlMap || {}).length;
+      throw googleDocsBrowserModelUnavailableError(
+        `Google Docs DOCS_modelChunk stream did not quiesce within ${maxWaitMs} ms for document ${documentId} (last chunks=${lastChunks}, payload_bytes=${fingerprint.payloadBytes}, cid_urls=${lastCidUrls}, poll_count=${pollCount}, stable_for_ms=${lastStableForMs})`
+      );
+    }
+
+    await waitForPage(page, Math.min(pollMs, maxWaitMs - elapsedMs));
+  }
+}
+
+export function resolveGoogleDocsModelWaitOptions(options) {
+  return {
+    stabilityMs: readDurationMs(
+      options.modelStabilityMs,
+      'WEB_CAPTURE_GDOCS_STABILITY_MS',
+      DEFAULT_GDOCS_MODEL_STABILITY_MS
+    ),
+    maxWaitMs: readDurationMs(
+      options.modelMaxWaitMs ?? options.waitMs,
+      'WEB_CAPTURE_GDOCS_MAX_WAIT_MS',
+      DEFAULT_GDOCS_MODEL_MAX_WAIT_MS
+    ),
+    pollMs: Math.max(
+      1,
+      readDurationMs(
+        options.modelPollMs,
+        'WEB_CAPTURE_GDOCS_POLL_MS',
+        DEFAULT_GDOCS_MODEL_POLL_MS
+      )
+    ),
+  };
+}
+
+async function readGoogleDocsModelData(page) {
+  return await evaluateOnPage(page, () => {
+    const chunks = [...(window.__captured_chunks || [])];
+    if (
+      window.DOCS_modelChunk &&
+      chunks.length === 0 &&
+      !chunks.includes(window.DOCS_modelChunk)
+    ) {
+      chunks.push(window.DOCS_modelChunk);
+    }
+    const cidUrlMap = {};
+    const scripts = document.querySelectorAll('script');
+    for (const script of scripts) {
+      const text = script.textContent || '';
+      if (!text.includes('docs-images-rt')) {
+        continue;
+      }
+      const regex =
+        /"([A-Za-z0-9_-]{20,})"\s*:\s*"(https:\/\/docs\.google\.com\/docs-images-rt\/[^"]+)"/g;
+      let match;
+      while ((match = regex.exec(text)) !== null) {
+        cidUrlMap[match[1]] = match[2]
+          .replace(/\\u003d/g, '=')
+          .replace(/\\u0026/g, '&')
+          .replace(/\\\//g, '/');
+      }
+    }
+    return { chunks, cidUrlMap };
+  });
+}
+
+function googleDocsModelFingerprint(modelData) {
+  const chunks = modelData?.chunks || [];
+  return {
+    chunks: chunks.length,
+    payloadBytes: chunks.reduce(
+      (total, chunk) => total + jsonPayloadByteLength(chunk),
+      0
+    ),
+  };
+}
+
+function sameModelFingerprint(left, right) {
+  return (
+    Boolean(left) &&
+    Boolean(right) &&
+    left.chunks === right.chunks &&
+    left.payloadBytes === right.payloadBytes
+  );
+}
+
+function jsonPayloadByteLength(value) {
+  try {
+    return Buffer.byteLength(JSON.stringify(value) || '');
+  } catch {
+    return 0;
+  }
+}
+
+function readDurationMs(optionValue, envName, defaultValue) {
+  const rawValue = optionValue ?? process.env[envName];
+  if (rawValue === undefined || rawValue === null || rawValue === '') {
+    return defaultValue;
+  }
+  const parsed = Number(rawValue);
+  if (!Number.isFinite(parsed) || parsed < 0) {
+    return defaultValue;
+  }
+  return Math.floor(parsed);
+}
+
+async function waitForPage(page, waitMs) {
+  if (waitMs <= 0) {
+    return;
+  }
+  const rawPage = page.rawPage || page;
+  if (typeof rawPage.waitForTimeout === 'function') {
+    await rawPage.waitForTimeout(waitMs);
+    return;
+  }
+  if (typeof page.waitForTimeout === 'function') {
+    await page.waitForTimeout(waitMs);
+    return;
+  }
+  await new Promise((resolve) => setTimeout(resolve, waitMs));
+}
+
+async function evaluateOnPage(page, fn) {
+  const rawPage = page.rawPage || page;
+  if (rawPage !== page && typeof rawPage.evaluate === 'function') {
+    return await rawPage.evaluate(fn);
+  }
+  return await page.evaluate(fn);
+}

package/src/gdocs-render-markdown.js

@@ -138,9 +138,27 @@ function renderTextRunsMarkdown(nodes = []) {
       italic: Boolean(node.italic),
       strike: Boolean(node.strike),
     };
-    output += markdownStyleTransition(active, next);
-    output += node.text || '';
-    active = next;
+    const text = node.text || '';
+    let start = 0;
+    for (let idx = 0; idx < text.length; idx++) {
+      if (text[idx] !== '\n') {
+        continue;
+      }
+      if (idx > start) {
+        output += markdownStyleTransition(active, next);
+        output += text.slice(start, idx);
+        active = next;
+      }
+      output += markdownStyleTransition(active, inactive);
+      output += '\n';
+      active = inactive;
+      start = idx + 1;
+    }
+    if (start < text.length) {
+      output += markdownStyleTransition(active, next);
+      output += text.slice(start);
+      active = next;
+    }
   }
   output += markdownStyleTransition(active, inactive);
   return output;

package/src/gdocs.js

@@ -1,4 +1,4 @@
-/* global document, window */
+/* global window */
 // Google Docs capture module.
 //
 // Public export capture uses:
@@ -23,6 +23,10 @@ import {
   fetchGoogleDocByExportFormat as fetchGoogleDocByExportFormatImpl,
   captureGoogleDocWithBrowserOrFallback as captureGoogleDocWithBrowserOrFallbackImpl,
 } from './gdocs-fallback.js';
+import {
+  resolveGoogleDocsModelWaitOptions,
+  waitForGoogleDocsModelQuiescence,
+} from './gdocs-model-wait.js';
 
 export {
   normalizeGoogleDocsExportMarkdown,
@@ -369,7 +373,10 @@ export function captureGoogleDocWithBrowserOrFallback(url, options = {}) {
  * @param {Object} [options] - Capture options
  * @param {string} [options.engine='playwright'] - Browser engine
  * @param {string} [options.apiToken] - Optional Authorization header token
- * @param {number} [options.waitMs=8000] - Post-load wait for chunks
+ * @param {number} [options.waitMs] - Deprecated alias for modelMaxWaitMs
+ * @param {number} [options.modelStabilityMs=1500] - Required quiet period after the last model change
+ * @param {number} [options.modelMaxWaitMs=30000] - Hard timeout for model quiescence
+ * @param {number} [options.modelPollMs=250] - Poll interval while waiting for model quiescence
  * @param {Function} [options.createBrowser] - Browser factory for tests
  * @returns {Promise<{capture: Object, markdown: string, html: string, text: string, documentId: string, exportUrl: string}>}
  */
@@ -377,10 +384,10 @@ export async function captureGoogleDocWithBrowser(url, options = {}) {
   const {
     engine = 'playwright',
     apiToken,
-    waitMs = 8000,
     createBrowser = defaultCreateBrowser,
     log,
   } = options;
+  const modelWait = resolveGoogleDocsModelWaitOptions(options);
   const documentId = extractDocumentId(url);
   if (!documentId) {
     throw new Error(`Not a valid Google Docs URL: ${url}`);
@@ -392,7 +399,9 @@ export async function captureGoogleDocWithBrowser(url, options = {}) {
     documentId,
     editUrl,
     engine,
-    waitMs,
+    modelStabilityMs: modelWait.stabilityMs,
+    modelMaxWaitMs: modelWait.maxWaitMs,
+    modelPollMs: modelWait.pollMs,
     hasApiToken: Boolean(apiToken),
   }));
   const browser = await createBrowser(engine);
@@ -416,35 +425,10 @@ export async function captureGoogleDocWithBrowser(url, options = {}) {
       documentId,
       editUrl,
     }));
-    await waitForPage(page, waitMs);
-
-    const modelData = await evaluateOnPage(page, () => {
-      const chunks = [...(window.__captured_chunks || [])];
-      if (
-        window.DOCS_modelChunk &&
-        chunks.length === 0 &&
-        !chunks.includes(window.DOCS_modelChunk)
-      ) {
-        chunks.push(window.DOCS_modelChunk);
-      }
-      const cidUrlMap = {};
-      const scripts = document.querySelectorAll('script');
-      for (const script of scripts) {
-        const text = script.textContent || '';
-        if (!text.includes('docs-images-rt')) {
-          continue;
-        }
-        const regex =
-          /"([A-Za-z0-9_-]{20,})"\s*:\s*"(https:\/\/docs\.google\.com\/docs-images-rt\/[^"]+)"/g;
-        let match;
-        while ((match = regex.exec(text)) !== null) {
-          cidUrlMap[match[1]] = match[2]
-            .replace(/\\u003d/g, '=')
-            .replace(/\\u0026/g, '&')
-            .replace(/\\\//g, '/');
-        }
-      }
-      return { chunks, cidUrlMap };
+    const modelData = await waitForGoogleDocsModelQuiescence(page, {
+      ...modelWait,
+      documentId,
+      log,
     });
 
     const capture = parseGoogleDocsModelChunks(
@@ -457,6 +441,9 @@ export async function captureGoogleDocWithBrowser(url, options = {}) {
       documentId,
       chunks: modelData.chunks?.length || 0,
       cidUrls: Object.keys(modelData.cidUrlMap || {}).length,
+      chunkPayloadBytes: modelData.chunkPayloadBytes,
+      pollCount: modelData.pollCount,
+      stableForMs: modelData.stableForMs,
       blocks: capture.blocks.length,
       tables: capture.tables.length,
       images: capture.images.length,
@@ -474,6 +461,11 @@ export async function captureGoogleDocWithBrowser(url, options = {}) {
       text: renderGoogleDocsCapture(capture, 'txt'),
       documentId,
       exportUrl: editUrl,
+      diagnostics: {
+        modelChunkPayloadBytes: modelData.chunkPayloadBytes,
+        modelPollCount: modelData.pollCount,
+        modelStableForMs: modelData.stableForMs,
+      },
     };
   } finally {
     if (page) {
@@ -548,30 +540,6 @@ async function installDocsModelCapture(page) {
   }
 }
 
-async function waitForPage(page, waitMs) {
-  if (waitMs <= 0) {
-    return;
-  }
-  const rawPage = page.rawPage || page;
-  if (typeof rawPage.waitForTimeout === 'function') {
-    await rawPage.waitForTimeout(waitMs);
-    return;
-  }
-  if (typeof page.waitForTimeout === 'function') {
-    await page.waitForTimeout(waitMs);
-    return;
-  }
-  await new Promise((resolve) => setTimeout(resolve, waitMs));
-}
-
-async function evaluateOnPage(page, fn) {
-  const rawPage = page.rawPage || page;
-  if (rawPage !== page && typeof rawPage.evaluate === 'function') {
-    return await rawPage.evaluate(fn);
-  }
-  return await page.evaluate(fn);
-}
-
 /**
  * Extract a CID-to-image URL map from Google Docs editor HTML.
  *
@@ -1219,7 +1187,17 @@ function renderContentHtml(content = []) {
 }
 
 function renderMarkedHtml(node) {
-  let output = escapeHtml(node.text || '').replace(/\n/g, '<br>');
+  return String(node.text || '')
+    .split('\n')
+    .map((segment) => renderMarkedHtmlSegment(node, segment))
+    .join('<br>');
+}
+
+function renderMarkedHtmlSegment(node, text) {
+  if (!text) {
+    return '';
+  }
+  let output = escapeHtml(text);
   if (node.bold) {
     output = `<strong>${output}</strong>`;
   }

package/tests/integration/gdocs-public-doc.test.js

@@ -182,7 +182,7 @@ describe('Google Docs public test document (issue #90)', () => {
       });
 
     const captureBrowserWithRetry = (url) =>
-      retry(() => captureGoogleDocWithBrowser(url, { waitMs: 3000 }), {
+      retry(() => captureGoogleDocWithBrowser(url), {
         retries: 2,
         baseDelay: 2000,
       });

package/tests/unit/gdocs.test.js

@@ -7,6 +7,7 @@ import {
   buildDocsApiUrl,
   extractBase64Images,
   captureGoogleDocWithBrowserOrFallback,
+  captureGoogleDocWithBrowser,
   isGoogleDocsBrowserModelUnavailableError,
   parseGoogleDocsModelChunks,
   renderGoogleDocsCapture,
@@ -26,10 +27,26 @@ import {
 import fs from 'fs';
 import os from 'os';
 import path from 'path';
+import { fileURLToPath } from 'url';
 import nock from 'nock';
 
 jest.setTimeout(30000);
 
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = path.dirname(__filename);
+const REPO_ROOT = path.resolve(__dirname, '..', '..', '..');
+const ISSUE_104_FIXTURE_DIR = path.join(
+  REPO_ROOT,
+  'docs',
+  'case-studies',
+  'issue-104',
+  'fixtures'
+);
+
+function readIssue104Fixture(filename) {
+  return fs.readFileSync(path.join(ISSUE_104_FIXTURE_DIR, filename), 'utf-8');
+}
+
 describe('gdocs', () => {
   describe('isGoogleDocsUrl', () => {
     it('returns true for standard Google Docs edit URL', () => {
@@ -648,6 +665,74 @@ describe('gdocs', () => {
       );
     });
 
+    it('renders soft breaks outside marks and preserves image dimensions (issue #104)', () => {
+      const fixture = JSON.parse(
+        readIssue104Fixture('multiline-marked-inline-image-model.json')
+      );
+      const capture = parseGoogleDocsModelChunks(
+        fixture.chunks,
+        fixture.cidUrlMap
+      );
+
+      expect(capture.blocks[0].content).toEqual([
+        expect.objectContaining({
+          type: 'text',
+          text: 'Line one of bold text.',
+          bold: true,
+        }),
+        expect.objectContaining({ type: 'text', text: '\n', bold: false }),
+        expect.objectContaining({
+          type: 'text',
+          text: 'Line two of bold text.',
+          bold: true,
+        }),
+        expect.objectContaining({
+          type: 'image',
+          alt: 'Inline diagram',
+          width: 320,
+          height: 180,
+        }),
+        expect.objectContaining({ type: 'text', text: '\n\n', bold: false }),
+        expect.objectContaining({
+          type: 'text',
+          text: 'Line three of bold text.',
+          bold: true,
+        }),
+      ]);
+      expect(renderGoogleDocsCapture(capture, 'html')).toBe(
+        readIssue104Fixture(
+          'multiline-marked-inline-image.expected.html'
+        ).trimEnd()
+      );
+      expect(renderGoogleDocsCapture(capture, 'markdown')).toBe(
+        readIssue104Fixture('multiline-marked-inline-image.expected.md')
+      );
+    });
+
+    it('does not keep inline marks open across embedded newlines (issue #104)', () => {
+      const capture = {
+        blocks: [
+          {
+            type: 'paragraph',
+            content: [
+              {
+                type: 'text',
+                text: 'Alpha\nBeta',
+                bold: true,
+              },
+            ],
+          },
+        ],
+      };
+
+      expect(renderGoogleDocsCapture(capture, 'html')).toBe(
+        '<!doctype html><html><body><p><strong>Alpha</strong><br><strong>Beta</strong></p></body></html>'
+      );
+      expect(renderGoogleDocsCapture(capture, 'markdown')).toBe(
+        '**Alpha**\n**Beta**\n'
+      );
+    });
+
     it('keeps nested bold and italic markers balanced (issue #96)', () => {
       const text = 'Bold text with italic inside and back to bold\n';
       const chunks = [
@@ -864,6 +949,57 @@ describe('gdocs', () => {
     });
   });
 
+  describe('captureGoogleDocWithBrowser (issue #106)', () => {
+    it('waits for DOCS_modelChunk data to stop changing before parsing', async () => {
+      const browser = fakeBrowserReturningModelSequence([
+        {
+          chunks: [{ chunk: [{ ty: 'is', s: 'First paragraph\n' }] }],
+          cidUrlMap: {},
+        },
+        {
+          chunks: [{ chunk: [{ ty: 'is', s: 'First paragraph\n' }] }],
+          cidUrlMap: {},
+        },
+        {
+          chunks: [
+            { chunk: [{ ty: 'is', s: 'First paragraph\n' }] },
+            { chunk: [{ ty: 'is', s: 'Second paragraph\n' }] },
+          ],
+          cidUrlMap: {},
+        },
+        {
+          chunks: [
+            { chunk: [{ ty: 'is', s: 'First paragraph\n' }] },
+            { chunk: [{ ty: 'is', s: 'Second paragraph\n' }] },
+          ],
+          cidUrlMap: {},
+        },
+        {
+          chunks: [
+            { chunk: [{ ty: 'is', s: 'First paragraph\n' }] },
+            { chunk: [{ ty: 'is', s: 'Second paragraph\n' }] },
+          ],
+          cidUrlMap: {},
+        },
+      ]);
+
+      const result = await captureGoogleDocWithBrowser(
+        'https://docs.google.com/document/d/quiescence-doc/edit',
+        {
+          waitMs: 0,
+          modelMaxWaitMs: 200,
+          modelPollMs: 2,
+          modelStabilityMs: 2,
+          createBrowser: async () => browser,
+        }
+      );
+
+      expect(result.markdown).toContain('First paragraph');
+      expect(result.markdown).toContain('Second paragraph');
+      expect(browser.evaluateCalls).toBeGreaterThan(2);
+    });
+  });
+
   describe('GDOCS_EXPORT_FORMATS', () => {
     it('contains all expected formats', () => {
       expect(GDOCS_EXPORT_FORMATS).toEqual({
@@ -1219,3 +1355,31 @@ function fakeBrowserReturningModel(modelData) {
     async close() {},
   };
 }
+
+function fakeBrowserReturningModelSequence(modelDataSequence) {
+  let evaluateCalls = 0;
+  return {
+    get evaluateCalls() {
+      return evaluateCalls;
+    },
+    async newPage() {
+      return {
+        async addInitScript() {},
+        async setUserAgent() {},
+        async setExtraHTTPHeaders() {},
+        async setViewport() {},
+        async goto() {},
+        async waitForTimeout(waitMs) {
+          await new Promise((resolve) => setTimeout(resolve, waitMs));
+        },
+        async evaluate() {
+          const index = Math.min(evaluateCalls, modelDataSequence.length - 1);
+          evaluateCalls += 1;
+          return JSON.parse(JSON.stringify(modelDataSequence[index]));
+        },
+        async close() {},
+      };
+    },
+    async close() {},
+  };
+}