@link-assistant/web-capture 1.7.16 → 1.7.17

package/CHANGELOG.md

@@ -1,5 +1,11 @@
 # @link-assistant/web-capture
 
+## 1.7.17
+
+### Patch Changes
+
+- 86b6fc5: Wait for Google Docs browser-model chunks to quiesce before parsing captures.
+
 ## 1.7.16
 
 ### Patch Changes

package/README.md

@@ -102,6 +102,11 @@ web-capture https://docs.google.com/document/d/DOC_ID/edit --capture api
 web-capture https://docs.google.com/document/d/DOC_ID/edit --capture api --apiToken YOUR_TOKEN
 ```
 
+Browser-model capture waits until `DOCS_modelChunk` data stops changing before
+parsing. For very large or slow documents, tune the wait with
+`WEB_CAPTURE_GDOCS_STABILITY_MS` (default `1500`) and
+`WEB_CAPTURE_GDOCS_MAX_WAIT_MS` (default `30000`).
+
 ## API Endpoints (Server Mode)
 
 Start the server with `web-capture --serve` and use the endpoints below.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@link-assistant/web-capture",
-  "version": "1.7.16",
+  "version": "1.7.17",
   "description": "CLI and microservice to render web pages as HTML, Markdown, or PNG",
   "main": "src/index.js",
   "bin": {

package/src/gdocs-model-wait.js

@@ -0,0 +1,186 @@
+/* global document, window */
+import { googleDocsBrowserModelUnavailableError } from './gdocs-fallback.js';
+
+const DEFAULT_GDOCS_MODEL_STABILITY_MS = 1500;
+const DEFAULT_GDOCS_MODEL_MAX_WAIT_MS = 30000;
+const DEFAULT_GDOCS_MODEL_POLL_MS = 250;
+
+export async function waitForGoogleDocsModelQuiescence(
+  page,
+  { documentId, maxWaitMs, stabilityMs, pollMs, log }
+) {
+  const started = Date.now();
+  let lastFingerprint = null;
+  let stableSince = null;
+  let lastModelData = null;
+  let lastStableForMs = 0;
+  let pollCount = 0;
+
+  while (true) {
+    const modelData = await readGoogleDocsModelData(page);
+    pollCount += 1;
+    const fingerprint = googleDocsModelFingerprint(modelData);
+    lastModelData = modelData;
+
+    if (
+      fingerprint.chunks > 0 &&
+      sameModelFingerprint(fingerprint, lastFingerprint)
+    ) {
+      if (stableSince === null) {
+        stableSince = Date.now();
+      }
+      lastStableForMs = Date.now() - stableSince;
+      if (lastStableForMs >= stabilityMs) {
+        log?.debug?.(() => ({
+          event: 'gdocs.browser-model.quiesced',
+          documentId,
+          chunks: fingerprint.chunks,
+          chunkPayloadBytes: fingerprint.payloadBytes,
+          cidUrls: Object.keys(modelData.cidUrlMap || {}).length,
+          pollCount,
+          stableForMs: lastStableForMs,
+          elapsedMs: Date.now() - started,
+        }));
+        return {
+          ...modelData,
+          chunkPayloadBytes: fingerprint.payloadBytes,
+          pollCount,
+          stableForMs: lastStableForMs,
+        };
+      }
+    } else {
+      lastFingerprint = fingerprint;
+      stableSince = null;
+      lastStableForMs = 0;
+    }
+
+    const elapsedMs = Date.now() - started;
+    if (elapsedMs >= maxWaitMs) {
+      const lastChunks = lastModelData?.chunks?.length || 0;
+      const lastCidUrls = Object.keys(lastModelData?.cidUrlMap || {}).length;
+      throw googleDocsBrowserModelUnavailableError(
+        `Google Docs DOCS_modelChunk stream did not quiesce within ${maxWaitMs} ms for document ${documentId} (last chunks=${lastChunks}, payload_bytes=${fingerprint.payloadBytes}, cid_urls=${lastCidUrls}, poll_count=${pollCount}, stable_for_ms=${lastStableForMs})`
+      );
+    }
+
+    await waitForPage(page, Math.min(pollMs, maxWaitMs - elapsedMs));
+  }
+}
+
+export function resolveGoogleDocsModelWaitOptions(options) {
+  return {
+    stabilityMs: readDurationMs(
+      options.modelStabilityMs,
+      'WEB_CAPTURE_GDOCS_STABILITY_MS',
+      DEFAULT_GDOCS_MODEL_STABILITY_MS
+    ),
+    maxWaitMs: readDurationMs(
+      options.modelMaxWaitMs ?? options.waitMs,
+      'WEB_CAPTURE_GDOCS_MAX_WAIT_MS',
+      DEFAULT_GDOCS_MODEL_MAX_WAIT_MS
+    ),
+    pollMs: Math.max(
+      1,
+      readDurationMs(
+        options.modelPollMs,
+        'WEB_CAPTURE_GDOCS_POLL_MS',
+        DEFAULT_GDOCS_MODEL_POLL_MS
+      )
+    ),
+  };
+}
+
+async function readGoogleDocsModelData(page) {
+  return await evaluateOnPage(page, () => {
+    const chunks = [...(window.__captured_chunks || [])];
+    if (
+      window.DOCS_modelChunk &&
+      chunks.length === 0 &&
+      !chunks.includes(window.DOCS_modelChunk)
+    ) {
+      chunks.push(window.DOCS_modelChunk);
+    }
+    const cidUrlMap = {};
+    const scripts = document.querySelectorAll('script');
+    for (const script of scripts) {
+      const text = script.textContent || '';
+      if (!text.includes('docs-images-rt')) {
+        continue;
+      }
+      const regex =
+        /"([A-Za-z0-9_-]{20,})"\s*:\s*"(https:\/\/docs\.google\.com\/docs-images-rt\/[^"]+)"/g;
+      let match;
+      while ((match = regex.exec(text)) !== null) {
+        cidUrlMap[match[1]] = match[2]
+          .replace(/\\u003d/g, '=')
+          .replace(/\\u0026/g, '&')
+          .replace(/\\\//g, '/');
+      }
+    }
+    return { chunks, cidUrlMap };
+  });
+}
+
+function googleDocsModelFingerprint(modelData) {
+  const chunks = modelData?.chunks || [];
+  return {
+    chunks: chunks.length,
+    payloadBytes: chunks.reduce(
+      (total, chunk) => total + jsonPayloadByteLength(chunk),
+      0
+    ),
+  };
+}
+
+function sameModelFingerprint(left, right) {
+  return (
+    Boolean(left) &&
+    Boolean(right) &&
+    left.chunks === right.chunks &&
+    left.payloadBytes === right.payloadBytes
+  );
+}
+
+function jsonPayloadByteLength(value) {
+  try {
+    return Buffer.byteLength(JSON.stringify(value) || '');
+  } catch {
+    return 0;
+  }
+}
+
+function readDurationMs(optionValue, envName, defaultValue) {
+  const rawValue = optionValue ?? process.env[envName];
+  if (rawValue === undefined || rawValue === null || rawValue === '') {
+    return defaultValue;
+  }
+  const parsed = Number(rawValue);
+  if (!Number.isFinite(parsed) || parsed < 0) {
+    return defaultValue;
+  }
+  return Math.floor(parsed);
+}
+
+async function waitForPage(page, waitMs) {
+  if (waitMs <= 0) {
+    return;
+  }
+  const rawPage = page.rawPage || page;
+  if (typeof rawPage.waitForTimeout === 'function') {
+    await rawPage.waitForTimeout(waitMs);
+    return;
+  }
+  if (typeof page.waitForTimeout === 'function') {
+    await page.waitForTimeout(waitMs);
+    return;
+  }
+  await new Promise((resolve) => setTimeout(resolve, waitMs));
+}
+
+async function evaluateOnPage(page, fn) {
+  const rawPage = page.rawPage || page;
+  if (rawPage !== page && typeof rawPage.evaluate === 'function') {
+    return await rawPage.evaluate(fn);
+  }
+  return await page.evaluate(fn);
+}

package/src/gdocs.js

@@ -1,4 +1,4 @@
-/* global document, window */
+/* global window */
 // Google Docs capture module.
 //
 // Public export capture uses:
@@ -23,6 +23,10 @@ import {
   fetchGoogleDocByExportFormat as fetchGoogleDocByExportFormatImpl,
   captureGoogleDocWithBrowserOrFallback as captureGoogleDocWithBrowserOrFallbackImpl,
 } from './gdocs-fallback.js';
+import {
+  resolveGoogleDocsModelWaitOptions,
+  waitForGoogleDocsModelQuiescence,
+} from './gdocs-model-wait.js';
 
 export {
   normalizeGoogleDocsExportMarkdown,
@@ -369,7 +373,10 @@ export function captureGoogleDocWithBrowserOrFallback(url, options = {}) {
  * @param {Object} [options] - Capture options
  * @param {string} [options.engine='playwright'] - Browser engine
  * @param {string} [options.apiToken] - Optional Authorization header token
- * @param {number} [options.waitMs=8000] - Post-load wait for chunks
+ * @param {number} [options.waitMs] - Deprecated alias for modelMaxWaitMs
+ * @param {number} [options.modelStabilityMs=1500] - Required quiet period after the last model change
+ * @param {number} [options.modelMaxWaitMs=30000] - Hard timeout for model quiescence
+ * @param {number} [options.modelPollMs=250] - Poll interval while waiting for model quiescence
  * @param {Function} [options.createBrowser] - Browser factory for tests
  * @returns {Promise<{capture: Object, markdown: string, html: string, text: string, documentId: string, exportUrl: string}>}
  */
@@ -377,10 +384,10 @@ export async function captureGoogleDocWithBrowser(url, options = {}) {
   const {
     engine = 'playwright',
     apiToken,
-    waitMs = 8000,
     createBrowser = defaultCreateBrowser,
     log,
   } = options;
+  const modelWait = resolveGoogleDocsModelWaitOptions(options);
   const documentId = extractDocumentId(url);
   if (!documentId) {
     throw new Error(`Not a valid Google Docs URL: ${url}`);
@@ -392,7 +399,9 @@ export async function captureGoogleDocWithBrowser(url, options = {}) {
     documentId,
     editUrl,
     engine,
-    waitMs,
+    modelStabilityMs: modelWait.stabilityMs,
+    modelMaxWaitMs: modelWait.maxWaitMs,
+    modelPollMs: modelWait.pollMs,
     hasApiToken: Boolean(apiToken),
   }));
   const browser = await createBrowser(engine);
@@ -416,35 +425,10 @@ export async function captureGoogleDocWithBrowser(url, options = {}) {
       documentId,
       editUrl,
     }));
-    await waitForPage(page, waitMs);
-
-    const modelData = await evaluateOnPage(page, () => {
-      const chunks = [...(window.__captured_chunks || [])];
-      if (
-        window.DOCS_modelChunk &&
-        chunks.length === 0 &&
-        !chunks.includes(window.DOCS_modelChunk)
-      ) {
-        chunks.push(window.DOCS_modelChunk);
-      }
-      const cidUrlMap = {};
-      const scripts = document.querySelectorAll('script');
-      for (const script of scripts) {
-        const text = script.textContent || '';
-        if (!text.includes('docs-images-rt')) {
-          continue;
-        }
-        const regex =
-          /"([A-Za-z0-9_-]{20,})"\s*:\s*"(https:\/\/docs\.google\.com\/docs-images-rt\/[^"]+)"/g;
-        let match;
-        while ((match = regex.exec(text)) !== null) {
-          cidUrlMap[match[1]] = match[2]
-            .replace(/\\u003d/g, '=')
-            .replace(/\\u0026/g, '&')
-            .replace(/\\\//g, '/');
-        }
-      }
-      return { chunks, cidUrlMap };
+    const modelData = await waitForGoogleDocsModelQuiescence(page, {
+      ...modelWait,
+      documentId,
+      log,
     });
 
     const capture = parseGoogleDocsModelChunks(
@@ -457,6 +441,9 @@ export async function captureGoogleDocWithBrowser(url, options = {}) {
       documentId,
       chunks: modelData.chunks?.length || 0,
       cidUrls: Object.keys(modelData.cidUrlMap || {}).length,
+      chunkPayloadBytes: modelData.chunkPayloadBytes,
+      pollCount: modelData.pollCount,
+      stableForMs: modelData.stableForMs,
       blocks: capture.blocks.length,
       tables: capture.tables.length,
       images: capture.images.length,
@@ -474,6 +461,11 @@ export async function captureGoogleDocWithBrowser(url, options = {}) {
       text: renderGoogleDocsCapture(capture, 'txt'),
       documentId,
       exportUrl: editUrl,
+      diagnostics: {
+        modelChunkPayloadBytes: modelData.chunkPayloadBytes,
+        modelPollCount: modelData.pollCount,
+        modelStableForMs: modelData.stableForMs,
+      },
     };
   } finally {
     if (page) {
@@ -548,30 +540,6 @@ async function installDocsModelCapture(page) {
   }
 }
 
-async function waitForPage(page, waitMs) {
-  if (waitMs <= 0) {
-    return;
-  }
-  const rawPage = page.rawPage || page;
-  if (typeof rawPage.waitForTimeout === 'function') {
-    await rawPage.waitForTimeout(waitMs);
-    return;
-  }
-  if (typeof page.waitForTimeout === 'function') {
-    await page.waitForTimeout(waitMs);
-    return;
-  }
-  await new Promise((resolve) => setTimeout(resolve, waitMs));
-}
-
-async function evaluateOnPage(page, fn) {
-  const rawPage = page.rawPage || page;
-  if (rawPage !== page && typeof rawPage.evaluate === 'function') {
-    return await rawPage.evaluate(fn);
-  }
-  return await page.evaluate(fn);
-}
-
 /**
  * Extract a CID-to-image URL map from Google Docs editor HTML.
  *

package/tests/integration/gdocs-public-doc.test.js

@@ -182,7 +182,7 @@ describe('Google Docs public test document (issue #90)', () => {
       });
 
     const captureBrowserWithRetry = (url) =>
-      retry(() => captureGoogleDocWithBrowser(url, { waitMs: 3000 }), {
+      retry(() => captureGoogleDocWithBrowser(url), {
         retries: 2,
         baseDelay: 2000,
       });

package/tests/unit/gdocs.test.js

@@ -7,6 +7,7 @@ import {
   buildDocsApiUrl,
   extractBase64Images,
   captureGoogleDocWithBrowserOrFallback,
+  captureGoogleDocWithBrowser,
   isGoogleDocsBrowserModelUnavailableError,
   parseGoogleDocsModelChunks,
   renderGoogleDocsCapture,
@@ -948,6 +949,57 @@ describe('gdocs', () => {
     });
   });
 
+  describe('captureGoogleDocWithBrowser (issue #106)', () => {
+    it('waits for DOCS_modelChunk data to stop changing before parsing', async () => {
+      const browser = fakeBrowserReturningModelSequence([
+        {
+          chunks: [{ chunk: [{ ty: 'is', s: 'First paragraph\n' }] }],
+          cidUrlMap: {},
+        },
+        {
+          chunks: [{ chunk: [{ ty: 'is', s: 'First paragraph\n' }] }],
+          cidUrlMap: {},
+        },
+        {
+          chunks: [
+            { chunk: [{ ty: 'is', s: 'First paragraph\n' }] },
+            { chunk: [{ ty: 'is', s: 'Second paragraph\n' }] },
+          ],
+          cidUrlMap: {},
+        },
+        {
+          chunks: [
+            { chunk: [{ ty: 'is', s: 'First paragraph\n' }] },
+            { chunk: [{ ty: 'is', s: 'Second paragraph\n' }] },
+          ],
+          cidUrlMap: {},
+        },
+        {
+          chunks: [
+            { chunk: [{ ty: 'is', s: 'First paragraph\n' }] },
+            { chunk: [{ ty: 'is', s: 'Second paragraph\n' }] },
+          ],
+          cidUrlMap: {},
+        },
+      ]);
+
+      const result = await captureGoogleDocWithBrowser(
+        'https://docs.google.com/document/d/quiescence-doc/edit',
+        {
+          waitMs: 0,
+          modelMaxWaitMs: 200,
+          modelPollMs: 2,
+          modelStabilityMs: 2,
+          createBrowser: async () => browser,
+        }
+      );
+
+      expect(result.markdown).toContain('First paragraph');
+      expect(result.markdown).toContain('Second paragraph');
+      expect(browser.evaluateCalls).toBeGreaterThan(2);
+    });
+  });
+
   describe('GDOCS_EXPORT_FORMATS', () => {
     it('contains all expected formats', () => {
       expect(GDOCS_EXPORT_FORMATS).toEqual({
@@ -1303,3 +1355,31 @@ function fakeBrowserReturningModel(modelData) {
     async close() {},
   };
 }
+
+function fakeBrowserReturningModelSequence(modelDataSequence) {
+  let evaluateCalls = 0;
+  return {
+    get evaluateCalls() {
+      return evaluateCalls;
+    },
+    async newPage() {
+      return {
+        async addInitScript() {},
+        async setUserAgent() {},
+        async setExtraHTTPHeaders() {},
+        async setViewport() {},
+        async goto() {},
+        async waitForTimeout(waitMs) {
+          await new Promise((resolve) => setTimeout(resolve, waitMs));
+        },
+        async evaluate() {
+          const index = Math.min(evaluateCalls, modelDataSequence.length - 1);
+          evaluateCalls += 1;
+          return JSON.parse(JSON.stringify(modelDataSequence[index]));
+        },
+        async close() {},
+      };
+    },
+    async close() {},
+  };
+}