@link-assistant/web-capture 1.7.16 → 1.7.18

package/CHANGELOG.md

@@ -1,5 +1,17 @@
 # @link-assistant/web-capture
 
+## 1.7.18
+
+### Patch Changes
+
+- 2c69a73: Fix Google Docs browser capture list semantics and continuation paragraph handling.
+
+## 1.7.17
+
+### Patch Changes
+
+- 86b6fc5: Wait for Google Docs browser-model chunks to quiesce before parsing captures.
+
 ## 1.7.16
 
 ### Patch Changes

package/README.md

@@ -102,6 +102,11 @@ web-capture https://docs.google.com/document/d/DOC_ID/edit --capture api
 web-capture https://docs.google.com/document/d/DOC_ID/edit --capture api --apiToken YOUR_TOKEN
 ```
 
+Browser-model capture waits until `DOCS_modelChunk` data stops changing before
+parsing. For very large or slow documents, tune the wait with
+`WEB_CAPTURE_GDOCS_STABILITY_MS` (default `1500`) and
+`WEB_CAPTURE_GDOCS_MAX_WAIT_MS` (default `30000`).
+
 ## API Endpoints (Server Mode)
 
 Start the server with `web-capture --serve` and use the endpoints below.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@link-assistant/web-capture",
-  "version": "1.7.16",
+  "version": "1.7.18",
   "description": "CLI and microservice to render web pages as HTML, Markdown, or PNG",
   "main": "src/index.js",
   "bin": {

package/src/gdocs-model-wait.js

@@ -0,0 +1,186 @@
+/* global document, window */
+import { googleDocsBrowserModelUnavailableError } from './gdocs-fallback.js';
+
+const DEFAULT_GDOCS_MODEL_STABILITY_MS = 1500;
+const DEFAULT_GDOCS_MODEL_MAX_WAIT_MS = 30000;
+const DEFAULT_GDOCS_MODEL_POLL_MS = 250;
+
+export async function waitForGoogleDocsModelQuiescence(
+  page,
+  { documentId, maxWaitMs, stabilityMs, pollMs, log }
+) {
+  const started = Date.now();
+  let lastFingerprint = null;
+  let stableSince = null;
+  let lastModelData = null;
+  let lastStableForMs = 0;
+  let pollCount = 0;
+
+  while (true) {
+    const modelData = await readGoogleDocsModelData(page);
+    pollCount += 1;
+    const fingerprint = googleDocsModelFingerprint(modelData);
+    lastModelData = modelData;
+
+    if (
+      fingerprint.chunks > 0 &&
+      sameModelFingerprint(fingerprint, lastFingerprint)
+    ) {
+      if (stableSince === null) {
+        stableSince = Date.now();
+      }
+      lastStableForMs = Date.now() - stableSince;
+      if (lastStableForMs >= stabilityMs) {
+        log?.debug?.(() => ({
+          event: 'gdocs.browser-model.quiesced',
+          documentId,
+          chunks: fingerprint.chunks,
+          chunkPayloadBytes: fingerprint.payloadBytes,
+          cidUrls: Object.keys(modelData.cidUrlMap || {}).length,
+          pollCount,
+          stableForMs: lastStableForMs,
+          elapsedMs: Date.now() - started,
+        }));
+        return {
+          ...modelData,
+          chunkPayloadBytes: fingerprint.payloadBytes,
+          pollCount,
+          stableForMs: lastStableForMs,
+        };
+      }
+    } else {
+      lastFingerprint = fingerprint;
+      stableSince = null;
+      lastStableForMs = 0;
+    }
+
+    const elapsedMs = Date.now() - started;
+    if (elapsedMs >= maxWaitMs) {
+      const lastChunks = lastModelData?.chunks?.length || 0;
+      const lastCidUrls = Object.keys(lastModelData?.cidUrlMap || {}).length;
+      throw googleDocsBrowserModelUnavailableError(
+        `Google Docs DOCS_modelChunk stream did not quiesce within ${maxWaitMs} ms for document ${documentId} (last chunks=${lastChunks}, payload_bytes=${fingerprint.payloadBytes}, cid_urls=${lastCidUrls}, poll_count=${pollCount}, stable_for_ms=${lastStableForMs})`
+      );
+    }
+
+    await waitForPage(page, Math.min(pollMs, maxWaitMs - elapsedMs));
+  }
+}
+
+export function resolveGoogleDocsModelWaitOptions(options) {
+  return {
+    stabilityMs: readDurationMs(
+      options.modelStabilityMs,
+      'WEB_CAPTURE_GDOCS_STABILITY_MS',
+      DEFAULT_GDOCS_MODEL_STABILITY_MS
+    ),
+    maxWaitMs: readDurationMs(
+      options.modelMaxWaitMs ?? options.waitMs,
+      'WEB_CAPTURE_GDOCS_MAX_WAIT_MS',
+      DEFAULT_GDOCS_MODEL_MAX_WAIT_MS
+    ),
+    pollMs: Math.max(
+      1,
+      readDurationMs(
+        options.modelPollMs,
+        'WEB_CAPTURE_GDOCS_POLL_MS',
+        DEFAULT_GDOCS_MODEL_POLL_MS
+      )
+    ),
+  };
+}
+
+async function readGoogleDocsModelData(page) {
+  return await evaluateOnPage(page, () => {
+    const chunks = [...(window.__captured_chunks || [])];
+    if (
+      window.DOCS_modelChunk &&
+      chunks.length === 0 &&
+      !chunks.includes(window.DOCS_modelChunk)
+    ) {
+      chunks.push(window.DOCS_modelChunk);
+    }
+    const cidUrlMap = {};
+    const scripts = document.querySelectorAll('script');
+    for (const script of scripts) {
+      const text = script.textContent || '';
+      if (!text.includes('docs-images-rt')) {
+        continue;
+      }
+      const regex =
+        /"([A-Za-z0-9_-]{20,})"\s*:\s*"(https:\/\/docs\.google\.com\/docs-images-rt\/[^"]+)"/g;
+      let match;
+      while ((match = regex.exec(text)) !== null) {
+        cidUrlMap[match[1]] = match[2]
+          .replace(/\\u003d/g, '=')
+          .replace(/\\u0026/g, '&')
+          .replace(/\\\//g, '/');
+      }
+    }
+    return { chunks, cidUrlMap };
+  });
+}
+
+function googleDocsModelFingerprint(modelData) {
+  const chunks = modelData?.chunks || [];
+  return {
+    chunks: chunks.length,
+    payloadBytes: chunks.reduce(
+      (total, chunk) => total + jsonPayloadByteLength(chunk),
+      0
+    ),
+  };
+}
+
+function sameModelFingerprint(left, right) {
+  return (
+    Boolean(left) &&
+    Boolean(right) &&
+    left.chunks === right.chunks &&
+    left.payloadBytes === right.payloadBytes
+  );
+}
+
+function jsonPayloadByteLength(value) {
+  try {
+    return Buffer.byteLength(JSON.stringify(value) || '');
+  } catch {
+    return 0;
+  }
+}
+
+function readDurationMs(optionValue, envName, defaultValue) {
+  const rawValue = optionValue ?? process.env[envName];
+  if (rawValue === undefined || rawValue === null || rawValue === '') {
+    return defaultValue;
+  }
+  const parsed = Number(rawValue);
+  if (!Number.isFinite(parsed) || parsed < 0) {
+    return defaultValue;
+  }
+  return Math.floor(parsed);
+}
+
+async function waitForPage(page, waitMs) {
+  if (waitMs <= 0) {
+    return;
+  }
+  const rawPage = page.rawPage || page;
+  if (typeof rawPage.waitForTimeout === 'function') {
+    await rawPage.waitForTimeout(waitMs);
+    return;
+  }
+  if (typeof page.waitForTimeout === 'function') {
+    await page.waitForTimeout(waitMs);
+    return;
+  }
+  await new Promise((resolve) => setTimeout(resolve, waitMs));
+}
+
+async function evaluateOnPage(page, fn) {
+  const rawPage = page.rawPage || page;
+  if (rawPage !== page && typeof rawPage.evaluate === 'function') {
+    return await rawPage.evaluate(fn);
+  }
+  return await page.evaluate(fn);
+}

package/src/gdocs.js

@@ -1,4 +1,4 @@
-/* global document, window */
+/* global window */
 // Google Docs capture module.
 //
 // Public export capture uses:
@@ -9,6 +9,7 @@
 
 import fetch from 'node-fetch';
 import he from 'he';
+import * as cheerio from 'cheerio';
 import { convertHtmlToMarkdown } from './lib.js';
 import { createBrowser as defaultCreateBrowser } from './browser.js';
 import {
@@ -23,6 +24,10 @@ import {
   fetchGoogleDocByExportFormat as fetchGoogleDocByExportFormatImpl,
   captureGoogleDocWithBrowserOrFallback as captureGoogleDocWithBrowserOrFallbackImpl,
 } from './gdocs-fallback.js';
+import {
+  resolveGoogleDocsModelWaitOptions,
+  waitForGoogleDocsModelQuiescence,
+} from './gdocs-model-wait.js';
 
 export {
   normalizeGoogleDocsExportMarkdown,
@@ -196,11 +201,10 @@ export async function fetchGoogleDoc(url, options = {}) {
     bytes: Buffer.byteLength(rawContent),
   }));
 
-  // Decode HTML entities to unicode for text-based formats
+  // Keep HTML markup escaped so literal examples such as `<ol>` do not
+  // become real tags before the HTML parser sees the document.
   const content =
-    format === 'html' || format === 'txt' || format === 'md'
-      ? he.decode(rawContent)
-      : rawContent;
+    format === 'txt' || format === 'md' ? he.decode(rawContent) : rawContent;
 
   return {
     content,
@@ -369,7 +373,10 @@ export function captureGoogleDocWithBrowserOrFallback(url, options = {}) {
  * @param {Object} [options] - Capture options
  * @param {string} [options.engine='playwright'] - Browser engine
  * @param {string} [options.apiToken] - Optional Authorization header token
- * @param {number} [options.waitMs=8000] - Post-load wait for chunks
+ * @param {number} [options.waitMs] - Deprecated alias for modelMaxWaitMs
+ * @param {number} [options.modelStabilityMs=1500] - Required quiet period after the last model change
+ * @param {number} [options.modelMaxWaitMs=30000] - Hard timeout for model quiescence
+ * @param {number} [options.modelPollMs=250] - Poll interval while waiting for model quiescence
  * @param {Function} [options.createBrowser] - Browser factory for tests
  * @returns {Promise<{capture: Object, markdown: string, html: string, text: string, documentId: string, exportUrl: string}>}
  */
@@ -377,10 +384,10 @@ export async function captureGoogleDocWithBrowser(url, options = {}) {
   const {
     engine = 'playwright',
     apiToken,
-    waitMs = 8000,
     createBrowser = defaultCreateBrowser,
     log,
   } = options;
+  const modelWait = resolveGoogleDocsModelWaitOptions(options);
   const documentId = extractDocumentId(url);
   if (!documentId) {
     throw new Error(`Not a valid Google Docs URL: ${url}`);
@@ -392,7 +399,9 @@ export async function captureGoogleDocWithBrowser(url, options = {}) {
     documentId,
     editUrl,
     engine,
-    waitMs,
+    modelStabilityMs: modelWait.stabilityMs,
+    modelMaxWaitMs: modelWait.maxWaitMs,
+    modelPollMs: modelWait.pollMs,
     hasApiToken: Boolean(apiToken),
   }));
   const browser = await createBrowser(engine);
@@ -416,47 +425,30 @@ export async function captureGoogleDocWithBrowser(url, options = {}) {
       documentId,
       editUrl,
     }));
-    await waitForPage(page, waitMs);
-
-    const modelData = await evaluateOnPage(page, () => {
-      const chunks = [...(window.__captured_chunks || [])];
-      if (
-        window.DOCS_modelChunk &&
-        chunks.length === 0 &&
-        !chunks.includes(window.DOCS_modelChunk)
-      ) {
-        chunks.push(window.DOCS_modelChunk);
-      }
-      const cidUrlMap = {};
-      const scripts = document.querySelectorAll('script');
-      for (const script of scripts) {
-        const text = script.textContent || '';
-        if (!text.includes('docs-images-rt')) {
-          continue;
-        }
-        const regex =
-          /"([A-Za-z0-9_-]{20,})"\s*:\s*"(https:\/\/docs\.google\.com\/docs-images-rt\/[^"]+)"/g;
-        let match;
-        while ((match = regex.exec(text)) !== null) {
-          cidUrlMap[match[1]] = match[2]
-            .replace(/\\u003d/g, '=')
-            .replace(/\\u0026/g, '&')
-            .replace(/\\\//g, '/');
-        }
-      }
-      return { chunks, cidUrlMap };
+    const modelData = await waitForGoogleDocsModelQuiescence(page, {
+      ...modelWait,
+      documentId,
+      log,
     });
+    const exportHtml = await fetchGoogleDocsExportHtmlFromPage(
+      page,
+      documentId,
+      log
+    );
 
     const capture = parseGoogleDocsModelChunks(
       modelData.chunks,
       modelData.cidUrlMap,
-      { log }
+      { log, exportHtml }
     );
     log?.debug?.(() => ({
       event: 'gdocs.browser-model.parsed',
       documentId,
       chunks: modelData.chunks?.length || 0,
       cidUrls: Object.keys(modelData.cidUrlMap || {}).length,
+      chunkPayloadBytes: modelData.chunkPayloadBytes,
+      pollCount: modelData.pollCount,
+      stableForMs: modelData.stableForMs,
       blocks: capture.blocks.length,
       tables: capture.tables.length,
       images: capture.images.length,
@@ -474,6 +466,11 @@ export async function captureGoogleDocWithBrowser(url, options = {}) {
       text: renderGoogleDocsCapture(capture, 'txt'),
       documentId,
       exportUrl: editUrl,
+      diagnostics: {
+        modelChunkPayloadBytes: modelData.chunkPayloadBytes,
+        modelPollCount: modelData.pollCount,
+        modelStableForMs: modelData.stableForMs,
+      },
     };
   } finally {
     if (page) {
@@ -548,30 +545,6 @@ async function installDocsModelCapture(page) {
   }
 }
 
-async function waitForPage(page, waitMs) {
-  if (waitMs <= 0) {
-    return;
-  }
-  const rawPage = page.rawPage || page;
-  if (typeof rawPage.waitForTimeout === 'function') {
-    await rawPage.waitForTimeout(waitMs);
-    return;
-  }
-  if (typeof page.waitForTimeout === 'function') {
-    await page.waitForTimeout(waitMs);
-    return;
-  }
-  await new Promise((resolve) => setTimeout(resolve, waitMs));
-}
-
-async function evaluateOnPage(page, fn) {
-  const rawPage = page.rawPage || page;
-  if (rawPage !== page && typeof rawPage.evaluate === 'function') {
-    return await rawPage.evaluate(fn);
-  }
-  return await page.evaluate(fn);
-}
-
 /**
  * Extract a CID-to-image URL map from Google Docs editor HTML.
  *
@@ -599,6 +572,7 @@ export function extractCidUrlMapFromHtml(html) {
  * @param {Object<string,string>} [cidUrlMap={}] - CID-to-docs-images-rt URL map
  * @param {Object} [options] - Parse options
  * @param {Object} [options.log] - Optional verbose logger
+ * @param {string} [options.exportHtml] - Optional Google Docs export HTML used for list and quote semantics
  * @returns {{blocks: Array, tables: Array, images: Array, text: string}}
  */
 export function parseGoogleDocsModelChunks(
@@ -845,12 +819,17 @@ export function parseGoogleDocsModelChunks(
     }
   }
 
-  return {
+  const capture = {
     blocks,
     tables,
     images,
     text: contentBlocksToText(blocks),
   };
+  if (options.exportHtml) {
+    applyGoogleDocsExportSemanticHints(capture, options.exportHtml);
+    capture.text = contentBlocksToText(capture.blocks);
+  }
+  return capture;
 }
 
 function cellIsEmpty(cell) {
@@ -921,13 +900,13 @@ function applyInlineTextStyle(inlineStyles, start, end, patch) {
 
 function textStyleFromModel(item) {
   const style = {};
-  if (item.sm?.ts_bd) {
+  if (item.sm?.ts_bd && item.sm?.ts_bd_i !== true) {
     style.bold = true;
   }
-  if (item.sm?.ts_it) {
+  if (item.sm?.ts_it && item.sm?.ts_it_i !== true) {
     style.italic = true;
   }
-  if (item.sm?.ts_st) {
+  if (item.sm?.ts_st && item.sm?.ts_st_i !== true) {
     style.strike = true;
   }
   return style;
@@ -976,19 +955,136 @@ function paragraphMetaForEndPosition(styleMaps, endPos, text) {
 }
 
 function inferOrderedList(list, text) {
-  // Google Docs does not expose imported DOCX list marker type in the public
-  // model chunk for this document. Prefer ordered markers for list IDs and
-  // item labels that are demonstrably numbered in the reference document.
-  if (
-    /ordered|^\d+(?:\.|\))|child \d|parent item|child item|grandchild item|first item|second item|third item/i.test(
-      text
-    )
-  ) {
-    return /^kix\.list\.(7|8|9|10|11|13)$/u.test(list.id);
-  }
+  void list;
+  void text;
   return false;
 }
 
+async function fetchGoogleDocsExportHtmlFromPage(page, documentId, log) {
+  const exportUrl = buildExportUrl(documentId, 'html');
+  try {
+    const result = await page.evaluate(async (url) => {
+      const response = await fetch(url, {
+        credentials: 'include',
+        redirect: 'follow',
+      });
+      return {
+        ok: response.ok,
+        status: response.status,
+        statusText: response.statusText,
+        contentType: response.headers.get('content-type'),
+        body: await response.text(),
+      };
+    }, exportUrl);
+    log?.debug?.(() => ({
+      event: 'gdocs.browser-model.export-hints.response',
+      documentId,
+      exportUrl,
+      status: result.status,
+      ok: result.ok,
+      contentType: result.contentType,
+      bytes: Buffer.byteLength(result.body || ''),
+    }));
+    return result.ok ? result.body || '' : null;
+  } catch (error) {
+    log?.debug?.(() => ({
+      event: 'gdocs.browser-model.export-hints.failed',
+      documentId,
+      exportUrl,
+      error: error?.message || String(error),
+    }));
+    return null;
+  }
+}
+
+function applyGoogleDocsExportSemanticHints(capture, exportHtml) {
+  const hints = extractGoogleDocsExportSemanticHints(exportHtml);
+  let cursor = 0;
+  for (const block of capture.blocks || []) {
+    if (block.type !== 'paragraph') {
+      continue;
+    }
+    const text = normalizeSemanticText(contentToText(block.content));
+    if (!text) {
+      continue;
+    }
+    const match = findNextSemanticHint(
+      hints,
+      text,
+      cursor,
+      Boolean(block.list)
+    );
+    if (!match) {
+      continue;
+    }
+    cursor = match.index + 1;
+    if (block.list && match.hint.listOrdered !== null) {
+      block.list.ordered = match.hint.listOrdered;
+    } else if (!block.list) {
+      block.quote = match.hint.quote;
+    }
+  }
+}
+
+function findNextSemanticHint(hints, text, cursor, needsListHint) {
+  for (let index = cursor; index < hints.length; index++) {
+    const hint = hints[index];
+    if (hint.text !== text) {
+      continue;
+    }
+    if (needsListHint && hint.listOrdered === null) {
+      continue;
+    }
+    if (!needsListHint && hint.listOrdered !== null) {
+      continue;
+    }
+    return { index, hint };
+  }
+  return null;
+}
+
+function extractGoogleDocsExportSemanticHints(exportHtml) {
+  const preprocessed = preprocessGoogleDocsExportHtml(exportHtml).html;
+  const $ = cheerio.load(preprocessed);
+  const hints = [];
+  $('body')
+    .find('h1,h2,h3,h4,h5,h6,p,li')
+    .each((_, element) => {
+      const tag = element.tagName?.toLowerCase();
+      const text = getExportElementSemanticText($, element, tag);
+      if (!text) {
+        return;
+      }
+      const parentList = $(element).parents('ol,ul').first();
+      const listOrdered =
+        tag === 'li' && parentList.length
+          ? parentList[0].tagName?.toLowerCase() === 'ol'
+          : null;
+      hints.push({
+        text,
+        listOrdered,
+        quote: tag !== 'li' && $(element).parents('blockquote').length > 0,
+      });
+    });
+  return hints;
+}
+
+function getExportElementSemanticText($, element, tag) {
+  if (tag !== 'li') {
+    return normalizeSemanticText($(element).text());
+  }
+  const clone = $(element).clone();
+  clone.find('ol,ul').remove();
+  return normalizeSemanticText(clone.text());
+}
+
+function normalizeSemanticText(text) {
+  return String(text || '')
+    .replace(/\u00a0/g, ' ')
+    .replace(/\s+/g, ' ')
+    .trim();
+}
+
 function collectModelItems(chunks) {
   const items = [];
   for (const chunk of chunks || []) {

package/tests/integration/gdocs-public-doc.test.js

@@ -86,6 +86,33 @@ export const PUBLIC_TEST_DOCUMENT = {
   ],
 };
 
+// The issue #108 v2 document extends the public reference fixture with Section
+// 15, covering browser-model list type and continuation-paragraph regressions.
+export const PUBLIC_TEST_DOCUMENT_V2 = {
+  id: '1Rvaod_u2wgkAUNdXG-e29yQem-P36KAQ',
+  canonicalUrl:
+    'https://docs.google.com/document/d/1Rvaod_u2wgkAUNdXG-e29yQem-P36KAQ/edit',
+};
+
+function expectIssue108Markdown(markdown) {
+  expect(markdown).toContain(
+    '15. List & Blockquote Edge Cases (browser-mode reproducers)'
+  );
+  expect(markdown).toMatch(/1\.\s+Apple/u);
+  expect(markdown).toMatch(/2\.\s+Banana/u);
+  expect(markdown).toMatch(/3\.\s+Cherry/u);
+  expect(markdown).not.toMatch(/^-\s+Apple$/mu);
+  expect(markdown).toContain(
+    'Continuation paragraph that is not a list item. Same indent as Step one.'
+  );
+  expect(markdown).not.toContain(
+    '> Continuation paragraph that is not a list item.'
+  );
+  expect(markdown).toMatch(/^-\s+Red$/mu);
+  expect(markdown).toMatch(/^-\s+Green$/mu);
+  expect(markdown).toMatch(/^-\s+Blue$/mu);
+}
+
 describe('Google Docs public test document (issue #90)', () => {
   describe('URL variations', () => {
     it.each(PUBLIC_TEST_DOCUMENT.urlVariations)(
@@ -181,11 +208,19 @@ describe('Google Docs public test document (issue #90)', () => {
         baseDelay: 2000,
       });
 
+    const browserEngine =
+      process.env.GDOCS_BROWSER_ENGINE || process.env.BROWSER_ENGINE;
     const captureBrowserWithRetry = (url) =>
-      retry(() => captureGoogleDocWithBrowser(url, { waitMs: 3000 }), {
-        retries: 2,
-        baseDelay: 2000,
-      });
+      retry(
+        () =>
+          captureGoogleDocWithBrowser(url, {
+            ...(browserEngine ? { engine: browserEngine } : {}),
+          }),
+        {
+          retries: 2,
+          baseDelay: 2000,
+        }
+      );
 
     it('fetches the public document via --capture api and returns markdown', async () => {
       const { markdown, documentId, exportUrl } = await fetchWithRetry(
@@ -252,5 +287,29 @@ describe('Google Docs public test document (issue #90)', () => {
       expect(markdown).toContain('docs-images-rt/');
       expect(markdown).toContain('---');
     });
+
+    it('fetches the issue #108 v2 document via --capture api and preserves Section 15 semantics', async () => {
+      const { markdown, documentId, exportUrl } = await fetchWithRetry(
+        PUBLIC_TEST_DOCUMENT_V2.canonicalUrl
+      );
+
+      expect(documentId).toBe(PUBLIC_TEST_DOCUMENT_V2.id);
+      expect(exportUrl).toBe(
+        buildExportUrl(PUBLIC_TEST_DOCUMENT_V2.id, 'html')
+      );
+      expectIssue108Markdown(markdown);
+    });
+
+    it('captures the issue #108 v2 document via --capture browser and preserves Section 15 semantics', async () => {
+      const { markdown, documentId, exportUrl, capture } =
+        await captureBrowserWithRetry(PUBLIC_TEST_DOCUMENT_V2.canonicalUrl);
+
+      expect(documentId).toBe(PUBLIC_TEST_DOCUMENT_V2.id);
+      expect(exportUrl).toBe(buildEditUrl(PUBLIC_TEST_DOCUMENT_V2.id));
+      expect(capture.blocks.length).toBeGreaterThan(30);
+      expectIssue108Markdown(markdown);
+      expect(markdown).not.toContain('*Apple*');
+      expect(markdown).not.toContain('*Red*');
+    });
   });
 });

package/tests/unit/gdocs.test.js

@@ -7,6 +7,7 @@ import {
   buildDocsApiUrl,
   extractBase64Images,
   captureGoogleDocWithBrowserOrFallback,
+  captureGoogleDocWithBrowser,
   isGoogleDocsBrowserModelUnavailableError,
   parseGoogleDocsModelChunks,
   renderGoogleDocsCapture,
@@ -430,7 +431,16 @@ describe('gdocs', () => {
           ],
         },
       ];
-      const capture = parseGoogleDocsModelChunks(chunks);
+      const exportHtml = `
+        <html><body>
+          <ol>
+            <li>First item</li>
+            <li>Second item</li>
+            <li>Third item</li>
+          </ol>
+        </body></html>
+      `;
+      const capture = parseGoogleDocsModelChunks(chunks, {}, { exportHtml });
       const markdown = renderGoogleDocsCapture(capture, 'markdown');
       const lines = markdown.split('\n');
 
@@ -464,7 +474,15 @@ describe('gdocs', () => {
           ],
         },
       ];
-      const capture = parseGoogleDocsModelChunks(chunks);
+      const exportHtml = `
+        <html><body>
+          <ol>
+            <li>First item</li>
+            <li>Second item</li>
+          </ol>
+        </body></html>
+      `;
+      const capture = parseGoogleDocsModelChunks(chunks, {}, { exportHtml });
       const markdown = renderGoogleDocsCapture(capture, 'markdown');
 
       expect(markdown).not.toMatch(/First item\n\n2\. Second item/u);
@@ -539,7 +557,10 @@ describe('gdocs', () => {
           ],
         },
       ];
-      const capture = parseGoogleDocsModelChunks(chunks);
+      const exportHtml = `
+        <html><body><ol><li>Parent item 1<ol><li>Child item 1.1</li><li>Child item 1.2<ol><li>Grandchild item 1.2.1</li><li>Grandchild item 1.2.2</li></ol></li><li>Child item 1.3</li></ol></li><li>Parent item 2</li></ol></body></html>
+      `;
+      const capture = parseGoogleDocsModelChunks(chunks, {}, { exportHtml });
       const markdown = renderGoogleDocsCapture(capture, 'markdown');
 
       expect(markdown).toContain(
@@ -557,6 +578,91 @@ describe('gdocs', () => {
       expect(markdown).not.toContain('- Child item 1.1');
     });
 
+    it('uses export semantics for ambiguous browser-model lists and continuation paragraphs (issue #108)', () => {
+      const text = [
+        'Apple',
+        'Banana',
+        'Cherry',
+        'Step one',
+        'Continuation paragraph that is not a list item. Same indent as Step one.',
+        'Step two',
+        'Red',
+        'Green',
+        'Blue',
+        '',
+      ].join('\n');
+      const lineEnd = (needle) => text.indexOf('\n', text.indexOf(needle)) + 1;
+      const listStyle = (needle, id) => ({
+        ty: 'as',
+        st: 'list',
+        si: lineEnd(needle),
+        ei: lineEnd(needle),
+        sm: { ls_id: id },
+      });
+      const paragraphStyle = (needle, indentStart, indentFirstLine) => ({
+        ty: 'as',
+        st: 'paragraph',
+        si: lineEnd(needle),
+        ei: lineEnd(needle),
+        sm: { ps_il: indentStart, ps_ifl: indentFirstLine },
+      });
+      const chunks = [
+        {
+          chunk: [
+            { ty: 'is', s: text },
+            listStyle('Apple', 'kix.ordered'),
+            listStyle('Banana', 'kix.ordered'),
+            listStyle('Cherry', 'kix.ordered'),
+            listStyle('Step one', 'kix.step-one'),
+            paragraphStyle(
+              'Continuation paragraph that is not a list item',
+              36,
+              36
+            ),
+            listStyle('Step two', 'kix.step-two'),
+            listStyle('Red', 'kix.bullets'),
+            listStyle('Green', 'kix.bullets'),
+            listStyle('Blue', 'kix.bullets'),
+            {
+              ty: 'as',
+              st: 'text',
+              si: 1,
+              ei: text.length - 1,
+              sm: { ts_it: true, ts_it_i: true },
+            },
+          ],
+        },
+      ];
+      const exportHtml = `
+        <html><body>
+          <ol><li>Apple</li><li>Banana</li><li>Cherry</li></ol>
+          <ol><li>Step one</li></ol>
+          <p>Continuation paragraph that is not a list item. Same indent as Step one.</p>
+          <ol><li>Step two</li></ol>
+          <ul><li>Red</li><li>Green</li><li>Blue</li></ul>
+        </body></html>
+      `;
+      const capture = parseGoogleDocsModelChunks(chunks, {}, { exportHtml });
+      const markdown = renderGoogleDocsCapture(capture, 'markdown');
+      const html = renderGoogleDocsCapture(capture, 'html');
+
+      expect(markdown).toContain('1. Apple\n2. Banana\n3. Cherry');
+      expect(markdown).not.toContain('- Apple');
+      expect(markdown).toContain(
+        'Continuation paragraph that is not a list item. Same indent as Step one.'
+      );
+      expect(markdown).not.toContain(
+        '> Continuation paragraph that is not a list item.'
+      );
+      expect(markdown).toContain('- Red\n- Green\n- Blue');
+      expect(markdown).not.toContain('*Apple*');
+      expect(html).toContain('<ol><li>Apple</li></ol>');
+      expect(html).toContain('<ul><li>Red</li></ul>');
+      expect(html).not.toContain(
+        '<blockquote>Continuation paragraph that is not a list item.'
+      );
+    });
+
     it('renders model style records for headings, inline formatting, links, lists, blockquotes, rules, and images', () => {
       const text = [
         'Title',
@@ -948,6 +1054,57 @@ describe('gdocs', () => {
     });
   });
 
+  describe('captureGoogleDocWithBrowser (issue #106)', () => {
+    it('waits for DOCS_modelChunk data to stop changing before parsing', async () => {
+      const browser = fakeBrowserReturningModelSequence([
+        {
+          chunks: [{ chunk: [{ ty: 'is', s: 'First paragraph\n' }] }],
+          cidUrlMap: {},
+        },
+        {
+          chunks: [{ chunk: [{ ty: 'is', s: 'First paragraph\n' }] }],
+          cidUrlMap: {},
+        },
+        {
+          chunks: [
+            { chunk: [{ ty: 'is', s: 'First paragraph\n' }] },
+            { chunk: [{ ty: 'is', s: 'Second paragraph\n' }] },
+          ],
+          cidUrlMap: {},
+        },
+        {
+          chunks: [
+            { chunk: [{ ty: 'is', s: 'First paragraph\n' }] },
+            { chunk: [{ ty: 'is', s: 'Second paragraph\n' }] },
+          ],
+          cidUrlMap: {},
+        },
+        {
+          chunks: [
+            { chunk: [{ ty: 'is', s: 'First paragraph\n' }] },
+            { chunk: [{ ty: 'is', s: 'Second paragraph\n' }] },
+          ],
+          cidUrlMap: {},
+        },
+      ]);
+
+      const result = await captureGoogleDocWithBrowser(
+        'https://docs.google.com/document/d/quiescence-doc/edit',
+        {
+          waitMs: 0,
+          modelMaxWaitMs: 200,
+          modelPollMs: 2,
+          modelStabilityMs: 2,
+          createBrowser: async () => browser,
+        }
+      );
+
+      expect(result.markdown).toContain('First paragraph');
+      expect(result.markdown).toContain('Second paragraph');
+      expect(browser.evaluateCalls).toBeGreaterThan(2);
+    });
+  });
+
   describe('GDOCS_EXPORT_FORMATS', () => {
     it('contains all expected formats', () => {
       expect(GDOCS_EXPORT_FORMATS).toEqual({
@@ -1303,3 +1460,31 @@ function fakeBrowserReturningModel(modelData) {
     async close() {},
   };
 }
+
+function fakeBrowserReturningModelSequence(modelDataSequence) {
+  let evaluateCalls = 0;
+  return {
+    get evaluateCalls() {
+      return evaluateCalls;
+    },
+    async newPage() {
+      return {
+        async addInitScript() {},
+        async setUserAgent() {},
+        async setExtraHTTPHeaders() {},
+        async setViewport() {},
+        async goto() {},
+        async waitForTimeout(waitMs) {
+          await new Promise((resolve) => setTimeout(resolve, waitMs));
+        },
+        async evaluate() {
+          const index = Math.min(evaluateCalls, modelDataSequence.length - 1);
+          evaluateCalls += 1;
+          return JSON.parse(JSON.stringify(modelDataSequence[index]));
+        },
+        async close() {},
+      };
+    },
+    async close() {},
+  };
+}