@link-assistant/web-capture 1.7.17 → 1.7.18

package/CHANGELOG.md

@@ -1,5 +1,11 @@
 # @link-assistant/web-capture
 
+## 1.7.18
+
+### Patch Changes
+
+- 2c69a73: Fix Google Docs browser capture list semantics and continuation paragraph handling.
+
 ## 1.7.17
 
 ### Patch Changes

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@link-assistant/web-capture",
-  "version": "1.7.17",
+  "version": "1.7.18",
   "description": "CLI and microservice to render web pages as HTML, Markdown, or PNG",
   "main": "src/index.js",
   "bin": {

package/src/gdocs.js

@@ -9,6 +9,7 @@
 
 import fetch from 'node-fetch';
 import he from 'he';
+import * as cheerio from 'cheerio';
 import { convertHtmlToMarkdown } from './lib.js';
 import { createBrowser as defaultCreateBrowser } from './browser.js';
 import {
@@ -200,11 +201,10 @@ export async function fetchGoogleDoc(url, options = {}) {
     bytes: Buffer.byteLength(rawContent),
   }));
 
-  // Decode HTML entities to unicode for text-based formats
+  // Keep HTML markup escaped so literal examples such as `<ol>` do not
+  // become real tags before the HTML parser sees the document.
   const content =
-    format === 'html' || format === 'txt' || format === 'md'
-      ? he.decode(rawContent)
-      : rawContent;
+    format === 'txt' || format === 'md' ? he.decode(rawContent) : rawContent;
 
   return {
     content,
@@ -430,11 +430,16 @@ export async function captureGoogleDocWithBrowser(url, options = {}) {
       documentId,
       log,
     });
+    const exportHtml = await fetchGoogleDocsExportHtmlFromPage(
+      page,
+      documentId,
+      log
+    );
 
     const capture = parseGoogleDocsModelChunks(
       modelData.chunks,
       modelData.cidUrlMap,
-      { log }
+      { log, exportHtml }
     );
     log?.debug?.(() => ({
       event: 'gdocs.browser-model.parsed',
@@ -567,6 +572,7 @@ export function extractCidUrlMapFromHtml(html) {
  * @param {Object<string,string>} [cidUrlMap={}] - CID-to-docs-images-rt URL map
  * @param {Object} [options] - Parse options
  * @param {Object} [options.log] - Optional verbose logger
+ * @param {string} [options.exportHtml] - Optional Google Docs export HTML used for list and quote semantics
  * @returns {{blocks: Array, tables: Array, images: Array, text: string}}
  */
 export function parseGoogleDocsModelChunks(
@@ -813,12 +819,17 @@ export function parseGoogleDocsModelChunks(
     }
   }
 
-  return {
+  const capture = {
     blocks,
     tables,
     images,
     text: contentBlocksToText(blocks),
   };
+  if (options.exportHtml) {
+    applyGoogleDocsExportSemanticHints(capture, options.exportHtml);
+    capture.text = contentBlocksToText(capture.blocks);
+  }
+  return capture;
 }
 
 function cellIsEmpty(cell) {
@@ -889,13 +900,13 @@ function applyInlineTextStyle(inlineStyles, start, end, patch) {
 
 function textStyleFromModel(item) {
   const style = {};
-  if (item.sm?.ts_bd) {
+  if (item.sm?.ts_bd && item.sm?.ts_bd_i !== true) {
     style.bold = true;
   }
-  if (item.sm?.ts_it) {
+  if (item.sm?.ts_it && item.sm?.ts_it_i !== true) {
     style.italic = true;
   }
-  if (item.sm?.ts_st) {
+  if (item.sm?.ts_st && item.sm?.ts_st_i !== true) {
     style.strike = true;
   }
   return style;
@@ -944,19 +955,136 @@ function paragraphMetaForEndPosition(styleMaps, endPos, text) {
 }
 
 function inferOrderedList(list, text) {
-  // Google Docs does not expose imported DOCX list marker type in the public
-  // model chunk for this document. Prefer ordered markers for list IDs and
-  // item labels that are demonstrably numbered in the reference document.
-  if (
-    /ordered|^\d+(?:\.|\))|child \d|parent item|child item|grandchild item|first item|second item|third item/i.test(
-      text
-    )
-  ) {
-    return /^kix\.list\.(7|8|9|10|11|13)$/u.test(list.id);
-  }
+  void list;
+  void text;
   return false;
 }
 
+async function fetchGoogleDocsExportHtmlFromPage(page, documentId, log) {
+  const exportUrl = buildExportUrl(documentId, 'html');
+  try {
+    const result = await page.evaluate(async (url) => {
+      const response = await fetch(url, {
+        credentials: 'include',
+        redirect: 'follow',
+      });
+      return {
+        ok: response.ok,
+        status: response.status,
+        statusText: response.statusText,
+        contentType: response.headers.get('content-type'),
+        body: await response.text(),
+      };
+    }, exportUrl);
+    log?.debug?.(() => ({
+      event: 'gdocs.browser-model.export-hints.response',
+      documentId,
+      exportUrl,
+      status: result.status,
+      ok: result.ok,
+      contentType: result.contentType,
+      bytes: Buffer.byteLength(result.body || ''),
+    }));
+    return result.ok ? result.body || '' : null;
+  } catch (error) {
+    log?.debug?.(() => ({
+      event: 'gdocs.browser-model.export-hints.failed',
+      documentId,
+      exportUrl,
+      error: error?.message || String(error),
+    }));
+    return null;
+  }
+}
+
+function applyGoogleDocsExportSemanticHints(capture, exportHtml) {
+  const hints = extractGoogleDocsExportSemanticHints(exportHtml);
+  let cursor = 0;
+  for (const block of capture.blocks || []) {
+    if (block.type !== 'paragraph') {
+      continue;
+    }
+    const text = normalizeSemanticText(contentToText(block.content));
+    if (!text) {
+      continue;
+    }
+    const match = findNextSemanticHint(
+      hints,
+      text,
+      cursor,
+      Boolean(block.list)
+    );
+    if (!match) {
+      continue;
+    }
+    cursor = match.index + 1;
+    if (block.list && match.hint.listOrdered !== null) {
+      block.list.ordered = match.hint.listOrdered;
+    } else if (!block.list) {
+      block.quote = match.hint.quote;
+    }
+  }
+}
+
+function findNextSemanticHint(hints, text, cursor, needsListHint) {
+  for (let index = cursor; index < hints.length; index++) {
+    const hint = hints[index];
+    if (hint.text !== text) {
+      continue;
+    }
+    if (needsListHint && hint.listOrdered === null) {
+      continue;
+    }
+    if (!needsListHint && hint.listOrdered !== null) {
+      continue;
+    }
+    return { index, hint };
+  }
+  return null;
+}
+
+function extractGoogleDocsExportSemanticHints(exportHtml) {
+  const preprocessed = preprocessGoogleDocsExportHtml(exportHtml).html;
+  const $ = cheerio.load(preprocessed);
+  const hints = [];
+  $('body')
+    .find('h1,h2,h3,h4,h5,h6,p,li')
+    .each((_, element) => {
+      const tag = element.tagName?.toLowerCase();
+      const text = getExportElementSemanticText($, element, tag);
+      if (!text) {
+        return;
+      }
+      const parentList = $(element).parents('ol,ul').first();
+      const listOrdered =
+        tag === 'li' && parentList.length
+          ? parentList[0].tagName?.toLowerCase() === 'ol'
+          : null;
+      hints.push({
+        text,
+        listOrdered,
+        quote: tag !== 'li' && $(element).parents('blockquote').length > 0,
+      });
+    });
+  return hints;
+}
+
+function getExportElementSemanticText($, element, tag) {
+  if (tag !== 'li') {
+    return normalizeSemanticText($(element).text());
+  }
+  const clone = $(element).clone();
+  clone.find('ol,ul').remove();
+  return normalizeSemanticText(clone.text());
+}
+
+function normalizeSemanticText(text) {
+  return String(text || '')
+    .replace(/\u00a0/g, ' ')
+    .replace(/\s+/g, ' ')
+    .trim();
+}
+
 function collectModelItems(chunks) {
   const items = [];
   for (const chunk of chunks || []) {

package/tests/integration/gdocs-public-doc.test.js

@@ -86,6 +86,33 @@ export const PUBLIC_TEST_DOCUMENT = {
   ],
 };
 
+// The issue #108 v2 document extends the public reference fixture with Section
+// 15, covering browser-model list type and continuation-paragraph regressions.
+export const PUBLIC_TEST_DOCUMENT_V2 = {
+  id: '1Rvaod_u2wgkAUNdXG-e29yQem-P36KAQ',
+  canonicalUrl:
+    'https://docs.google.com/document/d/1Rvaod_u2wgkAUNdXG-e29yQem-P36KAQ/edit',
+};
+
+function expectIssue108Markdown(markdown) {
+  expect(markdown).toContain(
+    '15. List & Blockquote Edge Cases (browser-mode reproducers)'
+  );
+  expect(markdown).toMatch(/1\.\s+Apple/u);
+  expect(markdown).toMatch(/2\.\s+Banana/u);
+  expect(markdown).toMatch(/3\.\s+Cherry/u);
+  expect(markdown).not.toMatch(/^-\s+Apple$/mu);
+  expect(markdown).toContain(
+    'Continuation paragraph that is not a list item. Same indent as Step one.'
+  );
+  expect(markdown).not.toContain(
+    '> Continuation paragraph that is not a list item.'
+  );
+  expect(markdown).toMatch(/^-\s+Red$/mu);
+  expect(markdown).toMatch(/^-\s+Green$/mu);
+  expect(markdown).toMatch(/^-\s+Blue$/mu);
+}
+
 describe('Google Docs public test document (issue #90)', () => {
   describe('URL variations', () => {
     it.each(PUBLIC_TEST_DOCUMENT.urlVariations)(
@@ -181,11 +208,19 @@ describe('Google Docs public test document (issue #90)', () => {
         baseDelay: 2000,
       });
 
+    const browserEngine =
+      process.env.GDOCS_BROWSER_ENGINE || process.env.BROWSER_ENGINE;
     const captureBrowserWithRetry = (url) =>
-      retry(() => captureGoogleDocWithBrowser(url), {
-        retries: 2,
-        baseDelay: 2000,
-      });
+      retry(
+        () =>
+          captureGoogleDocWithBrowser(url, {
+            ...(browserEngine ? { engine: browserEngine } : {}),
+          }),
+        {
+          retries: 2,
+          baseDelay: 2000,
+        }
+      );
 
     it('fetches the public document via --capture api and returns markdown', async () => {
       const { markdown, documentId, exportUrl } = await fetchWithRetry(
@@ -252,5 +287,29 @@ describe('Google Docs public test document (issue #90)', () => {
       expect(markdown).toContain('docs-images-rt/');
       expect(markdown).toContain('---');
     });
+
+    it('fetches the issue #108 v2 document via --capture api and preserves Section 15 semantics', async () => {
+      const { markdown, documentId, exportUrl } = await fetchWithRetry(
+        PUBLIC_TEST_DOCUMENT_V2.canonicalUrl
+      );
+
+      expect(documentId).toBe(PUBLIC_TEST_DOCUMENT_V2.id);
+      expect(exportUrl).toBe(
+        buildExportUrl(PUBLIC_TEST_DOCUMENT_V2.id, 'html')
+      );
+      expectIssue108Markdown(markdown);
+    });
+
+    it('captures the issue #108 v2 document via --capture browser and preserves Section 15 semantics', async () => {
+      const { markdown, documentId, exportUrl, capture } =
+        await captureBrowserWithRetry(PUBLIC_TEST_DOCUMENT_V2.canonicalUrl);
+
+      expect(documentId).toBe(PUBLIC_TEST_DOCUMENT_V2.id);
+      expect(exportUrl).toBe(buildEditUrl(PUBLIC_TEST_DOCUMENT_V2.id));
+      expect(capture.blocks.length).toBeGreaterThan(30);
+      expectIssue108Markdown(markdown);
+      expect(markdown).not.toContain('*Apple*');
+      expect(markdown).not.toContain('*Red*');
+    });
   });
 });

package/tests/unit/gdocs.test.js

@@ -431,7 +431,16 @@ describe('gdocs', () => {
           ],
         },
       ];
-      const capture = parseGoogleDocsModelChunks(chunks);
+      const exportHtml = `
+        <html><body>
+          <ol>
+            <li>First item</li>
+            <li>Second item</li>
+            <li>Third item</li>
+          </ol>
+        </body></html>
+      `;
+      const capture = parseGoogleDocsModelChunks(chunks, {}, { exportHtml });
       const markdown = renderGoogleDocsCapture(capture, 'markdown');
       const lines = markdown.split('\n');
 
@@ -465,7 +474,15 @@ describe('gdocs', () => {
           ],
         },
       ];
-      const capture = parseGoogleDocsModelChunks(chunks);
+      const exportHtml = `
+        <html><body>
+          <ol>
+            <li>First item</li>
+            <li>Second item</li>
+          </ol>
+        </body></html>
+      `;
+      const capture = parseGoogleDocsModelChunks(chunks, {}, { exportHtml });
       const markdown = renderGoogleDocsCapture(capture, 'markdown');
 
       expect(markdown).not.toMatch(/First item\n\n2\. Second item/u);
@@ -540,7 +557,10 @@ describe('gdocs', () => {
           ],
         },
       ];
-      const capture = parseGoogleDocsModelChunks(chunks);
+      const exportHtml = `
+        <html><body><ol><li>Parent item 1<ol><li>Child item 1.1</li><li>Child item 1.2<ol><li>Grandchild item 1.2.1</li><li>Grandchild item 1.2.2</li></ol></li><li>Child item 1.3</li></ol></li><li>Parent item 2</li></ol></body></html>
+      `;
+      const capture = parseGoogleDocsModelChunks(chunks, {}, { exportHtml });
       const markdown = renderGoogleDocsCapture(capture, 'markdown');
 
       expect(markdown).toContain(
@@ -558,6 +578,91 @@ describe('gdocs', () => {
       expect(markdown).not.toContain('- Child item 1.1');
     });
 
+    it('uses export semantics for ambiguous browser-model lists and continuation paragraphs (issue #108)', () => {
+      const text = [
+        'Apple',
+        'Banana',
+        'Cherry',
+        'Step one',
+        'Continuation paragraph that is not a list item. Same indent as Step one.',
+        'Step two',
+        'Red',
+        'Green',
+        'Blue',
+        '',
+      ].join('\n');
+      const lineEnd = (needle) => text.indexOf('\n', text.indexOf(needle)) + 1;
+      const listStyle = (needle, id) => ({
+        ty: 'as',
+        st: 'list',
+        si: lineEnd(needle),
+        ei: lineEnd(needle),
+        sm: { ls_id: id },
+      });
+      const paragraphStyle = (needle, indentStart, indentFirstLine) => ({
+        ty: 'as',
+        st: 'paragraph',
+        si: lineEnd(needle),
+        ei: lineEnd(needle),
+        sm: { ps_il: indentStart, ps_ifl: indentFirstLine },
+      });
+      const chunks = [
+        {
+          chunk: [
+            { ty: 'is', s: text },
+            listStyle('Apple', 'kix.ordered'),
+            listStyle('Banana', 'kix.ordered'),
+            listStyle('Cherry', 'kix.ordered'),
+            listStyle('Step one', 'kix.step-one'),
+            paragraphStyle(
+              'Continuation paragraph that is not a list item',
+              36,
+              36
+            ),
+            listStyle('Step two', 'kix.step-two'),
+            listStyle('Red', 'kix.bullets'),
+            listStyle('Green', 'kix.bullets'),
+            listStyle('Blue', 'kix.bullets'),
+            {
+              ty: 'as',
+              st: 'text',
+              si: 1,
+              ei: text.length - 1,
+              sm: { ts_it: true, ts_it_i: true },
+            },
+          ],
+        },
+      ];
+      const exportHtml = `
+        <html><body>
+          <ol><li>Apple</li><li>Banana</li><li>Cherry</li></ol>
+          <ol><li>Step one</li></ol>
+          <p>Continuation paragraph that is not a list item. Same indent as Step one.</p>
+          <ol><li>Step two</li></ol>
+          <ul><li>Red</li><li>Green</li><li>Blue</li></ul>
+        </body></html>
+      `;
+      const capture = parseGoogleDocsModelChunks(chunks, {}, { exportHtml });
+      const markdown = renderGoogleDocsCapture(capture, 'markdown');
+      const html = renderGoogleDocsCapture(capture, 'html');
+
+      expect(markdown).toContain('1. Apple\n2. Banana\n3. Cherry');
+      expect(markdown).not.toContain('- Apple');
+      expect(markdown).toContain(
+        'Continuation paragraph that is not a list item. Same indent as Step one.'
+      );
+      expect(markdown).not.toContain(
+        '> Continuation paragraph that is not a list item.'
+      );
+      expect(markdown).toContain('- Red\n- Green\n- Blue');
+      expect(markdown).not.toContain('*Apple*');
+      expect(html).toContain('<ol><li>Apple</li></ol>');
+      expect(html).toContain('<ul><li>Red</li></ul>');
+      expect(html).not.toContain(
+        '<blockquote>Continuation paragraph that is not a list item.'
+      );
+    });
+
     it('renders model style records for headings, inline formatting, links, lists, blockquotes, rules, and images', () => {
       const text = [
         'Title',