@link-assistant/web-capture 1.7.5 → 1.7.7

package/src/lib.js

@@ -312,6 +312,7 @@ export function convertRelativeUrls(html, baseUrl) {
  * @param {boolean} [options.extractMetadata=true] - Extract article metadata
  * @param {boolean} [options.postProcess=true] - Apply post-processing pipeline
  * @param {boolean} [options.detectCodeLanguage=true] - Detect/correct code languages
+ * @param {boolean} [options.preserveCodeWhitespace=false] - Keep original whitespace inside code blocks
  * @returns {Object} Result with { markdown, metadata }
  */
 export function convertHtmlToMarkdownEnhanced(html, baseUrl, options = {}) {
@@ -320,6 +321,7 @@ export function convertHtmlToMarkdownEnhanced(html, baseUrl, options = {}) {
     extractMetadata: shouldExtractMetadata = true,
     postProcess = true,
     detectCodeLanguage = true,
+    preserveCodeWhitespace = false,
   } = options;
 
   // Ensure all URLs are absolute before Markdown conversion
@@ -422,7 +424,7 @@ export function convertHtmlToMarkdownEnhanced(html, baseUrl, options = {}) {
   // Handle code language detection/correction
   if (detectCodeLanguage) {
     $('pre code').each(function () {
-      const codeText = $(this).text().trim();
+      const codeText = $(this).text();
       const language =
         $(this)
           .attr('class')
@@ -444,6 +446,12 @@ export function convertHtmlToMarkdownEnhanced(html, baseUrl, options = {}) {
     });
   }
 
+  if (preserveCodeWhitespace) {
+    $('pre code').each(function () {
+      $(this).text($(this).text().replace(/\r\n?/g, '\n'));
+    });
+  }
+
   // Preprocess ARIA tables (same as original)
   $('[role="table"]').each(function () {
     const $table = $(this);

package/src/localize-images.js

@@ -24,7 +24,8 @@ import { retry } from './retry.js';
  * @returns {Object[]} Array of {fullMatch, altText, url}
  */
 export function extractImageReferences(markdownText) {
-  const imageRegex = /!\[([^\]]*)\]\((https?:\/\/[^)]+)\)/g;
+  const imageRegex =
+    /!\[([^\]]*)\]\((https?:\/\/[^\s)]+)(?:\s+["'][^"']*["'])?\)/g;
   const images = [];
   let match;
 
@@ -73,6 +74,18 @@ export function generateLocalFilename(url, index) {
   return `image-${String(index + 1).padStart(2, '0')}${ext}`;
 }
 
+/**
+ * Generate a stable figure-numbered filename.
+ *
+ * @param {string} url - Image URL
+ * @param {number} index - Zero-based index
+ * @returns {string} Local filename (e.g., 'figure-1.png')
+ */
+export function generateFigureFilename(url, index) {
+  const ext = getExtensionFromUrl(url);
+  return `figure-${index + 1}${ext}`;
+}
+
 /**
  * Localize images in markdown text by downloading external images
  * and replacing URLs with local paths.
@@ -83,6 +96,7 @@ export function generateLocalFilename(url, index) {
  * @param {boolean} [options.dryRun=false] - Only report what would be done
  * @param {Function} [options.onProgress] - Callback(index, total, status, url)
  * @param {string[]} [options.excludeDomains] - Domains to skip (already local)
+ * @param {Function} [options.filenameGenerator] - Callback(url, index, image) for local filenames
  * @returns {Promise<Object>} Result with updated markdown and metadata
  */
 export async function localizeImages(markdownText, options = {}) {
@@ -91,6 +105,7 @@ export async function localizeImages(markdownText, options = {}) {
     dryRun = false,
     onProgress,
     excludeDomains = [],
+    filenameGenerator = generateLocalFilename,
   } = options;
 
   const allImages = extractImageReferences(markdownText);
@@ -128,7 +143,7 @@ export async function localizeImages(markdownText, options = {}) {
 
   for (let i = 0; i < externalImages.length; i++) {
     const image = externalImages[i];
-    const localFilename = generateLocalFilename(image.url, i);
+    const localFilename = filenameGenerator(image.url, i, image);
     const relativePath = `${imagesDir}/${localFilename}`;
 
     if (onProgress) {

package/tests/unit/gdocs.test.js

@@ -3,7 +3,13 @@ import {
   isGoogleDocsUrl,
   extractDocumentId,
   buildExportUrl,
+  buildEditUrl,
+  buildDocsApiUrl,
   extractBase64Images,
+  parseGoogleDocsModelChunks,
+  renderGoogleDocsCapture,
+  renderDocsApiDocument,
+  selectGoogleDocsCaptureMethod,
   GDOCS_EXPORT_FORMATS,
 } from '../../src/gdocs.js';
 import {
@@ -149,6 +155,161 @@ describe('gdocs', () => {
     });
   });
 
+  describe('capture method selection (issue #72)', () => {
+    it('honors explicit browser capture for Google Docs URLs', () => {
+      expect(selectGoogleDocsCaptureMethod('browser')).toBe('browser-model');
+    });
+
+    it('uses public export for API capture without token', () => {
+      expect(selectGoogleDocsCaptureMethod('api')).toBe('public-export');
+    });
+
+    it('uses Docs REST API for API capture with token', () => {
+      expect(selectGoogleDocsCaptureMethod('api', 'token-123')).toBe(
+        'docs-api'
+      );
+    });
+  });
+
+  describe('buildEditUrl', () => {
+    it('builds the fresh Google Docs editor URL', () => {
+      expect(buildEditUrl('abc123')).toBe(
+        'https://docs.google.com/document/d/abc123/edit'
+      );
+    });
+  });
+
+  describe('buildDocsApiUrl', () => {
+    it('builds the Google Docs REST API URL', () => {
+      expect(buildDocsApiUrl('abc123')).toBe(
+        'https://docs.googleapis.com/v1/documents/abc123'
+      );
+    });
+  });
+
+  describe('parseGoogleDocsModelChunks (issue #72)', () => {
+    it('includes regular and suggested text from DOCS_modelChunk data', () => {
+      const capture = parseGoogleDocsModelChunks([
+        {
+          chunk: [
+            { ty: 'is', s: 'Stable ' },
+            { ty: 'iss', s: 'suggested\n' },
+          ],
+        },
+      ]);
+
+      expect(capture.text).toContain('Stable suggested');
+      expect(renderGoogleDocsCapture(capture, 'markdown')).toContain(
+        'Stable suggested'
+      );
+    });
+
+    it('extracts table cells and suggested images from DOCS_modelChunk data', () => {
+      const chunks = [
+        {
+          chunk: [
+            { ty: 'is', s: String.fromCharCode(0x10) },
+            { ty: 'is', s: String.fromCharCode(0x12) },
+            { ty: 'is', s: String.fromCharCode(0x1c) },
+            { ty: 'is', s: 'Cell A' },
+            { ty: 'is', s: String.fromCharCode(0x1c) },
+            { ty: 'is', s: '*' },
+            { ty: 'is', s: '\n' },
+            { ty: 'is', s: String.fromCharCode(0x11) },
+            {
+              ty: 'ase',
+              id: 'suggested-image',
+              epm: { ee_eo: { i_cid: 'cid_12345678901234567890' } },
+            },
+            { ty: 'ste', id: 'suggested-image', spi: 10 },
+          ],
+        },
+      ];
+      const capture = parseGoogleDocsModelChunks(chunks, {
+        cid_12345678901234567890:
+          'https://docs.google.com/docs-images-rt/image-id',
+      });
+      const markdown = renderGoogleDocsCapture(capture, 'markdown');
+
+      expect(capture.tables).toHaveLength(1);
+      expect(capture.tables[0].rows[0].cells).toHaveLength(2);
+      expect(markdown).toContain('Cell A');
+      expect(markdown).toContain(
+        '![suggested image](https://docs.google.com/docs-images-rt/image-id)'
+      );
+    });
+  });
+
+  describe('renderDocsApiDocument (issue #72)', () => {
+    it('renders Google Docs REST API paragraphs, tables, and inline images', () => {
+      const apiDocument = {
+        title: 'API Doc',
+        body: {
+          content: [
+            {
+              paragraph: {
+                elements: [{ textRun: { content: 'Intro paragraph\n' } }],
+              },
+            },
+            {
+              table: {
+                tableRows: [
+                  {
+                    tableCells: [
+                      {
+                        content: [
+                          {
+                            paragraph: {
+                              elements: [{ textRun: { content: 'Name\n' } }],
+                            },
+                          },
+                        ],
+                      },
+                      {
+                        content: [
+                          {
+                            paragraph: {
+                              elements: [
+                                {
+                                  inlineObjectElement: {
+                                    inlineObjectId: 'image-1',
+                                  },
+                                },
+                              ],
+                            },
+                          },
+                        ],
+                      },
+                    ],
+                  },
+                ],
+              },
+            },
+          ],
+        },
+        inlineObjects: {
+          'image-1': {
+            inlineObjectProperties: {
+              embeddedObject: {
+                title: 'Diagram',
+                imageProperties: {
+                  contentUri: 'https://example.com/diagram.png',
+                },
+              },
+            },
+          },
+        },
+      };
+
+      const rendered = renderDocsApiDocument(apiDocument);
+
+      expect(rendered.markdown).toContain('Intro paragraph');
+      expect(rendered.markdown).toContain('| Name | ![Diagram]');
+      expect(rendered.html).toContain('<table>');
+      expect(rendered.html).toContain('src="https://example.com/diagram.png"');
+    });
+  });
+
   describe('GDOCS_EXPORT_FORMATS', () => {
     it('contains all expected formats', () => {
       expect(GDOCS_EXPORT_FORMATS).toEqual({

package/tests/unit/html2md.test.js

@@ -1,4 +1,7 @@
-import { convertHtmlToMarkdown } from '../../src/lib.js';
+import {
+  convertHtmlToMarkdown,
+  convertHtmlToMarkdownEnhanced,
+} from '../../src/lib.js';
 
 describe('convertHtmlToMarkdown', () => {
   it('removes empty links and anchor headings', () => {
@@ -608,3 +611,44 @@ describe('convertHtmlToMarkdown', () => {
     );
   });
 });
+
+describe('convertHtmlToMarkdownEnhanced Habr compatibility', () => {
+  it('preserves article code line breaks and extracts formulas from a Habr fixture', () => {
+    const html = `
+      <article>
+        <h1>The Links Theory 0.0.2</h1>
+        <div class="article-formatted-body">
+          <p>A formula <img class="formula" source="\\mathbf{L}: L \\to L^2" alt="formula"> remains inline.</p>
+          <figure>
+            <img src="https://habrastorage.org/r/w1560/getpro/habr/upload_files/abc.png" alt="Figure 1">
+            <figcaption>Figure 1. Link multiplication.</figcaption>
+          </figure>
+          <pre><code>L = { 1 , 2 }
+L x L = {
+  (1, 1),
+  (1, 2),
+}</code></pre>
+        </div>
+      </article>
+    `;
+
+    const { markdown } = convertHtmlToMarkdownEnhanced(
+      html,
+      'https://habr.com/en/articles/895896/',
+      { preserveCodeWhitespace: true }
+    );
+
+    expect(markdown).toContain('$\\\\mathbf{L}: L \\\\to L^2$');
+    expect(markdown).toContain(
+      '![Figure 1](https://habrastorage.org/r/w1560/getpro/habr/upload_files/abc.png)'
+    );
+    expect(markdown).toContain(`\`\`\`
+L = { 1 , 2 }
+L x L = {
+  (1, 1),
+  (1, 2),
+}
+\`\`\``);
+    expect(markdown).not.toContain('L = { 1 , 2 }L x L');
+  });
+});

package/tests/unit/localize-images.test.js

@@ -2,6 +2,8 @@ import {
   extractImageReferences,
   getExtensionFromUrl,
   generateLocalFilename,
+  generateFigureFilename,
+  localizeImages,
 } from '../../src/localize-images.js';
 
 describe('localize-images module', () => {
@@ -75,4 +77,38 @@ describe('localize-images module', () => {
       );
     });
   });
+
+  describe('generateFigureFilename', () => {
+    it('generates figure-numbered filenames while preserving extensions', () => {
+      expect(
+        generateFigureFilename('https://habrastorage.org/getpro/habr/a.png', 0)
+      ).toBe('figure-1.png');
+      expect(
+        generateFigureFilename('https://habrastorage.org/getpro/habr/a.jpg', 9)
+      ).toBe('figure-10.jpg');
+    });
+  });
+
+  describe('localizeImages', () => {
+    it('accepts a custom filename generator for archive-compatible paths', async () => {
+      const markdown =
+        '![Figure 1](https://habrastorage.org/r/w1560/getpro/habr/figure.png "Figure 1")';
+
+      const result = await localizeImages(markdown, {
+        dryRun: true,
+        filenameGenerator: generateFigureFilename,
+      });
+
+      expect(result.markdown).toBe('![Figure 1](images/figure-1.png)');
+      expect(result.metadata).toEqual([
+        {
+          index: 1,
+          originalUrl:
+            'https://habrastorage.org/r/w1560/getpro/habr/figure.png',
+          altText: 'Figure 1',
+          localPath: 'images/figure-1.png',
+        },
+      ]);
+    });
+  });
 });