@link-assistant/web-capture 1.1.2 → 1.4.2

package/tests/unit/metadata.test.js

@@ -0,0 +1,140 @@
+import * as cheerio from 'cheerio';
+import {
+  extractMetadata,
+  formatMetadataBlock,
+  formatFooterBlock,
+} from '../../src/metadata.js';
+
+describe('metadata module', () => {
+  describe('extractMetadata', () => {
+    it('extracts author information', () => {
+      const $ = cheerio.load(
+        '<a class="tm-user-info__username" href="/user/john">john_doe</a>'
+      );
+      const meta = extractMetadata($);
+      expect(meta.author).toBe('john_doe');
+      expect(meta.authorUrl).toBe('/user/john');
+    });
+
+    it('extracts publication date', () => {
+      const $ = cheerio.load(
+        '<time datetime="2024-01-15T10:00:00Z">Jan 15</time>'
+      );
+      const meta = extractMetadata($);
+      expect(meta.publishDate).toBe('2024-01-15T10:00:00Z');
+      expect(meta.publishDateText).toBe('Jan 15');
+    });
+
+    it('extracts hubs', () => {
+      const $ = cheerio.load(`
+        <a class="tm-publication-hub__link"><span>Math</span></a>
+        <a class="tm-publication-hub__link"><span>Science</span></a>
+      `);
+      const meta = extractMetadata($);
+      expect(meta.hubs).toEqual(['Math', 'Science']);
+    });
+
+    it('extracts tags from meta keywords', () => {
+      const $ = cheerio.load(
+        '<meta name="keywords" content="math, science, theory">'
+      );
+      const meta = extractMetadata($);
+      expect(meta.tags).toEqual(['math', 'science', 'theory']);
+    });
+
+    it('extracts LD+JSON author name', () => {
+      const $ = cheerio.load(
+        '<script type="application/ld+json">{"author":{"name":"John Doe"},"dateModified":"2024-02-01"}</script>'
+      );
+      const meta = extractMetadata($);
+      expect(meta.authorFullName).toBe('John Doe');
+      expect(meta.dateModified).toBe('2024-02-01');
+    });
+
+    it('extracts votes and comments', () => {
+      const $ = cheerio.load(`
+        <span class="tm-votes-meter__value">+42</span>
+        <span class="tm-article-comments-counter-link__value">15</span>
+      `);
+      const meta = extractMetadata($);
+      expect(meta.votes).toBe('+42');
+      expect(meta.comments).toBe('15');
+    });
+
+    it('returns empty object for minimal HTML', () => {
+      const $ = cheerio.load('<div>plain content</div>');
+      const meta = extractMetadata($);
+      expect(meta).toBeDefined();
+      expect(meta.author).toBeUndefined();
+    });
+  });
+
+  describe('formatMetadataBlock', () => {
+    it('formats author line with link', () => {
+      const lines = formatMetadataBlock({
+        author: 'john',
+        authorUrl: '/user/john',
+      });
+      expect(lines).toContain('**Author:** [john](/user/john)');
+    });
+
+    it('formats author with full name', () => {
+      const lines = formatMetadataBlock({
+        author: 'john',
+        authorFullName: 'John Doe',
+        authorUrl: '/user/john',
+      });
+      expect(lines[0]).toContain('John Doe (john)');
+    });
+
+    it('formats publication date', () => {
+      const lines = formatMetadataBlock({
+        publishDate: '2024-01-15T10:00:00Z',
+      });
+      expect(lines.some((l) => l.includes('Published:'))).toBe(true);
+      expect(lines.some((l) => l.includes('January 15, 2024'))).toBe(true);
+    });
+
+    it('includes hubs and tags', () => {
+      const lines = formatMetadataBlock({
+        hubs: ['Math', 'Science'],
+        tags: ['theory', 'links'],
+      });
+      expect(lines.some((l) => l.includes('Math, Science'))).toBe(true);
+      expect(lines.some((l) => l.includes('theory, links'))).toBe(true);
+    });
+
+    it('returns empty array for null metadata', () => {
+      expect(formatMetadataBlock(null)).toEqual([]);
+    });
+  });
+
+  describe('formatFooterBlock', () => {
+    it('includes separator line', () => {
+      const lines = formatFooterBlock({ author: 'test' });
+      expect(lines[0]).toBe('---');
+    });
+
+    it('formats tag links', () => {
+      const lines = formatFooterBlock({
+        tagLinks: [{ name: 'math', url: '/tag/math' }],
+      });
+      expect(lines.some((l) => l.includes('[math](/tag/math)'))).toBe(true);
+    });
+
+    it('formats article stats', () => {
+      const lines = formatFooterBlock({
+        votes: '+10',
+        views: '5000',
+        comments: '3',
+      });
+      const statsLine = lines.find((l) => l.includes('Votes:'));
+      expect(statsLine).toContain('+10');
+      expect(statsLine).toContain('5000');
+    });
+
+    it('returns empty array for null metadata', () => {
+      expect(formatFooterBlock(null)).toEqual([]);
+    });
+  });
+});

package/tests/unit/postprocess.test.js

@@ -0,0 +1,108 @@
+import {
+  postProcessMarkdown,
+  applyUnicodeNormalization,
+  applyLatexSpacingFixes,
+  applyPercentSignFix,
+  applyBoldFormattingFixes,
+} from '../../src/postprocess.js';
+
+describe('postprocess module', () => {
+  describe('applyUnicodeNormalization', () => {
+    it('replaces non-breaking spaces with regular spaces', () => {
+      expect(applyUnicodeNormalization('hello\u00A0world')).toBe('hello world');
+    });
+
+    it('normalizes curly quotes to straight quotes', () => {
+      expect(applyUnicodeNormalization('\u2018hello\u2019')).toBe("'hello'");
+      expect(applyUnicodeNormalization('\u201Chello\u201D')).toBe('"hello"');
+    });
+
+    it('normalizes en-dash to hyphen', () => {
+      expect(applyUnicodeNormalization('2\u20133')).toBe('2-3');
+    });
+
+    it('normalizes ellipsis', () => {
+      expect(applyUnicodeNormalization('wait\u2026')).toBe('wait...');
+    });
+  });
+
+  describe('applyLatexSpacingFixes', () => {
+    it('adds space before formula after word character', () => {
+      const result = applyLatexSpacingFixes('where$x$is');
+      expect(result).toBe('where $x$ is');
+    });
+
+    it('trims whitespace inside formula delimiters', () => {
+      const result = applyLatexSpacingFixes('$ x + y $');
+      expect(result).toBe('$x + y$');
+    });
+
+    it('does not modify block formulas', () => {
+      const result = applyLatexSpacingFixes('$$x + y$$');
+      expect(result).toBe('$$x + y$$');
+    });
+
+    it('does not modify blockquote block formulas', () => {
+      const result = applyLatexSpacingFixes('> $$x + y$$');
+      expect(result).toBe('> $$x + y$$');
+    });
+
+    it('handles multiple formulas on one line', () => {
+      const result = applyLatexSpacingFixes('$a$and$b$');
+      expect(result).toBe('$a$ and $b$');
+    });
+  });
+
+  describe('applyPercentSignFix', () => {
+    it('fixes percent sign in inline formulas', () => {
+      const result = applyPercentSignFix('$50\\%$');
+      expect(result).toBe('$50\\\\%$');
+    });
+
+    it('fixes \\text{%} notation', () => {
+      const result = applyPercentSignFix('$50\\text{%}$');
+      expect(result).toBe('$50\\\\%$');
+    });
+  });
+
+  describe('applyBoldFormattingFixes', () => {
+    it('removes empty bold markers', () => {
+      const result = applyBoldFormattingFixes('hello ** ** world');
+      expect(result).toBe('hello  world');
+    });
+
+    it('trims content inside bold markers', () => {
+      const result = applyBoldFormattingFixes('** hello **');
+      expect(result).toBe('**hello**');
+    });
+
+    it('adds space between word and bold', () => {
+      const result = applyBoldFormattingFixes('word**bold**next');
+      expect(result).toBe('word **bold** next');
+    });
+  });
+
+  describe('postProcessMarkdown', () => {
+    it('applies all transformations by default', () => {
+      const input = 'hello\u00A0$x$world';
+      const result = postProcessMarkdown(input);
+      expect(result).toContain('hello');
+      expect(result).toContain('$x$');
+      expect(result).not.toContain('\u00A0');
+    });
+
+    it('respects option flags', () => {
+      const input = 'hello\u00A0world';
+      const result = postProcessMarkdown(input, {
+        normalizeUnicode: false,
+      });
+      expect(result).toContain('\u00A0');
+    });
+
+    it('removes stray standalone $ signs', () => {
+      const input = 'line1\n$\nline2';
+      const result = postProcessMarkdown(input);
+      expect(result).toBe('line1\n\nline2');
+    });
+  });
+});

package/tests/unit/retry.test.js

@@ -0,0 +1,111 @@
+import { jest } from '@jest/globals';
+import { retry } from '../../src/retry.js';
+
+jest.setTimeout(30000);
+
+describe('retry', () => {
+  it('returns result on first success', async () => {
+    const fn = jest.fn().mockResolvedValue('ok');
+    const result = await retry(fn, { retries: 3 });
+    expect(result).toBe('ok');
+    expect(fn).toHaveBeenCalledTimes(1);
+  });
+
+  it('retries on failure and succeeds eventually', async () => {
+    let calls = 0;
+    const fn = jest.fn().mockImplementation(async () => {
+      calls++;
+      if (calls < 3) {
+        throw new Error('fail');
+      }
+      return 'success';
+    });
+
+    const result = await retry(fn, { retries: 3, baseDelay: 50 });
+    expect(result).toBe('success');
+    expect(fn).toHaveBeenCalledTimes(3);
+  });
+
+  it('throws after exhausting retries', async () => {
+    const fn = jest.fn().mockRejectedValue(new Error('persistent failure'));
+
+    await expect(retry(fn, { retries: 2, baseDelay: 50 })).rejects.toThrow(
+      'persistent failure'
+    );
+    expect(fn).toHaveBeenCalledTimes(3); // initial + 2 retries
+  });
+
+  it('calls onRetry callback before each retry', async () => {
+    let calls = 0;
+    const fn = jest.fn().mockImplementation(async () => {
+      calls++;
+      if (calls < 2) {
+        throw new Error('fail');
+      }
+      return 'ok';
+    });
+    const onRetry = jest.fn();
+
+    await retry(fn, { retries: 3, baseDelay: 50, onRetry });
+    expect(onRetry).toHaveBeenCalledTimes(1);
+    expect(onRetry).toHaveBeenCalledWith(expect.any(Error), 1, 50);
+  });
+
+  it('uses exponential backoff delays', async () => {
+    const delays = [];
+    const onRetry = jest.fn((err, attempt, delay) => {
+      delays.push(delay);
+    });
+
+    const fn = jest.fn().mockRejectedValue(new Error('fail'));
+
+    await expect(
+      retry(fn, {
+        retries: 3,
+        baseDelay: 100,
+        factor: 2,
+        maxDelay: 1000,
+        onRetry,
+      })
+    ).rejects.toThrow();
+
+    // Delays should be: 100, 200, 400
+    expect(delays).toEqual([100, 200, 400]);
+  });
+
+  it('caps delay at maxDelay', async () => {
+    const delays = [];
+    const onRetry = jest.fn((err, attempt, delay) => {
+      delays.push(delay);
+    });
+
+    const fn = jest.fn().mockRejectedValue(new Error('fail'));
+
+    await expect(
+      retry(fn, {
+        retries: 3,
+        baseDelay: 100,
+        factor: 10,
+        maxDelay: 500,
+        onRetry,
+      })
+    ).rejects.toThrow();
+
+    // Delays: min(100, 500)=100, min(1000, 500)=500, min(10000, 500)=500
+    expect(delays).toEqual([100, 500, 500]);
+  });
+
+  it('passes attempt number to fn', async () => {
+    const attempts = [];
+    const fn = jest.fn().mockImplementation(async (attempt) => {
+      attempts.push(attempt);
+      if (attempt < 2) {
+        throw new Error('fail');
+      }
+      return 'ok';
+    });
+
+    await retry(fn, { retries: 3, baseDelay: 50 });
+    expect(attempts).toEqual([0, 1, 2]);
+  });
+});

package/tests/unit/verify.test.js

@@ -0,0 +1,182 @@
+import {
+  normalizeText,
+  normalizeCode,
+  verifyMarkdownContent,
+} from '../../src/verify.js';
+
+describe('verify module', () => {
+  describe('normalizeText', () => {
+    it('lowercases text', () => {
+      expect(normalizeText('Hello World')).toBe('hello world');
+    });
+
+    it('normalizes unicode spaces', () => {
+      expect(normalizeText('hello\u00A0world')).toBe('hello world');
+    });
+
+    it('normalizes curly quotes', () => {
+      expect(normalizeText('\u201Chello\u201D')).toBe('"hello"');
+    });
+
+    it('removes LaTeX delimiters', () => {
+      expect(normalizeText('$x + y$')).toBe('x + y');
+      expect(normalizeText('$$x + y$$')).toBe('x + y');
+    });
+
+    it('normalizes arrows', () => {
+      expect(normalizeText('\u2192')).toBe('->');
+      expect(normalizeText('\\to')).toBe('->');
+    });
+
+    it('removes \\displaystyle', () => {
+      expect(normalizeText('\\displaystyle x')).toBe('x');
+    });
+
+    it('extracts text from \\text{}', () => {
+      expect(normalizeText('\\text{hello}')).toBe('hello');
+    });
+  });
+
+  describe('normalizeCode', () => {
+    it('lowercases and collapses whitespace', () => {
+      expect(normalizeCode('  Hello   World  ')).toBe('hello world');
+    });
+
+    it('normalizes multiplication sign', () => {
+      expect(normalizeCode('2\u00D73')).toBe('2x3');
+    });
+  });
+
+  describe('verifyMarkdownContent', () => {
+    it('verifies title presence', () => {
+      const webContent = {
+        title: 'My Article',
+        headings: [],
+        paragraphs: [],
+        codeBlocks: [],
+        formulas: [],
+        blockquoteFormulas: [],
+        listItems: [],
+        links: [],
+        figures: [],
+      };
+      const markdown = '# My Article\n\nSome content here.';
+      const result = verifyMarkdownContent(webContent, markdown);
+      expect(result.missing.title).toBe(false);
+      expect(result.passedChecks).toBeGreaterThan(0);
+    });
+
+    it('detects missing title', () => {
+      const webContent = {
+        title: 'Missing Title',
+        headings: [],
+        paragraphs: [],
+        codeBlocks: [],
+        formulas: [],
+        blockquoteFormulas: [],
+        listItems: [],
+        links: [],
+        figures: [],
+      };
+      const markdown = '# Different Title\n\nContent.';
+      const result = verifyMarkdownContent(webContent, markdown);
+      expect(result.missing.title).toBe(true);
+    });
+
+    it('verifies headings', () => {
+      const webContent = {
+        title: '',
+        headings: [
+          { level: 'h2', text: 'Introduction' },
+          { level: 'h3', text: 'Background' },
+        ],
+        paragraphs: [],
+        codeBlocks: [],
+        formulas: [],
+        blockquoteFormulas: [],
+        listItems: [],
+        links: [],
+        figures: [],
+      };
+      const markdown = '## Introduction\n\n### Background\n\nText.';
+      const result = verifyMarkdownContent(webContent, markdown);
+      expect(result.missing.headings).toHaveLength(0);
+    });
+
+    it('detects missing headings', () => {
+      const webContent = {
+        title: '',
+        headings: [{ level: 'h2', text: 'Unique Heading XYZ' }],
+        paragraphs: [],
+        codeBlocks: [],
+        formulas: [],
+        blockquoteFormulas: [],
+        listItems: [],
+        links: [],
+        figures: [],
+      };
+      const markdown = '## Different Heading\n\nText.';
+      const result = verifyMarkdownContent(webContent, markdown);
+      expect(result.missing.headings).toContain('Unique Heading XYZ');
+    });
+
+    it('verifies code blocks with fuzzy matching', () => {
+      const webContent = {
+        title: '',
+        headings: [],
+        paragraphs: [],
+        codeBlocks: [
+          'function hello() {\n  console.log("hello");\n  return true;\n}',
+        ],
+        formulas: [],
+        blockquoteFormulas: [],
+        listItems: [],
+        links: [],
+        figures: [],
+      };
+      const markdown =
+        '```javascript\nfunction hello() {\n  console.log("hello");\n  return true;\n}\n```';
+      const result = verifyMarkdownContent(webContent, markdown);
+      expect(result.missing.codeBlocks).toHaveLength(0);
+    });
+
+    it('calculates pass rate', () => {
+      const webContent = {
+        title: 'Test',
+        headings: [{ level: 'h2', text: 'Found' }],
+        paragraphs: [],
+        codeBlocks: [],
+        formulas: [],
+        blockquoteFormulas: [],
+        listItems: [],
+        links: [],
+        figures: [],
+      };
+      const markdown = '# Test\n\n## Found\n\nContent.';
+      const result = verifyMarkdownContent(webContent, markdown);
+      expect(result.passRate).toBe(1.0);
+      expect(result.success).toBe(true);
+    });
+
+    it('checks figure images when hasLocalImages is set', () => {
+      const webContent = {
+        title: '',
+        headings: [],
+        paragraphs: [],
+        codeBlocks: [],
+        formulas: [],
+        blockquoteFormulas: [],
+        listItems: [],
+        links: [],
+        figures: [1, 2, 3],
+      };
+      const markdown =
+        '![Figure 1](images/figure-1.png)\n![Figure 2](images/figure-2.png)\n![Figure 3](images/figure-3.png)';
+      const result = verifyMarkdownContent(webContent, markdown, {
+        hasLocalImages: true,
+        expectedFigures: 3,
+      });
+      expect(result.missing.images).toBe(0);
+    });
+  });
+});