@link-assistant/web-capture 1.1.2 → 1.7.1

package/tests/unit/postprocess.test.js

@@ -0,0 +1,110 @@
+import {
+  postProcessMarkdown,
+  applyUnicodeNormalization,
+  applyLatexSpacingFixes,
+  applyPercentSignFix,
+  applyBoldFormattingFixes,
+} from '../../src/postprocess.js';
+
+describe('postprocess module', () => {
+  describe('applyUnicodeNormalization', () => {
+    it('preserves non-breaking spaces as   entities', () => {
+      expect(applyUnicodeNormalization('hello\u00A0world')).toBe(
+        'hello world'
+      );
+    });
+
+    it('normalizes curly quotes to straight quotes', () => {
+      expect(applyUnicodeNormalization('\u2018hello\u2019')).toBe("'hello'");
+      expect(applyUnicodeNormalization('\u201Chello\u201D')).toBe('"hello"');
+    });
+
+    it('normalizes en-dash to hyphen', () => {
+      expect(applyUnicodeNormalization('2\u20133')).toBe('2-3');
+    });
+
+    it('normalizes ellipsis', () => {
+      expect(applyUnicodeNormalization('wait\u2026')).toBe('wait...');
+    });
+  });
+
+  describe('applyLatexSpacingFixes', () => {
+    it('adds space before formula after word character', () => {
+      const result = applyLatexSpacingFixes('where$x$is');
+      expect(result).toBe('where $x$ is');
+    });
+
+    it('trims whitespace inside formula delimiters', () => {
+      const result = applyLatexSpacingFixes('$ x + y $');
+      expect(result).toBe('$x + y$');
+    });
+
+    it('does not modify block formulas', () => {
+      const result = applyLatexSpacingFixes('$$x + y$$');
+      expect(result).toBe('$$x + y$$');
+    });
+
+    it('does not modify blockquote block formulas', () => {
+      const result = applyLatexSpacingFixes('> $$x + y$$');
+      expect(result).toBe('> $$x + y$$');
+    });
+
+    it('handles multiple formulas on one line', () => {
+      const result = applyLatexSpacingFixes('$a$and$b$');
+      expect(result).toBe('$a$ and $b$');
+    });
+  });
+
+  describe('applyPercentSignFix', () => {
+    it('fixes percent sign in inline formulas', () => {
+      const result = applyPercentSignFix('$50\\%$');
+      expect(result).toBe('$50\\\\%$');
+    });
+
+    it('fixes \\text{%} notation', () => {
+      const result = applyPercentSignFix('$50\\text{%}$');
+      expect(result).toBe('$50\\\\%$');
+    });
+  });
+
+  describe('applyBoldFormattingFixes', () => {
+    it('removes empty bold markers', () => {
+      const result = applyBoldFormattingFixes('hello ** ** world');
+      expect(result).toBe('hello  world');
+    });
+
+    it('trims content inside bold markers', () => {
+      const result = applyBoldFormattingFixes('** hello **');
+      expect(result).toBe('**hello**');
+    });
+
+    it('adds space between word and bold', () => {
+      const result = applyBoldFormattingFixes('word**bold**next');
+      expect(result).toBe('word **bold** next');
+    });
+  });
+
+  describe('postProcessMarkdown', () => {
+    it('applies all transformations by default', () => {
+      const input = 'hello\u00A0$x$world';
+      const result = postProcessMarkdown(input);
+      expect(result).toContain('hello');
+      expect(result).toContain('$x$');
+      expect(result).not.toContain('\u00A0');
+    });
+
+    it('respects option flags', () => {
+      const input = 'hello\u00A0world';
+      const result = postProcessMarkdown(input, {
+        normalizeUnicode: false,
+      });
+      expect(result).toContain('\u00A0');
+    });
+
+    it('removes stray standalone $ signs', () => {
+      const input = 'line1\n$\nline2';
+      const result = postProcessMarkdown(input);
+      expect(result).toBe('line1\n\nline2');
+    });
+  });
+});

package/tests/unit/retry.test.js

@@ -0,0 +1,111 @@
+import { jest } from '@jest/globals';
+import { retry } from '../../src/retry.js';
+
+jest.setTimeout(30000);
+
+describe('retry', () => {
+  it('returns result on first success', async () => {
+    const fn = jest.fn().mockResolvedValue('ok');
+    const result = await retry(fn, { retries: 3 });
+    expect(result).toBe('ok');
+    expect(fn).toHaveBeenCalledTimes(1);
+  });
+
+  it('retries on failure and succeeds eventually', async () => {
+    let calls = 0;
+    const fn = jest.fn().mockImplementation(async () => {
+      calls++;
+      if (calls < 3) {
+        throw new Error('fail');
+      }
+      return 'success';
+    });
+
+    const result = await retry(fn, { retries: 3, baseDelay: 50 });
+    expect(result).toBe('success');
+    expect(fn).toHaveBeenCalledTimes(3);
+  });
+
+  it('throws after exhausting retries', async () => {
+    const fn = jest.fn().mockRejectedValue(new Error('persistent failure'));
+
+    await expect(retry(fn, { retries: 2, baseDelay: 50 })).rejects.toThrow(
+      'persistent failure'
+    );
+    expect(fn).toHaveBeenCalledTimes(3); // initial + 2 retries
+  });
+
+  it('calls onRetry callback before each retry', async () => {
+    let calls = 0;
+    const fn = jest.fn().mockImplementation(async () => {
+      calls++;
+      if (calls < 2) {
+        throw new Error('fail');
+      }
+      return 'ok';
+    });
+    const onRetry = jest.fn();
+
+    await retry(fn, { retries: 3, baseDelay: 50, onRetry });
+    expect(onRetry).toHaveBeenCalledTimes(1);
+    expect(onRetry).toHaveBeenCalledWith(expect.any(Error), 1, 50);
+  });
+
+  it('uses exponential backoff delays', async () => {
+    const delays = [];
+    const onRetry = jest.fn((err, attempt, delay) => {
+      delays.push(delay);
+    });
+
+    const fn = jest.fn().mockRejectedValue(new Error('fail'));
+
+    await expect(
+      retry(fn, {
+        retries: 3,
+        baseDelay: 100,
+        factor: 2,
+        maxDelay: 1000,
+        onRetry,
+      })
+    ).rejects.toThrow();
+
+    // Delays should be: 100, 200, 400
+    expect(delays).toEqual([100, 200, 400]);
+  });
+
+  it('caps delay at maxDelay', async () => {
+    const delays = [];
+    const onRetry = jest.fn((err, attempt, delay) => {
+      delays.push(delay);
+    });
+
+    const fn = jest.fn().mockRejectedValue(new Error('fail'));
+
+    await expect(
+      retry(fn, {
+        retries: 3,
+        baseDelay: 100,
+        factor: 10,
+        maxDelay: 500,
+        onRetry,
+      })
+    ).rejects.toThrow();
+
+    // Delays: min(100, 500)=100, min(1000, 500)=500, min(10000, 500)=500
+    expect(delays).toEqual([100, 500, 500]);
+  });
+
+  it('passes attempt number to fn', async () => {
+    const attempts = [];
+    const fn = jest.fn().mockImplementation(async (attempt) => {
+      attempts.push(attempt);
+      if (attempt < 2) {
+        throw new Error('fail');
+      }
+      return 'ok';
+    });
+
+    await retry(fn, { retries: 3, baseDelay: 50 });
+    expect(attempts).toEqual([0, 1, 2]);
+  });
+});

package/tests/unit/verify.test.js

@@ -0,0 +1,182 @@
+import {
+  normalizeText,
+  normalizeCode,
+  verifyMarkdownContent,
+} from '../../src/verify.js';
+
+describe('verify module', () => {
+  describe('normalizeText', () => {
+    it('lowercases text', () => {
+      expect(normalizeText('Hello World')).toBe('hello world');
+    });
+
+    it('normalizes unicode spaces', () => {
+      expect(normalizeText('hello\u00A0world')).toBe('hello world');
+    });
+
+    it('normalizes curly quotes', () => {
+      expect(normalizeText('\u201Chello\u201D')).toBe('"hello"');
+    });
+
+    it('removes LaTeX delimiters', () => {
+      expect(normalizeText('$x + y$')).toBe('x + y');
+      expect(normalizeText('$$x + y$$')).toBe('x + y');
+    });
+
+    it('normalizes arrows', () => {
+      expect(normalizeText('\u2192')).toBe('->');
+      expect(normalizeText('\\to')).toBe('->');
+    });
+
+    it('removes \\displaystyle', () => {
+      expect(normalizeText('\\displaystyle x')).toBe('x');
+    });
+
+    it('extracts text from \\text{}', () => {
+      expect(normalizeText('\\text{hello}')).toBe('hello');
+    });
+  });
+
+  describe('normalizeCode', () => {
+    it('lowercases and collapses whitespace', () => {
+      expect(normalizeCode('  Hello   World  ')).toBe('hello world');
+    });
+
+    it('normalizes multiplication sign', () => {
+      expect(normalizeCode('2\u00D73')).toBe('2x3');
+    });
+  });
+
+  describe('verifyMarkdownContent', () => {
+    it('verifies title presence', () => {
+      const webContent = {
+        title: 'My Article',
+        headings: [],
+        paragraphs: [],
+        codeBlocks: [],
+        formulas: [],
+        blockquoteFormulas: [],
+        listItems: [],
+        links: [],
+        figures: [],
+      };
+      const markdown = '# My Article\n\nSome content here.';
+      const result = verifyMarkdownContent(webContent, markdown);
+      expect(result.missing.title).toBe(false);
+      expect(result.passedChecks).toBeGreaterThan(0);
+    });
+
+    it('detects missing title', () => {
+      const webContent = {
+        title: 'Missing Title',
+        headings: [],
+        paragraphs: [],
+        codeBlocks: [],
+        formulas: [],
+        blockquoteFormulas: [],
+        listItems: [],
+        links: [],
+        figures: [],
+      };
+      const markdown = '# Different Title\n\nContent.';
+      const result = verifyMarkdownContent(webContent, markdown);
+      expect(result.missing.title).toBe(true);
+    });
+
+    it('verifies headings', () => {
+      const webContent = {
+        title: '',
+        headings: [
+          { level: 'h2', text: 'Introduction' },
+          { level: 'h3', text: 'Background' },
+        ],
+        paragraphs: [],
+        codeBlocks: [],
+        formulas: [],
+        blockquoteFormulas: [],
+        listItems: [],
+        links: [],
+        figures: [],
+      };
+      const markdown = '## Introduction\n\n### Background\n\nText.';
+      const result = verifyMarkdownContent(webContent, markdown);
+      expect(result.missing.headings).toHaveLength(0);
+    });
+
+    it('detects missing headings', () => {
+      const webContent = {
+        title: '',
+        headings: [{ level: 'h2', text: 'Unique Heading XYZ' }],
+        paragraphs: [],
+        codeBlocks: [],
+        formulas: [],
+        blockquoteFormulas: [],
+        listItems: [],
+        links: [],
+        figures: [],
+      };
+      const markdown = '## Different Heading\n\nText.';
+      const result = verifyMarkdownContent(webContent, markdown);
+      expect(result.missing.headings).toContain('Unique Heading XYZ');
+    });
+
+    it('verifies code blocks with fuzzy matching', () => {
+      const webContent = {
+        title: '',
+        headings: [],
+        paragraphs: [],
+        codeBlocks: [
+          'function hello() {\n  console.log("hello");\n  return true;\n}',
+        ],
+        formulas: [],
+        blockquoteFormulas: [],
+        listItems: [],
+        links: [],
+        figures: [],
+      };
+      const markdown =
+        '```javascript\nfunction hello() {\n  console.log("hello");\n  return true;\n}\n```';
+      const result = verifyMarkdownContent(webContent, markdown);
+      expect(result.missing.codeBlocks).toHaveLength(0);
+    });
+
+    it('calculates pass rate', () => {
+      const webContent = {
+        title: 'Test',
+        headings: [{ level: 'h2', text: 'Found' }],
+        paragraphs: [],
+        codeBlocks: [],
+        formulas: [],
+        blockquoteFormulas: [],
+        listItems: [],
+        links: [],
+        figures: [],
+      };
+      const markdown = '# Test\n\n## Found\n\nContent.';
+      const result = verifyMarkdownContent(webContent, markdown);
+      expect(result.passRate).toBe(1.0);
+      expect(result.success).toBe(true);
+    });
+
+    it('checks figure images when hasLocalImages is set', () => {
+      const webContent = {
+        title: '',
+        headings: [],
+        paragraphs: [],
+        codeBlocks: [],
+        formulas: [],
+        blockquoteFormulas: [],
+        listItems: [],
+        links: [],
+        figures: [1, 2, 3],
+      };
+      const markdown =
+        '![Figure 1](images/figure-1.png)\n![Figure 2](images/figure-2.png)\n![Figure 3](images/figure-3.png)';
+      const result = verifyMarkdownContent(webContent, markdown, {
+        hasLocalImages: true,
+        expectedFigures: 3,
+      });
+      expect(result.missing.images).toBe(0);
+    });
+  });
+});

package/.github/workflows/release.yml

@@ -1,294 +0,0 @@
-name: Checks and release
-
-on:
-  push:
-    branches:
-      - main
-  pull_request:
-    types: [opened, synchronize, reopened]
-  # Manual release support - consolidated here to work with npm trusted publishing
-  # npm only allows ONE workflow file as trusted publisher, so all publishing
-  # must go through this workflow (release.yml)
-  workflow_dispatch:
-    inputs:
-      release_mode:
-        description: 'Manual release mode'
-        required: true
-        type: choice
-        default: 'instant'
-        options:
-          - instant
-          - changeset-pr
-      bump_type:
-        description: 'Manual release type'
-        required: true
-        type: choice
-        options:
-          - patch
-          - minor
-          - major
-      description:
-        description: 'Manual release description (optional)'
-        required: false
-        type: string
-
-concurrency: ${{ github.workflow }}-${{ github.ref }}
-
-jobs:
-  # Changeset check - only runs on PRs
-  changeset-check:
-    name: Check for Changesets
-    runs-on: ubuntu-latest
-    if: github.event_name == 'pull_request'
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-
-      - name: Setup Node.js
-        uses: actions/setup-node@v4
-        with:
-          node-version: '20.x'
-
-      - name: Install dependencies
-        run: npm install
-
-      - name: Check for changesets
-        run: |
-          # Skip changeset check for automated version PRs
-          if [[ "${{ github.head_ref }}" == "changeset-release/"* ]]; then
-            echo "Skipping changeset check for automated release PR"
-            exit 0
-          fi
-
-          # Run changeset validation script
-          node scripts/validate-changeset.mjs
-
-  # Linting and formatting - runs after changeset check on PRs, immediately on main
-  lint:
-    name: Lint and Format Check
-    runs-on: ubuntu-latest
-    needs: [changeset-check]
-    if: always() && (github.event_name == 'push' || needs.changeset-check.result == 'success')
-    steps:
-      - uses: actions/checkout@v4
-
-      - name: Setup Node.js
-        uses: actions/setup-node@v4
-        with:
-          node-version: '20.x'
-
-      - name: Install dependencies
-        run: npm install
-
-      - name: Run ESLint
-        run: npm run lint
-
-      - name: Check formatting
-        run: npm run format:check
-
-      - name: Check code duplication
-        run: npm run check:duplication
-
-  # Test matrix: Node.js on Ubuntu with unit tests and e2e tests
-  test:
-    name: Test (Node.js on ${{ matrix.os }})
-    runs-on: ${{ matrix.os }}
-    needs: [changeset-check]
-    if: always() && (github.event_name == 'push' || needs.changeset-check.result == 'success')
-    strategy:
-      fail-fast: false
-      matrix:
-        os: [ubuntu-latest]
-    steps:
-      - uses: actions/checkout@v4
-
-      - name: Setup Node.js
-        uses: actions/setup-node@v4
-        with:
-          node-version: '22.x'
-
-      - name: Install dependencies
-        run: npm install
-
-      - name: Install Playwright browsers
-        run: npx playwright install --with-deps chromium
-
-      - name: Install Puppeteer browsers
-        run: npx puppeteer browsers install chrome
-
-      - name: Run unit, integration, and e2e process tests
-        run: npm test -- --testPathIgnorePatterns="docker.test.js"
-
-      - name: Build Docker image for e2e tests
-        run: docker compose build
-
-      - name: Run e2e Docker tests
-        run: npm test -- tests/e2e/docker.test.js
-
-  # Release - only runs on main after tests pass (for push events)
-  release:
-    name: Release
-    needs: [lint, test]
-    # Use always() to ensure this job runs even if changeset-check was skipped
-    # This is needed because lint/test jobs have a transitive dependency on changeset-check
-    if: always() && github.ref == 'refs/heads/main' && github.event_name == 'push' && needs.lint.result == 'success' && needs.test.result == 'success'
-    runs-on: ubuntu-latest
-    # Permissions required for npm OIDC trusted publishing
-    permissions:
-      contents: write
-      pull-requests: write
-      id-token: write
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-
-      - name: Setup Node.js
-        uses: actions/setup-node@v4
-        with:
-          node-version: '22.x'
-          registry-url: 'https://registry.npmjs.org'
-
-      - name: Install dependencies
-        run: npm install
-
-      - name: Update npm for OIDC trusted publishing
-        run: node scripts/setup-npm.mjs
-
-      - name: Check for changesets
-        id: check_changesets
-        run: |
-          # Count changeset files (excluding README.md and config.json)
-          CHANGESET_COUNT=$(find .changeset -name "*.md" ! -name "README.md" | wc -l)
-          echo "Found $CHANGESET_COUNT changeset file(s)"
-          echo "has_changesets=$([[ $CHANGESET_COUNT -gt 0 ]] && echo 'true' || echo 'false')" >> $GITHUB_OUTPUT
-
-      - name: Version packages and commit to main
-        if: steps.check_changesets.outputs.has_changesets == 'true'
-        id: version
-        run: node scripts/version-and-commit.mjs --mode changeset
-
-      - name: Publish to npm
-        # Run if version was committed OR if a previous attempt already committed (for re-runs)
-        if: steps.version.outputs.version_committed == 'true' || steps.version.outputs.already_released == 'true'
-        id: publish
-        run: node scripts/publish-to-npm.mjs --should-pull
-
-      - name: Create GitHub Release
-        if: steps.publish.outputs.published == 'true'
-        env:
-          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        run: node scripts/create-github-release.mjs --release-version "${{ steps.publish.outputs.published_version }}" --repository "${{ github.repository }}"
-
-      - name: Format GitHub release notes
-        if: steps.publish.outputs.published == 'true'
-        env:
-          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        run: node scripts/format-github-release.mjs --release-version "${{ steps.publish.outputs.published_version }}" --repository "${{ github.repository }}" --commit-sha "${{ github.sha }}"
-
-  # Manual Instant Release - triggered via workflow_dispatch with instant mode
-  # This job is in release.yml because npm trusted publishing
-  # only allows one workflow file to be registered as a trusted publisher
-  instant-release:
-    name: Instant Release
-    if: github.event_name == 'workflow_dispatch' && github.event.inputs.release_mode == 'instant'
-    runs-on: ubuntu-latest
-    # Permissions required for npm OIDC trusted publishing
-    permissions:
-      contents: write
-      pull-requests: write
-      id-token: write
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-
-      - name: Setup Node.js
-        uses: actions/setup-node@v4
-        with:
-          node-version: '22.x'
-          registry-url: 'https://registry.npmjs.org'
-
-      - name: Install dependencies
-        run: npm install
-
-      - name: Update npm for OIDC trusted publishing
-        run: node scripts/setup-npm.mjs
-
-      - name: Version packages and commit to main
-        id: version
-        run: node scripts/version-and-commit.mjs --mode instant --bump-type "${{ github.event.inputs.bump_type }}" --description "${{ github.event.inputs.description }}"
-
-      - name: Publish to npm
-        # Run if version was committed OR if a previous attempt already committed (for re-runs)
-        if: steps.version.outputs.version_committed == 'true' || steps.version.outputs.already_released == 'true'
-        id: publish
-        run: node scripts/publish-to-npm.mjs
-
-      - name: Create GitHub Release
-        if: steps.publish.outputs.published == 'true'
-        env:
-          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        run: node scripts/create-github-release.mjs --release-version "${{ steps.publish.outputs.published_version }}" --repository "${{ github.repository }}"
-
-      - name: Format GitHub release notes
-        if: steps.publish.outputs.published == 'true'
-        env:
-          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        run: node scripts/format-github-release.mjs --release-version "${{ steps.publish.outputs.published_version }}" --repository "${{ github.repository }}" --commit-sha "${{ github.sha }}"
-
-  # Manual Changeset PR - creates a pull request with the changeset for review
-  changeset-pr:
-    name: Create Changeset PR
-    if: github.event_name == 'workflow_dispatch' && github.event.inputs.release_mode == 'changeset-pr'
-    runs-on: ubuntu-latest
-    permissions:
-      contents: write
-      pull-requests: write
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-
-      - name: Setup Node.js
-        uses: actions/setup-node@v4
-        with:
-          node-version: '22.x'
-
-      - name: Install dependencies
-        run: npm install
-
-      - name: Create changeset file
-        run: node scripts/create-manual-changeset.mjs --bump-type "${{ github.event.inputs.bump_type }}" --description "${{ github.event.inputs.description }}"
-
-      - name: Format changeset with Prettier
-        run: |
-          # Run Prettier on the changeset file to ensure it matches project style
-          npx prettier --write ".changeset/*.md" || true
-
-          echo "Formatted changeset files"
-
-      - name: Create Pull Request
-        uses: peter-evans/create-pull-request@v7
-        with:
-          token: ${{ secrets.GITHUB_TOKEN }}
-          commit-message: 'chore: add changeset for manual ${{ github.event.inputs.bump_type }} release'
-          branch: changeset-manual-release-${{ github.run_id }}
-          delete-branch: true
-          title: 'chore: manual ${{ github.event.inputs.bump_type }} release'
-          body: |
-            ## Manual Release Request
-
-            This PR was created by a manual workflow trigger to prepare a **${{ github.event.inputs.bump_type }}** release.
-
-            ### Release Details
-            - **Type:** ${{ github.event.inputs.bump_type }}
-            - **Description:** ${{ github.event.inputs.description || 'Manual release' }}
-            - **Triggered by:** @${{ github.actor }}
-
-            ### Next Steps
-            1. Review the changeset in this PR
-            2. Merge this PR to main
-            3. The automated release workflow will create a version PR
-            4. Merge the version PR to publish to npm and create a GitHub release