html-to-gutenberg 4.2.9 → 4.2.10

package/.env.example

@@ -1,3 +1,20 @@
-# Example .env file for html-to-gutenberg
-# Copy this file to .env and set your real API key
-SNAPAPI_KEY=sk_live_your_example_key_here
+# Copy this file to .env and keep the real values private.
+# Never commit your real tokens or keys.
+
+# Optional: SnapAPI preview generation
+SNAPAPI_KEY=sk_live_replace_me
+
+# Optional: Cloudflare API token for verification/management workflows
+CLOUDFLARE_API_TOKEN=cfut_replace_me
+
+# Required for R2 uploads in job mode
+CLOUDFLARE_R2_ACCOUNT_ID=replace_me
+CLOUDFLARE_R2_BUCKET=replace_me
+CLOUDFLARE_R2_ACCESS_KEY_ID=replace_me
+CLOUDFLARE_R2_SECRET_ACCESS_KEY=replace_me
+
+# Public base URL for generated assets and bundles
+CLOUDFLARE_R2_PUBLIC_BASE_URL=https://storage.example.com
+
+# Optional override for advanced setups
+# CLOUDFLARE_R2_ENDPOINT=https://<account_id>.r2.cloudflarestorage.com

package/.github/workflows/sync-npm.yml

@@ -0,0 +1,154 @@
+name: Sync And Publish NPM Packages
+
+on:
+  push:
+    branches: [main]
+  release:
+    types: [published]
+  workflow_dispatch:
+    inputs:
+      html_to_gutenberg_version:
+        description: Specific html-to-gutenberg npm version to sync. Leave empty to use the dist-tag.
+        required: false
+        type: string
+      fetch_page_assets_version:
+        description: Specific fetch-page-assets npm version to sync. Leave empty to use the dist-tag.
+        required: false
+        type: string
+      dist_tag:
+        description: npm dist-tag to sync when no explicit version is provided.
+        required: false
+        default: latest
+        type: string
+  schedule:
+    - cron: '0 9 * * *'
+
+concurrency:
+  group: sync-npm
+  cancel-in-progress: false
+
+jobs:
+  sync:
+    if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Set up Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version: '20'
+
+      - name: Sync html-to-gutenberg from npm
+        env:
+          NPM_SYNC_VERSION: ${{ inputs.html_to_gutenberg_version }}
+          NPM_SYNC_DIST_TAG: ${{ inputs.dist_tag }}
+          NPM_SYNC_PRESERVE_PATHS: .git,.github,node_modules,.env,scripts,vendor
+        run: node ./scripts/sync-from-npm.mjs
+
+      - name: Sync fetch-page-assets from npm
+        env:
+          NPM_PACKAGE_NAME: fetch-page-assets
+          NPM_SYNC_TARGET_DIR: vendor/fetch-page-assets
+          NPM_SYNC_VERSION: ${{ inputs.fetch_page_assets_version }}
+          NPM_SYNC_DIST_TAG: ${{ inputs.dist_tag }}
+        run: node ./scripts/sync-from-npm.mjs
+
+      - name: Detect changes
+        id: changes
+        run: |
+          if git diff --quiet; then
+            echo "changed=false" >> "$GITHUB_OUTPUT"
+          else
+            echo "changed=true" >> "$GITHUB_OUTPUT"
+          fi
+
+      - name: Read synced version
+        if: steps.changes.outputs.changed == 'true'
+        id: versions
+        run: |
+          echo "html_to_gutenberg=$(node -p \"require('./package.json').version\")" >> "$GITHUB_OUTPUT"
+          echo "fetch_page_assets=$(node -p \"require('./vendor/fetch-page-assets/package.json').version\")" >> "$GITHUB_OUTPUT"
+
+      - name: Commit changes
+        if: steps.changes.outputs.changed == 'true'
+        run: |
+          git config user.name "github-actions[bot]"
+          git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
+          git add -A
+          git commit -m "chore: sync npm packages html-to-gutenberg v${{ steps.versions.outputs.html_to_gutenberg }} and fetch-page-assets v${{ steps.versions.outputs.fetch_page_assets }}"
+
+      - name: Push changes
+        if: steps.changes.outputs.changed == 'true'
+        run: git push
+
+  publish_html_to_gutenberg:
+    if: github.event_name == 'release' || (github.event_name == 'push' && github.actor != 'github-actions[bot]')
+    runs-on: ubuntu-latest
+    environment: .env
+    permissions:
+      contents: write
+      id-token: write
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Set up Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version: '24'
+          registry-url: 'https://registry.npmjs.org'
+          cache: npm
+
+      - name: Install dependencies
+        run: npm ci
+
+      - name: Build package
+        run: npm run build
+
+      - name: Run tests
+        run: npm test
+
+      - name: Bump patch version on main pushes
+        if: github.event_name == 'push'
+        id: bump_version
+        run: |
+          npm version patch --no-git-tag-version
+          PACKAGE_VERSION=$(node -e "console.log(require('./package.json').version)")
+          echo "package_version=$PACKAGE_VERSION" >> "$GITHUB_OUTPUT"
+
+      - name: Check published version
+        id: version_check
+        run: |
+          PACKAGE_VERSION=$(node -p "require('./package.json').version")
+          PUBLISHED_VERSION=$(npm view html-to-gutenberg version 2>/dev/null || true)
+          echo "package_version=$PACKAGE_VERSION" >> "$GITHUB_OUTPUT"
+          echo "published_version=$PUBLISHED_VERSION" >> "$GITHUB_OUTPUT"
+          if [ "$PACKAGE_VERSION" = "$PUBLISHED_VERSION" ]; then
+            echo "should_publish=false" >> "$GITHUB_OUTPUT"
+          else
+            echo "should_publish=true" >> "$GITHUB_OUTPUT"
+          fi
+
+      - name: Publish to npm
+        if: steps.version_check.outputs.should_publish == 'true'
+        run: npm publish --provenance
+
+      - name: Commit version bump
+        if: github.event_name == 'push'
+        run: |
+          git config user.name "github-actions[bot]"
+          git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
+          git add package.json package-lock.json
+          git commit -m "chore: release html-to-gutenberg v${{ steps.version_check.outputs.package_version }}" || echo "No version changes to commit"
+
+      - name: Push version bump
+        if: github.event_name == 'push'
+        run: git push

package/fetch-page-assets.test.ts

@@ -0,0 +1,448 @@
+import { expect } from 'chai';
+import fs from 'fs';
+import http from 'http';
+import os from 'os';
+import path from 'path';
+
+import extractAssets, {
+  h,
+  p,
+  d,
+  m,
+  u,
+  w,
+  $,
+  E,
+  g,
+  U,
+  v,
+  A,
+  F,
+  x,
+  P,
+  R,
+  D,
+} from './vendor/fetch-page-assets/index.js';
+
+const tinyPng = Buffer.from(
+  '89504e470d0a1a0a0000000d4948445200000001000000010802000000907753de0000000c4944415408d763f8ffff3f0005fe02fea557a90000000049454e44ae426082',
+  'hex'
+);
+
+const listen = (server: http.Server) => {
+  return new Promise<{ baseUrl: string; close: () => Promise<void> }>((resolve, reject) => {
+    server.listen(0, '127.0.0.1', () => {
+      const address = server.address();
+
+      if (!address || typeof address === 'string') {
+        reject(new Error('Unable to determine test server address.'));
+        return;
+      }
+
+      resolve({
+        baseUrl: `http://127.0.0.1:${address.port}`,
+        close: () =>
+          new Promise<void>((closeResolve, closeReject) => {
+            server.close((error) => {
+              if (error) {
+                closeReject(error);
+                return;
+              }
+
+              closeResolve();
+            });
+          }),
+      });
+    });
+
+    server.on('error', reject);
+  });
+};
+
+describe('fetch-page-assets helpers', () => {
+  it('logs through h and p only when verbose', () => {
+    const errors: string[] = [];
+    const logs: string[] = [];
+    const originalError = console.error;
+    const originalLog = console.log;
+
+    console.error = (message?: any) => errors.push(String(message));
+    console.log = (message?: any) => logs.push(String(message));
+
+    try {
+      h('failed', true);
+      h('ignored', false);
+      p('ok', true);
+      p('ignored', false);
+    } finally {
+      console.error = originalError;
+      console.log = originalLog;
+    }
+
+    expect(errors).to.deep.equal(['[Error] failed']);
+    expect(logs).to.deep.equal(['[Success] ok']);
+  });
+
+  it('covers the URL and path helpers', () => {
+    expect(d('//cdn.example.com')).to.equal(true);
+    expect(d('https://example.com')).to.equal(false);
+    expect(m('//cdn.example.com/image.png', 'http')).to.equal('http://cdn.example.com/image.png');
+    expect(m('https://example.com/image.png')).to.equal('https://example.com/image.png');
+    expect(u('/tmp/')).to.equal(true);
+    expect(u('/tmp')).to.equal(false);
+    expect(w('images/logo.png')).to.equal(true);
+    expect(w('//cdn.example.com/logo.png')).to.equal(false);
+    expect($(' "hello" ')).to.equal('hello');
+    expect(E('images/logo.png', 'https://example.com/base/page')).to.equal('https://example.com/base/images/logo.png');
+    expect(E('//cdn.example.com/logo.png', '', 'http')).to.equal('http://cdn.example.com/logo.png');
+    expect(E('images/logo.png', 'not a valid url')).to.equal('images/logo.png');
+    expect(g(['a', 'b', 'c'])).to.equal(path.join('a', 'b', 'c'));
+    expect(A('image.png?foo=1#bar')).to.equal('image.png');
+    expect(F('/tmp', 'asset.png?foo=1#bar')).to.equal(path.join('/tmp', 'asset.png'));
+    expect(x('archive.tar.gz')).to.deep.equal(['archive', 'tar', 'gz']);
+    expect(P({ Header: 'A', header: 'B' }, 'Header')).to.equal('A');
+    expect(P({ header: 'B' }, 'Header')).to.equal('B');
+  });
+
+  it('covers file utilities and header parsing helpers', () => {
+    const tempRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'fetch-assets-helpers-'));
+    const nestedDir = path.join(tempRoot, 'nested', 'dir');
+    const html = '<img src="https://example.com/images/photo.png">';
+    const destinationPath = path.join(tempRoot, 'images');
+    const destinationFilePath = path.join(destinationPath, 'photo.png');
+
+    try {
+      U(nestedDir, false);
+      expect(fs.existsSync(nestedDir)).to.equal(true);
+
+      fs.mkdirSync(destinationPath, { recursive: true });
+      v(
+        {
+          parsedUrl: 'https://example.com/images/photo.png',
+          destinationFilePath,
+        },
+        html,
+        tempRoot,
+        true,
+        false
+      );
+
+      const writtenHtml = fs.readFileSync(path.join(tempRoot, 'index.html'), 'utf8');
+      expect(writtenHtml).to.include('images/photo.png');
+      expect(R({ 'Content-Disposition': 'attachment; filename="report"' }, 'fallback')).to.equal('report');
+      expect(R({ 'Content-Type': 'image/png' }, 'fallback')).to.equal('fallback.png');
+    } finally {
+      fs.rmSync(tempRoot, { recursive: true, force: true });
+    }
+  });
+
+  it('covers download progress logging for valid and invalid ratios', () => {
+    const logs: string[] = [];
+    const originalLog = console.log;
+    console.log = (message?: any) => logs.push(String(message));
+
+    try {
+      D({ loaded: 50, total: 100 });
+      D({ loaded: Number.NaN, total: 100 });
+    } finally {
+      console.log = originalLog;
+    }
+
+    expect(logs).to.deep.equal(['Download progress: 50%', 'Download progress: 0%']);
+  });
+});
+
+describe('fetch-page-assets integration', () => {
+  it('downloads local file assets from file URLs', async () => {
+    const tempRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'fetch-assets-local-'));
+    const sourceDir = fs.mkdtempSync(path.join(os.tmpdir(), 'fetch-assets-source-'));
+    const localFile = path.join(sourceDir, 'asset.txt');
+
+    fs.writeFileSync(localFile, 'local file content');
+
+    try {
+      const html = `<img src="file://${localFile}" />`;
+      const output = await extractAssets(html, {
+        basePath: tempRoot,
+        verbose: false,
+        maxRetryAttempts: 1,
+        retryDelay: 0,
+      });
+
+      expect(output).to.equal(html);
+      expect(fs.existsSync(path.join(tempRoot, localFile.replace(/^\//, '')))).to.equal(true);
+    } finally {
+      fs.rmSync(tempRoot, { recursive: true, force: true });
+      fs.rmSync(sourceDir, { recursive: true, force: true });
+    }
+  });
+
+  it('fetches remote HTML, follows redirects, and de-duplicates repeated asset downloads', async () => {
+    const tempRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'fetch-assets-remote-'));
+    const requestCounts: Record<string, number> = {};
+    const server = http.createServer((req, res) => {
+      const url = req.url || '/';
+      requestCounts[url] = (requestCounts[url] || 0) + 1;
+
+      if (url === '/redirect-html') {
+        res.writeHead(302, { Location: '/page' });
+        res.end();
+        return;
+      }
+
+      if (url === '/page') {
+        res.writeHead(200, { 'Content-Type': 'text/html' });
+        res.end(`
+          <html>
+            <body>
+              <img src="/redirect-image.png" srcset="/ignored-1x.png 1x" sizes="100vw" />
+              <img src="/redirect-image.png" />
+              <source src="/video.mp4" />
+              <script src="/script.js"></script>
+              <link rel="stylesheet" href="/styles/site.css" />
+              <div style="background-image:url('/backgrounds/bg.png')"></div>
+            </body>
+          </html>
+        `);
+        return;
+      }
+
+      if (url === '/redirect-image.png') {
+        res.writeHead(302, { Location: '/images/logo-download' });
+        res.end();
+        return;
+      }
+
+      if (url === '/images/logo-download') {
+        res.writeHead(200, {
+          'Content-Type': 'image/png',
+          'Content-Disposition': 'attachment; filename="logo-file"',
+        });
+        res.end(tinyPng);
+        return;
+      }
+
+      if (url === '/video.mp4') {
+        res.writeHead(200, { 'Content-Type': 'video/mp4' });
+        res.end(Buffer.from('video'));
+        return;
+      }
+
+      if (url === '/script.js') {
+        res.writeHead(200, { 'Content-Type': 'application/javascript' });
+        res.end('window.__fetchedScript = true;');
+        return;
+      }
+
+      if (url === '/styles/site.css') {
+        res.writeHead(200, { 'Content-Type': 'text/css' });
+        res.end(".hero { background-image: url('/backgrounds/bg.png'); }");
+        return;
+      }
+
+      if (url === '/backgrounds/bg.png') {
+        res.writeHead(200, { 'Content-Type': 'image/png' });
+        res.end(tinyPng);
+        return;
+      }
+
+      res.writeHead(404, { 'Content-Type': 'text/plain' });
+      res.end('not found');
+    });
+
+    const { baseUrl, close } = await listen(server);
+
+    try {
+      const output = await extractAssets(`${baseUrl}/redirect-html`, {
+        basePath: tempRoot,
+        protocol: 'http',
+        verbose: false,
+        maxRetryAttempts: 1,
+        retryDelay: 0,
+        concurrency: 4,
+      });
+
+      expect(output).to.include('<img src="/redirect-image.png"');
+      expect(output).to.not.include('srcset=');
+      expect(output).to.not.include('sizes=');
+      expect(fs.existsSync(path.join(tempRoot, 'logo-file.png'))).to.equal(true);
+      expect(fs.existsSync(path.join(tempRoot, 'styles', 'site.css'))).to.equal(true);
+      expect(fs.existsSync(path.join(tempRoot, 'script.js'))).to.equal(true);
+      expect(fs.existsSync(path.join(tempRoot, 'backgrounds', 'bg.png'))).to.equal(true);
+      expect(requestCounts['/redirect-image.png']).to.equal(1);
+      expect(requestCounts['/images/logo-download']).to.equal(1);
+    } finally {
+      await close();
+      fs.rmSync(tempRoot, { recursive: true, force: true });
+    }
+  });
+
+  it('handles invalid inputs, unsupported remote URLs, and failed asset downloads gracefully', async () => {
+    const tempRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'fetch-assets-errors-'));
+    const errors: string[] = [];
+    const originalError = console.error;
+    console.error = (message?: any) => errors.push(String(message));
+
+    const server = http.createServer((req, res) => {
+      if (req.url === '/broken.png') {
+        res.writeHead(500, { 'Content-Type': 'text/plain' });
+        res.end('broken');
+        return;
+      }
+
+      res.writeHead(404, { 'Content-Type': 'text/plain' });
+      res.end('not found');
+    });
+
+    const { baseUrl, close } = await listen(server);
+
+    try {
+      const invalidInput = await extractAssets(123 as any, {
+        basePath: tempRoot,
+        verbose: true,
+      } as any);
+      const unsupportedProtocol = await extractAssets('ftp://example.com/file.html', {
+        basePath: tempRoot,
+        verbose: true,
+      });
+      const unresolvedRelative = await extractAssets('<img src="/missing.png" />', {
+        basePath: tempRoot,
+        verbose: true,
+        maxRetryAttempts: 1,
+        retryDelay: 0,
+      });
+      const failedDownload = await extractAssets(`<img src="${baseUrl}/broken.png" />`, {
+        basePath: tempRoot,
+        verbose: true,
+        maxRetryAttempts: 1,
+        retryDelay: 0,
+      });
+
+      expect(invalidInput).to.equal('');
+      expect(unsupportedProtocol).to.equal('');
+      expect(unresolvedRelative).to.equal('<img src="/missing.png" />');
+      expect(failedDownload).to.equal(`<img src="${baseUrl}/broken.png" />`);
+      expect(errors.some((message) => message.includes('Invalid user input'))).to.equal(true);
+      expect(errors.some((message) => message.includes('Invalid baseUrl'))).to.equal(true);
+      expect(errors.some((message) => message.includes('A source URL is required'))).to.equal(true);
+      expect(errors.some((message) => message.includes('HTTP error! Status: 500'))).to.equal(true);
+    } finally {
+      console.error = originalError;
+      await close();
+      fs.rmSync(tempRoot, { recursive: true, force: true });
+    }
+  });
+
+  it('reuses a shared asset task cache and retries transient download failures', async () => {
+    const tempRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'fetch-assets-cache-'));
+    const sharedCache = new Map();
+    const requestCounts: Record<string, number> = {};
+
+    const server = http.createServer((req, res) => {
+      const url = req.url || '/';
+      requestCounts[url] = (requestCounts[url] || 0) + 1;
+
+      if (url === '/cached.png') {
+        res.writeHead(200, { 'Content-Type': 'image/png' });
+        res.end(tinyPng);
+        return;
+      }
+
+      if (url === '/flaky.png') {
+        if (requestCounts[url] === 1) {
+          req.socket.destroy();
+          return;
+        }
+
+        res.writeHead(200, { 'Content-Type': 'image/png' });
+        res.end(tinyPng);
+        return;
+      }
+
+      if (url === '/broken-page') {
+        res.writeHead(500, { 'Content-Type': 'text/plain' });
+        res.end('broken');
+        return;
+      }
+
+      res.writeHead(404, { 'Content-Type': 'text/plain' });
+      res.end('not found');
+    });
+
+    const { baseUrl, close } = await listen(server);
+
+    try {
+      await extractAssets(`<img src="${baseUrl}/cached.png" />`, {
+        basePath: tempRoot,
+        verbose: false,
+        maxRetryAttempts: 1,
+        retryDelay: 0,
+        _assetTaskCache: sharedCache,
+      });
+      await extractAssets(`<img src="${baseUrl}/cached.png" />`, {
+        basePath: tempRoot,
+        verbose: false,
+        maxRetryAttempts: 1,
+        retryDelay: 0,
+        _assetTaskCache: sharedCache,
+      });
+      await extractAssets(`<img src="${baseUrl}/flaky.png" />`, {
+        basePath: tempRoot,
+        verbose: false,
+        maxRetryAttempts: 2,
+        retryDelay: 0,
+      });
+      const brokenHtml = await extractAssets(`${baseUrl}/broken-page`, {
+        basePath: tempRoot,
+        verbose: false,
+      });
+
+      expect(requestCounts['/cached.png']).to.equal(1);
+      expect(requestCounts['/flaky.png']).to.equal(2);
+      expect(fs.existsSync(path.join(tempRoot, 'cached.png'))).to.equal(true);
+      expect(fs.existsSync(path.join(tempRoot, 'flaky.png'))).to.equal(true);
+      expect(brokenHtml).to.equal('');
+    } finally {
+      await close();
+      fs.rmSync(tempRoot, { recursive: true, force: true });
+    }
+  });
+
+  it('can upload extracted assets to mocked R2 and return asset metadata', async () => {
+    const server = http.createServer((req, res) => {
+      if (req.url === '/asset.png') {
+        res.writeHead(200, { 'Content-Type': 'image/png' });
+        res.end(tinyPng);
+        return;
+      }
+
+      res.writeHead(404, { 'Content-Type': 'text/plain' });
+      res.end('not found');
+    });
+
+    const { baseUrl, close } = await listen(server);
+    process.env.HTG_R2_MOCK = '1';
+    process.env.CLOUDFLARE_R2_PUBLIC_BASE_URL = 'https://storage.example.com';
+
+    try {
+      const result = await extractAssets(`<img src="${baseUrl}/asset.png" />`, {
+        basePath: process.cwd(),
+        verbose: false,
+        uploadToR2: true,
+        returnDetails: true,
+        jobId: 'conv_123',
+        r2Prefix: 'generated/conv_123/assets',
+      });
+
+      expect(result.html).to.include(`${baseUrl}/asset.png`);
+      expect(result.assets).to.have.length(1);
+      expect(result.assets[0].name).to.equal('asset.png');
+      expect(result.assets[0].url).to.equal('https://storage.example.com/generated/conv_123/assets/asset.png');
+      expect(result.assets[0].path).to.equal('/generated/conv_123/assets/asset.png');
+    } finally {
+      delete process.env.HTG_R2_MOCK;
+      delete process.env.CLOUDFLARE_R2_PUBLIC_BASE_URL;
+      await close();
+    }
+  });
+});