@link-assistant/web-capture 1.12.0 → 2.0.1

package/eslint.config.js

@@ -1,112 +0,0 @@
-import js from '@eslint/js';
-import prettierConfig from 'eslint-config-prettier';
-import prettierPlugin from 'eslint-plugin-prettier';
-
-export default [
-  js.configs.recommended,
-  prettierConfig,
-  {
-    files: ['**/*.js', '**/*.mjs'],
-    plugins: {
-      prettier: prettierPlugin,
-    },
-    languageOptions: {
-      ecmaVersion: 'latest',
-      sourceType: 'module',
-      globals: {
-        // Node.js globals
-        console: 'readonly',
-        process: 'readonly',
-        Buffer: 'readonly',
-        __dirname: 'readonly',
-        __filename: 'readonly',
-        global: 'readonly',
-        setTimeout: 'readonly',
-        clearTimeout: 'readonly',
-        setInterval: 'readonly',
-        clearInterval: 'readonly',
-        // Node.js 18+ globals
-        fetch: 'readonly',
-        // Runtime-specific globals
-        Bun: 'readonly',
-        Deno: 'readonly',
-      },
-    },
-    rules: {
-      // Prettier integration
-      'prettier/prettier': 'error',
-
-      // Code quality rules
-      'no-unused-vars': 'error',
-      'no-console': 'off', // Allow console in this project
-      'no-debugger': 'error',
-
-      // Best practices
-      eqeqeq: ['error', 'always'],
-      curly: ['error', 'all'],
-      'no-var': 'error',
-      'prefer-const': 'error',
-      'prefer-arrow-callback': 'error',
-      'no-duplicate-imports': 'error',
-
-      // ES6+ features
-      'arrow-body-style': ['error', 'as-needed'],
-      'object-shorthand': ['error', 'always'],
-      'prefer-template': 'error',
-
-      // Async/await
-      'no-async-promise-executor': 'error',
-      'require-await': 'warn',
-
-      // Comments and documentation
-      'spaced-comment': ['error', 'always', { markers: ['/'] }],
-
-      // Complexity rules - reasonable thresholds for maintainability
-      complexity: ['warn', 15], // Cyclomatic complexity - allow more complex logic than strict 8
-      'max-depth': ['warn', 5], // Maximum nesting depth - slightly more lenient than strict 4
-      'max-lines-per-function': [
-        'warn',
-        {
-          max: 150, // More reasonable than strict 50 lines per function
-          skipBlankLines: true,
-          skipComments: true,
-        },
-      ],
-      'max-params': ['warn', 6], // Maximum function parameters - slightly more lenient than strict 5
-      'max-statements': ['warn', 60], // Maximum statements per function - reasonable limit for orchestration functions
-      'max-lines': ['error', 1500], // Maximum lines per file - counts all lines including blank lines and comments
-    },
-  },
-  {
-    // Test files have different requirements
-    files: ['tests/**/*.js', '**/*.test.js', '**/*.test.mjs'],
-    languageOptions: {
-      globals: {
-        // Jest globals
-        describe: 'readonly',
-        it: 'readonly',
-        test: 'readonly',
-        expect: 'readonly',
-        beforeAll: 'readonly',
-        afterAll: 'readonly',
-        beforeEach: 'readonly',
-        afterEach: 'readonly',
-        jest: 'readonly',
-      },
-    },
-    rules: {
-      'require-await': 'off', // Async functions without await are common in tests
-      'max-lines-per-function': 'off', // Test files can have long test functions
-      'max-statements': 'off', // Test files can have many statements
-    },
-  },
-  {
-    ignores: [
-      'node_modules/**',
-      'coverage/**',
-      'dist/**',
-      '*.min.js',
-      '.eslintcache',
-    ],
-  },
-];

package/examples/js/capture_cache.js

@@ -1,12 +0,0 @@
-import { CaptureStore, CachedTransport } from '../../src/cache.js';
-import { fetchTransport } from '../../src/transport.js';
-
-const store = new CaptureStore('.captures');
-const request = { url: 'https://example.com', method: 'GET', headers: {} };
-const online = new CachedTransport(fetchTransport, store);
-const first = await online.capture(request);
-const offline = new CachedTransport(fetchTransport, store, { offline: true });
-const replay = await offline.capture(request);
-console.log(replay.sha256, replay.cached, replay.stale);
-console.log('Exact replay:', first.receipt.body.equals(replay.receipt.body));
-console.log('Portable fixture:', await store.export(request));

package/examples/js/engine_comparison.js

@@ -1,39 +0,0 @@
-import fetch from 'node-fetch';
-import fs from 'fs';
-
-const baseUrl = 'http://localhost:3000';
-const targetUrl = 'https://example.com';
-
-console.log('Comparing Puppeteer and Playwright engines...\n');
-
-// Fetch using Puppeteer
-console.log('1. Fetching HTML using Puppeteer (default)...');
-const startPuppeteer = Date.now();
-const puppeteerResponse = await fetch(
-  `${baseUrl}/html?url=${encodeURIComponent(targetUrl)}`
-);
-const puppeteerHtml = await puppeteerResponse.text();
-const puppeteerTime = Date.now() - startPuppeteer;
-fs.writeFileSync('output/puppeteer_comparison.html', puppeteerHtml);
-console.log(`   Time: ${puppeteerTime}ms`);
-console.log(`   Size: ${puppeteerHtml.length} bytes`);
-
-// Fetch using Playwright
-console.log('\n2. Fetching HTML using Playwright...');
-const startPlaywright = Date.now();
-const playwrightResponse = await fetch(
-  `${baseUrl}/html?url=${encodeURIComponent(targetUrl)}&engine=playwright`
-);
-const playwrightHtml = await playwrightResponse.text();
-const playwrightTime = Date.now() - startPlaywright;
-fs.writeFileSync('output/playwright_comparison.html', playwrightHtml);
-console.log(`   Time: ${playwrightTime}ms`);
-console.log(`   Size: ${playwrightHtml.length} bytes`);
-
-// Summary
-console.log('\n=== Summary ===');
-console.log(`Puppeteer: ${puppeteerTime}ms, ${puppeteerHtml.length} bytes`);
-console.log(`Playwright: ${playwrightTime}ms, ${playwrightHtml.length} bytes`);
-console.log(`Difference: ${Math.abs(puppeteerTime - playwrightTime)}ms`);
-
-console.log('\nResults saved to output/ directory.');

package/examples/js/html_download.js

@@ -1,23 +0,0 @@
-import fs from 'fs';
-import path from 'path';
-import { fileURLToPath } from 'url';
-
-const __dirname = path.dirname(fileURLToPath(import.meta.url));
-const url = process.argv[2] || 'https://example.com';
-const endpoint = `http://localhost:3000/html?url=${encodeURIComponent(url)}`;
-
-fetch(endpoint)
-  .then((res) => {
-    if (!res.ok) {
-      throw new Error(`HTTP error! status: ${res.status}`);
-    }
-    return res.text();
-  })
-  .then((text) => {
-    const outputPath = path.join(__dirname, 'downloaded.html');
-    fs.writeFileSync(outputPath, text, 'utf-8');
-    console.log(`HTML saved to ${outputPath}`);
-  })
-  .catch((err) => {
-    console.error('Error:', err);
-  });

package/examples/js/image_download.js

@@ -1,36 +0,0 @@
-import fs from 'fs';
-import path from 'path';
-import { fileURLToPath } from 'url';
-
-const __dirname = path.dirname(fileURLToPath(import.meta.url));
-const url = process.argv[2] || 'https://example.com';
-const endpoint = `http://localhost:3000/image?url=${encodeURIComponent(url)}`;
-
-fetch(endpoint)
-  .then((res) => {
-    console.log('Status:', res.status);
-    console.log('Content-Type:', res.headers.get('content-type'));
-    if (!res.ok) {
-      throw new Error(`HTTP error! status: ${res.status}`);
-    }
-    return res.arrayBuffer();
-  })
-  .then((buffer) => {
-    const buf = Buffer.from(buffer);
-    const pngSignature = Buffer.from([
-      0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a,
-    ]);
-    if (buf.slice(0, 8).equals(pngSignature)) {
-      const outputPath = path.join(__dirname, 'downloaded.png');
-      fs.writeFileSync(outputPath, buf);
-      console.log(`Image saved to ${outputPath} (valid PNG)`);
-    } else {
-      console.log(
-        'Response is not a valid PNG! First bytes:',
-        buf.slice(0, 16)
-      );
-    }
-  })
-  .catch((err) => {
-    console.error('Error:', err);
-  });

package/examples/js/markdown_download.js

@@ -1,23 +0,0 @@
-import fs from 'fs';
-import path from 'path';
-import { fileURLToPath } from 'url';
-
-const __dirname = path.dirname(fileURLToPath(import.meta.url));
-const url = process.argv[2] || 'https://example.com';
-const endpoint = `http://localhost:3000/markdown?url=${encodeURIComponent(url)}`;
-
-fetch(endpoint)
-  .then((res) => {
-    if (!res.ok) {
-      throw new Error(`HTTP error! status: ${res.status}`);
-    }
-    return res.text();
-  })
-  .then((text) => {
-    const outputPath = path.join(__dirname, 'downloaded.md');
-    fs.writeFileSync(outputPath, text, 'utf-8');
-    console.log(`Markdown saved to ${outputPath}`);
-  })
-  .catch((err) => {
-    console.error('Error:', err);
-  });

package/examples/js/playwright_example.js

@@ -1,23 +0,0 @@
-import fetch from 'node-fetch';
-import fs from 'fs';
-
-const baseUrl = 'http://localhost:3000';
-const targetUrl = 'https://example.com';
-
-console.log('Fetching HTML using Playwright engine...');
-const htmlResponse = await fetch(
-  `${baseUrl}/html?url=${encodeURIComponent(targetUrl)}&engine=playwright`
-);
-const html = await htmlResponse.text();
-fs.writeFileSync('output/playwright_html.html', html);
-console.log('HTML saved to output/playwright_html.html');
-
-console.log('Fetching screenshot using Playwright engine...');
-const imageResponse = await fetch(
-  `${baseUrl}/image?url=${encodeURIComponent(targetUrl)}&engine=playwright`
-);
-const imageBuffer = await imageResponse.buffer();
-fs.writeFileSync('output/playwright_screenshot.png', imageBuffer);
-console.log('Screenshot saved to output/playwright_screenshot.png');
-
-console.log('Done! Both requests used Playwright engine.');

package/examples/python/html_download.py

@@ -1,15 +0,0 @@
-import sys
-import requests
-import os
-
-url = sys.argv[1] if len(sys.argv) > 1 else 'https://example.com'
-endpoint = f'http://localhost:3000/html?url={url}'
-
-response = requests.get(endpoint)
-response.raise_for_status()
-
-output_path = os.path.join(os.path.dirname(__file__), 'downloaded.html')
-with open(output_path, 'w', encoding='utf-8') as f:
-    f.write(response.text)
-
-print(f'HTML saved to {output_path}') 

package/examples/python/image_download.py

@@ -1,20 +0,0 @@
-import sys
-import requests
-import os
-
-url = sys.argv[1] if len(sys.argv) > 1 else 'https://example.com'
-endpoint = f'http://localhost:3000/image?url={url}'
-
-response = requests.get(endpoint)
-print('Status:', response.status_code)
-print('Content-Type:', response.headers.get('content-type'))
-
-# PNG signature: first 8 bytes
-png_signature = b'\x89PNG\r\n\x1a\n'
-if response.content[:8] == png_signature:
-    output_path = os.path.join(os.path.dirname(__file__), 'downloaded.png')
-    with open(output_path, 'wb') as f:
-        f.write(response.content)
-    print(f'Image saved to {output_path} (valid PNG)')
-else:
-    print('Response is not a valid PNG! First bytes:', response.content[:16]) 

package/examples/python/markdown_download.py

@@ -1,15 +0,0 @@
-import sys
-import requests
-import os
-
-url = sys.argv[1] if len(sys.argv) > 1 else 'https://example.com'
-endpoint = f'http://localhost:3000/markdown?url={url}'
-
-response = requests.get(endpoint)
-response.raise_for_status()
-
-output_path = os.path.join(os.path.dirname(__file__), 'downloaded.md')
-with open(output_path, 'w', encoding='utf-8') as f:
-    f.write(response.text)
-
-print(f'Markdown saved to {output_path}') 

package/examples/python/playwright_example.py

@@ -1,22 +0,0 @@
-import requests
-import os
-
-base_url = 'http://localhost:3000'
-target_url = 'https://example.com'
-
-# Create output directory if it doesn't exist
-os.makedirs('output', exist_ok=True)
-
-print('Fetching HTML using Playwright engine...')
-html_response = requests.get(f'{base_url}/html', params={'url': target_url, 'engine': 'playwright'})
-with open('output/playwright_html.html', 'w') as f:
-    f.write(html_response.text)
-print('HTML saved to output/playwright_html.html')
-
-print('Fetching screenshot using Playwright engine...')
-image_response = requests.get(f'{base_url}/image', params={'url': target_url, 'engine': 'playwright'})
-with open('output/playwright_screenshot.png', 'wb') as f:
-    f.write(image_response.content)
-print('Screenshot saved to output/playwright_screenshot.png')
-
-print('Done! Both requests used Playwright engine.')

package/jest.config.mjs

@@ -1,20 +0,0 @@
-export default {
-  transform: {
-    '^.+\\.js$': ['babel-jest', { configFile: './babel.config.cjs' }],
-  },
-  transformIgnorePatterns: ['node_modules/(?!(turndown)/)'],
-  moduleNameMapper: {
-    '^(\\.{1,2}/.*)\\.js$': '$1',
-  },
-  testEnvironment: 'node',
-  verbose: true,
-  testMatch: [
-    '**/tests/ci/**/*.test.js',
-    '**/tests/unit/**/*.test.js',
-    '**/tests/mock/**/*.test.js',
-    '**/tests/e2e/**/*.test.js',
-    '**/tests/integration/**/*.test.js',
-    '**/tests/xpaste/**/*.test.js',
-  ],
-  setupFiles: ['./tests/jest.setup.mjs'],
-};

package/tests/ci/dependency-freshness.test.js

@@ -1,74 +0,0 @@
-import { jest } from '@jest/globals';
-import { checkVersions } from '../../../scripts/check-dependency-freshness.mjs';
-
-const latest = async () => '2.0.0';
-const dependencies = [{ name: 'example', current: '1.0.0' }];
-const blockers = {
-  example: {
-    issue: 'https://github.com/example/repo/issues/1',
-    reason: 'Upstream API breaks packaged consumers',
-  },
-};
-
-test('freshness rejects old releases, missing packages and undocumented exceptions', async () => {
-  for (const current of ['1.0.0', '(missing)']) {
-    await expect(
-      checkVersions([{ name: 'example', current }], {}, latest, jest.fn())
-    ).rejects.toThrow('behind 2.0.0');
-  }
-});
-test('freshness permits only documented open issues', async () => {
-  await expect(
-    checkVersions(dependencies, blockers, latest, async () => ({
-      state: 'open',
-    }))
-  ).resolves.toBeUndefined();
-  await expect(
-    checkVersions(dependencies, blockers, latest, async () => ({
-      state: 'closed',
-    }))
-  ).rejects.toThrow('is closed');
-  await expect(
-    checkVersions(
-      dependencies,
-      { example: { ...blockers.example, reason: '' } },
-      latest,
-      jest.fn()
-    )
-  ).rejects.toThrow('document an open issue');
-  await expect(
-    checkVersions(
-      dependencies,
-      { example: { ...blockers.example, reason: '   ' } },
-      latest,
-      jest.fn()
-    )
-  ).rejects.toThrow('document an open issue');
-});
-test('registry and issue failures fail the check rather than treating dependencies as fresh', async () => {
-  await expect(
-    checkVersions(
-      dependencies,
-      {},
-      async () => {
-        throw new Error('registry down');
-      },
-      jest.fn()
-    )
-  ).rejects.toThrow('registry down');
-  await expect(
-    checkVersions(dependencies, blockers, latest, async () => {
-      throw new Error('issue unavailable');
-    })
-  ).rejects.toThrow('issue unavailable');
-});
-test('current releases need no exception', async () => {
-  const getIssue = jest.fn();
-  await checkVersions(
-    [{ name: 'example', current: '2.0.0' }],
-    {},
-    latest,
-    getIssue
-  );
-  expect(getIssue).not.toHaveBeenCalled();
-});

package/tests/ci/detect-code-changes.test.js

@@ -1,174 +0,0 @@
-import { execFileSync } from 'child_process';
-import {
-  mkdirSync,
-  mkdtempSync,
-  readFileSync,
-  rmSync,
-  writeFileSync,
-} from 'fs';
-import { tmpdir } from 'os';
-import { dirname, join, resolve } from 'path';
-import { fileURLToPath } from 'url';
-
-const __filename = fileURLToPath(import.meta.url);
-const __dirname = dirname(__filename);
-const repoRoot = resolve(__dirname, '../../..');
-const detectorPath = resolve(repoRoot, 'scripts/detect-code-changes.mjs');
-
-function run(command, args, cwd, options = {}) {
-  return execFileSync(command, args, {
-    cwd,
-    encoding: 'utf8',
-    stdio: ['ignore', 'pipe', 'pipe'],
-    ...options,
-  });
-}
-
-function git(repoPath, args) {
-  return run('git', args, repoPath);
-}
-
-function writeTrackedFile(repoPath, relativePath, contents) {
-  const filePath = join(repoPath, relativePath);
-  mkdirSync(dirname(filePath), { recursive: true });
-  writeFileSync(filePath, contents);
-}
-
-function commitAll(repoPath, message) {
-  git(repoPath, ['add', '.']);
-  git(repoPath, ['commit', '-m', message]);
-}
-
-function createMergedFeatureRepo() {
-  const repoPath = mkdtempSync(join(tmpdir(), 'web-capture-detect-'));
-
-  initializeRepo(repoPath);
-
-  writeTrackedFile(repoPath, 'README.md', '# fixture\n');
-  commitAll(repoPath, 'initial commit');
-
-  git(repoPath, ['checkout', '-b', 'feature']);
-  writeTrackedFile(repoPath, 'rust/src/lib.rs', 'pub fn capture() {}\n');
-  commitAll(repoPath, 'add rust code');
-
-  writeTrackedFile(repoPath, 'docs/notes.md', '# docs only tail commit\n');
-  commitAll(repoPath, 'document rust change');
-
-  git(repoPath, ['checkout', 'main']);
-  git(repoPath, ['merge', '--no-ff', 'feature', '-m', 'merge feature']);
-
-  return repoPath;
-}
-
-function createSingleCommitRepo() {
-  return createSingleCommitRepoWithFiles({
-    'js/src/index.js': 'export const ok = true;\n',
-  });
-}
-
-function createSingleCommitRepoWithFiles(files) {
-  const repoPath = mkdtempSync(join(tmpdir(), 'web-capture-detect-'));
-
-  initializeRepo(repoPath);
-  for (const [relativePath, contents] of Object.entries(files)) {
-    writeTrackedFile(repoPath, relativePath, contents);
-  }
-  commitAll(repoPath, 'initial commit');
-
-  return repoPath;
-}
-
-function initializeRepo(repoPath) {
-  git(repoPath, ['init']);
-  git(repoPath, ['config', 'user.email', 'ci@example.test']);
-  git(repoPath, ['config', 'user.name', 'CI Test']);
-  git(repoPath, ['checkout', '-b', 'main']);
-}
-
-function parseOutputs(outputFile) {
-  return Object.fromEntries(
-    readFileSync(outputFile, 'utf8')
-      .trim()
-      .split('\n')
-      .filter(Boolean)
-      .map((line) => {
-        const separatorIndex = line.indexOf('=');
-        return [line.slice(0, separatorIndex), line.slice(separatorIndex + 1)];
-      })
-  );
-}
-
-function runDetector(repoPath, eventName) {
-  const outputFile = join(repoPath, `github-output-${eventName}.txt`);
-  const env = {
-    ...process.env,
-    GITHUB_EVENT_NAME: eventName,
-    GITHUB_OUTPUT: outputFile,
-  };
-  delete env.GITHUB_BASE_SHA;
-  delete env.GITHUB_HEAD_SHA;
-
-  run(process.execPath, [detectorPath], repoPath, { env });
-
-  return parseOutputs(outputFile);
-}
-
-describe('detect-code-changes', () => {
-  let repoPath;
-
-  afterEach(() => {
-    if (repoPath) {
-      rmSync(repoPath, { recursive: true, force: true });
-      repoPath = undefined;
-    }
-  });
-
-  test('detects code introduced by a real merge commit pushed to main', () => {
-    repoPath = createMergedFeatureRepo();
-
-    const outputs = runDetector(repoPath, 'push');
-
-    expect(outputs['rust-code-changed']).toBe('true');
-    expect(outputs['rust-changed']).toBe('true');
-    expect(outputs['any-rust-code-changed']).toBe('true');
-    expect(outputs['any-code-changed']).toBe('true');
-    expect(outputs['docs-changed']).toBe('true');
-  });
-
-  test('keeps pull request merge commits scoped to the PR head commit', () => {
-    repoPath = createMergedFeatureRepo();
-
-    const outputs = runDetector(repoPath, 'pull_request');
-
-    expect(outputs['rust-code-changed']).toBe('false');
-    expect(outputs['rust-changed']).toBe('false');
-    expect(outputs['any-rust-code-changed']).toBe('false');
-    expect(outputs['any-code-changed']).toBe('false');
-    expect(outputs['docs-changed']).toBe('true');
-  });
-
-  test('detects files when a push only has one commit', () => {
-    repoPath = createSingleCommitRepo();
-
-    const outputs = runDetector(repoPath, 'push');
-
-    expect(outputs['js-code-changed']).toBe('true');
-    expect(outputs['js-changed']).toBe('true');
-    expect(outputs['any-js-code-changed']).toBe('true');
-    expect(outputs['any-code-changed']).toBe('true');
-  });
-
-  test('runs JS checks for CI tests without requiring a changeset', () => {
-    repoPath = createSingleCommitRepoWithFiles({
-      'js/jest.config.mjs': 'export default {};\n',
-      'js/tests/ci/workflow-policy.test.js': 'test("ok", () => {});\n',
-    });
-
-    const outputs = runDetector(repoPath, 'push');
-
-    expect(outputs['js-code-changed']).toBe('false');
-    expect(outputs['js-changed']).toBe('true');
-    expect(outputs['any-js-code-changed']).toBe('true');
-    expect(outputs['any-code-changed']).toBe('true');
-  });
-});

package/tests/ci/docker-install.test.js

@@ -1,31 +0,0 @@
-import { readFileSync } from 'node:fs';
-import { spawnSync } from 'node:child_process';
-
-test.each(['js', 'rust'])(
-  '%s Docker build fails when every browser package installation attempt fails',
-  (implementation) => {
-    const dockerfile = readFileSync(
-      new globalThis.URL(
-        `../../../${implementation}/Dockerfile`,
-        import.meta.url
-      ),
-      'utf8'
-    );
-    const loop = dockerfile.match(/for i in 1 2 3; do[\s\S]*?done/)[0];
-    // Execute only the finite retry loop; stub installation and waiting.
-    const result = spawnSync(
-      'bash',
-      [
-        '-c',
-        `
-      apt-get() { printf '%s\\n' "$1"; [ "$1" = update ]; }
-      sleep() { return 0; }
-      ${loop}
-    `,
-      ],
-      { encoding: 'utf8' }
-    );
-    expect(result.stdout.match(/^install$/gm)).toHaveLength(3);
-    expect(result.status).toBe(1);
-  }
-);