@link-assistant/web-capture 1.1.2

package/bin/web-capture.js

@@ -0,0 +1,285 @@
+#!/usr/bin/env node
+// CLI entry point for web-capture
+// Supports two modes:
+// 1. Server mode: web-capture --serve [--port 3000]
+// 2. Capture mode: web-capture <url> [options]
+
+import fs from 'fs';
+import { URL } from 'node:url';
+import { makeConfig } from 'lino-arguments';
+
+// Create configuration using lino-arguments pattern
+const config = makeConfig({
+  yargs: ({ yargs, getenv }) =>
+    yargs
+      .usage(
+        'web-capture - Capture web pages as HTML, Markdown, or PNG\n\nUsage:\n  web-capture --serve [--port <port>]       Start as API server\n  web-capture <url> [options]               Capture a URL to file/stdout'
+      )
+      .option('serve', {
+        alias: 's',
+        type: 'boolean',
+        description: 'Start as HTTP API server',
+        default: false,
+      })
+      .option('port', {
+        alias: 'p',
+        type: 'number',
+        description: 'Port to listen on (default: 3000, or PORT env)',
+        // Don't use getenv here - let yargs read from process.env.PORT directly
+        // which will be set by the test or by lino-arguments from .lenv
+        default: 3000,
+      })
+      .option('format', {
+        alias: 'f',
+        type: 'string',
+        description: 'Output format: html, markdown, md, image, png',
+        default: 'html',
+      })
+      .option('output', {
+        alias: 'o',
+        type: 'string',
+        description:
+          'Output file path (default: stdout for text, auto-generated for images)',
+      })
+      .option('engine', {
+        alias: 'e',
+        type: 'string',
+        description: 'Browser engine: puppeteer, playwright',
+        default: getenv('BROWSER_ENGINE', 'puppeteer'),
+      })
+      .option('configuration', {
+        type: 'string',
+        description: 'Path to .lenv configuration file',
+      })
+      .help('help')
+      .alias('help', 'h')
+      .version()
+      .alias('version', 'v')
+      .example('web-capture --serve', 'Start API server on port 3000')
+      .example(
+        'web-capture --serve --port 8080',
+        'Start API server on custom port'
+      )
+      .example(
+        'web-capture https://example.com',
+        'Capture URL as HTML to stdout'
+      )
+      .example(
+        'web-capture https://example.com --format markdown --output page.md',
+        'Capture URL as Markdown to file'
+      )
+      .example(
+        'web-capture https://example.com --format png --engine playwright -o screenshot.png',
+        'Capture screenshot using Playwright'
+      )
+      .epilogue(
+        'API Endpoints (in server mode):\n  GET /html?url=<URL>&engine=<ENGINE>       Get rendered HTML\n  GET /markdown?url=<URL>                   Get Markdown conversion\n  GET /image?url=<URL>&engine=<ENGINE>      Get PNG screenshot\n  GET /fetch?url=<URL>                      Proxy fetch\n  GET /stream?url=<URL>                     Streaming proxy'
+      )
+      .strict(),
+  lenv: {
+    enabled: true,
+    path: '.lenv',
+  },
+});
+
+async function startServer(port) {
+  // Import the Express app
+  const { app } = await import('../src/index.js');
+
+  return new Promise((resolve, reject) => {
+    const server = app.listen(port, () => {
+      console.log(`web-capture server listening on http://localhost:${port}`);
+      console.log('');
+      console.log('Available endpoints:');
+      console.log(`  GET /html?url=<URL>       - Render page as HTML`);
+      console.log(`  GET /markdown?url=<URL>   - Convert page to Markdown`);
+      console.log(`  GET /image?url=<URL>      - Screenshot page as PNG`);
+      console.log(`  GET /fetch?url=<URL>      - Proxy fetch content`);
+      console.log(`  GET /stream?url=<URL>     - Stream content`);
+      console.log('');
+      console.log('Press Ctrl+C to stop the server');
+      resolve(server);
+    });
+
+    server.on('error', reject);
+
+    // Handle graceful shutdown
+    function shutdown(signal) {
+      console.log(`\nReceived ${signal}, shutting down...`);
+      server.close(() => {
+        console.log('Server closed');
+        process.exit(0);
+      });
+      // Force exit if not closed in 2 seconds
+      setTimeout(() => {
+        console.error('Force exiting after 2s');
+        process.exit(1);
+      }, 2000);
+    }
+
+    process.on('SIGTERM', () => shutdown('SIGTERM'));
+    process.on('SIGINT', () => shutdown('SIGINT'));
+  });
+}
+
+async function captureUrl(url, options) {
+  const { format, output, engine } = options;
+
+  // Ensure URL is absolute
+  let absoluteUrl = url;
+  if (!url.startsWith('http://') && !url.startsWith('https://')) {
+    absoluteUrl = `https://${url}`;
+  }
+
+  // Validate URL
+  try {
+    new URL(absoluteUrl);
+  } catch {
+    console.error(`Error: Invalid URL "${url}"`);
+    process.exit(1);
+  }
+
+  // Import required modules
+  const {
+    fetchHtml,
+    convertHtmlToMarkdown,
+    convertToUtf8,
+    convertRelativeUrls,
+  } = await import('../src/lib.js');
+  const { createBrowser } = await import('../src/browser.js');
+
+  const normalizedFormat = format.toLowerCase();
+
+  try {
+    if (normalizedFormat === 'markdown' || normalizedFormat === 'md') {
+      // Markdown format
+      const html = await fetchHtml(absoluteUrl);
+      const markdown = convertHtmlToMarkdown(html, absoluteUrl);
+
+      if (output) {
+        fs.writeFileSync(output, markdown, 'utf-8');
+        console.error(`Markdown saved to: ${output}`);
+      } else {
+        process.stdout.write(markdown);
+      }
+    } else if (
+      normalizedFormat === 'image' ||
+      normalizedFormat === 'png' ||
+      normalizedFormat === 'screenshot'
+    ) {
+      // Image/screenshot format
+      const browser = await createBrowser(engine);
+      try {
+        const page = await browser.newPage();
+        await page.setExtraHTTPHeaders({
+          'Accept-Language': 'en-US,en;q=0.9',
+          'Accept-Charset': 'utf-8',
+        });
+        await page.setUserAgent(
+          'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36'
+        );
+        await page.setViewport({ width: 1280, height: 800 });
+        await page.goto(absoluteUrl, {
+          waitUntil: 'networkidle0',
+          timeout: 30000,
+        });
+        // Wait for 5 seconds after page load
+        await new Promise((resolve) => setTimeout(resolve, 5000));
+
+        const buffer = await page.screenshot({ type: 'png' });
+
+        if (output) {
+          fs.writeFileSync(output, buffer);
+          console.error(`Screenshot saved to: ${output}`);
+        } else {
+          // Generate default filename based on URL
+          const urlObj = new URL(absoluteUrl);
+          const defaultFilename = `${urlObj.hostname.replace(/\./g, '_')}_${Date.now()}.png`;
+          fs.writeFileSync(defaultFilename, buffer);
+          console.error(`Screenshot saved to: ${defaultFilename}`);
+        }
+      } finally {
+        await browser.close();
+      }
+    } else {
+      // HTML format (default)
+      const html = await fetchHtml(absoluteUrl);
+      const hasJavaScript =
+        /<script[^>]*>[\s\S]*?<\/script>|<script[^>]*\/>|javascript:/i.test(
+          html
+        );
+      const isHtml = /<html[^>]*>[\s\S]*?<\/html>/i.test(html);
+
+      let resultHtml;
+
+      if (!isHtml || hasJavaScript) {
+        // Use browser to get rendered HTML
+        const browser = await createBrowser(engine);
+        try {
+          const page = await browser.newPage();
+          await page.setExtraHTTPHeaders({
+            'Accept-Language': 'en-US,en;q=0.9',
+            'Accept-Charset': 'utf-8',
+          });
+          await page.setUserAgent(
+            'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36'
+          );
+          await page.setViewport({ width: 1280, height: 800 });
+          await page.goto(absoluteUrl, {
+            waitUntil: 'networkidle0',
+            timeout: 30000,
+          });
+          // Wait for 5 seconds after page load
+          await new Promise((resolve) => setTimeout(resolve, 5000));
+
+          const renderedHtml = await page.content();
+          const utf8Html = convertToUtf8(renderedHtml);
+          resultHtml = convertRelativeUrls(utf8Html, absoluteUrl);
+        } finally {
+          await browser.close();
+        }
+      } else {
+        // Plain HTML without JavaScript
+        const utf8Html = convertToUtf8(html);
+        resultHtml = convertRelativeUrls(utf8Html, absoluteUrl);
+      }
+
+      if (output) {
+        fs.writeFileSync(output, resultHtml, 'utf-8');
+        console.error(`HTML saved to: ${output}`);
+      } else {
+        process.stdout.write(resultHtml);
+      }
+    }
+  } catch (err) {
+    console.error(`Error capturing ${absoluteUrl}: ${err.message}`);
+    process.exit(1);
+  }
+}
+
+async function main() {
+  // Get positional arguments (non-option arguments)
+  const url = config._ && config._.length > 0 ? config._[0] : null;
+
+  if (config.serve) {
+    // Server mode
+    await startServer(config.port);
+  } else if (url) {
+    // Capture mode
+    await captureUrl(url, {
+      format: config.format,
+      output: config.output,
+      engine: config.engine,
+    });
+  } else {
+    // No arguments - show error
+    console.error('Error: Missing URL or --serve flag');
+    console.error('Run with --help for usage information');
+    process.exit(1);
+  }
+}
+
+main().catch((err) => {
+  console.error('Fatal error:', err.message);
+  process.exit(1);
+});

package/docker-compose.yml

@@ -0,0 +1,9 @@
+services:
+  web-capture:
+    build:
+      context: .
+      dockerfile: Dockerfile
+    ports:
+      - '3000:3000'
+    environment:
+      - PORT=3000

package/docs/market-analysis-2025-05-13.md

@@ -0,0 +1,64 @@
+# Market Analysis Report: 13 May 2025
+
+This report analyzes open-source projects similar to "web-capture," a microservice for capturing web content in HTML, Markdown, and PNG formats. The projects are grouped into **microservices** and **libraries**, with each evaluated for its features and relevance to web content capture. Below are two tables summarizing the findings, followed by detailed feature lists for each project.
+
+---
+
+## Microservices for Web Capture
+
+| **Project**             | **License** | **GitHub Link**                                                                           |
+| ----------------------- | ----------- | ----------------------------------------------------------------------------------------- |
+| **screeenly**           | MIT         | [stefanzweifel/screeenly](https://github.com/stefanzweifel/screeenly)                     |
+| **capture-website-api** | ISC         | [robvanderleek/capture-website-api](https://github.com/robvanderleek/capture-website-api) |
+| **captino**             | MIT         | [pistatium/captino](https://github.com/pistatium/captino)                                 |
+| **gowitness**           | MIT         | [gowitness/gowitness](https://github.com/gowitness/gowitness)                             |
+
+### Features of Microservices
+
+- **screeenly**
+  - API for screenshots (PNG), PDFs, and HTML rendering
+  - Customizable via pixel density and image loading options
+  - Self-hosting support via Heroku, Zeet, or Docker
+
+- **capture-website-api**
+  - API for website screenshots (PNG)
+  - Customizable via query parameters (e.g., width, height, scale)
+  - Uses Puppeteer for rendering complex sites
+  - Configurable via environment variables
+
+- **captino**
+  - API for webpage screenshots (PNG)
+  - Supports full-page captures
+  - Easy deployment with Docker
+
+- **gowitness**
+  - Command-line tool for website screenshots (PNG)
+  - Batch processing and report generation
+  - Built with Golang and Chrome Headless
+
+---
+
+## Libraries for Web Capture
+
+| **Project**                | **License** | **GitHub Link**                                                                                                   |
+| -------------------------- | ----------- | ----------------------------------------------------------------------------------------------------------------- |
+| **webpage-capture**        | MIT         | [b4dnewz/webpage-capture](https://github.com/b4dnewz/webpage-capture)                                             |
+| **html-screen-capture-js** | MIT         | [html-screen-capture-js/html-screen-capture-js](https://github.com/html-screen-capture-js/html-screen-capture-js) |
+
+### Features of Libraries
+
+- **webpage-capture**
+  - Captures web pages in PNG, JPEG, PDF, HTML, and other formats
+  - Customizable via options like timeout and headers
+  - Supports single and batch captures
+
+- **html-screen-capture-js**
+  - Captures webpages as self-contained HTML
+  - Removes external dependencies while preserving appearance
+  - Lightweight (12KB)
+
+---
+
+## Summary
+
+The report identifies four microservices (**screeenly**, **capture-website-api**, **captino**, **gowitness**) and two libraries (**webpage-capture**, **html-screen-capture-js**) that align with the functionality of "web-capture." Microservices provide API-driven solutions, while libraries offer flexibility for custom implementations. All projects are open-source, licensed under permissive terms (MIT or ISC), and actively maintained as of May 13, 2025.

package/eslint.config.js

@@ -0,0 +1,112 @@
+import js from '@eslint/js';
+import prettierConfig from 'eslint-config-prettier';
+import prettierPlugin from 'eslint-plugin-prettier';
+
+export default [
+  js.configs.recommended,
+  prettierConfig,
+  {
+    files: ['**/*.js', '**/*.mjs'],
+    plugins: {
+      prettier: prettierPlugin,
+    },
+    languageOptions: {
+      ecmaVersion: 'latest',
+      sourceType: 'module',
+      globals: {
+        // Node.js globals
+        console: 'readonly',
+        process: 'readonly',
+        Buffer: 'readonly',
+        __dirname: 'readonly',
+        __filename: 'readonly',
+        global: 'readonly',
+        setTimeout: 'readonly',
+        clearTimeout: 'readonly',
+        setInterval: 'readonly',
+        clearInterval: 'readonly',
+        // Node.js 18+ globals
+        fetch: 'readonly',
+        // Runtime-specific globals
+        Bun: 'readonly',
+        Deno: 'readonly',
+      },
+    },
+    rules: {
+      // Prettier integration
+      'prettier/prettier': 'error',
+
+      // Code quality rules
+      'no-unused-vars': 'error',
+      'no-console': 'off', // Allow console in this project
+      'no-debugger': 'error',
+
+      // Best practices
+      eqeqeq: ['error', 'always'],
+      curly: ['error', 'all'],
+      'no-var': 'error',
+      'prefer-const': 'error',
+      'prefer-arrow-callback': 'error',
+      'no-duplicate-imports': 'error',
+
+      // ES6+ features
+      'arrow-body-style': ['error', 'as-needed'],
+      'object-shorthand': ['error', 'always'],
+      'prefer-template': 'error',
+
+      // Async/await
+      'no-async-promise-executor': 'error',
+      'require-await': 'warn',
+
+      // Comments and documentation
+      'spaced-comment': ['error', 'always', { markers: ['/'] }],
+
+      // Complexity rules - reasonable thresholds for maintainability
+      complexity: ['warn', 15], // Cyclomatic complexity - allow more complex logic than strict 8
+      'max-depth': ['warn', 5], // Maximum nesting depth - slightly more lenient than strict 4
+      'max-lines-per-function': [
+        'warn',
+        {
+          max: 150, // More reasonable than strict 50 lines per function
+          skipBlankLines: true,
+          skipComments: true,
+        },
+      ],
+      'max-params': ['warn', 6], // Maximum function parameters - slightly more lenient than strict 5
+      'max-statements': ['warn', 60], // Maximum statements per function - reasonable limit for orchestration functions
+      'max-lines': ['error', 1500], // Maximum lines per file - counts all lines including blank lines and comments
+    },
+  },
+  {
+    // Test files have different requirements
+    files: ['tests/**/*.js', '**/*.test.js', '**/*.test.mjs'],
+    languageOptions: {
+      globals: {
+        // Jest globals
+        describe: 'readonly',
+        it: 'readonly',
+        test: 'readonly',
+        expect: 'readonly',
+        beforeAll: 'readonly',
+        afterAll: 'readonly',
+        beforeEach: 'readonly',
+        afterEach: 'readonly',
+        jest: 'readonly',
+      },
+    },
+    rules: {
+      'require-await': 'off', // Async functions without await are common in tests
+      'max-lines-per-function': 'off', // Test files can have long test functions
+      'max-statements': 'off', // Test files can have many statements
+    },
+  },
+  {
+    ignores: [
+      'node_modules/**',
+      'coverage/**',
+      'dist/**',
+      '*.min.js',
+      '.eslintcache',
+    ],
+  },
+];

package/examples/js/engine_comparison.js

@@ -0,0 +1,39 @@
+import fetch from 'node-fetch';
+import fs from 'fs';
+
+const baseUrl = 'http://localhost:3000';
+const targetUrl = 'https://example.com';
+
+console.log('Comparing Puppeteer and Playwright engines...\n');
+
+// Fetch using Puppeteer
+console.log('1. Fetching HTML using Puppeteer (default)...');
+const startPuppeteer = Date.now();
+const puppeteerResponse = await fetch(
+  `${baseUrl}/html?url=${encodeURIComponent(targetUrl)}`
+);
+const puppeteerHtml = await puppeteerResponse.text();
+const puppeteerTime = Date.now() - startPuppeteer;
+fs.writeFileSync('output/puppeteer_comparison.html', puppeteerHtml);
+console.log(`   Time: ${puppeteerTime}ms`);
+console.log(`   Size: ${puppeteerHtml.length} bytes`);
+
+// Fetch using Playwright
+console.log('\n2. Fetching HTML using Playwright...');
+const startPlaywright = Date.now();
+const playwrightResponse = await fetch(
+  `${baseUrl}/html?url=${encodeURIComponent(targetUrl)}&engine=playwright`
+);
+const playwrightHtml = await playwrightResponse.text();
+const playwrightTime = Date.now() - startPlaywright;
+fs.writeFileSync('output/playwright_comparison.html', playwrightHtml);
+console.log(`   Time: ${playwrightTime}ms`);
+console.log(`   Size: ${playwrightHtml.length} bytes`);
+
+// Summary
+console.log('\n=== Summary ===');
+console.log(`Puppeteer: ${puppeteerTime}ms, ${puppeteerHtml.length} bytes`);
+console.log(`Playwright: ${playwrightTime}ms, ${playwrightHtml.length} bytes`);
+console.log(`Difference: ${Math.abs(puppeteerTime - playwrightTime)}ms`);
+
+console.log('\nResults saved to output/ directory.');

package/examples/js/html_download.js

@@ -0,0 +1,23 @@
+import fs from 'fs';
+import path from 'path';
+import { fileURLToPath } from 'url';
+
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
+const url = process.argv[2] || 'https://example.com';
+const endpoint = `http://localhost:3000/html?url=${encodeURIComponent(url)}`;
+
+fetch(endpoint)
+  .then((res) => {
+    if (!res.ok) {
+      throw new Error(`HTTP error! status: ${res.status}`);
+    }
+    return res.text();
+  })
+  .then((text) => {
+    const outputPath = path.join(__dirname, 'downloaded.html');
+    fs.writeFileSync(outputPath, text, 'utf-8');
+    console.log(`HTML saved to ${outputPath}`);
+  })
+  .catch((err) => {
+    console.error('Error:', err);
+  });

package/examples/js/image_download.js

@@ -0,0 +1,36 @@
+import fs from 'fs';
+import path from 'path';
+import { fileURLToPath } from 'url';
+
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
+const url = process.argv[2] || 'https://example.com';
+const endpoint = `http://localhost:3000/image?url=${encodeURIComponent(url)}`;
+
+fetch(endpoint)
+  .then((res) => {
+    console.log('Status:', res.status);
+    console.log('Content-Type:', res.headers.get('content-type'));
+    if (!res.ok) {
+      throw new Error(`HTTP error! status: ${res.status}`);
+    }
+    return res.arrayBuffer();
+  })
+  .then((buffer) => {
+    const buf = Buffer.from(buffer);
+    const pngSignature = Buffer.from([
+      0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a,
+    ]);
+    if (buf.slice(0, 8).equals(pngSignature)) {
+      const outputPath = path.join(__dirname, 'downloaded.png');
+      fs.writeFileSync(outputPath, buf);
+      console.log(`Image saved to ${outputPath} (valid PNG)`);
+    } else {
+      console.log(
+        'Response is not a valid PNG! First bytes:',
+        buf.slice(0, 16)
+      );
+    }
+  })
+  .catch((err) => {
+    console.error('Error:', err);
+  });

package/examples/js/markdown_download.js

@@ -0,0 +1,23 @@
+import fs from 'fs';
+import path from 'path';
+import { fileURLToPath } from 'url';
+
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
+const url = process.argv[2] || 'https://example.com';
+const endpoint = `http://localhost:3000/markdown?url=${encodeURIComponent(url)}`;
+
+fetch(endpoint)
+  .then((res) => {
+    if (!res.ok) {
+      throw new Error(`HTTP error! status: ${res.status}`);
+    }
+    return res.text();
+  })
+  .then((text) => {
+    const outputPath = path.join(__dirname, 'downloaded.md');
+    fs.writeFileSync(outputPath, text, 'utf-8');
+    console.log(`Markdown saved to ${outputPath}`);
+  })
+  .catch((err) => {
+    console.error('Error:', err);
+  });

package/examples/js/playwright_example.js

@@ -0,0 +1,23 @@
+import fetch from 'node-fetch';
+import fs from 'fs';
+
+const baseUrl = 'http://localhost:3000';
+const targetUrl = 'https://example.com';
+
+console.log('Fetching HTML using Playwright engine...');
+const htmlResponse = await fetch(
+  `${baseUrl}/html?url=${encodeURIComponent(targetUrl)}&engine=playwright`
+);
+const html = await htmlResponse.text();
+fs.writeFileSync('output/playwright_html.html', html);
+console.log('HTML saved to output/playwright_html.html');
+
+console.log('Fetching screenshot using Playwright engine...');
+const imageResponse = await fetch(
+  `${baseUrl}/image?url=${encodeURIComponent(targetUrl)}&engine=playwright`
+);
+const imageBuffer = await imageResponse.buffer();
+fs.writeFileSync('output/playwright_screenshot.png', imageBuffer);
+console.log('Screenshot saved to output/playwright_screenshot.png');
+
+console.log('Done! Both requests used Playwright engine.');

package/examples/python/html_download.py

@@ -0,0 +1,15 @@
+import sys
+import requests
+import os
+
+url = sys.argv[1] if len(sys.argv) > 1 else 'https://example.com'
+endpoint = f'http://localhost:3000/html?url={url}'
+
+response = requests.get(endpoint)
+response.raise_for_status()
+
+output_path = os.path.join(os.path.dirname(__file__), 'downloaded.html')
+with open(output_path, 'w', encoding='utf-8') as f:
+    f.write(response.text)
+
+print(f'HTML saved to {output_path}') 

package/examples/python/image_download.py

@@ -0,0 +1,20 @@
+import sys
+import requests
+import os
+
+url = sys.argv[1] if len(sys.argv) > 1 else 'https://example.com'
+endpoint = f'http://localhost:3000/image?url={url}'
+
+response = requests.get(endpoint)
+print('Status:', response.status_code)
+print('Content-Type:', response.headers.get('content-type'))
+
+# PNG signature: first 8 bytes
+png_signature = b'\x89PNG\r\n\x1a\n'
+if response.content[:8] == png_signature:
+    output_path = os.path.join(os.path.dirname(__file__), 'downloaded.png')
+    with open(output_path, 'wb') as f:
+        f.write(response.content)
+    print(f'Image saved to {output_path} (valid PNG)')
+else:
+    print('Response is not a valid PNG! First bytes:', response.content[:16]) 

package/examples/python/markdown_download.py

@@ -0,0 +1,15 @@
+import sys
+import requests
+import os
+
+url = sys.argv[1] if len(sys.argv) > 1 else 'https://example.com'
+endpoint = f'http://localhost:3000/markdown?url={url}'
+
+response = requests.get(endpoint)
+response.raise_for_status()
+
+output_path = os.path.join(os.path.dirname(__file__), 'downloaded.md')
+with open(output_path, 'w', encoding='utf-8') as f:
+    f.write(response.text)
+
+print(f'Markdown saved to {output_path}')