@link-assistant/web-capture 1.1.2 → 1.4.2

package/LICENSE

@@ -1,24 +0,0 @@
-This is free and unencumbered software released into the public domain.
-
-Anyone is free to copy, modify, publish, use, compile, sell, or
-distribute this software, either in source code form or as a compiled
-binary, for any purpose, commercial or non-commercial, and by any
-means.
-
-In jurisdictions that recognize copyright laws, the author or authors
-of this software dedicate any and all copyright interest in the
-software to the public domain. We make this dedication for the benefit
-of the public at large and to the detriment of our heirs and
-successors. We intend this dedication to be an overt act of
-relinquishment in perpetuity of all present and future rights to this
-software under copyright law.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
-EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
-IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR
-OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
-ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
-OTHER DEALINGS IN THE SOFTWARE.
-
-For more information, please refer to <https://unlicense.org>

package/apply.sh

@@ -1,166 +0,0 @@
-#!/usr/bin/env bash
-set -euo pipefail
-
-# Usage: ./init.sh [directory]
-DIR=${1:-web-capture}
-echo "Initializing web-capture microservice in '$DIR'..."
-
-# Create project directory and enter
-mkdir -p "$DIR"
-cd "$DIR"
-
-# Initialize git
-git init
-
-echo "# Creating package.json"
-cat > package.json << 'EOF'
-{
-  "name": "web-capture",
-  "version": "1.0.0",
-  "description": "Microservice to render web pages as HTML, Markdown, or PNG",
-  "main": "index.js",
-  "scripts": {
-    "start": "node index.js",
-    "dev": "nodemon index.js"
-  },
-  "dependencies": {
-    "express": "^4.18.2",
-    "turndown": "^7.1.1",
-    "capture-website": "^4.0.0"
-  },
-  "type": "module",
-  "keywords": ["render", "markdown", "screenshot", "microservice"],
-  "license": "MIT"
-}
-EOF
-
-# Install dependencies
-if command -v yarn >/dev/null 2>&1; then
-  echo "Installing dependencies with yarn..."
-  yarn install
-else
-  echo "Installing dependencies with npm..."
-  npm install
-fi
-
-echo "# Writing index.js"
-cat > index.js << 'EOF'
-import express from 'express';
-import fetch from 'node:fetch';               // Node 18+ has native fetch
-import TurndownService from 'turndown';
-import captureWebsite from 'capture-website';
-
-const app = express();
-const port = process.env.PORT || 3000;
-
-// Route: Fetch raw HTML
-app.get('/html', async (req, res) => {
-  const url = req.query.url;
-  if (!url) return res.status(400).send('Missing `url` parameter');
-  try {
-    const response = await fetch(url);
-    const html = await response.text();
-    res.type('text/html').send(html);
-  } catch (err) {
-    console.error(err);
-    res.status(500).send('Error fetching HTML');
-  }
-});
-
-// Route: Convert HTML to Markdown
-app.get('/markdown', async (req, res) => {
-  const url = req.query.url;
-  if (!url) return res.status(400).send('Missing `url` parameter');
-  try {
-    const response = await fetch(url);
-    const html = await response.text();
-    const turndown = new TurndownService();
-    const markdown = turndown.turndown(html);
-    res.type('text/markdown').send(markdown);
-  } catch (err) {
-    console.error(err);
-    res.status(500).send('Error converting to Markdown');
-  }
-});
-
-// Route: Capture screenshot as PNG
-app.get('/image', async (req, res) => {
-  const url = req.query.url;
-  if (!url) return res.status(400).send('Missing `url` parameter');
-  try {
-    const buffer = await captureWebsite.buffer(url, { fullPage: true });
-    res.type('image/png').send(buffer);
-  } catch (err) {
-    console.error(err);
-    res.status(500).send('Error capturing screenshot');
-  }
-});
-
-app.listen(port, () => {
-  console.log(`Renderer service listening on http://localhost:${port}`);
-});
-EOF
-
-echo "# Writing Dockerfile"
-cat > Dockerfile << 'EOF'
-FROM node:20-slim
-
-WORKDIR /usr/src/app
-COPY package.json yarn.lock* ./
-
-# Install production dependencies and Chrome deps
-RUN apt-get update && apt-get install -y \
-    gconf-service libasound2 libatk1.0-0 libc6 libcairo2 \
-    libexpat1 libfontconfig1 libgcc1 libgconf-2-4 libgdk-pixbuf2.0-0 \
-    libglib2.0-0 libgtk-3-0 libnspr4 libpango-1.0-0 libstdc++6 \
-    libx11-6 libx11-xcb1 libxcb1 libxcomposite1 libxcursor1 libxdamage1 \
-    libxext6 libxfixes3 libxi6 libxrandr2 libxrender1 libxss1 libxtst6 \
-    ca-certificates fonts-liberation lsb-release xdg-utils wget \
-    --no-install-recommends && rm -rf /var/lib/apt/lists/* \
- && npm install --production
-
-COPY . .
-ENV NODE_ENV=production
-EXPOSE 3000
-CMD ["node", "index.js"]
-EOF
-
-echo "# Creating .gitignore"
-cat > .gitignore << 'EOF'
-node_modules
-.env
-EOF
-
-echo "# Creating README.md"
-cat > README.md << 'EOF'
-# web-capture
-
-A microservice to fetch URLs and render them as:
-
-- **HTML**: GET /html?url=<URL>
-- **Markdown**: GET /markdown?url=<URL>
-- **PNG screenshot**: GET /image?url=<URL>
-
-## Usage
-
-```bash
-npm start
-# or
-yarn start
-curl http://localhost:3000/html?url=https://example.com
-```
-
-## Docker
-
-```bash
-docker build -t web-capture .
-docker run -p 3000:3000 web-capture
-```
-EOF
-
-echo "
-✔ Project 'web-capture' scaffolded successfully in $(pwd)
-Next steps:
-  cd "$(pwd)"
-  npm start
-"

package/docs/market-analysis-2025-05-13.md

@@ -1,64 +0,0 @@
-# Market Analysis Report: 13 May 2025
-
-This report analyzes open-source projects similar to "web-capture," a microservice for capturing web content in HTML, Markdown, and PNG formats. The projects are grouped into **microservices** and **libraries**, with each evaluated for its features and relevance to web content capture. Below are two tables summarizing the findings, followed by detailed feature lists for each project.
-
----
-
-## Microservices for Web Capture
-
-| **Project**             | **License** | **GitHub Link**                                                                           |
-| ----------------------- | ----------- | ----------------------------------------------------------------------------------------- |
-| **screeenly**           | MIT         | [stefanzweifel/screeenly](https://github.com/stefanzweifel/screeenly)                     |
-| **capture-website-api** | ISC         | [robvanderleek/capture-website-api](https://github.com/robvanderleek/capture-website-api) |
-| **captino**             | MIT         | [pistatium/captino](https://github.com/pistatium/captino)                                 |
-| **gowitness**           | MIT         | [gowitness/gowitness](https://github.com/gowitness/gowitness)                             |
-
-### Features of Microservices
-
-- **screeenly**
-  - API for screenshots (PNG), PDFs, and HTML rendering
-  - Customizable via pixel density and image loading options
-  - Self-hosting support via Heroku, Zeet, or Docker
-
-- **capture-website-api**
-  - API for website screenshots (PNG)
-  - Customizable via query parameters (e.g., width, height, scale)
-  - Uses Puppeteer for rendering complex sites
-  - Configurable via environment variables
-
-- **captino**
-  - API for webpage screenshots (PNG)
-  - Supports full-page captures
-  - Easy deployment with Docker
-
-- **gowitness**
-  - Command-line tool for website screenshots (PNG)
-  - Batch processing and report generation
-  - Built with Golang and Chrome Headless
-
----
-
-## Libraries for Web Capture
-
-| **Project**                | **License** | **GitHub Link**                                                                                                   |
-| -------------------------- | ----------- | ----------------------------------------------------------------------------------------------------------------- |
-| **webpage-capture**        | MIT         | [b4dnewz/webpage-capture](https://github.com/b4dnewz/webpage-capture)                                             |
-| **html-screen-capture-js** | MIT         | [html-screen-capture-js/html-screen-capture-js](https://github.com/html-screen-capture-js/html-screen-capture-js) |
-
-### Features of Libraries
-
-- **webpage-capture**
-  - Captures web pages in PNG, JPEG, PDF, HTML, and other formats
-  - Customizable via options like timeout and headers
-  - Supports single and batch captures
-
-- **html-screen-capture-js**
-  - Captures webpages as self-contained HTML
-  - Removes external dependencies while preserving appearance
-  - Lightweight (12KB)
-
----
-
-## Summary
-
-The report identifies four microservices (**screeenly**, **capture-website-api**, **captino**, **gowitness**) and two libraries (**webpage-capture**, **html-screen-capture-js**) that align with the functionality of "web-capture." Microservices provide API-driven solutions, while libraries offer flexibility for custom implementations. All projects are open-source, licensed under permissive terms (MIT or ISC), and actively maintained as of May 13, 2025.

package/scripts/changeset-version.mjs

@@ -1,38 +0,0 @@
-#!/usr/bin/env node
-
-/**
- * Custom changeset version script that ensures package-lock.json is synchronized
- * with package.json after version bumps.
- *
- * This script:
- * 1. Runs `changeset version` to update package versions
- * 2. Runs `npm install` to synchronize package-lock.json with the new versions
- *
- * Uses link-foundation libraries:
- * - use-m: Dynamic package loading without package.json dependencies
- * - command-stream: Modern shell command execution with streaming support
- */
-
-// Load use-m dynamically
-const { use } = eval(
-  await (await fetch('https://unpkg.com/use-m/use.js')).text()
-);
-
-// Import command-stream for shell command execution
-const { $ } = await use('command-stream');
-
-try {
-  console.log('Running changeset version...');
-  await $`npx changeset version`;
-
-  console.log('\nSynchronizing package-lock.json...');
-  await $`npm install --package-lock-only`;
-
-  console.log('\n✅ Version bump complete with synchronized package-lock.json');
-} catch (error) {
-  console.error('Error during version bump:', error.message);
-  if (process.env.DEBUG) {
-    console.error('Stack trace:', error.stack);
-  }
-  process.exit(1);
-}

package/scripts/create-github-release.mjs

@@ -1,93 +0,0 @@
-#!/usr/bin/env node
-
-/**
- * Create GitHub Release from CHANGELOG.md
- * Usage: node scripts/create-github-release.mjs --release-version <version> --repository <repository>
- *   release-version: Version number (e.g., 1.0.0)
- *   repository: GitHub repository (e.g., owner/repo)
- *
- * Uses link-foundation libraries:
- * - use-m: Dynamic package loading without package.json dependencies
- * - command-stream: Modern shell command execution with streaming support
- * - lino-arguments: Unified configuration from CLI args, env vars, and .lenv files
- */
-
-import { readFileSync } from 'fs';
-
-// Load use-m dynamically
-const { use } = eval(
-  await (await fetch('https://unpkg.com/use-m/use.js')).text()
-);
-
-// Import link-foundation libraries
-const { $ } = await use('command-stream');
-const { makeConfig } = await use('lino-arguments');
-
-// Parse CLI arguments using lino-arguments
-// Note: Using --release-version instead of --version to avoid conflict with yargs' built-in --version flag
-const config = makeConfig({
-  yargs: ({ yargs, getenv }) =>
-    yargs
-      .option('release-version', {
-        type: 'string',
-        default: getenv('VERSION', ''),
-        describe: 'Version number (e.g., 1.0.0)',
-      })
-      .option('repository', {
-        type: 'string',
-        default: getenv('REPOSITORY', ''),
-        describe: 'GitHub repository (e.g., owner/repo)',
-      }),
-});
-
-const { releaseVersion: version, repository } = config;
-
-if (!version || !repository) {
-  console.error('Error: Missing required arguments');
-  console.error(
-    'Usage: node scripts/create-github-release.mjs --release-version <version> --repository <repository>'
-  );
-  process.exit(1);
-}
-
-const tag = `v${version}`;
-
-console.log(`Creating GitHub release for ${tag}...`);
-
-try {
-  // Read CHANGELOG.md
-  const changelog = readFileSync('./CHANGELOG.md', 'utf8');
-
-  // Extract changelog entry for this version
-  // Read from CHANGELOG.md between this version header and the next version header
-  const versionHeaderRegex = new RegExp(`## ${version}[\\s\\S]*?(?=## \\d|$)`);
-  const match = changelog.match(versionHeaderRegex);
-
-  let releaseNotes = '';
-  if (match) {
-    // Remove the version header itself and trim
-    releaseNotes = match[0].replace(`## ${version}`, '').trim();
-  }
-
-  if (!releaseNotes) {
-    releaseNotes = `Release ${version}`;
-  }
-
-  // Create release using GitHub API with JSON input
-  // This avoids shell escaping issues that occur when passing text via command-line arguments
-  // (Previously caused apostrophes like "didn't" to appear as "didn'''" in releases)
-  const payload = JSON.stringify({
-    tag_name: tag,
-    name: version,
-    body: releaseNotes,
-  });
-
-  await $`gh api repos/${repository}/releases -X POST --input -`.run({
-    stdin: payload,
-  });
-
-  console.log(`\u2705 Created GitHub release: ${tag}`);
-} catch (error) {
-  console.error('Error creating release:', error.message);
-  process.exit(1);
-}

package/scripts/create-manual-changeset.mjs

@@ -1,88 +0,0 @@
-#!/usr/bin/env node
-
-/**
- * Create a changeset file for manual releases
- * Usage: node scripts/create-manual-changeset.mjs --bump-type <major|minor|patch> [--description <description>]
- *
- * IMPORTANT: Update the PACKAGE_NAME constant below to match your package.json
- *
- * Uses link-foundation libraries:
- * - use-m: Dynamic package loading without package.json dependencies
- * - command-stream: Modern shell command execution with streaming support
- * - lino-arguments: Unified configuration from CLI args, env vars, and .lenv files
- */
-
-import { writeFileSync } from 'fs';
-import { randomBytes } from 'crypto';
-
-const PACKAGE_NAME = '@link-assistant/web-capture';
-
-// Load use-m dynamically
-const { use } = eval(
-  await (await fetch('https://unpkg.com/use-m/use.js')).text()
-);
-
-// Import link-foundation libraries
-const { $ } = await use('command-stream');
-const { makeConfig } = await use('lino-arguments');
-
-// Parse CLI arguments using lino-arguments
-const config = makeConfig({
-  yargs: ({ yargs, getenv }) =>
-    yargs
-      .option('bump-type', {
-        type: 'string',
-        default: getenv('BUMP_TYPE', ''),
-        describe: 'Version bump type: major, minor, or patch',
-        choices: ['major', 'minor', 'patch'],
-      })
-      .option('description', {
-        type: 'string',
-        default: getenv('DESCRIPTION', ''),
-        describe: 'Description for the changeset',
-      }),
-});
-
-try {
-  const { bumpType, description: descriptionArg } = config;
-
-  // Use provided description or default based on bump type
-  const description = descriptionArg || `Manual ${bumpType} release`;
-
-  if (!bumpType || !['major', 'minor', 'patch'].includes(bumpType)) {
-    console.error(
-      'Usage: node scripts/create-manual-changeset.mjs --bump-type <major|minor|patch> [--description <description>]'
-    );
-    process.exit(1);
-  }
-
-  // Generate a random changeset ID
-  const changesetId = randomBytes(4).toString('hex');
-  const changesetFile = `.changeset/manual-release-${changesetId}.md`;
-
-  // Create the changeset file with single quotes to match Prettier config
-  const content = `---
-'${PACKAGE_NAME}': ${bumpType}
----
-
-${description}
-`;
-
-  writeFileSync(changesetFile, content, 'utf-8');
-
-  console.log(`Created changeset: ${changesetFile}`);
-  console.log('Content:');
-  console.log(content);
-
-  // Format with Prettier
-  console.log('\nFormatting with Prettier...');
-  await $`npx prettier --write "${changesetFile}"`;
-
-  console.log('\n✅ Changeset created and formatted successfully');
-} catch (error) {
-  console.error('Error creating changeset:', error.message);
-  if (process.env.DEBUG) {
-    console.error('Stack trace:', error.stack);
-  }
-  process.exit(1);
-}

package/scripts/format-github-release.mjs

@@ -1,83 +0,0 @@
-#!/usr/bin/env node
-
-/**
- * Format GitHub release notes using the format-release-notes.mjs script
- * Usage: node scripts/format-github-release.mjs --release-version <version> --repository <repository> --commit-sha <commit_sha>
- *   release-version: Version number (e.g., 1.0.0)
- *   repository: GitHub repository (e.g., owner/repo)
- *   commit_sha: Commit SHA for PR detection
- *
- * Uses link-foundation libraries:
- * - use-m: Dynamic package loading without package.json dependencies
- * - command-stream: Modern shell command execution with streaming support
- * - lino-arguments: Unified configuration from CLI args, env vars, and .lenv files
- */
-
-// Load use-m dynamically
-const { use } = eval(
-  await (await fetch('https://unpkg.com/use-m/use.js')).text()
-);
-
-// Import link-foundation libraries
-const { $ } = await use('command-stream');
-const { makeConfig } = await use('lino-arguments');
-
-// Parse CLI arguments using lino-arguments
-// Note: Using --release-version instead of --version to avoid conflict with yargs' built-in --version flag
-const config = makeConfig({
-  yargs: ({ yargs, getenv }) =>
-    yargs
-      .option('release-version', {
-        type: 'string',
-        default: getenv('VERSION', ''),
-        describe: 'Version number (e.g., 1.0.0)',
-      })
-      .option('repository', {
-        type: 'string',
-        default: getenv('REPOSITORY', ''),
-        describe: 'GitHub repository (e.g., owner/repo)',
-      })
-      .option('commit-sha', {
-        type: 'string',
-        default: getenv('COMMIT_SHA', ''),
-        describe: 'Commit SHA for PR detection',
-      }),
-});
-
-const { releaseVersion: version, repository, commitSha } = config;
-
-if (!version || !repository || !commitSha) {
-  console.error('Error: Missing required arguments');
-  console.error(
-    'Usage: node scripts/format-github-release.mjs --release-version <version> --repository <repository> --commit-sha <commit_sha>'
-  );
-  process.exit(1);
-}
-
-const tag = `v${version}`;
-
-try {
-  // Get the release ID for this version
-  let releaseId = '';
-  try {
-    const result =
-      await $`gh api "repos/${repository}/releases/tags/${tag}" --jq '.id'`.run(
-        { capture: true }
-      );
-    releaseId = result.stdout.trim();
-  } catch {
-    console.log(`\u26A0\uFE0F Could not find release for ${tag}`);
-    process.exit(0);
-  }
-
-  if (releaseId) {
-    console.log(`Formatting release notes for ${tag}...`);
-    // Pass the trigger commit SHA for PR detection
-    // This allows proper PR lookup even if the changelog doesn't have a commit hash
-    await $`node scripts/format-release-notes.mjs --release-id "${releaseId}" --release-version "${tag}" --repository "${repository}" --commit-sha "${commitSha}"`;
-    console.log(`\u2705 Formatted release notes for ${tag}`);
-  }
-} catch (error) {
-  console.error('Error formatting release:', error.message);
-  process.exit(1);
-}