@link-assistant/web-capture 1.1.2

package/examples/python/playwright_example.py

@@ -0,0 +1,22 @@
+import requests
+import os
+
+base_url = 'http://localhost:3000'
+target_url = 'https://example.com'
+
+# Create output directory if it doesn't exist
+os.makedirs('output', exist_ok=True)
+
+print('Fetching HTML using Playwright engine...')
+html_response = requests.get(f'{base_url}/html', params={'url': target_url, 'engine': 'playwright'})
+with open('output/playwright_html.html', 'w') as f:
+    f.write(html_response.text)
+print('HTML saved to output/playwright_html.html')
+
+print('Fetching screenshot using Playwright engine...')
+image_response = requests.get(f'{base_url}/image', params={'url': target_url, 'engine': 'playwright'})
+with open('output/playwright_screenshot.png', 'wb') as f:
+    f.write(image_response.content)
+print('Screenshot saved to output/playwright_screenshot.png')
+
+print('Done! Both requests used Playwright engine.')

package/jest.config.mjs

@@ -0,0 +1,18 @@
+export default {
+  transform: {
+    '^.+\\.js$': ['babel-jest', { configFile: './babel.config.cjs' }],
+  },
+  transformIgnorePatterns: ['node_modules/(?!(turndown)/)'],
+  moduleNameMapper: {
+    '^(\\.{1,2}/.*)\\.js$': '$1',
+  },
+  testEnvironment: 'node',
+  verbose: true,
+  testMatch: [
+    '**/tests/unit/**/*.test.js',
+    '**/tests/mock/**/*.test.js',
+    '**/tests/e2e/**/*.test.js',
+    '**/tests/integration/**/*.test.js',
+  ],
+  setupFiles: ['./tests/jest.setup.mjs'],
+};

package/package.json

@@ -0,0 +1,83 @@
+{
+  "name": "@link-assistant/web-capture",
+  "version": "1.1.2",
+  "description": "CLI and microservice to render web pages as HTML, Markdown, or PNG",
+  "main": "src/index.js",
+  "bin": {
+    "web-capture": "./bin/web-capture.js"
+  },
+  "scripts": {
+    "start": "docker compose up -d",
+    "dev": "nodemon bin/web-capture.js -- --serve",
+    "test": "node --experimental-vm-modules ./node_modules/.bin/jest",
+    "test:watch": "node --experimental-vm-modules ./node_modules/.bin/jest --watch",
+    "test:e2e:process": "node --experimental-vm-modules ./node_modules/.bin/jest tests/e2e.process.test.js",
+    "test:e2e:docker": "node --experimental-vm-modules ./node_modules/.bin/jest tests/e2e.docker.test.js",
+    "test:e2e": "yarn test:e2e:process && yarn test:e2e:docker",
+    "test:all": "yarn build && yarn test && yarn test:e2e",
+    "build": "docker compose down && docker compose build && yarn start",
+    "examples:python": "python3 examples/python/html_download.py && python3 examples/python/markdown_download.py && python3 examples/python/image_download.py",
+    "examples:javascript": "node examples/js/html_download.js && node examples/js/markdown_download.js && node examples/js/image_download.js",
+    "examples": "yarn build && yarn examples:javascript && yarn examples:python",
+    "lint": "eslint .",
+    "lint:fix": "eslint . --fix",
+    "format": "prettier --write .",
+    "format:check": "prettier --check .",
+    "check:duplication": "jscpd .",
+    "check": "npm run lint && npm run format:check && npm run check:duplication",
+    "prepare": "husky || true",
+    "changeset": "changeset",
+    "changeset:version": "node scripts/changeset-version.mjs",
+    "changeset:publish": "changeset publish",
+    "changeset:status": "changeset status --since=origin/main"
+  },
+  "dependencies": {
+    "cheerio": "^1.0.0",
+    "express": "^4.18.2",
+    "lino-arguments": "^0.2.1",
+    "node-fetch": "^2.7.0",
+    "playwright": "^1.49.0",
+    "puppeteer": "^24.8.2",
+    "turndown": "^7.1.1"
+  },
+  "devDependencies": {
+    "@babel/core": "^7.24.0",
+    "@babel/preset-env": "^7.24.0",
+    "@changesets/cli": "^2.29.7",
+    "babel-jest": "^29.7.0",
+    "eslint": "^9.38.0",
+    "eslint-config-prettier": "^10.1.8",
+    "eslint-plugin-prettier": "^5.5.4",
+    "get-port": "^7.1.0",
+    "husky": "^9.1.7",
+    "jest": "^29.7.0",
+    "jscpd": "^4.0.5",
+    "lint-staged": "^16.2.6",
+    "nock": "^13.5.4",
+    "prettier": "^3.6.2",
+    "supertest": "^6.3.4",
+    "turndown-plugin-gfm": "^1.0.2"
+  },
+  "type": "module",
+  "keywords": [
+    "render",
+    "markdown",
+    "screenshot",
+    "microservice"
+  ],
+  "license": "UNLICENSED",
+  "engines": {
+    "node": ">=22.0.0 <23.0.0"
+  },
+  "lint-staged": {
+    "*.{js,mjs,cjs}": [
+      "eslint --fix --max-warnings 0",
+      "prettier --write",
+      "prettier --check"
+    ],
+    "*.md": [
+      "prettier --write",
+      "prettier --check"
+    ]
+  }
+}

package/scripts/changeset-version.mjs

@@ -0,0 +1,38 @@
+#!/usr/bin/env node
+
+/**
+ * Custom changeset version script that ensures package-lock.json is synchronized
+ * with package.json after version bumps.
+ *
+ * This script:
+ * 1. Runs `changeset version` to update package versions
+ * 2. Runs `npm install` to synchronize package-lock.json with the new versions
+ *
+ * Uses link-foundation libraries:
+ * - use-m: Dynamic package loading without package.json dependencies
+ * - command-stream: Modern shell command execution with streaming support
+ */
+
+// Load use-m dynamically
+const { use } = eval(
+  await (await fetch('https://unpkg.com/use-m/use.js')).text()
+);
+
+// Import command-stream for shell command execution
+const { $ } = await use('command-stream');
+
+try {
+  console.log('Running changeset version...');
+  await $`npx changeset version`;
+
+  console.log('\nSynchronizing package-lock.json...');
+  await $`npm install --package-lock-only`;
+
+  console.log('\n✅ Version bump complete with synchronized package-lock.json');
+} catch (error) {
+  console.error('Error during version bump:', error.message);
+  if (process.env.DEBUG) {
+    console.error('Stack trace:', error.stack);
+  }
+  process.exit(1);
+}

package/scripts/create-github-release.mjs

@@ -0,0 +1,93 @@
+#!/usr/bin/env node
+
+/**
+ * Create GitHub Release from CHANGELOG.md
+ * Usage: node scripts/create-github-release.mjs --release-version <version> --repository <repository>
+ *   release-version: Version number (e.g., 1.0.0)
+ *   repository: GitHub repository (e.g., owner/repo)
+ *
+ * Uses link-foundation libraries:
+ * - use-m: Dynamic package loading without package.json dependencies
+ * - command-stream: Modern shell command execution with streaming support
+ * - lino-arguments: Unified configuration from CLI args, env vars, and .lenv files
+ */
+
+import { readFileSync } from 'fs';
+
+// Load use-m dynamically
+const { use } = eval(
+  await (await fetch('https://unpkg.com/use-m/use.js')).text()
+);
+
+// Import link-foundation libraries
+const { $ } = await use('command-stream');
+const { makeConfig } = await use('lino-arguments');
+
+// Parse CLI arguments using lino-arguments
+// Note: Using --release-version instead of --version to avoid conflict with yargs' built-in --version flag
+const config = makeConfig({
+  yargs: ({ yargs, getenv }) =>
+    yargs
+      .option('release-version', {
+        type: 'string',
+        default: getenv('VERSION', ''),
+        describe: 'Version number (e.g., 1.0.0)',
+      })
+      .option('repository', {
+        type: 'string',
+        default: getenv('REPOSITORY', ''),
+        describe: 'GitHub repository (e.g., owner/repo)',
+      }),
+});
+
+const { releaseVersion: version, repository } = config;
+
+if (!version || !repository) {
+  console.error('Error: Missing required arguments');
+  console.error(
+    'Usage: node scripts/create-github-release.mjs --release-version <version> --repository <repository>'
+  );
+  process.exit(1);
+}
+
+const tag = `v${version}`;
+
+console.log(`Creating GitHub release for ${tag}...`);
+
+try {
+  // Read CHANGELOG.md
+  const changelog = readFileSync('./CHANGELOG.md', 'utf8');
+
+  // Extract changelog entry for this version
+  // Read from CHANGELOG.md between this version header and the next version header
+  const versionHeaderRegex = new RegExp(`## ${version}[\\s\\S]*?(?=## \\d|$)`);
+  const match = changelog.match(versionHeaderRegex);
+
+  let releaseNotes = '';
+  if (match) {
+    // Remove the version header itself and trim
+    releaseNotes = match[0].replace(`## ${version}`, '').trim();
+  }
+
+  if (!releaseNotes) {
+    releaseNotes = `Release ${version}`;
+  }
+
+  // Create release using GitHub API with JSON input
+  // This avoids shell escaping issues that occur when passing text via command-line arguments
+  // (Previously caused apostrophes like "didn't" to appear as "didn'''" in releases)
+  const payload = JSON.stringify({
+    tag_name: tag,
+    name: version,
+    body: releaseNotes,
+  });
+
+  await $`gh api repos/${repository}/releases -X POST --input -`.run({
+    stdin: payload,
+  });
+
+  console.log(`\u2705 Created GitHub release: ${tag}`);
+} catch (error) {
+  console.error('Error creating release:', error.message);
+  process.exit(1);
+}

package/scripts/create-manual-changeset.mjs

@@ -0,0 +1,88 @@
+#!/usr/bin/env node
+
+/**
+ * Create a changeset file for manual releases
+ * Usage: node scripts/create-manual-changeset.mjs --bump-type <major|minor|patch> [--description <description>]
+ *
+ * IMPORTANT: Update the PACKAGE_NAME constant below to match your package.json
+ *
+ * Uses link-foundation libraries:
+ * - use-m: Dynamic package loading without package.json dependencies
+ * - command-stream: Modern shell command execution with streaming support
+ * - lino-arguments: Unified configuration from CLI args, env vars, and .lenv files
+ */
+
+import { writeFileSync } from 'fs';
+import { randomBytes } from 'crypto';
+
+const PACKAGE_NAME = '@link-assistant/web-capture';
+
+// Load use-m dynamically
+const { use } = eval(
+  await (await fetch('https://unpkg.com/use-m/use.js')).text()
+);
+
+// Import link-foundation libraries
+const { $ } = await use('command-stream');
+const { makeConfig } = await use('lino-arguments');
+
+// Parse CLI arguments using lino-arguments
+const config = makeConfig({
+  yargs: ({ yargs, getenv }) =>
+    yargs
+      .option('bump-type', {
+        type: 'string',
+        default: getenv('BUMP_TYPE', ''),
+        describe: 'Version bump type: major, minor, or patch',
+        choices: ['major', 'minor', 'patch'],
+      })
+      .option('description', {
+        type: 'string',
+        default: getenv('DESCRIPTION', ''),
+        describe: 'Description for the changeset',
+      }),
+});
+
+try {
+  const { bumpType, description: descriptionArg } = config;
+
+  // Use provided description or default based on bump type
+  const description = descriptionArg || `Manual ${bumpType} release`;
+
+  if (!bumpType || !['major', 'minor', 'patch'].includes(bumpType)) {
+    console.error(
+      'Usage: node scripts/create-manual-changeset.mjs --bump-type <major|minor|patch> [--description <description>]'
+    );
+    process.exit(1);
+  }
+
+  // Generate a random changeset ID
+  const changesetId = randomBytes(4).toString('hex');
+  const changesetFile = `.changeset/manual-release-${changesetId}.md`;
+
+  // Create the changeset file with single quotes to match Prettier config
+  const content = `---
+'${PACKAGE_NAME}': ${bumpType}
+---
+
+${description}
+`;
+
+  writeFileSync(changesetFile, content, 'utf-8');
+
+  console.log(`Created changeset: ${changesetFile}`);
+  console.log('Content:');
+  console.log(content);
+
+  // Format with Prettier
+  console.log('\nFormatting with Prettier...');
+  await $`npx prettier --write "${changesetFile}"`;
+
+  console.log('\n✅ Changeset created and formatted successfully');
+} catch (error) {
+  console.error('Error creating changeset:', error.message);
+  if (process.env.DEBUG) {
+    console.error('Stack trace:', error.stack);
+  }
+  process.exit(1);
+}

package/scripts/format-github-release.mjs

@@ -0,0 +1,83 @@
+#!/usr/bin/env node
+
+/**
+ * Format GitHub release notes using the format-release-notes.mjs script
+ * Usage: node scripts/format-github-release.mjs --release-version <version> --repository <repository> --commit-sha <commit_sha>
+ *   release-version: Version number (e.g., 1.0.0)
+ *   repository: GitHub repository (e.g., owner/repo)
+ *   commit_sha: Commit SHA for PR detection
+ *
+ * Uses link-foundation libraries:
+ * - use-m: Dynamic package loading without package.json dependencies
+ * - command-stream: Modern shell command execution with streaming support
+ * - lino-arguments: Unified configuration from CLI args, env vars, and .lenv files
+ */
+
+// Load use-m dynamically
+const { use } = eval(
+  await (await fetch('https://unpkg.com/use-m/use.js')).text()
+);
+
+// Import link-foundation libraries
+const { $ } = await use('command-stream');
+const { makeConfig } = await use('lino-arguments');
+
+// Parse CLI arguments using lino-arguments
+// Note: Using --release-version instead of --version to avoid conflict with yargs' built-in --version flag
+const config = makeConfig({
+  yargs: ({ yargs, getenv }) =>
+    yargs
+      .option('release-version', {
+        type: 'string',
+        default: getenv('VERSION', ''),
+        describe: 'Version number (e.g., 1.0.0)',
+      })
+      .option('repository', {
+        type: 'string',
+        default: getenv('REPOSITORY', ''),
+        describe: 'GitHub repository (e.g., owner/repo)',
+      })
+      .option('commit-sha', {
+        type: 'string',
+        default: getenv('COMMIT_SHA', ''),
+        describe: 'Commit SHA for PR detection',
+      }),
+});
+
+const { releaseVersion: version, repository, commitSha } = config;
+
+if (!version || !repository || !commitSha) {
+  console.error('Error: Missing required arguments');
+  console.error(
+    'Usage: node scripts/format-github-release.mjs --release-version <version> --repository <repository> --commit-sha <commit_sha>'
+  );
+  process.exit(1);
+}
+
+const tag = `v${version}`;
+
+try {
+  // Get the release ID for this version
+  let releaseId = '';
+  try {
+    const result =
+      await $`gh api "repos/${repository}/releases/tags/${tag}" --jq '.id'`.run(
+        { capture: true }
+      );
+    releaseId = result.stdout.trim();
+  } catch {
+    console.log(`\u26A0\uFE0F Could not find release for ${tag}`);
+    process.exit(0);
+  }
+
+  if (releaseId) {
+    console.log(`Formatting release notes for ${tag}...`);
+    // Pass the trigger commit SHA for PR detection
+    // This allows proper PR lookup even if the changelog doesn't have a commit hash
+    await $`node scripts/format-release-notes.mjs --release-id "${releaseId}" --release-version "${tag}" --repository "${repository}" --commit-sha "${commitSha}"`;
+    console.log(`\u2705 Formatted release notes for ${tag}`);
+  }
+} catch (error) {
+  console.error('Error formatting release:', error.message);
+  process.exit(1);
+}

package/scripts/format-release-notes.mjs

@@ -0,0 +1,218 @@
+#!/usr/bin/env node
+
+/**
+ * Script to format GitHub release notes with proper formatting:
+ * - Fix special characters like \n
+ * - Add link to PR that contains the release commit (if found)
+ * - Add shields.io NPM version badge
+ * - Format nicely with proper markdown
+ *
+ * IMPORTANT: Update the PACKAGE_NAME constant below to match your package.json
+ *
+ * PR Detection Logic:
+ * 1. Extract commit hash from changelog entry (if present)
+ * 2. Fall back to --commit-sha argument (passed from workflow)
+ * 3. Look up PRs that contain the commit via GitHub API
+ * 4. If no PR found, simply don't display any PR link (no guessing)
+ *
+ * Uses link-foundation libraries:
+ * - use-m: Dynamic package loading without package.json dependencies
+ * - command-stream: Modern shell command execution with streaming support
+ * - lino-arguments: Unified configuration from CLI args, env vars, and .lenv files
+ *
+ * Note: Uses --release-version instead of --version to avoid conflict with yargs' built-in --version flag.
+ */
+
+const PACKAGE_NAME = '@link-assistant/web-capture';
+
+// Load use-m dynamically
+const { use } = eval(
+  await (await fetch('https://unpkg.com/use-m/use.js')).text()
+);
+
+// Import link-foundation libraries
+const { $ } = await use('command-stream');
+const { makeConfig } = await use('lino-arguments');
+
+// Parse CLI arguments using lino-arguments
+// Note: Using --release-version instead of --version to avoid conflict with yargs' built-in --version flag
+const config = makeConfig({
+  yargs: ({ yargs, getenv }) =>
+    yargs
+      .option('release-version', {
+        type: 'string',
+        default: getenv('VERSION', ''),
+        describe: 'Version number (e.g., v0.8.36)',
+      })
+      .option('release-id', {
+        type: 'string',
+        default: getenv('RELEASE_ID', ''),
+        describe: 'GitHub release ID',
+      })
+      .option('repository', {
+        type: 'string',
+        default: getenv('REPOSITORY', ''),
+        describe: 'GitHub repository (e.g., owner/repo)',
+      })
+      .option('commit-sha', {
+        type: 'string',
+        default: getenv('COMMIT_SHA', ''),
+        describe: 'Commit SHA for PR detection',
+      }),
+});
+
+const releaseId = config.releaseId;
+const version = config.releaseVersion;
+const repository = config.repository;
+const passedCommitSha = config.commitSha;
+
+if (!releaseId || !version || !repository) {
+  console.error(
+    'Usage: format-release-notes.mjs --release-id <releaseId> --release-version <version> --repository <repository> [--commit-sha <sha>]'
+  );
+  process.exit(1);
+}
+
+try {
+  // Get current release body
+  const result = await $`gh api repos/${repository}/releases/${releaseId}`.run({
+    capture: true,
+  });
+  const releaseData = JSON.parse(result.stdout);
+
+  const currentBody = releaseData.body || '';
+
+  // Skip if already formatted (has shields.io badge image)
+  if (currentBody.includes('img.shields.io')) {
+    console.log('ℹ️ Release notes already formatted');
+    process.exit(0);
+  }
+
+  // Extract changes section (Major, Minor, or Patch)
+  // This regex handles multiple formats:
+  // 1. With commit hash: "### [Major|Minor|Patch] Changes\n- abc1234: Description"
+  // 2. Without commit hash: "### [Major|Minor|Patch] Changes\n- Description"
+  const changesPattern =
+    /### (Major|Minor|Patch) Changes\s*\n\s*-\s+(?:([a-f0-9]+):\s+)?(.+?)$/s;
+  const changesMatch = currentBody.match(changesPattern);
+
+  let commitHash = null;
+  let rawDescription = null;
+  let changeType = null;
+
+  if (changesMatch) {
+    // Extract: [full match, changeType, commitHash (optional), description]
+    [, changeType, commitHash, rawDescription] = changesMatch;
+    console.log(`ℹ️ Found ${changeType} Changes section`);
+
+    // If commitHash is undefined and description contains it, try to extract
+    if (!commitHash && rawDescription) {
+      // This handles the case where description itself might be null/undefined
+      // and we need to safely check for commit hash at the start
+      const descWithHashMatch = rawDescription.match(/^([a-f0-9]+):\s+(.+)$/s);
+      if (descWithHashMatch) {
+        [, commitHash, rawDescription] = descWithHashMatch;
+      }
+    }
+  } else {
+    console.log('⚠️ Could not parse changes from release notes');
+    console.log('   Looking for pattern: ### [Major|Minor|Patch] Changes');
+    process.exit(0);
+  }
+
+  // Clean up the description:
+  // 1. Convert literal \n sequences (escaped newlines from GitHub API) to actual newlines
+  // 2. Remove leading/trailing quotes (including escaped quotes from command-stream shell escaping)
+  // 3. Remove any trailing npm package links or markdown that might be there
+  // 4. Normalize whitespace while preserving line breaks
+  const cleanDescription = rawDescription
+    .replace(/\\n/g, '\n') // Convert escaped \n to actual newlines
+    .replace(/^(\\['"])+/g, '') // Remove leading escaped quotes (e.g., \', \", \'', \'')
+    .replace(/(['"])+$/g, '') // Remove trailing unescaped quotes (e.g., ', ", '', '')
+    .replace(/^(['"])+/g, '') // Remove leading unescaped quotes
+    .replace(/📦.*$/s, '') // Remove any existing npm package info
+    .replace(/---.*$/s, '') // Remove any existing separators and everything after
+    .trim()
+    .split('\n') // Split by lines
+    .map((line) => line.trim()) // Trim whitespace from each line
+    .join('\n') // Rejoin with newlines
+    .replace(/\n{3,}/g, '\n\n'); // Normalize excessive blank lines (3+ becomes 2)
+
+  // Find the PR that contains the release commit
+  // Uses commit hash from changelog or passed commit SHA from workflow
+  let prNumber = null;
+
+  // Determine which commit SHA to use for PR lookup
+  const commitShaToLookup = commitHash || passedCommitSha;
+
+  if (commitShaToLookup) {
+    const source = commitHash ? 'changelog' : 'workflow';
+    console.log(
+      `ℹ️ Looking up PR for commit ${commitShaToLookup} (from ${source})`
+    );
+
+    try {
+      const prResult =
+        await $`gh api "repos/${repository}/commits/${commitShaToLookup}/pulls"`.run(
+          { capture: true }
+        );
+      const prsData = JSON.parse(prResult.stdout);
+
+      // Find the PR that's not the version bump PR (not "chore: version packages")
+      const relevantPr = prsData.find(
+        (pr) => !pr.title.includes('version packages')
+      );
+
+      if (relevantPr) {
+        prNumber = relevantPr.number;
+        console.log(`✅ Found PR #${prNumber} containing commit`);
+      } else if (prsData.length > 0) {
+        console.log(
+          '⚠️ Found PRs but all are version bump PRs, not linking any'
+        );
+      } else {
+        console.log(
+          'ℹ️ No PR found containing this commit - not adding PR link'
+        );
+      }
+    } catch (error) {
+      console.log('⚠️ Could not find PR for commit', commitShaToLookup);
+      console.log('   Error:', error.message);
+      if (process.env.DEBUG) {
+        console.error(error);
+      }
+    }
+  } else {
+    // No commit hash available from any source
+    console.log('ℹ️ No commit SHA available - not adding PR link');
+  }
+
+  // Build formatted release notes
+  const versionWithoutV = version.replace(/^v/, '');
+  const npmBadge = `[![npm version](https://img.shields.io/badge/npm-${versionWithoutV}-blue.svg)](https://www.npmjs.com/package/${PACKAGE_NAME}/v/${versionWithoutV})`;
+
+  let formattedBody = `${cleanDescription}`;
+
+  // Add PR link if available
+  if (prNumber) {
+    formattedBody += `\n\n**Related Pull Request:** #${prNumber}`;
+  }
+
+  formattedBody += `\n\n---\n\n${npmBadge}`;
+
+  // Update the release using JSON input to properly handle special characters
+  const updatePayload = JSON.stringify({ body: formattedBody });
+  await $`gh api repos/${repository}/releases/${releaseId} -X PATCH --input -`.run(
+    { stdin: updatePayload }
+  );
+
+  console.log(`✅ Formatted release notes for v${versionWithoutV}`);
+  if (prNumber) {
+    console.log(`   - Added link to PR #${prNumber}`);
+  }
+  console.log('   - Added shields.io npm badge');
+  console.log('   - Cleaned up formatting');
+} catch (error) {
+  console.error('❌ Error formatting release notes:', error.message);
+  process.exit(1);
+}