@link-assistant/web-capture 1.4.2 → 1.7.2

package/bin/web-capture.js

@@ -5,6 +5,7 @@
 // 2. Capture mode: web-capture <url> [options]
 
 import fs from 'fs';
+import path from 'path';
 import { URL } from 'node:url';
 import { makeConfig } from 'lino-arguments';
 
@@ -33,8 +34,8 @@ const config = makeConfig({
         alias: 'f',
         type: 'string',
         description:
-          'Output format: html, markdown, md, image, png, jpeg, pdf, docx, archive',
-        default: 'html',
+          'Output format: markdown, md, html, image, png, jpeg, pdf, docx, archive',
+        default: 'markdown',
       })
       .option('theme', {
         alias: 't',
@@ -89,35 +90,65 @@ const config = makeConfig({
         type: 'string',
         description: 'Path to .lenv configuration file',
       })
-      .option('enhanced', {
-        type: 'boolean',
-        description:
-          'Use enhanced markdown conversion with LaTeX extraction, metadata, and post-processing (default: false)',
-        default: false,
-      })
       .option('extractLatex', {
         type: 'boolean',
         description:
-          'Extract LaTeX formulas from img.formula, KaTeX, MathJax (default: true when --enhanced)',
-        default: true,
+          'Extract LaTeX formulas from img.formula, KaTeX, MathJax (default: true). Use --no-extract-latex to disable.',
+        default: getenv('WEB_CAPTURE_EXTRACT_LATEX', true),
       })
       .option('extractMetadata', {
         type: 'boolean',
         description:
-          'Extract article metadata (author, date, hubs, tags) (default: true when --enhanced)',
-        default: true,
+          'Extract article metadata (author, date, hubs, tags) (default: true). Use --no-extract-metadata to disable.',
+        default: getenv('WEB_CAPTURE_EXTRACT_METADATA', true),
       })
       .option('postProcess', {
         type: 'boolean',
         description:
-          'Apply post-processing (unicode normalization, LaTeX spacing) (default: true when --enhanced)',
-        default: true,
+          'Apply post-processing (unicode normalization, LaTeX spacing) (default: true). Use --no-post-process to disable.',
+        default: getenv('WEB_CAPTURE_POST_PROCESS', true),
       })
       .option('detectCodeLanguage', {
         type: 'boolean',
         description:
-          'Detect and correct code block languages (default: true when --enhanced)',
-        default: true,
+          'Detect and correct code block languages (default: true). Use --no-detect-code-language to disable.',
+        default: getenv('WEB_CAPTURE_DETECT_CODE_LANGUAGE', true),
+      })
+      .option('embedImages', {
+        type: 'boolean',
+        description:
+          'Keep images as inline base64 data URIs instead of extracting to files (default: false). Use --embed-images to enable.',
+        default: getenv('WEB_CAPTURE_EMBED_IMAGES', false),
+      })
+      .option('imagesDir', {
+        type: 'string',
+        description:
+          'Directory name for extracted images, relative to output file (default: images)',
+        default: getenv('WEB_CAPTURE_IMAGES_DIR', 'images'),
+      })
+      .option('dataDir', {
+        type: 'string',
+        description:
+          'Base directory for auto-derived output paths when -o is omitted ' +
+          '(default: ./data/web-capture)',
+        default: getenv('WEB_CAPTURE_DATA_DIR', './data/web-capture'),
+      })
+      .option('archive', {
+        type: 'string',
+        description:
+          'Create archive output. Formats: zip (default), 7z, tar.gz (alias gz), tar. Use without value for zip.',
+      })
+      .option('noExtractImages', {
+        type: 'boolean',
+        description: 'Alias for --embed-images: keep images inline as base64',
+        default: false,
+      })
+      .option('keepOriginalLinks', {
+        type: 'boolean',
+        description:
+          'Keep original remote image URLs instead of downloading or extracting. ' +
+          'Base64 data URIs are stripped (no original URL to restore).',
+        default: getenv('WEB_CAPTURE_KEEP_ORIGINAL_LINKS', false),
       })
       .option('dualTheme', {
         type: 'boolean',
@@ -150,6 +181,12 @@ const config = makeConfig({
           'API token for authenticated capture (e.g., Google Docs private documents). Can also be set via API_TOKEN env variable.',
         default: getenv('API_TOKEN', undefined),
       })
+      .option('noPrettyHtml', {
+        type: 'boolean',
+        description:
+          'Disable HTML pretty-printing in archive output (output minified HTML instead of indented).',
+        default: getenv('WEB_CAPTURE_NO_PRETTY_HTML', false),
+      })
       .help('help')
       .alias('help', 'h')
       .version()
@@ -241,10 +278,20 @@ async function startServer(port) {
   });
 }
 
+function deriveOutputPath(absoluteUrl, ext, dataDir) {
+  const parsed = new URL(absoluteUrl);
+  const host = parsed.hostname;
+  let urlPath = parsed.pathname.replace(/\/+$/, '') || '';
+  urlPath = urlPath.replace(/^\//, '');
+  const dir = path.join(dataDir, host, urlPath);
+  fs.mkdirSync(dir, { recursive: true });
+  return path.join(dir, `document.${ext}`);
+}
+
 async function captureUrl(url, options) {
   const {
     format,
-    output,
+    output: explicitOutput,
     engine,
     theme,
     width,
@@ -254,6 +301,11 @@ async function captureUrl(url, options) {
     // localImages is used by archive format
     // eslint-disable-next-line no-unused-vars
     localImages,
+    embedImages,
+    // eslint-disable-next-line no-unused-vars
+    imagesDir,
+    dataDir,
+    keepOriginalLinks,
   } = options;
 
   // Ensure URL is absolute
@@ -271,15 +323,12 @@ async function captureUrl(url, options) {
   }
 
   // Import required modules
-  const {
-    fetchHtml,
-    convertHtmlToMarkdown,
-    convertToUtf8,
-    convertRelativeUrls,
-  } = await import('../src/lib.js');
+  const { fetchHtml, convertToUtf8, convertRelativeUrls } =
+    await import('../src/lib.js');
   const { createBrowser } = await import('../src/browser.js');
   const { isGoogleDocsUrl, fetchGoogleDoc, fetchGoogleDocAsMarkdown } =
     await import('../src/gdocs.js');
+  const { stripBase64Images } = await import('../src/extract-images.js');
 
   const normalizedFormat = format.toLowerCase();
 
@@ -287,15 +336,105 @@ async function captureUrl(url, options) {
   if (isGoogleDocsUrl(absoluteUrl)) {
     try {
       const apiToken = options.apiToken;
-      if (normalizedFormat === 'markdown' || normalizedFormat === 'md') {
+      const { extractAndSaveImages } = await import('../src/extract-images.js');
+      if (normalizedFormat === 'archive' || normalizedFormat === 'zip') {
+        const { fetchGoogleDocAsArchive } = await import('../src/gdocs.js');
+        const { default: archiver } = await import('archiver');
+        const archiveFormat = options.archiveFormat || 'zip';
+        const ext =
+          archiveFormat === 'tar.gz' || archiveFormat === 'gz'
+            ? 'tar.gz'
+            : archiveFormat === '7z'
+              ? '7z'
+              : archiveFormat === 'tar'
+                ? 'tar'
+                : 'zip';
+        const archiveResult = await fetchGoogleDocAsArchive(absoluteUrl, {
+          apiToken,
+        });
+        const output =
+          explicitOutput === '-'
+            ? null
+            : explicitOutput || deriveOutputPath(absoluteUrl, ext, dataDir);
+        if (output) {
+          const outStream = fs.createWriteStream(output);
+          const archive = archiver.default
+            ? archiver.default('zip', { zlib: { level: 9 } })
+            : archiver('zip', { zlib: { level: 9 } });
+          archive.pipe(outStream);
+          const { prettyPrintHtml: ppHtml } = await import('../src/lib.js');
+          archive.append(archiveResult.markdown, { name: 'document.md' });
+          const htmlContent = options.prettyHtml
+            ? ppHtml(archiveResult.html)
+            : archiveResult.html;
+          archive.append(htmlContent, {
+            name: 'document.html',
+          });
+          for (const img of archiveResult.images) {
+            archive.append(img.data, { name: `images/${img.filename}` });
+          }
+          await archive.finalize();
+          await new Promise((resolve) => outStream.on('close', resolve));
+          console.error(`Google Doc (archive) saved to: ${output}`);
+        } else {
+          const { PassThrough } = await import('stream');
+          const passthrough = new PassThrough();
+          const archiverMod = await import('archiver');
+          const archiverFn =
+            archiverMod.default?.default || archiverMod.default;
+          const archive = archiverFn('zip', { zlib: { level: 9 } });
+          archive.pipe(passthrough);
+          passthrough.pipe(process.stdout);
+          const { prettyPrintHtml: ppHtml2 } = await import('../src/lib.js');
+          archive.append(archiveResult.markdown, { name: 'document.md' });
+          const htmlContent2 = options.prettyHtml
+            ? ppHtml2(archiveResult.html)
+            : archiveResult.html;
+          archive.append(htmlContent2, {
+            name: 'document.html',
+          });
+          for (const img of archiveResult.images) {
+            archive.append(img.data, { name: `images/${img.filename}` });
+          }
+          await archive.finalize();
+        }
+      } else if (normalizedFormat === 'markdown' || normalizedFormat === 'md') {
         const result = await fetchGoogleDocAsMarkdown(absoluteUrl, {
           apiToken,
         });
+        let { markdown } = result;
+        const output =
+          explicitOutput === '-'
+            ? null
+            : explicitOutput || deriveOutputPath(absoluteUrl, 'md', dataDir);
         if (output) {
-          fs.writeFileSync(output, result.markdown, 'utf-8');
+          if (embedImages) {
+            // Keep base64 data URIs inline
+          } else if (keepOriginalLinks) {
+            const strip = stripBase64Images(markdown);
+            if (strip.stripped > 0) {
+              markdown = strip.markdown;
+              console.error(
+                `Stripped ${strip.stripped} base64 images (keeping original links)`
+              );
+            }
+          } else {
+            const outputDir = path.dirname(output);
+            const extracted = extractAndSaveImages(markdown, outputDir, {
+              imagesDir: options.imagesDir || 'images',
+            });
+            if (extracted.extracted > 0) {
+              markdown = extracted.markdown;
+              console.error(
+                `Extracted ${extracted.extracted} images to ${options.imagesDir || 'images'}/`
+              );
+            }
+          }
+          fs.mkdirSync(path.dirname(output), { recursive: true });
+          fs.writeFileSync(output, markdown, 'utf-8');
           console.error(`Google Doc Markdown saved to: ${output}`);
         } else {
-          process.stdout.write(result.markdown);
+          process.stdout.write(markdown);
         }
       } else {
         const gdocsFormat =
@@ -306,7 +445,13 @@ async function captureUrl(url, options) {
           format: gdocsFormat,
           apiToken,
         });
+        const output =
+          explicitOutput === '-'
+            ? null
+            : explicitOutput ||
+              deriveOutputPath(absoluteUrl, gdocsFormat, dataDir);
         if (output) {
+          fs.mkdirSync(path.dirname(output), { recursive: true });
           fs.writeFileSync(output, result.content, 'utf-8');
           console.error(`Google Doc (${gdocsFormat}) saved to: ${output}`);
         } else {
@@ -346,7 +491,7 @@ async function captureUrl(url, options) {
           fullPage,
         });
         const outPath =
-          output ||
+          explicitOutput ||
           `${new URL(absoluteUrl).hostname.replace(/\./g, '_')}_${Date.now()}.jpg`;
         fs.writeFileSync(outPath, buffer);
         console.error(`JPEG screenshot saved to: ${outPath}`);
@@ -376,7 +521,7 @@ async function captureUrl(url, options) {
           printBackground: true,
           margin: { top: '1cm', right: '1cm', bottom: '1cm', left: '1cm' },
         });
-        const outPath = output || 'page.pdf';
+        const outPath = explicitOutput || 'page.pdf';
         fs.writeFileSync(outPath, pdfBuffer);
         console.error(`PDF saved to: ${outPath}`);
       } finally {
@@ -413,7 +558,7 @@ async function captureUrl(url, options) {
       }
       const doc = new Document({ sections: [{ children }] });
       const buffer = await Packer.toBuffer(doc);
-      const outPath = output || 'page.docx';
+      const outPath = explicitOutput || 'page.docx';
       fs.writeFileSync(outPath, buffer);
       console.error(`DOCX saved to: ${outPath}`);
     } else if (normalizedFormat === 'archive' || normalizedFormat === 'zip') {
@@ -435,7 +580,7 @@ async function captureUrl(url, options) {
 
       const docFormat = options.documentFormat === 'html' ? 'html' : 'markdown';
       const outPath =
-        output ||
+        explicitOutput ||
         `${new URL(absoluteUrl).hostname.replace(/\./g, '-')}-archive.zip`;
       const outStream = fs.createWriteStream(outPath);
       const archive = archiver.default
@@ -460,7 +605,11 @@ async function captureUrl(url, options) {
 
       // Build image map for local downloads
       const imageMap = new Map();
-      if (options.localImages && uniqueImages.length > 0) {
+      if (
+        options.localImages &&
+        !keepOriginalLinks &&
+        uniqueImages.length > 0
+      ) {
         let idx = 1;
         for (const imgUrl of uniqueImages) {
           const ext = imgUrl.match(/\.(jpe?g|gif|webp|svg|png)/i)?.[1] || 'png';
@@ -488,7 +637,11 @@ async function captureUrl(url, options) {
             }
           });
         }
-        archive.append($out.html(), { name: 'article.html' });
+        const { prettyPrintHtml } = await import('../src/lib.js');
+        const formattedHtml = options.prettyHtml
+          ? prettyPrintHtml($out.html())
+          : $out.html();
+        archive.append(formattedHtml, { name: 'document.html' });
       } else {
         let markdown = convertHtmlToMarkdown(html, absoluteUrl);
         if (imageMap.size > 0) {
@@ -496,7 +649,7 @@ async function captureUrl(url, options) {
             markdown = markdown.split(remoteUrl).join(localPath);
           }
         }
-        archive.append(markdown, { name: 'article.md' });
+        archive.append(markdown, { name: 'document.md' });
       }
 
       // Download images
@@ -522,24 +675,49 @@ async function captureUrl(url, options) {
       await new Promise((resolve) => outStream.on('close', resolve));
       console.error(`Archive saved to: ${outPath}`);
     } else if (normalizedFormat === 'markdown' || normalizedFormat === 'md') {
-      // Markdown format
+      // Markdown format — enhanced conversion is now the default
       const html = await fetchHtml(absoluteUrl);
-      let markdown;
-
-      if (options.enhanced) {
-        const { convertHtmlToMarkdownEnhanced } = await import('../src/lib.js');
-        const result = convertHtmlToMarkdownEnhanced(html, absoluteUrl, {
-          extractLatex: options.extractLatex,
-          extractMetadata: options.extractMetadata,
-          postProcess: options.postProcess,
-          detectCodeLanguage: options.detectCodeLanguage,
-        });
-        markdown = result.markdown;
-      } else {
-        markdown = convertHtmlToMarkdown(html, absoluteUrl);
+      const { convertHtmlToMarkdownEnhanced } = await import('../src/lib.js');
+      const { extractAndSaveImages } = await import('../src/extract-images.js');
+      const result = convertHtmlToMarkdownEnhanced(html, absoluteUrl, {
+        extractLatex: options.extractLatex,
+        extractMetadata: options.extractMetadata,
+        postProcess: options.postProcess,
+        detectCodeLanguage: options.detectCodeLanguage,
+      });
+      let markdown = result.markdown;
+
+      const output =
+        explicitOutput === '-'
+          ? null
+          : explicitOutput || deriveOutputPath(absoluteUrl, 'md', dataDir);
+      if (output) {
+        if (embedImages) {
+          // Keep base64 data URIs inline
+        } else if (keepOriginalLinks) {
+          const strip = stripBase64Images(markdown);
+          if (strip.stripped > 0) {
+            markdown = strip.markdown;
+            console.error(
+              `Stripped ${strip.stripped} base64 images (keeping original links)`
+            );
+          }
+        } else {
+          const outputDir = path.dirname(output);
+          const extracted = extractAndSaveImages(markdown, outputDir, {
+            imagesDir: options.imagesDir || 'images',
+          });
+          if (extracted.extracted > 0) {
+            markdown = extracted.markdown;
+            console.error(
+              `Extracted ${extracted.extracted} images to ${options.imagesDir || 'images'}/`
+            );
+          }
+        }
       }
 
       if (output) {
+        fs.mkdirSync(path.dirname(output), { recursive: true });
         fs.writeFileSync(output, markdown, 'utf-8');
         console.error(`Markdown saved to: ${output}`);
       } else {
@@ -571,9 +749,9 @@ async function captureUrl(url, options) {
 
         const buffer = await page.screenshot({ type: 'png' });
 
-        if (output) {
-          fs.writeFileSync(output, buffer);
-          console.error(`Screenshot saved to: ${output}`);
+        if (explicitOutput) {
+          fs.writeFileSync(explicitOutput, buffer);
+          console.error(`Screenshot saved to: ${explicitOutput}`);
         } else {
           // Generate default filename based on URL
           const urlObj = new URL(absoluteUrl);
@@ -627,7 +805,12 @@ async function captureUrl(url, options) {
         resultHtml = convertRelativeUrls(utf8Html, absoluteUrl);
       }
 
+      const output =
+        explicitOutput === '-'
+          ? null
+          : explicitOutput || deriveOutputPath(absoluteUrl, 'html', dataDir);
       if (output) {
+        fs.mkdirSync(path.dirname(output), { recursive: true });
         fs.writeFileSync(output, resultHtml, 'utf-8');
         console.error(`HTML saved to: ${output}`);
       } else {
@@ -648,6 +831,26 @@ async function main() {
     // Server mode
     await startServer(config.port);
   } else if (url) {
+    // --no-extract-images is an alias for --embed-images
+    if (config.noExtractImages) {
+      config.embedImages = true;
+    }
+    // --archive flag overrides format
+    if (config.archive !== undefined) {
+      const archiveFormat =
+        config.archive === true || config.archive === ''
+          ? 'zip'
+          : config.archive;
+      const validFormats = ['zip', '7z', 'tar.gz', 'gz', 'tar'];
+      if (!validFormats.includes(archiveFormat)) {
+        console.error(
+          `Error: Unsupported archive format "${archiveFormat}". Supported: ${validFormats.join(', ')}`
+        );
+        process.exit(1);
+      }
+      config.archiveFormat = archiveFormat === 'gz' ? 'tar.gz' : archiveFormat;
+      config.format = 'archive';
+    }
     // Capture mode
     await captureUrl(url, {
       format: config.format,
@@ -660,13 +863,18 @@ async function main() {
       fullPage: config.fullPage,
       localImages: config.localImages,
       documentFormat: config.documentFormat,
-      enhanced: config.enhanced,
       extractLatex: config.extractLatex,
       extractMetadata: config.extractMetadata,
       postProcess: config.postProcess,
       detectCodeLanguage: config.detectCodeLanguage,
       dualTheme: config.dualTheme,
       apiToken: config.apiToken,
+      embedImages: config.embedImages,
+      imagesDir: config.imagesDir,
+      dataDir: config.dataDir,
+      archiveFormat: config.archiveFormat,
+      keepOriginalLinks: config.keepOriginalLinks,
+      prettyHtml: !config.noPrettyHtml,
     });
   } else {
     // No arguments - show error

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@link-assistant/web-capture",
-  "version": "1.4.2",
+  "version": "1.7.2",
   "description": "CLI and microservice to render web pages as HTML, Markdown, or PNG",
   "main": "src/index.js",
   "bin": {
@@ -73,6 +73,10 @@
     "microservice"
   ],
   "license": "Unlicense",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/link-assistant/web-capture"
+  },
   "engines": {
     "node": ">=22.0.0 <23.0.0"
   },

package/src/archive.js

@@ -2,7 +2,7 @@
  * ZIP archive handler.
  *
  * Downloads a web page as a self-contained ZIP archive containing:
- * - article.md or article.html  (document with either remote or local asset links)
+ * - document.md or document.html  (document with either remote or local asset links)
  * - images/     (directory of downloaded images, when localImages=true)
  * - css/        (directory for stylesheets, when documentFormat=html and localImages=true)
  * - js/         (directory for scripts, when documentFormat=html and localImages=true)
@@ -23,8 +23,10 @@ import {
   fetchHtml,
   convertHtmlToMarkdown,
   convertRelativeUrls,
+  prettyPrintHtml,
 } from './lib.js';
 import { retry } from './retry.js';
+import { extractBase64ToBuffers } from './extract-images.js';
 
 export async function archiveHandler(req, res) {
   const url = req.query.url;
@@ -32,7 +34,11 @@ export async function archiveHandler(req, res) {
     return res.status(400).send('Missing `url` parameter');
   }
 
-  const localImages = req.query.localImages !== 'false'; // default true
+  const keepOriginalLinks = req.query.keepOriginalLinks === 'true';
+  const localImages = keepOriginalLinks
+    ? false
+    : req.query.localImages !== 'false';
+  const embedImages = req.query.embedImages === 'true';
   const documentFormat =
     req.query.documentFormat === 'html' ? 'html' : 'markdown';
 
@@ -135,8 +141,8 @@ export async function archiveHandler(req, res) {
         });
       }
 
-      outputHtml = $out.html();
-      archive.append(outputHtml, { name: 'article.html' });
+      outputHtml = prettyPrintHtml($out.html());
+      archive.append(outputHtml, { name: 'document.html' });
 
       // Download and add CSS files
       for (const { url: cssUrl, localPath } of cssFiles) {
@@ -158,13 +164,20 @@ export async function archiveHandler(req, res) {
       let markdown = convertHtmlToMarkdown(html, absoluteUrl);
 
       if (localImages && imageMap.size > 0) {
-        // Rewrite image URLs in markdown to local paths
         for (const [remoteUrl, localPath] of imageMap) {
           markdown = markdown.split(remoteUrl).join(localPath);
         }
       }
 
-      archive.append(markdown, { name: 'article.md' });
+      if (!embedImages && !keepOriginalLinks) {
+        const b64 = extractBase64ToBuffers(markdown, 'images');
+        markdown = b64.markdown;
+        for (const img of b64.images) {
+          archive.append(img.buffer, { name: `images/${img.filename}` });
+        }
+      }
+
+      archive.append(markdown, { name: 'document.md' });
     }
 
     // Download and add images if local mode

package/src/batch.js

@@ -139,9 +139,9 @@ export function createConfigFromUrls(urls, defaults = {}) {
       url,
       title: `Article ${id}`,
       archivePath: `archive/${hostname.replace(/\./g, '-')}/${id}`,
-      markdownFile: 'article.md',
-      screenshotLightFile: 'article-light.png',
-      screenshotDarkFile: 'article-dark.png',
+      markdownFile: 'document.md',
+      screenshotLightFile: 'document-light.png',
+      screenshotDarkFile: 'document-dark.png',
       imagesDir: 'images',
       hasLocalImages: true,
     };