@link-assistant/web-capture 1.1.2 → 1.4.2

package/bin/web-capture.js

@@ -32,9 +32,47 @@ const config = makeConfig({
       .option('format', {
         alias: 'f',
         type: 'string',
-        description: 'Output format: html, markdown, md, image, png',
+        description:
+          'Output format: html, markdown, md, image, png, jpeg, pdf, docx, archive',
         default: 'html',
       })
+      .option('theme', {
+        alias: 't',
+        type: 'string',
+        description:
+          'Color scheme for screenshots/PDF: light, dark, no-preference',
+      })
+      .option('width', {
+        type: 'number',
+        description: 'Viewport width in pixels (default: 1280)',
+        default: 1280,
+      })
+      .option('height', {
+        type: 'number',
+        description: 'Viewport height in pixels (default: 800)',
+        default: 800,
+      })
+      .option('quality', {
+        type: 'number',
+        description: 'JPEG quality 0-100 (default: 80, only for jpeg format)',
+        default: 80,
+      })
+      .option('fullPage', {
+        type: 'boolean',
+        description: 'Capture full scrollable page (default: false)',
+        default: false,
+      })
+      .option('localImages', {
+        type: 'boolean',
+        description: 'Download images locally in archive mode (default: true)',
+        default: true,
+      })
+      .option('documentFormat', {
+        type: 'string',
+        description:
+          'Document format in archive mode: markdown (default) or html',
+        default: 'markdown',
+      })
       .option('output', {
         alias: 'o',
         type: 'string',
@@ -51,6 +89,67 @@ const config = makeConfig({
         type: 'string',
         description: 'Path to .lenv configuration file',
       })
+      .option('enhanced', {
+        type: 'boolean',
+        description:
+          'Use enhanced markdown conversion with LaTeX extraction, metadata, and post-processing (default: false)',
+        default: false,
+      })
+      .option('extractLatex', {
+        type: 'boolean',
+        description:
+          'Extract LaTeX formulas from img.formula, KaTeX, MathJax (default: true when --enhanced)',
+        default: true,
+      })
+      .option('extractMetadata', {
+        type: 'boolean',
+        description:
+          'Extract article metadata (author, date, hubs, tags) (default: true when --enhanced)',
+        default: true,
+      })
+      .option('postProcess', {
+        type: 'boolean',
+        description:
+          'Apply post-processing (unicode normalization, LaTeX spacing) (default: true when --enhanced)',
+        default: true,
+      })
+      .option('detectCodeLanguage', {
+        type: 'boolean',
+        description:
+          'Detect and correct code block languages (default: true when --enhanced)',
+        default: true,
+      })
+      .option('dualTheme', {
+        type: 'boolean',
+        description:
+          'Capture both light and dark theme screenshots (default: false)',
+        default: false,
+      })
+      .option('configFile', {
+        type: 'string',
+        description: 'Path to batch configuration file (JSON or MJS)',
+      })
+      .option('all', {
+        type: 'boolean',
+        description: 'Process all articles in batch configuration',
+        default: false,
+      })
+      .option('dryRun', {
+        type: 'boolean',
+        description: 'Show what would be done without making changes',
+        default: false,
+      })
+      .option('verbose', {
+        type: 'boolean',
+        description: 'Show detailed output',
+        default: false,
+      })
+      .option('apiToken', {
+        type: 'string',
+        description:
+          'API token for authenticated capture (e.g., Google Docs private documents). Can also be set via API_TOKEN env variable.',
+        default: getenv('API_TOKEN', undefined),
+      })
       .help('help')
       .alias('help', 'h')
       .version()
@@ -68,12 +167,26 @@ const config = makeConfig({
         'web-capture https://example.com --format markdown --output page.md',
         'Capture URL as Markdown to file'
       )
+      .option('capture', {
+        type: 'string',
+        description:
+          'Capture method: browser (default, uses Puppeteer/Playwright) or api (direct HTTP fetch, for Google Docs etc.)',
+        default: 'browser',
+      })
       .example(
         'web-capture https://example.com --format png --engine playwright -o screenshot.png',
         'Capture screenshot using Playwright'
       )
+      .example(
+        'web-capture https://docs.google.com/document/d/DOC_ID/edit --format markdown',
+        'Capture Google Doc as Markdown (API-based, public doc)'
+      )
+      .example(
+        'web-capture https://docs.google.com/document/d/DOC_ID/edit --format markdown --apiToken YOUR_TOKEN',
+        'Capture private Google Doc with API token'
+      )
       .epilogue(
-        'API Endpoints (in server mode):\n  GET /html?url=<URL>&engine=<ENGINE>       Get rendered HTML\n  GET /markdown?url=<URL>                   Get Markdown conversion\n  GET /image?url=<URL>&engine=<ENGINE>      Get PNG screenshot\n  GET /fetch?url=<URL>                      Proxy fetch\n  GET /stream?url=<URL>                     Streaming proxy'
+        'API Endpoints (in server mode):\n  GET /html?url=<URL>&engine=<ENGINE>           Get rendered HTML\n  GET /markdown?url=<URL>                       Get Markdown conversion\n  GET /image?url=<URL>&format=png|jpeg&theme=light|dark  Screenshot\n  GET /archive?url=<URL>&localImages=true&documentFormat=markdown|html  ZIP archive\n  GET /pdf?url=<URL>&theme=light|dark           PDF with embedded images\n  GET /docx?url=<URL>                           DOCX with embedded images\n  GET /fetch?url=<URL>                          Proxy fetch\n  GET /stream?url=<URL>                         Streaming proxy\n  GET /gdocs?url=<URL>&format=markdown|html|txt  Google Docs capture'
       )
       .strict(),
   lenv: {
@@ -93,9 +206,15 @@ async function startServer(port) {
       console.log('Available endpoints:');
       console.log(`  GET /html?url=<URL>       - Render page as HTML`);
       console.log(`  GET /markdown?url=<URL>   - Convert page to Markdown`);
-      console.log(`  GET /image?url=<URL>      - Screenshot page as PNG`);
+      console.log(`  GET /image?url=<URL>      - Screenshot (PNG/JPEG)`);
+      console.log(
+        `  GET /archive?url=<URL>    - ZIP archive with markdown + images`
+      );
+      console.log(`  GET /pdf?url=<URL>        - PDF with embedded images`);
+      console.log(`  GET /docx?url=<URL>       - DOCX with embedded images`);
       console.log(`  GET /fetch?url=<URL>      - Proxy fetch content`);
       console.log(`  GET /stream?url=<URL>     - Stream content`);
+      console.log(`  GET /gdocs?url=<URL>      - Capture Google Docs document`);
       console.log('');
       console.log('Press Ctrl+C to stop the server');
       resolve(server);
@@ -123,7 +242,19 @@ async function startServer(port) {
 }
 
 async function captureUrl(url, options) {
-  const { format, output, engine } = options;
+  const {
+    format,
+    output,
+    engine,
+    theme,
+    width,
+    height,
+    quality,
+    fullPage,
+    // localImages is used by archive format
+    // eslint-disable-next-line no-unused-vars
+    localImages,
+  } = options;
 
   // Ensure URL is absolute
   let absoluteUrl = url;
@@ -147,14 +278,266 @@ async function captureUrl(url, options) {
     convertRelativeUrls,
   } = await import('../src/lib.js');
   const { createBrowser } = await import('../src/browser.js');
+  const { isGoogleDocsUrl, fetchGoogleDoc, fetchGoogleDocAsMarkdown } =
+    await import('../src/gdocs.js');
 
   const normalizedFormat = format.toLowerCase();
 
+  // Auto-detect Google Docs URLs and use API-based capture
+  if (isGoogleDocsUrl(absoluteUrl)) {
+    try {
+      const apiToken = options.apiToken;
+      if (normalizedFormat === 'markdown' || normalizedFormat === 'md') {
+        const result = await fetchGoogleDocAsMarkdown(absoluteUrl, {
+          apiToken,
+        });
+        if (output) {
+          fs.writeFileSync(output, result.markdown, 'utf-8');
+          console.error(`Google Doc Markdown saved to: ${output}`);
+        } else {
+          process.stdout.write(result.markdown);
+        }
+      } else {
+        const gdocsFormat =
+          normalizedFormat === 'png' || normalizedFormat === 'image'
+            ? 'html'
+            : normalizedFormat;
+        const result = await fetchGoogleDoc(absoluteUrl, {
+          format: gdocsFormat,
+          apiToken,
+        });
+        if (output) {
+          fs.writeFileSync(output, result.content, 'utf-8');
+          console.error(`Google Doc (${gdocsFormat}) saved to: ${output}`);
+        } else {
+          process.stdout.write(result.content);
+        }
+      }
+      return;
+    } catch (err) {
+      console.error(`Error capturing Google Doc: ${err.message}`);
+      process.exit(1);
+    }
+  }
+
   try {
-    if (normalizedFormat === 'markdown' || normalizedFormat === 'md') {
+    if (normalizedFormat === 'jpeg') {
+      // JPEG screenshot
+      const { createBrowser } = await import('../src/browser.js');
+      const { dismissPopups, scrollToLoadContent } =
+        await import('../src/popups.js');
+      const browserOpts = theme ? { colorScheme: theme } : {};
+      const browser = await createBrowser(engine, browserOpts);
+      try {
+        const page = await browser.newPage();
+        await page.setViewport({ width, height });
+        await page.goto(absoluteUrl, {
+          waitUntil: 'networkidle0',
+          timeout: 30000,
+        });
+        await new Promise((resolve) => setTimeout(resolve, 5000));
+        if (fullPage) {
+          await scrollToLoadContent(page);
+        }
+        await dismissPopups(page);
+        const buffer = await page.screenshot({
+          type: 'jpeg',
+          quality,
+          fullPage,
+        });
+        const outPath =
+          output ||
+          `${new URL(absoluteUrl).hostname.replace(/\./g, '_')}_${Date.now()}.jpg`;
+        fs.writeFileSync(outPath, buffer);
+        console.error(`JPEG screenshot saved to: ${outPath}`);
+      } finally {
+        await browser.close();
+      }
+    } else if (normalizedFormat === 'pdf') {
+      // PDF export
+      const { createBrowser } = await import('../src/browser.js');
+      const { dismissPopups, scrollToLoadContent } =
+        await import('../src/popups.js');
+      const browserOpts = theme ? { colorScheme: theme } : {};
+      const browser = await createBrowser(engine, browserOpts);
+      try {
+        const page = await browser.newPage();
+        await page.setViewport({ width, height });
+        await page.goto(absoluteUrl, {
+          waitUntil: 'networkidle0',
+          timeout: 30000,
+        });
+        await new Promise((resolve) => setTimeout(resolve, 5000));
+        await scrollToLoadContent(page);
+        await dismissPopups(page);
+        const rawPage = page.rawPage || page;
+        const pdfBuffer = await rawPage.pdf({
+          format: 'A4',
+          printBackground: true,
+          margin: { top: '1cm', right: '1cm', bottom: '1cm', left: '1cm' },
+        });
+        const outPath = output || 'page.pdf';
+        fs.writeFileSync(outPath, pdfBuffer);
+        console.error(`PDF saved to: ${outPath}`);
+      } finally {
+        await browser.close();
+      }
+    } else if (normalizedFormat === 'docx') {
+      // DOCX export – delegate to the docx handler logic
+      const { fetchHtml, convertRelativeUrls } = await import('../src/lib.js');
+      const cheerio = await import('cheerio');
+      // eslint-disable-next-line no-unused-vars
+      const { Document, Packer, Paragraph, TextRun, HeadingLevel } =
+        await import('docx');
+      const html = await fetchHtml(absoluteUrl);
+      const absHtml = convertRelativeUrls(html, absoluteUrl);
+      const $ = cheerio.load(absHtml);
+      $('style, script, noscript, nav, footer, header').remove();
+      const children = [];
+      const titleText =
+        $('h1').first().text().trim() || $('title').text().trim();
+      if (titleText) {
+        children.push(
+          new Paragraph({ text: titleText, heading: HeadingLevel.TITLE })
+        );
+      }
+      const body = $('article').length ? $('article') : $('body');
+      for (const el of body.children().toArray()) {
+        const text = $(el).text().trim();
+        if (text) {
+          children.push(new Paragraph({ text }));
+        }
+      }
+      if (children.length === 0) {
+        children.push(new Paragraph({ text: 'No content extracted.' }));
+      }
+      const doc = new Document({ sections: [{ children }] });
+      const buffer = await Packer.toBuffer(doc);
+      const outPath = output || 'page.docx';
+      fs.writeFileSync(outPath, buffer);
+      console.error(`DOCX saved to: ${outPath}`);
+    } else if (normalizedFormat === 'archive' || normalizedFormat === 'zip') {
+      // ZIP archive
+      const { default: archiver } = await import('archiver');
+      const { fetchHtml, convertHtmlToMarkdown, convertRelativeUrls } =
+        await import('../src/lib.js');
+      const { retry } = await import('../src/retry.js');
+      const cheerio = await import('cheerio');
+      const nodeFetch = await import('node-fetch');
+
+      const html = await retry(() => fetchHtml(absoluteUrl), {
+        retries: 3,
+        baseDelay: 1000,
+        onRetry: (err, attempt) => {
+          console.error(`Retry ${attempt} fetching page: ${err.message}`);
+        },
+      });
+
+      const docFormat = options.documentFormat === 'html' ? 'html' : 'markdown';
+      const outPath =
+        output ||
+        `${new URL(absoluteUrl).hostname.replace(/\./g, '-')}-archive.zip`;
+      const outStream = fs.createWriteStream(outPath);
+      const archive = archiver.default
+        ? archiver.default('zip', { zlib: { level: 9 } })
+        : archiver('zip', { zlib: { level: 9 } });
+      archive.pipe(outStream);
+
+      // Collect images
+      const $ = cheerio.load(html);
+      const images = [];
+      $('img').each(function () {
+        const src = $(this).attr('src');
+        if (src && !src.startsWith('data:') && !src.startsWith('blob:')) {
+          try {
+            images.push(new URL(src, absoluteUrl).href);
+          } catch {
+            /* skip */
+          }
+        }
+      });
+      const uniqueImages = [...new Set(images)];
+
+      // Build image map for local downloads
+      const imageMap = new Map();
+      if (options.localImages && uniqueImages.length > 0) {
+        let idx = 1;
+        for (const imgUrl of uniqueImages) {
+          const ext = imgUrl.match(/\.(jpe?g|gif|webp|svg|png)/i)?.[1] || 'png';
+          imageMap.set(imgUrl, `images/image-${idx}.${ext}`);
+          idx++;
+        }
+      }
+
+      if (docFormat === 'html') {
+        const outputHtml = convertRelativeUrls(html, absoluteUrl);
+        const $out = cheerio.load(outputHtml);
+        $out('script, noscript').remove();
+        if (imageMap.size > 0) {
+          $out('img').each(function () {
+            const src = $out(this).attr('src');
+            if (src) {
+              try {
+                const resolved = new URL(src, absoluteUrl).href;
+                if (imageMap.has(resolved)) {
+                  $out(this).attr('src', imageMap.get(resolved));
+                }
+              } catch {
+                /* skip */
+              }
+            }
+          });
+        }
+        archive.append($out.html(), { name: 'article.html' });
+      } else {
+        let markdown = convertHtmlToMarkdown(html, absoluteUrl);
+        if (imageMap.size > 0) {
+          for (const [remoteUrl, localPath] of imageMap) {
+            markdown = markdown.split(remoteUrl).join(localPath);
+          }
+        }
+        archive.append(markdown, { name: 'article.md' });
+      }
+
+      // Download images
+      if (imageMap.size > 0) {
+        for (const [imgUrl, localPath] of imageMap) {
+          try {
+            const fetchFn = nodeFetch.default || nodeFetch;
+            const resp = await retry(() => fetchFn(imgUrl), {
+              retries: 2,
+              baseDelay: 500,
+            });
+            if (resp.ok) {
+              const buffer = await resp.buffer();
+              archive.append(buffer, { name: localPath });
+            }
+          } catch {
+            /* skip failed image downloads */
+          }
+        }
+      }
+
+      await archive.finalize();
+      await new Promise((resolve) => outStream.on('close', resolve));
+      console.error(`Archive saved to: ${outPath}`);
+    } else if (normalizedFormat === 'markdown' || normalizedFormat === 'md') {
       // Markdown format
       const html = await fetchHtml(absoluteUrl);
-      const markdown = convertHtmlToMarkdown(html, absoluteUrl);
+      let markdown;
+
+      if (options.enhanced) {
+        const { convertHtmlToMarkdownEnhanced } = await import('../src/lib.js');
+        const result = convertHtmlToMarkdownEnhanced(html, absoluteUrl, {
+          extractLatex: options.extractLatex,
+          extractMetadata: options.extractMetadata,
+          postProcess: options.postProcess,
+          detectCodeLanguage: options.detectCodeLanguage,
+        });
+        markdown = result.markdown;
+      } else {
+        markdown = convertHtmlToMarkdown(html, absoluteUrl);
+      }
 
       if (output) {
         fs.writeFileSync(output, markdown, 'utf-8');
@@ -270,6 +653,20 @@ async function main() {
       format: config.format,
       output: config.output,
       engine: config.engine,
+      theme: config.theme,
+      width: config.width,
+      height: config.height,
+      quality: config.quality,
+      fullPage: config.fullPage,
+      localImages: config.localImages,
+      documentFormat: config.documentFormat,
+      enhanced: config.enhanced,
+      extractLatex: config.extractLatex,
+      extractMetadata: config.extractMetadata,
+      postProcess: config.postProcess,
+      detectCodeLanguage: config.detectCodeLanguage,
+      dualTheme: config.dualTheme,
+      apiToken: config.apiToken,
     });
   } else {
     // No arguments - show error

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@link-assistant/web-capture",
-  "version": "1.1.2",
+  "version": "1.4.2",
   "description": "CLI and microservice to render web pages as HTML, Markdown, or PNG",
   "main": "src/index.js",
   "bin": {
@@ -27,23 +27,30 @@
     "check": "npm run lint && npm run format:check && npm run check:duplication",
     "prepare": "husky || true",
     "changeset": "changeset",
-    "changeset:version": "node scripts/changeset-version.mjs",
+    "changeset:version": "node ../scripts/changeset-version.mjs",
     "changeset:publish": "changeset publish",
     "changeset:status": "changeset status --since=origin/main"
   },
   "dependencies": {
+    "archiver": "^7.0.1",
+    "browser-commander": "^0.8.0",
     "cheerio": "^1.0.0",
+    "docx": "^9.6.1",
     "express": "^4.18.2",
-    "lino-arguments": "^0.2.1",
+    "he": "^1.2.0",
+    "iconv-lite": "^0.6.3",
+    "lino-arguments": "^0.3.0",
     "node-fetch": "^2.7.0",
     "playwright": "^1.49.0",
     "puppeteer": "^24.8.2",
+    "puppeteer-core": "^24.40.0",
     "turndown": "^7.1.1"
   },
   "devDependencies": {
     "@babel/core": "^7.24.0",
     "@babel/preset-env": "^7.24.0",
     "@changesets/cli": "^2.29.7",
+    "@types/archiver": "^7.0.0",
     "babel-jest": "^29.7.0",
     "eslint": "^9.38.0",
     "eslint-config-prettier": "^10.1.8",
@@ -65,7 +72,7 @@
     "screenshot",
     "microservice"
   ],
-  "license": "UNLICENSED",
+  "license": "Unlicense",
   "engines": {
     "node": ">=22.0.0 <23.0.0"
   },