@link-assistant/web-capture 1.7.6 → 1.7.8

package/CHANGELOG.md

@@ -1,5 +1,18 @@
 # @link-assistant/web-capture
 
+## 1.7.8
+
+### Patch Changes
+
+- e951b65: Fix JavaScript CLI version output when run from another npm project.
+- aa4888d: Add content selector options for article-only Markdown capture while preserving full-page metadata extraction.
+
+## 1.7.7
+
+### Patch Changes
+
+- 9034d01: Add Habr archive-compatible markdown controls for figure-numbered image localization and preserved code block whitespace.
+
 ## 1.7.6
 
 ### Patch Changes

package/bin/web-capture.js

@@ -9,6 +9,7 @@ import path from 'path';
 import { URL } from 'node:url';
 import { makeConfig } from 'lino-arguments';
 import makeLog from 'log-lazy';
+import packageJson from '../package.json' with { type: 'json' };
 
 function makeVerboseLog(enabled) {
   return makeLog({
@@ -138,6 +139,18 @@ const config = makeConfig({
           'Detect and correct code block languages (default: true). Use --no-detect-code-language to disable.',
         default: getenv('WEB_CAPTURE_DETECT_CODE_LANGUAGE', true),
       })
+      .option('contentSelector', {
+        type: 'string',
+        description:
+          'CSS selector used to scope markdown conversion while preserving full-page metadata extraction.',
+        default: getenv('WEB_CAPTURE_CONTENT_SELECTOR', undefined),
+      })
+      .option('bodySelector', {
+        type: 'string',
+        description:
+          'CSS selector for article body markdown; prepends the selected article title when available.',
+        default: getenv('WEB_CAPTURE_BODY_SELECTOR', undefined),
+      })
       .option('embedImages', {
         type: 'boolean',
         description:
@@ -213,7 +226,7 @@ const config = makeConfig({
       })
       .help('help')
       .alias('help', 'h')
-      .version()
+      .version(packageJson.version)
       .alias('version', 'v')
       .example('web-capture --serve', 'Start API server on port 3000')
       .example(
@@ -956,6 +969,8 @@ async function captureUrl(url, options) {
         extractMetadata: options.extractMetadata,
         postProcess: options.postProcess,
         detectCodeLanguage: options.detectCodeLanguage,
+        contentSelector: options.contentSelector,
+        bodySelector: options.bodySelector,
       });
       let markdown = result.markdown;
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@link-assistant/web-capture",
-  "version": "1.7.6",
+  "version": "1.7.8",
   "description": "CLI and microservice to render web pages as HTML, Markdown, or PNG",
   "main": "src/index.js",
   "bin": {

package/src/lib.js

@@ -171,6 +171,14 @@ export function convertHtmlToMarkdown(html, baseUrl) {
   return he.decode(turndown.turndown($.html())).replace(/\u00A0/g, ' ');
 }
 
+function selectedHtml($, selector) {
+  if (!selector) {
+    return null;
+  }
+  const $selected = $(selector).first();
+  return $selected.length ? $.html($selected) : null;
+}
+
 // Convert relative URLs to absolute URLs in HTML content
 export function convertRelativeUrls(html, baseUrl) {
   const base = new URL(baseUrl);
@@ -312,6 +320,9 @@ export function convertRelativeUrls(html, baseUrl) {
  * @param {boolean} [options.extractMetadata=true] - Extract article metadata
  * @param {boolean} [options.postProcess=true] - Apply post-processing pipeline
  * @param {boolean} [options.detectCodeLanguage=true] - Detect/correct code languages
+ * @param {boolean} [options.preserveCodeWhitespace=false] - Keep original whitespace inside code blocks
+ * @param {string} [options.contentSelector] - CSS selector to scope Markdown conversion
+ * @param {string} [options.bodySelector] - CSS selector appended after the selected article title
  * @returns {Object} Result with { markdown, metadata }
  */
 export function convertHtmlToMarkdownEnhanced(html, baseUrl, options = {}) {
@@ -320,6 +331,9 @@ export function convertHtmlToMarkdownEnhanced(html, baseUrl, options = {}) {
     extractMetadata: shouldExtractMetadata = true,
     postProcess = true,
     detectCodeLanguage = true,
+    preserveCodeWhitespace = false,
+    contentSelector,
+    bodySelector,
   } = options;
 
   // Ensure all URLs are absolute before Markdown conversion
@@ -327,7 +341,7 @@ export function convertHtmlToMarkdownEnhanced(html, baseUrl, options = {}) {
     html = convertRelativeUrls(html, baseUrl);
   }
 
-  const $ = cheerio.load(html);
+  let $ = cheerio.load(html);
 
   // Extract metadata before cleaning
   let metadata = null;
@@ -335,6 +349,16 @@ export function convertHtmlToMarkdownEnhanced(html, baseUrl, options = {}) {
     metadata = extractMetadata($);
   }
 
+  const bodyHtml = selectedHtml($, bodySelector);
+  const contentHtml = selectedHtml($, contentSelector);
+  if (bodyHtml || contentHtml) {
+    const titleSelector = contentSelector ? `${contentSelector} h1, h1` : 'h1';
+    const titleHtml = bodyHtml ? selectedHtml($, titleSelector) : null;
+    $ = cheerio.load(
+      [titleHtml, bodyHtml || contentHtml].filter(Boolean).join('\n')
+    );
+  }
+
   // Remove unwanted elements
   $('style, script, noscript').remove();
   $('*').each(function () {
@@ -422,7 +446,7 @@ export function convertHtmlToMarkdownEnhanced(html, baseUrl, options = {}) {
   // Handle code language detection/correction
   if (detectCodeLanguage) {
     $('pre code').each(function () {
-      const codeText = $(this).text().trim();
+      const codeText = $(this).text();
       const language =
         $(this)
           .attr('class')
@@ -444,6 +468,12 @@ export function convertHtmlToMarkdownEnhanced(html, baseUrl, options = {}) {
     });
   }
 
+  if (preserveCodeWhitespace) {
+    $('pre code').each(function () {
+      $(this).text($(this).text().replace(/\r\n?/g, '\n'));
+    });
+  }
+
   // Preprocess ARIA tables (same as original)
   $('[role="table"]').each(function () {
     const $table = $(this);

package/src/localize-images.js

@@ -24,7 +24,8 @@ import { retry } from './retry.js';
  * @returns {Object[]} Array of {fullMatch, altText, url}
  */
 export function extractImageReferences(markdownText) {
-  const imageRegex = /!\[([^\]]*)\]\((https?:\/\/[^)]+)\)/g;
+  const imageRegex =
+    /!\[([^\]]*)\]\((https?:\/\/[^\s)]+)(?:\s+["'][^"']*["'])?\)/g;
   const images = [];
   let match;
 
@@ -73,6 +74,18 @@ export function generateLocalFilename(url, index) {
   return `image-${String(index + 1).padStart(2, '0')}${ext}`;
 }
 
+/**
+ * Generate a stable figure-numbered filename.
+ *
+ * @param {string} url - Image URL
+ * @param {number} index - Zero-based index
+ * @returns {string} Local filename (e.g., 'figure-1.png')
+ */
+export function generateFigureFilename(url, index) {
+  const ext = getExtensionFromUrl(url);
+  return `figure-${index + 1}${ext}`;
+}
+
 /**
  * Localize images in markdown text by downloading external images
  * and replacing URLs with local paths.
@@ -83,6 +96,7 @@ export function generateLocalFilename(url, index) {
  * @param {boolean} [options.dryRun=false] - Only report what would be done
  * @param {Function} [options.onProgress] - Callback(index, total, status, url)
  * @param {string[]} [options.excludeDomains] - Domains to skip (already local)
+ * @param {Function} [options.filenameGenerator] - Callback(url, index, image) for local filenames
  * @returns {Promise<Object>} Result with updated markdown and metadata
  */
 export async function localizeImages(markdownText, options = {}) {
@@ -91,6 +105,7 @@ export async function localizeImages(markdownText, options = {}) {
     dryRun = false,
     onProgress,
     excludeDomains = [],
+    filenameGenerator = generateLocalFilename,
   } = options;
 
   const allImages = extractImageReferences(markdownText);
@@ -128,7 +143,7 @@ export async function localizeImages(markdownText, options = {}) {
 
   for (let i = 0; i < externalImages.length; i++) {
     const image = externalImages[i];
-    const localFilename = generateLocalFilename(image.url, i);
+    const localFilename = filenameGenerator(image.url, i, image);
     const relativePath = `${imagesDir}/${localFilename}`;
 
     if (onProgress) {

package/src/markdown.js

@@ -1,4 +1,4 @@
-import { fetchHtml, convertHtmlToMarkdown } from './lib.js';
+import { fetchHtml, convertHtmlToMarkdownEnhanced } from './lib.js';
 import { stripBase64Images } from './extract-images.js';
 
 export async function markdownHandler(req, res) {
@@ -9,7 +9,10 @@ export async function markdownHandler(req, res) {
   const embedImages = req.query.embedImages === 'true';
   try {
     const html = await fetchHtml(url);
-    let markdown = convertHtmlToMarkdown(html, url);
+    let { markdown } = convertHtmlToMarkdownEnhanced(html, url, {
+      contentSelector: req.query.contentSelector,
+      bodySelector: req.query.bodySelector,
+    });
     if (!embedImages) {
       const strip = stripBase64Images(markdown);
       markdown = strip.markdown;

package/tests/unit/cli.test.js

@@ -1,8 +1,10 @@
 // Unit tests for CLI argument parsing and functionality
 import { spawn } from 'child_process';
 import http from 'node:http';
+import { mkdtempSync, rmSync, writeFileSync } from 'fs';
 import { fileURLToPath } from 'url';
-import { dirname, resolve } from 'path';
+import { dirname, join, resolve } from 'path';
+import packageJson from '../../package.json' with { type: 'json' };
 
 const __filename = fileURLToPath(import.meta.url);
 const __dirname = dirname(__filename);
@@ -108,6 +110,26 @@ describe('CLI', () => {
       // yargs outputs just the version number
       expect(result.stdout).toMatch(/\d+\.\d+\.\d+/);
     });
+
+    test('shows web-capture version when run from another npm project', async () => {
+      const callerProjectDir = mkdtempSync(join(process.cwd(), 'caller-'));
+      writeFileSync(
+        join(callerProjectDir, 'package.json'),
+        JSON.stringify({
+          name: 'caller-project',
+          version: '1.0.0',
+        })
+      );
+
+      try {
+        const result = await runCli(['--version'], { cwd: callerProjectDir });
+
+        expect(result.code).toBe(0);
+        expect(result.stdout.trim()).toBe(packageJson.version);
+      } finally {
+        rmSync(callerProjectDir, { recursive: true, force: true });
+      }
+    });
   });
 
   describe('URL validation', () => {

package/tests/unit/html2md.test.js

@@ -1,4 +1,7 @@
-import { convertHtmlToMarkdown } from '../../src/lib.js';
+import {
+  convertHtmlToMarkdown,
+  convertHtmlToMarkdownEnhanced,
+} from '../../src/lib.js';
 
 describe('convertHtmlToMarkdown', () => {
   it('removes empty links and anchor headings', () => {
@@ -608,3 +611,85 @@ describe('convertHtmlToMarkdown', () => {
     );
   });
 });
+
+describe('convertHtmlToMarkdownEnhanced content selectors', () => {
+  it('scopes Habr-shaped article markdown while keeping full-page metadata', () => {
+    const html = `
+      <html>
+        <head><meta name="keywords" content="links, theory"></head>
+        <body>
+          <nav><a href="/en/feed">Habr</a><a href="/en/search">Search</a></nav>
+          <a href="/en/sandbox/start/">Write a publication</a>
+          <article>
+            <header>
+              <h1>The Links Theory 0.0.2</h1>
+              <a class="tm-user-info__username" href="/users/links">links</a>
+              <time datetime="2026-04-01T00:00:00Z">April 1</time>
+            </header>
+            <div class="article-formatted-body">
+              <p>Last April 1st, as you might have guessed, the project shipped.</p>
+            </div>
+          </article>
+        </body>
+      </html>
+    `;
+
+    const result = convertHtmlToMarkdownEnhanced(
+      html,
+      'https://habr.com/en/articles/895896/',
+      {
+        contentSelector: 'article',
+        bodySelector: '.article-formatted-body',
+      }
+    );
+
+    expect(result.markdown.trim()).toMatch(/^# The Links Theory 0\.0\.2/);
+    expect(result.markdown).toContain('Last April 1st');
+    expect(result.markdown).not.toContain('Habr');
+    expect(result.markdown).not.toContain('Search');
+    expect(result.markdown).not.toContain('Write a publication');
+    expect(result.metadata.author).toBe('links');
+    expect(result.metadata.tags).toEqual(['links', 'theory']);
+  });
+});
+
+describe('convertHtmlToMarkdownEnhanced Habr compatibility', () => {
+  it('preserves article code line breaks and extracts formulas from a Habr fixture', () => {
+    const html = `
+      <article>
+        <h1>The Links Theory 0.0.2</h1>
+        <div class="article-formatted-body">
+          <p>A formula <img class="formula" source="\\mathbf{L}: L \\to L^2" alt="formula"> remains inline.</p>
+          <figure>
+            <img src="https://habrastorage.org/r/w1560/getpro/habr/upload_files/abc.png" alt="Figure 1">
+            <figcaption>Figure 1. Link multiplication.</figcaption>
+          </figure>
+          <pre><code>L = { 1 , 2 }
+L x L = {
+  (1, 1),
+  (1, 2),
+}</code></pre>
+        </div>
+      </article>
+    `;
+
+    const { markdown } = convertHtmlToMarkdownEnhanced(
+      html,
+      'https://habr.com/en/articles/895896/',
+      { preserveCodeWhitespace: true }
+    );
+
+    expect(markdown).toContain('$\\\\mathbf{L}: L \\\\to L^2$');
+    expect(markdown).toContain(
+      '![Figure 1](https://habrastorage.org/r/w1560/getpro/habr/upload_files/abc.png)'
+    );
+    expect(markdown).toContain(`\`\`\`
+L = { 1 , 2 }
+L x L = {
+  (1, 1),
+  (1, 2),
+}
+\`\`\``);
+    expect(markdown).not.toContain('L = { 1 , 2 }L x L');
+  });
+});

package/tests/unit/localize-images.test.js

@@ -2,6 +2,8 @@ import {
   extractImageReferences,
   getExtensionFromUrl,
   generateLocalFilename,
+  generateFigureFilename,
+  localizeImages,
 } from '../../src/localize-images.js';
 
 describe('localize-images module', () => {
@@ -75,4 +77,38 @@ describe('localize-images module', () => {
       );
     });
   });
+
+  describe('generateFigureFilename', () => {
+    it('generates figure-numbered filenames while preserving extensions', () => {
+      expect(
+        generateFigureFilename('https://habrastorage.org/getpro/habr/a.png', 0)
+      ).toBe('figure-1.png');
+      expect(
+        generateFigureFilename('https://habrastorage.org/getpro/habr/a.jpg', 9)
+      ).toBe('figure-10.jpg');
+    });
+  });
+
+  describe('localizeImages', () => {
+    it('accepts a custom filename generator for archive-compatible paths', async () => {
+      const markdown =
+        '![Figure 1](https://habrastorage.org/r/w1560/getpro/habr/figure.png "Figure 1")';
+
+      const result = await localizeImages(markdown, {
+        dryRun: true,
+        filenameGenerator: generateFigureFilename,
+      });
+
+      expect(result.markdown).toBe('![Figure 1](images/figure-1.png)');
+      expect(result.metadata).toEqual([
+        {
+          index: 1,
+          originalUrl:
+            'https://habrastorage.org/r/w1560/getpro/habr/figure.png',
+          altText: 'Figure 1',
+          localPath: 'images/figure-1.png',
+        },
+      ]);
+    });
+  });
 });