@link-assistant/web-capture 1.7.7 → 1.7.8

package/CHANGELOG.md

@@ -1,5 +1,12 @@
 # @link-assistant/web-capture
 
+## 1.7.8
+
+### Patch Changes
+
+- e951b65: Fix JavaScript CLI version output when run from another npm project.
+- aa4888d: Add content selector options for article-only Markdown capture while preserving full-page metadata extraction.
+
 ## 1.7.7
 
 ### Patch Changes

package/bin/web-capture.js

@@ -9,6 +9,7 @@ import path from 'path';
 import { URL } from 'node:url';
 import { makeConfig } from 'lino-arguments';
 import makeLog from 'log-lazy';
+import packageJson from '../package.json' with { type: 'json' };
 
 function makeVerboseLog(enabled) {
   return makeLog({
@@ -138,6 +139,18 @@ const config = makeConfig({
           'Detect and correct code block languages (default: true). Use --no-detect-code-language to disable.',
         default: getenv('WEB_CAPTURE_DETECT_CODE_LANGUAGE', true),
       })
+      .option('contentSelector', {
+        type: 'string',
+        description:
+          'CSS selector used to scope markdown conversion while preserving full-page metadata extraction.',
+        default: getenv('WEB_CAPTURE_CONTENT_SELECTOR', undefined),
+      })
+      .option('bodySelector', {
+        type: 'string',
+        description:
+          'CSS selector for article body markdown; prepends the selected article title when available.',
+        default: getenv('WEB_CAPTURE_BODY_SELECTOR', undefined),
+      })
       .option('embedImages', {
         type: 'boolean',
         description:
@@ -213,7 +226,7 @@ const config = makeConfig({
       })
       .help('help')
       .alias('help', 'h')
-      .version()
+      .version(packageJson.version)
       .alias('version', 'v')
       .example('web-capture --serve', 'Start API server on port 3000')
       .example(
@@ -956,6 +969,8 @@ async function captureUrl(url, options) {
         extractMetadata: options.extractMetadata,
         postProcess: options.postProcess,
         detectCodeLanguage: options.detectCodeLanguage,
+        contentSelector: options.contentSelector,
+        bodySelector: options.bodySelector,
       });
       let markdown = result.markdown;
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@link-assistant/web-capture",
-  "version": "1.7.7",
+  "version": "1.7.8",
   "description": "CLI and microservice to render web pages as HTML, Markdown, or PNG",
   "main": "src/index.js",
   "bin": {

package/src/lib.js

@@ -171,6 +171,14 @@ export function convertHtmlToMarkdown(html, baseUrl) {
   return he.decode(turndown.turndown($.html())).replace(/\u00A0/g, ' ');
 }
 
+function selectedHtml($, selector) {
+  if (!selector) {
+    return null;
+  }
+  const $selected = $(selector).first();
+  return $selected.length ? $.html($selected) : null;
+}
+
 // Convert relative URLs to absolute URLs in HTML content
 export function convertRelativeUrls(html, baseUrl) {
   const base = new URL(baseUrl);
@@ -313,6 +321,8 @@ export function convertRelativeUrls(html, baseUrl) {
  * @param {boolean} [options.postProcess=true] - Apply post-processing pipeline
  * @param {boolean} [options.detectCodeLanguage=true] - Detect/correct code languages
  * @param {boolean} [options.preserveCodeWhitespace=false] - Keep original whitespace inside code blocks
+ * @param {string} [options.contentSelector] - CSS selector to scope Markdown conversion
+ * @param {string} [options.bodySelector] - CSS selector appended after the selected article title
  * @returns {Object} Result with { markdown, metadata }
  */
 export function convertHtmlToMarkdownEnhanced(html, baseUrl, options = {}) {
@@ -322,6 +332,8 @@ export function convertHtmlToMarkdownEnhanced(html, baseUrl, options = {}) {
     postProcess = true,
     detectCodeLanguage = true,
     preserveCodeWhitespace = false,
+    contentSelector,
+    bodySelector,
   } = options;
 
   // Ensure all URLs are absolute before Markdown conversion
@@ -329,7 +341,7 @@ export function convertHtmlToMarkdownEnhanced(html, baseUrl, options = {}) {
     html = convertRelativeUrls(html, baseUrl);
   }
 
-  const $ = cheerio.load(html);
+  let $ = cheerio.load(html);
 
   // Extract metadata before cleaning
   let metadata = null;
@@ -337,6 +349,16 @@ export function convertHtmlToMarkdownEnhanced(html, baseUrl, options = {}) {
     metadata = extractMetadata($);
   }
 
+  const bodyHtml = selectedHtml($, bodySelector);
+  const contentHtml = selectedHtml($, contentSelector);
+  if (bodyHtml || contentHtml) {
+    const titleSelector = contentSelector ? `${contentSelector} h1, h1` : 'h1';
+    const titleHtml = bodyHtml ? selectedHtml($, titleSelector) : null;
+    $ = cheerio.load(
+      [titleHtml, bodyHtml || contentHtml].filter(Boolean).join('\n')
+    );
+  }
+
   // Remove unwanted elements
   $('style, script, noscript').remove();
   $('*').each(function () {

package/src/markdown.js

@@ -1,4 +1,4 @@
-import { fetchHtml, convertHtmlToMarkdown } from './lib.js';
+import { fetchHtml, convertHtmlToMarkdownEnhanced } from './lib.js';
 import { stripBase64Images } from './extract-images.js';
 
 export async function markdownHandler(req, res) {
@@ -9,7 +9,10 @@ export async function markdownHandler(req, res) {
   const embedImages = req.query.embedImages === 'true';
   try {
     const html = await fetchHtml(url);
-    let markdown = convertHtmlToMarkdown(html, url);
+    let { markdown } = convertHtmlToMarkdownEnhanced(html, url, {
+      contentSelector: req.query.contentSelector,
+      bodySelector: req.query.bodySelector,
+    });
     if (!embedImages) {
       const strip = stripBase64Images(markdown);
       markdown = strip.markdown;

package/tests/unit/cli.test.js

@@ -1,8 +1,10 @@
 // Unit tests for CLI argument parsing and functionality
 import { spawn } from 'child_process';
 import http from 'node:http';
+import { mkdtempSync, rmSync, writeFileSync } from 'fs';
 import { fileURLToPath } from 'url';
-import { dirname, resolve } from 'path';
+import { dirname, join, resolve } from 'path';
+import packageJson from '../../package.json' with { type: 'json' };
 
 const __filename = fileURLToPath(import.meta.url);
 const __dirname = dirname(__filename);
@@ -108,6 +110,26 @@ describe('CLI', () => {
       // yargs outputs just the version number
       expect(result.stdout).toMatch(/\d+\.\d+\.\d+/);
     });
+
+    test('shows web-capture version when run from another npm project', async () => {
+      const callerProjectDir = mkdtempSync(join(process.cwd(), 'caller-'));
+      writeFileSync(
+        join(callerProjectDir, 'package.json'),
+        JSON.stringify({
+          name: 'caller-project',
+          version: '1.0.0',
+        })
+      );
+
+      try {
+        const result = await runCli(['--version'], { cwd: callerProjectDir });
+
+        expect(result.code).toBe(0);
+        expect(result.stdout.trim()).toBe(packageJson.version);
+      } finally {
+        rmSync(callerProjectDir, { recursive: true, force: true });
+      }
+    });
   });
 
   describe('URL validation', () => {

package/tests/unit/html2md.test.js

@@ -612,6 +612,47 @@ describe('convertHtmlToMarkdown', () => {
   });
 });
 
+describe('convertHtmlToMarkdownEnhanced content selectors', () => {
+  it('scopes Habr-shaped article markdown while keeping full-page metadata', () => {
+    const html = `
+      <html>
+        <head><meta name="keywords" content="links, theory"></head>
+        <body>
+          <nav><a href="/en/feed">Habr</a><a href="/en/search">Search</a></nav>
+          <a href="/en/sandbox/start/">Write a publication</a>
+          <article>
+            <header>
+              <h1>The Links Theory 0.0.2</h1>
+              <a class="tm-user-info__username" href="/users/links">links</a>
+              <time datetime="2026-04-01T00:00:00Z">April 1</time>
+            </header>
+            <div class="article-formatted-body">
+              <p>Last April 1st, as you might have guessed, the project shipped.</p>
+            </div>
+          </article>
+        </body>
+      </html>
+    `;
+
+    const result = convertHtmlToMarkdownEnhanced(
+      html,
+      'https://habr.com/en/articles/895896/',
+      {
+        contentSelector: 'article',
+        bodySelector: '.article-formatted-body',
+      }
+    );
+
+    expect(result.markdown.trim()).toMatch(/^# The Links Theory 0\.0\.2/);
+    expect(result.markdown).toContain('Last April 1st');
+    expect(result.markdown).not.toContain('Habr');
+    expect(result.markdown).not.toContain('Search');
+    expect(result.markdown).not.toContain('Write a publication');
+    expect(result.metadata.author).toBe('links');
+    expect(result.metadata.tags).toEqual(['links', 'theory']);
+  });
+});
+
 describe('convertHtmlToMarkdownEnhanced Habr compatibility', () => {
   it('preserves article code line breaks and extracts formulas from a Habr fixture', () => {
     const html = `