@link-assistant/web-capture 1.1.2 → 1.4.2

package/src/verify.js

@@ -0,0 +1,312 @@
+/**
+ * Content verification module (R6).
+ *
+ * Compares captured markdown content against the original web page
+ * to verify completeness and accuracy.
+ *
+ * Checks: title, headings, paragraphs, code blocks, formulas,
+ * blockquote formulas, list items, links, and figure images.
+ *
+ * Based on reference implementation from:
+ * https://github.com/link-foundation/meta-theory/blob/main/scripts/verify.mjs
+ *
+ * @module verify
+ */
+
+/**
+ * Normalize text for comparison.
+ * Removes extra whitespace and normalizes unicode characters,
+ * LaTeX delimiters, and common symbol substitutions.
+ *
+ * @param {string} text - Input text
+ * @returns {string} Normalized text
+ */
+export function normalizeText(text) {
+  return text
+    .trim()
+    .replace(/\s+/g, ' ')
+    .replace(/[\u2000-\u200F\u2028-\u202F]/g, ' ')
+    .replace(/\u00A0/g, ' ')
+    .replace(/[\u2018\u2019]/g, "'")
+    .replace(/[\u201C\u201D]/g, '"')
+    .replace(/[\u00D7]/g, 'x')
+    .replace(/\\times/g, 'x')
+    .replace(/[\u2192\u21A6]/g, '->')
+    .replace(/\\to/g, '->')
+    .replace(/[\u2212]/g, '-')
+    .replace(/\$\$/g, '')
+    .replace(/\$/g, '')
+    .replace(/\\displaystyle\s*/g, '')
+    .replace(/\\text\{([^}]*)\}/g, '$1')
+    .replace(/\\\\%/g, '%')
+    .replace(/\\%/g, '%')
+    .replace(/\\subseteq/g, '\u2286')
+    .replace(/\\mathbb\{n\}_0/gi, '\u2115\u2080')
+    .replace(/\\in/g, '\u2208')
+    .replace(/\\emptyset/g, '\u2205')
+    .replace(/\^2/g, '\u00B2')
+    .replace(/\^n/g, '\u207F')
+    .toLowerCase();
+}
+
+/**
+ * Normalize code for comparison (more lenient than text).
+ *
+ * @param {string} text - Code text
+ * @returns {string} Normalized code
+ */
+export function normalizeCode(text) {
+  return text
+    .trim()
+    .replace(/\s+/g, ' ')
+    .replace(/[\u2000-\u200F\u2028-\u202F]/g, ' ')
+    .replace(/\u00A0/g, ' ')
+    .replace(/[\u00D7]/g, 'x')
+    .replace(/\\times/g, 'x')
+    .replace(/\$\$/g, '')
+    .replace(/\$/g, '')
+    .toLowerCase();
+}
+
+/**
+ * Verify that markdown contains the expected web page content.
+ *
+ * @param {Object} webContent - Extracted web page content
+ * @param {string} webContent.title - Article title
+ * @param {Object[]} webContent.headings - Array of {level, text}
+ * @param {string[]} webContent.paragraphs - Paragraph texts
+ * @param {string[]} webContent.codeBlocks - Code block texts
+ * @param {string[]} webContent.formulas - Formula texts
+ * @param {string[]} webContent.blockquoteFormulas - Formulas in blockquotes
+ * @param {string[]} webContent.listItems - List item texts
+ * @param {Object[]} webContent.links - Array of {text, href}
+ * @param {number[]} webContent.figures - Figure numbers
+ * @param {string} markdownText - The markdown to verify
+ * @param {Object} [options] - Verification options
+ * @param {boolean} [options.verbose=false] - Detailed output
+ * @param {number} [options.expectedFigures] - Expected figure count
+ * @param {boolean} [options.hasLocalImages=false] - Whether images are localized
+ * @returns {Object} Verification result
+ */
+export function verifyMarkdownContent(webContent, markdownText, options = {}) {
+  const { verbose = false, expectedFigures, hasLocalImages = false } = options;
+  const normalizedMarkdown = normalizeText(markdownText);
+  const missing = {
+    title: false,
+    headings: [],
+    paragraphs: [],
+    codeBlocks: [],
+    formulas: [],
+    blockquoteFormulas: [],
+    listItems: [],
+    images: 0,
+  };
+
+  let totalChecks = 0;
+  let passedChecks = 0;
+  const details = [];
+
+  // Check title
+  if (webContent.title) {
+    totalChecks++;
+    const normalizedTitle = normalizeText(webContent.title);
+    if (normalizedMarkdown.includes(normalizedTitle)) {
+      passedChecks++;
+      if (verbose) {
+        details.push({ type: 'title', status: 'pass' });
+      }
+    } else {
+      missing.title = true;
+      if (verbose) {
+        details.push({
+          type: 'title',
+          status: 'fail',
+          text: webContent.title,
+        });
+      }
+    }
+  }
+
+  // Check headings
+  for (const heading of webContent.headings || []) {
+    totalChecks++;
+    const normalized = normalizeText(heading.text);
+    if (normalizedMarkdown.includes(normalized)) {
+      passedChecks++;
+    } else {
+      missing.headings.push(heading.text);
+      if (verbose) {
+        details.push({
+          type: 'heading',
+          status: 'fail',
+          text: heading.text,
+        });
+      }
+    }
+  }
+
+  // Check paragraphs (sample first 5 and last 5)
+  const paragraphs = webContent.paragraphs || [];
+  const paragraphsToCheck = [
+    ...paragraphs.slice(0, 5),
+    ...paragraphs.slice(-5),
+  ];
+
+  for (const paragraph of paragraphsToCheck) {
+    totalChecks++;
+    const normalized = normalizeText(paragraph);
+    const words = normalized.split(' ').filter((w) => w.length > 2);
+    const matchingWords = words.filter((word) =>
+      normalizedMarkdown.includes(word)
+    );
+    const matchRate =
+      words.length > 0 ? matchingWords.length / words.length : 0;
+
+    const substringMatch =
+      normalized.length > 20 &&
+      normalizedMarkdown.includes(
+        normalized.substring(0, Math.min(50, normalized.length))
+      );
+
+    if (matchRate >= 0.6 || substringMatch) {
+      passedChecks++;
+    } else {
+      missing.paragraphs.push(`${paragraph.substring(0, 100)}...`);
+    }
+  }
+
+  // Check code blocks (fuzzy matching)
+  const normalizedMarkdownForCode = normalizeCode(markdownText);
+  for (const code of webContent.codeBlocks || []) {
+    totalChecks++;
+    const normalizedCodeFull = normalizeCode(code);
+
+    const lines = code
+      .split('\n')
+      .map((l) => l.trim())
+      .filter((l) => l.length > 3 && !/^[{}[\](),;]+$/.test(l));
+
+    const matchingLines = lines.filter((line) => {
+      const normalizedLine = normalizeCode(line);
+      return normalizedMarkdownForCode.includes(normalizedLine);
+    });
+
+    const matchRate =
+      lines.length > 0 ? matchingLines.length / lines.length : 1;
+
+    if (
+      matchRate >= 0.6 ||
+      normalizedMarkdownForCode.includes(normalizedCodeFull)
+    ) {
+      passedChecks++;
+    } else {
+      missing.codeBlocks.push(`${code.substring(0, 100)}...`);
+    }
+  }
+
+  // Check list items (sample first 10)
+  const listItemsToCheck = (webContent.listItems || []).slice(0, 10);
+  for (const item of listItemsToCheck) {
+    totalChecks++;
+    const normalized = normalizeText(item);
+    const words = normalized.split(' ').filter((w) => w.length > 2);
+    const matchingWords = words.filter((word) =>
+      normalizedMarkdown.includes(word)
+    );
+    const matchRate =
+      words.length > 0 ? matchingWords.length / words.length : 0;
+
+    const substringMatch =
+      normalized.length > 15 &&
+      normalizedMarkdown.includes(
+        normalized.substring(0, Math.min(40, normalized.length))
+      );
+
+    if (matchRate >= 0.6 || substringMatch) {
+      passedChecks++;
+    } else {
+      missing.listItems.push(`${item.substring(0, 100)}...`);
+    }
+  }
+
+  // Check blockquote formulas
+  if (
+    webContent.blockquoteFormulas &&
+    webContent.blockquoteFormulas.length > 0
+  ) {
+    for (const formula of webContent.blockquoteFormulas) {
+      totalChecks++;
+      const normalizedFormula = formula.replace(/\s+/g, ' ').trim();
+
+      const keyParts = normalizedFormula
+        .replace(/\\mathbf\{([^}]*)\}/g, '$1')
+        .replace(/\\textbf\{([^}]*)\}/g, '$1')
+        .replace(/[{}\\]/g, '')
+        .split(/\s+/)
+        .filter((part) => part.length > 1);
+
+      const blockquoteLines = markdownText.match(/^>.*$/gm) || [];
+      let foundInBlockquote = false;
+
+      for (const line of blockquoteLines) {
+        if (line.includes('$')) {
+          const matchingParts = keyParts.filter((part) =>
+            line.toLowerCase().includes(part.toLowerCase())
+          );
+          if (
+            keyParts.length > 0 &&
+            matchingParts.length >= Math.min(2, keyParts.length)
+          ) {
+            foundInBlockquote = true;
+            break;
+          }
+          if (
+            line.includes(normalizedFormula) ||
+            line.includes(formula) ||
+            (formula.length < 20 && line.includes(formula.replace(/\s/g, '')))
+          ) {
+            foundInBlockquote = true;
+            break;
+          }
+        }
+      }
+
+      if (foundInBlockquote) {
+        passedChecks++;
+      } else {
+        missing.blockquoteFormulas.push(formula.substring(0, 100));
+      }
+    }
+  }
+
+  // Check for figure images
+  if (hasLocalImages && expectedFigures) {
+    const figurePattern =
+      /!\[(?:\*\*)?(?:Figure|Рис\.?|Рисунок)\s*\d+[\s\S]*?\]\(images\/figure-\d+\.(png|jpg)\)/gi;
+    const figureMatches = markdownText.match(figurePattern) || [];
+
+    totalChecks++;
+    if (figureMatches.length >= expectedFigures) {
+      passedChecks++;
+    } else {
+      missing.images = expectedFigures - figureMatches.length;
+    }
+  }
+
+  // Calculate results
+  const passRate = totalChecks > 0 ? passedChecks / totalChecks : 0;
+  const hasMissingContent =
+    missing.title ||
+    missing.images > 0 ||
+    Object.values(missing).some((arr) => Array.isArray(arr) && arr.length > 0);
+
+  return {
+    totalChecks,
+    passedChecks,
+    passRate,
+    hasMissingContent,
+    missing,
+    success: !hasMissingContent || passRate >= 0.85,
+    details: verbose ? details : undefined,
+  };
+}

package/tests/e2e/docker.test.js

@@ -1,4 +1,6 @@
 import fetch from 'node-fetch';
+import http from 'http';
+import getPort from 'get-port';
 import { exec } from 'child_process';
 import { promisify } from 'util';
 
@@ -8,10 +10,18 @@ const PORT = 3000; // Use the same port as in docker-compose.yml
 const baseUrl = `http://localhost:${PORT}`;
 
 const timings = {};
+let mockServer;
+let mockUrl;
+
+const MOCK_HTML = `<!DOCTYPE html>
+<html lang="en">
+<head><meta charset="utf-8"><title>Example Domain</title></head>
+<body><h1>Example Domain</h1><p>This domain is for use in illustrative examples.</p></body>
+</html>`;
 
 async function isServiceRunning() {
   try {
-    const res = await fetch(`${baseUrl}/html?url=https://example.com`);
+    const res = await fetch(`${baseUrl}/health`);
     return res.status === 200;
   } catch {
     return false;
@@ -20,6 +30,20 @@ async function isServiceRunning() {
 
 beforeAll(async () => {
   timings.start = Date.now();
+
+  // Start a local mock server to avoid depending on external network
+  const mockPort = await getPort();
+  mockServer = http.createServer((req, res) => {
+    res.writeHead(200, { 'Content-Type': 'text/html; charset=utf-8' });
+    res.end(MOCK_HTML);
+  });
+  await new Promise((resolve) =>
+    mockServer.listen(mockPort, '0.0.0.0', resolve)
+  );
+  // Use host.docker.internal or the host IP to make it reachable from Docker
+  // On Linux (CI), Docker containers can reach the host via 172.17.0.1
+  mockUrl = `http://172.17.0.1:${mockPort}`;
+
   console.log('Checking if Docker service is already running...');
   const alreadyRunning = await isServiceRunning();
   if (!alreadyRunning) {
@@ -44,7 +68,7 @@ beforeAll(async () => {
         const checkReady = async () => {
           try {
             console.log(`Checking if service is ready at ${baseUrl}...`);
-            const res = await fetch(`${baseUrl}/html?url=https://example.com`);
+            const res = await fetch(`${baseUrl}/health`);
             if (res.status === 200) {
               console.log('Service is ready!');
               clearTimeout(timeout);
@@ -79,11 +103,24 @@ beforeAll(async () => {
   }
 }, 30000); // Reduced timeout for beforeAll
 
+afterAll(async () => {
+  if (mockServer) {
+    mockServer.close();
+  }
+  // Stop Docker containers
+  try {
+    await execAsync('docker compose down');
+  } catch {
+    // Ignore errors during cleanup
+  }
+});
+
 describe('E2E (Docker): Web Capture Microservice', () => {
   it('should return HTML from /html endpoint', async () => {
     const htmlStart = Date.now();
-    const url = 'https://example.com';
-    const res = await fetch(`${baseUrl}/html?url=${encodeURIComponent(url)}`);
+    const res = await fetch(
+      `${baseUrl}/html?url=${encodeURIComponent(mockUrl)}`
+    );
     expect(res.status).toBe(200);
     const text = await res.text();
     expect(text).toMatch(/<html/i);
@@ -93,9 +130,8 @@ describe('E2E (Docker): Web Capture Microservice', () => {
 
   it('should return Markdown from /markdown endpoint', async () => {
     const mdStart = Date.now();
-    const url = 'https://example.com';
     const res = await fetch(
-      `${baseUrl}/markdown?url=${encodeURIComponent(url)}`
+      `${baseUrl}/markdown?url=${encodeURIComponent(mockUrl)}`
     );
     expect(res.status).toBe(200);
     const text = await res.text();
@@ -106,8 +142,9 @@ describe('E2E (Docker): Web Capture Microservice', () => {
 
   it('should return PNG from /image endpoint', async () => {
     const pngStart = Date.now();
-    const url = 'https://example.com';
-    const res = await fetch(`${baseUrl}/image?url=${encodeURIComponent(url)}`);
+    const res = await fetch(
+      `${baseUrl}/image?url=${encodeURIComponent(mockUrl)}`
+    );
     expect(res.status).toBe(200);
     expect(res.headers.get('content-type')).toMatch(/^image\/png/);
     const buf = Buffer.from(await res.arrayBuffer());
@@ -123,8 +160,9 @@ describe('E2E (Docker): Web Capture Microservice', () => {
 
   it('should stream content from /stream endpoint', async () => {
     const startTime = Date.now();
-    const url = 'https://example.com';
-    const res = await fetch(`${baseUrl}/stream?url=${encodeURIComponent(url)}`);
+    const res = await fetch(
+      `${baseUrl}/stream?url=${encodeURIComponent(mockUrl)}`
+    );
     expect(res.status).toBe(200);
     // Get the response as text
     const text = await res.text();
@@ -136,8 +174,9 @@ describe('E2E (Docker): Web Capture Microservice', () => {
 
   it('should return content from /fetch endpoint', async () => {
     const startTime = Date.now();
-    const url = 'https://example.com';
-    const res = await fetch(`${baseUrl}/fetch?url=${encodeURIComponent(url)}`);
+    const res = await fetch(
+      `${baseUrl}/fetch?url=${encodeURIComponent(mockUrl)}`
+    );
     expect(res.status).toBe(200);
     const text = await res.text();
     expect(text).toMatch(/<html/i);

package/tests/e2e/process.test.js

@@ -1,4 +1,5 @@
 import { spawn } from 'child_process';
+import http from 'http';
 import fetch from 'node-fetch';
 import getPort from 'get-port';
 import path from 'path';
@@ -6,8 +7,25 @@ import path from 'path';
 const WAIT_FOR_READY = 5000; // ms
 let serverProcess;
 let baseUrl;
+let mockServer;
+let mockUrl;
+
+const MOCK_HTML = `<!DOCTYPE html>
+<html lang="en">
+<head><meta charset="utf-8"><title>Example Domain</title></head>
+<body><h1>Example Domain</h1><p>This domain is for use in illustrative examples.</p></body>
+</html>`;
 
 beforeAll(async () => {
+  // Start a local mock server to avoid depending on external network
+  const mockPort = await getPort();
+  mockServer = http.createServer((req, res) => {
+    res.writeHead(200, { 'Content-Type': 'text/html; charset=utf-8' });
+    res.end(MOCK_HTML);
+  });
+  await new Promise((resolve) => mockServer.listen(mockPort, resolve));
+  mockUrl = `http://localhost:${mockPort}`;
+
   const port = await getPort();
   baseUrl = `http://localhost:${port}`;
 
@@ -61,21 +79,24 @@ afterAll(() => {
   if (serverProcess) {
     serverProcess.kill();
   }
+  if (mockServer) {
+    mockServer.close();
+  }
 });
 
 describe('E2E: Web Capture Microservice', () => {
   it('should return HTML from /html endpoint', async () => {
-    const url = 'https://example.com';
-    const res = await fetch(`${baseUrl}/html?url=${encodeURIComponent(url)}`);
+    const res = await fetch(
+      `${baseUrl}/html?url=${encodeURIComponent(mockUrl)}`
+    );
     expect(res.status).toBe(200);
     const text = await res.text();
     expect(text).toMatch(/<html/i);
   });
 
   it('should return Markdown from /markdown endpoint', async () => {
-    const url = 'https://example.com';
     const res = await fetch(
-      `${baseUrl}/markdown?url=${encodeURIComponent(url)}`
+      `${baseUrl}/markdown?url=${encodeURIComponent(mockUrl)}`
     );
     expect(res.status).toBe(200);
     const text = await res.text();
@@ -83,8 +104,9 @@ describe('E2E: Web Capture Microservice', () => {
   });
 
   it('should return PNG from /image endpoint', async () => {
-    const url = 'https://example.com';
-    const res = await fetch(`${baseUrl}/image?url=${encodeURIComponent(url)}`);
+    const res = await fetch(
+      `${baseUrl}/image?url=${encodeURIComponent(mockUrl)}`
+    );
     expect(res.status).toBe(200);
     expect(res.headers.get('content-type')).toMatch(/^image\/png/);
     const buf = Buffer.from(await res.arrayBuffer());
@@ -97,8 +119,9 @@ describe('E2E: Web Capture Microservice', () => {
   }, 60000);
 
   it('should stream content from /stream endpoint', async () => {
-    const url = 'https://example.com';
-    const res = await fetch(`${baseUrl}/stream?url=${encodeURIComponent(url)}`);
+    const res = await fetch(
+      `${baseUrl}/stream?url=${encodeURIComponent(mockUrl)}`
+    );
     expect(res.status).toBe(200);
     // Get the response as text
     const text = await res.text();
@@ -107,8 +130,9 @@ describe('E2E: Web Capture Microservice', () => {
   }, 20000);
 
   it('should return content from /fetch endpoint', async () => {
-    const url = 'https://example.com';
-    const res = await fetch(`${baseUrl}/fetch?url=${encodeURIComponent(url)}`);
+    const res = await fetch(
+      `${baseUrl}/fetch?url=${encodeURIComponent(mockUrl)}`
+    );
     expect(res.status).toBe(200);
     const text = await res.text();
     expect(text).toMatch(/<html/i);

package/tests/fixtures/habr-articles.js

@@ -0,0 +1,45 @@
+/**
+ * Configuration for Habr test articles.
+ *
+ * These are the same three articles archived in the link-foundation/meta-theory repo.
+ * Each entry provides article metadata for use in integration tests.
+ */
+
+export const HABR_ARTICLES = {
+  '0.0.0': {
+    version: '0.0.0',
+    title: 'Math introduction to Deep Theory',
+    url: 'https://habr.com/en/companies/deepfoundation/articles/658705/',
+    language: 'en',
+    expectedFigures: 12,
+  },
+  '0.0.1': {
+    version: '0.0.1',
+    title: 'Глубокая Теория Связей 0.0.1',
+    titleEnglish: 'Deep Theory of Links 0.0.1',
+    url: 'https://habr.com/ru/companies/deepfoundation/articles/804617/',
+    language: 'ru',
+    expectedFigures: 10,
+  },
+  '0.0.2': {
+    version: '0.0.2',
+    title: 'The Links Theory 0.0.2',
+    url: 'https://habr.com/en/articles/895896/',
+    language: 'en',
+    expectedFigures: 13,
+  },
+};
+
+export function getArticle(version) {
+  const article = HABR_ARTICLES[version];
+  if (!article) {
+    throw new Error(
+      `Unknown article version: ${version}. Available: ${Object.keys(HABR_ARTICLES).join(', ')}`
+    );
+  }
+  return article;
+}
+
+export function getAllArticles() {
+  return Object.values(HABR_ARTICLES);
+}