@link-assistant/web-capture 1.10.3 → 1.10.4

package/CHANGELOG.md

@@ -1,5 +1,11 @@
 # @link-assistant/web-capture
 
+## 1.10.4
+
+### Patch Changes
+
+- Add live StackOverflow download coverage across JavaScript output formats and browser engines.
+
 ## 1.10.3
 
 ### Patch Changes

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@link-assistant/web-capture",
-  "version": "1.10.3",
+  "version": "1.10.4",
   "description": "CLI and microservice to render web pages as HTML, Markdown, or PNG",
   "main": "src/index.js",
   "bin": {

package/src/html.js

@@ -1,4 +1,9 @@
-import { fetchHtml, convertToUtf8, convertRelativeUrls } from './lib.js';
+import {
+  fetchHtml,
+  convertToUtf8,
+  convertRelativeUrls,
+  isStackOverflowQuestionUrl,
+} from './lib.js';
 import { createBrowser, getBrowserEngine } from './browser.js';
 
 export async function htmlHandler(req, res) {
@@ -12,6 +17,11 @@ export async function htmlHandler(req, res) {
 
     // First try to fetch HTML directly
     const html = await fetchHtml(absoluteUrl);
+    if (isStackOverflowQuestionUrl(absoluteUrl)) {
+      const utf8Html = convertToUtf8(html);
+      const absoluteHtml = convertRelativeUrls(utf8Html, absoluteUrl);
+      return res.type('text/html; charset=utf-8').send(absoluteHtml);
+    }
 
     // Check if it's valid HTML and contains JavaScript
     const hasJavaScript =

package/src/image.js

@@ -1,4 +1,5 @@
 import { createBrowser, getBrowserEngine } from './browser.js';
+import { isStackOverflowQuestionUrl } from './lib.js';
 import { dismissPopups, scrollToLoadContent } from './popups.js';
 
 /**
@@ -39,6 +40,10 @@ export async function imageHandler(req, res) {
   try {
     const absoluteUrl = url.startsWith('http') ? url : `https://${url}`;
     const engine = getBrowserEngine(req);
+    const waitUntil =
+      engine === 'playwright' && isStackOverflowQuestionUrl(absoluteUrl)
+        ? 'load'
+        : 'networkidle0';
     const browserOpts = theme ? { colorScheme: theme } : {};
     const browser = await createBrowser(engine, browserOpts);
     try {
@@ -52,7 +57,7 @@ export async function imageHandler(req, res) {
       );
       await page.setViewport({ width, height });
       await page.goto(absoluteUrl, {
-        waitUntil: 'networkidle0',
+        waitUntil,
         timeout: 30000,
       });
 

package/src/lib.js

@@ -4,27 +4,87 @@ import fetch from 'node-fetch';
 import * as cheerio from 'cheerio';
 import TurndownService from 'turndown';
 import iconv from 'iconv-lite';
-import { URL } from 'url';
+import { URL, URLSearchParams } from 'url';
 import turndownPluginGfm from 'turndown-plugin-gfm';
 import he from 'he';
 import { isFormulaImage, isMathElement, extractFormula } from './latex.js';
 import { extractMetadata } from './metadata.js';
 import { postProcessMarkdown } from './postprocess.js';
+import { retry } from './retry.js';
+
+const STACKPRINTER_RETRIES = 3;
+const STACKPRINTER_RETRY_BASE_DELAY_MS = 1000;
 
 export async function fetchHtml(url) {
   if (!url) {
     throw new Error('Missing URL parameter');
   }
+  const stackPrinterUrl = stackPrinterUrlForQuestion(url);
+  if (stackPrinterUrl) {
+    return await fetchStackPrinterHtml(stackPrinterUrl);
+  }
+
   const response = await fetch(url);
   return response.text();
 }
 
+async function fetchStackPrinterHtml(url) {
+  return await retry(
+    async () => {
+      const response = await fetch(url);
+      if (!response.ok) {
+        throw new Error(`StackPrinter returned HTTP ${response.status}`);
+      }
+
+      const html = await response.text();
+      if (isStackPrinterTransientError(html)) {
+        throw new Error('StackPrinter returned a transient error page');
+      }
+      return html;
+    },
+    {
+      retries: STACKPRINTER_RETRIES,
+      baseDelay: STACKPRINTER_RETRY_BASE_DELAY_MS,
+    }
+  );
+}
+
+function isStackPrinterTransientError(html) {
+  return /Ooooops|Please try again later/i.test(html);
+}
+
+export function isStackOverflowQuestionUrl(url) {
+  return Boolean(getStackOverflowQuestionId(url));
+}
+
+export function stackPrinterUrlForQuestion(url) {
+  const questionId = getStackOverflowQuestionId(url);
+  if (!questionId) {
+    return null;
+  }
+
+  const params = new URLSearchParams({
+    question: questionId,
+    service: 'stackoverflow',
+    language: 'en',
+    hideAnswers: 'false',
+    showAll: 'true',
+    width: '640',
+  });
+  return `https://stackprinter.appspot.com/export?${params.toString()}`;
+}
+
 export function isTextPasteUrl(url) {
   const parsed = parseTextPasteUrl(url);
   return Boolean(parsed);
 }
 
 export function normalizeUrlForTextContent(url) {
+  const stackPrinterUrl = stackPrinterUrlForQuestion(url);
+  if (stackPrinterUrl) {
+    return stackPrinterUrl;
+  }
+
   const parsed = parseTextPasteUrl(url);
   if (!parsed) {
     return url;
@@ -49,6 +109,25 @@ export function getTextPasteFilename(url) {
   return pasteId ? `xpaste-pro-${pasteId}.txt` : null;
 }
 
+function getStackOverflowQuestionId(url) {
+  try {
+    const parsed = new URL(url);
+    const hostname = parsed.hostname.replace(/^www\./, '');
+    if (hostname !== 'stackoverflow.com') {
+      return null;
+    }
+
+    const [, prefix, questionId] = parsed.pathname.split('/');
+    if (prefix !== 'questions' || !/^\d+$/.test(questionId || '')) {
+      return null;
+    }
+
+    return questionId;
+  } catch {
+    return null;
+  }
+}
+
 export function appendTextPasteMarkdownAttachment(markdown, url, rawText) {
   const textFilename = getTextPasteFilename(url) || 'download.txt';
   const normalizedRawText = normalizeMarkdownAttachmentText(rawText);

package/src/txt.js

@@ -1,6 +1,11 @@
 import fetch from 'node-fetch';
 import { URL } from 'node:url';
-import { getTextPasteFilename, normalizeUrlForTextContent } from './lib.js';
+import {
+  fetchHtml,
+  getTextPasteFilename,
+  isStackOverflowQuestionUrl,
+  normalizeUrlForTextContent,
+} from './lib.js';
 import {
   fetchGithubRepositorySnapshot,
   formatGithubRepositoryText,
@@ -21,6 +26,11 @@ export async function txtHandler(req, res) {
       return sendTextResponse(res, url, text);
     }
 
+    if (isStackOverflowQuestionUrl(url)) {
+      const text = await fetchHtml(url);
+      return sendTextResponse(res, url, text);
+    }
+
     // Normalize URL to get text content (e.g., xpaste.pro -> xpaste.pro/raw)
     const textUrl = normalizeUrlForTextContent(url);
 

package/tests/integration/stackoverflow-download.test.js

@@ -0,0 +1,173 @@
+/**
+ * Integration tests for downloading the StackOverflow page from issue #11.
+ *
+ * These tests hit the live StackOverflow page:
+ * https://stackoverflow.com/questions/927358/how-do-i-undo-the-most-recent-local-commits-in-git
+ *
+ * Set STACKOVERFLOW_INTEGRATION=true to run them. The suite is skipped by
+ * default so offline/local test runs stay deterministic.
+ */
+
+import { jest } from '@jest/globals';
+import request from 'supertest';
+import unzipper from 'unzipper';
+import { app } from '../../src/index.js';
+import { retry } from '../../src/retry.js';
+
+const STACKOVERFLOW_URL =
+  'https://stackoverflow.com/questions/927358/how-do-i-undo-the-most-recent-local-commits-in-git';
+const STACKOVERFLOW_TITLE =
+  'How do I undo the most recent local commits in Git';
+
+const PNG_SIGNATURE = Buffer.from([
+  0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a,
+]);
+const ZIP_SIGNATURE = Buffer.from([0x50, 0x4b]);
+const PDF_SIGNATURE = Buffer.from([0x25, 0x50, 0x44, 0x46]);
+
+const SKIP_LIVE =
+  !process.env.STACKOVERFLOW_INTEGRATION ||
+  process.env.STACKOVERFLOW_INTEGRATION === 'false';
+const describeIfLive = SKIP_LIVE ? describe.skip : describe;
+
+jest.setTimeout(180000);
+
+function parseBinary(res, callback) {
+  const chunks = [];
+  res.on('data', (chunk) => chunks.push(Buffer.from(chunk)));
+  res.on('end', () => callback(null, Buffer.concat(chunks)));
+}
+
+async function getText(path, query = {}) {
+  return await retry(
+    () =>
+      request(app)
+        .get(path)
+        .query({ url: STACKOVERFLOW_URL, ...query })
+        .expect(200),
+    {
+      retries: 2,
+      baseDelay: 2000,
+      onRetry: (err, attempt, delay) => {
+        console.log(
+          `Retry ${attempt} for ${path} after ${delay}ms: ${err.message}`
+        );
+      },
+    }
+  );
+}
+
+async function getBinary(path, query = {}) {
+  return await retry(
+    () =>
+      request(app)
+        .get(path)
+        .query({ url: STACKOVERFLOW_URL, ...query })
+        .buffer(true)
+        .parse(parseBinary)
+        .expect(200),
+    {
+      retries: 2,
+      baseDelay: 2000,
+      onRetry: (err, attempt, delay) => {
+        console.log(
+          `Retry ${attempt} for ${path} after ${delay}ms: ${err.message}`
+        );
+      },
+    }
+  );
+}
+
+function expectStackOverflowContent(text) {
+  expect(text).toContain(STACKOVERFLOW_TITLE);
+  expect(text).toMatch(/Stack Overflow/i);
+  expect(text).toMatch(/\bgit\b/i);
+}
+
+function expectBufferWithSignature(buffer, signature) {
+  expect(Buffer.isBuffer(buffer)).toBe(true);
+  expect(buffer.length).toBeGreaterThan(1000);
+  expect(buffer.subarray(0, signature.length)).toEqual(signature);
+}
+
+describeIfLive('StackOverflow page downloads (JavaScript)', () => {
+  it('downloads the page as Markdown and plain text', async () => {
+    const markdown = await getText('/markdown');
+    expect(markdown.headers['content-type']).toMatch(/text\/markdown/);
+    expectStackOverflowContent(markdown.text);
+    expect(markdown.text).not.toMatch(/<html/i);
+
+    const text = await getText('/txt');
+    expect(text.headers['content-type']).toMatch(/text\/plain/);
+    expect(text.headers['content-disposition']).toContain('stackoverflow-com');
+    expectStackOverflowContent(text.text);
+  });
+
+  it('downloads Markdown and HTML archives with the expected files', async () => {
+    const markdownArchive = await getBinary('/archive', {
+      documentFormat: 'markdown',
+      localImages: 'false',
+    });
+    expect(markdownArchive.headers['content-type']).toMatch(/application\/zip/);
+    expectBufferWithSignature(markdownArchive.body, ZIP_SIGNATURE);
+
+    const markdownZip = await unzipper.Open.buffer(markdownArchive.body);
+    const markdownEntries = new Map(
+      markdownZip.files.map((file) => [file.path, file])
+    );
+    expect(markdownEntries.has('document.md')).toBe(true);
+    expect(markdownEntries.has('document.html')).toBe(true);
+    expectStackOverflowContent(
+      (await markdownEntries.get('document.md').buffer()).toString('utf8')
+    );
+
+    const htmlArchive = await getBinary('/archive', {
+      documentFormat: 'html',
+      localImages: 'false',
+    });
+    expect(htmlArchive.headers['content-type']).toMatch(/application\/zip/);
+    expectBufferWithSignature(htmlArchive.body, ZIP_SIGNATURE);
+
+    const htmlZip = await unzipper.Open.buffer(htmlArchive.body);
+    const htmlEntries = new Map(htmlZip.files.map((file) => [file.path, file]));
+    expect(htmlEntries.has('document.html')).toBe(true);
+    expectStackOverflowContent(
+      (await htmlEntries.get('document.html').buffer()).toString('utf8')
+    );
+  });
+
+  it('downloads the page as DOCX and PDF', async () => {
+    const docx = await getBinary('/docx');
+    expect(docx.headers['content-type']).toMatch(
+      /application\/vnd\.openxmlformats-officedocument\.wordprocessingml\.document/
+    );
+    expectBufferWithSignature(docx.body, ZIP_SIGNATURE);
+
+    const pdf = await getBinary('/pdf');
+    expect(pdf.headers['content-type']).toMatch(/application\/pdf/);
+    expectBufferWithSignature(pdf.body, PDF_SIGNATURE);
+  }, 180000);
+});
+
+describe.each(['puppeteer', 'playwright'])(
+  'StackOverflow browser downloads (%s engine)',
+  (engine) => {
+    describeIfLive(`${engine}`, () => {
+      it('downloads rendered HTML for the page', async () => {
+        const html = await getText('/html', { engine });
+        expect(html.headers['content-type']).toMatch(/text\/html/);
+        expectStackOverflowContent(html.text);
+        expect(html.text).toMatch(/<html/i);
+      }, 120000);
+
+      it('downloads a PNG screenshot for the page', async () => {
+        const image = await getBinary('/image', {
+          engine,
+          dismissPopups: 'true',
+        });
+        expect(image.headers['content-type']).toMatch(/image\/png/);
+        expectBufferWithSignature(image.body, PNG_SIGNATURE);
+      }, 120000);
+    });
+  }
+);

package/tests/unit/stackoverflow.test.js

@@ -0,0 +1,32 @@
+import {
+  isStackOverflowQuestionUrl,
+  stackPrinterUrlForQuestion,
+} from '../../src/lib.js';
+
+describe('Stack Overflow URL helpers', () => {
+  it('recognizes Stack Overflow question URLs', () => {
+    expect(
+      isStackOverflowQuestionUrl(
+        'https://stackoverflow.com/questions/927358/how-do-i-undo-the-most-recent-local-commits-in-git'
+      )
+    ).toBe(true);
+    expect(
+      isStackOverflowQuestionUrl(
+        'https://stackoverflow.com/questions/tagged/git'
+      )
+    ).toBe(false);
+    expect(
+      isStackOverflowQuestionUrl('https://serverfault.com/questions/927358')
+    ).toBe(false);
+  });
+
+  it('builds a StackPrinter URL for direct captures', () => {
+    const stackPrinterUrl = stackPrinterUrlForQuestion(
+      'https://stackoverflow.com/questions/927358/how-do-i-undo-the-most-recent-local-commits-in-git'
+    );
+
+    expect(stackPrinterUrl).toBe(
+      'https://stackprinter.appspot.com/export?question=927358&service=stackoverflow&language=en&hideAnswers=false&showAll=true&width=640'
+    );
+  });
+});