@link-assistant/web-capture 1.10.1 → 1.10.3

package/src/lib.js

@@ -19,6 +19,95 @@ export async function fetchHtml(url) {
   return response.text();
 }
 
+export function isTextPasteUrl(url) {
+  const parsed = parseTextPasteUrl(url);
+  return Boolean(parsed);
+}
+
+export function normalizeUrlForTextContent(url) {
+  const parsed = parseTextPasteUrl(url);
+  if (!parsed) {
+    return url;
+  }
+  return `https://${parsed.host}${parsed.pathPrefix}/p/${parsed.pasteId}/raw`;
+}
+
+export function normalizeUrlForTextPage(url) {
+  const parsed = parseTextPasteUrl(url);
+  if (!parsed) {
+    return url;
+  }
+  return `https://${parsed.host}${parsed.pathPrefix}/p/${parsed.pasteId}`;
+}
+
+export function getTextPasteId(url) {
+  return parseTextPasteUrl(url)?.pasteId || null;
+}
+
+export function getTextPasteFilename(url) {
+  const pasteId = getTextPasteId(url);
+  return pasteId ? `xpaste-pro-${pasteId}.txt` : null;
+}
+
+export function appendTextPasteMarkdownAttachment(markdown, url, rawText) {
+  const textFilename = getTextPasteFilename(url) || 'download.txt';
+  const normalizedRawText = normalizeMarkdownAttachmentText(rawText);
+  const fence = markdownFenceFor(normalizedRawText);
+  const attachmentText = normalizedRawText.endsWith('\n')
+    ? normalizedRawText
+    : `${normalizedRawText}\n`;
+
+  return [
+    markdown.trimEnd(),
+    '',
+    `## ${textFilename}`,
+    '',
+    `${fence}text`,
+    `${attachmentText}${fence}`,
+    '',
+  ].join('\n');
+}
+
+function normalizeMarkdownAttachmentText(text) {
+  return String(text).replace(/\r\n/g, '\n').replace(/\r/g, '\n');
+}
+
+function markdownFenceFor(text) {
+  const runs = text.match(/`{3,}/g) || [];
+  const length = runs.reduce((max, run) => Math.max(max, run.length + 1), 3);
+  return '`'.repeat(length);
+}
+
+function parseTextPasteUrl(url) {
+  try {
+    const parsed = new URL(url);
+    const host = parsed.hostname.toLowerCase();
+    if (host !== 'xpaste.pro' && host !== 'www.xpaste.pro') {
+      return null;
+    }
+
+    const parts = parsed.pathname.split('/').filter(Boolean);
+    const index = parts.indexOf('p');
+    let pathPrefix = '';
+    if (index === -1 || !parts[index + 1]) {
+      return null;
+    }
+    if (index === 1 && ['en', 'ru'].includes(parts[0])) {
+      pathPrefix = `/${parts[0]}`;
+    } else if (index !== 0) {
+      return null;
+    }
+    const tail = parts.slice(index + 2);
+    if (tail.length > 1 || (tail[0] && tail[0] !== 'raw')) {
+      return null;
+    }
+
+    return { host: 'xpaste.pro', pathPrefix, pasteId: parts[index + 1] };
+  } catch {
+    return null;
+  }
+}
+
 export function convertHtmlToMarkdown(html, baseUrl) {
   // Ensure all URLs are absolute before Markdown conversion
   if (baseUrl) {
@@ -27,6 +116,8 @@ export function convertHtmlToMarkdown(html, baseUrl) {
   // Load HTML into Cheerio
   const $ = cheerio.load(html);
 
+  reorderVisualLayoutElements($);
+
   // Remove <style>, <script>, and <noscript> tags
   $('style, script, noscript').remove();
 
@@ -199,6 +290,20 @@ function coalesceBrRunsToParagraphBreak(markdown) {
   return markdown.replace(/(?: {2,}\n){2,}/g, '\n\n');
 }
 
+function reorderVisualLayoutElements($) {
+  const $header = $('header').first();
+  const $main = $('main').first();
+  const $footer = $('footer').first();
+
+  if ($header.length && $main.length && $header.index() > $main.index()) {
+    $main.before($header);
+  }
+
+  if ($footer.length && $main.length && $footer.index() < $main.index()) {
+    $main.after($footer);
+  }
+}
+
 // Lift <br> nodes out of inline parents when they sit at the leading or
 // trailing edge.
 //
@@ -598,6 +703,8 @@ export function convertHtmlToMarkdownEnhanced(html, baseUrl, options = {}) {
   html = scopeHtmlForMarkdown(html, { contentSelector, bodySelector });
   $ = cheerio.load(html);
 
+  reorderVisualLayoutElements($);
+
   // Remove unwanted elements
   $('style, script, noscript').remove();
   $('*').each(function () {

package/src/markdown.js

@@ -1,10 +1,24 @@
 import {
+  appendTextPasteMarkdownAttachment,
   fetchHtml,
   convertHtmlToMarkdownEnhanced,
+  getTextPasteId,
+  isTextPasteUrl,
+  normalizeUrlForTextContent,
+  normalizeUrlForTextPage,
   scopeHtmlForMarkdown,
 } from './lib.js';
+import {
+  fetchGithubRepositorySnapshot,
+  formatGithubRepositoryMarkdown,
+  isGithubRepositoryUrl,
+} from './github.js';
 import { convertWithKreuzberg, isKreuzbergAvailable } from './kreuzberg.js';
 import { applyImageMode } from './extract-images.js';
+import archiver from 'archiver';
+import fetch from 'node-fetch';
+
+const INLINE_MARKDOWN_LINE_LIMIT = 1500;
 
 export async function markdownHandler(req, res) {
   const url = req.query.url;
@@ -29,7 +43,23 @@ export async function markdownHandler(req, res) {
   }
 
   try {
-    const html = await fetchHtml(url);
+    const pageUrl = normalizeUrlForTextPage(url);
+    if (
+      format === 'text' &&
+      !req.query.contentSelector &&
+      !req.query.bodySelector &&
+      isGithubRepositoryUrl(pageUrl)
+    ) {
+      const snapshot = await fetchGithubRepositorySnapshot(pageUrl);
+      let markdown = formatGithubRepositoryMarkdown(snapshot);
+      const result = await applyImageMode(markdown, {
+        mode: embedImages ? 'embed' : 'default',
+      });
+      markdown = result.markdown;
+      return await sendMarkdownResponse(res, url, markdown);
+    }
+
+    const html = await fetchHtml(pageUrl);
 
     if (converter === 'kreuzberg') {
       const available = await isKreuzbergAvailable();
@@ -45,7 +75,9 @@ export async function markdownHandler(req, res) {
         contentSelector: req.query.contentSelector,
         bodySelector: req.query.bodySelector,
       });
-      const result = await convertWithKreuzberg(scopedHtml, { baseUrl: url });
+      const result = await convertWithKreuzberg(scopedHtml, {
+        baseUrl: pageUrl,
+      });
       const imageResult = await applyImageMode(result.content, {
         mode: embedImages ? 'embed' : 'default',
       });
@@ -54,10 +86,10 @@ export async function markdownHandler(req, res) {
       if (format === 'json') {
         return res.json(result);
       }
-      return res.type('text/markdown').send(result.content);
+      return await sendMarkdownResponse(res, url, result.content);
     }
 
-    const { markdown } = convertHtmlToMarkdownEnhanced(html, url, {
+    let { markdown } = convertHtmlToMarkdownEnhanced(html, pageUrl, {
       contentSelector: req.query.contentSelector,
       bodySelector: req.query.bodySelector,
     });
@@ -67,9 +99,64 @@ export async function markdownHandler(req, res) {
     const result = await applyImageMode(markdown, {
       mode: embedImages ? 'embed' : 'default',
     });
-    res.type('text/markdown').send(result.markdown);
+    markdown = result.markdown;
+    return await sendMarkdownResponse(res, url, markdown);
   } catch (err) {
     console.error(err);
     res.status(500).send('Error converting to Markdown');
   }
 }
+
+async function sendMarkdownResponse(res, url, markdown) {
+  if (!isTextPasteUrl(url)) {
+    return res.type('text/markdown').send(markdown);
+  }
+
+  const rawText = await fetchRawText(url);
+  const markdownWithText = appendTextPasteMarkdownAttachment(
+    markdown,
+    url,
+    rawText
+  );
+  if (markdownWithText.split('\n').length >= INLINE_MARKDOWN_LINE_LIMIT) {
+    return await sendTextPasteMarkdownArchive(res, url, markdown, rawText);
+  }
+
+  return res.type('text/markdown').send(markdownWithText);
+}
+
+async function sendTextPasteMarkdownArchive(res, url, markdown, rawText) {
+  const pasteId = getTextPasteId(url) || 'paste';
+  const markdownFilename = `xpaste-pro-${pasteId}.md`;
+  const textFilename = `xpaste-pro-${pasteId}.txt`;
+  const lineCount = markdown.split('\n').length;
+
+  res.setHeader('Content-Type', 'application/zip');
+  res.setHeader('Content-Disposition', `attachment; filename="${pasteId}.zip"`);
+
+  const archive = archiver('zip', { zlib: { level: 9 } });
+  archive.pipe(res);
+  archive.append(
+    [
+      `# ${url}`,
+      '',
+      `Content from: ${url}`,
+      '',
+      `The page markdown is available in [${markdownFilename}](${markdownFilename}) (${lineCount} lines).`,
+      `The raw text content is available in [${textFilename}](${textFilename}).`,
+      '',
+    ].join('\n'),
+    { name: 'index.md' }
+  );
+  archive.append(markdown, { name: markdownFilename });
+  archive.append(rawText, { name: textFilename });
+  await archive.finalize();
+}
+
+async function fetchRawText(url) {
+  const response = await fetch(normalizeUrlForTextContent(url));
+  if (!response.ok) {
+    throw new Error(`HTTP ${response.status}: ${response.statusText}`);
+  }
+  return response.text();
+}

package/src/txt.js

@@ -0,0 +1,80 @@
+import fetch from 'node-fetch';
+import { URL } from 'node:url';
+import { getTextPasteFilename, normalizeUrlForTextContent } from './lib.js';
+import {
+  fetchGithubRepositorySnapshot,
+  formatGithubRepositoryText,
+  getGithubRepositoryTextFilename,
+  isGithubRepositoryUrl,
+} from './github.js';
+
+export async function txtHandler(req, res) {
+  const url = req.query.url;
+  if (!url) {
+    return res.status(400).send('Missing `url` parameter');
+  }
+
+  try {
+    if (isGithubRepositoryUrl(url)) {
+      const snapshot = await fetchGithubRepositorySnapshot(url);
+      const text = formatGithubRepositoryText(snapshot);
+      return sendTextResponse(res, url, text);
+    }
+
+    // Normalize URL to get text content (e.g., xpaste.pro -> xpaste.pro/raw)
+    const textUrl = normalizeUrlForTextContent(url);
+
+    const response = await fetch(textUrl);
+
+    if (!response.ok) {
+      throw new Error(`HTTP ${response.status}: ${response.statusText}`);
+    }
+
+    const contentType = response.headers.get('content-type') || 'text/plain';
+
+    // Only accept text content types
+    if (!contentType.includes('text/')) {
+      throw new Error(`Expected text content, got ${contentType}`);
+    }
+
+    const text = await response.text();
+
+    sendTextResponse(res, url, text);
+  } catch (err) {
+    console.error('Text fetch error:', err);
+    res.status(500).send('Error fetching text content');
+  }
+}
+
+function sendTextResponse(res, url, text) {
+  res.setHeader('Content-Type', 'text/plain; charset=utf-8');
+  res.setHeader(
+    'Content-Disposition',
+    `attachment; filename="${getFilenameFromUrl(url)}"`
+  );
+  res.send(text);
+}
+
+function getFilenameFromUrl(url) {
+  const githubRepositoryFilename = getGithubRepositoryTextFilename(url);
+  if (githubRepositoryFilename) {
+    return githubRepositoryFilename;
+  }
+
+  const textPasteFilename = getTextPasteFilename(url);
+  if (textPasteFilename) {
+    return textPasteFilename;
+  }
+
+  try {
+    const urlObj = new URL(url);
+    const hostname = urlObj.hostname.replace(/\./g, '-');
+    const path = urlObj.pathname
+      .replace(/\//g, '-')
+      .replace(/^-/, '')
+      .replace(/-raw$/, '');
+    return `${hostname}${path}.txt`;
+  } catch {
+    return 'download.txt';
+  }
+}

package/tests/integration/api-endpoints.test.js

@@ -25,6 +25,50 @@ const MOCK_HTML = `<!DOCTYPE html>
 <img src="https://example.com/image.png" alt="test">
 </body></html>`;
 
+function mockGithubRepository(owner = 'octocat', repo = 'Hello-World') {
+  const readme = '# Hello World\n\nThis README came from GitHub.';
+  nock('https://api.github.com')
+    .get(`/repos/${owner}/${repo}`)
+    .reply(200, {
+      full_name: `${owner}/${repo}`,
+      description: 'A friendly test repository',
+      html_url: `https://github.com/${owner}/${repo}`,
+      default_branch: 'master',
+      language: 'JavaScript',
+      stargazers_count: 42,
+      forks_count: 7,
+      open_issues_count: 3,
+      license: { spdx_id: 'MIT' },
+      topics: ['demo', 'capture'],
+    })
+    .get(`/repos/${owner}/${repo}/readme`)
+    .query({ ref: 'master' })
+    .reply(200, {
+      name: 'README.md',
+      path: 'README.md',
+      encoding: 'base64',
+      content: Buffer.from(readme, 'utf8').toString('base64'),
+      html_url: `https://github.com/${owner}/${repo}/blob/master/README.md`,
+    })
+    .get(`/repos/${owner}/${repo}/contents`)
+    .query({ ref: 'master' })
+    .reply(200, [
+      {
+        name: 'src',
+        path: 'src',
+        type: 'dir',
+        html_url: `https://github.com/${owner}/${repo}/tree/master/src`,
+      },
+      {
+        name: 'README.md',
+        path: 'README.md',
+        type: 'file',
+        size: readme.length,
+        html_url: `https://github.com/${owner}/${repo}/blob/master/README.md`,
+      },
+    ]);
+}
+
 afterEach(() => {
   nock.cleanAll();
 });
@@ -70,6 +114,44 @@ describe('API Endpoint Tests', () => {
       expect(res.text).toContain('Hello world');
     });
 
+    it('returns a compact GitHub repository markdown snapshot', async () => {
+      mockGithubRepository();
+
+      const res = await request(app)
+        .get('/markdown')
+        .query({ url: 'https://github.com/octocat/Hello-World' })
+        .expect(200);
+
+      expect(res.headers['content-type']).toMatch(/text\/markdown/);
+      expect(res.text).toContain('# octocat/Hello-World');
+      expect(res.text).toContain('## Repository');
+      expect(res.text).toContain('## Files');
+      expect(res.text).toContain('- [src/](');
+      expect(res.text).toContain('## README.md');
+      expect(res.text).toContain('This README came from GitHub.');
+      expect(res.text).not.toContain('Skip to content');
+      expect(nock.isDone()).toBe(true);
+    });
+
+    it('returns a compact GitHub repository markdown snapshot for kreuzberg text format', async () => {
+      mockGithubRepository();
+
+      const res = await request(app)
+        .get('/markdown')
+        .query({
+          url: 'https://github.com/octocat/Hello-World',
+          converter: 'kreuzberg',
+          format: 'text',
+        })
+        .expect(200);
+
+      expect(res.headers['content-type']).toMatch(/text\/markdown/);
+      expect(res.text).toContain('# octocat/Hello-World');
+      expect(res.text).toContain('## Files');
+      expect(res.text).toContain('This README came from GitHub.');
+      expect(nock.isDone()).toBe(true);
+    });
+
     it('rejects unsupported converter names', async () => {
       await request(app)
         .get('/markdown')
@@ -133,6 +215,28 @@ describe('API Endpoint Tests', () => {
     });
   });
 
+  describe('GET /txt', () => {
+    it('returns a compact GitHub repository plain-text snapshot', async () => {
+      mockGithubRepository();
+
+      const res = await request(app)
+        .get('/txt')
+        .query({ url: 'https://github.com/octocat/Hello-World' })
+        .expect(200);
+
+      expect(res.headers['content-type']).toMatch(/text\/plain/);
+      expect(res.headers['content-disposition']).toContain(
+        'filename="octocat-Hello-World.txt"'
+      );
+      expect(res.text).toContain('Repository: octocat/Hello-World');
+      expect(res.text).toContain('Files:');
+      expect(res.text).toContain('- src/');
+      expect(res.text).toContain('README.md:');
+      expect(res.text).toContain('This README came from GitHub.');
+      expect(nock.isDone()).toBe(true);
+    });
+  });
+
   describe('GET /archive', () => {
     it('returns a ZIP archive with remote images (markdown format)', async () => {
       nock('https://example.com').get('/archive-test').reply(200, MOCK_HTML, {

package/tests/integration/github-readme.test.js

@@ -0,0 +1,137 @@
+/**
+ * Integration tests for capturing GitHub repository pages (issue #5).
+ *
+ * Live tests are gated behind GITHUB_REPOSITORY_INTEGRATION so default/offline
+ * runs stay deterministic. The mocked API endpoint tests cover the default
+ * behavior without network access; this suite proves the real GitHub repository
+ * page can still be captured as compact text/markdown, original HTML, and PNG
+ * screenshots in every supported JavaScript browser engine.
+ */
+
+import { jest } from '@jest/globals';
+import { createBrowser } from '../../src/browser.js';
+import { fetchHtml } from '../../src/lib.js';
+import { retry } from '../../src/retry.js';
+import {
+  fetchGithubRepositorySnapshot,
+  formatGithubRepositoryMarkdown,
+  formatGithubRepositoryText,
+  parseGithubRepositoryUrl,
+} from '../../src/github.js';
+
+const GITHUB_REPOSITORY_URL =
+  process.env.GITHUB_REPOSITORY_URL ||
+  'https://github.com/link-assistant/web-capture';
+const PNG_SIGNATURE = Buffer.from([
+  0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a,
+]);
+
+const SKIP_LIVE =
+  !process.env.GITHUB_REPOSITORY_INTEGRATION ||
+  process.env.GITHUB_REPOSITORY_INTEGRATION === 'false';
+const describeIfLive = SKIP_LIVE ? describe.skip : describe;
+
+jest.setTimeout(120000);
+
+async function navigateWithRetry(page, url) {
+  await retry(
+    async () => {
+      await page.goto(url, {
+        waitUntil: 'domcontentloaded',
+        timeout: 30000,
+      });
+    },
+    {
+      retries: 3,
+      baseDelay: 2000,
+      onRetry: (err, attempt, delay) => {
+        console.log(
+          `Navigation retry ${attempt} for ${url} after ${delay}ms: ${err.message}`
+        );
+      },
+    }
+  );
+  await page.waitForTimeout(5000);
+}
+
+describeIfLive('GitHub repository capture', () => {
+  it('downloads compact repository text and markdown with README plus file tree', async () => {
+    const parsed = parseGithubRepositoryUrl(GITHUB_REPOSITORY_URL);
+    expect(parsed).not.toBeNull();
+
+    const snapshot = await retry(
+      () => fetchGithubRepositorySnapshot(GITHUB_REPOSITORY_URL),
+      {
+        retries: 3,
+        baseDelay: 2000,
+      }
+    );
+
+    const text = formatGithubRepositoryText(snapshot);
+    const markdown = formatGithubRepositoryMarkdown(snapshot);
+
+    expect(text).toContain(`Repository: ${parsed.fullName}`);
+    expect(text).toContain('Files:');
+    expect(text).toMatch(/README/i);
+    expect(text.length).toBeGreaterThan(500);
+
+    expect(markdown).toContain(`# ${parsed.fullName}`);
+    expect(markdown).toContain('## Files');
+    expect(markdown).toMatch(/## .*README/i);
+    expect(markdown.length).toBeGreaterThan(500);
+    expect(markdown).not.toMatch(/<html/i);
+  });
+
+  it('downloads original GitHub repository HTML', async () => {
+    const parsed = parseGithubRepositoryUrl(GITHUB_REPOSITORY_URL);
+    expect(parsed).not.toBeNull();
+
+    const html = await retry(() => fetchHtml(GITHUB_REPOSITORY_URL), {
+      retries: 3,
+      baseDelay: 2000,
+    });
+
+    expect(html).toMatch(/<html/i);
+    expect(html).toContain(parsed.owner);
+    expect(html).toContain(parsed.repo);
+    expect(html.length).toBeGreaterThan(1000);
+  });
+});
+
+describe.each(['puppeteer', 'playwright'])(
+  'GitHub repository screenshot (%s engine)',
+  (engine) => {
+    describeIfLive(`${engine}`, () => {
+      let browser;
+
+      beforeAll(async () => {
+        browser = await createBrowser(engine);
+      });
+
+      afterAll(async () => {
+        if (browser) {
+          await browser.close();
+        }
+      });
+
+      it('captures the GitHub repository page as a PNG screenshot', async () => {
+        const page = await browser.newPage();
+        await page.setExtraHTTPHeaders({
+          'Accept-Language': 'en-US,en;q=0.9',
+          'Accept-Charset': 'utf-8',
+        });
+        await page.setUserAgent(
+          'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36'
+        );
+        await page.setViewport({ width: 1280, height: 800 });
+        await navigateWithRetry(page, GITHUB_REPOSITORY_URL);
+
+        const screenshot = await page.screenshot({ type: 'png' });
+
+        expect(screenshot).toBeInstanceOf(Buffer);
+        expect(screenshot.length).toBeGreaterThan(1000);
+        expect(screenshot.slice(0, 8)).toEqual(PNG_SIGNATURE);
+      });
+    });
+  }
+);