@link-assistant/web-capture 1.10.1 → 1.10.3

package/CHANGELOG.md

@@ -1,5 +1,17 @@
 # @link-assistant/web-capture
 
+## 1.10.3
+
+### Patch Changes
+
+- Add compact GitHub repository capture for txt and markdown output, including repository metadata, the root file tree, and README content.
+
+## 1.10.2
+
+### Patch Changes
+
+- Add xpaste.pro raw text capture and large-paste markdown archive handling.
+
 ## 1.10.1
 
 ### Patch Changes

package/README.md

@@ -10,6 +10,7 @@ A CLI and microservice to fetch URLs and render them as:
 
 - **Markdown**: Converted from HTML with image extraction (default)
 - **HTML**: Rendered page content
+- **Plain text**: Raw text downloads for paste-like URLs such as xpaste.pro
 - **PNG/JPEG screenshot**: Viewport or full-page capture with theme support
 - **ZIP archive**: Markdown + locally downloaded images
 - **PDF**: Print-quality document with embedded images
@@ -48,6 +49,13 @@ web-capture https://example.com -o -
 # Capture as HTML
 web-capture https://example.com --format html -o page.html
 
+# Capture raw paste text
+web-capture https://xpaste.pro/p/t4q0Lsp0 --format txt -o paste.txt
+
+# Capture a GitHub repository as compact text or Markdown
+web-capture https://github.com/link-assistant/web-capture --format txt -o repository.txt
+web-capture https://github.com/link-assistant/web-capture --format markdown -o repository.md
+
 # Take a PNG screenshot
 web-capture https://example.com --format png -o screenshot.png
 
@@ -141,6 +149,19 @@ GET /markdown?url=<URL>
 
 Converts the HTML content of the specified URL to Markdown format. By default, original remote image URLs are preserved and base64 data URIs are stripped (clean single-file output). Use `keepOriginalLinks=false` to strip all images, or `embedImages=true` to keep base64 images inline.
 
+For xpaste.pro paste URLs, `/markdown` fetches the visual paste page, keeps
+visible header, language, metadata, paste body, and footer text in visual order,
+and appends the raw paste text as a fenced block named `xpaste-pro-<id>.txt`
+when the final Markdown stays under 1500 lines. Larger paste pages return a ZIP
+containing `index.md`, `xpaste-pro-<id>.md`, and `xpaste-pro-<id>.txt`.
+Canonical `/p/<id>`, `/p/<id>/raw`, `/ru/p/<id>`, and `/en/p/<id>` URLs are
+normalized before capture.
+
+For plain GitHub repository URLs such as `https://github.com/owner/repo`,
+`/markdown` returns a compact repository snapshot with repository metadata, the
+root file tree, and README content. GitHub subpages continue through the regular
+HTML-to-Markdown conversion path.
+
 | Parameter           | Required | Description                                                               | Default  |
 | ------------------- | -------- | ------------------------------------------------------------------------- | -------- |
 | `url`               | Yes      | URL to fetch                                                              | -        |
@@ -149,6 +170,23 @@ Converts the HTML content of the specified URL to Markdown format. By default, o
 | `embedImages`       | No       | Keep base64 images inline (`true`/`false`)                                | `false`  |
 | `keepOriginalLinks` | No       | Keep original remote URLs, strip base64                                   | `true`   |
 
+### Text Endpoint
+
+```
+GET /txt?url=<URL>
+```
+
+Returns raw text content as a `.txt` attachment. xpaste.pro paste URLs are
+normalized to their `/raw` endpoint, including localized `/ru/p/<id>` and
+`/en/p/<id>` URLs.
+
+Plain GitHub repository URLs return a compact `.txt` snapshot with repository
+metadata, the root file tree, and README content.
+
+| Parameter | Required | Description  | Default |
+| --------- | -------- | ------------ | ------- |
+| `url`     | Yes      | URL to fetch | -       |
+
 ### Image Endpoint
 
 ```
@@ -334,6 +372,7 @@ web-capture <url> [options]
 | ----------------- | ---------------------------------------- |
 | `markdown` / `md` | Markdown conversion (default)            |
 | `html`            | Rendered HTML                            |
+| `txt` / `text`    | Raw text download                        |
 | `image` / `png`   | PNG screenshot (lossless)                |
 | `jpeg`            | JPEG screenshot (configurable quality)   |
 | `pdf`             | PDF with embedded images                 |
@@ -463,6 +502,9 @@ with environment variables:
 # Download the Wikipedia page (markdown + image) in every supported engine
 WIKIPEDIA_INTEGRATION=true npm test -- --testPathPattern="wikipedia-download"
 
+# Download a GitHub repository page as compact txt/markdown, original HTML, and screenshots
+GITHUB_REPOSITORY_INTEGRATION=true npm test -- --testPathPattern="github-readme"
+
 # Habr articles and public Google Docs live suites
 HABR_INTEGRATION=true npm test -- --testPathPattern="habr-article"
 GDOCS_INTEGRATION=true npm test -- --testPathPattern="gdocs-public-doc"

package/bin/web-capture.js

@@ -78,7 +78,7 @@ const config = makeConfig({
         alias: 'f',
         type: 'string',
         description:
-          'Output format: markdown, md, html, image, png, jpeg, pdf, docx, archive',
+          'Output format: markdown, md, html, txt, text, image, png, jpeg, pdf, docx, archive',
         default: 'markdown',
       })
       .option('theme', {
@@ -289,7 +289,7 @@ const config = makeConfig({
         'Capture private Google Doc with API token'
       )
       .epilogue(
-        'API Endpoints (in server mode):\n  GET /html?url=<URL>&engine=<ENGINE>           Get rendered HTML\n  GET /markdown?url=<URL>                       Get Markdown conversion\n  GET /image?url=<URL>&format=png|jpeg&theme=light|dark  Screenshot\n  GET /archive?url=<URL>&localImages=true&documentFormat=markdown|html  ZIP archive\n  GET /pdf?url=<URL>&theme=light|dark           PDF with embedded images\n  GET /docx?url=<URL>                           DOCX with embedded images\n  GET /fetch?url=<URL>                          Proxy fetch\n  GET /stream?url=<URL>                         Streaming proxy'
+        'API Endpoints (in server mode):\n  GET /html?url=<URL>&engine=<ENGINE>           Get rendered HTML\n  GET /markdown?url=<URL>                       Get Markdown conversion\n  GET /txt?url=<URL>                            Get text content\n  GET /image?url=<URL>&format=png|jpeg&theme=light|dark  Screenshot\n  GET /archive?url=<URL>&localImages=true&documentFormat=markdown|html  ZIP archive\n  GET /pdf?url=<URL>&theme=light|dark           PDF with embedded images\n  GET /docx?url=<URL>                           DOCX with embedded images\n  GET /fetch?url=<URL>                          Proxy fetch\n  GET /stream?url=<URL>                         Streaming proxy'
       )
       .strict(),
   lenv: {
@@ -322,6 +322,7 @@ async function startServer(port) {
       console.log('Available endpoints:');
       console.log(`  GET /html?url=<URL>       - Render page as HTML`);
       console.log(`  GET /markdown?url=<URL>   - Convert page to Markdown`);
+      console.log(`  GET /txt?url=<URL>        - Fetch text content`);
       console.log(`  GET /image?url=<URL>      - Screenshot (PNG/JPEG)`);
       console.log(
         `  GET /archive?url=<URL>    - ZIP archive with markdown + images`
@@ -598,8 +599,12 @@ async function captureUrl(url, options) {
   }
 
   // Import required modules
-  const { fetchHtml, convertToUtf8, convertRelativeUrls } =
-    await import('../src/lib.js');
+  const {
+    fetchHtml,
+    convertToUtf8,
+    convertRelativeUrls,
+    normalizeUrlForTextContent,
+  } = await import('../src/lib.js');
   const { createBrowser } = await import('../src/browser.js');
   const {
     isGoogleDocsUrl,
@@ -609,6 +614,12 @@ async function captureUrl(url, options) {
     captureGoogleDocWithBrowserOrFallback,
     selectGoogleDocsCaptureMethod,
   } = await import('../src/gdocs.js');
+  const {
+    fetchGithubRepositorySnapshot,
+    formatGithubRepositoryMarkdown,
+    formatGithubRepositoryText,
+    isGithubRepositoryUrl,
+  } = await import('../src/github.js');
 
   const normalizedFormat = format.toLowerCase();
   log.debug(() => ({
@@ -915,7 +926,35 @@ async function captureUrl(url, options) {
   }
 
   try {
-    if (normalizedFormat === 'jpeg') {
+    if (normalizedFormat === 'txt' || normalizedFormat === 'text') {
+      let text;
+      if (isGithubRepositoryUrl(absoluteUrl)) {
+        const snapshot = await fetchGithubRepositorySnapshot(absoluteUrl);
+        text = formatGithubRepositoryText(snapshot);
+      } else {
+        const response = await fetch(normalizeUrlForTextContent(absoluteUrl));
+        if (!response.ok) {
+          throw new Error(`HTTP ${response.status}: ${response.statusText}`);
+        }
+        const contentType =
+          response.headers.get('content-type') || 'text/plain';
+        if (!contentType.includes('text/')) {
+          throw new Error(`Expected text content, got ${contentType}`);
+        }
+        text = await response.text();
+      }
+      const output =
+        explicitOutput === '-'
+          ? null
+          : explicitOutput || deriveOutputPath(absoluteUrl, 'txt', dataDir);
+      if (output) {
+        fs.mkdirSync(path.dirname(output), { recursive: true });
+        fs.writeFileSync(output, text, 'utf-8');
+        console.error(`Text saved to: ${output}`);
+      } else {
+        process.stdout.write(text);
+      }
+    } else if (normalizedFormat === 'jpeg') {
       // JPEG screenshot
       const { createBrowser } = await import('../src/browser.js');
       const { dismissPopups, scrollToLoadContent } =
@@ -1125,17 +1164,27 @@ async function captureUrl(url, options) {
       console.error(`Archive saved to: ${outPath}`);
     } else if (normalizedFormat === 'markdown' || normalizedFormat === 'md') {
       // Markdown format — enhanced conversion is now the default
-      const html = await fetchHtml(absoluteUrl);
-      const { convertHtmlToMarkdownEnhanced } = await import('../src/lib.js');
-      const result = convertHtmlToMarkdownEnhanced(html, absoluteUrl, {
-        extractLatex: options.extractLatex,
-        extractMetadata: options.extractMetadata,
-        postProcess: options.postProcess,
-        detectCodeLanguage: options.detectCodeLanguage,
-        contentSelector: options.contentSelector,
-        bodySelector: options.bodySelector,
-      });
-      const markdown = result.markdown;
+      let markdown;
+      if (
+        isGithubRepositoryUrl(absoluteUrl) &&
+        !options.contentSelector &&
+        !options.bodySelector
+      ) {
+        const snapshot = await fetchGithubRepositorySnapshot(absoluteUrl);
+        markdown = formatGithubRepositoryMarkdown(snapshot);
+      } else {
+        const html = await fetchHtml(absoluteUrl);
+        const { convertHtmlToMarkdownEnhanced } = await import('../src/lib.js');
+        const result = convertHtmlToMarkdownEnhanced(html, absoluteUrl, {
+          extractLatex: options.extractLatex,
+          extractMetadata: options.extractMetadata,
+          postProcess: options.postProcess,
+          detectCodeLanguage: options.detectCodeLanguage,
+          contentSelector: options.contentSelector,
+          bodySelector: options.bodySelector,
+        });
+        markdown = result.markdown;
+      }
 
       const output =
         explicitOutput === '-'
@@ -1160,6 +1209,8 @@ async function captureUrl(url, options) {
       normalizedFormat === 'screenshot'
     ) {
       // Image/screenshot format
+      const { dismissPopups, scrollToLoadContent } =
+        await import('../src/popups.js');
       const browser = await createBrowser(engine);
       try {
         const page = await browser.newPage();
@@ -1177,8 +1228,12 @@ async function captureUrl(url, options) {
         });
         // Wait for 5 seconds after page load
         await new Promise((resolve) => setTimeout(resolve, 5000));
+        if (fullPage) {
+          await scrollToLoadContent(page);
+        }
+        await dismissPopups(page);
 
-        const buffer = await page.screenshot({ type: 'png' });
+        const buffer = await page.screenshot({ type: 'png', fullPage });
 
         if (explicitOutput) {
           fs.writeFileSync(explicitOutput, buffer);

package/jest.config.mjs

@@ -13,6 +13,7 @@ export default {
     '**/tests/mock/**/*.test.js',
     '**/tests/e2e/**/*.test.js',
     '**/tests/integration/**/*.test.js',
+    '**/tests/xpaste/**/*.test.js',
   ],
   setupFiles: ['./tests/jest.setup.mjs'],
 };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@link-assistant/web-capture",
-  "version": "1.10.1",
+  "version": "1.10.3",
   "description": "CLI and microservice to render web pages as HTML, Markdown, or PNG",
   "main": "src/index.js",
   "bin": {

package/src/github.js

@@ -0,0 +1,285 @@
+import fetch from 'node-fetch';
+import { URL } from 'node:url';
+
+const GITHUB_API_BASE = 'https://api.github.com';
+const GITHUB_USER_AGENT = 'web-capture';
+
+export function parseGithubRepositoryUrl(url) {
+  try {
+    const parsed = new URL(url);
+    const host = parsed.hostname.toLowerCase();
+    if (host !== 'github.com' && host !== 'www.github.com') {
+      return null;
+    }
+
+    const parts = parsed.pathname.split('/').filter(Boolean);
+    if (parts.length !== 2) {
+      return null;
+    }
+
+    const [owner, repo] = parts;
+    if (!owner || !repo) {
+      return null;
+    }
+
+    return {
+      owner,
+      repo,
+      fullName: `${owner}/${repo}`,
+      htmlUrl: `https://github.com/${owner}/${repo}`,
+    };
+  } catch {
+    return null;
+  }
+}
+
+export function isGithubRepositoryUrl(url) {
+  return Boolean(parseGithubRepositoryUrl(url));
+}
+
+export function getGithubRepositoryTextFilename(url) {
+  const parsed = parseGithubRepositoryUrl(url);
+  return parsed ? `${parsed.owner}-${parsed.repo}.txt` : null;
+}
+
+export async function fetchGithubRepositorySnapshot(url) {
+  const parsed = parseGithubRepositoryUrl(url);
+  if (!parsed) {
+    throw new Error(`Not a GitHub repository URL: ${url}`);
+  }
+
+  const repository = await fetchGithubJson(
+    `${GITHUB_API_BASE}/repos/${encodePath(parsed.owner)}/${encodePath(parsed.repo)}`
+  );
+  const defaultBranch = repository.default_branch || 'main';
+
+  const [readme, tree] = await Promise.all([
+    fetchGithubReadme(parsed, defaultBranch),
+    fetchGithubRootTree(parsed, defaultBranch),
+  ]);
+
+  return {
+    sourceUrl: parsed.htmlUrl,
+    repository,
+    defaultBranch,
+    readme,
+    tree,
+  };
+}
+
+export function formatGithubRepositoryMarkdown(snapshot) {
+  const { repository, defaultBranch, readme, tree, sourceUrl } = snapshot;
+  const lines = [
+    `# ${repository.full_name}`,
+    '',
+    repository.description ? `> ${repository.description}` : null,
+    '',
+    '## Repository',
+    '',
+    `- URL: ${repository.html_url || sourceUrl}`,
+    `- Default branch: \`${defaultBranch}\``,
+    repository.language ? `- Primary language: ${repository.language}` : null,
+    numberLine('Stars', repository.stargazers_count),
+    numberLine('Forks', repository.forks_count),
+    numberLine('Open issues', repository.open_issues_count),
+    repository.license?.spdx_id
+      ? `- License: ${repository.license.spdx_id}`
+      : null,
+    topicsLine(repository.topics),
+    '',
+    '## Files',
+    '',
+  ].filter((line) => line !== null);
+
+  if (tree.length > 0) {
+    for (const item of tree) {
+      const label = item.type === 'dir' ? `${item.name}/` : item.name;
+      const suffix =
+        item.type === 'file' && typeof item.size === 'number'
+          ? ` (${formatBytes(item.size)})`
+          : '';
+      lines.push(`- [${label}](${item.html_url})${suffix}`);
+    }
+  } else {
+    lines.push('- No root files returned by the GitHub API.');
+  }
+
+  lines.push('', `## ${readme?.path || 'README'}`, '');
+  if (readme?.content) {
+    lines.push(readme.content.trimEnd(), '');
+  } else {
+    lines.push('README content was not returned by the GitHub API.', '');
+  }
+
+  return lines.join('\n');
+}
+
+export function formatGithubRepositoryText(snapshot) {
+  const { repository, defaultBranch, readme, tree, sourceUrl } = snapshot;
+  const lines = [
+    `Repository: ${repository.full_name}`,
+    repository.description ? `Description: ${repository.description}` : null,
+    `URL: ${repository.html_url || sourceUrl}`,
+    `Default branch: ${defaultBranch}`,
+    repository.language ? `Primary language: ${repository.language}` : null,
+    plainNumberLine('Stars', repository.stargazers_count),
+    plainNumberLine('Forks', repository.forks_count),
+    plainNumberLine('Open issues', repository.open_issues_count),
+    repository.license?.spdx_id
+      ? `License: ${repository.license.spdx_id}`
+      : null,
+    repository.topics?.length
+      ? `Topics: ${repository.topics.join(', ')}`
+      : null,
+    '',
+    'Files:',
+  ].filter((line) => line !== null);
+
+  if (tree.length > 0) {
+    for (const item of tree) {
+      const label = item.type === 'dir' ? `${item.name}/` : item.name;
+      const suffix =
+        item.type === 'file' && typeof item.size === 'number'
+          ? ` (${formatBytes(item.size)})`
+          : '';
+      lines.push(`- ${label}${suffix}`);
+    }
+  } else {
+    lines.push('- No root files returned by the GitHub API.');
+  }
+
+  lines.push('', `${readme?.path || 'README'}:`, '');
+  if (readme?.content) {
+    lines.push(readme.content.trimEnd(), '');
+  } else {
+    lines.push('README content was not returned by the GitHub API.', '');
+  }
+
+  return lines.join('\n');
+}
+
+async function fetchGithubReadme(parsed, defaultBranch) {
+  const readme = await fetchGithubJson(
+    `${GITHUB_API_BASE}/repos/${encodePath(parsed.owner)}/${encodePath(parsed.repo)}/readme?ref=${encodeURIComponent(defaultBranch)}`,
+    { optional: true }
+  );
+  if (!readme) {
+    return null;
+  }
+
+  let content = null;
+  if (readme.content && readme.encoding === 'base64') {
+    content = Buffer.from(
+      readme.content.replace(/\s+/g, ''),
+      'base64'
+    ).toString('utf8');
+  } else if (readme.download_url) {
+    content = await fetchGithubText(readme.download_url, { optional: true });
+  }
+
+  return {
+    name: readme.name || 'README',
+    path: readme.path || readme.name || 'README',
+    htmlUrl: readme.html_url,
+    content,
+  };
+}
+
+async function fetchGithubRootTree(parsed, defaultBranch) {
+  const tree = await fetchGithubJson(
+    `${GITHUB_API_BASE}/repos/${encodePath(parsed.owner)}/${encodePath(parsed.repo)}/contents?ref=${encodeURIComponent(defaultBranch)}`,
+    { optional: true }
+  );
+  if (!Array.isArray(tree)) {
+    return [];
+  }
+
+  return tree
+    .map((item) => {
+      const pathKind = item.type === 'dir' ? 'tree' : 'blob';
+      return {
+        name: item.name,
+        path: item.path,
+        type: item.type,
+        size: item.size,
+        html_url:
+          item.html_url ||
+          `https://github.com/${parsed.owner}/${parsed.repo}/${pathKind}/${defaultBranch}/${item.path}`,
+      };
+    })
+    .sort((a, b) => {
+      if (a.type !== b.type) {
+        return a.type === 'dir' ? -1 : 1;
+      }
+      return a.name.localeCompare(b.name);
+    });
+}
+
+async function fetchGithubJson(url, options = {}) {
+  const response = await fetch(url, {
+    headers: githubHeaders('application/vnd.github+json'),
+  });
+  if (options.optional && response.status === 404) {
+    return null;
+  }
+  if (!response.ok) {
+    throw new Error(`GitHub API ${response.status}: ${response.statusText}`);
+  }
+  return response.json();
+}
+
+async function fetchGithubText(url, options = {}) {
+  const response = await fetch(url, {
+    headers: githubHeaders('text/plain'),
+  });
+  if (options.optional && response.status === 404) {
+    return null;
+  }
+  if (!response.ok) {
+    throw new Error(`GitHub raw ${response.status}: ${response.statusText}`);
+  }
+  return response.text();
+}
+
+function githubHeaders(accept) {
+  const headers = {
+    Accept: accept,
+    'User-Agent': GITHUB_USER_AGENT,
+    'X-GitHub-Api-Version': '2022-11-28',
+  };
+  const token = process.env.GITHUB_TOKEN || process.env.GH_TOKEN;
+  if (token) {
+    headers.Authorization = `Bearer ${token}`;
+  }
+  return headers;
+}
+
+function encodePath(value) {
+  return encodeURIComponent(value);
+}
+
+function numberLine(label, value) {
+  return typeof value === 'number'
+    ? `- ${label}: ${value.toLocaleString()}`
+    : null;
+}
+
+function plainNumberLine(label, value) {
+  return typeof value === 'number'
+    ? `${label}: ${value.toLocaleString()}`
+    : null;
+}
+
+function topicsLine(topics) {
+  return topics?.length ? `- Topics: ${topics.join(', ')}` : null;
+}
+
+function formatBytes(size) {
+  if (size < 1024) {
+    return `${size} B`;
+  }
+  if (size < 1024 * 1024) {
+    return `${(size / 1024).toFixed(1)} KB`;
+  }
+  return `${(size / (1024 * 1024)).toFixed(1)} MB`;
+}

package/src/index.js

@@ -5,6 +5,7 @@ import { markdownHandler } from './markdown.js';
 import { imageHandler } from './image.js';
 import { streamHandler } from './stream.js';
 import { fetchHandler } from './fetch.js';
+import { txtHandler } from './txt.js';
 import { archiveHandler } from './archive.js';
 import { pdfHandler } from './pdf.js';
 import { docxHandler } from './docx.js';
@@ -22,6 +23,7 @@ app.get('/markdown', markdownHandler);
 app.get('/image', imageHandler);
 app.get('/stream', streamHandler);
 app.get('/fetch', fetchHandler);
+app.get('/txt', txtHandler);
 app.get('/archive', archiveHandler);
 app.get('/pdf', pdfHandler);
 app.get('/docx', docxHandler);