@link-assistant/web-capture 1.10.2 → 1.10.3

package/CHANGELOG.md

@@ -1,5 +1,11 @@
 # @link-assistant/web-capture
 
+## 1.10.3
+
+### Patch Changes
+
+- Add compact GitHub repository capture for txt and markdown output, including repository metadata, the root file tree, and README content.
+
 ## 1.10.2
 
 ### Patch Changes

package/README.md

@@ -52,6 +52,10 @@ web-capture https://example.com --format html -o page.html
 # Capture raw paste text
 web-capture https://xpaste.pro/p/t4q0Lsp0 --format txt -o paste.txt
 
+# Capture a GitHub repository as compact text or Markdown
+web-capture https://github.com/link-assistant/web-capture --format txt -o repository.txt
+web-capture https://github.com/link-assistant/web-capture --format markdown -o repository.md
+
 # Take a PNG screenshot
 web-capture https://example.com --format png -o screenshot.png
 
@@ -153,6 +157,11 @@ containing `index.md`, `xpaste-pro-<id>.md`, and `xpaste-pro-<id>.txt`.
 Canonical `/p/<id>`, `/p/<id>/raw`, `/ru/p/<id>`, and `/en/p/<id>` URLs are
 normalized before capture.
 
+For plain GitHub repository URLs such as `https://github.com/owner/repo`,
+`/markdown` returns a compact repository snapshot with repository metadata, the
+root file tree, and README content. GitHub subpages continue through the regular
+HTML-to-Markdown conversion path.
+
 | Parameter           | Required | Description                                                               | Default  |
 | ------------------- | -------- | ------------------------------------------------------------------------- | -------- |
 | `url`               | Yes      | URL to fetch                                                              | -        |
@@ -171,6 +180,9 @@ Returns raw text content as a `.txt` attachment. xpaste.pro paste URLs are
 normalized to their `/raw` endpoint, including localized `/ru/p/<id>` and
 `/en/p/<id>` URLs.
 
+Plain GitHub repository URLs return a compact `.txt` snapshot with repository
+metadata, the root file tree, and README content.
+
 | Parameter | Required | Description  | Default |
 | --------- | -------- | ------------ | ------- |
 | `url`     | Yes      | URL to fetch | -       |
@@ -490,6 +502,9 @@ with environment variables:
 # Download the Wikipedia page (markdown + image) in every supported engine
 WIKIPEDIA_INTEGRATION=true npm test -- --testPathPattern="wikipedia-download"
 
+# Download a GitHub repository page as compact txt/markdown, original HTML, and screenshots
+GITHUB_REPOSITORY_INTEGRATION=true npm test -- --testPathPattern="github-readme"
+
 # Habr articles and public Google Docs live suites
 HABR_INTEGRATION=true npm test -- --testPathPattern="habr-article"
 GDOCS_INTEGRATION=true npm test -- --testPathPattern="gdocs-public-doc"

package/bin/web-capture.js

@@ -614,6 +614,12 @@ async function captureUrl(url, options) {
     captureGoogleDocWithBrowserOrFallback,
     selectGoogleDocsCaptureMethod,
   } = await import('../src/gdocs.js');
+  const {
+    fetchGithubRepositorySnapshot,
+    formatGithubRepositoryMarkdown,
+    formatGithubRepositoryText,
+    isGithubRepositoryUrl,
+  } = await import('../src/github.js');
 
   const normalizedFormat = format.toLowerCase();
   log.debug(() => ({
@@ -921,15 +927,22 @@ async function captureUrl(url, options) {
 
   try {
     if (normalizedFormat === 'txt' || normalizedFormat === 'text') {
-      const response = await fetch(normalizeUrlForTextContent(absoluteUrl));
-      if (!response.ok) {
-        throw new Error(`HTTP ${response.status}: ${response.statusText}`);
-      }
-      const contentType = response.headers.get('content-type') || 'text/plain';
-      if (!contentType.includes('text/')) {
-        throw new Error(`Expected text content, got ${contentType}`);
+      let text;
+      if (isGithubRepositoryUrl(absoluteUrl)) {
+        const snapshot = await fetchGithubRepositorySnapshot(absoluteUrl);
+        text = formatGithubRepositoryText(snapshot);
+      } else {
+        const response = await fetch(normalizeUrlForTextContent(absoluteUrl));
+        if (!response.ok) {
+          throw new Error(`HTTP ${response.status}: ${response.statusText}`);
+        }
+        const contentType =
+          response.headers.get('content-type') || 'text/plain';
+        if (!contentType.includes('text/')) {
+          throw new Error(`Expected text content, got ${contentType}`);
+        }
+        text = await response.text();
       }
-      const text = await response.text();
       const output =
         explicitOutput === '-'
           ? null
@@ -1151,17 +1164,27 @@ async function captureUrl(url, options) {
       console.error(`Archive saved to: ${outPath}`);
     } else if (normalizedFormat === 'markdown' || normalizedFormat === 'md') {
       // Markdown format — enhanced conversion is now the default
-      const html = await fetchHtml(absoluteUrl);
-      const { convertHtmlToMarkdownEnhanced } = await import('../src/lib.js');
-      const result = convertHtmlToMarkdownEnhanced(html, absoluteUrl, {
-        extractLatex: options.extractLatex,
-        extractMetadata: options.extractMetadata,
-        postProcess: options.postProcess,
-        detectCodeLanguage: options.detectCodeLanguage,
-        contentSelector: options.contentSelector,
-        bodySelector: options.bodySelector,
-      });
-      const markdown = result.markdown;
+      let markdown;
+      if (
+        isGithubRepositoryUrl(absoluteUrl) &&
+        !options.contentSelector &&
+        !options.bodySelector
+      ) {
+        const snapshot = await fetchGithubRepositorySnapshot(absoluteUrl);
+        markdown = formatGithubRepositoryMarkdown(snapshot);
+      } else {
+        const html = await fetchHtml(absoluteUrl);
+        const { convertHtmlToMarkdownEnhanced } = await import('../src/lib.js');
+        const result = convertHtmlToMarkdownEnhanced(html, absoluteUrl, {
+          extractLatex: options.extractLatex,
+          extractMetadata: options.extractMetadata,
+          postProcess: options.postProcess,
+          detectCodeLanguage: options.detectCodeLanguage,
+          contentSelector: options.contentSelector,
+          bodySelector: options.bodySelector,
+        });
+        markdown = result.markdown;
+      }
 
       const output =
         explicitOutput === '-'

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@link-assistant/web-capture",
-  "version": "1.10.2",
+  "version": "1.10.3",
   "description": "CLI and microservice to render web pages as HTML, Markdown, or PNG",
   "main": "src/index.js",
   "bin": {

package/src/github.js

@@ -0,0 +1,285 @@
+import fetch from 'node-fetch';
+import { URL } from 'node:url';
+
+const GITHUB_API_BASE = 'https://api.github.com';
+const GITHUB_USER_AGENT = 'web-capture';
+
+export function parseGithubRepositoryUrl(url) {
+  try {
+    const parsed = new URL(url);
+    const host = parsed.hostname.toLowerCase();
+    if (host !== 'github.com' && host !== 'www.github.com') {
+      return null;
+    }
+
+    const parts = parsed.pathname.split('/').filter(Boolean);
+    if (parts.length !== 2) {
+      return null;
+    }
+
+    const [owner, repo] = parts;
+    if (!owner || !repo) {
+      return null;
+    }
+
+    return {
+      owner,
+      repo,
+      fullName: `${owner}/${repo}`,
+      htmlUrl: `https://github.com/${owner}/${repo}`,
+    };
+  } catch {
+    return null;
+  }
+}
+
+export function isGithubRepositoryUrl(url) {
+  return Boolean(parseGithubRepositoryUrl(url));
+}
+
+export function getGithubRepositoryTextFilename(url) {
+  const parsed = parseGithubRepositoryUrl(url);
+  return parsed ? `${parsed.owner}-${parsed.repo}.txt` : null;
+}
+
+export async function fetchGithubRepositorySnapshot(url) {
+  const parsed = parseGithubRepositoryUrl(url);
+  if (!parsed) {
+    throw new Error(`Not a GitHub repository URL: ${url}`);
+  }
+
+  const repository = await fetchGithubJson(
+    `${GITHUB_API_BASE}/repos/${encodePath(parsed.owner)}/${encodePath(parsed.repo)}`
+  );
+  const defaultBranch = repository.default_branch || 'main';
+
+  const [readme, tree] = await Promise.all([
+    fetchGithubReadme(parsed, defaultBranch),
+    fetchGithubRootTree(parsed, defaultBranch),
+  ]);
+
+  return {
+    sourceUrl: parsed.htmlUrl,
+    repository,
+    defaultBranch,
+    readme,
+    tree,
+  };
+}
+
+export function formatGithubRepositoryMarkdown(snapshot) {
+  const { repository, defaultBranch, readme, tree, sourceUrl } = snapshot;
+  const lines = [
+    `# ${repository.full_name}`,
+    '',
+    repository.description ? `> ${repository.description}` : null,
+    '',
+    '## Repository',
+    '',
+    `- URL: ${repository.html_url || sourceUrl}`,
+    `- Default branch: \`${defaultBranch}\``,
+    repository.language ? `- Primary language: ${repository.language}` : null,
+    numberLine('Stars', repository.stargazers_count),
+    numberLine('Forks', repository.forks_count),
+    numberLine('Open issues', repository.open_issues_count),
+    repository.license?.spdx_id
+      ? `- License: ${repository.license.spdx_id}`
+      : null,
+    topicsLine(repository.topics),
+    '',
+    '## Files',
+    '',
+  ].filter((line) => line !== null);
+
+  if (tree.length > 0) {
+    for (const item of tree) {
+      const label = item.type === 'dir' ? `${item.name}/` : item.name;
+      const suffix =
+        item.type === 'file' && typeof item.size === 'number'
+          ? ` (${formatBytes(item.size)})`
+          : '';
+      lines.push(`- [${label}](${item.html_url})${suffix}`);
+    }
+  } else {
+    lines.push('- No root files returned by the GitHub API.');
+  }
+
+  lines.push('', `## ${readme?.path || 'README'}`, '');
+  if (readme?.content) {
+    lines.push(readme.content.trimEnd(), '');
+  } else {
+    lines.push('README content was not returned by the GitHub API.', '');
+  }
+
+  return lines.join('\n');
+}
+
+export function formatGithubRepositoryText(snapshot) {
+  const { repository, defaultBranch, readme, tree, sourceUrl } = snapshot;
+  const lines = [
+    `Repository: ${repository.full_name}`,
+    repository.description ? `Description: ${repository.description}` : null,
+    `URL: ${repository.html_url || sourceUrl}`,
+    `Default branch: ${defaultBranch}`,
+    repository.language ? `Primary language: ${repository.language}` : null,
+    plainNumberLine('Stars', repository.stargazers_count),
+    plainNumberLine('Forks', repository.forks_count),
+    plainNumberLine('Open issues', repository.open_issues_count),
+    repository.license?.spdx_id
+      ? `License: ${repository.license.spdx_id}`
+      : null,
+    repository.topics?.length
+      ? `Topics: ${repository.topics.join(', ')}`
+      : null,
+    '',
+    'Files:',
+  ].filter((line) => line !== null);
+
+  if (tree.length > 0) {
+    for (const item of tree) {
+      const label = item.type === 'dir' ? `${item.name}/` : item.name;
+      const suffix =
+        item.type === 'file' && typeof item.size === 'number'
+          ? ` (${formatBytes(item.size)})`
+          : '';
+      lines.push(`- ${label}${suffix}`);
+    }
+  } else {
+    lines.push('- No root files returned by the GitHub API.');
+  }
+
+  lines.push('', `${readme?.path || 'README'}:`, '');
+  if (readme?.content) {
+    lines.push(readme.content.trimEnd(), '');
+  } else {
+    lines.push('README content was not returned by the GitHub API.', '');
+  }
+
+  return lines.join('\n');
+}
+
+async function fetchGithubReadme(parsed, defaultBranch) {
+  const readme = await fetchGithubJson(
+    `${GITHUB_API_BASE}/repos/${encodePath(parsed.owner)}/${encodePath(parsed.repo)}/readme?ref=${encodeURIComponent(defaultBranch)}`,
+    { optional: true }
+  );
+  if (!readme) {
+    return null;
+  }
+
+  let content = null;
+  if (readme.content && readme.encoding === 'base64') {
+    content = Buffer.from(
+      readme.content.replace(/\s+/g, ''),
+      'base64'
+    ).toString('utf8');
+  } else if (readme.download_url) {
+    content = await fetchGithubText(readme.download_url, { optional: true });
+  }
+
+  return {
+    name: readme.name || 'README',
+    path: readme.path || readme.name || 'README',
+    htmlUrl: readme.html_url,
+    content,
+  };
+}
+
+async function fetchGithubRootTree(parsed, defaultBranch) {
+  const tree = await fetchGithubJson(
+    `${GITHUB_API_BASE}/repos/${encodePath(parsed.owner)}/${encodePath(parsed.repo)}/contents?ref=${encodeURIComponent(defaultBranch)}`,
+    { optional: true }
+  );
+  if (!Array.isArray(tree)) {
+    return [];
+  }
+
+  return tree
+    .map((item) => {
+      const pathKind = item.type === 'dir' ? 'tree' : 'blob';
+      return {
+        name: item.name,
+        path: item.path,
+        type: item.type,
+        size: item.size,
+        html_url:
+          item.html_url ||
+          `https://github.com/${parsed.owner}/${parsed.repo}/${pathKind}/${defaultBranch}/${item.path}`,
+      };
+    })
+    .sort((a, b) => {
+      if (a.type !== b.type) {
+        return a.type === 'dir' ? -1 : 1;
+      }
+      return a.name.localeCompare(b.name);
+    });
+}
+
+async function fetchGithubJson(url, options = {}) {
+  const response = await fetch(url, {
+    headers: githubHeaders('application/vnd.github+json'),
+  });
+  if (options.optional && response.status === 404) {
+    return null;
+  }
+  if (!response.ok) {
+    throw new Error(`GitHub API ${response.status}: ${response.statusText}`);
+  }
+  return response.json();
+}
+
+async function fetchGithubText(url, options = {}) {
+  const response = await fetch(url, {
+    headers: githubHeaders('text/plain'),
+  });
+  if (options.optional && response.status === 404) {
+    return null;
+  }
+  if (!response.ok) {
+    throw new Error(`GitHub raw ${response.status}: ${response.statusText}`);
+  }
+  return response.text();
+}
+
+function githubHeaders(accept) {
+  const headers = {
+    Accept: accept,
+    'User-Agent': GITHUB_USER_AGENT,
+    'X-GitHub-Api-Version': '2022-11-28',
+  };
+  const token = process.env.GITHUB_TOKEN || process.env.GH_TOKEN;
+  if (token) {
+    headers.Authorization = `Bearer ${token}`;
+  }
+  return headers;
+}
+
+function encodePath(value) {
+  return encodeURIComponent(value);
+}
+
+function numberLine(label, value) {
+  return typeof value === 'number'
+    ? `- ${label}: ${value.toLocaleString()}`
+    : null;
+}
+
+function plainNumberLine(label, value) {
+  return typeof value === 'number'
+    ? `${label}: ${value.toLocaleString()}`
+    : null;
+}
+
+function topicsLine(topics) {
+  return topics?.length ? `- Topics: ${topics.join(', ')}` : null;
+}
+
+function formatBytes(size) {
+  if (size < 1024) {
+    return `${size} B`;
+  }
+  if (size < 1024 * 1024) {
+    return `${(size / 1024).toFixed(1)} KB`;
+  }
+  return `${(size / (1024 * 1024)).toFixed(1)} MB`;
+}

package/src/markdown.js

@@ -8,6 +8,11 @@ import {
   normalizeUrlForTextPage,
   scopeHtmlForMarkdown,
 } from './lib.js';
+import {
+  fetchGithubRepositorySnapshot,
+  formatGithubRepositoryMarkdown,
+  isGithubRepositoryUrl,
+} from './github.js';
 import { convertWithKreuzberg, isKreuzbergAvailable } from './kreuzberg.js';
 import { applyImageMode } from './extract-images.js';
 import archiver from 'archiver';
@@ -39,6 +44,21 @@ export async function markdownHandler(req, res) {
 
   try {
     const pageUrl = normalizeUrlForTextPage(url);
+    if (
+      format === 'text' &&
+      !req.query.contentSelector &&
+      !req.query.bodySelector &&
+      isGithubRepositoryUrl(pageUrl)
+    ) {
+      const snapshot = await fetchGithubRepositorySnapshot(pageUrl);
+      let markdown = formatGithubRepositoryMarkdown(snapshot);
+      const result = await applyImageMode(markdown, {
+        mode: embedImages ? 'embed' : 'default',
+      });
+      markdown = result.markdown;
+      return await sendMarkdownResponse(res, url, markdown);
+    }
+
     const html = await fetchHtml(pageUrl);
 
     if (converter === 'kreuzberg') {

package/src/txt.js

@@ -1,6 +1,12 @@
 import fetch from 'node-fetch';
 import { URL } from 'node:url';
 import { getTextPasteFilename, normalizeUrlForTextContent } from './lib.js';
+import {
+  fetchGithubRepositorySnapshot,
+  formatGithubRepositoryText,
+  getGithubRepositoryTextFilename,
+  isGithubRepositoryUrl,
+} from './github.js';
 
 export async function txtHandler(req, res) {
   const url = req.query.url;
@@ -9,6 +15,12 @@ export async function txtHandler(req, res) {
   }
 
   try {
+    if (isGithubRepositoryUrl(url)) {
+      const snapshot = await fetchGithubRepositorySnapshot(url);
+      const text = formatGithubRepositoryText(snapshot);
+      return sendTextResponse(res, url, text);
+    }
+
     // Normalize URL to get text content (e.g., xpaste.pro -> xpaste.pro/raw)
     const textUrl = normalizeUrlForTextContent(url);
 
@@ -27,20 +39,28 @@ export async function txtHandler(req, res) {
 
     const text = await response.text();
 
-    // Set appropriate headers for text file download
-    res.setHeader('Content-Type', 'text/plain; charset=utf-8');
-    res.setHeader(
-      'Content-Disposition',
-      `attachment; filename="${getFilenameFromUrl(url)}"`
-    );
-    res.send(text);
+    sendTextResponse(res, url, text);
   } catch (err) {
     console.error('Text fetch error:', err);
     res.status(500).send('Error fetching text content');
   }
 }
 
+function sendTextResponse(res, url, text) {
+  res.setHeader('Content-Type', 'text/plain; charset=utf-8');
+  res.setHeader(
+    'Content-Disposition',
+    `attachment; filename="${getFilenameFromUrl(url)}"`
+  );
+  res.send(text);
+}
+
 function getFilenameFromUrl(url) {
+  const githubRepositoryFilename = getGithubRepositoryTextFilename(url);
+  if (githubRepositoryFilename) {
+    return githubRepositoryFilename;
+  }
+
   const textPasteFilename = getTextPasteFilename(url);
   if (textPasteFilename) {
     return textPasteFilename;

package/tests/integration/api-endpoints.test.js

@@ -25,6 +25,50 @@ const MOCK_HTML = `<!DOCTYPE html>
 <img src="https://example.com/image.png" alt="test">
 </body></html>`;
 
+function mockGithubRepository(owner = 'octocat', repo = 'Hello-World') {
+  const readme = '# Hello World\n\nThis README came from GitHub.';
+  nock('https://api.github.com')
+    .get(`/repos/${owner}/${repo}`)
+    .reply(200, {
+      full_name: `${owner}/${repo}`,
+      description: 'A friendly test repository',
+      html_url: `https://github.com/${owner}/${repo}`,
+      default_branch: 'master',
+      language: 'JavaScript',
+      stargazers_count: 42,
+      forks_count: 7,
+      open_issues_count: 3,
+      license: { spdx_id: 'MIT' },
+      topics: ['demo', 'capture'],
+    })
+    .get(`/repos/${owner}/${repo}/readme`)
+    .query({ ref: 'master' })
+    .reply(200, {
+      name: 'README.md',
+      path: 'README.md',
+      encoding: 'base64',
+      content: Buffer.from(readme, 'utf8').toString('base64'),
+      html_url: `https://github.com/${owner}/${repo}/blob/master/README.md`,
+    })
+    .get(`/repos/${owner}/${repo}/contents`)
+    .query({ ref: 'master' })
+    .reply(200, [
+      {
+        name: 'src',
+        path: 'src',
+        type: 'dir',
+        html_url: `https://github.com/${owner}/${repo}/tree/master/src`,
+      },
+      {
+        name: 'README.md',
+        path: 'README.md',
+        type: 'file',
+        size: readme.length,
+        html_url: `https://github.com/${owner}/${repo}/blob/master/README.md`,
+      },
+    ]);
+}
+
 afterEach(() => {
   nock.cleanAll();
 });
@@ -70,6 +114,44 @@ describe('API Endpoint Tests', () => {
       expect(res.text).toContain('Hello world');
     });
 
+    it('returns a compact GitHub repository markdown snapshot', async () => {
+      mockGithubRepository();
+
+      const res = await request(app)
+        .get('/markdown')
+        .query({ url: 'https://github.com/octocat/Hello-World' })
+        .expect(200);
+
+      expect(res.headers['content-type']).toMatch(/text\/markdown/);
+      expect(res.text).toContain('# octocat/Hello-World');
+      expect(res.text).toContain('## Repository');
+      expect(res.text).toContain('## Files');
+      expect(res.text).toContain('- [src/](');
+      expect(res.text).toContain('## README.md');
+      expect(res.text).toContain('This README came from GitHub.');
+      expect(res.text).not.toContain('Skip to content');
+      expect(nock.isDone()).toBe(true);
+    });
+
+    it('returns a compact GitHub repository markdown snapshot for kreuzberg text format', async () => {
+      mockGithubRepository();
+
+      const res = await request(app)
+        .get('/markdown')
+        .query({
+          url: 'https://github.com/octocat/Hello-World',
+          converter: 'kreuzberg',
+          format: 'text',
+        })
+        .expect(200);
+
+      expect(res.headers['content-type']).toMatch(/text\/markdown/);
+      expect(res.text).toContain('# octocat/Hello-World');
+      expect(res.text).toContain('## Files');
+      expect(res.text).toContain('This README came from GitHub.');
+      expect(nock.isDone()).toBe(true);
+    });
+
     it('rejects unsupported converter names', async () => {
       await request(app)
         .get('/markdown')
@@ -133,6 +215,28 @@ describe('API Endpoint Tests', () => {
     });
   });
 
+  describe('GET /txt', () => {
+    it('returns a compact GitHub repository plain-text snapshot', async () => {
+      mockGithubRepository();
+
+      const res = await request(app)
+        .get('/txt')
+        .query({ url: 'https://github.com/octocat/Hello-World' })
+        .expect(200);
+
+      expect(res.headers['content-type']).toMatch(/text\/plain/);
+      expect(res.headers['content-disposition']).toContain(
+        'filename="octocat-Hello-World.txt"'
+      );
+      expect(res.text).toContain('Repository: octocat/Hello-World');
+      expect(res.text).toContain('Files:');
+      expect(res.text).toContain('- src/');
+      expect(res.text).toContain('README.md:');
+      expect(res.text).toContain('This README came from GitHub.');
+      expect(nock.isDone()).toBe(true);
+    });
+  });
+
   describe('GET /archive', () => {
     it('returns a ZIP archive with remote images (markdown format)', async () => {
       nock('https://example.com').get('/archive-test').reply(200, MOCK_HTML, {

package/tests/integration/github-readme.test.js

@@ -0,0 +1,137 @@
+/**
+ * Integration tests for capturing GitHub repository pages (issue #5).
+ *
+ * Live tests are gated behind GITHUB_REPOSITORY_INTEGRATION so default/offline
+ * runs stay deterministic. The mocked API endpoint tests cover the default
+ * behavior without network access; this suite proves the real GitHub repository
+ * page can still be captured as compact text/markdown, original HTML, and PNG
+ * screenshots in every supported JavaScript browser engine.
+ */
+
+import { jest } from '@jest/globals';
+import { createBrowser } from '../../src/browser.js';
+import { fetchHtml } from '../../src/lib.js';
+import { retry } from '../../src/retry.js';
+import {
+  fetchGithubRepositorySnapshot,
+  formatGithubRepositoryMarkdown,
+  formatGithubRepositoryText,
+  parseGithubRepositoryUrl,
+} from '../../src/github.js';
+
+const GITHUB_REPOSITORY_URL =
+  process.env.GITHUB_REPOSITORY_URL ||
+  'https://github.com/link-assistant/web-capture';
+const PNG_SIGNATURE = Buffer.from([
+  0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a,
+]);
+
+const SKIP_LIVE =
+  !process.env.GITHUB_REPOSITORY_INTEGRATION ||
+  process.env.GITHUB_REPOSITORY_INTEGRATION === 'false';
+const describeIfLive = SKIP_LIVE ? describe.skip : describe;
+
+jest.setTimeout(120000);
+
+async function navigateWithRetry(page, url) {
+  await retry(
+    async () => {
+      await page.goto(url, {
+        waitUntil: 'domcontentloaded',
+        timeout: 30000,
+      });
+    },
+    {
+      retries: 3,
+      baseDelay: 2000,
+      onRetry: (err, attempt, delay) => {
+        console.log(
+          `Navigation retry ${attempt} for ${url} after ${delay}ms: ${err.message}`
+        );
+      },
+    }
+  );
+  await page.waitForTimeout(5000);
+}
+
+describeIfLive('GitHub repository capture', () => {
+  it('downloads compact repository text and markdown with README plus file tree', async () => {
+    const parsed = parseGithubRepositoryUrl(GITHUB_REPOSITORY_URL);
+    expect(parsed).not.toBeNull();
+
+    const snapshot = await retry(
+      () => fetchGithubRepositorySnapshot(GITHUB_REPOSITORY_URL),
+      {
+        retries: 3,
+        baseDelay: 2000,
+      }
+    );
+
+    const text = formatGithubRepositoryText(snapshot);
+    const markdown = formatGithubRepositoryMarkdown(snapshot);
+
+    expect(text).toContain(`Repository: ${parsed.fullName}`);
+    expect(text).toContain('Files:');
+    expect(text).toMatch(/README/i);
+    expect(text.length).toBeGreaterThan(500);
+
+    expect(markdown).toContain(`# ${parsed.fullName}`);
+    expect(markdown).toContain('## Files');
+    expect(markdown).toMatch(/## .*README/i);
+    expect(markdown.length).toBeGreaterThan(500);
+    expect(markdown).not.toMatch(/<html/i);
+  });
+
+  it('downloads original GitHub repository HTML', async () => {
+    const parsed = parseGithubRepositoryUrl(GITHUB_REPOSITORY_URL);
+    expect(parsed).not.toBeNull();
+
+    const html = await retry(() => fetchHtml(GITHUB_REPOSITORY_URL), {
+      retries: 3,
+      baseDelay: 2000,
+    });
+
+    expect(html).toMatch(/<html/i);
+    expect(html).toContain(parsed.owner);
+    expect(html).toContain(parsed.repo);
+    expect(html.length).toBeGreaterThan(1000);
+  });
+});
+
+describe.each(['puppeteer', 'playwright'])(
+  'GitHub repository screenshot (%s engine)',
+  (engine) => {
+    describeIfLive(`${engine}`, () => {
+      let browser;
+
+      beforeAll(async () => {
+        browser = await createBrowser(engine);
+      });
+
+      afterAll(async () => {
+        if (browser) {
+          await browser.close();
+        }
+      });
+
+      it('captures the GitHub repository page as a PNG screenshot', async () => {
+        const page = await browser.newPage();
+        await page.setExtraHTTPHeaders({
+          'Accept-Language': 'en-US,en;q=0.9',
+          'Accept-Charset': 'utf-8',
+        });
+        await page.setUserAgent(
+          'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36'
+        );
+        await page.setViewport({ width: 1280, height: 800 });
+        await navigateWithRetry(page, GITHUB_REPOSITORY_URL);
+
+        const screenshot = await page.screenshot({ type: 'png' });
+
+        expect(screenshot).toBeInstanceOf(Buffer);
+        expect(screenshot.length).toBeGreaterThan(1000);
+        expect(screenshot.slice(0, 8)).toEqual(PNG_SIGNATURE);
+      });
+    });
+  }
+);

package/tests/unit/github.test.js

@@ -0,0 +1,97 @@
+import {
+  formatGithubRepositoryMarkdown,
+  formatGithubRepositoryText,
+  getGithubRepositoryTextFilename,
+  isGithubRepositoryUrl,
+  parseGithubRepositoryUrl,
+} from '../../src/github.js';
+
+const SNAPSHOT = {
+  sourceUrl: 'https://github.com/octocat/Hello-World',
+  defaultBranch: 'master',
+  repository: {
+    full_name: 'octocat/Hello-World',
+    description: 'A friendly test repository',
+    html_url: 'https://github.com/octocat/Hello-World',
+    default_branch: 'master',
+    language: 'JavaScript',
+    stargazers_count: 42,
+    forks_count: 7,
+    open_issues_count: 3,
+    license: { spdx_id: 'MIT' },
+    topics: ['demo', 'capture'],
+  },
+  tree: [
+    {
+      name: 'src',
+      path: 'src',
+      type: 'dir',
+      html_url: 'https://github.com/octocat/Hello-World/tree/master/src',
+    },
+    {
+      name: 'README.md',
+      path: 'README.md',
+      type: 'file',
+      size: 37,
+      html_url: 'https://github.com/octocat/Hello-World/blob/master/README.md',
+    },
+  ],
+  readme: {
+    path: 'README.md',
+    content: '# Hello World\n\nThis is the README.',
+  },
+};
+
+describe('GitHub repository URLs', () => {
+  it('detects plain GitHub repository pages', () => {
+    expect(
+      parseGithubRepositoryUrl('https://github.com/octocat/Hello-World')
+    ).toEqual({
+      owner: 'octocat',
+      repo: 'Hello-World',
+      fullName: 'octocat/Hello-World',
+      htmlUrl: 'https://github.com/octocat/Hello-World',
+    });
+    expect(
+      isGithubRepositoryUrl('https://github.com/octocat/Hello-World')
+    ).toBe(true);
+    expect(
+      isGithubRepositoryUrl('https://github.com/octocat/Hello-World/issues')
+    ).toBe(false);
+    expect(
+      isGithubRepositoryUrl('https://example.com/octocat/Hello-World')
+    ).toBe(false);
+  });
+
+  it('derives repository text filenames', () => {
+    expect(
+      getGithubRepositoryTextFilename('https://github.com/octocat/Hello-World')
+    ).toBe('octocat-Hello-World.txt');
+  });
+});
+
+describe('GitHub repository snapshot formatting', () => {
+  it('formats a compact markdown snapshot with metadata, file tree, and README', () => {
+    const markdown = formatGithubRepositoryMarkdown(SNAPSHOT);
+
+    expect(markdown).toContain('# octocat/Hello-World');
+    expect(markdown).toContain('> A friendly test repository');
+    expect(markdown).toContain('- Default branch: `master`');
+    expect(markdown).toContain('- [src/](');
+    expect(markdown).toContain('- [README.md](');
+    expect(markdown).toContain('## README.md');
+    expect(markdown).toContain('# Hello World');
+  });
+
+  it('formats a compact plain-text snapshot with metadata, file tree, and README', () => {
+    const text = formatGithubRepositoryText(SNAPSHOT);
+
+    expect(text).toContain('Repository: octocat/Hello-World');
+    expect(text).toContain('Description: A friendly test repository');
+    expect(text).toContain('Files:');
+    expect(text).toContain('- src/');
+    expect(text).toContain('- README.md');
+    expect(text).toContain('README.md:');
+    expect(text).toContain('This is the README.');
+  });
+});