@link-assistant/web-capture 1.10.1 → 1.10.3

package/tests/unit/github.test.js

@@ -0,0 +1,97 @@
+import {
+  formatGithubRepositoryMarkdown,
+  formatGithubRepositoryText,
+  getGithubRepositoryTextFilename,
+  isGithubRepositoryUrl,
+  parseGithubRepositoryUrl,
+} from '../../src/github.js';
+
+const SNAPSHOT = {
+  sourceUrl: 'https://github.com/octocat/Hello-World',
+  defaultBranch: 'master',
+  repository: {
+    full_name: 'octocat/Hello-World',
+    description: 'A friendly test repository',
+    html_url: 'https://github.com/octocat/Hello-World',
+    default_branch: 'master',
+    language: 'JavaScript',
+    stargazers_count: 42,
+    forks_count: 7,
+    open_issues_count: 3,
+    license: { spdx_id: 'MIT' },
+    topics: ['demo', 'capture'],
+  },
+  tree: [
+    {
+      name: 'src',
+      path: 'src',
+      type: 'dir',
+      html_url: 'https://github.com/octocat/Hello-World/tree/master/src',
+    },
+    {
+      name: 'README.md',
+      path: 'README.md',
+      type: 'file',
+      size: 37,
+      html_url: 'https://github.com/octocat/Hello-World/blob/master/README.md',
+    },
+  ],
+  readme: {
+    path: 'README.md',
+    content: '# Hello World\n\nThis is the README.',
+  },
+};
+
+describe('GitHub repository URLs', () => {
+  it('detects plain GitHub repository pages', () => {
+    expect(
+      parseGithubRepositoryUrl('https://github.com/octocat/Hello-World')
+    ).toEqual({
+      owner: 'octocat',
+      repo: 'Hello-World',
+      fullName: 'octocat/Hello-World',
+      htmlUrl: 'https://github.com/octocat/Hello-World',
+    });
+    expect(
+      isGithubRepositoryUrl('https://github.com/octocat/Hello-World')
+    ).toBe(true);
+    expect(
+      isGithubRepositoryUrl('https://github.com/octocat/Hello-World/issues')
+    ).toBe(false);
+    expect(
+      isGithubRepositoryUrl('https://example.com/octocat/Hello-World')
+    ).toBe(false);
+  });
+
+  it('derives repository text filenames', () => {
+    expect(
+      getGithubRepositoryTextFilename('https://github.com/octocat/Hello-World')
+    ).toBe('octocat-Hello-World.txt');
+  });
+});
+
+describe('GitHub repository snapshot formatting', () => {
+  it('formats a compact markdown snapshot with metadata, file tree, and README', () => {
+    const markdown = formatGithubRepositoryMarkdown(SNAPSHOT);
+
+    expect(markdown).toContain('# octocat/Hello-World');
+    expect(markdown).toContain('> A friendly test repository');
+    expect(markdown).toContain('- Default branch: `master`');
+    expect(markdown).toContain('- [src/](');
+    expect(markdown).toContain('- [README.md](');
+    expect(markdown).toContain('## README.md');
+    expect(markdown).toContain('# Hello World');
+  });
+
+  it('formats a compact plain-text snapshot with metadata, file tree, and README', () => {
+    const text = formatGithubRepositoryText(SNAPSHOT);
+
+    expect(text).toContain('Repository: octocat/Hello-World');
+    expect(text).toContain('Description: A friendly test repository');
+    expect(text).toContain('Files:');
+    expect(text).toContain('- src/');
+    expect(text).toContain('- README.md');
+    expect(text).toContain('README.md:');
+    expect(text).toContain('This is the README.');
+  });
+});

package/tests/xpaste/xpaste-screenshot-markdown.mock.test.js

@@ -0,0 +1,375 @@
+/**
+ * Integration tests to verify that markdown extraction from xpaste.pro
+ * includes ALL text visible in screenshots of the same page
+ *
+ * This ensures 100% coverage of visual content extraction as required in issue #15
+ */
+
+import request from 'supertest';
+import nock from 'nock';
+import fs from 'fs';
+import path from 'path';
+import { fileURLToPath } from 'url';
+
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
+
+let app;
+
+beforeAll(async () => {
+  app = (await import('../../src/index.js')).app;
+});
+
+describe('xpaste.pro screenshot-to-markdown content matching', () => {
+  beforeEach(() => {
+    nock.cleanAll();
+  });
+
+  describe('GET /markdown for t4q0Lsp0', () => {
+    const testUrl = 'https://xpaste.pro/p/t4q0Lsp0';
+
+    // Load the actual HTML page content that was saved
+    const htmlPath = path.join(
+      __dirname,
+      '../../../tests/xpaste/data/t4q0Lsp0-page.html'
+    );
+    const rawTextPath = path.join(
+      __dirname,
+      '../../../tests/xpaste/data/t4q0Lsp0-actual-content.txt'
+    );
+    let testHtml;
+    let rawText;
+
+    beforeAll(() => {
+      if (fs.existsSync(htmlPath)) {
+        testHtml = fs.readFileSync(htmlPath, 'utf-8');
+      }
+      if (fs.existsSync(rawTextPath)) {
+        rawText = fs.readFileSync(rawTextPath, 'utf-8');
+      }
+    });
+
+    beforeEach(() => {
+      if (rawText) {
+        nock('https://xpaste.pro')
+          .persist()
+          .get('/p/t4q0Lsp0/raw')
+          .reply(200, rawText, {
+            'content-type': 'text/plain; charset=utf-8',
+          });
+      }
+    });
+
+    it('should include all header/metadata text visible in screenshot', async () => {
+      if (!testHtml) {
+        console.warn('Skipping test - HTML file not found');
+        return;
+      }
+
+      nock('https://xpaste.pro')
+        .get('/p/t4q0Lsp0')
+        .reply(200, testHtml, { 'content-type': 'text/html; charset=utf-8' });
+
+      const response = await request(app)
+        .get('/markdown')
+        .query({ url: testUrl });
+
+      expect(response.status).toBe(200);
+      expect(response.type).toBe('text/markdown');
+
+      // Verify page title/branding (top of screenshot)
+      expect(response.text).toContain('Xpaste');
+      expect(response.text).toContain('xPaste');
+
+      // Verify metadata section (visible in screenshot)
+      expect(response.text).toContain('Формат');
+      expect(response.text).toContain('text');
+      expect(response.text).toContain('Время создания');
+      expect(response.text).toContain('07.07.2021');
+      expect(response.text).toContain('12:48 UTC');
+      expect(response.text).toContain('Будет удалена');
+      expect(response.text).toContain('05.07.2031');
+    });
+
+    it('should include all SQL query content visible in screenshot', async () => {
+      if (!testHtml) {
+        console.warn('Skipping test - HTML file not found');
+        return;
+      }
+
+      nock('https://xpaste.pro')
+        .get('/p/t4q0Lsp0')
+        .reply(200, testHtml, { 'content-type': 'text/html; charset=utf-8' });
+
+      const response = await request(app)
+        .get('/markdown')
+        .query({ url: testUrl });
+
+      expect(response.status).toBe(200);
+
+      // Verify all 4 SQL queries visible in screenshot are present
+
+      // Query #1 metadata (lines 1-8 in screenshot)
+      expect(response.text).toContain('# 1');
+      expect(response.text).toContain('Time: 210707 15:39:36');
+      expect(response.text).toMatch(
+        /User@Host:\s+1703313381\[1703313381\]\s+@\s+\[136\.243\.53\.188\]/
+      );
+      expect(response.text).toContain('Id: 1138102510');
+      expect(response.text).toContain('Schema: xxxxxx');
+      expect(response.text).toContain('Query_time: 2.182754');
+      expect(response.text).toContain('Rows_examined: 324036');
+
+      // Query #2 metadata (lines 19-28 in screenshot)
+      expect(response.text).toContain('# 2');
+      expect(response.text).toContain('Time: 210707 15:38:11');
+      expect(response.text).toContain('Id: 1138100591');
+      expect(response.text).toContain('Query_time: 9.259469');
+      expect(response.text).toContain('DELETE FROM phpbb_post_revisions');
+      expect(response.text).toContain('WHERE post_id = 10472158');
+
+      // Query #3 metadata (lines 30-46 in screenshot)
+      expect(response.text).toContain('# 3');
+      expect(response.text).toContain('Time: 210707 15:32:18');
+      expect(response.text).toContain('Id: 1138094492');
+      expect(response.text).toContain('Query_time: 1.113014');
+      expect(response.text).toContain('Rows_examined: 521597');
+
+      // Query #4 metadata (lines 48-64 in screenshot)
+      expect(response.text).toContain('# 4');
+      expect(response.text).toContain('Time: 210707 15:27:56');
+      expect(response.text).toContain('Id: 1138089558');
+      expect(response.text).toContain('Query_time: 4.682828');
+      expect(response.text).toContain('Rows_examined: 2457916');
+
+      // Verify SQL statements contain key elements
+      expect(response.text).toContain('SELECT f.*, t.*, p.*, u.*');
+      expect(response.text).toContain('phpbb_posts');
+      expect(response.text).toContain('phpbb_users');
+      expect(response.text).toContain('phpbb_topics');
+      expect(response.text).toContain('phpbb_forums');
+    });
+
+    it('should include all footer text visible in screenshot', async () => {
+      if (!testHtml) {
+        console.warn('Skipping test - HTML file not found');
+        return;
+      }
+
+      nock('https://xpaste.pro')
+        .get('/p/t4q0Lsp0')
+        .reply(200, testHtml, { 'content-type': 'text/html; charset=utf-8' });
+
+      const response = await request(app)
+        .get('/markdown')
+        .query({ url: testUrl });
+
+      expect(response.status).toBe(200);
+
+      // Verify footer elements (bottom of screenshot)
+      expect(response.text).toContain('xPaste');
+      expect(response.text).toContain('Упакуем пароль или код в');
+      expect(response.text).toMatch(/cсылку для передачи/); // Note: 'c' can be different Unicode char
+      expect(response.text).toContain('Сделано в');
+      expect(response.text).toContain('Southbridge');
+      expect(response.text).toContain('Справка');
+      expect(response.text).toContain('Политика конфиденциальности');
+    });
+
+    it('should include language selector links visible in screenshot', async () => {
+      if (!testHtml) {
+        console.warn('Skipping test - HTML file not found');
+        return;
+      }
+
+      nock('https://xpaste.pro')
+        .get('/p/t4q0Lsp0')
+        .reply(200, testHtml, { 'content-type': 'text/html; charset=utf-8' });
+
+      const response = await request(app)
+        .get('/markdown')
+        .query({ url: testUrl });
+
+      expect(response.status).toBe(200);
+
+      // Verify language links (top right of screenshot)
+      expect(response.text).toContain('Ru');
+      expect(response.text).toContain('En');
+    });
+
+    it('should include RAW link visible in screenshot', async () => {
+      if (!testHtml) {
+        console.warn('Skipping test - HTML file not found');
+        return;
+      }
+
+      nock('https://xpaste.pro')
+        .get('/p/t4q0Lsp0')
+        .reply(200, testHtml, { 'content-type': 'text/html; charset=utf-8' });
+
+      const response = await request(app)
+        .get('/markdown')
+        .query({ url: testUrl });
+
+      expect(response.status).toBe(200);
+
+      // Verify RAW button (visible in screenshot)
+      expect(response.text).toContain('RAW');
+    });
+
+    it('should embed the raw text file as a markdown text block', async () => {
+      if (!testHtml || !rawText) {
+        console.warn('Skipping test - fixture files not found');
+        return;
+      }
+
+      nock('https://xpaste.pro')
+        .get('/p/t4q0Lsp0')
+        .reply(200, testHtml, { 'content-type': 'text/html; charset=utf-8' });
+
+      const response = await request(app)
+        .get('/markdown')
+        .query({ url: testUrl });
+
+      expect(response.status).toBe(200);
+      expect(response.type).toBe('text/markdown');
+      expect(response.text).toContain('## xpaste-pro-t4q0Lsp0.txt');
+      expect(response.text).toContain('```text');
+      expect(response.text).toContain(
+        rawText.split('\n')[0].replace(/\r/g, '')
+      );
+      expect(response.text).toContain('DELETE FROM phpbb_post_revisions');
+    });
+
+    it('should verify correct element ordering - heading and languages at top', async () => {
+      if (!testHtml) {
+        console.warn('Skipping test - HTML file not found');
+        return;
+      }
+
+      nock('https://xpaste.pro')
+        .get('/p/t4q0Lsp0')
+        .reply(200, testHtml, { 'content-type': 'text/html; charset=utf-8' });
+
+      const response = await request(app)
+        .get('/markdown')
+        .query({ url: testUrl });
+
+      expect(response.status).toBe(200);
+
+      const lines = response.text.split('\n');
+
+      // Find line numbers of key elements
+      const headingLine = lines.findIndex((l) => l.includes('Упакуем пароль'));
+      const languageLine = lines.findIndex(
+        (l) => l.includes('[Ru]') || l.includes('[En]')
+      );
+      const formatLine = lines.findIndex((l) => l.includes('Формат:'));
+      const firstQueryLine = lines.findIndex(
+        (l) => l.includes('# 1') && l.includes('#')
+      );
+
+      // Verify heading comes before metadata (as shown in screenshot)
+      expect(headingLine).toBeGreaterThan(0);
+      expect(headingLine).toBeLessThan(formatLine);
+
+      // Verify languages come before metadata (as shown in screenshot)
+      expect(languageLine).toBeGreaterThan(0);
+      expect(languageLine).toBeLessThan(formatLine);
+
+      // Verify heading comes before main content
+      expect(headingLine).toBeLessThan(firstQueryLine);
+
+      // Verify metadata comes before main content
+      expect(formatLine).toBeLessThan(firstQueryLine);
+
+      console.log(
+        `✓ Element ordering verified: heading (line ${headingLine + 1}) -> languages (line ${languageLine + 1}) -> metadata (line ${formatLine + 1}) -> content (line ${firstQueryLine + 1})`
+      );
+    });
+
+    it('should verify markdown extraction matches screenshot - comprehensive check', async () => {
+      if (!testHtml) {
+        console.warn('Skipping test - HTML file not found');
+        return;
+      }
+
+      nock('https://xpaste.pro')
+        .get('/p/t4q0Lsp0')
+        .reply(200, testHtml, { 'content-type': 'text/html; charset=utf-8' });
+
+      const response = await request(app)
+        .get('/markdown')
+        .query({ url: testUrl });
+
+      expect(response.status).toBe(200);
+
+      // Define all key text elements visible in the screenshot
+      const requiredElements = [
+        // Header/branding
+        'Xpaste',
+
+        // Metadata section
+        'Формат:',
+        'text',
+        'Время создания:',
+        '07.07.2021, 12:48 UTC',
+        'Будет удалена:',
+        '05.07.2031, 12:48 UTC',
+
+        // Query #1
+        '# 1',
+        'Time: 210707 15:39:36',
+        'User@Host: 1703313381[1703313381]',
+        '[136.243.53.188]',
+        'Id: 1138102510',
+        'Query_time: 2.182754',
+
+        // Query #2
+        '# 2',
+        'Time: 210707 15:38:11',
+        'Id: 1138100591',
+        'DELETE FROM phpbb_post_revisions',
+
+        // Query #3
+        '# 3',
+        'Time: 210707 15:32:18',
+        'Id: 1138094492',
+
+        // Query #4
+        '# 4',
+        'Time: 210707 15:27:56',
+        'Id: 1138089558',
+
+        // Footer
+        'Упакуем пароль',
+        'Southbridge',
+        'Справка',
+        'Политика конфиденциальности',
+
+        // Language links
+        'Ru',
+        'En',
+
+        // RAW button
+        'RAW',
+      ];
+
+      // Verify ALL elements are present
+      const missingElements = [];
+      requiredElements.forEach((element) => {
+        if (!response.text.includes(element)) {
+          missingElements.push(element);
+        }
+      });
+
+      expect(missingElements).toEqual([]);
+
+      // Log success with coverage stats
+      if (missingElements.length === 0) {
+        console.log(
+          `✓ All ${requiredElements.length} screenshot text elements found in markdown`
+        );
+      }
+    });
+  });
+});

package/tests/xpaste/xpaste.e2e.test.js

@@ -0,0 +1,36 @@
+import getPort from 'get-port';
+import nock from 'nock';
+import { app } from '../../src/index.js';
+
+describe('xpaste.pro HTTP e2e', () => {
+  let server;
+
+  afterEach(async () => {
+    nock.cleanAll();
+    if (server) {
+      await new Promise((resolve) => server.close(resolve));
+      server = null;
+    }
+  });
+
+  it('serves raw paste text through the real HTTP server', async () => {
+    const pasteText = 'console.log("xpaste fixture");\n';
+    nock('https://xpaste.pro')
+      .get('/p/js123/raw')
+      .reply(200, pasteText, { 'content-type': 'text/plain; charset=utf-8' });
+
+    const port = await getPort();
+    server = app.listen(port);
+
+    const response = await fetch(
+      `http://127.0.0.1:${port}/txt?url=${encodeURIComponent('https://xpaste.pro/p/js123')}`
+    );
+
+    expect(response.status).toBe(200);
+    expect(response.headers.get('content-type')).toMatch(/text\/plain/);
+    expect(response.headers.get('content-disposition')).toContain(
+      'xpaste-pro-js123.txt'
+    );
+    expect(await response.text()).toBe(pasteText);
+  });
+});

package/tests/xpaste/xpaste.integration.test.js

@@ -0,0 +1,41 @@
+import fs from 'fs';
+import path from 'path';
+import { fileURLToPath } from 'url';
+import { convertHtmlToMarkdownEnhanced } from '../../src/lib.js';
+
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
+const DATA_DIR = path.join(__dirname, '../../../tests/xpaste/data');
+
+describe('xpaste.pro fixture integration', () => {
+  it('converts the captured page fixture with the same visual ordering as the screenshot', () => {
+    const html = fs.readFileSync(
+      path.join(DATA_DIR, 't4q0Lsp0-page.html'),
+      'utf-8'
+    );
+    const { markdown } = convertHtmlToMarkdownEnhanced(
+      html,
+      'https://xpaste.pro/p/t4q0Lsp0',
+      {
+        extractMetadata: false,
+        postProcess: false,
+      }
+    );
+
+    const headingIndex = markdown.indexOf('Упакуем пароль');
+    const languageIndexes = ['[Ru]', '[En]']
+      .map((label) => markdown.indexOf(label))
+      .filter((index) => index >= 0);
+    const languageIndex = Math.min(...languageIndexes);
+    const formatIndex = markdown.indexOf('Формат:');
+    const firstQueryIndex = markdown.indexOf('# 1');
+
+    expect(headingIndex).toBeGreaterThanOrEqual(0);
+    expect(languageIndexes).not.toHaveLength(0);
+    expect(languageIndex).toBeGreaterThanOrEqual(0);
+    expect(formatIndex).toBeGreaterThanOrEqual(0);
+    expect(firstQueryIndex).toBeGreaterThanOrEqual(0);
+    expect(headingIndex).toBeLessThan(formatIndex);
+    expect(languageIndex).toBeLessThan(formatIndex);
+    expect(formatIndex).toBeLessThan(firstQueryIndex);
+  });
+});