@link-assistant/web-capture 1.12.0 → 2.0.1

package/tests/fixtures/habr-articles.js

@@ -1,45 +0,0 @@
-/**
- * Configuration for Habr test articles.
- *
- * These are the same three articles archived in the link-foundation/meta-theory repo.
- * Each entry provides article metadata for use in integration tests.
- */
-
-export const HABR_ARTICLES = {
-  '0.0.0': {
-    version: '0.0.0',
-    title: 'Math introduction to Deep Theory',
-    url: 'https://habr.com/en/companies/deepfoundation/articles/658705/',
-    language: 'en',
-    expectedFigures: 12,
-  },
-  '0.0.1': {
-    version: '0.0.1',
-    title: 'Глубокая Теория Связей 0.0.1',
-    titleEnglish: 'Deep Theory of Links 0.0.1',
-    url: 'https://habr.com/ru/companies/deepfoundation/articles/804617/',
-    language: 'ru',
-    expectedFigures: 10,
-  },
-  '0.0.2': {
-    version: '0.0.2',
-    title: 'The Links Theory 0.0.2',
-    url: 'https://habr.com/en/articles/895896/',
-    language: 'en',
-    expectedFigures: 13,
-  },
-};
-
-export function getArticle(version) {
-  const article = HABR_ARTICLES[version];
-  if (!article) {
-    throw new Error(
-      `Unknown article version: ${version}. Available: ${Object.keys(HABR_ARTICLES).join(', ')}`
-    );
-  }
-  return article;
-}
-
-export function getAllArticles() {
-  return Object.values(HABR_ARTICLES);
-}

package/tests/fixtures/heading-numbering.html

@@ -1,19 +0,0 @@
-<!doctype html>
-<html>
-  <body>
-    <ol>
-      <li>
-        <h4><strong>13. Top-level numbered heading in source</strong></h4>
-      </li>
-    </ol>
-
-    <h5><strong>13.1 First subsection heading</strong></h5>
-    <p>Where it activates: ...</p>
-
-    <h5><strong>13.2 Second subsection heading</strong></h5>
-    <p>Where it activates: ...</p>
-
-    <h5><strong>13.3 Third subsection heading</strong></h5>
-    <p>Where it activates: ...</p>
-  </body>
-</html>

package/tests/fixtures/list-item-with-br.html

@@ -1,8 +0,0 @@
-<ol>
-  <li>
-    <strong>Glossary entry.</strong><span> Some lead text.<br /></span
-    ><strong>TAG1</strong><span> – Definition one.<br /></span
-    ><strong>TAG2</strong><span> – Definition two.<br /></span
-    ><strong>TAG3</strong><span> – Definition three.</span>
-  </li>
-</ol>

package/tests/fixtures/ol-continuous-numbering.html

@@ -1,12 +0,0 @@
-<ol>
-  <li><strong>First</strong></li>
-  <li><strong>Second</strong></li>
-</ol>
-<p>Some prose between lists.</p>
-<ol>
-  <li><strong>Third</strong></li>
-  <li><strong>Fourth</strong></li>
-</ol>
-<ol start="13">
-  <li><strong>Top-level numbered heading.</strong></li>
-</ol>

package/tests/fixtures/paragraph-vs-line-break.html

@@ -1,11 +0,0 @@
-<!doctype html>
-<html>
-  <body>
-    <p>
-      <span>Caption A:</span><br />
-      <img alt="" src="data:image/png;base64,iVBORw0KGgo=" /><br /><br />
-      <span>Caption B:</span><br />
-      <img alt="" src="data:image/png;base64,iVBORw0KGgo=" />
-    </p>
-  </body>
-</html>

package/tests/helpers/nock.js

@@ -1,10 +0,0 @@
-import nock from 'nock';
-
-// Nock 15 interceptors survive Jest VM contexts unless explicitly restored.
-// https://github.com/nock/nock#memory-issues-with-jest
-afterAll(() => {
-  nock.cleanAll();
-  nock.restore();
-});
-
-export default nock;

package/tests/helpers/search-fixture.cjs

@@ -1,17 +0,0 @@
-const { default: nock } = require('nock');
-
-nock('https://en.wikipedia.org')
-  .persist()
-  .get('/w/rest.php/v1/search/page')
-  .query(true)
-  .reply(200, {
-    pages: [
-      {
-        id: 1,
-        key: 'Formal_methods',
-        title: 'Formal methods',
-        excerpt: 'the study of <b>formal</b> methods',
-        description: 'rigorous techniques',
-      },
-    ],
-  });

package/tests/helpers/temporary-directory.js

@@ -1,12 +0,0 @@
-import { mkdtemp, rm } from 'node:fs/promises';
-import { tmpdir } from 'node:os';
-import path from 'node:path';
-
-export async function withTemporaryDirectory(prefix, operation) {
-  const directory = await mkdtemp(path.join(tmpdir(), prefix));
-  try {
-    return await operation(directory);
-  } finally {
-    await rm(directory, { recursive: true, force: true });
-  }
-}

package/tests/integration/api-endpoints.test.js

@@ -1,418 +0,0 @@
-/**
- * Integration tests for the API endpoints.
- *
- * Tests /image, /archive, /pdf, /docx endpoints with various options.
- * Uses nock to mock external requests for fast, reliable unit-level testing.
- * These tests validate endpoint behavior and response formats.
- */
-
-import { jest } from '@jest/globals';
-import request from 'supertest';
-import nock from '../helpers/nock.js';
-import unzipper from 'unzipper';
-import { app } from '../../src/index.js';
-import { isKreuzbergAvailable } from '../../src/kreuzberg.js';
-
-jest.setTimeout(60000);
-
-const MOCK_HTML = `<!DOCTYPE html>
-<html><head><title>Test</title>
-<link rel="stylesheet" href="https://example.com/style.css">
-</head>
-<body>
-<h1>Test Page</h1>
-<p>Hello world</p>
-<img src="https://example.com/image.png" alt="test">
-</body></html>`;
-
-function mockGithubRepository(owner = 'octocat', repo = 'Hello-World') {
-  const readme = '# Hello World\n\nThis README came from GitHub.';
-  nock('https://api.github.com')
-    .get(`/repos/${owner}/${repo}`)
-    .reply(200, {
-      full_name: `${owner}/${repo}`,
-      description: 'A friendly test repository',
-      html_url: `https://github.com/${owner}/${repo}`,
-      default_branch: 'master',
-      language: 'JavaScript',
-      stargazers_count: 42,
-      forks_count: 7,
-      open_issues_count: 3,
-      license: { spdx_id: 'MIT' },
-      topics: ['demo', 'capture'],
-    })
-    .get(`/repos/${owner}/${repo}/readme`)
-    .query({ ref: 'master' })
-    .reply(200, {
-      name: 'README.md',
-      path: 'README.md',
-      encoding: 'base64',
-      content: Buffer.from(readme, 'utf8').toString('base64'),
-      html_url: `https://github.com/${owner}/${repo}/blob/master/README.md`,
-    })
-    .get(`/repos/${owner}/${repo}/contents`)
-    .query({ ref: 'master' })
-    .reply(200, [
-      {
-        name: 'src',
-        path: 'src',
-        type: 'dir',
-        html_url: `https://github.com/${owner}/${repo}/tree/master/src`,
-      },
-      {
-        name: 'README.md',
-        path: 'README.md',
-        type: 'file',
-        size: readme.length,
-        html_url: `https://github.com/${owner}/${repo}/blob/master/README.md`,
-      },
-    ]);
-}
-
-afterEach(() => {
-  nock.cleanAll();
-});
-
-describe('API Endpoint Tests', () => {
-  describe('unsupported routes', () => {
-    it('does not expose /gdocs as an output-format endpoint', async () => {
-      await request(app)
-        .get('/gdocs')
-        .query({
-          url: 'https://docs.google.com/document/d/test-doc/edit',
-        })
-        .expect(404);
-    });
-  });
-
-  describe('GET /image', () => {
-    it('rejects invalid format', async () => {
-      await request(app)
-        .get('/image')
-        .query({ url: 'https://example.com/test', format: 'bmp' })
-        .expect(400);
-    });
-
-    it('returns 400 without url', async () => {
-      await request(app).get('/image').expect(400);
-    });
-  });
-
-  describe('GET /markdown', () => {
-    it('returns markdown', async () => {
-      nock('https://example.com').get('/md-test').reply(200, MOCK_HTML, {
-        'content-type': 'text/html',
-      });
-
-      const res = await request(app)
-        .get('/markdown')
-        .query({ url: 'https://example.com/md-test' })
-        .expect(200);
-
-      expect(res.headers['content-type']).toMatch(/text\/markdown/);
-      expect(res.text).toContain('Test Page');
-      expect(res.text).toContain('Hello world');
-    });
-
-    it('returns a compact GitHub repository markdown snapshot', async () => {
-      mockGithubRepository();
-
-      const res = await request(app)
-        .get('/markdown')
-        .query({ url: 'https://github.com/octocat/Hello-World' })
-        .expect(200);
-
-      expect(res.headers['content-type']).toMatch(/text\/markdown/);
-      expect(res.text).toContain('# octocat/Hello-World');
-      expect(res.text).toContain('## Repository');
-      expect(res.text).toContain('## Files');
-      expect(res.text).toContain('- [src/](');
-      expect(res.text).toContain('## README.md');
-      expect(res.text).toContain('This README came from GitHub.');
-      expect(res.text).not.toContain('Skip to content');
-      expect(nock.isDone()).toBe(true);
-    });
-
-    it('returns a compact GitHub repository markdown snapshot for kreuzberg text format', async () => {
-      mockGithubRepository();
-
-      const res = await request(app)
-        .get('/markdown')
-        .query({
-          url: 'https://github.com/octocat/Hello-World',
-          converter: 'kreuzberg',
-          format: 'text',
-        })
-        .expect(200);
-
-      expect(res.headers['content-type']).toMatch(/text\/markdown/);
-      expect(res.text).toContain('# octocat/Hello-World');
-      expect(res.text).toContain('## Files');
-      expect(res.text).toContain('This README came from GitHub.');
-      expect(nock.isDone()).toBe(true);
-    });
-
-    it('rejects unsupported converter names', async () => {
-      await request(app)
-        .get('/markdown')
-        .query({
-          url: 'https://example.com/md-test',
-          converter: 'unknown',
-        })
-        .expect(400);
-    });
-
-    it('rejects JSON format without the kreuzberg converter', async () => {
-      await request(app)
-        .get('/markdown')
-        .query({
-          url: 'https://example.com/md-test',
-          format: 'json',
-        })
-        .expect(400);
-    });
-
-    it('returns structured kreuzberg JSON with selectors and absolute links', async () => {
-      if (!(await isKreuzbergAvailable())) {
-        return;
-      }
-
-      nock('https://example.com')
-        .get('/kreuzberg-json')
-        .reply(
-          200,
-          `<!DOCTYPE html><html><body>
-            <main>
-              <h1>Article Title</h1>
-              <nav>Navigation</nav>
-              <article><p>Wanted body with <a href="/about">relative link</a>.</p></article>
-            </main>
-          </body></html>`,
-          { 'content-type': 'text/html' }
-        );
-
-      const res = await request(app)
-        .get('/markdown')
-        .query({
-          url: 'https://example.com/kreuzberg-json',
-          converter: 'kreuzberg',
-          format: 'json',
-          contentSelector: 'main',
-          bodySelector: 'article',
-        })
-        .expect(200);
-
-      expect(res.headers['content-type']).toMatch(/application\/json/);
-      expect(res.body).toHaveProperty('content');
-      expect(res.body).toHaveProperty('metadata');
-      expect(res.body).toHaveProperty('tables');
-      expect(res.body).toHaveProperty('images');
-      expect(res.body).toHaveProperty('warnings');
-      expect(res.body.content).toContain('Article Title');
-      expect(res.body.content).toContain('Wanted body');
-      expect(res.body.content).toContain('https://example.com/about');
-      expect(res.body.content).not.toContain('Navigation');
-    });
-  });
-
-  describe('GET /txt', () => {
-    it('returns a compact GitHub repository plain-text snapshot', async () => {
-      mockGithubRepository();
-
-      const res = await request(app)
-        .get('/txt')
-        .query({ url: 'https://github.com/octocat/Hello-World' })
-        .expect(200);
-
-      expect(res.headers['content-type']).toMatch(/text\/plain/);
-      expect(res.headers['content-disposition']).toContain(
-        'filename="octocat-Hello-World.txt"'
-      );
-      expect(res.text).toContain('Repository: octocat/Hello-World');
-      expect(res.text).toContain('Files:');
-      expect(res.text).toContain('- src/');
-      expect(res.text).toContain('README.md:');
-      expect(res.text).toContain('This README came from GitHub.');
-      expect(nock.isDone()).toBe(true);
-    });
-  });
-
-  describe('GET /archive', () => {
-    it('returns a ZIP archive with remote images (markdown format)', async () => {
-      nock('https://example.com').get('/archive-test').reply(200, MOCK_HTML, {
-        'content-type': 'text/html',
-      });
-
-      const res = await request(app)
-        .get('/archive')
-        .query({
-          url: 'https://example.com/archive-test',
-          localImages: 'false',
-          documentFormat: 'markdown',
-        })
-        .buffer(true)
-        .parse((res, callback) => {
-          const chunks = [];
-          res.on('data', (chunk) => chunks.push(chunk));
-          res.on('end', () => callback(null, Buffer.concat(chunks)));
-        })
-        .expect(200);
-
-      expect(res.body).toBeInstanceOf(Buffer);
-      expect(res.body.length).toBeGreaterThan(50);
-      // ZIP signature: PK (0x50 0x4B)
-      expect(res.body[0]).toBe(0x50);
-      expect(res.body[1]).toBe(0x4b);
-
-      // Default archive layout contract (issue #113): the markdown archive must
-      // bundle document.md AND document.html for reference.
-      const dir = await unzipper.Open.buffer(res.body);
-      const names = dir.files.map((f) => f.path);
-      expect(names).toEqual(
-        expect.arrayContaining(['document.md', 'document.html'])
-      );
-    });
-
-    it('returns a ZIP archive with HTML format', async () => {
-      nock('https://example.com')
-        .get('/archive-html-test')
-        .reply(200, MOCK_HTML, {
-          'content-type': 'text/html',
-        });
-
-      const res = await request(app)
-        .get('/archive')
-        .query({
-          url: 'https://example.com/archive-html-test',
-          localImages: 'false',
-          documentFormat: 'html',
-        })
-        .buffer(true)
-        .parse((res, callback) => {
-          const chunks = [];
-          res.on('data', (chunk) => chunks.push(chunk));
-          res.on('end', () => callback(null, Buffer.concat(chunks)));
-        })
-        .expect(200);
-
-      expect(res.body).toBeInstanceOf(Buffer);
-      expect(res.body.length).toBeGreaterThan(50);
-      // ZIP signature
-      expect(res.body[0]).toBe(0x50);
-      expect(res.body[1]).toBe(0x4b);
-    });
-
-    it('returns a ZIP archive with local images and folders', async () => {
-      const mockPng = Buffer.from([
-        0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a,
-      ]);
-
-      nock('https://example.com')
-        .get('/archive-local-test')
-        .reply(200, MOCK_HTML, { 'content-type': 'text/html' });
-
-      nock('https://example.com')
-        .get('/image.png')
-        .reply(200, mockPng, { 'content-type': 'image/png' });
-
-      const res = await request(app)
-        .get('/archive')
-        .query({
-          url: 'https://example.com/archive-local-test',
-          localImages: 'true',
-        })
-        .buffer(true)
-        .parse((res, callback) => {
-          const chunks = [];
-          res.on('data', (chunk) => chunks.push(chunk));
-          res.on('end', () => callback(null, Buffer.concat(chunks)));
-        })
-        .expect(200);
-
-      expect(res.body).toBeInstanceOf(Buffer);
-      expect(res.body.length).toBeGreaterThan(50);
-      // ZIP signature
-      expect(res.body[0]).toBe(0x50);
-      expect(res.body[1]).toBe(0x4b);
-    });
-
-    it('returns a ZIP archive with HTML format and local CSS', async () => {
-      const mockCss = 'body { background: #fff; }';
-      const mockPng = Buffer.from([
-        0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a,
-      ]);
-
-      nock('https://example.com')
-        .get('/archive-html-local')
-        .reply(200, MOCK_HTML, { 'content-type': 'text/html' });
-
-      nock('https://example.com')
-        .get('/style.css')
-        .reply(200, mockCss, { 'content-type': 'text/css' });
-
-      nock('https://example.com')
-        .get('/image.png')
-        .reply(200, mockPng, { 'content-type': 'image/png' });
-
-      const res = await request(app)
-        .get('/archive')
-        .query({
-          url: 'https://example.com/archive-html-local',
-          localImages: 'true',
-          documentFormat: 'html',
-        })
-        .buffer(true)
-        .parse((res, callback) => {
-          const chunks = [];
-          res.on('data', (chunk) => chunks.push(chunk));
-          res.on('end', () => callback(null, Buffer.concat(chunks)));
-        })
-        .expect(200);
-
-      expect(res.body).toBeInstanceOf(Buffer);
-      expect(res.body.length).toBeGreaterThan(50);
-      // ZIP signature
-      expect(res.body[0]).toBe(0x50);
-      expect(res.body[1]).toBe(0x4b);
-    });
-
-    it('returns 400 without url', async () => {
-      await request(app).get('/archive').expect(400);
-    });
-  });
-
-  describe('GET /docx', () => {
-    it('returns a DOCX document', async () => {
-      nock('https://example.com').get('/docx-test').reply(200, MOCK_HTML, {
-        'content-type': 'text/html',
-      });
-
-      const res = await request(app)
-        .get('/docx')
-        .query({ url: 'https://example.com/docx-test' })
-        .buffer(true)
-        .parse((res, callback) => {
-          const chunks = [];
-          res.on('data', (chunk) => chunks.push(chunk));
-          res.on('end', () => callback(null, Buffer.concat(chunks)));
-        })
-        .expect(200);
-
-      expect(res.body).toBeInstanceOf(Buffer);
-      expect(res.body.length).toBeGreaterThan(100);
-      // DOCX is a ZIP file
-      expect(res.body[0]).toBe(0x50);
-      expect(res.body[1]).toBe(0x4b);
-    });
-
-    it('returns 400 without url', async () => {
-      await request(app).get('/docx').expect(400);
-    });
-  });
-
-  describe('GET /pdf', () => {
-    it('returns 400 without url', async () => {
-      await request(app).get('/pdf').expect(400);
-    });
-  });
-});

package/tests/integration/archive-default-layout.test.js

@@ -1,37 +0,0 @@
-/**
- * Pin the default `--format archive` layout contract (issue #113):
- * the zip contains exactly `document.md` + `document.html` + `images/`.
- */
-import unzipper from 'unzipper';
-import { buildArchiveFromHtml } from '../../src/archive.js';
-
-const TINY_PNG =
-  'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8/5+hHgAHggJ/PchI7wAAAABJRU5ErkJggg==';
-const FIXTURE_HTML = `<!doctype html><html><body>
-  <h1>Hi</h1><p>Para 1.</p>
-  <p><img alt="" src="data:image/png;base64,${TINY_PNG}"></p>
-  <p>Para 2.</p>
-</body></html>`;
-
-it('default archive contains document.md, document.html, and images/ folder', async () => {
-  const buf = await buildArchiveFromHtml(
-    FIXTURE_HTML,
-    'https://example.invalid/'
-  );
-  const dir = await unzipper.Open.buffer(buf);
-  const names = dir.files.map((f) => f.path);
-
-  expect(names).toEqual(
-    expect.arrayContaining(['document.md', 'document.html'])
-  );
-  expect(names.some((n) => n.startsWith('images/'))).toBe(true);
-
-  const mdEntry = dir.files.find((f) => f.path === 'document.md');
-  const md = (await mdEntry.buffer()).toString();
-  expect(md).toMatch(/images\//);
-  expect(md).not.toMatch(/data:image/);
-
-  const htmlEntry = dir.files.find((f) => f.path === 'document.html');
-  const html = (await htmlEntry.buffer()).toString();
-  expect(html).toMatch(/<h1>/);
-});