@v1nvn/readability 0.45.0 → 0.45.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (64) hide show
  1. package/README.md +3 -3
  2. package/package.json +4 -4
  3. package/src/cli.js +99 -0
  4. package/src/config.js +29 -0
  5. package/src/dev.js +55 -0
  6. package/src/errors.js +16 -0
  7. package/src/handler.js +4 -0
  8. package/src/host-sampling.js +57 -0
  9. package/src/index.js +23 -0
  10. package/src/index.ts +0 -0
  11. package/src/logger.js +2 -0
  12. package/src/output/format.js +92 -0
  13. package/src/pipeline/cell-text.js +51 -0
  14. package/src/pipeline/context.js +1 -0
  15. package/src/pipeline/dom.js +16 -0
  16. package/src/pipeline/footnotes.js +148 -0
  17. package/src/pipeline/math.js +113 -0
  18. package/src/pipeline/normalize.js +507 -0
  19. package/src/pipeline/readability.js +10 -0
  20. package/src/pipeline/resolver.js +113 -0
  21. package/src/pipeline/sanitize.js +26 -0
  22. package/src/pipeline/tables.js +239 -0
  23. package/src/pipeline/turndown.js +153 -0
  24. package/src/pipeline/urls.js +14 -0
  25. package/src/policy/chunk.js +365 -0
  26. package/src/policy/diagnostics.js +53 -0
  27. package/src/policy/explain.js +104 -0
  28. package/src/policy/fallback.js +68 -0
  29. package/src/policy/gating.js +90 -0
  30. package/src/policy/grid-detector.js +207 -0
  31. package/src/policy/identifiers.js +62 -0
  32. package/src/policy/images.js +49 -0
  33. package/src/policy/list-detector.js +232 -0
  34. package/src/policy/lost-signal.js +44 -0
  35. package/src/policy/markdown.js +58 -0
  36. package/src/policy/metadata.js +216 -0
  37. package/src/policy/outline-chains.js +165 -0
  38. package/src/policy/outline.js +71 -0
  39. package/src/policy/pagination.js +79 -0
  40. package/src/policy/presets.js +100 -0
  41. package/src/policy/section.js +111 -0
  42. package/src/policy/selector-lint.js +143 -0
  43. package/src/policy/sibling-scan.js +92 -0
  44. package/src/policy/text.js +18 -0
  45. package/src/policy/truncate.js +45 -0
  46. package/src/preset-cache.js +165 -0
  47. package/src/resources.js +227 -0
  48. package/src/sampling.js +45 -0
  49. package/src/server.js +73 -0
  50. package/src/tools/chunk_text.js +40 -0
  51. package/src/tools/explain.js +200 -0
  52. package/src/tools/extract.js +263 -0
  53. package/src/tools/extract_grid.js +56 -0
  54. package/src/tools/extract_links.js +98 -0
  55. package/src/tools/extract_list.js +66 -0
  56. package/src/tools/extract_metadata.js +67 -0
  57. package/src/tools/extract_section.js +56 -0
  58. package/src/tools/extract_tables.js +59 -0
  59. package/src/tools/html-source.js +30 -0
  60. package/src/tools/html_to_markdown.js +113 -0
  61. package/src/tools/outline.js +47 -0
  62. package/src/tools/output-schema.js +504 -0
  63. package/src/tools/schemas.js +288 -0
  64. package/src/tools/suggest-preset.js +504 -0
package/README.md CHANGED
@@ -10,9 +10,9 @@ User-facing docs: [root README](../../README.md).
10
10
  ## Install
11
11
 
12
12
  ```bash
13
- npm install @v1nvn/readability@0.45.0
13
+ npm install @v1nvn/readability@0.45.1
14
14
  # or run on demand:
15
- npx @v1nvn/readability@0.45.0
15
+ npx @v1nvn/readability@0.45.1
16
16
  ```
17
17
 
18
18
  Requires Node >= 22.22.2 (jsdom 30's floor). Build from source:
@@ -58,7 +58,7 @@ Add to your MCP client config (Claude Code, Claude Desktop, etc.):
58
58
  "mcpServers": {
59
59
  "readability": {
60
60
  "command": "npx",
61
- "args": ["-y", "@v1nvn/readability@0.45.0"],
61
+ "args": ["-y", "@v1nvn/readability@0.45.1"],
62
62
  },
63
63
  },
64
64
  }
package/package.json CHANGED
@@ -1,10 +1,10 @@
1
1
  {
2
2
  "name": "@v1nvn/readability",
3
- "version": "0.45.0",
3
+ "version": "0.45.1",
4
4
  "description": "MCP server that turns rendered (post-JS) HTML into clean Markdown + metadata via Readability, Turndown, and DOMPurify.",
5
5
  "type": "module",
6
- "main": "./src/index.ts",
7
- "bin": "./src/index.ts",
6
+ "main": "./src/index.js",
7
+ "bin": "./src/index.js",
8
8
  "scripts": {
9
9
  "bench": "vite-node test/bench/run.ts",
10
10
  "dev": "vite-node src/dev.ts",
@@ -43,7 +43,7 @@
43
43
  "dependencies": {
44
44
  "@modelcontextprotocol/sdk": "^1.29.0",
45
45
  "@mozilla/readability": "^0.6.0",
46
- "@v1nvn/agentic-core": "^0.45.0",
46
+ "@v1nvn/agentic-core": "^0.45.1",
47
47
  "dompurify": "^3.4.12",
48
48
  "jsdom": "^30.0.1",
49
49
  "remark-gfm": "^4.0.1",
package/src/cli.js ADDED
@@ -0,0 +1,99 @@
1
+ import { describeError, HelpRequested, parseCli, runCli, } from '@v1nvn/agentic-core';
2
+ import { extractArticleFromHtml } from "./tools/extract.js";
3
+ import { readHtmlFile } from "./tools/html-source.js";
4
+ const FORMATS = ['html', 'json', 'md'];
5
+ function parseMaxChars(value) {
6
+ const n = Number(value);
7
+ if (!Number.isInteger(n)) {
8
+ throw new Error('must be an integer');
9
+ }
10
+ return n;
11
+ }
12
+ const spec = {
13
+ name: 'readability',
14
+ usageExit: 2,
15
+ commands: [
16
+ {
17
+ name: 'extract',
18
+ description: 'extract the readable article from an HTML file',
19
+ argument: {
20
+ flags: '[file]',
21
+ description: 'HTML file; stdin when omitted',
22
+ },
23
+ options: [
24
+ {
25
+ flags: '--format <fmt>',
26
+ description: 'output format',
27
+ choices: FORMATS,
28
+ default: 'md',
29
+ },
30
+ {
31
+ flags: '--max-chars <n>',
32
+ description: 'truncate the output',
33
+ parse: parseMaxChars,
34
+ },
35
+ ],
36
+ run: async (options, args) => {
37
+ try {
38
+ const html = await readHtml(args.at(0), process.stdin);
39
+ // json reuses the markdown pipeline; the structured object is serialized below.
40
+ const pipelineFormat = options.format === 'html' ? 'html' : 'markdown';
41
+ const result = extractArticleFromHtml({
42
+ html,
43
+ format: pipelineFormat,
44
+ ...(typeof options.maxChars === 'number'
45
+ ? { maxChars: options.maxChars }
46
+ : {}),
47
+ });
48
+ if (result.isError) {
49
+ process.stderr.write(`${payloadText(result)}\n`);
50
+ return 1;
51
+ }
52
+ if (options.format === 'json') {
53
+ process.stdout.write(`${JSON.stringify(result.structuredContent, null, 2)}\n`);
54
+ }
55
+ else {
56
+ process.stdout.write(`${payloadText(result)}\n`);
57
+ }
58
+ return 0;
59
+ }
60
+ catch (err) {
61
+ process.stderr.write(`${describeError(err)}\n`);
62
+ return 1;
63
+ }
64
+ },
65
+ },
66
+ ],
67
+ };
68
+ export function parseArgs(argv) {
69
+ const parsed = parseCli(spec, argv);
70
+ if (parsed === undefined || parsed instanceof HelpRequested) {
71
+ return parsed;
72
+ }
73
+ const { format, maxChars } = parsed.options;
74
+ return { file: parsed.args.at(0), format, maxChars };
75
+ }
76
+ // The stream is injected rather than reading process.stdin directly so the
77
+ // path is testable; chunks may be Buffer or string.
78
+ export async function readHtml(file, stream) {
79
+ if (file !== undefined) {
80
+ return readHtmlFile(file);
81
+ }
82
+ const chunks = [];
83
+ for await (const chunk of stream) {
84
+ if (typeof chunk === 'string') {
85
+ chunks.push(chunk);
86
+ }
87
+ else {
88
+ chunks.push(Buffer.from(chunk).toString('utf8'));
89
+ }
90
+ }
91
+ return chunks.join('');
92
+ }
93
+ function payloadText(result) {
94
+ const first = result.content.at(0);
95
+ return first !== undefined && 'text' in first ? first.text : '';
96
+ }
97
+ export function runExtract(argv) {
98
+ return runCli(spec, argv);
99
+ }
package/src/config.js ADDED
@@ -0,0 +1,29 @@
1
+ import pkg from '../package.json' with { type: 'json' };
2
+ const SERVER_TITLE = 'Readability MCP';
3
+ const SERVER_DESCRIPTION = 'Turn already-rendered (post-JavaScript) HTML into clean, LLM-friendly Markdown plus metadata, via Mozilla Readability, Turndown, and DOMPurify. Makes no outbound requests — input is the rendered HTML, read from a file path (localPath) so the page bytes never enter the model context.';
4
+ const SERVER_INSTRUCTIONS = `Every tool reads already-rendered HTML from a file path (\`localPath\`) — e.g. \`document.documentElement.outerHTML\` written to disk by a browser/devtools capture — so the page bytes never enter the model context; \`chunk_text\` alone takes text. The server never fetches URLs: \`baseUrl\` only absolutizes links.
5
+
6
+ Pick the tool by page shape:
7
+ - extract: the default for article-like pages → Markdown + metadata + diagnostics. \`cache: true\` exposes the result as a \`readability://page/{hash}\` resource; \`chunk\` adds token-bounded chunks.
8
+ - extract_list: feed, index, search, or HN-style pages Readability cannot turn into one article.
9
+ - extract_tables: every \`<table>\` on the page, including ones outside the article body.
10
+ - extract_grid: div-rendered tables in SPAs.
11
+ - extract_section: one section, by CSS selector or heading text.
12
+ - extract_links: anchors from the raw DOM, for crawl and navigation decisions.
13
+ - extract_metadata, outline: cheap pre-checks (bibliographic metadata; h1-h6 heading TOC) before full extraction.
14
+ - html_to_markdown: a fragment already isolated, converted with no Readability scoring.
15
+ - explain: post-mortem when \`extract\` picked the wrong root.
16
+ - chunk_text: split already-extracted text for embedding/RAG.
17
+
18
+ When the client advertises MCP \`sampling\`, two more tools appear: \`summarize\` (the host's model summarizes text) and \`suggest_preset\` (after \`extract\` reports gating, a fallback, a near-empty result, or debris, the host's model proposes site selectors; the server verifies them and caches a per-site preset).
19
+
20
+ Failures come back as \`isError: true\` results, never thrown.`;
21
+ export function loadConfig() {
22
+ return {
23
+ name: 'readability',
24
+ version: pkg.version,
25
+ title: SERVER_TITLE,
26
+ description: SERVER_DESCRIPTION,
27
+ instructions: SERVER_INSTRUCTIONS,
28
+ };
29
+ }
package/src/dev.js ADDED
@@ -0,0 +1,55 @@
1
+ // Dev-only hot reload: tool implementations reload on file change without
2
+ // restarting the process; the harness in core owns the vite/runner/watch loop.
3
+ // Never bundled into dist (build entry is src/index.ts). Diagnostics to stderr.
4
+ import { StdioServerTransport } from '@modelcontextprotocol/sdk/server/stdio.js';
5
+ import { describeError } from '@v1nvn/agentic-core';
6
+ import { runDevServer } from '@v1nvn/agentic-core/vite';
7
+ import { dirname, resolve } from 'node:path';
8
+ import { fileURLToPath } from 'node:url';
9
+ import { logger } from "./logger.js";
10
+ const here = dirname(fileURLToPath(import.meta.url));
11
+ runDevServer({
12
+ entry: resolve(here, 'server.ts'),
13
+ log: logger,
14
+ start: async (mod) => {
15
+ const server = mod.createMcpServer();
16
+ mod.loadPresets();
17
+ let handles = [
18
+ ...mod.registerTools(server),
19
+ ...mod.registerResources(server),
20
+ ];
21
+ // Capability-gated tools (sampling) defer to the `initialized`
22
+ // notification: `connect()` returns before the client advertises
23
+ // capabilities, so a direct call here would observe
24
+ // `getClientCapabilities() === undefined`. The hook fires once per
25
+ // connection, after the initialize handshake but before any `tools/list`.
26
+ let gatedHandles = [];
27
+ server.server.oninitialized = () => {
28
+ gatedHandles = mod.registerCapabilityGatedTools(server);
29
+ };
30
+ const transport = new StdioServerTransport();
31
+ await server.connect(transport);
32
+ return {
33
+ closers: [server],
34
+ reload: next => {
35
+ for (const handle of [...handles, ...gatedHandles]) {
36
+ handle.remove();
37
+ }
38
+ handles = [
39
+ ...next.registerTools(server),
40
+ ...next.registerResources(server),
41
+ ];
42
+ // The re-imported module graph starts with an empty preset store;
43
+ // refill it alongside the re-registrations. The client is already
44
+ // past `initialized` by the time a file change fires, so the gate
45
+ // can re-evaluate directly.
46
+ next.loadPresets();
47
+ gatedHandles = next.registerCapabilityGatedTools(server);
48
+ },
49
+ };
50
+ },
51
+ }).catch((error) => {
52
+ logger.error(`dev failed: ${describeError(error)}`);
53
+ // eslint-disable-next-line n/no-process-exit
54
+ process.exit(1);
55
+ });
package/src/errors.js ADDED
@@ -0,0 +1,16 @@
1
+ import { describeError } from '@v1nvn/agentic-core';
2
+ export class ExtractionError extends Error {
3
+ cause;
4
+ constructor(message, options = {}) {
5
+ super(message);
6
+ this.name = 'ExtractionError';
7
+ this.cause = options.cause;
8
+ }
9
+ }
10
+ export function toErrorResult(err) {
11
+ const label = err instanceof ExtractionError ? err.name : 'Error';
12
+ return {
13
+ isError: true,
14
+ content: [{ type: 'text', text: `${label}: ${describeError(err)}` }],
15
+ };
16
+ }
package/src/handler.js ADDED
@@ -0,0 +1,4 @@
1
+ import { toolHandler } from '@v1nvn/agentic-core';
2
+ import { toErrorResult } from "./errors.js";
3
+ import { logger } from "./logger.js";
4
+ export const tool = toolHandler(logger, toErrorResult);
@@ -0,0 +1,57 @@
1
+ // The one seam to the HOST's model via MCP `sampling/createMessage`
2
+ // (server→client request). The server never embeds a model and never calls a
3
+ // provider directly — every LLM call is delegated to the connected client,
4
+ // which picks the model and may prompt the user first. Gating on the
5
+ // capability happens at registration (server.ts), not here.
6
+ // Host models — and the human-in-the-loop approval MCP allows before each
7
+ // call — routinely take longer than the SDK's 60s request default.
8
+ const SAMPLING_TIMEOUT_MS = 300_000;
9
+ export async function sampleText(server, args) {
10
+ const result = await server.server.createMessage({
11
+ messages: [
12
+ {
13
+ role: 'user',
14
+ content: { type: 'text', text: args.userText },
15
+ },
16
+ ],
17
+ systemPrompt: args.systemPrompt,
18
+ maxTokens: args.maxTokens,
19
+ }, { timeout: SAMPLING_TIMEOUT_MS });
20
+ if (result.content.type !== 'text') {
21
+ throw new Error(`host sampling returned non-text content (${result.content.type})`);
22
+ }
23
+ return result.content.text;
24
+ }
25
+ // createMessage has no structured-output field, so JSON proposals are
26
+ // negotiated in prose: the prompt demands strict JSON, and the reply is
27
+ // tolerated past code fences before the caller's schema takes over.
28
+ export class SuggestParseError extends Error {
29
+ rawText;
30
+ constructor(message, rawText) {
31
+ super(message);
32
+ this.rawText = rawText;
33
+ }
34
+ }
35
+ export function extractJsonReply(text) {
36
+ const withoutFences = text.replace(/^[\s\S]*?```(?:json)?\s*\n?([\s\S]*?)\n?```[\s\S]*$/, '$1');
37
+ const trimmed = withoutFences.trim();
38
+ try {
39
+ return JSON.parse(trimmed);
40
+ }
41
+ catch {
42
+ const start = trimmed.indexOf('{');
43
+ const end = trimmed.lastIndexOf('}');
44
+ if (start !== -1 && end > start) {
45
+ try {
46
+ return JSON.parse(trimmed.slice(start, end + 1));
47
+ }
48
+ catch {
49
+ // Fall through to the error below.
50
+ }
51
+ }
52
+ throw new SuggestParseError(`host sampling returned text that does not parse as JSON (${trimmed.slice(0, 80)}…)`, trimmed);
53
+ }
54
+ }
55
+ export async function sampleJson(server, args) {
56
+ return extractJsonReply(await sampleText(server, args));
57
+ }
package/src/index.js ADDED
@@ -0,0 +1,23 @@
1
+ #!/usr/bin/env node
2
+ import { StdioServerTransport } from '@modelcontextprotocol/sdk/server/stdio.js';
3
+ import { describeError, shutdownOnSignals } from '@v1nvn/agentic-core';
4
+ import { createServer } from "./server.js";
5
+ if (process.argv[2] === 'extract') {
6
+ void import("./cli.js")
7
+ .then(m => m.runExtract(process.argv.slice(2)))
8
+ .then(code => {
9
+ // eslint-disable-next-line n/no-process-exit
10
+ process.exit(code);
11
+ })
12
+ .catch((err) => {
13
+ process.stderr.write(`${describeError(err)}\n`);
14
+ // eslint-disable-next-line n/no-process-exit
15
+ process.exit(1);
16
+ });
17
+ }
18
+ else {
19
+ const server = createServer();
20
+ const transport = new StdioServerTransport();
21
+ await server.connect(transport);
22
+ shutdownOnSignals([server]);
23
+ }
package/src/index.ts CHANGED
File without changes
package/src/logger.js ADDED
@@ -0,0 +1,2 @@
1
+ import { loggerFromEnv } from '@v1nvn/agentic-core';
2
+ export const logger = loggerFromEnv(process.env, 'READABILITY_LOG_LEVEL');
@@ -0,0 +1,92 @@
1
+ import { stringify as stringifyYaml } from 'yaml';
2
+ import { headingText, parseBlocks } from "../policy/markdown.js";
3
+ // Readability demotes the article <h1> to <h2> inside `content` and mirrors its
4
+ // text into the title; drop the echoed sub-heading so the title prints once.
5
+ function dropEchoedTitle(body, title) {
6
+ const blocks = parseBlocks(body);
7
+ if (blocks.length === 0) {
8
+ return body;
9
+ }
10
+ const first = blocks[0];
11
+ if (first.kind === 'heading' &&
12
+ headingText(body.slice(first.start, first.end)) === title.trim()) {
13
+ return body.slice(first.end).replace(/^\n+/, '');
14
+ }
15
+ return body;
16
+ }
17
+ function renderMarkdown(input) {
18
+ const title = input.metadata.title?.trim();
19
+ let body = input.markdown;
20
+ if (title) {
21
+ body = dropEchoedTitle(body, title);
22
+ return `# ${title}\n\n${body}`.replace(/\n+$/, '\n');
23
+ }
24
+ return body.replace(/\n+$/, '\n');
25
+ }
26
+ // Stable key order for deterministic frontmatter. `structured` is deliberately
27
+ // absent: it is a nested object, not a scalar, and would dump a whole JSON-LD
28
+ // graph into frontmatter. It reaches the host via format=json and
29
+ // structuredContent.metadata, which serialize the full metadata object.
30
+ const METADATA_KEYS = [
31
+ 'title',
32
+ 'byline',
33
+ 'siteName',
34
+ 'lang',
35
+ 'publishedTime',
36
+ 'excerpt',
37
+ 'canonical',
38
+ 'baseUrl',
39
+ 'wordCount',
40
+ 'readingTimeMin',
41
+ 'tokenEstimate',
42
+ 'estimator',
43
+ ];
44
+ function pickMetadata(metadata) {
45
+ const picked = {};
46
+ for (const key of METADATA_KEYS) {
47
+ const value = metadata[key];
48
+ if (value !== undefined) {
49
+ picked[key] = value;
50
+ }
51
+ }
52
+ return picked;
53
+ }
54
+ // lineWidth: 0 disables folding so each field stays on one line.
55
+ function yamlFrontmatter(metadata) {
56
+ return `---\n${stringifyYaml(pickMetadata(metadata), { lineWidth: 0 })}---\n`;
57
+ }
58
+ function jsonFrontmatter(metadata) {
59
+ return ('```json\n' + JSON.stringify(pickMetadata(metadata), null, 2) + '\n```\n');
60
+ }
61
+ function withFrontmatter(payload, mode, metadata) {
62
+ if (mode === 'yaml') {
63
+ return `${yamlFrontmatter(metadata)}${payload}`;
64
+ }
65
+ if (mode === 'json') {
66
+ return `${jsonFrontmatter(metadata)}\n${payload}`;
67
+ }
68
+ return payload;
69
+ }
70
+ export function formatPayload(input) {
71
+ let payload;
72
+ switch (input.format) {
73
+ case 'html':
74
+ return input.sanitizedHtml;
75
+ case 'json': {
76
+ const body = {
77
+ metadata: input.metadata,
78
+ content: input.markdown,
79
+ diagnostics: input.diagnostics,
80
+ };
81
+ return JSON.stringify(body, null, 2);
82
+ }
83
+ case 'text':
84
+ payload = input.textContent;
85
+ break;
86
+ case 'markdown':
87
+ default:
88
+ payload = renderMarkdown(input);
89
+ break;
90
+ }
91
+ return withFrontmatter(payload, input.metadataMode, input.metadata);
92
+ }
@@ -0,0 +1,51 @@
1
+ // Resolve a table/grid cell element to the string that represents it in the
2
+ // matrix. Shared by tables.ts (the <table> serializer) and
3
+ // policy/grid-detector.ts (the div-grid detector) so both paths project a
4
+ // cell the same way: whitespace-normalized visible text, with two fallbacks.
5
+ //
6
+ // 1. Tooltip/badge/aria wrappers carry chrome, not data — they are stripped
7
+ // from a clone so a label cell reads "Market Capitalization" rather than
8
+ // concatenating every descendant. But stripping must never empty a cell: a
9
+ // badge/aria element that holds a cell's only text is data, not chrome, so
10
+ // when stripping would blank the cell the full text is kept.
11
+ // 2. A cell whose only content is an icon link has no visible text at all; its
12
+ // value lives in the anchor's href, which is surfaced as a last resort.
13
+ const CELL_CHROME_SELECTOR = '[aria-label], [data-tooltip], [data-toggle="tooltip"], .tooltip, .badge';
14
+ function normalize(text) {
15
+ return text.replace(/\s+/g, ' ').trim();
16
+ }
17
+ // All non-empty hrefs reachable from the cell, including the cell itself when it
18
+ // is an anchor (a div-grid "cell" can be an <a>). Descendant-only search would
19
+ // miss a link-cell whose element IS the anchor.
20
+ function nonEmptyHrefs(cell) {
21
+ const anchors = cell.tagName === 'A'
22
+ ? [cell, ...Array.from(cell.querySelectorAll('a[href]'))]
23
+ : Array.from(cell.querySelectorAll('a[href]'));
24
+ const hrefs = [];
25
+ for (const a of anchors) {
26
+ const href = (a.getAttribute('href') ?? '').trim();
27
+ if (href !== '') {
28
+ hrefs.push(href);
29
+ }
30
+ }
31
+ return hrefs;
32
+ }
33
+ export function resolveCellText(cell) {
34
+ const full = normalize(cell.textContent);
35
+ let resolved = full;
36
+ if (full !== '') {
37
+ const clone = cell.cloneNode(true);
38
+ clone.querySelectorAll(CELL_CHROME_SELECTOR).forEach(el => {
39
+ el.remove();
40
+ });
41
+ const stripped = normalize(clone.textContent);
42
+ resolved = stripped !== '' ? stripped : full;
43
+ }
44
+ if (resolved === '') {
45
+ const hrefs = nonEmptyHrefs(cell);
46
+ if (hrefs.length > 0) {
47
+ return hrefs.join(' ');
48
+ }
49
+ }
50
+ return resolved;
51
+ }
@@ -0,0 +1 @@
1
+ export {};
@@ -0,0 +1,16 @@
1
+ import { JSDOM } from 'jsdom';
2
+ // Built only from caller-supplied `html`; `baseUrl` is origin context for
3
+ // absolutization, never fetched. Parse and the downstream Readability walk run
4
+ // synchronously on the event loop — no worker isolation or wall-clock timeout
5
+ // guards them. A worker pool was deferred rather than rejected: the synchronous
6
+ // parse can't be interrupted in-process, and `worker_threads` can't load the dev
7
+ // hot-reload loop's in-memory Vite modules. `maxNodes` is the only input-size
8
+ // bound until a holistic worker strategy lands; a standalone `timeout` option
9
+ // would mislead (it can't interrupt the parse), so it ships with the worker.
10
+ export function buildDocument(html, baseUrl) {
11
+ const dom = new JSDOM(html, { url: baseUrl });
12
+ return { document: dom.window.document, window: dom.window };
13
+ }
14
+ export function isElement(node) {
15
+ return node.nodeType === 1;
16
+ }
@@ -0,0 +1,148 @@
1
+ import { buildDocument } from "./dom.js";
2
+ // jsdom-parsing every turndown call would dominate hot paths; this pre-check
3
+ // gates on a string scan so footnote-less articles pay nothing.
4
+ const FOOTNOTE_SIGNAL_RE = /cite_note|cite_ref|class="footnotes"|class="references"|data-footnote|role="doc-endnote|<sup\b[^>]*>\s*<a\s[^>]*href="#/i;
5
+ // First match wins per group; an `<ol class="footnotes">` is the common case.
6
+ const DEFINITION_CONTAINER_SELECTORS = [
7
+ 'ol.footnotes',
8
+ 'ol[class*="footnotes"]',
9
+ 'ol.references',
10
+ 'ol[class*="references"]',
11
+ 'section[class*="footnote"]',
12
+ 'div[class*="footnote"]',
13
+ '[role="doc-endnotes"]',
14
+ '[role="doc-bibliography"]',
15
+ ];
16
+ const STANDALONE_DEF_ID_RE = /^(?:fn|cite_note|footnote|note)[:_-]/i;
17
+ // Some sites prepend a back-to-text caret or "Jump to" label to each definition.
18
+ // Stripping only a leading occurrence keeps accidental mid-text carets intact.
19
+ const BACKREF_LEADING_RE = /^(?:↑\s?|↩\s?|\^\s|Jump to\s*)/;
20
+ function cleanDefText(text) {
21
+ return text.replace(/\s+/g, ' ').trim().replace(BACKREF_LEADING_RE, '');
22
+ }
23
+ function collectDefinitions(document) {
24
+ const containers = new Set();
25
+ const defs = new Map();
26
+ const standaloneIds = new Set();
27
+ for (const selector of DEFINITION_CONTAINER_SELECTORS) {
28
+ let matched;
29
+ try {
30
+ matched = document.querySelectorAll(selector);
31
+ }
32
+ catch {
33
+ continue;
34
+ }
35
+ for (const container of Array.from(matched)) {
36
+ if (!containers.has(container)) {
37
+ containers.add(container);
38
+ }
39
+ for (const item of Array.from(container.querySelectorAll('li, [role="doc-endnote"]'))) {
40
+ const id = item.id;
41
+ if (!id || defs.has(id)) {
42
+ continue;
43
+ }
44
+ const text = cleanDefText(item.textContent);
45
+ if (text) {
46
+ defs.set(id, text);
47
+ }
48
+ }
49
+ }
50
+ }
51
+ // Some sites scatter `<li id="fn1">` outside any container; collect those too.
52
+ for (const li of Array.from(document.querySelectorAll('li[id]'))) {
53
+ const id = li.id;
54
+ if (!id || defs.has(id) || !STANDALONE_DEF_ID_RE.test(id)) {
55
+ continue;
56
+ }
57
+ const text = cleanDefText(li.textContent);
58
+ if (text) {
59
+ defs.set(id, text);
60
+ standaloneIds.add(id);
61
+ }
62
+ }
63
+ return { containers, defs, standaloneIds };
64
+ }
65
+ export function processFootnotes(html) {
66
+ if (!html || !FOOTNOTE_SIGNAL_RE.test(html)) {
67
+ return null;
68
+ }
69
+ let document;
70
+ try {
71
+ document = buildDocument(html).document;
72
+ }
73
+ catch {
74
+ return null;
75
+ }
76
+ const { containers, defs, standaloneIds } = collectDefinitions(document);
77
+ if (defs.size === 0) {
78
+ return null;
79
+ }
80
+ const refHits = [];
81
+ const defIdToNumber = new Map();
82
+ for (const sup of Array.from(document.querySelectorAll('sup'))) {
83
+ if (!sup.isConnected) {
84
+ continue;
85
+ }
86
+ try {
87
+ const anchor = sup.querySelector('a[href^="#"]');
88
+ if (!anchor) {
89
+ continue;
90
+ }
91
+ const frag = (anchor.getAttribute('href') ?? '').slice(1);
92
+ if (!frag || !defs.has(frag)) {
93
+ continue;
94
+ }
95
+ let n = defIdToNumber.get(frag);
96
+ if (n === undefined) {
97
+ n = defIdToNumber.size + 1;
98
+ defIdToNumber.set(frag, n);
99
+ }
100
+ refHits.push({ defId: frag, n, sup });
101
+ }
102
+ catch {
103
+ // Defensive: malformed <sup> must not abort the whole pass.
104
+ }
105
+ }
106
+ if (refHits.length === 0) {
107
+ return null;
108
+ }
109
+ for (const { n, sup } of refHits) {
110
+ sup.replaceWith(document.createTextNode(`[^${n}]`));
111
+ }
112
+ // Drop the rendered references list so it doesn't duplicate the appended
113
+ // `[^N]:` block. Removing containers also takes care of the standalone <li>
114
+ // elements that lived inside them; only out-of-container standalones need
115
+ // individual removal here.
116
+ for (const container of containers) {
117
+ if (container.isConnected) {
118
+ container.remove();
119
+ }
120
+ }
121
+ for (const id of standaloneIds) {
122
+ const el = document.getElementById(id);
123
+ if (el?.isConnected) {
124
+ el.remove();
125
+ }
126
+ }
127
+ const numberToDefId = new Map();
128
+ for (const [id, n] of defIdToNumber) {
129
+ numberToDefId.set(n, id);
130
+ }
131
+ const footnoteDefs = [];
132
+ for (let n = 1; n <= numberToDefId.size; n++) {
133
+ const id = numberToDefId.get(n);
134
+ if (id === undefined) {
135
+ break;
136
+ }
137
+ const text = defs.get(id);
138
+ if (text !== undefined) {
139
+ footnoteDefs.push(text);
140
+ }
141
+ }
142
+ // buildDocument wraps both full documents and bare fragments in a <body>;
143
+ // serializing its innerHTML returns the equivalent of the input region.
144
+ return {
145
+ footnoteDefs,
146
+ html: document.body.innerHTML,
147
+ };
148
+ }