@v1nvn/readability-mcp 0.14.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +310 -0
- package/dist/assets/cli-BwKCixh6.js +83 -0
- package/dist/assets/cli-BwKCixh6.js.map +1 -0
- package/dist/assets/extract-BKl4PzEI.js +2734 -0
- package/dist/assets/extract-BKl4PzEI.js.map +1 -0
- package/dist/index.js +1425 -0
- package/dist/index.js.map +1 -0
- package/package.json +62 -0
package/dist/index.js
ADDED
|
@@ -0,0 +1,1425 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
import { $ as outlineInputShape, A as extractListOutputShape, B as extractLinksInputShape, C as resolveLazyImages, D as chunkTextOutputShape, E as isElement, F as chunkTextInputSchema, G as extractSectionInputSchema, H as extractListInputShape, I as chunkTextInputShape, J as extractTablesInputShape, K as extractSectionInputShape, L as extractGridInputSchema, M as extractTablesOutputShape, N as outlineOutputShape, O as extractGridOutputShape, P as outputSchemaShape, Q as outlineInputSchema, R as extractGridInputShape, S as normalizeDocument, T as buildDocument, U as extractMetadataInputSchema, V as extractListInputSchema, W as extractMetadataInputShape, X as htmlToMarkdownInputShape, Y as htmlToMarkdownInputSchema, Z as localPathField, _ as detectGating, a as sanitizeHtml, at as registerResources, b as isReaderable, c as computeTextMetrics, d as renderTable, et as selectorsSchema, f as resolveHeaderKeys, g as absolutize, h as detectPagination, i as toMarkdown, it as toErrorResult, j as extractMetadataOutputShape, k as extractLinksOutputShape, l as nonEmpty, m as readHtmlFile, n as registerExtractTool, nt as logger, o as formatPayload, ot as loadConfig, p as resolveCellText, q as extractTablesInputSchema, r as truncateMarkdown, rt as ExtractionError, s as resolveMetadata, t as extractArticleFromHtml, tt as chunkMarkdown, u as parseTableMatrix, v as TraceCollector, w as resolveReadabilityOptions, x as applySelectors, y as assembleDiagnostics, z as extractLinksInputSchema } from "./assets/extract-BKl4PzEI.js";
|
|
3
|
+
import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js";
|
|
4
|
+
import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
|
|
5
|
+
import { z } from "zod";
|
|
6
|
+
import { Readability } from "@mozilla/readability";
|
|
7
|
+
//#region src/sampling.ts
|
|
8
|
+
var SUMMARIZE_SYSTEM_PROMPT = "Summarize the user-supplied text concisely while preserving its key points, entities, and any decisive conclusions. Output only the summary prose — no preamble, no headings unless the source had them.";
|
|
9
|
+
var summarizeInputShape = {
|
|
10
|
+
text: z.string().describe("The markdown or text to summarize — typically the output of `extract`, `extract_section`, `html_to_markdown`, or `chunk_text`. Passed through to the host model verbatim; the server does not parse or modify it."),
|
|
11
|
+
maxTokens: z.number().int().min(1).describe("Upper bound on the summary length in tokens, forwarded to the host as `sampling/createMessage` maxTokens. The host chooses the actual length.").default(512)
|
|
12
|
+
};
|
|
13
|
+
var summarizeInputSchema = z.object(summarizeInputShape);
|
|
14
|
+
var SUMMARIZE_TOOL_DESCRIPTION = `Summarize text using the HOST's model via MCP \`sampling/createMessage\` — the server embeds no model and calls no provider directly. Hand it the output of \`extract\`, \`extract_section\`, \`html_to_markdown\`, or any markdown/text string; the host picks the model and may ask the user to approve the sampling request (human-in-the-loop per MCP). The tool is only listed when the connected client advertises the sampling capability.`;
|
|
15
|
+
async function summarizeWithHost(server, args) {
|
|
16
|
+
const result = await server.server.createMessage({
|
|
17
|
+
messages: [{
|
|
18
|
+
role: "user",
|
|
19
|
+
content: {
|
|
20
|
+
type: "text",
|
|
21
|
+
text: args.text
|
|
22
|
+
}
|
|
23
|
+
}],
|
|
24
|
+
systemPrompt: SUMMARIZE_SYSTEM_PROMPT,
|
|
25
|
+
maxTokens: args.maxTokens
|
|
26
|
+
});
|
|
27
|
+
if (result.content.type !== "text") throw new Error(`host sampling returned non-text content (${result.content.type}); summarize expects a text response`);
|
|
28
|
+
return result.content.text;
|
|
29
|
+
}
|
|
30
|
+
function registerSummarizeTool(server) {
|
|
31
|
+
return server.registerTool("summarize", {
|
|
32
|
+
title: "Summarize text using the host model",
|
|
33
|
+
description: SUMMARIZE_TOOL_DESCRIPTION,
|
|
34
|
+
inputSchema: summarizeInputShape
|
|
35
|
+
}, async (rawArgs) => {
|
|
36
|
+
const args = summarizeInputSchema.parse(rawArgs);
|
|
37
|
+
try {
|
|
38
|
+
return { content: [{
|
|
39
|
+
type: "text",
|
|
40
|
+
text: await summarizeWithHost(server, args)
|
|
41
|
+
}] };
|
|
42
|
+
} catch (err) {
|
|
43
|
+
logger.error(`summarize failed: ${err instanceof Error ? err.message : String(err)}`);
|
|
44
|
+
return toErrorResult(err);
|
|
45
|
+
}
|
|
46
|
+
});
|
|
47
|
+
}
|
|
48
|
+
function registerSamplingTools(server) {
|
|
49
|
+
return [registerSummarizeTool(server)];
|
|
50
|
+
}
|
|
51
|
+
//#endregion
|
|
52
|
+
//#region src/tools/chunk_text.ts
|
|
53
|
+
function renderChunkIndex(chunks) {
|
|
54
|
+
if (chunks.length === 0) return "(no chunks emitted — input had no non-whitespace content)";
|
|
55
|
+
return chunks.map((chunk) => {
|
|
56
|
+
const head = chunk.headingContext ? ` [${chunk.headingContext}]` : "";
|
|
57
|
+
return `## Chunk ${chunk.index}${head}\n\n${chunk.text}`;
|
|
58
|
+
}).join("\n\n");
|
|
59
|
+
}
|
|
60
|
+
function chunkTextDocument(rawArgs) {
|
|
61
|
+
const { text, maxTokens, overlap, strategy } = chunkTextInputSchema.parse(rawArgs);
|
|
62
|
+
const chunks = chunkMarkdown(text, {
|
|
63
|
+
maxTokens,
|
|
64
|
+
overlap,
|
|
65
|
+
strategy
|
|
66
|
+
});
|
|
67
|
+
const content = renderChunkIndex(chunks);
|
|
68
|
+
return {
|
|
69
|
+
content: [{
|
|
70
|
+
text: content,
|
|
71
|
+
type: "text"
|
|
72
|
+
}],
|
|
73
|
+
structuredContent: {
|
|
74
|
+
schemaVersion: 1,
|
|
75
|
+
content,
|
|
76
|
+
chunks
|
|
77
|
+
}
|
|
78
|
+
};
|
|
79
|
+
}
|
|
80
|
+
var CHUNK_TEXT_TOOL_DESCRIPTION = `Split already-extracted text into token-bounded chunks for embedding/RAG. Each chunk carries its index, tokenCount (chars/4), and the nearest preceding markdown heading as headingContext. Operates on any text — pair with the \`chunk\` option on \`extract\` when you want chunks inline with the extraction. The server fetches nothing: \`text\` is the only input.`;
|
|
81
|
+
function chunkTextHandler(args) {
|
|
82
|
+
try {
|
|
83
|
+
return chunkTextDocument(args);
|
|
84
|
+
} catch (err) {
|
|
85
|
+
logger.error(`chunk_text failed: ${err instanceof Error ? err.message : String(err)}`);
|
|
86
|
+
return toErrorResult(err);
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
function registerChunkTextTool(server) {
|
|
90
|
+
return server.registerTool("chunk_text", {
|
|
91
|
+
title: "Chunk text for RAG/embedding",
|
|
92
|
+
description: CHUNK_TEXT_TOOL_DESCRIPTION,
|
|
93
|
+
inputSchema: chunkTextInputShape,
|
|
94
|
+
outputSchema: chunkTextOutputShape
|
|
95
|
+
}, chunkTextHandler);
|
|
96
|
+
}
|
|
97
|
+
//#endregion
|
|
98
|
+
//#region src/policy/explain.ts
|
|
99
|
+
var DEFAULT_SNAPSHOT_MAX = 4e3;
|
|
100
|
+
var DEFAULT_TOP_N = 5;
|
|
101
|
+
function describeSelector$1(parts) {
|
|
102
|
+
const idPart = parts.id ? `#${parts.id}` : "";
|
|
103
|
+
const cls = parts.className.trim().split(/\s+/).filter(Boolean).slice(0, 2).join(".");
|
|
104
|
+
const classPart = cls ? `.${cls}` : "";
|
|
105
|
+
return `${parts.tag.toLowerCase()}${idPart}${classPart}`;
|
|
106
|
+
}
|
|
107
|
+
function readScored(el) {
|
|
108
|
+
const score = el.readability?.contentScore;
|
|
109
|
+
if (typeof score !== "number") return null;
|
|
110
|
+
const candidate = {
|
|
111
|
+
className: typeof el.className === "string" ? el.className : "",
|
|
112
|
+
id: el.id,
|
|
113
|
+
score,
|
|
114
|
+
tag: el.tagName,
|
|
115
|
+
textLength: el.textContent.trim().length
|
|
116
|
+
};
|
|
117
|
+
return {
|
|
118
|
+
...candidate,
|
|
119
|
+
selector: describeSelector$1(candidate)
|
|
120
|
+
};
|
|
121
|
+
}
|
|
122
|
+
function truncateSnapshot(html, max) {
|
|
123
|
+
if (html.length <= max) return {
|
|
124
|
+
html,
|
|
125
|
+
truncated: false
|
|
126
|
+
};
|
|
127
|
+
return {
|
|
128
|
+
html: html.slice(0, max),
|
|
129
|
+
truncated: true
|
|
130
|
+
};
|
|
131
|
+
}
|
|
132
|
+
function buildExplainReport(options) {
|
|
133
|
+
const { html, selectors, snapshotMaxChars = DEFAULT_SNAPSHOT_MAX, topN = DEFAULT_TOP_N, baseUrl } = options;
|
|
134
|
+
const { document, window } = buildDocument(html, baseUrl);
|
|
135
|
+
const gating = detectGating(document);
|
|
136
|
+
const documentElementCount = document.querySelectorAll("*").length;
|
|
137
|
+
const normalizeCounts = normalizeDocument(document);
|
|
138
|
+
resolveLazyImages(document);
|
|
139
|
+
const pagination = detectPagination(document, baseUrl);
|
|
140
|
+
applySelectors(document, selectors);
|
|
141
|
+
const snapshot = truncateSnapshot(document.body.innerHTML, snapshotMaxChars);
|
|
142
|
+
const readerable = isReaderable(document);
|
|
143
|
+
const readabilityOptions = resolveReadabilityOptions({});
|
|
144
|
+
const clone = document.cloneNode(true);
|
|
145
|
+
const heldNodes = Array.from(clone.querySelectorAll("*"));
|
|
146
|
+
const article = new Readability(clone, readabilityOptions).parse();
|
|
147
|
+
const parseSucceeded = !!article?.content;
|
|
148
|
+
const scored = [];
|
|
149
|
+
for (const el of heldNodes) {
|
|
150
|
+
const entry = readScored(el);
|
|
151
|
+
if (entry) scored.push(entry);
|
|
152
|
+
}
|
|
153
|
+
scored.sort((a, b) => b.score - a.score);
|
|
154
|
+
const candidates = scored.slice(0, Math.max(0, topN));
|
|
155
|
+
const chosenRoot = scored[0] ?? null;
|
|
156
|
+
const diagnostics = assembleDiagnostics({
|
|
157
|
+
articleHtml: article?.content ?? "",
|
|
158
|
+
boilerplateRemoved: normalizeCounts.boilerplateRemoved,
|
|
159
|
+
chromeRemoved: normalizeCounts.chromeRemoved,
|
|
160
|
+
documentElementCount,
|
|
161
|
+
extractedNode: "readability",
|
|
162
|
+
fallbackUsed: false,
|
|
163
|
+
gated: gating,
|
|
164
|
+
pagination,
|
|
165
|
+
readerable,
|
|
166
|
+
window
|
|
167
|
+
});
|
|
168
|
+
return {
|
|
169
|
+
candidates,
|
|
170
|
+
chosenRoot,
|
|
171
|
+
fallbackUsed: diagnostics.fallbackUsed,
|
|
172
|
+
gating: diagnostics.gated,
|
|
173
|
+
pagination: diagnostics.pagination,
|
|
174
|
+
parseSucceeded,
|
|
175
|
+
readerable: diagnostics.readerable ?? false,
|
|
176
|
+
removedNodes: {
|
|
177
|
+
boilerplate: diagnostics.boilerplateRemoved ?? 0,
|
|
178
|
+
chrome: diagnostics.chromeRemoved ?? 0,
|
|
179
|
+
total: diagnostics.removedNodes ?? 0
|
|
180
|
+
},
|
|
181
|
+
snapshot
|
|
182
|
+
};
|
|
183
|
+
}
|
|
184
|
+
//#endregion
|
|
185
|
+
//#region src/tools/explain.ts
|
|
186
|
+
var explainInputShape = {
|
|
187
|
+
localPath: localPathField,
|
|
188
|
+
baseUrl: z.url().describe("Base URL for absolutizing relative links during pagination/gating detection. NEVER fetched — origin context only.").optional(),
|
|
189
|
+
selectors: selectorsSchema,
|
|
190
|
+
topN: z.number().int().min(1).max(20).describe("Maximum number of scored candidate nodes to return (highest first). Default 5.").default(5)
|
|
191
|
+
};
|
|
192
|
+
var explainInputSchema = z.object(explainInputShape);
|
|
193
|
+
var DEFAULTS$4 = explainInputSchema.parse({ localPath: "" });
|
|
194
|
+
var candidateSchema = z.object({
|
|
195
|
+
className: z.string().describe("The candidate's class attribute (raw, unsplit). Empty string when absent."),
|
|
196
|
+
id: z.string().describe("The candidate's id attribute, or empty string."),
|
|
197
|
+
score: z.number().describe("Readability's actual contentScore for this node (link-density-scaled). Higher is better; the top entry is Readability's raw top candidate before its parent-walking/only-child adjustments."),
|
|
198
|
+
selector: z.string().describe("A CSS-ish hint (tag#id.class1.class2) for locating the node in the host DOM. NOT a unique locator — Readability's score lives on a JS expando invisible to CSS."),
|
|
199
|
+
tag: z.string().describe("Uppercase DOM tag name (e.g. \"ARTICLE\", \"MAIN\", \"DIV\")."),
|
|
200
|
+
textLength: z.number().int().min(0).describe("Trimmed textContent length of the candidate node, for eyeballing content density.")
|
|
201
|
+
}).describe("One scored candidate node Readability considered, with its real contentScore.");
|
|
202
|
+
var explainOutputShape = {
|
|
203
|
+
schemaVersion: z.literal(1).describe("Structured-content schema version. Bumps only on breaking shape changes to this object."),
|
|
204
|
+
content: z.string().describe("Readable rendering of the report (chosen root, ranked candidates, removal counts, gating/pagination, snapshot head) so content[0].text is always scannable."),
|
|
205
|
+
chosenRoot: candidateSchema.nullable().describe("The highest-scoring candidate Readability computed — its raw top pick before parent-walking/only-child post-processing. Null when Readability scored nothing (e.g. empty input)."),
|
|
206
|
+
candidates: z.array(candidateSchema).describe("Scored candidate nodes (highest first), capped at topN. These are Readability's real contentScore values, not a self-computed heuristic."),
|
|
207
|
+
readerable: z.boolean().describe("Readability isProbablyReaderable verdict on the normalized document."),
|
|
208
|
+
parseSucceeded: z.boolean().describe("True when reader.parse() returned article content. False signals that `extract` would fall back to its selector cascade."),
|
|
209
|
+
fallbackUsed: z.boolean().describe("Always false for explain — this tool runs only the Readability path it is diagnosing, never the fallback cascade."),
|
|
210
|
+
gating: z.object({
|
|
211
|
+
likely: z.boolean().describe("True when heuristics strongly suggest the content is paywalled or truncated."),
|
|
212
|
+
reason: z.string().describe("Short label naming the detected signal (e.g. \"paywall overlay\").")
|
|
213
|
+
}).nullable().describe("Likely paywall / gating signal detected before normalization. Null when none."),
|
|
214
|
+
pagination: z.object({
|
|
215
|
+
type: z.enum(["infinite", "paginated"]).describe("Kind of pagination signal detected."),
|
|
216
|
+
nextUrl: z.string().optional().describe("Absolute URL of the detected next page (paginated only). Never fetched."),
|
|
217
|
+
selector: z.string().optional().describe("CSS selector of the load-more / infinite-scroll sentinel (infinite only).")
|
|
218
|
+
}).nullable().describe("Detected pagination / infinite-scroll signal. Null when none."),
|
|
219
|
+
removedNodes: z.object({
|
|
220
|
+
boilerplate: z.number().int().min(0).describe("Boilerplate blocks (related-posts, newsletter signup) stripped before Readability."),
|
|
221
|
+
chrome: z.number().int().min(0).describe("Browser-chrome nodes stripped (scrollbars, consent banners, overlays)."),
|
|
222
|
+
total: z.number().int().min(0).describe("Net element count removed across the whole pipeline (delta vs. the parsed document).")
|
|
223
|
+
}).describe("Breakdown of nodes removed before Readability saw the document, reused from the extract diagnostics path."),
|
|
224
|
+
snapshot: z.object({
|
|
225
|
+
html: z.string().describe("The sanitized-by-normalization (post chrome/boilerplate/script strip) HTML fed to Readability — \"what Readability saw\". Not DOMPurify-sanitized (that runs on Readability's output in `extract`), so it may still carry inline event handlers (`onerror`/`onclick`/…); it is diagnostic data — do not render verbatim."),
|
|
226
|
+
truncated: z.boolean().describe("True when the snapshot was cut at snapshotMaxChars (default 4000).")
|
|
227
|
+
}).describe("Pre-Readability HTML snapshot of the normalized document body.")
|
|
228
|
+
};
|
|
229
|
+
var explainOutput = z.object(explainOutputShape);
|
|
230
|
+
function formatGating(report) {
|
|
231
|
+
const g = report.gating;
|
|
232
|
+
if (!g) return "none";
|
|
233
|
+
return `${g.reason}${g.likely ? "" : " (weak)"}`;
|
|
234
|
+
}
|
|
235
|
+
function formatPagination(report) {
|
|
236
|
+
const p = report.pagination;
|
|
237
|
+
if (!p) return "none";
|
|
238
|
+
if (p.type === "paginated") return `paginated -> ${p.nextUrl ?? "(no href)"}`;
|
|
239
|
+
return `infinite (${p.selector ?? "sentinel"})`;
|
|
240
|
+
}
|
|
241
|
+
function renderText(report) {
|
|
242
|
+
const lines = [];
|
|
243
|
+
const root = report.chosenRoot;
|
|
244
|
+
lines.push(`readerable: ${report.readerable ? "yes" : "no"} parse: ${report.parseSucceeded ? "ok" : "fail"} fallback: ${report.fallbackUsed ? "yes" : "no"}`);
|
|
245
|
+
lines.push(`chosen root: ${root ? `${root.selector} (score ${root.score.toFixed(2)}, ${root.textLength} chars)` : "(no candidate scored)"}`);
|
|
246
|
+
lines.push(`top candidates (${report.candidates.length}):`);
|
|
247
|
+
if (report.candidates.length === 0) lines.push(" (none)");
|
|
248
|
+
report.candidates.forEach((c, i) => {
|
|
249
|
+
lines.push(` ${i + 1}. ${c.selector} score ${c.score.toFixed(2)} (${c.textLength} chars)`);
|
|
250
|
+
});
|
|
251
|
+
const r = report.removedNodes;
|
|
252
|
+
lines.push(`removed: total=${r.total} (chrome=${r.chrome}, boilerplate=${r.boilerplate})`);
|
|
253
|
+
lines.push(`gating: ${formatGating(report)}`);
|
|
254
|
+
lines.push(`pagination: ${formatPagination(report)}`);
|
|
255
|
+
lines.push(`snapshot (${report.snapshot.html.length} chars${report.snapshot.truncated ? ", truncated" : ""}):`);
|
|
256
|
+
lines.push(report.snapshot.html);
|
|
257
|
+
return lines.join("\n");
|
|
258
|
+
}
|
|
259
|
+
function explain(rawArgs) {
|
|
260
|
+
const { localPath, ...rest } = explainInputSchema.parse(rawArgs);
|
|
261
|
+
return explainFromHtml({
|
|
262
|
+
html: readHtmlFile(localPath),
|
|
263
|
+
...rest
|
|
264
|
+
});
|
|
265
|
+
}
|
|
266
|
+
function explainFromHtml(input) {
|
|
267
|
+
const { html, baseUrl, selectors, topN } = {
|
|
268
|
+
...DEFAULTS$4,
|
|
269
|
+
...input
|
|
270
|
+
};
|
|
271
|
+
const report = buildExplainReport({
|
|
272
|
+
html,
|
|
273
|
+
selectors,
|
|
274
|
+
topN,
|
|
275
|
+
baseUrl
|
|
276
|
+
});
|
|
277
|
+
const content = renderText(report);
|
|
278
|
+
return {
|
|
279
|
+
content: [{
|
|
280
|
+
text: content,
|
|
281
|
+
type: "text"
|
|
282
|
+
}],
|
|
283
|
+
structuredContent: {
|
|
284
|
+
schemaVersion: 1,
|
|
285
|
+
content,
|
|
286
|
+
chosenRoot: report.chosenRoot,
|
|
287
|
+
candidates: report.candidates,
|
|
288
|
+
readerable: report.readerable,
|
|
289
|
+
parseSucceeded: report.parseSucceeded,
|
|
290
|
+
fallbackUsed: report.fallbackUsed,
|
|
291
|
+
gating: report.gating ?? null,
|
|
292
|
+
pagination: report.pagination ?? null,
|
|
293
|
+
removedNodes: report.removedNodes,
|
|
294
|
+
snapshot: report.snapshot
|
|
295
|
+
}
|
|
296
|
+
};
|
|
297
|
+
}
|
|
298
|
+
var EXPLAIN_TOOL_DESCRIPTION = `Post-mortem diagnostics for extraction: shows WHY Readability picked what it picked. Returns the chosen root, the ranked candidate nodes with their REAL Readability contentScore values (read off the DOM expando Readability stamps during scoring), a categorized removed-nodes breakdown, gating/pagination signals, and a snapshot of the normalized HTML fed to Readability. Runs the same normalize + Readability pipeline as \`extract\` (no fallback cascade, no Turndown). The server fetches nothing: \`localPath\` is the only source, and \`baseUrl\` (optional) is origin context only.`;
|
|
299
|
+
function explainHandler(args) {
|
|
300
|
+
try {
|
|
301
|
+
return explain(args);
|
|
302
|
+
} catch (err) {
|
|
303
|
+
logger.error(`explain failed: ${err instanceof Error ? err.message : String(err)}`);
|
|
304
|
+
return toErrorResult(err);
|
|
305
|
+
}
|
|
306
|
+
}
|
|
307
|
+
function registerExplainTool(server) {
|
|
308
|
+
return server.registerTool("explain", {
|
|
309
|
+
title: "Explain why Readability picked what it picked",
|
|
310
|
+
description: EXPLAIN_TOOL_DESCRIPTION,
|
|
311
|
+
inputSchema: explainInputShape,
|
|
312
|
+
outputSchema: explainOutput
|
|
313
|
+
}, explainHandler);
|
|
314
|
+
}
|
|
315
|
+
//#endregion
|
|
316
|
+
//#region src/policy/sibling-scan.ts
|
|
317
|
+
var CONTAINER_TAGS = /* @__PURE__ */ new Set([
|
|
318
|
+
"ARTICLE",
|
|
319
|
+
"DIV",
|
|
320
|
+
"MAIN",
|
|
321
|
+
"OL",
|
|
322
|
+
"SECTION",
|
|
323
|
+
"TABLE",
|
|
324
|
+
"TBODY",
|
|
325
|
+
"UL"
|
|
326
|
+
]);
|
|
327
|
+
var LANDMARK_CHROME_SELECTOR = "nav, header, footer, aside, [role=\"navigation\"], [role=\"banner\"], [role=\"contentinfo\"], [role=\"complementary\"], [role=\"search\"], [role=\"menu\"], [role=\"menubar\"]";
|
|
328
|
+
function stripLandmarkChrome(document) {
|
|
329
|
+
for (const el of document.querySelectorAll(LANDMARK_CHROME_SELECTOR)) el.remove();
|
|
330
|
+
for (const el of document.querySelectorAll("script, style, template")) el.remove();
|
|
331
|
+
}
|
|
332
|
+
function shapeKey(el) {
|
|
333
|
+
const raw = el.getAttribute("class");
|
|
334
|
+
if (!raw) return el.tagName;
|
|
335
|
+
const normalized = raw.trim().split(/\s+/).sort().join(" ");
|
|
336
|
+
return normalized ? `${el.tagName}|${normalized}` : el.tagName;
|
|
337
|
+
}
|
|
338
|
+
function describeSelector(el) {
|
|
339
|
+
const parts = [el.tagName.toLowerCase()];
|
|
340
|
+
const id = el.getAttribute("id");
|
|
341
|
+
if (id) parts.push(`#${id}`);
|
|
342
|
+
const cls = el.getAttribute("class");
|
|
343
|
+
if (cls) {
|
|
344
|
+
for (const token of cls.trim().split(/\s+/)) if (token) parts.push(`.${token}`);
|
|
345
|
+
}
|
|
346
|
+
return parts.join("");
|
|
347
|
+
}
|
|
348
|
+
//#endregion
|
|
349
|
+
//#region src/policy/grid-detector.ts
|
|
350
|
+
var MIN_ROWS = 3;
|
|
351
|
+
var HIGH_CONF_ROWS = 6;
|
|
352
|
+
var MIN_CELLS_PER_ROW = 2;
|
|
353
|
+
function rowCells(row) {
|
|
354
|
+
return Array.from(row.children).map(resolveCellText);
|
|
355
|
+
}
|
|
356
|
+
function notDetected$1(note) {
|
|
357
|
+
return {
|
|
358
|
+
confidence: "low",
|
|
359
|
+
containerSelector: "",
|
|
360
|
+
detected: false,
|
|
361
|
+
rowCount: 0,
|
|
362
|
+
colCount: 0,
|
|
363
|
+
rows: [],
|
|
364
|
+
rowTag: "",
|
|
365
|
+
note
|
|
366
|
+
};
|
|
367
|
+
}
|
|
368
|
+
function confidenceFor$1(dataRowCount) {
|
|
369
|
+
return dataRowCount >= HIGH_CONF_ROWS ? "high" : "medium";
|
|
370
|
+
}
|
|
371
|
+
function buildResult(raggedRows, rowTag, containerSelector, selectorHint, evidence) {
|
|
372
|
+
let maxCols = 0;
|
|
373
|
+
for (const row of raggedRows) if (row.length > maxCols) maxCols = row.length;
|
|
374
|
+
if (maxCols === 0) return notDetected$1(selectorHint ? `not a grid: rows matched ${selectorHint} but no cells were found` : "not a grid: rows matched but no cells were found");
|
|
375
|
+
const rowCount = raggedRows.length;
|
|
376
|
+
const dataRowCount = evidence?.dataRowCount ?? rowCount;
|
|
377
|
+
const headerCount = evidence?.headerCount ?? 0;
|
|
378
|
+
const rows = raggedRows.map((row) => ({ cells: Array.from({ length: maxCols }, (_, i) => row[i] ?? "") }));
|
|
379
|
+
const where = containerSelector || selectorHint || "document";
|
|
380
|
+
const descriptor = headerCount > 0 ? `detected ${dataRowCount} ${rowTag} data rows plus ${headerCount} header row${headerCount > 1 ? "s" : ""} (${maxCols} cols) in ${where}` : `detected ${dataRowCount} ${rowTag} rows (${maxCols} cols) in ${where}`;
|
|
381
|
+
return {
|
|
382
|
+
confidence: confidenceFor$1(dataRowCount),
|
|
383
|
+
containerSelector,
|
|
384
|
+
detected: true,
|
|
385
|
+
rowCount,
|
|
386
|
+
colCount: maxCols,
|
|
387
|
+
rows,
|
|
388
|
+
rowTag,
|
|
389
|
+
note: descriptor
|
|
390
|
+
};
|
|
391
|
+
}
|
|
392
|
+
function detectSelectorMode(document, rowSelector, cellSelector, minRows) {
|
|
393
|
+
const rowEls = Array.from(document.querySelectorAll(rowSelector));
|
|
394
|
+
if (rowEls.length < minRows) return notDetected$1(`not a grid: rowSelector "${rowSelector}" matched ${rowEls.length} row(s) (min ${minRows})`);
|
|
395
|
+
return buildResult(rowEls.map((row) => Array.from(row.querySelectorAll(cellSelector)).map(resolveCellText)), rowEls[0].tagName, "", rowSelector);
|
|
396
|
+
}
|
|
397
|
+
function modalWidth(members) {
|
|
398
|
+
const counts = /* @__PURE__ */ new Map();
|
|
399
|
+
for (const member of members) {
|
|
400
|
+
const width = member.children.length;
|
|
401
|
+
counts.set(width, (counts.get(width) ?? 0) + 1);
|
|
402
|
+
}
|
|
403
|
+
let best = members[0].children.length;
|
|
404
|
+
let bestCount = 0;
|
|
405
|
+
for (const [width, count] of counts) if (count > bestCount) {
|
|
406
|
+
best = width;
|
|
407
|
+
bestCount = count;
|
|
408
|
+
}
|
|
409
|
+
return best;
|
|
410
|
+
}
|
|
411
|
+
function classTokens(el) {
|
|
412
|
+
const cls = el.getAttribute("class");
|
|
413
|
+
return cls ? new Set(cls.trim().split(/\s+/)) : /* @__PURE__ */ new Set();
|
|
414
|
+
}
|
|
415
|
+
function looksLikeHeader(child, rowTag, dataClass, dataWidth) {
|
|
416
|
+
if (child.tagName !== rowTag || child.children.length !== dataWidth) return false;
|
|
417
|
+
if (child.getAttribute("role") === "row" && Array.from(child.children).some((c) => c.getAttribute("role") === "columnheader")) return true;
|
|
418
|
+
const childClass = classTokens(child);
|
|
419
|
+
return dataClass.size > 0 && [...dataClass].every((token) => childClass.has(token));
|
|
420
|
+
}
|
|
421
|
+
function collectHeaderRows(container, members) {
|
|
422
|
+
const memberSet = new Set(members);
|
|
423
|
+
const rowTag = members[0].tagName;
|
|
424
|
+
const dataWidth = modalWidth(members);
|
|
425
|
+
const dataClass = classTokens(members[0]);
|
|
426
|
+
const headers = [];
|
|
427
|
+
for (const child of Array.from(container.children)) {
|
|
428
|
+
if (memberSet.has(child)) break;
|
|
429
|
+
if (looksLikeHeader(child, rowTag, dataClass, dataWidth)) headers.push(rowCells(child));
|
|
430
|
+
}
|
|
431
|
+
return headers;
|
|
432
|
+
}
|
|
433
|
+
function detectAuto(document, minRows) {
|
|
434
|
+
stripLandmarkChrome(document);
|
|
435
|
+
const candidates = [];
|
|
436
|
+
for (const container of document.querySelectorAll("*")) {
|
|
437
|
+
if (!CONTAINER_TAGS.has(container.tagName)) continue;
|
|
438
|
+
const groups = /* @__PURE__ */ new Map();
|
|
439
|
+
for (const child of Array.from(container.childNodes)) {
|
|
440
|
+
if (!isElement(child)) continue;
|
|
441
|
+
const key = shapeKey(child);
|
|
442
|
+
const bucket = groups.get(key);
|
|
443
|
+
if (bucket) bucket.push(child);
|
|
444
|
+
else groups.set(key, [child]);
|
|
445
|
+
}
|
|
446
|
+
for (const members of groups.values()) {
|
|
447
|
+
if (members.length < minRows) continue;
|
|
448
|
+
if (!members.every((m) => m.children.length >= MIN_CELLS_PER_ROW)) continue;
|
|
449
|
+
const rows = members.map(rowCells);
|
|
450
|
+
const totalCellText = rows.reduce((sum, row) => sum + row.reduce((s, c) => s + c.length, 0), 0);
|
|
451
|
+
candidates.push({
|
|
452
|
+
container,
|
|
453
|
+
containerSelector: describeSelector(container),
|
|
454
|
+
members,
|
|
455
|
+
rows,
|
|
456
|
+
rowTag: members[0].tagName,
|
|
457
|
+
memberCount: members.length,
|
|
458
|
+
totalCellText
|
|
459
|
+
});
|
|
460
|
+
}
|
|
461
|
+
}
|
|
462
|
+
if (candidates.length === 0) return notDetected$1("not a grid: no repeating row structure (≥3 same-shape siblings each with ≥2 direct element-children, outside nav/header/footer/aside)");
|
|
463
|
+
let winner = candidates[0];
|
|
464
|
+
for (let i = 1; i < candidates.length; i++) {
|
|
465
|
+
const candidate = candidates[i];
|
|
466
|
+
if (candidate.memberCount > winner.memberCount || candidate.memberCount === winner.memberCount && candidate.totalCellText > winner.totalCellText) winner = candidate;
|
|
467
|
+
}
|
|
468
|
+
const headerRows = collectHeaderRows(winner.container, winner.members);
|
|
469
|
+
return buildResult([...headerRows, ...winner.rows], winner.rowTag, winner.containerSelector, void 0, {
|
|
470
|
+
dataRowCount: winner.memberCount,
|
|
471
|
+
headerCount: headerRows.length
|
|
472
|
+
});
|
|
473
|
+
}
|
|
474
|
+
function detectGrid(document, opts) {
|
|
475
|
+
const minRows = opts?.minRows ?? MIN_ROWS;
|
|
476
|
+
if (opts?.rowSelector && opts.cellSelector) return detectSelectorMode(document, opts.rowSelector, opts.cellSelector, minRows);
|
|
477
|
+
return detectAuto(document, minRows);
|
|
478
|
+
}
|
|
479
|
+
//#endregion
|
|
480
|
+
//#region src/tools/extract_grid.ts
|
|
481
|
+
var NO_GRID = "(no repeating grid found)";
|
|
482
|
+
function extractGrid(rawArgs) {
|
|
483
|
+
const { localPath, ...rest } = extractGridInputSchema.parse(rawArgs);
|
|
484
|
+
return extractGridFromHtml({
|
|
485
|
+
html: readHtmlFile(localPath),
|
|
486
|
+
...rest
|
|
487
|
+
});
|
|
488
|
+
}
|
|
489
|
+
var DEFAULTS$3 = extractGridInputSchema.parse({ localPath: "" });
|
|
490
|
+
function extractGridFromHtml(input) {
|
|
491
|
+
const { html, baseUrl, format, selectors, rowSelector, cellSelector } = {
|
|
492
|
+
...DEFAULTS$3,
|
|
493
|
+
...input
|
|
494
|
+
};
|
|
495
|
+
const { document } = buildDocument(html, baseUrl);
|
|
496
|
+
applySelectors(document, selectors);
|
|
497
|
+
const result = detectGrid(document, {
|
|
498
|
+
rowSelector,
|
|
499
|
+
cellSelector
|
|
500
|
+
});
|
|
501
|
+
let content = NO_GRID;
|
|
502
|
+
let grid = {
|
|
503
|
+
rows: 0,
|
|
504
|
+
cols: 0,
|
|
505
|
+
markdown: ""
|
|
506
|
+
};
|
|
507
|
+
if (result.detected && result.rows.length > 0) {
|
|
508
|
+
const matrix = result.rows.map((row) => [...row.cells]);
|
|
509
|
+
const markdown = renderTable(matrix, format);
|
|
510
|
+
content = markdown;
|
|
511
|
+
grid = {
|
|
512
|
+
rows: result.rowCount,
|
|
513
|
+
cols: result.colCount,
|
|
514
|
+
markdown
|
|
515
|
+
};
|
|
516
|
+
}
|
|
517
|
+
return {
|
|
518
|
+
content: [{
|
|
519
|
+
text: content,
|
|
520
|
+
type: "text"
|
|
521
|
+
}],
|
|
522
|
+
structuredContent: {
|
|
523
|
+
schemaVersion: 1,
|
|
524
|
+
content,
|
|
525
|
+
grid,
|
|
526
|
+
diagnostics: {
|
|
527
|
+
confidence: result.confidence,
|
|
528
|
+
containerSelector: result.containerSelector,
|
|
529
|
+
detected: result.detected,
|
|
530
|
+
rowCount: result.rowCount,
|
|
531
|
+
colCount: result.colCount,
|
|
532
|
+
rowTag: result.rowTag,
|
|
533
|
+
note: result.note
|
|
534
|
+
},
|
|
535
|
+
metadata: {
|
|
536
|
+
baseUrl,
|
|
537
|
+
format,
|
|
538
|
+
detected: result.detected
|
|
539
|
+
}
|
|
540
|
+
}
|
|
541
|
+
};
|
|
542
|
+
}
|
|
543
|
+
var EXTRACT_GRID_TOOL_DESCRIPTION = `Detect and extract a CSS-grid / div "table" from already-rendered (post-JavaScript) HTML — the div equivalent of \`extract_tables\` for SPAs that render data into repeating \`<div>\` rows instead of \`<table>\`. Supports auto-detect (find the container whose direct children form the largest same-shape sibling group of ≥3 rows, each row a set of ≥2 direct element-children) and explicit \`rowSelector\` + \`cellSelector\` selector mode (cells scoped to each row subtree). Renders the matrix through the SAME gfm/csv/json renderer as \`extract_tables\`. Runs no Readability, no Turndown, no sanitization — the server fetches nothing: \`localPath\` is the only source, and \`baseUrl\` (optional) is origin context only.`;
|
|
544
|
+
function extractGridHandler(args) {
|
|
545
|
+
try {
|
|
546
|
+
return extractGrid(args);
|
|
547
|
+
} catch (err) {
|
|
548
|
+
logger.error(`extract_grid failed: ${err instanceof Error ? err.message : String(err)}`);
|
|
549
|
+
return toErrorResult(err);
|
|
550
|
+
}
|
|
551
|
+
}
|
|
552
|
+
function registerExtractGridTool(server) {
|
|
553
|
+
return server.registerTool("extract_grid", {
|
|
554
|
+
title: "Extract a CSS-grid / div table (the div equivalent of extract_tables)",
|
|
555
|
+
description: EXTRACT_GRID_TOOL_DESCRIPTION,
|
|
556
|
+
inputSchema: extractGridInputShape,
|
|
557
|
+
outputSchema: extractGridOutputShape
|
|
558
|
+
}, extractGridHandler);
|
|
559
|
+
}
|
|
560
|
+
//#endregion
|
|
561
|
+
//#region src/tools/extract_links.ts
|
|
562
|
+
var MAX_TEXT_LENGTH = 300;
|
|
563
|
+
function clipText(raw) {
|
|
564
|
+
const collapsed = raw.replace(/\s+/g, " ").trim();
|
|
565
|
+
return collapsed.length > MAX_TEXT_LENGTH ? `${collapsed.slice(0, MAX_TEXT_LENGTH)}…` : collapsed;
|
|
566
|
+
}
|
|
567
|
+
function isWebOrigin(parsed) {
|
|
568
|
+
return parsed.protocol === "http:" || parsed.protocol === "https:";
|
|
569
|
+
}
|
|
570
|
+
function resolveExternal(absolutizedHref, baseUrl) {
|
|
571
|
+
let parsedHref;
|
|
572
|
+
try {
|
|
573
|
+
parsedHref = new URL(absolutizedHref);
|
|
574
|
+
} catch {
|
|
575
|
+
return false;
|
|
576
|
+
}
|
|
577
|
+
if (!isWebOrigin(parsedHref)) return false;
|
|
578
|
+
try {
|
|
579
|
+
return new URL(baseUrl).origin !== parsedHref.origin;
|
|
580
|
+
} catch {
|
|
581
|
+
return false;
|
|
582
|
+
}
|
|
583
|
+
}
|
|
584
|
+
function pruneUnsafeRoots(document) {
|
|
585
|
+
for (const el of document.querySelectorAll("script, template")) el.remove();
|
|
586
|
+
}
|
|
587
|
+
function extractLinks(rawArgs) {
|
|
588
|
+
const { localPath, ...rest } = extractLinksInputSchema.parse(rawArgs);
|
|
589
|
+
return extractLinksFromHtml({
|
|
590
|
+
html: readHtmlFile(localPath),
|
|
591
|
+
...rest
|
|
592
|
+
});
|
|
593
|
+
}
|
|
594
|
+
var DEFAULTS$2 = extractLinksInputSchema.parse({ localPath: "" });
|
|
595
|
+
function extractLinksFromHtml(input) {
|
|
596
|
+
const { html, baseUrl, sameOriginOnly, selectors } = {
|
|
597
|
+
...DEFAULTS$2,
|
|
598
|
+
...input
|
|
599
|
+
};
|
|
600
|
+
const { document } = buildDocument(html, baseUrl);
|
|
601
|
+
applySelectors(document, selectors);
|
|
602
|
+
pruneUnsafeRoots(document);
|
|
603
|
+
const links = [];
|
|
604
|
+
for (const anchor of document.querySelectorAll("a")) {
|
|
605
|
+
const rawHref = anchor.getAttribute("href");
|
|
606
|
+
if (!rawHref) continue;
|
|
607
|
+
const href = absolutize(rawHref, baseUrl);
|
|
608
|
+
const isExternal = baseUrl ? resolveExternal(href, baseUrl) : false;
|
|
609
|
+
if (sameOriginOnly && isExternal) continue;
|
|
610
|
+
links.push({
|
|
611
|
+
text: clipText(anchor.textContent),
|
|
612
|
+
href,
|
|
613
|
+
rel: anchor.getAttribute("rel") ?? "",
|
|
614
|
+
isExternal
|
|
615
|
+
});
|
|
616
|
+
}
|
|
617
|
+
const content = renderLinksIndex(links);
|
|
618
|
+
return {
|
|
619
|
+
content: [{
|
|
620
|
+
text: content,
|
|
621
|
+
type: "text"
|
|
622
|
+
}],
|
|
623
|
+
structuredContent: {
|
|
624
|
+
schemaVersion: 1,
|
|
625
|
+
content,
|
|
626
|
+
links,
|
|
627
|
+
metadata: { baseUrl }
|
|
628
|
+
}
|
|
629
|
+
};
|
|
630
|
+
}
|
|
631
|
+
function renderLinksIndex(links) {
|
|
632
|
+
if (links.length === 0) return "(no links found)";
|
|
633
|
+
return links.map((link) => `- [${link.text || "(no text)"}](${link.href})`).join("\n");
|
|
634
|
+
}
|
|
635
|
+
var EXTRACT_LINKS_TOOL_DESCRIPTION = `Return a structured list of anchor links from already-rendered (post-JavaScript) HTML — \`[{text, href, rel, isExternal}]\` in document order, hrefs absolutized against \`baseUrl\`. No Readability scoring, Turndown, or sanitization — links are gathered from the raw parsed DOM so nav/footer/main links survive. Pairs with chrome-devtools for crawl/navigation decisions. The server fetches nothing: \`localPath\` is the only source, and \`baseUrl\` is origin context only (never fetched).`;
|
|
636
|
+
function extractLinksHandler(args) {
|
|
637
|
+
try {
|
|
638
|
+
return extractLinks(args);
|
|
639
|
+
} catch (err) {
|
|
640
|
+
logger.error(`extract_links failed: ${err instanceof Error ? err.message : String(err)}`);
|
|
641
|
+
return toErrorResult(err);
|
|
642
|
+
}
|
|
643
|
+
}
|
|
644
|
+
function registerExtractLinksTool(server) {
|
|
645
|
+
return server.registerTool("extract_links", {
|
|
646
|
+
title: "Extract anchor links",
|
|
647
|
+
description: EXTRACT_LINKS_TOOL_DESCRIPTION,
|
|
648
|
+
inputSchema: extractLinksInputShape,
|
|
649
|
+
outputSchema: extractLinksOutputShape
|
|
650
|
+
}, extractLinksHandler);
|
|
651
|
+
}
|
|
652
|
+
//#endregion
|
|
653
|
+
//#region src/policy/list-detector.ts
|
|
654
|
+
var MIN_ITEMS = 3;
|
|
655
|
+
var HIGH_CONF_ITEMS = 6;
|
|
656
|
+
var HIGH_CONF_AVG_SCORE = 30;
|
|
657
|
+
var MAX_SNIPPET_CHARS = 200;
|
|
658
|
+
function isNavigationHref(href) {
|
|
659
|
+
if (!href || href === "#") return false;
|
|
660
|
+
const lower = href.toLowerCase();
|
|
661
|
+
return !lower.startsWith("javascript:") && !lower.startsWith("mailto:") && !lower.startsWith("tel:");
|
|
662
|
+
}
|
|
663
|
+
function anchorText(anchor) {
|
|
664
|
+
return anchor.textContent.replace(/\s+/g, " ").trim();
|
|
665
|
+
}
|
|
666
|
+
function navigationAnchors(child) {
|
|
667
|
+
const anchors = [];
|
|
668
|
+
for (const el of child.querySelectorAll("a[href]")) {
|
|
669
|
+
const anchor = el;
|
|
670
|
+
if (!isNavigationHref(anchor.getAttribute("href") ?? "")) continue;
|
|
671
|
+
if (anchorText(anchor).length === 0) continue;
|
|
672
|
+
anchors.push(anchor);
|
|
673
|
+
}
|
|
674
|
+
return anchors;
|
|
675
|
+
}
|
|
676
|
+
function pickPrimaryAnchor(anchors) {
|
|
677
|
+
let primary = anchors[0];
|
|
678
|
+
let best = anchorText(primary).length;
|
|
679
|
+
for (let i = 1; i < anchors.length; i++) {
|
|
680
|
+
const len = anchorText(anchors[i]).length;
|
|
681
|
+
if (len > best) {
|
|
682
|
+
primary = anchors[i];
|
|
683
|
+
best = len;
|
|
684
|
+
}
|
|
685
|
+
}
|
|
686
|
+
return primary;
|
|
687
|
+
}
|
|
688
|
+
function clipSnippet(raw) {
|
|
689
|
+
const collapsed = raw.replace(/\s+/g, " ").trim();
|
|
690
|
+
if (collapsed.length <= MAX_SNIPPET_CHARS) return collapsed;
|
|
691
|
+
return `${collapsed.slice(0, MAX_SNIPPET_CHARS)}…`;
|
|
692
|
+
}
|
|
693
|
+
function scoreItem(textLength, linkTextLength, primaryAnchorTextLength) {
|
|
694
|
+
if (textLength === 0) return 0;
|
|
695
|
+
return primaryAnchorTextLength + Math.max(0, textLength - linkTextLength);
|
|
696
|
+
}
|
|
697
|
+
function extractItem(child, baseUrl) {
|
|
698
|
+
const anchors = navigationAnchors(child);
|
|
699
|
+
if (anchors.length === 0) return null;
|
|
700
|
+
const primary = pickPrimaryAnchor(anchors);
|
|
701
|
+
const title = anchorText(primary);
|
|
702
|
+
if (!title) return null;
|
|
703
|
+
const href = primary.getAttribute("href") ?? "";
|
|
704
|
+
const url = absolutize(href, baseUrl);
|
|
705
|
+
if (!url) return null;
|
|
706
|
+
const fullText = child.textContent.replace(/\s+/g, " ").trim();
|
|
707
|
+
const snippet = fullText === title ? "" : fullText.startsWith(title) ? clipSnippet(fullText.slice(title.length + 1)) : clipSnippet(fullText);
|
|
708
|
+
const linkTextLength = anchors.reduce((sum, anchor) => sum + anchorText(anchor).length, 0);
|
|
709
|
+
return {
|
|
710
|
+
score: scoreItem(fullText.length, linkTextLength, title.length),
|
|
711
|
+
snippet,
|
|
712
|
+
title,
|
|
713
|
+
url
|
|
714
|
+
};
|
|
715
|
+
}
|
|
716
|
+
function distinctPathnames(items) {
|
|
717
|
+
const paths = /* @__PURE__ */ new Set();
|
|
718
|
+
for (const item of items) try {
|
|
719
|
+
paths.add(new URL(item.url).pathname);
|
|
720
|
+
} catch {
|
|
721
|
+
paths.add(item.url);
|
|
722
|
+
}
|
|
723
|
+
return paths.size;
|
|
724
|
+
}
|
|
725
|
+
function collectCandidates(document, baseUrl) {
|
|
726
|
+
const candidates = [];
|
|
727
|
+
for (const container of document.querySelectorAll("*")) {
|
|
728
|
+
if (!CONTAINER_TAGS.has(container.tagName)) continue;
|
|
729
|
+
const groups = /* @__PURE__ */ new Map();
|
|
730
|
+
for (const child of Array.from(container.childNodes)) {
|
|
731
|
+
if (!isElement(child)) continue;
|
|
732
|
+
const key = shapeKey(child);
|
|
733
|
+
const bucket = groups.get(key);
|
|
734
|
+
if (bucket) bucket.push(child);
|
|
735
|
+
else groups.set(key, [child]);
|
|
736
|
+
}
|
|
737
|
+
for (const children of groups.values()) {
|
|
738
|
+
if (children.length < MIN_ITEMS) continue;
|
|
739
|
+
if (!children.every((child) => navigationAnchors(child).length > 0)) continue;
|
|
740
|
+
const items = [];
|
|
741
|
+
for (const child of children) {
|
|
742
|
+
const item = extractItem(child, baseUrl);
|
|
743
|
+
if (item) items.push(item);
|
|
744
|
+
}
|
|
745
|
+
if (items.length < MIN_ITEMS) continue;
|
|
746
|
+
candidates.push({
|
|
747
|
+
containerSelector: describeSelector(container),
|
|
748
|
+
distinctPathnames: distinctPathnames(items),
|
|
749
|
+
itemTag: children[0].tagName,
|
|
750
|
+
items,
|
|
751
|
+
totalScore: items.reduce((sum, item) => sum + item.score, 0)
|
|
752
|
+
});
|
|
753
|
+
}
|
|
754
|
+
}
|
|
755
|
+
return candidates;
|
|
756
|
+
}
|
|
757
|
+
function confidenceFor(items) {
|
|
758
|
+
if (items.length < MIN_ITEMS) return "low";
|
|
759
|
+
if (items.length >= HIGH_CONF_ITEMS) return items.reduce((sum, item) => sum + item.score, 0) / items.length >= HIGH_CONF_AVG_SCORE ? "high" : "medium";
|
|
760
|
+
return "medium";
|
|
761
|
+
}
|
|
762
|
+
function notDetected(note) {
|
|
763
|
+
return {
|
|
764
|
+
confidence: "low",
|
|
765
|
+
containerSelector: "",
|
|
766
|
+
detected: false,
|
|
767
|
+
itemCount: 0,
|
|
768
|
+
itemTag: "",
|
|
769
|
+
items: [],
|
|
770
|
+
note
|
|
771
|
+
};
|
|
772
|
+
}
|
|
773
|
+
function detectList(document, baseUrl) {
|
|
774
|
+
stripLandmarkChrome(document);
|
|
775
|
+
const candidates = collectCandidates(document, baseUrl);
|
|
776
|
+
if (candidates.length === 0) return notDetected("not a list: no repeated item structure with links (≥3 same-shape siblings each carrying an anchor, outside nav/header/footer/aside)");
|
|
777
|
+
let winner = candidates[0];
|
|
778
|
+
for (let i = 1; i < candidates.length; i++) {
|
|
779
|
+
const candidate = candidates[i];
|
|
780
|
+
if (candidate.distinctPathnames > winner.distinctPathnames || candidate.distinctPathnames === winner.distinctPathnames && candidate.items.length > winner.items.length || candidate.distinctPathnames === winner.distinctPathnames && candidate.items.length === winner.items.length && candidate.totalScore > winner.totalScore) winner = candidate;
|
|
781
|
+
}
|
|
782
|
+
if (winner.items.length < MIN_ITEMS) return notDetected("not a list: best candidate had fewer than 3 extracted items");
|
|
783
|
+
const items = winner.items;
|
|
784
|
+
return {
|
|
785
|
+
confidence: confidenceFor(items),
|
|
786
|
+
containerSelector: winner.containerSelector,
|
|
787
|
+
detected: true,
|
|
788
|
+
itemCount: items.length,
|
|
789
|
+
itemTag: winner.itemTag,
|
|
790
|
+
items,
|
|
791
|
+
note: `detected ${items.length} ${winner.itemTag} items in ${winner.containerSelector}`
|
|
792
|
+
};
|
|
793
|
+
}
|
|
794
|
+
//#endregion
|
|
795
|
+
//#region src/tools/extract_list.ts
|
|
796
|
+
var NOT_A_LIST = "not a list: no repeated item structure with links";
|
|
797
|
+
function renderItems(result) {
|
|
798
|
+
if (!result.detected || result.items.length === 0) return result.note || NOT_A_LIST;
|
|
799
|
+
return result.items.map((item, index) => {
|
|
800
|
+
const head = `${index + 1}. ${item.title} — ${item.url}`;
|
|
801
|
+
return item.snippet ? `${head}\n ${item.snippet}` : head;
|
|
802
|
+
}).join("\n");
|
|
803
|
+
}
|
|
804
|
+
function toStructuredItem(item) {
|
|
805
|
+
return {
|
|
806
|
+
score: item.score,
|
|
807
|
+
snippet: item.snippet,
|
|
808
|
+
title: item.title,
|
|
809
|
+
url: item.url
|
|
810
|
+
};
|
|
811
|
+
}
|
|
812
|
+
function extractList(rawArgs) {
|
|
813
|
+
const { localPath, ...rest } = extractListInputSchema.parse(rawArgs);
|
|
814
|
+
return extractListFromHtml({
|
|
815
|
+
html: readHtmlFile(localPath),
|
|
816
|
+
...rest
|
|
817
|
+
});
|
|
818
|
+
}
|
|
819
|
+
function extractListFromHtml(input) {
|
|
820
|
+
const { html, baseUrl, selectors } = input;
|
|
821
|
+
const { document } = buildDocument(html, baseUrl);
|
|
822
|
+
applySelectors(document, selectors);
|
|
823
|
+
const result = detectList(document, baseUrl);
|
|
824
|
+
const content = renderItems(result);
|
|
825
|
+
return {
|
|
826
|
+
content: [{
|
|
827
|
+
text: content,
|
|
828
|
+
type: "text"
|
|
829
|
+
}],
|
|
830
|
+
structuredContent: {
|
|
831
|
+
schemaVersion: 1,
|
|
832
|
+
content,
|
|
833
|
+
items: result.items.map(toStructuredItem),
|
|
834
|
+
diagnostics: {
|
|
835
|
+
confidence: result.confidence,
|
|
836
|
+
containerSelector: result.containerSelector,
|
|
837
|
+
detected: result.detected,
|
|
838
|
+
itemCount: result.itemCount,
|
|
839
|
+
itemTag: result.itemTag,
|
|
840
|
+
note: result.note
|
|
841
|
+
},
|
|
842
|
+
metadata: { baseUrl }
|
|
843
|
+
}
|
|
844
|
+
};
|
|
845
|
+
}
|
|
846
|
+
var EXTRACT_LIST_TOOL_DESCRIPTION = `Detect and extract a list/feed/index structure from already-rendered (post-JavaScript) HTML — for HN-style, search-result, and blog-index pages that Readability cannot turn into one article. Returns \`{items: [{title, url, snippet, score}], diagnostics}\` instead of one article. Strips nav/header/footer/aside + ARIA chrome roles first (the false-positive guard so an article's nav menu doesn't look like a 4-item feed), then finds the container whose direct children form a same-shape sibling cluster of ≥3 elements each carrying a navigation anchor, and the cluster with the most items wins. No Readability, no Turndown, no sanitization. The server fetches nothing: \`localPath\` is the only source, and \`baseUrl\` (optional) is origin context for absolutizing item hrefs.`;
|
|
847
|
+
function extractListHandler(args) {
|
|
848
|
+
try {
|
|
849
|
+
return extractList(args);
|
|
850
|
+
} catch (err) {
|
|
851
|
+
logger.error(`extract_list failed: ${err instanceof Error ? err.message : String(err)}`);
|
|
852
|
+
return toErrorResult(err);
|
|
853
|
+
}
|
|
854
|
+
}
|
|
855
|
+
function registerExtractListTool(server) {
|
|
856
|
+
return server.registerTool("extract_list", {
|
|
857
|
+
title: "Extract a list/feed (HN/search/blog-index pages)",
|
|
858
|
+
description: EXTRACT_LIST_TOOL_DESCRIPTION,
|
|
859
|
+
inputSchema: extractListInputShape,
|
|
860
|
+
outputSchema: extractListOutputShape
|
|
861
|
+
}, extractListHandler);
|
|
862
|
+
}
|
|
863
|
+
//#endregion
|
|
864
|
+
//#region src/tools/extract_metadata.ts
|
|
865
|
+
var BIBLIOGRAPHIC_KEYS = [
|
|
866
|
+
"title",
|
|
867
|
+
"byline",
|
|
868
|
+
"siteName",
|
|
869
|
+
"lang",
|
|
870
|
+
"publishedTime",
|
|
871
|
+
"excerpt",
|
|
872
|
+
"canonical",
|
|
873
|
+
"baseUrl"
|
|
874
|
+
];
|
|
875
|
+
function pickBibliographic(metadata) {
|
|
876
|
+
const out = {};
|
|
877
|
+
for (const key of BIBLIOGRAPHIC_KEYS) {
|
|
878
|
+
const value = metadata[key];
|
|
879
|
+
if (value !== void 0) out[key] = value;
|
|
880
|
+
}
|
|
881
|
+
return out;
|
|
882
|
+
}
|
|
883
|
+
function renderMetadataLines(metadata) {
|
|
884
|
+
const lines = Object.entries(metadata).map(([key, value]) => `${key}: ${value}`);
|
|
885
|
+
return lines.length > 0 ? lines.join("\n") : "(no metadata found)";
|
|
886
|
+
}
|
|
887
|
+
function extractMetadataDocument(rawArgs) {
|
|
888
|
+
const { localPath, ...rest } = extractMetadataInputSchema.parse(rawArgs);
|
|
889
|
+
return extractMetadataDocumentFromHtml({
|
|
890
|
+
html: readHtmlFile(localPath),
|
|
891
|
+
...rest
|
|
892
|
+
});
|
|
893
|
+
}
|
|
894
|
+
function extractMetadataDocumentFromHtml(input) {
|
|
895
|
+
const { html, baseUrl } = input;
|
|
896
|
+
const { document } = buildDocument(html, baseUrl);
|
|
897
|
+
normalizeDocument(document);
|
|
898
|
+
const metadata = pickBibliographic(resolveMetadata({
|
|
899
|
+
document,
|
|
900
|
+
textContent: "",
|
|
901
|
+
wordCount: 0,
|
|
902
|
+
readingTimeMin: 0,
|
|
903
|
+
baseUrl
|
|
904
|
+
}));
|
|
905
|
+
const content = renderMetadataLines(metadata);
|
|
906
|
+
return {
|
|
907
|
+
content: [{
|
|
908
|
+
text: content,
|
|
909
|
+
type: "text"
|
|
910
|
+
}],
|
|
911
|
+
structuredContent: {
|
|
912
|
+
schemaVersion: 1,
|
|
913
|
+
content,
|
|
914
|
+
metadata
|
|
915
|
+
}
|
|
916
|
+
};
|
|
917
|
+
}
|
|
918
|
+
var EXTRACT_METADATA_TOOL_DESCRIPTION = `Return only the bibliographic metadata (title, byline, siteName, lang, publishedTime, excerpt, canonical, baseUrl) of already-rendered (post-JavaScript) HTML without running Readability/Turndown — a fast pre-check for crawlers and citation. Resolves the same metadata cascade as \`extract\` (JSON-LD → OpenGraph → Twitter → <meta> → <time> → <title>), plus <link rel="canonical"> → og:url. The server fetches nothing: \`localPath\` is the only source, and \`baseUrl\` is origin context only (never fetched).`;
|
|
919
|
+
function extractMetadataHandler(args) {
|
|
920
|
+
try {
|
|
921
|
+
return extractMetadataDocument(args);
|
|
922
|
+
} catch (err) {
|
|
923
|
+
logger.error(`extract_metadata failed: ${err instanceof Error ? err.message : String(err)}`);
|
|
924
|
+
return toErrorResult(err);
|
|
925
|
+
}
|
|
926
|
+
}
|
|
927
|
+
function registerExtractMetadataTool(server) {
|
|
928
|
+
return server.registerTool("extract_metadata", {
|
|
929
|
+
title: "Extract metadata only (no Readability)",
|
|
930
|
+
description: EXTRACT_METADATA_TOOL_DESCRIPTION,
|
|
931
|
+
inputSchema: extractMetadataInputShape,
|
|
932
|
+
outputSchema: extractMetadataOutputShape
|
|
933
|
+
}, extractMetadataHandler);
|
|
934
|
+
}
|
|
935
|
+
//#endregion
|
|
936
|
+
//#region src/policy/outline.ts
|
|
937
|
+
function normalizeHeadingText(raw) {
|
|
938
|
+
return raw.replace(/\s+/g, " ").trim().replace(/^#+|#+$/g, "").trim();
|
|
939
|
+
}
|
|
940
|
+
function slugify(text) {
|
|
941
|
+
return text.toLowerCase().replace(/[^\p{L}\p{M}\p{N}\p{Pc} -]/gu, "").replace(/ /g, "-") || "section";
|
|
942
|
+
}
|
|
943
|
+
function linkAnchor(heading) {
|
|
944
|
+
for (const link of heading.querySelectorAll("a[href^=\"#\"]")) {
|
|
945
|
+
const href = link.getAttribute("href");
|
|
946
|
+
if (!href) continue;
|
|
947
|
+
const fragment = href.slice(1);
|
|
948
|
+
if (fragment) return fragment;
|
|
949
|
+
}
|
|
950
|
+
}
|
|
951
|
+
function resolveCandidate(heading, text) {
|
|
952
|
+
const id = heading.getAttribute("id");
|
|
953
|
+
if (id) return {
|
|
954
|
+
anchor: id,
|
|
955
|
+
explicit: true
|
|
956
|
+
};
|
|
957
|
+
const link = linkAnchor(heading);
|
|
958
|
+
if (link) return {
|
|
959
|
+
anchor: link,
|
|
960
|
+
explicit: true
|
|
961
|
+
};
|
|
962
|
+
return {
|
|
963
|
+
anchor: slugify(text),
|
|
964
|
+
explicit: false
|
|
965
|
+
};
|
|
966
|
+
}
|
|
967
|
+
function dedupe(candidate, explicit, used) {
|
|
968
|
+
if (explicit || !used.has(candidate)) return candidate;
|
|
969
|
+
let suffix = 1;
|
|
970
|
+
while (used.has(`${candidate}-${suffix}`)) suffix++;
|
|
971
|
+
return `${candidate}-${suffix}`;
|
|
972
|
+
}
|
|
973
|
+
function resolveOutline(document) {
|
|
974
|
+
const entries = [];
|
|
975
|
+
const used = /* @__PURE__ */ new Set();
|
|
976
|
+
document.querySelectorAll("h1, h2, h3, h4, h5, h6").forEach((heading) => {
|
|
977
|
+
const rawText = heading.textContent;
|
|
978
|
+
if (!rawText.trim()) return;
|
|
979
|
+
const text = normalizeHeadingText(rawText);
|
|
980
|
+
const level = Number.parseInt(heading.tagName.slice(1), 10);
|
|
981
|
+
const { anchor: candidate, explicit } = resolveCandidate(heading, text);
|
|
982
|
+
const anchor = dedupe(candidate, explicit, used);
|
|
983
|
+
used.add(anchor);
|
|
984
|
+
entries.push({
|
|
985
|
+
anchor,
|
|
986
|
+
level,
|
|
987
|
+
text
|
|
988
|
+
});
|
|
989
|
+
});
|
|
990
|
+
return entries;
|
|
991
|
+
}
|
|
992
|
+
//#endregion
|
|
993
|
+
//#region src/policy/section.ts
|
|
994
|
+
function findHeading(document, query) {
|
|
995
|
+
const needle = normalizeHeadingText(query).toLowerCase();
|
|
996
|
+
if (!needle) return;
|
|
997
|
+
const headings = document.querySelectorAll("h1, h2, h3, h4, h5, h6");
|
|
998
|
+
let substring;
|
|
999
|
+
for (const heading of headings) {
|
|
1000
|
+
const text = normalizeHeadingText(heading.textContent).toLowerCase();
|
|
1001
|
+
if (!text) continue;
|
|
1002
|
+
const level = Number.parseInt(heading.tagName.slice(1), 10);
|
|
1003
|
+
if (text === needle) return {
|
|
1004
|
+
heading,
|
|
1005
|
+
level
|
|
1006
|
+
};
|
|
1007
|
+
if (!substring && text.includes(needle)) substring = {
|
|
1008
|
+
heading,
|
|
1009
|
+
level
|
|
1010
|
+
};
|
|
1011
|
+
}
|
|
1012
|
+
return substring;
|
|
1013
|
+
}
|
|
1014
|
+
var FLOW_CONTAINERS = /* @__PURE__ */ new Set([
|
|
1015
|
+
"ARTICLE",
|
|
1016
|
+
"ASIDE",
|
|
1017
|
+
"BLOCKQUOTE",
|
|
1018
|
+
"BODY",
|
|
1019
|
+
"DD",
|
|
1020
|
+
"DETAILS",
|
|
1021
|
+
"DT",
|
|
1022
|
+
"FOOTER",
|
|
1023
|
+
"HEADER",
|
|
1024
|
+
"LI",
|
|
1025
|
+
"MAIN",
|
|
1026
|
+
"NAV",
|
|
1027
|
+
"SECTION",
|
|
1028
|
+
"TD"
|
|
1029
|
+
]);
|
|
1030
|
+
function findFlowContainer(heading) {
|
|
1031
|
+
let ancestor = heading.parentElement;
|
|
1032
|
+
while (ancestor) {
|
|
1033
|
+
if (FLOW_CONTAINERS.has(ancestor.tagName)) return ancestor;
|
|
1034
|
+
ancestor = ancestor.parentElement;
|
|
1035
|
+
}
|
|
1036
|
+
return heading.ownerDocument.body;
|
|
1037
|
+
}
|
|
1038
|
+
function firstHeadingLevel(el) {
|
|
1039
|
+
const direct = /^H([1-6])$/.exec(el.tagName);
|
|
1040
|
+
if (direct) return Number.parseInt(direct[1], 10);
|
|
1041
|
+
const inner = el.querySelector("h1, h2, h3, h4, h5, h6");
|
|
1042
|
+
if (inner) return Number.parseInt(inner.tagName.slice(1), 10);
|
|
1043
|
+
}
|
|
1044
|
+
function scopeToHeading(document, headingText) {
|
|
1045
|
+
const match = findHeading(document, headingText);
|
|
1046
|
+
if (!match) return false;
|
|
1047
|
+
const { heading, level } = match;
|
|
1048
|
+
const container = findFlowContainer(heading);
|
|
1049
|
+
let startChild = heading;
|
|
1050
|
+
while (startChild.parentElement && startChild.parentElement !== container) startChild = startChild.parentElement;
|
|
1051
|
+
const wrap = document.createElement("section");
|
|
1052
|
+
wrap.setAttribute("data-rdrm-section-scope", "");
|
|
1053
|
+
container.insertBefore(wrap, startChild);
|
|
1054
|
+
let node = startChild;
|
|
1055
|
+
while (node) {
|
|
1056
|
+
const next = node.nextSibling;
|
|
1057
|
+
wrap.appendChild(node);
|
|
1058
|
+
if (next !== null && isElement(next)) {
|
|
1059
|
+
const nextLevel = firstHeadingLevel(next);
|
|
1060
|
+
if (nextLevel !== void 0 && nextLevel <= level) break;
|
|
1061
|
+
}
|
|
1062
|
+
node = next;
|
|
1063
|
+
}
|
|
1064
|
+
return true;
|
|
1065
|
+
}
|
|
1066
|
+
//#endregion
|
|
1067
|
+
//#region src/tools/extract_section.ts
|
|
1068
|
+
var SECTION_SCOPE_SELECTOR = "[data-rdrm-section-scope]";
|
|
1069
|
+
function extractSection(rawArgs) {
|
|
1070
|
+
const { localPath, ...rest } = extractSectionInputSchema.parse(rawArgs);
|
|
1071
|
+
return extractSectionFromHtml({
|
|
1072
|
+
html: readHtmlFile(localPath),
|
|
1073
|
+
...rest
|
|
1074
|
+
});
|
|
1075
|
+
}
|
|
1076
|
+
function extractSectionFromHtml(input) {
|
|
1077
|
+
const { html, baseUrl, selector, heading } = input;
|
|
1078
|
+
if (selector !== void 0) return extractArticleFromHtml({
|
|
1079
|
+
html,
|
|
1080
|
+
baseUrl,
|
|
1081
|
+
selectors: { include: selector }
|
|
1082
|
+
});
|
|
1083
|
+
if (heading === void 0) throw new ExtractionError("Provide exactly one of `selector` or `heading`.");
|
|
1084
|
+
const { document } = buildDocument(html, baseUrl);
|
|
1085
|
+
normalizeDocument(document);
|
|
1086
|
+
if (!scopeToHeading(document, heading)) throw new ExtractionError(`no heading matched: ${heading}`);
|
|
1087
|
+
const scoped = `<!DOCTYPE html><html><head></head><body>${document.body.innerHTML}</body></html>`;
|
|
1088
|
+
return extractArticleFromHtml({
|
|
1089
|
+
html: scoped,
|
|
1090
|
+
baseUrl,
|
|
1091
|
+
selectors: { include: SECTION_SCOPE_SELECTOR }
|
|
1092
|
+
});
|
|
1093
|
+
}
|
|
1094
|
+
var EXTRACT_SECTION_TOOL_DESCRIPTION = `Extract one section of an already-rendered (post-JavaScript) HTML document and return its Markdown + metadata + diagnostics — a thin resolver over extract’s \`selectors.include\` path, not a new extractor. Pick the section by CSS \`selector\` (passed straight through) OR by \`heading\` text (case-insensitive, first match wins; the section spans from the matched heading to the next same-or-higher-level heading). Exactly one of \`selector\`/\`heading\` is required. The server fetches nothing: \`localPath\` is the only source, and \`baseUrl\` (optional) is origin context only (never fetched).`;
|
|
1095
|
+
function extractSectionHandler(args) {
|
|
1096
|
+
try {
|
|
1097
|
+
return extractSection(args);
|
|
1098
|
+
} catch (err) {
|
|
1099
|
+
logger.error(`extract_section failed: ${err instanceof Error ? err.message : String(err)}`);
|
|
1100
|
+
return toErrorResult(err);
|
|
1101
|
+
}
|
|
1102
|
+
}
|
|
1103
|
+
function registerExtractSectionTool(server) {
|
|
1104
|
+
return server.registerTool("extract_section", {
|
|
1105
|
+
title: "Extract one section by selector or heading",
|
|
1106
|
+
description: EXTRACT_SECTION_TOOL_DESCRIPTION,
|
|
1107
|
+
inputSchema: extractSectionInputShape,
|
|
1108
|
+
outputSchema: outputSchemaShape
|
|
1109
|
+
}, extractSectionHandler);
|
|
1110
|
+
}
|
|
1111
|
+
//#endregion
|
|
1112
|
+
//#region src/tools/extract_tables.ts
|
|
1113
|
+
var NO_TABLES = "(no tables found)";
|
|
1114
|
+
function extractTables(rawArgs) {
|
|
1115
|
+
const { localPath, ...rest } = extractTablesInputSchema.parse(rawArgs);
|
|
1116
|
+
return extractTablesFromHtml({
|
|
1117
|
+
html: readHtmlFile(localPath),
|
|
1118
|
+
...rest
|
|
1119
|
+
});
|
|
1120
|
+
}
|
|
1121
|
+
var DEFAULTS$1 = extractTablesInputSchema.parse({ localPath: "" });
|
|
1122
|
+
function extractTablesFromHtml(input) {
|
|
1123
|
+
const { html, baseUrl, format, selectors } = {
|
|
1124
|
+
...DEFAULTS$1,
|
|
1125
|
+
...input
|
|
1126
|
+
};
|
|
1127
|
+
const { document } = buildDocument(html, baseUrl);
|
|
1128
|
+
applySelectors(document, selectors);
|
|
1129
|
+
const tables = [];
|
|
1130
|
+
let index = 0;
|
|
1131
|
+
for (const table of document.querySelectorAll("table")) {
|
|
1132
|
+
const matrix = parseTableMatrix(table);
|
|
1133
|
+
if (matrix.length === 0) continue;
|
|
1134
|
+
const keys = format === "json" ? resolveHeaderKeys(table, matrix) : void 0;
|
|
1135
|
+
const markdown = renderTable(matrix, format, keys);
|
|
1136
|
+
tables.push({
|
|
1137
|
+
index,
|
|
1138
|
+
rows: matrix.length,
|
|
1139
|
+
cols: matrix[0]?.length ?? 0,
|
|
1140
|
+
markdown
|
|
1141
|
+
});
|
|
1142
|
+
index++;
|
|
1143
|
+
}
|
|
1144
|
+
const content = tables.length > 0 ? tables.map((entry) => entry.markdown).join("\n\n") : NO_TABLES;
|
|
1145
|
+
return {
|
|
1146
|
+
content: [{
|
|
1147
|
+
text: content,
|
|
1148
|
+
type: "text"
|
|
1149
|
+
}],
|
|
1150
|
+
structuredContent: {
|
|
1151
|
+
schemaVersion: 1,
|
|
1152
|
+
content,
|
|
1153
|
+
tables,
|
|
1154
|
+
metadata: {
|
|
1155
|
+
baseUrl,
|
|
1156
|
+
format,
|
|
1157
|
+
tableCount: tables.length
|
|
1158
|
+
}
|
|
1159
|
+
}
|
|
1160
|
+
};
|
|
1161
|
+
}
|
|
1162
|
+
var EXTRACT_TABLES_TOOL_DESCRIPTION = `Extract every <table> on the page from already-rendered (post-JavaScript) HTML and return each as GFM / CSV / JSON (caller picks). Runs no Readability, Turndown, or sanitization — a page-wide \`querySelectorAll('table')\` walk in front of the same rowspan/colspan-aware matrix serializer used by the \`tables\` option on \`extract\`. Captures tables outside the article body (nav, aside, boilerplate) that the \`tables\` option never sees. The server fetches nothing: \`localPath\` is the only source, and \`baseUrl\` (optional) is origin context only (never fetched).`;
|
|
1163
|
+
function extractTablesHandler(args) {
|
|
1164
|
+
try {
|
|
1165
|
+
return extractTables(args);
|
|
1166
|
+
} catch (err) {
|
|
1167
|
+
logger.error(`extract_tables failed: ${err instanceof Error ? err.message : String(err)}`);
|
|
1168
|
+
return toErrorResult(err);
|
|
1169
|
+
}
|
|
1170
|
+
}
|
|
1171
|
+
function registerExtractTablesTool(server) {
|
|
1172
|
+
return server.registerTool("extract_tables", {
|
|
1173
|
+
title: "Extract every table on the page",
|
|
1174
|
+
description: EXTRACT_TABLES_TOOL_DESCRIPTION,
|
|
1175
|
+
inputSchema: extractTablesInputShape,
|
|
1176
|
+
outputSchema: extractTablesOutputShape
|
|
1177
|
+
}, extractTablesHandler);
|
|
1178
|
+
}
|
|
1179
|
+
//#endregion
|
|
1180
|
+
//#region src/tools/html_to_markdown.ts
|
|
1181
|
+
var EXTRACTED_NODE = "fragment";
|
|
1182
|
+
function htmlToMarkdown(rawArgs) {
|
|
1183
|
+
const { localPath, ...rest } = htmlToMarkdownInputSchema.parse(rawArgs);
|
|
1184
|
+
return htmlToMarkdownFromHtml({
|
|
1185
|
+
html: readHtmlFile(localPath),
|
|
1186
|
+
...rest
|
|
1187
|
+
});
|
|
1188
|
+
}
|
|
1189
|
+
var DEFAULTS = htmlToMarkdownInputSchema.parse({ localPath: "" });
|
|
1190
|
+
function htmlToMarkdownFromHtml(input) {
|
|
1191
|
+
const { html, baseUrl, selectors, format, metadataMode, gfm, headingStyle, codeBlockStyle, images, sanitize: shouldSanitize, maxChars, wordsPerMinute, cleanChrome, tables, debug } = {
|
|
1192
|
+
...DEFAULTS,
|
|
1193
|
+
...input
|
|
1194
|
+
};
|
|
1195
|
+
const trace = new TraceCollector(debug);
|
|
1196
|
+
const { document, window } = buildDocument(html, baseUrl);
|
|
1197
|
+
const { documentElementCount, normalizeCounts, imagesResolved, textContent, rawHtml } = trace.run("normalize", () => {
|
|
1198
|
+
const documentElementCount = document.querySelectorAll("*").length;
|
|
1199
|
+
const normalizeCounts = normalizeDocument(document, { cleanChrome });
|
|
1200
|
+
const imagesResolved = resolveLazyImages(document);
|
|
1201
|
+
applySelectors(document, selectors);
|
|
1202
|
+
const body = document.body;
|
|
1203
|
+
return {
|
|
1204
|
+
documentElementCount,
|
|
1205
|
+
imagesResolved,
|
|
1206
|
+
normalizeCounts,
|
|
1207
|
+
rawHtml: body.innerHTML,
|
|
1208
|
+
textContent: body.textContent
|
|
1209
|
+
};
|
|
1210
|
+
});
|
|
1211
|
+
const { html: sanitizedHtml, counts: sanitizeCounts } = trace.run("sanitize", () => {
|
|
1212
|
+
if (!shouldSanitize) return {
|
|
1213
|
+
counts: {
|
|
1214
|
+
iframes: 0,
|
|
1215
|
+
scripts: 0
|
|
1216
|
+
},
|
|
1217
|
+
html: rawHtml
|
|
1218
|
+
};
|
|
1219
|
+
const res = sanitizeHtml(rawHtml, window);
|
|
1220
|
+
return {
|
|
1221
|
+
counts: {
|
|
1222
|
+
iframes: res.iframesRemoved,
|
|
1223
|
+
scripts: res.scriptsRemoved
|
|
1224
|
+
},
|
|
1225
|
+
html: res.html
|
|
1226
|
+
};
|
|
1227
|
+
});
|
|
1228
|
+
const markdown = trace.run("turndown", () => toMarkdown(sanitizedHtml, {
|
|
1229
|
+
codeBlockStyle,
|
|
1230
|
+
gfm,
|
|
1231
|
+
headingStyle,
|
|
1232
|
+
images,
|
|
1233
|
+
tables,
|
|
1234
|
+
baseUrl
|
|
1235
|
+
}));
|
|
1236
|
+
const { metadata } = trace.run("metadata", () => {
|
|
1237
|
+
return { metadata: {
|
|
1238
|
+
title: nonEmpty(document.body.querySelector("h1, h2, h3, h4, h5, h6")?.textContent),
|
|
1239
|
+
baseUrl,
|
|
1240
|
+
...computeTextMetrics(textContent, wordsPerMinute)
|
|
1241
|
+
} };
|
|
1242
|
+
});
|
|
1243
|
+
const sanitization = {
|
|
1244
|
+
iframes: normalizeCounts.iframes + sanitizeCounts.iframes,
|
|
1245
|
+
scripts: normalizeCounts.scripts + sanitizeCounts.scripts
|
|
1246
|
+
};
|
|
1247
|
+
const baseDiagnostics = assembleDiagnostics({
|
|
1248
|
+
articleHtml: sanitizedHtml,
|
|
1249
|
+
boilerplateRemoved: normalizeCounts.boilerplateRemoved,
|
|
1250
|
+
chromeRemoved: normalizeCounts.chromeRemoved,
|
|
1251
|
+
documentElementCount,
|
|
1252
|
+
extractedNode: EXTRACTED_NODE,
|
|
1253
|
+
fallbackUsed: true,
|
|
1254
|
+
imagesResolved,
|
|
1255
|
+
sanitization,
|
|
1256
|
+
trace: trace.collect(),
|
|
1257
|
+
truncated: false,
|
|
1258
|
+
window
|
|
1259
|
+
});
|
|
1260
|
+
let payload = formatPayload({
|
|
1261
|
+
diagnostics: baseDiagnostics,
|
|
1262
|
+
format,
|
|
1263
|
+
markdown,
|
|
1264
|
+
metadata,
|
|
1265
|
+
metadataMode,
|
|
1266
|
+
sanitizedHtml,
|
|
1267
|
+
textContent
|
|
1268
|
+
});
|
|
1269
|
+
let truncated = false;
|
|
1270
|
+
if (maxChars !== void 0 && (format === "markdown" || format === "text")) {
|
|
1271
|
+
const res = truncateMarkdown(payload, maxChars);
|
|
1272
|
+
payload = res.text;
|
|
1273
|
+
truncated = res.truncated;
|
|
1274
|
+
}
|
|
1275
|
+
const diagnostics = truncated ? {
|
|
1276
|
+
...baseDiagnostics,
|
|
1277
|
+
truncated
|
|
1278
|
+
} : baseDiagnostics;
|
|
1279
|
+
return {
|
|
1280
|
+
content: [{
|
|
1281
|
+
text: payload,
|
|
1282
|
+
type: "text"
|
|
1283
|
+
}],
|
|
1284
|
+
structuredContent: {
|
|
1285
|
+
schemaVersion: 1,
|
|
1286
|
+
content: payload,
|
|
1287
|
+
metadata,
|
|
1288
|
+
diagnostics
|
|
1289
|
+
}
|
|
1290
|
+
};
|
|
1291
|
+
}
|
|
1292
|
+
var HTML_TO_MARKDOWN_TOOL_DESCRIPTION = `Convert an arbitrary HTML fragment to Markdown WITHOUT Readability article extraction (e.g. a snippet already isolated via chrome-devtools). Same Turndown + DOMPurify path as \`extract\`. The server fetches nothing: \`localPath\` is the only source, and \`baseUrl\` (optional) absolutizes relative links.`;
|
|
1293
|
+
function htmlToMarkdownHandler(args) {
|
|
1294
|
+
try {
|
|
1295
|
+
return htmlToMarkdown(args);
|
|
1296
|
+
} catch (err) {
|
|
1297
|
+
logger.error(`html_to_markdown failed: ${err instanceof Error ? err.message : String(err)}`);
|
|
1298
|
+
return toErrorResult(err);
|
|
1299
|
+
}
|
|
1300
|
+
}
|
|
1301
|
+
function registerHtmlToMarkdownTool(server) {
|
|
1302
|
+
return server.registerTool("html_to_markdown", {
|
|
1303
|
+
title: "Convert HTML fragment to Markdown",
|
|
1304
|
+
description: HTML_TO_MARKDOWN_TOOL_DESCRIPTION,
|
|
1305
|
+
inputSchema: htmlToMarkdownInputShape,
|
|
1306
|
+
outputSchema: outputSchemaShape
|
|
1307
|
+
}, htmlToMarkdownHandler);
|
|
1308
|
+
}
|
|
1309
|
+
//#endregion
|
|
1310
|
+
//#region src/tools/outline.ts
|
|
1311
|
+
function renderOutlineToc(outline) {
|
|
1312
|
+
if (outline.length === 0) return "(no headings found)";
|
|
1313
|
+
return outline.map((entry) => `${" ".repeat(entry.level - 1)}- ${entry.text}`).join("\n");
|
|
1314
|
+
}
|
|
1315
|
+
function outlineDocument(rawArgs) {
|
|
1316
|
+
const { localPath, ...rest } = outlineInputSchema.parse(rawArgs);
|
|
1317
|
+
return outlineDocumentFromHtml({
|
|
1318
|
+
html: readHtmlFile(localPath),
|
|
1319
|
+
...rest
|
|
1320
|
+
});
|
|
1321
|
+
}
|
|
1322
|
+
function outlineDocumentFromHtml(input) {
|
|
1323
|
+
const { html, baseUrl, selectors } = input;
|
|
1324
|
+
const { document } = buildDocument(html, baseUrl);
|
|
1325
|
+
normalizeDocument(document);
|
|
1326
|
+
applySelectors(document, selectors);
|
|
1327
|
+
const outline = resolveOutline(document);
|
|
1328
|
+
const metadata = {
|
|
1329
|
+
title: document.title.trim() || document.querySelector("h1")?.textContent.replace(/\s+/g, " ").trim() || void 0,
|
|
1330
|
+
baseUrl
|
|
1331
|
+
};
|
|
1332
|
+
const content = renderOutlineToc(outline);
|
|
1333
|
+
return {
|
|
1334
|
+
content: [{
|
|
1335
|
+
text: content,
|
|
1336
|
+
type: "text"
|
|
1337
|
+
}],
|
|
1338
|
+
structuredContent: {
|
|
1339
|
+
schemaVersion: 1,
|
|
1340
|
+
content,
|
|
1341
|
+
outline,
|
|
1342
|
+
metadata
|
|
1343
|
+
}
|
|
1344
|
+
};
|
|
1345
|
+
}
|
|
1346
|
+
var OUTLINE_TOOL_DESCRIPTION = `Return the document outline (h1-h6 headings with stable anchor ids) of already-rendered (post-JavaScript) HTML as a cheap pre-check before full extraction. No Readability scoring, no Turndown, no sanitization — a pure heading walk. The server fetches nothing: \`localPath\` is the only source, and \`baseUrl\` is origin context only (never fetched).`;
|
|
1347
|
+
function outlineHandler(args) {
|
|
1348
|
+
try {
|
|
1349
|
+
return outlineDocument(args);
|
|
1350
|
+
} catch (err) {
|
|
1351
|
+
logger.error(`outline failed: ${err instanceof Error ? err.message : String(err)}`);
|
|
1352
|
+
return toErrorResult(err);
|
|
1353
|
+
}
|
|
1354
|
+
}
|
|
1355
|
+
function registerOutlineTool(server) {
|
|
1356
|
+
return server.registerTool("outline", {
|
|
1357
|
+
title: "Get document outline (heading TOC)",
|
|
1358
|
+
description: OUTLINE_TOOL_DESCRIPTION,
|
|
1359
|
+
inputSchema: outlineInputShape,
|
|
1360
|
+
outputSchema: outlineOutputShape
|
|
1361
|
+
}, outlineHandler);
|
|
1362
|
+
}
|
|
1363
|
+
//#endregion
|
|
1364
|
+
//#region src/server.ts
|
|
1365
|
+
function createMcpServer() {
|
|
1366
|
+
const { name, version, title, description, instructions } = loadConfig();
|
|
1367
|
+
return new McpServer({
|
|
1368
|
+
name,
|
|
1369
|
+
version,
|
|
1370
|
+
title,
|
|
1371
|
+
description
|
|
1372
|
+
}, { instructions });
|
|
1373
|
+
}
|
|
1374
|
+
function registerTools(server) {
|
|
1375
|
+
return [
|
|
1376
|
+
registerChunkTextTool(server),
|
|
1377
|
+
registerExplainTool(server),
|
|
1378
|
+
registerExtractGridTool(server),
|
|
1379
|
+
registerExtractLinksTool(server),
|
|
1380
|
+
registerExtractListTool(server),
|
|
1381
|
+
registerExtractTool(server),
|
|
1382
|
+
registerExtractMetadataTool(server),
|
|
1383
|
+
registerExtractSectionTool(server),
|
|
1384
|
+
registerExtractTablesTool(server),
|
|
1385
|
+
registerHtmlToMarkdownTool(server),
|
|
1386
|
+
registerOutlineTool(server)
|
|
1387
|
+
];
|
|
1388
|
+
}
|
|
1389
|
+
function registerCapabilityGatedTools(server) {
|
|
1390
|
+
if (!server.server.getClientCapabilities()?.sampling) return [];
|
|
1391
|
+
return registerSamplingTools(server);
|
|
1392
|
+
}
|
|
1393
|
+
function createServer() {
|
|
1394
|
+
const server = createMcpServer();
|
|
1395
|
+
registerTools(server);
|
|
1396
|
+
registerResources(server);
|
|
1397
|
+
server.server.oninitialized = () => {
|
|
1398
|
+
registerCapabilityGatedTools(server);
|
|
1399
|
+
};
|
|
1400
|
+
return server;
|
|
1401
|
+
}
|
|
1402
|
+
//#endregion
|
|
1403
|
+
//#region src/index.ts
|
|
1404
|
+
if (process.argv[2] === "extract") import("./assets/cli-BwKCixh6.js").then((m) => m.runCli(process.argv.slice(2))).then((code) => {
|
|
1405
|
+
process.exit(code);
|
|
1406
|
+
}).catch((err) => {
|
|
1407
|
+
process.stderr.write(`${err instanceof Error ? err.message : String(err)}\n`);
|
|
1408
|
+
process.exit(1);
|
|
1409
|
+
});
|
|
1410
|
+
else {
|
|
1411
|
+
const server = createServer();
|
|
1412
|
+
const transport = new StdioServerTransport();
|
|
1413
|
+
await server.connect(transport);
|
|
1414
|
+
function shutdown() {
|
|
1415
|
+
server.close().catch(() => {}).finally(() => {
|
|
1416
|
+
process.exit(0);
|
|
1417
|
+
});
|
|
1418
|
+
}
|
|
1419
|
+
process.on("SIGINT", shutdown);
|
|
1420
|
+
process.on("SIGTERM", shutdown);
|
|
1421
|
+
}
|
|
1422
|
+
//#endregion
|
|
1423
|
+
export {};
|
|
1424
|
+
|
|
1425
|
+
//# sourceMappingURL=index.js.map
|