@v1nvn/readability 0.43.0 → 0.45.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (71) hide show
  1. package/README.md +3 -3
  2. package/package.json +7 -9
  3. package/src/cli.ts +134 -0
  4. package/src/config.ts +42 -0
  5. package/src/dev.ts +77 -0
  6. package/src/errors.ts +19 -0
  7. package/src/handler.ts +8 -0
  8. package/src/host-sampling.ts +86 -0
  9. package/src/index.ts +27 -0
  10. package/src/logger.ts +3 -0
  11. package/src/output/format.ts +132 -0
  12. package/src/pipeline/cell-text.ts +56 -0
  13. package/src/pipeline/context.ts +88 -0
  14. package/src/pipeline/dom.ts +23 -0
  15. package/src/pipeline/footnotes.ts +181 -0
  16. package/src/pipeline/math.ts +130 -0
  17. package/src/pipeline/normalize.ts +594 -0
  18. package/src/pipeline/readability.ts +36 -0
  19. package/src/pipeline/resolver.ts +143 -0
  20. package/src/pipeline/sanitize.ts +42 -0
  21. package/src/pipeline/tables.ts +286 -0
  22. package/src/pipeline/turndown.ts +191 -0
  23. package/src/pipeline/urls.ts +13 -0
  24. package/src/policy/chunk.ts +482 -0
  25. package/src/policy/diagnostics.ts +96 -0
  26. package/src/policy/explain.ts +182 -0
  27. package/src/policy/fallback.ts +110 -0
  28. package/src/policy/gating.ts +106 -0
  29. package/src/policy/grid-detector.ts +313 -0
  30. package/src/policy/identifiers.ts +75 -0
  31. package/src/policy/images.ts +64 -0
  32. package/src/policy/list-detector.ts +308 -0
  33. package/src/policy/lost-signal.ts +69 -0
  34. package/src/policy/markdown.ts +76 -0
  35. package/src/policy/metadata.ts +312 -0
  36. package/src/policy/outline-chains.ts +215 -0
  37. package/src/policy/outline.ts +89 -0
  38. package/src/policy/pagination.ts +104 -0
  39. package/src/policy/presets.ts +142 -0
  40. package/src/policy/section.ts +127 -0
  41. package/src/policy/selector-lint.ts +186 -0
  42. package/src/policy/sibling-scan.ts +99 -0
  43. package/src/policy/text.ts +36 -0
  44. package/src/policy/truncate.ts +59 -0
  45. package/src/preset-cache.ts +221 -0
  46. package/src/resources.ts +317 -0
  47. package/src/sampling.ts +74 -0
  48. package/src/server.ts +85 -0
  49. package/src/tools/chunk_text.ts +65 -0
  50. package/src/tools/explain.ts +299 -0
  51. package/src/tools/extract.ts +368 -0
  52. package/src/tools/extract_grid.ts +90 -0
  53. package/src/tools/extract_links.ts +138 -0
  54. package/src/tools/extract_list.ts +97 -0
  55. package/src/tools/extract_metadata.ts +102 -0
  56. package/src/tools/extract_section.ts +84 -0
  57. package/src/tools/extract_tables.ts +101 -0
  58. package/src/tools/html-source.ts +40 -0
  59. package/src/tools/html_to_markdown.ts +180 -0
  60. package/src/tools/outline.ts +77 -0
  61. package/src/tools/output-schema.ts +715 -0
  62. package/src/tools/schemas.ts +442 -0
  63. package/src/tools/suggest-preset.ts +708 -0
  64. package/src/types/jsdom.d.ts +15 -0
  65. package/src/types/turndown-plugin-gfm.d.ts +11 -0
  66. package/dist/assets/cli-BE_Ml3rh.js +0 -76
  67. package/dist/assets/cli-BE_Ml3rh.js.map +0 -1
  68. package/dist/assets/extract-Cyv5qfGN.js +0 -2880
  69. package/dist/assets/extract-Cyv5qfGN.js.map +0 -1
  70. package/dist/index.js +0 -2267
  71. package/dist/index.js.map +0 -1
package/dist/index.js DELETED
@@ -1,2267 +0,0 @@
1
- #!/usr/bin/env node
2
- import { $ as resolveLazyImages, A as selectorsSchema, B as absolutize, C as extractTablesInputSchema, D as localPathField, E as htmlToMarkdownInputShape, F as detectGating, G as sanitizeHtml, H as renderTable, I as TraceCollector, J as computeTextMetrics, K as isReaderable, L as assembleDiagnostics, M as truncateMarkdown, N as detectPagination, O as outlineInputSchema, P as resolveMetadata, Q as normalizeDocument, R as chunkMarkdown, S as extractSectionInputShape, T as htmlToMarkdownInputSchema, U as resolveHeaderKeys, V as parseTableMatrix, W as resolveCellText, X as nonEmpty, Y as countWords, Z as applySelectors, _ as extractListInputSchema, a as extractLinksOutputShape, at as normalizeSiteKey, b as extractMetadataInputShape, c as extractTablesOutputShape, ct as selectorMisses, d as chunkTextInputSchema, dt as ExtractionError, et as resolveReadabilityOptions, f as chunkTextInputShape, ft as describeError, g as extractLinksInputShape, gt as shutdownOnSignals, h as extractLinksInputSchema, i as extractGridOutputShape, it as addPreset, j as readHtmlFile, k as outlineInputShape, l as outlineOutputShape, lt as logger, m as extractGridInputShape, n as registerExtractTool, nt as isElement, o as extractListOutputShape, ot as presetForSite, p as extractGridInputSchema, pt as toErrorResult, q as formatPayload, r as chunkTextOutputShape, rt as registerResources, s as extractMetadataOutputShape, st as removePreset, t as extractArticleFromHtml, tt as buildDocument, u as outputSchemaShape, ut as loadConfig, v as extractListInputShape, w as extractTablesInputShape, x as extractSectionInputSchema, y as extractMetadataInputSchema, z as toMarkdown } from "./assets/extract-Cyv5qfGN.js";
3
- import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js";
4
- import { mkdirSync, readFileSync, readdirSync, statSync, unlinkSync, writeFileSync } from "node:fs";
5
- import { homedir } from "node:os";
6
- import { join } from "node:path";
7
- import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
8
- import { z } from "zod";
9
- import { Readability } from "@mozilla/readability";
10
- //#region src/preset-cache.ts
11
- var PRESETS_DIR_ENV = "READABILITY_PRESETS_DIR";
12
- var presetFileSchema = z.object({
13
- site: z.string().min(1),
14
- detectors: z.array(z.string().min(1)).min(1),
15
- scope: z.object({
16
- include: z.string().min(1).optional(),
17
- exclude: z.array(z.string().min(1)).min(1).optional()
18
- }).refine((scope) => scope.include !== void 0 || scope.exclude !== void 0, { message: "scope must carry include or exclude" })
19
- });
20
- function resolvePresetsDir(env = process.env) {
21
- const override = env[PRESETS_DIR_ENV];
22
- if (override !== void 0) return override.trim() === "" ? void 0 : override;
23
- const root = env.XDG_CACHE_HOME || join(homedir(), process.platform === "darwin" ? "Library/Caches" : ".cache");
24
- return join(root, "readability", "presets");
25
- }
26
- function loadPresetDir(dir) {
27
- let names;
28
- try {
29
- names = readdirSync(dir).filter((name) => name.endsWith(".json")).sort();
30
- } catch (err) {
31
- if (err.code === "ENOENT") logger.debug(`no preset directory at ${dir}`);
32
- else logger.warn(`preset directory unreadable: ${dir}`);
33
- return {
34
- loaded: 0,
35
- pruned: 0,
36
- skipped: 0
37
- };
38
- }
39
- const ranked = rankPresetFiles(dir, names);
40
- const pruned = pruneRanked(dir, ranked.slice(64));
41
- let loaded = 0;
42
- let skipped = 0;
43
- for (const { name } of ranked.slice(0, 64)) if (loadPresetFile(join(dir, name))) loaded++;
44
- else skipped++;
45
- if (loaded > 0) logger.info(`loaded ${loaded} site preset(s) from ${dir}`);
46
- if (pruned > 0) logger.info(`pruned ${pruned} preset file(s) beyond the 64-file bound`);
47
- return {
48
- loaded,
49
- pruned,
50
- skipped
51
- };
52
- }
53
- function loadPresets(env = process.env) {
54
- const dir = resolvePresetsDir(env);
55
- return dir ? loadPresetDir(dir) : void 0;
56
- }
57
- function rankPresetFiles(dir, names) {
58
- const ranked = [];
59
- for (const name of names) try {
60
- ranked.push({
61
- name,
62
- mtimeMs: statSync(join(dir, name)).mtimeMs
63
- });
64
- } catch {}
65
- ranked.sort((a, b) => b.mtimeMs - a.mtimeMs);
66
- return ranked;
67
- }
68
- function pruneRanked(dir, ranked) {
69
- let pruned = 0;
70
- for (const { name } of ranked) try {
71
- unlinkSync(join(dir, name));
72
- pruned++;
73
- } catch (err) {
74
- logger.warn(`could not prune preset file ${name}: ${describeError(err)}`);
75
- }
76
- return pruned;
77
- }
78
- function savePreset(preset, env = process.env) {
79
- const dir = resolvePresetsDir(env);
80
- if (!dir) return {
81
- persisted: false,
82
- reason: "preset-directory-disabled"
83
- };
84
- const shape = presetFileSchema.safeParse(preset);
85
- if (!shape.success) return {
86
- persisted: false,
87
- reason: shape.error.issues[0]?.message ?? "invalid preset shape"
88
- };
89
- const key = normalizeSiteKey(preset.site);
90
- if (!key) return {
91
- persisted: false,
92
- reason: "site-is-not-a-hostname"
93
- };
94
- const path = join(dir, `${key}.json`);
95
- try {
96
- mkdirSync(dir, { recursive: true });
97
- writeFileSync(path, `${JSON.stringify(shape.data, null, 2)}\n`);
98
- } catch (err) {
99
- return {
100
- persisted: false,
101
- reason: describeError(err)
102
- };
103
- }
104
- pruneRanked(dir, rankPresetFiles(dir, readdirSync(dir).filter((name) => name.endsWith(".json"))).slice(64));
105
- return {
106
- path,
107
- persisted: true
108
- };
109
- }
110
- function loadPresetFile(path) {
111
- let parsed;
112
- try {
113
- parsed = JSON.parse(readFileSync(path, "utf8"));
114
- } catch (err) {
115
- logger.warn(`${path}: not readable JSON (${describeError(err)}), preset skipped`);
116
- return false;
117
- }
118
- const result = presetFileSchema.safeParse(parsed);
119
- if (!result.success) {
120
- logger.warn(`${path}: ${result.error.issues[0]?.message}, preset skipped`);
121
- return false;
122
- }
123
- if (!normalizeSiteKey(result.data.site)) {
124
- logger.warn(`${path}: site is not a hostname, preset skipped`);
125
- return false;
126
- }
127
- addPreset(result.data);
128
- return true;
129
- }
130
- //#endregion
131
- //#region src/host-sampling.ts
132
- var SAMPLING_TIMEOUT_MS = 3e5;
133
- async function sampleText$1(server, args) {
134
- const result = await server.server.createMessage({
135
- messages: [{
136
- role: "user",
137
- content: {
138
- type: "text",
139
- text: args.userText
140
- }
141
- }],
142
- systemPrompt: args.systemPrompt,
143
- maxTokens: args.maxTokens
144
- }, { timeout: SAMPLING_TIMEOUT_MS });
145
- if (result.content.type !== "text") throw new Error(`host sampling returned non-text content (${result.content.type})`);
146
- return result.content.text;
147
- }
148
- var SuggestParseError = class extends Error {
149
- rawText;
150
- constructor(message, rawText) {
151
- super(message);
152
- this.rawText = rawText;
153
- }
154
- };
155
- function extractJsonReply(text) {
156
- const trimmed = text.replace(/^[\s\S]*?```(?:json)?\s*\n?([\s\S]*?)\n?```[\s\S]*$/, "$1").trim();
157
- try {
158
- return JSON.parse(trimmed);
159
- } catch {
160
- const start = trimmed.indexOf("{");
161
- const end = trimmed.lastIndexOf("}");
162
- if (start !== -1 && end > start) try {
163
- return JSON.parse(trimmed.slice(start, end + 1));
164
- } catch {}
165
- throw new SuggestParseError(`host sampling returned text that does not parse as JSON (${trimmed.slice(0, 80)}…)`, trimmed);
166
- }
167
- }
168
- async function sampleJson(server, args) {
169
- return extractJsonReply(await sampleText$1(server, args));
170
- }
171
- var DEBRIS_PROBES = [
172
- {
173
- label: "player-controls",
174
- pattern: /Loaded:\s*\d+%/
175
- },
176
- {
177
- label: "player-controls",
178
- pattern: /Duration Time \d+:\d\d/
179
- },
180
- {
181
- label: "metered-barrier",
182
- pattern: /read your last free article|subscribe to continue reading/i
183
- }
184
- ];
185
- function assessLostSignal(input) {
186
- const wordCount = countWords(input.contentText);
187
- const nearEmpty = wordCount < 120;
188
- const debrisProbes = [...new Set(DEBRIS_PROBES.filter((probe) => probe.pattern.test(input.contentText)).map((probe) => probe.label))];
189
- const reasons = [
190
- ...input.fallbackUsed ? ["fallback-used"] : [],
191
- ...nearEmpty ? ["near-empty"] : [],
192
- ...debrisProbes.map((label) => `debris:${label}`)
193
- ];
194
- return {
195
- debrisProbes,
196
- fallbackUsed: input.fallbackUsed,
197
- gatedReason: input.gated?.likely ? input.gated.reason : void 0,
198
- nearEmpty,
199
- reasons,
200
- wordCount
201
- };
202
- }
203
- var PRICE_TABLE = {
204
- "Upper:Lower": .2,
205
- "Lower:Upper": .5,
206
- "Symbol:Upper": .4,
207
- "Symbol:Lower": .3,
208
- "Symbol:Digit": .9,
209
- "Upper:Digit": 1.4,
210
- "Lower:Digit": 1.3,
211
- "Digit:Upper": 1.4,
212
- "Digit:Lower": 1.5,
213
- "Digit:Symbol": 1.2,
214
- "Upper:Symbol": .2,
215
- "Lower:Symbol": .2,
216
- "Digit:Digit": 1.2,
217
- "Symbol:Symbol": .3,
218
- "Upper:Upper": .1,
219
- "Lower:Lower": .1,
220
- "Other:Other": 0
221
- };
222
- function classifyChar(ch) {
223
- if (/[A-Z]/.test(ch)) return "Upper";
224
- if (/[a-z]/.test(ch)) return "Lower";
225
- if (/[0-9]/.test(ch)) return "Digit";
226
- if (/[-_]/.test(ch)) return "Symbol";
227
- return "Other";
228
- }
229
- function transitionPrice(from, to) {
230
- return PRICE_TABLE[`${from}:${to}`] ?? 1;
231
- }
232
- function gibberishScore(value) {
233
- if (value.length < 2) return 0;
234
- let absolute = 0;
235
- let previous = null;
236
- for (const ch of value) {
237
- const current = classifyChar(ch);
238
- if (previous !== null) absolute += transitionPrice(previous, current);
239
- previous = current;
240
- }
241
- return absolute / Math.max(1, value.length - 1);
242
- }
243
- function exceedsThreshold(value) {
244
- return value.length >= 4 && gibberishScore(value) >= .3;
245
- }
246
- function isGeneratedIdentifier(value) {
247
- if (exceedsThreshold(value)) return true;
248
- return value.split(/[-_]/).some((segment) => exceedsThreshold(segment));
249
- }
250
- var SAMPLE_MAX_CHARS = 80;
251
- var NON_CONTENT_SELECTOR = "script, style, noscript, template";
252
- var HOP_ATTRIBUTE_WHITELIST = [
253
- "itemprop",
254
- "role",
255
- "data-testid"
256
- ];
257
- function buildChainOutline(input) {
258
- const { document } = buildDocument(input.html, input.baseUrl);
259
- normalizeDocument(document, { cleanChrome: input.cleanChrome });
260
- resolveLazyImages(document);
261
- const scope = input.mode === "page" ? void 0 : input.mode.scope;
262
- if (scope) applySelectors(document, scope);
263
- const scopeRoot = scope?.include ? document.body.querySelector(scope.include) ?? void 0 : void 0;
264
- const min = scope ? 40 : 200;
265
- const candidates = [];
266
- for (const element of (scopeRoot ?? document.body).querySelectorAll("*")) {
267
- if (element.closest(NON_CONTENT_SELECTOR)) continue;
268
- const own = ownTextLength(element);
269
- if (own >= min) candidates.push({
270
- element,
271
- own
272
- });
273
- }
274
- return scope ? groupedOutline(candidates, scopeRoot) : flatOutline(candidates);
275
- }
276
- function flatOutline(candidates) {
277
- const lines = [];
278
- let truncated = false;
279
- for (const candidate of [...candidates].sort((a, b) => b.own - a.own)) {
280
- if (lines.length >= 60) {
281
- truncated = true;
282
- break;
283
- }
284
- const line = ancestorsUpTo(candidate.element, void 0).map((element) => hopWithSizes(element)).join(" > ");
285
- if (lines.join("\n---\n").length + line.length > 6e3) {
286
- truncated = true;
287
- break;
288
- }
289
- lines.push(line);
290
- }
291
- return {
292
- chainCount: lines.length,
293
- text: lines.join("\n---\n"),
294
- truncated
295
- };
296
- }
297
- function groupedOutline(candidates, scopeRoot) {
298
- const groups = /* @__PURE__ */ new Map();
299
- for (const candidate of candidates) {
300
- const chain = ancestorsUpTo(candidate.element, scopeRoot).map((element) => renderHop(element));
301
- if (scopeRoot) chain.push(renderHop(scopeRoot));
302
- const key = chain.join(" > ");
303
- const existing = groups.get(key);
304
- if (existing) {
305
- existing.count += 1;
306
- existing.min = Math.min(existing.min, candidate.own);
307
- existing.max = Math.max(existing.max, candidate.own);
308
- } else groups.set(key, {
309
- chain: key,
310
- count: 1,
311
- min: candidate.own,
312
- max: candidate.own,
313
- sample: sampleText(candidate.element)
314
- });
315
- }
316
- const lines = [];
317
- let truncated = false;
318
- const sorted = [...groups.values()].sort((a, b) => a.min - b.min);
319
- for (const group of sorted) {
320
- if (lines.length >= 60) {
321
- truncated = true;
322
- break;
323
- }
324
- const line = `${group.chain} (own:${group.min}..${group.max}, ×${group.count}) sample: "${group.sample}"`;
325
- if (lines.join("\n").length + line.length > 6e3) {
326
- truncated = true;
327
- break;
328
- }
329
- lines.push(line);
330
- }
331
- return {
332
- chainCount: lines.length,
333
- text: lines.join("\n"),
334
- truncated
335
- };
336
- }
337
- function ancestorsUpTo(element, stop) {
338
- const hops = [];
339
- let node = element;
340
- const body = element.ownerDocument.body;
341
- while (node && node !== stop && node !== body) {
342
- hops.push(node);
343
- node = node.parentElement;
344
- }
345
- return hops;
346
- }
347
- function hopWithSizes(element) {
348
- const own = ownTextLength(element);
349
- const all = element.textContent.length;
350
- return `${renderHop(element)} (own:${own}, all:${all})`;
351
- }
352
- function ownTextLength(element) {
353
- let own = 0;
354
- for (const node of element.childNodes) if (node.nodeType === node.TEXT_NODE) own += (node.textContent ?? "").length;
355
- return own;
356
- }
357
- function renderHop(element) {
358
- let hop = element.tagName.toLowerCase();
359
- const id = element.id;
360
- if (id && !isGeneratedIdentifier(id)) hop += `#${id}`;
361
- const classes = (element.getAttribute("class") ?? "").split(/\s+/).filter(Boolean).filter((className) => !isGeneratedIdentifier(className));
362
- if (classes.length > 0) hop += `.${classes.join(".")}`;
363
- for (const attribute of HOP_ATTRIBUTE_WHITELIST) {
364
- const value = element.getAttribute(attribute);
365
- if (value) hop += `[${attribute}="${value}"]`;
366
- }
367
- return hop;
368
- }
369
- function sampleText(element) {
370
- const text = element.textContent.trim().replace(/\s+/g, " ");
371
- return text.length > SAMPLE_MAX_CHARS ? `${text.slice(0, SAMPLE_MAX_CHARS)}…` : text;
372
- }
373
- //#endregion
374
- //#region src/policy/selector-lint.ts
375
- var POSITIONAL_PSEUDO_RE = /:(?:nth(?:-last)?(?:-child|-of-type)|first(?:-child|-of-type)|last(?:-child|-of-type)|only(?:-child|-of-type))\b/i;
376
- var CONTAINS_PSEUDO_RE = /:contains\s*\(/i;
377
- function quotedSegmentsOut(selector) {
378
- return selector.replace(/(["'])(?:\\.|(?!\1)[\s\S])*\1/g, "\"\"");
379
- }
380
- function identifierTokens(selector) {
381
- return [...quotedSegmentsOut(selector).matchAll(/[.#]([A-Za-z0-9_-]+)/g)].map((match) => match[1]);
382
- }
383
- function lintSelectorText(selector) {
384
- if (CONTAINS_PSEUDO_RE.test(selector)) return {
385
- kind: "contains-pseudo",
386
- selector,
387
- detail: ":contains is not standard CSS and changes the result silently"
388
- };
389
- const positional = POSITIONAL_PSEUDO_RE.exec(selector);
390
- if (positional) return {
391
- kind: "positional-pseudo",
392
- selector,
393
- detail: `${positional[0]} depends on this page's child order, not the site's layout`
394
- };
395
- for (const token of identifierTokens(selector)) if (isGeneratedIdentifier(token)) return {
396
- kind: "generated-identifier",
397
- selector,
398
- detail: `"${token}" reads as a generated hash that changes on deploy`
399
- };
400
- }
401
- function lintProposal({ document, detectors, scope }) {
402
- const violations = [];
403
- const selectors = [
404
- ...detectors,
405
- ...scope.include ? [scope.include] : [],
406
- ...scope.exclude ?? []
407
- ];
408
- for (const selector of selectors) {
409
- const violation = lintSelectorText(selector);
410
- if (violation) violations.push(violation);
411
- }
412
- let includeRoot;
413
- if (scope.include && !violations.some((violation) => violation.selector === scope.include)) try {
414
- includeRoot = document.body.querySelector(scope.include) ?? void 0;
415
- } catch {
416
- violations.push({
417
- kind: "unparseable",
418
- selector: scope.include,
419
- detail: "the selector engine rejects this selector"
420
- });
421
- }
422
- for (const detector of detectors) {
423
- if (violations.some((violation) => violation.selector === detector)) continue;
424
- if (selectorMisses(document, detector)) violations.push({
425
- kind: "no-match",
426
- selector: detector,
427
- detail: "detector matches nothing on this page"
428
- });
429
- }
430
- if (scope.include && !includeRoot && !violations.some((violation) => violation.selector === scope.include)) violations.push({
431
- kind: "no-match",
432
- selector: scope.include,
433
- detail: "include matches nothing inside <body>, where applySelectors searches"
434
- });
435
- for (const selector of scope.exclude ?? []) {
436
- if (violations.some((violation) => violation.selector === selector)) continue;
437
- try {
438
- if (document.querySelectorAll(selector).length === 0) violations.push({
439
- kind: "no-match",
440
- selector,
441
- detail: "exclude matches nothing on this page"
442
- });
443
- } catch {
444
- violations.push({
445
- kind: "unparseable",
446
- selector,
447
- detail: "the selector engine rejects this selector"
448
- });
449
- }
450
- }
451
- if (includeRoot) for (const selector of scope.exclude ?? []) {
452
- if (violations.some((violation) => violation.selector === selector)) continue;
453
- try {
454
- for (const match of document.querySelectorAll(selector)) if (match.contains(includeRoot)) {
455
- violations.push({
456
- kind: "exclude-shadows-include",
457
- selector,
458
- detail: "this exclude removes the include root itself"
459
- });
460
- break;
461
- }
462
- } catch {}
463
- }
464
- return violations;
465
- }
466
- //#endregion
467
- //#region src/tools/suggest-preset.ts
468
- var SUGGEST_SYSTEM_PROMPT = `You propose site-level CSS presets for a web-article extractor. The extractor removes every "exclude" match document-wide, then keeps the FIRST match of "include" in document order as the entire page body (the search runs inside <body>). An exclude matching an ancestor of the include root destroys it.
469
-
470
- Rules:
471
- - "include": exactly ONE selector that matches an element inside <body> on THIS page. A non-matching selector silently does nothing.
472
- - "exclude": selectors naming debris containers (video players, carousels, caption figures, comment blocks, promo modules).
473
- - "detectors": 1-3 selectors naming stable site-template containers that also match on other pages of the same site. Their loss is what retires the preset, so they must be structural, not per-page.
474
- - Forbidden — statically rejected and fed back to you: :nth-child and every positional pseudo (:first-child, :last-of-type, :only-child, ...), :contains, and generated hash class names (css-1a2b3c, content_1tsiE, BoxStyles_x__Wo6Z4).
475
- - Prefer itemprop, role, data-testid, ids, and stable class-name stems; quote attribute values exactly as rendered in the material.
476
- - The material shows every text-bearing node as a CSS chain with text sizes. It deliberately carries no article prose.
477
- Return strict JSON only — no prose, no code fences.`;
478
- var suggestPresetInputShape = {
479
- localPath: localPathField,
480
- baseUrl: z.url().describe("A URL of the site the page belongs to — its host names the preset. Required: without a host there is no site key to store a preset under, and presets only resolve for pages whose site matches."),
481
- secondPath: localPathField.optional().describe("A second capture of the SAME site. The converged preset must verify on it too — detectors match, extraction clean — before anything is persisted; without it the preset is verified on the proposal page alone."),
482
- maxSamplingCalls: z.number().int().min(1).max(8).describe("Upper bound on host-model sampling calls for this run. Round one (include + detectors) and round two (excludes) each consume a call, rejected proposals consume retries. A budget of 1 skips round two.").default(4)
483
- };
484
- var suggestPresetInputSchema = z.object(suggestPresetInputShape);
485
- var proposalSchema = z.object({
486
- detectors: z.array(z.string().min(1)).min(1).max(6),
487
- scope: z.object({
488
- include: z.string().min(1),
489
- exclude: z.array(z.string().min(1)).max(12).optional()
490
- })
491
- });
492
- var excludeProposalSchema = z.object({ exclude: z.array(z.string().min(1)).max(12) });
493
- var MAX_SAMPLING_TOKENS = 8192;
494
- var SUGGEST_PRESET_TOOL_DESCRIPTION = `Ask the HOST's model to propose a site preset (detectors + selectors) for a page whose extraction was lost, then verify it: proposals are validated deterministically, applied through the real pipeline, and a converged preset is stored in memory and persisted to the local preset cache so later extractions of the same site apply it automatically. Runs a bounded two-round loop over MCP \`sampling/createMessage\` — the server embeds no model. The tool is only listed when the connected client advertises the sampling capability, and it refuses to run when the baseline extraction looks healthy: call it after \`extract\` reports gated content, a fallback extraction, a near-empty result, or visible debris such as video-player controls.`;
495
- var triggerShape = {
496
- fired: z.boolean().describe("Whether the suggest loop ran at all."),
497
- reasons: z.array(z.string()).describe("Why the loop ran (the lost signals observed on the baseline extraction), or why it refused."),
498
- wordCount: z.number().describe("Word count of the baseline extraction; 0 when the loop never ran."),
499
- debrisProbes: z.array(z.string()).describe("Debris signatures matched in the baseline extraction text."),
500
- gatedReason: z.string().optional().describe("The gating signal reported by the baseline extraction, if any.")
501
- };
502
- var suggestPresetOutputShape = {
503
- schemaVersion: z.literal(1).describe("Structured-content schema version."),
504
- content: z.string().describe("Human-readable report of the run."),
505
- trigger: z.object(triggerShape).describe("Baseline lost-signal verdict."),
506
- preset: z.object({
507
- site: z.string().describe("The host the preset is keyed by."),
508
- detectors: z.array(z.string()).describe("Template fingerprints that must keep matching."),
509
- scope: z.object({
510
- include: z.string().optional(),
511
- exclude: z.array(z.string()).optional()
512
- }).describe("The accepted selector scope.")
513
- }).optional().describe("The preset that converged, present only when one was accepted and verified."),
514
- verification: z.object({
515
- applied: z.boolean().describe("Whether the stored preset resolved and applied during the verification extraction."),
516
- converged: z.boolean().describe("Whether the verification extraction came back clean."),
517
- verifiedPages: z.number().int().min(1).max(2).describe("Pages the preset verified on: 2 only when secondPath was given and the preset verified on that capture too."),
518
- wordCountBefore: z.number().describe("Baseline extraction word count."),
519
- wordCountAfter: z.number().describe("Verification extraction word count.")
520
- }).optional().describe("Result of extracting with the preset actually stored."),
521
- secondVerification: z.object({
522
- applied: z.boolean().describe("Whether the preset resolved and applied on the second capture."),
523
- converged: z.boolean().describe("Whether the second-capture extraction came back clean."),
524
- wordCountAfter: z.number().describe("Second-capture extraction word count.")
525
- }).optional().describe("Verification on the secondPath capture; absent when secondPath was not given or the capture belongs to another site."),
526
- persistence: z.object({
527
- persisted: z.boolean().describe("Whether the preset file was written to the local preset cache directory."),
528
- path: z.string().optional().describe("The file written, when persisted."),
529
- reason: z.string().optional().describe("Why nothing was persisted, when it was not.")
530
- }).optional().describe("Outcome of the disk write; absent when the run never accepted a preset."),
531
- sampling: z.object({
532
- calls: z.number().int().describe("Sampling calls actually made."),
533
- budget: z.number().int().describe("The budget this run was given."),
534
- budgetExhausted: z.boolean().describe("Whether the budget ran out before the loop could converge.")
535
- }).describe("Sampling accounting for this run.")
536
- };
537
- function registerSuggestPresetTool(server) {
538
- return server.registerTool("suggest_preset", {
539
- title: "Suggest a site preset using the host model",
540
- description: SUGGEST_PRESET_TOOL_DESCRIPTION,
541
- inputSchema: suggestPresetInputShape,
542
- outputSchema: suggestPresetOutputShape
543
- }, async (rawArgs) => {
544
- const args = suggestPresetInputSchema.parse(rawArgs);
545
- try {
546
- return await runSuggestLoop(args, server);
547
- } catch (err) {
548
- logger.error(`suggest_preset failed: ${describeError(err)}`);
549
- return toErrorResult(err);
550
- }
551
- });
552
- }
553
- async function runSuggestLoop(args, server) {
554
- const site = normalizeSiteKey(args.baseUrl);
555
- if (!site) throw new Error(`baseUrl does not name a site: ${args.baseUrl}`);
556
- const html = readHtmlFile(args.localPath);
557
- const secondHtml = args.secondPath ? readHtmlFile(args.secondPath) : void 0;
558
- const state = {
559
- budget: args.maxSamplingCalls,
560
- calls: 0,
561
- exhausted: false
562
- };
563
- if (presetForSite(site)) return report(state, { trigger: {
564
- debrisProbes: [],
565
- fired: false,
566
- reasons: ["preset-exists"],
567
- wordCount: 0
568
- } });
569
- const baseline = extractText(html, args.baseUrl);
570
- const canonicalSite = normalizeSiteKey(baseline.metadata.canonical);
571
- if (canonicalSite && canonicalSite !== site) throw new Error(`baseUrl names ${site} but the page's canonical URL names ${canonicalSite} — the capture does not belong to the site the preset would be keyed by.`);
572
- const baselineEvidence = assessLostSignal({
573
- contentText: baseline.content,
574
- fallbackUsed: baseline.diagnostics.fallbackUsed ?? false,
575
- gated: baseline.diagnostics.gated
576
- });
577
- const trigger = {
578
- debrisProbes: baselineEvidence.debrisProbes,
579
- fired: baselineEvidence.reasons.length > 0,
580
- gatedReason: baselineEvidence.gatedReason,
581
- reasons: baselineEvidence.reasons,
582
- wordCount: baselineEvidence.wordCount
583
- };
584
- if (!trigger.fired) return report(state, { trigger });
585
- const core = await proposeCore(html, args.baseUrl, site, baseline, trigger, state, server);
586
- if (!core) return report(state, { trigger });
587
- const excludes = await proposeExcludes(html, args.baseUrl, core, state, server);
588
- const preset = {
589
- detectors: core.detectors,
590
- site,
591
- scope: {
592
- exclude: excludes.length > 0 ? excludes : void 0,
593
- include: core.scope.include
594
- }
595
- };
596
- addPreset({
597
- detectors: preset.detectors,
598
- scope: preset.scope,
599
- site: preset.site
600
- });
601
- const verification = extractText(html, args.baseUrl);
602
- const verificationEvidence = assessLostSignal({
603
- contentText: verification.content,
604
- fallbackUsed: verification.diagnostics.fallbackUsed ?? false,
605
- gated: verification.diagnostics.gated
606
- });
607
- const applied = verification.presetSignal?.applied === true;
608
- const converged = applied && verificationEvidence.reasons.length === 0;
609
- const verdict = {
610
- applied,
611
- converged,
612
- verifiedPages: 1,
613
- wordCountAfter: verificationEvidence.wordCount,
614
- wordCountBefore: trigger.wordCount
615
- };
616
- if (!converged) {
617
- removePreset(preset.site);
618
- state.persistence = {
619
- persisted: false,
620
- reason: "not-converged"
621
- };
622
- return report(state, {
623
- preset,
624
- trigger,
625
- verification: verdict
626
- });
627
- }
628
- let secondVerification;
629
- if (secondHtml) {
630
- const second = extractText(secondHtml, args.baseUrl);
631
- const secondCanonicalSite = normalizeSiteKey(second.metadata.canonical);
632
- if (secondCanonicalSite && secondCanonicalSite !== site) {
633
- removePreset(preset.site);
634
- state.persistence = {
635
- persisted: false,
636
- reason: "second-page-canonical-mismatch"
637
- };
638
- return report(state, {
639
- preset,
640
- trigger,
641
- verification: verdict
642
- });
643
- }
644
- const secondEvidence = assessLostSignal({
645
- contentText: second.content,
646
- fallbackUsed: second.diagnostics.fallbackUsed ?? false,
647
- gated: second.diagnostics.gated
648
- });
649
- const secondApplied = second.presetSignal?.applied === true;
650
- const secondConverged = secondApplied && secondEvidence.reasons.length === 0;
651
- secondVerification = {
652
- applied: secondApplied,
653
- converged: secondConverged,
654
- wordCountAfter: secondEvidence.wordCount
655
- };
656
- if (!secondConverged) {
657
- removePreset(preset.site);
658
- state.persistence = {
659
- persisted: false,
660
- reason: "second-page-not-converged"
661
- };
662
- return report(state, {
663
- preset,
664
- secondVerification,
665
- trigger,
666
- verification: verdict
667
- });
668
- }
669
- verdict.verifiedPages = 2;
670
- }
671
- state.persistence = savePreset(preset);
672
- return report(state, {
673
- preset,
674
- secondVerification,
675
- trigger,
676
- verification: verdict
677
- });
678
- }
679
- function report(state, parts) {
680
- const lines = [];
681
- if (!parts.trigger.fired) lines.push(parts.trigger.reasons.length > 0 ? `No suggest loop run: ${parts.trigger.reasons.join(", ")}.` : `No suggest loop run: the baseline extraction looks healthy (${parts.trigger.wordCount} words, no lost signal).`);
682
- else lines.push(`Lost signals on the baseline extraction (${parts.trigger.wordCount} words): ${parts.trigger.reasons.join(", ")}.`);
683
- if (state.calls > 0) lines.push(`Sampling: ${state.calls}/${state.budget} call(s)${state.exhausted ? " — budget exhausted" : ""}.`);
684
- if (parts.verification) lines.push(`Verification with the preset stored: applied=${parts.verification.applied}, converged=${parts.verification.converged}, ${parts.verification.wordCountAfter} words, pages verified: ${parts.verification.verifiedPages}.`);
685
- if (parts.secondVerification) lines.push(`Second-page verification: applied=${parts.secondVerification.applied}, converged=${parts.secondVerification.converged}, ${parts.secondVerification.wordCountAfter} words.`);
686
- if (parts.preset) lines.push(`Accepted preset for ${parts.preset.site}: include=${parts.preset.scope.include ?? "(none)"}, exclude=${(parts.preset.scope.exclude ?? []).join(", ") || "(none)"}, detectors=${parts.preset.detectors.join(", ")}.`);
687
- if (state.persistence) lines.push(state.persistence.persisted ? `Persisted to ${state.persistence.path}.` : `Not persisted: ${state.persistence.reason}.`);
688
- return {
689
- content: [{
690
- text: lines.join("\n"),
691
- type: "text"
692
- }],
693
- structuredContent: {
694
- content: lines.join("\n"),
695
- persistence: state.persistence,
696
- preset: parts.preset,
697
- sampling: {
698
- budget: state.budget,
699
- budgetExhausted: state.exhausted,
700
- calls: state.calls
701
- },
702
- schemaVersion: 1,
703
- secondVerification: parts.secondVerification,
704
- trigger: parts.trigger,
705
- verification: parts.verification
706
- }
707
- };
708
- }
709
- function coreProposalUserText(site, title, evidence, outline, rejection) {
710
- return [
711
- `SITE: ${site}`,
712
- `PAGE: ${title || "(untitled)"}`,
713
- `BASELINE: ${evidence.wordCount} words; debris probes: ${evidence.debrisProbes.join(", ") || "none"}; gating: ${evidence.gatedReason ?? "none"}. The extraction lost to page junk — propose a preset.`,
714
- "",
715
- "MATERIAL — ancestor chains of every node holding at least 200 chars of its own text, hop format tag#id.classes[attr=\"value\"] (own:X, all:Y):",
716
- outline.text,
717
- ...outline.truncated ? ["(material truncated at the budget — the largest own-text chains come first)"] : [],
718
- ...rejection ? [
719
- "",
720
- "YOUR PREVIOUS PROPOSAL WAS REJECTED:",
721
- rejection,
722
- "Return corrected strict JSON."
723
- ] : [""],
724
- "Return JSON: {\"detectors\": [\"...\"], \"scope\": {\"include\": \"...\", \"exclude\": [\"...\"]}}"
725
- ].join("\n");
726
- }
727
- async function proposeCore(html, baseUrl, site, baseline, evidence, state, server) {
728
- const outline = buildChainOutline({
729
- baseUrl,
730
- cleanChrome: true,
731
- html,
732
- mode: "page"
733
- });
734
- let rejection;
735
- while (state.calls < state.budget) {
736
- state.calls += 1;
737
- let reply;
738
- try {
739
- reply = await sampleJson(server, {
740
- maxTokens: MAX_SAMPLING_TOKENS,
741
- systemPrompt: SUGGEST_SYSTEM_PROMPT,
742
- userText: coreProposalUserText(site, baseline.metadata.title ?? "", evidence, outline, rejection)
743
- });
744
- } catch (err) {
745
- if (err instanceof SuggestParseError) {
746
- rejection = err.message;
747
- continue;
748
- }
749
- throw err;
750
- }
751
- const parsed = proposalSchema.safeParse(reply);
752
- if (!parsed.success) {
753
- rejection = `the reply must parse as {"detectors": [...], "scope": {"include": "...", "exclude": [...]}} — ${parsed.error.issues[0]?.message}`;
754
- continue;
755
- }
756
- const violations = lintProposal({
757
- detectors: parsed.data.detectors,
758
- document: normalizedDocument(html, baseUrl),
759
- scope: parsed.data.scope
760
- });
761
- if (violations.length > 0) {
762
- rejection = renderViolations(violations);
763
- continue;
764
- }
765
- return {
766
- detectors: parsed.data.detectors,
767
- scope: {
768
- exclude: parsed.data.scope.exclude,
769
- include: parsed.data.scope.include
770
- }
771
- };
772
- }
773
- state.exhausted = true;
774
- }
775
- async function proposeExcludes(html, baseUrl, core, state, server) {
776
- if (state.calls >= state.budget) return core.scope.exclude ?? [];
777
- const outline = buildChainOutline({
778
- baseUrl,
779
- cleanChrome: true,
780
- html,
781
- mode: { scope: { include: core.scope.include } }
782
- });
783
- let rejection;
784
- while (state.calls < state.budget) {
785
- state.calls += 1;
786
- let reply;
787
- try {
788
- reply = await sampleJson(server, {
789
- maxTokens: MAX_SAMPLING_TOKENS,
790
- systemPrompt: SUGGEST_SYSTEM_PROMPT,
791
- userText: [
792
- `ACCEPTED SO FAR: {"detectors": ${JSON.stringify(core.detectors)}, "scope": {"include": ${JSON.stringify(core.scope.include)}}}`,
793
- "",
794
- "MATERIAL — text blocks remaining inside the included subtree after the include was applied, grouped by chain, smallest own text first, each with a text sample:",
795
- outline.text,
796
- ...rejection ? [
797
- "",
798
- "YOUR PREVIOUS PROPOSAL WAS REJECTED:",
799
- rejection,
800
- "Return corrected strict JSON."
801
- ] : [""],
802
- "Propose excludes for the debris groups (video players, carousels, caption figures, comment blocks, promos) — never for prose paragraphs. Return [] if none. Return JSON: {\"exclude\": [\"...\"]}"
803
- ].join("\n")
804
- });
805
- } catch (err) {
806
- if (err instanceof SuggestParseError) {
807
- rejection = err.message;
808
- continue;
809
- }
810
- throw err;
811
- }
812
- const parsed = excludeProposalSchema.safeParse(reply);
813
- if (!parsed.success) {
814
- rejection = `the reply must parse as {"exclude": ["..."]} — ${parsed.error.issues[0]?.message}`;
815
- continue;
816
- }
817
- const violations = lintProposal({
818
- detectors: core.detectors,
819
- document: normalizedDocument(html, baseUrl),
820
- scope: {
821
- exclude: parsed.data.exclude,
822
- include: core.scope.include
823
- }
824
- });
825
- if (violations.length > 0) {
826
- rejection = renderViolations(violations);
827
- continue;
828
- }
829
- const merged = [...core.scope.exclude ?? []];
830
- for (const selector of parsed.data.exclude) if (!merged.includes(selector)) merged.push(selector);
831
- return merged;
832
- }
833
- state.exhausted = true;
834
- return core.scope.exclude ?? [];
835
- }
836
- function renderViolations(violations) {
837
- return violations.map((violation) => `- ${violation.kind}: ${violation.selector} — ${violation.detail}`).join("\n");
838
- }
839
- function normalizedDocument(html, baseUrl) {
840
- const { document } = buildDocument(html, baseUrl);
841
- normalizeDocument(document, { cleanChrome: true });
842
- resolveLazyImages(document);
843
- return document;
844
- }
845
- function extractText(html, baseUrl) {
846
- const result = extractArticleFromHtml({
847
- baseUrl,
848
- cache: false,
849
- format: "text",
850
- html
851
- });
852
- if (result.isError || !result.structuredContent) throw new Error("extraction failed inside the suggest loop");
853
- const structured = result.structuredContent;
854
- return {
855
- content: structured.content,
856
- diagnostics: {
857
- fallbackUsed: structured.diagnostics.fallbackUsed,
858
- gated: structured.diagnostics.gated
859
- },
860
- metadata: structured.metadata,
861
- presetSignal: structured.diagnostics.preset
862
- };
863
- }
864
- //#endregion
865
- //#region src/sampling.ts
866
- var SUMMARIZE_SYSTEM_PROMPT = "Summarize the user-supplied text concisely while preserving its key points, entities, and any decisive conclusions. Output only the summary prose — no preamble, no headings unless the source had them.";
867
- var summarizeInputShape = {
868
- text: z.string().describe("The markdown or text to summarize — typically the output of `extract`, `extract_section`, `html_to_markdown`, or `chunk_text`. Passed through to the host model verbatim; the server does not parse or modify it."),
869
- maxTokens: z.number().int().min(1).describe("Ceiling on tokens the host model may generate, forwarded as `sampling/createMessage` maxTokens. Thinking counts toward it on thinking models, so a low value truncates the summary; the system prompt already asks for a concise one.").default(4096)
870
- };
871
- var summarizeInputSchema = z.object(summarizeInputShape);
872
- var SUMMARIZE_TOOL_DESCRIPTION = `Summarize text using the HOST's model via MCP \`sampling/createMessage\` — the server embeds no model and calls no provider directly. Hand it the output of \`extract\`, \`extract_section\`, \`html_to_markdown\`, or any markdown/text string; the host picks the model and may ask the user to approve the sampling request (human-in-the-loop per MCP). The tool is only listed when the connected client advertises the sampling capability.`;
873
- async function summarizeWithHost(server, args) {
874
- return sampleText$1(server, {
875
- maxTokens: args.maxTokens,
876
- systemPrompt: SUMMARIZE_SYSTEM_PROMPT,
877
- userText: args.text
878
- });
879
- }
880
- function registerSummarizeTool(server) {
881
- return server.registerTool("summarize", {
882
- title: "Summarize text using the host model",
883
- description: SUMMARIZE_TOOL_DESCRIPTION,
884
- inputSchema: summarizeInputShape
885
- }, async (rawArgs) => {
886
- const args = summarizeInputSchema.parse(rawArgs);
887
- try {
888
- return { content: [{
889
- type: "text",
890
- text: await summarizeWithHost(server, args)
891
- }] };
892
- } catch (err) {
893
- logger.error(`summarize failed: ${describeError(err)}`);
894
- return toErrorResult(err);
895
- }
896
- });
897
- }
898
- function registerSamplingTools(server) {
899
- return [registerSummarizeTool(server), registerSuggestPresetTool(server)];
900
- }
901
- //#endregion
902
- //#region src/tools/chunk_text.ts
903
- function renderChunkIndex(chunks) {
904
- if (chunks.length === 0) return "(no chunks emitted — input had no non-whitespace content)";
905
- return chunks.map((chunk) => {
906
- const head = chunk.headingContext ? ` [${chunk.headingContext}]` : "";
907
- return `## Chunk ${chunk.index}${head}\n\n${chunk.text}`;
908
- }).join("\n\n");
909
- }
910
- function chunkTextDocument(rawArgs) {
911
- const { text, maxTokens, overlap, strategy } = chunkTextInputSchema.parse(rawArgs);
912
- const chunks = chunkMarkdown(text, {
913
- maxTokens,
914
- overlap,
915
- strategy
916
- });
917
- const content = renderChunkIndex(chunks);
918
- return {
919
- content: [{
920
- text: content,
921
- type: "text"
922
- }],
923
- structuredContent: {
924
- schemaVersion: 1,
925
- content,
926
- chunks
927
- }
928
- };
929
- }
930
- var CHUNK_TEXT_TOOL_DESCRIPTION = `Split already-extracted text into token-bounded chunks for embedding/RAG. Each chunk carries its index, tokenCount (chars/4), and the nearest preceding markdown heading as headingContext. Operates on any text — pair with the \`chunk\` option on \`extract\` when you want chunks inline with the extraction. The server fetches nothing: \`text\` is the only input.`;
931
- function chunkTextHandler(args) {
932
- try {
933
- return chunkTextDocument(args);
934
- } catch (err) {
935
- logger.error(`chunk_text failed: ${describeError(err)}`);
936
- return toErrorResult(err);
937
- }
938
- }
939
- function registerChunkTextTool(server) {
940
- return server.registerTool("chunk_text", {
941
- title: "Chunk text for RAG/embedding",
942
- description: CHUNK_TEXT_TOOL_DESCRIPTION,
943
- inputSchema: chunkTextInputShape,
944
- outputSchema: chunkTextOutputShape
945
- }, chunkTextHandler);
946
- }
947
- //#endregion
948
- //#region src/policy/sibling-scan.ts
949
- var CONTAINER_TAGS = /* @__PURE__ */ new Set([
950
- "ARTICLE",
951
- "DIV",
952
- "MAIN",
953
- "OL",
954
- "SECTION",
955
- "TABLE",
956
- "TBODY",
957
- "UL"
958
- ]);
959
- var LANDMARK_CHROME_SELECTOR = "nav, header, footer, aside, [role=\"navigation\"], [role=\"banner\"], [role=\"contentinfo\"], [role=\"complementary\"], [role=\"search\"], [role=\"menu\"], [role=\"menubar\"]";
960
- function stripLandmarkChrome(document) {
961
- for (const el of document.querySelectorAll(LANDMARK_CHROME_SELECTOR)) el.remove();
962
- for (const el of document.querySelectorAll("script, style, template")) el.remove();
963
- }
964
- function shapeKey(el) {
965
- const raw = el.getAttribute("class");
966
- if (!raw) return el.tagName;
967
- const normalized = raw.trim().split(/\s+/).sort().join(" ");
968
- return normalized ? `${el.tagName}|${normalized}` : el.tagName;
969
- }
970
- function groupChildrenByShape(container) {
971
- const groups = /* @__PURE__ */ new Map();
972
- for (const child of Array.from(container.childNodes)) {
973
- if (!isElement(child)) continue;
974
- const key = shapeKey(child);
975
- const bucket = groups.get(key);
976
- if (bucket) bucket.push(child);
977
- else groups.set(key, [child]);
978
- }
979
- return groups;
980
- }
981
- function describeSelector(el, maxClasses) {
982
- const parts = [el.tagName.toLowerCase()];
983
- const id = el.getAttribute("id");
984
- if (id) parts.push(`#${id}`);
985
- const cls = el.getAttribute("class");
986
- if (cls) {
987
- let used = 0;
988
- for (const token of cls.trim().split(/\s+/)) {
989
- if (token) {
990
- parts.push(`.${token}`);
991
- used += 1;
992
- }
993
- if (maxClasses !== void 0 && used >= maxClasses) break;
994
- }
995
- }
996
- return parts.join("");
997
- }
998
- //#endregion
999
- //#region src/policy/explain.ts
1000
- var DEFAULT_SNAPSHOT_MAX = 4e3;
1001
- var DEFAULT_TOP_N = 5;
1002
- function readScored(el) {
1003
- const score = el.readability?.contentScore;
1004
- if (typeof score !== "number") return null;
1005
- return {
1006
- className: typeof el.className === "string" ? el.className : "",
1007
- id: el.id,
1008
- score,
1009
- tag: el.tagName,
1010
- textLength: el.textContent.trim().length,
1011
- selector: describeSelector(el, 2)
1012
- };
1013
- }
1014
- function truncateSnapshot(html, max) {
1015
- if (html.length <= max) return {
1016
- html,
1017
- truncated: false
1018
- };
1019
- return {
1020
- html: html.slice(0, max),
1021
- truncated: true
1022
- };
1023
- }
1024
- function buildExplainReport(options) {
1025
- const { html, selectors, snapshotMaxChars = DEFAULT_SNAPSHOT_MAX, topN = DEFAULT_TOP_N, baseUrl } = options;
1026
- const { document, window } = buildDocument(html, baseUrl);
1027
- const gating = detectGating(document);
1028
- const documentElementCount = document.querySelectorAll("*").length;
1029
- const normalizeCounts = normalizeDocument(document);
1030
- resolveLazyImages(document);
1031
- const pagination = detectPagination(document, baseUrl);
1032
- applySelectors(document, selectors);
1033
- const snapshot = truncateSnapshot(document.body.innerHTML, snapshotMaxChars);
1034
- const readerable = isReaderable(document);
1035
- const readabilityOptions = resolveReadabilityOptions({});
1036
- const clone = document.cloneNode(true);
1037
- const heldNodes = Array.from(clone.querySelectorAll("*"));
1038
- const article = new Readability(clone, readabilityOptions).parse();
1039
- const parseSucceeded = !!article?.content;
1040
- const scored = [];
1041
- for (const el of heldNodes) {
1042
- const entry = readScored(el);
1043
- if (entry) scored.push(entry);
1044
- }
1045
- scored.sort((a, b) => b.score - a.score);
1046
- const candidates = scored.slice(0, Math.max(0, topN));
1047
- const chosenRoot = scored[0] ?? null;
1048
- const diagnostics = assembleDiagnostics({
1049
- articleHtml: article?.content ?? "",
1050
- boilerplateRemoved: normalizeCounts.boilerplateRemoved,
1051
- chromeRemoved: normalizeCounts.chromeRemoved,
1052
- documentElementCount,
1053
- extractedNode: "readability",
1054
- fallbackUsed: false,
1055
- gated: gating,
1056
- pagination,
1057
- readerable,
1058
- window
1059
- });
1060
- return {
1061
- candidates,
1062
- chosenRoot,
1063
- fallbackUsed: diagnostics.fallbackUsed,
1064
- gating: diagnostics.gated,
1065
- pagination: diagnostics.pagination,
1066
- parseSucceeded,
1067
- readerable: diagnostics.readerable ?? false,
1068
- removedNodes: {
1069
- boilerplate: diagnostics.boilerplateRemoved ?? 0,
1070
- chrome: diagnostics.chromeRemoved ?? 0,
1071
- total: diagnostics.removedNodes ?? 0
1072
- },
1073
- snapshot
1074
- };
1075
- }
1076
- //#endregion
1077
- //#region src/tools/explain.ts
1078
- var explainInputShape = {
1079
- localPath: localPathField,
1080
- baseUrl: z.url().describe("Base URL for absolutizing relative links during pagination/gating detection. NEVER fetched — origin context only.").optional(),
1081
- selectors: selectorsSchema,
1082
- topN: z.number().int().min(1).max(20).describe("Maximum number of scored candidate nodes to return (highest first). Default 5.").default(5)
1083
- };
1084
- var explainInputSchema = z.object(explainInputShape);
1085
- var DEFAULTS$4 = explainInputSchema.parse({ localPath: "" });
1086
- var candidateSchema = z.object({
1087
- className: z.string().describe("The candidate's class attribute (raw, unsplit). Empty string when absent."),
1088
- id: z.string().describe("The candidate's id attribute, or empty string."),
1089
- score: z.number().describe("Readability's actual contentScore for this node (link-density-scaled). Higher is better; the top entry is Readability's raw top candidate before its parent-walking/only-child adjustments."),
1090
- selector: z.string().describe("A CSS-ish hint (tag#id.class1.class2) for locating the node in the host DOM. NOT a unique locator — Readability's score lives on a JS expando invisible to CSS."),
1091
- tag: z.string().describe("Uppercase DOM tag name (e.g. \"ARTICLE\", \"MAIN\", \"DIV\")."),
1092
- textLength: z.number().int().min(0).describe("Trimmed textContent length of the candidate node, for eyeballing content density.")
1093
- }).describe("One scored candidate node Readability considered, with its real contentScore.");
1094
- var explainOutputShape = {
1095
- schemaVersion: z.literal(1).describe("Structured-content schema version. Bumps only on breaking shape changes to this object."),
1096
- content: z.string().describe("Readable rendering of the report (chosen root, ranked candidates, removal counts, gating/pagination, snapshot head) so content[0].text is always scannable."),
1097
- chosenRoot: candidateSchema.nullable().describe("The highest-scoring candidate Readability computed — its raw top pick before parent-walking/only-child post-processing. Null when Readability scored nothing (e.g. empty input)."),
1098
- candidates: z.array(candidateSchema).describe("Scored candidate nodes (highest first), capped at topN. These are Readability's real contentScore values, not a self-computed heuristic."),
1099
- readerable: z.boolean().describe("Readability isProbablyReaderable verdict on the normalized document."),
1100
- parseSucceeded: z.boolean().describe("True when reader.parse() returned article content. False signals that `extract` would fall back to its selector cascade."),
1101
- fallbackUsed: z.boolean().describe("Always false for explain — this tool runs only the Readability path it is diagnosing, never the fallback cascade."),
1102
- gating: z.object({
1103
- likely: z.boolean().describe("True when heuristics strongly suggest the content is paywalled or truncated."),
1104
- reason: z.string().describe("Short label naming the detected signal (e.g. \"paywall overlay\").")
1105
- }).nullable().describe("Likely paywall / gating signal detected before normalization. Null when none."),
1106
- pagination: z.object({
1107
- type: z.enum(["infinite", "paginated"]).describe("Kind of pagination signal detected."),
1108
- nextUrl: z.string().optional().describe("Absolute URL of the detected next page (paginated only). Never fetched."),
1109
- selector: z.string().optional().describe("CSS selector of the load-more / infinite-scroll sentinel (infinite only).")
1110
- }).nullable().describe("Detected pagination / infinite-scroll signal. Null when none."),
1111
- removedNodes: z.object({
1112
- boilerplate: z.number().int().min(0).describe("Boilerplate blocks (related-posts, newsletter signup) stripped before Readability."),
1113
- chrome: z.number().int().min(0).describe("Browser-chrome nodes stripped (scrollbars, consent banners, overlays)."),
1114
- total: z.number().int().min(0).describe("Net element count removed across the whole pipeline (delta vs. the parsed document).")
1115
- }).describe("Breakdown of nodes removed before Readability saw the document, reused from the extract diagnostics path."),
1116
- snapshot: z.object({
1117
- html: z.string().describe("The sanitized-by-normalization (post chrome/boilerplate/script strip) HTML fed to Readability — \"what Readability saw\". Not DOMPurify-sanitized (that runs on Readability's output in `extract`), so it may still carry inline event handlers (`onerror`/`onclick`/…); it is diagnostic data — do not render verbatim."),
1118
- truncated: z.boolean().describe("True when the snapshot was cut at snapshotMaxChars (default 4000).")
1119
- }).describe("Pre-Readability HTML snapshot of the normalized document body.")
1120
- };
1121
- var explainOutput = z.object(explainOutputShape);
1122
- function formatGating(report) {
1123
- const g = report.gating;
1124
- if (!g) return "none";
1125
- return `${g.reason}${g.likely ? "" : " (weak)"}`;
1126
- }
1127
- function formatPagination(report) {
1128
- const p = report.pagination;
1129
- if (!p) return "none";
1130
- if (p.type === "paginated") return `paginated -> ${p.nextUrl ?? "(no href)"}`;
1131
- return `infinite (${p.selector ?? "sentinel"})`;
1132
- }
1133
- function renderText(report) {
1134
- const lines = [];
1135
- const root = report.chosenRoot;
1136
- lines.push(`readerable: ${report.readerable ? "yes" : "no"} parse: ${report.parseSucceeded ? "ok" : "fail"} fallback: ${report.fallbackUsed ? "yes" : "no"}`);
1137
- lines.push(`chosen root: ${root ? `${root.selector} (score ${root.score.toFixed(2)}, ${root.textLength} chars)` : "(no candidate scored)"}`);
1138
- lines.push(`top candidates (${report.candidates.length}):`);
1139
- if (report.candidates.length === 0) lines.push(" (none)");
1140
- report.candidates.forEach((c, i) => {
1141
- lines.push(` ${i + 1}. ${c.selector} score ${c.score.toFixed(2)} (${c.textLength} chars)`);
1142
- });
1143
- const r = report.removedNodes;
1144
- lines.push(`removed: total=${r.total} (chrome=${r.chrome}, boilerplate=${r.boilerplate})`);
1145
- lines.push(`gating: ${formatGating(report)}`);
1146
- lines.push(`pagination: ${formatPagination(report)}`);
1147
- lines.push(`snapshot (${report.snapshot.html.length} chars${report.snapshot.truncated ? ", truncated" : ""}):`);
1148
- lines.push(report.snapshot.html);
1149
- return lines.join("\n");
1150
- }
1151
- function explain(rawArgs) {
1152
- const { localPath, ...rest } = explainInputSchema.parse(rawArgs);
1153
- return explainFromHtml({
1154
- html: readHtmlFile(localPath),
1155
- ...rest
1156
- });
1157
- }
1158
- function explainFromHtml(input) {
1159
- const { html, baseUrl, selectors, topN } = {
1160
- ...DEFAULTS$4,
1161
- ...input
1162
- };
1163
- const report = buildExplainReport({
1164
- html,
1165
- selectors,
1166
- topN,
1167
- baseUrl
1168
- });
1169
- const content = renderText(report);
1170
- return {
1171
- content: [{
1172
- text: content,
1173
- type: "text"
1174
- }],
1175
- structuredContent: {
1176
- schemaVersion: 1,
1177
- content,
1178
- chosenRoot: report.chosenRoot,
1179
- candidates: report.candidates,
1180
- readerable: report.readerable,
1181
- parseSucceeded: report.parseSucceeded,
1182
- fallbackUsed: report.fallbackUsed,
1183
- gating: report.gating ?? null,
1184
- pagination: report.pagination ?? null,
1185
- removedNodes: report.removedNodes,
1186
- snapshot: report.snapshot
1187
- }
1188
- };
1189
- }
1190
- var EXPLAIN_TOOL_DESCRIPTION = `Post-mortem diagnostics for extraction: shows WHY Readability picked what it picked. Returns the chosen root, the ranked candidate nodes with their REAL Readability contentScore values (read off the DOM expando Readability stamps during scoring), a categorized removed-nodes breakdown, gating/pagination signals, and a snapshot of the normalized HTML fed to Readability. Runs the same normalize + Readability pipeline as \`extract\` (no fallback cascade, no Turndown). The server fetches nothing: \`localPath\` is the only source, and \`baseUrl\` (optional) is origin context only.`;
1191
- function explainHandler(args) {
1192
- try {
1193
- return explain(args);
1194
- } catch (err) {
1195
- logger.error(`explain failed: ${describeError(err)}`);
1196
- return toErrorResult(err);
1197
- }
1198
- }
1199
- function registerExplainTool(server) {
1200
- return server.registerTool("explain", {
1201
- title: "Explain why Readability picked what it picked",
1202
- description: EXPLAIN_TOOL_DESCRIPTION,
1203
- inputSchema: explainInputShape,
1204
- outputSchema: explainOutput
1205
- }, explainHandler);
1206
- }
1207
- //#endregion
1208
- //#region src/policy/grid-detector.ts
1209
- var MIN_ROWS = 3;
1210
- var HIGH_CONF_ROWS = 6;
1211
- var MIN_CELLS_PER_ROW = 2;
1212
- function rowCells(row) {
1213
- return Array.from(row.children).map(resolveCellText);
1214
- }
1215
- function notDetected$1(note) {
1216
- return {
1217
- confidence: "low",
1218
- containerSelector: "",
1219
- detected: false,
1220
- rowCount: 0,
1221
- colCount: 0,
1222
- rows: [],
1223
- rowTag: "",
1224
- note
1225
- };
1226
- }
1227
- function confidenceFor$1(dataRowCount) {
1228
- return dataRowCount >= HIGH_CONF_ROWS ? "high" : "medium";
1229
- }
1230
- function buildResult(raggedRows, rowTag, containerSelector, selectorHint, evidence) {
1231
- let maxCols = 0;
1232
- for (const row of raggedRows) if (row.length > maxCols) maxCols = row.length;
1233
- if (maxCols === 0) return notDetected$1(selectorHint ? `not a grid: rows matched ${selectorHint} but no cells were found` : "not a grid: rows matched but no cells were found");
1234
- const rowCount = raggedRows.length;
1235
- const dataRowCount = evidence?.dataRowCount ?? rowCount;
1236
- const headerCount = evidence?.headerCount ?? 0;
1237
- const rows = raggedRows.map((row) => ({ cells: Array.from({ length: maxCols }, (_, i) => row[i] ?? "") }));
1238
- const where = containerSelector || selectorHint || "document";
1239
- const descriptor = headerCount > 0 ? `detected ${dataRowCount} ${rowTag} data rows plus ${headerCount} header row${headerCount > 1 ? "s" : ""} (${maxCols} cols) in ${where}` : `detected ${dataRowCount} ${rowTag} rows (${maxCols} cols) in ${where}`;
1240
- return {
1241
- confidence: confidenceFor$1(dataRowCount),
1242
- containerSelector,
1243
- detected: true,
1244
- rowCount,
1245
- colCount: maxCols,
1246
- rows,
1247
- rowTag,
1248
- note: descriptor
1249
- };
1250
- }
1251
- function detectSelectorMode(document, rowSelector, cellSelector, minRows) {
1252
- const rowEls = Array.from(document.querySelectorAll(rowSelector));
1253
- if (rowEls.length < minRows) return notDetected$1(`not a grid: rowSelector "${rowSelector}" matched ${rowEls.length} row(s) (min ${minRows})`);
1254
- return buildResult(rowEls.map((row) => Array.from(row.querySelectorAll(cellSelector)).map(resolveCellText)), rowEls[0].tagName, "", rowSelector);
1255
- }
1256
- function modalWidth(members) {
1257
- const counts = /* @__PURE__ */ new Map();
1258
- for (const member of members) {
1259
- const width = member.children.length;
1260
- counts.set(width, (counts.get(width) ?? 0) + 1);
1261
- }
1262
- let best = members[0].children.length;
1263
- let bestCount = 0;
1264
- for (const [width, count] of counts) if (count > bestCount) {
1265
- best = width;
1266
- bestCount = count;
1267
- }
1268
- return best;
1269
- }
1270
- function classTokens(el) {
1271
- const cls = el.getAttribute("class");
1272
- return cls ? new Set(cls.trim().split(/\s+/)) : /* @__PURE__ */ new Set();
1273
- }
1274
- function looksLikeHeader(child, rowTag, dataClass, dataWidth) {
1275
- if (child.tagName !== rowTag || child.children.length !== dataWidth) return false;
1276
- if (child.getAttribute("role") === "row" && Array.from(child.children).some((c) => c.getAttribute("role") === "columnheader")) return true;
1277
- const childClass = classTokens(child);
1278
- return dataClass.size > 0 && [...dataClass].every((token) => childClass.has(token));
1279
- }
1280
- function collectHeaderRows(container, members) {
1281
- const memberSet = new Set(members);
1282
- const rowTag = members[0].tagName;
1283
- const dataWidth = modalWidth(members);
1284
- const dataClass = classTokens(members[0]);
1285
- const headers = [];
1286
- for (const child of Array.from(container.children)) {
1287
- if (memberSet.has(child)) break;
1288
- if (looksLikeHeader(child, rowTag, dataClass, dataWidth)) headers.push(rowCells(child));
1289
- }
1290
- return headers;
1291
- }
1292
- function detectAuto(document, minRows) {
1293
- stripLandmarkChrome(document);
1294
- const candidates = [];
1295
- for (const container of document.querySelectorAll("*")) {
1296
- if (!CONTAINER_TAGS.has(container.tagName)) continue;
1297
- const groups = groupChildrenByShape(container);
1298
- for (const members of groups.values()) {
1299
- if (members.length < minRows) continue;
1300
- if (!members.every((m) => m.children.length >= MIN_CELLS_PER_ROW)) continue;
1301
- const rows = members.map(rowCells);
1302
- const totalCellText = rows.reduce((sum, row) => sum + row.reduce((s, c) => s + c.length, 0), 0);
1303
- candidates.push({
1304
- container,
1305
- containerSelector: describeSelector(container),
1306
- members,
1307
- rows,
1308
- rowTag: members[0].tagName,
1309
- memberCount: members.length,
1310
- totalCellText
1311
- });
1312
- }
1313
- }
1314
- if (candidates.length === 0) return notDetected$1("not a grid: no repeating row structure (≥3 same-shape siblings each with ≥2 direct element-children, outside nav/header/footer/aside)");
1315
- let winner = candidates[0];
1316
- for (let i = 1; i < candidates.length; i++) {
1317
- const candidate = candidates[i];
1318
- if (candidate.memberCount > winner.memberCount || candidate.memberCount === winner.memberCount && candidate.totalCellText > winner.totalCellText) winner = candidate;
1319
- }
1320
- const headerRows = collectHeaderRows(winner.container, winner.members);
1321
- return buildResult([...headerRows, ...winner.rows], winner.rowTag, winner.containerSelector, void 0, {
1322
- dataRowCount: winner.memberCount,
1323
- headerCount: headerRows.length
1324
- });
1325
- }
1326
- function detectGrid(document, opts) {
1327
- const minRows = opts?.minRows ?? MIN_ROWS;
1328
- if (opts?.rowSelector && opts.cellSelector) return detectSelectorMode(document, opts.rowSelector, opts.cellSelector, minRows);
1329
- return detectAuto(document, minRows);
1330
- }
1331
- //#endregion
1332
- //#region src/tools/extract_grid.ts
1333
- var NO_GRID = "(no repeating grid found)";
1334
- function extractGrid(rawArgs) {
1335
- const { localPath, ...rest } = extractGridInputSchema.parse(rawArgs);
1336
- return extractGridFromHtml({
1337
- html: readHtmlFile(localPath),
1338
- ...rest
1339
- });
1340
- }
1341
- var DEFAULTS$3 = extractGridInputSchema.parse({ localPath: "" });
1342
- function extractGridFromHtml(input) {
1343
- const { html, baseUrl, format, selectors, rowSelector, cellSelector } = {
1344
- ...DEFAULTS$3,
1345
- ...input
1346
- };
1347
- const { document } = buildDocument(html, baseUrl);
1348
- applySelectors(document, selectors);
1349
- const result = detectGrid(document, {
1350
- rowSelector,
1351
- cellSelector
1352
- });
1353
- let content = NO_GRID;
1354
- let grid = {
1355
- rows: 0,
1356
- cols: 0,
1357
- markdown: ""
1358
- };
1359
- if (result.detected && result.rows.length > 0) {
1360
- const matrix = result.rows.map((row) => [...row.cells]);
1361
- const markdown = renderTable(matrix, format);
1362
- content = markdown;
1363
- grid = {
1364
- rows: result.rowCount,
1365
- cols: result.colCount,
1366
- markdown
1367
- };
1368
- }
1369
- return {
1370
- content: [{
1371
- text: content,
1372
- type: "text"
1373
- }],
1374
- structuredContent: {
1375
- schemaVersion: 1,
1376
- content,
1377
- grid,
1378
- diagnostics: {
1379
- confidence: result.confidence,
1380
- containerSelector: result.containerSelector,
1381
- detected: result.detected,
1382
- rowCount: result.rowCount,
1383
- colCount: result.colCount,
1384
- rowTag: result.rowTag,
1385
- note: result.note
1386
- },
1387
- metadata: {
1388
- baseUrl,
1389
- format,
1390
- detected: result.detected
1391
- }
1392
- }
1393
- };
1394
- }
1395
- var EXTRACT_GRID_TOOL_DESCRIPTION = `Detect and extract a CSS-grid / div "table" from already-rendered (post-JavaScript) HTML — the div equivalent of \`extract_tables\` for SPAs that render data into repeating \`<div>\` rows instead of \`<table>\`. Supports auto-detect (find the container whose direct children form the largest same-shape sibling group of ≥3 rows, each row a set of ≥2 direct element-children) and explicit \`rowSelector\` + \`cellSelector\` selector mode (cells scoped to each row subtree). Renders the matrix through the SAME gfm/csv/json renderer as \`extract_tables\`. Runs no Readability, no Turndown, no sanitization — the server fetches nothing: \`localPath\` is the only source, and \`baseUrl\` (optional) is origin context only.`;
1396
- function extractGridHandler(args) {
1397
- try {
1398
- return extractGrid(args);
1399
- } catch (err) {
1400
- logger.error(`extract_grid failed: ${describeError(err)}`);
1401
- return toErrorResult(err);
1402
- }
1403
- }
1404
- function registerExtractGridTool(server) {
1405
- return server.registerTool("extract_grid", {
1406
- title: "Extract a CSS-grid / div table (the div equivalent of extract_tables)",
1407
- description: EXTRACT_GRID_TOOL_DESCRIPTION,
1408
- inputSchema: extractGridInputShape,
1409
- outputSchema: extractGridOutputShape
1410
- }, extractGridHandler);
1411
- }
1412
- //#endregion
1413
- //#region src/tools/extract_links.ts
1414
- var MAX_TEXT_LENGTH = 300;
1415
- function clipText(raw) {
1416
- const collapsed = raw.replace(/\s+/g, " ").trim();
1417
- return collapsed.length > MAX_TEXT_LENGTH ? `${collapsed.slice(0, MAX_TEXT_LENGTH)}…` : collapsed;
1418
- }
1419
- function isWebOrigin(parsed) {
1420
- return parsed.protocol === "http:" || parsed.protocol === "https:";
1421
- }
1422
- function resolveExternal(absolutizedHref, baseUrl) {
1423
- let parsedHref;
1424
- try {
1425
- parsedHref = new URL(absolutizedHref);
1426
- } catch {
1427
- return false;
1428
- }
1429
- if (!isWebOrigin(parsedHref)) return false;
1430
- try {
1431
- return new URL(baseUrl).origin !== parsedHref.origin;
1432
- } catch {
1433
- return false;
1434
- }
1435
- }
1436
- function pruneUnsafeRoots(document) {
1437
- for (const el of document.querySelectorAll("script, template")) el.remove();
1438
- }
1439
- function extractLinks(rawArgs) {
1440
- const { localPath, ...rest } = extractLinksInputSchema.parse(rawArgs);
1441
- return extractLinksFromHtml({
1442
- html: readHtmlFile(localPath),
1443
- ...rest
1444
- });
1445
- }
1446
- var DEFAULTS$2 = extractLinksInputSchema.parse({ localPath: "" });
1447
- function extractLinksFromHtml(input) {
1448
- const { html, baseUrl, sameOriginOnly, selectors } = {
1449
- ...DEFAULTS$2,
1450
- ...input
1451
- };
1452
- const { document } = buildDocument(html, baseUrl);
1453
- applySelectors(document, selectors);
1454
- pruneUnsafeRoots(document);
1455
- const links = [];
1456
- for (const anchor of document.querySelectorAll("a")) {
1457
- const rawHref = anchor.getAttribute("href");
1458
- if (!rawHref) continue;
1459
- const href = absolutize(rawHref, baseUrl);
1460
- const isExternal = baseUrl ? resolveExternal(href, baseUrl) : false;
1461
- if (sameOriginOnly && isExternal) continue;
1462
- links.push({
1463
- text: clipText(anchor.textContent),
1464
- href,
1465
- rel: anchor.getAttribute("rel") ?? "",
1466
- isExternal
1467
- });
1468
- }
1469
- const content = renderLinksIndex(links);
1470
- return {
1471
- content: [{
1472
- text: content,
1473
- type: "text"
1474
- }],
1475
- structuredContent: {
1476
- schemaVersion: 1,
1477
- content,
1478
- links,
1479
- metadata: { baseUrl }
1480
- }
1481
- };
1482
- }
1483
- function renderLinksIndex(links) {
1484
- if (links.length === 0) return "(no links found)";
1485
- return links.map((link) => `- [${link.text || "(no text)"}](${link.href})`).join("\n");
1486
- }
1487
- var EXTRACT_LINKS_TOOL_DESCRIPTION = `Return a structured list of anchor links from already-rendered (post-JavaScript) HTML — \`[{text, href, rel, isExternal}]\` in document order, hrefs absolutized against \`baseUrl\`. No Readability scoring, Turndown, or sanitization — links are gathered from the raw parsed DOM so nav/footer/main links survive. Pairs with chrome-devtools for crawl/navigation decisions. The server fetches nothing: \`localPath\` is the only source, and \`baseUrl\` is origin context only (never fetched).`;
1488
- function extractLinksHandler(args) {
1489
- try {
1490
- return extractLinks(args);
1491
- } catch (err) {
1492
- logger.error(`extract_links failed: ${describeError(err)}`);
1493
- return toErrorResult(err);
1494
- }
1495
- }
1496
- function registerExtractLinksTool(server) {
1497
- return server.registerTool("extract_links", {
1498
- title: "Extract anchor links",
1499
- description: EXTRACT_LINKS_TOOL_DESCRIPTION,
1500
- inputSchema: extractLinksInputShape,
1501
- outputSchema: extractLinksOutputShape
1502
- }, extractLinksHandler);
1503
- }
1504
- //#endregion
1505
- //#region src/policy/list-detector.ts
1506
- var MIN_ITEMS = 3;
1507
- var HIGH_CONF_ITEMS = 6;
1508
- var HIGH_CONF_AVG_SCORE = 30;
1509
- var MAX_SNIPPET_CHARS = 200;
1510
- function isNavigationHref(href) {
1511
- if (!href || href === "#") return false;
1512
- const lower = href.toLowerCase();
1513
- return !lower.startsWith("javascript:") && !lower.startsWith("mailto:") && !lower.startsWith("tel:");
1514
- }
1515
- function anchorText(anchor) {
1516
- return anchor.textContent.replace(/\s+/g, " ").trim();
1517
- }
1518
- function navigationAnchors(child) {
1519
- const anchors = [];
1520
- for (const el of child.querySelectorAll("a[href]")) {
1521
- const anchor = el;
1522
- if (!isNavigationHref(anchor.getAttribute("href") ?? "")) continue;
1523
- if (anchorText(anchor).length === 0) continue;
1524
- anchors.push(anchor);
1525
- }
1526
- return anchors;
1527
- }
1528
- function pickPrimaryAnchor(anchors) {
1529
- let primary = anchors[0];
1530
- let best = anchorText(primary).length;
1531
- for (let i = 1; i < anchors.length; i++) {
1532
- const len = anchorText(anchors[i]).length;
1533
- if (len > best) {
1534
- primary = anchors[i];
1535
- best = len;
1536
- }
1537
- }
1538
- return primary;
1539
- }
1540
- function clipSnippet(raw) {
1541
- const collapsed = raw.replace(/\s+/g, " ").trim();
1542
- if (collapsed.length <= MAX_SNIPPET_CHARS) return collapsed;
1543
- return `${collapsed.slice(0, MAX_SNIPPET_CHARS)}…`;
1544
- }
1545
- function scoreItem(textLength, linkTextLength, primaryAnchorTextLength) {
1546
- if (textLength === 0) return 0;
1547
- return primaryAnchorTextLength + Math.max(0, textLength - linkTextLength);
1548
- }
1549
- function extractItem(child, baseUrl) {
1550
- const anchors = navigationAnchors(child);
1551
- if (anchors.length === 0) return null;
1552
- const primary = pickPrimaryAnchor(anchors);
1553
- const title = anchorText(primary);
1554
- if (!title) return null;
1555
- const href = primary.getAttribute("href") ?? "";
1556
- const url = absolutize(href, baseUrl);
1557
- if (!url) return null;
1558
- const fullText = child.textContent.replace(/\s+/g, " ").trim();
1559
- const snippet = fullText === title ? "" : fullText.startsWith(title) ? clipSnippet(fullText.slice(title.length + 1)) : clipSnippet(fullText);
1560
- const linkTextLength = anchors.reduce((sum, anchor) => sum + anchorText(anchor).length, 0);
1561
- return {
1562
- score: scoreItem(fullText.length, linkTextLength, title.length),
1563
- snippet,
1564
- title,
1565
- url
1566
- };
1567
- }
1568
- function distinctPathnames(items) {
1569
- const paths = /* @__PURE__ */ new Set();
1570
- for (const item of items) try {
1571
- paths.add(new URL(item.url).pathname);
1572
- } catch {
1573
- paths.add(item.url);
1574
- }
1575
- return paths.size;
1576
- }
1577
- function collectCandidates(document, baseUrl) {
1578
- const candidates = [];
1579
- for (const container of document.querySelectorAll("*")) {
1580
- if (!CONTAINER_TAGS.has(container.tagName)) continue;
1581
- const groups = groupChildrenByShape(container);
1582
- for (const children of groups.values()) {
1583
- if (children.length < MIN_ITEMS) continue;
1584
- if (!children.every((child) => navigationAnchors(child).length > 0)) continue;
1585
- const items = [];
1586
- for (const child of children) {
1587
- const item = extractItem(child, baseUrl);
1588
- if (item) items.push(item);
1589
- }
1590
- if (items.length < MIN_ITEMS) continue;
1591
- candidates.push({
1592
- containerSelector: describeSelector(container),
1593
- distinctPathnames: distinctPathnames(items),
1594
- itemTag: children[0].tagName,
1595
- items,
1596
- totalScore: items.reduce((sum, item) => sum + item.score, 0)
1597
- });
1598
- }
1599
- }
1600
- return candidates;
1601
- }
1602
- function confidenceFor(items) {
1603
- if (items.length < MIN_ITEMS) return "low";
1604
- if (items.length >= HIGH_CONF_ITEMS) return items.reduce((sum, item) => sum + item.score, 0) / items.length >= HIGH_CONF_AVG_SCORE ? "high" : "medium";
1605
- return "medium";
1606
- }
1607
- function notDetected(note) {
1608
- return {
1609
- confidence: "low",
1610
- containerSelector: "",
1611
- detected: false,
1612
- itemCount: 0,
1613
- itemTag: "",
1614
- items: [],
1615
- note
1616
- };
1617
- }
1618
- function detectList(document, baseUrl) {
1619
- stripLandmarkChrome(document);
1620
- const candidates = collectCandidates(document, baseUrl);
1621
- if (candidates.length === 0) return notDetected("not a list: no repeated item structure with links (≥3 same-shape siblings each carrying an anchor, outside nav/header/footer/aside)");
1622
- let winner = candidates[0];
1623
- for (let i = 1; i < candidates.length; i++) {
1624
- const candidate = candidates[i];
1625
- if (candidate.distinctPathnames > winner.distinctPathnames || candidate.distinctPathnames === winner.distinctPathnames && candidate.items.length > winner.items.length || candidate.distinctPathnames === winner.distinctPathnames && candidate.items.length === winner.items.length && candidate.totalScore > winner.totalScore) winner = candidate;
1626
- }
1627
- if (winner.items.length < MIN_ITEMS) return notDetected("not a list: best candidate had fewer than 3 extracted items");
1628
- const items = winner.items;
1629
- return {
1630
- confidence: confidenceFor(items),
1631
- containerSelector: winner.containerSelector,
1632
- detected: true,
1633
- itemCount: items.length,
1634
- itemTag: winner.itemTag,
1635
- items,
1636
- note: `detected ${items.length} ${winner.itemTag} items in ${winner.containerSelector}`
1637
- };
1638
- }
1639
- //#endregion
1640
- //#region src/tools/extract_list.ts
1641
- var NOT_A_LIST = "not a list: no repeated item structure with links";
1642
- function renderItems(result) {
1643
- if (!result.detected || result.items.length === 0) return result.note || NOT_A_LIST;
1644
- return result.items.map((item, index) => {
1645
- const head = `${index + 1}. ${item.title} — ${item.url}`;
1646
- return item.snippet ? `${head}\n ${item.snippet}` : head;
1647
- }).join("\n");
1648
- }
1649
- function toStructuredItem(item) {
1650
- return {
1651
- score: item.score,
1652
- snippet: item.snippet,
1653
- title: item.title,
1654
- url: item.url
1655
- };
1656
- }
1657
- function extractList(rawArgs) {
1658
- const { localPath, ...rest } = extractListInputSchema.parse(rawArgs);
1659
- return extractListFromHtml({
1660
- html: readHtmlFile(localPath),
1661
- ...rest
1662
- });
1663
- }
1664
- function extractListFromHtml(input) {
1665
- const { html, baseUrl, selectors } = input;
1666
- const { document } = buildDocument(html, baseUrl);
1667
- applySelectors(document, selectors);
1668
- const result = detectList(document, baseUrl);
1669
- const content = renderItems(result);
1670
- return {
1671
- content: [{
1672
- text: content,
1673
- type: "text"
1674
- }],
1675
- structuredContent: {
1676
- schemaVersion: 1,
1677
- content,
1678
- items: result.items.map(toStructuredItem),
1679
- diagnostics: {
1680
- confidence: result.confidence,
1681
- containerSelector: result.containerSelector,
1682
- detected: result.detected,
1683
- itemCount: result.itemCount,
1684
- itemTag: result.itemTag,
1685
- note: result.note
1686
- },
1687
- metadata: { baseUrl }
1688
- }
1689
- };
1690
- }
1691
- var EXTRACT_LIST_TOOL_DESCRIPTION = `Detect and extract a list/feed/index structure from already-rendered (post-JavaScript) HTML — for HN-style, search-result, and blog-index pages that Readability cannot turn into one article. Returns \`{items: [{title, url, snippet, score}], diagnostics}\` instead of one article. Strips nav/header/footer/aside + ARIA chrome roles first (the false-positive guard so an article's nav menu doesn't look like a 4-item feed), then finds the container whose direct children form a same-shape sibling cluster of ≥3 elements each carrying a navigation anchor, and the cluster with the most items wins. No Readability, no Turndown, no sanitization. The server fetches nothing: \`localPath\` is the only source, and \`baseUrl\` (optional) is origin context for absolutizing item hrefs.`;
1692
- function extractListHandler(args) {
1693
- try {
1694
- return extractList(args);
1695
- } catch (err) {
1696
- logger.error(`extract_list failed: ${describeError(err)}`);
1697
- return toErrorResult(err);
1698
- }
1699
- }
1700
- function registerExtractListTool(server) {
1701
- return server.registerTool("extract_list", {
1702
- title: "Extract a list/feed (HN/search/blog-index pages)",
1703
- description: EXTRACT_LIST_TOOL_DESCRIPTION,
1704
- inputSchema: extractListInputShape,
1705
- outputSchema: extractListOutputShape
1706
- }, extractListHandler);
1707
- }
1708
- //#endregion
1709
- //#region src/tools/extract_metadata.ts
1710
- var BIBLIOGRAPHIC_KEYS = [
1711
- "title",
1712
- "byline",
1713
- "siteName",
1714
- "lang",
1715
- "publishedTime",
1716
- "excerpt",
1717
- "canonical",
1718
- "baseUrl"
1719
- ];
1720
- function pickBibliographic(metadata) {
1721
- const out = {};
1722
- for (const key of BIBLIOGRAPHIC_KEYS) {
1723
- const value = metadata[key];
1724
- if (value !== void 0) out[key] = value;
1725
- }
1726
- return out;
1727
- }
1728
- function renderMetadataLines(metadata) {
1729
- const lines = Object.entries(metadata).map(([key, value]) => `${key}: ${value}`);
1730
- return lines.length > 0 ? lines.join("\n") : "(no metadata found)";
1731
- }
1732
- function extractMetadataDocument(rawArgs) {
1733
- const { localPath, ...rest } = extractMetadataInputSchema.parse(rawArgs);
1734
- return extractMetadataDocumentFromHtml({
1735
- html: readHtmlFile(localPath),
1736
- ...rest
1737
- });
1738
- }
1739
- function extractMetadataDocumentFromHtml(input) {
1740
- const { html, baseUrl } = input;
1741
- const { document } = buildDocument(html, baseUrl);
1742
- normalizeDocument(document);
1743
- const metadata = pickBibliographic(resolveMetadata({
1744
- document,
1745
- textContent: "",
1746
- wordCount: 0,
1747
- readingTimeMin: 0,
1748
- baseUrl
1749
- }));
1750
- const content = renderMetadataLines(metadata);
1751
- return {
1752
- content: [{
1753
- text: content,
1754
- type: "text"
1755
- }],
1756
- structuredContent: {
1757
- schemaVersion: 1,
1758
- content,
1759
- metadata
1760
- }
1761
- };
1762
- }
1763
- var EXTRACT_METADATA_TOOL_DESCRIPTION = `Return only the bibliographic metadata (title, byline, siteName, lang, publishedTime, excerpt, canonical, baseUrl) of already-rendered (post-JavaScript) HTML without running Readability/Turndown — a fast pre-check for crawlers and citation. Resolves the same metadata cascade as \`extract\` (JSON-LD → OpenGraph → Twitter → <meta> → <time> → <title>), plus <link rel="canonical"> → og:url. The server fetches nothing: \`localPath\` is the only source, and \`baseUrl\` is origin context only (never fetched).`;
1764
- function extractMetadataHandler(args) {
1765
- try {
1766
- return extractMetadataDocument(args);
1767
- } catch (err) {
1768
- logger.error(`extract_metadata failed: ${describeError(err)}`);
1769
- return toErrorResult(err);
1770
- }
1771
- }
1772
- function registerExtractMetadataTool(server) {
1773
- return server.registerTool("extract_metadata", {
1774
- title: "Extract metadata only (no Readability)",
1775
- description: EXTRACT_METADATA_TOOL_DESCRIPTION,
1776
- inputSchema: extractMetadataInputShape,
1777
- outputSchema: extractMetadataOutputShape
1778
- }, extractMetadataHandler);
1779
- }
1780
- //#endregion
1781
- //#region src/policy/outline.ts
1782
- function normalizeHeadingText(raw) {
1783
- return raw.replace(/\s+/g, " ").trim().replace(/^#+|#+$/g, "").trim();
1784
- }
1785
- function slugify(text) {
1786
- return text.toLowerCase().replace(/[^\p{L}\p{M}\p{N}\p{Pc} -]/gu, "").replace(/ /g, "-") || "section";
1787
- }
1788
- function linkAnchor(heading) {
1789
- for (const link of heading.querySelectorAll("a[href^=\"#\"]")) {
1790
- const href = link.getAttribute("href");
1791
- if (!href) continue;
1792
- const fragment = href.slice(1);
1793
- if (fragment) return fragment;
1794
- }
1795
- }
1796
- function resolveCandidate(heading, text) {
1797
- const id = heading.getAttribute("id");
1798
- if (id) return {
1799
- anchor: id,
1800
- explicit: true
1801
- };
1802
- const link = linkAnchor(heading);
1803
- if (link) return {
1804
- anchor: link,
1805
- explicit: true
1806
- };
1807
- return {
1808
- anchor: slugify(text),
1809
- explicit: false
1810
- };
1811
- }
1812
- function dedupe(candidate, explicit, used) {
1813
- if (explicit || !used.has(candidate)) return candidate;
1814
- let suffix = 1;
1815
- while (used.has(`${candidate}-${suffix}`)) suffix++;
1816
- return `${candidate}-${suffix}`;
1817
- }
1818
- function resolveOutline(document) {
1819
- const entries = [];
1820
- const used = /* @__PURE__ */ new Set();
1821
- document.querySelectorAll("h1, h2, h3, h4, h5, h6").forEach((heading) => {
1822
- const rawText = heading.textContent;
1823
- if (!rawText.trim()) return;
1824
- const text = normalizeHeadingText(rawText);
1825
- const level = Number.parseInt(heading.tagName.slice(1), 10);
1826
- const { anchor: candidate, explicit } = resolveCandidate(heading, text);
1827
- const anchor = dedupe(candidate, explicit, used);
1828
- used.add(anchor);
1829
- entries.push({
1830
- anchor,
1831
- level,
1832
- text
1833
- });
1834
- });
1835
- return entries;
1836
- }
1837
- //#endregion
1838
- //#region src/policy/section.ts
1839
- function findHeading(document, query) {
1840
- const needle = normalizeHeadingText(query).toLowerCase();
1841
- if (!needle) return;
1842
- const headings = document.querySelectorAll("h1, h2, h3, h4, h5, h6");
1843
- let substring;
1844
- for (const heading of headings) {
1845
- const text = normalizeHeadingText(heading.textContent).toLowerCase();
1846
- if (!text) continue;
1847
- const level = Number.parseInt(heading.tagName.slice(1), 10);
1848
- if (text === needle) return {
1849
- heading,
1850
- level
1851
- };
1852
- if (!substring && text.includes(needle)) substring = {
1853
- heading,
1854
- level
1855
- };
1856
- }
1857
- return substring;
1858
- }
1859
- var FLOW_CONTAINERS = /* @__PURE__ */ new Set([
1860
- "ARTICLE",
1861
- "ASIDE",
1862
- "BLOCKQUOTE",
1863
- "BODY",
1864
- "DD",
1865
- "DETAILS",
1866
- "DT",
1867
- "FOOTER",
1868
- "HEADER",
1869
- "LI",
1870
- "MAIN",
1871
- "NAV",
1872
- "SECTION",
1873
- "TD"
1874
- ]);
1875
- function findFlowContainer(heading) {
1876
- let ancestor = heading.parentElement;
1877
- while (ancestor) {
1878
- if (FLOW_CONTAINERS.has(ancestor.tagName)) return ancestor;
1879
- ancestor = ancestor.parentElement;
1880
- }
1881
- return heading.ownerDocument.body;
1882
- }
1883
- function firstHeadingLevel(el) {
1884
- const direct = /^H([1-6])$/.exec(el.tagName);
1885
- if (direct) return Number.parseInt(direct[1], 10);
1886
- const inner = el.querySelector("h1, h2, h3, h4, h5, h6");
1887
- if (inner) return Number.parseInt(inner.tagName.slice(1), 10);
1888
- }
1889
- function scopeToHeading(document, headingText) {
1890
- const match = findHeading(document, headingText);
1891
- if (!match) return false;
1892
- const { heading, level } = match;
1893
- const container = findFlowContainer(heading);
1894
- let startChild = heading;
1895
- while (startChild.parentElement && startChild.parentElement !== container) startChild = startChild.parentElement;
1896
- const wrap = document.createElement("section");
1897
- wrap.setAttribute("data-rdrm-section-scope", "");
1898
- container.insertBefore(wrap, startChild);
1899
- let node = startChild;
1900
- while (node) {
1901
- const next = node.nextSibling;
1902
- wrap.appendChild(node);
1903
- if (next !== null && isElement(next)) {
1904
- const nextLevel = firstHeadingLevel(next);
1905
- if (nextLevel !== void 0 && nextLevel <= level) break;
1906
- }
1907
- node = next;
1908
- }
1909
- return true;
1910
- }
1911
- //#endregion
1912
- //#region src/tools/extract_section.ts
1913
- var SECTION_SCOPE_SELECTOR = "[data-rdrm-section-scope]";
1914
- function extractSection(rawArgs) {
1915
- const { localPath, ...rest } = extractSectionInputSchema.parse(rawArgs);
1916
- return extractSectionFromHtml({
1917
- html: readHtmlFile(localPath),
1918
- ...rest
1919
- });
1920
- }
1921
- function extractSectionFromHtml(input) {
1922
- const { html, baseUrl, selector, heading } = input;
1923
- if (selector !== void 0) return extractArticleFromHtml({
1924
- html,
1925
- baseUrl,
1926
- selectors: { include: selector },
1927
- resolvePreset: false
1928
- });
1929
- if (heading === void 0) throw new ExtractionError("Provide exactly one of `selector` or `heading`.");
1930
- const { document } = buildDocument(html, baseUrl);
1931
- normalizeDocument(document);
1932
- if (!scopeToHeading(document, heading)) throw new ExtractionError(`no heading matched: ${heading}`);
1933
- const scoped = `<!DOCTYPE html><html><head></head><body>${document.body.innerHTML}</body></html>`;
1934
- return extractArticleFromHtml({
1935
- html: scoped,
1936
- baseUrl,
1937
- selectors: { include: SECTION_SCOPE_SELECTOR },
1938
- resolvePreset: false
1939
- });
1940
- }
1941
- var EXTRACT_SECTION_TOOL_DESCRIPTION = `Extract one section of an already-rendered (post-JavaScript) HTML document and return its Markdown + metadata + diagnostics — a thin resolver over extract’s \`selectors.include\` path, not a new extractor. Pick the section by CSS \`selector\` (passed straight through) OR by \`heading\` text (case-insensitive, first match wins; the section spans from the matched heading to the next same-or-higher-level heading). Exactly one of \`selector\`/\`heading\` is required. The server fetches nothing: \`localPath\` is the only source, and \`baseUrl\` (optional) is origin context only (never fetched).`;
1942
- function extractSectionHandler(args) {
1943
- try {
1944
- return extractSection(args);
1945
- } catch (err) {
1946
- logger.error(`extract_section failed: ${describeError(err)}`);
1947
- return toErrorResult(err);
1948
- }
1949
- }
1950
- function registerExtractSectionTool(server) {
1951
- return server.registerTool("extract_section", {
1952
- title: "Extract one section by selector or heading",
1953
- description: EXTRACT_SECTION_TOOL_DESCRIPTION,
1954
- inputSchema: extractSectionInputShape,
1955
- outputSchema: outputSchemaShape
1956
- }, extractSectionHandler);
1957
- }
1958
- //#endregion
1959
- //#region src/tools/extract_tables.ts
1960
- var NO_TABLES = "(no tables found)";
1961
- function extractTables(rawArgs) {
1962
- const { localPath, ...rest } = extractTablesInputSchema.parse(rawArgs);
1963
- return extractTablesFromHtml({
1964
- html: readHtmlFile(localPath),
1965
- ...rest
1966
- });
1967
- }
1968
- var DEFAULTS$1 = extractTablesInputSchema.parse({ localPath: "" });
1969
- function extractTablesFromHtml(input) {
1970
- const { html, baseUrl, format, selectors } = {
1971
- ...DEFAULTS$1,
1972
- ...input
1973
- };
1974
- const { document } = buildDocument(html, baseUrl);
1975
- applySelectors(document, selectors);
1976
- const tables = [];
1977
- let index = 0;
1978
- for (const table of document.querySelectorAll("table")) {
1979
- const matrix = parseTableMatrix(table);
1980
- if (matrix.length === 0) continue;
1981
- const keys = format === "json" ? resolveHeaderKeys(table, matrix) : void 0;
1982
- const markdown = renderTable(matrix, format, keys);
1983
- tables.push({
1984
- index,
1985
- rows: matrix.length,
1986
- cols: matrix[0]?.length ?? 0,
1987
- markdown
1988
- });
1989
- index++;
1990
- }
1991
- const content = tables.length > 0 ? tables.map((entry) => entry.markdown).join("\n\n") : NO_TABLES;
1992
- return {
1993
- content: [{
1994
- text: content,
1995
- type: "text"
1996
- }],
1997
- structuredContent: {
1998
- schemaVersion: 1,
1999
- content,
2000
- tables,
2001
- metadata: {
2002
- baseUrl,
2003
- format,
2004
- tableCount: tables.length
2005
- }
2006
- }
2007
- };
2008
- }
2009
- var EXTRACT_TABLES_TOOL_DESCRIPTION = `Extract every <table> on the page from already-rendered (post-JavaScript) HTML and return each as GFM / CSV / JSON (caller picks). Runs no Readability, Turndown, or sanitization — a page-wide \`querySelectorAll('table')\` walk in front of the same rowspan/colspan-aware matrix serializer used by the \`tables\` option on \`extract\`. Captures tables outside the article body (nav, aside, boilerplate) that the \`tables\` option never sees. The server fetches nothing: \`localPath\` is the only source, and \`baseUrl\` (optional) is origin context only (never fetched).`;
2010
- function extractTablesHandler(args) {
2011
- try {
2012
- return extractTables(args);
2013
- } catch (err) {
2014
- logger.error(`extract_tables failed: ${describeError(err)}`);
2015
- return toErrorResult(err);
2016
- }
2017
- }
2018
- function registerExtractTablesTool(server) {
2019
- return server.registerTool("extract_tables", {
2020
- title: "Extract every table on the page",
2021
- description: EXTRACT_TABLES_TOOL_DESCRIPTION,
2022
- inputSchema: extractTablesInputShape,
2023
- outputSchema: extractTablesOutputShape
2024
- }, extractTablesHandler);
2025
- }
2026
- //#endregion
2027
- //#region src/tools/html_to_markdown.ts
2028
- var EXTRACTED_NODE = "fragment";
2029
- function htmlToMarkdown(rawArgs) {
2030
- const { localPath, ...rest } = htmlToMarkdownInputSchema.parse(rawArgs);
2031
- return htmlToMarkdownFromHtml({
2032
- html: readHtmlFile(localPath),
2033
- ...rest
2034
- });
2035
- }
2036
- var DEFAULTS = htmlToMarkdownInputSchema.parse({ localPath: "" });
2037
- function htmlToMarkdownFromHtml(input) {
2038
- const { html, baseUrl, selectors, format, metadataMode, gfm, headingStyle, codeBlockStyle, images, sanitize: shouldSanitize, maxChars, wordsPerMinute, cleanChrome, tables, debug } = {
2039
- ...DEFAULTS,
2040
- ...input
2041
- };
2042
- const trace = new TraceCollector(debug);
2043
- const { document, window } = buildDocument(html, baseUrl);
2044
- const { documentElementCount, normalizeCounts, imagesResolved, textContent, rawHtml } = trace.run("normalize", () => {
2045
- const documentElementCount = document.querySelectorAll("*").length;
2046
- const normalizeCounts = normalizeDocument(document, { cleanChrome });
2047
- const imagesResolved = resolveLazyImages(document);
2048
- applySelectors(document, selectors);
2049
- const body = document.body;
2050
- return {
2051
- documentElementCount,
2052
- imagesResolved,
2053
- normalizeCounts,
2054
- rawHtml: body.innerHTML,
2055
- textContent: body.textContent
2056
- };
2057
- });
2058
- const { html: sanitizedHtml, counts: sanitizeCounts } = trace.run("sanitize", () => {
2059
- if (!shouldSanitize) return {
2060
- counts: {
2061
- iframes: 0,
2062
- scripts: 0
2063
- },
2064
- html: rawHtml
2065
- };
2066
- const res = sanitizeHtml(rawHtml, window);
2067
- return {
2068
- counts: {
2069
- iframes: res.iframesRemoved,
2070
- scripts: res.scriptsRemoved
2071
- },
2072
- html: res.html
2073
- };
2074
- });
2075
- const markdown = trace.run("turndown", () => toMarkdown(sanitizedHtml, {
2076
- codeBlockStyle,
2077
- gfm,
2078
- headingStyle,
2079
- images,
2080
- tables,
2081
- baseUrl
2082
- }));
2083
- const { metadata } = trace.run("metadata", () => {
2084
- return { metadata: {
2085
- title: nonEmpty(document.body.querySelector("h1, h2, h3, h4, h5, h6")?.textContent),
2086
- baseUrl,
2087
- ...computeTextMetrics(textContent, wordsPerMinute)
2088
- } };
2089
- });
2090
- const sanitization = {
2091
- iframes: normalizeCounts.iframes + sanitizeCounts.iframes,
2092
- scripts: normalizeCounts.scripts + sanitizeCounts.scripts
2093
- };
2094
- const baseDiagnostics = assembleDiagnostics({
2095
- articleHtml: sanitizedHtml,
2096
- boilerplateRemoved: normalizeCounts.boilerplateRemoved,
2097
- chromeRemoved: normalizeCounts.chromeRemoved,
2098
- documentElementCount,
2099
- extractedNode: EXTRACTED_NODE,
2100
- fallbackUsed: true,
2101
- imagesResolved,
2102
- sanitization,
2103
- trace: trace.collect(),
2104
- truncated: false,
2105
- window
2106
- });
2107
- let payload = formatPayload({
2108
- diagnostics: baseDiagnostics,
2109
- format,
2110
- markdown,
2111
- metadata,
2112
- metadataMode,
2113
- sanitizedHtml,
2114
- textContent
2115
- });
2116
- let truncated = false;
2117
- if (maxChars !== void 0 && (format === "markdown" || format === "text")) {
2118
- const res = truncateMarkdown(payload, maxChars);
2119
- payload = res.text;
2120
- truncated = res.truncated;
2121
- }
2122
- const diagnostics = truncated ? {
2123
- ...baseDiagnostics,
2124
- truncated
2125
- } : baseDiagnostics;
2126
- return {
2127
- content: [{
2128
- text: payload,
2129
- type: "text"
2130
- }],
2131
- structuredContent: {
2132
- schemaVersion: 1,
2133
- content: payload,
2134
- metadata,
2135
- diagnostics
2136
- }
2137
- };
2138
- }
2139
- var HTML_TO_MARKDOWN_TOOL_DESCRIPTION = `Convert an arbitrary HTML fragment to Markdown WITHOUT Readability article extraction (e.g. a snippet already isolated via chrome-devtools). Same Turndown + DOMPurify path as \`extract\`. The server fetches nothing: \`localPath\` is the only source, and \`baseUrl\` (optional) absolutizes relative links.`;
2140
- function htmlToMarkdownHandler(args) {
2141
- try {
2142
- return htmlToMarkdown(args);
2143
- } catch (err) {
2144
- logger.error(`html_to_markdown failed: ${describeError(err)}`);
2145
- return toErrorResult(err);
2146
- }
2147
- }
2148
- function registerHtmlToMarkdownTool(server) {
2149
- return server.registerTool("html_to_markdown", {
2150
- title: "Convert HTML fragment to Markdown",
2151
- description: HTML_TO_MARKDOWN_TOOL_DESCRIPTION,
2152
- inputSchema: htmlToMarkdownInputShape,
2153
- outputSchema: outputSchemaShape
2154
- }, htmlToMarkdownHandler);
2155
- }
2156
- //#endregion
2157
- //#region src/tools/outline.ts
2158
- function renderOutlineToc(outline) {
2159
- if (outline.length === 0) return "(no headings found)";
2160
- return outline.map((entry) => `${" ".repeat(entry.level - 1)}- ${entry.text}`).join("\n");
2161
- }
2162
- function outlineDocument(rawArgs) {
2163
- const { localPath, ...rest } = outlineInputSchema.parse(rawArgs);
2164
- return outlineDocumentFromHtml({
2165
- html: readHtmlFile(localPath),
2166
- ...rest
2167
- });
2168
- }
2169
- function outlineDocumentFromHtml(input) {
2170
- const { html, baseUrl, selectors } = input;
2171
- const { document } = buildDocument(html, baseUrl);
2172
- normalizeDocument(document);
2173
- applySelectors(document, selectors);
2174
- const outline = resolveOutline(document);
2175
- const metadata = {
2176
- title: document.title.trim() || document.querySelector("h1")?.textContent.replace(/\s+/g, " ").trim() || void 0,
2177
- baseUrl
2178
- };
2179
- const content = renderOutlineToc(outline);
2180
- return {
2181
- content: [{
2182
- text: content,
2183
- type: "text"
2184
- }],
2185
- structuredContent: {
2186
- schemaVersion: 1,
2187
- content,
2188
- outline,
2189
- metadata
2190
- }
2191
- };
2192
- }
2193
- var OUTLINE_TOOL_DESCRIPTION = `Return the document outline (h1-h6 headings with stable anchor ids) of already-rendered (post-JavaScript) HTML as a cheap pre-check before full extraction. No Readability scoring, no Turndown, no sanitization — a pure heading walk. The server fetches nothing: \`localPath\` is the only source, and \`baseUrl\` is origin context only (never fetched).`;
2194
- function outlineHandler(args) {
2195
- try {
2196
- return outlineDocument(args);
2197
- } catch (err) {
2198
- logger.error(`outline failed: ${describeError(err)}`);
2199
- return toErrorResult(err);
2200
- }
2201
- }
2202
- function registerOutlineTool(server) {
2203
- return server.registerTool("outline", {
2204
- title: "Get document outline (heading TOC)",
2205
- description: OUTLINE_TOOL_DESCRIPTION,
2206
- inputSchema: outlineInputShape,
2207
- outputSchema: outlineOutputShape
2208
- }, outlineHandler);
2209
- }
2210
- //#endregion
2211
- //#region src/server.ts
2212
- function createMcpServer() {
2213
- const { name, version, title, description, instructions } = loadConfig();
2214
- return new McpServer({
2215
- name,
2216
- version,
2217
- title,
2218
- description
2219
- }, { instructions });
2220
- }
2221
- function registerTools(server) {
2222
- return [
2223
- registerChunkTextTool(server),
2224
- registerExplainTool(server),
2225
- registerExtractGridTool(server),
2226
- registerExtractLinksTool(server),
2227
- registerExtractListTool(server),
2228
- registerExtractTool(server),
2229
- registerExtractMetadataTool(server),
2230
- registerExtractSectionTool(server),
2231
- registerExtractTablesTool(server),
2232
- registerHtmlToMarkdownTool(server),
2233
- registerOutlineTool(server)
2234
- ];
2235
- }
2236
- function registerCapabilityGatedTools(server) {
2237
- if (!server.server.getClientCapabilities()?.sampling) return [];
2238
- return registerSamplingTools(server);
2239
- }
2240
- function createServer() {
2241
- const server = createMcpServer();
2242
- loadPresets();
2243
- registerTools(server);
2244
- registerResources(server);
2245
- server.server.oninitialized = () => {
2246
- registerCapabilityGatedTools(server);
2247
- };
2248
- return server;
2249
- }
2250
- //#endregion
2251
- //#region src/index.ts
2252
- if (process.argv[2] === "extract") import("./assets/cli-BE_Ml3rh.js").then((m) => m.runCli(process.argv.slice(2))).then((code) => {
2253
- process.exit(code);
2254
- }).catch((err) => {
2255
- process.stderr.write(`${describeError(err)}\n`);
2256
- process.exit(1);
2257
- });
2258
- else {
2259
- const server = createServer();
2260
- const transport = new StdioServerTransport();
2261
- await server.connect(transport);
2262
- shutdownOnSignals([server]);
2263
- }
2264
- //#endregion
2265
- export {};
2266
-
2267
- //# sourceMappingURL=index.js.map