@v1nvn/readability 0.44.0 → 0.45.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (132) hide show
  1. package/README.md +3 -3
  2. package/package.json +7 -9
  3. package/src/cli.js +99 -0
  4. package/src/cli.ts +134 -0
  5. package/src/config.js +29 -0
  6. package/src/config.ts +42 -0
  7. package/src/dev.js +55 -0
  8. package/src/dev.ts +77 -0
  9. package/src/errors.js +16 -0
  10. package/src/errors.ts +19 -0
  11. package/src/handler.js +4 -0
  12. package/src/handler.ts +8 -0
  13. package/src/host-sampling.js +57 -0
  14. package/src/host-sampling.ts +86 -0
  15. package/src/index.js +23 -0
  16. package/src/index.ts +27 -0
  17. package/src/logger.js +2 -0
  18. package/src/logger.ts +3 -0
  19. package/src/output/format.js +92 -0
  20. package/src/output/format.ts +132 -0
  21. package/src/pipeline/cell-text.js +51 -0
  22. package/src/pipeline/cell-text.ts +56 -0
  23. package/src/pipeline/context.js +1 -0
  24. package/src/pipeline/context.ts +88 -0
  25. package/src/pipeline/dom.js +16 -0
  26. package/src/pipeline/dom.ts +23 -0
  27. package/src/pipeline/footnotes.js +148 -0
  28. package/src/pipeline/footnotes.ts +181 -0
  29. package/src/pipeline/math.js +113 -0
  30. package/src/pipeline/math.ts +130 -0
  31. package/src/pipeline/normalize.js +507 -0
  32. package/src/pipeline/normalize.ts +594 -0
  33. package/src/pipeline/readability.js +10 -0
  34. package/src/pipeline/readability.ts +36 -0
  35. package/src/pipeline/resolver.js +113 -0
  36. package/src/pipeline/resolver.ts +143 -0
  37. package/src/pipeline/sanitize.js +26 -0
  38. package/src/pipeline/sanitize.ts +42 -0
  39. package/src/pipeline/tables.js +239 -0
  40. package/src/pipeline/tables.ts +286 -0
  41. package/src/pipeline/turndown.js +153 -0
  42. package/src/pipeline/turndown.ts +191 -0
  43. package/src/pipeline/urls.js +14 -0
  44. package/src/pipeline/urls.ts +13 -0
  45. package/src/policy/chunk.js +365 -0
  46. package/src/policy/chunk.ts +482 -0
  47. package/src/policy/diagnostics.js +53 -0
  48. package/src/policy/diagnostics.ts +96 -0
  49. package/src/policy/explain.js +104 -0
  50. package/src/policy/explain.ts +182 -0
  51. package/src/policy/fallback.js +68 -0
  52. package/src/policy/fallback.ts +110 -0
  53. package/src/policy/gating.js +90 -0
  54. package/src/policy/gating.ts +106 -0
  55. package/src/policy/grid-detector.js +207 -0
  56. package/src/policy/grid-detector.ts +313 -0
  57. package/src/policy/identifiers.js +62 -0
  58. package/src/policy/identifiers.ts +75 -0
  59. package/src/policy/images.js +49 -0
  60. package/src/policy/images.ts +64 -0
  61. package/src/policy/list-detector.js +232 -0
  62. package/src/policy/list-detector.ts +308 -0
  63. package/src/policy/lost-signal.js +44 -0
  64. package/src/policy/lost-signal.ts +69 -0
  65. package/src/policy/markdown.js +58 -0
  66. package/src/policy/markdown.ts +76 -0
  67. package/src/policy/metadata.js +216 -0
  68. package/src/policy/metadata.ts +312 -0
  69. package/src/policy/outline-chains.js +165 -0
  70. package/src/policy/outline-chains.ts +215 -0
  71. package/src/policy/outline.js +71 -0
  72. package/src/policy/outline.ts +89 -0
  73. package/src/policy/pagination.js +79 -0
  74. package/src/policy/pagination.ts +104 -0
  75. package/src/policy/presets.js +100 -0
  76. package/src/policy/presets.ts +142 -0
  77. package/src/policy/section.js +111 -0
  78. package/src/policy/section.ts +127 -0
  79. package/src/policy/selector-lint.js +143 -0
  80. package/src/policy/selector-lint.ts +186 -0
  81. package/src/policy/sibling-scan.js +92 -0
  82. package/src/policy/sibling-scan.ts +99 -0
  83. package/src/policy/text.js +18 -0
  84. package/src/policy/text.ts +36 -0
  85. package/src/policy/truncate.js +45 -0
  86. package/src/policy/truncate.ts +59 -0
  87. package/src/preset-cache.js +165 -0
  88. package/src/preset-cache.ts +221 -0
  89. package/src/resources.js +227 -0
  90. package/src/resources.ts +317 -0
  91. package/src/sampling.js +45 -0
  92. package/src/sampling.ts +74 -0
  93. package/src/server.js +73 -0
  94. package/src/server.ts +85 -0
  95. package/src/tools/chunk_text.js +40 -0
  96. package/src/tools/chunk_text.ts +65 -0
  97. package/src/tools/explain.js +200 -0
  98. package/src/tools/explain.ts +299 -0
  99. package/src/tools/extract.js +263 -0
  100. package/src/tools/extract.ts +368 -0
  101. package/src/tools/extract_grid.js +56 -0
  102. package/src/tools/extract_grid.ts +90 -0
  103. package/src/tools/extract_links.js +98 -0
  104. package/src/tools/extract_links.ts +138 -0
  105. package/src/tools/extract_list.js +66 -0
  106. package/src/tools/extract_list.ts +97 -0
  107. package/src/tools/extract_metadata.js +67 -0
  108. package/src/tools/extract_metadata.ts +102 -0
  109. package/src/tools/extract_section.js +56 -0
  110. package/src/tools/extract_section.ts +84 -0
  111. package/src/tools/extract_tables.js +59 -0
  112. package/src/tools/extract_tables.ts +101 -0
  113. package/src/tools/html-source.js +30 -0
  114. package/src/tools/html-source.ts +40 -0
  115. package/src/tools/html_to_markdown.js +113 -0
  116. package/src/tools/html_to_markdown.ts +180 -0
  117. package/src/tools/outline.js +47 -0
  118. package/src/tools/outline.ts +77 -0
  119. package/src/tools/output-schema.js +504 -0
  120. package/src/tools/output-schema.ts +715 -0
  121. package/src/tools/schemas.js +288 -0
  122. package/src/tools/schemas.ts +442 -0
  123. package/src/tools/suggest-preset.js +504 -0
  124. package/src/tools/suggest-preset.ts +708 -0
  125. package/src/types/jsdom.d.ts +15 -0
  126. package/src/types/turndown-plugin-gfm.d.ts +11 -0
  127. package/dist/assets/cli-Dfjg7dOQ.js +0 -76
  128. package/dist/assets/cli-Dfjg7dOQ.js.map +0 -1
  129. package/dist/assets/extract-CdDp-KwX.js +0 -2880
  130. package/dist/assets/extract-CdDp-KwX.js.map +0 -1
  131. package/dist/index.js +0 -2267
  132. package/dist/index.js.map +0 -1
@@ -0,0 +1,227 @@
1
+ // In-memory cache of extract results keyed by a volatility-normalized hash of
2
+ // the HTML plus an args fingerprint, exposed to clients as addressable MCP
3
+ // Resources (`readability://page/{hash}`). Re-renders that differ only in
4
+ // nonce/CSP/generated ids collapse to the same key; the original hash rides
5
+ // alongside each entry so a should-have-hit-but-didn't miss points at a
6
+ // normalizer bug rather than a genuinely different page.
7
+ import { ResourceTemplate } from '@modelcontextprotocol/sdk/server/mcp.js';
8
+ import { createHash } from 'node:crypto';
9
+ import { presetGeneration } from "./policy/presets.js";
10
+ const MAX_ENTRIES = 256;
11
+ const TTL_MS = 30 * 60 * 1000;
12
+ const RESOURCE_SCHEME = 'readability://page/';
13
+ const entries = new Map();
14
+ export function resetCache() {
15
+ entries.clear();
16
+ }
17
+ function evictExpired(now) {
18
+ for (const [key, entry] of entries) {
19
+ if (entry.expiresAt <= now) {
20
+ entries.delete(key);
21
+ }
22
+ }
23
+ }
24
+ // Bounded LRU: Map preserves insertion order, so re-insert on access to surface
25
+ // recently-used entries to the end and drop the oldest when over capacity.
26
+ function touch(key, entry) {
27
+ entries.delete(key);
28
+ entries.set(key, entry);
29
+ }
30
+ function pruneToMax() {
31
+ while (entries.size > MAX_ENTRIES) {
32
+ const oldest = entries.keys().next();
33
+ if (oldest.done) {
34
+ break;
35
+ }
36
+ entries.delete(oldest.value);
37
+ }
38
+ }
39
+ // --- Hashing ---------------------------------------------------------------
40
+ // Conservative volatility normalization — over-stripping collides distinct
41
+ // pages. Each rule targets a specific known-volatile source.
42
+ function normalizeForHash(html) {
43
+ let s = html;
44
+ // Non-JSON-LD scripts carry CSP nonces, build hashes, A/B test buckets —
45
+ // same page, different bytes on every render — so strip them. Preserve
46
+ // <script type="application/ld+json">: structured metadata is content, so
47
+ // a changed datePublished must bust the cache. Mirrors the extraction
48
+ // normalizer in pipeline/normalize.ts.
49
+ s = s.replace(/<script\b(?![^>]*type\s*=\s*["']application\/ld\+json["'])[^>]*>[\s\S]*?<\/script>/gi, '');
50
+ s = s.replace(/<script\b(?![^>]*type\s*=\s*["']application\/ld\+json["'])[^>]*\/?>/gi, '');
51
+ // CSP <meta http-equiv="Content-Security-Policy"> carries the per-response
52
+ // nonce directive — strip the whole tag.
53
+ s = s.replace(/<meta\b[^>]*http-equiv=["']?content-security-policy["']?[^>]*>/gi, '');
54
+ // Per-render nonce attributes on any tag (script, link, style, …).
55
+ s = s.replace(/\snonce\s*=\s*("[^"]*"|'[^']*'|[^\s>]+)/gi, '');
56
+ // Build-tool generated attribute names: Vue `data-v-1a2b3c4d`,
57
+ // CSS-modules `data-css-1a2b3c4d`, Svelte `data-svelte-1a2b3c4d`. The
58
+ // value-part is optional — Vue's compiler emits the bare `data-v-<hash>`
59
+ // form with no `="..."`.
60
+ s = s.replace(/\sdata-(?:v|css|svelte|h)-[a-z0-9]{6,}(?:\s*=\s*("[^"]*"|'[^']*'|[^\s>]+))?/gi, '');
61
+ // React useId / RSC / Next.js generated ids: id=":R1:", ':r1:',
62
+ // __next_internal_action_…, react internal identifiers.
63
+ s = s.replace(/\sid\s*=\s*(":[A-Za-z0-9_-]+:"|':[A-Za-z0-9_-]+:'|__next_[A-Za-z0-9_-]+)/g, '');
64
+ s = s.replace(/\sid\s*=\s*("react[A-Z]_[A-Za-z0-9_]+"|'react[A-Z]_[A-Za-z0-9_]+')/gi, '');
65
+ // Whitespace runs and indentation differ by minifier/pretty-printer.
66
+ s = s.replace(/\s+/g, ' ');
67
+ return s.trim();
68
+ }
69
+ function sha256(text) {
70
+ return createHash('sha256').update(text).digest('hex');
71
+ }
72
+ export function normalizedHashOf(html) {
73
+ return sha256(normalizeForHash(html));
74
+ }
75
+ export function originalHashOf(html) {
76
+ return sha256(html.trim());
77
+ }
78
+ // Output-affecting options participate in the fingerprint so the same page with
79
+ // format:'html' vs format:'markdown' does not collide. `baseUrl` absolutizes
80
+ // links in the output, so it must participate too. The preset generation rides
81
+ // in the fingerprint rather than the resolved scope: an entry extracted before
82
+ // a preset landed must never be served after it, and hashing the scope would
83
+ // need a second document parse while conflating applied and missed runs under
84
+ // one key.
85
+ function buildArgsFingerprint(args) {
86
+ const sel = args.selectors
87
+ ? {
88
+ exclude: args.selectors.exclude ?? [],
89
+ include: args.selectors.include ?? '',
90
+ }
91
+ : null;
92
+ const chunk = args.chunk
93
+ ? {
94
+ maxTokens: args.chunk.maxTokens,
95
+ overlap: args.chunk.overlap,
96
+ strategy: args.chunk.strategy,
97
+ }
98
+ : null;
99
+ const fp = {
100
+ chunk,
101
+ cleanChrome: args.cleanChrome,
102
+ codeBlockStyle: args.codeBlockStyle,
103
+ debug: args.debug,
104
+ extraction: args.extraction,
105
+ format: args.format,
106
+ gfm: args.gfm,
107
+ headingStyle: args.headingStyle,
108
+ imageInventory: args.imageInventory,
109
+ images: args.images,
110
+ keepClasses: args.keepClasses,
111
+ maxChars: args.maxChars ?? null,
112
+ maxNodes: args.maxNodes ?? null,
113
+ metadataMode: args.metadataMode,
114
+ minArticleLength: args.minArticleLength ?? null,
115
+ presetGeneration: presetGeneration(),
116
+ readabilityOverrides: args.readabilityOverrides ?? null,
117
+ sanitize: args.sanitize,
118
+ selectors: sel,
119
+ tables: args.tables ?? null,
120
+ baseUrl: args.baseUrl ?? null,
121
+ wordsPerMinute: args.wordsPerMinute,
122
+ };
123
+ return JSON.stringify(fp);
124
+ }
125
+ function computeHashes(html) {
126
+ return {
127
+ normalizedHash: normalizedHashOf(html),
128
+ originalHash: originalHashOf(html),
129
+ };
130
+ }
131
+ function combineKey(normalizedHash, argsFingerprint) {
132
+ return sha256(`${normalizedHash}:${argsFingerprint}`);
133
+ }
134
+ // --- Store API -------------------------------------------------------------
135
+ export function lookup(html, args) {
136
+ const now = Date.now();
137
+ evictExpired(now);
138
+ const { normalizedHash, originalHash } = computeHashes(html);
139
+ const cacheKey = combineKey(normalizedHash, buildArgsFingerprint(args));
140
+ const entry = entries.get(cacheKey);
141
+ if (!entry || entry.expiresAt <= now) {
142
+ if (entry) {
143
+ entries.delete(cacheKey);
144
+ }
145
+ return undefined;
146
+ }
147
+ touch(cacheKey, entry);
148
+ return { entry, normalizedHash, originalHash };
149
+ }
150
+ export function storeResult(html, args, result) {
151
+ const now = Date.now();
152
+ evictExpired(now);
153
+ const { normalizedHash, originalHash } = computeHashes(html);
154
+ const argsFingerprint = buildArgsFingerprint(args);
155
+ const cacheKey = combineKey(normalizedHash, argsFingerprint);
156
+ const entry = {
157
+ argsFingerprint,
158
+ cacheKey,
159
+ contentText: result.contentText,
160
+ expiresAt: now + TTL_MS,
161
+ normalizedHash,
162
+ originalHash,
163
+ structuredContent: result.structuredContent,
164
+ };
165
+ entries.set(cacheKey, entry);
166
+ pruneToMax();
167
+ return { normalizedHash, originalHash };
168
+ }
169
+ export function getEntryByHash(hash) {
170
+ for (const entry of entries.values()) {
171
+ if (entry.normalizedHash === hash || entry.cacheKey === hash) {
172
+ return entry;
173
+ }
174
+ }
175
+ return undefined;
176
+ }
177
+ export function listEntries() {
178
+ const now = Date.now();
179
+ evictExpired(now);
180
+ return [...entries.values()];
181
+ }
182
+ // --- Resource registration -------------------------------------------------
183
+ // "page-cache" names the registered template; the readability:// URI scheme
184
+ // identifies the cache namespace and {hash} is the combined cache key.
185
+ const PAGE_CACHE_TEMPLATE = new ResourceTemplate('readability://page/{hash}', {
186
+ // Dynamic set: the cache comes and goes, so we enumerate current entries.
187
+ list: () => ({
188
+ resources: listEntries().map(entry => ({
189
+ description: `Cached extract for normalized hash ${entry.normalizedHash.slice(0, 12)}…`,
190
+ mimeType: 'text/markdown',
191
+ name: entry.normalizedHash,
192
+ uri: `${RESOURCE_SCHEME}${entry.cacheKey}`,
193
+ })),
194
+ }),
195
+ });
196
+ export function registerCacheResources(server) {
197
+ return server.registerResource('page-cache', PAGE_CACHE_TEMPLATE, {
198
+ title: 'Cached page extractions',
199
+ description: 'Addressable cache of `extract` results called with cache:true. Each entry is keyed by the volatility-normalized hash of the HTML plus the output options; the URI path segment is the combined cache key.',
200
+ mimeType: 'text/markdown',
201
+ }, (uri, variables) => {
202
+ const hash = String(variables.hash);
203
+ const entry = getEntryByHash(hash);
204
+ if (!entry) {
205
+ return {
206
+ contents: [
207
+ {
208
+ uri: uri.href,
209
+ text: '',
210
+ },
211
+ ],
212
+ };
213
+ }
214
+ return {
215
+ contents: [
216
+ {
217
+ uri: uri.href,
218
+ mimeType: 'text/markdown',
219
+ text: entry.contentText,
220
+ },
221
+ ],
222
+ };
223
+ });
224
+ }
225
+ export function registerResources(server) {
226
+ return [registerCacheResources(server)];
227
+ }
@@ -0,0 +1,317 @@
1
+ // In-memory cache of extract results keyed by a volatility-normalized hash of
2
+ // the HTML plus an args fingerprint, exposed to clients as addressable MCP
3
+ // Resources (`readability://page/{hash}`). Re-renders that differ only in
4
+ // nonce/CSP/generated ids collapse to the same key; the original hash rides
5
+ // alongside each entry so a should-have-hit-but-didn't miss points at a
6
+ // normalizer bug rather than a genuinely different page.
7
+
8
+ import { ResourceTemplate } from '@modelcontextprotocol/sdk/server/mcp.js';
9
+ import { createHash } from 'node:crypto';
10
+
11
+ import type { ToolHandle } from './server.ts';
12
+ import type { ExtractFromHtmlInput } from './tools/schemas.ts';
13
+
14
+ import { presetGeneration } from './policy/presets.ts';
15
+
16
+ import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
17
+ import type {
18
+ ListResourcesResult,
19
+ ReadResourceResult,
20
+ } from '@modelcontextprotocol/sdk/types.js';
21
+
22
+ const MAX_ENTRIES = 256;
23
+ const TTL_MS = 30 * 60 * 1000;
24
+ const RESOURCE_SCHEME = 'readability://page/';
25
+
26
+ interface CacheEntry {
27
+ readonly argsFingerprint: string;
28
+ readonly cacheKey: string;
29
+ readonly contentText: string;
30
+ readonly expiresAt: number;
31
+ readonly normalizedHash: string;
32
+ readonly originalHash: string;
33
+ // Held opaquely: cloned on hit and returned as CallToolResult.structuredContent
34
+ // without re-validation, so a wider element type than the strict output schema
35
+ // (e.g. readonly trace arrays) is fine here.
36
+ readonly structuredContent: unknown;
37
+ }
38
+
39
+ interface CacheLookup {
40
+ readonly entry: CacheEntry;
41
+ readonly normalizedHash: string;
42
+ readonly originalHash: string;
43
+ }
44
+
45
+ const entries = new Map<string, CacheEntry>();
46
+
47
+ export function resetCache(): void {
48
+ entries.clear();
49
+ }
50
+
51
+ function evictExpired(now: number): void {
52
+ for (const [key, entry] of entries) {
53
+ if (entry.expiresAt <= now) {
54
+ entries.delete(key);
55
+ }
56
+ }
57
+ }
58
+
59
+ // Bounded LRU: Map preserves insertion order, so re-insert on access to surface
60
+ // recently-used entries to the end and drop the oldest when over capacity.
61
+ function touch(key: string, entry: CacheEntry): void {
62
+ entries.delete(key);
63
+ entries.set(key, entry);
64
+ }
65
+
66
+ function pruneToMax(): void {
67
+ while (entries.size > MAX_ENTRIES) {
68
+ const oldest = entries.keys().next();
69
+ if (oldest.done) {
70
+ break;
71
+ }
72
+ entries.delete(oldest.value);
73
+ }
74
+ }
75
+
76
+ // --- Hashing ---------------------------------------------------------------
77
+
78
+ // Conservative volatility normalization — over-stripping collides distinct
79
+ // pages. Each rule targets a specific known-volatile source.
80
+ function normalizeForHash(html: string): string {
81
+ let s = html;
82
+ // Non-JSON-LD scripts carry CSP nonces, build hashes, A/B test buckets —
83
+ // same page, different bytes on every render — so strip them. Preserve
84
+ // <script type="application/ld+json">: structured metadata is content, so
85
+ // a changed datePublished must bust the cache. Mirrors the extraction
86
+ // normalizer in pipeline/normalize.ts.
87
+ s = s.replace(
88
+ /<script\b(?![^>]*type\s*=\s*["']application\/ld\+json["'])[^>]*>[\s\S]*?<\/script>/gi,
89
+ '',
90
+ );
91
+ s = s.replace(
92
+ /<script\b(?![^>]*type\s*=\s*["']application\/ld\+json["'])[^>]*\/?>/gi,
93
+ '',
94
+ );
95
+ // CSP <meta http-equiv="Content-Security-Policy"> carries the per-response
96
+ // nonce directive — strip the whole tag.
97
+ s = s.replace(
98
+ /<meta\b[^>]*http-equiv=["']?content-security-policy["']?[^>]*>/gi,
99
+ '',
100
+ );
101
+ // Per-render nonce attributes on any tag (script, link, style, …).
102
+ s = s.replace(/\snonce\s*=\s*("[^"]*"|'[^']*'|[^\s>]+)/gi, '');
103
+ // Build-tool generated attribute names: Vue `data-v-1a2b3c4d`,
104
+ // CSS-modules `data-css-1a2b3c4d`, Svelte `data-svelte-1a2b3c4d`. The
105
+ // value-part is optional — Vue's compiler emits the bare `data-v-<hash>`
106
+ // form with no `="..."`.
107
+ s = s.replace(
108
+ /\sdata-(?:v|css|svelte|h)-[a-z0-9]{6,}(?:\s*=\s*("[^"]*"|'[^']*'|[^\s>]+))?/gi,
109
+ '',
110
+ );
111
+ // React useId / RSC / Next.js generated ids: id=":R1:", ':r1:',
112
+ // __next_internal_action_…, react internal identifiers.
113
+ s = s.replace(
114
+ /\sid\s*=\s*(":[A-Za-z0-9_-]+:"|':[A-Za-z0-9_-]+:'|__next_[A-Za-z0-9_-]+)/g,
115
+ '',
116
+ );
117
+ s = s.replace(
118
+ /\sid\s*=\s*("react[A-Z]_[A-Za-z0-9_]+"|'react[A-Z]_[A-Za-z0-9_]+')/gi,
119
+ '',
120
+ );
121
+ // Whitespace runs and indentation differ by minifier/pretty-printer.
122
+ s = s.replace(/\s+/g, ' ');
123
+ return s.trim();
124
+ }
125
+
126
+ function sha256(text: string): string {
127
+ return createHash('sha256').update(text).digest('hex');
128
+ }
129
+
130
+ export function normalizedHashOf(html: string): string {
131
+ return sha256(normalizeForHash(html));
132
+ }
133
+
134
+ export function originalHashOf(html: string): string {
135
+ return sha256(html.trim());
136
+ }
137
+
138
+ // Output-affecting options participate in the fingerprint so the same page with
139
+ // format:'html' vs format:'markdown' does not collide. `baseUrl` absolutizes
140
+ // links in the output, so it must participate too. The preset generation rides
141
+ // in the fingerprint rather than the resolved scope: an entry extracted before
142
+ // a preset landed must never be served after it, and hashing the scope would
143
+ // need a second document parse while conflating applied and missed runs under
144
+ // one key.
145
+ function buildArgsFingerprint(args: ExtractFromHtmlInput): string {
146
+ const sel = args.selectors
147
+ ? {
148
+ exclude: args.selectors.exclude ?? [],
149
+ include: args.selectors.include ?? '',
150
+ }
151
+ : null;
152
+ const chunk = args.chunk
153
+ ? {
154
+ maxTokens: args.chunk.maxTokens,
155
+ overlap: args.chunk.overlap,
156
+ strategy: args.chunk.strategy,
157
+ }
158
+ : null;
159
+ const fp = {
160
+ chunk,
161
+ cleanChrome: args.cleanChrome,
162
+ codeBlockStyle: args.codeBlockStyle,
163
+ debug: args.debug,
164
+ extraction: args.extraction,
165
+ format: args.format,
166
+ gfm: args.gfm,
167
+ headingStyle: args.headingStyle,
168
+ imageInventory: args.imageInventory,
169
+ images: args.images,
170
+ keepClasses: args.keepClasses,
171
+ maxChars: args.maxChars ?? null,
172
+ maxNodes: args.maxNodes ?? null,
173
+ metadataMode: args.metadataMode,
174
+ minArticleLength: args.minArticleLength ?? null,
175
+ presetGeneration: presetGeneration(),
176
+ readabilityOverrides: args.readabilityOverrides ?? null,
177
+ sanitize: args.sanitize,
178
+ selectors: sel,
179
+ tables: args.tables ?? null,
180
+ baseUrl: args.baseUrl ?? null,
181
+ wordsPerMinute: args.wordsPerMinute,
182
+ };
183
+ return JSON.stringify(fp);
184
+ }
185
+
186
+ function computeHashes(html: string): {
187
+ normalizedHash: string;
188
+ originalHash: string;
189
+ } {
190
+ return {
191
+ normalizedHash: normalizedHashOf(html),
192
+ originalHash: originalHashOf(html),
193
+ };
194
+ }
195
+
196
+ function combineKey(normalizedHash: string, argsFingerprint: string): string {
197
+ return sha256(`${normalizedHash}:${argsFingerprint}`);
198
+ }
199
+
200
+ // --- Store API -------------------------------------------------------------
201
+
202
+ export function lookup(
203
+ html: string,
204
+ args: ExtractFromHtmlInput,
205
+ ): CacheLookup | undefined {
206
+ const now = Date.now();
207
+ evictExpired(now);
208
+ const { normalizedHash, originalHash } = computeHashes(html);
209
+ const cacheKey = combineKey(normalizedHash, buildArgsFingerprint(args));
210
+ const entry = entries.get(cacheKey);
211
+ if (!entry || entry.expiresAt <= now) {
212
+ if (entry) {
213
+ entries.delete(cacheKey);
214
+ }
215
+ return undefined;
216
+ }
217
+ touch(cacheKey, entry);
218
+ return { entry, normalizedHash, originalHash };
219
+ }
220
+
221
+ export function storeResult(
222
+ html: string,
223
+ args: ExtractFromHtmlInput,
224
+ result: {
225
+ contentText: string;
226
+ structuredContent: unknown;
227
+ },
228
+ ): { normalizedHash: string; originalHash: string } {
229
+ const now = Date.now();
230
+ evictExpired(now);
231
+ const { normalizedHash, originalHash } = computeHashes(html);
232
+ const argsFingerprint = buildArgsFingerprint(args);
233
+ const cacheKey = combineKey(normalizedHash, argsFingerprint);
234
+ const entry: CacheEntry = {
235
+ argsFingerprint,
236
+ cacheKey,
237
+ contentText: result.contentText,
238
+ expiresAt: now + TTL_MS,
239
+ normalizedHash,
240
+ originalHash,
241
+ structuredContent: result.structuredContent,
242
+ };
243
+ entries.set(cacheKey, entry);
244
+ pruneToMax();
245
+ return { normalizedHash, originalHash };
246
+ }
247
+
248
+ export function getEntryByHash(hash: string): CacheEntry | undefined {
249
+ for (const entry of entries.values()) {
250
+ if (entry.normalizedHash === hash || entry.cacheKey === hash) {
251
+ return entry;
252
+ }
253
+ }
254
+ return undefined;
255
+ }
256
+
257
+ export function listEntries(): readonly CacheEntry[] {
258
+ const now = Date.now();
259
+ evictExpired(now);
260
+ return [...entries.values()];
261
+ }
262
+
263
+ // --- Resource registration -------------------------------------------------
264
+
265
+ // "page-cache" names the registered template; the readability:// URI scheme
266
+ // identifies the cache namespace and {hash} is the combined cache key.
267
+ const PAGE_CACHE_TEMPLATE = new ResourceTemplate('readability://page/{hash}', {
268
+ // Dynamic set: the cache comes and goes, so we enumerate current entries.
269
+ list: (): ListResourcesResult => ({
270
+ resources: listEntries().map(entry => ({
271
+ description: `Cached extract for normalized hash ${entry.normalizedHash.slice(0, 12)}…`,
272
+ mimeType: 'text/markdown',
273
+ name: entry.normalizedHash,
274
+ uri: `${RESOURCE_SCHEME}${entry.cacheKey}`,
275
+ })),
276
+ }),
277
+ });
278
+
279
+ export function registerCacheResources(server: McpServer): ToolHandle {
280
+ return server.registerResource(
281
+ 'page-cache',
282
+ PAGE_CACHE_TEMPLATE,
283
+ {
284
+ title: 'Cached page extractions',
285
+ description:
286
+ 'Addressable cache of `extract` results called with cache:true. Each entry is keyed by the volatility-normalized hash of the HTML plus the output options; the URI path segment is the combined cache key.',
287
+ mimeType: 'text/markdown',
288
+ },
289
+ (uri, variables): ReadResourceResult => {
290
+ const hash = String(variables.hash);
291
+ const entry = getEntryByHash(hash);
292
+ if (!entry) {
293
+ return {
294
+ contents: [
295
+ {
296
+ uri: uri.href,
297
+ text: '',
298
+ },
299
+ ],
300
+ };
301
+ }
302
+ return {
303
+ contents: [
304
+ {
305
+ uri: uri.href,
306
+ mimeType: 'text/markdown',
307
+ text: entry.contentText,
308
+ },
309
+ ],
310
+ };
311
+ },
312
+ );
313
+ }
314
+
315
+ export function registerResources(server: McpServer): ToolHandle[] {
316
+ return [registerCacheResources(server)];
317
+ }
@@ -0,0 +1,45 @@
1
+ // Optional, capability-gated tools backed by the HOST's model. The sampling
2
+ // seam itself lives in host-sampling.ts; this file owns the family's tool
3
+ // registrations.
4
+ import { z } from 'zod';
5
+ import { tool } from "./handler.js";
6
+ import { sampleText } from "./host-sampling.js";
7
+ import { registerSuggestPresetTool } from "./tools/suggest-preset.js";
8
+ const SUMMARIZE_SYSTEM_PROMPT = 'Summarize the user-supplied text concisely while preserving its key points, entities, and any decisive conclusions. Output only the summary prose — no preamble, no headings unless the source had them.';
9
+ const summarizeInputShape = {
10
+ text: z
11
+ .string()
12
+ .describe('The markdown or text to summarize — typically the output of `extract`, `extract_section`, `html_to_markdown`, or `chunk_text`. Passed through to the host model verbatim; the server does not parse or modify it.'),
13
+ maxTokens: z
14
+ .number()
15
+ .int()
16
+ .min(1)
17
+ .describe('Ceiling on tokens the host model may generate, forwarded as `sampling/createMessage` maxTokens. Thinking counts toward it on thinking models, so a low value truncates the summary; the system prompt already asks for a concise one.')
18
+ .default(4096),
19
+ };
20
+ const summarizeInputSchema = z.object(summarizeInputShape);
21
+ export const SUMMARIZE_TOOL_DESCRIPTION = `Summarize text using the HOST's model via MCP \`sampling/createMessage\` — the server embeds no model and calls no provider directly. Hand it the output of \`extract\`, \`extract_section\`, \`html_to_markdown\`, or any markdown/text string; the host picks the model and may ask the user to approve the sampling request (human-in-the-loop per MCP). The tool is only listed when the connected client advertises the sampling capability.`;
22
+ async function summarizeWithHost(server, args) {
23
+ return sampleText(server, {
24
+ maxTokens: args.maxTokens,
25
+ systemPrompt: SUMMARIZE_SYSTEM_PROMPT,
26
+ userText: args.text,
27
+ });
28
+ }
29
+ export function registerSummarizeTool(server) {
30
+ return server.registerTool('summarize', {
31
+ title: 'Summarize text using the host model',
32
+ description: SUMMARIZE_TOOL_DESCRIPTION,
33
+ inputSchema: summarizeInputShape,
34
+ }, tool('summarize', async (args) => {
35
+ const summary = await summarizeWithHost(server, args);
36
+ return {
37
+ content: [{ type: 'text', text: summary }],
38
+ };
39
+ }, summarizeInputSchema));
40
+ }
41
+ // Mirrors registerTools/registerResources so the dev reload
42
+ // loop and capability gate can treat sampling as one registration family.
43
+ export function registerSamplingTools(server) {
44
+ return [registerSummarizeTool(server), registerSuggestPresetTool(server)];
45
+ }
@@ -0,0 +1,74 @@
1
+ // Optional, capability-gated tools backed by the HOST's model. The sampling
2
+ // seam itself lives in host-sampling.ts; this file owns the family's tool
3
+ // registrations.
4
+
5
+ import { z } from 'zod';
6
+
7
+ import type { ToolHandle } from './server.ts';
8
+
9
+ import { tool } from './handler.ts';
10
+ import { sampleText } from './host-sampling.ts';
11
+ import { registerSuggestPresetTool } from './tools/suggest-preset.ts';
12
+
13
+ import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
14
+
15
+ const SUMMARIZE_SYSTEM_PROMPT =
16
+ 'Summarize the user-supplied text concisely while preserving its key points, entities, and any decisive conclusions. Output only the summary prose — no preamble, no headings unless the source had them.';
17
+
18
+ const summarizeInputShape = {
19
+ text: z
20
+ .string()
21
+ .describe(
22
+ 'The markdown or text to summarize — typically the output of `extract`, `extract_section`, `html_to_markdown`, or `chunk_text`. Passed through to the host model verbatim; the server does not parse or modify it.',
23
+ ),
24
+ maxTokens: z
25
+ .number()
26
+ .int()
27
+ .min(1)
28
+ .describe(
29
+ 'Ceiling on tokens the host model may generate, forwarded as `sampling/createMessage` maxTokens. Thinking counts toward it on thinking models, so a low value truncates the summary; the system prompt already asks for a concise one.',
30
+ )
31
+ .default(4096),
32
+ } as const;
33
+
34
+ const summarizeInputSchema = z.object(summarizeInputShape);
35
+
36
+ export const SUMMARIZE_TOOL_DESCRIPTION = `Summarize text using the HOST's model via MCP \`sampling/createMessage\` — the server embeds no model and calls no provider directly. Hand it the output of \`extract\`, \`extract_section\`, \`html_to_markdown\`, or any markdown/text string; the host picks the model and may ask the user to approve the sampling request (human-in-the-loop per MCP). The tool is only listed when the connected client advertises the sampling capability.`;
37
+
38
+ async function summarizeWithHost(
39
+ server: McpServer,
40
+ args: { maxTokens: number; text: string },
41
+ ): Promise<string> {
42
+ return sampleText(server, {
43
+ maxTokens: args.maxTokens,
44
+ systemPrompt: SUMMARIZE_SYSTEM_PROMPT,
45
+ userText: args.text,
46
+ });
47
+ }
48
+
49
+ export function registerSummarizeTool(server: McpServer): ToolHandle {
50
+ return server.registerTool(
51
+ 'summarize',
52
+ {
53
+ title: 'Summarize text using the host model',
54
+ description: SUMMARIZE_TOOL_DESCRIPTION,
55
+ inputSchema: summarizeInputShape,
56
+ },
57
+ tool(
58
+ 'summarize',
59
+ async args => {
60
+ const summary = await summarizeWithHost(server, args);
61
+ return {
62
+ content: [{ type: 'text', text: summary }],
63
+ };
64
+ },
65
+ summarizeInputSchema,
66
+ ),
67
+ );
68
+ }
69
+
70
+ // Mirrors registerTools/registerResources so the dev reload
71
+ // loop and capability gate can treat sampling as one registration family.
72
+ export function registerSamplingTools(server: McpServer): ToolHandle[] {
73
+ return [registerSummarizeTool(server), registerSuggestPresetTool(server)];
74
+ }