@j0hanz/fetch-url-mcp 1.4.0 → 1.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (215) hide show
  1. package/dist/cli.d.ts +2 -3
  2. package/dist/cli.js +1 -2
  3. package/dist/http/auth.d.ts +5 -3
  4. package/dist/http/auth.js +64 -15
  5. package/dist/http/health.d.ts +1 -2
  6. package/dist/http/health.js +7 -18
  7. package/dist/http/helpers.d.ts +3 -4
  8. package/dist/http/helpers.js +21 -21
  9. package/dist/http/native.d.ts +0 -1
  10. package/dist/http/native.js +34 -26
  11. package/dist/http/rate-limit.d.ts +0 -1
  12. package/dist/http/rate-limit.js +3 -4
  13. package/dist/index.d.ts +0 -1
  14. package/dist/index.js +17 -18
  15. package/dist/lib/{markdown-cleanup.d.ts → content.d.ts} +4 -2
  16. package/dist/lib/content.js +1356 -0
  17. package/dist/lib/core.d.ts +253 -0
  18. package/dist/lib/core.js +1228 -0
  19. package/dist/lib/{tool-pipeline.d.ts → fetch-pipeline.d.ts} +1 -2
  20. package/dist/lib/{tool-pipeline.js → fetch-pipeline.js} +10 -19
  21. package/dist/lib/{fetch.d.ts → http.d.ts} +7 -9
  22. package/dist/lib/{fetch.js → http.js} +706 -944
  23. package/dist/lib/mcp-tools.d.ts +28 -0
  24. package/dist/lib/mcp-tools.js +107 -0
  25. package/dist/lib/{tool-progress.d.ts → progress.d.ts} +0 -1
  26. package/dist/lib/{tool-progress.js → progress.js} +8 -13
  27. package/dist/lib/task-handlers.d.ts +5 -0
  28. package/dist/lib/{mcp.js → task-handlers.js} +56 -12
  29. package/dist/lib/url.d.ts +70 -0
  30. package/dist/lib/url.js +686 -0
  31. package/dist/lib/utils.d.ts +58 -0
  32. package/dist/lib/utils.js +304 -0
  33. package/dist/prompts/index.d.ts +0 -1
  34. package/dist/prompts/index.js +0 -1
  35. package/dist/resources/index.d.ts +0 -1
  36. package/dist/resources/index.js +74 -33
  37. package/dist/resources/instructions.d.ts +0 -1
  38. package/dist/resources/instructions.js +2 -2
  39. package/dist/schemas/inputs.d.ts +0 -1
  40. package/dist/schemas/inputs.js +2 -3
  41. package/dist/schemas/outputs.d.ts +0 -1
  42. package/dist/schemas/outputs.js +1 -2
  43. package/dist/server.d.ts +0 -1
  44. package/dist/server.js +16 -26
  45. package/dist/tasks/execution.d.ts +0 -1
  46. package/dist/tasks/execution.js +27 -24
  47. package/dist/tasks/manager.d.ts +7 -3
  48. package/dist/tasks/manager.js +53 -34
  49. package/dist/tasks/owner.d.ts +1 -2
  50. package/dist/tasks/owner.js +1 -2
  51. package/dist/tasks/tool-registry.d.ts +1 -2
  52. package/dist/tasks/tool-registry.js +0 -1
  53. package/dist/tools/fetch-url.d.ts +1 -2
  54. package/dist/tools/fetch-url.js +39 -31
  55. package/dist/tools/index.d.ts +0 -1
  56. package/dist/tools/index.js +0 -1
  57. package/dist/transform/html-translators.d.ts +1 -0
  58. package/dist/transform/html-translators.js +454 -0
  59. package/dist/transform/metadata.d.ts +4 -0
  60. package/dist/transform/metadata.js +183 -0
  61. package/dist/transform/transform.d.ts +0 -1
  62. package/dist/transform/transform.js +24 -641
  63. package/dist/transform/types.d.ts +9 -11
  64. package/dist/transform/types.js +0 -1
  65. package/dist/transform/worker-pool.d.ts +0 -1
  66. package/dist/transform/worker-pool.js +7 -16
  67. package/dist/transform/workers/shared.d.ts +0 -1
  68. package/dist/transform/workers/shared.js +1 -2
  69. package/dist/transform/workers/transform-child.d.ts +0 -1
  70. package/dist/transform/workers/transform-child.js +0 -1
  71. package/dist/transform/workers/transform-worker.d.ts +0 -1
  72. package/dist/transform/workers/transform-worker.js +0 -1
  73. package/package.json +6 -3
  74. package/dist/cli.d.ts.map +0 -1
  75. package/dist/cli.js.map +0 -1
  76. package/dist/http/auth.d.ts.map +0 -1
  77. package/dist/http/auth.js.map +0 -1
  78. package/dist/http/health.d.ts.map +0 -1
  79. package/dist/http/health.js.map +0 -1
  80. package/dist/http/helpers.d.ts.map +0 -1
  81. package/dist/http/helpers.js.map +0 -1
  82. package/dist/http/native.d.ts.map +0 -1
  83. package/dist/http/native.js.map +0 -1
  84. package/dist/http/rate-limit.d.ts.map +0 -1
  85. package/dist/http/rate-limit.js.map +0 -1
  86. package/dist/index.d.ts.map +0 -1
  87. package/dist/index.js.map +0 -1
  88. package/dist/lib/cache.d.ts +0 -54
  89. package/dist/lib/cache.d.ts.map +0 -1
  90. package/dist/lib/cache.js +0 -264
  91. package/dist/lib/cache.js.map +0 -1
  92. package/dist/lib/config.d.ts +0 -143
  93. package/dist/lib/config.d.ts.map +0 -1
  94. package/dist/lib/config.js +0 -476
  95. package/dist/lib/config.js.map +0 -1
  96. package/dist/lib/crypto.d.ts +0 -4
  97. package/dist/lib/crypto.d.ts.map +0 -1
  98. package/dist/lib/crypto.js +0 -56
  99. package/dist/lib/crypto.js.map +0 -1
  100. package/dist/lib/dom-noise-removal.d.ts +0 -2
  101. package/dist/lib/dom-noise-removal.d.ts.map +0 -1
  102. package/dist/lib/dom-noise-removal.js +0 -494
  103. package/dist/lib/dom-noise-removal.js.map +0 -1
  104. package/dist/lib/download.d.ts +0 -4
  105. package/dist/lib/download.d.ts.map +0 -1
  106. package/dist/lib/download.js +0 -106
  107. package/dist/lib/download.js.map +0 -1
  108. package/dist/lib/errors.d.ts +0 -14
  109. package/dist/lib/errors.d.ts.map +0 -1
  110. package/dist/lib/errors.js +0 -72
  111. package/dist/lib/errors.js.map +0 -1
  112. package/dist/lib/fetch-content.d.ts +0 -5
  113. package/dist/lib/fetch-content.d.ts.map +0 -1
  114. package/dist/lib/fetch-content.js +0 -164
  115. package/dist/lib/fetch-content.js.map +0 -1
  116. package/dist/lib/fetch-stream.d.ts +0 -5
  117. package/dist/lib/fetch-stream.d.ts.map +0 -1
  118. package/dist/lib/fetch-stream.js +0 -29
  119. package/dist/lib/fetch-stream.js.map +0 -1
  120. package/dist/lib/fetch.d.ts.map +0 -1
  121. package/dist/lib/fetch.js.map +0 -1
  122. package/dist/lib/host-normalization.d.ts +0 -2
  123. package/dist/lib/host-normalization.d.ts.map +0 -1
  124. package/dist/lib/host-normalization.js +0 -91
  125. package/dist/lib/host-normalization.js.map +0 -1
  126. package/dist/lib/ip-blocklist.d.ts +0 -9
  127. package/dist/lib/ip-blocklist.d.ts.map +0 -1
  128. package/dist/lib/ip-blocklist.js +0 -79
  129. package/dist/lib/ip-blocklist.js.map +0 -1
  130. package/dist/lib/json.d.ts +0 -2
  131. package/dist/lib/json.d.ts.map +0 -1
  132. package/dist/lib/json.js +0 -45
  133. package/dist/lib/json.js.map +0 -1
  134. package/dist/lib/language-detection.d.ts +0 -3
  135. package/dist/lib/language-detection.d.ts.map +0 -1
  136. package/dist/lib/language-detection.js +0 -355
  137. package/dist/lib/language-detection.js.map +0 -1
  138. package/dist/lib/markdown-cleanup.d.ts.map +0 -1
  139. package/dist/lib/markdown-cleanup.js +0 -532
  140. package/dist/lib/markdown-cleanup.js.map +0 -1
  141. package/dist/lib/mcp-lifecycle.d.ts +0 -5
  142. package/dist/lib/mcp-lifecycle.d.ts.map +0 -1
  143. package/dist/lib/mcp-lifecycle.js +0 -51
  144. package/dist/lib/mcp-lifecycle.js.map +0 -1
  145. package/dist/lib/mcp-validator.d.ts +0 -17
  146. package/dist/lib/mcp-validator.d.ts.map +0 -1
  147. package/dist/lib/mcp-validator.js +0 -45
  148. package/dist/lib/mcp-validator.js.map +0 -1
  149. package/dist/lib/mcp.d.ts +0 -4
  150. package/dist/lib/mcp.d.ts.map +0 -1
  151. package/dist/lib/mcp.js.map +0 -1
  152. package/dist/lib/observability.d.ts +0 -23
  153. package/dist/lib/observability.d.ts.map +0 -1
  154. package/dist/lib/observability.js +0 -238
  155. package/dist/lib/observability.js.map +0 -1
  156. package/dist/lib/server-tuning.d.ts +0 -15
  157. package/dist/lib/server-tuning.d.ts.map +0 -1
  158. package/dist/lib/server-tuning.js +0 -49
  159. package/dist/lib/server-tuning.js.map +0 -1
  160. package/dist/lib/session.d.ts +0 -45
  161. package/dist/lib/session.d.ts.map +0 -1
  162. package/dist/lib/session.js +0 -263
  163. package/dist/lib/session.js.map +0 -1
  164. package/dist/lib/timer-utils.d.ts +0 -13
  165. package/dist/lib/timer-utils.d.ts.map +0 -1
  166. package/dist/lib/timer-utils.js +0 -44
  167. package/dist/lib/timer-utils.js.map +0 -1
  168. package/dist/lib/tool-errors.d.ts +0 -12
  169. package/dist/lib/tool-errors.d.ts.map +0 -1
  170. package/dist/lib/tool-errors.js +0 -55
  171. package/dist/lib/tool-errors.js.map +0 -1
  172. package/dist/lib/tool-pipeline.d.ts.map +0 -1
  173. package/dist/lib/tool-pipeline.js.map +0 -1
  174. package/dist/lib/tool-progress.d.ts.map +0 -1
  175. package/dist/lib/tool-progress.js.map +0 -1
  176. package/dist/lib/type-guards.d.ts +0 -16
  177. package/dist/lib/type-guards.d.ts.map +0 -1
  178. package/dist/lib/type-guards.js +0 -13
  179. package/dist/lib/type-guards.js.map +0 -1
  180. package/dist/prompts/index.d.ts.map +0 -1
  181. package/dist/prompts/index.js.map +0 -1
  182. package/dist/resources/index.d.ts.map +0 -1
  183. package/dist/resources/index.js.map +0 -1
  184. package/dist/resources/instructions.d.ts.map +0 -1
  185. package/dist/resources/instructions.js.map +0 -1
  186. package/dist/schemas/inputs.d.ts.map +0 -1
  187. package/dist/schemas/inputs.js.map +0 -1
  188. package/dist/schemas/outputs.d.ts.map +0 -1
  189. package/dist/schemas/outputs.js.map +0 -1
  190. package/dist/server.d.ts.map +0 -1
  191. package/dist/server.js.map +0 -1
  192. package/dist/tasks/execution.d.ts.map +0 -1
  193. package/dist/tasks/execution.js.map +0 -1
  194. package/dist/tasks/manager.d.ts.map +0 -1
  195. package/dist/tasks/manager.js.map +0 -1
  196. package/dist/tasks/owner.d.ts.map +0 -1
  197. package/dist/tasks/owner.js.map +0 -1
  198. package/dist/tasks/tool-registry.d.ts.map +0 -1
  199. package/dist/tasks/tool-registry.js.map +0 -1
  200. package/dist/tools/fetch-url.d.ts.map +0 -1
  201. package/dist/tools/fetch-url.js.map +0 -1
  202. package/dist/tools/index.d.ts.map +0 -1
  203. package/dist/tools/index.js.map +0 -1
  204. package/dist/transform/transform.d.ts.map +0 -1
  205. package/dist/transform/transform.js.map +0 -1
  206. package/dist/transform/types.d.ts.map +0 -1
  207. package/dist/transform/types.js.map +0 -1
  208. package/dist/transform/worker-pool.d.ts.map +0 -1
  209. package/dist/transform/worker-pool.js.map +0 -1
  210. package/dist/transform/workers/shared.d.ts.map +0 -1
  211. package/dist/transform/workers/shared.js.map +0 -1
  212. package/dist/transform/workers/transform-child.d.ts.map +0 -1
  213. package/dist/transform/workers/transform-child.js.map +0 -1
  214. package/dist/transform/workers/transform-worker.d.ts.map +0 -1
  215. package/dist/transform/workers/transform-worker.js.map +0 -1
@@ -1,494 +0,0 @@
1
- import { parseHTML } from 'linkedom';
2
- import { config } from './config.js';
3
- import { logDebug } from './observability.js';
4
- // --- Constants & Pre-compiled Regex ---
5
- const NOISE_SCAN_LIMIT = 50_000;
6
- const MIN_BODY_CONTENT_LENGTH = 100;
7
- const DIALOG_MIN_CHARS_FOR_PRESERVATION = 500;
8
- const NAV_FOOTER_MIN_CHARS_FOR_PRESERVATION = 500;
9
- // Merged markers for fast rejection
10
- const HTML_DOCUMENT_MARKERS = /<\s*(?:!doctype|html|head|body)\b/i;
11
- const HTML_FRAGMENT_MARKERS = /<\s*(?:article|main|section|div|nav|footer|header|aside|table|ul|ol)\b/i;
12
- // Split into smaller regexes to stay within sonarjs/regex-complexity limit
13
- const NOISE_PATTERNS = [
14
- /<\s*(?:script|style|noscript|iframe|nav|footer|header|form|button|input|select|textarea|svg|canvas)\b/i,
15
- /[\s"']role\s*=\s*['"]?(?:navigation|banner|complementary|contentinfo|tree|menubar|menu)['"]?/i,
16
- /[\s"'](?:aria-hidden\s*=\s*['"]?true['"]?|hidden)/i,
17
- /[\s"'](?:banner|promo|announcement|cta|advert|newsletter|subscribe|cookie|consent|popup|modal|overlay|toast)\b/i,
18
- /[\s"'](?:fixed|sticky|z-50|z-4|isolate|breadcrumb|pagination)\b/i,
19
- ];
20
- const HEADER_NOISE_PATTERN = /\b(site-header|masthead|topbar|navbar|nav(?:bar)?|menu|header-nav)\b/i;
21
- const FIXED_OR_HIGH_Z_PATTERN = /\b(?:fixed|sticky|z-(?:4\d|50)|isolate)\b/;
22
- const SKIP_URL_PREFIXES = [
23
- '#',
24
- 'java' + 'script:',
25
- 'mailto:',
26
- 'tel:',
27
- 'data:',
28
- 'blob:',
29
- ];
30
- const BASE_STRUCTURAL_TAGS = new Set([
31
- 'script',
32
- 'style',
33
- 'noscript',
34
- 'iframe',
35
- 'form',
36
- 'button',
37
- 'input',
38
- 'select',
39
- 'textarea',
40
- ]);
41
- const ALWAYS_NOISE_TAGS = new Set(['nav', 'footer']);
42
- const NAVIGATION_ROLES = new Set([
43
- 'navigation',
44
- 'banner',
45
- 'complementary',
46
- 'contentinfo',
47
- 'tree',
48
- 'menubar',
49
- 'menu',
50
- 'dialog',
51
- 'alertdialog',
52
- 'search',
53
- ]);
54
- const INTERACTIVE_CONTENT_ROLES = new Set([
55
- 'tabpanel',
56
- 'tab',
57
- 'tablist',
58
- 'dialog',
59
- 'alertdialog',
60
- 'menu',
61
- 'menuitem',
62
- 'option',
63
- 'listbox',
64
- 'combobox',
65
- 'tooltip',
66
- 'alert',
67
- ]);
68
- const PROMO_TOKENS_ALWAYS = [
69
- 'banner',
70
- 'promo',
71
- 'announcement',
72
- 'cta',
73
- 'advert',
74
- 'ads',
75
- 'sponsor',
76
- 'recommend',
77
- 'breadcrumb',
78
- 'pagination',
79
- 'pager',
80
- 'taglist',
81
- ];
82
- const PROMO_TOKENS_AGGRESSIVE = ['ad', 'related', 'comment'];
83
- const PROMO_TOKENS_BY_CATEGORY = {
84
- 'cookie-banners': ['cookie', 'consent', 'popup', 'modal', 'overlay', 'toast'],
85
- newsletters: ['newsletter', 'subscribe'],
86
- 'social-share': ['share', 'social'],
87
- };
88
- const BASE_NOISE_SELECTORS = {
89
- navFooter: 'nav,footer,header[class*="site"],header[class*="nav"],header[class*="menu"],[role="banner"],[role="navigation"]',
90
- cookieBanners: '[role="dialog"]',
91
- hidden: '[style*="display: none"],[style*="display:none"],[hidden],[aria-hidden="true"]',
92
- };
93
- const NO_MATCH_REGEX = /a^/i;
94
- // --- State Cache ---
95
- let cachedContext;
96
- let lastConfigRef;
97
- // --- Helpers Inlined/Optimized ---
98
- function escapeRegexLiteral(value) {
99
- return value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
100
- }
101
- function buildTokenRegex(tokens) {
102
- if (tokens.size === 0)
103
- return NO_MATCH_REGEX;
104
- return new RegExp(`(?:^|[^a-z0-9])(?:${[...tokens].map(escapeRegexLiteral).join('|')})(?:$|[^a-z0-9])`, 'i');
105
- }
106
- function addTokens(target, tokens) {
107
- for (const token of tokens)
108
- target.add(token);
109
- }
110
- function getPromoMatchers(currentConfig, flags) {
111
- const baseTokens = new Set(PROMO_TOKENS_ALWAYS);
112
- const aggressiveTokens = new Set();
113
- if (currentConfig.aggressiveMode) {
114
- addTokens(aggressiveTokens, PROMO_TOKENS_AGGRESSIVE);
115
- }
116
- if (flags.cookieBanners) {
117
- addTokens(baseTokens, PROMO_TOKENS_BY_CATEGORY['cookie-banners']);
118
- }
119
- if (flags.newsletters) {
120
- addTokens(baseTokens, PROMO_TOKENS_BY_CATEGORY['newsletters']);
121
- }
122
- if (flags.socialShare) {
123
- addTokens(baseTokens, PROMO_TOKENS_BY_CATEGORY['social-share']);
124
- }
125
- for (const t of currentConfig.extraTokens) {
126
- const n = t.toLowerCase().trim();
127
- if (n)
128
- baseTokens.add(n);
129
- }
130
- return {
131
- base: buildTokenRegex(baseTokens),
132
- aggressive: buildTokenRegex(aggressiveTokens),
133
- };
134
- }
135
- function getContext() {
136
- const currentConfig = config.noiseRemoval;
137
- if (cachedContext !== undefined && lastConfigRef === currentConfig)
138
- return cachedContext;
139
- const enabled = new Set(currentConfig.enabledCategories
140
- .map((c) => {
141
- const s = c.toLowerCase().trim();
142
- const { locale } = config.i18n;
143
- return locale ? s.toLocaleLowerCase(locale) : s;
144
- })
145
- .filter(Boolean));
146
- const isEnabled = (cat) => enabled.has(cat);
147
- const flags = {
148
- navFooter: isEnabled('nav-footer'),
149
- cookieBanners: isEnabled('cookie-banners'),
150
- newsletters: isEnabled('newsletters'),
151
- socialShare: isEnabled('social-share'),
152
- };
153
- const structuralTags = new Set(BASE_STRUCTURAL_TAGS);
154
- if (!currentConfig.preserveSvgCanvas) {
155
- structuralTags.add('svg');
156
- structuralTags.add('canvas');
157
- }
158
- const promoMatchers = getPromoMatchers(currentConfig, flags);
159
- const extraSelectors = currentConfig.extraSelectors
160
- .map((s) => s.trim())
161
- .filter((s) => s.length > 0);
162
- // Pre-build selectors
163
- const selectors = [BASE_NOISE_SELECTORS.hidden];
164
- if (flags.navFooter)
165
- selectors.push(BASE_NOISE_SELECTORS.navFooter);
166
- if (flags.cookieBanners)
167
- selectors.push(BASE_NOISE_SELECTORS.cookieBanners);
168
- const baseSelector = selectors.join(',');
169
- const candidateSelector = [
170
- ...structuralTags,
171
- ...ALWAYS_NOISE_TAGS,
172
- 'aside',
173
- 'header',
174
- '[class]',
175
- '[id]',
176
- '[role]',
177
- '[style]',
178
- ].join(',');
179
- cachedContext = {
180
- flags,
181
- structuralTags,
182
- weights: currentConfig.weights,
183
- promoMatchers,
184
- promoEnabled: flags.cookieBanners || flags.newsletters || flags.socialShare,
185
- extraSelectors,
186
- baseSelector,
187
- candidateSelector,
188
- };
189
- lastConfigRef = currentConfig;
190
- return cachedContext;
191
- }
192
- // --- Hot Path Logic ---
193
- function isInteractive(element, role) {
194
- if (role && INTERACTIVE_CONTENT_ROLES.has(role))
195
- return true;
196
- const ds = element.getAttribute('data-state');
197
- if (ds === 'inactive' || ds === 'closed')
198
- return true;
199
- const dataOrientation = element.getAttribute('data-orientation');
200
- if (dataOrientation === 'horizontal' || dataOrientation === 'vertical')
201
- return true;
202
- return (element.hasAttribute('data-accordion-item') ||
203
- element.hasAttribute('data-radix-collection-item'));
204
- }
205
- function isWithinPrimaryContent(element) {
206
- let current = element;
207
- while (current) {
208
- const tagName = current.tagName.toLowerCase();
209
- if (tagName === 'article' || tagName === 'main')
210
- return true;
211
- if (current.getAttribute('role') === 'main')
212
- return true;
213
- current = current.parentElement;
214
- }
215
- return false;
216
- }
217
- function shouldPreserve(element, tagName) {
218
- // Check Dialog
219
- const role = element.getAttribute('role');
220
- if (role === 'dialog' || role === 'alertdialog') {
221
- if (isWithinPrimaryContent(element))
222
- return true;
223
- const textLen = (element.textContent || '').length;
224
- if (textLen > DIALOG_MIN_CHARS_FOR_PRESERVATION)
225
- return true;
226
- return element.querySelector('h1,h2,h3,h4,h5,h6') !== null;
227
- }
228
- // Check Nav/Footer
229
- if (tagName === 'nav' || tagName === 'footer') {
230
- if (element.querySelector('article,main,section,[role="main"]'))
231
- return true;
232
- return ((element.textContent || '').trim().length >=
233
- NAV_FOOTER_MIN_CHARS_FOR_PRESERVATION);
234
- }
235
- return false;
236
- }
237
- function removeNodes(nodes) {
238
- for (let i = nodes.length - 1; i >= 0; i--) {
239
- const node = nodes[i];
240
- if (node?.parentNode && !shouldPreserve(node, node.tagName.toLowerCase())) {
241
- node.remove();
242
- }
243
- }
244
- }
245
- function scoreNavFooter(tagName, role, className, id, weights) {
246
- let score = 0;
247
- if (ALWAYS_NOISE_TAGS.has(tagName))
248
- score += weights.structural;
249
- // Header Boilerplate
250
- if (tagName === 'header') {
251
- if ((role && NAVIGATION_ROLES.has(role)) ||
252
- HEADER_NOISE_PATTERN.test(`${className} ${id}`)) {
253
- score += weights.structural;
254
- }
255
- }
256
- // Role Noise
257
- if (role && NAVIGATION_ROLES.has(role)) {
258
- if (tagName !== 'aside' || role !== 'complementary') {
259
- score += weights.structural;
260
- }
261
- }
262
- return score;
263
- }
264
- function extractElementMetadata(element) {
265
- const tagName = element.tagName.toLowerCase();
266
- const className = element.getAttribute('class') ?? '';
267
- const id = element.getAttribute('id') ?? '';
268
- const role = element.getAttribute('role');
269
- const style = element.getAttribute('style');
270
- const _isInteractive = isInteractive(element, role);
271
- const isHidden = element.hasAttribute('hidden') ||
272
- element.getAttribute('aria-hidden') === 'true' ||
273
- (style !== null &&
274
- /\b(?:display\s*:\s*none|visibility\s*:\s*hidden)\b/i.test(style));
275
- return {
276
- tagName,
277
- className,
278
- id,
279
- role,
280
- style,
281
- isInteractive: _isInteractive,
282
- isHidden,
283
- };
284
- }
285
- function isNoiseElement(element, context) {
286
- const meta = extractElementMetadata(element);
287
- let score = 0;
288
- const { weights } = context;
289
- // Structural
290
- if (context.structuralTags.has(meta.tagName) && !meta.isInteractive) {
291
- score += weights.structural;
292
- }
293
- // Nav/Footer Scoring
294
- if (context.flags.navFooter) {
295
- score += scoreNavFooter(meta.tagName, meta.role, meta.className, meta.id, weights);
296
- }
297
- // Hidden
298
- if (meta.isHidden && !meta.isInteractive) {
299
- score += weights.hidden;
300
- }
301
- // Sticky/Fixed
302
- if (FIXED_OR_HIGH_Z_PATTERN.test(meta.className)) {
303
- score += weights.stickyFixed;
304
- }
305
- // Promo
306
- if (context.promoEnabled) {
307
- const aggTest = context.promoMatchers.aggressive.test(meta.className) ||
308
- context.promoMatchers.aggressive.test(meta.id);
309
- const isAggressiveMatch = aggTest && !isWithinPrimaryContent(element);
310
- const isBaseMatch = !aggTest &&
311
- (context.promoMatchers.base.test(meta.className) ||
312
- context.promoMatchers.base.test(meta.id));
313
- if (isAggressiveMatch || isBaseMatch) {
314
- score += weights.promo;
315
- }
316
- }
317
- return score >= weights.threshold;
318
- }
319
- function cleanHeadingWrapperDivs(h) {
320
- const divs = h.querySelectorAll('div');
321
- for (let j = divs.length - 1; j >= 0; j--) {
322
- const d = divs[j];
323
- if (!d?.parentNode)
324
- continue;
325
- const cls = d.getAttribute('class') ?? '';
326
- const stl = d.getAttribute('style') ?? '';
327
- if (cls.includes('absolute') ||
328
- stl.includes('position') ||
329
- d.getAttribute('tabindex') === '-1') {
330
- d.remove();
331
- }
332
- }
333
- }
334
- function cleanHeadingAnchors(h) {
335
- const anchors = h.querySelectorAll('a');
336
- for (let j = anchors.length - 1; j >= 0; j--) {
337
- const a = anchors[j];
338
- if (!a?.parentNode)
339
- continue;
340
- const href = a.getAttribute('href') ?? '';
341
- const txt = (a.textContent || '').replace(/[\u200B\s]/g, '');
342
- if (href.startsWith('#') && txt.length === 0) {
343
- a.remove();
344
- }
345
- }
346
- }
347
- function cleanHeadingZeroWidth(h, document) {
348
- const walker = document.createTreeWalker(h, 4); // SHOW_TEXT
349
- let node;
350
- while ((node = walker.nextNode())) {
351
- if (node.textContent?.includes('\u200B')) {
352
- node.textContent = node.textContent.replace(/\u200B/g, '');
353
- }
354
- }
355
- }
356
- function cleanHeadings(document) {
357
- // Clean Heading Anchors
358
- const headings = document.querySelectorAll('h1,h2,h3,h4,h5,h6');
359
- for (const h of headings) {
360
- if (!h.parentNode)
361
- continue;
362
- cleanHeadingWrapperDivs(h);
363
- cleanHeadingAnchors(h);
364
- cleanHeadingZeroWidth(h, document);
365
- }
366
- }
367
- function stripNoise(document, context, signal) {
368
- cleanHeadings(document);
369
- // Remove Base & Extra
370
- const { baseSelector, extraSelectors } = context;
371
- // Base
372
- const baseNodes = document.querySelectorAll(baseSelector);
373
- removeNodes(baseNodes);
374
- // Extra
375
- if (extraSelectors.length > 0) {
376
- const combinedExtra = extraSelectors.join(',');
377
- const extraNodes = document.querySelectorAll(combinedExtra);
378
- removeNodes(extraNodes);
379
- }
380
- // Candidates
381
- const candidates = document.querySelectorAll(context.candidateSelector);
382
- for (let i = candidates.length - 1; i >= 0; i--) {
383
- if (i % 500 === 0 && signal?.aborted) {
384
- throw new Error('Noise removal aborted');
385
- }
386
- const node = candidates[i];
387
- if (!node)
388
- continue;
389
- if (!node.parentNode)
390
- continue;
391
- if (shouldPreserve(node, node.tagName.toLowerCase()))
392
- continue;
393
- if (isNoiseElement(node, context)) {
394
- node.remove();
395
- }
396
- }
397
- }
398
- function processUrlElement(el, attr, base, isSrcset) {
399
- if (!el.parentNode)
400
- return;
401
- if (isSrcset) {
402
- const val = el.getAttribute(attr);
403
- if (val) {
404
- const newVal = val
405
- .split(',')
406
- .map((entry) => {
407
- const parts = entry.trim().split(/\s+/);
408
- if (!parts[0])
409
- return entry;
410
- try {
411
- parts[0] = new URL(parts[0], base).href;
412
- }
413
- catch {
414
- /* ignore */
415
- }
416
- return parts.join(' ');
417
- })
418
- .join(', ');
419
- el.setAttribute(attr, newVal);
420
- }
421
- return;
422
- }
423
- const val = el.getAttribute(attr);
424
- if (val &&
425
- !SKIP_URL_PREFIXES.some((p) => val.trim().toLowerCase().startsWith(p))) {
426
- try {
427
- el.setAttribute(attr, new URL(val, base).href);
428
- }
429
- catch {
430
- /* ignore */
431
- }
432
- }
433
- }
434
- function resolveUrls(document, baseUrlStr) {
435
- let base;
436
- try {
437
- base = new URL(baseUrlStr);
438
- }
439
- catch {
440
- return;
441
- }
442
- const elements = document.querySelectorAll('a[href],img[src],source[srcset]');
443
- for (const el of elements) {
444
- const tag = el.tagName.toLowerCase();
445
- if (tag === 'a')
446
- processUrlElement(el, 'href', base, false);
447
- else if (tag === 'img')
448
- processUrlElement(el, 'src', base, false);
449
- else if (tag === 'source')
450
- processUrlElement(el, 'srcset', base, true);
451
- }
452
- }
453
- function serialize(document, fallback) {
454
- const bodyHtml = document.body.innerHTML;
455
- if (bodyHtml.trim().length > MIN_BODY_CONTENT_LENGTH)
456
- return bodyHtml;
457
- const outerHtml = document.documentElement.outerHTML;
458
- if (outerHtml.trim().length > MIN_BODY_CONTENT_LENGTH)
459
- return outerHtml;
460
- return fallback;
461
- }
462
- function isFullDocumentHtml(html) {
463
- return HTML_DOCUMENT_MARKERS.test(html);
464
- }
465
- function mayContainNoise(html) {
466
- const sample = html.length <= NOISE_SCAN_LIMIT
467
- ? html
468
- : `${html.substring(0, NOISE_SCAN_LIMIT)}\n${html.substring(html.length - NOISE_SCAN_LIMIT)}`;
469
- return NOISE_PATTERNS.some((re) => re.test(sample));
470
- }
471
- export function removeNoiseFromHtml(html, document, baseUrl, signal) {
472
- const shouldParse = isFullDocumentHtml(html) ||
473
- mayContainNoise(html) ||
474
- HTML_FRAGMENT_MARKERS.test(html);
475
- if (!shouldParse)
476
- return html;
477
- try {
478
- const context = getContext();
479
- if (config.noiseRemoval.debug) {
480
- logDebug('Noise removal audit enabled', {
481
- categories: [...(context.flags.navFooter ? ['nav-footer'] : [])],
482
- });
483
- }
484
- const doc = document ?? parseHTML(html).document;
485
- stripNoise(doc, context, signal);
486
- if (baseUrl)
487
- resolveUrls(doc, baseUrl);
488
- return serialize(doc, html);
489
- }
490
- catch {
491
- return html;
492
- }
493
- }
494
- //# sourceMappingURL=dom-noise-removal.js.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"dom-noise-removal.js","sourceRoot":"","sources":["../../src/lib/dom-noise-removal.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,UAAU,CAAC;AAErC,OAAO,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AACrC,OAAO,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AAE9C,yCAAyC;AAEzC,MAAM,gBAAgB,GAAG,MAAM,CAAC;AAChC,MAAM,uBAAuB,GAAG,GAAG,CAAC;AACpC,MAAM,iCAAiC,GAAG,GAAG,CAAC;AAC9C,MAAM,qCAAqC,GAAG,GAAG,CAAC;AAElD,oCAAoC;AACpC,MAAM,qBAAqB,GAAG,oCAAoC,CAAC;AACnE,MAAM,qBAAqB,GACzB,yEAAyE,CAAC;AAE5E,2EAA2E;AAC3E,MAAM,cAAc,GAAsB;IACxC,wGAAwG;IACxG,+FAA+F;IAC/F,oDAAoD;IACpD,iHAAiH;IACjH,kEAAkE;CACnE,CAAC;AAEF,MAAM,oBAAoB,GACxB,uEAAuE,CAAC;AAC1E,MAAM,uBAAuB,GAAG,2CAA2C,CAAC;AAE5E,MAAM,iBAAiB,GAAG;IACxB,GAAG;IACH,MAAM,GAAG,SAAS;IAClB,SAAS;IACT,MAAM;IACN,OAAO;IACP,OAAO;CACR,CAAC;AACF,MAAM,oBAAoB,GAAG,IAAI,GAAG,CAAC;IACnC,QAAQ;IACR,OAAO;IACP,UAAU;IACV,QAAQ;IACR,MAAM;IACN,QAAQ;IACR,OAAO;IACP,QAAQ;IACR,UAAU;CACX,CAAC,CAAC;AACH,MAAM,iBAAiB,GAAG,IAAI,GAAG,CAAC,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC,CAAC;AAErD,MAAM,gBAAgB,GAAG,IAAI,GAAG,CAAC;IAC/B,YAAY;IACZ,QAAQ;IACR,eAAe;IACf,aAAa;IACb,MAAM;IACN,SAAS;IACT,MAAM;IACN,QAAQ;IACR,aAAa;IACb,QAAQ;CACT,CAAC,CAAC;AAEH,MAAM,yBAAyB,GAAG,IAAI,GAAG,CAAC;IACxC,UAAU;IACV,KAAK;IACL,SAAS;IACT,QAAQ;IACR,aAAa;IACb,MAAM;IACN,UAAU;IACV,QAAQ;IACR,SAAS;IACT,UAAU;IACV,SAAS;IACT,OAAO;CACR,CAAC,CAAC;AAEH,MAAM,mBAAmB,GAAG;IAC1B,QAAQ;IACR,OAAO;IACP,cAAc;IACd,KAAK;IACL,QAAQ;IACR,KAAK;IACL,SAAS;IACT,WAAW;IACX,YAAY;IACZ,YAAY;IACZ,OAAO;IACP,SAAS;CACV,CAAC;AACF,MAAM,uBAAuB,GAAG,CAAC,IAAI,EAAE,SAAS,EAAE,SAAS,CAAC,CAAC;AAC7D,MAAM,wBAAwB,GAAG;IAC/B,gBAAgB,EAAE,CAAC,QAAQ,EAAE,SAAS,EAAE,OAAO,EAAE,OAAO,EAAE,SAAS,EAAE,OAAO,CAAC;IAC7E,WAAW,EAAE,CAAC,YAAY,EAAE,WAAW,CAAC;IACxC,cAAc,EAAE,CAAC,OAAO,EAAE,QAAQ,CAAC;CACpC,CAAC;AAEF,MAAM,oBAAoB,GAAG;IAC3B,SAAS,EACP,iHAAiH;IACnH,aAAa,EAAE,iBAAiB;IAChC,MAAM,EACJ,gFAAgF;CACnF,CAAC;AAEF,MAAM,cAAc,GAAG,KAAK,CAAC;AA4B7B,sBAAsB;AAEtB,IAAI,aAAuC,CAAC;AAC5C,IAAI,aAA6C,CAAC;AAElD,oCAAoC;AAEpC,SAAS,kBAAkB,CAAC,KAAa;IACvC,OAAO,KAAK,CAAC,OAAO,CAAC,qBAAqB,EAAE,MAAM,CAAC,CAAC;AACtD,CAAC;AAED,SAAS,eAAe,CAAC,MAAmB;IAC1C,IAAI,MAAM,CAAC,IAAI,KAAK,CAAC;QAAE,OAAO,cAAc,CAAC;IAC7C,OAAO,IAAI,MAAM,CACf,qBAAqB,CAAC,GAAG,MAAM,CAAC,CAAC,GAAG,CAAC,kBAAkB,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,kBAAkB,EACpF,GAAG,CACJ,CAAC;AACJ,CAAC;AAED,SAAS,SAAS,CAAC,MAAmB,EAAE,MAAyB;IAC/D,KAAK,MAAM,KAAK,IAAI,MAAM;QAAE,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;AAChD,CAAC;AAED,SAAS,gBAAgB,CACvB,aAAiC,EACjC,KAA4B;IAE5B,MAAM,UAAU,GAAG,IAAI,GAAG,CAAC,mBAAmB,CAAC,CAAC;IAChD,MAAM,gBAAgB,GAAG,IAAI,GAAG,EAAU,CAAC;IAE3C,IAAI,aAAa,CAAC,cAAc,EAAE,CAAC;QACjC,SAAS,CAAC,gBAAgB,EAAE,uBAAuB,CAAC,CAAC;IACvD,CAAC;IAED,IAAI,KAAK,CAAC,aAAa,EAAE,CAAC;QACxB,SAAS,CAAC,UAAU,EAAE,wBAAwB,CAAC,gBAAgB,CAAC,CAAC,CAAC;IACpE,CAAC;IACD,IAAI,KAAK,CAAC,WAAW,EAAE,CAAC;QACtB,SAAS,CAAC,UAAU,EAAE,wBAAwB,CAAC,aAAa,CAAC,CAAC,CAAC;IACjE,CAAC;IACD,IAAI,KAAK,CAAC,WAAW,EAAE,CAAC;QACtB,SAAS,CAAC,UAAU,EAAE,wBAAwB,CAAC,cAAc,CAAC,CAAC,CAAC;IAClE,CAAC;IAED,KAAK,MAAM,CAAC,IAAI,aAAa,CAAC,WAAW,EAAE,CAAC;QAC1C,MAAM,CAAC,GAAG,CAAC,CAAC,WAAW,EAAE,CAAC,IAAI,EAAE,CAAC;QACjC,IAAI,CAAC;YAAE,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;IAC3B,CAAC;IAED,OAAO;QACL,IAAI,EAAE,eAAe,CAAC,UAAU,CAAC;QACjC,UAAU,EAAE,eAAe,CAAC,gBAAgB,CAAC;KAC9C,CAAC;AACJ,CAAC;AAED,SAAS,UAAU;IACjB,MAAM,aAAa,GAAG,MAAM,CAAC,YAAY,CAAC;IAC1C,IAAI,aAAa,KAAK,SAAS,IAAI,aAAa,KAAK,aAAa;QAChE,OAAO,aAAa,CAAC;IAEvB,MAAM,OAAO,GAAG,IAAI,GAAG,CACrB,aAAa,CAAC,iBAAiB;SAC5B,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;QACT,MAAM,CAAC,GAAG,CAAC,CAAC,WAAW,EAAE,CAAC,IAAI,EAAE,CAAC;QACjC,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,CAAC,IAAI,CAAC;QAC/B,OAAO,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,iBAAiB,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAClD,CAAC,CAAC;SACD,MAAM,CAAC,OAAO,CAAC,CACnB,CAAC;IAEF,MAAM,SAAS,GAAG,CAAC,GAAW,EAAW,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IAC7D,MAAM,KAAK,GAAG;QACZ,SAAS,EAAE,SAAS,CAAC,YAAY,CAAC;QAClC,aAAa,EAAE,SAAS,CAAC,gBAAgB,CAAC;QAC1C,WAAW,EAAE,SAAS,CAAC,aAAa,CAAC;QACrC,WAAW,EAAE,SAAS,CAAC,cAAc,CAAC;KACvC,CAAC;IAEF,MAAM,cAAc,GAAG,IAAI,GAAG,CAAC,oBAAoB,CAAC,CAAC;IACrD,IAAI,CAAC,aAAa,CAAC,iBAAiB,EAAE,CAAC;QACrC,cAAc,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;QAC1B,cAAc,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;IAC/B,CAAC;IAED,MAAM,aAAa,GAAG,gBAAgB,CAAC,aAAa,EAAE,KAAK,CAAC,CAAC;IAC7D,MAAM,cAAc,GAAG,aAAa,CAAC,cAAc;SAChD,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;SACpB,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IAE/B,sBAAsB;IACtB,MAAM,SAAS,GAAG,CAAC,oBAAoB,CAAC,MAAM,CAAC,CAAC;IAChD,IAAI,KAAK,CAAC,SAAS;QAAE,SAAS,CAAC,IAAI,CAAC,oBAAoB,CAAC,SAAS,CAAC,CAAC;IACpE,IAAI,KAAK,CAAC,aAAa;QAAE,SAAS,CAAC,IAAI,CAAC,oBAAoB,CAAC,aAAa,CAAC,CAAC;IAC5E,MAAM,YAAY,GAAG,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAEzC,MAAM,iBAAiB,GAAG;QACxB,GAAG,cAAc;QACjB,GAAG,iBAAiB;QACpB,OAAO;QACP,QAAQ;QACR,SAAS;QACT,MAAM;QACN,QAAQ;QACR,SAAS;KACV,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAEZ,aAAa,GAAG;QACd,KAAK;QACL,cAAc;QACd,OAAO,EAAE,aAAa,CAAC,OAAO;QAC9B,aAAa;QACb,YAAY,EAAE,KAAK,CAAC,aAAa,IAAI,KAAK,CAAC,WAAW,IAAI,KAAK,CAAC,WAAW;QAC3E,cAAc;QACd,YAAY;QACZ,iBAAiB;KAClB,CAAC;IACF,aAAa,GAAG,aAAa,CAAC;IAC9B,OAAO,aAAa,CAAC;AACvB,CAAC;AAED,yBAAyB;AAEzB,SAAS,aAAa,CAAC,OAAgB,EAAE,IAAmB;IAC1D,IAAI,IAAI,IAAI,yBAAyB,CAAC,GAAG,CAAC,IAAI,CAAC;QAAE,OAAO,IAAI,CAAC;IAC7D,MAAM,EAAE,GAAG,OAAO,CAAC,YAAY,CAAC,YAAY,CAAC,CAAC;IAC9C,IAAI,EAAE,KAAK,UAAU,IAAI,EAAE,KAAK,QAAQ;QAAE,OAAO,IAAI,CAAC;IACtD,MAAM,eAAe,GAAG,OAAO,CAAC,YAAY,CAAC,kBAAkB,CAAC,CAAC;IACjE,IAAI,eAAe,KAAK,YAAY,IAAI,eAAe,KAAK,UAAU;QACpE,OAAO,IAAI,CAAC;IACd,OAAO,CACL,OAAO,CAAC,YAAY,CAAC,qBAAqB,CAAC;QAC3C,OAAO,CAAC,YAAY,CAAC,4BAA4B,CAAC,CACnD,CAAC;AACJ,CAAC;AAED,SAAS,sBAAsB,CAAC,OAAgB;IAC9C,IAAI,OAAO,GAAmB,OAAO,CAAC;IACtC,OAAO,OAAO,EAAE,CAAC;QACf,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC;QAC9C,IAAI,OAAO,KAAK,SAAS,IAAI,OAAO,KAAK,MAAM;YAAE,OAAO,IAAI,CAAC;QAC7D,IAAI,OAAO,CAAC,YAAY,CAAC,MAAM,CAAC,KAAK,MAAM;YAAE,OAAO,IAAI,CAAC;QACzD,OAAO,GAAG,OAAO,CAAC,aAAa,CAAC;IAClC,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAS,cAAc,CAAC,OAAgB,EAAE,OAAe;IACvD,eAAe;IACf,MAAM,IAAI,GAAG,OAAO,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC;IAC1C,IAAI,IAAI,KAAK,QAAQ,IAAI,IAAI,KAAK,aAAa,EAAE,CAAC;QAChD,IAAI,sBAAsB,CAAC,OAAO,CAAC;YAAE,OAAO,IAAI,CAAC;QACjD,MAAM,OAAO,GAAG,CAAC,OAAO,CAAC,WAAW,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC;QACnD,IAAI,OAAO,GAAG,iCAAiC;YAAE,OAAO,IAAI,CAAC;QAC7D,OAAO,OAAO,CAAC,aAAa,CAAC,mBAAmB,CAAC,KAAK,IAAI,CAAC;IAC7D,CAAC;IAED,mBAAmB;IACnB,IAAI,OAAO,KAAK,KAAK,IAAI,OAAO,KAAK,QAAQ,EAAE,CAAC;QAC9C,IAAI,OAAO,CAAC,aAAa,CAAC,oCAAoC,CAAC;YAC7D,OAAO,IAAI,CAAC;QACd,OAAO,CACL,CAAC,OAAO,CAAC,WAAW,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,MAAM;YACzC,qCAAqC,CACtC,CAAC;IACJ,CAAC;IAED,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAS,WAAW,CAAC,KAAyB;IAC5C,KAAK,IAAI,CAAC,GAAG,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;QAC3C,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;QACtB,IAAI,IAAI,EAAE,UAAU,IAAI,CAAC,cAAc,CAAC,IAAI,EAAE,IAAI,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC,EAAE,CAAC;YAC1E,IAAI,CAAC,MAAM,EAAE,CAAC;QAChB,CAAC;IACH,CAAC;AACH,CAAC;AAED,SAAS,cAAc,CACrB,OAAe,EACf,IAAmB,EACnB,SAAiB,EACjB,EAAU,EACV,OAAqB;IAErB,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,IAAI,iBAAiB,CAAC,GAAG,CAAC,OAAO,CAAC;QAAE,KAAK,IAAI,OAAO,CAAC,UAAU,CAAC;IAEhE,qBAAqB;IACrB,IAAI,OAAO,KAAK,QAAQ,EAAE,CAAC;QACzB,IACE,CAAC,IAAI,IAAI,gBAAgB,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;YACpC,oBAAoB,CAAC,IAAI,CAAC,GAAG,SAAS,IAAI,EAAE,EAAE,CAAC,EAC/C,CAAC;YACD,KAAK,IAAI,OAAO,CAAC,UAAU,CAAC;QAC9B,CAAC;IACH,CAAC;IAED,aAAa;IACb,IAAI,IAAI,IAAI,gBAAgB,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;QACvC,IAAI,OAAO,KAAK,OAAO,IAAI,IAAI,KAAK,eAAe,EAAE,CAAC;YACpD,KAAK,IAAI,OAAO,CAAC,UAAU,CAAC;QAC9B,CAAC;IACH,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAYD,SAAS,sBAAsB,CAAC,OAAgB;IAC9C,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC;IAC9C,MAAM,SAAS,GAAG,OAAO,CAAC,YAAY,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC;IACtD,MAAM,EAAE,GAAG,OAAO,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;IAC5C,MAAM,IAAI,GAAG,OAAO,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC;IAC1C,MAAM,KAAK,GAAG,OAAO,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC;IAC5C,MAAM,cAAc,GAAG,aAAa,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC;IACpD,MAAM,QAAQ,GACZ,OAAO,CAAC,YAAY,CAAC,QAAQ,CAAC;QAC9B,OAAO,CAAC,YAAY,CAAC,aAAa,CAAC,KAAK,MAAM;QAC9C,CAAC,KAAK,KAAK,IAAI;YACb,qDAAqD,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC;IAEvE,OAAO;QACL,OAAO;QACP,SAAS;QACT,EAAE;QACF,IAAI;QACJ,KAAK;QACL,aAAa,EAAE,cAAc;QAC7B,QAAQ;KACT,CAAC;AACJ,CAAC;AAED,SAAS,cAAc,CAAC,OAAgB,EAAE,OAAqB;IAC7D,MAAM,IAAI,GAAG,sBAAsB,CAAC,OAAO,CAAC,CAAC;IAE7C,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,MAAM,EAAE,OAAO,EAAE,GAAG,OAAO,CAAC;IAE5B,aAAa;IACb,IAAI,OAAO,CAAC,cAAc,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,CAAC;QACpE,KAAK,IAAI,OAAO,CAAC,UAAU,CAAC;IAC9B,CAAC;IAED,qBAAqB;IACrB,IAAI,OAAO,CAAC,KAAK,CAAC,SAAS,EAAE,CAAC;QAC5B,KAAK,IAAI,cAAc,CACrB,IAAI,CAAC,OAAO,EACZ,IAAI,CAAC,IAAI,EACT,IAAI,CAAC,SAAS,EACd,IAAI,CAAC,EAAE,EACP,OAAO,CACR,CAAC;IACJ,CAAC;IAED,SAAS;IACT,IAAI,IAAI,CAAC,QAAQ,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,CAAC;QACzC,KAAK,IAAI,OAAO,CAAC,MAAM,CAAC;IAC1B,CAAC;IAED,eAAe;IACf,IAAI,uBAAuB,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC;QACjD,KAAK,IAAI,OAAO,CAAC,WAAW,CAAC;IAC/B,CAAC;IAED,QAAQ;IACR,IAAI,OAAO,CAAC,YAAY,EAAE,CAAC;QACzB,MAAM,OAAO,GACX,OAAO,CAAC,aAAa,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC;YACrD,OAAO,CAAC,aAAa,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACjD,MAAM,iBAAiB,GAAG,OAAO,IAAI,CAAC,sBAAsB,CAAC,OAAO,CAAC,CAAC;QACtE,MAAM,WAAW,GACf,CAAC,OAAO;YACR,CAAC,OAAO,CAAC,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC;gBAC9C,OAAO,CAAC,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC;QAE9C,IAAI,iBAAiB,IAAI,WAAW,EAAE,CAAC;YACrC,KAAK,IAAI,OAAO,CAAC,KAAK,CAAC;QACzB,CAAC;IACH,CAAC;IAED,OAAO,KAAK,IAAI,OAAO,CAAC,SAAS,CAAC;AACpC,CAAC;AAED,SAAS,uBAAuB,CAAC,CAAU;IACzC,MAAM,IAAI,GAAG,CAAC,CAAC,gBAAgB,CAAC,KAAK,CAAC,CAAC;IACvC,KAAK,IAAI,CAAC,GAAG,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;QAC1C,MAAM,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC;QAClB,IAAI,CAAC,CAAC,EAAE,UAAU;YAAE,SAAS;QAC7B,MAAM,GAAG,GAAG,CAAC,CAAC,YAAY,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC;QAC1C,MAAM,GAAG,GAAG,CAAC,CAAC,YAAY,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC;QAC1C,IACE,GAAG,CAAC,QAAQ,CAAC,UAAU,CAAC;YACxB,GAAG,CAAC,QAAQ,CAAC,UAAU,CAAC;YACxB,CAAC,CAAC,YAAY,CAAC,UAAU,CAAC,KAAK,IAAI,EACnC,CAAC;YACD,CAAC,CAAC,MAAM,EAAE,CAAC;QACb,CAAC;IACH,CAAC;AACH,CAAC;AAED,SAAS,mBAAmB,CAAC,CAAU;IACrC,MAAM,OAAO,GAAG,CAAC,CAAC,gBAAgB,CAAC,GAAG,CAAC,CAAC;IACxC,KAAK,IAAI,CAAC,GAAG,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;QAC7C,MAAM,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;QACrB,IAAI,CAAC,CAAC,EAAE,UAAU;YAAE,SAAS;QAC7B,MAAM,IAAI,GAAG,CAAC,CAAC,YAAY,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;QAC1C,MAAM,GAAG,GAAG,CAAC,CAAC,CAAC,WAAW,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,aAAa,EAAE,EAAE,CAAC,CAAC;QAC7D,IAAI,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,GAAG,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC7C,CAAC,CAAC,MAAM,EAAE,CAAC;QACb,CAAC;IACH,CAAC;AACH,CAAC;AAED,SAAS,qBAAqB,CAAC,CAAU,EAAE,QAAkB;IAC3D,MAAM,MAAM,GAAG,QAAQ,CAAC,gBAAgB,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,YAAY;IAC5D,IAAI,IAAiB,CAAC;IACtB,OAAO,CAAC,IAAI,GAAG,MAAM,CAAC,QAAQ,EAAE,CAAC,EAAE,CAAC;QAClC,IAAI,IAAI,CAAC,WAAW,EAAE,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC;YACzC,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,WAAW,CAAC,OAAO,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;QAC7D,CAAC;IACH,CAAC;AACH,CAAC;AAED,SAAS,aAAa,CAAC,QAAkB;IACvC,wBAAwB;IACxB,MAAM,QAAQ,GAAG,QAAQ,CAAC,gBAAgB,CAAC,mBAAmB,CAAC,CAAC;IAChE,KAAK,MAAM,CAAC,IAAI,QAAQ,EAAE,CAAC;QACzB,IAAI,CAAC,CAAC,CAAC,UAAU;YAAE,SAAS;QAC5B,uBAAuB,CAAC,CAAC,CAAC,CAAC;QAC3B,mBAAmB,CAAC,CAAC,CAAC,CAAC;QACvB,qBAAqB,CAAC,CAAC,EAAE,QAAQ,CAAC,CAAC;IACrC,CAAC;AACH,CAAC;AAED,SAAS,UAAU,CACjB,QAAkB,EAClB,OAAqB,EACrB,MAAoB;IAEpB,aAAa,CAAC,QAAQ,CAAC,CAAC;IAExB,sBAAsB;IACtB,MAAM,EAAE,YAAY,EAAE,cAAc,EAAE,GAAG,OAAO,CAAC;IAEjD,OAAO;IACP,MAAM,SAAS,GAAG,QAAQ,CAAC,gBAAgB,CAAC,YAAY,CAAC,CAAC;IAC1D,WAAW,CAAC,SAAS,CAAC,CAAC;IAEvB,QAAQ;IACR,IAAI,cAAc,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC9B,MAAM,aAAa,GAAG,cAAc,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAC/C,MAAM,UAAU,GAAG,QAAQ,CAAC,gBAAgB,CAAC,aAAa,CAAC,CAAC;QAC5D,WAAW,CAAC,UAAU,CAAC,CAAC;IAC1B,CAAC;IAED,aAAa;IACb,MAAM,UAAU,GAAG,QAAQ,CAAC,gBAAgB,CAAC,OAAO,CAAC,iBAAiB,CAAC,CAAC;IACxE,KAAK,IAAI,CAAC,GAAG,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;QAChD,IAAI,CAAC,GAAG,GAAG,KAAK,CAAC,IAAI,MAAM,EAAE,OAAO,EAAE,CAAC;YACrC,MAAM,IAAI,KAAK,CAAC,uBAAuB,CAAC,CAAC;QAC3C,CAAC;QACD,MAAM,IAAI,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC;QAC3B,IAAI,CAAC,IAAI;YAAE,SAAS;QACpB,IAAI,CAAC,IAAI,CAAC,UAAU;YAAE,SAAS;QAE/B,IAAI,cAAc,CAAC,IAAI,EAAE,IAAI,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC;YAAE,SAAS;QAC/D,IAAI,cAAc,CAAC,IAAI,EAAE,OAAO,CAAC,EAAE,CAAC;YAClC,IAAI,CAAC,MAAM,EAAE,CAAC;QAChB,CAAC;IACH,CAAC;AACH,CAAC;AAED,SAAS,iBAAiB,CACxB,EAAW,EACX,IAAY,EACZ,IAAS,EACT,QAAiB;IAEjB,IAAI,CAAC,EAAE,CAAC,UAAU;QAAE,OAAO;IAC3B,IAAI,QAAQ,EAAE,CAAC;QACb,MAAM,GAAG,GAAG,EAAE,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC;QAClC,IAAI,GAAG,EAAE,CAAC;YACR,MAAM,MAAM,GAAG,GAAG;iBACf,KAAK,CAAC,GAAG,CAAC;iBACV,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE;gBACb,MAAM,KAAK,GAAG,KAAK,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;gBACxC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC;oBAAE,OAAO,KAAK,CAAC;gBAC5B,IAAI,CAAC;oBACH,KAAK,CAAC,CAAC,CAAC,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,IAAI,CAAC;gBAC1C,CAAC;gBAAC,MAAM,CAAC;oBACP,YAAY;gBACd,CAAC;gBACD,OAAO,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACzB,CAAC,CAAC;iBACD,IAAI,CAAC,IAAI,CAAC,CAAC;YACd,EAAE,CAAC,YAAY,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;QAChC,CAAC;QACD,OAAO;IACT,CAAC;IAED,MAAM,GAAG,GAAG,EAAE,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC;IAClC,IACE,GAAG;QACH,CAAC,iBAAiB,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,EACtE,CAAC;QACD,IAAI,CAAC;YACH,EAAE,CAAC,YAAY,CAAC,IAAI,EAAE,IAAI,GAAG,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC;QACjD,CAAC;QAAC,MAAM,CAAC;YACP,YAAY;QACd,CAAC;IACH,CAAC;AACH,CAAC;AAED,SAAS,WAAW,CAAC,QAAkB,EAAE,UAAkB;IACzD,IAAI,IAAS,CAAC;IACd,IAAI,CAAC;QACH,IAAI,GAAG,IAAI,GAAG,CAAC,UAAU,CAAC,CAAC;IAC7B,CAAC;IAAC,MAAM,CAAC;QACP,OAAO;IACT,CAAC;IAED,MAAM,QAAQ,GAAG,QAAQ,CAAC,gBAAgB,CAAC,iCAAiC,CAAC,CAAC;IAC9E,KAAK,MAAM,EAAE,IAAI,QAAQ,EAAE,CAAC;QAC1B,MAAM,GAAG,GAAG,EAAE,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC;QACrC,IAAI,GAAG,KAAK,GAAG;YAAE,iBAAiB,CAAC,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,KAAK,CAAC,CAAC;aACvD,IAAI,GAAG,KAAK,KAAK;YAAE,iBAAiB,CAAC,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,KAAK,CAAC,CAAC;aAC7D,IAAI,GAAG,KAAK,QAAQ;YAAE,iBAAiB,CAAC,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC;IACzE,CAAC;AACH,CAAC;AAED,SAAS,SAAS,CAAC,QAAkB,EAAE,QAAgB;IACrD,MAAM,QAAQ,GAAG,QAAQ,CAAC,IAAI,CAAC,SAAS,CAAC;IACzC,IAAI,QAAQ,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,uBAAuB;QAAE,OAAO,QAAQ,CAAC;IAEtE,MAAM,SAAS,GAAG,QAAQ,CAAC,eAAe,CAAC,SAAS,CAAC;IACrD,IAAI,SAAS,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,uBAAuB;QAAE,OAAO,SAAS,CAAC;IAExE,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED,SAAS,kBAAkB,CAAC,IAAY;IACtC,OAAO,qBAAqB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1C,CAAC;AAED,SAAS,eAAe,CAAC,IAAY;IACnC,MAAM,MAAM,GACV,IAAI,CAAC,MAAM,IAAI,gBAAgB;QAC7B,CAAC,CAAC,IAAI;QACN,CAAC,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC,EAAE,gBAAgB,CAAC,KAAK,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,MAAM,GAAG,gBAAgB,CAAC,EAAE,CAAC;IAClG,OAAO,cAAc,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC;AACtD,CAAC;AAED,MAAM,UAAU,mBAAmB,CACjC,IAAY,EACZ,QAAmB,EACnB,OAAgB,EAChB,MAAoB;IAEpB,MAAM,WAAW,GACf,kBAAkB,CAAC,IAAI,CAAC;QACxB,eAAe,CAAC,IAAI,CAAC;QACrB,qBAAqB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACnC,IAAI,CAAC,WAAW;QAAE,OAAO,IAAI,CAAC;IAE9B,IAAI,CAAC;QACH,MAAM,OAAO,GAAG,UAAU,EAAE,CAAC;QAE7B,IAAI,MAAM,CAAC,YAAY,CAAC,KAAK,EAAE,CAAC;YAC9B,QAAQ,CAAC,6BAA6B,EAAE;gBACtC,UAAU,EAAE,CAAC,GAAG,CAAC,OAAO,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;aACjE,CAAC,CAAC;QACL,CAAC;QAED,MAAM,GAAG,GAAG,QAAQ,IAAI,SAAS,CAAC,IAAI,CAAC,CAAC,QAAQ,CAAC;QAEjD,UAAU,CAAC,GAAG,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC;QAEjC,IAAI,OAAO;YAAE,WAAW,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;QAEvC,OAAO,SAAS,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;IAC9B,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC"}
@@ -1,4 +0,0 @@
1
- import type { ServerResponse } from 'node:http';
2
- export declare function generateSafeFilename(url: string, title?: string, hashFallback?: string, extension?: string): string;
3
- export declare function handleDownload(res: ServerResponse, namespace: string, hash: string): void;
4
- //# sourceMappingURL=download.d.ts.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"download.d.ts","sourceRoot":"","sources":["../../src/lib/download.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,WAAW,CAAC;AA2EhD,wBAAgB,oBAAoB,CAClC,GAAG,EAAE,MAAM,EACX,KAAK,CAAC,EAAE,MAAM,EACd,YAAY,CAAC,EAAE,MAAM,EACrB,SAAS,SAAQ,GAChB,MAAM,CAIR;AAyBD,wBAAgB,cAAc,CAC5B,GAAG,EAAE,cAAc,EACnB,SAAS,EAAE,MAAM,EACjB,IAAI,EAAE,MAAM,GACX,IAAI,CAwCN"}
@@ -1,106 +0,0 @@
1
- import { posix as pathPosix } from 'node:path';
2
- import { z } from 'zod';
3
- import { get as cacheGet, parseCachedPayload, resolveCachedPayloadContent, } from './cache.js';
4
- import { config } from './config.js';
5
- /* -------------------------------------------------------------------------------------------------
6
- * Utils: Filename Logic
7
- * ------------------------------------------------------------------------------------------------- */
8
- const FILENAME_RULES = {
9
- MAX_LEN: 200,
10
- UNSAFE_CHARS: /[<>:"/\\|?*\p{C}]/gu,
11
- WHITESPACE: /\s+/g,
12
- EXTENSIONS: /\.(html?|php|aspx?|jsp)$/i,
13
- };
14
- function sanitizeString(input) {
15
- return input
16
- .toLowerCase()
17
- .replace(FILENAME_RULES.UNSAFE_CHARS, '')
18
- .replace(FILENAME_RULES.WHITESPACE, '-')
19
- .replace(/-+/g, '-')
20
- .replace(/(?:^-|-$)/g, '');
21
- }
22
- function resolveUrlFilenameCandidate(url) {
23
- const parsed = new URL(url);
24
- if (parsed.protocol !== 'http:' && parsed.protocol !== 'https:')
25
- return null;
26
- const basename = pathPosix.basename(parsed.pathname);
27
- if (!basename || basename === 'index')
28
- return null;
29
- const cleaned = basename.replace(FILENAME_RULES.EXTENSIONS, '');
30
- const sanitized = sanitizeString(cleaned);
31
- if (sanitized === 'index')
32
- return null;
33
- return sanitized || null;
34
- }
35
- function truncateFilenameBase(name, extension) {
36
- const maxBase = FILENAME_RULES.MAX_LEN - extension.length;
37
- return name.length > maxBase ? name.substring(0, maxBase) : name;
38
- }
39
- function resolveTitleFilenameCandidate(title) {
40
- if (!title)
41
- return null;
42
- return sanitizeString(title) || null;
43
- }
44
- function resolveFilenameBase(url, title, hashFallback) {
45
- try {
46
- const fromUrl = resolveUrlFilenameCandidate(url);
47
- if (fromUrl)
48
- return fromUrl;
49
- }
50
- catch {
51
- // Ignore URL parsing errors and continue fallbacks.
52
- }
53
- const fromTitle = resolveTitleFilenameCandidate(title);
54
- if (fromTitle)
55
- return fromTitle;
56
- if (hashFallback)
57
- return hashFallback.substring(0, 16);
58
- return `download-${Date.now()}`;
59
- }
60
- export function generateSafeFilename(url, title, hashFallback, extension = '.md') {
61
- const name = resolveFilenameBase(url, title, hashFallback);
62
- return `${truncateFilenameBase(name, extension)}${extension}`;
63
- }
64
- /* -------------------------------------------------------------------------------------------------
65
- * Adapter: Download Handler
66
- * ------------------------------------------------------------------------------------------------- */
67
- const DownloadParamsSchema = z.strictObject({
68
- namespace: z.literal('markdown'),
69
- hash: z
70
- .string()
71
- .regex(/^[a-f0-9.]+$/i)
72
- .min(8)
73
- .max(64),
74
- });
75
- function writeJsonError(res, status, message, code) {
76
- res.writeHead(status, { 'Content-Type': 'application/json' });
77
- res.end(JSON.stringify({ error: message, code }));
78
- }
79
- export function handleDownload(res, namespace, hash) {
80
- const parsed = DownloadParamsSchema.safeParse({ namespace, hash });
81
- if (!parsed.success) {
82
- writeJsonError(res, 400, 'Invalid namespace or hash', 'BAD_REQUEST');
83
- return;
84
- }
85
- const cacheKey = `${parsed.data.namespace}:${parsed.data.hash}`;
86
- const entry = cacheGet(cacheKey, { force: true });
87
- if (!entry) {
88
- writeJsonError(res, 404, 'Not found or expired', 'NOT_FOUND');
89
- return;
90
- }
91
- const payload = parseCachedPayload(entry.content);
92
- const content = payload ? resolveCachedPayloadContent(payload) : null;
93
- if (!content) {
94
- writeJsonError(res, 404, 'Content missing', 'NOT_FOUND');
95
- return;
96
- }
97
- const fileName = generateSafeFilename(entry.url, payload?.title, parsed.data.hash);
98
- // Safe header generation — RFC 5987 encoding for non-ASCII filenames
99
- const encoded = encodeURIComponent(fileName).replace(/'/g, '%27');
100
- res.setHeader('Content-Type', 'text/markdown; charset=utf-8');
101
- res.setHeader('Content-Disposition', `attachment; filename="${fileName}"; filename*=UTF-8''${encoded}`);
102
- res.setHeader('Cache-Control', `private, max-age=${config.cache.ttl}`);
103
- res.setHeader('X-Content-Type-Options', 'nosniff');
104
- res.end(content);
105
- }
106
- //# sourceMappingURL=download.js.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"download.js","sourceRoot":"","sources":["../../src/lib/download.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,KAAK,IAAI,SAAS,EAAE,MAAM,WAAW,CAAC;AAE/C,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,OAAO,EACL,GAAG,IAAI,QAAQ,EACf,kBAAkB,EAClB,2BAA2B,GAC5B,MAAM,YAAY,CAAC;AACpB,OAAO,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAErC;;uGAEuG;AAEvG,MAAM,cAAc,GAAG;IACrB,OAAO,EAAE,GAAG;IACZ,YAAY,EAAE,qBAAqB;IACnC,UAAU,EAAE,MAAM;IAClB,UAAU,EAAE,2BAA2B;CAC/B,CAAC;AAEX,SAAS,cAAc,CAAC,KAAa;IACnC,OAAO,KAAK;SACT,WAAW,EAAE;SACb,OAAO,CAAC,cAAc,CAAC,YAAY,EAAE,EAAE,CAAC;SACxC,OAAO,CAAC,cAAc,CAAC,UAAU,EAAE,GAAG,CAAC;SACvC,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC;SACnB,OAAO,CAAC,YAAY,EAAE,EAAE,CAAC,CAAC;AAC/B,CAAC;AAED,SAAS,2BAA2B,CAAC,GAAW;IAC9C,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;IAC5B,IAAI,MAAM,CAAC,QAAQ,KAAK,OAAO,IAAI,MAAM,CAAC,QAAQ,KAAK,QAAQ;QAAE,OAAO,IAAI,CAAC;IAE7E,MAAM,QAAQ,GAAG,SAAS,CAAC,QAAQ,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;IACrD,IAAI,CAAC,QAAQ,IAAI,QAAQ,KAAK,OAAO;QAAE,OAAO,IAAI,CAAC;IAEnD,MAAM,OAAO,GAAG,QAAQ,CAAC,OAAO,CAAC,cAAc,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC;IAChE,MAAM,SAAS,GAAG,cAAc,CAAC,OAAO,CAAC,CAAC;IAE1C,IAAI,SAAS,KAAK,OAAO;QAAE,OAAO,IAAI,CAAC;IACvC,OAAO,SAAS,IAAI,IAAI,CAAC;AAC3B,CAAC;AAED,SAAS,oBAAoB,CAAC,IAAY,EAAE,SAAiB;IAC3D,MAAM,OAAO,GAAG,cAAc,CAAC,OAAO,GAAG,SAAS,CAAC,MAAM,CAAC;IAC1D,OAAO,IAAI,CAAC,MAAM,GAAG,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;AACnE,CAAC;AAED,SAAS,6BAA6B,CAAC,KAAc;IACnD,IAAI,CAAC,KAAK;QAAE,OAAO,IAAI,CAAC;IACxB,OAAO,cAAc,CAAC,KAAK,CAAC,IAAI,IAAI,CAAC;AACvC,CAAC;AAED,SAAS,mBAAmB,CAC1B,GAAW,EACX,KAAc,EACd,YAAqB;IAErB,IAAI,CAAC;QACH,MAAM,OAAO,GAAG,2BAA2B,CAAC,GAAG,CAAC,CAAC;QACjD,IAAI,OAAO;YAAE,OAAO,OAAO,CAAC;IAC9B,CAAC;IAAC,MAAM,CAAC;QACP,oDAAoD;IACtD,CAAC;IAED,MAAM,SAAS,GAAG,6BAA6B,CAAC,KAAK,CAAC,CAAC;IACvD,IAAI,SAAS;QAAE,OAAO,SAAS,CAAC;IAEhC,IAAI,YAAY;QAAE,OAAO,YAAY,CAAC,SAAS,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IACvD,OAAO,YAAY,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC;AAClC,CAAC;AAED,MAAM,UAAU,oBAAoB,CAClC,GAAW,EACX,KAAc,EACd,YAAqB,EACrB,SAAS,GAAG,KAAK;IAEjB,MAAM,IAAI,GAAG,mBAAmB,CAAC,GAAG,EAAE,KAAK,EAAE,YAAY,CAAC,CAAC;IAE3D,OAAO,GAAG,oBAAoB,CAAC,IAAI,EAAE,SAAS,CAAC,GAAG,SAAS,EAAE,CAAC;AAChE,CAAC;AAED;;uGAEuG;AAEvG,MAAM,oBAAoB,GAAG,CAAC,CAAC,YAAY,CAAC;IAC1C,SAAS,EAAE,CAAC,CAAC,OAAO,CAAC,UAAU,CAAC;IAChC,IAAI,EAAE,CAAC;SACJ,MAAM,EAAE;SACR,KAAK,CAAC,eAAe,CAAC;SACtB,GAAG,CAAC,CAAC,CAAC;SACN,GAAG,CAAC,EAAE,CAAC;CACX,CAAC,CAAC;AAEH,SAAS,cAAc,CACrB,GAAmB,EACnB,MAAc,EACd,OAAe,EACf,IAAY;IAEZ,GAAG,CAAC,SAAS,CAAC,MAAM,EAAE,EAAE,cAAc,EAAE,kBAAkB,EAAE,CAAC,CAAC;IAC9D,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,KAAK,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;AACpD,CAAC;AAED,MAAM,UAAU,cAAc,CAC5B,GAAmB,EACnB,SAAiB,EACjB,IAAY;IAEZ,MAAM,MAAM,GAAG,oBAAoB,CAAC,SAAS,CAAC,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IACnE,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;QACpB,cAAc,CAAC,GAAG,EAAE,GAAG,EAAE,2BAA2B,EAAE,aAAa,CAAC,CAAC;QACrE,OAAO;IACT,CAAC;IAED,MAAM,QAAQ,GAAG,GAAG,MAAM,CAAC,IAAI,CAAC,SAAS,IAAI,MAAM,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;IAChE,MAAM,KAAK,GAAG,QAAQ,CAAC,QAAQ,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;IAElD,IAAI,CAAC,KAAK,EAAE,CAAC;QACX,cAAc,CAAC,GAAG,EAAE,GAAG,EAAE,sBAAsB,EAAE,WAAW,CAAC,CAAC;QAC9D,OAAO;IACT,CAAC;IAED,MAAM,OAAO,GAAG,kBAAkB,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;IAClD,MAAM,OAAO,GAAG,OAAO,CAAC,CAAC,CAAC,2BAA2B,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IAEtE,IAAI,CAAC,OAAO,EAAE,CAAC;QACb,cAAc,CAAC,GAAG,EAAE,GAAG,EAAE,iBAAiB,EAAE,WAAW,CAAC,CAAC;QACzD,OAAO;IACT,CAAC;IAED,MAAM,QAAQ,GAAG,oBAAoB,CACnC,KAAK,CAAC,GAAG,EACT,OAAO,EAAE,KAAK,EACd,MAAM,CAAC,IAAI,CAAC,IAAI,CACjB,CAAC;IAEF,qEAAqE;IACrE,MAAM,OAAO,GAAG,kBAAkB,CAAC,QAAQ,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;IAElE,GAAG,CAAC,SAAS,CAAC,cAAc,EAAE,8BAA8B,CAAC,CAAC;IAC9D,GAAG,CAAC,SAAS,CACX,qBAAqB,EACrB,yBAAyB,QAAQ,uBAAuB,OAAO,EAAE,CAClE,CAAC;IACF,GAAG,CAAC,SAAS,CAAC,eAAe,EAAE,oBAAoB,MAAM,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,CAAC;IACvE,GAAG,CAAC,SAAS,CAAC,wBAAwB,EAAE,SAAS,CAAC,CAAC;IACnD,GAAG,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;AACnB,CAAC"}
@@ -1,14 +0,0 @@
1
- export declare class FetchError extends Error {
2
- readonly url: string;
3
- readonly statusCode: number;
4
- readonly code: string;
5
- readonly details: Readonly<Record<string, unknown>>;
6
- constructor(message: string, url: string, httpStatus?: number, details?: Record<string, unknown>, options?: ErrorOptions);
7
- }
8
- export declare function getErrorMessage(error: unknown): string;
9
- export declare function toError(error: unknown): Error;
10
- export declare function isAbortError(error: unknown): boolean;
11
- export declare function createErrorWithCode(message: string, code: string, options?: ErrorOptions): NodeJS.ErrnoException;
12
- export declare function isSystemError(error: unknown): error is NodeJS.ErrnoException;
13
- export declare const RESOURCE_NOT_FOUND_ERROR_CODE = -32002;
14
- //# sourceMappingURL=errors.d.ts.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"errors.d.ts","sourceRoot":"","sources":["../../src/lib/errors.ts"],"names":[],"mappings":"AAMA,qBAAa,UAAW,SAAQ,KAAK;IAOjC,QAAQ,CAAC,GAAG,EAAE,MAAM;IANtB,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,OAAO,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC;gBAGlD,OAAO,EAAE,MAAM,EACN,GAAG,EAAE,MAAM,EACpB,UAAU,CAAC,EAAE,MAAM,EACnB,OAAO,GAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAM,EACrC,OAAO,CAAC,EAAE,YAAY;CASzB;AAED,wBAAgB,eAAe,CAAC,KAAK,EAAE,OAAO,GAAG,MAAM,CAKtD;AAED,wBAAgB,OAAO,CAAC,KAAK,EAAE,OAAO,GAAG,KAAK,CAE7C;AAED,wBAAgB,YAAY,CAAC,KAAK,EAAE,OAAO,GAAG,OAAO,CAEpD;AA2BD,wBAAgB,mBAAmB,CACjC,OAAO,EAAE,MAAM,EACf,IAAI,EAAE,MAAM,EACZ,OAAO,CAAC,EAAE,YAAY,GACrB,MAAM,CAAC,cAAc,CAGvB;AAED,wBAAgB,aAAa,CAAC,KAAK,EAAE,OAAO,GAAG,KAAK,IAAI,MAAM,CAAC,cAAc,CAK5E;AAED,eAAO,MAAM,6BAA6B,SAAS,CAAC"}