@j0hanz/fetch-url-mcp 1.4.0 → 1.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli.d.ts +2 -3
- package/dist/cli.js +1 -2
- package/dist/http/auth.d.ts +5 -3
- package/dist/http/auth.js +64 -15
- package/dist/http/health.d.ts +1 -2
- package/dist/http/health.js +7 -18
- package/dist/http/helpers.d.ts +3 -4
- package/dist/http/helpers.js +21 -21
- package/dist/http/native.d.ts +0 -1
- package/dist/http/native.js +34 -26
- package/dist/http/rate-limit.d.ts +0 -1
- package/dist/http/rate-limit.js +3 -4
- package/dist/index.d.ts +0 -1
- package/dist/index.js +17 -18
- package/dist/lib/{markdown-cleanup.d.ts → content.d.ts} +4 -2
- package/dist/lib/content.js +1356 -0
- package/dist/lib/core.d.ts +253 -0
- package/dist/lib/core.js +1228 -0
- package/dist/lib/{tool-pipeline.d.ts → fetch-pipeline.d.ts} +1 -2
- package/dist/lib/{tool-pipeline.js → fetch-pipeline.js} +10 -19
- package/dist/lib/{fetch.d.ts → http.d.ts} +7 -9
- package/dist/lib/{fetch.js → http.js} +706 -944
- package/dist/lib/mcp-tools.d.ts +28 -0
- package/dist/lib/mcp-tools.js +107 -0
- package/dist/lib/{tool-progress.d.ts → progress.d.ts} +0 -1
- package/dist/lib/{tool-progress.js → progress.js} +8 -13
- package/dist/lib/task-handlers.d.ts +5 -0
- package/dist/lib/{mcp.js → task-handlers.js} +56 -12
- package/dist/lib/url.d.ts +70 -0
- package/dist/lib/url.js +686 -0
- package/dist/lib/utils.d.ts +58 -0
- package/dist/lib/utils.js +304 -0
- package/dist/prompts/index.d.ts +0 -1
- package/dist/prompts/index.js +0 -1
- package/dist/resources/index.d.ts +0 -1
- package/dist/resources/index.js +74 -33
- package/dist/resources/instructions.d.ts +0 -1
- package/dist/resources/instructions.js +2 -2
- package/dist/schemas/inputs.d.ts +0 -1
- package/dist/schemas/inputs.js +2 -3
- package/dist/schemas/outputs.d.ts +0 -1
- package/dist/schemas/outputs.js +1 -2
- package/dist/server.d.ts +0 -1
- package/dist/server.js +16 -26
- package/dist/tasks/execution.d.ts +0 -1
- package/dist/tasks/execution.js +27 -24
- package/dist/tasks/manager.d.ts +7 -3
- package/dist/tasks/manager.js +53 -34
- package/dist/tasks/owner.d.ts +1 -2
- package/dist/tasks/owner.js +1 -2
- package/dist/tasks/tool-registry.d.ts +1 -2
- package/dist/tasks/tool-registry.js +0 -1
- package/dist/tools/fetch-url.d.ts +1 -2
- package/dist/tools/fetch-url.js +39 -31
- package/dist/tools/index.d.ts +0 -1
- package/dist/tools/index.js +0 -1
- package/dist/transform/html-translators.d.ts +1 -0
- package/dist/transform/html-translators.js +454 -0
- package/dist/transform/metadata.d.ts +4 -0
- package/dist/transform/metadata.js +183 -0
- package/dist/transform/transform.d.ts +0 -1
- package/dist/transform/transform.js +24 -641
- package/dist/transform/types.d.ts +9 -11
- package/dist/transform/types.js +0 -1
- package/dist/transform/worker-pool.d.ts +0 -1
- package/dist/transform/worker-pool.js +7 -16
- package/dist/transform/workers/shared.d.ts +0 -1
- package/dist/transform/workers/shared.js +1 -2
- package/dist/transform/workers/transform-child.d.ts +0 -1
- package/dist/transform/workers/transform-child.js +0 -1
- package/dist/transform/workers/transform-worker.d.ts +0 -1
- package/dist/transform/workers/transform-worker.js +0 -1
- package/package.json +6 -3
- package/dist/cli.d.ts.map +0 -1
- package/dist/cli.js.map +0 -1
- package/dist/http/auth.d.ts.map +0 -1
- package/dist/http/auth.js.map +0 -1
- package/dist/http/health.d.ts.map +0 -1
- package/dist/http/health.js.map +0 -1
- package/dist/http/helpers.d.ts.map +0 -1
- package/dist/http/helpers.js.map +0 -1
- package/dist/http/native.d.ts.map +0 -1
- package/dist/http/native.js.map +0 -1
- package/dist/http/rate-limit.d.ts.map +0 -1
- package/dist/http/rate-limit.js.map +0 -1
- package/dist/index.d.ts.map +0 -1
- package/dist/index.js.map +0 -1
- package/dist/lib/cache.d.ts +0 -54
- package/dist/lib/cache.d.ts.map +0 -1
- package/dist/lib/cache.js +0 -264
- package/dist/lib/cache.js.map +0 -1
- package/dist/lib/config.d.ts +0 -143
- package/dist/lib/config.d.ts.map +0 -1
- package/dist/lib/config.js +0 -476
- package/dist/lib/config.js.map +0 -1
- package/dist/lib/crypto.d.ts +0 -4
- package/dist/lib/crypto.d.ts.map +0 -1
- package/dist/lib/crypto.js +0 -56
- package/dist/lib/crypto.js.map +0 -1
- package/dist/lib/dom-noise-removal.d.ts +0 -2
- package/dist/lib/dom-noise-removal.d.ts.map +0 -1
- package/dist/lib/dom-noise-removal.js +0 -494
- package/dist/lib/dom-noise-removal.js.map +0 -1
- package/dist/lib/download.d.ts +0 -4
- package/dist/lib/download.d.ts.map +0 -1
- package/dist/lib/download.js +0 -106
- package/dist/lib/download.js.map +0 -1
- package/dist/lib/errors.d.ts +0 -14
- package/dist/lib/errors.d.ts.map +0 -1
- package/dist/lib/errors.js +0 -72
- package/dist/lib/errors.js.map +0 -1
- package/dist/lib/fetch-content.d.ts +0 -5
- package/dist/lib/fetch-content.d.ts.map +0 -1
- package/dist/lib/fetch-content.js +0 -164
- package/dist/lib/fetch-content.js.map +0 -1
- package/dist/lib/fetch-stream.d.ts +0 -5
- package/dist/lib/fetch-stream.d.ts.map +0 -1
- package/dist/lib/fetch-stream.js +0 -29
- package/dist/lib/fetch-stream.js.map +0 -1
- package/dist/lib/fetch.d.ts.map +0 -1
- package/dist/lib/fetch.js.map +0 -1
- package/dist/lib/host-normalization.d.ts +0 -2
- package/dist/lib/host-normalization.d.ts.map +0 -1
- package/dist/lib/host-normalization.js +0 -91
- package/dist/lib/host-normalization.js.map +0 -1
- package/dist/lib/ip-blocklist.d.ts +0 -9
- package/dist/lib/ip-blocklist.d.ts.map +0 -1
- package/dist/lib/ip-blocklist.js +0 -79
- package/dist/lib/ip-blocklist.js.map +0 -1
- package/dist/lib/json.d.ts +0 -2
- package/dist/lib/json.d.ts.map +0 -1
- package/dist/lib/json.js +0 -45
- package/dist/lib/json.js.map +0 -1
- package/dist/lib/language-detection.d.ts +0 -3
- package/dist/lib/language-detection.d.ts.map +0 -1
- package/dist/lib/language-detection.js +0 -355
- package/dist/lib/language-detection.js.map +0 -1
- package/dist/lib/markdown-cleanup.d.ts.map +0 -1
- package/dist/lib/markdown-cleanup.js +0 -532
- package/dist/lib/markdown-cleanup.js.map +0 -1
- package/dist/lib/mcp-lifecycle.d.ts +0 -5
- package/dist/lib/mcp-lifecycle.d.ts.map +0 -1
- package/dist/lib/mcp-lifecycle.js +0 -51
- package/dist/lib/mcp-lifecycle.js.map +0 -1
- package/dist/lib/mcp-validator.d.ts +0 -17
- package/dist/lib/mcp-validator.d.ts.map +0 -1
- package/dist/lib/mcp-validator.js +0 -45
- package/dist/lib/mcp-validator.js.map +0 -1
- package/dist/lib/mcp.d.ts +0 -4
- package/dist/lib/mcp.d.ts.map +0 -1
- package/dist/lib/mcp.js.map +0 -1
- package/dist/lib/observability.d.ts +0 -23
- package/dist/lib/observability.d.ts.map +0 -1
- package/dist/lib/observability.js +0 -238
- package/dist/lib/observability.js.map +0 -1
- package/dist/lib/server-tuning.d.ts +0 -15
- package/dist/lib/server-tuning.d.ts.map +0 -1
- package/dist/lib/server-tuning.js +0 -49
- package/dist/lib/server-tuning.js.map +0 -1
- package/dist/lib/session.d.ts +0 -45
- package/dist/lib/session.d.ts.map +0 -1
- package/dist/lib/session.js +0 -263
- package/dist/lib/session.js.map +0 -1
- package/dist/lib/timer-utils.d.ts +0 -13
- package/dist/lib/timer-utils.d.ts.map +0 -1
- package/dist/lib/timer-utils.js +0 -44
- package/dist/lib/timer-utils.js.map +0 -1
- package/dist/lib/tool-errors.d.ts +0 -12
- package/dist/lib/tool-errors.d.ts.map +0 -1
- package/dist/lib/tool-errors.js +0 -55
- package/dist/lib/tool-errors.js.map +0 -1
- package/dist/lib/tool-pipeline.d.ts.map +0 -1
- package/dist/lib/tool-pipeline.js.map +0 -1
- package/dist/lib/tool-progress.d.ts.map +0 -1
- package/dist/lib/tool-progress.js.map +0 -1
- package/dist/lib/type-guards.d.ts +0 -16
- package/dist/lib/type-guards.d.ts.map +0 -1
- package/dist/lib/type-guards.js +0 -13
- package/dist/lib/type-guards.js.map +0 -1
- package/dist/prompts/index.d.ts.map +0 -1
- package/dist/prompts/index.js.map +0 -1
- package/dist/resources/index.d.ts.map +0 -1
- package/dist/resources/index.js.map +0 -1
- package/dist/resources/instructions.d.ts.map +0 -1
- package/dist/resources/instructions.js.map +0 -1
- package/dist/schemas/inputs.d.ts.map +0 -1
- package/dist/schemas/inputs.js.map +0 -1
- package/dist/schemas/outputs.d.ts.map +0 -1
- package/dist/schemas/outputs.js.map +0 -1
- package/dist/server.d.ts.map +0 -1
- package/dist/server.js.map +0 -1
- package/dist/tasks/execution.d.ts.map +0 -1
- package/dist/tasks/execution.js.map +0 -1
- package/dist/tasks/manager.d.ts.map +0 -1
- package/dist/tasks/manager.js.map +0 -1
- package/dist/tasks/owner.d.ts.map +0 -1
- package/dist/tasks/owner.js.map +0 -1
- package/dist/tasks/tool-registry.d.ts.map +0 -1
- package/dist/tasks/tool-registry.js.map +0 -1
- package/dist/tools/fetch-url.d.ts.map +0 -1
- package/dist/tools/fetch-url.js.map +0 -1
- package/dist/tools/index.d.ts.map +0 -1
- package/dist/tools/index.js.map +0 -1
- package/dist/transform/transform.d.ts.map +0 -1
- package/dist/transform/transform.js.map +0 -1
- package/dist/transform/types.d.ts.map +0 -1
- package/dist/transform/types.js.map +0 -1
- package/dist/transform/worker-pool.d.ts.map +0 -1
- package/dist/transform/worker-pool.js.map +0 -1
- package/dist/transform/workers/shared.d.ts.map +0 -1
- package/dist/transform/workers/shared.js.map +0 -1
- package/dist/transform/workers/transform-child.d.ts.map +0 -1
- package/dist/transform/workers/transform-child.js.map +0 -1
- package/dist/transform/workers/transform-worker.d.ts.map +0 -1
- package/dist/transform/workers/transform-worker.js.map +0 -1
|
@@ -1,494 +0,0 @@
|
|
|
1
|
-
import { parseHTML } from 'linkedom';
|
|
2
|
-
import { config } from './config.js';
|
|
3
|
-
import { logDebug } from './observability.js';
|
|
4
|
-
// --- Constants & Pre-compiled Regex ---
|
|
5
|
-
const NOISE_SCAN_LIMIT = 50_000;
|
|
6
|
-
const MIN_BODY_CONTENT_LENGTH = 100;
|
|
7
|
-
const DIALOG_MIN_CHARS_FOR_PRESERVATION = 500;
|
|
8
|
-
const NAV_FOOTER_MIN_CHARS_FOR_PRESERVATION = 500;
|
|
9
|
-
// Merged markers for fast rejection
|
|
10
|
-
const HTML_DOCUMENT_MARKERS = /<\s*(?:!doctype|html|head|body)\b/i;
|
|
11
|
-
const HTML_FRAGMENT_MARKERS = /<\s*(?:article|main|section|div|nav|footer|header|aside|table|ul|ol)\b/i;
|
|
12
|
-
// Split into smaller regexes to stay within sonarjs/regex-complexity limit
|
|
13
|
-
const NOISE_PATTERNS = [
|
|
14
|
-
/<\s*(?:script|style|noscript|iframe|nav|footer|header|form|button|input|select|textarea|svg|canvas)\b/i,
|
|
15
|
-
/[\s"']role\s*=\s*['"]?(?:navigation|banner|complementary|contentinfo|tree|menubar|menu)['"]?/i,
|
|
16
|
-
/[\s"'](?:aria-hidden\s*=\s*['"]?true['"]?|hidden)/i,
|
|
17
|
-
/[\s"'](?:banner|promo|announcement|cta|advert|newsletter|subscribe|cookie|consent|popup|modal|overlay|toast)\b/i,
|
|
18
|
-
/[\s"'](?:fixed|sticky|z-50|z-4|isolate|breadcrumb|pagination)\b/i,
|
|
19
|
-
];
|
|
20
|
-
const HEADER_NOISE_PATTERN = /\b(site-header|masthead|topbar|navbar|nav(?:bar)?|menu|header-nav)\b/i;
|
|
21
|
-
const FIXED_OR_HIGH_Z_PATTERN = /\b(?:fixed|sticky|z-(?:4\d|50)|isolate)\b/;
|
|
22
|
-
const SKIP_URL_PREFIXES = [
|
|
23
|
-
'#',
|
|
24
|
-
'java' + 'script:',
|
|
25
|
-
'mailto:',
|
|
26
|
-
'tel:',
|
|
27
|
-
'data:',
|
|
28
|
-
'blob:',
|
|
29
|
-
];
|
|
30
|
-
const BASE_STRUCTURAL_TAGS = new Set([
|
|
31
|
-
'script',
|
|
32
|
-
'style',
|
|
33
|
-
'noscript',
|
|
34
|
-
'iframe',
|
|
35
|
-
'form',
|
|
36
|
-
'button',
|
|
37
|
-
'input',
|
|
38
|
-
'select',
|
|
39
|
-
'textarea',
|
|
40
|
-
]);
|
|
41
|
-
const ALWAYS_NOISE_TAGS = new Set(['nav', 'footer']);
|
|
42
|
-
const NAVIGATION_ROLES = new Set([
|
|
43
|
-
'navigation',
|
|
44
|
-
'banner',
|
|
45
|
-
'complementary',
|
|
46
|
-
'contentinfo',
|
|
47
|
-
'tree',
|
|
48
|
-
'menubar',
|
|
49
|
-
'menu',
|
|
50
|
-
'dialog',
|
|
51
|
-
'alertdialog',
|
|
52
|
-
'search',
|
|
53
|
-
]);
|
|
54
|
-
const INTERACTIVE_CONTENT_ROLES = new Set([
|
|
55
|
-
'tabpanel',
|
|
56
|
-
'tab',
|
|
57
|
-
'tablist',
|
|
58
|
-
'dialog',
|
|
59
|
-
'alertdialog',
|
|
60
|
-
'menu',
|
|
61
|
-
'menuitem',
|
|
62
|
-
'option',
|
|
63
|
-
'listbox',
|
|
64
|
-
'combobox',
|
|
65
|
-
'tooltip',
|
|
66
|
-
'alert',
|
|
67
|
-
]);
|
|
68
|
-
const PROMO_TOKENS_ALWAYS = [
|
|
69
|
-
'banner',
|
|
70
|
-
'promo',
|
|
71
|
-
'announcement',
|
|
72
|
-
'cta',
|
|
73
|
-
'advert',
|
|
74
|
-
'ads',
|
|
75
|
-
'sponsor',
|
|
76
|
-
'recommend',
|
|
77
|
-
'breadcrumb',
|
|
78
|
-
'pagination',
|
|
79
|
-
'pager',
|
|
80
|
-
'taglist',
|
|
81
|
-
];
|
|
82
|
-
const PROMO_TOKENS_AGGRESSIVE = ['ad', 'related', 'comment'];
|
|
83
|
-
const PROMO_TOKENS_BY_CATEGORY = {
|
|
84
|
-
'cookie-banners': ['cookie', 'consent', 'popup', 'modal', 'overlay', 'toast'],
|
|
85
|
-
newsletters: ['newsletter', 'subscribe'],
|
|
86
|
-
'social-share': ['share', 'social'],
|
|
87
|
-
};
|
|
88
|
-
const BASE_NOISE_SELECTORS = {
|
|
89
|
-
navFooter: 'nav,footer,header[class*="site"],header[class*="nav"],header[class*="menu"],[role="banner"],[role="navigation"]',
|
|
90
|
-
cookieBanners: '[role="dialog"]',
|
|
91
|
-
hidden: '[style*="display: none"],[style*="display:none"],[hidden],[aria-hidden="true"]',
|
|
92
|
-
};
|
|
93
|
-
const NO_MATCH_REGEX = /a^/i;
|
|
94
|
-
// --- State Cache ---
|
|
95
|
-
let cachedContext;
|
|
96
|
-
let lastConfigRef;
|
|
97
|
-
// --- Helpers Inlined/Optimized ---
|
|
98
|
-
function escapeRegexLiteral(value) {
|
|
99
|
-
return value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
|
|
100
|
-
}
|
|
101
|
-
function buildTokenRegex(tokens) {
|
|
102
|
-
if (tokens.size === 0)
|
|
103
|
-
return NO_MATCH_REGEX;
|
|
104
|
-
return new RegExp(`(?:^|[^a-z0-9])(?:${[...tokens].map(escapeRegexLiteral).join('|')})(?:$|[^a-z0-9])`, 'i');
|
|
105
|
-
}
|
|
106
|
-
function addTokens(target, tokens) {
|
|
107
|
-
for (const token of tokens)
|
|
108
|
-
target.add(token);
|
|
109
|
-
}
|
|
110
|
-
function getPromoMatchers(currentConfig, flags) {
|
|
111
|
-
const baseTokens = new Set(PROMO_TOKENS_ALWAYS);
|
|
112
|
-
const aggressiveTokens = new Set();
|
|
113
|
-
if (currentConfig.aggressiveMode) {
|
|
114
|
-
addTokens(aggressiveTokens, PROMO_TOKENS_AGGRESSIVE);
|
|
115
|
-
}
|
|
116
|
-
if (flags.cookieBanners) {
|
|
117
|
-
addTokens(baseTokens, PROMO_TOKENS_BY_CATEGORY['cookie-banners']);
|
|
118
|
-
}
|
|
119
|
-
if (flags.newsletters) {
|
|
120
|
-
addTokens(baseTokens, PROMO_TOKENS_BY_CATEGORY['newsletters']);
|
|
121
|
-
}
|
|
122
|
-
if (flags.socialShare) {
|
|
123
|
-
addTokens(baseTokens, PROMO_TOKENS_BY_CATEGORY['social-share']);
|
|
124
|
-
}
|
|
125
|
-
for (const t of currentConfig.extraTokens) {
|
|
126
|
-
const n = t.toLowerCase().trim();
|
|
127
|
-
if (n)
|
|
128
|
-
baseTokens.add(n);
|
|
129
|
-
}
|
|
130
|
-
return {
|
|
131
|
-
base: buildTokenRegex(baseTokens),
|
|
132
|
-
aggressive: buildTokenRegex(aggressiveTokens),
|
|
133
|
-
};
|
|
134
|
-
}
|
|
135
|
-
function getContext() {
|
|
136
|
-
const currentConfig = config.noiseRemoval;
|
|
137
|
-
if (cachedContext !== undefined && lastConfigRef === currentConfig)
|
|
138
|
-
return cachedContext;
|
|
139
|
-
const enabled = new Set(currentConfig.enabledCategories
|
|
140
|
-
.map((c) => {
|
|
141
|
-
const s = c.toLowerCase().trim();
|
|
142
|
-
const { locale } = config.i18n;
|
|
143
|
-
return locale ? s.toLocaleLowerCase(locale) : s;
|
|
144
|
-
})
|
|
145
|
-
.filter(Boolean));
|
|
146
|
-
const isEnabled = (cat) => enabled.has(cat);
|
|
147
|
-
const flags = {
|
|
148
|
-
navFooter: isEnabled('nav-footer'),
|
|
149
|
-
cookieBanners: isEnabled('cookie-banners'),
|
|
150
|
-
newsletters: isEnabled('newsletters'),
|
|
151
|
-
socialShare: isEnabled('social-share'),
|
|
152
|
-
};
|
|
153
|
-
const structuralTags = new Set(BASE_STRUCTURAL_TAGS);
|
|
154
|
-
if (!currentConfig.preserveSvgCanvas) {
|
|
155
|
-
structuralTags.add('svg');
|
|
156
|
-
structuralTags.add('canvas');
|
|
157
|
-
}
|
|
158
|
-
const promoMatchers = getPromoMatchers(currentConfig, flags);
|
|
159
|
-
const extraSelectors = currentConfig.extraSelectors
|
|
160
|
-
.map((s) => s.trim())
|
|
161
|
-
.filter((s) => s.length > 0);
|
|
162
|
-
// Pre-build selectors
|
|
163
|
-
const selectors = [BASE_NOISE_SELECTORS.hidden];
|
|
164
|
-
if (flags.navFooter)
|
|
165
|
-
selectors.push(BASE_NOISE_SELECTORS.navFooter);
|
|
166
|
-
if (flags.cookieBanners)
|
|
167
|
-
selectors.push(BASE_NOISE_SELECTORS.cookieBanners);
|
|
168
|
-
const baseSelector = selectors.join(',');
|
|
169
|
-
const candidateSelector = [
|
|
170
|
-
...structuralTags,
|
|
171
|
-
...ALWAYS_NOISE_TAGS,
|
|
172
|
-
'aside',
|
|
173
|
-
'header',
|
|
174
|
-
'[class]',
|
|
175
|
-
'[id]',
|
|
176
|
-
'[role]',
|
|
177
|
-
'[style]',
|
|
178
|
-
].join(',');
|
|
179
|
-
cachedContext = {
|
|
180
|
-
flags,
|
|
181
|
-
structuralTags,
|
|
182
|
-
weights: currentConfig.weights,
|
|
183
|
-
promoMatchers,
|
|
184
|
-
promoEnabled: flags.cookieBanners || flags.newsletters || flags.socialShare,
|
|
185
|
-
extraSelectors,
|
|
186
|
-
baseSelector,
|
|
187
|
-
candidateSelector,
|
|
188
|
-
};
|
|
189
|
-
lastConfigRef = currentConfig;
|
|
190
|
-
return cachedContext;
|
|
191
|
-
}
|
|
192
|
-
// --- Hot Path Logic ---
|
|
193
|
-
function isInteractive(element, role) {
|
|
194
|
-
if (role && INTERACTIVE_CONTENT_ROLES.has(role))
|
|
195
|
-
return true;
|
|
196
|
-
const ds = element.getAttribute('data-state');
|
|
197
|
-
if (ds === 'inactive' || ds === 'closed')
|
|
198
|
-
return true;
|
|
199
|
-
const dataOrientation = element.getAttribute('data-orientation');
|
|
200
|
-
if (dataOrientation === 'horizontal' || dataOrientation === 'vertical')
|
|
201
|
-
return true;
|
|
202
|
-
return (element.hasAttribute('data-accordion-item') ||
|
|
203
|
-
element.hasAttribute('data-radix-collection-item'));
|
|
204
|
-
}
|
|
205
|
-
function isWithinPrimaryContent(element) {
|
|
206
|
-
let current = element;
|
|
207
|
-
while (current) {
|
|
208
|
-
const tagName = current.tagName.toLowerCase();
|
|
209
|
-
if (tagName === 'article' || tagName === 'main')
|
|
210
|
-
return true;
|
|
211
|
-
if (current.getAttribute('role') === 'main')
|
|
212
|
-
return true;
|
|
213
|
-
current = current.parentElement;
|
|
214
|
-
}
|
|
215
|
-
return false;
|
|
216
|
-
}
|
|
217
|
-
function shouldPreserve(element, tagName) {
|
|
218
|
-
// Check Dialog
|
|
219
|
-
const role = element.getAttribute('role');
|
|
220
|
-
if (role === 'dialog' || role === 'alertdialog') {
|
|
221
|
-
if (isWithinPrimaryContent(element))
|
|
222
|
-
return true;
|
|
223
|
-
const textLen = (element.textContent || '').length;
|
|
224
|
-
if (textLen > DIALOG_MIN_CHARS_FOR_PRESERVATION)
|
|
225
|
-
return true;
|
|
226
|
-
return element.querySelector('h1,h2,h3,h4,h5,h6') !== null;
|
|
227
|
-
}
|
|
228
|
-
// Check Nav/Footer
|
|
229
|
-
if (tagName === 'nav' || tagName === 'footer') {
|
|
230
|
-
if (element.querySelector('article,main,section,[role="main"]'))
|
|
231
|
-
return true;
|
|
232
|
-
return ((element.textContent || '').trim().length >=
|
|
233
|
-
NAV_FOOTER_MIN_CHARS_FOR_PRESERVATION);
|
|
234
|
-
}
|
|
235
|
-
return false;
|
|
236
|
-
}
|
|
237
|
-
function removeNodes(nodes) {
|
|
238
|
-
for (let i = nodes.length - 1; i >= 0; i--) {
|
|
239
|
-
const node = nodes[i];
|
|
240
|
-
if (node?.parentNode && !shouldPreserve(node, node.tagName.toLowerCase())) {
|
|
241
|
-
node.remove();
|
|
242
|
-
}
|
|
243
|
-
}
|
|
244
|
-
}
|
|
245
|
-
function scoreNavFooter(tagName, role, className, id, weights) {
|
|
246
|
-
let score = 0;
|
|
247
|
-
if (ALWAYS_NOISE_TAGS.has(tagName))
|
|
248
|
-
score += weights.structural;
|
|
249
|
-
// Header Boilerplate
|
|
250
|
-
if (tagName === 'header') {
|
|
251
|
-
if ((role && NAVIGATION_ROLES.has(role)) ||
|
|
252
|
-
HEADER_NOISE_PATTERN.test(`${className} ${id}`)) {
|
|
253
|
-
score += weights.structural;
|
|
254
|
-
}
|
|
255
|
-
}
|
|
256
|
-
// Role Noise
|
|
257
|
-
if (role && NAVIGATION_ROLES.has(role)) {
|
|
258
|
-
if (tagName !== 'aside' || role !== 'complementary') {
|
|
259
|
-
score += weights.structural;
|
|
260
|
-
}
|
|
261
|
-
}
|
|
262
|
-
return score;
|
|
263
|
-
}
|
|
264
|
-
function extractElementMetadata(element) {
|
|
265
|
-
const tagName = element.tagName.toLowerCase();
|
|
266
|
-
const className = element.getAttribute('class') ?? '';
|
|
267
|
-
const id = element.getAttribute('id') ?? '';
|
|
268
|
-
const role = element.getAttribute('role');
|
|
269
|
-
const style = element.getAttribute('style');
|
|
270
|
-
const _isInteractive = isInteractive(element, role);
|
|
271
|
-
const isHidden = element.hasAttribute('hidden') ||
|
|
272
|
-
element.getAttribute('aria-hidden') === 'true' ||
|
|
273
|
-
(style !== null &&
|
|
274
|
-
/\b(?:display\s*:\s*none|visibility\s*:\s*hidden)\b/i.test(style));
|
|
275
|
-
return {
|
|
276
|
-
tagName,
|
|
277
|
-
className,
|
|
278
|
-
id,
|
|
279
|
-
role,
|
|
280
|
-
style,
|
|
281
|
-
isInteractive: _isInteractive,
|
|
282
|
-
isHidden,
|
|
283
|
-
};
|
|
284
|
-
}
|
|
285
|
-
function isNoiseElement(element, context) {
|
|
286
|
-
const meta = extractElementMetadata(element);
|
|
287
|
-
let score = 0;
|
|
288
|
-
const { weights } = context;
|
|
289
|
-
// Structural
|
|
290
|
-
if (context.structuralTags.has(meta.tagName) && !meta.isInteractive) {
|
|
291
|
-
score += weights.structural;
|
|
292
|
-
}
|
|
293
|
-
// Nav/Footer Scoring
|
|
294
|
-
if (context.flags.navFooter) {
|
|
295
|
-
score += scoreNavFooter(meta.tagName, meta.role, meta.className, meta.id, weights);
|
|
296
|
-
}
|
|
297
|
-
// Hidden
|
|
298
|
-
if (meta.isHidden && !meta.isInteractive) {
|
|
299
|
-
score += weights.hidden;
|
|
300
|
-
}
|
|
301
|
-
// Sticky/Fixed
|
|
302
|
-
if (FIXED_OR_HIGH_Z_PATTERN.test(meta.className)) {
|
|
303
|
-
score += weights.stickyFixed;
|
|
304
|
-
}
|
|
305
|
-
// Promo
|
|
306
|
-
if (context.promoEnabled) {
|
|
307
|
-
const aggTest = context.promoMatchers.aggressive.test(meta.className) ||
|
|
308
|
-
context.promoMatchers.aggressive.test(meta.id);
|
|
309
|
-
const isAggressiveMatch = aggTest && !isWithinPrimaryContent(element);
|
|
310
|
-
const isBaseMatch = !aggTest &&
|
|
311
|
-
(context.promoMatchers.base.test(meta.className) ||
|
|
312
|
-
context.promoMatchers.base.test(meta.id));
|
|
313
|
-
if (isAggressiveMatch || isBaseMatch) {
|
|
314
|
-
score += weights.promo;
|
|
315
|
-
}
|
|
316
|
-
}
|
|
317
|
-
return score >= weights.threshold;
|
|
318
|
-
}
|
|
319
|
-
function cleanHeadingWrapperDivs(h) {
|
|
320
|
-
const divs = h.querySelectorAll('div');
|
|
321
|
-
for (let j = divs.length - 1; j >= 0; j--) {
|
|
322
|
-
const d = divs[j];
|
|
323
|
-
if (!d?.parentNode)
|
|
324
|
-
continue;
|
|
325
|
-
const cls = d.getAttribute('class') ?? '';
|
|
326
|
-
const stl = d.getAttribute('style') ?? '';
|
|
327
|
-
if (cls.includes('absolute') ||
|
|
328
|
-
stl.includes('position') ||
|
|
329
|
-
d.getAttribute('tabindex') === '-1') {
|
|
330
|
-
d.remove();
|
|
331
|
-
}
|
|
332
|
-
}
|
|
333
|
-
}
|
|
334
|
-
function cleanHeadingAnchors(h) {
|
|
335
|
-
const anchors = h.querySelectorAll('a');
|
|
336
|
-
for (let j = anchors.length - 1; j >= 0; j--) {
|
|
337
|
-
const a = anchors[j];
|
|
338
|
-
if (!a?.parentNode)
|
|
339
|
-
continue;
|
|
340
|
-
const href = a.getAttribute('href') ?? '';
|
|
341
|
-
const txt = (a.textContent || '').replace(/[\u200B\s]/g, '');
|
|
342
|
-
if (href.startsWith('#') && txt.length === 0) {
|
|
343
|
-
a.remove();
|
|
344
|
-
}
|
|
345
|
-
}
|
|
346
|
-
}
|
|
347
|
-
function cleanHeadingZeroWidth(h, document) {
|
|
348
|
-
const walker = document.createTreeWalker(h, 4); // SHOW_TEXT
|
|
349
|
-
let node;
|
|
350
|
-
while ((node = walker.nextNode())) {
|
|
351
|
-
if (node.textContent?.includes('\u200B')) {
|
|
352
|
-
node.textContent = node.textContent.replace(/\u200B/g, '');
|
|
353
|
-
}
|
|
354
|
-
}
|
|
355
|
-
}
|
|
356
|
-
function cleanHeadings(document) {
|
|
357
|
-
// Clean Heading Anchors
|
|
358
|
-
const headings = document.querySelectorAll('h1,h2,h3,h4,h5,h6');
|
|
359
|
-
for (const h of headings) {
|
|
360
|
-
if (!h.parentNode)
|
|
361
|
-
continue;
|
|
362
|
-
cleanHeadingWrapperDivs(h);
|
|
363
|
-
cleanHeadingAnchors(h);
|
|
364
|
-
cleanHeadingZeroWidth(h, document);
|
|
365
|
-
}
|
|
366
|
-
}
|
|
367
|
-
function stripNoise(document, context, signal) {
|
|
368
|
-
cleanHeadings(document);
|
|
369
|
-
// Remove Base & Extra
|
|
370
|
-
const { baseSelector, extraSelectors } = context;
|
|
371
|
-
// Base
|
|
372
|
-
const baseNodes = document.querySelectorAll(baseSelector);
|
|
373
|
-
removeNodes(baseNodes);
|
|
374
|
-
// Extra
|
|
375
|
-
if (extraSelectors.length > 0) {
|
|
376
|
-
const combinedExtra = extraSelectors.join(',');
|
|
377
|
-
const extraNodes = document.querySelectorAll(combinedExtra);
|
|
378
|
-
removeNodes(extraNodes);
|
|
379
|
-
}
|
|
380
|
-
// Candidates
|
|
381
|
-
const candidates = document.querySelectorAll(context.candidateSelector);
|
|
382
|
-
for (let i = candidates.length - 1; i >= 0; i--) {
|
|
383
|
-
if (i % 500 === 0 && signal?.aborted) {
|
|
384
|
-
throw new Error('Noise removal aborted');
|
|
385
|
-
}
|
|
386
|
-
const node = candidates[i];
|
|
387
|
-
if (!node)
|
|
388
|
-
continue;
|
|
389
|
-
if (!node.parentNode)
|
|
390
|
-
continue;
|
|
391
|
-
if (shouldPreserve(node, node.tagName.toLowerCase()))
|
|
392
|
-
continue;
|
|
393
|
-
if (isNoiseElement(node, context)) {
|
|
394
|
-
node.remove();
|
|
395
|
-
}
|
|
396
|
-
}
|
|
397
|
-
}
|
|
398
|
-
function processUrlElement(el, attr, base, isSrcset) {
|
|
399
|
-
if (!el.parentNode)
|
|
400
|
-
return;
|
|
401
|
-
if (isSrcset) {
|
|
402
|
-
const val = el.getAttribute(attr);
|
|
403
|
-
if (val) {
|
|
404
|
-
const newVal = val
|
|
405
|
-
.split(',')
|
|
406
|
-
.map((entry) => {
|
|
407
|
-
const parts = entry.trim().split(/\s+/);
|
|
408
|
-
if (!parts[0])
|
|
409
|
-
return entry;
|
|
410
|
-
try {
|
|
411
|
-
parts[0] = new URL(parts[0], base).href;
|
|
412
|
-
}
|
|
413
|
-
catch {
|
|
414
|
-
/* ignore */
|
|
415
|
-
}
|
|
416
|
-
return parts.join(' ');
|
|
417
|
-
})
|
|
418
|
-
.join(', ');
|
|
419
|
-
el.setAttribute(attr, newVal);
|
|
420
|
-
}
|
|
421
|
-
return;
|
|
422
|
-
}
|
|
423
|
-
const val = el.getAttribute(attr);
|
|
424
|
-
if (val &&
|
|
425
|
-
!SKIP_URL_PREFIXES.some((p) => val.trim().toLowerCase().startsWith(p))) {
|
|
426
|
-
try {
|
|
427
|
-
el.setAttribute(attr, new URL(val, base).href);
|
|
428
|
-
}
|
|
429
|
-
catch {
|
|
430
|
-
/* ignore */
|
|
431
|
-
}
|
|
432
|
-
}
|
|
433
|
-
}
|
|
434
|
-
function resolveUrls(document, baseUrlStr) {
|
|
435
|
-
let base;
|
|
436
|
-
try {
|
|
437
|
-
base = new URL(baseUrlStr);
|
|
438
|
-
}
|
|
439
|
-
catch {
|
|
440
|
-
return;
|
|
441
|
-
}
|
|
442
|
-
const elements = document.querySelectorAll('a[href],img[src],source[srcset]');
|
|
443
|
-
for (const el of elements) {
|
|
444
|
-
const tag = el.tagName.toLowerCase();
|
|
445
|
-
if (tag === 'a')
|
|
446
|
-
processUrlElement(el, 'href', base, false);
|
|
447
|
-
else if (tag === 'img')
|
|
448
|
-
processUrlElement(el, 'src', base, false);
|
|
449
|
-
else if (tag === 'source')
|
|
450
|
-
processUrlElement(el, 'srcset', base, true);
|
|
451
|
-
}
|
|
452
|
-
}
|
|
453
|
-
function serialize(document, fallback) {
|
|
454
|
-
const bodyHtml = document.body.innerHTML;
|
|
455
|
-
if (bodyHtml.trim().length > MIN_BODY_CONTENT_LENGTH)
|
|
456
|
-
return bodyHtml;
|
|
457
|
-
const outerHtml = document.documentElement.outerHTML;
|
|
458
|
-
if (outerHtml.trim().length > MIN_BODY_CONTENT_LENGTH)
|
|
459
|
-
return outerHtml;
|
|
460
|
-
return fallback;
|
|
461
|
-
}
|
|
462
|
-
function isFullDocumentHtml(html) {
|
|
463
|
-
return HTML_DOCUMENT_MARKERS.test(html);
|
|
464
|
-
}
|
|
465
|
-
function mayContainNoise(html) {
|
|
466
|
-
const sample = html.length <= NOISE_SCAN_LIMIT
|
|
467
|
-
? html
|
|
468
|
-
: `${html.substring(0, NOISE_SCAN_LIMIT)}\n${html.substring(html.length - NOISE_SCAN_LIMIT)}`;
|
|
469
|
-
return NOISE_PATTERNS.some((re) => re.test(sample));
|
|
470
|
-
}
|
|
471
|
-
export function removeNoiseFromHtml(html, document, baseUrl, signal) {
|
|
472
|
-
const shouldParse = isFullDocumentHtml(html) ||
|
|
473
|
-
mayContainNoise(html) ||
|
|
474
|
-
HTML_FRAGMENT_MARKERS.test(html);
|
|
475
|
-
if (!shouldParse)
|
|
476
|
-
return html;
|
|
477
|
-
try {
|
|
478
|
-
const context = getContext();
|
|
479
|
-
if (config.noiseRemoval.debug) {
|
|
480
|
-
logDebug('Noise removal audit enabled', {
|
|
481
|
-
categories: [...(context.flags.navFooter ? ['nav-footer'] : [])],
|
|
482
|
-
});
|
|
483
|
-
}
|
|
484
|
-
const doc = document ?? parseHTML(html).document;
|
|
485
|
-
stripNoise(doc, context, signal);
|
|
486
|
-
if (baseUrl)
|
|
487
|
-
resolveUrls(doc, baseUrl);
|
|
488
|
-
return serialize(doc, html);
|
|
489
|
-
}
|
|
490
|
-
catch {
|
|
491
|
-
return html;
|
|
492
|
-
}
|
|
493
|
-
}
|
|
494
|
-
//# sourceMappingURL=dom-noise-removal.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"dom-noise-removal.js","sourceRoot":"","sources":["../../src/lib/dom-noise-removal.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,UAAU,CAAC;AAErC,OAAO,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AACrC,OAAO,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AAE9C,yCAAyC;AAEzC,MAAM,gBAAgB,GAAG,MAAM,CAAC;AAChC,MAAM,uBAAuB,GAAG,GAAG,CAAC;AACpC,MAAM,iCAAiC,GAAG,GAAG,CAAC;AAC9C,MAAM,qCAAqC,GAAG,GAAG,CAAC;AAElD,oCAAoC;AACpC,MAAM,qBAAqB,GAAG,oCAAoC,CAAC;AACnE,MAAM,qBAAqB,GACzB,yEAAyE,CAAC;AAE5E,2EAA2E;AAC3E,MAAM,cAAc,GAAsB;IACxC,wGAAwG;IACxG,+FAA+F;IAC/F,oDAAoD;IACpD,iHAAiH;IACjH,kEAAkE;CACnE,CAAC;AAEF,MAAM,oBAAoB,GACxB,uEAAuE,CAAC;AAC1E,MAAM,uBAAuB,GAAG,2CAA2C,CAAC;AAE5E,MAAM,iBAAiB,GAAG;IACxB,GAAG;IACH,MAAM,GAAG,SAAS;IAClB,SAAS;IACT,MAAM;IACN,OAAO;IACP,OAAO;CACR,CAAC;AACF,MAAM,oBAAoB,GAAG,IAAI,GAAG,CAAC;IACnC,QAAQ;IACR,OAAO;IACP,UAAU;IACV,QAAQ;IACR,MAAM;IACN,QAAQ;IACR,OAAO;IACP,QAAQ;IACR,UAAU;CACX,CAAC,CAAC;AACH,MAAM,iBAAiB,GAAG,IAAI,GAAG,CAAC,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC,CAAC;AAErD,MAAM,gBAAgB,GAAG,IAAI,GAAG,CAAC;IAC/B,YAAY;IACZ,QAAQ;IACR,eAAe;IACf,aAAa;IACb,MAAM;IACN,SAAS;IACT,MAAM;IACN,QAAQ;IACR,aAAa;IACb,QAAQ;CACT,CAAC,CAAC;AAEH,MAAM,yBAAyB,GAAG,IAAI,GAAG,CAAC;IACxC,UAAU;IACV,KAAK;IACL,SAAS;IACT,QAAQ;IACR,aAAa;IACb,MAAM;IACN,UAAU;IACV,QAAQ;IACR,SAAS;IACT,UAAU;IACV,SAAS;IACT,OAAO;CACR,CAAC,CAAC;AAEH,MAAM,mBAAmB,GAAG;IAC1B,QAAQ;IACR,OAAO;IACP,cAAc;IACd,KAAK;IACL,QAAQ;IACR,KAAK;IACL,SAAS;IACT,WAAW;IACX,YAAY;IACZ,YAAY;IACZ,OAAO;IACP,SAAS;CACV,CAAC;AACF,MAAM,uBAAuB,GAAG,CAAC,IAAI,EAAE,SAAS,EAAE,SAAS,CAAC,CAAC;AAC7D,MAAM,wBAAwB,GAAG;IAC/B,gBAAgB,EAAE,CAAC,QAAQ,EAAE,SAAS,EAAE,OAAO,EAAE,OAAO,EAAE,SAAS,EAAE,OAAO,CAAC;IAC7E,WAAW,EAAE,CAAC,YAAY,EAAE,WAAW,CAAC;IACxC,cAAc,EAAE,CAAC,OAAO,EAAE,QAAQ,CAAC;CACpC,CAAC;AAEF,MAAM,oBAAoB,GAAG;IAC3B,SAAS,EACP,iHAAiH;IACnH,aAAa,EAAE,iBAAiB;IAChC,MAAM,EACJ,gFAAgF;CACnF,CAAC;AAEF,MAAM,cAAc,GAAG,KAAK,CAAC;AA4B7B,sBAAsB;AAEtB,IAAI,aAAuC,CAAC;AAC5C,IAAI,aAA6C,CAAC;AAElD,oCAAoC;AAEpC,SAAS,kBAAkB,CAAC,KAAa;IACvC,OAAO,KAAK,CAAC,OAAO,CAAC,qBAAqB,EAAE,MAAM,CAAC,CAAC;AACtD,CAAC;AAED,SAAS,eAAe,CAAC,MAAmB;IAC1C,IAAI,MAAM,CAAC,IAAI,KAAK,CAAC;QAAE,OAAO,cAAc,CAAC;IAC7C,OAAO,IAAI,MAAM,CACf,qBAAqB,CAAC,GAAG,MAAM,CAAC,CAAC,GAAG,CAAC,kBAAkB,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,kBAAkB,EACpF,GAAG,CACJ,CAAC;AACJ,CAAC;AAED,SAAS,SAAS,CAAC,MAAmB,EAAE,MAAyB;IAC/D,KAAK,MAAM,KAAK,IAAI,MAAM;QAAE,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;AAChD,CAAC;AAED,SAAS,gBAAgB,CACvB,aAAiC,EACjC,KAA4B;IAE5B,MAAM,UAAU,GAAG,IAAI,GAAG,CAAC,mBAAmB,CAAC,CAAC;IAChD,MAAM,gBAAgB,GAAG,IAAI,GAAG,EAAU,CAAC;IAE3C,IAAI,aAAa,CAAC,cAAc,EAAE,CAAC;QACjC,SAAS,CAAC,gBAAgB,EAAE,uBAAuB,CAAC,CAAC;IACvD,CAAC;IAED,IAAI,KAAK,CAAC,aAAa,EAAE,CAAC;QACxB,SAAS,CAAC,UAAU,EAAE,wBAAwB,CAAC,gBAAgB,CAAC,CAAC,CAAC;IACpE,CAAC;IACD,IAAI,KAAK,CAAC,WAAW,EAAE,CAAC;QACtB,SAAS,CAAC,UAAU,EAAE,wBAAwB,CAAC,aAAa,CAAC,CAAC,CAAC;IACjE,CAAC;IACD,IAAI,KAAK,CAAC,WAAW,EAAE,CAAC;QACtB,SAAS,CAAC,UAAU,EAAE,wBAAwB,CAAC,cAAc,CAAC,CAAC,CAAC;IAClE,CAAC;IAED,KAAK,MAAM,CAAC,IAAI,aAAa,CAAC,WAAW,EAAE,CAAC;QAC1C,MAAM,CAAC,GAAG,CAAC,CAAC,WAAW,EAAE,CAAC,IAAI,EAAE,CAAC;QACjC,IAAI,CAAC;YAAE,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;IAC3B,CAAC;IAED,OAAO;QACL,IAAI,EAAE,eAAe,CAAC,UAAU,CAAC;QACjC,UAAU,EAAE,eAAe,CAAC,gBAAgB,CAAC;KAC9C,CAAC;AACJ,CAAC;AAED,SAAS,UAAU;IACjB,MAAM,aAAa,GAAG,MAAM,CAAC,YAAY,CAAC;IAC1C,IAAI,aAAa,KAAK,SAAS,IAAI,aAAa,KAAK,aAAa;QAChE,OAAO,aAAa,CAAC;IAEvB,MAAM,OAAO,GAAG,IAAI,GAAG,CACrB,aAAa,CAAC,iBAAiB;SAC5B,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;QACT,MAAM,CAAC,GAAG,CAAC,CAAC,WAAW,EAAE,CAAC,IAAI,EAAE,CAAC;QACjC,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,CAAC,IAAI,CAAC;QAC/B,OAAO,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,iBAAiB,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAClD,CAAC,CAAC;SACD,MAAM,CAAC,OAAO,CAAC,CACnB,CAAC;IAEF,MAAM,SAAS,GAAG,CAAC,GAAW,EAAW,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IAC7D,MAAM,KAAK,GAAG;QACZ,SAAS,EAAE,SAAS,CAAC,YAAY,CAAC;QAClC,aAAa,EAAE,SAAS,CAAC,gBAAgB,CAAC;QAC1C,WAAW,EAAE,SAAS,CAAC,aAAa,CAAC;QACrC,WAAW,EAAE,SAAS,CAAC,cAAc,CAAC;KACvC,CAAC;IAEF,MAAM,cAAc,GAAG,IAAI,GAAG,CAAC,oBAAoB,CAAC,CAAC;IACrD,IAAI,CAAC,aAAa,CAAC,iBAAiB,EAAE,CAAC;QACrC,cAAc,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;QAC1B,cAAc,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;IAC/B,CAAC;IAED,MAAM,aAAa,GAAG,gBAAgB,CAAC,aAAa,EAAE,KAAK,CAAC,CAAC;IAC7D,MAAM,cAAc,GAAG,aAAa,CAAC,cAAc;SAChD,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;SACpB,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IAE/B,sBAAsB;IACtB,MAAM,SAAS,GAAG,CAAC,oBAAoB,CAAC,MAAM,CAAC,CAAC;IAChD,IAAI,KAAK,CAAC,SAAS;QAAE,SAAS,CAAC,IAAI,CAAC,oBAAoB,CAAC,SAAS,CAAC,CAAC;IACpE,IAAI,KAAK,CAAC,aAAa;QAAE,SAAS,CAAC,IAAI,CAAC,oBAAoB,CAAC,aAAa,CAAC,CAAC;IAC5E,MAAM,YAAY,GAAG,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAEzC,MAAM,iBAAiB,GAAG;QACxB,GAAG,cAAc;QACjB,GAAG,iBAAiB;QACpB,OAAO;QACP,QAAQ;QACR,SAAS;QACT,MAAM;QACN,QAAQ;QACR,SAAS;KACV,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAEZ,aAAa,GAAG;QACd,KAAK;QACL,cAAc;QACd,OAAO,EAAE,aAAa,CAAC,OAAO;QAC9B,aAAa;QACb,YAAY,EAAE,KAAK,CAAC,aAAa,IAAI,KAAK,CAAC,WAAW,IAAI,KAAK,CAAC,WAAW;QAC3E,cAAc;QACd,YAAY;QACZ,iBAAiB;KAClB,CAAC;IACF,aAAa,GAAG,aAAa,CAAC;IAC9B,OAAO,aAAa,CAAC;AACvB,CAAC;AAED,yBAAyB;AAEzB,SAAS,aAAa,CAAC,OAAgB,EAAE,IAAmB;IAC1D,IAAI,IAAI,IAAI,yBAAyB,CAAC,GAAG,CAAC,IAAI,CAAC;QAAE,OAAO,IAAI,CAAC;IAC7D,MAAM,EAAE,GAAG,OAAO,CAAC,YAAY,CAAC,YAAY,CAAC,CAAC;IAC9C,IAAI,EAAE,KAAK,UAAU,IAAI,EAAE,KAAK,QAAQ;QAAE,OAAO,IAAI,CAAC;IACtD,MAAM,eAAe,GAAG,OAAO,CAAC,YAAY,CAAC,kBAAkB,CAAC,CAAC;IACjE,IAAI,eAAe,KAAK,YAAY,IAAI,eAAe,KAAK,UAAU;QACpE,OAAO,IAAI,CAAC;IACd,OAAO,CACL,OAAO,CAAC,YAAY,CAAC,qBAAqB,CAAC;QAC3C,OAAO,CAAC,YAAY,CAAC,4BAA4B,CAAC,CACnD,CAAC;AACJ,CAAC;AAED,SAAS,sBAAsB,CAAC,OAAgB;IAC9C,IAAI,OAAO,GAAmB,OAAO,CAAC;IACtC,OAAO,OAAO,EAAE,CAAC;QACf,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC;QAC9C,IAAI,OAAO,KAAK,SAAS,IAAI,OAAO,KAAK,MAAM;YAAE,OAAO,IAAI,CAAC;QAC7D,IAAI,OAAO,CAAC,YAAY,CAAC,MAAM,CAAC,KAAK,MAAM;YAAE,OAAO,IAAI,CAAC;QACzD,OAAO,GAAG,OAAO,CAAC,aAAa,CAAC;IAClC,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAS,cAAc,CAAC,OAAgB,EAAE,OAAe;IACvD,eAAe;IACf,MAAM,IAAI,GAAG,OAAO,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC;IAC1C,IAAI,IAAI,KAAK,QAAQ,IAAI,IAAI,KAAK,aAAa,EAAE,CAAC;QAChD,IAAI,sBAAsB,CAAC,OAAO,CAAC;YAAE,OAAO,IAAI,CAAC;QACjD,MAAM,OAAO,GAAG,CAAC,OAAO,CAAC,WAAW,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC;QACnD,IAAI,OAAO,GAAG,iCAAiC;YAAE,OAAO,IAAI,CAAC;QAC7D,OAAO,OAAO,CAAC,aAAa,CAAC,mBAAmB,CAAC,KAAK,IAAI,CAAC;IAC7D,CAAC;IAED,mBAAmB;IACnB,IAAI,OAAO,KAAK,KAAK,IAAI,OAAO,KAAK,QAAQ,EAAE,CAAC;QAC9C,IAAI,OAAO,CAAC,aAAa,CAAC,oCAAoC,CAAC;YAC7D,OAAO,IAAI,CAAC;QACd,OAAO,CACL,CAAC,OAAO,CAAC,WAAW,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,MAAM;YACzC,qCAAqC,CACtC,CAAC;IACJ,CAAC;IAED,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAS,WAAW,CAAC,KAAyB;IAC5C,KAAK,IAAI,CAAC,GAAG,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;QAC3C,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;QACtB,IAAI,IAAI,EAAE,UAAU,IAAI,CAAC,cAAc,CAAC,IAAI,EAAE,IAAI,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC,EAAE,CAAC;YAC1E,IAAI,CAAC,MAAM,EAAE,CAAC;QAChB,CAAC;IACH,CAAC;AACH,CAAC;AAED,SAAS,cAAc,CACrB,OAAe,EACf,IAAmB,EACnB,SAAiB,EACjB,EAAU,EACV,OAAqB;IAErB,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,IAAI,iBAAiB,CAAC,GAAG,CAAC,OAAO,CAAC;QAAE,KAAK,IAAI,OAAO,CAAC,UAAU,CAAC;IAEhE,qBAAqB;IACrB,IAAI,OAAO,KAAK,QAAQ,EAAE,CAAC;QACzB,IACE,CAAC,IAAI,IAAI,gBAAgB,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;YACpC,oBAAoB,CAAC,IAAI,CAAC,GAAG,SAAS,IAAI,EAAE,EAAE,CAAC,EAC/C,CAAC;YACD,KAAK,IAAI,OAAO,CAAC,UAAU,CAAC;QAC9B,CAAC;IACH,CAAC;IAED,aAAa;IACb,IAAI,IAAI,IAAI,gBAAgB,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;QACvC,IAAI,OAAO,KAAK,OAAO,IAAI,IAAI,KAAK,eAAe,EAAE,CAAC;YACpD,KAAK,IAAI,OAAO,CAAC,UAAU,CAAC;QAC9B,CAAC;IACH,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAYD,SAAS,sBAAsB,CAAC,OAAgB;IAC9C,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC;IAC9C,MAAM,SAAS,GAAG,OAAO,CAAC,YAAY,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC;IACtD,MAAM,EAAE,GAAG,OAAO,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;IAC5C,MAAM,IAAI,GAAG,OAAO,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC;IAC1C,MAAM,KAAK,GAAG,OAAO,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC;IAC5C,MAAM,cAAc,GAAG,aAAa,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC;IACpD,MAAM,QAAQ,GACZ,OAAO,CAAC,YAAY,CAAC,QAAQ,CAAC;QAC9B,OAAO,CAAC,YAAY,CAAC,aAAa,CAAC,KAAK,MAAM;QAC9C,CAAC,KAAK,KAAK,IAAI;YACb,qDAAqD,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC;IAEvE,OAAO;QACL,OAAO;QACP,SAAS;QACT,EAAE;QACF,IAAI;QACJ,KAAK;QACL,aAAa,EAAE,cAAc;QAC7B,QAAQ;KACT,CAAC;AACJ,CAAC;AAED,SAAS,cAAc,CAAC,OAAgB,EAAE,OAAqB;IAC7D,MAAM,IAAI,GAAG,sBAAsB,CAAC,OAAO,CAAC,CAAC;IAE7C,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,MAAM,EAAE,OAAO,EAAE,GAAG,OAAO,CAAC;IAE5B,aAAa;IACb,IAAI,OAAO,CAAC,cAAc,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,CAAC;QACpE,KAAK,IAAI,OAAO,CAAC,UAAU,CAAC;IAC9B,CAAC;IAED,qBAAqB;IACrB,IAAI,OAAO,CAAC,KAAK,CAAC,SAAS,EAAE,CAAC;QAC5B,KAAK,IAAI,cAAc,CACrB,IAAI,CAAC,OAAO,EACZ,IAAI,CAAC,IAAI,EACT,IAAI,CAAC,SAAS,EACd,IAAI,CAAC,EAAE,EACP,OAAO,CACR,CAAC;IACJ,CAAC;IAED,SAAS;IACT,IAAI,IAAI,CAAC,QAAQ,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,CAAC;QACzC,KAAK,IAAI,OAAO,CAAC,MAAM,CAAC;IAC1B,CAAC;IAED,eAAe;IACf,IAAI,uBAAuB,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC;QACjD,KAAK,IAAI,OAAO,CAAC,WAAW,CAAC;IAC/B,CAAC;IAED,QAAQ;IACR,IAAI,OAAO,CAAC,YAAY,EAAE,CAAC;QACzB,MAAM,OAAO,GACX,OAAO,CAAC,aAAa,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC;YACrD,OAAO,CAAC,aAAa,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACjD,MAAM,iBAAiB,GAAG,OAAO,IAAI,CAAC,sBAAsB,CAAC,OAAO,CAAC,CAAC;QACtE,MAAM,WAAW,GACf,CAAC,OAAO;YACR,CAAC,OAAO,CAAC,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC;gBAC9C,OAAO,CAAC,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC;QAE9C,IAAI,iBAAiB,IAAI,WAAW,EAAE,CAAC;YACrC,KAAK,IAAI,OAAO,CAAC,KAAK,CAAC;QACzB,CAAC;IACH,CAAC;IAED,OAAO,KAAK,IAAI,OAAO,CAAC,SAAS,CAAC;AACpC,CAAC;AAED,SAAS,uBAAuB,CAAC,CAAU;IACzC,MAAM,IAAI,GAAG,CAAC,CAAC,gBAAgB,CAAC,KAAK,CAAC,CAAC;IACvC,KAAK,IAAI,CAAC,GAAG,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;QAC1C,MAAM,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC;QAClB,IAAI,CAAC,CAAC,EAAE,UAAU;YAAE,SAAS;QAC7B,MAAM,GAAG,GAAG,CAAC,CAAC,YAAY,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC;QAC1C,MAAM,GAAG,GAAG,CAAC,CAAC,YAAY,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC;QAC1C,IACE,GAAG,CAAC,QAAQ,CAAC,UAAU,CAAC;YACxB,GAAG,CAAC,QAAQ,CAAC,UAAU,CAAC;YACxB,CAAC,CAAC,YAAY,CAAC,UAAU,CAAC,KAAK,IAAI,EACnC,CAAC;YACD,CAAC,CAAC,MAAM,EAAE,CAAC;QACb,CAAC;IACH,CAAC;AACH,CAAC;AAED,SAAS,mBAAmB,CAAC,CAAU;IACrC,MAAM,OAAO,GAAG,CAAC,CAAC,gBAAgB,CAAC,GAAG,CAAC,CAAC;IACxC,KAAK,IAAI,CAAC,GAAG,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;QAC7C,MAAM,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;QACrB,IAAI,CAAC,CAAC,EAAE,UAAU;YAAE,SAAS;QAC7B,MAAM,IAAI,GAAG,CAAC,CAAC,YAAY,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;QAC1C,MAAM,GAAG,GAAG,CAAC,CAAC,CAAC,WAAW,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,aAAa,EAAE,EAAE,CAAC,CAAC;QAC7D,IAAI,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,GAAG,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC7C,CAAC,CAAC,MAAM,EAAE,CAAC;QACb,CAAC;IACH,CAAC;AACH,CAAC;AAED,SAAS,qBAAqB,CAAC,CAAU,EAAE,QAAkB;IAC3D,MAAM,MAAM,GAAG,QAAQ,CAAC,gBAAgB,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,YAAY;IAC5D,IAAI,IAAiB,CAAC;IACtB,OAAO,CAAC,IAAI,GAAG,MAAM,CAAC,QAAQ,EAAE,CAAC,EAAE,CAAC;QAClC,IAAI,IAAI,CAAC,WAAW,EAAE,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC;YACzC,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,WAAW,CAAC,OAAO,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;QAC7D,CAAC;IACH,CAAC;AACH,CAAC;AAED,SAAS,aAAa,CAAC,QAAkB;IACvC,wBAAwB;IACxB,MAAM,QAAQ,GAAG,QAAQ,CAAC,gBAAgB,CAAC,mBAAmB,CAAC,CAAC;IAChE,KAAK,MAAM,CAAC,IAAI,QAAQ,EAAE,CAAC;QACzB,IAAI,CAAC,CAAC,CAAC,UAAU;YAAE,SAAS;QAC5B,uBAAuB,CAAC,CAAC,CAAC,CAAC;QAC3B,mBAAmB,CAAC,CAAC,CAAC,CAAC;QACvB,qBAAqB,CAAC,CAAC,EAAE,QAAQ,CAAC,CAAC;IACrC,CAAC;AACH,CAAC;AAED,SAAS,UAAU,CACjB,QAAkB,EAClB,OAAqB,EACrB,MAAoB;IAEpB,aAAa,CAAC,QAAQ,CAAC,CAAC;IAExB,sBAAsB;IACtB,MAAM,EAAE,YAAY,EAAE,cAAc,EAAE,GAAG,OAAO,CAAC;IAEjD,OAAO;IACP,MAAM,SAAS,GAAG,QAAQ,CAAC,gBAAgB,CAAC,YAAY,CAAC,CAAC;IAC1D,WAAW,CAAC,SAAS,CAAC,CAAC;IAEvB,QAAQ;IACR,IAAI,cAAc,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC9B,MAAM,aAAa,GAAG,cAAc,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAC/C,MAAM,UAAU,GAAG,QAAQ,CAAC,gBAAgB,CAAC,aAAa,CAAC,CAAC;QAC5D,WAAW,CAAC,UAAU,CAAC,CAAC;IAC1B,CAAC;IAED,aAAa;IACb,MAAM,UAAU,GAAG,QAAQ,CAAC,gBAAgB,CAAC,OAAO,CAAC,iBAAiB,CAAC,CAAC;IACxE,KAAK,IAAI,CAAC,GAAG,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;QAChD,IAAI,CAAC,GAAG,GAAG,KAAK,CAAC,IAAI,MAAM,EAAE,OAAO,EAAE,CAAC;YACrC,MAAM,IAAI,KAAK,CAAC,uBAAuB,CAAC,CAAC;QAC3C,CAAC;QACD,MAAM,IAAI,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC;QAC3B,IAAI,CAAC,IAAI;YAAE,SAAS;QACpB,IAAI,CAAC,IAAI,CAAC,UAAU;YAAE,SAAS;QAE/B,IAAI,cAAc,CAAC,IAAI,EAAE,IAAI,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC;YAAE,SAAS;QAC/D,IAAI,cAAc,CAAC,IAAI,EAAE,OAAO,CAAC,EAAE,CAAC;YAClC,IAAI,CAAC,MAAM,EAAE,CAAC;QAChB,CAAC;IACH,CAAC;AACH,CAAC;AAED,SAAS,iBAAiB,CACxB,EAAW,EACX,IAAY,EACZ,IAAS,EACT,QAAiB;IAEjB,IAAI,CAAC,EAAE,CAAC,UAAU;QAAE,OAAO;IAC3B,IAAI,QAAQ,EAAE,CAAC;QACb,MAAM,GAAG,GAAG,EAAE,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC;QAClC,IAAI,GAAG,EAAE,CAAC;YACR,MAAM,MAAM,GAAG,GAAG;iBACf,KAAK,CAAC,GAAG,CAAC;iBACV,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE;gBACb,MAAM,KAAK,GAAG,KAAK,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;gBACxC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC;oBAAE,OAAO,KAAK,CAAC;gBAC5B,IAAI,CAAC;oBACH,KAAK,CAAC,CAAC,CAAC,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,IAAI,CAAC;gBAC1C,CAAC;gBAAC,MAAM,CAAC;oBACP,YAAY;gBACd,CAAC;gBACD,OAAO,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACzB,CAAC,CAAC;iBACD,IAAI,CAAC,IAAI,CAAC,CAAC;YACd,EAAE,CAAC,YAAY,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;QAChC,CAAC;QACD,OAAO;IACT,CAAC;IAED,MAAM,GAAG,GAAG,EAAE,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC;IAClC,IACE,GAAG;QACH,CAAC,iBAAiB,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,EACtE,CAAC;QACD,IAAI,CAAC;YACH,EAAE,CAAC,YAAY,CAAC,IAAI,EAAE,IAAI,GAAG,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC;QACjD,CAAC;QAAC,MAAM,CAAC;YACP,YAAY;QACd,CAAC;IACH,CAAC;AACH,CAAC;AAED,SAAS,WAAW,CAAC,QAAkB,EAAE,UAAkB;IACzD,IAAI,IAAS,CAAC;IACd,IAAI,CAAC;QACH,IAAI,GAAG,IAAI,GAAG,CAAC,UAAU,CAAC,CAAC;IAC7B,CAAC;IAAC,MAAM,CAAC;QACP,OAAO;IACT,CAAC;IAED,MAAM,QAAQ,GAAG,QAAQ,CAAC,gBAAgB,CAAC,iCAAiC,CAAC,CAAC;IAC9E,KAAK,MAAM,EAAE,IAAI,QAAQ,EAAE,CAAC;QAC1B,MAAM,GAAG,GAAG,EAAE,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC;QACrC,IAAI,GAAG,KAAK,GAAG;YAAE,iBAAiB,CAAC,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,KAAK,CAAC,CAAC;aACvD,IAAI,GAAG,KAAK,KAAK;YAAE,iBAAiB,CAAC,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,KAAK,CAAC,CAAC;aAC7D,IAAI,GAAG,KAAK,QAAQ;YAAE,iBAAiB,CAAC,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC;IACzE,CAAC;AACH,CAAC;AAED,SAAS,SAAS,CAAC,QAAkB,EAAE,QAAgB;IACrD,MAAM,QAAQ,GAAG,QAAQ,CAAC,IAAI,CAAC,SAAS,CAAC;IACzC,IAAI,QAAQ,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,uBAAuB;QAAE,OAAO,QAAQ,CAAC;IAEtE,MAAM,SAAS,GAAG,QAAQ,CAAC,eAAe,CAAC,SAAS,CAAC;IACrD,IAAI,SAAS,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,uBAAuB;QAAE,OAAO,SAAS,CAAC;IAExE,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED,SAAS,kBAAkB,CAAC,IAAY;IACtC,OAAO,qBAAqB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1C,CAAC;AAED,SAAS,eAAe,CAAC,IAAY;IACnC,MAAM,MAAM,GACV,IAAI,CAAC,MAAM,IAAI,gBAAgB;QAC7B,CAAC,CAAC,IAAI;QACN,CAAC,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC,EAAE,gBAAgB,CAAC,KAAK,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,MAAM,GAAG,gBAAgB,CAAC,EAAE,CAAC;IAClG,OAAO,cAAc,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC;AACtD,CAAC;AAED,MAAM,UAAU,mBAAmB,CACjC,IAAY,EACZ,QAAmB,EACnB,OAAgB,EAChB,MAAoB;IAEpB,MAAM,WAAW,GACf,kBAAkB,CAAC,IAAI,CAAC;QACxB,eAAe,CAAC,IAAI,CAAC;QACrB,qBAAqB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACnC,IAAI,CAAC,WAAW;QAAE,OAAO,IAAI,CAAC;IAE9B,IAAI,CAAC;QACH,MAAM,OAAO,GAAG,UAAU,EAAE,CAAC;QAE7B,IAAI,MAAM,CAAC,YAAY,CAAC,KAAK,EAAE,CAAC;YAC9B,QAAQ,CAAC,6BAA6B,EAAE;gBACtC,UAAU,EAAE,CAAC,GAAG,CAAC,OAAO,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;aACjE,CAAC,CAAC;QACL,CAAC;QAED,MAAM,GAAG,GAAG,QAAQ,IAAI,SAAS,CAAC,IAAI,CAAC,CAAC,QAAQ,CAAC;QAEjD,UAAU,CAAC,GAAG,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC;QAEjC,IAAI,OAAO;YAAE,WAAW,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;QAEvC,OAAO,SAAS,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;IAC9B,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC"}
|
package/dist/lib/download.d.ts
DELETED
|
@@ -1,4 +0,0 @@
|
|
|
1
|
-
import type { ServerResponse } from 'node:http';
|
|
2
|
-
export declare function generateSafeFilename(url: string, title?: string, hashFallback?: string, extension?: string): string;
|
|
3
|
-
export declare function handleDownload(res: ServerResponse, namespace: string, hash: string): void;
|
|
4
|
-
//# sourceMappingURL=download.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"download.d.ts","sourceRoot":"","sources":["../../src/lib/download.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,WAAW,CAAC;AA2EhD,wBAAgB,oBAAoB,CAClC,GAAG,EAAE,MAAM,EACX,KAAK,CAAC,EAAE,MAAM,EACd,YAAY,CAAC,EAAE,MAAM,EACrB,SAAS,SAAQ,GAChB,MAAM,CAIR;AAyBD,wBAAgB,cAAc,CAC5B,GAAG,EAAE,cAAc,EACnB,SAAS,EAAE,MAAM,EACjB,IAAI,EAAE,MAAM,GACX,IAAI,CAwCN"}
|
package/dist/lib/download.js
DELETED
|
@@ -1,106 +0,0 @@
|
|
|
1
|
-
import { posix as pathPosix } from 'node:path';
|
|
2
|
-
import { z } from 'zod';
|
|
3
|
-
import { get as cacheGet, parseCachedPayload, resolveCachedPayloadContent, } from './cache.js';
|
|
4
|
-
import { config } from './config.js';
|
|
5
|
-
/* -------------------------------------------------------------------------------------------------
|
|
6
|
-
* Utils: Filename Logic
|
|
7
|
-
* ------------------------------------------------------------------------------------------------- */
|
|
8
|
-
const FILENAME_RULES = {
|
|
9
|
-
MAX_LEN: 200,
|
|
10
|
-
UNSAFE_CHARS: /[<>:"/\\|?*\p{C}]/gu,
|
|
11
|
-
WHITESPACE: /\s+/g,
|
|
12
|
-
EXTENSIONS: /\.(html?|php|aspx?|jsp)$/i,
|
|
13
|
-
};
|
|
14
|
-
function sanitizeString(input) {
|
|
15
|
-
return input
|
|
16
|
-
.toLowerCase()
|
|
17
|
-
.replace(FILENAME_RULES.UNSAFE_CHARS, '')
|
|
18
|
-
.replace(FILENAME_RULES.WHITESPACE, '-')
|
|
19
|
-
.replace(/-+/g, '-')
|
|
20
|
-
.replace(/(?:^-|-$)/g, '');
|
|
21
|
-
}
|
|
22
|
-
function resolveUrlFilenameCandidate(url) {
|
|
23
|
-
const parsed = new URL(url);
|
|
24
|
-
if (parsed.protocol !== 'http:' && parsed.protocol !== 'https:')
|
|
25
|
-
return null;
|
|
26
|
-
const basename = pathPosix.basename(parsed.pathname);
|
|
27
|
-
if (!basename || basename === 'index')
|
|
28
|
-
return null;
|
|
29
|
-
const cleaned = basename.replace(FILENAME_RULES.EXTENSIONS, '');
|
|
30
|
-
const sanitized = sanitizeString(cleaned);
|
|
31
|
-
if (sanitized === 'index')
|
|
32
|
-
return null;
|
|
33
|
-
return sanitized || null;
|
|
34
|
-
}
|
|
35
|
-
function truncateFilenameBase(name, extension) {
|
|
36
|
-
const maxBase = FILENAME_RULES.MAX_LEN - extension.length;
|
|
37
|
-
return name.length > maxBase ? name.substring(0, maxBase) : name;
|
|
38
|
-
}
|
|
39
|
-
function resolveTitleFilenameCandidate(title) {
|
|
40
|
-
if (!title)
|
|
41
|
-
return null;
|
|
42
|
-
return sanitizeString(title) || null;
|
|
43
|
-
}
|
|
44
|
-
function resolveFilenameBase(url, title, hashFallback) {
|
|
45
|
-
try {
|
|
46
|
-
const fromUrl = resolveUrlFilenameCandidate(url);
|
|
47
|
-
if (fromUrl)
|
|
48
|
-
return fromUrl;
|
|
49
|
-
}
|
|
50
|
-
catch {
|
|
51
|
-
// Ignore URL parsing errors and continue fallbacks.
|
|
52
|
-
}
|
|
53
|
-
const fromTitle = resolveTitleFilenameCandidate(title);
|
|
54
|
-
if (fromTitle)
|
|
55
|
-
return fromTitle;
|
|
56
|
-
if (hashFallback)
|
|
57
|
-
return hashFallback.substring(0, 16);
|
|
58
|
-
return `download-${Date.now()}`;
|
|
59
|
-
}
|
|
60
|
-
export function generateSafeFilename(url, title, hashFallback, extension = '.md') {
|
|
61
|
-
const name = resolveFilenameBase(url, title, hashFallback);
|
|
62
|
-
return `${truncateFilenameBase(name, extension)}${extension}`;
|
|
63
|
-
}
|
|
64
|
-
/* -------------------------------------------------------------------------------------------------
|
|
65
|
-
* Adapter: Download Handler
|
|
66
|
-
* ------------------------------------------------------------------------------------------------- */
|
|
67
|
-
const DownloadParamsSchema = z.strictObject({
|
|
68
|
-
namespace: z.literal('markdown'),
|
|
69
|
-
hash: z
|
|
70
|
-
.string()
|
|
71
|
-
.regex(/^[a-f0-9.]+$/i)
|
|
72
|
-
.min(8)
|
|
73
|
-
.max(64),
|
|
74
|
-
});
|
|
75
|
-
function writeJsonError(res, status, message, code) {
|
|
76
|
-
res.writeHead(status, { 'Content-Type': 'application/json' });
|
|
77
|
-
res.end(JSON.stringify({ error: message, code }));
|
|
78
|
-
}
|
|
79
|
-
export function handleDownload(res, namespace, hash) {
|
|
80
|
-
const parsed = DownloadParamsSchema.safeParse({ namespace, hash });
|
|
81
|
-
if (!parsed.success) {
|
|
82
|
-
writeJsonError(res, 400, 'Invalid namespace or hash', 'BAD_REQUEST');
|
|
83
|
-
return;
|
|
84
|
-
}
|
|
85
|
-
const cacheKey = `${parsed.data.namespace}:${parsed.data.hash}`;
|
|
86
|
-
const entry = cacheGet(cacheKey, { force: true });
|
|
87
|
-
if (!entry) {
|
|
88
|
-
writeJsonError(res, 404, 'Not found or expired', 'NOT_FOUND');
|
|
89
|
-
return;
|
|
90
|
-
}
|
|
91
|
-
const payload = parseCachedPayload(entry.content);
|
|
92
|
-
const content = payload ? resolveCachedPayloadContent(payload) : null;
|
|
93
|
-
if (!content) {
|
|
94
|
-
writeJsonError(res, 404, 'Content missing', 'NOT_FOUND');
|
|
95
|
-
return;
|
|
96
|
-
}
|
|
97
|
-
const fileName = generateSafeFilename(entry.url, payload?.title, parsed.data.hash);
|
|
98
|
-
// Safe header generation — RFC 5987 encoding for non-ASCII filenames
|
|
99
|
-
const encoded = encodeURIComponent(fileName).replace(/'/g, '%27');
|
|
100
|
-
res.setHeader('Content-Type', 'text/markdown; charset=utf-8');
|
|
101
|
-
res.setHeader('Content-Disposition', `attachment; filename="${fileName}"; filename*=UTF-8''${encoded}`);
|
|
102
|
-
res.setHeader('Cache-Control', `private, max-age=${config.cache.ttl}`);
|
|
103
|
-
res.setHeader('X-Content-Type-Options', 'nosniff');
|
|
104
|
-
res.end(content);
|
|
105
|
-
}
|
|
106
|
-
//# sourceMappingURL=download.js.map
|
package/dist/lib/download.js.map
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"download.js","sourceRoot":"","sources":["../../src/lib/download.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,KAAK,IAAI,SAAS,EAAE,MAAM,WAAW,CAAC;AAE/C,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,OAAO,EACL,GAAG,IAAI,QAAQ,EACf,kBAAkB,EAClB,2BAA2B,GAC5B,MAAM,YAAY,CAAC;AACpB,OAAO,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAErC;;uGAEuG;AAEvG,MAAM,cAAc,GAAG;IACrB,OAAO,EAAE,GAAG;IACZ,YAAY,EAAE,qBAAqB;IACnC,UAAU,EAAE,MAAM;IAClB,UAAU,EAAE,2BAA2B;CAC/B,CAAC;AAEX,SAAS,cAAc,CAAC,KAAa;IACnC,OAAO,KAAK;SACT,WAAW,EAAE;SACb,OAAO,CAAC,cAAc,CAAC,YAAY,EAAE,EAAE,CAAC;SACxC,OAAO,CAAC,cAAc,CAAC,UAAU,EAAE,GAAG,CAAC;SACvC,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC;SACnB,OAAO,CAAC,YAAY,EAAE,EAAE,CAAC,CAAC;AAC/B,CAAC;AAED,SAAS,2BAA2B,CAAC,GAAW;IAC9C,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;IAC5B,IAAI,MAAM,CAAC,QAAQ,KAAK,OAAO,IAAI,MAAM,CAAC,QAAQ,KAAK,QAAQ;QAAE,OAAO,IAAI,CAAC;IAE7E,MAAM,QAAQ,GAAG,SAAS,CAAC,QAAQ,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;IACrD,IAAI,CAAC,QAAQ,IAAI,QAAQ,KAAK,OAAO;QAAE,OAAO,IAAI,CAAC;IAEnD,MAAM,OAAO,GAAG,QAAQ,CAAC,OAAO,CAAC,cAAc,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC;IAChE,MAAM,SAAS,GAAG,cAAc,CAAC,OAAO,CAAC,CAAC;IAE1C,IAAI,SAAS,KAAK,OAAO;QAAE,OAAO,IAAI,CAAC;IACvC,OAAO,SAAS,IAAI,IAAI,CAAC;AAC3B,CAAC;AAED,SAAS,oBAAoB,CAAC,IAAY,EAAE,SAAiB;IAC3D,MAAM,OAAO,GAAG,cAAc,CAAC,OAAO,GAAG,SAAS,CAAC,MAAM,CAAC;IAC1D,OAAO,IAAI,CAAC,MAAM,GAAG,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;AACnE,CAAC;AAED,SAAS,6BAA6B,CAAC,KAAc;IACnD,IAAI,CAAC,KAAK;QAAE,OAAO,IAAI,CAAC;IACxB,OAAO,cAAc,CAAC,KAAK,CAAC,IAAI,IAAI,CAAC;AACvC,CAAC;AAED,SAAS,mBAAmB,CAC1B,GAAW,EACX,KAAc,EACd,YAAqB;IAErB,IAAI,CAAC;QACH,MAAM,OAAO,GAAG,2BAA2B,CAAC,GAAG,CAAC,CAAC;QACjD,IAAI,OAAO;YAAE,OAAO,OAAO,CAAC;IAC9B,CAAC;IAAC,MAAM,CAAC;QACP,oDAAoD;IACtD,CAAC;IAED,MAAM,SAAS,GAAG,6BAA6B,CAAC,KAAK,CAAC,CAAC;IACvD,IAAI,SAAS;QAAE,OAAO,SAAS,CAAC;IAEhC,IAAI,YAAY;QAAE,OAAO,YAAY,CAAC,SAAS,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IACvD,OAAO,YAAY,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC;AAClC,CAAC;AAED,MAAM,UAAU,oBAAoB,CAClC,GAAW,EACX,KAAc,EACd,YAAqB,EACrB,SAAS,GAAG,KAAK;IAEjB,MAAM,IAAI,GAAG,mBAAmB,CAAC,GAAG,EAAE,KAAK,EAAE,YAAY,CAAC,CAAC;IAE3D,OAAO,GAAG,oBAAoB,CAAC,IAAI,EAAE,SAAS,CAAC,GAAG,SAAS,EAAE,CAAC;AAChE,CAAC;AAED;;uGAEuG;AAEvG,MAAM,oBAAoB,GAAG,CAAC,CAAC,YAAY,CAAC;IAC1C,SAAS,EAAE,CAAC,CAAC,OAAO,CAAC,UAAU,CAAC;IAChC,IAAI,EAAE,CAAC;SACJ,MAAM,EAAE;SACR,KAAK,CAAC,eAAe,CAAC;SACtB,GAAG,CAAC,CAAC,CAAC;SACN,GAAG,CAAC,EAAE,CAAC;CACX,CAAC,CAAC;AAEH,SAAS,cAAc,CACrB,GAAmB,EACnB,MAAc,EACd,OAAe,EACf,IAAY;IAEZ,GAAG,CAAC,SAAS,CAAC,MAAM,EAAE,EAAE,cAAc,EAAE,kBAAkB,EAAE,CAAC,CAAC;IAC9D,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,KAAK,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;AACpD,CAAC;AAED,MAAM,UAAU,cAAc,CAC5B,GAAmB,EACnB,SAAiB,EACjB,IAAY;IAEZ,MAAM,MAAM,GAAG,oBAAoB,CAAC,SAAS,CAAC,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IACnE,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;QACpB,cAAc,CAAC,GAAG,EAAE,GAAG,EAAE,2BAA2B,EAAE,aAAa,CAAC,CAAC;QACrE,OAAO;IACT,CAAC;IAED,MAAM,QAAQ,GAAG,GAAG,MAAM,CAAC,IAAI,CAAC,SAAS,IAAI,MAAM,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;IAChE,MAAM,KAAK,GAAG,QAAQ,CAAC,QAAQ,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;IAElD,IAAI,CAAC,KAAK,EAAE,CAAC;QACX,cAAc,CAAC,GAAG,EAAE,GAAG,EAAE,sBAAsB,EAAE,WAAW,CAAC,CAAC;QAC9D,OAAO;IACT,CAAC;IAED,MAAM,OAAO,GAAG,kBAAkB,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;IAClD,MAAM,OAAO,GAAG,OAAO,CAAC,CAAC,CAAC,2BAA2B,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IAEtE,IAAI,CAAC,OAAO,EAAE,CAAC;QACb,cAAc,CAAC,GAAG,EAAE,GAAG,EAAE,iBAAiB,EAAE,WAAW,CAAC,CAAC;QACzD,OAAO;IACT,CAAC;IAED,MAAM,QAAQ,GAAG,oBAAoB,CACnC,KAAK,CAAC,GAAG,EACT,OAAO,EAAE,KAAK,EACd,MAAM,CAAC,IAAI,CAAC,IAAI,CACjB,CAAC;IAEF,qEAAqE;IACrE,MAAM,OAAO,GAAG,kBAAkB,CAAC,QAAQ,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;IAElE,GAAG,CAAC,SAAS,CAAC,cAAc,EAAE,8BAA8B,CAAC,CAAC;IAC9D,GAAG,CAAC,SAAS,CACX,qBAAqB,EACrB,yBAAyB,QAAQ,uBAAuB,OAAO,EAAE,CAClE,CAAC;IACF,GAAG,CAAC,SAAS,CAAC,eAAe,EAAE,oBAAoB,MAAM,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,CAAC;IACvE,GAAG,CAAC,SAAS,CAAC,wBAAwB,EAAE,SAAS,CAAC,CAAC;IACnD,GAAG,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;AACnB,CAAC"}
|
package/dist/lib/errors.d.ts
DELETED
|
@@ -1,14 +0,0 @@
|
|
|
1
|
-
export declare class FetchError extends Error {
|
|
2
|
-
readonly url: string;
|
|
3
|
-
readonly statusCode: number;
|
|
4
|
-
readonly code: string;
|
|
5
|
-
readonly details: Readonly<Record<string, unknown>>;
|
|
6
|
-
constructor(message: string, url: string, httpStatus?: number, details?: Record<string, unknown>, options?: ErrorOptions);
|
|
7
|
-
}
|
|
8
|
-
export declare function getErrorMessage(error: unknown): string;
|
|
9
|
-
export declare function toError(error: unknown): Error;
|
|
10
|
-
export declare function isAbortError(error: unknown): boolean;
|
|
11
|
-
export declare function createErrorWithCode(message: string, code: string, options?: ErrorOptions): NodeJS.ErrnoException;
|
|
12
|
-
export declare function isSystemError(error: unknown): error is NodeJS.ErrnoException;
|
|
13
|
-
export declare const RESOURCE_NOT_FOUND_ERROR_CODE = -32002;
|
|
14
|
-
//# sourceMappingURL=errors.d.ts.map
|
package/dist/lib/errors.d.ts.map
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"errors.d.ts","sourceRoot":"","sources":["../../src/lib/errors.ts"],"names":[],"mappings":"AAMA,qBAAa,UAAW,SAAQ,KAAK;IAOjC,QAAQ,CAAC,GAAG,EAAE,MAAM;IANtB,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,OAAO,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC;gBAGlD,OAAO,EAAE,MAAM,EACN,GAAG,EAAE,MAAM,EACpB,UAAU,CAAC,EAAE,MAAM,EACnB,OAAO,GAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAM,EACrC,OAAO,CAAC,EAAE,YAAY;CASzB;AAED,wBAAgB,eAAe,CAAC,KAAK,EAAE,OAAO,GAAG,MAAM,CAKtD;AAED,wBAAgB,OAAO,CAAC,KAAK,EAAE,OAAO,GAAG,KAAK,CAE7C;AAED,wBAAgB,YAAY,CAAC,KAAK,EAAE,OAAO,GAAG,OAAO,CAEpD;AA2BD,wBAAgB,mBAAmB,CACjC,OAAO,EAAE,MAAM,EACf,IAAI,EAAE,MAAM,EACZ,OAAO,CAAC,EAAE,YAAY,GACrB,MAAM,CAAC,cAAc,CAGvB;AAED,wBAAgB,aAAa,CAAC,KAAK,EAAE,OAAO,GAAG,KAAK,IAAI,MAAM,CAAC,cAAc,CAK5E;AAED,eAAO,MAAM,6BAA6B,SAAS,CAAC"}
|