@j0hanz/superfetch 1.1.1 → 1.1.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (206) hide show
  1. package/README.md +57 -32
  2. package/dist/config/formatting.d.ts +9 -0
  3. package/dist/config/formatting.d.ts.map +1 -0
  4. package/dist/config/formatting.js +11 -0
  5. package/dist/config/formatting.js.map +1 -0
  6. package/dist/config/index.d.ts +16 -2
  7. package/dist/config/index.d.ts.map +1 -1
  8. package/dist/config/index.js +43 -14
  9. package/dist/config/index.js.map +1 -1
  10. package/dist/config/types/content.d.ts +107 -0
  11. package/dist/config/types/content.d.ts.map +1 -0
  12. package/dist/config/types/content.js +2 -0
  13. package/dist/config/types/content.js.map +1 -0
  14. package/dist/config/types/runtime.d.ts +78 -0
  15. package/dist/config/types/runtime.d.ts.map +1 -0
  16. package/dist/config/types/runtime.js +2 -0
  17. package/dist/config/types/runtime.js.map +1 -0
  18. package/dist/config/types/tools.d.ts +99 -0
  19. package/dist/config/types/tools.d.ts.map +1 -0
  20. package/dist/config/types/tools.js +2 -0
  21. package/dist/config/types/tools.js.map +1 -0
  22. package/dist/config/types.d.ts +3 -296
  23. package/dist/config/types.d.ts.map +1 -1
  24. package/dist/http/auth.d.ts +3 -0
  25. package/dist/http/auth.d.ts.map +1 -0
  26. package/dist/http/auth.js +34 -0
  27. package/dist/http/auth.js.map +1 -0
  28. package/dist/http/cors.d.ts +8 -0
  29. package/dist/http/cors.d.ts.map +1 -0
  30. package/dist/http/cors.js +47 -0
  31. package/dist/http/cors.js.map +1 -0
  32. package/dist/http/mcp-routes.d.ts +5 -0
  33. package/dist/http/mcp-routes.d.ts.map +1 -0
  34. package/dist/http/mcp-routes.js +110 -0
  35. package/dist/http/mcp-routes.js.map +1 -0
  36. package/dist/http/mcp-session.d.ts +12 -0
  37. package/dist/http/mcp-session.d.ts.map +1 -0
  38. package/dist/http/mcp-session.js +209 -0
  39. package/dist/http/mcp-session.js.map +1 -0
  40. package/dist/http/mcp-validation.d.ts +3 -0
  41. package/dist/http/mcp-validation.d.ts.map +1 -0
  42. package/dist/http/mcp-validation.js +34 -0
  43. package/dist/http/mcp-validation.js.map +1 -0
  44. package/dist/http/rate-limit.d.ts +13 -0
  45. package/dist/http/rate-limit.d.ts.map +1 -0
  46. package/dist/http/rate-limit.js +91 -0
  47. package/dist/http/rate-limit.js.map +1 -0
  48. package/dist/http/server.d.ts +4 -0
  49. package/dist/http/server.d.ts.map +1 -0
  50. package/dist/http/server.js +183 -0
  51. package/dist/http/server.js.map +1 -0
  52. package/dist/http/sessions.d.ts +15 -0
  53. package/dist/http/sessions.d.ts.map +1 -0
  54. package/dist/http/sessions.js +64 -0
  55. package/dist/http/sessions.js.map +1 -0
  56. package/dist/index.js +26 -223
  57. package/dist/index.js.map +1 -1
  58. package/dist/middleware/error-handler.d.ts +2 -2
  59. package/dist/middleware/error-handler.d.ts.map +1 -1
  60. package/dist/middleware/error-handler.js +46 -15
  61. package/dist/middleware/error-handler.js.map +1 -1
  62. package/dist/resources/cached-content.d.ts.map +1 -1
  63. package/dist/resources/cached-content.js +104 -44
  64. package/dist/resources/cached-content.js.map +1 -1
  65. package/dist/resources/index.d.ts.map +1 -1
  66. package/dist/resources/index.js +77 -69
  67. package/dist/resources/index.js.map +1 -1
  68. package/dist/server.d.ts.map +1 -1
  69. package/dist/server.js +9 -3
  70. package/dist/server.js.map +1 -1
  71. package/dist/services/cache.d.ts +13 -1
  72. package/dist/services/cache.d.ts.map +1 -1
  73. package/dist/services/cache.js +90 -13
  74. package/dist/services/cache.js.map +1 -1
  75. package/dist/services/context.d.ts +9 -0
  76. package/dist/services/context.d.ts.map +1 -0
  77. package/dist/services/context.js +9 -0
  78. package/dist/services/context.js.map +1 -0
  79. package/dist/services/extractor.d.ts.map +1 -1
  80. package/dist/services/extractor.js +122 -87
  81. package/dist/services/extractor.js.map +1 -1
  82. package/dist/services/fetcher/agents.d.ts +4 -0
  83. package/dist/services/fetcher/agents.d.ts.map +1 -0
  84. package/dist/services/fetcher/agents.js +111 -0
  85. package/dist/services/fetcher/agents.js.map +1 -0
  86. package/dist/services/fetcher/errors.d.ts +5 -0
  87. package/dist/services/fetcher/errors.d.ts.map +1 -0
  88. package/dist/services/fetcher/errors.js +71 -0
  89. package/dist/services/fetcher/errors.js.map +1 -0
  90. package/dist/services/fetcher/headers.d.ts +2 -0
  91. package/dist/services/fetcher/headers.d.ts.map +1 -0
  92. package/dist/services/fetcher/headers.js +28 -0
  93. package/dist/services/fetcher/headers.js.map +1 -0
  94. package/dist/services/fetcher/interceptors.d.ts +10 -0
  95. package/dist/services/fetcher/interceptors.d.ts.map +1 -0
  96. package/dist/services/fetcher/interceptors.js +82 -0
  97. package/dist/services/fetcher/interceptors.js.map +1 -0
  98. package/dist/services/fetcher/redirects.d.ts +6 -0
  99. package/dist/services/fetcher/redirects.d.ts.map +1 -0
  100. package/dist/services/fetcher/redirects.js +67 -0
  101. package/dist/services/fetcher/redirects.js.map +1 -0
  102. package/dist/services/fetcher/response.d.ts +5 -0
  103. package/dist/services/fetcher/response.d.ts.map +1 -0
  104. package/dist/services/fetcher/response.js +39 -0
  105. package/dist/services/fetcher/response.js.map +1 -0
  106. package/dist/services/fetcher/retry-policy.d.ts +28 -0
  107. package/dist/services/fetcher/retry-policy.d.ts.map +1 -0
  108. package/dist/services/fetcher/retry-policy.js +138 -0
  109. package/dist/services/fetcher/retry-policy.js.map +1 -0
  110. package/dist/services/fetcher.d.ts +2 -1
  111. package/dist/services/fetcher.d.ts.map +1 -1
  112. package/dist/services/fetcher.js +61 -254
  113. package/dist/services/fetcher.js.map +1 -1
  114. package/dist/services/logger.d.ts.map +1 -1
  115. package/dist/services/logger.js +14 -5
  116. package/dist/services/logger.js.map +1 -1
  117. package/dist/services/parser.d.ts +1 -0
  118. package/dist/services/parser.d.ts.map +1 -1
  119. package/dist/services/parser.js +55 -35
  120. package/dist/services/parser.js.map +1 -1
  121. package/dist/tools/handlers/fetch-links/link-extractor.d.ts +4 -0
  122. package/dist/tools/handlers/fetch-links/link-extractor.d.ts.map +1 -0
  123. package/dist/tools/handlers/fetch-links/link-extractor.js +163 -0
  124. package/dist/tools/handlers/fetch-links/link-extractor.js.map +1 -0
  125. package/dist/tools/handlers/fetch-links.tool.d.ts.map +1 -1
  126. package/dist/tools/handlers/fetch-links.tool.js +78 -116
  127. package/dist/tools/handlers/fetch-links.tool.js.map +1 -1
  128. package/dist/tools/handlers/fetch-markdown.tool.d.ts +3 -13
  129. package/dist/tools/handlers/fetch-markdown.tool.d.ts.map +1 -1
  130. package/dist/tools/handlers/fetch-markdown.tool.js +74 -83
  131. package/dist/tools/handlers/fetch-markdown.tool.js.map +1 -1
  132. package/dist/tools/handlers/fetch-single.shared.d.ts +26 -0
  133. package/dist/tools/handlers/fetch-single.shared.d.ts.map +1 -0
  134. package/dist/tools/handlers/fetch-single.shared.js +49 -0
  135. package/dist/tools/handlers/fetch-single.shared.js.map +1 -0
  136. package/dist/tools/handlers/fetch-url.tool.d.ts.map +1 -1
  137. package/dist/tools/handlers/fetch-url.tool.js +82 -54
  138. package/dist/tools/handlers/fetch-url.tool.js.map +1 -1
  139. package/dist/tools/handlers/fetch-urls/processor.d.ts +13 -0
  140. package/dist/tools/handlers/fetch-urls/processor.d.ts.map +1 -0
  141. package/dist/tools/handlers/fetch-urls/processor.js +153 -0
  142. package/dist/tools/handlers/fetch-urls/processor.js.map +1 -0
  143. package/dist/tools/handlers/fetch-urls/response.d.ts +3 -0
  144. package/dist/tools/handlers/fetch-urls/response.d.ts.map +1 -0
  145. package/dist/tools/handlers/fetch-urls/response.js +58 -0
  146. package/dist/tools/handlers/fetch-urls/response.js.map +1 -0
  147. package/dist/tools/handlers/fetch-urls/validation.d.ts +6 -0
  148. package/dist/tools/handlers/fetch-urls/validation.d.ts.map +1 -0
  149. package/dist/tools/handlers/fetch-urls/validation.js +18 -0
  150. package/dist/tools/handlers/fetch-urls/validation.js.map +1 -0
  151. package/dist/tools/handlers/fetch-urls.tool.d.ts.map +1 -1
  152. package/dist/tools/handlers/fetch-urls.tool.js +104 -197
  153. package/dist/tools/handlers/fetch-urls.tool.js.map +1 -1
  154. package/dist/tools/index.d.ts.map +1 -1
  155. package/dist/tools/index.js +36 -237
  156. package/dist/tools/index.js.map +1 -1
  157. package/dist/tools/schemas.d.ts +357 -0
  158. package/dist/tools/schemas.d.ts.map +1 -0
  159. package/dist/tools/schemas.js +272 -0
  160. package/dist/tools/schemas.js.map +1 -0
  161. package/dist/tools/utils/cache-vary.d.ts +3 -0
  162. package/dist/tools/utils/cache-vary.d.ts.map +1 -0
  163. package/dist/tools/utils/cache-vary.js +44 -0
  164. package/dist/tools/utils/cache-vary.js.map +1 -0
  165. package/dist/tools/utils/common.d.ts +2 -2
  166. package/dist/tools/utils/common.d.ts.map +1 -1
  167. package/dist/tools/utils/common.js +5 -1
  168. package/dist/tools/utils/common.js.map +1 -1
  169. package/dist/tools/utils/content-transform.d.ts +16 -0
  170. package/dist/tools/utils/content-transform.d.ts.map +1 -0
  171. package/dist/tools/utils/content-transform.js +49 -0
  172. package/dist/tools/utils/content-transform.js.map +1 -0
  173. package/dist/tools/utils/fetch-pipeline.d.ts.map +1 -1
  174. package/dist/tools/utils/fetch-pipeline.js +32 -18
  175. package/dist/tools/utils/fetch-pipeline.js.map +1 -1
  176. package/dist/tools/utils/inline-content.d.ts +11 -0
  177. package/dist/tools/utils/inline-content.d.ts.map +1 -0
  178. package/dist/tools/utils/inline-content.js +39 -0
  179. package/dist/tools/utils/inline-content.js.map +1 -0
  180. package/dist/tools/utils/markdown-toc.d.ts +3 -0
  181. package/dist/tools/utils/markdown-toc.d.ts.map +1 -0
  182. package/dist/tools/utils/markdown-toc.js +35 -0
  183. package/dist/tools/utils/markdown-toc.js.map +1 -0
  184. package/dist/tools/utils/tool-response.d.ts +9 -0
  185. package/dist/tools/utils/tool-response.d.ts.map +1 -0
  186. package/dist/tools/utils/tool-response.js +19 -0
  187. package/dist/tools/utils/tool-response.js.map +1 -0
  188. package/dist/transformers/jsonl.transformer.d.ts.map +1 -1
  189. package/dist/transformers/jsonl.transformer.js +51 -28
  190. package/dist/transformers/jsonl.transformer.js.map +1 -1
  191. package/dist/transformers/markdown.transformer.d.ts.map +1 -1
  192. package/dist/transformers/markdown.transformer.js +82 -111
  193. package/dist/transformers/markdown.transformer.js.map +1 -1
  194. package/dist/utils/header-normalizer.d.ts +5 -0
  195. package/dist/utils/header-normalizer.d.ts.map +1 -0
  196. package/dist/utils/header-normalizer.js +25 -0
  197. package/dist/utils/header-normalizer.js.map +1 -0
  198. package/dist/utils/tool-error-handler.d.ts +1 -0
  199. package/dist/utils/tool-error-handler.d.ts.map +1 -1
  200. package/dist/utils/tool-error-handler.js +29 -1
  201. package/dist/utils/tool-error-handler.js.map +1 -1
  202. package/dist/utils/url-validator.d.ts +0 -3
  203. package/dist/utils/url-validator.d.ts.map +1 -1
  204. package/dist/utils/url-validator.js +98 -18
  205. package/dist/utils/url-validator.js.map +1 -1
  206. package/package.json +11 -6
@@ -1,273 +1,80 @@
1
- import axios, { isCancel, } from 'axios';
2
- import crypto from 'crypto';
3
- import dns from 'dns';
4
- import http from 'http';
5
- import https from 'https';
6
- import os from 'os';
7
1
  import { config } from '../config/index.js';
8
- import { FetchError } from '../errors/app-error.js';
9
- import { isBlockedIp } from '../utils/url-validator.js';
10
- import { logDebug, logError, logWarn } from './logger.js';
11
- // --- Constants & Types ---
12
- const REQUEST_START_TIME = Symbol('requestStartTime');
13
- const REQUEST_ID = Symbol('requestId');
14
- // --- Helper Functions ---
15
- function sanitizeHeaders(headers) {
16
- if (!headers || Object.keys(headers).length === 0) {
17
- return undefined;
18
- }
19
- const { blockedHeaders } = config.security;
20
- const crlfRegex = /[\r\n]/;
21
- const sanitized = {};
22
- for (const [key, value] of Object.entries(headers)) {
23
- if (!blockedHeaders.has(key.toLowerCase()) &&
24
- !crlfRegex.test(key) &&
25
- !crlfRegex.test(value)) {
26
- sanitized[key] = value;
2
+ import { validateAndNormalizeUrl } from '../utils/url-validator.js';
3
+ import { destroyAgents, dispatcher } from './fetcher/agents.js';
4
+ import { createHttpError, createRateLimitError, mapFetchError, } from './fetcher/errors.js';
5
+ import { sanitizeHeaders } from './fetcher/headers.js';
6
+ import { recordFetchError, recordFetchResponse, startFetchTelemetry, } from './fetcher/interceptors.js';
7
+ import { fetchWithRedirects } from './fetcher/redirects.js';
8
+ import { readResponseText } from './fetcher/response.js';
9
+ import { RetryPolicy } from './fetcher/retry-policy.js';
10
+ export { destroyAgents };
11
+ const DEFAULT_HEADERS = {
12
+ 'User-Agent': config.fetcher.userAgent,
13
+ Accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
14
+ 'Accept-Language': 'en-US,en;q=0.5',
15
+ 'Accept-Encoding': 'gzip, deflate, br',
16
+ Connection: 'keep-alive',
17
+ };
18
+ function buildHeaders(customHeaders) {
19
+ const headers = new Headers(DEFAULT_HEADERS);
20
+ const sanitized = sanitizeHeaders(customHeaders);
21
+ if (sanitized) {
22
+ for (const [key, value] of Object.entries(sanitized)) {
23
+ headers.set(key, value);
27
24
  }
28
25
  }
29
- return Object.keys(sanitized).length > 0 ? sanitized : undefined;
26
+ return headers;
30
27
  }
31
- function resolveDns(hostname, options, callback) {
32
- dns.lookup(hostname, options, (err, address, family) => {
33
- if (err) {
34
- callback(err, address, family);
35
- return;
36
- }
37
- const addresses = Array.isArray(address) ? address : [{ address, family }];
38
- for (const addr of addresses) {
39
- const ip = typeof addr === 'string' ? addr : addr.address;
40
- if (isBlockedIp(ip)) {
41
- const error = new Error(`Blocked IP detected for ${hostname}`);
42
- error.code = 'EBLOCKED';
43
- callback(error, address, family);
44
- return;
45
- }
46
- }
47
- callback(null, address, family);
48
- });
28
+ function buildRequestSignal(timeoutMs, external) {
29
+ const timeoutSignal = AbortSignal.timeout(timeoutMs);
30
+ if (!external)
31
+ return timeoutSignal;
32
+ return AbortSignal.any([external, timeoutSignal]);
49
33
  }
50
- function getAgentOptions() {
51
- const cpuCount = os.cpus().length;
34
+ function buildRequestInit(headers, signal) {
52
35
  return {
53
- keepAlive: true,
54
- maxSockets: Math.max(cpuCount * 2, 25),
55
- maxFreeSockets: Math.max(Math.floor(cpuCount * 0.5), 10),
56
- timeout: 60000,
57
- scheduling: 'fifo',
58
- lookup: resolveDns,
36
+ method: 'GET',
37
+ headers,
38
+ signal,
39
+ dispatcher,
59
40
  };
60
41
  }
61
- function handleRequest(config) {
62
- const timedConfig = config;
63
- timedConfig[REQUEST_START_TIME] = Date.now();
64
- timedConfig[REQUEST_ID] = crypto.randomUUID().substring(0, 8);
65
- logDebug('HTTP Request', {
66
- requestId: timedConfig[REQUEST_ID],
67
- method: config.method?.toUpperCase(),
68
- url: config.url,
69
- });
70
- return config;
71
- }
72
- function handleRequestError(error) {
73
- logError('HTTP Request Error', error);
74
- throw error;
75
- }
76
- function calculateDuration(config) {
77
- const startTime = config[REQUEST_START_TIME];
78
- return startTime ? Date.now() - startTime : 0;
79
- }
80
- function logResponse(response, requestId, duration) {
81
- const headers = response.headers;
82
- const contentType = headers['content-type'];
83
- const contentTypeStr = typeof contentType === 'string' ? contentType : undefined;
84
- logDebug('HTTP Response', {
85
- requestId,
86
- status: response.status,
87
- url: response.config.url ?? 'unknown',
88
- contentType: contentTypeStr,
89
- duration: `${duration}ms`,
90
- size: headers['content-length'],
91
- });
92
- if (duration > 5000) {
93
- logWarn('Slow HTTP request detected', {
94
- requestId,
95
- url: response.config.url ?? 'unknown',
96
- duration: `${duration}ms`,
97
- });
42
+ async function handleFetchResponse(response, finalUrl, telemetry) {
43
+ if (response.status === 429) {
44
+ void response.body?.cancel();
45
+ throw createRateLimitError(finalUrl, response.headers.get('retry-after'));
98
46
  }
99
- }
100
- function parseRetryAfter(header) {
101
- if (!header)
102
- return 60;
103
- const parsed = typeof header === 'string' ? parseInt(header, 10) : Number(header);
104
- return isNaN(parsed) ? 60 : parsed;
105
- }
106
- function handleResponseError(error) {
107
- const url = error.config?.url ?? 'unknown';
108
- if (isCancel(error) ||
109
- error.name === 'AbortError' ||
110
- error.name === 'CanceledError') {
111
- logDebug('HTTP Request Aborted/Canceled', { url });
112
- throw new FetchError('Request was canceled', url, 499, {
113
- reason: 'aborted',
114
- });
115
- }
116
- if (error.code === 'ECONNABORTED' || error.code === 'ETIMEDOUT') {
117
- logError('HTTP Timeout', { url, timeout: config.fetcher.timeout });
118
- throw new FetchError(`Request timeout after ${config.fetcher.timeout}ms`, url, 504, { timeout: config.fetcher.timeout });
119
- }
120
- if (error.response) {
121
- const { status, statusText, headers } = error.response;
122
- const responseHeaders = headers;
123
- if (status === 429) {
124
- const retryAfter = parseRetryAfter(responseHeaders['retry-after']);
125
- logWarn('Rate limited by server', { url, retryAfter: `${retryAfter}s` });
126
- throw new FetchError('Too many requests', url, 429, { retryAfter });
127
- }
128
- logError('HTTP Error Response', { url, status, statusText });
129
- throw new FetchError(`HTTP ${status}: ${statusText}`, url, status);
130
- }
131
- if (error.request) {
132
- logError('HTTP Network Error', { url, code: error.code });
133
- throw new FetchError(`Network error: Could not reach ${url}`, url);
47
+ if (!response.ok) {
48
+ void response.body?.cancel();
49
+ throw createHttpError(finalUrl, response.status, response.statusText);
134
50
  }
135
- logError('HTTP Unknown Error', { url, message: error.message });
136
- throw new FetchError(error.message, url);
137
- }
138
- function handleResponse(response) {
139
- const timedConfig = response.config;
140
- const duration = calculateDuration(timedConfig);
141
- const requestId = timedConfig[REQUEST_ID];
142
- // Cleanup symbols safely
143
- timedConfig[REQUEST_START_TIME] = undefined;
144
- timedConfig[REQUEST_ID] = undefined;
145
- logResponse(response, requestId, duration);
146
- return response;
51
+ const { text, size } = await readResponseText(response, finalUrl, config.fetcher.maxContentLength);
52
+ recordFetchResponse(telemetry, response, size);
53
+ return text;
147
54
  }
148
- class RetryPolicy {
149
- maxRetries;
150
- url;
151
- static BASE_DELAY_MS = 1000;
152
- static MAX_DELAY_MS = 10000;
153
- static JITTER_FACTOR = 0.25;
154
- constructor(maxRetries, url) {
155
- this.maxRetries = maxRetries;
156
- this.url = url;
55
+ async function fetchWithTelemetry(normalizedUrl, requestInit, timeoutMs) {
56
+ const telemetry = startFetchTelemetry(normalizedUrl, 'GET');
57
+ try {
58
+ const { response, url: finalUrl } = await fetchWithRedirects(normalizedUrl, requestInit, config.fetcher.maxRedirects);
59
+ telemetry.url = finalUrl;
60
+ return await handleFetchResponse(response, finalUrl, telemetry);
157
61
  }
158
- async execute(operation, signal) {
159
- let lastError = new Error(`Failed to fetch ${this.url}`);
160
- const retries = Math.min(Math.max(1, this.maxRetries), 10);
161
- for (let attempt = 1; attempt <= retries; attempt++) {
162
- if (signal?.aborted) {
163
- throw new FetchError('Request was aborted before execution', this.url);
164
- }
165
- try {
166
- return await operation();
167
- }
168
- catch (error) {
169
- lastError = error instanceof Error ? error : new Error(String(error));
170
- if (!this.shouldRetry(attempt, retries, lastError)) {
171
- throw lastError;
172
- }
173
- await this.wait(attempt, lastError);
174
- }
175
- }
176
- throw new FetchError(`Failed after ${retries} attempts: ${lastError.message}`, this.url);
177
- }
178
- shouldRetry(attempt, maxRetries, error) {
179
- if (attempt >= maxRetries)
180
- return false;
181
- if (error instanceof FetchError) {
182
- if (error.details.reason === 'aborted')
183
- return false;
184
- if (error.details.httpStatus === 429)
185
- return true;
186
- const status = error.details.httpStatus;
187
- if (status && status >= 400 && status < 500)
188
- return false;
189
- }
190
- return true;
191
- }
192
- async wait(attempt, error) {
193
- let delay;
194
- if (error instanceof FetchError && error.details.httpStatus === 429) {
195
- const retryAfter = error.details.retryAfter || 60;
196
- delay = Math.min(retryAfter * 1000, 30000);
197
- logWarn('Rate limited, waiting before retry', {
198
- url: this.url,
199
- attempt,
200
- waitTime: `${delay}ms`,
201
- });
202
- }
203
- else {
204
- delay = this.calculateBackoff(attempt);
205
- logDebug('Retrying request', {
206
- url: this.url,
207
- attempt,
208
- delay: `${delay}ms`,
209
- });
210
- }
211
- await new Promise((resolve) => setTimeout(resolve, delay));
212
- }
213
- calculateBackoff(attempt) {
214
- const exponentialDelay = Math.min(RetryPolicy.BASE_DELAY_MS * Math.pow(2, attempt - 1), RetryPolicy.MAX_DELAY_MS);
215
- const jitter = exponentialDelay * RetryPolicy.JITTER_FACTOR * (Math.random() * 2 - 1);
216
- return Math.round(exponentialDelay + jitter);
62
+ catch (error) {
63
+ const mapped = mapFetchError(error, normalizedUrl, timeoutMs);
64
+ telemetry.url = mapped.url;
65
+ recordFetchError(telemetry, mapped, mapped.statusCode);
66
+ throw mapped;
217
67
  }
218
68
  }
219
- const httpAgent = new http.Agent(getAgentOptions());
220
- const httpsAgent = new https.Agent(getAgentOptions());
221
- const client = axios.create({
222
- timeout: config.fetcher.timeout,
223
- maxRedirects: config.fetcher.maxRedirects,
224
- maxContentLength: config.fetcher.maxContentLength,
225
- httpAgent,
226
- httpsAgent,
227
- headers: {
228
- 'User-Agent': config.fetcher.userAgent,
229
- Accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
230
- 'Accept-Language': 'en-US,en;q=0.5',
231
- 'Accept-Encoding': 'gzip, deflate, br',
232
- Connection: 'keep-alive',
233
- },
234
- validateStatus: (status) => status >= 200 && status < 300,
235
- });
236
- client.interceptors.request.use(handleRequest, handleRequestError);
237
- client.interceptors.response.use(handleResponse, handleResponseError);
238
- export function destroyAgents() {
239
- httpAgent.destroy();
240
- httpsAgent.destroy();
241
- }
242
69
  export async function fetchUrlWithRetry(url, options, maxRetries = 3) {
243
- const policy = new RetryPolicy(maxRetries, url);
70
+ const normalizedUrl = validateAndNormalizeUrl(url);
71
+ const policy = new RetryPolicy(maxRetries, normalizedUrl);
72
+ const timeoutMs = options?.timeout ?? config.fetcher.timeout;
73
+ const headers = buildHeaders(options?.customHeaders);
244
74
  return policy.execute(async () => {
245
- const requestConfig = {
246
- method: 'GET',
247
- url,
248
- responseType: 'text',
249
- };
250
- if (options?.signal) {
251
- requestConfig.signal = options.signal;
252
- }
253
- else if (options?.timeout) {
254
- requestConfig.signal = AbortSignal.timeout(options.timeout);
255
- }
256
- const sanitizedHeaders = sanitizeHeaders(options?.customHeaders);
257
- if (sanitizedHeaders) {
258
- const existingHeaders = (requestConfig.headers ?? {});
259
- requestConfig.headers = { ...existingHeaders, ...sanitizedHeaders };
260
- }
261
- try {
262
- const response = await client.request(requestConfig);
263
- return response.data;
264
- }
265
- catch (error) {
266
- if (error instanceof FetchError) {
267
- throw error;
268
- }
269
- throw new FetchError(`Unexpected error: ${error instanceof Error ? error.message : 'Unknown'}`, url);
270
- }
75
+ const signal = buildRequestSignal(timeoutMs, options?.signal);
76
+ const requestInit = buildRequestInit(headers, signal);
77
+ return fetchWithTelemetry(normalizedUrl, requestInit, timeoutMs);
271
78
  }, options?.signal);
272
79
  }
273
80
  //# sourceMappingURL=fetcher.js.map
@@ -1 +1 @@
1
- {"version":3,"file":"fetcher.js","sourceRoot":"","sources":["../../src/services/fetcher.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,EAOZ,QAAQ,GAET,MAAM,OAAO,CAAC;AACf,OAAO,MAAM,MAAM,QAAQ,CAAC;AAC5B,OAAO,GAAG,MAAM,KAAK,CAAC;AACtB,OAAO,IAAI,MAAM,MAAM,CAAC;AACxB,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,EAAE,MAAM,IAAI,CAAC;AAEpB,OAAO,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAG5C,OAAO,EAAE,UAAU,EAAE,MAAM,wBAAwB,CAAC;AAEpD,OAAO,EAAE,WAAW,EAAE,MAAM,2BAA2B,CAAC;AAExD,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,aAAa,CAAC;AAE1D,4BAA4B;AAE5B,MAAM,kBAAkB,GAAG,MAAM,CAAC,kBAAkB,CAAC,CAAC;AACtD,MAAM,UAAU,GAAG,MAAM,CAAC,WAAW,CAAC,CAAC;AAOvC,2BAA2B;AAE3B,SAAS,eAAe,CACtB,OAAgC;IAEhC,IAAI,CAAC,OAAO,IAAI,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAClD,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,MAAM,EAAE,cAAc,EAAE,GAAG,MAAM,CAAC,QAAQ,CAAC;IAC3C,MAAM,SAAS,GAAG,QAAQ,CAAC;IAC3B,MAAM,SAAS,GAA2B,EAAE,CAAC;IAE7C,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;QACnD,IACE,CAAC,cAAc,CAAC,GAAG,CAAC,GAAG,CAAC,WAAW,EAAE,CAAC;YACtC,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC;YACpB,CAAC,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,EACtB,CAAC;YACD,SAAS,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC;QACzB,CAAC;IACH,CAAC;IAED,OAAO,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC;AACnE,CAAC;AAED,SAAS,UAAU,CACjB,QAAgB,EAChB,OAA0B,EAC1B,QAIS;IAET,GAAG,CAAC,MAAM,CAAC,QAAQ,EAAE,OAAO,EAAE,CAAC,GAAG,EAAE,OAAO,EAAE,MAAM,EAAE,EAAE;QACrD,IAAI,GAAG,EAAE,CAAC;YACR,QAAQ,CAAC,GAAG,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC;YAC/B,OAAO;QACT,CAAC;QAED,MAAM,SAAS,GAAG,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,CAAC;QAE3E,KAAK,MAAM,IAAI,IAAI,SAAS,EAAE,CAAC;YAC7B,MAAM,EAAE,GAAG,OAAO,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC;YAC1D,IAAI,WAAW,CAAC,EAAE,CAAC,EAAE,CAAC;gBACpB,MAAM,KAAK,GAAG,IAAI,KAAK,CACrB,2BAA2B,QAAQ,EAAE,CACb,CAAC;gBAC3B,KAAK,CAAC,IAAI,GAAG,UAAU,CAAC;gBACxB,QAAQ,CAAC,KAAK,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC;gBACjC,OAAO;YACT,CAAC;QACH,CAAC;QAED,QAAQ,CAAC,IAAI,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC;IAClC,CAAC,CAAC,CAAC;AACL,CAAC;AAED,SAAS,eAAe;IACtB,MAAM,QAAQ,GAAG,EAAE,CAAC,IAAI,EAAE,CAAC,MAAM,CAAC;IAClC,OAAO;QACL,SAAS,EAAE,IAAI;QACf,UAAU,EAAE,IAAI,CAAC,GAAG,CAAC,QAAQ,GAAG,CAAC,EAAE,EAAE,CAAC;QACtC,cAAc,EAAE,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ,GAAG,GAAG,CAAC,EAAE,EAAE,CAAC;QACxD,OAAO,EAAE,KAAK;QACd,UAAU,EAAE,MAAM;QAClB,MAAM,EAAE,UAAU;KACnB,CAAC;AACJ,CAAC;AAED,SAAS,aAAa,CACpB,MAAkC;IAElC,MAAM,WAAW,GAAG,MAAiC,CAAC;IACtD,WAAW,CAAC,kBAAkB,CAAC,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAC7C,WAAW,CAAC,UAAU,CAAC,GAAG,MAAM,CAAC,UAAU,EAAE,CAAC,SAAS,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IAE9D,QAAQ,CAAC,cAAc,EAAE;QACvB,SAAS,EAAE,WAAW,CAAC,UAAU,CAAC;QAClC,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,WAAW,EAAE;QACpC,GAAG,EAAE,MAAM,CAAC,GAAG;KAChB,CAAC,CAAC;IAEH,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,kBAAkB,CAAC,KAAiB;IAC3C,QAAQ,CAAC,oBAAoB,EAAE,KAAK,CAAC,CAAC;IACtC,MAAM,KAAK,CAAC;AACd,CAAC;AAED,SAAS,iBAAiB,CAAC,MAA+B;IACxD,MAAM,SAAS,GAAG,MAAM,CAAC,kBAAkB,CAAC,CAAC;IAC7C,OAAO,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC;AAChD,CAAC;AAED,SAAS,WAAW,CAClB,QAAuB,EACvB,SAA6B,EAC7B,QAAgB;IAEhB,MAAM,OAAO,GAAG,QAAQ,CAAC,OAEE,CAAC;IAC5B,MAAM,WAAW,GAAG,OAAO,CAAC,cAAc,CAAY,CAAC;IACvD,MAAM,cAAc,GAClB,OAAO,WAAW,KAAK,QAAQ,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,SAAS,CAAC;IAE5D,QAAQ,CAAC,eAAe,EAAE;QACxB,SAAS;QACT,MAAM,EAAE,QAAQ,CAAC,MAAM;QACvB,GAAG,EAAE,QAAQ,CAAC,MAAM,CAAC,GAAG,IAAI,SAAS;QACrC,WAAW,EAAE,cAAc;QAC3B,QAAQ,EAAE,GAAG,QAAQ,IAAI;QACzB,IAAI,EAAE,OAAO,CAAC,gBAAgB,CAAC;KAChC,CAAC,CAAC;IAEH,IAAI,QAAQ,GAAG,IAAI,EAAE,CAAC;QACpB,OAAO,CAAC,4BAA4B,EAAE;YACpC,SAAS;YACT,GAAG,EAAE,QAAQ,CAAC,MAAM,CAAC,GAAG,IAAI,SAAS;YACrC,QAAQ,EAAE,GAAG,QAAQ,IAAI;SAC1B,CAAC,CAAC;IACL,CAAC;AACH,CAAC;AAED,SAAS,eAAe,CAAC,MAAe;IACtC,IAAI,CAAC,MAAM;QAAE,OAAO,EAAE,CAAC;IACvB,MAAM,MAAM,GACV,OAAO,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;IACrE,OAAO,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC;AACrC,CAAC;AAED,SAAS,mBAAmB,CAAC,KAAiB;IAC5C,MAAM,GAAG,GAAG,KAAK,CAAC,MAAM,EAAE,GAAG,IAAI,SAAS,CAAC;IAE3C,IACE,QAAQ,CAAC,KAAK,CAAC;QACf,KAAK,CAAC,IAAI,KAAK,YAAY;QAC3B,KAAK,CAAC,IAAI,KAAK,eAAe,EAC9B,CAAC;QACD,QAAQ,CAAC,+BAA+B,EAAE,EAAE,GAAG,EAAE,CAAC,CAAC;QACnD,MAAM,IAAI,UAAU,CAAC,sBAAsB,EAAE,GAAG,EAAE,GAAG,EAAE;YACrD,MAAM,EAAE,SAAS;SAClB,CAAC,CAAC;IACL,CAAC;IAED,IAAI,KAAK,CAAC,IAAI,KAAK,cAAc,IAAI,KAAK,CAAC,IAAI,KAAK,WAAW,EAAE,CAAC;QAChE,QAAQ,CAAC,cAAc,EAAE,EAAE,GAAG,EAAE,OAAO,EAAE,MAAM,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC;QACnE,MAAM,IAAI,UAAU,CAClB,yBAAyB,MAAM,CAAC,OAAO,CAAC,OAAO,IAAI,EACnD,GAAG,EACH,GAAG,EACH,EAAE,OAAO,EAAE,MAAM,CAAC,OAAO,CAAC,OAAO,EAAE,CACpC,CAAC;IACJ,CAAC;IAED,IAAI,KAAK,CAAC,QAAQ,EAAE,CAAC;QACnB,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,OAAO,EAAE,GAAG,KAAK,CAAC,QAAQ,CAAC;QACvD,MAAM,eAAe,GAAG,OAEG,CAAC;QAE5B,IAAI,MAAM,KAAK,GAAG,EAAE,CAAC;YACnB,MAAM,UAAU,GAAG,eAAe,CAAC,eAAe,CAAC,aAAa,CAAC,CAAC,CAAC;YACnE,OAAO,CAAC,wBAAwB,EAAE,EAAE,GAAG,EAAE,UAAU,EAAE,GAAG,UAAU,GAAG,EAAE,CAAC,CAAC;YACzE,MAAM,IAAI,UAAU,CAAC,mBAAmB,EAAE,GAAG,EAAE,GAAG,EAAE,EAAE,UAAU,EAAE,CAAC,CAAC;QACtE,CAAC;QAED,QAAQ,CAAC,qBAAqB,EAAE,EAAE,GAAG,EAAE,MAAM,EAAE,UAAU,EAAE,CAAC,CAAC;QAC7D,MAAM,IAAI,UAAU,CAAC,QAAQ,MAAM,KAAK,UAAU,EAAE,EAAE,GAAG,EAAE,MAAM,CAAC,CAAC;IACrE,CAAC;IAED,IAAI,KAAK,CAAC,OAAO,EAAE,CAAC;QAClB,QAAQ,CAAC,oBAAoB,EAAE,EAAE,GAAG,EAAE,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC;QAC1D,MAAM,IAAI,UAAU,CAAC,kCAAkC,GAAG,EAAE,EAAE,GAAG,CAAC,CAAC;IACrE,CAAC;IAED,QAAQ,CAAC,oBAAoB,EAAE,EAAE,GAAG,EAAE,OAAO,EAAE,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;IAChE,MAAM,IAAI,UAAU,CAAC,KAAK,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;AAC3C,CAAC;AAED,SAAS,cAAc,CAAC,QAAuB;IAC7C,MAAM,WAAW,GAAG,QAAQ,CAAC,MAAiC,CAAC;IAC/D,MAAM,QAAQ,GAAG,iBAAiB,CAAC,WAAW,CAAC,CAAC;IAChD,MAAM,SAAS,GAAG,WAAW,CAAC,UAAU,CAAC,CAAC;IAE1C,yBAAyB;IACzB,WAAW,CAAC,kBAAkB,CAAC,GAAG,SAAS,CAAC;IAC5C,WAAW,CAAC,UAAU,CAAC,GAAG,SAAS,CAAC;IAEpC,WAAW,CAAC,QAAQ,EAAE,SAAS,EAAE,QAAQ,CAAC,CAAC;IAE3C,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED,MAAM,WAAW;IAMI;IACA;IANX,MAAM,CAAU,aAAa,GAAG,IAAI,CAAC;IACrC,MAAM,CAAU,YAAY,GAAG,KAAK,CAAC;IACrC,MAAM,CAAU,aAAa,GAAG,IAAI,CAAC;IAE7C,YACmB,UAAkB,EAClB,GAAW;QADX,eAAU,GAAV,UAAU,CAAQ;QAClB,QAAG,GAAH,GAAG,CAAQ;IAC3B,CAAC;IAEJ,KAAK,CAAC,OAAO,CACX,SAA2B,EAC3B,MAAoB;QAEpB,IAAI,SAAS,GAAU,IAAI,KAAK,CAAC,mBAAmB,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC;QAChE,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,UAAU,CAAC,EAAE,EAAE,CAAC,CAAC;QAE3D,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,IAAI,OAAO,EAAE,OAAO,EAAE,EAAE,CAAC;YACpD,IAAI,MAAM,EAAE,OAAO,EAAE,CAAC;gBACpB,MAAM,IAAI,UAAU,CAAC,sCAAsC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC;YACzE,CAAC;YAED,IAAI,CAAC;gBACH,OAAO,MAAM,SAAS,EAAE,CAAC;YAC3B,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,SAAS,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;gBAEtE,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,OAAO,EAAE,OAAO,EAAE,SAAS,CAAC,EAAE,CAAC;oBACnD,MAAM,SAAS,CAAC;gBAClB,CAAC;gBAED,MAAM,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC;YACtC,CAAC;QACH,CAAC;QAED,MAAM,IAAI,UAAU,CAClB,gBAAgB,OAAO,cAAc,SAAS,CAAC,OAAO,EAAE,EACxD,IAAI,CAAC,GAAG,CACT,CAAC;IACJ,CAAC;IAEO,WAAW,CACjB,OAAe,EACf,UAAkB,EAClB,KAAY;QAEZ,IAAI,OAAO,IAAI,UAAU;YAAE,OAAO,KAAK,CAAC;QAExC,IAAI,KAAK,YAAY,UAAU,EAAE,CAAC;YAChC,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,KAAK,SAAS;gBAAE,OAAO,KAAK,CAAC;YACrD,IAAI,KAAK,CAAC,OAAO,CAAC,UAAU,KAAK,GAAG;gBAAE,OAAO,IAAI,CAAC;YAElD,MAAM,MAAM,GAAG,KAAK,CAAC,OAAO,CAAC,UAAgC,CAAC;YAC9D,IAAI,MAAM,IAAI,MAAM,IAAI,GAAG,IAAI,MAAM,GAAG,GAAG;gBAAE,OAAO,KAAK,CAAC;QAC5D,CAAC;QAED,OAAO,IAAI,CAAC;IACd,CAAC;IAEO,KAAK,CAAC,IAAI,CAAC,OAAe,EAAE,KAAY;QAC9C,IAAI,KAAa,CAAC;QAElB,IAAI,KAAK,YAAY,UAAU,IAAI,KAAK,CAAC,OAAO,CAAC,UAAU,KAAK,GAAG,EAAE,CAAC;YACpE,MAAM,UAAU,GAAI,KAAK,CAAC,OAAO,CAAC,UAAqB,IAAI,EAAE,CAAC;YAC9D,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,UAAU,GAAG,IAAI,EAAE,KAAK,CAAC,CAAC;YAC3C,OAAO,CAAC,oCAAoC,EAAE;gBAC5C,GAAG,EAAE,IAAI,CAAC,GAAG;gBACb,OAAO;gBACP,QAAQ,EAAE,GAAG,KAAK,IAAI;aACvB,CAAC,CAAC;QACL,CAAC;aAAM,CAAC;YACN,KAAK,GAAG,IAAI,CAAC,gBAAgB,CAAC,OAAO,CAAC,CAAC;YACvC,QAAQ,CAAC,kBAAkB,EAAE;gBAC3B,GAAG,EAAE,IAAI,CAAC,GAAG;gBACb,OAAO;gBACP,KAAK,EAAE,GAAG,KAAK,IAAI;aACpB,CAAC,CAAC;QACL,CAAC;QAED,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC,CAAC;IAC7D,CAAC;IAEO,gBAAgB,CAAC,OAAe;QACtC,MAAM,gBAAgB,GAAG,IAAI,CAAC,GAAG,CAC/B,WAAW,CAAC,aAAa,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,GAAG,CAAC,CAAC,EACpD,WAAW,CAAC,YAAY,CACzB,CAAC;QACF,MAAM,MAAM,GACV,gBAAgB,GAAG,WAAW,CAAC,aAAa,GAAG,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC;QACzE,OAAO,IAAI,CAAC,KAAK,CAAC,gBAAgB,GAAG,MAAM,CAAC,CAAC;IAC/C,CAAC;;AAGH,MAAM,SAAS,GAAG,IAAI,IAAI,CAAC,KAAK,CAAC,eAAe,EAAE,CAAC,CAAC;AACpD,MAAM,UAAU,GAAG,IAAI,KAAK,CAAC,KAAK,CAAC,eAAe,EAAE,CAAC,CAAC;AAEtD,MAAM,MAAM,GAAkB,KAAK,CAAC,MAAM,CAAC;IACzC,OAAO,EAAE,MAAM,CAAC,OAAO,CAAC,OAAO;IAC/B,YAAY,EAAE,MAAM,CAAC,OAAO,CAAC,YAAY;IACzC,gBAAgB,EAAE,MAAM,CAAC,OAAO,CAAC,gBAAgB;IACjD,SAAS;IACT,UAAU;IACV,OAAO,EAAE;QACP,YAAY,EAAE,MAAM,CAAC,OAAO,CAAC,SAAS;QACtC,MAAM,EACJ,4EAA4E;QAC9E,iBAAiB,EAAE,gBAAgB;QACnC,iBAAiB,EAAE,mBAAmB;QACtC,UAAU,EAAE,YAAY;KACzB;IACD,cAAc,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,IAAI,GAAG,IAAI,MAAM,GAAG,GAAG;CAC1D,CAAC,CAAC;AAEH,MAAM,CAAC,YAAY,CAAC,OAAO,CAAC,GAAG,CAAC,aAAa,EAAE,kBAAkB,CAAC,CAAC;AACnE,MAAM,CAAC,YAAY,CAAC,QAAQ,CAAC,GAAG,CAAC,cAAc,EAAE,mBAAmB,CAAC,CAAC;AAEtE,MAAM,UAAU,aAAa;IAC3B,SAAS,CAAC,OAAO,EAAE,CAAC;IACpB,UAAU,CAAC,OAAO,EAAE,CAAC;AACvB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,iBAAiB,CACrC,GAAW,EACX,OAAsB,EACtB,UAAU,GAAG,CAAC;IAEd,MAAM,MAAM,GAAG,IAAI,WAAW,CAAC,UAAU,EAAE,GAAG,CAAC,CAAC;IAEhD,OAAO,MAAM,CAAC,OAAO,CAAC,KAAK,IAAI,EAAE;QAC/B,MAAM,aAAa,GAAuB;YACxC,MAAM,EAAE,KAAK;YACb,GAAG;YACH,YAAY,EAAE,MAAM;SACrB,CAAC;QAEF,IAAI,OAAO,EAAE,MAAM,EAAE,CAAC;YACpB,aAAa,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;QACxC,CAAC;aAAM,IAAI,OAAO,EAAE,OAAO,EAAE,CAAC;YAC5B,aAAa,CAAC,MAAM,GAAG,WAAW,CAAC,OAAO,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;QAC9D,CAAC;QAED,MAAM,gBAAgB,GAAG,eAAe,CAAC,OAAO,EAAE,aAAa,CAAC,CAAC;QACjE,IAAI,gBAAgB,EAAE,CAAC;YACrB,MAAM,eAAe,GAAG,CAAC,aAAa,CAAC,OAAO,IAAI,EAAE,CAGnD,CAAC;YACF,aAAa,CAAC,OAAO,GAAG,EAAE,GAAG,eAAe,EAAE,GAAG,gBAAgB,EAAE,CAAC;QACtE,CAAC;QAED,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,OAAO,CAAS,aAAa,CAAC,CAAC;YAC7D,OAAO,QAAQ,CAAC,IAAI,CAAC;QACvB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,KAAK,YAAY,UAAU,EAAE,CAAC;gBAChC,MAAM,KAAK,CAAC;YACd,CAAC;YACD,MAAM,IAAI,UAAU,CAClB,qBAAqB,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,SAAS,EAAE,EACzE,GAAG,CACJ,CAAC;QACJ,CAAC;IACH,CAAC,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC;AACtB,CAAC"}
1
+ {"version":3,"file":"fetcher.js","sourceRoot":"","sources":["../../src/services/fetcher.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAG5C,OAAO,EAAE,uBAAuB,EAAE,MAAM,2BAA2B,CAAC;AAEpE,OAAO,EAAE,aAAa,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAC;AAChE,OAAO,EACL,eAAe,EACf,oBAAoB,EACpB,aAAa,GACd,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EAAE,eAAe,EAAE,MAAM,sBAAsB,CAAC;AACvD,OAAO,EACL,gBAAgB,EAChB,mBAAmB,EACnB,mBAAmB,GACpB,MAAM,2BAA2B,CAAC;AACnC,OAAO,EAAE,kBAAkB,EAAE,MAAM,wBAAwB,CAAC;AAC5D,OAAO,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AACzD,OAAO,EAAE,WAAW,EAAE,MAAM,2BAA2B,CAAC;AAExD,OAAO,EAAE,aAAa,EAAE,CAAC;AAEzB,MAAM,eAAe,GAAG;IACtB,YAAY,EAAE,MAAM,CAAC,OAAO,CAAC,SAAS;IACtC,MAAM,EACJ,4EAA4E;IAC9E,iBAAiB,EAAE,gBAAgB;IACnC,iBAAiB,EAAE,mBAAmB;IACtC,UAAU,EAAE,YAAY;CAChB,CAAC;AAEX,SAAS,YAAY,CAAC,aAAsC;IAC1D,MAAM,OAAO,GAAG,IAAI,OAAO,CAAC,eAAe,CAAC,CAAC;IAC7C,MAAM,SAAS,GAAG,eAAe,CAAC,aAAa,CAAC,CAAC;IACjD,IAAI,SAAS,EAAE,CAAC;QACd,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,CAAC;YACrD,OAAO,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;QAC1B,CAAC;IACH,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,SAAS,kBAAkB,CACzB,SAAiB,EACjB,QAAsB;IAEtB,MAAM,aAAa,GAAG,WAAW,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;IACrD,IAAI,CAAC,QAAQ;QAAE,OAAO,aAAa,CAAC;IACpC,OAAO,WAAW,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,aAAa,CAAC,CAAC,CAAC;AACpD,CAAC;AAED,SAAS,gBAAgB,CACvB,OAAgB,EAChB,MAAmB;IAEnB,OAAO;QACL,MAAM,EAAE,KAAK;QACb,OAAO;QACP,MAAM;QACN,UAAU;KACX,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,mBAAmB,CAChC,QAAkB,EAClB,QAAgB,EAChB,SAAiD;IAEjD,IAAI,QAAQ,CAAC,MAAM,KAAK,GAAG,EAAE,CAAC;QAC5B,KAAK,QAAQ,CAAC,IAAI,EAAE,MAAM,EAAE,CAAC;QAC7B,MAAM,oBAAoB,CAAC,QAAQ,EAAE,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC,CAAC;IAC5E,CAAC;IAED,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;QACjB,KAAK,QAAQ,CAAC,IAAI,EAAE,MAAM,EAAE,CAAC;QAC7B,MAAM,eAAe,CAAC,QAAQ,EAAE,QAAQ,CAAC,MAAM,EAAE,QAAQ,CAAC,UAAU,CAAC,CAAC;IACxE,CAAC;IAED,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,GAAG,MAAM,gBAAgB,CAC3C,QAAQ,EACR,QAAQ,EACR,MAAM,CAAC,OAAO,CAAC,gBAAgB,CAChC,CAAC;IACF,mBAAmB,CAAC,SAAS,EAAE,QAAQ,EAAE,IAAI,CAAC,CAAC;IAC/C,OAAO,IAAI,CAAC;AACd,CAAC;AAED,KAAK,UAAU,kBAAkB,CAC/B,aAAqB,EACrB,WAAwB,EACxB,SAAiB;IAEjB,MAAM,SAAS,GAAG,mBAAmB,CAAC,aAAa,EAAE,KAAK,CAAC,CAAC;IAE5D,IAAI,CAAC;QACH,MAAM,EAAE,QAAQ,EAAE,GAAG,EAAE,QAAQ,EAAE,GAAG,MAAM,kBAAkB,CAC1D,aAAa,EACb,WAAW,EACX,MAAM,CAAC,OAAO,CAAC,YAAY,CAC5B,CAAC;QAEF,SAAS,CAAC,GAAG,GAAG,QAAQ,CAAC;QACzB,OAAO,MAAM,mBAAmB,CAAC,QAAQ,EAAE,QAAQ,EAAE,SAAS,CAAC,CAAC;IAClE,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,MAAM,GAAG,aAAa,CAAC,KAAK,EAAE,aAAa,EAAE,SAAS,CAAC,CAAC;QAC9D,SAAS,CAAC,GAAG,GAAG,MAAM,CAAC,GAAG,CAAC;QAC3B,gBAAgB,CAAC,SAAS,EAAE,MAAM,EAAE,MAAM,CAAC,UAAU,CAAC,CAAC;QACvD,MAAM,MAAM,CAAC;IACf,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,iBAAiB,CACrC,GAAW,EACX,OAAsB,EACtB,UAAU,GAAG,CAAC;IAEd,MAAM,aAAa,GAAG,uBAAuB,CAAC,GAAG,CAAC,CAAC;IACnD,MAAM,MAAM,GAAG,IAAI,WAAW,CAAC,UAAU,EAAE,aAAa,CAAC,CAAC;IAC1D,MAAM,SAAS,GAAG,OAAO,EAAE,OAAO,IAAI,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC;IAC7D,MAAM,OAAO,GAAG,YAAY,CAAC,OAAO,EAAE,aAAa,CAAC,CAAC;IAErD,OAAO,MAAM,CAAC,OAAO,CAAC,KAAK,IAAI,EAAE;QAC/B,MAAM,MAAM,GAAG,kBAAkB,CAAC,SAAS,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC;QAC9D,MAAM,WAAW,GAAG,gBAAgB,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;QACtD,OAAO,kBAAkB,CAAC,aAAa,EAAE,WAAW,EAAE,SAAS,CAAC,CAAC;IACnE,CAAC,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC;AACtB,CAAC"}
@@ -1 +1 @@
1
- {"version":3,"file":"logger.d.ts","sourceRoot":"","sources":["../../src/services/logger.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAY,WAAW,EAAE,MAAM,oBAAoB,CAAC;AAwBhE,wBAAgB,OAAO,CAAC,OAAO,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,WAAW,GAAG,IAAI,CAIjE;AAED,wBAAgB,QAAQ,CAAC,OAAO,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,WAAW,GAAG,IAAI,CAIlE;AAED,wBAAgB,OAAO,CAAC,OAAO,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,WAAW,GAAG,IAAI,CAIjE;AAED,wBAAgB,QAAQ,CAAC,OAAO,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,KAAK,GAAG,WAAW,GAAG,IAAI,CAS3E"}
1
+ {"version":3,"file":"logger.d.ts","sourceRoot":"","sources":["../../src/services/logger.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAY,WAAW,EAAE,MAAM,oBAAoB,CAAC;AAkChE,wBAAgB,OAAO,CAAC,OAAO,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,WAAW,GAAG,IAAI,CAIjE;AAED,wBAAgB,QAAQ,CAAC,OAAO,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,WAAW,GAAG,IAAI,CAIlE;AAED,wBAAgB,OAAO,CAAC,OAAO,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,WAAW,GAAG,IAAI,CAIjE;AAED,wBAAgB,QAAQ,CAAC,OAAO,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,KAAK,GAAG,WAAW,GAAG,IAAI,CAS3E"}
@@ -1,6 +1,15 @@
1
1
  import { config } from '../config/index.js';
2
+ import { getRequestId, getSessionId } from './context.js';
2
3
  function formatMetadata(meta) {
3
- return meta && Object.keys(meta).length > 0 ? ` ${JSON.stringify(meta)}` : '';
4
+ const requestId = getRequestId();
5
+ const sessionId = getSessionId();
6
+ const contextMeta = {};
7
+ if (requestId)
8
+ contextMeta.requestId = requestId;
9
+ if (sessionId)
10
+ contextMeta.sessionId = sessionId;
11
+ const merged = { ...contextMeta, ...meta };
12
+ return Object.keys(merged).length > 0 ? ` ${JSON.stringify(merged)}` : '';
4
13
  }
5
14
  function createTimestamp() {
6
15
  return new Date().toISOString();
@@ -17,17 +26,17 @@ function shouldLog(level) {
17
26
  }
18
27
  export function logInfo(message, meta) {
19
28
  if (shouldLog('info')) {
20
- console.log(formatLogEntry('info', message, meta));
29
+ process.stderr.write(`${formatLogEntry('info', message, meta)}\n`);
21
30
  }
22
31
  }
23
32
  export function logDebug(message, meta) {
24
33
  if (shouldLog('debug')) {
25
- console.debug(formatLogEntry('debug', message, meta));
34
+ process.stderr.write(`${formatLogEntry('debug', message, meta)}\n`);
26
35
  }
27
36
  }
28
37
  export function logWarn(message, meta) {
29
38
  if (shouldLog('warn')) {
30
- console.warn(formatLogEntry('warn', message, meta));
39
+ process.stderr.write(`${formatLogEntry('warn', message, meta)}\n`);
31
40
  }
32
41
  }
33
42
  export function logError(message, error) {
@@ -36,6 +45,6 @@ export function logError(message, error) {
36
45
  const errorMeta = error instanceof Error
37
46
  ? { error: error.message, stack: error.stack }
38
47
  : (error ?? {});
39
- console.error(formatLogEntry('error', message, errorMeta));
48
+ process.stderr.write(`${formatLogEntry('error', message, errorMeta)}\n`);
40
49
  }
41
50
  //# sourceMappingURL=logger.js.map
@@ -1 +1 @@
1
- {"version":3,"file":"logger.js","sourceRoot":"","sources":["../../src/services/logger.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAG5C,SAAS,cAAc,CAAC,IAAkB;IACxC,OAAO,IAAI,IAAI,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;AAChF,CAAC;AAED,SAAS,eAAe;IACtB,OAAO,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;AAClC,CAAC;AAED,SAAS,cAAc,CACrB,KAAe,EACf,OAAe,EACf,IAAkB;IAElB,OAAO,IAAI,eAAe,EAAE,KAAK,KAAK,CAAC,WAAW,EAAE,KAAK,OAAO,GAAG,cAAc,CAAC,IAAI,CAAC,EAAE,CAAC;AAC5F,CAAC;AAED,SAAS,SAAS,CAAC,KAAe;IAChC,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,OAAO;QAAE,OAAO,KAAK,CAAC;IAC1C,IAAI,KAAK,KAAK,OAAO;QAAE,OAAO,MAAM,CAAC,OAAO,CAAC,KAAK,KAAK,OAAO,CAAC;IAC/D,OAAO,IAAI,CAAC;AACd,CAAC;AAED,MAAM,UAAU,OAAO,CAAC,OAAe,EAAE,IAAkB;IACzD,IAAI,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC;QACtB,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,MAAM,EAAE,OAAO,EAAE,IAAI,CAAC,CAAC,CAAC;IACrD,CAAC;AACH,CAAC;AAED,MAAM,UAAU,QAAQ,CAAC,OAAe,EAAE,IAAkB;IAC1D,IAAI,SAAS,CAAC,OAAO,CAAC,EAAE,CAAC;QACvB,OAAO,CAAC,KAAK,CAAC,cAAc,CAAC,OAAO,EAAE,OAAO,EAAE,IAAI,CAAC,CAAC,CAAC;IACxD,CAAC;AACH,CAAC;AAED,MAAM,UAAU,OAAO,CAAC,OAAe,EAAE,IAAkB;IACzD,IAAI,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC;QACtB,OAAO,CAAC,IAAI,CAAC,cAAc,CAAC,MAAM,EAAE,OAAO,EAAE,IAAI,CAAC,CAAC,CAAC;IACtD,CAAC;AACH,CAAC;AAED,MAAM,UAAU,QAAQ,CAAC,OAAe,EAAE,KAA2B;IACnE,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC;QAAE,OAAO;IAEhC,MAAM,SAAS,GACb,KAAK,YAAY,KAAK;QACpB,CAAC,CAAC,EAAE,KAAK,EAAE,KAAK,CAAC,OAAO,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE;QAC9C,CAAC,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC;IAEpB,OAAO,CAAC,KAAK,CAAC,cAAc,CAAC,OAAO,EAAE,OAAO,EAAE,SAAS,CAAC,CAAC,CAAC;AAC7D,CAAC"}
1
+ {"version":3,"file":"logger.js","sourceRoot":"","sources":["../../src/services/logger.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAG5C,OAAO,EAAE,YAAY,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAE1D,SAAS,cAAc,CAAC,IAAkB;IACxC,MAAM,SAAS,GAAG,YAAY,EAAE,CAAC;IACjC,MAAM,SAAS,GAAG,YAAY,EAAE,CAAC;IAEjC,MAAM,WAAW,GAAgB,EAAE,CAAC;IACpC,IAAI,SAAS;QAAE,WAAW,CAAC,SAAS,GAAG,SAAS,CAAC;IACjD,IAAI,SAAS;QAAE,WAAW,CAAC,SAAS,GAAG,SAAS,CAAC;IAEjD,MAAM,MAAM,GAAG,EAAE,GAAG,WAAW,EAAE,GAAG,IAAI,EAAE,CAAC;IAC3C,OAAO,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;AAC5E,CAAC;AAED,SAAS,eAAe;IACtB,OAAO,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;AAClC,CAAC;AAED,SAAS,cAAc,CACrB,KAAe,EACf,OAAe,EACf,IAAkB;IAElB,OAAO,IAAI,eAAe,EAAE,KAAK,KAAK,CAAC,WAAW,EAAE,KAAK,OAAO,GAAG,cAAc,CAAC,IAAI,CAAC,EAAE,CAAC;AAC5F,CAAC;AAED,SAAS,SAAS,CAAC,KAAe;IAChC,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,OAAO;QAAE,OAAO,KAAK,CAAC;IAC1C,IAAI,KAAK,KAAK,OAAO;QAAE,OAAO,MAAM,CAAC,OAAO,CAAC,KAAK,KAAK,OAAO,CAAC;IAC/D,OAAO,IAAI,CAAC;AACd,CAAC;AAED,MAAM,UAAU,OAAO,CAAC,OAAe,EAAE,IAAkB;IACzD,IAAI,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC;QACtB,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,cAAc,CAAC,MAAM,EAAE,OAAO,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC;IACrE,CAAC;AACH,CAAC;AAED,MAAM,UAAU,QAAQ,CAAC,OAAe,EAAE,IAAkB;IAC1D,IAAI,SAAS,CAAC,OAAO,CAAC,EAAE,CAAC;QACvB,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,cAAc,CAAC,OAAO,EAAE,OAAO,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC;IACtE,CAAC;AACH,CAAC;AAED,MAAM,UAAU,OAAO,CAAC,OAAe,EAAE,IAAkB;IACzD,IAAI,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC;QACtB,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,cAAc,CAAC,MAAM,EAAE,OAAO,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC;IACrE,CAAC;AACH,CAAC;AAED,MAAM,UAAU,QAAQ,CAAC,OAAe,EAAE,KAA2B;IACnE,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC;QAAE,OAAO;IAEhC,MAAM,SAAS,GACb,KAAK,YAAY,KAAK;QACpB,CAAC,CAAC,EAAE,KAAK,EAAE,KAAK,CAAC,OAAO,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE;QAC9C,CAAC,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC;IAEpB,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,cAAc,CAAC,OAAO,EAAE,OAAO,EAAE,SAAS,CAAC,IAAI,CAAC,CAAC;AAC3E,CAAC"}
@@ -1,3 +1,4 @@
1
1
  import type { ContentBlockUnion } from '../config/types.js';
2
+ export declare function detectLanguageFromCode(code: string): string | undefined;
2
3
  export declare function parseHtml(html: string): ContentBlockUnion[];
3
4
  //# sourceMappingURL=parser.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"parser.d.ts","sourceRoot":"","sources":["../../src/services/parser.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EAGV,iBAAiB,EAOlB,MAAM,oBAAoB,CAAC;AAuO5B,wBAAgB,SAAS,CAAC,IAAI,EAAE,MAAM,GAAG,iBAAiB,EAAE,CA8B3D"}
1
+ {"version":3,"file":"parser.d.ts","sourceRoot":"","sources":["../../src/services/parser.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EAGV,iBAAiB,EAOlB,MAAM,oBAAoB,CAAC;AAc5B,wBAAgB,sBAAsB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS,CA8BvE;AAwOD,wBAAgB,SAAS,CAAC,IAAI,EAAE,MAAM,GAAG,iBAAiB,EAAE,CAS3D"}
@@ -4,7 +4,7 @@ import { cleanCodeBlock, cleanHeading, cleanListItems, cleanParagraph, removeInl
4
4
  import { truncateHtml } from '../utils/html-truncator.js';
5
5
  import { sanitizeText } from '../utils/sanitizer.js';
6
6
  import { logWarn } from './logger.js';
7
- function detectLanguageFromCode(code) {
7
+ export function detectLanguageFromCode(code) {
8
8
  // Common language patterns for code block detection
9
9
  const patterns = [
10
10
  [
@@ -174,48 +174,68 @@ function parseElement($, node) {
174
174
  return ELEMENT_PARSERS[tagName]($, node);
175
175
  }
176
176
  function filterBlocks(blocks) {
177
- return blocks.filter((block) => {
178
- switch (block.type) {
179
- case 'paragraph':
180
- case 'heading':
181
- case 'code':
182
- case 'blockquote':
183
- return block.text.length > 0;
184
- case 'list':
185
- return block.items.length > 0;
186
- default:
187
- return true;
188
- }
189
- });
177
+ return blocks.filter(shouldIncludeBlock);
190
178
  }
191
- export function parseHtml(html) {
192
- if (!html || typeof html !== 'string')
193
- return [];
194
- const processedHtml = truncateHtml(html);
179
+ const TEXT_BLOCK_TYPES = new Set([
180
+ 'paragraph',
181
+ 'heading',
182
+ 'code',
183
+ 'blockquote',
184
+ ]);
185
+ function isTextBlock(block) {
186
+ return TEXT_BLOCK_TYPES.has(block.type);
187
+ }
188
+ function shouldIncludeBlock(block) {
189
+ if (isTextBlock(block)) {
190
+ return block.text.length > 0;
191
+ }
192
+ if (block.type === 'list') {
193
+ return block.items.length > 0;
194
+ }
195
+ return true;
196
+ }
197
+ function loadHtml(html) {
195
198
  try {
196
- const $ = cheerio.load(processedHtml);
197
- const blocks = [];
198
- $('script, style, noscript, iframe, svg').remove();
199
- $('body')
200
- .find(CONTENT_SELECTOR)
201
- .each((_, element) => {
202
- try {
203
- const block = parseElement($, element);
204
- if (block)
205
- blocks.push(block);
206
- }
207
- catch {
208
- /* skip */
209
- }
210
- });
211
- return filterBlocks(blocks);
199
+ return cheerio.load(html);
212
200
  }
213
201
  catch (error) {
214
202
  logWarn('Failed to parse HTML', {
215
203
  error: error instanceof Error ? error.message : 'Unknown error',
216
204
  htmlLength: html.length,
217
205
  });
218
- return [];
206
+ return null;
207
+ }
208
+ }
209
+ function removeNoiseElements($) {
210
+ $('script, style, noscript, iframe, svg').remove();
211
+ }
212
+ function collectBlocks($) {
213
+ const blocks = [];
214
+ $('body')
215
+ .find(CONTENT_SELECTOR)
216
+ .each((_, element) => {
217
+ const block = safeParseElement($, element);
218
+ if (block)
219
+ blocks.push(block);
220
+ });
221
+ return blocks;
222
+ }
223
+ function safeParseElement($, element) {
224
+ try {
225
+ return parseElement($, element);
226
+ }
227
+ catch {
228
+ return null;
219
229
  }
220
230
  }
231
+ export function parseHtml(html) {
232
+ if (!html || typeof html !== 'string')
233
+ return [];
234
+ const processedHtml = truncateHtml(html);
235
+ const $ = loadHtml(processedHtml);
236
+ if (!$)
237
+ return [];
238
+ removeNoiseElements($);
239
+ return filterBlocks(collectBlocks($));
240
+ }
221
241
  //# sourceMappingURL=parser.js.map
@@ -1 +1 @@
1
- {"version":3,"file":"parser.js","sourceRoot":"","sources":["../../src/services/parser.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,OAAO,MAAM,SAAS,CAAC;AAInC,OAAO,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAa5C,OAAO,EACL,cAAc,EACd,YAAY,EACZ,cAAc,EACd,cAAc,EACd,sBAAsB,GACvB,MAAM,6BAA6B,CAAC;AACrC,OAAO,EAAE,YAAY,EAAE,MAAM,4BAA4B,CAAC;AAC1D,OAAO,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAC;AAErD,OAAO,EAAE,OAAO,EAAE,MAAM,aAAa,CAAC;AAEtC,SAAS,sBAAsB,CAAC,IAAY;IAC1C,oDAAoD;IACpD,MAAM,QAAQ,GAAgC;QAC5C;YACE,kFAAkF;YAClF,KAAK;SACN;QACD;YACE,sFAAsF;YACtF,YAAY;SACb;QACD,CAAC,kDAAkD,EAAE,MAAM,CAAC;QAC5D;YACE,+EAA+E;YAC/E,YAAY;SACb;QACD,CAAC,kDAAkD,EAAE,QAAQ,CAAC;QAC9D;YACE,iFAAiF;YACjF,MAAM;SACP;QACD,CAAC,iEAAiE,EAAE,MAAM,CAAC;QAC3E,CAAC,yDAAyD,EAAE,KAAK,CAAC;QAClE,CAAC,8DAA8D,EAAE,MAAM,CAAC;QACxE,CAAC,6CAA6C,EAAE,MAAM,CAAC;QACvD,CAAC,oBAAoB,EAAE,MAAM,CAAC;QAC9B,CAAC,0DAA0D,EAAE,KAAK,CAAC;QACnE,CAAC,gCAAgC,EAAE,IAAI,CAAC;KACzC,CAAC;IAEF,OAAO,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;AAC/D,CAAC;AAED,MAAM,gBAAgB,GACpB,oFAAoF,CAAC;AAEvF,SAAS,YAAY,CAAC,CAAa,EAAE,OAAgB;IACnD,MAAM,OAAO,GAAG,YAAY,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;IAChD,MAAM,IAAI,GAAG,YAAY,CAAC,OAAO,CAAC,CAAC;IACnC,IAAI,CAAC,IAAI;QAAE,OAAO,IAAI,CAAC;IAEvB,OAAO;QACL,IAAI,EAAE,SAAS;QACf,KAAK,EAAE,QAAQ,CAAC,OAAO,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC;QACjD,IAAI;KACL,CAAC;AACJ,CAAC;AAED,SAAS,cAAc,CACrB,CAAa,EACb,OAAgB;IAEhB,IAAI,OAAO,GAAG,YAAY,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;IAC9C,8DAA8D;IAC9D,OAAO,GAAG,sBAAsB,CAAC,OAAO,CAAC,CAAC;IAC1C,MAAM,IAAI,GAAG,cAAc,CAAC,OAAO,CAAC,CAAC;IACrC,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,UAAU,CAAC,kBAAkB;QAAE,OAAO,IAAI,CAAC;IAE7E,OAAO,EAAE,IAAI,EAAE,WAAW,EAAE,IAAI,EAAE,CAAC;AACrC,CAAC;AAED,SAAS,SAAS,CAAC,CAAa,EAAE,OAAgB;IAChD,MAAM,QAAQ,GAAa,EAAE,CAAC;IAE9B,CAAC,CAAC,OAAO,CAAC;SACP,IAAI,CAAC,IAAI,CAAC;SACV,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,EAAE;QACd,MAAM,IAAI,GAAG,YAAY,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;QACxC,IAAI,IAAI;YAAE,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAChC,CAAC,CAAC,CAAC;IAEL,mCAAmC;IACnC,MAAM,KAAK,GAAG,cAAc,CAAC,QAAQ,CAAC,CAAC;IACvC,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IAEpC,OAAO;QACL,IAAI,EAAE,MAAM;QACZ,OAAO,EAAE,OAAO,CAAC,OAAO,CAAC,WAAW,EAAE,KAAK,IAAI;QAC/C,KAAK;KACN,CAAC;AACJ,CAAC;AAED,SAAS,SAAS,CAAC,CAAa,EAAE,OAAgB;IAChD,MAAM,OAAO,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;IACzC,MAAM,IAAI,GAAG,cAAc,CAAC,OAAO,CAAC,CAAC;IACrC,IAAI,CAAC,IAAI;QAAE,OAAO,IAAI,CAAC;IAEvB,iDAAiD;IACjD,MAAM,SAAS,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC;IACjD,MAAM,QAAQ,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,IAAI,EAAE,CAAC;IAExD,MAAM,aAAa,GACjB,gBAAgB,CAAC,IAAI,CAAC,SAAS,CAAC;QAChC,YAAY,CAAC,IAAI,CAAC,SAAS,CAAC;QAC5B,iBAAiB,CAAC,IAAI,CAAC,SAAS,CAAC;QACjC,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IAE3B,MAAM,QAAQ,GAAG,aAAa,EAAE,CAAC,CAAC,CAAC,IAAI,sBAAsB,CAAC,IAAI,CAAC,CAAC;IAEpE,OAAO;QACL,IAAI,EAAE,MAAM;QACZ,QAAQ;QACR,IAAI;KACL,CAAC;AACJ,CAAC;AAED,SAAS,UAAU,CAAC,CAAa,EAAE,OAAgB;IACjD,MAAM,OAAO,GAAa,EAAE,CAAC;IAC7B,MAAM,IAAI,GAAe,EAAE,CAAC;IAC5B,MAAM,MAAM,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC;IAE1B,MAAM,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,EAAE;QACjD,OAAO,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;IAC7C,CAAC,CAAC,CAAC;IAEH,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACzB,MAAM;aACH,IAAI,CAAC,IAAI,CAAC;aACV,KAAK,EAAE;aACP,IAAI,CAAC,QAAQ,CAAC;aACd,IAAI,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,EAAE;YAChB,OAAO,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;QAC7C,CAAC,CAAC,CAAC;IACP,CAAC;IAED,MAAM,YAAY,GAChB,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,0BAA0B,CAAC,CAAC,CAAC,cAAc,CAAC;IAEnE,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,EAAE;QACxC,MAAM,KAAK,GAAa,EAAE,CAAC;QAC3B,CAAC,CAAC,GAAG,CAAC;aACH,IAAI,CAAC,QAAQ,CAAC;aACd,IAAI,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,EAAE;YAChB,KAAK,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;QAC3C,CAAC,CAAC,CAAC;QACL,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC;YAAE,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IACzC,CAAC,CAAC,CAAC;IAEH,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IAEnC,OAAO;QACL,IAAI,EAAE,OAAO;QACb,OAAO,EAAE,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,SAAS;QACjD,IAAI;KACL,CAAC;AACJ,CAAC;AAED,SAAS,UAAU,CAAC,CAAa,EAAE,OAAgB;IACjD,MAAM,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IACnC,IAAI,CAAC,GAAG;QAAE,OAAO,IAAI,CAAC;IAEtB,OAAO;QACL,IAAI,EAAE,OAAO;QACb,GAAG;QACH,GAAG,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,SAAS;KACzC,CAAC;AACJ,CAAC;AAED,SAAS,eAAe,CACtB,CAAa,EACb,OAAgB;IAEhB,MAAM,OAAO,GAAG,YAAY,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;IAChD,MAAM,IAAI,GAAG,cAAc,CAAC,OAAO,CAAC,CAAC;IACrC,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,UAAU,CAAC,kBAAkB;QAAE,OAAO,IAAI,CAAC;IAE7E,OAAO,EAAE,IAAI,EAAE,YAAY,EAAE,IAAI,EAAE,CAAC;AACtC,CAAC;AAED,MAAM,eAAe,GAAG;IACtB,EAAE,EAAE,YAAY;IAChB,EAAE,EAAE,YAAY;IAChB,EAAE,EAAE,YAAY;IAChB,EAAE,EAAE,YAAY;IAChB,EAAE,EAAE,YAAY;IAChB,EAAE,EAAE,YAAY;IAChB,CAAC,EAAE,cAAc;IACjB,EAAE,EAAE,SAAS;IACb,EAAE,EAAE,SAAS;IACb,GAAG,EAAE,SAAS;IACd,IAAI,EAAE,SAAS;IACf,KAAK,EAAE,UAAU;IACjB,GAAG,EAAE,UAAU;IACf,UAAU,EAAE,eAAe;CAI5B,CAAC;AAEF,SAAS,cAAc,CAAC,GAAW;IACjC,OAAO,GAAG,IAAI,eAAe,CAAC;AAChC,CAAC;AAED,SAAS,YAAY,CAAC,CAAa,EAAE,IAAa;IAChD,IAAI,CAAC,CAAC,SAAS,IAAI,IAAI,CAAC,IAAI,OAAO,IAAI,CAAC,OAAO,KAAK,QAAQ;QAAE,OAAO,IAAI,CAAC;IAE1E,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC;IAC3C,IAAI,CAAC,cAAc,CAAC,OAAO,CAAC;QAAE,OAAO,IAAI,CAAC;IAC1C,OAAO,eAAe,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC;AAC3C,CAAC;AAED,SAAS,YAAY,CAAC,MAA2B;IAC/C,OAAO,MAAM,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE;QAC7B,QAAQ,KAAK,CAAC,IAAI,EAAE,CAAC;YACnB,KAAK,WAAW,CAAC;YACjB,KAAK,SAAS,CAAC;YACf,KAAK,MAAM,CAAC;YACZ,KAAK,YAAY;gBACf,OAAO,KAAK,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC;YAC/B,KAAK,MAAM;gBACT,OAAO,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC;YAChC;gBACE,OAAO,IAAI,CAAC;QAChB,CAAC;IACH,CAAC,CAAC,CAAC;AACL,CAAC;AAED,MAAM,UAAU,SAAS,CAAC,IAAY;IACpC,IAAI,CAAC,IAAI,IAAI,OAAO,IAAI,KAAK,QAAQ;QAAE,OAAO,EAAE,CAAC;IAEjD,MAAM,aAAa,GAAG,YAAY,CAAC,IAAI,CAAC,CAAC;IAEzC,IAAI,CAAC;QACH,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;QACtC,MAAM,MAAM,GAAwB,EAAE,CAAC;QAEvC,CAAC,CAAC,sCAAsC,CAAC,CAAC,MAAM,EAAE,CAAC;QAEnD,CAAC,CAAC,MAAM,CAAC;aACN,IAAI,CAAC,gBAAgB,CAAC;aACtB,IAAI,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,EAAE;YACnB,IAAI,CAAC;gBACH,MAAM,KAAK,GAAG,YAAY,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC;gBACvC,IAAI,KAAK;oBAAE,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAChC,CAAC;YAAC,MAAM,CAAC;gBACP,UAAU;YACZ,CAAC;QACH,CAAC,CAAC,CAAC;QAEL,OAAO,YAAY,CAAC,MAAM,CAAC,CAAC;IAC9B,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,OAAO,CAAC,sBAAsB,EAAE;YAC9B,KAAK,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe;YAC/D,UAAU,EAAE,IAAI,CAAC,MAAM;SACxB,CAAC,CAAC;QACH,OAAO,EAAE,CAAC;IACZ,CAAC;AACH,CAAC"}
1
+ {"version":3,"file":"parser.js","sourceRoot":"","sources":["../../src/services/parser.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,OAAO,MAAM,SAAS,CAAC;AAInC,OAAO,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAa5C,OAAO,EACL,cAAc,EACd,YAAY,EACZ,cAAc,EACd,cAAc,EACd,sBAAsB,GACvB,MAAM,6BAA6B,CAAC;AACrC,OAAO,EAAE,YAAY,EAAE,MAAM,4BAA4B,CAAC;AAC1D,OAAO,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAC;AAErD,OAAO,EAAE,OAAO,EAAE,MAAM,aAAa,CAAC;AAEtC,MAAM,UAAU,sBAAsB,CAAC,IAAY;IACjD,oDAAoD;IACpD,MAAM,QAAQ,GAAgC;QAC5C;YACE,kFAAkF;YAClF,KAAK;SACN;QACD;YACE,sFAAsF;YACtF,YAAY;SACb;QACD,CAAC,kDAAkD,EAAE,MAAM,CAAC;QAC5D;YACE,+EAA+E;YAC/E,YAAY;SACb;QACD,CAAC,kDAAkD,EAAE,QAAQ,CAAC;QAC9D;YACE,iFAAiF;YACjF,MAAM;SACP;QACD,CAAC,iEAAiE,EAAE,MAAM,CAAC;QAC3E,CAAC,yDAAyD,EAAE,KAAK,CAAC;QAClE,CAAC,8DAA8D,EAAE,MAAM,CAAC;QACxE,CAAC,6CAA6C,EAAE,MAAM,CAAC;QACvD,CAAC,oBAAoB,EAAE,MAAM,CAAC;QAC9B,CAAC,0DAA0D,EAAE,KAAK,CAAC;QACnE,CAAC,gCAAgC,EAAE,IAAI,CAAC;KACzC,CAAC;IACF,OAAO,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;AAC/D,CAAC;AACD,MAAM,gBAAgB,GACpB,oFAAoF,CAAC;AACvF,SAAS,YAAY,CAAC,CAAa,EAAE,OAAgB;IACnD,MAAM,OAAO,GAAG,YAAY,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;IAChD,MAAM,IAAI,GAAG,YAAY,CAAC,OAAO,CAAC,CAAC;IACnC,IAAI,CAAC,IAAI;QAAE,OAAO,IAAI,CAAC;IAEvB,OAAO;QACL,IAAI,EAAE,SAAS;QACf,KAAK,EAAE,QAAQ,CAAC,OAAO,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC;QACjD,IAAI;KACL,CAAC;AACJ,CAAC;AACD,SAAS,cAAc,CACrB,CAAa,EACb,OAAgB;IAEhB,IAAI,OAAO,GAAG,YAAY,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;IAC9C,8DAA8D;IAC9D,OAAO,GAAG,sBAAsB,CAAC,OAAO,CAAC,CAAC;IAC1C,MAAM,IAAI,GAAG,cAAc,CAAC,OAAO,CAAC,CAAC;IACrC,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,UAAU,CAAC,kBAAkB;QAAE,OAAO,IAAI,CAAC;IAE7E,OAAO,EAAE,IAAI,EAAE,WAAW,EAAE,IAAI,EAAE,CAAC;AACrC,CAAC;AACD,SAAS,SAAS,CAAC,CAAa,EAAE,OAAgB;IAChD,MAAM,QAAQ,GAAa,EAAE,CAAC;IAC9B,CAAC,CAAC,OAAO,CAAC;SACP,IAAI,CAAC,IAAI,CAAC;SACV,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,EAAE;QACd,MAAM,IAAI,GAAG,YAAY,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;QACxC,IAAI,IAAI;YAAE,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAChC,CAAC,CAAC,CAAC;IAEL,mCAAmC;IACnC,MAAM,KAAK,GAAG,cAAc,CAAC,QAAQ,CAAC,CAAC;IACvC,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IAEpC,OAAO;QACL,IAAI,EAAE,MAAM;QACZ,OAAO,EAAE,OAAO,CAAC,OAAO,CAAC,WAAW,EAAE,KAAK,IAAI;QAC/C,KAAK;KACN,CAAC;AACJ,CAAC;AAED,SAAS,SAAS,CAAC,CAAa,EAAE,OAAgB;IAChD,MAAM,OAAO,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;IACzC,MAAM,IAAI,GAAG,cAAc,CAAC,OAAO,CAAC,CAAC;IACrC,IAAI,CAAC,IAAI;QAAE,OAAO,IAAI,CAAC;IAEvB,iDAAiD;IACjD,MAAM,SAAS,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC;IACjD,MAAM,QAAQ,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,IAAI,EAAE,CAAC;IAExD,MAAM,aAAa,GACjB,gBAAgB,CAAC,IAAI,CAAC,SAAS,CAAC;QAChC,YAAY,CAAC,IAAI,CAAC,SAAS,CAAC;QAC5B,iBAAiB,CAAC,IAAI,CAAC,SAAS,CAAC;QACjC,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IAE3B,MAAM,QAAQ,GAAG,aAAa,EAAE,CAAC,CAAC,CAAC,IAAI,sBAAsB,CAAC,IAAI,CAAC,CAAC;IAEpE,OAAO;QACL,IAAI,EAAE,MAAM;QACZ,QAAQ;QACR,IAAI;KACL,CAAC;AACJ,CAAC;AAED,SAAS,UAAU,CAAC,CAAa,EAAE,OAAgB;IACjD,MAAM,OAAO,GAAa,EAAE,CAAC;IAC7B,MAAM,IAAI,GAAe,EAAE,CAAC;IAC5B,MAAM,MAAM,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC;IAE1B,MAAM,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,EAAE;QACjD,OAAO,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;IAC7C,CAAC,CAAC,CAAC;IAEH,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACzB,MAAM;aACH,IAAI,CAAC,IAAI,CAAC;aACV,KAAK,EAAE;aACP,IAAI,CAAC,QAAQ,CAAC;aACd,IAAI,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,EAAE;YAChB,OAAO,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;QAC7C,CAAC,CAAC,CAAC;IACP,CAAC;IAED,MAAM,YAAY,GAChB,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,0BAA0B,CAAC,CAAC,CAAC,cAAc,CAAC;IAEnE,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,EAAE;QACxC,MAAM,KAAK,GAAa,EAAE,CAAC;QAC3B,CAAC,CAAC,GAAG,CAAC;aACH,IAAI,CAAC,QAAQ,CAAC;aACd,IAAI,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,EAAE;YAChB,KAAK,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;QAC3C,CAAC,CAAC,CAAC;QACL,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC;YAAE,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IACzC,CAAC,CAAC,CAAC;IAEH,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IAEnC,OAAO;QACL,IAAI,EAAE,OAAO;QACb,OAAO,EAAE,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,SAAS;QACjD,IAAI;KACL,CAAC;AACJ,CAAC;AAED,SAAS,UAAU,CAAC,CAAa,EAAE,OAAgB;IACjD,MAAM,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IACnC,IAAI,CAAC,GAAG;QAAE,OAAO,IAAI,CAAC;IAEtB,OAAO;QACL,IAAI,EAAE,OAAO;QACb,GAAG;QACH,GAAG,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,SAAS;KACzC,CAAC;AACJ,CAAC;AAED,SAAS,eAAe,CACtB,CAAa,EACb,OAAgB;IAEhB,MAAM,OAAO,GAAG,YAAY,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;IAChD,MAAM,IAAI,GAAG,cAAc,CAAC,OAAO,CAAC,CAAC;IACrC,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,UAAU,CAAC,kBAAkB;QAAE,OAAO,IAAI,CAAC;IAE7E,OAAO,EAAE,IAAI,EAAE,YAAY,EAAE,IAAI,EAAE,CAAC;AACtC,CAAC;AAED,MAAM,eAAe,GAAG;IACtB,EAAE,EAAE,YAAY;IAChB,EAAE,EAAE,YAAY;IAChB,EAAE,EAAE,YAAY;IAChB,EAAE,EAAE,YAAY;IAChB,EAAE,EAAE,YAAY;IAChB,EAAE,EAAE,YAAY;IAChB,CAAC,EAAE,cAAc;IACjB,EAAE,EAAE,SAAS;IACb,EAAE,EAAE,SAAS;IACb,GAAG,EAAE,SAAS;IACd,IAAI,EAAE,SAAS;IACf,KAAK,EAAE,UAAU;IACjB,GAAG,EAAE,UAAU;IACf,UAAU,EAAE,eAAe;CAI5B,CAAC;AAEF,SAAS,cAAc,CAAC,GAAW;IACjC,OAAO,GAAG,IAAI,eAAe,CAAC;AAChC,CAAC;AAED,SAAS,YAAY,CAAC,CAAa,EAAE,IAAa;IAChD,IAAI,CAAC,CAAC,SAAS,IAAI,IAAI,CAAC,IAAI,OAAO,IAAI,CAAC,OAAO,KAAK,QAAQ;QAAE,OAAO,IAAI,CAAC;IAE1E,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC;IAC3C,IAAI,CAAC,cAAc,CAAC,OAAO,CAAC;QAAE,OAAO,IAAI,CAAC;IAC1C,OAAO,eAAe,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC;AAC3C,CAAC;AAED,SAAS,YAAY,CAAC,MAA2B;IAC/C,OAAO,MAAM,CAAC,MAAM,CAAC,kBAAkB,CAAC,CAAC;AAC3C,CAAC;AAED,MAAM,gBAAgB,GAAG,IAAI,GAAG,CAAC;IAC/B,WAAW;IACX,SAAS;IACT,MAAM;IACN,YAAY;CACb,CAAC,CAAC;AAEH,SAAS,WAAW,CAClB,KAAwB;IAExB,OAAO,gBAAgB,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;AAC1C,CAAC;AAED,SAAS,kBAAkB,CAAC,KAAwB;IAClD,IAAI,WAAW,CAAC,KAAK,CAAC,EAAE,CAAC;QACvB,OAAO,KAAK,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC;IAC/B,CAAC;IACD,IAAI,KAAK,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;QAC1B,OAAO,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC;IAChC,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,SAAS,QAAQ,CAAC,IAAY;IAC5B,IAAI,CAAC;QACH,OAAO,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC5B,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,OAAO,CAAC,sBAAsB,EAAE;YAC9B,KAAK,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe;YAC/D,UAAU,EAAE,IAAI,CAAC,MAAM;SACxB,CAAC,CAAC;QACH,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED,SAAS,mBAAmB,CAAC,CAAa;IACxC,CAAC,CAAC,sCAAsC,CAAC,CAAC,MAAM,EAAE,CAAC;AACrD,CAAC;AAED,SAAS,aAAa,CAAC,CAAa;IAClC,MAAM,MAAM,GAAwB,EAAE,CAAC;IAEvC,CAAC,CAAC,MAAM,CAAC;SACN,IAAI,CAAC,gBAAgB,CAAC;SACtB,IAAI,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,EAAE;QACnB,MAAM,KAAK,GAAG,gBAAgB,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC;QAC3C,IAAI,KAAK;YAAE,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAChC,CAAC,CAAC,CAAC;IAEL,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,gBAAgB,CACvB,CAAa,EACb,OAAgB;IAEhB,IAAI,CAAC;QACH,OAAO,YAAY,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC;IAClC,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED,MAAM,UAAU,SAAS,CAAC,IAAY;IACpC,IAAI,CAAC,IAAI,IAAI,OAAO,IAAI,KAAK,QAAQ;QAAE,OAAO,EAAE,CAAC;IAEjD,MAAM,aAAa,GAAG,YAAY,CAAC,IAAI,CAAC,CAAC;IACzC,MAAM,CAAC,GAAG,QAAQ,CAAC,aAAa,CAAC,CAAC;IAClC,IAAI,CAAC,CAAC;QAAE,OAAO,EAAE,CAAC;IAElB,mBAAmB,CAAC,CAAC,CAAC,CAAC;IACvB,OAAO,YAAY,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,CAAC;AACxC,CAAC"}
@@ -0,0 +1,4 @@
1
+ import type { ExtractLinksOptions, LinksTransformResult, ToolResponseBase } from '../../../config/types.js';
2
+ export declare function resolveFilterPattern(pattern: string | undefined, url: string): RegExp | undefined | ToolResponseBase;
3
+ export declare function extractLinks(html: string, baseUrl: string, options: ExtractLinksOptions): LinksTransformResult;
4
+ //# sourceMappingURL=link-extractor.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"link-extractor.d.ts","sourceRoot":"","sources":["../../../../src/tools/handlers/fetch-links/link-extractor.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAEV,mBAAmB,EACnB,oBAAoB,EAEpB,gBAAgB,EACjB,MAAM,0BAA0B,CAAC;AA0ClC,wBAAgB,oBAAoB,CAClC,OAAO,EAAE,MAAM,GAAG,SAAS,EAC3B,GAAG,EAAE,MAAM,GACV,MAAM,GAAG,SAAS,GAAG,gBAAgB,CAavC;AA6JD,wBAAgB,YAAY,CAC1B,IAAI,EAAE,MAAM,EACZ,OAAO,EAAE,MAAM,EACf,OAAO,EAAE,mBAAmB,GAC3B,oBAAoB,CAiBtB"}