@j0hanz/superfetch 1.1.2 → 1.1.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (205) hide show
  1. package/README.md +52 -30
  2. package/dist/config/formatting.d.ts +9 -0
  3. package/dist/config/formatting.d.ts.map +1 -0
  4. package/dist/config/formatting.js +11 -0
  5. package/dist/config/formatting.js.map +1 -0
  6. package/dist/config/index.d.ts +11 -2
  7. package/dist/config/index.d.ts.map +1 -1
  8. package/dist/config/index.js +35 -13
  9. package/dist/config/index.js.map +1 -1
  10. package/dist/config/types/content.d.ts +107 -0
  11. package/dist/config/types/content.d.ts.map +1 -0
  12. package/dist/config/types/content.js +2 -0
  13. package/dist/config/types/content.js.map +1 -0
  14. package/dist/config/types/runtime.d.ts +78 -0
  15. package/dist/config/types/runtime.d.ts.map +1 -0
  16. package/dist/config/types/runtime.js +2 -0
  17. package/dist/config/types/runtime.js.map +1 -0
  18. package/dist/config/types/tools.d.ts +99 -0
  19. package/dist/config/types/tools.d.ts.map +1 -0
  20. package/dist/config/types/tools.js +2 -0
  21. package/dist/config/types/tools.js.map +1 -0
  22. package/dist/config/types.d.ts +3 -297
  23. package/dist/config/types.d.ts.map +1 -1
  24. package/dist/http/auth.d.ts +3 -0
  25. package/dist/http/auth.d.ts.map +1 -0
  26. package/dist/http/auth.js +34 -0
  27. package/dist/http/auth.js.map +1 -0
  28. package/dist/http/cors.d.ts +8 -0
  29. package/dist/http/cors.d.ts.map +1 -0
  30. package/dist/http/cors.js +47 -0
  31. package/dist/http/cors.js.map +1 -0
  32. package/dist/http/mcp-routes.d.ts +5 -0
  33. package/dist/http/mcp-routes.d.ts.map +1 -0
  34. package/dist/http/mcp-routes.js +110 -0
  35. package/dist/http/mcp-routes.js.map +1 -0
  36. package/dist/http/mcp-session.d.ts +12 -0
  37. package/dist/http/mcp-session.d.ts.map +1 -0
  38. package/dist/http/mcp-session.js +209 -0
  39. package/dist/http/mcp-session.js.map +1 -0
  40. package/dist/http/mcp-validation.d.ts +3 -0
  41. package/dist/http/mcp-validation.d.ts.map +1 -0
  42. package/dist/http/mcp-validation.js +34 -0
  43. package/dist/http/mcp-validation.js.map +1 -0
  44. package/dist/http/rate-limit.d.ts +13 -0
  45. package/dist/http/rate-limit.d.ts.map +1 -0
  46. package/dist/http/rate-limit.js +91 -0
  47. package/dist/http/rate-limit.js.map +1 -0
  48. package/dist/http/server.d.ts +4 -0
  49. package/dist/http/server.d.ts.map +1 -0
  50. package/dist/http/server.js +183 -0
  51. package/dist/http/server.js.map +1 -0
  52. package/dist/http/sessions.d.ts +15 -0
  53. package/dist/http/sessions.d.ts.map +1 -0
  54. package/dist/http/sessions.js +64 -0
  55. package/dist/http/sessions.js.map +1 -0
  56. package/dist/index.js +26 -341
  57. package/dist/index.js.map +1 -1
  58. package/dist/middleware/error-handler.d.ts +2 -2
  59. package/dist/middleware/error-handler.d.ts.map +1 -1
  60. package/dist/middleware/error-handler.js +46 -15
  61. package/dist/middleware/error-handler.js.map +1 -1
  62. package/dist/resources/cached-content.d.ts.map +1 -1
  63. package/dist/resources/cached-content.js +104 -44
  64. package/dist/resources/cached-content.js.map +1 -1
  65. package/dist/resources/index.d.ts.map +1 -1
  66. package/dist/resources/index.js +77 -69
  67. package/dist/resources/index.js.map +1 -1
  68. package/dist/server.d.ts.map +1 -1
  69. package/dist/server.js +9 -3
  70. package/dist/server.js.map +1 -1
  71. package/dist/services/cache.d.ts +13 -1
  72. package/dist/services/cache.d.ts.map +1 -1
  73. package/dist/services/cache.js +89 -16
  74. package/dist/services/cache.js.map +1 -1
  75. package/dist/services/context.d.ts +1 -1
  76. package/dist/services/context.d.ts.map +1 -1
  77. package/dist/services/context.js +1 -1
  78. package/dist/services/context.js.map +1 -1
  79. package/dist/services/extractor.d.ts.map +1 -1
  80. package/dist/services/extractor.js +122 -87
  81. package/dist/services/extractor.js.map +1 -1
  82. package/dist/services/fetcher/agents.d.ts +4 -0
  83. package/dist/services/fetcher/agents.d.ts.map +1 -0
  84. package/dist/services/fetcher/agents.js +111 -0
  85. package/dist/services/fetcher/agents.js.map +1 -0
  86. package/dist/services/fetcher/errors.d.ts +5 -0
  87. package/dist/services/fetcher/errors.d.ts.map +1 -0
  88. package/dist/services/fetcher/errors.js +71 -0
  89. package/dist/services/fetcher/errors.js.map +1 -0
  90. package/dist/services/fetcher/headers.d.ts +2 -0
  91. package/dist/services/fetcher/headers.d.ts.map +1 -0
  92. package/dist/services/fetcher/headers.js +28 -0
  93. package/dist/services/fetcher/headers.js.map +1 -0
  94. package/dist/services/fetcher/interceptors.d.ts +10 -0
  95. package/dist/services/fetcher/interceptors.d.ts.map +1 -0
  96. package/dist/services/fetcher/interceptors.js +82 -0
  97. package/dist/services/fetcher/interceptors.js.map +1 -0
  98. package/dist/services/fetcher/redirects.d.ts +6 -0
  99. package/dist/services/fetcher/redirects.d.ts.map +1 -0
  100. package/dist/services/fetcher/redirects.js +67 -0
  101. package/dist/services/fetcher/redirects.js.map +1 -0
  102. package/dist/services/fetcher/response.d.ts +5 -0
  103. package/dist/services/fetcher/response.d.ts.map +1 -0
  104. package/dist/services/fetcher/response.js +39 -0
  105. package/dist/services/fetcher/response.js.map +1 -0
  106. package/dist/services/fetcher/retry-policy.d.ts +28 -0
  107. package/dist/services/fetcher/retry-policy.d.ts.map +1 -0
  108. package/dist/services/fetcher/retry-policy.js +138 -0
  109. package/dist/services/fetcher/retry-policy.js.map +1 -0
  110. package/dist/services/fetcher.d.ts +2 -1
  111. package/dist/services/fetcher.d.ts.map +1 -1
  112. package/dist/services/fetcher.js +62 -315
  113. package/dist/services/fetcher.js.map +1 -1
  114. package/dist/services/logger.js +4 -4
  115. package/dist/services/logger.js.map +1 -1
  116. package/dist/services/parser.d.ts +1 -0
  117. package/dist/services/parser.d.ts.map +1 -1
  118. package/dist/services/parser.js +55 -35
  119. package/dist/services/parser.js.map +1 -1
  120. package/dist/tools/handlers/fetch-links/link-extractor.d.ts +4 -0
  121. package/dist/tools/handlers/fetch-links/link-extractor.d.ts.map +1 -0
  122. package/dist/tools/handlers/fetch-links/link-extractor.js +163 -0
  123. package/dist/tools/handlers/fetch-links/link-extractor.js.map +1 -0
  124. package/dist/tools/handlers/fetch-links.tool.d.ts.map +1 -1
  125. package/dist/tools/handlers/fetch-links.tool.js +78 -116
  126. package/dist/tools/handlers/fetch-links.tool.js.map +1 -1
  127. package/dist/tools/handlers/fetch-markdown.tool.d.ts +3 -13
  128. package/dist/tools/handlers/fetch-markdown.tool.d.ts.map +1 -1
  129. package/dist/tools/handlers/fetch-markdown.tool.js +74 -83
  130. package/dist/tools/handlers/fetch-markdown.tool.js.map +1 -1
  131. package/dist/tools/handlers/fetch-single.shared.d.ts +26 -0
  132. package/dist/tools/handlers/fetch-single.shared.d.ts.map +1 -0
  133. package/dist/tools/handlers/fetch-single.shared.js +49 -0
  134. package/dist/tools/handlers/fetch-single.shared.js.map +1 -0
  135. package/dist/tools/handlers/fetch-url.tool.d.ts.map +1 -1
  136. package/dist/tools/handlers/fetch-url.tool.js +82 -54
  137. package/dist/tools/handlers/fetch-url.tool.js.map +1 -1
  138. package/dist/tools/handlers/fetch-urls/processor.d.ts +13 -0
  139. package/dist/tools/handlers/fetch-urls/processor.d.ts.map +1 -0
  140. package/dist/tools/handlers/fetch-urls/processor.js +153 -0
  141. package/dist/tools/handlers/fetch-urls/processor.js.map +1 -0
  142. package/dist/tools/handlers/fetch-urls/response.d.ts +3 -0
  143. package/dist/tools/handlers/fetch-urls/response.d.ts.map +1 -0
  144. package/dist/tools/handlers/fetch-urls/response.js +58 -0
  145. package/dist/tools/handlers/fetch-urls/response.js.map +1 -0
  146. package/dist/tools/handlers/fetch-urls/validation.d.ts +6 -0
  147. package/dist/tools/handlers/fetch-urls/validation.d.ts.map +1 -0
  148. package/dist/tools/handlers/fetch-urls/validation.js +18 -0
  149. package/dist/tools/handlers/fetch-urls/validation.js.map +1 -0
  150. package/dist/tools/handlers/fetch-urls.tool.d.ts.map +1 -1
  151. package/dist/tools/handlers/fetch-urls.tool.js +104 -202
  152. package/dist/tools/handlers/fetch-urls.tool.js.map +1 -1
  153. package/dist/tools/index.d.ts.map +1 -1
  154. package/dist/tools/index.js +36 -237
  155. package/dist/tools/index.js.map +1 -1
  156. package/dist/tools/schemas.d.ts +357 -0
  157. package/dist/tools/schemas.d.ts.map +1 -0
  158. package/dist/tools/schemas.js +272 -0
  159. package/dist/tools/schemas.js.map +1 -0
  160. package/dist/tools/utils/cache-vary.d.ts +3 -0
  161. package/dist/tools/utils/cache-vary.d.ts.map +1 -0
  162. package/dist/tools/utils/cache-vary.js +44 -0
  163. package/dist/tools/utils/cache-vary.js.map +1 -0
  164. package/dist/tools/utils/common.d.ts +2 -2
  165. package/dist/tools/utils/common.d.ts.map +1 -1
  166. package/dist/tools/utils/common.js +5 -1
  167. package/dist/tools/utils/common.js.map +1 -1
  168. package/dist/tools/utils/content-transform.d.ts +16 -0
  169. package/dist/tools/utils/content-transform.d.ts.map +1 -0
  170. package/dist/tools/utils/content-transform.js +49 -0
  171. package/dist/tools/utils/content-transform.js.map +1 -0
  172. package/dist/tools/utils/fetch-pipeline.d.ts.map +1 -1
  173. package/dist/tools/utils/fetch-pipeline.js +32 -18
  174. package/dist/tools/utils/fetch-pipeline.js.map +1 -1
  175. package/dist/tools/utils/inline-content.d.ts +11 -0
  176. package/dist/tools/utils/inline-content.d.ts.map +1 -0
  177. package/dist/tools/utils/inline-content.js +39 -0
  178. package/dist/tools/utils/inline-content.js.map +1 -0
  179. package/dist/tools/utils/markdown-toc.d.ts +3 -0
  180. package/dist/tools/utils/markdown-toc.d.ts.map +1 -0
  181. package/dist/tools/utils/markdown-toc.js +35 -0
  182. package/dist/tools/utils/markdown-toc.js.map +1 -0
  183. package/dist/tools/utils/tool-response.d.ts +9 -0
  184. package/dist/tools/utils/tool-response.d.ts.map +1 -0
  185. package/dist/tools/utils/tool-response.js +19 -0
  186. package/dist/tools/utils/tool-response.js.map +1 -0
  187. package/dist/transformers/jsonl.transformer.d.ts.map +1 -1
  188. package/dist/transformers/jsonl.transformer.js +51 -28
  189. package/dist/transformers/jsonl.transformer.js.map +1 -1
  190. package/dist/transformers/markdown.transformer.d.ts.map +1 -1
  191. package/dist/transformers/markdown.transformer.js +82 -111
  192. package/dist/transformers/markdown.transformer.js.map +1 -1
  193. package/dist/utils/header-normalizer.d.ts +5 -0
  194. package/dist/utils/header-normalizer.d.ts.map +1 -0
  195. package/dist/utils/header-normalizer.js +25 -0
  196. package/dist/utils/header-normalizer.js.map +1 -0
  197. package/dist/utils/tool-error-handler.d.ts +1 -0
  198. package/dist/utils/tool-error-handler.d.ts.map +1 -1
  199. package/dist/utils/tool-error-handler.js +29 -1
  200. package/dist/utils/tool-error-handler.js.map +1 -1
  201. package/dist/utils/url-validator.d.ts +0 -3
  202. package/dist/utils/url-validator.d.ts.map +1 -1
  203. package/dist/utils/url-validator.js +97 -18
  204. package/dist/utils/url-validator.js.map +1 -1
  205. package/package.json +11 -6
@@ -1,333 +1,80 @@
1
- import axios, { isCancel, } from 'axios';
2
- import crypto from 'crypto';
3
- import diagnosticsChannel from 'diagnostics_channel';
4
- import dns from 'dns';
5
- import http from 'http';
6
- import https from 'https';
7
- import os from 'os';
8
- import { performance } from 'perf_hooks';
9
- import { setTimeout } from 'timers/promises';
10
1
  import { config } from '../config/index.js';
11
- import { FetchError } from '../errors/app-error.js';
12
- import { isBlockedIp, validateAndNormalizeUrl, } from '../utils/url-validator.js';
13
- import { logDebug, logError, logWarn } from './logger.js';
14
- // --- Constants & Types ---
15
- const REQUEST_START_TIME = Symbol('requestStartTime');
16
- const REQUEST_ID = Symbol('requestId');
17
- const fetchChannel = diagnosticsChannel.channel('superfetch.fetch');
18
- // --- Helper Functions ---
19
- function sanitizeHeaders(headers) {
20
- if (!headers || Object.keys(headers).length === 0) {
21
- return undefined;
22
- }
23
- const { blockedHeaders } = config.security;
24
- const crlfRegex = /[\r\n]/;
25
- const sanitized = {};
26
- for (const [key, value] of Object.entries(headers)) {
27
- if (!blockedHeaders.has(key.toLowerCase()) &&
28
- !crlfRegex.test(key) &&
29
- !crlfRegex.test(value)) {
30
- sanitized[key] = value;
31
- }
32
- }
33
- return Object.keys(sanitized).length > 0 ? sanitized : undefined;
34
- }
35
- function resolveDns(hostname, options, callback) {
36
- dns.lookup(hostname, options, (err, address, family) => {
37
- if (err) {
38
- callback(err, address, family);
39
- return;
40
- }
41
- const addresses = Array.isArray(address) ? address : [{ address, family }];
42
- for (const addr of addresses) {
43
- const ip = typeof addr === 'string' ? addr : addr.address;
44
- if (isBlockedIp(ip)) {
45
- const error = new Error(`Blocked IP detected for ${hostname}`);
46
- error.code = 'EBLOCKED';
47
- callback(error, address, family);
48
- return;
49
- }
2
+ import { validateAndNormalizeUrl } from '../utils/url-validator.js';
3
+ import { destroyAgents, dispatcher } from './fetcher/agents.js';
4
+ import { createHttpError, createRateLimitError, mapFetchError, } from './fetcher/errors.js';
5
+ import { sanitizeHeaders } from './fetcher/headers.js';
6
+ import { recordFetchError, recordFetchResponse, startFetchTelemetry, } from './fetcher/interceptors.js';
7
+ import { fetchWithRedirects } from './fetcher/redirects.js';
8
+ import { readResponseText } from './fetcher/response.js';
9
+ import { RetryPolicy } from './fetcher/retry-policy.js';
10
+ export { destroyAgents };
11
+ const DEFAULT_HEADERS = {
12
+ 'User-Agent': config.fetcher.userAgent,
13
+ Accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
14
+ 'Accept-Language': 'en-US,en;q=0.5',
15
+ 'Accept-Encoding': 'gzip, deflate, br',
16
+ Connection: 'keep-alive',
17
+ };
18
+ function buildHeaders(customHeaders) {
19
+ const headers = new Headers(DEFAULT_HEADERS);
20
+ const sanitized = sanitizeHeaders(customHeaders);
21
+ if (sanitized) {
22
+ for (const [key, value] of Object.entries(sanitized)) {
23
+ headers.set(key, value);
50
24
  }
51
- callback(null, address, family);
52
- });
53
- }
54
- function buildRedirectUrl(options) {
55
- if (typeof options.href === 'string' && options.href.length > 0) {
56
- return options.href;
57
25
  }
58
- const protocol = options.protocol ?? 'http:';
59
- const hostname = options.hostname ?? options.host;
60
- if (!hostname)
61
- return null;
62
- const port = options.port ? `:${options.port}` : '';
63
- const path = options.path ?? '/';
64
- return `${protocol}//${hostname}${port}${path}`;
26
+ return headers;
65
27
  }
66
- function validateRedirectTarget(options) {
67
- if (options.auth) {
68
- const error = new Error('Redirect target includes credentials');
69
- error.code = 'EBADREDIRECT';
70
- throw error;
71
- }
72
- const targetUrl = buildRedirectUrl(options);
73
- if (!targetUrl) {
74
- const error = new Error('Invalid redirect target');
75
- error.code = 'EBADREDIRECT';
76
- throw error;
77
- }
78
- validateAndNormalizeUrl(targetUrl);
28
+ function buildRequestSignal(timeoutMs, external) {
29
+ const timeoutSignal = AbortSignal.timeout(timeoutMs);
30
+ if (!external)
31
+ return timeoutSignal;
32
+ return AbortSignal.any([external, timeoutSignal]);
79
33
  }
80
- function getAgentOptions() {
81
- const cpuCount = os.cpus().length;
34
+ function buildRequestInit(headers, signal) {
82
35
  return {
83
- keepAlive: true,
84
- maxSockets: Math.max(cpuCount * 2, 25),
85
- maxFreeSockets: Math.max(Math.floor(cpuCount * 0.5), 10),
86
- timeout: 60000,
87
- scheduling: 'fifo',
88
- lookup: resolveDns,
89
- };
90
- }
91
- function handleRequest(config) {
92
- const timedConfig = config;
93
- timedConfig[REQUEST_START_TIME] = performance.now();
94
- timedConfig[REQUEST_ID] = crypto.randomUUID().substring(0, 8);
95
- const eventData = {
96
- requestId: timedConfig[REQUEST_ID],
97
- method: config.method?.toUpperCase(),
98
- url: config.url,
36
+ method: 'GET',
37
+ headers,
38
+ signal,
39
+ dispatcher,
99
40
  };
100
- if (fetchChannel.hasSubscribers) {
101
- fetchChannel.publish({ type: 'start', ...eventData });
102
- }
103
- logDebug('HTTP Request', eventData);
104
- return config;
105
- }
106
- function handleRequestError(error) {
107
- logError('HTTP Request Error', error);
108
- throw error;
109
- }
110
- function calculateDuration(config) {
111
- const startTime = config[REQUEST_START_TIME];
112
- return startTime ? performance.now() - startTime : 0;
113
- }
114
- function logResponse(response, requestId, duration) {
115
- const headers = response.headers;
116
- const contentType = headers['content-type'];
117
- const contentTypeStr = typeof contentType === 'string' ? contentType : undefined;
118
- logDebug('HTTP Response', {
119
- requestId,
120
- status: response.status,
121
- url: response.config.url ?? 'unknown',
122
- contentType: contentTypeStr,
123
- duration: `${duration}ms`,
124
- size: headers['content-length'],
125
- });
126
- if (duration > 5000) {
127
- logWarn('Slow HTTP request detected', {
128
- requestId,
129
- url: response.config.url ?? 'unknown',
130
- duration: `${duration}ms`,
131
- });
132
- }
133
- }
134
- function parseRetryAfter(header) {
135
- if (!header)
136
- return 60;
137
- const parsed = typeof header === 'string' ? parseInt(header, 10) : Number(header);
138
- return isNaN(parsed) ? 60 : parsed;
139
- }
140
- function handleResponseError(error) {
141
- const url = error.config?.url ?? 'unknown';
142
- if (fetchChannel.hasSubscribers) {
143
- const timedConfig = error.config;
144
- const requestId = timedConfig?.[REQUEST_ID];
145
- const duration = timedConfig ? calculateDuration(timedConfig) : 0;
146
- fetchChannel.publish({
147
- type: 'error',
148
- requestId,
149
- url,
150
- error: error.message,
151
- code: error.code,
152
- status: error.response?.status,
153
- duration,
154
- });
155
- }
156
- if (isCancel(error) ||
157
- error.name === 'AbortError' ||
158
- error.name === 'CanceledError') {
159
- logDebug('HTTP Request Aborted/Canceled', { url });
160
- throw new FetchError('Request was canceled', url, 499, {
161
- reason: 'aborted',
162
- });
163
- }
164
- if (error.code === 'ECONNABORTED' || error.code === 'ETIMEDOUT') {
165
- logError('HTTP Timeout', { url, timeout: config.fetcher.timeout });
166
- throw new FetchError(`Request timeout after ${config.fetcher.timeout}ms`, url, 504, { timeout: config.fetcher.timeout });
167
- }
168
- if (error.response) {
169
- const { status, statusText, headers } = error.response;
170
- const responseHeaders = headers;
171
- if (status === 429) {
172
- const retryAfter = parseRetryAfter(responseHeaders['retry-after']);
173
- logWarn('Rate limited by server', { url, retryAfter: `${retryAfter}s` });
174
- throw new FetchError('Too many requests', url, 429, { retryAfter });
175
- }
176
- logError('HTTP Error Response', { url, status, statusText });
177
- throw new FetchError(`HTTP ${status}: ${statusText}`, url, status);
178
- }
179
- if (error.request) {
180
- logError('HTTP Network Error', { url, code: error.code });
181
- throw new FetchError(`Network error: Could not reach ${url}`, url);
182
- }
183
- logError('HTTP Unknown Error', { url, message: error.message });
184
- throw new FetchError(error.message, url);
185
41
  }
186
- function handleResponse(response) {
187
- const timedConfig = response.config;
188
- const duration = calculateDuration(timedConfig);
189
- const requestId = timedConfig[REQUEST_ID];
190
- // Cleanup symbols safely
191
- timedConfig[REQUEST_START_TIME] = undefined;
192
- timedConfig[REQUEST_ID] = undefined;
193
- if (fetchChannel.hasSubscribers) {
194
- fetchChannel.publish({
195
- type: 'end',
196
- requestId,
197
- status: response.status,
198
- duration,
199
- });
200
- }
201
- logResponse(response, requestId, duration);
202
- return response;
42
+ async function handleFetchResponse(response, finalUrl, telemetry) {
43
+ if (response.status === 429) {
44
+ void response.body?.cancel();
45
+ throw createRateLimitError(finalUrl, response.headers.get('retry-after'));
46
+ }
47
+ if (!response.ok) {
48
+ void response.body?.cancel();
49
+ throw createHttpError(finalUrl, response.status, response.statusText);
50
+ }
51
+ const { text, size } = await readResponseText(response, finalUrl, config.fetcher.maxContentLength);
52
+ recordFetchResponse(telemetry, response, size);
53
+ return text;
203
54
  }
204
- class RetryPolicy {
205
- maxRetries;
206
- url;
207
- static BASE_DELAY_MS = 1000;
208
- static MAX_DELAY_MS = 10000;
209
- static JITTER_FACTOR = 0.25;
210
- constructor(maxRetries, url) {
211
- this.maxRetries = maxRetries;
212
- this.url = url;
213
- }
214
- async execute(operation, signal) {
215
- let lastError = new Error(`Failed to fetch ${this.url}`);
216
- const retries = Math.min(Math.max(1, this.maxRetries), 10);
217
- for (let attempt = 1; attempt <= retries; attempt++) {
218
- if (signal?.aborted) {
219
- throw new FetchError('Request was aborted before execution', this.url);
220
- }
221
- try {
222
- return await operation();
223
- }
224
- catch (error) {
225
- lastError = error instanceof Error ? error : new Error(String(error));
226
- if (!this.shouldRetry(attempt, retries, lastError)) {
227
- throw lastError;
228
- }
229
- await this.wait(attempt, lastError);
230
- }
231
- }
232
- throw new FetchError(`Failed after ${retries} attempts: ${lastError.message}`, this.url);
233
- }
234
- shouldRetry(attempt, maxRetries, error) {
235
- if (attempt >= maxRetries)
236
- return false;
237
- if (error instanceof FetchError) {
238
- if (error.details.reason === 'aborted')
239
- return false;
240
- if (error.details.httpStatus === 429)
241
- return true;
242
- const status = error.details.httpStatus;
243
- if (status && status >= 400 && status < 500)
244
- return false;
245
- }
246
- return true;
247
- }
248
- async wait(attempt, error) {
249
- let delay;
250
- if (error instanceof FetchError && error.details.httpStatus === 429) {
251
- const retryAfter = error.details.retryAfter || 60;
252
- delay = Math.min(retryAfter * 1000, 30000);
253
- logWarn('Rate limited, waiting before retry', {
254
- url: this.url,
255
- attempt,
256
- waitTime: `${delay}ms`,
257
- });
258
- }
259
- else {
260
- delay = this.calculateBackoff(attempt);
261
- logDebug('Retrying request', {
262
- url: this.url,
263
- attempt,
264
- delay: `${delay}ms`,
265
- });
266
- }
267
- await setTimeout(delay);
268
- }
269
- calculateBackoff(attempt) {
270
- const exponentialDelay = Math.min(RetryPolicy.BASE_DELAY_MS * Math.pow(2, attempt - 1), RetryPolicy.MAX_DELAY_MS);
271
- const jitter = exponentialDelay * RetryPolicy.JITTER_FACTOR * (Math.random() * 2 - 1);
272
- return Math.round(exponentialDelay + jitter);
55
+ async function fetchWithTelemetry(normalizedUrl, requestInit, timeoutMs) {
56
+ const telemetry = startFetchTelemetry(normalizedUrl, 'GET');
57
+ try {
58
+ const { response, url: finalUrl } = await fetchWithRedirects(normalizedUrl, requestInit, config.fetcher.maxRedirects);
59
+ telemetry.url = finalUrl;
60
+ return await handleFetchResponse(response, finalUrl, telemetry);
61
+ }
62
+ catch (error) {
63
+ const mapped = mapFetchError(error, normalizedUrl, timeoutMs);
64
+ telemetry.url = mapped.url;
65
+ recordFetchError(telemetry, mapped, mapped.statusCode);
66
+ throw mapped;
273
67
  }
274
68
  }
275
- const httpAgent = new http.Agent(getAgentOptions());
276
- const httpsAgent = new https.Agent(getAgentOptions());
277
- const client = axios.create({
278
- timeout: config.fetcher.timeout,
279
- maxRedirects: config.fetcher.maxRedirects,
280
- maxContentLength: config.fetcher.maxContentLength,
281
- httpAgent,
282
- httpsAgent,
283
- beforeRedirect: (options) => {
284
- validateRedirectTarget(options);
285
- },
286
- headers: {
287
- 'User-Agent': config.fetcher.userAgent,
288
- Accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
289
- 'Accept-Language': 'en-US,en;q=0.5',
290
- 'Accept-Encoding': 'gzip, deflate, br',
291
- Connection: 'keep-alive',
292
- },
293
- validateStatus: (status) => status >= 200 && status < 300,
294
- });
295
- client.interceptors.request.use(handleRequest, handleRequestError);
296
- client.interceptors.response.use(handleResponse, handleResponseError);
297
- export function destroyAgents() {
298
- httpAgent.destroy();
299
- httpsAgent.destroy();
300
- }
301
69
  export async function fetchUrlWithRetry(url, options, maxRetries = 3) {
302
70
  const normalizedUrl = validateAndNormalizeUrl(url);
303
71
  const policy = new RetryPolicy(maxRetries, normalizedUrl);
72
+ const timeoutMs = options?.timeout ?? config.fetcher.timeout;
73
+ const headers = buildHeaders(options?.customHeaders);
304
74
  return policy.execute(async () => {
305
- const requestConfig = {
306
- method: 'GET',
307
- url: normalizedUrl,
308
- responseType: 'text',
309
- };
310
- if (options?.signal) {
311
- requestConfig.signal = options.signal;
312
- }
313
- else if (options?.timeout) {
314
- requestConfig.signal = AbortSignal.timeout(options.timeout);
315
- }
316
- const sanitizedHeaders = sanitizeHeaders(options?.customHeaders);
317
- if (sanitizedHeaders) {
318
- const existingHeaders = (requestConfig.headers ?? {});
319
- requestConfig.headers = { ...existingHeaders, ...sanitizedHeaders };
320
- }
321
- try {
322
- const response = await client.request(requestConfig);
323
- return response.data;
324
- }
325
- catch (error) {
326
- if (error instanceof FetchError) {
327
- throw error;
328
- }
329
- throw new FetchError(`Unexpected error: ${error instanceof Error ? error.message : 'Unknown'}`, normalizedUrl);
330
- }
75
+ const signal = buildRequestSignal(timeoutMs, options?.signal);
76
+ const requestInit = buildRequestInit(headers, signal);
77
+ return fetchWithTelemetry(normalizedUrl, requestInit, timeoutMs);
331
78
  }, options?.signal);
332
79
  }
333
80
  //# sourceMappingURL=fetcher.js.map
@@ -1 +1 @@
1
- {"version":3,"file":"fetcher.js","sourceRoot":"","sources":["../../src/services/fetcher.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,EAOZ,QAAQ,GAET,MAAM,OAAO,CAAC;AACf,OAAO,MAAM,MAAM,QAAQ,CAAC;AAC5B,OAAO,kBAAkB,MAAM,qBAAqB,CAAC;AACrD,OAAO,GAAG,MAAM,KAAK,CAAC;AACtB,OAAO,IAAI,MAAM,MAAM,CAAC;AACxB,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,EAAE,MAAM,IAAI,CAAC;AACpB,OAAO,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AACzC,OAAO,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAE7C,OAAO,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAG5C,OAAO,EAAE,UAAU,EAAE,MAAM,wBAAwB,CAAC;AAEpD,OAAO,EACL,WAAW,EACX,uBAAuB,GACxB,MAAM,2BAA2B,CAAC;AAEnC,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,aAAa,CAAC;AAE1D,4BAA4B;AAE5B,MAAM,kBAAkB,GAAG,MAAM,CAAC,kBAAkB,CAAC,CAAC;AACtD,MAAM,UAAU,GAAG,MAAM,CAAC,WAAW,CAAC,CAAC;AACvC,MAAM,YAAY,GAAG,kBAAkB,CAAC,OAAO,CAAC,kBAAkB,CAAC,CAAC;AAOpE,2BAA2B;AAE3B,SAAS,eAAe,CACtB,OAAgC;IAEhC,IAAI,CAAC,OAAO,IAAI,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAClD,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,MAAM,EAAE,cAAc,EAAE,GAAG,MAAM,CAAC,QAAQ,CAAC;IAC3C,MAAM,SAAS,GAAG,QAAQ,CAAC;IAC3B,MAAM,SAAS,GAA2B,EAAE,CAAC;IAE7C,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;QACnD,IACE,CAAC,cAAc,CAAC,GAAG,CAAC,GAAG,CAAC,WAAW,EAAE,CAAC;YACtC,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC;YACpB,CAAC,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,EACtB,CAAC;YACD,SAAS,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC;QACzB,CAAC;IACH,CAAC;IAED,OAAO,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC;AACnE,CAAC;AAED,SAAS,UAAU,CACjB,QAAgB,EAChB,OAA0B,EAC1B,QAIS;IAET,GAAG,CAAC,MAAM,CAAC,QAAQ,EAAE,OAAO,EAAE,CAAC,GAAG,EAAE,OAAO,EAAE,MAAM,EAAE,EAAE;QACrD,IAAI,GAAG,EAAE,CAAC;YACR,QAAQ,CAAC,GAAG,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC;YAC/B,OAAO;QACT,CAAC;QAED,MAAM,SAAS,GAAG,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,CAAC;QAE3E,KAAK,MAAM,IAAI,IAAI,SAAS,EAAE,CAAC;YAC7B,MAAM,EAAE,GAAG,OAAO,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC;YAC1D,IAAI,WAAW,CAAC,EAAE,CAAC,EAAE,CAAC;gBACpB,MAAM,KAAK,GAAG,IAAI,KAAK,CACrB,2BAA2B,QAAQ,EAAE,CACb,CAAC;gBAC3B,KAAK,CAAC,IAAI,GAAG,UAAU,CAAC;gBACxB,QAAQ,CAAC,KAAK,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC;gBACjC,OAAO;YACT,CAAC;QACH,CAAC;QAED,QAAQ,CAAC,IAAI,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC;IAClC,CAAC,CAAC,CAAC;AACL,CAAC;AAYD,SAAS,gBAAgB,CAAC,OAAwB;IAChD,IAAI,OAAO,OAAO,CAAC,IAAI,KAAK,QAAQ,IAAI,OAAO,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAChE,OAAO,OAAO,CAAC,IAAI,CAAC;IACtB,CAAC;IAED,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,IAAI,OAAO,CAAC;IAC7C,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,IAAI,OAAO,CAAC,IAAI,CAAC;IAClD,IAAI,CAAC,QAAQ;QAAE,OAAO,IAAI,CAAC;IAE3B,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;IACpD,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,IAAI,GAAG,CAAC;IAEjC,OAAO,GAAG,QAAQ,KAAK,QAAQ,GAAG,IAAI,GAAG,IAAI,EAAE,CAAC;AAClD,CAAC;AAED,SAAS,sBAAsB,CAAC,OAAwB;IACtD,IAAI,OAAO,CAAC,IAAI,EAAE,CAAC;QACjB,MAAM,KAAK,GAAG,IAAI,KAAK,CACrB,sCAAsC,CACd,CAAC;QAC3B,KAAK,CAAC,IAAI,GAAG,cAAc,CAAC;QAC5B,MAAM,KAAK,CAAC;IACd,CAAC;IAED,MAAM,SAAS,GAAG,gBAAgB,CAAC,OAAO,CAAC,CAAC;IAC5C,IAAI,CAAC,SAAS,EAAE,CAAC;QACf,MAAM,KAAK,GAAG,IAAI,KAAK,CAAC,yBAAyB,CAA0B,CAAC;QAC5E,KAAK,CAAC,IAAI,GAAG,cAAc,CAAC;QAC5B,MAAM,KAAK,CAAC;IACd,CAAC;IAED,uBAAuB,CAAC,SAAS,CAAC,CAAC;AACrC,CAAC;AAED,SAAS,eAAe;IACtB,MAAM,QAAQ,GAAG,EAAE,CAAC,IAAI,EAAE,CAAC,MAAM,CAAC;IAClC,OAAO;QACL,SAAS,EAAE,IAAI;QACf,UAAU,EAAE,IAAI,CAAC,GAAG,CAAC,QAAQ,GAAG,CAAC,EAAE,EAAE,CAAC;QACtC,cAAc,EAAE,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ,GAAG,GAAG,CAAC,EAAE,EAAE,CAAC;QACxD,OAAO,EAAE,KAAK;QACd,UAAU,EAAE,MAAM;QAClB,MAAM,EAAE,UAAU;KACnB,CAAC;AACJ,CAAC;AAED,SAAS,aAAa,CACpB,MAAkC;IAElC,MAAM,WAAW,GAAG,MAAiC,CAAC;IACtD,WAAW,CAAC,kBAAkB,CAAC,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC;IACpD,WAAW,CAAC,UAAU,CAAC,GAAG,MAAM,CAAC,UAAU,EAAE,CAAC,SAAS,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IAE9D,MAAM,SAAS,GAAG;QAChB,SAAS,EAAE,WAAW,CAAC,UAAU,CAAC;QAClC,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,WAAW,EAAE;QACpC,GAAG,EAAE,MAAM,CAAC,GAAG;KAChB,CAAC;IAEF,IAAI,YAAY,CAAC,cAAc,EAAE,CAAC;QAChC,YAAY,CAAC,OAAO,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,GAAG,SAAS,EAAE,CAAC,CAAC;IACxD,CAAC;IAED,QAAQ,CAAC,cAAc,EAAE,SAAS,CAAC,CAAC;IAEpC,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,kBAAkB,CAAC,KAAiB;IAC3C,QAAQ,CAAC,oBAAoB,EAAE,KAAK,CAAC,CAAC;IACtC,MAAM,KAAK,CAAC;AACd,CAAC;AAED,SAAS,iBAAiB,CAAC,MAA+B;IACxD,MAAM,SAAS,GAAG,MAAM,CAAC,kBAAkB,CAAC,CAAC;IAC7C,OAAO,SAAS,CAAC,CAAC,CAAC,WAAW,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC;AACvD,CAAC;AAED,SAAS,WAAW,CAClB,QAAuB,EACvB,SAA6B,EAC7B,QAAgB;IAEhB,MAAM,OAAO,GAAG,QAAQ,CAAC,OAEE,CAAC;IAC5B,MAAM,WAAW,GAAG,OAAO,CAAC,cAAc,CAAY,CAAC;IACvD,MAAM,cAAc,GAClB,OAAO,WAAW,KAAK,QAAQ,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,SAAS,CAAC;IAE5D,QAAQ,CAAC,eAAe,EAAE;QACxB,SAAS;QACT,MAAM,EAAE,QAAQ,CAAC,MAAM;QACvB,GAAG,EAAE,QAAQ,CAAC,MAAM,CAAC,GAAG,IAAI,SAAS;QACrC,WAAW,EAAE,cAAc;QAC3B,QAAQ,EAAE,GAAG,QAAQ,IAAI;QACzB,IAAI,EAAE,OAAO,CAAC,gBAAgB,CAAC;KAChC,CAAC,CAAC;IAEH,IAAI,QAAQ,GAAG,IAAI,EAAE,CAAC;QACpB,OAAO,CAAC,4BAA4B,EAAE;YACpC,SAAS;YACT,GAAG,EAAE,QAAQ,CAAC,MAAM,CAAC,GAAG,IAAI,SAAS;YACrC,QAAQ,EAAE,GAAG,QAAQ,IAAI;SAC1B,CAAC,CAAC;IACL,CAAC;AACH,CAAC;AAED,SAAS,eAAe,CAAC,MAAe;IACtC,IAAI,CAAC,MAAM;QAAE,OAAO,EAAE,CAAC;IACvB,MAAM,MAAM,GACV,OAAO,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;IACrE,OAAO,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC;AACrC,CAAC;AAED,SAAS,mBAAmB,CAAC,KAAiB;IAC5C,MAAM,GAAG,GAAG,KAAK,CAAC,MAAM,EAAE,GAAG,IAAI,SAAS,CAAC;IAE3C,IAAI,YAAY,CAAC,cAAc,EAAE,CAAC;QAChC,MAAM,WAAW,GAAG,KAAK,CAAC,MAA6C,CAAC;QACxE,MAAM,SAAS,GAAG,WAAW,EAAE,CAAC,UAAU,CAAC,CAAC;QAC5C,MAAM,QAAQ,GAAG,WAAW,CAAC,CAAC,CAAC,iBAAiB,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAElE,YAAY,CAAC,OAAO,CAAC;YACnB,IAAI,EAAE,OAAO;YACb,SAAS;YACT,GAAG;YACH,KAAK,EAAE,KAAK,CAAC,OAAO;YACpB,IAAI,EAAE,KAAK,CAAC,IAAI;YAChB,MAAM,EAAE,KAAK,CAAC,QAAQ,EAAE,MAAM;YAC9B,QAAQ;SACT,CAAC,CAAC;IACL,CAAC;IAED,IACE,QAAQ,CAAC,KAAK,CAAC;QACf,KAAK,CAAC,IAAI,KAAK,YAAY;QAC3B,KAAK,CAAC,IAAI,KAAK,eAAe,EAC9B,CAAC;QACD,QAAQ,CAAC,+BAA+B,EAAE,EAAE,GAAG,EAAE,CAAC,CAAC;QACnD,MAAM,IAAI,UAAU,CAAC,sBAAsB,EAAE,GAAG,EAAE,GAAG,EAAE;YACrD,MAAM,EAAE,SAAS;SAClB,CAAC,CAAC;IACL,CAAC;IAED,IAAI,KAAK,CAAC,IAAI,KAAK,cAAc,IAAI,KAAK,CAAC,IAAI,KAAK,WAAW,EAAE,CAAC;QAChE,QAAQ,CAAC,cAAc,EAAE,EAAE,GAAG,EAAE,OAAO,EAAE,MAAM,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC;QACnE,MAAM,IAAI,UAAU,CAClB,yBAAyB,MAAM,CAAC,OAAO,CAAC,OAAO,IAAI,EACnD,GAAG,EACH,GAAG,EACH,EAAE,OAAO,EAAE,MAAM,CAAC,OAAO,CAAC,OAAO,EAAE,CACpC,CAAC;IACJ,CAAC;IAED,IAAI,KAAK,CAAC,QAAQ,EAAE,CAAC;QACnB,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,OAAO,EAAE,GAAG,KAAK,CAAC,QAAQ,CAAC;QACvD,MAAM,eAAe,GAAG,OAEG,CAAC;QAE5B,IAAI,MAAM,KAAK,GAAG,EAAE,CAAC;YACnB,MAAM,UAAU,GAAG,eAAe,CAAC,eAAe,CAAC,aAAa,CAAC,CAAC,CAAC;YACnE,OAAO,CAAC,wBAAwB,EAAE,EAAE,GAAG,EAAE,UAAU,EAAE,GAAG,UAAU,GAAG,EAAE,CAAC,CAAC;YACzE,MAAM,IAAI,UAAU,CAAC,mBAAmB,EAAE,GAAG,EAAE,GAAG,EAAE,EAAE,UAAU,EAAE,CAAC,CAAC;QACtE,CAAC;QAED,QAAQ,CAAC,qBAAqB,EAAE,EAAE,GAAG,EAAE,MAAM,EAAE,UAAU,EAAE,CAAC,CAAC;QAC7D,MAAM,IAAI,UAAU,CAAC,QAAQ,MAAM,KAAK,UAAU,EAAE,EAAE,GAAG,EAAE,MAAM,CAAC,CAAC;IACrE,CAAC;IAED,IAAI,KAAK,CAAC,OAAO,EAAE,CAAC;QAClB,QAAQ,CAAC,oBAAoB,EAAE,EAAE,GAAG,EAAE,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC;QAC1D,MAAM,IAAI,UAAU,CAAC,kCAAkC,GAAG,EAAE,EAAE,GAAG,CAAC,CAAC;IACrE,CAAC;IAED,QAAQ,CAAC,oBAAoB,EAAE,EAAE,GAAG,EAAE,OAAO,EAAE,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;IAChE,MAAM,IAAI,UAAU,CAAC,KAAK,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;AAC3C,CAAC;AAED,SAAS,cAAc,CAAC,QAAuB;IAC7C,MAAM,WAAW,GAAG,QAAQ,CAAC,MAAiC,CAAC;IAC/D,MAAM,QAAQ,GAAG,iBAAiB,CAAC,WAAW,CAAC,CAAC;IAChD,MAAM,SAAS,GAAG,WAAW,CAAC,UAAU,CAAC,CAAC;IAE1C,yBAAyB;IACzB,WAAW,CAAC,kBAAkB,CAAC,GAAG,SAAS,CAAC;IAC5C,WAAW,CAAC,UAAU,CAAC,GAAG,SAAS,CAAC;IAEpC,IAAI,YAAY,CAAC,cAAc,EAAE,CAAC;QAChC,YAAY,CAAC,OAAO,CAAC;YACnB,IAAI,EAAE,KAAK;YACX,SAAS;YACT,MAAM,EAAE,QAAQ,CAAC,MAAM;YACvB,QAAQ;SACT,CAAC,CAAC;IACL,CAAC;IAED,WAAW,CAAC,QAAQ,EAAE,SAAS,EAAE,QAAQ,CAAC,CAAC;IAE3C,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED,MAAM,WAAW;IAMI;IACA;IANX,MAAM,CAAU,aAAa,GAAG,IAAI,CAAC;IACrC,MAAM,CAAU,YAAY,GAAG,KAAK,CAAC;IACrC,MAAM,CAAU,aAAa,GAAG,IAAI,CAAC;IAE7C,YACmB,UAAkB,EAClB,GAAW;QADX,eAAU,GAAV,UAAU,CAAQ;QAClB,QAAG,GAAH,GAAG,CAAQ;IAC3B,CAAC;IAEJ,KAAK,CAAC,OAAO,CACX,SAA2B,EAC3B,MAAoB;QAEpB,IAAI,SAAS,GAAU,IAAI,KAAK,CAAC,mBAAmB,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC;QAChE,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,UAAU,CAAC,EAAE,EAAE,CAAC,CAAC;QAE3D,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,IAAI,OAAO,EAAE,OAAO,EAAE,EAAE,CAAC;YACpD,IAAI,MAAM,EAAE,OAAO,EAAE,CAAC;gBACpB,MAAM,IAAI,UAAU,CAAC,sCAAsC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC;YACzE,CAAC;YAED,IAAI,CAAC;gBACH,OAAO,MAAM,SAAS,EAAE,CAAC;YAC3B,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,SAAS,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;gBAEtE,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,OAAO,EAAE,OAAO,EAAE,SAAS,CAAC,EAAE,CAAC;oBACnD,MAAM,SAAS,CAAC;gBAClB,CAAC;gBAED,MAAM,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC;YACtC,CAAC;QACH,CAAC;QAED,MAAM,IAAI,UAAU,CAClB,gBAAgB,OAAO,cAAc,SAAS,CAAC,OAAO,EAAE,EACxD,IAAI,CAAC,GAAG,CACT,CAAC;IACJ,CAAC;IAEO,WAAW,CACjB,OAAe,EACf,UAAkB,EAClB,KAAY;QAEZ,IAAI,OAAO,IAAI,UAAU;YAAE,OAAO,KAAK,CAAC;QAExC,IAAI,KAAK,YAAY,UAAU,EAAE,CAAC;YAChC,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,KAAK,SAAS;gBAAE,OAAO,KAAK,CAAC;YACrD,IAAI,KAAK,CAAC,OAAO,CAAC,UAAU,KAAK,GAAG;gBAAE,OAAO,IAAI,CAAC;YAElD,MAAM,MAAM,GAAG,KAAK,CAAC,OAAO,CAAC,UAAgC,CAAC;YAC9D,IAAI,MAAM,IAAI,MAAM,IAAI,GAAG,IAAI,MAAM,GAAG,GAAG;gBAAE,OAAO,KAAK,CAAC;QAC5D,CAAC;QAED,OAAO,IAAI,CAAC;IACd,CAAC;IAEO,KAAK,CAAC,IAAI,CAAC,OAAe,EAAE,KAAY;QAC9C,IAAI,KAAa,CAAC;QAElB,IAAI,KAAK,YAAY,UAAU,IAAI,KAAK,CAAC,OAAO,CAAC,UAAU,KAAK,GAAG,EAAE,CAAC;YACpE,MAAM,UAAU,GAAI,KAAK,CAAC,OAAO,CAAC,UAAqB,IAAI,EAAE,CAAC;YAC9D,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,UAAU,GAAG,IAAI,EAAE,KAAK,CAAC,CAAC;YAC3C,OAAO,CAAC,oCAAoC,EAAE;gBAC5C,GAAG,EAAE,IAAI,CAAC,GAAG;gBACb,OAAO;gBACP,QAAQ,EAAE,GAAG,KAAK,IAAI;aACvB,CAAC,CAAC;QACL,CAAC;aAAM,CAAC;YACN,KAAK,GAAG,IAAI,CAAC,gBAAgB,CAAC,OAAO,CAAC,CAAC;YACvC,QAAQ,CAAC,kBAAkB,EAAE;gBAC3B,GAAG,EAAE,IAAI,CAAC,GAAG;gBACb,OAAO;gBACP,KAAK,EAAE,GAAG,KAAK,IAAI;aACpB,CAAC,CAAC;QACL,CAAC;QAED,MAAM,UAAU,CAAC,KAAK,CAAC,CAAC;IAC1B,CAAC;IAEO,gBAAgB,CAAC,OAAe;QACtC,MAAM,gBAAgB,GAAG,IAAI,CAAC,GAAG,CAC/B,WAAW,CAAC,aAAa,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,GAAG,CAAC,CAAC,EACpD,WAAW,CAAC,YAAY,CACzB,CAAC;QACF,MAAM,MAAM,GACV,gBAAgB,GAAG,WAAW,CAAC,aAAa,GAAG,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC;QACzE,OAAO,IAAI,CAAC,KAAK,CAAC,gBAAgB,GAAG,MAAM,CAAC,CAAC;IAC/C,CAAC;;AAGH,MAAM,SAAS,GAAG,IAAI,IAAI,CAAC,KAAK,CAAC,eAAe,EAAE,CAAC,CAAC;AACpD,MAAM,UAAU,GAAG,IAAI,KAAK,CAAC,KAAK,CAAC,eAAe,EAAE,CAAC,CAAC;AAEtD,MAAM,MAAM,GAAkB,KAAK,CAAC,MAAM,CAAC;IACzC,OAAO,EAAE,MAAM,CAAC,OAAO,CAAC,OAAO;IAC/B,YAAY,EAAE,MAAM,CAAC,OAAO,CAAC,YAAY;IACzC,gBAAgB,EAAE,MAAM,CAAC,OAAO,CAAC,gBAAgB;IACjD,SAAS;IACT,UAAU;IACV,cAAc,EAAE,CAAC,OAAO,EAAE,EAAE;QAC1B,sBAAsB,CAAC,OAA0B,CAAC,CAAC;IACrD,CAAC;IACD,OAAO,EAAE;QACP,YAAY,EAAE,MAAM,CAAC,OAAO,CAAC,SAAS;QACtC,MAAM,EACJ,4EAA4E;QAC9E,iBAAiB,EAAE,gBAAgB;QACnC,iBAAiB,EAAE,mBAAmB;QACtC,UAAU,EAAE,YAAY;KACzB;IACD,cAAc,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,IAAI,GAAG,IAAI,MAAM,GAAG,GAAG;CAC1D,CAAC,CAAC;AAEH,MAAM,CAAC,YAAY,CAAC,OAAO,CAAC,GAAG,CAAC,aAAa,EAAE,kBAAkB,CAAC,CAAC;AACnE,MAAM,CAAC,YAAY,CAAC,QAAQ,CAAC,GAAG,CAAC,cAAc,EAAE,mBAAmB,CAAC,CAAC;AAEtE,MAAM,UAAU,aAAa;IAC3B,SAAS,CAAC,OAAO,EAAE,CAAC;IACpB,UAAU,CAAC,OAAO,EAAE,CAAC;AACvB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,iBAAiB,CACrC,GAAW,EACX,OAAsB,EACtB,UAAU,GAAG,CAAC;IAEd,MAAM,aAAa,GAAG,uBAAuB,CAAC,GAAG,CAAC,CAAC;IACnD,MAAM,MAAM,GAAG,IAAI,WAAW,CAAC,UAAU,EAAE,aAAa,CAAC,CAAC;IAE1D,OAAO,MAAM,CAAC,OAAO,CAAC,KAAK,IAAI,EAAE;QAC/B,MAAM,aAAa,GAAuB;YACxC,MAAM,EAAE,KAAK;YACb,GAAG,EAAE,aAAa;YAClB,YAAY,EAAE,MAAM;SACrB,CAAC;QAEF,IAAI,OAAO,EAAE,MAAM,EAAE,CAAC;YACpB,aAAa,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;QACxC,CAAC;aAAM,IAAI,OAAO,EAAE,OAAO,EAAE,CAAC;YAC5B,aAAa,CAAC,MAAM,GAAG,WAAW,CAAC,OAAO,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;QAC9D,CAAC;QAED,MAAM,gBAAgB,GAAG,eAAe,CAAC,OAAO,EAAE,aAAa,CAAC,CAAC;QACjE,IAAI,gBAAgB,EAAE,CAAC;YACrB,MAAM,eAAe,GAAG,CAAC,aAAa,CAAC,OAAO,IAAI,EAAE,CAGnD,CAAC;YACF,aAAa,CAAC,OAAO,GAAG,EAAE,GAAG,eAAe,EAAE,GAAG,gBAAgB,EAAE,CAAC;QACtE,CAAC;QAED,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,OAAO,CAAS,aAAa,CAAC,CAAC;YAC7D,OAAO,QAAQ,CAAC,IAAI,CAAC;QACvB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,KAAK,YAAY,UAAU,EAAE,CAAC;gBAChC,MAAM,KAAK,CAAC;YACd,CAAC;YACD,MAAM,IAAI,UAAU,CAClB,qBAAqB,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,SAAS,EAAE,EACzE,aAAa,CACd,CAAC;QACJ,CAAC;IACH,CAAC,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC;AACtB,CAAC"}
1
+ {"version":3,"file":"fetcher.js","sourceRoot":"","sources":["../../src/services/fetcher.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAG5C,OAAO,EAAE,uBAAuB,EAAE,MAAM,2BAA2B,CAAC;AAEpE,OAAO,EAAE,aAAa,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAC;AAChE,OAAO,EACL,eAAe,EACf,oBAAoB,EACpB,aAAa,GACd,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EAAE,eAAe,EAAE,MAAM,sBAAsB,CAAC;AACvD,OAAO,EACL,gBAAgB,EAChB,mBAAmB,EACnB,mBAAmB,GACpB,MAAM,2BAA2B,CAAC;AACnC,OAAO,EAAE,kBAAkB,EAAE,MAAM,wBAAwB,CAAC;AAC5D,OAAO,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AACzD,OAAO,EAAE,WAAW,EAAE,MAAM,2BAA2B,CAAC;AAExD,OAAO,EAAE,aAAa,EAAE,CAAC;AAEzB,MAAM,eAAe,GAAG;IACtB,YAAY,EAAE,MAAM,CAAC,OAAO,CAAC,SAAS;IACtC,MAAM,EACJ,4EAA4E;IAC9E,iBAAiB,EAAE,gBAAgB;IACnC,iBAAiB,EAAE,mBAAmB;IACtC,UAAU,EAAE,YAAY;CAChB,CAAC;AAEX,SAAS,YAAY,CAAC,aAAsC;IAC1D,MAAM,OAAO,GAAG,IAAI,OAAO,CAAC,eAAe,CAAC,CAAC;IAC7C,MAAM,SAAS,GAAG,eAAe,CAAC,aAAa,CAAC,CAAC;IACjD,IAAI,SAAS,EAAE,CAAC;QACd,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,CAAC;YACrD,OAAO,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;QAC1B,CAAC;IACH,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,SAAS,kBAAkB,CACzB,SAAiB,EACjB,QAAsB;IAEtB,MAAM,aAAa,GAAG,WAAW,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;IACrD,IAAI,CAAC,QAAQ;QAAE,OAAO,aAAa,CAAC;IACpC,OAAO,WAAW,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,aAAa,CAAC,CAAC,CAAC;AACpD,CAAC;AAED,SAAS,gBAAgB,CACvB,OAAgB,EAChB,MAAmB;IAEnB,OAAO;QACL,MAAM,EAAE,KAAK;QACb,OAAO;QACP,MAAM;QACN,UAAU;KACX,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,mBAAmB,CAChC,QAAkB,EAClB,QAAgB,EAChB,SAAiD;IAEjD,IAAI,QAAQ,CAAC,MAAM,KAAK,GAAG,EAAE,CAAC;QAC5B,KAAK,QAAQ,CAAC,IAAI,EAAE,MAAM,EAAE,CAAC;QAC7B,MAAM,oBAAoB,CAAC,QAAQ,EAAE,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC,CAAC;IAC5E,CAAC;IAED,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;QACjB,KAAK,QAAQ,CAAC,IAAI,EAAE,MAAM,EAAE,CAAC;QAC7B,MAAM,eAAe,CAAC,QAAQ,EAAE,QAAQ,CAAC,MAAM,EAAE,QAAQ,CAAC,UAAU,CAAC,CAAC;IACxE,CAAC;IAED,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,GAAG,MAAM,gBAAgB,CAC3C,QAAQ,EACR,QAAQ,EACR,MAAM,CAAC,OAAO,CAAC,gBAAgB,CAChC,CAAC;IACF,mBAAmB,CAAC,SAAS,EAAE,QAAQ,EAAE,IAAI,CAAC,CAAC;IAC/C,OAAO,IAAI,CAAC;AACd,CAAC;AAED,KAAK,UAAU,kBAAkB,CAC/B,aAAqB,EACrB,WAAwB,EACxB,SAAiB;IAEjB,MAAM,SAAS,GAAG,mBAAmB,CAAC,aAAa,EAAE,KAAK,CAAC,CAAC;IAE5D,IAAI,CAAC;QACH,MAAM,EAAE,QAAQ,EAAE,GAAG,EAAE,QAAQ,EAAE,GAAG,MAAM,kBAAkB,CAC1D,aAAa,EACb,WAAW,EACX,MAAM,CAAC,OAAO,CAAC,YAAY,CAC5B,CAAC;QAEF,SAAS,CAAC,GAAG,GAAG,QAAQ,CAAC;QACzB,OAAO,MAAM,mBAAmB,CAAC,QAAQ,EAAE,QAAQ,EAAE,SAAS,CAAC,CAAC;IAClE,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,MAAM,GAAG,aAAa,CAAC,KAAK,EAAE,aAAa,EAAE,SAAS,CAAC,CAAC;QAC9D,SAAS,CAAC,GAAG,GAAG,MAAM,CAAC,GAAG,CAAC;QAC3B,gBAAgB,CAAC,SAAS,EAAE,MAAM,EAAE,MAAM,CAAC,UAAU,CAAC,CAAC;QACvD,MAAM,MAAM,CAAC;IACf,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,iBAAiB,CACrC,GAAW,EACX,OAAsB,EACtB,UAAU,GAAG,CAAC;IAEd,MAAM,aAAa,GAAG,uBAAuB,CAAC,GAAG,CAAC,CAAC;IACnD,MAAM,MAAM,GAAG,IAAI,WAAW,CAAC,UAAU,EAAE,aAAa,CAAC,CAAC;IAC1D,MAAM,SAAS,GAAG,OAAO,EAAE,OAAO,IAAI,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC;IAC7D,MAAM,OAAO,GAAG,YAAY,CAAC,OAAO,EAAE,aAAa,CAAC,CAAC;IAErD,OAAO,MAAM,CAAC,OAAO,CAAC,KAAK,IAAI,EAAE;QAC/B,MAAM,MAAM,GAAG,kBAAkB,CAAC,SAAS,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC;QAC9D,MAAM,WAAW,GAAG,gBAAgB,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;QACtD,OAAO,kBAAkB,CAAC,aAAa,EAAE,WAAW,EAAE,SAAS,CAAC,CAAC;IACnE,CAAC,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC;AACtB,CAAC"}
@@ -26,17 +26,17 @@ function shouldLog(level) {
26
26
  }
27
27
  export function logInfo(message, meta) {
28
28
  if (shouldLog('info')) {
29
- console.log(formatLogEntry('info', message, meta));
29
+ process.stderr.write(`${formatLogEntry('info', message, meta)}\n`);
30
30
  }
31
31
  }
32
32
  export function logDebug(message, meta) {
33
33
  if (shouldLog('debug')) {
34
- console.debug(formatLogEntry('debug', message, meta));
34
+ process.stderr.write(`${formatLogEntry('debug', message, meta)}\n`);
35
35
  }
36
36
  }
37
37
  export function logWarn(message, meta) {
38
38
  if (shouldLog('warn')) {
39
- console.warn(formatLogEntry('warn', message, meta));
39
+ process.stderr.write(`${formatLogEntry('warn', message, meta)}\n`);
40
40
  }
41
41
  }
42
42
  export function logError(message, error) {
@@ -45,6 +45,6 @@ export function logError(message, error) {
45
45
  const errorMeta = error instanceof Error
46
46
  ? { error: error.message, stack: error.stack }
47
47
  : (error ?? {});
48
- console.error(formatLogEntry('error', message, errorMeta));
48
+ process.stderr.write(`${formatLogEntry('error', message, errorMeta)}\n`);
49
49
  }
50
50
  //# sourceMappingURL=logger.js.map
@@ -1 +1 @@
1
- {"version":3,"file":"logger.js","sourceRoot":"","sources":["../../src/services/logger.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAG5C,OAAO,EAAE,YAAY,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAE1D,SAAS,cAAc,CAAC,IAAkB;IACxC,MAAM,SAAS,GAAG,YAAY,EAAE,CAAC;IACjC,MAAM,SAAS,GAAG,YAAY,EAAE,CAAC;IAEjC,MAAM,WAAW,GAAgB,EAAE,CAAC;IACpC,IAAI,SAAS;QAAE,WAAW,CAAC,SAAS,GAAG,SAAS,CAAC;IACjD,IAAI,SAAS;QAAE,WAAW,CAAC,SAAS,GAAG,SAAS,CAAC;IAEjD,MAAM,MAAM,GAAG,EAAE,GAAG,WAAW,EAAE,GAAG,IAAI,EAAE,CAAC;IAC3C,OAAO,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;AAC5E,CAAC;AAED,SAAS,eAAe;IACtB,OAAO,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;AAClC,CAAC;AAED,SAAS,cAAc,CACrB,KAAe,EACf,OAAe,EACf,IAAkB;IAElB,OAAO,IAAI,eAAe,EAAE,KAAK,KAAK,CAAC,WAAW,EAAE,KAAK,OAAO,GAAG,cAAc,CAAC,IAAI,CAAC,EAAE,CAAC;AAC5F,CAAC;AAED,SAAS,SAAS,CAAC,KAAe;IAChC,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,OAAO;QAAE,OAAO,KAAK,CAAC;IAC1C,IAAI,KAAK,KAAK,OAAO;QAAE,OAAO,MAAM,CAAC,OAAO,CAAC,KAAK,KAAK,OAAO,CAAC;IAC/D,OAAO,IAAI,CAAC;AACd,CAAC;AAED,MAAM,UAAU,OAAO,CAAC,OAAe,EAAE,IAAkB;IACzD,IAAI,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC;QACtB,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,MAAM,EAAE,OAAO,EAAE,IAAI,CAAC,CAAC,CAAC;IACrD,CAAC;AACH,CAAC;AAED,MAAM,UAAU,QAAQ,CAAC,OAAe,EAAE,IAAkB;IAC1D,IAAI,SAAS,CAAC,OAAO,CAAC,EAAE,CAAC;QACvB,OAAO,CAAC,KAAK,CAAC,cAAc,CAAC,OAAO,EAAE,OAAO,EAAE,IAAI,CAAC,CAAC,CAAC;IACxD,CAAC;AACH,CAAC;AAED,MAAM,UAAU,OAAO,CAAC,OAAe,EAAE,IAAkB;IACzD,IAAI,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC;QACtB,OAAO,CAAC,IAAI,CAAC,cAAc,CAAC,MAAM,EAAE,OAAO,EAAE,IAAI,CAAC,CAAC,CAAC;IACtD,CAAC;AACH,CAAC;AAED,MAAM,UAAU,QAAQ,CAAC,OAAe,EAAE,KAA2B;IACnE,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC;QAAE,OAAO;IAEhC,MAAM,SAAS,GACb,KAAK,YAAY,KAAK;QACpB,CAAC,CAAC,EAAE,KAAK,EAAE,KAAK,CAAC,OAAO,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE;QAC9C,CAAC,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC;IAEpB,OAAO,CAAC,KAAK,CAAC,cAAc,CAAC,OAAO,EAAE,OAAO,EAAE,SAAS,CAAC,CAAC,CAAC;AAC7D,CAAC"}
1
+ {"version":3,"file":"logger.js","sourceRoot":"","sources":["../../src/services/logger.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAG5C,OAAO,EAAE,YAAY,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAE1D,SAAS,cAAc,CAAC,IAAkB;IACxC,MAAM,SAAS,GAAG,YAAY,EAAE,CAAC;IACjC,MAAM,SAAS,GAAG,YAAY,EAAE,CAAC;IAEjC,MAAM,WAAW,GAAgB,EAAE,CAAC;IACpC,IAAI,SAAS;QAAE,WAAW,CAAC,SAAS,GAAG,SAAS,CAAC;IACjD,IAAI,SAAS;QAAE,WAAW,CAAC,SAAS,GAAG,SAAS,CAAC;IAEjD,MAAM,MAAM,GAAG,EAAE,GAAG,WAAW,EAAE,GAAG,IAAI,EAAE,CAAC;IAC3C,OAAO,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;AAC5E,CAAC;AAED,SAAS,eAAe;IACtB,OAAO,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;AAClC,CAAC;AAED,SAAS,cAAc,CACrB,KAAe,EACf,OAAe,EACf,IAAkB;IAElB,OAAO,IAAI,eAAe,EAAE,KAAK,KAAK,CAAC,WAAW,EAAE,KAAK,OAAO,GAAG,cAAc,CAAC,IAAI,CAAC,EAAE,CAAC;AAC5F,CAAC;AAED,SAAS,SAAS,CAAC,KAAe;IAChC,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,OAAO;QAAE,OAAO,KAAK,CAAC;IAC1C,IAAI,KAAK,KAAK,OAAO;QAAE,OAAO,MAAM,CAAC,OAAO,CAAC,KAAK,KAAK,OAAO,CAAC;IAC/D,OAAO,IAAI,CAAC;AACd,CAAC;AAED,MAAM,UAAU,OAAO,CAAC,OAAe,EAAE,IAAkB;IACzD,IAAI,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC;QACtB,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,cAAc,CAAC,MAAM,EAAE,OAAO,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC;IACrE,CAAC;AACH,CAAC;AAED,MAAM,UAAU,QAAQ,CAAC,OAAe,EAAE,IAAkB;IAC1D,IAAI,SAAS,CAAC,OAAO,CAAC,EAAE,CAAC;QACvB,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,cAAc,CAAC,OAAO,EAAE,OAAO,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC;IACtE,CAAC;AACH,CAAC;AAED,MAAM,UAAU,OAAO,CAAC,OAAe,EAAE,IAAkB;IACzD,IAAI,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC;QACtB,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,cAAc,CAAC,MAAM,EAAE,OAAO,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC;IACrE,CAAC;AACH,CAAC;AAED,MAAM,UAAU,QAAQ,CAAC,OAAe,EAAE,KAA2B;IACnE,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC;QAAE,OAAO;IAEhC,MAAM,SAAS,GACb,KAAK,YAAY,KAAK;QACpB,CAAC,CAAC,EAAE,KAAK,EAAE,KAAK,CAAC,OAAO,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE;QAC9C,CAAC,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC;IAEpB,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,cAAc,CAAC,OAAO,EAAE,OAAO,EAAE,SAAS,CAAC,IAAI,CAAC,CAAC;AAC3E,CAAC"}
@@ -1,3 +1,4 @@
1
1
  import type { ContentBlockUnion } from '../config/types.js';
2
+ export declare function detectLanguageFromCode(code: string): string | undefined;
2
3
  export declare function parseHtml(html: string): ContentBlockUnion[];
3
4
  //# sourceMappingURL=parser.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"parser.d.ts","sourceRoot":"","sources":["../../src/services/parser.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EAGV,iBAAiB,EAOlB,MAAM,oBAAoB,CAAC;AAuO5B,wBAAgB,SAAS,CAAC,IAAI,EAAE,MAAM,GAAG,iBAAiB,EAAE,CA8B3D"}
1
+ {"version":3,"file":"parser.d.ts","sourceRoot":"","sources":["../../src/services/parser.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EAGV,iBAAiB,EAOlB,MAAM,oBAAoB,CAAC;AAc5B,wBAAgB,sBAAsB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS,CA8BvE;AAwOD,wBAAgB,SAAS,CAAC,IAAI,EAAE,MAAM,GAAG,iBAAiB,EAAE,CAS3D"}
@@ -4,7 +4,7 @@ import { cleanCodeBlock, cleanHeading, cleanListItems, cleanParagraph, removeInl
4
4
  import { truncateHtml } from '../utils/html-truncator.js';
5
5
  import { sanitizeText } from '../utils/sanitizer.js';
6
6
  import { logWarn } from './logger.js';
7
- function detectLanguageFromCode(code) {
7
+ export function detectLanguageFromCode(code) {
8
8
  // Common language patterns for code block detection
9
9
  const patterns = [
10
10
  [
@@ -174,48 +174,68 @@ function parseElement($, node) {
174
174
  return ELEMENT_PARSERS[tagName]($, node);
175
175
  }
176
176
  function filterBlocks(blocks) {
177
- return blocks.filter((block) => {
178
- switch (block.type) {
179
- case 'paragraph':
180
- case 'heading':
181
- case 'code':
182
- case 'blockquote':
183
- return block.text.length > 0;
184
- case 'list':
185
- return block.items.length > 0;
186
- default:
187
- return true;
188
- }
189
- });
177
+ return blocks.filter(shouldIncludeBlock);
190
178
  }
191
- export function parseHtml(html) {
192
- if (!html || typeof html !== 'string')
193
- return [];
194
- const processedHtml = truncateHtml(html);
179
+ const TEXT_BLOCK_TYPES = new Set([
180
+ 'paragraph',
181
+ 'heading',
182
+ 'code',
183
+ 'blockquote',
184
+ ]);
185
+ function isTextBlock(block) {
186
+ return TEXT_BLOCK_TYPES.has(block.type);
187
+ }
188
+ function shouldIncludeBlock(block) {
189
+ if (isTextBlock(block)) {
190
+ return block.text.length > 0;
191
+ }
192
+ if (block.type === 'list') {
193
+ return block.items.length > 0;
194
+ }
195
+ return true;
196
+ }
197
+ function loadHtml(html) {
195
198
  try {
196
- const $ = cheerio.load(processedHtml);
197
- const blocks = [];
198
- $('script, style, noscript, iframe, svg').remove();
199
- $('body')
200
- .find(CONTENT_SELECTOR)
201
- .each((_, element) => {
202
- try {
203
- const block = parseElement($, element);
204
- if (block)
205
- blocks.push(block);
206
- }
207
- catch {
208
- /* skip */
209
- }
210
- });
211
- return filterBlocks(blocks);
199
+ return cheerio.load(html);
212
200
  }
213
201
  catch (error) {
214
202
  logWarn('Failed to parse HTML', {
215
203
  error: error instanceof Error ? error.message : 'Unknown error',
216
204
  htmlLength: html.length,
217
205
  });
218
- return [];
206
+ return null;
207
+ }
208
+ }
209
+ function removeNoiseElements($) {
210
+ $('script, style, noscript, iframe, svg').remove();
211
+ }
212
+ function collectBlocks($) {
213
+ const blocks = [];
214
+ $('body')
215
+ .find(CONTENT_SELECTOR)
216
+ .each((_, element) => {
217
+ const block = safeParseElement($, element);
218
+ if (block)
219
+ blocks.push(block);
220
+ });
221
+ return blocks;
222
+ }
223
+ function safeParseElement($, element) {
224
+ try {
225
+ return parseElement($, element);
226
+ }
227
+ catch {
228
+ return null;
219
229
  }
220
230
  }
231
+ export function parseHtml(html) {
232
+ if (!html || typeof html !== 'string')
233
+ return [];
234
+ const processedHtml = truncateHtml(html);
235
+ const $ = loadHtml(processedHtml);
236
+ if (!$)
237
+ return [];
238
+ removeNoiseElements($);
239
+ return filterBlocks(collectBlocks($));
240
+ }
221
241
  //# sourceMappingURL=parser.js.map
@@ -1 +1 @@
1
- {"version":3,"file":"parser.js","sourceRoot":"","sources":["../../src/services/parser.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,OAAO,MAAM,SAAS,CAAC;AAInC,OAAO,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAa5C,OAAO,EACL,cAAc,EACd,YAAY,EACZ,cAAc,EACd,cAAc,EACd,sBAAsB,GACvB,MAAM,6BAA6B,CAAC;AACrC,OAAO,EAAE,YAAY,EAAE,MAAM,4BAA4B,CAAC;AAC1D,OAAO,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAC;AAErD,OAAO,EAAE,OAAO,EAAE,MAAM,aAAa,CAAC;AAEtC,SAAS,sBAAsB,CAAC,IAAY;IAC1C,oDAAoD;IACpD,MAAM,QAAQ,GAAgC;QAC5C;YACE,kFAAkF;YAClF,KAAK;SACN;QACD;YACE,sFAAsF;YACtF,YAAY;SACb;QACD,CAAC,kDAAkD,EAAE,MAAM,CAAC;QAC5D;YACE,+EAA+E;YAC/E,YAAY;SACb;QACD,CAAC,kDAAkD,EAAE,QAAQ,CAAC;QAC9D;YACE,iFAAiF;YACjF,MAAM;SACP;QACD,CAAC,iEAAiE,EAAE,MAAM,CAAC;QAC3E,CAAC,yDAAyD,EAAE,KAAK,CAAC;QAClE,CAAC,8DAA8D,EAAE,MAAM,CAAC;QACxE,CAAC,6CAA6C,EAAE,MAAM,CAAC;QACvD,CAAC,oBAAoB,EAAE,MAAM,CAAC;QAC9B,CAAC,0DAA0D,EAAE,KAAK,CAAC;QACnE,CAAC,gCAAgC,EAAE,IAAI,CAAC;KACzC,CAAC;IAEF,OAAO,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;AAC/D,CAAC;AAED,MAAM,gBAAgB,GACpB,oFAAoF,CAAC;AAEvF,SAAS,YAAY,CAAC,CAAa,EAAE,OAAgB;IACnD,MAAM,OAAO,GAAG,YAAY,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;IAChD,MAAM,IAAI,GAAG,YAAY,CAAC,OAAO,CAAC,CAAC;IACnC,IAAI,CAAC,IAAI;QAAE,OAAO,IAAI,CAAC;IAEvB,OAAO;QACL,IAAI,EAAE,SAAS;QACf,KAAK,EAAE,QAAQ,CAAC,OAAO,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC;QACjD,IAAI;KACL,CAAC;AACJ,CAAC;AAED,SAAS,cAAc,CACrB,CAAa,EACb,OAAgB;IAEhB,IAAI,OAAO,GAAG,YAAY,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;IAC9C,8DAA8D;IAC9D,OAAO,GAAG,sBAAsB,CAAC,OAAO,CAAC,CAAC;IAC1C,MAAM,IAAI,GAAG,cAAc,CAAC,OAAO,CAAC,CAAC;IACrC,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,UAAU,CAAC,kBAAkB;QAAE,OAAO,IAAI,CAAC;IAE7E,OAAO,EAAE,IAAI,EAAE,WAAW,EAAE,IAAI,EAAE,CAAC;AACrC,CAAC;AAED,SAAS,SAAS,CAAC,CAAa,EAAE,OAAgB;IAChD,MAAM,QAAQ,GAAa,EAAE,CAAC;IAE9B,CAAC,CAAC,OAAO,CAAC;SACP,IAAI,CAAC,IAAI,CAAC;SACV,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,EAAE;QACd,MAAM,IAAI,GAAG,YAAY,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;QACxC,IAAI,IAAI;YAAE,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAChC,CAAC,CAAC,CAAC;IAEL,mCAAmC;IACnC,MAAM,KAAK,GAAG,cAAc,CAAC,QAAQ,CAAC,CAAC;IACvC,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IAEpC,OAAO;QACL,IAAI,EAAE,MAAM;QACZ,OAAO,EAAE,OAAO,CAAC,OAAO,CAAC,WAAW,EAAE,KAAK,IAAI;QAC/C,KAAK;KACN,CAAC;AACJ,CAAC;AAED,SAAS,SAAS,CAAC,CAAa,EAAE,OAAgB;IAChD,MAAM,OAAO,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;IACzC,MAAM,IAAI,GAAG,cAAc,CAAC,OAAO,CAAC,CAAC;IACrC,IAAI,CAAC,IAAI;QAAE,OAAO,IAAI,CAAC;IAEvB,iDAAiD;IACjD,MAAM,SAAS,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC;IACjD,MAAM,QAAQ,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,IAAI,EAAE,CAAC;IAExD,MAAM,aAAa,GACjB,gBAAgB,CAAC,IAAI,CAAC,SAAS,CAAC;QAChC,YAAY,CAAC,IAAI,CAAC,SAAS,CAAC;QAC5B,iBAAiB,CAAC,IAAI,CAAC,SAAS,CAAC;QACjC,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IAE3B,MAAM,QAAQ,GAAG,aAAa,EAAE,CAAC,CAAC,CAAC,IAAI,sBAAsB,CAAC,IAAI,CAAC,CAAC;IAEpE,OAAO;QACL,IAAI,EAAE,MAAM;QACZ,QAAQ;QACR,IAAI;KACL,CAAC;AACJ,CAAC;AAED,SAAS,UAAU,CAAC,CAAa,EAAE,OAAgB;IACjD,MAAM,OAAO,GAAa,EAAE,CAAC;IAC7B,MAAM,IAAI,GAAe,EAAE,CAAC;IAC5B,MAAM,MAAM,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC;IAE1B,MAAM,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,EAAE;QACjD,OAAO,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;IAC7C,CAAC,CAAC,CAAC;IAEH,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACzB,MAAM;aACH,IAAI,CAAC,IAAI,CAAC;aACV,KAAK,EAAE;aACP,IAAI,CAAC,QAAQ,CAAC;aACd,IAAI,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,EAAE;YAChB,OAAO,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;QAC7C,CAAC,CAAC,CAAC;IACP,CAAC;IAED,MAAM,YAAY,GAChB,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,0BAA0B,CAAC,CAAC,CAAC,cAAc,CAAC;IAEnE,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,EAAE;QACxC,MAAM,KAAK,GAAa,EAAE,CAAC;QAC3B,CAAC,CAAC,GAAG,CAAC;aACH,IAAI,CAAC,QAAQ,CAAC;aACd,IAAI,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,EAAE;YAChB,KAAK,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;QAC3C,CAAC,CAAC,CAAC;QACL,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC;YAAE,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IACzC,CAAC,CAAC,CAAC;IAEH,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IAEnC,OAAO;QACL,IAAI,EAAE,OAAO;QACb,OAAO,EAAE,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,SAAS;QACjD,IAAI;KACL,CAAC;AACJ,CAAC;AAED,SAAS,UAAU,CAAC,CAAa,EAAE,OAAgB;IACjD,MAAM,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IACnC,IAAI,CAAC,GAAG;QAAE,OAAO,IAAI,CAAC;IAEtB,OAAO;QACL,IAAI,EAAE,OAAO;QACb,GAAG;QACH,GAAG,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,SAAS;KACzC,CAAC;AACJ,CAAC;AAED,SAAS,eAAe,CACtB,CAAa,EACb,OAAgB;IAEhB,MAAM,OAAO,GAAG,YAAY,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;IAChD,MAAM,IAAI,GAAG,cAAc,CAAC,OAAO,CAAC,CAAC;IACrC,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,UAAU,CAAC,kBAAkB;QAAE,OAAO,IAAI,CAAC;IAE7E,OAAO,EAAE,IAAI,EAAE,YAAY,EAAE,IAAI,EAAE,CAAC;AACtC,CAAC;AAED,MAAM,eAAe,GAAG;IACtB,EAAE,EAAE,YAAY;IAChB,EAAE,EAAE,YAAY;IAChB,EAAE,EAAE,YAAY;IAChB,EAAE,EAAE,YAAY;IAChB,EAAE,EAAE,YAAY;IAChB,EAAE,EAAE,YAAY;IAChB,CAAC,EAAE,cAAc;IACjB,EAAE,EAAE,SAAS;IACb,EAAE,EAAE,SAAS;IACb,GAAG,EAAE,SAAS;IACd,IAAI,EAAE,SAAS;IACf,KAAK,EAAE,UAAU;IACjB,GAAG,EAAE,UAAU;IACf,UAAU,EAAE,eAAe;CAI5B,CAAC;AAEF,SAAS,cAAc,CAAC,GAAW;IACjC,OAAO,GAAG,IAAI,eAAe,CAAC;AAChC,CAAC;AAED,SAAS,YAAY,CAAC,CAAa,EAAE,IAAa;IAChD,IAAI,CAAC,CAAC,SAAS,IAAI,IAAI,CAAC,IAAI,OAAO,IAAI,CAAC,OAAO,KAAK,QAAQ;QAAE,OAAO,IAAI,CAAC;IAE1E,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC;IAC3C,IAAI,CAAC,cAAc,CAAC,OAAO,CAAC;QAAE,OAAO,IAAI,CAAC;IAC1C,OAAO,eAAe,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC;AAC3C,CAAC;AAED,SAAS,YAAY,CAAC,MAA2B;IAC/C,OAAO,MAAM,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE;QAC7B,QAAQ,KAAK,CAAC,IAAI,EAAE,CAAC;YACnB,KAAK,WAAW,CAAC;YACjB,KAAK,SAAS,CAAC;YACf,KAAK,MAAM,CAAC;YACZ,KAAK,YAAY;gBACf,OAAO,KAAK,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC;YAC/B,KAAK,MAAM;gBACT,OAAO,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC;YAChC;gBACE,OAAO,IAAI,CAAC;QAChB,CAAC;IACH,CAAC,CAAC,CAAC;AACL,CAAC;AAED,MAAM,UAAU,SAAS,CAAC,IAAY;IACpC,IAAI,CAAC,IAAI,IAAI,OAAO,IAAI,KAAK,QAAQ;QAAE,OAAO,EAAE,CAAC;IAEjD,MAAM,aAAa,GAAG,YAAY,CAAC,IAAI,CAAC,CAAC;IAEzC,IAAI,CAAC;QACH,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;QACtC,MAAM,MAAM,GAAwB,EAAE,CAAC;QAEvC,CAAC,CAAC,sCAAsC,CAAC,CAAC,MAAM,EAAE,CAAC;QAEnD,CAAC,CAAC,MAAM,CAAC;aACN,IAAI,CAAC,gBAAgB,CAAC;aACtB,IAAI,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,EAAE;YACnB,IAAI,CAAC;gBACH,MAAM,KAAK,GAAG,YAAY,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC;gBACvC,IAAI,KAAK;oBAAE,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAChC,CAAC;YAAC,MAAM,CAAC;gBACP,UAAU;YACZ,CAAC;QACH,CAAC,CAAC,CAAC;QAEL,OAAO,YAAY,CAAC,MAAM,CAAC,CAAC;IAC9B,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,OAAO,CAAC,sBAAsB,EAAE;YAC9B,KAAK,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe;YAC/D,UAAU,EAAE,IAAI,CAAC,MAAM;SACxB,CAAC,CAAC;QACH,OAAO,EAAE,CAAC;IACZ,CAAC;AACH,CAAC"}
1
+ {"version":3,"file":"parser.js","sourceRoot":"","sources":["../../src/services/parser.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,OAAO,MAAM,SAAS,CAAC;AAInC,OAAO,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAa5C,OAAO,EACL,cAAc,EACd,YAAY,EACZ,cAAc,EACd,cAAc,EACd,sBAAsB,GACvB,MAAM,6BAA6B,CAAC;AACrC,OAAO,EAAE,YAAY,EAAE,MAAM,4BAA4B,CAAC;AAC1D,OAAO,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAC;AAErD,OAAO,EAAE,OAAO,EAAE,MAAM,aAAa,CAAC;AAEtC,MAAM,UAAU,sBAAsB,CAAC,IAAY;IACjD,oDAAoD;IACpD,MAAM,QAAQ,GAAgC;QAC5C;YACE,kFAAkF;YAClF,KAAK;SACN;QACD;YACE,sFAAsF;YACtF,YAAY;SACb;QACD,CAAC,kDAAkD,EAAE,MAAM,CAAC;QAC5D;YACE,+EAA+E;YAC/E,YAAY;SACb;QACD,CAAC,kDAAkD,EAAE,QAAQ,CAAC;QAC9D;YACE,iFAAiF;YACjF,MAAM;SACP;QACD,CAAC,iEAAiE,EAAE,MAAM,CAAC;QAC3E,CAAC,yDAAyD,EAAE,KAAK,CAAC;QAClE,CAAC,8DAA8D,EAAE,MAAM,CAAC;QACxE,CAAC,6CAA6C,EAAE,MAAM,CAAC;QACvD,CAAC,oBAAoB,EAAE,MAAM,CAAC;QAC9B,CAAC,0DAA0D,EAAE,KAAK,CAAC;QACnE,CAAC,gCAAgC,EAAE,IAAI,CAAC;KACzC,CAAC;IACF,OAAO,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;AAC/D,CAAC;AACD,MAAM,gBAAgB,GACpB,oFAAoF,CAAC;AACvF,SAAS,YAAY,CAAC,CAAa,EAAE,OAAgB;IACnD,MAAM,OAAO,GAAG,YAAY,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;IAChD,MAAM,IAAI,GAAG,YAAY,CAAC,OAAO,CAAC,CAAC;IACnC,IAAI,CAAC,IAAI;QAAE,OAAO,IAAI,CAAC;IAEvB,OAAO;QACL,IAAI,EAAE,SAAS;QACf,KAAK,EAAE,QAAQ,CAAC,OAAO,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC;QACjD,IAAI;KACL,CAAC;AACJ,CAAC;AACD,SAAS,cAAc,CACrB,CAAa,EACb,OAAgB;IAEhB,IAAI,OAAO,GAAG,YAAY,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;IAC9C,8DAA8D;IAC9D,OAAO,GAAG,sBAAsB,CAAC,OAAO,CAAC,CAAC;IAC1C,MAAM,IAAI,GAAG,cAAc,CAAC,OAAO,CAAC,CAAC;IACrC,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,UAAU,CAAC,kBAAkB;QAAE,OAAO,IAAI,CAAC;IAE7E,OAAO,EAAE,IAAI,EAAE,WAAW,EAAE,IAAI,EAAE,CAAC;AACrC,CAAC;AACD,SAAS,SAAS,CAAC,CAAa,EAAE,OAAgB;IAChD,MAAM,QAAQ,GAAa,EAAE,CAAC;IAC9B,CAAC,CAAC,OAAO,CAAC;SACP,IAAI,CAAC,IAAI,CAAC;SACV,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,EAAE;QACd,MAAM,IAAI,GAAG,YAAY,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;QACxC,IAAI,IAAI;YAAE,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAChC,CAAC,CAAC,CAAC;IAEL,mCAAmC;IACnC,MAAM,KAAK,GAAG,cAAc,CAAC,QAAQ,CAAC,CAAC;IACvC,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IAEpC,OAAO;QACL,IAAI,EAAE,MAAM;QACZ,OAAO,EAAE,OAAO,CAAC,OAAO,CAAC,WAAW,EAAE,KAAK,IAAI;QAC/C,KAAK;KACN,CAAC;AACJ,CAAC;AAED,SAAS,SAAS,CAAC,CAAa,EAAE,OAAgB;IAChD,MAAM,OAAO,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;IACzC,MAAM,IAAI,GAAG,cAAc,CAAC,OAAO,CAAC,CAAC;IACrC,IAAI,CAAC,IAAI;QAAE,OAAO,IAAI,CAAC;IAEvB,iDAAiD;IACjD,MAAM,SAAS,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC;IACjD,MAAM,QAAQ,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,IAAI,EAAE,CAAC;IAExD,MAAM,aAAa,GACjB,gBAAgB,CAAC,IAAI,CAAC,SAAS,CAAC;QAChC,YAAY,CAAC,IAAI,CAAC,SAAS,CAAC;QAC5B,iBAAiB,CAAC,IAAI,CAAC,SAAS,CAAC;QACjC,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IAE3B,MAAM,QAAQ,GAAG,aAAa,EAAE,CAAC,CAAC,CAAC,IAAI,sBAAsB,CAAC,IAAI,CAAC,CAAC;IAEpE,OAAO;QACL,IAAI,EAAE,MAAM;QACZ,QAAQ;QACR,IAAI;KACL,CAAC;AACJ,CAAC;AAED,SAAS,UAAU,CAAC,CAAa,EAAE,OAAgB;IACjD,MAAM,OAAO,GAAa,EAAE,CAAC;IAC7B,MAAM,IAAI,GAAe,EAAE,CAAC;IAC5B,MAAM,MAAM,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC;IAE1B,MAAM,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,EAAE;QACjD,OAAO,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;IAC7C,CAAC,CAAC,CAAC;IAEH,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACzB,MAAM;aACH,IAAI,CAAC,IAAI,CAAC;aACV,KAAK,EAAE;aACP,IAAI,CAAC,QAAQ,CAAC;aACd,IAAI,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,EAAE;YAChB,OAAO,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;QAC7C,CAAC,CAAC,CAAC;IACP,CAAC;IAED,MAAM,YAAY,GAChB,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,0BAA0B,CAAC,CAAC,CAAC,cAAc,CAAC;IAEnE,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,EAAE;QACxC,MAAM,KAAK,GAAa,EAAE,CAAC;QAC3B,CAAC,CAAC,GAAG,CAAC;aACH,IAAI,CAAC,QAAQ,CAAC;aACd,IAAI,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,EAAE;YAChB,KAAK,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;QAC3C,CAAC,CAAC,CAAC;QACL,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC;YAAE,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IACzC,CAAC,CAAC,CAAC;IAEH,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IAEnC,OAAO;QACL,IAAI,EAAE,OAAO;QACb,OAAO,EAAE,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,SAAS;QACjD,IAAI;KACL,CAAC;AACJ,CAAC;AAED,SAAS,UAAU,CAAC,CAAa,EAAE,OAAgB;IACjD,MAAM,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IACnC,IAAI,CAAC,GAAG;QAAE,OAAO,IAAI,CAAC;IAEtB,OAAO;QACL,IAAI,EAAE,OAAO;QACb,GAAG;QACH,GAAG,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,SAAS;KACzC,CAAC;AACJ,CAAC;AAED,SAAS,eAAe,CACtB,CAAa,EACb,OAAgB;IAEhB,MAAM,OAAO,GAAG,YAAY,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;IAChD,MAAM,IAAI,GAAG,cAAc,CAAC,OAAO,CAAC,CAAC;IACrC,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,UAAU,CAAC,kBAAkB;QAAE,OAAO,IAAI,CAAC;IAE7E,OAAO,EAAE,IAAI,EAAE,YAAY,EAAE,IAAI,EAAE,CAAC;AACtC,CAAC;AAED,MAAM,eAAe,GAAG;IACtB,EAAE,EAAE,YAAY;IAChB,EAAE,EAAE,YAAY;IAChB,EAAE,EAAE,YAAY;IAChB,EAAE,EAAE,YAAY;IAChB,EAAE,EAAE,YAAY;IAChB,EAAE,EAAE,YAAY;IAChB,CAAC,EAAE,cAAc;IACjB,EAAE,EAAE,SAAS;IACb,EAAE,EAAE,SAAS;IACb,GAAG,EAAE,SAAS;IACd,IAAI,EAAE,SAAS;IACf,KAAK,EAAE,UAAU;IACjB,GAAG,EAAE,UAAU;IACf,UAAU,EAAE,eAAe;CAI5B,CAAC;AAEF,SAAS,cAAc,CAAC,GAAW;IACjC,OAAO,GAAG,IAAI,eAAe,CAAC;AAChC,CAAC;AAED,SAAS,YAAY,CAAC,CAAa,EAAE,IAAa;IAChD,IAAI,CAAC,CAAC,SAAS,IAAI,IAAI,CAAC,IAAI,OAAO,IAAI,CAAC,OAAO,KAAK,QAAQ;QAAE,OAAO,IAAI,CAAC;IAE1E,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC;IAC3C,IAAI,CAAC,cAAc,CAAC,OAAO,CAAC;QAAE,OAAO,IAAI,CAAC;IAC1C,OAAO,eAAe,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC;AAC3C,CAAC;AAED,SAAS,YAAY,CAAC,MAA2B;IAC/C,OAAO,MAAM,CAAC,MAAM,CAAC,kBAAkB,CAAC,CAAC;AAC3C,CAAC;AAED,MAAM,gBAAgB,GAAG,IAAI,GAAG,CAAC;IAC/B,WAAW;IACX,SAAS;IACT,MAAM;IACN,YAAY;CACb,CAAC,CAAC;AAEH,SAAS,WAAW,CAClB,KAAwB;IAExB,OAAO,gBAAgB,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;AAC1C,CAAC;AAED,SAAS,kBAAkB,CAAC,KAAwB;IAClD,IAAI,WAAW,CAAC,KAAK,CAAC,EAAE,CAAC;QACvB,OAAO,KAAK,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC;IAC/B,CAAC;IACD,IAAI,KAAK,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;QAC1B,OAAO,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC;IAChC,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,SAAS,QAAQ,CAAC,IAAY;IAC5B,IAAI,CAAC;QACH,OAAO,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC5B,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,OAAO,CAAC,sBAAsB,EAAE;YAC9B,KAAK,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe;YAC/D,UAAU,EAAE,IAAI,CAAC,MAAM;SACxB,CAAC,CAAC;QACH,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED,SAAS,mBAAmB,CAAC,CAAa;IACxC,CAAC,CAAC,sCAAsC,CAAC,CAAC,MAAM,EAAE,CAAC;AACrD,CAAC;AAED,SAAS,aAAa,CAAC,CAAa;IAClC,MAAM,MAAM,GAAwB,EAAE,CAAC;IAEvC,CAAC,CAAC,MAAM,CAAC;SACN,IAAI,CAAC,gBAAgB,CAAC;SACtB,IAAI,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,EAAE;QACnB,MAAM,KAAK,GAAG,gBAAgB,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC;QAC3C,IAAI,KAAK;YAAE,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAChC,CAAC,CAAC,CAAC;IAEL,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,gBAAgB,CACvB,CAAa,EACb,OAAgB;IAEhB,IAAI,CAAC;QACH,OAAO,YAAY,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC;IAClC,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED,MAAM,UAAU,SAAS,CAAC,IAAY;IACpC,IAAI,CAAC,IAAI,IAAI,OAAO,IAAI,KAAK,QAAQ;QAAE,OAAO,EAAE,CAAC;IAEjD,MAAM,aAAa,GAAG,YAAY,CAAC,IAAI,CAAC,CAAC;IACzC,MAAM,CAAC,GAAG,QAAQ,CAAC,aAAa,CAAC,CAAC;IAClC,IAAI,CAAC,CAAC;QAAE,OAAO,EAAE,CAAC;IAElB,mBAAmB,CAAC,CAAC,CAAC,CAAC;IACvB,OAAO,YAAY,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,CAAC;AACxC,CAAC"}
@@ -0,0 +1,4 @@
1
+ import type { ExtractLinksOptions, LinksTransformResult, ToolResponseBase } from '../../../config/types.js';
2
+ export declare function resolveFilterPattern(pattern: string | undefined, url: string): RegExp | undefined | ToolResponseBase;
3
+ export declare function extractLinks(html: string, baseUrl: string, options: ExtractLinksOptions): LinksTransformResult;
4
+ //# sourceMappingURL=link-extractor.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"link-extractor.d.ts","sourceRoot":"","sources":["../../../../src/tools/handlers/fetch-links/link-extractor.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAEV,mBAAmB,EACnB,oBAAoB,EAEpB,gBAAgB,EACjB,MAAM,0BAA0B,CAAC;AA0ClC,wBAAgB,oBAAoB,CAClC,OAAO,EAAE,MAAM,GAAG,SAAS,EAC3B,GAAG,EAAE,MAAM,GACV,MAAM,GAAG,SAAS,GAAG,gBAAgB,CAavC;AA6JD,wBAAgB,YAAY,CAC1B,IAAI,EAAE,MAAM,EACZ,OAAO,EAAE,MAAM,EACf,OAAO,EAAE,mBAAmB,GAC3B,oBAAoB,CAiBtB"}