@j0hanz/superfetch 1.1.1 → 1.1.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +57 -32
- package/dist/config/formatting.d.ts +9 -0
- package/dist/config/formatting.d.ts.map +1 -0
- package/dist/config/formatting.js +11 -0
- package/dist/config/formatting.js.map +1 -0
- package/dist/config/index.d.ts +16 -2
- package/dist/config/index.d.ts.map +1 -1
- package/dist/config/index.js +43 -14
- package/dist/config/index.js.map +1 -1
- package/dist/config/types/content.d.ts +107 -0
- package/dist/config/types/content.d.ts.map +1 -0
- package/dist/config/types/content.js +2 -0
- package/dist/config/types/content.js.map +1 -0
- package/dist/config/types/runtime.d.ts +78 -0
- package/dist/config/types/runtime.d.ts.map +1 -0
- package/dist/config/types/runtime.js +2 -0
- package/dist/config/types/runtime.js.map +1 -0
- package/dist/config/types/tools.d.ts +99 -0
- package/dist/config/types/tools.d.ts.map +1 -0
- package/dist/config/types/tools.js +2 -0
- package/dist/config/types/tools.js.map +1 -0
- package/dist/config/types.d.ts +3 -296
- package/dist/config/types.d.ts.map +1 -1
- package/dist/http/auth.d.ts +3 -0
- package/dist/http/auth.d.ts.map +1 -0
- package/dist/http/auth.js +34 -0
- package/dist/http/auth.js.map +1 -0
- package/dist/http/cors.d.ts +8 -0
- package/dist/http/cors.d.ts.map +1 -0
- package/dist/http/cors.js +47 -0
- package/dist/http/cors.js.map +1 -0
- package/dist/http/mcp-routes.d.ts +5 -0
- package/dist/http/mcp-routes.d.ts.map +1 -0
- package/dist/http/mcp-routes.js +110 -0
- package/dist/http/mcp-routes.js.map +1 -0
- package/dist/http/mcp-session.d.ts +12 -0
- package/dist/http/mcp-session.d.ts.map +1 -0
- package/dist/http/mcp-session.js +209 -0
- package/dist/http/mcp-session.js.map +1 -0
- package/dist/http/mcp-validation.d.ts +3 -0
- package/dist/http/mcp-validation.d.ts.map +1 -0
- package/dist/http/mcp-validation.js +34 -0
- package/dist/http/mcp-validation.js.map +1 -0
- package/dist/http/rate-limit.d.ts +13 -0
- package/dist/http/rate-limit.d.ts.map +1 -0
- package/dist/http/rate-limit.js +91 -0
- package/dist/http/rate-limit.js.map +1 -0
- package/dist/http/server.d.ts +4 -0
- package/dist/http/server.d.ts.map +1 -0
- package/dist/http/server.js +183 -0
- package/dist/http/server.js.map +1 -0
- package/dist/http/sessions.d.ts +15 -0
- package/dist/http/sessions.d.ts.map +1 -0
- package/dist/http/sessions.js +64 -0
- package/dist/http/sessions.js.map +1 -0
- package/dist/index.js +26 -223
- package/dist/index.js.map +1 -1
- package/dist/middleware/error-handler.d.ts +2 -2
- package/dist/middleware/error-handler.d.ts.map +1 -1
- package/dist/middleware/error-handler.js +46 -15
- package/dist/middleware/error-handler.js.map +1 -1
- package/dist/resources/cached-content.d.ts.map +1 -1
- package/dist/resources/cached-content.js +104 -44
- package/dist/resources/cached-content.js.map +1 -1
- package/dist/resources/index.d.ts.map +1 -1
- package/dist/resources/index.js +77 -69
- package/dist/resources/index.js.map +1 -1
- package/dist/server.d.ts.map +1 -1
- package/dist/server.js +9 -3
- package/dist/server.js.map +1 -1
- package/dist/services/cache.d.ts +13 -1
- package/dist/services/cache.d.ts.map +1 -1
- package/dist/services/cache.js +90 -13
- package/dist/services/cache.js.map +1 -1
- package/dist/services/context.d.ts +9 -0
- package/dist/services/context.d.ts.map +1 -0
- package/dist/services/context.js +9 -0
- package/dist/services/context.js.map +1 -0
- package/dist/services/extractor.d.ts.map +1 -1
- package/dist/services/extractor.js +122 -87
- package/dist/services/extractor.js.map +1 -1
- package/dist/services/fetcher/agents.d.ts +4 -0
- package/dist/services/fetcher/agents.d.ts.map +1 -0
- package/dist/services/fetcher/agents.js +111 -0
- package/dist/services/fetcher/agents.js.map +1 -0
- package/dist/services/fetcher/errors.d.ts +5 -0
- package/dist/services/fetcher/errors.d.ts.map +1 -0
- package/dist/services/fetcher/errors.js +71 -0
- package/dist/services/fetcher/errors.js.map +1 -0
- package/dist/services/fetcher/headers.d.ts +2 -0
- package/dist/services/fetcher/headers.d.ts.map +1 -0
- package/dist/services/fetcher/headers.js +28 -0
- package/dist/services/fetcher/headers.js.map +1 -0
- package/dist/services/fetcher/interceptors.d.ts +10 -0
- package/dist/services/fetcher/interceptors.d.ts.map +1 -0
- package/dist/services/fetcher/interceptors.js +82 -0
- package/dist/services/fetcher/interceptors.js.map +1 -0
- package/dist/services/fetcher/redirects.d.ts +6 -0
- package/dist/services/fetcher/redirects.d.ts.map +1 -0
- package/dist/services/fetcher/redirects.js +67 -0
- package/dist/services/fetcher/redirects.js.map +1 -0
- package/dist/services/fetcher/response.d.ts +5 -0
- package/dist/services/fetcher/response.d.ts.map +1 -0
- package/dist/services/fetcher/response.js +39 -0
- package/dist/services/fetcher/response.js.map +1 -0
- package/dist/services/fetcher/retry-policy.d.ts +28 -0
- package/dist/services/fetcher/retry-policy.d.ts.map +1 -0
- package/dist/services/fetcher/retry-policy.js +138 -0
- package/dist/services/fetcher/retry-policy.js.map +1 -0
- package/dist/services/fetcher.d.ts +2 -1
- package/dist/services/fetcher.d.ts.map +1 -1
- package/dist/services/fetcher.js +61 -254
- package/dist/services/fetcher.js.map +1 -1
- package/dist/services/logger.d.ts.map +1 -1
- package/dist/services/logger.js +14 -5
- package/dist/services/logger.js.map +1 -1
- package/dist/services/parser.d.ts +1 -0
- package/dist/services/parser.d.ts.map +1 -1
- package/dist/services/parser.js +55 -35
- package/dist/services/parser.js.map +1 -1
- package/dist/tools/handlers/fetch-links/link-extractor.d.ts +4 -0
- package/dist/tools/handlers/fetch-links/link-extractor.d.ts.map +1 -0
- package/dist/tools/handlers/fetch-links/link-extractor.js +163 -0
- package/dist/tools/handlers/fetch-links/link-extractor.js.map +1 -0
- package/dist/tools/handlers/fetch-links.tool.d.ts.map +1 -1
- package/dist/tools/handlers/fetch-links.tool.js +78 -116
- package/dist/tools/handlers/fetch-links.tool.js.map +1 -1
- package/dist/tools/handlers/fetch-markdown.tool.d.ts +3 -13
- package/dist/tools/handlers/fetch-markdown.tool.d.ts.map +1 -1
- package/dist/tools/handlers/fetch-markdown.tool.js +74 -83
- package/dist/tools/handlers/fetch-markdown.tool.js.map +1 -1
- package/dist/tools/handlers/fetch-single.shared.d.ts +26 -0
- package/dist/tools/handlers/fetch-single.shared.d.ts.map +1 -0
- package/dist/tools/handlers/fetch-single.shared.js +49 -0
- package/dist/tools/handlers/fetch-single.shared.js.map +1 -0
- package/dist/tools/handlers/fetch-url.tool.d.ts.map +1 -1
- package/dist/tools/handlers/fetch-url.tool.js +82 -54
- package/dist/tools/handlers/fetch-url.tool.js.map +1 -1
- package/dist/tools/handlers/fetch-urls/processor.d.ts +13 -0
- package/dist/tools/handlers/fetch-urls/processor.d.ts.map +1 -0
- package/dist/tools/handlers/fetch-urls/processor.js +153 -0
- package/dist/tools/handlers/fetch-urls/processor.js.map +1 -0
- package/dist/tools/handlers/fetch-urls/response.d.ts +3 -0
- package/dist/tools/handlers/fetch-urls/response.d.ts.map +1 -0
- package/dist/tools/handlers/fetch-urls/response.js +58 -0
- package/dist/tools/handlers/fetch-urls/response.js.map +1 -0
- package/dist/tools/handlers/fetch-urls/validation.d.ts +6 -0
- package/dist/tools/handlers/fetch-urls/validation.d.ts.map +1 -0
- package/dist/tools/handlers/fetch-urls/validation.js +18 -0
- package/dist/tools/handlers/fetch-urls/validation.js.map +1 -0
- package/dist/tools/handlers/fetch-urls.tool.d.ts.map +1 -1
- package/dist/tools/handlers/fetch-urls.tool.js +104 -197
- package/dist/tools/handlers/fetch-urls.tool.js.map +1 -1
- package/dist/tools/index.d.ts.map +1 -1
- package/dist/tools/index.js +36 -237
- package/dist/tools/index.js.map +1 -1
- package/dist/tools/schemas.d.ts +357 -0
- package/dist/tools/schemas.d.ts.map +1 -0
- package/dist/tools/schemas.js +272 -0
- package/dist/tools/schemas.js.map +1 -0
- package/dist/tools/utils/cache-vary.d.ts +3 -0
- package/dist/tools/utils/cache-vary.d.ts.map +1 -0
- package/dist/tools/utils/cache-vary.js +44 -0
- package/dist/tools/utils/cache-vary.js.map +1 -0
- package/dist/tools/utils/common.d.ts +2 -2
- package/dist/tools/utils/common.d.ts.map +1 -1
- package/dist/tools/utils/common.js +5 -1
- package/dist/tools/utils/common.js.map +1 -1
- package/dist/tools/utils/content-transform.d.ts +16 -0
- package/dist/tools/utils/content-transform.d.ts.map +1 -0
- package/dist/tools/utils/content-transform.js +49 -0
- package/dist/tools/utils/content-transform.js.map +1 -0
- package/dist/tools/utils/fetch-pipeline.d.ts.map +1 -1
- package/dist/tools/utils/fetch-pipeline.js +32 -18
- package/dist/tools/utils/fetch-pipeline.js.map +1 -1
- package/dist/tools/utils/inline-content.d.ts +11 -0
- package/dist/tools/utils/inline-content.d.ts.map +1 -0
- package/dist/tools/utils/inline-content.js +39 -0
- package/dist/tools/utils/inline-content.js.map +1 -0
- package/dist/tools/utils/markdown-toc.d.ts +3 -0
- package/dist/tools/utils/markdown-toc.d.ts.map +1 -0
- package/dist/tools/utils/markdown-toc.js +35 -0
- package/dist/tools/utils/markdown-toc.js.map +1 -0
- package/dist/tools/utils/tool-response.d.ts +9 -0
- package/dist/tools/utils/tool-response.d.ts.map +1 -0
- package/dist/tools/utils/tool-response.js +19 -0
- package/dist/tools/utils/tool-response.js.map +1 -0
- package/dist/transformers/jsonl.transformer.d.ts.map +1 -1
- package/dist/transformers/jsonl.transformer.js +51 -28
- package/dist/transformers/jsonl.transformer.js.map +1 -1
- package/dist/transformers/markdown.transformer.d.ts.map +1 -1
- package/dist/transformers/markdown.transformer.js +82 -111
- package/dist/transformers/markdown.transformer.js.map +1 -1
- package/dist/utils/header-normalizer.d.ts +5 -0
- package/dist/utils/header-normalizer.d.ts.map +1 -0
- package/dist/utils/header-normalizer.js +25 -0
- package/dist/utils/header-normalizer.js.map +1 -0
- package/dist/utils/tool-error-handler.d.ts +1 -0
- package/dist/utils/tool-error-handler.d.ts.map +1 -1
- package/dist/utils/tool-error-handler.js +29 -1
- package/dist/utils/tool-error-handler.js.map +1 -1
- package/dist/utils/url-validator.d.ts +0 -3
- package/dist/utils/url-validator.d.ts.map +1 -1
- package/dist/utils/url-validator.js +98 -18
- package/dist/utils/url-validator.js.map +1 -1
- package/package.json +11 -6
package/dist/services/fetcher.js
CHANGED
|
@@ -1,273 +1,80 @@
|
|
|
1
|
-
import axios, { isCancel, } from 'axios';
|
|
2
|
-
import crypto from 'crypto';
|
|
3
|
-
import dns from 'dns';
|
|
4
|
-
import http from 'http';
|
|
5
|
-
import https from 'https';
|
|
6
|
-
import os from 'os';
|
|
7
1
|
import { config } from '../config/index.js';
|
|
8
|
-
import {
|
|
9
|
-
import {
|
|
10
|
-
import {
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
2
|
+
import { validateAndNormalizeUrl } from '../utils/url-validator.js';
|
|
3
|
+
import { destroyAgents, dispatcher } from './fetcher/agents.js';
|
|
4
|
+
import { createHttpError, createRateLimitError, mapFetchError, } from './fetcher/errors.js';
|
|
5
|
+
import { sanitizeHeaders } from './fetcher/headers.js';
|
|
6
|
+
import { recordFetchError, recordFetchResponse, startFetchTelemetry, } from './fetcher/interceptors.js';
|
|
7
|
+
import { fetchWithRedirects } from './fetcher/redirects.js';
|
|
8
|
+
import { readResponseText } from './fetcher/response.js';
|
|
9
|
+
import { RetryPolicy } from './fetcher/retry-policy.js';
|
|
10
|
+
export { destroyAgents };
|
|
11
|
+
const DEFAULT_HEADERS = {
|
|
12
|
+
'User-Agent': config.fetcher.userAgent,
|
|
13
|
+
Accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
|
|
14
|
+
'Accept-Language': 'en-US,en;q=0.5',
|
|
15
|
+
'Accept-Encoding': 'gzip, deflate, br',
|
|
16
|
+
Connection: 'keep-alive',
|
|
17
|
+
};
|
|
18
|
+
function buildHeaders(customHeaders) {
|
|
19
|
+
const headers = new Headers(DEFAULT_HEADERS);
|
|
20
|
+
const sanitized = sanitizeHeaders(customHeaders);
|
|
21
|
+
if (sanitized) {
|
|
22
|
+
for (const [key, value] of Object.entries(sanitized)) {
|
|
23
|
+
headers.set(key, value);
|
|
27
24
|
}
|
|
28
25
|
}
|
|
29
|
-
return
|
|
26
|
+
return headers;
|
|
30
27
|
}
|
|
31
|
-
function
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
}
|
|
37
|
-
const addresses = Array.isArray(address) ? address : [{ address, family }];
|
|
38
|
-
for (const addr of addresses) {
|
|
39
|
-
const ip = typeof addr === 'string' ? addr : addr.address;
|
|
40
|
-
if (isBlockedIp(ip)) {
|
|
41
|
-
const error = new Error(`Blocked IP detected for ${hostname}`);
|
|
42
|
-
error.code = 'EBLOCKED';
|
|
43
|
-
callback(error, address, family);
|
|
44
|
-
return;
|
|
45
|
-
}
|
|
46
|
-
}
|
|
47
|
-
callback(null, address, family);
|
|
48
|
-
});
|
|
28
|
+
function buildRequestSignal(timeoutMs, external) {
|
|
29
|
+
const timeoutSignal = AbortSignal.timeout(timeoutMs);
|
|
30
|
+
if (!external)
|
|
31
|
+
return timeoutSignal;
|
|
32
|
+
return AbortSignal.any([external, timeoutSignal]);
|
|
49
33
|
}
|
|
50
|
-
function
|
|
51
|
-
const cpuCount = os.cpus().length;
|
|
34
|
+
function buildRequestInit(headers, signal) {
|
|
52
35
|
return {
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
scheduling: 'fifo',
|
|
58
|
-
lookup: resolveDns,
|
|
36
|
+
method: 'GET',
|
|
37
|
+
headers,
|
|
38
|
+
signal,
|
|
39
|
+
dispatcher,
|
|
59
40
|
};
|
|
60
41
|
}
|
|
61
|
-
function
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
logDebug('HTTP Request', {
|
|
66
|
-
requestId: timedConfig[REQUEST_ID],
|
|
67
|
-
method: config.method?.toUpperCase(),
|
|
68
|
-
url: config.url,
|
|
69
|
-
});
|
|
70
|
-
return config;
|
|
71
|
-
}
|
|
72
|
-
function handleRequestError(error) {
|
|
73
|
-
logError('HTTP Request Error', error);
|
|
74
|
-
throw error;
|
|
75
|
-
}
|
|
76
|
-
function calculateDuration(config) {
|
|
77
|
-
const startTime = config[REQUEST_START_TIME];
|
|
78
|
-
return startTime ? Date.now() - startTime : 0;
|
|
79
|
-
}
|
|
80
|
-
function logResponse(response, requestId, duration) {
|
|
81
|
-
const headers = response.headers;
|
|
82
|
-
const contentType = headers['content-type'];
|
|
83
|
-
const contentTypeStr = typeof contentType === 'string' ? contentType : undefined;
|
|
84
|
-
logDebug('HTTP Response', {
|
|
85
|
-
requestId,
|
|
86
|
-
status: response.status,
|
|
87
|
-
url: response.config.url ?? 'unknown',
|
|
88
|
-
contentType: contentTypeStr,
|
|
89
|
-
duration: `${duration}ms`,
|
|
90
|
-
size: headers['content-length'],
|
|
91
|
-
});
|
|
92
|
-
if (duration > 5000) {
|
|
93
|
-
logWarn('Slow HTTP request detected', {
|
|
94
|
-
requestId,
|
|
95
|
-
url: response.config.url ?? 'unknown',
|
|
96
|
-
duration: `${duration}ms`,
|
|
97
|
-
});
|
|
42
|
+
async function handleFetchResponse(response, finalUrl, telemetry) {
|
|
43
|
+
if (response.status === 429) {
|
|
44
|
+
void response.body?.cancel();
|
|
45
|
+
throw createRateLimitError(finalUrl, response.headers.get('retry-after'));
|
|
98
46
|
}
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
return 60;
|
|
103
|
-
const parsed = typeof header === 'string' ? parseInt(header, 10) : Number(header);
|
|
104
|
-
return isNaN(parsed) ? 60 : parsed;
|
|
105
|
-
}
|
|
106
|
-
function handleResponseError(error) {
|
|
107
|
-
const url = error.config?.url ?? 'unknown';
|
|
108
|
-
if (isCancel(error) ||
|
|
109
|
-
error.name === 'AbortError' ||
|
|
110
|
-
error.name === 'CanceledError') {
|
|
111
|
-
logDebug('HTTP Request Aborted/Canceled', { url });
|
|
112
|
-
throw new FetchError('Request was canceled', url, 499, {
|
|
113
|
-
reason: 'aborted',
|
|
114
|
-
});
|
|
115
|
-
}
|
|
116
|
-
if (error.code === 'ECONNABORTED' || error.code === 'ETIMEDOUT') {
|
|
117
|
-
logError('HTTP Timeout', { url, timeout: config.fetcher.timeout });
|
|
118
|
-
throw new FetchError(`Request timeout after ${config.fetcher.timeout}ms`, url, 504, { timeout: config.fetcher.timeout });
|
|
119
|
-
}
|
|
120
|
-
if (error.response) {
|
|
121
|
-
const { status, statusText, headers } = error.response;
|
|
122
|
-
const responseHeaders = headers;
|
|
123
|
-
if (status === 429) {
|
|
124
|
-
const retryAfter = parseRetryAfter(responseHeaders['retry-after']);
|
|
125
|
-
logWarn('Rate limited by server', { url, retryAfter: `${retryAfter}s` });
|
|
126
|
-
throw new FetchError('Too many requests', url, 429, { retryAfter });
|
|
127
|
-
}
|
|
128
|
-
logError('HTTP Error Response', { url, status, statusText });
|
|
129
|
-
throw new FetchError(`HTTP ${status}: ${statusText}`, url, status);
|
|
130
|
-
}
|
|
131
|
-
if (error.request) {
|
|
132
|
-
logError('HTTP Network Error', { url, code: error.code });
|
|
133
|
-
throw new FetchError(`Network error: Could not reach ${url}`, url);
|
|
47
|
+
if (!response.ok) {
|
|
48
|
+
void response.body?.cancel();
|
|
49
|
+
throw createHttpError(finalUrl, response.status, response.statusText);
|
|
134
50
|
}
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
function handleResponse(response) {
|
|
139
|
-
const timedConfig = response.config;
|
|
140
|
-
const duration = calculateDuration(timedConfig);
|
|
141
|
-
const requestId = timedConfig[REQUEST_ID];
|
|
142
|
-
// Cleanup symbols safely
|
|
143
|
-
timedConfig[REQUEST_START_TIME] = undefined;
|
|
144
|
-
timedConfig[REQUEST_ID] = undefined;
|
|
145
|
-
logResponse(response, requestId, duration);
|
|
146
|
-
return response;
|
|
51
|
+
const { text, size } = await readResponseText(response, finalUrl, config.fetcher.maxContentLength);
|
|
52
|
+
recordFetchResponse(telemetry, response, size);
|
|
53
|
+
return text;
|
|
147
54
|
}
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
constructor(maxRetries, url) {
|
|
155
|
-
this.maxRetries = maxRetries;
|
|
156
|
-
this.url = url;
|
|
55
|
+
async function fetchWithTelemetry(normalizedUrl, requestInit, timeoutMs) {
|
|
56
|
+
const telemetry = startFetchTelemetry(normalizedUrl, 'GET');
|
|
57
|
+
try {
|
|
58
|
+
const { response, url: finalUrl } = await fetchWithRedirects(normalizedUrl, requestInit, config.fetcher.maxRedirects);
|
|
59
|
+
telemetry.url = finalUrl;
|
|
60
|
+
return await handleFetchResponse(response, finalUrl, telemetry);
|
|
157
61
|
}
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
throw new FetchError('Request was aborted before execution', this.url);
|
|
164
|
-
}
|
|
165
|
-
try {
|
|
166
|
-
return await operation();
|
|
167
|
-
}
|
|
168
|
-
catch (error) {
|
|
169
|
-
lastError = error instanceof Error ? error : new Error(String(error));
|
|
170
|
-
if (!this.shouldRetry(attempt, retries, lastError)) {
|
|
171
|
-
throw lastError;
|
|
172
|
-
}
|
|
173
|
-
await this.wait(attempt, lastError);
|
|
174
|
-
}
|
|
175
|
-
}
|
|
176
|
-
throw new FetchError(`Failed after ${retries} attempts: ${lastError.message}`, this.url);
|
|
177
|
-
}
|
|
178
|
-
shouldRetry(attempt, maxRetries, error) {
|
|
179
|
-
if (attempt >= maxRetries)
|
|
180
|
-
return false;
|
|
181
|
-
if (error instanceof FetchError) {
|
|
182
|
-
if (error.details.reason === 'aborted')
|
|
183
|
-
return false;
|
|
184
|
-
if (error.details.httpStatus === 429)
|
|
185
|
-
return true;
|
|
186
|
-
const status = error.details.httpStatus;
|
|
187
|
-
if (status && status >= 400 && status < 500)
|
|
188
|
-
return false;
|
|
189
|
-
}
|
|
190
|
-
return true;
|
|
191
|
-
}
|
|
192
|
-
async wait(attempt, error) {
|
|
193
|
-
let delay;
|
|
194
|
-
if (error instanceof FetchError && error.details.httpStatus === 429) {
|
|
195
|
-
const retryAfter = error.details.retryAfter || 60;
|
|
196
|
-
delay = Math.min(retryAfter * 1000, 30000);
|
|
197
|
-
logWarn('Rate limited, waiting before retry', {
|
|
198
|
-
url: this.url,
|
|
199
|
-
attempt,
|
|
200
|
-
waitTime: `${delay}ms`,
|
|
201
|
-
});
|
|
202
|
-
}
|
|
203
|
-
else {
|
|
204
|
-
delay = this.calculateBackoff(attempt);
|
|
205
|
-
logDebug('Retrying request', {
|
|
206
|
-
url: this.url,
|
|
207
|
-
attempt,
|
|
208
|
-
delay: `${delay}ms`,
|
|
209
|
-
});
|
|
210
|
-
}
|
|
211
|
-
await new Promise((resolve) => setTimeout(resolve, delay));
|
|
212
|
-
}
|
|
213
|
-
calculateBackoff(attempt) {
|
|
214
|
-
const exponentialDelay = Math.min(RetryPolicy.BASE_DELAY_MS * Math.pow(2, attempt - 1), RetryPolicy.MAX_DELAY_MS);
|
|
215
|
-
const jitter = exponentialDelay * RetryPolicy.JITTER_FACTOR * (Math.random() * 2 - 1);
|
|
216
|
-
return Math.round(exponentialDelay + jitter);
|
|
62
|
+
catch (error) {
|
|
63
|
+
const mapped = mapFetchError(error, normalizedUrl, timeoutMs);
|
|
64
|
+
telemetry.url = mapped.url;
|
|
65
|
+
recordFetchError(telemetry, mapped, mapped.statusCode);
|
|
66
|
+
throw mapped;
|
|
217
67
|
}
|
|
218
68
|
}
|
|
219
|
-
const httpAgent = new http.Agent(getAgentOptions());
|
|
220
|
-
const httpsAgent = new https.Agent(getAgentOptions());
|
|
221
|
-
const client = axios.create({
|
|
222
|
-
timeout: config.fetcher.timeout,
|
|
223
|
-
maxRedirects: config.fetcher.maxRedirects,
|
|
224
|
-
maxContentLength: config.fetcher.maxContentLength,
|
|
225
|
-
httpAgent,
|
|
226
|
-
httpsAgent,
|
|
227
|
-
headers: {
|
|
228
|
-
'User-Agent': config.fetcher.userAgent,
|
|
229
|
-
Accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
|
|
230
|
-
'Accept-Language': 'en-US,en;q=0.5',
|
|
231
|
-
'Accept-Encoding': 'gzip, deflate, br',
|
|
232
|
-
Connection: 'keep-alive',
|
|
233
|
-
},
|
|
234
|
-
validateStatus: (status) => status >= 200 && status < 300,
|
|
235
|
-
});
|
|
236
|
-
client.interceptors.request.use(handleRequest, handleRequestError);
|
|
237
|
-
client.interceptors.response.use(handleResponse, handleResponseError);
|
|
238
|
-
export function destroyAgents() {
|
|
239
|
-
httpAgent.destroy();
|
|
240
|
-
httpsAgent.destroy();
|
|
241
|
-
}
|
|
242
69
|
export async function fetchUrlWithRetry(url, options, maxRetries = 3) {
|
|
243
|
-
const
|
|
70
|
+
const normalizedUrl = validateAndNormalizeUrl(url);
|
|
71
|
+
const policy = new RetryPolicy(maxRetries, normalizedUrl);
|
|
72
|
+
const timeoutMs = options?.timeout ?? config.fetcher.timeout;
|
|
73
|
+
const headers = buildHeaders(options?.customHeaders);
|
|
244
74
|
return policy.execute(async () => {
|
|
245
|
-
const
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
responseType: 'text',
|
|
249
|
-
};
|
|
250
|
-
if (options?.signal) {
|
|
251
|
-
requestConfig.signal = options.signal;
|
|
252
|
-
}
|
|
253
|
-
else if (options?.timeout) {
|
|
254
|
-
requestConfig.signal = AbortSignal.timeout(options.timeout);
|
|
255
|
-
}
|
|
256
|
-
const sanitizedHeaders = sanitizeHeaders(options?.customHeaders);
|
|
257
|
-
if (sanitizedHeaders) {
|
|
258
|
-
const existingHeaders = (requestConfig.headers ?? {});
|
|
259
|
-
requestConfig.headers = { ...existingHeaders, ...sanitizedHeaders };
|
|
260
|
-
}
|
|
261
|
-
try {
|
|
262
|
-
const response = await client.request(requestConfig);
|
|
263
|
-
return response.data;
|
|
264
|
-
}
|
|
265
|
-
catch (error) {
|
|
266
|
-
if (error instanceof FetchError) {
|
|
267
|
-
throw error;
|
|
268
|
-
}
|
|
269
|
-
throw new FetchError(`Unexpected error: ${error instanceof Error ? error.message : 'Unknown'}`, url);
|
|
270
|
-
}
|
|
75
|
+
const signal = buildRequestSignal(timeoutMs, options?.signal);
|
|
76
|
+
const requestInit = buildRequestInit(headers, signal);
|
|
77
|
+
return fetchWithTelemetry(normalizedUrl, requestInit, timeoutMs);
|
|
271
78
|
}, options?.signal);
|
|
272
79
|
}
|
|
273
80
|
//# sourceMappingURL=fetcher.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"fetcher.js","sourceRoot":"","sources":["../../src/services/fetcher.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,EAOZ,QAAQ,GAET,MAAM,OAAO,CAAC;AACf,OAAO,MAAM,MAAM,QAAQ,CAAC;AAC5B,OAAO,GAAG,MAAM,KAAK,CAAC;AACtB,OAAO,IAAI,MAAM,MAAM,CAAC;AACxB,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,EAAE,MAAM,IAAI,CAAC;AAEpB,OAAO,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAG5C,OAAO,EAAE,UAAU,EAAE,MAAM,wBAAwB,CAAC;AAEpD,OAAO,EAAE,WAAW,EAAE,MAAM,2BAA2B,CAAC;AAExD,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,aAAa,CAAC;AAE1D,4BAA4B;AAE5B,MAAM,kBAAkB,GAAG,MAAM,CAAC,kBAAkB,CAAC,CAAC;AACtD,MAAM,UAAU,GAAG,MAAM,CAAC,WAAW,CAAC,CAAC;AAOvC,2BAA2B;AAE3B,SAAS,eAAe,CACtB,OAAgC;IAEhC,IAAI,CAAC,OAAO,IAAI,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAClD,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,MAAM,EAAE,cAAc,EAAE,GAAG,MAAM,CAAC,QAAQ,CAAC;IAC3C,MAAM,SAAS,GAAG,QAAQ,CAAC;IAC3B,MAAM,SAAS,GAA2B,EAAE,CAAC;IAE7C,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;QACnD,IACE,CAAC,cAAc,CAAC,GAAG,CAAC,GAAG,CAAC,WAAW,EAAE,CAAC;YACtC,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC;YACpB,CAAC,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,EACtB,CAAC;YACD,SAAS,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC;QACzB,CAAC;IACH,CAAC;IAED,OAAO,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC;AACnE,CAAC;AAED,SAAS,UAAU,CACjB,QAAgB,EAChB,OAA0B,EAC1B,QAIS;IAET,GAAG,CAAC,MAAM,CAAC,QAAQ,EAAE,OAAO,EAAE,CAAC,GAAG,EAAE,OAAO,EAAE,MAAM,EAAE,EAAE;QACrD,IAAI,GAAG,EAAE,CAAC;YACR,QAAQ,CAAC,GAAG,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC;YAC/B,OAAO;QACT,CAAC;QAED,MAAM,SAAS,GAAG,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,CAAC;QAE3E,KAAK,MAAM,IAAI,IAAI,SAAS,EAAE,CAAC;YAC7B,MAAM,EAAE,GAAG,OAAO,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC;YAC1D,IAAI,WAAW,CAAC,EAAE,CAAC,EAAE,CAAC;gBACpB,MAAM,KAAK,GAAG,IAAI,KAAK,CACrB,2BAA2B,QAAQ,EAAE,CACb,CAAC;gBAC3B,KAAK,CAAC,IAAI,GAAG,UAAU,CAAC;gBACxB,QAAQ,CAAC,KAAK,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC;gBACjC,OAAO;YACT,CAAC;QACH,CAAC;QAED,QAAQ,CAAC,IAAI,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC;IAClC,CAAC,CAAC,CAAC;AACL,CAAC;AAED,SAAS,eAAe;IACtB,MAAM,QAAQ,GAAG,EAAE,CAAC,IAAI,EAAE,CAAC,MAAM,CAAC;IAClC,OAAO;QACL,SAAS,EAAE,IAAI;QACf,UAAU,EAAE,IAAI,CAAC,GAAG,CAAC,QAAQ,GAAG,CAAC,EAAE,EAAE,CAAC;QACtC,cAAc,EAAE,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ,GAAG,GAAG,CAAC,EAAE,EAAE,CAAC;QACxD,OAAO,EAAE,KAAK;QACd,UAAU,EAAE,MAAM;QAClB,MAAM,EAAE,UAAU;KACnB,CAAC;AACJ,CAAC;AAED,SAAS,aAAa,CACpB,MAAkC;IAElC,MAAM,WAAW,GAAG,MAAiC,CAAC;IACtD,WAAW,CAAC,kBAAkB,CAAC,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAC7C,WAAW,CAAC,UAAU,CAAC,GAAG,MAAM,CAAC,UAAU,EAAE,CAAC,SAAS,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IAE9D,QAAQ,CAAC,cAAc,EAAE;QACvB,SAAS,EAAE,WAAW,CAAC,UAAU,CAAC;QAClC,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,WAAW,EAAE;QACpC,GAAG,EAAE,MAAM,CAAC,GAAG;KAChB,CAAC,CAAC;IAEH,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,kBAAkB,CAAC,KAAiB;IAC3C,QAAQ,CAAC,oBAAoB,EAAE,KAAK,CAAC,CAAC;IACtC,MAAM,KAAK,CAAC;AACd,CAAC;AAED,SAAS,iBAAiB,CAAC,MAA+B;IACxD,MAAM,SAAS,GAAG,MAAM,CAAC,kBAAkB,CAAC,CAAC;IAC7C,OAAO,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC;AAChD,CAAC;AAED,SAAS,WAAW,CAClB,QAAuB,EACvB,SAA6B,EAC7B,QAAgB;IAEhB,MAAM,OAAO,GAAG,QAAQ,CAAC,OAEE,CAAC;IAC5B,MAAM,WAAW,GAAG,OAAO,CAAC,cAAc,CAAY,CAAC;IACvD,MAAM,cAAc,GAClB,OAAO,WAAW,KAAK,QAAQ,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,SAAS,CAAC;IAE5D,QAAQ,CAAC,eAAe,EAAE;QACxB,SAAS;QACT,MAAM,EAAE,QAAQ,CAAC,MAAM;QACvB,GAAG,EAAE,QAAQ,CAAC,MAAM,CAAC,GAAG,IAAI,SAAS;QACrC,WAAW,EAAE,cAAc;QAC3B,QAAQ,EAAE,GAAG,QAAQ,IAAI;QACzB,IAAI,EAAE,OAAO,CAAC,gBAAgB,CAAC;KAChC,CAAC,CAAC;IAEH,IAAI,QAAQ,GAAG,IAAI,EAAE,CAAC;QACpB,OAAO,CAAC,4BAA4B,EAAE;YACpC,SAAS;YACT,GAAG,EAAE,QAAQ,CAAC,MAAM,CAAC,GAAG,IAAI,SAAS;YACrC,QAAQ,EAAE,GAAG,QAAQ,IAAI;SAC1B,CAAC,CAAC;IACL,CAAC;AACH,CAAC;AAED,SAAS,eAAe,CAAC,MAAe;IACtC,IAAI,CAAC,MAAM;QAAE,OAAO,EAAE,CAAC;IACvB,MAAM,MAAM,GACV,OAAO,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;IACrE,OAAO,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC;AACrC,CAAC;AAED,SAAS,mBAAmB,CAAC,KAAiB;IAC5C,MAAM,GAAG,GAAG,KAAK,CAAC,MAAM,EAAE,GAAG,IAAI,SAAS,CAAC;IAE3C,IACE,QAAQ,CAAC,KAAK,CAAC;QACf,KAAK,CAAC,IAAI,KAAK,YAAY;QAC3B,KAAK,CAAC,IAAI,KAAK,eAAe,EAC9B,CAAC;QACD,QAAQ,CAAC,+BAA+B,EAAE,EAAE,GAAG,EAAE,CAAC,CAAC;QACnD,MAAM,IAAI,UAAU,CAAC,sBAAsB,EAAE,GAAG,EAAE,GAAG,EAAE;YACrD,MAAM,EAAE,SAAS;SAClB,CAAC,CAAC;IACL,CAAC;IAED,IAAI,KAAK,CAAC,IAAI,KAAK,cAAc,IAAI,KAAK,CAAC,IAAI,KAAK,WAAW,EAAE,CAAC;QAChE,QAAQ,CAAC,cAAc,EAAE,EAAE,GAAG,EAAE,OAAO,EAAE,MAAM,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC;QACnE,MAAM,IAAI,UAAU,CAClB,yBAAyB,MAAM,CAAC,OAAO,CAAC,OAAO,IAAI,EACnD,GAAG,EACH,GAAG,EACH,EAAE,OAAO,EAAE,MAAM,CAAC,OAAO,CAAC,OAAO,EAAE,CACpC,CAAC;IACJ,CAAC;IAED,IAAI,KAAK,CAAC,QAAQ,EAAE,CAAC;QACnB,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,OAAO,EAAE,GAAG,KAAK,CAAC,QAAQ,CAAC;QACvD,MAAM,eAAe,GAAG,OAEG,CAAC;QAE5B,IAAI,MAAM,KAAK,GAAG,EAAE,CAAC;YACnB,MAAM,UAAU,GAAG,eAAe,CAAC,eAAe,CAAC,aAAa,CAAC,CAAC,CAAC;YACnE,OAAO,CAAC,wBAAwB,EAAE,EAAE,GAAG,EAAE,UAAU,EAAE,GAAG,UAAU,GAAG,EAAE,CAAC,CAAC;YACzE,MAAM,IAAI,UAAU,CAAC,mBAAmB,EAAE,GAAG,EAAE,GAAG,EAAE,EAAE,UAAU,EAAE,CAAC,CAAC;QACtE,CAAC;QAED,QAAQ,CAAC,qBAAqB,EAAE,EAAE,GAAG,EAAE,MAAM,EAAE,UAAU,EAAE,CAAC,CAAC;QAC7D,MAAM,IAAI,UAAU,CAAC,QAAQ,MAAM,KAAK,UAAU,EAAE,EAAE,GAAG,EAAE,MAAM,CAAC,CAAC;IACrE,CAAC;IAED,IAAI,KAAK,CAAC,OAAO,EAAE,CAAC;QAClB,QAAQ,CAAC,oBAAoB,EAAE,EAAE,GAAG,EAAE,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC;QAC1D,MAAM,IAAI,UAAU,CAAC,kCAAkC,GAAG,EAAE,EAAE,GAAG,CAAC,CAAC;IACrE,CAAC;IAED,QAAQ,CAAC,oBAAoB,EAAE,EAAE,GAAG,EAAE,OAAO,EAAE,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;IAChE,MAAM,IAAI,UAAU,CAAC,KAAK,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;AAC3C,CAAC;AAED,SAAS,cAAc,CAAC,QAAuB;IAC7C,MAAM,WAAW,GAAG,QAAQ,CAAC,MAAiC,CAAC;IAC/D,MAAM,QAAQ,GAAG,iBAAiB,CAAC,WAAW,CAAC,CAAC;IAChD,MAAM,SAAS,GAAG,WAAW,CAAC,UAAU,CAAC,CAAC;IAE1C,yBAAyB;IACzB,WAAW,CAAC,kBAAkB,CAAC,GAAG,SAAS,CAAC;IAC5C,WAAW,CAAC,UAAU,CAAC,GAAG,SAAS,CAAC;IAEpC,WAAW,CAAC,QAAQ,EAAE,SAAS,EAAE,QAAQ,CAAC,CAAC;IAE3C,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED,MAAM,WAAW;IAMI;IACA;IANX,MAAM,CAAU,aAAa,GAAG,IAAI,CAAC;IACrC,MAAM,CAAU,YAAY,GAAG,KAAK,CAAC;IACrC,MAAM,CAAU,aAAa,GAAG,IAAI,CAAC;IAE7C,YACmB,UAAkB,EAClB,GAAW;QADX,eAAU,GAAV,UAAU,CAAQ;QAClB,QAAG,GAAH,GAAG,CAAQ;IAC3B,CAAC;IAEJ,KAAK,CAAC,OAAO,CACX,SAA2B,EAC3B,MAAoB;QAEpB,IAAI,SAAS,GAAU,IAAI,KAAK,CAAC,mBAAmB,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC;QAChE,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,UAAU,CAAC,EAAE,EAAE,CAAC,CAAC;QAE3D,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,IAAI,OAAO,EAAE,OAAO,EAAE,EAAE,CAAC;YACpD,IAAI,MAAM,EAAE,OAAO,EAAE,CAAC;gBACpB,MAAM,IAAI,UAAU,CAAC,sCAAsC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC;YACzE,CAAC;YAED,IAAI,CAAC;gBACH,OAAO,MAAM,SAAS,EAAE,CAAC;YAC3B,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,SAAS,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;gBAEtE,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,OAAO,EAAE,OAAO,EAAE,SAAS,CAAC,EAAE,CAAC;oBACnD,MAAM,SAAS,CAAC;gBAClB,CAAC;gBAED,MAAM,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC;YACtC,CAAC;QACH,CAAC;QAED,MAAM,IAAI,UAAU,CAClB,gBAAgB,OAAO,cAAc,SAAS,CAAC,OAAO,EAAE,EACxD,IAAI,CAAC,GAAG,CACT,CAAC;IACJ,CAAC;IAEO,WAAW,CACjB,OAAe,EACf,UAAkB,EAClB,KAAY;QAEZ,IAAI,OAAO,IAAI,UAAU;YAAE,OAAO,KAAK,CAAC;QAExC,IAAI,KAAK,YAAY,UAAU,EAAE,CAAC;YAChC,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,KAAK,SAAS;gBAAE,OAAO,KAAK,CAAC;YACrD,IAAI,KAAK,CAAC,OAAO,CAAC,UAAU,KAAK,GAAG;gBAAE,OAAO,IAAI,CAAC;YAElD,MAAM,MAAM,GAAG,KAAK,CAAC,OAAO,CAAC,UAAgC,CAAC;YAC9D,IAAI,MAAM,IAAI,MAAM,IAAI,GAAG,IAAI,MAAM,GAAG,GAAG;gBAAE,OAAO,KAAK,CAAC;QAC5D,CAAC;QAED,OAAO,IAAI,CAAC;IACd,CAAC;IAEO,KAAK,CAAC,IAAI,CAAC,OAAe,EAAE,KAAY;QAC9C,IAAI,KAAa,CAAC;QAElB,IAAI,KAAK,YAAY,UAAU,IAAI,KAAK,CAAC,OAAO,CAAC,UAAU,KAAK,GAAG,EAAE,CAAC;YACpE,MAAM,UAAU,GAAI,KAAK,CAAC,OAAO,CAAC,UAAqB,IAAI,EAAE,CAAC;YAC9D,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,UAAU,GAAG,IAAI,EAAE,KAAK,CAAC,CAAC;YAC3C,OAAO,CAAC,oCAAoC,EAAE;gBAC5C,GAAG,EAAE,IAAI,CAAC,GAAG;gBACb,OAAO;gBACP,QAAQ,EAAE,GAAG,KAAK,IAAI;aACvB,CAAC,CAAC;QACL,CAAC;aAAM,CAAC;YACN,KAAK,GAAG,IAAI,CAAC,gBAAgB,CAAC,OAAO,CAAC,CAAC;YACvC,QAAQ,CAAC,kBAAkB,EAAE;gBAC3B,GAAG,EAAE,IAAI,CAAC,GAAG;gBACb,OAAO;gBACP,KAAK,EAAE,GAAG,KAAK,IAAI;aACpB,CAAC,CAAC;QACL,CAAC;QAED,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC,CAAC;IAC7D,CAAC;IAEO,gBAAgB,CAAC,OAAe;QACtC,MAAM,gBAAgB,GAAG,IAAI,CAAC,GAAG,CAC/B,WAAW,CAAC,aAAa,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,GAAG,CAAC,CAAC,EACpD,WAAW,CAAC,YAAY,CACzB,CAAC;QACF,MAAM,MAAM,GACV,gBAAgB,GAAG,WAAW,CAAC,aAAa,GAAG,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC;QACzE,OAAO,IAAI,CAAC,KAAK,CAAC,gBAAgB,GAAG,MAAM,CAAC,CAAC;IAC/C,CAAC;;AAGH,MAAM,SAAS,GAAG,IAAI,IAAI,CAAC,KAAK,CAAC,eAAe,EAAE,CAAC,CAAC;AACpD,MAAM,UAAU,GAAG,IAAI,KAAK,CAAC,KAAK,CAAC,eAAe,EAAE,CAAC,CAAC;AAEtD,MAAM,MAAM,GAAkB,KAAK,CAAC,MAAM,CAAC;IACzC,OAAO,EAAE,MAAM,CAAC,OAAO,CAAC,OAAO;IAC/B,YAAY,EAAE,MAAM,CAAC,OAAO,CAAC,YAAY;IACzC,gBAAgB,EAAE,MAAM,CAAC,OAAO,CAAC,gBAAgB;IACjD,SAAS;IACT,UAAU;IACV,OAAO,EAAE;QACP,YAAY,EAAE,MAAM,CAAC,OAAO,CAAC,SAAS;QACtC,MAAM,EACJ,4EAA4E;QAC9E,iBAAiB,EAAE,gBAAgB;QACnC,iBAAiB,EAAE,mBAAmB;QACtC,UAAU,EAAE,YAAY;KACzB;IACD,cAAc,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,IAAI,GAAG,IAAI,MAAM,GAAG,GAAG;CAC1D,CAAC,CAAC;AAEH,MAAM,CAAC,YAAY,CAAC,OAAO,CAAC,GAAG,CAAC,aAAa,EAAE,kBAAkB,CAAC,CAAC;AACnE,MAAM,CAAC,YAAY,CAAC,QAAQ,CAAC,GAAG,CAAC,cAAc,EAAE,mBAAmB,CAAC,CAAC;AAEtE,MAAM,UAAU,aAAa;IAC3B,SAAS,CAAC,OAAO,EAAE,CAAC;IACpB,UAAU,CAAC,OAAO,EAAE,CAAC;AACvB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,iBAAiB,CACrC,GAAW,EACX,OAAsB,EACtB,UAAU,GAAG,CAAC;IAEd,MAAM,MAAM,GAAG,IAAI,WAAW,CAAC,UAAU,EAAE,GAAG,CAAC,CAAC;IAEhD,OAAO,MAAM,CAAC,OAAO,CAAC,KAAK,IAAI,EAAE;QAC/B,MAAM,aAAa,GAAuB;YACxC,MAAM,EAAE,KAAK;YACb,GAAG;YACH,YAAY,EAAE,MAAM;SACrB,CAAC;QAEF,IAAI,OAAO,EAAE,MAAM,EAAE,CAAC;YACpB,aAAa,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;QACxC,CAAC;aAAM,IAAI,OAAO,EAAE,OAAO,EAAE,CAAC;YAC5B,aAAa,CAAC,MAAM,GAAG,WAAW,CAAC,OAAO,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;QAC9D,CAAC;QAED,MAAM,gBAAgB,GAAG,eAAe,CAAC,OAAO,EAAE,aAAa,CAAC,CAAC;QACjE,IAAI,gBAAgB,EAAE,CAAC;YACrB,MAAM,eAAe,GAAG,CAAC,aAAa,CAAC,OAAO,IAAI,EAAE,CAGnD,CAAC;YACF,aAAa,CAAC,OAAO,GAAG,EAAE,GAAG,eAAe,EAAE,GAAG,gBAAgB,EAAE,CAAC;QACtE,CAAC;QAED,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,OAAO,CAAS,aAAa,CAAC,CAAC;YAC7D,OAAO,QAAQ,CAAC,IAAI,CAAC;QACvB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,KAAK,YAAY,UAAU,EAAE,CAAC;gBAChC,MAAM,KAAK,CAAC;YACd,CAAC;YACD,MAAM,IAAI,UAAU,CAClB,qBAAqB,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,SAAS,EAAE,EACzE,GAAG,CACJ,CAAC;QACJ,CAAC;IACH,CAAC,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC;AACtB,CAAC"}
|
|
1
|
+
{"version":3,"file":"fetcher.js","sourceRoot":"","sources":["../../src/services/fetcher.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAG5C,OAAO,EAAE,uBAAuB,EAAE,MAAM,2BAA2B,CAAC;AAEpE,OAAO,EAAE,aAAa,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAC;AAChE,OAAO,EACL,eAAe,EACf,oBAAoB,EACpB,aAAa,GACd,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EAAE,eAAe,EAAE,MAAM,sBAAsB,CAAC;AACvD,OAAO,EACL,gBAAgB,EAChB,mBAAmB,EACnB,mBAAmB,GACpB,MAAM,2BAA2B,CAAC;AACnC,OAAO,EAAE,kBAAkB,EAAE,MAAM,wBAAwB,CAAC;AAC5D,OAAO,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AACzD,OAAO,EAAE,WAAW,EAAE,MAAM,2BAA2B,CAAC;AAExD,OAAO,EAAE,aAAa,EAAE,CAAC;AAEzB,MAAM,eAAe,GAAG;IACtB,YAAY,EAAE,MAAM,CAAC,OAAO,CAAC,SAAS;IACtC,MAAM,EACJ,4EAA4E;IAC9E,iBAAiB,EAAE,gBAAgB;IACnC,iBAAiB,EAAE,mBAAmB;IACtC,UAAU,EAAE,YAAY;CAChB,CAAC;AAEX,SAAS,YAAY,CAAC,aAAsC;IAC1D,MAAM,OAAO,GAAG,IAAI,OAAO,CAAC,eAAe,CAAC,CAAC;IAC7C,MAAM,SAAS,GAAG,eAAe,CAAC,aAAa,CAAC,CAAC;IACjD,IAAI,SAAS,EAAE,CAAC;QACd,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,CAAC;YACrD,OAAO,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;QAC1B,CAAC;IACH,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,SAAS,kBAAkB,CACzB,SAAiB,EACjB,QAAsB;IAEtB,MAAM,aAAa,GAAG,WAAW,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;IACrD,IAAI,CAAC,QAAQ;QAAE,OAAO,aAAa,CAAC;IACpC,OAAO,WAAW,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,aAAa,CAAC,CAAC,CAAC;AACpD,CAAC;AAED,SAAS,gBAAgB,CACvB,OAAgB,EAChB,MAAmB;IAEnB,OAAO;QACL,MAAM,EAAE,KAAK;QACb,OAAO;QACP,MAAM;QACN,UAAU;KACX,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,mBAAmB,CAChC,QAAkB,EAClB,QAAgB,EAChB,SAAiD;IAEjD,IAAI,QAAQ,CAAC,MAAM,KAAK,GAAG,EAAE,CAAC;QAC5B,KAAK,QAAQ,CAAC,IAAI,EAAE,MAAM,EAAE,CAAC;QAC7B,MAAM,oBAAoB,CAAC,QAAQ,EAAE,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC,CAAC;IAC5E,CAAC;IAED,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;QACjB,KAAK,QAAQ,CAAC,IAAI,EAAE,MAAM,EAAE,CAAC;QAC7B,MAAM,eAAe,CAAC,QAAQ,EAAE,QAAQ,CAAC,MAAM,EAAE,QAAQ,CAAC,UAAU,CAAC,CAAC;IACxE,CAAC;IAED,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,GAAG,MAAM,gBAAgB,CAC3C,QAAQ,EACR,QAAQ,EACR,MAAM,CAAC,OAAO,CAAC,gBAAgB,CAChC,CAAC;IACF,mBAAmB,CAAC,SAAS,EAAE,QAAQ,EAAE,IAAI,CAAC,CAAC;IAC/C,OAAO,IAAI,CAAC;AACd,CAAC;AAED,KAAK,UAAU,kBAAkB,CAC/B,aAAqB,EACrB,WAAwB,EACxB,SAAiB;IAEjB,MAAM,SAAS,GAAG,mBAAmB,CAAC,aAAa,EAAE,KAAK,CAAC,CAAC;IAE5D,IAAI,CAAC;QACH,MAAM,EAAE,QAAQ,EAAE,GAAG,EAAE,QAAQ,EAAE,GAAG,MAAM,kBAAkB,CAC1D,aAAa,EACb,WAAW,EACX,MAAM,CAAC,OAAO,CAAC,YAAY,CAC5B,CAAC;QAEF,SAAS,CAAC,GAAG,GAAG,QAAQ,CAAC;QACzB,OAAO,MAAM,mBAAmB,CAAC,QAAQ,EAAE,QAAQ,EAAE,SAAS,CAAC,CAAC;IAClE,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,MAAM,GAAG,aAAa,CAAC,KAAK,EAAE,aAAa,EAAE,SAAS,CAAC,CAAC;QAC9D,SAAS,CAAC,GAAG,GAAG,MAAM,CAAC,GAAG,CAAC;QAC3B,gBAAgB,CAAC,SAAS,EAAE,MAAM,EAAE,MAAM,CAAC,UAAU,CAAC,CAAC;QACvD,MAAM,MAAM,CAAC;IACf,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,iBAAiB,CACrC,GAAW,EACX,OAAsB,EACtB,UAAU,GAAG,CAAC;IAEd,MAAM,aAAa,GAAG,uBAAuB,CAAC,GAAG,CAAC,CAAC;IACnD,MAAM,MAAM,GAAG,IAAI,WAAW,CAAC,UAAU,EAAE,aAAa,CAAC,CAAC;IAC1D,MAAM,SAAS,GAAG,OAAO,EAAE,OAAO,IAAI,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC;IAC7D,MAAM,OAAO,GAAG,YAAY,CAAC,OAAO,EAAE,aAAa,CAAC,CAAC;IAErD,OAAO,MAAM,CAAC,OAAO,CAAC,KAAK,IAAI,EAAE;QAC/B,MAAM,MAAM,GAAG,kBAAkB,CAAC,SAAS,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC;QAC9D,MAAM,WAAW,GAAG,gBAAgB,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;QACtD,OAAO,kBAAkB,CAAC,aAAa,EAAE,WAAW,EAAE,SAAS,CAAC,CAAC;IACnE,CAAC,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC;AACtB,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"logger.d.ts","sourceRoot":"","sources":["../../src/services/logger.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAY,WAAW,EAAE,MAAM,oBAAoB,CAAC;
|
|
1
|
+
{"version":3,"file":"logger.d.ts","sourceRoot":"","sources":["../../src/services/logger.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAY,WAAW,EAAE,MAAM,oBAAoB,CAAC;AAkChE,wBAAgB,OAAO,CAAC,OAAO,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,WAAW,GAAG,IAAI,CAIjE;AAED,wBAAgB,QAAQ,CAAC,OAAO,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,WAAW,GAAG,IAAI,CAIlE;AAED,wBAAgB,OAAO,CAAC,OAAO,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,WAAW,GAAG,IAAI,CAIjE;AAED,wBAAgB,QAAQ,CAAC,OAAO,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,KAAK,GAAG,WAAW,GAAG,IAAI,CAS3E"}
|
package/dist/services/logger.js
CHANGED
|
@@ -1,6 +1,15 @@
|
|
|
1
1
|
import { config } from '../config/index.js';
|
|
2
|
+
import { getRequestId, getSessionId } from './context.js';
|
|
2
3
|
function formatMetadata(meta) {
|
|
3
|
-
|
|
4
|
+
const requestId = getRequestId();
|
|
5
|
+
const sessionId = getSessionId();
|
|
6
|
+
const contextMeta = {};
|
|
7
|
+
if (requestId)
|
|
8
|
+
contextMeta.requestId = requestId;
|
|
9
|
+
if (sessionId)
|
|
10
|
+
contextMeta.sessionId = sessionId;
|
|
11
|
+
const merged = { ...contextMeta, ...meta };
|
|
12
|
+
return Object.keys(merged).length > 0 ? ` ${JSON.stringify(merged)}` : '';
|
|
4
13
|
}
|
|
5
14
|
function createTimestamp() {
|
|
6
15
|
return new Date().toISOString();
|
|
@@ -17,17 +26,17 @@ function shouldLog(level) {
|
|
|
17
26
|
}
|
|
18
27
|
export function logInfo(message, meta) {
|
|
19
28
|
if (shouldLog('info')) {
|
|
20
|
-
|
|
29
|
+
process.stderr.write(`${formatLogEntry('info', message, meta)}\n`);
|
|
21
30
|
}
|
|
22
31
|
}
|
|
23
32
|
export function logDebug(message, meta) {
|
|
24
33
|
if (shouldLog('debug')) {
|
|
25
|
-
|
|
34
|
+
process.stderr.write(`${formatLogEntry('debug', message, meta)}\n`);
|
|
26
35
|
}
|
|
27
36
|
}
|
|
28
37
|
export function logWarn(message, meta) {
|
|
29
38
|
if (shouldLog('warn')) {
|
|
30
|
-
|
|
39
|
+
process.stderr.write(`${formatLogEntry('warn', message, meta)}\n`);
|
|
31
40
|
}
|
|
32
41
|
}
|
|
33
42
|
export function logError(message, error) {
|
|
@@ -36,6 +45,6 @@ export function logError(message, error) {
|
|
|
36
45
|
const errorMeta = error instanceof Error
|
|
37
46
|
? { error: error.message, stack: error.stack }
|
|
38
47
|
: (error ?? {});
|
|
39
|
-
|
|
48
|
+
process.stderr.write(`${formatLogEntry('error', message, errorMeta)}\n`);
|
|
40
49
|
}
|
|
41
50
|
//# sourceMappingURL=logger.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"logger.js","sourceRoot":"","sources":["../../src/services/logger.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAG5C,SAAS,cAAc,CAAC,IAAkB;IACxC,
|
|
1
|
+
{"version":3,"file":"logger.js","sourceRoot":"","sources":["../../src/services/logger.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAG5C,OAAO,EAAE,YAAY,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAE1D,SAAS,cAAc,CAAC,IAAkB;IACxC,MAAM,SAAS,GAAG,YAAY,EAAE,CAAC;IACjC,MAAM,SAAS,GAAG,YAAY,EAAE,CAAC;IAEjC,MAAM,WAAW,GAAgB,EAAE,CAAC;IACpC,IAAI,SAAS;QAAE,WAAW,CAAC,SAAS,GAAG,SAAS,CAAC;IACjD,IAAI,SAAS;QAAE,WAAW,CAAC,SAAS,GAAG,SAAS,CAAC;IAEjD,MAAM,MAAM,GAAG,EAAE,GAAG,WAAW,EAAE,GAAG,IAAI,EAAE,CAAC;IAC3C,OAAO,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;AAC5E,CAAC;AAED,SAAS,eAAe;IACtB,OAAO,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;AAClC,CAAC;AAED,SAAS,cAAc,CACrB,KAAe,EACf,OAAe,EACf,IAAkB;IAElB,OAAO,IAAI,eAAe,EAAE,KAAK,KAAK,CAAC,WAAW,EAAE,KAAK,OAAO,GAAG,cAAc,CAAC,IAAI,CAAC,EAAE,CAAC;AAC5F,CAAC;AAED,SAAS,SAAS,CAAC,KAAe;IAChC,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,OAAO;QAAE,OAAO,KAAK,CAAC;IAC1C,IAAI,KAAK,KAAK,OAAO;QAAE,OAAO,MAAM,CAAC,OAAO,CAAC,KAAK,KAAK,OAAO,CAAC;IAC/D,OAAO,IAAI,CAAC;AACd,CAAC;AAED,MAAM,UAAU,OAAO,CAAC,OAAe,EAAE,IAAkB;IACzD,IAAI,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC;QACtB,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,cAAc,CAAC,MAAM,EAAE,OAAO,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC;IACrE,CAAC;AACH,CAAC;AAED,MAAM,UAAU,QAAQ,CAAC,OAAe,EAAE,IAAkB;IAC1D,IAAI,SAAS,CAAC,OAAO,CAAC,EAAE,CAAC;QACvB,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,cAAc,CAAC,OAAO,EAAE,OAAO,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC;IACtE,CAAC;AACH,CAAC;AAED,MAAM,UAAU,OAAO,CAAC,OAAe,EAAE,IAAkB;IACzD,IAAI,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC;QACtB,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,cAAc,CAAC,MAAM,EAAE,OAAO,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC;IACrE,CAAC;AACH,CAAC;AAED,MAAM,UAAU,QAAQ,CAAC,OAAe,EAAE,KAA2B;IACnE,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC;QAAE,OAAO;IAEhC,MAAM,SAAS,GACb,KAAK,YAAY,KAAK;QACpB,CAAC,CAAC,EAAE,KAAK,EAAE,KAAK,CAAC,OAAO,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE;QAC9C,CAAC,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC;IAEpB,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,cAAc,CAAC,OAAO,EAAE,OAAO,EAAE,SAAS,CAAC,IAAI,CAAC,CAAC;AAC3E,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"parser.d.ts","sourceRoot":"","sources":["../../src/services/parser.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EAGV,iBAAiB,EAOlB,MAAM,oBAAoB,CAAC;
|
|
1
|
+
{"version":3,"file":"parser.d.ts","sourceRoot":"","sources":["../../src/services/parser.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EAGV,iBAAiB,EAOlB,MAAM,oBAAoB,CAAC;AAc5B,wBAAgB,sBAAsB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS,CA8BvE;AAwOD,wBAAgB,SAAS,CAAC,IAAI,EAAE,MAAM,GAAG,iBAAiB,EAAE,CAS3D"}
|
package/dist/services/parser.js
CHANGED
|
@@ -4,7 +4,7 @@ import { cleanCodeBlock, cleanHeading, cleanListItems, cleanParagraph, removeInl
|
|
|
4
4
|
import { truncateHtml } from '../utils/html-truncator.js';
|
|
5
5
|
import { sanitizeText } from '../utils/sanitizer.js';
|
|
6
6
|
import { logWarn } from './logger.js';
|
|
7
|
-
function detectLanguageFromCode(code) {
|
|
7
|
+
export function detectLanguageFromCode(code) {
|
|
8
8
|
// Common language patterns for code block detection
|
|
9
9
|
const patterns = [
|
|
10
10
|
[
|
|
@@ -174,48 +174,68 @@ function parseElement($, node) {
|
|
|
174
174
|
return ELEMENT_PARSERS[tagName]($, node);
|
|
175
175
|
}
|
|
176
176
|
function filterBlocks(blocks) {
|
|
177
|
-
return blocks.filter(
|
|
178
|
-
switch (block.type) {
|
|
179
|
-
case 'paragraph':
|
|
180
|
-
case 'heading':
|
|
181
|
-
case 'code':
|
|
182
|
-
case 'blockquote':
|
|
183
|
-
return block.text.length > 0;
|
|
184
|
-
case 'list':
|
|
185
|
-
return block.items.length > 0;
|
|
186
|
-
default:
|
|
187
|
-
return true;
|
|
188
|
-
}
|
|
189
|
-
});
|
|
177
|
+
return blocks.filter(shouldIncludeBlock);
|
|
190
178
|
}
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
179
|
+
const TEXT_BLOCK_TYPES = new Set([
|
|
180
|
+
'paragraph',
|
|
181
|
+
'heading',
|
|
182
|
+
'code',
|
|
183
|
+
'blockquote',
|
|
184
|
+
]);
|
|
185
|
+
function isTextBlock(block) {
|
|
186
|
+
return TEXT_BLOCK_TYPES.has(block.type);
|
|
187
|
+
}
|
|
188
|
+
function shouldIncludeBlock(block) {
|
|
189
|
+
if (isTextBlock(block)) {
|
|
190
|
+
return block.text.length > 0;
|
|
191
|
+
}
|
|
192
|
+
if (block.type === 'list') {
|
|
193
|
+
return block.items.length > 0;
|
|
194
|
+
}
|
|
195
|
+
return true;
|
|
196
|
+
}
|
|
197
|
+
function loadHtml(html) {
|
|
195
198
|
try {
|
|
196
|
-
|
|
197
|
-
const blocks = [];
|
|
198
|
-
$('script, style, noscript, iframe, svg').remove();
|
|
199
|
-
$('body')
|
|
200
|
-
.find(CONTENT_SELECTOR)
|
|
201
|
-
.each((_, element) => {
|
|
202
|
-
try {
|
|
203
|
-
const block = parseElement($, element);
|
|
204
|
-
if (block)
|
|
205
|
-
blocks.push(block);
|
|
206
|
-
}
|
|
207
|
-
catch {
|
|
208
|
-
/* skip */
|
|
209
|
-
}
|
|
210
|
-
});
|
|
211
|
-
return filterBlocks(blocks);
|
|
199
|
+
return cheerio.load(html);
|
|
212
200
|
}
|
|
213
201
|
catch (error) {
|
|
214
202
|
logWarn('Failed to parse HTML', {
|
|
215
203
|
error: error instanceof Error ? error.message : 'Unknown error',
|
|
216
204
|
htmlLength: html.length,
|
|
217
205
|
});
|
|
218
|
-
return
|
|
206
|
+
return null;
|
|
207
|
+
}
|
|
208
|
+
}
|
|
209
|
+
function removeNoiseElements($) {
|
|
210
|
+
$('script, style, noscript, iframe, svg').remove();
|
|
211
|
+
}
|
|
212
|
+
function collectBlocks($) {
|
|
213
|
+
const blocks = [];
|
|
214
|
+
$('body')
|
|
215
|
+
.find(CONTENT_SELECTOR)
|
|
216
|
+
.each((_, element) => {
|
|
217
|
+
const block = safeParseElement($, element);
|
|
218
|
+
if (block)
|
|
219
|
+
blocks.push(block);
|
|
220
|
+
});
|
|
221
|
+
return blocks;
|
|
222
|
+
}
|
|
223
|
+
function safeParseElement($, element) {
|
|
224
|
+
try {
|
|
225
|
+
return parseElement($, element);
|
|
226
|
+
}
|
|
227
|
+
catch {
|
|
228
|
+
return null;
|
|
219
229
|
}
|
|
220
230
|
}
|
|
231
|
+
export function parseHtml(html) {
|
|
232
|
+
if (!html || typeof html !== 'string')
|
|
233
|
+
return [];
|
|
234
|
+
const processedHtml = truncateHtml(html);
|
|
235
|
+
const $ = loadHtml(processedHtml);
|
|
236
|
+
if (!$)
|
|
237
|
+
return [];
|
|
238
|
+
removeNoiseElements($);
|
|
239
|
+
return filterBlocks(collectBlocks($));
|
|
240
|
+
}
|
|
221
241
|
//# sourceMappingURL=parser.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"parser.js","sourceRoot":"","sources":["../../src/services/parser.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,OAAO,MAAM,SAAS,CAAC;AAInC,OAAO,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAa5C,OAAO,EACL,cAAc,EACd,YAAY,EACZ,cAAc,EACd,cAAc,EACd,sBAAsB,GACvB,MAAM,6BAA6B,CAAC;AACrC,OAAO,EAAE,YAAY,EAAE,MAAM,4BAA4B,CAAC;AAC1D,OAAO,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAC;AAErD,OAAO,EAAE,OAAO,EAAE,MAAM,aAAa,CAAC;AAEtC,
|
|
1
|
+
{"version":3,"file":"parser.js","sourceRoot":"","sources":["../../src/services/parser.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,OAAO,MAAM,SAAS,CAAC;AAInC,OAAO,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAa5C,OAAO,EACL,cAAc,EACd,YAAY,EACZ,cAAc,EACd,cAAc,EACd,sBAAsB,GACvB,MAAM,6BAA6B,CAAC;AACrC,OAAO,EAAE,YAAY,EAAE,MAAM,4BAA4B,CAAC;AAC1D,OAAO,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAC;AAErD,OAAO,EAAE,OAAO,EAAE,MAAM,aAAa,CAAC;AAEtC,MAAM,UAAU,sBAAsB,CAAC,IAAY;IACjD,oDAAoD;IACpD,MAAM,QAAQ,GAAgC;QAC5C;YACE,kFAAkF;YAClF,KAAK;SACN;QACD;YACE,sFAAsF;YACtF,YAAY;SACb;QACD,CAAC,kDAAkD,EAAE,MAAM,CAAC;QAC5D;YACE,+EAA+E;YAC/E,YAAY;SACb;QACD,CAAC,kDAAkD,EAAE,QAAQ,CAAC;QAC9D;YACE,iFAAiF;YACjF,MAAM;SACP;QACD,CAAC,iEAAiE,EAAE,MAAM,CAAC;QAC3E,CAAC,yDAAyD,EAAE,KAAK,CAAC;QAClE,CAAC,8DAA8D,EAAE,MAAM,CAAC;QACxE,CAAC,6CAA6C,EAAE,MAAM,CAAC;QACvD,CAAC,oBAAoB,EAAE,MAAM,CAAC;QAC9B,CAAC,0DAA0D,EAAE,KAAK,CAAC;QACnE,CAAC,gCAAgC,EAAE,IAAI,CAAC;KACzC,CAAC;IACF,OAAO,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;AAC/D,CAAC;AACD,MAAM,gBAAgB,GACpB,oFAAoF,CAAC;AACvF,SAAS,YAAY,CAAC,CAAa,EAAE,OAAgB;IACnD,MAAM,OAAO,GAAG,YAAY,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;IAChD,MAAM,IAAI,GAAG,YAAY,CAAC,OAAO,CAAC,CAAC;IACnC,IAAI,CAAC,IAAI;QAAE,OAAO,IAAI,CAAC;IAEvB,OAAO;QACL,IAAI,EAAE,SAAS;QACf,KAAK,EAAE,QAAQ,CAAC,OAAO,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC;QACjD,IAAI;KACL,CAAC;AACJ,CAAC;AACD,SAAS,cAAc,CACrB,CAAa,EACb,OAAgB;IAEhB,IAAI,OAAO,GAAG,YAAY,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;IAC9C,8DAA8D;IAC9D,OAAO,GAAG,sBAAsB,CAAC,OAAO,CAAC,CAAC;IAC1C,MAAM,IAAI,GAAG,cAAc,CAAC,OAAO,CAAC,CAAC;IACrC,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,UAAU,CAAC,kBAAkB;QAAE,OAAO,IAAI,CAAC;IAE7E,OAAO,EAAE,IAAI,EAAE,WAAW,EAAE,IAAI,EAAE,CAAC;AACrC,CAAC;AACD,SAAS,SAAS,CAAC,CAAa,EAAE,OAAgB;IAChD,MAAM,QAAQ,GAAa,EAAE,CAAC;IAC9B,CAAC,CAAC,OAAO,CAAC;SACP,IAAI,CAAC,IAAI,CAAC;SACV,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,EAAE;QACd,MAAM,IAAI,GAAG,YAAY,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;QACxC,IAAI,IAAI;YAAE,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAChC,CAAC,CAAC,CAAC;IAEL,mCAAmC;IACnC,MAAM,KAAK,GAAG,cAAc,CAAC,QAAQ,CAAC,CAAC;IACvC,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IAEpC,OAAO;QACL,IAAI,EAAE,MAAM;QACZ,OAAO,EAAE,OAAO,CAAC,OAAO,CAAC,WAAW,EAAE,KAAK,IAAI;QAC/C,KAAK;KACN,CAAC;AACJ,CAAC;AAED,SAAS,SAAS,CAAC,CAAa,EAAE,OAAgB;IAChD,MAAM,OAAO,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;IACzC,MAAM,IAAI,GAAG,cAAc,CAAC,OAAO,CAAC,CAAC;IACrC,IAAI,CAAC,IAAI;QAAE,OAAO,IAAI,CAAC;IAEvB,iDAAiD;IACjD,MAAM,SAAS,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC;IACjD,MAAM,QAAQ,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,IAAI,EAAE,CAAC;IAExD,MAAM,aAAa,GACjB,gBAAgB,CAAC,IAAI,CAAC,SAAS,CAAC;QAChC,YAAY,CAAC,IAAI,CAAC,SAAS,CAAC;QAC5B,iBAAiB,CAAC,IAAI,CAAC,SAAS,CAAC;QACjC,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IAE3B,MAAM,QAAQ,GAAG,aAAa,EAAE,CAAC,CAAC,CAAC,IAAI,sBAAsB,CAAC,IAAI,CAAC,CAAC;IAEpE,OAAO;QACL,IAAI,EAAE,MAAM;QACZ,QAAQ;QACR,IAAI;KACL,CAAC;AACJ,CAAC;AAED,SAAS,UAAU,CAAC,CAAa,EAAE,OAAgB;IACjD,MAAM,OAAO,GAAa,EAAE,CAAC;IAC7B,MAAM,IAAI,GAAe,EAAE,CAAC;IAC5B,MAAM,MAAM,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC;IAE1B,MAAM,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,EAAE;QACjD,OAAO,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;IAC7C,CAAC,CAAC,CAAC;IAEH,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACzB,MAAM;aACH,IAAI,CAAC,IAAI,CAAC;aACV,KAAK,EAAE;aACP,IAAI,CAAC,QAAQ,CAAC;aACd,IAAI,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,EAAE;YAChB,OAAO,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;QAC7C,CAAC,CAAC,CAAC;IACP,CAAC;IAED,MAAM,YAAY,GAChB,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,0BAA0B,CAAC,CAAC,CAAC,cAAc,CAAC;IAEnE,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,EAAE;QACxC,MAAM,KAAK,GAAa,EAAE,CAAC;QAC3B,CAAC,CAAC,GAAG,CAAC;aACH,IAAI,CAAC,QAAQ,CAAC;aACd,IAAI,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,EAAE;YAChB,KAAK,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;QAC3C,CAAC,CAAC,CAAC;QACL,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC;YAAE,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IACzC,CAAC,CAAC,CAAC;IAEH,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IAEnC,OAAO;QACL,IAAI,EAAE,OAAO;QACb,OAAO,EAAE,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,SAAS;QACjD,IAAI;KACL,CAAC;AACJ,CAAC;AAED,SAAS,UAAU,CAAC,CAAa,EAAE,OAAgB;IACjD,MAAM,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IACnC,IAAI,CAAC,GAAG;QAAE,OAAO,IAAI,CAAC;IAEtB,OAAO;QACL,IAAI,EAAE,OAAO;QACb,GAAG;QACH,GAAG,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,SAAS;KACzC,CAAC;AACJ,CAAC;AAED,SAAS,eAAe,CACtB,CAAa,EACb,OAAgB;IAEhB,MAAM,OAAO,GAAG,YAAY,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;IAChD,MAAM,IAAI,GAAG,cAAc,CAAC,OAAO,CAAC,CAAC;IACrC,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,UAAU,CAAC,kBAAkB;QAAE,OAAO,IAAI,CAAC;IAE7E,OAAO,EAAE,IAAI,EAAE,YAAY,EAAE,IAAI,EAAE,CAAC;AACtC,CAAC;AAED,MAAM,eAAe,GAAG;IACtB,EAAE,EAAE,YAAY;IAChB,EAAE,EAAE,YAAY;IAChB,EAAE,EAAE,YAAY;IAChB,EAAE,EAAE,YAAY;IAChB,EAAE,EAAE,YAAY;IAChB,EAAE,EAAE,YAAY;IAChB,CAAC,EAAE,cAAc;IACjB,EAAE,EAAE,SAAS;IACb,EAAE,EAAE,SAAS;IACb,GAAG,EAAE,SAAS;IACd,IAAI,EAAE,SAAS;IACf,KAAK,EAAE,UAAU;IACjB,GAAG,EAAE,UAAU;IACf,UAAU,EAAE,eAAe;CAI5B,CAAC;AAEF,SAAS,cAAc,CAAC,GAAW;IACjC,OAAO,GAAG,IAAI,eAAe,CAAC;AAChC,CAAC;AAED,SAAS,YAAY,CAAC,CAAa,EAAE,IAAa;IAChD,IAAI,CAAC,CAAC,SAAS,IAAI,IAAI,CAAC,IAAI,OAAO,IAAI,CAAC,OAAO,KAAK,QAAQ;QAAE,OAAO,IAAI,CAAC;IAE1E,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC;IAC3C,IAAI,CAAC,cAAc,CAAC,OAAO,CAAC;QAAE,OAAO,IAAI,CAAC;IAC1C,OAAO,eAAe,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC;AAC3C,CAAC;AAED,SAAS,YAAY,CAAC,MAA2B;IAC/C,OAAO,MAAM,CAAC,MAAM,CAAC,kBAAkB,CAAC,CAAC;AAC3C,CAAC;AAED,MAAM,gBAAgB,GAAG,IAAI,GAAG,CAAC;IAC/B,WAAW;IACX,SAAS;IACT,MAAM;IACN,YAAY;CACb,CAAC,CAAC;AAEH,SAAS,WAAW,CAClB,KAAwB;IAExB,OAAO,gBAAgB,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;AAC1C,CAAC;AAED,SAAS,kBAAkB,CAAC,KAAwB;IAClD,IAAI,WAAW,CAAC,KAAK,CAAC,EAAE,CAAC;QACvB,OAAO,KAAK,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC;IAC/B,CAAC;IACD,IAAI,KAAK,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;QAC1B,OAAO,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC;IAChC,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,SAAS,QAAQ,CAAC,IAAY;IAC5B,IAAI,CAAC;QACH,OAAO,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC5B,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,OAAO,CAAC,sBAAsB,EAAE;YAC9B,KAAK,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe;YAC/D,UAAU,EAAE,IAAI,CAAC,MAAM;SACxB,CAAC,CAAC;QACH,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED,SAAS,mBAAmB,CAAC,CAAa;IACxC,CAAC,CAAC,sCAAsC,CAAC,CAAC,MAAM,EAAE,CAAC;AACrD,CAAC;AAED,SAAS,aAAa,CAAC,CAAa;IAClC,MAAM,MAAM,GAAwB,EAAE,CAAC;IAEvC,CAAC,CAAC,MAAM,CAAC;SACN,IAAI,CAAC,gBAAgB,CAAC;SACtB,IAAI,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,EAAE;QACnB,MAAM,KAAK,GAAG,gBAAgB,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC;QAC3C,IAAI,KAAK;YAAE,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAChC,CAAC,CAAC,CAAC;IAEL,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,gBAAgB,CACvB,CAAa,EACb,OAAgB;IAEhB,IAAI,CAAC;QACH,OAAO,YAAY,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC;IAClC,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED,MAAM,UAAU,SAAS,CAAC,IAAY;IACpC,IAAI,CAAC,IAAI,IAAI,OAAO,IAAI,KAAK,QAAQ;QAAE,OAAO,EAAE,CAAC;IAEjD,MAAM,aAAa,GAAG,YAAY,CAAC,IAAI,CAAC,CAAC;IACzC,MAAM,CAAC,GAAG,QAAQ,CAAC,aAAa,CAAC,CAAC;IAClC,IAAI,CAAC,CAAC;QAAE,OAAO,EAAE,CAAC;IAElB,mBAAmB,CAAC,CAAC,CAAC,CAAC;IACvB,OAAO,YAAY,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,CAAC;AACxC,CAAC"}
|
|
@@ -0,0 +1,4 @@
|
|
|
1
|
+
import type { ExtractLinksOptions, LinksTransformResult, ToolResponseBase } from '../../../config/types.js';
|
|
2
|
+
export declare function resolveFilterPattern(pattern: string | undefined, url: string): RegExp | undefined | ToolResponseBase;
|
|
3
|
+
export declare function extractLinks(html: string, baseUrl: string, options: ExtractLinksOptions): LinksTransformResult;
|
|
4
|
+
//# sourceMappingURL=link-extractor.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"link-extractor.d.ts","sourceRoot":"","sources":["../../../../src/tools/handlers/fetch-links/link-extractor.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAEV,mBAAmB,EACnB,oBAAoB,EAEpB,gBAAgB,EACjB,MAAM,0BAA0B,CAAC;AA0ClC,wBAAgB,oBAAoB,CAClC,OAAO,EAAE,MAAM,GAAG,SAAS,EAC3B,GAAG,EAAE,MAAM,GACV,MAAM,GAAG,SAAS,GAAG,gBAAgB,CAavC;AA6JD,wBAAgB,YAAY,CAC1B,IAAI,EAAE,MAAM,EACZ,OAAO,EAAE,MAAM,EACf,OAAO,EAAE,mBAAmB,GAC3B,oBAAoB,CAiBtB"}
|