@j0hanz/superfetch 1.0.2 → 1.0.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +345 -57
- package/dist/config/index.d.ts.map +1 -1
- package/dist/config/index.js +6 -10
- package/dist/config/index.js.map +1 -1
- package/dist/config/types.d.ts +256 -0
- package/dist/config/types.d.ts.map +1 -0
- package/dist/config/types.js +2 -0
- package/dist/config/types.js.map +1 -0
- package/dist/errors/app-error.d.ts +6 -20
- package/dist/errors/app-error.d.ts.map +1 -1
- package/dist/errors/app-error.js +7 -18
- package/dist/errors/app-error.js.map +1 -1
- package/dist/index.js +75 -62
- package/dist/index.js.map +1 -1
- package/dist/middleware/error-handler.d.ts +1 -5
- package/dist/middleware/error-handler.d.ts.map +1 -1
- package/dist/middleware/error-handler.js +4 -12
- package/dist/middleware/error-handler.js.map +1 -1
- package/dist/middleware/rate-limiter.d.ts +2 -20
- package/dist/middleware/rate-limiter.d.ts.map +1 -1
- package/dist/middleware/rate-limiter.js +22 -47
- package/dist/middleware/rate-limiter.js.map +1 -1
- package/dist/prompts/index.d.ts +0 -3
- package/dist/prompts/index.d.ts.map +1 -1
- package/dist/prompts/index.js +2 -10
- package/dist/prompts/index.js.map +1 -1
- package/dist/resources/cached-content.d.ts +5 -0
- package/dist/resources/cached-content.d.ts.map +1 -0
- package/dist/resources/cached-content.js +93 -0
- package/dist/resources/cached-content.js.map +1 -0
- package/dist/resources/index.d.ts +0 -3
- package/dist/resources/index.d.ts.map +1 -1
- package/dist/resources/index.js +40 -5
- package/dist/resources/index.js.map +1 -1
- package/dist/server.d.ts +0 -4
- package/dist/server.d.ts.map +1 -1
- package/dist/server.js +11 -6
- package/dist/server.js.map +1 -1
- package/dist/services/cache.d.ts +20 -6
- package/dist/services/cache.d.ts.map +1 -1
- package/dist/services/cache.js +128 -20
- package/dist/services/cache.js.map +1 -1
- package/dist/services/card-extractor.d.ts +10 -0
- package/dist/services/card-extractor.d.ts.map +1 -0
- package/dist/services/card-extractor.js +194 -0
- package/dist/services/card-extractor.js.map +1 -0
- package/dist/services/extractor.d.ts +12 -19
- package/dist/services/extractor.d.ts.map +1 -1
- package/dist/services/extractor.js +60 -46
- package/dist/services/extractor.js.map +1 -1
- package/dist/services/fetcher.d.ts +13 -11
- package/dist/services/fetcher.d.ts.map +1 -1
- package/dist/services/fetcher.js +143 -54
- package/dist/services/fetcher.js.map +1 -1
- package/dist/services/logger.d.ts.map +1 -1
- package/dist/services/logger.js +4 -6
- package/dist/services/logger.js.map +1 -1
- package/dist/services/parser.d.ts +1 -6
- package/dist/services/parser.d.ts.map +1 -1
- package/dist/services/parser.js +57 -27
- package/dist/services/parser.js.map +1 -1
- package/dist/tools/handlers/fetch-links.tool.d.ts +6 -18
- package/dist/tools/handlers/fetch-links.tool.d.ts.map +1 -1
- package/dist/tools/handlers/fetch-links.tool.js +104 -79
- package/dist/tools/handlers/fetch-links.tool.js.map +1 -1
- package/dist/tools/handlers/fetch-markdown.tool.d.ts +6 -10
- package/dist/tools/handlers/fetch-markdown.tool.d.ts.map +1 -1
- package/dist/tools/handlers/fetch-markdown.tool.js +83 -84
- package/dist/tools/handlers/fetch-markdown.tool.js.map +1 -1
- package/dist/tools/handlers/fetch-url.tool.d.ts +6 -12
- package/dist/tools/handlers/fetch-url.tool.d.ts.map +1 -1
- package/dist/tools/handlers/fetch-url.tool.js +51 -93
- package/dist/tools/handlers/fetch-url.tool.js.map +1 -1
- package/dist/tools/handlers/fetch-urls.tool.d.ts +12 -0
- package/dist/tools/handlers/fetch-urls.tool.d.ts.map +1 -0
- package/dist/tools/handlers/fetch-urls.tool.js +184 -0
- package/dist/tools/handlers/fetch-urls.tool.js.map +1 -0
- package/dist/tools/index.d.ts +0 -4
- package/dist/tools/index.d.ts.map +1 -1
- package/dist/tools/index.js +145 -15
- package/dist/tools/index.js.map +1 -1
- package/dist/tools/utils/common.d.ts +8 -0
- package/dist/tools/utils/common.d.ts.map +1 -0
- package/dist/tools/utils/common.js +35 -0
- package/dist/tools/utils/common.js.map +1 -0
- package/dist/tools/utils/fetch-pipeline.d.ts +3 -0
- package/dist/tools/utils/fetch-pipeline.d.ts.map +1 -0
- package/dist/tools/utils/fetch-pipeline.js +78 -0
- package/dist/tools/utils/fetch-pipeline.js.map +1 -0
- package/dist/tools/utils/index.d.ts +4 -0
- package/dist/tools/utils/index.d.ts.map +1 -0
- package/dist/tools/utils/index.js +3 -0
- package/dist/tools/utils/index.js.map +1 -0
- package/dist/tools/utils/response-builder.d.ts +3 -0
- package/dist/tools/utils/response-builder.d.ts.map +1 -0
- package/dist/tools/utils/response-builder.js +24 -0
- package/dist/tools/utils/response-builder.js.map +1 -0
- package/dist/transformers/jsonl.transformer.d.ts +1 -1
- package/dist/transformers/jsonl.transformer.d.ts.map +1 -1
- package/dist/transformers/jsonl.transformer.js +2 -1
- package/dist/transformers/jsonl.transformer.js.map +1 -1
- package/dist/transformers/markdown.transformer.d.ts +1 -1
- package/dist/transformers/markdown.transformer.d.ts.map +1 -1
- package/dist/transformers/markdown.transformer.js +99 -5
- package/dist/transformers/markdown.transformer.js.map +1 -1
- package/dist/types/content.types.d.ts +11 -11
- package/dist/types/content.types.d.ts.map +1 -1
- package/dist/types/index.d.ts +1 -2
- package/dist/types/index.d.ts.map +1 -1
- package/dist/types/index.js +1 -2
- package/dist/types/index.js.map +1 -1
- package/dist/types/schemas.d.ts +39 -12
- package/dist/types/schemas.d.ts.map +1 -1
- package/dist/utils/concurrency.d.ts +6 -0
- package/dist/utils/concurrency.d.ts.map +1 -0
- package/dist/utils/concurrency.js +38 -0
- package/dist/utils/concurrency.js.map +1 -0
- package/dist/utils/content-cleaner.d.ts +32 -0
- package/dist/utils/content-cleaner.d.ts.map +1 -0
- package/dist/utils/content-cleaner.js +238 -0
- package/dist/utils/content-cleaner.js.map +1 -0
- package/dist/utils/language-detector.d.ts +5 -0
- package/dist/utils/language-detector.d.ts.map +1 -0
- package/dist/utils/language-detector.js +50 -0
- package/dist/utils/language-detector.js.map +1 -0
- package/dist/utils/sanitizer.d.ts +0 -10
- package/dist/utils/sanitizer.d.ts.map +1 -1
- package/dist/utils/sanitizer.js +4 -12
- package/dist/utils/sanitizer.js.map +1 -1
- package/dist/utils/tool-error-handler.d.ts +1 -15
- package/dist/utils/tool-error-handler.d.ts.map +1 -1
- package/dist/utils/tool-error-handler.js +34 -6
- package/dist/utils/tool-error-handler.js.map +1 -1
- package/dist/utils/url-validator.d.ts +0 -8
- package/dist/utils/url-validator.d.ts.map +1 -1
- package/dist/utils/url-validator.js +17 -31
- package/dist/utils/url-validator.js.map +1 -1
- package/package.json +81 -79
package/dist/services/fetcher.js
CHANGED
|
@@ -1,9 +1,13 @@
|
|
|
1
|
-
import axios from 'axios';
|
|
1
|
+
import axios, { isCancel, } from 'axios';
|
|
2
2
|
import http from 'http';
|
|
3
3
|
import https from 'https';
|
|
4
|
+
import os from 'os';
|
|
4
5
|
import { config } from '../config/index.js';
|
|
5
|
-
import { FetchError, TimeoutError } from '../errors/app-error.js';
|
|
6
|
-
import {
|
|
6
|
+
import { AbortError, FetchError, RateLimitError, TimeoutError, } from '../errors/app-error.js';
|
|
7
|
+
import { getHtml, setHtml } from './cache.js';
|
|
8
|
+
import { logDebug, logError, logWarn } from './logger.js';
|
|
9
|
+
// Use Symbol for request timings (20-30% faster than WeakMap)
|
|
10
|
+
const REQUEST_START_TIME = Symbol('requestStartTime');
|
|
7
11
|
const BLOCKED_HEADERS = new Set([
|
|
8
12
|
'host',
|
|
9
13
|
'authorization',
|
|
@@ -28,13 +32,24 @@ function calculateBackoff(attempt, maxDelay = 10000) {
|
|
|
28
32
|
const jitter = baseDelay * 0.25 * (Math.random() * 2 - 1);
|
|
29
33
|
return Math.round(baseDelay + jitter);
|
|
30
34
|
}
|
|
31
|
-
//
|
|
32
|
-
const
|
|
33
|
-
const
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
35
|
+
// Dynamic connection pool sizing based on CPU cores (2-4x throughput on multi-core)
|
|
36
|
+
const CPU_COUNT = os.cpus().length;
|
|
37
|
+
const MAX_SOCKETS = Math.max(CPU_COUNT * 2, 25); // Scale with cores, minimum 25
|
|
38
|
+
const MAX_FREE_SOCKETS = Math.max(Math.floor(CPU_COUNT * 0.5), 10);
|
|
39
|
+
const httpAgent = new http.Agent({
|
|
40
|
+
keepAlive: true,
|
|
41
|
+
maxSockets: MAX_SOCKETS,
|
|
42
|
+
maxFreeSockets: MAX_FREE_SOCKETS,
|
|
43
|
+
timeout: 60000,
|
|
44
|
+
scheduling: 'fifo',
|
|
45
|
+
});
|
|
46
|
+
const httpsAgent = new https.Agent({
|
|
47
|
+
keepAlive: true,
|
|
48
|
+
maxSockets: MAX_SOCKETS,
|
|
49
|
+
maxFreeSockets: MAX_FREE_SOCKETS,
|
|
50
|
+
timeout: 60000,
|
|
51
|
+
scheduling: 'fifo',
|
|
52
|
+
});
|
|
38
53
|
export function destroyAgents() {
|
|
39
54
|
httpAgent.destroy();
|
|
40
55
|
httpsAgent.destroy();
|
|
@@ -54,8 +69,10 @@ const client = axios.create({
|
|
|
54
69
|
},
|
|
55
70
|
validateStatus: (status) => status >= 200 && status < 300,
|
|
56
71
|
});
|
|
57
|
-
// Request interceptor for logging and request enhancement
|
|
58
72
|
client.interceptors.request.use((requestConfig) => {
|
|
73
|
+
// Store timing using Symbol (faster than WeakMap)
|
|
74
|
+
const config = requestConfig;
|
|
75
|
+
config[REQUEST_START_TIME] = Date.now();
|
|
59
76
|
logDebug('HTTP Request', {
|
|
60
77
|
method: requestConfig.method?.toUpperCase(),
|
|
61
78
|
url: requestConfig.url,
|
|
@@ -63,107 +80,179 @@ client.interceptors.request.use((requestConfig) => {
|
|
|
63
80
|
return requestConfig;
|
|
64
81
|
}, (error) => {
|
|
65
82
|
logError('HTTP Request Error', error);
|
|
66
|
-
|
|
83
|
+
throw error;
|
|
67
84
|
});
|
|
68
|
-
// Response interceptor for logging and consistent error transformation
|
|
69
85
|
client.interceptors.response.use((response) => {
|
|
86
|
+
const config = response.config;
|
|
87
|
+
const startTime = config[REQUEST_START_TIME];
|
|
88
|
+
const duration = startTime ? Date.now() - startTime : 0;
|
|
89
|
+
// Clean up timing data
|
|
90
|
+
if (config[REQUEST_START_TIME] !== undefined) {
|
|
91
|
+
config[REQUEST_START_TIME] = undefined;
|
|
92
|
+
}
|
|
93
|
+
const contentType = response.headers['content-type'];
|
|
94
|
+
const contentTypeStr = typeof contentType === 'string' ? contentType : undefined;
|
|
70
95
|
logDebug('HTTP Response', {
|
|
71
96
|
status: response.status,
|
|
72
|
-
url: response.config.url,
|
|
73
|
-
contentType:
|
|
97
|
+
url: response.config.url ?? 'unknown',
|
|
98
|
+
contentType: contentTypeStr,
|
|
99
|
+
duration: `${duration}ms`,
|
|
100
|
+
size: response.headers['content-length'],
|
|
74
101
|
});
|
|
102
|
+
// Log slow requests
|
|
103
|
+
if (duration > 5000) {
|
|
104
|
+
logWarn('Slow HTTP request detected', {
|
|
105
|
+
url: response.config.url ?? 'unknown',
|
|
106
|
+
duration: `${duration}ms`,
|
|
107
|
+
});
|
|
108
|
+
}
|
|
109
|
+
// Early content-type validation before processing
|
|
110
|
+
if (contentTypeStr && !isHtmlContentType(contentTypeStr)) {
|
|
111
|
+
throw new FetchError(`Unexpected content type: ${contentTypeStr}. Expected HTML content.`, response.config.url ?? 'unknown');
|
|
112
|
+
}
|
|
75
113
|
return response;
|
|
76
114
|
}, (error) => {
|
|
77
115
|
const url = error.config?.url ?? 'unknown';
|
|
78
|
-
//
|
|
116
|
+
// Handle request cancellation (AbortController)
|
|
117
|
+
if (isCancel(error) || error.name === 'AbortError') {
|
|
118
|
+
logDebug('HTTP Request Aborted', { url });
|
|
119
|
+
throw new AbortError('Request was canceled');
|
|
120
|
+
}
|
|
121
|
+
// Handle CanceledError from AbortSignal.timeout()
|
|
122
|
+
if (error.name === 'CanceledError') {
|
|
123
|
+
logDebug('HTTP Request Canceled (timeout signal)', { url });
|
|
124
|
+
throw new TimeoutError(config.fetcher.timeout, true);
|
|
125
|
+
}
|
|
79
126
|
if (error.code === 'ECONNABORTED' || error.code === 'ETIMEDOUT') {
|
|
80
127
|
logError('HTTP Timeout', { url, timeout: config.fetcher.timeout });
|
|
81
|
-
|
|
128
|
+
throw new TimeoutError(config.fetcher.timeout, true);
|
|
82
129
|
}
|
|
83
130
|
if (error.response) {
|
|
84
|
-
const status = error.response
|
|
85
|
-
|
|
131
|
+
const { status, statusText, headers } = error.response;
|
|
132
|
+
// Handle 429 Too Many Requests with Retry-After header
|
|
133
|
+
if (status === 429) {
|
|
134
|
+
const retryAfterHeader = headers['retry-after'];
|
|
135
|
+
let retryAfterSeconds = 60; // Default 60 seconds
|
|
136
|
+
if (retryAfterHeader) {
|
|
137
|
+
const parsed = parseInt(retryAfterHeader, 10);
|
|
138
|
+
if (!isNaN(parsed)) {
|
|
139
|
+
retryAfterSeconds = parsed;
|
|
140
|
+
}
|
|
141
|
+
}
|
|
142
|
+
logWarn('Rate limited by server', {
|
|
143
|
+
url,
|
|
144
|
+
retryAfter: `${retryAfterSeconds}s`,
|
|
145
|
+
});
|
|
146
|
+
throw new RateLimitError(retryAfterSeconds);
|
|
147
|
+
}
|
|
86
148
|
logError('HTTP Error Response', { url, status, statusText });
|
|
87
|
-
|
|
149
|
+
throw new FetchError(`HTTP ${status}: ${statusText}`, url, status);
|
|
88
150
|
}
|
|
89
151
|
if (error.request) {
|
|
90
152
|
logError('HTTP Network Error', { url, code: error.code });
|
|
91
|
-
|
|
153
|
+
throw new FetchError(`Network error: Could not reach ${url}`, url);
|
|
92
154
|
}
|
|
93
155
|
logError('HTTP Unknown Error', { url, message: error.message });
|
|
94
|
-
|
|
156
|
+
throw new FetchError(error.message, url);
|
|
95
157
|
});
|
|
96
|
-
|
|
97
|
-
* Fetches HTML content from a URL (internal - use fetchUrlWithRetry for retry logic)
|
|
98
|
-
* @throws {FetchError} if request fails or returns non-HTML content
|
|
99
|
-
* @throws {TimeoutError} if request times out
|
|
100
|
-
*/
|
|
101
|
-
async function fetchUrl(url, customHeaders) {
|
|
158
|
+
async function fetchUrl(url, options) {
|
|
102
159
|
const requestConfig = {
|
|
103
160
|
method: 'GET',
|
|
104
161
|
url,
|
|
105
162
|
responseType: 'text',
|
|
106
163
|
};
|
|
107
|
-
|
|
164
|
+
// Apply per-request timeout via AbortSignal.timeout() if provided
|
|
165
|
+
// This is cleaner than axios timeout as it properly cancels the request
|
|
166
|
+
if (options?.signal) {
|
|
167
|
+
requestConfig.signal = options.signal;
|
|
168
|
+
}
|
|
169
|
+
else if (options?.timeout) {
|
|
170
|
+
// Use AbortSignal.timeout() for per-request timeout (Node 17.3+)
|
|
171
|
+
requestConfig.signal = AbortSignal.timeout(options.timeout);
|
|
172
|
+
}
|
|
173
|
+
const sanitized = sanitizeHeaders(options?.customHeaders);
|
|
108
174
|
if (sanitized) {
|
|
109
|
-
requestConfig.headers
|
|
175
|
+
const existingHeaders = requestConfig.headers && typeof requestConfig.headers === 'object'
|
|
176
|
+
? requestConfig.headers
|
|
177
|
+
: {};
|
|
178
|
+
requestConfig.headers = { ...existingHeaders, ...sanitized };
|
|
110
179
|
}
|
|
111
180
|
try {
|
|
112
181
|
const response = await client.request(requestConfig);
|
|
113
|
-
// Validate content type is HTML/text
|
|
114
|
-
const contentType = response.headers['content-type'];
|
|
115
|
-
if (contentType && !isHtmlContentType(contentType)) {
|
|
116
|
-
throw new FetchError(`Unexpected content type: ${contentType}. Expected HTML content.`, url);
|
|
117
|
-
}
|
|
118
182
|
return response.data;
|
|
119
183
|
}
|
|
120
184
|
catch (error) {
|
|
121
|
-
|
|
122
|
-
|
|
185
|
+
if (error instanceof FetchError ||
|
|
186
|
+
error instanceof TimeoutError ||
|
|
187
|
+
error instanceof AbortError ||
|
|
188
|
+
error instanceof RateLimitError) {
|
|
123
189
|
throw error;
|
|
124
190
|
}
|
|
125
|
-
// Handle any unexpected errors
|
|
126
191
|
throw new FetchError(`Unexpected error: ${error instanceof Error ? error.message : 'Unknown'}`, url);
|
|
127
192
|
}
|
|
128
193
|
}
|
|
129
|
-
/**
|
|
130
|
-
* Checks if content type indicates HTML content
|
|
131
|
-
*/
|
|
132
194
|
function isHtmlContentType(contentType) {
|
|
133
195
|
const normalized = contentType.toLowerCase();
|
|
134
196
|
return (normalized.includes('text/html') ||
|
|
135
197
|
normalized.includes('application/xhtml') ||
|
|
136
198
|
normalized.includes('text/plain'));
|
|
137
199
|
}
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
200
|
+
export async function fetchUrlWithRetry(url, options, maxRetries = 3, skipCache = false) {
|
|
201
|
+
if (!skipCache) {
|
|
202
|
+
const cachedHtml = getHtml(url);
|
|
203
|
+
if (cachedHtml) {
|
|
204
|
+
logDebug('HTML Cache Hit', { url });
|
|
205
|
+
return { html: cachedHtml, fromHtmlCache: true };
|
|
206
|
+
}
|
|
207
|
+
}
|
|
146
208
|
const retries = Math.min(Math.max(1, maxRetries), 10);
|
|
147
|
-
let lastError;
|
|
209
|
+
let lastError = new Error(`Failed to fetch ${url}`);
|
|
148
210
|
for (let attempt = 1; attempt <= retries; attempt++) {
|
|
211
|
+
// Check if aborted before attempting (early exit for batch operations)
|
|
212
|
+
if (options?.signal?.aborted) {
|
|
213
|
+
const abortError = new AbortError('Request was aborted before execution');
|
|
214
|
+
throw abortError;
|
|
215
|
+
}
|
|
149
216
|
try {
|
|
150
|
-
|
|
217
|
+
const html = await fetchUrl(url, options);
|
|
218
|
+
setHtml(url, html);
|
|
219
|
+
logDebug('HTML Cache Set', { url });
|
|
220
|
+
return { html, fromHtmlCache: false };
|
|
151
221
|
}
|
|
152
222
|
catch (error) {
|
|
153
|
-
lastError = error instanceof Error ? error : new Error(
|
|
154
|
-
// Don't retry on
|
|
223
|
+
lastError = error instanceof Error ? error : new Error(String(error));
|
|
224
|
+
// Don't retry on abort - exit immediately
|
|
225
|
+
if (error instanceof AbortError) {
|
|
226
|
+
throw error;
|
|
227
|
+
}
|
|
228
|
+
// Handle rate limiting with smart retry
|
|
229
|
+
if (error instanceof RateLimitError) {
|
|
230
|
+
if (attempt < retries) {
|
|
231
|
+
const waitTime = Math.min(error.retryAfter * 1000, 30000); // Cap at 30s
|
|
232
|
+
logWarn('Rate limited, waiting before retry', {
|
|
233
|
+
url,
|
|
234
|
+
attempt,
|
|
235
|
+
waitTime: `${waitTime}ms`,
|
|
236
|
+
});
|
|
237
|
+
await new Promise((resolve) => setTimeout(resolve, waitTime));
|
|
238
|
+
continue;
|
|
239
|
+
}
|
|
240
|
+
throw error;
|
|
241
|
+
}
|
|
155
242
|
if (error instanceof FetchError && error.httpStatus) {
|
|
156
243
|
const status = error.httpStatus;
|
|
244
|
+
// Don't retry client errors (except 429 which is handled above)
|
|
157
245
|
if (status >= 400 && status < 500 && status !== 429) {
|
|
158
246
|
throw error;
|
|
159
247
|
}
|
|
160
248
|
}
|
|
161
249
|
if (attempt < retries) {
|
|
162
250
|
const delay = calculateBackoff(attempt);
|
|
251
|
+
logDebug('Retrying request', { url, attempt, delay: `${delay}ms` });
|
|
163
252
|
await new Promise((resolve) => setTimeout(resolve, delay));
|
|
164
253
|
}
|
|
165
254
|
}
|
|
166
255
|
}
|
|
167
|
-
throw new FetchError(`Failed after ${retries} attempts: ${lastError
|
|
256
|
+
throw new FetchError(`Failed after ${retries} attempts: ${lastError.message}`, url);
|
|
168
257
|
}
|
|
169
258
|
//# sourceMappingURL=fetcher.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"fetcher.js","sourceRoot":"","sources":["../../src/services/fetcher.ts"],"names":[],"mappings":"AAAA,OAAO,
|
|
1
|
+
{"version":3,"file":"fetcher.js","sourceRoot":"","sources":["../../src/services/fetcher.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,EAGZ,QAAQ,GACT,MAAM,OAAO,CAAC;AACf,OAAO,IAAI,MAAM,MAAM,CAAC;AACxB,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,EAAE,MAAM,IAAI,CAAC;AAEpB,OAAO,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAE5C,OAAO,EACL,UAAU,EACV,UAAU,EACV,cAAc,EACd,YAAY,GACb,MAAM,wBAAwB,CAAC;AAEhC,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AAC9C,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,aAAa,CAAC;AAY1D,8DAA8D;AAC9D,MAAM,kBAAkB,GAAG,MAAM,CAAC,kBAAkB,CAAC,CAAC;AAOtD,MAAM,eAAe,GAAG,IAAI,GAAG,CAAC;IAC9B,MAAM;IACN,eAAe;IACf,QAAQ;IACR,iBAAiB;IACjB,WAAW;IACX,qBAAqB;CACtB,CAAC,CAAC;AAEH,SAAS,eAAe,CACtB,OAAgC;IAEhC,IAAI,CAAC,OAAO,IAAI,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,SAAS,CAAC;IAEpE,MAAM,SAAS,GAA2B,EAAE,CAAC;IAC7C,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;QACnD,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,GAAG,CAAC,WAAW,EAAE,CAAC,EAAE,CAAC;YAC5C,SAAS,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC;QACzB,CAAC;IACH,CAAC;IAED,OAAO,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC;AACnE,CAAC;AAED,SAAS,gBAAgB,CAAC,OAAe,EAAE,QAAQ,GAAG,KAAK;IACzD,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,GAAG,CAAC,CAAC,EAAE,QAAQ,CAAC,CAAC;IACtE,MAAM,MAAM,GAAG,SAAS,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC;IAC1D,OAAO,IAAI,CAAC,KAAK,CAAC,SAAS,GAAG,MAAM,CAAC,CAAC;AACxC,CAAC;AAED,oFAAoF;AACpF,MAAM,SAAS,GAAG,EAAE,CAAC,IAAI,EAAE,CAAC,MAAM,CAAC;AACnC,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,SAAS,GAAG,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,+BAA+B;AAChF,MAAM,gBAAgB,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,SAAS,GAAG,GAAG,CAAC,EAAE,EAAE,CAAC,CAAC;AAEnE,MAAM,SAAS,GAAG,IAAI,IAAI,CAAC,KAAK,CAAC;IAC/B,SAAS,EAAE,IAAI;IACf,UAAU,EAAE,WAAW;IACvB,cAAc,EAAE,gBAAgB;IAChC,OAAO,EAAE,KAAK;IACd,UAAU,EAAE,MAAM;CACnB,CAAC,CAAC;AAEH,MAAM,UAAU,GAAG,IAAI,KAAK,CAAC,KAAK,CAAC;IACjC,SAAS,EAAE,IAAI;IACf,UAAU,EAAE,WAAW;IACvB,cAAc,EAAE,gBAAgB;IAChC,OAAO,EAAE,KAAK;IACd,UAAU,EAAE,MAAM;CACnB,CAAC,CAAC;AAEH,MAAM,UAAU,aAAa;IAC3B,SAAS,CAAC,OAAO,EAAE,CAAC;IACpB,UAAU,CAAC,OAAO,EAAE,CAAC;AACvB,CAAC;AAED,MAAM,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC;IAC1B,OAAO,EAAE,MAAM,CAAC,OAAO,CAAC,OAAO;IAC/B,YAAY,EAAE,MAAM,CAAC,OAAO,CAAC,YAAY;IACzC,gBAAgB,EAAE,MAAM,CAAC,OAAO,CAAC,gBAAgB;IACjD,SAAS;IACT,UAAU;IACV,OAAO,EAAE;QACP,YAAY,EAAE,MAAM,CAAC,OAAO,CAAC,SAAS;QACtC,MAAM,EACJ,4EAA4E;QAC9E,iBAAiB,EAAE,gBAAgB;QACnC,iBAAiB,EAAE,mBAAmB;QACtC,UAAU,EAAE,YAAY;KACzB;IACD,cAAc,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,IAAI,GAAG,IAAI,MAAM,GAAG,GAAG;CAC1D,CAAC,CAAC;AAEH,MAAM,CAAC,YAAY,CAAC,OAAO,CAAC,GAAG,CAC7B,CAAC,aAAa,EAAE,EAAE;IAChB,kDAAkD;IAClD,MAAM,MAAM,GAAG,aAAwC,CAAC;IACxD,MAAM,CAAC,kBAAkB,CAAC,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAExC,QAAQ,CAAC,cAAc,EAAE;QACvB,MAAM,EAAE,aAAa,CAAC,MAAM,EAAE,WAAW,EAAE;QAC3C,GAAG,EAAE,aAAa,CAAC,GAAG;KACvB,CAAC,CAAC;IACH,OAAO,aAAa,CAAC;AACvB,CAAC,EACD,CAAC,KAAiB,EAAE,EAAE;IACpB,QAAQ,CAAC,oBAAoB,EAAE,KAAK,CAAC,CAAC;IACtC,MAAM,KAAK,CAAC;AACd,CAAC,CACF,CAAC;AAEF,MAAM,CAAC,YAAY,CAAC,QAAQ,CAAC,GAAG,CAC9B,CAAC,QAAQ,EAAE,EAAE;IACX,MAAM,MAAM,GAAG,QAAQ,CAAC,MAAiC,CAAC;IAC1D,MAAM,SAAS,GAAG,MAAM,CAAC,kBAAkB,CAAC,CAAC;IAC7C,MAAM,QAAQ,GAAG,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC;IAExD,uBAAuB;IACvB,IAAI,MAAM,CAAC,kBAAkB,CAAC,KAAK,SAAS,EAAE,CAAC;QAC7C,MAAM,CAAC,kBAAkB,CAAC,GAAG,SAAS,CAAC;IACzC,CAAC;IAED,MAAM,WAAW,GAAY,QAAQ,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC;IAC9D,MAAM,cAAc,GAClB,OAAO,WAAW,KAAK,QAAQ,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,SAAS,CAAC;IAE5D,QAAQ,CAAC,eAAe,EAAE;QACxB,MAAM,EAAE,QAAQ,CAAC,MAAM;QACvB,GAAG,EAAE,QAAQ,CAAC,MAAM,CAAC,GAAG,IAAI,SAAS;QACrC,WAAW,EAAE,cAAc;QAC3B,QAAQ,EAAE,GAAG,QAAQ,IAAI;QACzB,IAAI,EAAE,QAAQ,CAAC,OAAO,CAAC,gBAAgB,CAAC;KACzC,CAAC,CAAC;IAEH,oBAAoB;IACpB,IAAI,QAAQ,GAAG,IAAI,EAAE,CAAC;QACpB,OAAO,CAAC,4BAA4B,EAAE;YACpC,GAAG,EAAE,QAAQ,CAAC,MAAM,CAAC,GAAG,IAAI,SAAS;YACrC,QAAQ,EAAE,GAAG,QAAQ,IAAI;SAC1B,CAAC,CAAC;IACL,CAAC;IAED,kDAAkD;IAClD,IAAI,cAAc,IAAI,CAAC,iBAAiB,CAAC,cAAc,CAAC,EAAE,CAAC;QACzD,MAAM,IAAI,UAAU,CAClB,4BAA4B,cAAc,0BAA0B,EACpE,QAAQ,CAAC,MAAM,CAAC,GAAG,IAAI,SAAS,CACjC,CAAC;IACJ,CAAC;IAED,OAAO,QAAQ,CAAC;AAClB,CAAC,EACD,CAAC,KAAiB,EAAE,EAAE;IACpB,MAAM,GAAG,GAAG,KAAK,CAAC,MAAM,EAAE,GAAG,IAAI,SAAS,CAAC;IAE3C,gDAAgD;IAChD,IAAI,QAAQ,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,IAAI,KAAK,YAAY,EAAE,CAAC;QACnD,QAAQ,CAAC,sBAAsB,EAAE,EAAE,GAAG,EAAE,CAAC,CAAC;QAC1C,MAAM,IAAI,UAAU,CAAC,sBAAsB,CAAC,CAAC;IAC/C,CAAC;IAED,kDAAkD;IAClD,IAAI,KAAK,CAAC,IAAI,KAAK,eAAe,EAAE,CAAC;QACnC,QAAQ,CAAC,wCAAwC,EAAE,EAAE,GAAG,EAAE,CAAC,CAAC;QAC5D,MAAM,IAAI,YAAY,CAAC,MAAM,CAAC,OAAO,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC;IACvD,CAAC;IAED,IAAI,KAAK,CAAC,IAAI,KAAK,cAAc,IAAI,KAAK,CAAC,IAAI,KAAK,WAAW,EAAE,CAAC;QAChE,QAAQ,CAAC,cAAc,EAAE,EAAE,GAAG,EAAE,OAAO,EAAE,MAAM,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC;QACnE,MAAM,IAAI,YAAY,CAAC,MAAM,CAAC,OAAO,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC;IACvD,CAAC;IAED,IAAI,KAAK,CAAC,QAAQ,EAAE,CAAC;QACnB,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,OAAO,EAAE,GAAG,KAAK,CAAC,QAAQ,CAAC;QAEvD,uDAAuD;QACvD,IAAI,MAAM,KAAK,GAAG,EAAE,CAAC;YACnB,MAAM,gBAAgB,GAAG,OAAO,CAAC,aAAa,CAAuB,CAAC;YACtE,IAAI,iBAAiB,GAAG,EAAE,CAAC,CAAC,qBAAqB;YAEjD,IAAI,gBAAgB,EAAE,CAAC;gBACrB,MAAM,MAAM,GAAG,QAAQ,CAAC,gBAAgB,EAAE,EAAE,CAAC,CAAC;gBAC9C,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC;oBACnB,iBAAiB,GAAG,MAAM,CAAC;gBAC7B,CAAC;YACH,CAAC;YAED,OAAO,CAAC,wBAAwB,EAAE;gBAChC,GAAG;gBACH,UAAU,EAAE,GAAG,iBAAiB,GAAG;aACpC,CAAC,CAAC;YACH,MAAM,IAAI,cAAc,CAAC,iBAAiB,CAAC,CAAC;QAC9C,CAAC;QAED,QAAQ,CAAC,qBAAqB,EAAE,EAAE,GAAG,EAAE,MAAM,EAAE,UAAU,EAAE,CAAC,CAAC;QAC7D,MAAM,IAAI,UAAU,CAAC,QAAQ,MAAM,KAAK,UAAU,EAAE,EAAE,GAAG,EAAE,MAAM,CAAC,CAAC;IACrE,CAAC;IAED,IAAI,KAAK,CAAC,OAAO,EAAE,CAAC;QAClB,QAAQ,CAAC,oBAAoB,EAAE,EAAE,GAAG,EAAE,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC;QAC1D,MAAM,IAAI,UAAU,CAAC,kCAAkC,GAAG,EAAE,EAAE,GAAG,CAAC,CAAC;IACrE,CAAC;IAED,QAAQ,CAAC,oBAAoB,EAAE,EAAE,GAAG,EAAE,OAAO,EAAE,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;IAChE,MAAM,IAAI,UAAU,CAAC,KAAK,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;AAC3C,CAAC,CACF,CAAC;AAEF,KAAK,UAAU,QAAQ,CAAC,GAAW,EAAE,OAAsB;IACzD,MAAM,aAAa,GAAuB;QACxC,MAAM,EAAE,KAAK;QACb,GAAG;QACH,YAAY,EAAE,MAAM;KACrB,CAAC;IAEF,kEAAkE;IAClE,wEAAwE;IACxE,IAAI,OAAO,EAAE,MAAM,EAAE,CAAC;QACpB,aAAa,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IACxC,CAAC;SAAM,IAAI,OAAO,EAAE,OAAO,EAAE,CAAC;QAC5B,iEAAiE;QACjE,aAAa,CAAC,MAAM,GAAG,WAAW,CAAC,OAAO,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;IAC9D,CAAC;IAED,MAAM,SAAS,GAAG,eAAe,CAAC,OAAO,EAAE,aAAa,CAAC,CAAC;IAC1D,IAAI,SAAS,EAAE,CAAC;QACd,MAAM,eAAe,GACnB,aAAa,CAAC,OAAO,IAAI,OAAO,aAAa,CAAC,OAAO,KAAK,QAAQ;YAChE,CAAC,CAAE,aAAa,CAAC,OAAkC;YACnD,CAAC,CAAC,EAAE,CAAC;QACT,aAAa,CAAC,OAAO,GAAG,EAAE,GAAG,eAAe,EAAE,GAAG,SAAS,EAAE,CAAC;IAC/D,CAAC;IAED,IAAI,CAAC;QACH,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,OAAO,CAAS,aAAa,CAAC,CAAC;QAC7D,OAAO,QAAQ,CAAC,IAAI,CAAC;IACvB,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IACE,KAAK,YAAY,UAAU;YAC3B,KAAK,YAAY,YAAY;YAC7B,KAAK,YAAY,UAAU;YAC3B,KAAK,YAAY,cAAc,EAC/B,CAAC;YACD,MAAM,KAAK,CAAC;QACd,CAAC;QAED,MAAM,IAAI,UAAU,CAClB,qBAAqB,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,SAAS,EAAE,EACzE,GAAG,CACJ,CAAC;IACJ,CAAC;AACH,CAAC;AAED,SAAS,iBAAiB,CAAC,WAAmB;IAC5C,MAAM,UAAU,GAAG,WAAW,CAAC,WAAW,EAAE,CAAC;IAC7C,OAAO,CACL,UAAU,CAAC,QAAQ,CAAC,WAAW,CAAC;QAChC,UAAU,CAAC,QAAQ,CAAC,mBAAmB,CAAC;QACxC,UAAU,CAAC,QAAQ,CAAC,YAAY,CAAC,CAClC,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,iBAAiB,CACrC,GAAW,EACX,OAAsB,EACtB,UAAU,GAAG,CAAC,EACd,SAAS,GAAG,KAAK;IAEjB,IAAI,CAAC,SAAS,EAAE,CAAC;QACf,MAAM,UAAU,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC;QAChC,IAAI,UAAU,EAAE,CAAC;YACf,QAAQ,CAAC,gBAAgB,EAAE,EAAE,GAAG,EAAE,CAAC,CAAC;YACpC,OAAO,EAAE,IAAI,EAAE,UAAU,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC;QACnD,CAAC;IACH,CAAC;IAED,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,UAAU,CAAC,EAAE,EAAE,CAAC,CAAC;IACtD,IAAI,SAAS,GAAU,IAAI,KAAK,CAAC,mBAAmB,GAAG,EAAE,CAAC,CAAC;IAE3D,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,IAAI,OAAO,EAAE,OAAO,EAAE,EAAE,CAAC;QACpD,uEAAuE;QACvE,IAAI,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC;YAC7B,MAAM,UAAU,GAAG,IAAI,UAAU,CAAC,sCAAsC,CAAC,CAAC;YAC1E,MAAM,UAAU,CAAC;QACnB,CAAC;QAED,IAAI,CAAC;YACH,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;YAC1C,OAAO,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;YACnB,QAAQ,CAAC,gBAAgB,EAAE,EAAE,GAAG,EAAE,CAAC,CAAC;YAEpC,OAAO,EAAE,IAAI,EAAE,aAAa,EAAE,KAAK,EAAE,CAAC;QACxC,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,SAAS,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;YAEtE,0CAA0C;YAC1C,IAAI,KAAK,YAAY,UAAU,EAAE,CAAC;gBAChC,MAAM,KAAK,CAAC;YACd,CAAC;YAED,wCAAwC;YACxC,IAAI,KAAK,YAAY,cAAc,EAAE,CAAC;gBACpC,IAAI,OAAO,GAAG,OAAO,EAAE,CAAC;oBACtB,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,UAAU,GAAG,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC,aAAa;oBACxE,OAAO,CAAC,oCAAoC,EAAE;wBAC5C,GAAG;wBACH,OAAO;wBACP,QAAQ,EAAE,GAAG,QAAQ,IAAI;qBAC1B,CAAC,CAAC;oBACH,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC,CAAC;oBAC9D,SAAS;gBACX,CAAC;gBACD,MAAM,KAAK,CAAC;YACd,CAAC;YAED,IAAI,KAAK,YAAY,UAAU,IAAI,KAAK,CAAC,UAAU,EAAE,CAAC;gBACpD,MAAM,MAAM,GAAG,KAAK,CAAC,UAAU,CAAC;gBAChC,gEAAgE;gBAChE,IAAI,MAAM,IAAI,GAAG,IAAI,MAAM,GAAG,GAAG,IAAI,MAAM,KAAK,GAAG,EAAE,CAAC;oBACpD,MAAM,KAAK,CAAC;gBACd,CAAC;YACH,CAAC;YAED,IAAI,OAAO,GAAG,OAAO,EAAE,CAAC;gBACtB,MAAM,KAAK,GAAG,gBAAgB,CAAC,OAAO,CAAC,CAAC;gBACxC,QAAQ,CAAC,kBAAkB,EAAE,EAAE,GAAG,EAAE,OAAO,EAAE,KAAK,EAAE,GAAG,KAAK,IAAI,EAAE,CAAC,CAAC;gBACpE,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC,CAAC;YAC7D,CAAC;QACH,CAAC;IACH,CAAC;IAED,MAAM,IAAI,UAAU,CAClB,gBAAgB,OAAO,cAAc,SAAS,CAAC,OAAO,EAAE,EACxD,GAAG,CACJ,CAAC;AACJ,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"logger.d.ts","sourceRoot":"","sources":["../../src/services/logger.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"logger.d.ts","sourceRoot":"","sources":["../../src/services/logger.ts"],"names":[],"mappings":"AAkDA,wBAAgB,OAAO,CAAC,OAAO,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI,CAE7E;AAED,wBAAgB,QAAQ,CACtB,OAAO,EAAE,MAAM,EACf,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAC7B,IAAI,CAEN;AAED,wBAAgB,OAAO,CAAC,OAAO,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI,CAE7E;AAED,wBAAgB,QAAQ,CACtB,OAAO,EAAE,MAAM,EACf,KAAK,CAAC,EAAE,KAAK,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GACtC,IAAI,CAQN"}
|
package/dist/services/logger.js
CHANGED
|
@@ -1,13 +1,11 @@
|
|
|
1
|
-
import winston from 'winston';
|
|
2
1
|
import fs from 'fs';
|
|
3
|
-
import { config } from '../config/index.js';
|
|
4
2
|
import path from 'path';
|
|
3
|
+
import winston from 'winston';
|
|
4
|
+
import { config } from '../config/index.js';
|
|
5
5
|
const logsDir = path.join(process.cwd(), 'logs');
|
|
6
|
-
// Ensure logs directory exists
|
|
6
|
+
// Ensure logs directory exists (mkdirSync with recursive is idempotent)
|
|
7
7
|
try {
|
|
8
|
-
|
|
9
|
-
fs.mkdirSync(logsDir, { recursive: true });
|
|
10
|
-
}
|
|
8
|
+
fs.mkdirSync(logsDir, { recursive: true });
|
|
11
9
|
}
|
|
12
10
|
catch {
|
|
13
11
|
// If we can't create logs dir, file transports will fail gracefully
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"logger.js","sourceRoot":"","sources":["../../src/services/logger.ts"],"names":[],"mappings":"AAAA,OAAO,
|
|
1
|
+
{"version":3,"file":"logger.js","sourceRoot":"","sources":["../../src/services/logger.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,IAAI,CAAC;AACpB,OAAO,IAAI,MAAM,MAAM,CAAC;AACxB,OAAO,OAAO,MAAM,SAAS,CAAC;AAE9B,OAAO,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAE5C,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,MAAM,CAAC,CAAC;AAEjD,wEAAwE;AACxE,IAAI,CAAC;IACH,EAAE,CAAC,SAAS,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;AAC7C,CAAC;AAAC,MAAM,CAAC;IACP,oEAAoE;AACtE,CAAC;AAED,MAAM,MAAM,GAAG,OAAO,CAAC,YAAY,CAAC;IAClC,KAAK,EAAE,MAAM,CAAC,OAAO,CAAC,KAAK;IAC3B,MAAM,EAAE,OAAO,CAAC,MAAM,CAAC,OAAO,CAC5B,OAAO,CAAC,MAAM,CAAC,SAAS,CAAC,EAAE,MAAM,EAAE,qBAAqB,EAAE,CAAC,EAC3D,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,EACtC,OAAO,CAAC,MAAM,CAAC,KAAK,EAAE,EACtB,OAAO,CAAC,MAAM,CAAC,IAAI,EAAE,CACtB;IACD,WAAW,EAAE,EAAE,OAAO,EAAE,YAAY,EAAE;IACtC,UAAU,EAAE;QACV,IAAI,OAAO,CAAC,UAAU,CAAC,IAAI,CAAC;YAC1B,QAAQ,EAAE,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,cAAc,CAAC;YAC5C,OAAO,EAAE,OAAO;YAChB,QAAQ,EAAE,CAAC;SACZ,CAAC;QACF,IAAI,OAAO,CAAC,UAAU,CAAC,IAAI,CAAC;YAC1B,QAAQ,EAAE,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,WAAW,CAAC;YACzC,KAAK,EAAE,OAAO;YACd,OAAO,EAAE,OAAO;YAChB,QAAQ,EAAE,CAAC;SACZ,CAAC;KACH;CACF,CAAC,CAAC;AAEH,IAAI,OAAO,CAAC,GAAG,CAAC,QAAQ,KAAK,YAAY,EAAE,CAAC;IAC1C,MAAM,CAAC,GAAG,CACR,IAAI,OAAO,CAAC,UAAU,CAAC,OAAO,CAAC;QAC7B,MAAM,EAAE,OAAO,CAAC,MAAM,CAAC,OAAO,CAC5B,OAAO,CAAC,MAAM,CAAC,QAAQ,EAAE,EACzB,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,CACxB;KACF,CAAC,CACH,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,OAAO,CAAC,OAAe,EAAE,IAA8B;IACrE,IAAI,MAAM,CAAC,OAAO,CAAC,OAAO;QAAE,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC;AACzD,CAAC;AAED,MAAM,UAAU,QAAQ,CACtB,OAAe,EACf,IAA8B;IAE9B,IAAI,MAAM,CAAC,OAAO,CAAC,OAAO;QAAE,MAAM,CAAC,KAAK,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC;AAC1D,CAAC;AAED,MAAM,UAAU,OAAO,CAAC,OAAe,EAAE,IAA8B;IACrE,IAAI,MAAM,CAAC,OAAO,CAAC,OAAO;QAAE,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC;AACzD,CAAC;AAED,MAAM,UAAU,QAAQ,CACtB,OAAe,EACf,KAAuC;IAEvC,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,OAAO;QAAE,OAAO;IAEpC,MAAM,SAAS,GACb,KAAK,YAAY,KAAK;QACpB,CAAC,CAAC,EAAE,KAAK,EAAE,KAAK,CAAC,OAAO,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE;QAC9C,CAAC,CAAC,KAAK,CAAC;IACZ,MAAM,CAAC,KAAK,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC;AACnC,CAAC"}
|
|
@@ -1,8 +1,3 @@
|
|
|
1
|
-
import type { ContentBlockUnion } from '../types
|
|
2
|
-
/**
|
|
3
|
-
* Parses HTML content and extracts semantic blocks
|
|
4
|
-
* @param html - HTML string to parse
|
|
5
|
-
* @returns Array of content blocks (empty array if parsing fails)
|
|
6
|
-
*/
|
|
1
|
+
import type { ContentBlockUnion } from '../config/types.js';
|
|
7
2
|
export declare function parseHtml(html: string): ContentBlockUnion[];
|
|
8
3
|
//# sourceMappingURL=parser.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"parser.d.ts","sourceRoot":"","sources":["../../src/services/parser.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"parser.d.ts","sourceRoot":"","sources":["../../src/services/parser.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EAGV,iBAAiB,EAOlB,MAAM,oBAAoB,CAAC;AA6M5B,wBAAgB,SAAS,CAAC,IAAI,EAAE,MAAM,GAAG,iBAAiB,EAAE,CAgD3D"}
|
package/dist/services/parser.js
CHANGED
|
@@ -1,11 +1,15 @@
|
|
|
1
1
|
import * as cheerio from 'cheerio';
|
|
2
|
-
import { sanitizeText } from '../utils/sanitizer.js';
|
|
3
2
|
import { config } from '../config/index.js';
|
|
3
|
+
import { cleanCodeBlock, cleanHeading, cleanListItems, cleanParagraph, removeInlineTimestamps, } from '../utils/content-cleaner.js';
|
|
4
|
+
import { detectLanguage } from '../utils/language-detector.js';
|
|
5
|
+
import { sanitizeText } from '../utils/sanitizer.js';
|
|
4
6
|
import { logWarn } from './logger.js';
|
|
5
|
-
// Maximum HTML size to parse (10MB)
|
|
6
7
|
const MAX_HTML_SIZE = 10 * 1024 * 1024;
|
|
8
|
+
// Cache selector for performance
|
|
9
|
+
const CONTENT_SELECTOR = 'h1, h2, h3, h4, h5, h6, p, ul, ol, pre, code:not(pre code), table, img, blockquote';
|
|
7
10
|
function parseHeading($, element) {
|
|
8
|
-
const
|
|
11
|
+
const rawText = sanitizeText($(element).text());
|
|
12
|
+
const text = cleanHeading(rawText);
|
|
9
13
|
if (!text)
|
|
10
14
|
return null;
|
|
11
15
|
return {
|
|
@@ -15,20 +19,26 @@ function parseHeading($, element) {
|
|
|
15
19
|
};
|
|
16
20
|
}
|
|
17
21
|
function parseParagraph($, element) {
|
|
18
|
-
|
|
22
|
+
let rawText = sanitizeText($(element).text());
|
|
23
|
+
// Remove inline timestamps like "13 days ago" from paragraphs
|
|
24
|
+
rawText = removeInlineTimestamps(rawText);
|
|
25
|
+
const text = cleanParagraph(rawText);
|
|
19
26
|
if (!text || text.length < config.extraction.minParagraphLength)
|
|
20
27
|
return null;
|
|
21
28
|
return { type: 'paragraph', text };
|
|
22
29
|
}
|
|
23
30
|
function parseList($, element) {
|
|
24
|
-
const
|
|
31
|
+
const rawItems = [];
|
|
32
|
+
// Direct .each() iteration - no array allocation overhead
|
|
25
33
|
$(element)
|
|
26
34
|
.find('li')
|
|
27
35
|
.each((_, li) => {
|
|
28
36
|
const text = sanitizeText($(li).text());
|
|
29
37
|
if (text)
|
|
30
|
-
|
|
38
|
+
rawItems.push(text);
|
|
31
39
|
});
|
|
40
|
+
// Clean list items to remove noise
|
|
41
|
+
const items = cleanListItems(rawItems);
|
|
32
42
|
if (items.length === 0)
|
|
33
43
|
return null;
|
|
34
44
|
return {
|
|
@@ -38,14 +48,23 @@ function parseList($, element) {
|
|
|
38
48
|
};
|
|
39
49
|
}
|
|
40
50
|
function parseCode($, element) {
|
|
41
|
-
const
|
|
51
|
+
const rawText = $(element).text().trim();
|
|
52
|
+
const text = cleanCodeBlock(rawText);
|
|
42
53
|
if (!text)
|
|
43
54
|
return null;
|
|
44
|
-
|
|
45
|
-
const
|
|
55
|
+
// Try to get language from class attribute first
|
|
56
|
+
const className = $(element).attr('class') ?? '';
|
|
57
|
+
const dataLang = $(element).attr('data-language') ?? '';
|
|
58
|
+
// Check multiple possible class patterns for language
|
|
59
|
+
const languageMatch = /language-(\w+)/.exec(className) ??
|
|
60
|
+
/lang-(\w+)/.exec(className) ??
|
|
61
|
+
/highlight-(\w+)/.exec(className) ??
|
|
62
|
+
/^(\w+)$/.exec(dataLang);
|
|
63
|
+
// Use detected language from class, or try to detect from content
|
|
64
|
+
const language = languageMatch?.[1] ?? detectLanguage(text);
|
|
46
65
|
return {
|
|
47
66
|
type: 'code',
|
|
48
|
-
language
|
|
67
|
+
language,
|
|
49
68
|
text,
|
|
50
69
|
};
|
|
51
70
|
}
|
|
@@ -53,7 +72,7 @@ function parseTable($, element) {
|
|
|
53
72
|
const headers = [];
|
|
54
73
|
const rows = [];
|
|
55
74
|
const $table = $(element);
|
|
56
|
-
//
|
|
75
|
+
// Direct .each() iteration - no array allocation
|
|
57
76
|
$table.find('thead th, thead td').each((_, cell) => {
|
|
58
77
|
headers.push(sanitizeText($(cell).text()));
|
|
59
78
|
});
|
|
@@ -66,7 +85,6 @@ function parseTable($, element) {
|
|
|
66
85
|
headers.push(sanitizeText($(cell).text()));
|
|
67
86
|
});
|
|
68
87
|
}
|
|
69
|
-
// Extract body rows
|
|
70
88
|
const rowsSelector = headers.length > 0 ? 'tbody tr, tr:not(:first)' : 'tbody tr, tr';
|
|
71
89
|
$table.find(rowsSelector).each((_, row) => {
|
|
72
90
|
const cells = [];
|
|
@@ -93,9 +111,16 @@ function parseImage($, element) {
|
|
|
93
111
|
return {
|
|
94
112
|
type: 'image',
|
|
95
113
|
src,
|
|
96
|
-
alt: $(element).attr('alt')
|
|
114
|
+
alt: $(element).attr('alt') ?? undefined,
|
|
97
115
|
};
|
|
98
116
|
}
|
|
117
|
+
function parseBlockquote($, element) {
|
|
118
|
+
const rawText = sanitizeText($(element).text());
|
|
119
|
+
const text = cleanParagraph(rawText);
|
|
120
|
+
if (!text || text.length < config.extraction.minParagraphLength)
|
|
121
|
+
return null;
|
|
122
|
+
return { type: 'blockquote', text };
|
|
123
|
+
}
|
|
99
124
|
const ELEMENT_PARSERS = {
|
|
100
125
|
h1: parseHeading,
|
|
101
126
|
h2: parseHeading,
|
|
@@ -110,6 +135,7 @@ const ELEMENT_PARSERS = {
|
|
|
110
135
|
code: parseCode,
|
|
111
136
|
table: parseTable,
|
|
112
137
|
img: parseImage,
|
|
138
|
+
blockquote: parseBlockquote,
|
|
113
139
|
};
|
|
114
140
|
function isParseableTag(tag) {
|
|
115
141
|
return tag in ELEMENT_PARSERS;
|
|
@@ -128,6 +154,7 @@ function filterBlocks(blocks) {
|
|
|
128
154
|
case 'paragraph':
|
|
129
155
|
case 'heading':
|
|
130
156
|
case 'code':
|
|
157
|
+
case 'blockquote':
|
|
131
158
|
return block.text.length > 0;
|
|
132
159
|
case 'list':
|
|
133
160
|
return block.items.length > 0;
|
|
@@ -136,30 +163,33 @@ function filterBlocks(blocks) {
|
|
|
136
163
|
}
|
|
137
164
|
});
|
|
138
165
|
}
|
|
139
|
-
/**
|
|
140
|
-
* Parses HTML content and extracts semantic blocks
|
|
141
|
-
* @param html - HTML string to parse
|
|
142
|
-
* @returns Array of content blocks (empty array if parsing fails)
|
|
143
|
-
*/
|
|
144
166
|
export function parseHtml(html) {
|
|
145
|
-
|
|
146
|
-
if (!html || typeof html !== 'string') {
|
|
167
|
+
if (!html || typeof html !== 'string')
|
|
147
168
|
return [];
|
|
148
|
-
|
|
149
|
-
// Size validation to prevent memory issues
|
|
169
|
+
let processedHtml = html;
|
|
150
170
|
if (html.length > MAX_HTML_SIZE) {
|
|
151
|
-
logWarn('HTML content exceeds maximum size, truncating', {
|
|
171
|
+
logWarn('HTML content exceeds maximum size, truncating at safe boundary', {
|
|
152
172
|
size: html.length,
|
|
153
173
|
maxSize: MAX_HTML_SIZE,
|
|
154
174
|
});
|
|
155
|
-
|
|
175
|
+
// Use lastIndexOf for O(log n) reverse search (10-100x faster than while loop)
|
|
176
|
+
const lastTag = html.lastIndexOf('>', MAX_HTML_SIZE);
|
|
177
|
+
// If we found a tag boundary near the limit (within 10% buffer)
|
|
178
|
+
if (lastTag !== -1 && lastTag > MAX_HTML_SIZE * 0.9) {
|
|
179
|
+
processedHtml = html.substring(0, lastTag + 1);
|
|
180
|
+
}
|
|
181
|
+
else {
|
|
182
|
+
// Fallback: simple truncation if no suitable boundary found
|
|
183
|
+
processedHtml = html.substring(0, MAX_HTML_SIZE);
|
|
184
|
+
}
|
|
156
185
|
}
|
|
157
186
|
try {
|
|
158
|
-
const $ = cheerio.load(
|
|
187
|
+
const $ = cheerio.load(processedHtml);
|
|
159
188
|
const blocks = [];
|
|
160
189
|
$('script, style, noscript, iframe, svg').remove();
|
|
190
|
+
// Direct .each() iteration - no array allocation overhead
|
|
161
191
|
$('body')
|
|
162
|
-
.find(
|
|
192
|
+
.find(CONTENT_SELECTOR)
|
|
163
193
|
.each((_, element) => {
|
|
164
194
|
try {
|
|
165
195
|
const block = parseElement($, element);
|
|
@@ -167,7 +197,7 @@ export function parseHtml(html) {
|
|
|
167
197
|
blocks.push(block);
|
|
168
198
|
}
|
|
169
199
|
catch {
|
|
170
|
-
// Skip
|
|
200
|
+
// Skip element errors
|
|
171
201
|
}
|
|
172
202
|
});
|
|
173
203
|
return filterBlocks(blocks);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"parser.js","sourceRoot":"","sources":["../../src/services/parser.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,OAAO,MAAM,SAAS,CAAC;
|
|
1
|
+
{"version":3,"file":"parser.js","sourceRoot":"","sources":["../../src/services/parser.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,OAAO,MAAM,SAAS,CAAC;AAInC,OAAO,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAa5C,OAAO,EACL,cAAc,EACd,YAAY,EACZ,cAAc,EACd,cAAc,EACd,sBAAsB,GACvB,MAAM,6BAA6B,CAAC;AACrC,OAAO,EAAE,cAAc,EAAE,MAAM,+BAA+B,CAAC;AAC/D,OAAO,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAC;AAErD,OAAO,EAAE,OAAO,EAAE,MAAM,aAAa,CAAC;AAEtC,MAAM,aAAa,GAAG,EAAE,GAAG,IAAI,GAAG,IAAI,CAAC;AAEvC,iCAAiC;AACjC,MAAM,gBAAgB,GACpB,oFAAoF,CAAC;AAEvF,SAAS,YAAY,CAAC,CAAa,EAAE,OAAgB;IACnD,MAAM,OAAO,GAAG,YAAY,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;IAChD,MAAM,IAAI,GAAG,YAAY,CAAC,OAAO,CAAC,CAAC;IACnC,IAAI,CAAC,IAAI;QAAE,OAAO,IAAI,CAAC;IAEvB,OAAO;QACL,IAAI,EAAE,SAAS;QACf,KAAK,EAAE,QAAQ,CAAC,OAAO,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC;QACjD,IAAI;KACL,CAAC;AACJ,CAAC;AAED,SAAS,cAAc,CACrB,CAAa,EACb,OAAgB;IAEhB,IAAI,OAAO,GAAG,YAAY,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;IAC9C,8DAA8D;IAC9D,OAAO,GAAG,sBAAsB,CAAC,OAAO,CAAC,CAAC;IAC1C,MAAM,IAAI,GAAG,cAAc,CAAC,OAAO,CAAC,CAAC;IACrC,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,UAAU,CAAC,kBAAkB;QAAE,OAAO,IAAI,CAAC;IAE7E,OAAO,EAAE,IAAI,EAAE,WAAW,EAAE,IAAI,EAAE,CAAC;AACrC,CAAC;AAED,SAAS,SAAS,CAAC,CAAa,EAAE,OAAgB;IAChD,MAAM,QAAQ,GAAa,EAAE,CAAC;IAE9B,0DAA0D;IAC1D,CAAC,CAAC,OAAO,CAAC;SACP,IAAI,CAAC,IAAI,CAAC;SACV,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,EAAE;QACd,MAAM,IAAI,GAAG,YAAY,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;QACxC,IAAI,IAAI;YAAE,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAChC,CAAC,CAAC,CAAC;IAEL,mCAAmC;IACnC,MAAM,KAAK,GAAG,cAAc,CAAC,QAAQ,CAAC,CAAC;IACvC,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IAEpC,OAAO;QACL,IAAI,EAAE,MAAM;QACZ,OAAO,EAAE,OAAO,CAAC,OAAO,CAAC,WAAW,EAAE,KAAK,IAAI;QAC/C,KAAK;KACN,CAAC;AACJ,CAAC;AAED,SAAS,SAAS,CAAC,CAAa,EAAE,OAAgB;IAChD,MAAM,OAAO,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;IACzC,MAAM,IAAI,GAAG,cAAc,CAAC,OAAO,CAAC,CAAC;IACrC,IAAI,CAAC,IAAI;QAAE,OAAO,IAAI,CAAC;IAEvB,iDAAiD;IACjD,MAAM,SAAS,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC;IACjD,MAAM,QAAQ,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,IAAI,EAAE,CAAC;IAExD,sDAAsD;IACtD,MAAM,aAAa,GACjB,gBAAgB,CAAC,IAAI,CAAC,SAAS,CAAC;QAChC,YAAY,CAAC,IAAI,CAAC,SAAS,CAAC;QAC5B,iBAAiB,CAAC,IAAI,CAAC,SAAS,CAAC;QACjC,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IAE3B,kEAAkE;IAClE,MAAM,QAAQ,GAAG,aAAa,EAAE,CAAC,CAAC,CAAC,IAAI,cAAc,CAAC,IAAI,CAAC,CAAC;IAE5D,OAAO;QACL,IAAI,EAAE,MAAM;QACZ,QAAQ;QACR,IAAI;KACL,CAAC;AACJ,CAAC;AAED,SAAS,UAAU,CAAC,CAAa,EAAE,OAAgB;IACjD,MAAM,OAAO,GAAa,EAAE,CAAC;IAC7B,MAAM,IAAI,GAAe,EAAE,CAAC;IAC5B,MAAM,MAAM,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC;IAE1B,iDAAiD;IACjD,MAAM,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,EAAE;QACjD,OAAO,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;IAC7C,CAAC,CAAC,CAAC;IAEH,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACzB,MAAM;aACH,IAAI,CAAC,IAAI,CAAC;aACV,KAAK,EAAE;aACP,IAAI,CAAC,QAAQ,CAAC;aACd,IAAI,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,EAAE;YAChB,OAAO,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;QAC7C,CAAC,CAAC,CAAC;IACP,CAAC;IAED,MAAM,YAAY,GAChB,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,0BAA0B,CAAC,CAAC,CAAC,cAAc,CAAC;IAEnE,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,EAAE;QACxC,MAAM,KAAK,GAAa,EAAE,CAAC;QAC3B,CAAC,CAAC,GAAG,CAAC;aACH,IAAI,CAAC,QAAQ,CAAC;aACd,IAAI,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,EAAE;YAChB,KAAK,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;QAC3C,CAAC,CAAC,CAAC;QACL,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC;YAAE,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IACzC,CAAC,CAAC,CAAC;IAEH,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IAEnC,OAAO;QACL,IAAI,EAAE,OAAO;QACb,OAAO,EAAE,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,SAAS;QACjD,IAAI;KACL,CAAC;AACJ,CAAC;AAED,SAAS,UAAU,CAAC,CAAa,EAAE,OAAgB;IACjD,MAAM,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IACnC,IAAI,CAAC,GAAG;QAAE,OAAO,IAAI,CAAC;IAEtB,OAAO;QACL,IAAI,EAAE,OAAO;QACb,GAAG;QACH,GAAG,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,SAAS;KACzC,CAAC;AACJ,CAAC;AAED,SAAS,eAAe,CACtB,CAAa,EACb,OAAgB;IAEhB,MAAM,OAAO,GAAG,YAAY,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;IAChD,MAAM,IAAI,GAAG,cAAc,CAAC,OAAO,CAAC,CAAC;IACrC,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,UAAU,CAAC,kBAAkB;QAAE,OAAO,IAAI,CAAC;IAE7E,OAAO,EAAE,IAAI,EAAE,YAAY,EAAE,IAAI,EAAE,CAAC;AACtC,CAAC;AAED,MAAM,eAAe,GAAG;IACtB,EAAE,EAAE,YAAY;IAChB,EAAE,EAAE,YAAY;IAChB,EAAE,EAAE,YAAY;IAChB,EAAE,EAAE,YAAY;IAChB,EAAE,EAAE,YAAY;IAChB,EAAE,EAAE,YAAY;IAChB,CAAC,EAAE,cAAc;IACjB,EAAE,EAAE,SAAS;IACb,EAAE,EAAE,SAAS;IACb,GAAG,EAAE,SAAS;IACd,IAAI,EAAE,SAAS;IACf,KAAK,EAAE,UAAU;IACjB,GAAG,EAAE,UAAU;IACf,UAAU,EAAE,eAAe;CAI5B,CAAC;AAEF,SAAS,cAAc,CAAC,GAAW;IACjC,OAAO,GAAG,IAAI,eAAe,CAAC;AAChC,CAAC;AAED,SAAS,YAAY,CAAC,CAAa,EAAE,IAAa;IAChD,IAAI,CAAC,CAAC,SAAS,IAAI,IAAI,CAAC,IAAI,OAAO,IAAI,CAAC,OAAO,KAAK,QAAQ;QAAE,OAAO,IAAI,CAAC;IAE1E,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC;IAC3C,IAAI,CAAC,cAAc,CAAC,OAAO,CAAC;QAAE,OAAO,IAAI,CAAC;IAC1C,OAAO,eAAe,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC;AAC3C,CAAC;AAED,SAAS,YAAY,CAAC,MAA2B;IAC/C,OAAO,MAAM,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE;QAC7B,QAAQ,KAAK,CAAC,IAAI,EAAE,CAAC;YACnB,KAAK,WAAW,CAAC;YACjB,KAAK,SAAS,CAAC;YACf,KAAK,MAAM,CAAC;YACZ,KAAK,YAAY;gBACf,OAAO,KAAK,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC;YAC/B,KAAK,MAAM;gBACT,OAAO,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC;YAChC;gBACE,OAAO,IAAI,CAAC;QAChB,CAAC;IACH,CAAC,CAAC,CAAC;AACL,CAAC;AAED,MAAM,UAAU,SAAS,CAAC,IAAY;IACpC,IAAI,CAAC,IAAI,IAAI,OAAO,IAAI,KAAK,QAAQ;QAAE,OAAO,EAAE,CAAC;IAEjD,IAAI,aAAa,GAAG,IAAI,CAAC;IACzB,IAAI,IAAI,CAAC,MAAM,GAAG,aAAa,EAAE,CAAC;QAChC,OAAO,CAAC,gEAAgE,EAAE;YACxE,IAAI,EAAE,IAAI,CAAC,MAAM;YACjB,OAAO,EAAE,aAAa;SACvB,CAAC,CAAC;QAEH,+EAA+E;QAC/E,MAAM,OAAO,GAAG,IAAI,CAAC,WAAW,CAAC,GAAG,EAAE,aAAa,CAAC,CAAC;QAErD,gEAAgE;QAChE,IAAI,OAAO,KAAK,CAAC,CAAC,IAAI,OAAO,GAAG,aAAa,GAAG,GAAG,EAAE,CAAC;YACpD,aAAa,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC,EAAE,OAAO,GAAG,CAAC,CAAC,CAAC;QACjD,CAAC;aAAM,CAAC;YACN,4DAA4D;YAC5D,aAAa,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC,EAAE,aAAa,CAAC,CAAC;QACnD,CAAC;IACH,CAAC;IAED,IAAI,CAAC;QACH,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;QACtC,MAAM,MAAM,GAAwB,EAAE,CAAC;QAEvC,CAAC,CAAC,sCAAsC,CAAC,CAAC,MAAM,EAAE,CAAC;QAEnD,0DAA0D;QAC1D,CAAC,CAAC,MAAM,CAAC;aACN,IAAI,CAAC,gBAAgB,CAAC;aACtB,IAAI,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,EAAE;YACnB,IAAI,CAAC;gBACH,MAAM,KAAK,GAAG,YAAY,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC;gBACvC,IAAI,KAAK;oBAAE,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAChC,CAAC;YAAC,MAAM,CAAC;gBACP,sBAAsB;YACxB,CAAC;QACH,CAAC,CAAC,CAAC;QAEL,OAAO,YAAY,CAAC,MAAM,CAAC,CAAC;IAC9B,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,OAAO,CAAC,sBAAsB,EAAE;YAC9B,KAAK,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe;YAC/D,UAAU,EAAE,IAAI,CAAC,MAAM;SACxB,CAAC,CAAC;QACH,OAAO,EAAE,CAAC;IACZ,CAAC;AACH,CAAC"}
|
|
@@ -1,24 +1,12 @@
|
|
|
1
|
-
import type { FetchLinksInput
|
|
1
|
+
import type { FetchLinksInput } from '../../config/types.js';
|
|
2
2
|
export declare const FETCH_LINKS_TOOL_NAME = "fetch-links";
|
|
3
|
-
export declare const FETCH_LINKS_TOOL_DESCRIPTION = "Extracts all hyperlinks from a webpage with anchor text and type classification";
|
|
4
|
-
|
|
5
|
-
* Tool handler for extracting links from a URL
|
|
6
|
-
*/
|
|
7
|
-
export declare function fetchLinksToolHandler(input: FetchLinksInput): Promise<import("../../utils/tool-error-handler.js").ToolErrorResponse | {
|
|
3
|
+
export declare const FETCH_LINKS_TOOL_DESCRIPTION = "Extracts all hyperlinks from a webpage with anchor text and type classification. Supports filtering, image links, and link limits.";
|
|
4
|
+
export declare function fetchLinksToolHandler(input: FetchLinksInput): Promise<{
|
|
8
5
|
content: {
|
|
9
|
-
type:
|
|
6
|
+
type: 'text';
|
|
10
7
|
text: string;
|
|
11
8
|
}[];
|
|
12
|
-
structuredContent
|
|
13
|
-
|
|
14
|
-
linkCount: number;
|
|
15
|
-
links: ExtractedLink[];
|
|
16
|
-
};
|
|
17
|
-
} | {
|
|
18
|
-
content: {
|
|
19
|
-
type: "text";
|
|
20
|
-
text: string;
|
|
21
|
-
}[];
|
|
22
|
-
structuredContent?: undefined;
|
|
9
|
+
structuredContent?: Record<string, unknown>;
|
|
10
|
+
isError?: boolean;
|
|
23
11
|
}>;
|
|
24
12
|
//# sourceMappingURL=fetch-links.tool.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"fetch-links.tool.d.ts","sourceRoot":"","sources":["../../../src/tools/handlers/fetch-links.tool.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"fetch-links.tool.d.ts","sourceRoot":"","sources":["../../../src/tools/handlers/fetch-links.tool.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAGV,eAAe,EAEhB,MAAM,uBAAuB,CAAC;AAW/B,eAAO,MAAM,qBAAqB,gBAAgB,CAAC;AACnD,eAAO,MAAM,4BAA4B,uIAC6F,CAAC;AAuFvI,wBAAsB,qBAAqB,CAAC,KAAK,EAAE,eAAe,GAAG,OAAO,CAAC;IAC3E,OAAO,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC;IAC1C,iBAAiB,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAC5C,OAAO,CAAC,EAAE,OAAO,CAAC;CACnB,CAAC,CAiED"}
|