@j0hanz/superfetch 1.0.3 → 1.0.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +615 -590
- package/dist/config/types.d.ts +5 -0
- package/dist/config/types.d.ts.map +1 -1
- package/dist/errors/app-error.d.ts +4 -0
- package/dist/errors/app-error.d.ts.map +1 -1
- package/dist/errors/app-error.js +7 -0
- package/dist/errors/app-error.js.map +1 -1
- package/dist/index.js +62 -15
- package/dist/index.js.map +1 -1
- package/dist/middleware/error-handler.d.ts.map +1 -1
- package/dist/middleware/error-handler.js +3 -1
- package/dist/middleware/error-handler.js.map +1 -1
- package/dist/middleware/rate-limiter.d.ts.map +1 -1
- package/dist/middleware/rate-limiter.js +11 -3
- package/dist/middleware/rate-limiter.js.map +1 -1
- package/dist/prompts/index.d.ts.map +1 -1
- package/dist/prompts/index.js +2 -7
- package/dist/prompts/index.js.map +1 -1
- package/dist/resources/cached-content.d.ts +5 -0
- package/dist/resources/cached-content.d.ts.map +1 -0
- package/dist/resources/cached-content.js +93 -0
- package/dist/resources/cached-content.js.map +1 -0
- package/dist/resources/index.d.ts.map +1 -1
- package/dist/resources/index.js +39 -1
- package/dist/resources/index.js.map +1 -1
- package/dist/server.d.ts.map +1 -1
- package/dist/server.js +9 -0
- package/dist/server.js.map +1 -1
- package/dist/services/cache.d.ts +11 -0
- package/dist/services/cache.d.ts.map +1 -1
- package/dist/services/cache.js +63 -6
- package/dist/services/cache.js.map +1 -1
- package/dist/services/card-extractor.d.ts.map +1 -1
- package/dist/services/card-extractor.js +11 -4
- package/dist/services/card-extractor.js.map +1 -1
- package/dist/services/extractor.d.ts +7 -1
- package/dist/services/extractor.d.ts.map +1 -1
- package/dist/services/extractor.js +12 -5
- package/dist/services/extractor.js.map +1 -1
- package/dist/services/fetcher.d.ts +10 -1
- package/dist/services/fetcher.d.ts.map +1 -1
- package/dist/services/fetcher.js +131 -36
- package/dist/services/fetcher.js.map +1 -1
- package/dist/services/parser.d.ts.map +1 -1
- package/dist/services/parser.js +38 -25
- package/dist/services/parser.js.map +1 -1
- package/dist/tools/handlers/fetch-links.tool.d.ts +5 -10
- package/dist/tools/handlers/fetch-links.tool.d.ts.map +1 -1
- package/dist/tools/handlers/fetch-links.tool.js.map +1 -1
- package/dist/tools/handlers/fetch-markdown.tool.d.ts +5 -12
- package/dist/tools/handlers/fetch-markdown.tool.d.ts.map +1 -1
- package/dist/tools/handlers/fetch-markdown.tool.js +1 -2
- package/dist/tools/handlers/fetch-markdown.tool.js.map +1 -1
- package/dist/tools/handlers/fetch-url.tool.d.ts +4 -12
- package/dist/tools/handlers/fetch-url.tool.d.ts.map +1 -1
- package/dist/tools/handlers/fetch-url.tool.js.map +1 -1
- package/dist/tools/handlers/fetch-urls.tool.d.ts +8 -1
- package/dist/tools/handlers/fetch-urls.tool.d.ts.map +1 -1
- package/dist/tools/handlers/fetch-urls.tool.js +52 -15
- package/dist/tools/handlers/fetch-urls.tool.js.map +1 -1
- package/dist/tools/utils/common.js +1 -1
- package/dist/tools/utils/common.js.map +1 -1
- package/dist/tools/utils/fetch-pipeline.d.ts.map +1 -1
- package/dist/tools/utils/fetch-pipeline.js +54 -13
- package/dist/tools/utils/fetch-pipeline.js.map +1 -1
- package/dist/transformers/markdown.transformer.d.ts.map +1 -1
- package/dist/transformers/markdown.transformer.js +7 -27
- package/dist/transformers/markdown.transformer.js.map +1 -1
- package/dist/utils/concurrency.d.ts +5 -1
- package/dist/utils/concurrency.d.ts.map +1 -1
- package/dist/utils/concurrency.js +15 -2
- package/dist/utils/concurrency.js.map +1 -1
- package/dist/utils/content-cleaner.d.ts.map +1 -1
- package/dist/utils/content-cleaner.js +106 -108
- package/dist/utils/content-cleaner.js.map +1 -1
- package/dist/utils/language-detector.d.ts +1 -1
- package/dist/utils/language-detector.d.ts.map +1 -1
- package/dist/utils/sanitizer.js +1 -1
- package/dist/utils/sanitizer.js.map +1 -1
- package/dist/utils/tool-error-handler.d.ts.map +1 -1
- package/dist/utils/tool-error-handler.js +34 -6
- package/dist/utils/tool-error-handler.js.map +1 -1
- package/package.json +81 -80
package/dist/services/fetcher.js
CHANGED
|
@@ -1,10 +1,13 @@
|
|
|
1
|
-
import axios, {} from 'axios';
|
|
1
|
+
import axios, { isCancel, } from 'axios';
|
|
2
2
|
import http from 'http';
|
|
3
3
|
import https from 'https';
|
|
4
|
+
import os from 'os';
|
|
4
5
|
import { config } from '../config/index.js';
|
|
5
|
-
import { FetchError, TimeoutError } from '../errors/app-error.js';
|
|
6
|
+
import { AbortError, FetchError, RateLimitError, TimeoutError, } from '../errors/app-error.js';
|
|
6
7
|
import { getHtml, setHtml } from './cache.js';
|
|
7
|
-
import { logDebug, logError } from './logger.js';
|
|
8
|
+
import { logDebug, logError, logWarn } from './logger.js';
|
|
9
|
+
// Use Symbol for request timings (20-30% faster than WeakMap)
|
|
10
|
+
const REQUEST_START_TIME = Symbol('requestStartTime');
|
|
8
11
|
const BLOCKED_HEADERS = new Set([
|
|
9
12
|
'host',
|
|
10
13
|
'authorization',
|
|
@@ -29,8 +32,24 @@ function calculateBackoff(attempt, maxDelay = 10000) {
|
|
|
29
32
|
const jitter = baseDelay * 0.25 * (Math.random() * 2 - 1);
|
|
30
33
|
return Math.round(baseDelay + jitter);
|
|
31
34
|
}
|
|
32
|
-
|
|
33
|
-
const
|
|
35
|
+
// Dynamic connection pool sizing based on CPU cores (2-4x throughput on multi-core)
|
|
36
|
+
const CPU_COUNT = os.cpus().length;
|
|
37
|
+
const MAX_SOCKETS = Math.max(CPU_COUNT * 2, 25); // Scale with cores, minimum 25
|
|
38
|
+
const MAX_FREE_SOCKETS = Math.max(Math.floor(CPU_COUNT * 0.5), 10);
|
|
39
|
+
const httpAgent = new http.Agent({
|
|
40
|
+
keepAlive: true,
|
|
41
|
+
maxSockets: MAX_SOCKETS,
|
|
42
|
+
maxFreeSockets: MAX_FREE_SOCKETS,
|
|
43
|
+
timeout: 60000,
|
|
44
|
+
scheduling: 'fifo',
|
|
45
|
+
});
|
|
46
|
+
const httpsAgent = new https.Agent({
|
|
47
|
+
keepAlive: true,
|
|
48
|
+
maxSockets: MAX_SOCKETS,
|
|
49
|
+
maxFreeSockets: MAX_FREE_SOCKETS,
|
|
50
|
+
timeout: 60000,
|
|
51
|
+
scheduling: 'fifo',
|
|
52
|
+
});
|
|
34
53
|
export function destroyAgents() {
|
|
35
54
|
httpAgent.destroy();
|
|
36
55
|
httpsAgent.destroy();
|
|
@@ -51,70 +70,122 @@ const client = axios.create({
|
|
|
51
70
|
validateStatus: (status) => status >= 200 && status < 300,
|
|
52
71
|
});
|
|
53
72
|
client.interceptors.request.use((requestConfig) => {
|
|
73
|
+
// Store timing using Symbol (faster than WeakMap)
|
|
74
|
+
const config = requestConfig;
|
|
75
|
+
config[REQUEST_START_TIME] = Date.now();
|
|
54
76
|
logDebug('HTTP Request', {
|
|
55
77
|
method: requestConfig.method?.toUpperCase(),
|
|
56
78
|
url: requestConfig.url,
|
|
57
79
|
});
|
|
58
80
|
return requestConfig;
|
|
59
|
-
},
|
|
81
|
+
}, (error) => {
|
|
60
82
|
logError('HTTP Request Error', error);
|
|
61
|
-
|
|
83
|
+
throw error;
|
|
62
84
|
});
|
|
63
85
|
client.interceptors.response.use((response) => {
|
|
86
|
+
const config = response.config;
|
|
87
|
+
const startTime = config[REQUEST_START_TIME];
|
|
88
|
+
const duration = startTime ? Date.now() - startTime : 0;
|
|
89
|
+
// Clean up timing data
|
|
90
|
+
if (config[REQUEST_START_TIME] !== undefined) {
|
|
91
|
+
config[REQUEST_START_TIME] = undefined;
|
|
92
|
+
}
|
|
93
|
+
const contentType = response.headers['content-type'];
|
|
94
|
+
const contentTypeStr = typeof contentType === 'string' ? contentType : undefined;
|
|
64
95
|
logDebug('HTTP Response', {
|
|
65
96
|
status: response.status,
|
|
66
|
-
url: response.config.url,
|
|
67
|
-
contentType:
|
|
97
|
+
url: response.config.url ?? 'unknown',
|
|
98
|
+
contentType: contentTypeStr,
|
|
99
|
+
duration: `${duration}ms`,
|
|
100
|
+
size: response.headers['content-length'],
|
|
68
101
|
});
|
|
102
|
+
// Log slow requests
|
|
103
|
+
if (duration > 5000) {
|
|
104
|
+
logWarn('Slow HTTP request detected', {
|
|
105
|
+
url: response.config.url ?? 'unknown',
|
|
106
|
+
duration: `${duration}ms`,
|
|
107
|
+
});
|
|
108
|
+
}
|
|
109
|
+
// Early content-type validation before processing
|
|
110
|
+
if (contentTypeStr && !isHtmlContentType(contentTypeStr)) {
|
|
111
|
+
throw new FetchError(`Unexpected content type: ${contentTypeStr}. Expected HTML content.`, response.config.url ?? 'unknown');
|
|
112
|
+
}
|
|
69
113
|
return response;
|
|
70
|
-
},
|
|
114
|
+
}, (error) => {
|
|
71
115
|
const url = error.config?.url ?? 'unknown';
|
|
116
|
+
// Handle request cancellation (AbortController)
|
|
117
|
+
if (isCancel(error) || error.name === 'AbortError') {
|
|
118
|
+
logDebug('HTTP Request Aborted', { url });
|
|
119
|
+
throw new AbortError('Request was canceled');
|
|
120
|
+
}
|
|
121
|
+
// Handle CanceledError from AbortSignal.timeout()
|
|
122
|
+
if (error.name === 'CanceledError') {
|
|
123
|
+
logDebug('HTTP Request Canceled (timeout signal)', { url });
|
|
124
|
+
throw new TimeoutError(config.fetcher.timeout, true);
|
|
125
|
+
}
|
|
72
126
|
if (error.code === 'ECONNABORTED' || error.code === 'ETIMEDOUT') {
|
|
73
127
|
logError('HTTP Timeout', { url, timeout: config.fetcher.timeout });
|
|
74
|
-
|
|
128
|
+
throw new TimeoutError(config.fetcher.timeout, true);
|
|
75
129
|
}
|
|
76
130
|
if (error.response) {
|
|
77
|
-
const status = error.response
|
|
78
|
-
|
|
131
|
+
const { status, statusText, headers } = error.response;
|
|
132
|
+
// Handle 429 Too Many Requests with Retry-After header
|
|
133
|
+
if (status === 429) {
|
|
134
|
+
const retryAfterHeader = headers['retry-after'];
|
|
135
|
+
let retryAfterSeconds = 60; // Default 60 seconds
|
|
136
|
+
if (retryAfterHeader) {
|
|
137
|
+
const parsed = parseInt(retryAfterHeader, 10);
|
|
138
|
+
if (!isNaN(parsed)) {
|
|
139
|
+
retryAfterSeconds = parsed;
|
|
140
|
+
}
|
|
141
|
+
}
|
|
142
|
+
logWarn('Rate limited by server', {
|
|
143
|
+
url,
|
|
144
|
+
retryAfter: `${retryAfterSeconds}s`,
|
|
145
|
+
});
|
|
146
|
+
throw new RateLimitError(retryAfterSeconds);
|
|
147
|
+
}
|
|
79
148
|
logError('HTTP Error Response', { url, status, statusText });
|
|
80
|
-
|
|
149
|
+
throw new FetchError(`HTTP ${status}: ${statusText}`, url, status);
|
|
81
150
|
}
|
|
82
151
|
if (error.request) {
|
|
83
152
|
logError('HTTP Network Error', { url, code: error.code });
|
|
84
|
-
|
|
153
|
+
throw new FetchError(`Network error: Could not reach ${url}`, url);
|
|
85
154
|
}
|
|
86
155
|
logError('HTTP Unknown Error', { url, message: error.message });
|
|
87
|
-
|
|
156
|
+
throw new FetchError(error.message, url);
|
|
88
157
|
});
|
|
89
|
-
async function fetchUrl(url,
|
|
158
|
+
async function fetchUrl(url, options) {
|
|
90
159
|
const requestConfig = {
|
|
91
160
|
method: 'GET',
|
|
92
161
|
url,
|
|
93
162
|
responseType: 'text',
|
|
94
163
|
};
|
|
95
|
-
|
|
164
|
+
// Apply per-request timeout via AbortSignal.timeout() if provided
|
|
165
|
+
// This is cleaner than axios timeout as it properly cancels the request
|
|
166
|
+
if (options?.signal) {
|
|
167
|
+
requestConfig.signal = options.signal;
|
|
168
|
+
}
|
|
169
|
+
else if (options?.timeout) {
|
|
170
|
+
// Use AbortSignal.timeout() for per-request timeout (Node 17.3+)
|
|
171
|
+
requestConfig.signal = AbortSignal.timeout(options.timeout);
|
|
172
|
+
}
|
|
173
|
+
const sanitized = sanitizeHeaders(options?.customHeaders);
|
|
96
174
|
if (sanitized) {
|
|
97
|
-
requestConfig.headers
|
|
175
|
+
const existingHeaders = requestConfig.headers && typeof requestConfig.headers === 'object'
|
|
176
|
+
? requestConfig.headers
|
|
177
|
+
: {};
|
|
178
|
+
requestConfig.headers = { ...existingHeaders, ...sanitized };
|
|
98
179
|
}
|
|
99
180
|
try {
|
|
100
181
|
const response = await client.request(requestConfig);
|
|
101
|
-
const contentTypeHeader = response.headers['content-type'];
|
|
102
|
-
let contentType;
|
|
103
|
-
if (Array.isArray(contentTypeHeader)) {
|
|
104
|
-
const first = contentTypeHeader[0];
|
|
105
|
-
if (typeof first === 'string')
|
|
106
|
-
contentType = first;
|
|
107
|
-
}
|
|
108
|
-
else if (typeof contentTypeHeader === 'string') {
|
|
109
|
-
contentType = contentTypeHeader;
|
|
110
|
-
}
|
|
111
|
-
if (contentType && !isHtmlContentType(contentType)) {
|
|
112
|
-
throw new FetchError(`Unexpected content type: ${contentType}. Expected HTML content.`, url);
|
|
113
|
-
}
|
|
114
182
|
return response.data;
|
|
115
183
|
}
|
|
116
184
|
catch (error) {
|
|
117
|
-
if (error instanceof FetchError ||
|
|
185
|
+
if (error instanceof FetchError ||
|
|
186
|
+
error instanceof TimeoutError ||
|
|
187
|
+
error instanceof AbortError ||
|
|
188
|
+
error instanceof RateLimitError) {
|
|
118
189
|
throw error;
|
|
119
190
|
}
|
|
120
191
|
throw new FetchError(`Unexpected error: ${error instanceof Error ? error.message : 'Unknown'}`, url);
|
|
@@ -126,7 +197,7 @@ function isHtmlContentType(contentType) {
|
|
|
126
197
|
normalized.includes('application/xhtml') ||
|
|
127
198
|
normalized.includes('text/plain'));
|
|
128
199
|
}
|
|
129
|
-
export async function fetchUrlWithRetry(url,
|
|
200
|
+
export async function fetchUrlWithRetry(url, options, maxRetries = 3, skipCache = false) {
|
|
130
201
|
if (!skipCache) {
|
|
131
202
|
const cachedHtml = getHtml(url);
|
|
132
203
|
if (cachedHtml) {
|
|
@@ -137,23 +208,47 @@ export async function fetchUrlWithRetry(url, customHeaders, maxRetries = 3, skip
|
|
|
137
208
|
const retries = Math.min(Math.max(1, maxRetries), 10);
|
|
138
209
|
let lastError = new Error(`Failed to fetch ${url}`);
|
|
139
210
|
for (let attempt = 1; attempt <= retries; attempt++) {
|
|
211
|
+
// Check if aborted before attempting (early exit for batch operations)
|
|
212
|
+
if (options?.signal?.aborted) {
|
|
213
|
+
const abortError = new AbortError('Request was aborted before execution');
|
|
214
|
+
throw abortError;
|
|
215
|
+
}
|
|
140
216
|
try {
|
|
141
|
-
const html = await fetchUrl(url,
|
|
217
|
+
const html = await fetchUrl(url, options);
|
|
142
218
|
setHtml(url, html);
|
|
143
219
|
logDebug('HTML Cache Set', { url });
|
|
144
220
|
return { html, fromHtmlCache: false };
|
|
145
221
|
}
|
|
146
222
|
catch (error) {
|
|
147
223
|
lastError = error instanceof Error ? error : new Error(String(error));
|
|
224
|
+
// Don't retry on abort - exit immediately
|
|
225
|
+
if (error instanceof AbortError) {
|
|
226
|
+
throw error;
|
|
227
|
+
}
|
|
228
|
+
// Handle rate limiting with smart retry
|
|
229
|
+
if (error instanceof RateLimitError) {
|
|
230
|
+
if (attempt < retries) {
|
|
231
|
+
const waitTime = Math.min(error.retryAfter * 1000, 30000); // Cap at 30s
|
|
232
|
+
logWarn('Rate limited, waiting before retry', {
|
|
233
|
+
url,
|
|
234
|
+
attempt,
|
|
235
|
+
waitTime: `${waitTime}ms`,
|
|
236
|
+
});
|
|
237
|
+
await new Promise((resolve) => setTimeout(resolve, waitTime));
|
|
238
|
+
continue;
|
|
239
|
+
}
|
|
240
|
+
throw error;
|
|
241
|
+
}
|
|
148
242
|
if (error instanceof FetchError && error.httpStatus) {
|
|
149
243
|
const status = error.httpStatus;
|
|
150
|
-
// Don't retry client errors (except 429
|
|
244
|
+
// Don't retry client errors (except 429 which is handled above)
|
|
151
245
|
if (status >= 400 && status < 500 && status !== 429) {
|
|
152
246
|
throw error;
|
|
153
247
|
}
|
|
154
248
|
}
|
|
155
249
|
if (attempt < retries) {
|
|
156
250
|
const delay = calculateBackoff(attempt);
|
|
251
|
+
logDebug('Retrying request', { url, attempt, delay: `${delay}ms` });
|
|
157
252
|
await new Promise((resolve) => setTimeout(resolve, delay));
|
|
158
253
|
}
|
|
159
254
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"fetcher.js","sourceRoot":"","sources":["../../src/services/fetcher.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,
|
|
1
|
+
{"version":3,"file":"fetcher.js","sourceRoot":"","sources":["../../src/services/fetcher.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,EAGZ,QAAQ,GACT,MAAM,OAAO,CAAC;AACf,OAAO,IAAI,MAAM,MAAM,CAAC;AACxB,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,EAAE,MAAM,IAAI,CAAC;AAEpB,OAAO,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAE5C,OAAO,EACL,UAAU,EACV,UAAU,EACV,cAAc,EACd,YAAY,GACb,MAAM,wBAAwB,CAAC;AAEhC,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AAC9C,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,aAAa,CAAC;AAY1D,8DAA8D;AAC9D,MAAM,kBAAkB,GAAG,MAAM,CAAC,kBAAkB,CAAC,CAAC;AAOtD,MAAM,eAAe,GAAG,IAAI,GAAG,CAAC;IAC9B,MAAM;IACN,eAAe;IACf,QAAQ;IACR,iBAAiB;IACjB,WAAW;IACX,qBAAqB;CACtB,CAAC,CAAC;AAEH,SAAS,eAAe,CACtB,OAAgC;IAEhC,IAAI,CAAC,OAAO,IAAI,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,SAAS,CAAC;IAEpE,MAAM,SAAS,GAA2B,EAAE,CAAC;IAC7C,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;QACnD,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,GAAG,CAAC,WAAW,EAAE,CAAC,EAAE,CAAC;YAC5C,SAAS,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC;QACzB,CAAC;IACH,CAAC;IAED,OAAO,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC;AACnE,CAAC;AAED,SAAS,gBAAgB,CAAC,OAAe,EAAE,QAAQ,GAAG,KAAK;IACzD,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,GAAG,CAAC,CAAC,EAAE,QAAQ,CAAC,CAAC;IACtE,MAAM,MAAM,GAAG,SAAS,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC;IAC1D,OAAO,IAAI,CAAC,KAAK,CAAC,SAAS,GAAG,MAAM,CAAC,CAAC;AACxC,CAAC;AAED,oFAAoF;AACpF,MAAM,SAAS,GAAG,EAAE,CAAC,IAAI,EAAE,CAAC,MAAM,CAAC;AACnC,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,SAAS,GAAG,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,+BAA+B;AAChF,MAAM,gBAAgB,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,SAAS,GAAG,GAAG,CAAC,EAAE,EAAE,CAAC,CAAC;AAEnE,MAAM,SAAS,GAAG,IAAI,IAAI,CAAC,KAAK,CAAC;IAC/B,SAAS,EAAE,IAAI;IACf,UAAU,EAAE,WAAW;IACvB,cAAc,EAAE,gBAAgB;IAChC,OAAO,EAAE,KAAK;IACd,UAAU,EAAE,MAAM;CACnB,CAAC,CAAC;AAEH,MAAM,UAAU,GAAG,IAAI,KAAK,CAAC,KAAK,CAAC;IACjC,SAAS,EAAE,IAAI;IACf,UAAU,EAAE,WAAW;IACvB,cAAc,EAAE,gBAAgB;IAChC,OAAO,EAAE,KAAK;IACd,UAAU,EAAE,MAAM;CACnB,CAAC,CAAC;AAEH,MAAM,UAAU,aAAa;IAC3B,SAAS,CAAC,OAAO,EAAE,CAAC;IACpB,UAAU,CAAC,OAAO,EAAE,CAAC;AACvB,CAAC;AAED,MAAM,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC;IAC1B,OAAO,EAAE,MAAM,CAAC,OAAO,CAAC,OAAO;IAC/B,YAAY,EAAE,MAAM,CAAC,OAAO,CAAC,YAAY;IACzC,gBAAgB,EAAE,MAAM,CAAC,OAAO,CAAC,gBAAgB;IACjD,SAAS;IACT,UAAU;IACV,OAAO,EAAE;QACP,YAAY,EAAE,MAAM,CAAC,OAAO,CAAC,SAAS;QACtC,MAAM,EACJ,4EAA4E;QAC9E,iBAAiB,EAAE,gBAAgB;QACnC,iBAAiB,EAAE,mBAAmB;QACtC,UAAU,EAAE,YAAY;KACzB;IACD,cAAc,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,IAAI,GAAG,IAAI,MAAM,GAAG,GAAG;CAC1D,CAAC,CAAC;AAEH,MAAM,CAAC,YAAY,CAAC,OAAO,CAAC,GAAG,CAC7B,CAAC,aAAa,EAAE,EAAE;IAChB,kDAAkD;IAClD,MAAM,MAAM,GAAG,aAAwC,CAAC;IACxD,MAAM,CAAC,kBAAkB,CAAC,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAExC,QAAQ,CAAC,cAAc,EAAE;QACvB,MAAM,EAAE,aAAa,CAAC,MAAM,EAAE,WAAW,EAAE;QAC3C,GAAG,EAAE,aAAa,CAAC,GAAG;KACvB,CAAC,CAAC;IACH,OAAO,aAAa,CAAC;AACvB,CAAC,EACD,CAAC,KAAiB,EAAE,EAAE;IACpB,QAAQ,CAAC,oBAAoB,EAAE,KAAK,CAAC,CAAC;IACtC,MAAM,KAAK,CAAC;AACd,CAAC,CACF,CAAC;AAEF,MAAM,CAAC,YAAY,CAAC,QAAQ,CAAC,GAAG,CAC9B,CAAC,QAAQ,EAAE,EAAE;IACX,MAAM,MAAM,GAAG,QAAQ,CAAC,MAAiC,CAAC;IAC1D,MAAM,SAAS,GAAG,MAAM,CAAC,kBAAkB,CAAC,CAAC;IAC7C,MAAM,QAAQ,GAAG,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC;IAExD,uBAAuB;IACvB,IAAI,MAAM,CAAC,kBAAkB,CAAC,KAAK,SAAS,EAAE,CAAC;QAC7C,MAAM,CAAC,kBAAkB,CAAC,GAAG,SAAS,CAAC;IACzC,CAAC;IAED,MAAM,WAAW,GAAY,QAAQ,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC;IAC9D,MAAM,cAAc,GAClB,OAAO,WAAW,KAAK,QAAQ,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,SAAS,CAAC;IAE5D,QAAQ,CAAC,eAAe,EAAE;QACxB,MAAM,EAAE,QAAQ,CAAC,MAAM;QACvB,GAAG,EAAE,QAAQ,CAAC,MAAM,CAAC,GAAG,IAAI,SAAS;QACrC,WAAW,EAAE,cAAc;QAC3B,QAAQ,EAAE,GAAG,QAAQ,IAAI;QACzB,IAAI,EAAE,QAAQ,CAAC,OAAO,CAAC,gBAAgB,CAAC;KACzC,CAAC,CAAC;IAEH,oBAAoB;IACpB,IAAI,QAAQ,GAAG,IAAI,EAAE,CAAC;QACpB,OAAO,CAAC,4BAA4B,EAAE;YACpC,GAAG,EAAE,QAAQ,CAAC,MAAM,CAAC,GAAG,IAAI,SAAS;YACrC,QAAQ,EAAE,GAAG,QAAQ,IAAI;SAC1B,CAAC,CAAC;IACL,CAAC;IAED,kDAAkD;IAClD,IAAI,cAAc,IAAI,CAAC,iBAAiB,CAAC,cAAc,CAAC,EAAE,CAAC;QACzD,MAAM,IAAI,UAAU,CAClB,4BAA4B,cAAc,0BAA0B,EACpE,QAAQ,CAAC,MAAM,CAAC,GAAG,IAAI,SAAS,CACjC,CAAC;IACJ,CAAC;IAED,OAAO,QAAQ,CAAC;AAClB,CAAC,EACD,CAAC,KAAiB,EAAE,EAAE;IACpB,MAAM,GAAG,GAAG,KAAK,CAAC,MAAM,EAAE,GAAG,IAAI,SAAS,CAAC;IAE3C,gDAAgD;IAChD,IAAI,QAAQ,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,IAAI,KAAK,YAAY,EAAE,CAAC;QACnD,QAAQ,CAAC,sBAAsB,EAAE,EAAE,GAAG,EAAE,CAAC,CAAC;QAC1C,MAAM,IAAI,UAAU,CAAC,sBAAsB,CAAC,CAAC;IAC/C,CAAC;IAED,kDAAkD;IAClD,IAAI,KAAK,CAAC,IAAI,KAAK,eAAe,EAAE,CAAC;QACnC,QAAQ,CAAC,wCAAwC,EAAE,EAAE,GAAG,EAAE,CAAC,CAAC;QAC5D,MAAM,IAAI,YAAY,CAAC,MAAM,CAAC,OAAO,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC;IACvD,CAAC;IAED,IAAI,KAAK,CAAC,IAAI,KAAK,cAAc,IAAI,KAAK,CAAC,IAAI,KAAK,WAAW,EAAE,CAAC;QAChE,QAAQ,CAAC,cAAc,EAAE,EAAE,GAAG,EAAE,OAAO,EAAE,MAAM,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC;QACnE,MAAM,IAAI,YAAY,CAAC,MAAM,CAAC,OAAO,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC;IACvD,CAAC;IAED,IAAI,KAAK,CAAC,QAAQ,EAAE,CAAC;QACnB,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,OAAO,EAAE,GAAG,KAAK,CAAC,QAAQ,CAAC;QAEvD,uDAAuD;QACvD,IAAI,MAAM,KAAK,GAAG,EAAE,CAAC;YACnB,MAAM,gBAAgB,GAAG,OAAO,CAAC,aAAa,CAAuB,CAAC;YACtE,IAAI,iBAAiB,GAAG,EAAE,CAAC,CAAC,qBAAqB;YAEjD,IAAI,gBAAgB,EAAE,CAAC;gBACrB,MAAM,MAAM,GAAG,QAAQ,CAAC,gBAAgB,EAAE,EAAE,CAAC,CAAC;gBAC9C,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC;oBACnB,iBAAiB,GAAG,MAAM,CAAC;gBAC7B,CAAC;YACH,CAAC;YAED,OAAO,CAAC,wBAAwB,EAAE;gBAChC,GAAG;gBACH,UAAU,EAAE,GAAG,iBAAiB,GAAG;aACpC,CAAC,CAAC;YACH,MAAM,IAAI,cAAc,CAAC,iBAAiB,CAAC,CAAC;QAC9C,CAAC;QAED,QAAQ,CAAC,qBAAqB,EAAE,EAAE,GAAG,EAAE,MAAM,EAAE,UAAU,EAAE,CAAC,CAAC;QAC7D,MAAM,IAAI,UAAU,CAAC,QAAQ,MAAM,KAAK,UAAU,EAAE,EAAE,GAAG,EAAE,MAAM,CAAC,CAAC;IACrE,CAAC;IAED,IAAI,KAAK,CAAC,OAAO,EAAE,CAAC;QAClB,QAAQ,CAAC,oBAAoB,EAAE,EAAE,GAAG,EAAE,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC;QAC1D,MAAM,IAAI,UAAU,CAAC,kCAAkC,GAAG,EAAE,EAAE,GAAG,CAAC,CAAC;IACrE,CAAC;IAED,QAAQ,CAAC,oBAAoB,EAAE,EAAE,GAAG,EAAE,OAAO,EAAE,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;IAChE,MAAM,IAAI,UAAU,CAAC,KAAK,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;AAC3C,CAAC,CACF,CAAC;AAEF,KAAK,UAAU,QAAQ,CAAC,GAAW,EAAE,OAAsB;IACzD,MAAM,aAAa,GAAuB;QACxC,MAAM,EAAE,KAAK;QACb,GAAG;QACH,YAAY,EAAE,MAAM;KACrB,CAAC;IAEF,kEAAkE;IAClE,wEAAwE;IACxE,IAAI,OAAO,EAAE,MAAM,EAAE,CAAC;QACpB,aAAa,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IACxC,CAAC;SAAM,IAAI,OAAO,EAAE,OAAO,EAAE,CAAC;QAC5B,iEAAiE;QACjE,aAAa,CAAC,MAAM,GAAG,WAAW,CAAC,OAAO,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;IAC9D,CAAC;IAED,MAAM,SAAS,GAAG,eAAe,CAAC,OAAO,EAAE,aAAa,CAAC,CAAC;IAC1D,IAAI,SAAS,EAAE,CAAC;QACd,MAAM,eAAe,GACnB,aAAa,CAAC,OAAO,IAAI,OAAO,aAAa,CAAC,OAAO,KAAK,QAAQ;YAChE,CAAC,CAAE,aAAa,CAAC,OAAkC;YACnD,CAAC,CAAC,EAAE,CAAC;QACT,aAAa,CAAC,OAAO,GAAG,EAAE,GAAG,eAAe,EAAE,GAAG,SAAS,EAAE,CAAC;IAC/D,CAAC;IAED,IAAI,CAAC;QACH,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,OAAO,CAAS,aAAa,CAAC,CAAC;QAC7D,OAAO,QAAQ,CAAC,IAAI,CAAC;IACvB,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IACE,KAAK,YAAY,UAAU;YAC3B,KAAK,YAAY,YAAY;YAC7B,KAAK,YAAY,UAAU;YAC3B,KAAK,YAAY,cAAc,EAC/B,CAAC;YACD,MAAM,KAAK,CAAC;QACd,CAAC;QAED,MAAM,IAAI,UAAU,CAClB,qBAAqB,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,SAAS,EAAE,EACzE,GAAG,CACJ,CAAC;IACJ,CAAC;AACH,CAAC;AAED,SAAS,iBAAiB,CAAC,WAAmB;IAC5C,MAAM,UAAU,GAAG,WAAW,CAAC,WAAW,EAAE,CAAC;IAC7C,OAAO,CACL,UAAU,CAAC,QAAQ,CAAC,WAAW,CAAC;QAChC,UAAU,CAAC,QAAQ,CAAC,mBAAmB,CAAC;QACxC,UAAU,CAAC,QAAQ,CAAC,YAAY,CAAC,CAClC,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,iBAAiB,CACrC,GAAW,EACX,OAAsB,EACtB,UAAU,GAAG,CAAC,EACd,SAAS,GAAG,KAAK;IAEjB,IAAI,CAAC,SAAS,EAAE,CAAC;QACf,MAAM,UAAU,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC;QAChC,IAAI,UAAU,EAAE,CAAC;YACf,QAAQ,CAAC,gBAAgB,EAAE,EAAE,GAAG,EAAE,CAAC,CAAC;YACpC,OAAO,EAAE,IAAI,EAAE,UAAU,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC;QACnD,CAAC;IACH,CAAC;IAED,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,UAAU,CAAC,EAAE,EAAE,CAAC,CAAC;IACtD,IAAI,SAAS,GAAU,IAAI,KAAK,CAAC,mBAAmB,GAAG,EAAE,CAAC,CAAC;IAE3D,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,IAAI,OAAO,EAAE,OAAO,EAAE,EAAE,CAAC;QACpD,uEAAuE;QACvE,IAAI,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC;YAC7B,MAAM,UAAU,GAAG,IAAI,UAAU,CAAC,sCAAsC,CAAC,CAAC;YAC1E,MAAM,UAAU,CAAC;QACnB,CAAC;QAED,IAAI,CAAC;YACH,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;YAC1C,OAAO,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;YACnB,QAAQ,CAAC,gBAAgB,EAAE,EAAE,GAAG,EAAE,CAAC,CAAC;YAEpC,OAAO,EAAE,IAAI,EAAE,aAAa,EAAE,KAAK,EAAE,CAAC;QACxC,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,SAAS,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;YAEtE,0CAA0C;YAC1C,IAAI,KAAK,YAAY,UAAU,EAAE,CAAC;gBAChC,MAAM,KAAK,CAAC;YACd,CAAC;YAED,wCAAwC;YACxC,IAAI,KAAK,YAAY,cAAc,EAAE,CAAC;gBACpC,IAAI,OAAO,GAAG,OAAO,EAAE,CAAC;oBACtB,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,UAAU,GAAG,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC,aAAa;oBACxE,OAAO,CAAC,oCAAoC,EAAE;wBAC5C,GAAG;wBACH,OAAO;wBACP,QAAQ,EAAE,GAAG,QAAQ,IAAI;qBAC1B,CAAC,CAAC;oBACH,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC,CAAC;oBAC9D,SAAS;gBACX,CAAC;gBACD,MAAM,KAAK,CAAC;YACd,CAAC;YAED,IAAI,KAAK,YAAY,UAAU,IAAI,KAAK,CAAC,UAAU,EAAE,CAAC;gBACpD,MAAM,MAAM,GAAG,KAAK,CAAC,UAAU,CAAC;gBAChC,gEAAgE;gBAChE,IAAI,MAAM,IAAI,GAAG,IAAI,MAAM,GAAG,GAAG,IAAI,MAAM,KAAK,GAAG,EAAE,CAAC;oBACpD,MAAM,KAAK,CAAC;gBACd,CAAC;YACH,CAAC;YAED,IAAI,OAAO,GAAG,OAAO,EAAE,CAAC;gBACtB,MAAM,KAAK,GAAG,gBAAgB,CAAC,OAAO,CAAC,CAAC;gBACxC,QAAQ,CAAC,kBAAkB,EAAE,EAAE,GAAG,EAAE,OAAO,EAAE,KAAK,EAAE,GAAG,KAAK,IAAI,EAAE,CAAC,CAAC;gBACpE,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC,CAAC;YAC7D,CAAC;QACH,CAAC;IACH,CAAC;IAED,MAAM,IAAI,UAAU,CAClB,gBAAgB,OAAO,cAAc,SAAS,CAAC,OAAO,EAAE,EACxD,GAAG,CACJ,CAAC;AACJ,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"parser.d.ts","sourceRoot":"","sources":["../../src/services/parser.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EAGV,iBAAiB,EAOlB,MAAM,oBAAoB,CAAC;
|
|
1
|
+
{"version":3,"file":"parser.d.ts","sourceRoot":"","sources":["../../src/services/parser.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EAGV,iBAAiB,EAOlB,MAAM,oBAAoB,CAAC;AA6M5B,wBAAgB,SAAS,CAAC,IAAI,EAAE,MAAM,GAAG,iBAAiB,EAAE,CAgD3D"}
|
package/dist/services/parser.js
CHANGED
|
@@ -5,6 +5,8 @@ import { detectLanguage } from '../utils/language-detector.js';
|
|
|
5
5
|
import { sanitizeText } from '../utils/sanitizer.js';
|
|
6
6
|
import { logWarn } from './logger.js';
|
|
7
7
|
const MAX_HTML_SIZE = 10 * 1024 * 1024;
|
|
8
|
+
// Cache selector for performance
|
|
9
|
+
const CONTENT_SELECTOR = 'h1, h2, h3, h4, h5, h6, p, ul, ol, pre, code:not(pre code), table, img, blockquote';
|
|
8
10
|
function parseHeading($, element) {
|
|
9
11
|
const rawText = sanitizeText($(element).text());
|
|
10
12
|
const text = cleanHeading(rawText);
|
|
@@ -26,14 +28,15 @@ function parseParagraph($, element) {
|
|
|
26
28
|
return { type: 'paragraph', text };
|
|
27
29
|
}
|
|
28
30
|
function parseList($, element) {
|
|
29
|
-
const listItems = $(element).find('li').toArray();
|
|
30
31
|
const rawItems = [];
|
|
31
|
-
//
|
|
32
|
-
|
|
32
|
+
// Direct .each() iteration - no array allocation overhead
|
|
33
|
+
$(element)
|
|
34
|
+
.find('li')
|
|
35
|
+
.each((_, li) => {
|
|
33
36
|
const text = sanitizeText($(li).text());
|
|
34
37
|
if (text)
|
|
35
38
|
rawItems.push(text);
|
|
36
|
-
}
|
|
39
|
+
});
|
|
37
40
|
// Clean list items to remove noise
|
|
38
41
|
const items = cleanListItems(rawItems);
|
|
39
42
|
if (items.length === 0)
|
|
@@ -69,28 +72,30 @@ function parseTable($, element) {
|
|
|
69
72
|
const headers = [];
|
|
70
73
|
const rows = [];
|
|
71
74
|
const $table = $(element);
|
|
72
|
-
//
|
|
73
|
-
|
|
74
|
-
for (const cell of headerCells) {
|
|
75
|
+
// Direct .each() iteration - no array allocation
|
|
76
|
+
$table.find('thead th, thead td').each((_, cell) => {
|
|
75
77
|
headers.push(sanitizeText($(cell).text()));
|
|
76
|
-
}
|
|
78
|
+
});
|
|
77
79
|
if (headers.length === 0) {
|
|
78
|
-
|
|
79
|
-
|
|
80
|
+
$table
|
|
81
|
+
.find('tr')
|
|
82
|
+
.first()
|
|
83
|
+
.find('th, td')
|
|
84
|
+
.each((_, cell) => {
|
|
80
85
|
headers.push(sanitizeText($(cell).text()));
|
|
81
|
-
}
|
|
86
|
+
});
|
|
82
87
|
}
|
|
83
88
|
const rowsSelector = headers.length > 0 ? 'tbody tr, tr:not(:first)' : 'tbody tr, tr';
|
|
84
|
-
|
|
85
|
-
for (const row of tableRows) {
|
|
86
|
-
const rowCells = $(row).find('td, th').toArray();
|
|
89
|
+
$table.find(rowsSelector).each((_, row) => {
|
|
87
90
|
const cells = [];
|
|
88
|
-
|
|
91
|
+
$(row)
|
|
92
|
+
.find('td, th')
|
|
93
|
+
.each((_, cell) => {
|
|
89
94
|
cells.push(sanitizeText($(cell).text()));
|
|
90
|
-
}
|
|
95
|
+
});
|
|
91
96
|
if (cells.length > 0)
|
|
92
97
|
rows.push(cells);
|
|
93
|
-
}
|
|
98
|
+
});
|
|
94
99
|
if (rows.length === 0)
|
|
95
100
|
return null;
|
|
96
101
|
return {
|
|
@@ -163,21 +168,29 @@ export function parseHtml(html) {
|
|
|
163
168
|
return [];
|
|
164
169
|
let processedHtml = html;
|
|
165
170
|
if (html.length > MAX_HTML_SIZE) {
|
|
166
|
-
logWarn('HTML content exceeds maximum size, truncating', {
|
|
171
|
+
logWarn('HTML content exceeds maximum size, truncating at safe boundary', {
|
|
167
172
|
size: html.length,
|
|
168
173
|
maxSize: MAX_HTML_SIZE,
|
|
169
174
|
});
|
|
170
|
-
|
|
175
|
+
// Use lastIndexOf for O(log n) reverse search (10-100x faster than while loop)
|
|
176
|
+
const lastTag = html.lastIndexOf('>', MAX_HTML_SIZE);
|
|
177
|
+
// If we found a tag boundary near the limit (within 10% buffer)
|
|
178
|
+
if (lastTag !== -1 && lastTag > MAX_HTML_SIZE * 0.9) {
|
|
179
|
+
processedHtml = html.substring(0, lastTag + 1);
|
|
180
|
+
}
|
|
181
|
+
else {
|
|
182
|
+
// Fallback: simple truncation if no suitable boundary found
|
|
183
|
+
processedHtml = html.substring(0, MAX_HTML_SIZE);
|
|
184
|
+
}
|
|
171
185
|
}
|
|
172
186
|
try {
|
|
173
187
|
const $ = cheerio.load(processedHtml);
|
|
174
188
|
const blocks = [];
|
|
175
189
|
$('script, style, noscript, iframe, svg').remove();
|
|
176
|
-
//
|
|
177
|
-
|
|
178
|
-
.find(
|
|
179
|
-
.
|
|
180
|
-
for (const element of elements) {
|
|
190
|
+
// Direct .each() iteration - no array allocation overhead
|
|
191
|
+
$('body')
|
|
192
|
+
.find(CONTENT_SELECTOR)
|
|
193
|
+
.each((_, element) => {
|
|
181
194
|
try {
|
|
182
195
|
const block = parseElement($, element);
|
|
183
196
|
if (block)
|
|
@@ -186,7 +199,7 @@ export function parseHtml(html) {
|
|
|
186
199
|
catch {
|
|
187
200
|
// Skip element errors
|
|
188
201
|
}
|
|
189
|
-
}
|
|
202
|
+
});
|
|
190
203
|
return filterBlocks(blocks);
|
|
191
204
|
}
|
|
192
205
|
catch (error) {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"parser.js","sourceRoot":"","sources":["../../src/services/parser.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,OAAO,MAAM,SAAS,CAAC;AAInC,OAAO,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAa5C,OAAO,EACL,cAAc,EACd,YAAY,EACZ,cAAc,EACd,cAAc,EACd,sBAAsB,GACvB,MAAM,6BAA6B,CAAC;AACrC,OAAO,EAAE,cAAc,EAAE,MAAM,+BAA+B,CAAC;AAC/D,OAAO,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAC;AAErD,OAAO,EAAE,OAAO,EAAE,MAAM,aAAa,CAAC;AAEtC,MAAM,aAAa,GAAG,EAAE,GAAG,IAAI,GAAG,IAAI,CAAC;AAEvC,SAAS,YAAY,CAAC,CAAa,EAAE,OAAgB;IACnD,MAAM,OAAO,GAAG,YAAY,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;IAChD,MAAM,IAAI,GAAG,YAAY,CAAC,OAAO,CAAC,CAAC;IACnC,IAAI,CAAC,IAAI;QAAE,OAAO,IAAI,CAAC;IAEvB,OAAO;QACL,IAAI,EAAE,SAAS;QACf,KAAK,EAAE,QAAQ,CAAC,OAAO,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC;QACjD,IAAI;KACL,CAAC;AACJ,CAAC;AAED,SAAS,cAAc,CACrB,CAAa,EACb,OAAgB;IAEhB,IAAI,OAAO,GAAG,YAAY,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;IAC9C,8DAA8D;IAC9D,OAAO,GAAG,sBAAsB,CAAC,OAAO,CAAC,CAAC;IAC1C,MAAM,IAAI,GAAG,cAAc,CAAC,OAAO,CAAC,CAAC;IACrC,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,UAAU,CAAC,kBAAkB;QAAE,OAAO,IAAI,CAAC;IAE7E,OAAO,EAAE,IAAI,EAAE,WAAW,EAAE,IAAI,EAAE,CAAC;AACrC,CAAC;AAED,SAAS,SAAS,CAAC,CAAa,EAAE,OAAgB;IAChD,MAAM,
|
|
1
|
+
{"version":3,"file":"parser.js","sourceRoot":"","sources":["../../src/services/parser.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,OAAO,MAAM,SAAS,CAAC;AAInC,OAAO,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAa5C,OAAO,EACL,cAAc,EACd,YAAY,EACZ,cAAc,EACd,cAAc,EACd,sBAAsB,GACvB,MAAM,6BAA6B,CAAC;AACrC,OAAO,EAAE,cAAc,EAAE,MAAM,+BAA+B,CAAC;AAC/D,OAAO,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAC;AAErD,OAAO,EAAE,OAAO,EAAE,MAAM,aAAa,CAAC;AAEtC,MAAM,aAAa,GAAG,EAAE,GAAG,IAAI,GAAG,IAAI,CAAC;AAEvC,iCAAiC;AACjC,MAAM,gBAAgB,GACpB,oFAAoF,CAAC;AAEvF,SAAS,YAAY,CAAC,CAAa,EAAE,OAAgB;IACnD,MAAM,OAAO,GAAG,YAAY,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;IAChD,MAAM,IAAI,GAAG,YAAY,CAAC,OAAO,CAAC,CAAC;IACnC,IAAI,CAAC,IAAI;QAAE,OAAO,IAAI,CAAC;IAEvB,OAAO;QACL,IAAI,EAAE,SAAS;QACf,KAAK,EAAE,QAAQ,CAAC,OAAO,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC;QACjD,IAAI;KACL,CAAC;AACJ,CAAC;AAED,SAAS,cAAc,CACrB,CAAa,EACb,OAAgB;IAEhB,IAAI,OAAO,GAAG,YAAY,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;IAC9C,8DAA8D;IAC9D,OAAO,GAAG,sBAAsB,CAAC,OAAO,CAAC,CAAC;IAC1C,MAAM,IAAI,GAAG,cAAc,CAAC,OAAO,CAAC,CAAC;IACrC,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,UAAU,CAAC,kBAAkB;QAAE,OAAO,IAAI,CAAC;IAE7E,OAAO,EAAE,IAAI,EAAE,WAAW,EAAE,IAAI,EAAE,CAAC;AACrC,CAAC;AAED,SAAS,SAAS,CAAC,CAAa,EAAE,OAAgB;IAChD,MAAM,QAAQ,GAAa,EAAE,CAAC;IAE9B,0DAA0D;IAC1D,CAAC,CAAC,OAAO,CAAC;SACP,IAAI,CAAC,IAAI,CAAC;SACV,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,EAAE;QACd,MAAM,IAAI,GAAG,YAAY,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;QACxC,IAAI,IAAI;YAAE,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAChC,CAAC,CAAC,CAAC;IAEL,mCAAmC;IACnC,MAAM,KAAK,GAAG,cAAc,CAAC,QAAQ,CAAC,CAAC;IACvC,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IAEpC,OAAO;QACL,IAAI,EAAE,MAAM;QACZ,OAAO,EAAE,OAAO,CAAC,OAAO,CAAC,WAAW,EAAE,KAAK,IAAI;QAC/C,KAAK;KACN,CAAC;AACJ,CAAC;AAED,SAAS,SAAS,CAAC,CAAa,EAAE,OAAgB;IAChD,MAAM,OAAO,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;IACzC,MAAM,IAAI,GAAG,cAAc,CAAC,OAAO,CAAC,CAAC;IACrC,IAAI,CAAC,IAAI;QAAE,OAAO,IAAI,CAAC;IAEvB,iDAAiD;IACjD,MAAM,SAAS,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC;IACjD,MAAM,QAAQ,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,IAAI,EAAE,CAAC;IAExD,sDAAsD;IACtD,MAAM,aAAa,GACjB,gBAAgB,CAAC,IAAI,CAAC,SAAS,CAAC;QAChC,YAAY,CAAC,IAAI,CAAC,SAAS,CAAC;QAC5B,iBAAiB,CAAC,IAAI,CAAC,SAAS,CAAC;QACjC,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IAE3B,kEAAkE;IAClE,MAAM,QAAQ,GAAG,aAAa,EAAE,CAAC,CAAC,CAAC,IAAI,cAAc,CAAC,IAAI,CAAC,CAAC;IAE5D,OAAO;QACL,IAAI,EAAE,MAAM;QACZ,QAAQ;QACR,IAAI;KACL,CAAC;AACJ,CAAC;AAED,SAAS,UAAU,CAAC,CAAa,EAAE,OAAgB;IACjD,MAAM,OAAO,GAAa,EAAE,CAAC;IAC7B,MAAM,IAAI,GAAe,EAAE,CAAC;IAC5B,MAAM,MAAM,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC;IAE1B,iDAAiD;IACjD,MAAM,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,EAAE;QACjD,OAAO,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;IAC7C,CAAC,CAAC,CAAC;IAEH,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACzB,MAAM;aACH,IAAI,CAAC,IAAI,CAAC;aACV,KAAK,EAAE;aACP,IAAI,CAAC,QAAQ,CAAC;aACd,IAAI,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,EAAE;YAChB,OAAO,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;QAC7C,CAAC,CAAC,CAAC;IACP,CAAC;IAED,MAAM,YAAY,GAChB,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,0BAA0B,CAAC,CAAC,CAAC,cAAc,CAAC;IAEnE,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,EAAE;QACxC,MAAM,KAAK,GAAa,EAAE,CAAC;QAC3B,CAAC,CAAC,GAAG,CAAC;aACH,IAAI,CAAC,QAAQ,CAAC;aACd,IAAI,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,EAAE;YAChB,KAAK,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;QAC3C,CAAC,CAAC,CAAC;QACL,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC;YAAE,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IACzC,CAAC,CAAC,CAAC;IAEH,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IAEnC,OAAO;QACL,IAAI,EAAE,OAAO;QACb,OAAO,EAAE,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,SAAS;QACjD,IAAI;KACL,CAAC;AACJ,CAAC;AAED,SAAS,UAAU,CAAC,CAAa,EAAE,OAAgB;IACjD,MAAM,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IACnC,IAAI,CAAC,GAAG;QAAE,OAAO,IAAI,CAAC;IAEtB,OAAO;QACL,IAAI,EAAE,OAAO;QACb,GAAG;QACH,GAAG,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,SAAS;KACzC,CAAC;AACJ,CAAC;AAED,SAAS,eAAe,CACtB,CAAa,EACb,OAAgB;IAEhB,MAAM,OAAO,GAAG,YAAY,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;IAChD,MAAM,IAAI,GAAG,cAAc,CAAC,OAAO,CAAC,CAAC;IACrC,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,UAAU,CAAC,kBAAkB;QAAE,OAAO,IAAI,CAAC;IAE7E,OAAO,EAAE,IAAI,EAAE,YAAY,EAAE,IAAI,EAAE,CAAC;AACtC,CAAC;AAED,MAAM,eAAe,GAAG;IACtB,EAAE,EAAE,YAAY;IAChB,EAAE,EAAE,YAAY;IAChB,EAAE,EAAE,YAAY;IAChB,EAAE,EAAE,YAAY;IAChB,EAAE,EAAE,YAAY;IAChB,EAAE,EAAE,YAAY;IAChB,CAAC,EAAE,cAAc;IACjB,EAAE,EAAE,SAAS;IACb,EAAE,EAAE,SAAS;IACb,GAAG,EAAE,SAAS;IACd,IAAI,EAAE,SAAS;IACf,KAAK,EAAE,UAAU;IACjB,GAAG,EAAE,UAAU;IACf,UAAU,EAAE,eAAe;CAI5B,CAAC;AAEF,SAAS,cAAc,CAAC,GAAW;IACjC,OAAO,GAAG,IAAI,eAAe,CAAC;AAChC,CAAC;AAED,SAAS,YAAY,CAAC,CAAa,EAAE,IAAa;IAChD,IAAI,CAAC,CAAC,SAAS,IAAI,IAAI,CAAC,IAAI,OAAO,IAAI,CAAC,OAAO,KAAK,QAAQ;QAAE,OAAO,IAAI,CAAC;IAE1E,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC;IAC3C,IAAI,CAAC,cAAc,CAAC,OAAO,CAAC;QAAE,OAAO,IAAI,CAAC;IAC1C,OAAO,eAAe,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC;AAC3C,CAAC;AAED,SAAS,YAAY,CAAC,MAA2B;IAC/C,OAAO,MAAM,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE;QAC7B,QAAQ,KAAK,CAAC,IAAI,EAAE,CAAC;YACnB,KAAK,WAAW,CAAC;YACjB,KAAK,SAAS,CAAC;YACf,KAAK,MAAM,CAAC;YACZ,KAAK,YAAY;gBACf,OAAO,KAAK,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC;YAC/B,KAAK,MAAM;gBACT,OAAO,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC;YAChC;gBACE,OAAO,IAAI,CAAC;QAChB,CAAC;IACH,CAAC,CAAC,CAAC;AACL,CAAC;AAED,MAAM,UAAU,SAAS,CAAC,IAAY;IACpC,IAAI,CAAC,IAAI,IAAI,OAAO,IAAI,KAAK,QAAQ;QAAE,OAAO,EAAE,CAAC;IAEjD,IAAI,aAAa,GAAG,IAAI,CAAC;IACzB,IAAI,IAAI,CAAC,MAAM,GAAG,aAAa,EAAE,CAAC;QAChC,OAAO,CAAC,gEAAgE,EAAE;YACxE,IAAI,EAAE,IAAI,CAAC,MAAM;YACjB,OAAO,EAAE,aAAa;SACvB,CAAC,CAAC;QAEH,+EAA+E;QAC/E,MAAM,OAAO,GAAG,IAAI,CAAC,WAAW,CAAC,GAAG,EAAE,aAAa,CAAC,CAAC;QAErD,gEAAgE;QAChE,IAAI,OAAO,KAAK,CAAC,CAAC,IAAI,OAAO,GAAG,aAAa,GAAG,GAAG,EAAE,CAAC;YACpD,aAAa,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC,EAAE,OAAO,GAAG,CAAC,CAAC,CAAC;QACjD,CAAC;aAAM,CAAC;YACN,4DAA4D;YAC5D,aAAa,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC,EAAE,aAAa,CAAC,CAAC;QACnD,CAAC;IACH,CAAC;IAED,IAAI,CAAC;QACH,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;QACtC,MAAM,MAAM,GAAwB,EAAE,CAAC;QAEvC,CAAC,CAAC,sCAAsC,CAAC,CAAC,MAAM,EAAE,CAAC;QAEnD,0DAA0D;QAC1D,CAAC,CAAC,MAAM,CAAC;aACN,IAAI,CAAC,gBAAgB,CAAC;aACtB,IAAI,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,EAAE;YACnB,IAAI,CAAC;gBACH,MAAM,KAAK,GAAG,YAAY,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC;gBACvC,IAAI,KAAK;oBAAE,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAChC,CAAC;YAAC,MAAM,CAAC;gBACP,sBAAsB;YACxB,CAAC;QACH,CAAC,CAAC,CAAC;QAEL,OAAO,YAAY,CAAC,MAAM,CAAC,CAAC;IAC9B,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,OAAO,CAAC,sBAAsB,EAAE;YAC9B,KAAK,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe;YAC/D,UAAU,EAAE,IAAI,CAAC,MAAM;SACxB,CAAC,CAAC;QACH,OAAO,EAAE,CAAC;IACZ,CAAC;AACH,CAAC"}
|
|
@@ -1,17 +1,12 @@
|
|
|
1
|
-
import type {
|
|
1
|
+
import type { FetchLinksInput } from '../../config/types.js';
|
|
2
2
|
export declare const FETCH_LINKS_TOOL_NAME = "fetch-links";
|
|
3
3
|
export declare const FETCH_LINKS_TOOL_DESCRIPTION = "Extracts all hyperlinks from a webpage with anchor text and type classification. Supports filtering, image links, and link limits.";
|
|
4
|
-
export declare function fetchLinksToolHandler(input: FetchLinksInput): Promise<
|
|
4
|
+
export declare function fetchLinksToolHandler(input: FetchLinksInput): Promise<{
|
|
5
5
|
content: {
|
|
6
|
-
type:
|
|
6
|
+
type: 'text';
|
|
7
7
|
text: string;
|
|
8
8
|
}[];
|
|
9
|
-
structuredContent
|
|
10
|
-
|
|
11
|
-
filtered?: number;
|
|
12
|
-
url: string;
|
|
13
|
-
linkCount: number;
|
|
14
|
-
links: ExtractedLink[];
|
|
15
|
-
};
|
|
9
|
+
structuredContent?: Record<string, unknown>;
|
|
10
|
+
isError?: boolean;
|
|
16
11
|
}>;
|
|
17
12
|
//# sourceMappingURL=fetch-links.tool.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"fetch-links.tool.d.ts","sourceRoot":"","sources":["../../../src/tools/handlers/fetch-links.tool.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,
|
|
1
|
+
{"version":3,"file":"fetch-links.tool.d.ts","sourceRoot":"","sources":["../../../src/tools/handlers/fetch-links.tool.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAGV,eAAe,EAEhB,MAAM,uBAAuB,CAAC;AAW/B,eAAO,MAAM,qBAAqB,gBAAgB,CAAC;AACnD,eAAO,MAAM,4BAA4B,uIAC6F,CAAC;AAuFvI,wBAAsB,qBAAqB,CAAC,KAAK,EAAE,eAAe,GAAG,OAAO,CAAC;IAC3E,OAAO,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC;IAC1C,iBAAiB,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAC5C,OAAO,CAAC,EAAE,OAAO,CAAC;CACnB,CAAC,CAiED"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"fetch-links.tool.js","sourceRoot":"","sources":["../../../src/tools/handlers/fetch-links.tool.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,OAAO,MAAM,SAAS,CAAC;AASnC,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,0BAA0B,CAAC;AAE9D,OAAO,EACL,uBAAuB,EACvB,eAAe,GAChB,MAAM,mCAAmC,CAAC;AAC3C,OAAO,EAAE,aAAa,EAAE,MAAM,8BAA8B,CAAC;AAC7D,OAAO,EAAE,oBAAoB,EAAE,MAAM,4BAA4B,CAAC;AAElE,MAAM,CAAC,MAAM,qBAAqB,GAAG,aAAa,CAAC;AACnD,MAAM,CAAC,MAAM,4BAA4B,GACvC,oIAAoI,CAAC;AAIvI,SAAS,aAAa,CAAC,IAAY,EAAE,OAAe;IAClD,IAAI,CAAC;QACH,OAAO,IAAI,GAAG,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC,IAAI,CAAC;IACrC,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED,SAAS,iBAAiB,CACxB,IAAc,EACd,GAAW,EACX,OAA4B,EAC5B,IAAiB;IAEjB,IAAI,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC;QAAE,OAAO,KAAK,CAAC;IAChC,IAAI,OAAO,CAAC,aAAa,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC,IAAI,CAAC,GAAG,CAAC;QAAE,OAAO,KAAK,CAAC;IAC5E,IAAI,IAAI,KAAK,UAAU,IAAI,CAAC,OAAO,CAAC,eAAe;QAAE,OAAO,KAAK,CAAC;IAClE,IAAI,IAAI,KAAK,UAAU,IAAI,CAAC,OAAO,CAAC,eAAe;QAAE,OAAO,KAAK,CAAC;IAClE,OAAO,IAAI,CAAC;AACd,CAAC;AAED,SAAS,YAAY,CACnB,IAAY,EACZ,OAAe,EACf,OAA4B;IAE5B,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC7B,MAAM,KAAK,GAAoB,EAAE,CAAC;IAClC,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAC/B,IAAI,QAAQ,GAAG,CAAC,CAAC;IAEjB,CAAC,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,EAAE;QAC1B,MAAM,IAAI,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAChC,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,aAAa,CAAC;YAAE,OAAO;QAE5E,MAAM,GAAG,GAAG,aAAa,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;QACzC,IAAI,CAAC,GAAG;YAAE,OAAO;QAEjB,MAAM,IAAI,GAAa,aAAa,CAAC,GAAG,EAAE,OAAO,CAAC;YAChD,CAAC,CAAC,UAAU;YACZ,CAAC,CAAC,UAAU,CAAC;QACf,IAAI,CAAC,iBAAiB,CAAC,IAAI,EAAE,GAAG,EAAE,OAAO,EAAE,IAAI,CAAC,EAAE,CAAC;YACjD,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC;gBAAE,QAAQ,EAAE,CAAC;YAC/B,OAAO;QACT,CAAC;QAED,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACd,KAAK,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,GAAG,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,IAAI,GAAG,EAAE,IAAI,EAAE,CAAC,CAAC;IACpE,CAAC,CAAC,CAAC;IAEH,IAAI,OAAO,CAAC,aAAa,EAAE,CAAC;QAC1B,CAAC,CAAC,UAAU,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,EAAE;YAC3B,MAAM,GAAG,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAC9B,IAAI,CAAC,GAAG,IAAI,GAAG,CAAC,UAAU,CAAC,OAAO,CAAC;gBAAE,OAAO;YAE5C,MAAM,GAAG,GAAG,aAAa,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;YACxC,IAAI,CAAC,GAAG;gBAAE,OAAO;YAEjB,IAAI,CAAC,iBAAiB,CAAC,OAAO,EAAE,GAAG,EAAE,OAAO,EAAE,IAAI,CAAC,EAAE,CAAC;gBACpD,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC;oBAAE,QAAQ,EAAE,CAAC;gBAC/B,OAAO;YACT,CAAC;YAED,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YACd,KAAK,CAAC,IAAI,CAAC;gBACT,IAAI,EAAE,GAAG;gBACT,IAAI,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,IAAI,EAAE,IAAI,GAAG;gBACtC,IAAI,EAAE,OAAO;aACd,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;IACL,CAAC;IAED,MAAM,SAAS,GAAG,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,GAAG,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC;IAC7E,MAAM,WAAW,GAAG,SAAS,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC;IAEzE,OAAO;QACL,KAAK,EAAE,WAAW;QAClB,SAAS,EAAE,WAAW,CAAC,MAAM;QAC7B,QAAQ;QACR,SAAS;KACV,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,qBAAqB,CAAC,KAAsB;
|
|
1
|
+
{"version":3,"file":"fetch-links.tool.js","sourceRoot":"","sources":["../../../src/tools/handlers/fetch-links.tool.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,OAAO,MAAM,SAAS,CAAC;AASnC,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,0BAA0B,CAAC;AAE9D,OAAO,EACL,uBAAuB,EACvB,eAAe,GAChB,MAAM,mCAAmC,CAAC;AAC3C,OAAO,EAAE,aAAa,EAAE,MAAM,8BAA8B,CAAC;AAC7D,OAAO,EAAE,oBAAoB,EAAE,MAAM,4BAA4B,CAAC;AAElE,MAAM,CAAC,MAAM,qBAAqB,GAAG,aAAa,CAAC;AACnD,MAAM,CAAC,MAAM,4BAA4B,GACvC,oIAAoI,CAAC;AAIvI,SAAS,aAAa,CAAC,IAAY,EAAE,OAAe;IAClD,IAAI,CAAC;QACH,OAAO,IAAI,GAAG,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC,IAAI,CAAC;IACrC,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED,SAAS,iBAAiB,CACxB,IAAc,EACd,GAAW,EACX,OAA4B,EAC5B,IAAiB;IAEjB,IAAI,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC;QAAE,OAAO,KAAK,CAAC;IAChC,IAAI,OAAO,CAAC,aAAa,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC,IAAI,CAAC,GAAG,CAAC;QAAE,OAAO,KAAK,CAAC;IAC5E,IAAI,IAAI,KAAK,UAAU,IAAI,CAAC,OAAO,CAAC,eAAe;QAAE,OAAO,KAAK,CAAC;IAClE,IAAI,IAAI,KAAK,UAAU,IAAI,CAAC,OAAO,CAAC,eAAe;QAAE,OAAO,KAAK,CAAC;IAClE,OAAO,IAAI,CAAC;AACd,CAAC;AAED,SAAS,YAAY,CACnB,IAAY,EACZ,OAAe,EACf,OAA4B;IAE5B,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC7B,MAAM,KAAK,GAAoB,EAAE,CAAC;IAClC,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAC/B,IAAI,QAAQ,GAAG,CAAC,CAAC;IAEjB,CAAC,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,EAAE;QAC1B,MAAM,IAAI,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAChC,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,aAAa,CAAC;YAAE,OAAO;QAE5E,MAAM,GAAG,GAAG,aAAa,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;QACzC,IAAI,CAAC,GAAG;YAAE,OAAO;QAEjB,MAAM,IAAI,GAAa,aAAa,CAAC,GAAG,EAAE,OAAO,CAAC;YAChD,CAAC,CAAC,UAAU;YACZ,CAAC,CAAC,UAAU,CAAC;QACf,IAAI,CAAC,iBAAiB,CAAC,IAAI,EAAE,GAAG,EAAE,OAAO,EAAE,IAAI,CAAC,EAAE,CAAC;YACjD,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC;gBAAE,QAAQ,EAAE,CAAC;YAC/B,OAAO;QACT,CAAC;QAED,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACd,KAAK,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,GAAG,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,IAAI,GAAG,EAAE,IAAI,EAAE,CAAC,CAAC;IACpE,CAAC,CAAC,CAAC;IAEH,IAAI,OAAO,CAAC,aAAa,EAAE,CAAC;QAC1B,CAAC,CAAC,UAAU,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,EAAE;YAC3B,MAAM,GAAG,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAC9B,IAAI,CAAC,GAAG,IAAI,GAAG,CAAC,UAAU,CAAC,OAAO,CAAC;gBAAE,OAAO;YAE5C,MAAM,GAAG,GAAG,aAAa,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;YACxC,IAAI,CAAC,GAAG;gBAAE,OAAO;YAEjB,IAAI,CAAC,iBAAiB,CAAC,OAAO,EAAE,GAAG,EAAE,OAAO,EAAE,IAAI,CAAC,EAAE,CAAC;gBACpD,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC;oBAAE,QAAQ,EAAE,CAAC;gBAC/B,OAAO;YACT,CAAC;YAED,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YACd,KAAK,CAAC,IAAI,CAAC;gBACT,IAAI,EAAE,GAAG;gBACT,IAAI,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,IAAI,EAAE,IAAI,GAAG;gBACtC,IAAI,EAAE,OAAO;aACd,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;IACL,CAAC;IAED,MAAM,SAAS,GAAG,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,GAAG,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC;IAC7E,MAAM,WAAW,GAAG,SAAS,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC;IAEzE,OAAO;QACL,KAAK,EAAE,WAAW;QAClB,SAAS,EAAE,WAAW,CAAC,MAAM;QAC7B,QAAQ;QACR,SAAS;KACV,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,qBAAqB,CAAC,KAAsB;IAKhE,IAAI,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC;QACf,OAAO,uBAAuB,CAAC,iBAAiB,EAAE,EAAE,EAAE,kBAAkB,CAAC,CAAC;IAC5E,CAAC;IAED,IAAI,aAAiC,CAAC;IACtC,IAAI,KAAK,CAAC,aAAa,EAAE,CAAC;QACxB,IAAI,CAAC;YACH,aAAa,GAAG,IAAI,MAAM,CAAC,KAAK,CAAC,aAAa,EAAE,GAAG,CAAC,CAAC;QACvD,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,uBAAuB,CAC5B,2BAA2B,KAAK,CAAC,aAAa,EAAE,EAChD,KAAK,CAAC,GAAG,EACT,kBAAkB,CACnB,CAAC;QACJ,CAAC;IACH,CAAC;IAED,IAAI,CAAC;QACH,MAAM,OAAO,GAAwB;YACnC,eAAe,EAAE,KAAK,CAAC,eAAe,IAAI,IAAI;YAC9C,eAAe,EAAE,KAAK,CAAC,eAAe,IAAI,IAAI;YAC9C,aAAa,EAAE,KAAK,CAAC,aAAa,IAAI,KAAK;YAC3C,QAAQ,EAAE,KAAK,CAAC,QAAQ;YACxB,aAAa;SACd,CAAC;QAEF,QAAQ,CAAC,kBAAkB,EAAE;YAC3B,GAAG,EAAE,KAAK,CAAC,GAAG;YACd,GAAG,OAAO;YACV,aAAa,EAAE,KAAK,CAAC,aAAa;SACnC,CAAC,CAAC;QAEH,MAAM,MAAM,GAAG,MAAM,oBAAoB,CAAuB;YAC9D,GAAG,EAAE,KAAK,CAAC,GAAG;YACd,cAAc,EAAE,OAAO;YACvB,aAAa,EAAE,KAAK,CAAC,aAAa;YAClC,OAAO,EAAE,KAAK,CAAC,OAAO;YACtB,SAAS,EAAE,CAAC,IAAI,EAAE,GAAG,EAAE,EAAE,CAAC,YAAY,CAAC,IAAI,EAAE,GAAG,EAAE,OAAO,CAAC;SAC3D,CAAC,CAAC;QAEH,MAAM,iBAAiB,GAAG;YACxB,GAAG,EAAE,MAAM,CAAC,GAAG;YACf,SAAS,EAAE,MAAM,CAAC,IAAI,CAAC,SAAS;YAChC,KAAK,EAAE,MAAM,CAAC,IAAI,CAAC,KAAK;YACxB,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,QAAQ,GAAG,CAAC,IAAI,EAAE,QAAQ,EAAE,MAAM,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC;YACnE,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,SAAS,IAAI,EAAE,SAAS,EAAE,MAAM,CAAC,IAAI,CAAC,SAAS,EAAE,CAAC;SACnE,CAAC;QAEF,OAAO;YACL,OAAO,EAAE;gBACP;oBACE,IAAI,EAAE,MAAe;oBACrB,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,iBAAiB,EAAE,IAAI,EAAE,CAAC,CAAC;iBACjD;aACF;YACD,iBAAiB;SAClB,CAAC;IACJ,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,QAAQ,CACN,wBAAwB,EACxB,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS,CAC3C,CAAC;QACF,OAAO,eAAe,CAAC,KAAK,EAAE,KAAK,CAAC,GAAG,EAAE,yBAAyB,CAAC,CAAC;IACtE,CAAC;AACH,CAAC"}
|
|
@@ -1,19 +1,12 @@
|
|
|
1
|
-
import type { FetchMarkdownInput
|
|
1
|
+
import type { FetchMarkdownInput } from '../../config/types.js';
|
|
2
2
|
export declare const FETCH_MARKDOWN_TOOL_NAME = "fetch-markdown";
|
|
3
3
|
export declare const FETCH_MARKDOWN_TOOL_DESCRIPTION = "Fetches a webpage and converts it to clean Markdown format with optional frontmatter, table of contents, and content length limits";
|
|
4
|
-
export declare function fetchMarkdownToolHandler(input: FetchMarkdownInput): Promise<
|
|
4
|
+
export declare function fetchMarkdownToolHandler(input: FetchMarkdownInput): Promise<{
|
|
5
5
|
content: {
|
|
6
|
-
type:
|
|
6
|
+
type: 'text';
|
|
7
7
|
text: string;
|
|
8
8
|
}[];
|
|
9
|
-
structuredContent
|
|
10
|
-
|
|
11
|
-
cached: boolean;
|
|
12
|
-
toc?: TocEntry[];
|
|
13
|
-
url: string;
|
|
14
|
-
title: string | undefined;
|
|
15
|
-
fetchedAt: string;
|
|
16
|
-
markdown: string;
|
|
17
|
-
};
|
|
9
|
+
structuredContent?: Record<string, unknown>;
|
|
10
|
+
isError?: boolean;
|
|
18
11
|
}>;
|
|
19
12
|
//# sourceMappingURL=fetch-markdown.tool.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"fetch-markdown.tool.d.ts","sourceRoot":"","sources":["../../../src/tools/handlers/fetch-markdown.tool.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,kBAAkB,
|
|
1
|
+
{"version":3,"file":"fetch-markdown.tool.d.ts","sourceRoot":"","sources":["../../../src/tools/handlers/fetch-markdown.tool.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,kBAAkB,EAInB,MAAM,uBAAuB,CAAC;AAe/B,eAAO,MAAM,wBAAwB,mBAAmB,CAAC;AACzD,eAAO,MAAM,+BAA+B,uIAC0F,CAAC;AAsEvI,wBAAsB,wBAAwB,CAC5C,KAAK,EAAE,kBAAkB,GACxB,OAAO,CAAC;IACT,OAAO,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC;IAC1C,iBAAiB,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAC5C,OAAO,CAAC,EAAE,OAAO,CAAC;CACnB,CAAC,CA4DD"}
|
|
@@ -52,8 +52,7 @@ function transformToMarkdown(html, url, options) {
|
|
|
52
52
|
const toc = options.generateToc ? extractToc(markdown) : undefined;
|
|
53
53
|
let truncated = false;
|
|
54
54
|
if (options.maxContentLength && markdown.length > options.maxContentLength) {
|
|
55
|
-
markdown =
|
|
56
|
-
markdown.substring(0, options.maxContentLength) + '\n\n...[truncated]';
|
|
55
|
+
markdown = `${markdown.substring(0, options.maxContentLength)}\n\n...[truncated]`;
|
|
57
56
|
truncated = true;
|
|
58
57
|
}
|
|
59
58
|
return { markdown, title, toc, truncated };
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"fetch-markdown.tool.js","sourceRoot":"","sources":["../../../src/tools/handlers/fetch-markdown.tool.ts"],"names":[],"mappings":"AAOA,OAAO,EAAE,cAAc,EAAE,MAAM,6BAA6B,CAAC;AAC7D,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,0BAA0B,CAAC;AAE9D,OAAO,EAAE,kBAAkB,EAAE,MAAM,gCAAgC,CAAC;AACpE,OAAO,EACL,uBAAuB,EACvB,eAAe,GAChB,MAAM,mCAAmC,CAAC;AAC3C,OAAO,EAAE,aAAa,EAAE,gBAAgB,EAAE,MAAM,oBAAoB,CAAC;AACrE,OAAO,EAAE,oBAAoB,EAAE,MAAM,4BAA4B,CAAC;AAElE,OAAO,EAAE,cAAc,EAAE,MAAM,4CAA4C,CAAC;AAE5E,MAAM,CAAC,MAAM,wBAAwB,GAAG,gBAAgB,CAAC;AACzD,MAAM,CAAC,MAAM,+BAA+B,GAC1C,oIAAoI,CAAC;AAEvI;;;GAGG;AACH,SAAS,OAAO,CAAC,IAAY;IAC3B,sDAAsD;IACtD,MAAM,SAAS,GAAG,kBAAkB,CAAC,IAAI,CAAC,CAAC;IAE3C,OAAO,SAAS;SACb,WAAW,EAAE;SACb,OAAO,CAAC,WAAW,EAAE,EAAE,CAAC;SACxB,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC;SACpB,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC;SACpB,IAAI,EAAE,CAAC;AACZ,CAAC;AAED;;;GAGG;AACH,SAAS,UAAU,CAAC,QAAgB;IAClC,MAAM,YAAY,GAAG,qBAAqB,CAAC;IAC3C,MAAM,GAAG,GAAe,EAAE,CAAC;IAC3B,IAAI,KAAK,CAAC;IAEV,OAAO,CAAC,KAAK,GAAG,YAAY,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;QACtD,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC;YAAE,SAAS;QACrC,MAAM,OAAO,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QAChC,+DAA+D;QAC/D,MAAM,IAAI,GAAG,kBAAkB,CAAC,OAAO,CAAC,CAAC;QACzC,GAAG,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;IACrE,CAAC;IAED,OAAO,GAAG,CAAC;AACb,CAAC;AAED,SAAS,mBAAmB,CAC1B,IAAY,EACZ,GAAW,EACX,OAAyB;IAEzB,gFAAgF;IAChF,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,aAAa,EAAE,GAAG,cAAc,CAAC,IAAI,EAAE,GAAG,EAAE;QACrE,cAAc,EAAE,OAAO,CAAC,kBAAkB;KAC3C,CAAC,CAAC;IACH,MAAM,UAAU,GAAG,gBAAgB,CAAC,OAAO,CAAC,kBAAkB,EAAE,OAAO,CAAC,CAAC;IACzE,MAAM,QAAQ,GAAG,aAAa,CAC5B,GAAG,EACH,OAAO,EACP,aAAa,EACb,UAAU,EACV,OAAO,CAAC,eAAe,CACxB,CAAC;IACF,MAAM,UAAU,GAAG,UAAU,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC;IACvD,MAAM,KAAK,GAAG,UAAU,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,aAAa,CAAC,KAAK,CAAC;IAE/D,IAAI,QAAQ,GAAG,cAAc,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAC;IACpD,MAAM,GAAG,GAAG,OAAO,CAAC,WAAW,CAAC,CAAC,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;IAEnE,IAAI,SAAS,GAAG,KAAK,CAAC;IACtB,IAAI,OAAO,CAAC,gBAAgB,IAAI,QAAQ,CAAC,MAAM,GAAG,OAAO,CAAC,gBAAgB,EAAE,CAAC;QAC3E,QAAQ
|
|
1
|
+
{"version":3,"file":"fetch-markdown.tool.js","sourceRoot":"","sources":["../../../src/tools/handlers/fetch-markdown.tool.ts"],"names":[],"mappings":"AAOA,OAAO,EAAE,cAAc,EAAE,MAAM,6BAA6B,CAAC;AAC7D,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,0BAA0B,CAAC;AAE9D,OAAO,EAAE,kBAAkB,EAAE,MAAM,gCAAgC,CAAC;AACpE,OAAO,EACL,uBAAuB,EACvB,eAAe,GAChB,MAAM,mCAAmC,CAAC;AAC3C,OAAO,EAAE,aAAa,EAAE,gBAAgB,EAAE,MAAM,oBAAoB,CAAC;AACrE,OAAO,EAAE,oBAAoB,EAAE,MAAM,4BAA4B,CAAC;AAElE,OAAO,EAAE,cAAc,EAAE,MAAM,4CAA4C,CAAC;AAE5E,MAAM,CAAC,MAAM,wBAAwB,GAAG,gBAAgB,CAAC;AACzD,MAAM,CAAC,MAAM,+BAA+B,GAC1C,oIAAoI,CAAC;AAEvI;;;GAGG;AACH,SAAS,OAAO,CAAC,IAAY;IAC3B,sDAAsD;IACtD,MAAM,SAAS,GAAG,kBAAkB,CAAC,IAAI,CAAC,CAAC;IAE3C,OAAO,SAAS;SACb,WAAW,EAAE;SACb,OAAO,CAAC,WAAW,EAAE,EAAE,CAAC;SACxB,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC;SACpB,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC;SACpB,IAAI,EAAE,CAAC;AACZ,CAAC;AAED;;;GAGG;AACH,SAAS,UAAU,CAAC,QAAgB;IAClC,MAAM,YAAY,GAAG,qBAAqB,CAAC;IAC3C,MAAM,GAAG,GAAe,EAAE,CAAC;IAC3B,IAAI,KAAK,CAAC;IAEV,OAAO,CAAC,KAAK,GAAG,YAAY,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;QACtD,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC;YAAE,SAAS;QACrC,MAAM,OAAO,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QAChC,+DAA+D;QAC/D,MAAM,IAAI,GAAG,kBAAkB,CAAC,OAAO,CAAC,CAAC;QACzC,GAAG,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;IACrE,CAAC;IAED,OAAO,GAAG,CAAC;AACb,CAAC;AAED,SAAS,mBAAmB,CAC1B,IAAY,EACZ,GAAW,EACX,OAAyB;IAEzB,gFAAgF;IAChF,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,aAAa,EAAE,GAAG,cAAc,CAAC,IAAI,EAAE,GAAG,EAAE;QACrE,cAAc,EAAE,OAAO,CAAC,kBAAkB;KAC3C,CAAC,CAAC;IACH,MAAM,UAAU,GAAG,gBAAgB,CAAC,OAAO,CAAC,kBAAkB,EAAE,OAAO,CAAC,CAAC;IACzE,MAAM,QAAQ,GAAG,aAAa,CAC5B,GAAG,EACH,OAAO,EACP,aAAa,EACb,UAAU,EACV,OAAO,CAAC,eAAe,CACxB,CAAC;IACF,MAAM,UAAU,GAAG,UAAU,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC;IACvD,MAAM,KAAK,GAAG,UAAU,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,aAAa,CAAC,KAAK,CAAC;IAE/D,IAAI,QAAQ,GAAG,cAAc,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAC;IACpD,MAAM,GAAG,GAAG,OAAO,CAAC,WAAW,CAAC,CAAC,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;IAEnE,IAAI,SAAS,GAAG,KAAK,CAAC;IACtB,IAAI,OAAO,CAAC,gBAAgB,IAAI,QAAQ,CAAC,MAAM,GAAG,OAAO,CAAC,gBAAgB,EAAE,CAAC;QAC3E,QAAQ,GAAG,GAAG,QAAQ,CAAC,SAAS,CAAC,CAAC,EAAE,OAAO,CAAC,gBAAgB,CAAC,oBAAoB,CAAC;QAClF,SAAS,GAAG,IAAI,CAAC;IACnB,CAAC;IAED,OAAO,EAAE,QAAQ,EAAE,KAAK,EAAE,GAAG,EAAE,SAAS,EAAE,CAAC;AAC7C,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,wBAAwB,CAC5C,KAAyB;IAMzB,IAAI,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC;QACf,OAAO,uBAAuB,CAAC,iBAAiB,EAAE,EAAE,EAAE,kBAAkB,CAAC,CAAC;IAC5E,CAAC;IAED,IAAI,CAAC;QACH,MAAM,OAAO,GAAqB;YAChC,kBAAkB,EAAE,KAAK,CAAC,kBAAkB,IAAI,IAAI;YACpD,eAAe,EAAE,KAAK,CAAC,eAAe,IAAI,IAAI;YAC9C,WAAW,EAAE,KAAK,CAAC,WAAW,IAAI,KAAK;YACvC,gBAAgB,EAAE,KAAK,CAAC,gBAAgB;SACzC,CAAC;QAEF,QAAQ,CAAC,mBAAmB,EAAE,EAAE,GAAG,EAAE,KAAK,CAAC,GAAG,EAAE,GAAG,OAAO,EAAE,CAAC,CAAC;QAE9D,MAAM,MAAM,GAAG,MAAM,oBAAoB,CAA0B;YACjE,GAAG,EAAE,KAAK,CAAC,GAAG;YACd,cAAc,EAAE,UAAU;YAC1B,aAAa,EAAE,KAAK,CAAC,aAAa;YAClC,OAAO,EAAE,KAAK,CAAC,OAAO;YACtB,SAAS,EAAE,CAAC,IAAI,EAAE,GAAG,EAAE,EAAE,CAAC,mBAAmB,CAAC,IAAI,EAAE,GAAG,EAAE,OAAO,CAAC;YACjE,SAAS,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,QAAQ;YAClC,WAAW,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;gBACxB,QAAQ,EAAE,MAAM;gBAChB,KAAK,EAAE,SAAS;gBAChB,GAAG,EAAE,SAAS;gBACd,SAAS,EAAE,KAAK;aACjB,CAAC;SACH,CAAC,CAAC;QAEH,MAAM,iBAAiB,GAAG;YACxB,GAAG,EAAE,MAAM,CAAC,GAAG;YACf,KAAK,EAAE,MAAM,CAAC,IAAI,CAAC,KAAK;YACxB,SAAS,EAAE,MAAM,CAAC,SAAS;YAC3B,QAAQ,EAAE,MAAM,CAAC,IAAI,CAAC,QAAQ;YAC9B,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,IAAI,EAAE,GAAG,EAAE,MAAM,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC;YAChD,MAAM,EAAE,MAAM,CAAC,SAAS;YACxB,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,SAAS,IAAI,EAAE,SAAS,EAAE,MAAM,CAAC,IAAI,CAAC,SAAS,EAAE,CAAC;SACnE,CAAC;QAEF,OAAO;YACL,OAAO,EAAE;gBACP;oBACE,IAAI,EAAE,MAAe;oBACrB,IAAI,EAAE,IAAI,CAAC,SAAS,CAClB,iBAAiB,EACjB,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,EACnC,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CACjC;iBACF;aACF;YACD,iBAAiB;SAClB,CAAC;IACJ,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,QAAQ,CACN,2BAA2B,EAC3B,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS,CAC3C,CAAC;QACF,OAAO,eAAe,CAAC,KAAK,EAAE,KAAK,CAAC,GAAG,EAAE,0BAA0B,CAAC,CAAC;IACvE,CAAC;AACH,CAAC"}
|
|
@@ -1,20 +1,12 @@
|
|
|
1
1
|
import type { FetchUrlInput } from '../../config/types.js';
|
|
2
2
|
export declare const FETCH_URL_TOOL_NAME = "fetch-url";
|
|
3
3
|
export declare const FETCH_URL_TOOL_DESCRIPTION = "Fetches a webpage and converts it to AI-readable JSONL format with semantic content blocks. Supports custom headers, retries, and content length limits.";
|
|
4
|
-
export declare function fetchUrlToolHandler(input: FetchUrlInput): Promise<
|
|
4
|
+
export declare function fetchUrlToolHandler(input: FetchUrlInput): Promise<{
|
|
5
5
|
content: {
|
|
6
|
-
type:
|
|
6
|
+
type: 'text';
|
|
7
7
|
text: string;
|
|
8
8
|
}[];
|
|
9
|
-
structuredContent
|
|
10
|
-
|
|
11
|
-
url: string;
|
|
12
|
-
title: string | undefined;
|
|
13
|
-
contentBlocks: number;
|
|
14
|
-
fetchedAt: string;
|
|
15
|
-
format: "jsonl";
|
|
16
|
-
content: string;
|
|
17
|
-
cached: boolean;
|
|
18
|
-
};
|
|
9
|
+
structuredContent?: Record<string, unknown>;
|
|
10
|
+
isError?: boolean;
|
|
19
11
|
}>;
|
|
20
12
|
//# sourceMappingURL=fetch-url.tool.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"fetch-url.tool.d.ts","sourceRoot":"","sources":["../../../src/tools/handlers/fetch-url.tool.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,aAAa,EAEd,MAAM,uBAAuB,CAAC;AAmB/B,eAAO,MAAM,mBAAmB,cAAc,CAAC;AAC/C,eAAO,MAAM,0BAA0B,6JACqH,CAAC;AA8B7J,wBAAsB,mBAAmB,CAAC,KAAK,EAAE,aAAa
|
|
1
|
+
{"version":3,"file":"fetch-url.tool.d.ts","sourceRoot":"","sources":["../../../src/tools/handlers/fetch-url.tool.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,aAAa,EAEd,MAAM,uBAAuB,CAAC;AAmB/B,eAAO,MAAM,mBAAmB,cAAc,CAAC;AAC/C,eAAO,MAAM,0BAA0B,6JACqH,CAAC;AA8B7J,wBAAsB,mBAAmB,CAAC,KAAK,EAAE,aAAa,GAAG,OAAO,CAAC;IACvE,OAAO,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC;IAC1C,iBAAiB,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAC5C,OAAO,CAAC,EAAE,OAAO,CAAC;CACnB,CAAC,CAkED"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"fetch-url.tool.js","sourceRoot":"","sources":["../../../src/tools/handlers/fetch-url.tool.ts"],"names":[],"mappings":"AAKA,OAAO,EAAE,cAAc,EAAE,MAAM,6BAA6B,CAAC;AAC7D,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,0BAA0B,CAAC;AAC9D,OAAO,EAAE,SAAS,EAAE,MAAM,0BAA0B,CAAC;AAErD,OAAO,EACL,uBAAuB,EACvB,eAAe,GAChB,MAAM,mCAAmC,CAAC;AAC3C,OAAO,EACL,aAAa,EACb,gBAAgB,EAChB,eAAe,GAChB,MAAM,oBAAoB,CAAC;AAC5B,OAAO,EAAE,oBAAoB,EAAE,MAAM,4BAA4B,CAAC;AAElE,OAAO,EAAE,OAAO,EAAE,MAAM,yCAAyC,CAAC;AAElE,MAAM,CAAC,MAAM,mBAAmB,GAAG,WAAW,CAAC;AAC/C,MAAM,CAAC,MAAM,0BAA0B,GACrC,0JAA0J,CAAC;AAE7J,SAAS,gBAAgB,CACvB,IAAY,EACZ,GAAW,EACX,OAAkE;IAElE,gFAAgF;IAChF,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,aAAa,EAAE,GAAG,cAAc,CAAC,IAAI,EAAE,GAAG,EAAE;QACrE,cAAc,EAAE,OAAO,CAAC,kBAAkB;KAC3C,CAAC,CAAC;IACH,MAAM,UAAU,GAAG,gBAAgB,CAAC,OAAO,CAAC,kBAAkB,EAAE,OAAO,CAAC,CAAC;IACzE,MAAM,UAAU,GAAG,UAAU,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC;IACvD,MAAM,aAAa,GAAG,SAAS,CAAC,UAAU,CAAC,CAAC;IAC5C,MAAM,QAAQ,GAAG,aAAa,CAC5B,GAAG,EACH,OAAO,EACP,aAAa,EACb,UAAU,EACV,OAAO,CAAC,eAAe,CACxB,CAAC;IACF,MAAM,KAAK,GAAG,UAAU,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,aAAa,CAAC,KAAK,CAAC;IAE/D,OAAO;QACL,OAAO,EAAE,OAAO,CAAC,aAAa,EAAE,QAAQ,CAAC;QACzC,aAAa,EAAE,aAAa,CAAC,MAAM;QACnC,KAAK;KACN,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,mBAAmB,CAAC,KAAoB;
|
|
1
|
+
{"version":3,"file":"fetch-url.tool.js","sourceRoot":"","sources":["../../../src/tools/handlers/fetch-url.tool.ts"],"names":[],"mappings":"AAKA,OAAO,EAAE,cAAc,EAAE,MAAM,6BAA6B,CAAC;AAC7D,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,0BAA0B,CAAC;AAC9D,OAAO,EAAE,SAAS,EAAE,MAAM,0BAA0B,CAAC;AAErD,OAAO,EACL,uBAAuB,EACvB,eAAe,GAChB,MAAM,mCAAmC,CAAC;AAC3C,OAAO,EACL,aAAa,EACb,gBAAgB,EAChB,eAAe,GAChB,MAAM,oBAAoB,CAAC;AAC5B,OAAO,EAAE,oBAAoB,EAAE,MAAM,4BAA4B,CAAC;AAElE,OAAO,EAAE,OAAO,EAAE,MAAM,yCAAyC,CAAC;AAElE,MAAM,CAAC,MAAM,mBAAmB,GAAG,WAAW,CAAC;AAC/C,MAAM,CAAC,MAAM,0BAA0B,GACrC,0JAA0J,CAAC;AAE7J,SAAS,gBAAgB,CACvB,IAAY,EACZ,GAAW,EACX,OAAkE;IAElE,gFAAgF;IAChF,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,aAAa,EAAE,GAAG,cAAc,CAAC,IAAI,EAAE,GAAG,EAAE;QACrE,cAAc,EAAE,OAAO,CAAC,kBAAkB;KAC3C,CAAC,CAAC;IACH,MAAM,UAAU,GAAG,gBAAgB,CAAC,OAAO,CAAC,kBAAkB,EAAE,OAAO,CAAC,CAAC;IACzE,MAAM,UAAU,GAAG,UAAU,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC;IACvD,MAAM,aAAa,GAAG,SAAS,CAAC,UAAU,CAAC,CAAC;IAC5C,MAAM,QAAQ,GAAG,aAAa,CAC5B,GAAG,EACH,OAAO,EACP,aAAa,EACb,UAAU,EACV,OAAO,CAAC,eAAe,CACxB,CAAC;IACF,MAAM,KAAK,GAAG,UAAU,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,aAAa,CAAC,KAAK,CAAC;IAE/D,OAAO;QACL,OAAO,EAAE,OAAO,CAAC,aAAa,EAAE,QAAQ,CAAC;QACzC,aAAa,EAAE,aAAa,CAAC,MAAM;QACnC,KAAK;KACN,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,mBAAmB,CAAC,KAAoB;IAK5D,IAAI,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC;QACf,OAAO,uBAAuB,CAAC,iBAAiB,EAAE,EAAE,EAAE,kBAAkB,CAAC,CAAC;IAC5E,CAAC;IAED,IAAI,CAAC;QACH,MAAM,kBAAkB,GAAG,KAAK,CAAC,kBAAkB,IAAI,IAAI,CAAC;QAC5D,MAAM,eAAe,GAAG,KAAK,CAAC,eAAe,IAAI,IAAI,CAAC;QAEtD,QAAQ,CAAC,cAAc,EAAE;YACvB,GAAG,EAAE,KAAK,CAAC,GAAG;YACd,kBAAkB;YAClB,eAAe;SAChB,CAAC,CAAC;QAEH,MAAM,MAAM,GAAG,MAAM,oBAAoB,CAAuB;YAC9D,GAAG,EAAE,KAAK,CAAC,GAAG;YACd,cAAc,EAAE,KAAK;YACrB,aAAa,EAAE,KAAK,CAAC,aAAa;YAClC,OAAO,EAAE,KAAK,CAAC,OAAO;YACtB,SAAS,EAAE,CAAC,IAAI,EAAE,GAAG,EAAE,EAAE,CACvB,gBAAgB,CAAC,IAAI,EAAE,GAAG,EAAE,EAAE,kBAAkB,EAAE,eAAe,EAAE,CAAC;YACtE,SAAS,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,OAAO;YACjC,WAAW,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;gBACxB,OAAO,EAAE,MAAM;gBACf,aAAa,EAAE,CAAC;gBAChB,KAAK,EAAE,SAAS;aACjB,CAAC;SACH,CAAC,CAAC;QAEH,MAAM,EAAE,OAAO,EAAE,SAAS,EAAE,GAAG,eAAe,CAC5C,MAAM,CAAC,IAAI,CAAC,OAAO,EACnB,KAAK,CAAC,gBAAgB,CACvB,CAAC;QAEF,MAAM,iBAAiB,GAAG;YACxB,GAAG,EAAE,MAAM,CAAC,GAAG;YACf,KAAK,EAAE,MAAM,CAAC,IAAI,CAAC,KAAK;YACxB,aAAa,EAAE,MAAM,CAAC,IAAI,CAAC,aAAa;YACxC,SAAS,EAAE,MAAM,CAAC,SAAS;YAC3B,MAAM,EAAE,OAAgB;YACxB,OAAO;YACP,MAAM,EAAE,MAAM,CAAC,SAAS;YACxB,GAAG,CAAC,SAAS,IAAI,EAAE,SAAS,EAAE,CAAC;SAChC,CAAC;QAEF,OAAO;YACL,OAAO,EAAE;gBACP;oBACE,IAAI,EAAE,MAAe;oBACrB,IAAI,EAAE,IAAI,CAAC,SAAS,CAClB,iBAAiB,EACjB,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,EACnC,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CACjC;iBACF;aACF;YACD,iBAAiB;SAClB,CAAC;IACJ,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,QAAQ,CACN,sBAAsB,EACtB,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS,CAC3C,CAAC;QACF,OAAO,eAAe,CAAC,KAAK,EAAE,KAAK,CAAC,GAAG,EAAE,qBAAqB,CAAC,CAAC;IAClE,CAAC;AACH,CAAC"}
|