@j0hanz/superfetch 1.0.6 → 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +200 -36
- package/dist/config/index.d.ts +5 -5
- package/dist/config/index.d.ts.map +1 -1
- package/dist/config/index.js +38 -19
- package/dist/config/index.js.map +1 -1
- package/dist/config/types.d.ts +98 -57
- package/dist/config/types.d.ts.map +1 -1
- package/dist/errors/app-error.d.ts +4 -28
- package/dist/errors/app-error.d.ts.map +1 -1
- package/dist/errors/app-error.js +10 -51
- package/dist/errors/app-error.js.map +1 -1
- package/dist/index.js +10 -55
- package/dist/index.js.map +1 -1
- package/dist/middleware/error-handler.d.ts +2 -2
- package/dist/middleware/error-handler.d.ts.map +1 -1
- package/dist/middleware/error-handler.js +12 -14
- package/dist/middleware/error-handler.js.map +1 -1
- package/dist/middleware/rate-limiter.d.ts.map +1 -1
- package/dist/middleware/rate-limiter.js +0 -8
- package/dist/middleware/rate-limiter.js.map +1 -1
- package/dist/parsers/base-html-element-parser.d.ts +43 -0
- package/dist/parsers/base-html-element-parser.d.ts.map +1 -0
- package/dist/parsers/base-html-element-parser.js +59 -0
- package/dist/parsers/base-html-element-parser.js.map +1 -0
- package/dist/parsers/heading-element-parser.d.ts +14 -0
- package/dist/parsers/heading-element-parser.d.ts.map +1 -0
- package/dist/parsers/heading-element-parser.js +26 -0
- package/dist/parsers/heading-element-parser.js.map +1 -0
- package/dist/parsers/image-element-parser.d.ts +16 -0
- package/dist/parsers/image-element-parser.d.ts.map +1 -0
- package/dist/parsers/image-element-parser.js +33 -0
- package/dist/parsers/image-element-parser.js.map +1 -0
- package/dist/parsers/link-element-parser.d.ts +15 -0
- package/dist/parsers/link-element-parser.d.ts.map +1 -0
- package/dist/parsers/link-element-parser.js +28 -0
- package/dist/parsers/link-element-parser.js.map +1 -0
- package/dist/parsers/open-graph-parser.d.ts +17 -0
- package/dist/parsers/open-graph-parser.d.ts.map +1 -0
- package/dist/parsers/open-graph-parser.js +41 -0
- package/dist/parsers/open-graph-parser.js.map +1 -0
- package/dist/parsers/schema-org-parser.d.ts +17 -0
- package/dist/parsers/schema-org-parser.d.ts.map +1 -0
- package/dist/parsers/schema-org-parser.js +32 -0
- package/dist/parsers/schema-org-parser.js.map +1 -0
- package/dist/parsers/standard-meta-parser.d.ts +18 -0
- package/dist/parsers/standard-meta-parser.d.ts.map +1 -0
- package/dist/parsers/standard-meta-parser.js +32 -0
- package/dist/parsers/standard-meta-parser.js.map +1 -0
- package/dist/parsers/twitter-card-parser.d.ts +17 -0
- package/dist/parsers/twitter-card-parser.d.ts.map +1 -0
- package/dist/parsers/twitter-card-parser.js +41 -0
- package/dist/parsers/twitter-card-parser.js.map +1 -0
- package/dist/resources/cached-content.d.ts +0 -1
- package/dist/resources/cached-content.d.ts.map +1 -1
- package/dist/resources/cached-content.js +3 -9
- package/dist/resources/cached-content.js.map +1 -1
- package/dist/resources/index.d.ts.map +1 -1
- package/dist/resources/index.js +8 -8
- package/dist/resources/index.js.map +1 -1
- package/dist/server.d.ts.map +1 -1
- package/dist/server.js +10 -10
- package/dist/server.js.map +1 -1
- package/dist/services/cache.d.ts +0 -28
- package/dist/services/cache.d.ts.map +1 -1
- package/dist/services/cache.js +10 -173
- package/dist/services/cache.js.map +1 -1
- package/dist/services/extractor.d.ts +1 -11
- package/dist/services/extractor.d.ts.map +1 -1
- package/dist/services/extractor.js +86 -84
- package/dist/services/extractor.js.map +1 -1
- package/dist/services/fetcher.d.ts +2 -13
- package/dist/services/fetcher.d.ts.map +1 -1
- package/dist/services/fetcher.js +48 -79
- package/dist/services/fetcher.js.map +1 -1
- package/dist/services/logger.d.ts +5 -4
- package/dist/services/logger.d.ts.map +1 -1
- package/dist/services/logger.js +27 -42
- package/dist/services/logger.js.map +1 -1
- package/dist/services/parser.d.ts.map +1 -1
- package/dist/services/parser.js +35 -26
- package/dist/services/parser.js.map +1 -1
- package/dist/services/session-manager.d.ts +18 -0
- package/dist/services/session-manager.d.ts.map +1 -0
- package/dist/services/session-manager.js +73 -0
- package/dist/services/session-manager.js.map +1 -0
- package/dist/strategies/exponential-backoff-strategy.d.ts +13 -0
- package/dist/strategies/exponential-backoff-strategy.d.ts.map +1 -0
- package/dist/strategies/exponential-backoff-strategy.js +32 -0
- package/dist/strategies/exponential-backoff-strategy.js.map +1 -0
- package/dist/tools/handlers/fetch-links.tool.d.ts +2 -9
- package/dist/tools/handlers/fetch-links.tool.d.ts.map +1 -1
- package/dist/tools/handlers/fetch-links.tool.js +0 -1
- package/dist/tools/handlers/fetch-links.tool.js.map +1 -1
- package/dist/tools/handlers/fetch-markdown.tool.d.ts +5 -2
- package/dist/tools/handlers/fetch-markdown.tool.d.ts.map +1 -1
- package/dist/tools/handlers/fetch-markdown.tool.js +23 -33
- package/dist/tools/handlers/fetch-markdown.tool.js.map +1 -1
- package/dist/tools/handlers/fetch-url.tool.d.ts +2 -9
- package/dist/tools/handlers/fetch-url.tool.d.ts.map +1 -1
- package/dist/tools/handlers/fetch-url.tool.js +15 -20
- package/dist/tools/handlers/fetch-url.tool.js.map +1 -1
- package/dist/tools/handlers/fetch-urls.tool.d.ts +2 -9
- package/dist/tools/handlers/fetch-urls.tool.d.ts.map +1 -1
- package/dist/tools/handlers/fetch-urls.tool.js +124 -105
- package/dist/tools/handlers/fetch-urls.tool.js.map +1 -1
- package/dist/tools/index.d.ts.map +1 -1
- package/dist/tools/index.js +0 -4
- package/dist/tools/index.js.map +1 -1
- package/dist/tools/utils/common.d.ts +6 -7
- package/dist/tools/utils/common.d.ts.map +1 -1
- package/dist/tools/utils/common.js +8 -8
- package/dist/tools/utils/common.js.map +1 -1
- package/dist/tools/utils/fetch-pipeline.d.ts +8 -0
- package/dist/tools/utils/fetch-pipeline.d.ts.map +1 -1
- package/dist/tools/utils/fetch-pipeline.js +47 -79
- package/dist/tools/utils/fetch-pipeline.js.map +1 -1
- package/dist/transformers/jsonl.transformer.d.ts +1 -1
- package/dist/transformers/jsonl.transformer.d.ts.map +1 -1
- package/dist/transformers/jsonl.transformer.js +15 -10
- package/dist/transformers/jsonl.transformer.js.map +1 -1
- package/dist/transformers/markdown.transformer.d.ts.map +1 -1
- package/dist/transformers/markdown.transformer.js +58 -62
- package/dist/transformers/markdown.transformer.js.map +1 -1
- package/dist/utils/concurrency.d.ts +2 -5
- package/dist/utils/concurrency.d.ts.map +1 -1
- package/dist/utils/concurrency.js +19 -19
- package/dist/utils/concurrency.js.map +1 -1
- package/dist/utils/content-cleaner.d.ts +0 -25
- package/dist/utils/content-cleaner.d.ts.map +1 -1
- package/dist/utils/content-cleaner.js +12 -187
- package/dist/utils/content-cleaner.js.map +1 -1
- package/dist/utils/html-truncator.d.ts +2 -0
- package/dist/utils/html-truncator.d.ts.map +1 -0
- package/dist/utils/html-truncator.js +14 -0
- package/dist/utils/html-truncator.js.map +1 -0
- package/dist/utils/language-detector.d.ts +0 -3
- package/dist/utils/language-detector.d.ts.map +1 -1
- package/dist/utils/language-detector.js +0 -11
- package/dist/utils/language-detector.js.map +1 -1
- package/dist/utils/sanitizer.d.ts.map +1 -1
- package/dist/utils/sanitizer.js +7 -5
- package/dist/utils/sanitizer.js.map +1 -1
- package/dist/utils/tool-error-handler.d.ts.map +1 -1
- package/dist/utils/tool-error-handler.js +15 -42
- package/dist/utils/tool-error-handler.js.map +1 -1
- package/dist/utils/url-validator.d.ts +0 -9
- package/dist/utils/url-validator.d.ts.map +1 -1
- package/dist/utils/url-validator.js +14 -33
- package/dist/utils/url-validator.js.map +1 -1
- package/package.json +3 -4
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"logger.js","sourceRoot":"","sources":["../../src/services/logger.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,
|
|
1
|
+
{"version":3,"file":"logger.js","sourceRoot":"","sources":["../../src/services/logger.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAG5C,SAAS,cAAc,CAAC,IAAkB;IACxC,OAAO,IAAI,IAAI,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;AAChF,CAAC;AAED,SAAS,eAAe;IACtB,OAAO,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;AAClC,CAAC;AAED,SAAS,cAAc,CACrB,KAAe,EACf,OAAe,EACf,IAAkB;IAElB,OAAO,IAAI,eAAe,EAAE,KAAK,KAAK,CAAC,WAAW,EAAE,KAAK,OAAO,GAAG,cAAc,CAAC,IAAI,CAAC,EAAE,CAAC;AAC5F,CAAC;AAED,SAAS,SAAS,CAAC,KAAe;IAChC,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,OAAO;QAAE,OAAO,KAAK,CAAC;IAC1C,IAAI,KAAK,KAAK,OAAO;QAAE,OAAO,MAAM,CAAC,OAAO,CAAC,KAAK,KAAK,OAAO,CAAC;IAC/D,OAAO,IAAI,CAAC;AACd,CAAC;AAED,MAAM,UAAU,OAAO,CAAC,OAAe,EAAE,IAAkB;IACzD,IAAI,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC;QACtB,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,MAAM,EAAE,OAAO,EAAE,IAAI,CAAC,CAAC,CAAC;IACrD,CAAC;AACH,CAAC;AAED,MAAM,UAAU,QAAQ,CAAC,OAAe,EAAE,IAAkB;IAC1D,IAAI,SAAS,CAAC,OAAO,CAAC,EAAE,CAAC;QACvB,OAAO,CAAC,KAAK,CAAC,cAAc,CAAC,OAAO,EAAE,OAAO,EAAE,IAAI,CAAC,CAAC,CAAC;IACxD,CAAC;AACH,CAAC;AAED,MAAM,UAAU,OAAO,CAAC,OAAe,EAAE,IAAkB;IACzD,IAAI,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC;QACtB,OAAO,CAAC,IAAI,CAAC,cAAc,CAAC,MAAM,EAAE,OAAO,EAAE,IAAI,CAAC,CAAC,CAAC;IACtD,CAAC;AACH,CAAC;AAED,MAAM,UAAU,QAAQ,CAAC,OAAe,EAAE,KAA2B;IACnE,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC;QAAE,OAAO;IAEhC,MAAM,SAAS,GACb,KAAK,YAAY,KAAK;QACpB,CAAC,CAAC,EAAE,KAAK,EAAE,KAAK,CAAC,OAAO,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE;QAC9C,CAAC,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC;IAEpB,OAAO,CAAC,KAAK,CAAC,cAAc,CAAC,OAAO,EAAE,OAAO,EAAE,SAAS,CAAC,CAAC,CAAC;AAC7D,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"parser.d.ts","sourceRoot":"","sources":["../../src/services/parser.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EAGV,iBAAiB,EAOlB,MAAM,oBAAoB,CAAC;
|
|
1
|
+
{"version":3,"file":"parser.d.ts","sourceRoot":"","sources":["../../src/services/parser.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EAGV,iBAAiB,EAOlB,MAAM,oBAAoB,CAAC;AAuO5B,wBAAgB,SAAS,CAAC,IAAI,EAAE,MAAM,GAAG,iBAAiB,EAAE,CA8B3D"}
|
package/dist/services/parser.js
CHANGED
|
@@ -1,10 +1,40 @@
|
|
|
1
1
|
import * as cheerio from 'cheerio';
|
|
2
2
|
import { config } from '../config/index.js';
|
|
3
3
|
import { cleanCodeBlock, cleanHeading, cleanListItems, cleanParagraph, removeInlineTimestamps, } from '../utils/content-cleaner.js';
|
|
4
|
-
import {
|
|
4
|
+
import { truncateHtml } from '../utils/html-truncator.js';
|
|
5
5
|
import { sanitizeText } from '../utils/sanitizer.js';
|
|
6
6
|
import { logWarn } from './logger.js';
|
|
7
|
-
|
|
7
|
+
function detectLanguageFromCode(code) {
|
|
8
|
+
// Common language patterns for code block detection
|
|
9
|
+
const patterns = [
|
|
10
|
+
[
|
|
11
|
+
/^\s*import\s+.*\s+from\s+['"]react['"]|<[A-Z][a-zA-Z]*[\s/>]|jsx\s*:|className=/m,
|
|
12
|
+
'jsx',
|
|
13
|
+
],
|
|
14
|
+
[
|
|
15
|
+
/:\s*(string|number|boolean|void|any|unknown|never)\b|interface\s+\w+|type\s+\w+\s*=/m,
|
|
16
|
+
'typescript',
|
|
17
|
+
],
|
|
18
|
+
[/^\s*(fn|let\s+mut|impl|struct|enum|use\s+\w+::)/m, 'rust'],
|
|
19
|
+
[
|
|
20
|
+
/^\s*(export|const|let|var|function|class|async|await)\b|^\s*import\s+.*['"]]/m,
|
|
21
|
+
'javascript',
|
|
22
|
+
],
|
|
23
|
+
[/^\s*(def|class|import|from|if __name__|print\()/m, 'python'],
|
|
24
|
+
[
|
|
25
|
+
/^\s*(npm|yarn|pnpm|npx|brew|apt|pip|cargo|go )\s+(install|add|run|build|start)/m,
|
|
26
|
+
'bash',
|
|
27
|
+
],
|
|
28
|
+
[/^\s*[$#]\s+\w+|^\s*#!|^\s*(sudo|chmod|mkdir|cd|ls|cat|echo)\s+/m, 'bash'],
|
|
29
|
+
[/^\s*[.#@]?[\w-]+\s*\{[^}]*\}|@media|@import|@keyframes/m, 'css'],
|
|
30
|
+
[/^\s*<(!DOCTYPE|html|head|body|div|span|p|a|script|style)\b/im, 'html'],
|
|
31
|
+
[/^\s*\{\s*"|^\s*\[\s*("|\d|true|false|null)/m, 'json'],
|
|
32
|
+
[/^\s*[\w-]+:\s*.+$/m, 'yaml'],
|
|
33
|
+
[/^\s*(SELECT|INSERT|UPDATE|DELETE|CREATE|ALTER|DROP)\s+/im, 'sql'],
|
|
34
|
+
[/^\s*(func|package|import\s+")/m, 'go'],
|
|
35
|
+
];
|
|
36
|
+
return patterns.find(([pattern]) => pattern.test(code))?.[1];
|
|
37
|
+
}
|
|
8
38
|
const CONTENT_SELECTOR = 'h1, h2, h3, h4, h5, h6, p, ul, ol, pre, code:not(pre code), table, img, blockquote';
|
|
9
39
|
function parseHeading($, element) {
|
|
10
40
|
const rawText = sanitizeText($(element).text());
|
|
@@ -28,7 +58,6 @@ function parseParagraph($, element) {
|
|
|
28
58
|
}
|
|
29
59
|
function parseList($, element) {
|
|
30
60
|
const rawItems = [];
|
|
31
|
-
// Direct .each() iteration - no array allocation overhead
|
|
32
61
|
$(element)
|
|
33
62
|
.find('li')
|
|
34
63
|
.each((_, li) => {
|
|
@@ -54,13 +83,11 @@ function parseCode($, element) {
|
|
|
54
83
|
// Try to get language from class attribute first
|
|
55
84
|
const className = $(element).attr('class') ?? '';
|
|
56
85
|
const dataLang = $(element).attr('data-language') ?? '';
|
|
57
|
-
// Check multiple possible class patterns for language
|
|
58
86
|
const languageMatch = /language-(\w+)/.exec(className) ??
|
|
59
87
|
/lang-(\w+)/.exec(className) ??
|
|
60
88
|
/highlight-(\w+)/.exec(className) ??
|
|
61
89
|
/^(\w+)$/.exec(dataLang);
|
|
62
|
-
|
|
63
|
-
const language = languageMatch?.[1] ?? detectLanguage(text);
|
|
90
|
+
const language = languageMatch?.[1] ?? detectLanguageFromCode(text);
|
|
64
91
|
return {
|
|
65
92
|
type: 'code',
|
|
66
93
|
language,
|
|
@@ -71,7 +98,6 @@ function parseTable($, element) {
|
|
|
71
98
|
const headers = [];
|
|
72
99
|
const rows = [];
|
|
73
100
|
const $table = $(element);
|
|
74
|
-
// Direct .each() iteration - no array allocation
|
|
75
101
|
$table.find('thead th, thead td').each((_, cell) => {
|
|
76
102
|
headers.push(sanitizeText($(cell).text()));
|
|
77
103
|
});
|
|
@@ -165,28 +191,11 @@ function filterBlocks(blocks) {
|
|
|
165
191
|
export function parseHtml(html) {
|
|
166
192
|
if (!html || typeof html !== 'string')
|
|
167
193
|
return [];
|
|
168
|
-
|
|
169
|
-
if (html.length > config.constants.maxHtmlSize) {
|
|
170
|
-
logWarn('HTML content exceeds maximum size, truncating at safe boundary', {
|
|
171
|
-
size: html.length,
|
|
172
|
-
maxSize: config.constants.maxHtmlSize,
|
|
173
|
-
});
|
|
174
|
-
// Use lastIndexOf for O(log n) reverse search (10-100x faster than while loop)
|
|
175
|
-
const lastTag = html.lastIndexOf('>', config.constants.maxHtmlSize);
|
|
176
|
-
// If we found a tag boundary near the limit (within 10% buffer)
|
|
177
|
-
if (lastTag !== -1 && lastTag > config.constants.maxHtmlSize * 0.9) {
|
|
178
|
-
processedHtml = html.substring(0, lastTag + 1);
|
|
179
|
-
}
|
|
180
|
-
else {
|
|
181
|
-
// Fallback: simple truncation if no suitable boundary found
|
|
182
|
-
processedHtml = html.substring(0, config.constants.maxHtmlSize);
|
|
183
|
-
}
|
|
184
|
-
}
|
|
194
|
+
const processedHtml = truncateHtml(html);
|
|
185
195
|
try {
|
|
186
196
|
const $ = cheerio.load(processedHtml);
|
|
187
197
|
const blocks = [];
|
|
188
198
|
$('script, style, noscript, iframe, svg').remove();
|
|
189
|
-
// Direct .each() iteration - no array allocation overhead
|
|
190
199
|
$('body')
|
|
191
200
|
.find(CONTENT_SELECTOR)
|
|
192
201
|
.each((_, element) => {
|
|
@@ -196,7 +205,7 @@ export function parseHtml(html) {
|
|
|
196
205
|
blocks.push(block);
|
|
197
206
|
}
|
|
198
207
|
catch {
|
|
199
|
-
|
|
208
|
+
/* skip */
|
|
200
209
|
}
|
|
201
210
|
});
|
|
202
211
|
return filterBlocks(blocks);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"parser.js","sourceRoot":"","sources":["../../src/services/parser.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,OAAO,MAAM,SAAS,CAAC;AAInC,OAAO,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAa5C,OAAO,EACL,cAAc,EACd,YAAY,EACZ,cAAc,EACd,cAAc,EACd,sBAAsB,GACvB,MAAM,6BAA6B,CAAC;AACrC,OAAO,EAAE,
|
|
1
|
+
{"version":3,"file":"parser.js","sourceRoot":"","sources":["../../src/services/parser.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,OAAO,MAAM,SAAS,CAAC;AAInC,OAAO,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAa5C,OAAO,EACL,cAAc,EACd,YAAY,EACZ,cAAc,EACd,cAAc,EACd,sBAAsB,GACvB,MAAM,6BAA6B,CAAC;AACrC,OAAO,EAAE,YAAY,EAAE,MAAM,4BAA4B,CAAC;AAC1D,OAAO,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAC;AAErD,OAAO,EAAE,OAAO,EAAE,MAAM,aAAa,CAAC;AAEtC,SAAS,sBAAsB,CAAC,IAAY;IAC1C,oDAAoD;IACpD,MAAM,QAAQ,GAAgC;QAC5C;YACE,kFAAkF;YAClF,KAAK;SACN;QACD;YACE,sFAAsF;YACtF,YAAY;SACb;QACD,CAAC,kDAAkD,EAAE,MAAM,CAAC;QAC5D;YACE,+EAA+E;YAC/E,YAAY;SACb;QACD,CAAC,kDAAkD,EAAE,QAAQ,CAAC;QAC9D;YACE,iFAAiF;YACjF,MAAM;SACP;QACD,CAAC,iEAAiE,EAAE,MAAM,CAAC;QAC3E,CAAC,yDAAyD,EAAE,KAAK,CAAC;QAClE,CAAC,8DAA8D,EAAE,MAAM,CAAC;QACxE,CAAC,6CAA6C,EAAE,MAAM,CAAC;QACvD,CAAC,oBAAoB,EAAE,MAAM,CAAC;QAC9B,CAAC,0DAA0D,EAAE,KAAK,CAAC;QACnE,CAAC,gCAAgC,EAAE,IAAI,CAAC;KACzC,CAAC;IAEF,OAAO,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;AAC/D,CAAC;AAED,MAAM,gBAAgB,GACpB,oFAAoF,CAAC;AAEvF,SAAS,YAAY,CAAC,CAAa,EAAE,OAAgB;IACnD,MAAM,OAAO,GAAG,YAAY,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;IAChD,MAAM,IAAI,GAAG,YAAY,CAAC,OAAO,CAAC,CAAC;IACnC,IAAI,CAAC,IAAI;QAAE,OAAO,IAAI,CAAC;IAEvB,OAAO;QACL,IAAI,EAAE,SAAS;QACf,KAAK,EAAE,QAAQ,CAAC,OAAO,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC;QACjD,IAAI;KACL,CAAC;AACJ,CAAC;AAED,SAAS,cAAc,CACrB,CAAa,EACb,OAAgB;IAEhB,IAAI,OAAO,GAAG,YAAY,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;IAC9C,8DAA8D;IAC9D,OAAO,GAAG,sBAAsB,CAAC,OAAO,CAAC,CAAC;IAC1C,MAAM,IAAI,GAAG,cAAc,CAAC,OAAO,CAAC,CAAC;IACrC,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,UAAU,CAAC,kBAAkB;QAAE,OAAO,IAAI,CAAC;IAE7E,OAAO,EAAE,IAAI,EAAE,WAAW,EAAE,IAAI,EAAE,CAAC;AACrC,CAAC;AAED,SAAS,SAAS,CAAC,CAAa,EAAE,OAAgB;IAChD,MAAM,QAAQ,GAAa,EAAE,CAAC;IAE9B,CAAC,CAAC,OAAO,CAAC;SACP,IAAI,CAAC,IAAI,CAAC;SACV,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,EAAE;QACd,MAAM,IAAI,GAAG,YAAY,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;QACxC,IAAI,IAAI;YAAE,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAChC,CAAC,CAAC,CAAC;IAEL,mCAAmC;IACnC,MAAM,KAAK,GAAG,cAAc,CAAC,QAAQ,CAAC,CAAC;IACvC,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IAEpC,OAAO;QACL,IAAI,EAAE,MAAM;QACZ,OAAO,EAAE,OAAO,CAAC,OAAO,CAAC,WAAW,EAAE,KAAK,IAAI;QAC/C,KAAK;KACN,CAAC;AACJ,CAAC;AAED,SAAS,SAAS,CAAC,CAAa,EAAE,OAAgB;IAChD,MAAM,OAAO,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;IACzC,MAAM,IAAI,GAAG,cAAc,CAAC,OAAO,CAAC,CAAC;IACrC,IAAI,CAAC,IAAI;QAAE,OAAO,IAAI,CAAC;IAEvB,iDAAiD;IACjD,MAAM,SAAS,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC;IACjD,MAAM,QAAQ,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,IAAI,EAAE,CAAC;IAExD,MAAM,aAAa,GACjB,gBAAgB,CAAC,IAAI,CAAC,SAAS,CAAC;QAChC,YAAY,CAAC,IAAI,CAAC,SAAS,CAAC;QAC5B,iBAAiB,CAAC,IAAI,CAAC,SAAS,CAAC;QACjC,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IAE3B,MAAM,QAAQ,GAAG,aAAa,EAAE,CAAC,CAAC,CAAC,IAAI,sBAAsB,CAAC,IAAI,CAAC,CAAC;IAEpE,OAAO;QACL,IAAI,EAAE,MAAM;QACZ,QAAQ;QACR,IAAI;KACL,CAAC;AACJ,CAAC;AAED,SAAS,UAAU,CAAC,CAAa,EAAE,OAAgB;IACjD,MAAM,OAAO,GAAa,EAAE,CAAC;IAC7B,MAAM,IAAI,GAAe,EAAE,CAAC;IAC5B,MAAM,MAAM,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC;IAE1B,MAAM,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,EAAE;QACjD,OAAO,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;IAC7C,CAAC,CAAC,CAAC;IAEH,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACzB,MAAM;aACH,IAAI,CAAC,IAAI,CAAC;aACV,KAAK,EAAE;aACP,IAAI,CAAC,QAAQ,CAAC;aACd,IAAI,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,EAAE;YAChB,OAAO,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;QAC7C,CAAC,CAAC,CAAC;IACP,CAAC;IAED,MAAM,YAAY,GAChB,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,0BAA0B,CAAC,CAAC,CAAC,cAAc,CAAC;IAEnE,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,EAAE;QACxC,MAAM,KAAK,GAAa,EAAE,CAAC;QAC3B,CAAC,CAAC,GAAG,CAAC;aACH,IAAI,CAAC,QAAQ,CAAC;aACd,IAAI,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,EAAE;YAChB,KAAK,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;QAC3C,CAAC,CAAC,CAAC;QACL,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC;YAAE,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IACzC,CAAC,CAAC,CAAC;IAEH,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IAEnC,OAAO;QACL,IAAI,EAAE,OAAO;QACb,OAAO,EAAE,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,SAAS;QACjD,IAAI;KACL,CAAC;AACJ,CAAC;AAED,SAAS,UAAU,CAAC,CAAa,EAAE,OAAgB;IACjD,MAAM,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IACnC,IAAI,CAAC,GAAG;QAAE,OAAO,IAAI,CAAC;IAEtB,OAAO;QACL,IAAI,EAAE,OAAO;QACb,GAAG;QACH,GAAG,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,SAAS;KACzC,CAAC;AACJ,CAAC;AAED,SAAS,eAAe,CACtB,CAAa,EACb,OAAgB;IAEhB,MAAM,OAAO,GAAG,YAAY,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;IAChD,MAAM,IAAI,GAAG,cAAc,CAAC,OAAO,CAAC,CAAC;IACrC,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,UAAU,CAAC,kBAAkB;QAAE,OAAO,IAAI,CAAC;IAE7E,OAAO,EAAE,IAAI,EAAE,YAAY,EAAE,IAAI,EAAE,CAAC;AACtC,CAAC;AAED,MAAM,eAAe,GAAG;IACtB,EAAE,EAAE,YAAY;IAChB,EAAE,EAAE,YAAY;IAChB,EAAE,EAAE,YAAY;IAChB,EAAE,EAAE,YAAY;IAChB,EAAE,EAAE,YAAY;IAChB,EAAE,EAAE,YAAY;IAChB,CAAC,EAAE,cAAc;IACjB,EAAE,EAAE,SAAS;IACb,EAAE,EAAE,SAAS;IACb,GAAG,EAAE,SAAS;IACd,IAAI,EAAE,SAAS;IACf,KAAK,EAAE,UAAU;IACjB,GAAG,EAAE,UAAU;IACf,UAAU,EAAE,eAAe;CAI5B,CAAC;AAEF,SAAS,cAAc,CAAC,GAAW;IACjC,OAAO,GAAG,IAAI,eAAe,CAAC;AAChC,CAAC;AAED,SAAS,YAAY,CAAC,CAAa,EAAE,IAAa;IAChD,IAAI,CAAC,CAAC,SAAS,IAAI,IAAI,CAAC,IAAI,OAAO,IAAI,CAAC,OAAO,KAAK,QAAQ;QAAE,OAAO,IAAI,CAAC;IAE1E,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC;IAC3C,IAAI,CAAC,cAAc,CAAC,OAAO,CAAC;QAAE,OAAO,IAAI,CAAC;IAC1C,OAAO,eAAe,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC;AAC3C,CAAC;AAED,SAAS,YAAY,CAAC,MAA2B;IAC/C,OAAO,MAAM,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE;QAC7B,QAAQ,KAAK,CAAC,IAAI,EAAE,CAAC;YACnB,KAAK,WAAW,CAAC;YACjB,KAAK,SAAS,CAAC;YACf,KAAK,MAAM,CAAC;YACZ,KAAK,YAAY;gBACf,OAAO,KAAK,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC;YAC/B,KAAK,MAAM;gBACT,OAAO,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC;YAChC;gBACE,OAAO,IAAI,CAAC;QAChB,CAAC;IACH,CAAC,CAAC,CAAC;AACL,CAAC;AAED,MAAM,UAAU,SAAS,CAAC,IAAY;IACpC,IAAI,CAAC,IAAI,IAAI,OAAO,IAAI,KAAK,QAAQ;QAAE,OAAO,EAAE,CAAC;IAEjD,MAAM,aAAa,GAAG,YAAY,CAAC,IAAI,CAAC,CAAC;IAEzC,IAAI,CAAC;QACH,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;QACtC,MAAM,MAAM,GAAwB,EAAE,CAAC;QAEvC,CAAC,CAAC,sCAAsC,CAAC,CAAC,MAAM,EAAE,CAAC;QAEnD,CAAC,CAAC,MAAM,CAAC;aACN,IAAI,CAAC,gBAAgB,CAAC;aACtB,IAAI,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,EAAE;YACnB,IAAI,CAAC;gBACH,MAAM,KAAK,GAAG,YAAY,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC;gBACvC,IAAI,KAAK;oBAAE,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAChC,CAAC;YAAC,MAAM,CAAC;gBACP,UAAU;YACZ,CAAC;QACH,CAAC,CAAC,CAAC;QAEL,OAAO,YAAY,CAAC,MAAM,CAAC,CAAC;IAC9B,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,OAAO,CAAC,sBAAsB,EAAE;YAC9B,KAAK,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe;YAC/D,UAAU,EAAE,IAAI,CAAC,MAAM;SACxB,CAAC,CAAC;QACH,OAAO,EAAE,CAAC;IACZ,CAAC;AACH,CAAC"}
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
import type { StreamableHTTPServerTransport } from '@modelcontextprotocol/sdk/server/streamableHttp.js';
|
|
2
|
+
import type { SessionEntry } from '../config/types.js';
|
|
3
|
+
export declare class SessionManager {
|
|
4
|
+
private readonly sessions;
|
|
5
|
+
private readonly sessionTtlMs;
|
|
6
|
+
private cleanupTimeout;
|
|
7
|
+
constructor(sessionTtlMs?: number);
|
|
8
|
+
get(sessionId: string): SessionEntry | undefined;
|
|
9
|
+
set(sessionId: string, transport: StreamableHTTPServerTransport): void;
|
|
10
|
+
update(sessionId: string): boolean;
|
|
11
|
+
delete(sessionId: string): void;
|
|
12
|
+
get size(): number;
|
|
13
|
+
closeAll(): Promise<void>;
|
|
14
|
+
private removeExpiredAndIdleSessions;
|
|
15
|
+
private scheduleCleanup;
|
|
16
|
+
destroy(): void;
|
|
17
|
+
}
|
|
18
|
+
//# sourceMappingURL=session-manager.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"session-manager.d.ts","sourceRoot":"","sources":["../../src/services/session-manager.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,6BAA6B,EAAE,MAAM,oDAAoD,CAAC;AAExG,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAOvD,qBAAa,cAAc;IACzB,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAmC;IAC5D,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAS;IACtC,OAAO,CAAC,cAAc,CAA+B;gBAEzC,YAAY,SAAyB;IAIjD,GAAG,CAAC,SAAS,EAAE,MAAM,GAAG,YAAY,GAAG,SAAS;IAIhD,GAAG,CAAC,SAAS,EAAE,MAAM,EAAE,SAAS,EAAE,6BAA6B,GAAG,IAAI;IAMtE,MAAM,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO;IAUlC,MAAM,CAAC,SAAS,EAAE,MAAM,GAAG,IAAI;IAK/B,IAAI,IAAI,IAAI,MAAM,CAEjB;IAEK,QAAQ,IAAI,OAAO,CAAC,IAAI,CAAC;YASjB,4BAA4B;IAiB1C,OAAO,CAAC,eAAe;IAiBvB,OAAO,IAAI,IAAI;CAMhB"}
|
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
import { logInfo } from './logger.js';
|
|
2
|
+
const DEFAULT_SESSION_TTL_MS = 30 * 60 * 1000;
|
|
3
|
+
const CLEANUP_INTERVAL_MS = 2 * 60 * 1000;
|
|
4
|
+
export class SessionManager {
|
|
5
|
+
sessions = new Map();
|
|
6
|
+
sessionTtlMs;
|
|
7
|
+
cleanupTimeout = null;
|
|
8
|
+
constructor(sessionTtlMs = DEFAULT_SESSION_TTL_MS) {
|
|
9
|
+
this.sessionTtlMs = sessionTtlMs;
|
|
10
|
+
}
|
|
11
|
+
get(sessionId) {
|
|
12
|
+
return this.sessions.get(sessionId);
|
|
13
|
+
}
|
|
14
|
+
set(sessionId, transport) {
|
|
15
|
+
this.sessions.set(sessionId, { transport, createdAt: Date.now() });
|
|
16
|
+
logInfo('Session initialized', { sessionId });
|
|
17
|
+
this.scheduleCleanup();
|
|
18
|
+
}
|
|
19
|
+
update(sessionId) {
|
|
20
|
+
const entry = this.sessions.get(sessionId);
|
|
21
|
+
if (!entry) {
|
|
22
|
+
return false;
|
|
23
|
+
}
|
|
24
|
+
entry.createdAt = Date.now();
|
|
25
|
+
return true;
|
|
26
|
+
}
|
|
27
|
+
delete(sessionId) {
|
|
28
|
+
this.sessions.delete(sessionId);
|
|
29
|
+
logInfo('Session closed', { sessionId });
|
|
30
|
+
}
|
|
31
|
+
get size() {
|
|
32
|
+
return this.sessions.size;
|
|
33
|
+
}
|
|
34
|
+
async closeAll() {
|
|
35
|
+
const closePromises = Array.from(this.sessions.values()).map((session) => session.transport.close());
|
|
36
|
+
await Promise.allSettled(closePromises);
|
|
37
|
+
this.sessions.clear();
|
|
38
|
+
}
|
|
39
|
+
async removeExpiredAndIdleSessions() {
|
|
40
|
+
const now = Date.now();
|
|
41
|
+
const closePromises = [];
|
|
42
|
+
for (const [sessionId, entry] of this.sessions) {
|
|
43
|
+
const isExpired = now - entry.createdAt > this.sessionTtlMs;
|
|
44
|
+
if (isExpired) {
|
|
45
|
+
logInfo('Cleaning up stale session', { sessionId });
|
|
46
|
+
closePromises.push(entry.transport.close());
|
|
47
|
+
this.sessions.delete(sessionId);
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
await Promise.allSettled(closePromises);
|
|
51
|
+
}
|
|
52
|
+
scheduleCleanup() {
|
|
53
|
+
if (this.cleanupTimeout) {
|
|
54
|
+
return;
|
|
55
|
+
}
|
|
56
|
+
this.cleanupTimeout = setTimeout(() => {
|
|
57
|
+
void this.removeExpiredAndIdleSessions().then(() => {
|
|
58
|
+
this.cleanupTimeout = null;
|
|
59
|
+
if (this.sessions.size > 0) {
|
|
60
|
+
this.scheduleCleanup();
|
|
61
|
+
}
|
|
62
|
+
});
|
|
63
|
+
}, CLEANUP_INTERVAL_MS);
|
|
64
|
+
this.cleanupTimeout.unref();
|
|
65
|
+
}
|
|
66
|
+
destroy() {
|
|
67
|
+
if (this.cleanupTimeout) {
|
|
68
|
+
clearTimeout(this.cleanupTimeout);
|
|
69
|
+
this.cleanupTimeout = null;
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
//# sourceMappingURL=session-manager.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"session-manager.js","sourceRoot":"","sources":["../../src/services/session-manager.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,OAAO,EAAE,MAAM,aAAa,CAAC;AAEtC,MAAM,sBAAsB,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC;AAC9C,MAAM,mBAAmB,GAAG,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC;AAE1C,MAAM,OAAO,cAAc;IACR,QAAQ,GAAG,IAAI,GAAG,EAAwB,CAAC;IAC3C,YAAY,CAAS;IAC9B,cAAc,GAA0B,IAAI,CAAC;IAErD,YAAY,YAAY,GAAG,sBAAsB;QAC/C,IAAI,CAAC,YAAY,GAAG,YAAY,CAAC;IACnC,CAAC;IAED,GAAG,CAAC,SAAiB;QACnB,OAAO,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;IACtC,CAAC;IAED,GAAG,CAAC,SAAiB,EAAE,SAAwC;QAC7D,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;QACnE,OAAO,CAAC,qBAAqB,EAAE,EAAE,SAAS,EAAE,CAAC,CAAC;QAC9C,IAAI,CAAC,eAAe,EAAE,CAAC;IACzB,CAAC;IAED,MAAM,CAAC,SAAiB;QACtB,MAAM,KAAK,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;QAC3C,IAAI,CAAC,KAAK,EAAE,CAAC;YACX,OAAO,KAAK,CAAC;QACf,CAAC;QAED,KAAK,CAAC,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC7B,OAAO,IAAI,CAAC;IACd,CAAC;IAED,MAAM,CAAC,SAAiB;QACtB,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;QAChC,OAAO,CAAC,gBAAgB,EAAE,EAAE,SAAS,EAAE,CAAC,CAAC;IAC3C,CAAC;IAED,IAAI,IAAI;QACN,OAAO,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC;IAC5B,CAAC;IAED,KAAK,CAAC,QAAQ;QACZ,MAAM,aAAa,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CACvE,OAAO,CAAC,SAAS,CAAC,KAAK,EAAE,CAC1B,CAAC;QAEF,MAAM,OAAO,CAAC,UAAU,CAAC,aAAa,CAAC,CAAC;QACxC,IAAI,CAAC,QAAQ,CAAC,KAAK,EAAE,CAAC;IACxB,CAAC;IAEO,KAAK,CAAC,4BAA4B;QACxC,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACvB,MAAM,aAAa,GAAoB,EAAE,CAAC;QAE1C,KAAK,MAAM,CAAC,SAAS,EAAE,KAAK,CAAC,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;YAC/C,MAAM,SAAS,GAAG,GAAG,GAAG,KAAK,CAAC,SAAS,GAAG,IAAI,CAAC,YAAY,CAAC;YAE5D,IAAI,SAAS,EAAE,CAAC;gBACd,OAAO,CAAC,2BAA2B,EAAE,EAAE,SAAS,EAAE,CAAC,CAAC;gBACpD,aAAa,CAAC,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,KAAK,EAAE,CAAC,CAAC;gBAC5C,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;YAClC,CAAC;QACH,CAAC;QAED,MAAM,OAAO,CAAC,UAAU,CAAC,aAAa,CAAC,CAAC;IAC1C,CAAC;IAEO,eAAe;QACrB,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;YACxB,OAAO;QACT,CAAC;QAED,IAAI,CAAC,cAAc,GAAG,UAAU,CAAC,GAAG,EAAE;YACpC,KAAK,IAAI,CAAC,4BAA4B,EAAE,CAAC,IAAI,CAAC,GAAG,EAAE;gBACjD,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC;gBAC3B,IAAI,IAAI,CAAC,QAAQ,CAAC,IAAI,GAAG,CAAC,EAAE,CAAC;oBAC3B,IAAI,CAAC,eAAe,EAAE,CAAC;gBACzB,CAAC;YACH,CAAC,CAAC,CAAC;QACL,CAAC,EAAE,mBAAmB,CAAC,CAAC;QAExB,IAAI,CAAC,cAAc,CAAC,KAAK,EAAE,CAAC;IAC9B,CAAC;IAED,OAAO;QACL,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;YACxB,YAAY,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;YAClC,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC;QAC7B,CAAC;IACH,CAAC;CACF"}
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
export interface RetryStrategy {
|
|
2
|
+
calculateDelay(attempt: number): number;
|
|
3
|
+
shouldRetry(attempt: number, maxRetries: number, error: Error): boolean;
|
|
4
|
+
}
|
|
5
|
+
export declare class ExponentialBackoffStrategy implements RetryStrategy {
|
|
6
|
+
private readonly baseDelayMs;
|
|
7
|
+
private readonly maxDelayMs;
|
|
8
|
+
private readonly jitterFactor;
|
|
9
|
+
constructor(baseDelayMs?: number, maxDelayMs?: number, jitterFactor?: number);
|
|
10
|
+
calculateDelay(attempt: number): number;
|
|
11
|
+
shouldRetry(attempt: number, maxRetries: number, error: Error): boolean;
|
|
12
|
+
}
|
|
13
|
+
//# sourceMappingURL=exponential-backoff-strategy.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"exponential-backoff-strategy.d.ts","sourceRoot":"","sources":["../../src/strategies/exponential-backoff-strategy.ts"],"names":[],"mappings":"AAAA,MAAM,WAAW,aAAa;IAC5B,cAAc,CAAC,OAAO,EAAE,MAAM,GAAG,MAAM,CAAC;IACxC,WAAW,CAAC,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,GAAG,OAAO,CAAC;CACzE;AAED,qBAAa,0BAA2B,YAAW,aAAa;IAE5D,OAAO,CAAC,QAAQ,CAAC,WAAW;IAC5B,OAAO,CAAC,QAAQ,CAAC,UAAU;IAC3B,OAAO,CAAC,QAAQ,CAAC,YAAY;gBAFZ,WAAW,SAAO,EAClB,UAAU,SAAQ,EAClB,YAAY,SAAO;IAGtC,cAAc,CAAC,OAAO,EAAE,MAAM,GAAG,MAAM;IAUvC,WAAW,CAAC,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,GAAG,OAAO;CAkBxE"}
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
export class ExponentialBackoffStrategy {
|
|
2
|
+
baseDelayMs;
|
|
3
|
+
maxDelayMs;
|
|
4
|
+
jitterFactor;
|
|
5
|
+
constructor(baseDelayMs = 1000, maxDelayMs = 10000, jitterFactor = 0.25) {
|
|
6
|
+
this.baseDelayMs = baseDelayMs;
|
|
7
|
+
this.maxDelayMs = maxDelayMs;
|
|
8
|
+
this.jitterFactor = jitterFactor;
|
|
9
|
+
}
|
|
10
|
+
calculateDelay(attempt) {
|
|
11
|
+
const exponentialDelay = Math.min(this.baseDelayMs * Math.pow(2, attempt - 1), this.maxDelayMs);
|
|
12
|
+
const jitter = exponentialDelay * this.jitterFactor * (Math.random() * 2 - 1);
|
|
13
|
+
return Math.round(exponentialDelay + jitter);
|
|
14
|
+
}
|
|
15
|
+
shouldRetry(attempt, maxRetries, error) {
|
|
16
|
+
if (attempt >= maxRetries)
|
|
17
|
+
return false;
|
|
18
|
+
// Don't retry on abort
|
|
19
|
+
if (error.name === 'AbortError' || error.name === 'CanceledError') {
|
|
20
|
+
return false;
|
|
21
|
+
}
|
|
22
|
+
// Don't retry on client errors (4xx except 429)
|
|
23
|
+
if ('httpStatus' in error) {
|
|
24
|
+
const status = error.httpStatus;
|
|
25
|
+
if (status && status >= 400 && status < 500 && status !== 429) {
|
|
26
|
+
return false;
|
|
27
|
+
}
|
|
28
|
+
}
|
|
29
|
+
return true;
|
|
30
|
+
}
|
|
31
|
+
}
|
|
32
|
+
//# sourceMappingURL=exponential-backoff-strategy.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"exponential-backoff-strategy.js","sourceRoot":"","sources":["../../src/strategies/exponential-backoff-strategy.ts"],"names":[],"mappings":"AAKA,MAAM,OAAO,0BAA0B;IAElB;IACA;IACA;IAHnB,YACmB,cAAc,IAAI,EAClB,aAAa,KAAK,EAClB,eAAe,IAAI;QAFnB,gBAAW,GAAX,WAAW,CAAO;QAClB,eAAU,GAAV,UAAU,CAAQ;QAClB,iBAAY,GAAZ,YAAY,CAAO;IACnC,CAAC;IAEJ,cAAc,CAAC,OAAe;QAC5B,MAAM,gBAAgB,GAAG,IAAI,CAAC,GAAG,CAC/B,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,GAAG,CAAC,CAAC,EAC3C,IAAI,CAAC,UAAU,CAChB,CAAC;QACF,MAAM,MAAM,GACV,gBAAgB,GAAG,IAAI,CAAC,YAAY,GAAG,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC;QACjE,OAAO,IAAI,CAAC,KAAK,CAAC,gBAAgB,GAAG,MAAM,CAAC,CAAC;IAC/C,CAAC;IAED,WAAW,CAAC,OAAe,EAAE,UAAkB,EAAE,KAAY;QAC3D,IAAI,OAAO,IAAI,UAAU;YAAE,OAAO,KAAK,CAAC;QAExC,uBAAuB;QACvB,IAAI,KAAK,CAAC,IAAI,KAAK,YAAY,IAAI,KAAK,CAAC,IAAI,KAAK,eAAe,EAAE,CAAC;YAClE,OAAO,KAAK,CAAC;QACf,CAAC;QAED,gDAAgD;QAChD,IAAI,YAAY,IAAI,KAAK,EAAE,CAAC;YAC1B,MAAM,MAAM,GAAI,KAAiC,CAAC,UAAU,CAAC;YAC7D,IAAI,MAAM,IAAI,MAAM,IAAI,GAAG,IAAI,MAAM,GAAG,GAAG,IAAI,MAAM,KAAK,GAAG,EAAE,CAAC;gBAC9D,OAAO,KAAK,CAAC;YACf,CAAC;QACH,CAAC;QAED,OAAO,IAAI,CAAC;IACd,CAAC;CACF"}
|
|
@@ -1,12 +1,5 @@
|
|
|
1
|
-
import type { FetchLinksInput } from '../../config/types.js';
|
|
1
|
+
import type { FetchLinksInput, ToolResponseBase } from '../../config/types.js';
|
|
2
2
|
export declare const FETCH_LINKS_TOOL_NAME = "fetch-links";
|
|
3
3
|
export declare const FETCH_LINKS_TOOL_DESCRIPTION = "Extracts all hyperlinks from a webpage with anchor text and type classification. Supports filtering, image links, and link limits.";
|
|
4
|
-
export declare function fetchLinksToolHandler(input: FetchLinksInput): Promise<
|
|
5
|
-
content: {
|
|
6
|
-
type: 'text';
|
|
7
|
-
text: string;
|
|
8
|
-
}[];
|
|
9
|
-
structuredContent?: Record<string, unknown>;
|
|
10
|
-
isError?: boolean;
|
|
11
|
-
}>;
|
|
4
|
+
export declare function fetchLinksToolHandler(input: FetchLinksInput): Promise<ToolResponseBase>;
|
|
12
5
|
//# sourceMappingURL=fetch-links.tool.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"fetch-links.tool.d.ts","sourceRoot":"","sources":["../../../src/tools/handlers/fetch-links.tool.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAGV,eAAe,
|
|
1
|
+
{"version":3,"file":"fetch-links.tool.d.ts","sourceRoot":"","sources":["../../../src/tools/handlers/fetch-links.tool.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAGV,eAAe,EAGf,gBAAgB,EACjB,MAAM,uBAAuB,CAAC;AAW/B,eAAO,MAAM,qBAAqB,gBAAgB,CAAC;AACnD,eAAO,MAAM,4BAA4B,uIAC6F,CAAC;AAqFvI,wBAAsB,qBAAqB,CACzC,KAAK,EAAE,eAAe,GACrB,OAAO,CAAC,gBAAgB,CAAC,CAwE3B"}
|
|
@@ -83,7 +83,6 @@ export async function fetchLinksToolHandler(input) {
|
|
|
83
83
|
}
|
|
84
84
|
let filterPattern;
|
|
85
85
|
if (input.filterPattern) {
|
|
86
|
-
// ReDoS protection: limit filter pattern length
|
|
87
86
|
if (input.filterPattern.length > 200) {
|
|
88
87
|
return createToolErrorResponse('Filter pattern too long (max 200 characters)', input.url, 'VALIDATION_ERROR');
|
|
89
88
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"fetch-links.tool.js","sourceRoot":"","sources":["../../../src/tools/handlers/fetch-links.tool.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,OAAO,MAAM,SAAS,CAAC;
|
|
1
|
+
{"version":3,"file":"fetch-links.tool.js","sourceRoot":"","sources":["../../../src/tools/handlers/fetch-links.tool.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,OAAO,MAAM,SAAS,CAAC;AAWnC,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,0BAA0B,CAAC;AAE9D,OAAO,EACL,uBAAuB,EACvB,eAAe,GAChB,MAAM,mCAAmC,CAAC;AAC3C,OAAO,EAAE,aAAa,EAAE,MAAM,8BAA8B,CAAC;AAC7D,OAAO,EAAE,oBAAoB,EAAE,MAAM,4BAA4B,CAAC;AAElE,MAAM,CAAC,MAAM,qBAAqB,GAAG,aAAa,CAAC;AACnD,MAAM,CAAC,MAAM,4BAA4B,GACvC,oIAAoI,CAAC;AAEvI,SAAS,aAAa,CAAC,IAAY,EAAE,OAAe;IAClD,IAAI,CAAC;QACH,OAAO,IAAI,GAAG,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC,IAAI,CAAC;IACrC,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED,SAAS,iBAAiB,CACxB,IAAc,EACd,GAAW,EACX,OAA4B,EAC5B,IAAiB;IAEjB,IAAI,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC;QAAE,OAAO,KAAK,CAAC;IAChC,IAAI,OAAO,CAAC,aAAa,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC,IAAI,CAAC,GAAG,CAAC;QAAE,OAAO,KAAK,CAAC;IAC5E,IAAI,IAAI,KAAK,UAAU,IAAI,CAAC,OAAO,CAAC,eAAe;QAAE,OAAO,KAAK,CAAC;IAClE,IAAI,IAAI,KAAK,UAAU,IAAI,CAAC,OAAO,CAAC,eAAe;QAAE,OAAO,KAAK,CAAC;IAClE,OAAO,IAAI,CAAC;AACd,CAAC;AAED,SAAS,YAAY,CACnB,IAAY,EACZ,OAAe,EACf,OAA4B;IAE5B,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC7B,MAAM,KAAK,GAAoB,EAAE,CAAC;IAClC,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAC/B,IAAI,QAAQ,GAAG,CAAC,CAAC;IAEjB,CAAC,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,EAAE;QAC1B,MAAM,IAAI,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAChC,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,aAAa,CAAC;YAAE,OAAO;QAE5E,MAAM,GAAG,GAAG,aAAa,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;QACzC,IAAI,CAAC,GAAG;YAAE,OAAO;QAEjB,MAAM,IAAI,GAAa,aAAa,CAAC,GAAG,EAAE,OAAO,CAAC;YAChD,CAAC,CAAC,UAAU;YACZ,CAAC,CAAC,UAAU,CAAC;QACf,IAAI,CAAC,iBAAiB,CAAC,IAAI,EAAE,GAAG,EAAE,OAAO,EAAE,IAAI,CAAC,EAAE,CAAC;YACjD,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC;gBAAE,QAAQ,EAAE,CAAC;YAC/B,OAAO;QACT,CAAC;QAED,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACd,KAAK,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,GAAG,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,IAAI,GAAG,EAAE,IAAI,EAAE,CAAC,CAAC;IACpE,CAAC,CAAC,CAAC;IAEH,IAAI,OAAO,CAAC,aAAa,EAAE,CAAC;QAC1B,CAAC,CAAC,UAAU,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,EAAE;YAC3B,MAAM,GAAG,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAC9B,IAAI,CAAC,GAAG,IAAI,GAAG,CAAC,UAAU,CAAC,OAAO,CAAC;gBAAE,OAAO;YAE5C,MAAM,GAAG,GAAG,aAAa,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;YACxC,IAAI,CAAC,GAAG;gBAAE,OAAO;YAEjB,IAAI,CAAC,iBAAiB,CAAC,OAAO,EAAE,GAAG,EAAE,OAAO,EAAE,IAAI,CAAC,EAAE,CAAC;gBACpD,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC;oBAAE,QAAQ,EAAE,CAAC;gBAC/B,OAAO;YACT,CAAC;YAED,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YACd,KAAK,CAAC,IAAI,CAAC;gBACT,IAAI,EAAE,GAAG;gBACT,IAAI,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,IAAI,EAAE,IAAI,GAAG;gBACtC,IAAI,EAAE,OAAO;aACd,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;IACL,CAAC;IAED,MAAM,SAAS,GAAG,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,GAAG,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC;IAC7E,MAAM,WAAW,GAAG,SAAS,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC;IAEzE,OAAO;QACL,KAAK,EAAE,WAAW;QAClB,SAAS,EAAE,WAAW,CAAC,MAAM;QAC7B,QAAQ;QACR,SAAS;KACV,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,qBAAqB,CACzC,KAAsB;IAEtB,IAAI,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC;QACf,OAAO,uBAAuB,CAAC,iBAAiB,EAAE,EAAE,EAAE,kBAAkB,CAAC,CAAC;IAC5E,CAAC;IAED,IAAI,aAAiC,CAAC;IACtC,IAAI,KAAK,CAAC,aAAa,EAAE,CAAC;QACxB,IAAI,KAAK,CAAC,aAAa,CAAC,MAAM,GAAG,GAAG,EAAE,CAAC;YACrC,OAAO,uBAAuB,CAC5B,8CAA8C,EAC9C,KAAK,CAAC,GAAG,EACT,kBAAkB,CACnB,CAAC;QACJ,CAAC;QACD,IAAI,CAAC;YACH,aAAa,GAAG,IAAI,MAAM,CAAC,KAAK,CAAC,aAAa,EAAE,GAAG,CAAC,CAAC;QACvD,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,uBAAuB,CAC5B,2BAA2B,KAAK,CAAC,aAAa,EAAE,EAChD,KAAK,CAAC,GAAG,EACT,kBAAkB,CACnB,CAAC;QACJ,CAAC;IACH,CAAC;IAED,IAAI,CAAC;QACH,MAAM,OAAO,GAAwB;YACnC,eAAe,EAAE,KAAK,CAAC,eAAe,IAAI,IAAI;YAC9C,eAAe,EAAE,KAAK,CAAC,eAAe,IAAI,IAAI;YAC9C,aAAa,EAAE,KAAK,CAAC,aAAa,IAAI,KAAK;YAC3C,QAAQ,EAAE,KAAK,CAAC,QAAQ;YACxB,aAAa;SACd,CAAC;QAEF,QAAQ,CAAC,kBAAkB,EAAE;YAC3B,GAAG,EAAE,KAAK,CAAC,GAAG;YACd,GAAG,OAAO;YACV,aAAa,EAAE,KAAK,CAAC,aAAa;SACnC,CAAC,CAAC;QAEH,MAAM,MAAM,GAAG,MAAM,oBAAoB,CAAuB;YAC9D,GAAG,EAAE,KAAK,CAAC,GAAG;YACd,cAAc,EAAE,OAAO;YACvB,aAAa,EAAE,KAAK,CAAC,aAAa;YAClC,OAAO,EAAE,KAAK,CAAC,OAAO;YACtB,SAAS,EAAE,CAAC,IAAI,EAAE,GAAG,EAAE,EAAE,CAAC,YAAY,CAAC,IAAI,EAAE,GAAG,EAAE,OAAO,CAAC;SAC3D,CAAC,CAAC;QAEH,MAAM,iBAAiB,GAAG;YACxB,GAAG,EAAE,MAAM,CAAC,GAAG;YACf,SAAS,EAAE,MAAM,CAAC,IAAI,CAAC,SAAS;YAChC,KAAK,EAAE,MAAM,CAAC,IAAI,CAAC,KAAK;YACxB,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,QAAQ,GAAG,CAAC,IAAI,EAAE,QAAQ,EAAE,MAAM,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC;YACnE,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,SAAS,IAAI,EAAE,SAAS,EAAE,MAAM,CAAC,IAAI,CAAC,SAAS,EAAE,CAAC;SACnE,CAAC;QAEF,OAAO;YACL,OAAO,EAAE;gBACP;oBACE,IAAI,EAAE,MAAe;oBACrB,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,iBAAiB,EAAE,IAAI,EAAE,CAAC,CAAC;iBACjD;aACF;YACD,iBAAiB;SAClB,CAAC;IACJ,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,QAAQ,CACN,wBAAwB,EACxB,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS,CAC3C,CAAC;QACF,OAAO,eAAe,CAAC,KAAK,EAAE,KAAK,CAAC,GAAG,EAAE,yBAAyB,CAAC,CAAC;IACtE,CAAC;AACH,CAAC"}
|
|
@@ -1,12 +1,15 @@
|
|
|
1
1
|
import type { FetchMarkdownInput } from '../../config/types.js';
|
|
2
2
|
export declare const FETCH_MARKDOWN_TOOL_NAME = "fetch-markdown";
|
|
3
3
|
export declare const FETCH_MARKDOWN_TOOL_DESCRIPTION = "Fetches a webpage and converts it to clean Markdown format with optional frontmatter, table of contents, and content length limits";
|
|
4
|
-
|
|
4
|
+
interface FetchMarkdownToolResponse {
|
|
5
|
+
[x: string]: unknown;
|
|
5
6
|
content: {
|
|
6
7
|
type: 'text';
|
|
7
8
|
text: string;
|
|
8
9
|
}[];
|
|
9
10
|
structuredContent?: Record<string, unknown>;
|
|
10
11
|
isError?: boolean;
|
|
11
|
-
}
|
|
12
|
+
}
|
|
13
|
+
export declare function fetchMarkdownToolHandler(input: FetchMarkdownInput): Promise<FetchMarkdownToolResponse>;
|
|
14
|
+
export {};
|
|
12
15
|
//# sourceMappingURL=fetch-markdown.tool.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"fetch-markdown.tool.d.ts","sourceRoot":"","sources":["../../../src/tools/handlers/fetch-markdown.tool.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,kBAAkB,EAInB,MAAM,uBAAuB,CAAC;
|
|
1
|
+
{"version":3,"file":"fetch-markdown.tool.d.ts","sourceRoot":"","sources":["../../../src/tools/handlers/fetch-markdown.tool.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,kBAAkB,EAInB,MAAM,uBAAuB,CAAC;AAkB/B,eAAO,MAAM,wBAAwB,mBAAmB,CAAC;AACzD,eAAO,MAAM,+BAA+B,uIAC0F,CAAC;AAEvI,UAAU,yBAAyB;IACjC,CAAC,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC;IACrB,OAAO,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC;IAC1C,iBAAiB,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAC5C,OAAO,CAAC,EAAE,OAAO,CAAC;CACnB;AAwED,wBAAsB,wBAAwB,CAC5C,KAAK,EAAE,kBAAkB,GACxB,OAAO,CAAC,yBAAyB,CAAC,CAyDpC"}
|
|
@@ -2,17 +2,12 @@ import { extractContent } from '../../services/extractor.js';
|
|
|
2
2
|
import { logDebug, logError } from '../../services/logger.js';
|
|
3
3
|
import { stripMarkdownLinks } from '../../utils/content-cleaner.js';
|
|
4
4
|
import { createToolErrorResponse, handleToolError, } from '../../utils/tool-error-handler.js';
|
|
5
|
-
import {
|
|
5
|
+
import { createContentMetadataBlock, determineContentExtractionSource, } from '../utils/common.js';
|
|
6
6
|
import { executeFetchPipeline } from '../utils/fetch-pipeline.js';
|
|
7
7
|
import { htmlToMarkdown } from '../../transformers/markdown.transformer.js';
|
|
8
8
|
export const FETCH_MARKDOWN_TOOL_NAME = 'fetch-markdown';
|
|
9
9
|
export const FETCH_MARKDOWN_TOOL_DESCRIPTION = 'Fetches a webpage and converts it to clean Markdown format with optional frontmatter, table of contents, and content length limits';
|
|
10
|
-
/**
|
|
11
|
-
* Generate URL-friendly slug from text
|
|
12
|
-
* Strips markdown link syntax before slugifying
|
|
13
|
-
*/
|
|
14
10
|
function slugify(text) {
|
|
15
|
-
// First strip markdown links: [Text](#anchor) -> Text
|
|
16
11
|
const cleanText = stripMarkdownLinks(text);
|
|
17
12
|
return cleanText
|
|
18
13
|
.toLowerCase()
|
|
@@ -21,33 +16,32 @@ function slugify(text) {
|
|
|
21
16
|
.replace(/--+/g, '-')
|
|
22
17
|
.trim();
|
|
23
18
|
}
|
|
24
|
-
/**
|
|
25
|
-
* Extract table of contents from markdown headings
|
|
26
|
-
* Returns clean text without markdown link syntax
|
|
27
|
-
*/
|
|
28
19
|
function extractToc(markdown) {
|
|
29
20
|
const headingRegex = /^(#{1,6})\s+(.+)$/gm;
|
|
30
21
|
const toc = [];
|
|
31
22
|
let match;
|
|
32
23
|
while ((match = headingRegex.exec(markdown)) !== null) {
|
|
33
|
-
|
|
24
|
+
const hashMarks = match[1];
|
|
25
|
+
const rawText = match[2];
|
|
26
|
+
if (!hashMarks || !rawText)
|
|
34
27
|
continue;
|
|
35
|
-
const
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
28
|
+
const text = stripMarkdownLinks(rawText.trim());
|
|
29
|
+
toc.push({
|
|
30
|
+
level: hashMarks.length,
|
|
31
|
+
text,
|
|
32
|
+
slug: slugify(rawText),
|
|
33
|
+
});
|
|
39
34
|
}
|
|
40
35
|
return toc;
|
|
41
36
|
}
|
|
42
37
|
function transformToMarkdown(html, url, options) {
|
|
43
|
-
// Only invoke JSDOM when extractMainContent is true (lazy loading optimization)
|
|
44
38
|
const { article, metadata: extractedMeta } = extractContent(html, url, {
|
|
45
39
|
extractArticle: options.extractMainContent,
|
|
46
40
|
});
|
|
47
|
-
const
|
|
48
|
-
const metadata =
|
|
49
|
-
const sourceHtml =
|
|
50
|
-
const title =
|
|
41
|
+
const shouldExtractFromArticle = determineContentExtractionSource(options.extractMainContent, article);
|
|
42
|
+
const metadata = createContentMetadataBlock(url, article, extractedMeta, shouldExtractFromArticle, options.includeMetadata);
|
|
43
|
+
const sourceHtml = shouldExtractFromArticle ? article.content : html;
|
|
44
|
+
const title = shouldExtractFromArticle ? article.title : extractedMeta.title;
|
|
51
45
|
let markdown = htmlToMarkdown(sourceHtml, metadata);
|
|
52
46
|
const toc = options.generateToc ? extractToc(markdown) : undefined;
|
|
53
47
|
let truncated = false;
|
|
@@ -61,14 +55,14 @@ export async function fetchMarkdownToolHandler(input) {
|
|
|
61
55
|
if (!input.url) {
|
|
62
56
|
return createToolErrorResponse('URL is required', '', 'VALIDATION_ERROR');
|
|
63
57
|
}
|
|
58
|
+
const options = {
|
|
59
|
+
extractMainContent: input.extractMainContent ?? true,
|
|
60
|
+
includeMetadata: input.includeMetadata ?? true,
|
|
61
|
+
generateToc: input.generateToc ?? false,
|
|
62
|
+
maxContentLength: input.maxContentLength,
|
|
63
|
+
};
|
|
64
|
+
logDebug('Fetching markdown', { url: input.url, ...options });
|
|
64
65
|
try {
|
|
65
|
-
const options = {
|
|
66
|
-
extractMainContent: input.extractMainContent ?? true,
|
|
67
|
-
includeMetadata: input.includeMetadata ?? true,
|
|
68
|
-
generateToc: input.generateToc ?? false,
|
|
69
|
-
maxContentLength: input.maxContentLength,
|
|
70
|
-
};
|
|
71
|
-
logDebug('Fetching markdown', { url: input.url, ...options });
|
|
72
66
|
const result = await executeFetchPipeline({
|
|
73
67
|
url: input.url,
|
|
74
68
|
cacheNamespace: 'markdown',
|
|
@@ -92,13 +86,9 @@ export async function fetchMarkdownToolHandler(input) {
|
|
|
92
86
|
cached: result.fromCache,
|
|
93
87
|
...(result.data.truncated && { truncated: result.data.truncated }),
|
|
94
88
|
};
|
|
89
|
+
const jsonOutput = JSON.stringify(structuredContent, result.fromCache ? undefined : null, result.fromCache ? undefined : 2);
|
|
95
90
|
return {
|
|
96
|
-
content: [
|
|
97
|
-
{
|
|
98
|
-
type: 'text',
|
|
99
|
-
text: JSON.stringify(structuredContent, result.fromCache ? undefined : null, result.fromCache ? undefined : 2),
|
|
100
|
-
},
|
|
101
|
-
],
|
|
91
|
+
content: [{ type: 'text', text: jsonOutput }],
|
|
102
92
|
structuredContent,
|
|
103
93
|
};
|
|
104
94
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"fetch-markdown.tool.js","sourceRoot":"","sources":["../../../src/tools/handlers/fetch-markdown.tool.ts"],"names":[],"mappings":"AAOA,OAAO,EAAE,cAAc,EAAE,MAAM,6BAA6B,CAAC;AAC7D,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,0BAA0B,CAAC;AAE9D,OAAO,EAAE,kBAAkB,EAAE,MAAM,gCAAgC,CAAC;AACpE,OAAO,EACL,uBAAuB,EACvB,eAAe,GAChB,MAAM,mCAAmC,CAAC;AAC3C,OAAO,
|
|
1
|
+
{"version":3,"file":"fetch-markdown.tool.js","sourceRoot":"","sources":["../../../src/tools/handlers/fetch-markdown.tool.ts"],"names":[],"mappings":"AAOA,OAAO,EAAE,cAAc,EAAE,MAAM,6BAA6B,CAAC;AAC7D,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,0BAA0B,CAAC;AAE9D,OAAO,EAAE,kBAAkB,EAAE,MAAM,gCAAgC,CAAC;AACpE,OAAO,EACL,uBAAuB,EACvB,eAAe,GAChB,MAAM,mCAAmC,CAAC;AAC3C,OAAO,EACL,0BAA0B,EAC1B,gCAAgC,GACjC,MAAM,oBAAoB,CAAC;AAC5B,OAAO,EAAE,oBAAoB,EAAE,MAAM,4BAA4B,CAAC;AAElE,OAAO,EAAE,cAAc,EAAE,MAAM,4CAA4C,CAAC;AAE5E,MAAM,CAAC,MAAM,wBAAwB,GAAG,gBAAgB,CAAC;AACzD,MAAM,CAAC,MAAM,+BAA+B,GAC1C,oIAAoI,CAAC;AASvI,SAAS,OAAO,CAAC,IAAY;IAC3B,MAAM,SAAS,GAAG,kBAAkB,CAAC,IAAI,CAAC,CAAC;IAE3C,OAAO,SAAS;SACb,WAAW,EAAE;SACb,OAAO,CAAC,WAAW,EAAE,EAAE,CAAC;SACxB,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC;SACpB,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC;SACpB,IAAI,EAAE,CAAC;AACZ,CAAC;AAED,SAAS,UAAU,CAAC,QAAgB;IAClC,MAAM,YAAY,GAAG,qBAAqB,CAAC;IAC3C,MAAM,GAAG,GAAe,EAAE,CAAC;IAC3B,IAAI,KAAK,CAAC;IAEV,OAAO,CAAC,KAAK,GAAG,YAAY,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;QACtD,MAAM,SAAS,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;QAC3B,MAAM,OAAO,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;QAEzB,IAAI,CAAC,SAAS,IAAI,CAAC,OAAO;YAAE,SAAS;QAErC,MAAM,IAAI,GAAG,kBAAkB,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC;QAChD,GAAG,CAAC,IAAI,CAAC;YACP,KAAK,EAAE,SAAS,CAAC,MAAM;YACvB,IAAI;YACJ,IAAI,EAAE,OAAO,CAAC,OAAO,CAAC;SACvB,CAAC,CAAC;IACL,CAAC;IAED,OAAO,GAAG,CAAC;AACb,CAAC;AAED,SAAS,mBAAmB,CAC1B,IAAY,EACZ,GAAW,EACX,OAAyB;IAEzB,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,aAAa,EAAE,GAAG,cAAc,CAAC,IAAI,EAAE,GAAG,EAAE;QACrE,cAAc,EAAE,OAAO,CAAC,kBAAkB;KAC3C,CAAC,CAAC;IAEH,MAAM,wBAAwB,GAAG,gCAAgC,CAC/D,OAAO,CAAC,kBAAkB,EAC1B,OAAO,CACR,CAAC;IAEF,MAAM,QAAQ,GAAG,0BAA0B,CACzC,GAAG,EACH,OAAO,EACP,aAAa,EACb,wBAAwB,EACxB,OAAO,CAAC,eAAe,CACxB,CAAC;IAEF,MAAM,UAAU,GAAG,wBAAwB,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC;IACrE,MAAM,KAAK,GAAG,wBAAwB,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,aAAa,CAAC,KAAK,CAAC;IAE7E,IAAI,QAAQ,GAAG,cAAc,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAC;IACpD,MAAM,GAAG,GAAG,OAAO,CAAC,WAAW,CAAC,CAAC,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;IAEnE,IAAI,SAAS,GAAG,KAAK,CAAC;IACtB,IAAI,OAAO,CAAC,gBAAgB,IAAI,QAAQ,CAAC,MAAM,GAAG,OAAO,CAAC,gBAAgB,EAAE,CAAC;QAC3E,QAAQ,GAAG,GAAG,QAAQ,CAAC,SAAS,CAAC,CAAC,EAAE,OAAO,CAAC,gBAAgB,CAAC,oBAAoB,CAAC;QAClF,SAAS,GAAG,IAAI,CAAC;IACnB,CAAC;IAED,OAAO,EAAE,QAAQ,EAAE,KAAK,EAAE,GAAG,EAAE,SAAS,EAAE,CAAC;AAC7C,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,wBAAwB,CAC5C,KAAyB;IAEzB,IAAI,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC;QACf,OAAO,uBAAuB,CAAC,iBAAiB,EAAE,EAAE,EAAE,kBAAkB,CAAC,CAAC;IAC5E,CAAC;IAED,MAAM,OAAO,GAAqB;QAChC,kBAAkB,EAAE,KAAK,CAAC,kBAAkB,IAAI,IAAI;QACpD,eAAe,EAAE,KAAK,CAAC,eAAe,IAAI,IAAI;QAC9C,WAAW,EAAE,KAAK,CAAC,WAAW,IAAI,KAAK;QACvC,gBAAgB,EAAE,KAAK,CAAC,gBAAgB;KACzC,CAAC;IAEF,QAAQ,CAAC,mBAAmB,EAAE,EAAE,GAAG,EAAE,KAAK,CAAC,GAAG,EAAE,GAAG,OAAO,EAAE,CAAC,CAAC;IAE9D,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,MAAM,oBAAoB,CAA0B;YACjE,GAAG,EAAE,KAAK,CAAC,GAAG;YACd,cAAc,EAAE,UAAU;YAC1B,aAAa,EAAE,KAAK,CAAC,aAAa;YAClC,OAAO,EAAE,KAAK,CAAC,OAAO;YACtB,SAAS,EAAE,CAAC,IAAI,EAAE,GAAG,EAAE,EAAE,CAAC,mBAAmB,CAAC,IAAI,EAAE,GAAG,EAAE,OAAO,CAAC;YACjE,SAAS,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,QAAQ;YAClC,WAAW,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;gBACxB,QAAQ,EAAE,MAAM;gBAChB,KAAK,EAAE,SAAS;gBAChB,GAAG,EAAE,SAAS;gBACd,SAAS,EAAE,KAAK;aACjB,CAAC;SACH,CAAC,CAAC;QAEH,MAAM,iBAAiB,GAAG;YACxB,GAAG,EAAE,MAAM,CAAC,GAAG;YACf,KAAK,EAAE,MAAM,CAAC,IAAI,CAAC,KAAK;YACxB,SAAS,EAAE,MAAM,CAAC,SAAS;YAC3B,QAAQ,EAAE,MAAM,CAAC,IAAI,CAAC,QAAQ;YAC9B,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,IAAI,EAAE,GAAG,EAAE,MAAM,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC;YAChD,MAAM,EAAE,MAAM,CAAC,SAAS;YACxB,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,SAAS,IAAI,EAAE,SAAS,EAAE,MAAM,CAAC,IAAI,CAAC,SAAS,EAAE,CAAC;SACnE,CAAC;QAEF,MAAM,UAAU,GAAG,IAAI,CAAC,SAAS,CAC/B,iBAAiB,EACjB,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,EACnC,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CACjC,CAAC;QAEF,OAAO;YACL,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAe,EAAE,IAAI,EAAE,UAAU,EAAE,CAAC;YACtD,iBAAiB;SAClB,CAAC;IACJ,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,QAAQ,CACN,2BAA2B,EAC3B,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS,CAC3C,CAAC;QACF,OAAO,eAAe,CAAC,KAAK,EAAE,KAAK,CAAC,GAAG,EAAE,0BAA0B,CAAC,CAAC;IACvE,CAAC;AACH,CAAC"}
|
|
@@ -1,12 +1,5 @@
|
|
|
1
|
-
import type { FetchUrlInput } from '../../config/types.js';
|
|
1
|
+
import type { FetchUrlInput, ToolResponseBase } from '../../config/types.js';
|
|
2
2
|
export declare const FETCH_URL_TOOL_NAME = "fetch-url";
|
|
3
3
|
export declare const FETCH_URL_TOOL_DESCRIPTION = "Fetches a webpage and converts it to AI-readable JSONL format with semantic content blocks. Supports custom headers, retries, and content length limits.";
|
|
4
|
-
export declare function fetchUrlToolHandler(input: FetchUrlInput): Promise<
|
|
5
|
-
content: {
|
|
6
|
-
type: 'text';
|
|
7
|
-
text: string;
|
|
8
|
-
}[];
|
|
9
|
-
structuredContent?: Record<string, unknown>;
|
|
10
|
-
isError?: boolean;
|
|
11
|
-
}>;
|
|
4
|
+
export declare function fetchUrlToolHandler(input: FetchUrlInput): Promise<ToolResponseBase>;
|
|
12
5
|
//# sourceMappingURL=fetch-url.tool.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"fetch-url.tool.d.ts","sourceRoot":"","sources":["../../../src/tools/handlers/fetch-url.tool.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,
|
|
1
|
+
{"version":3,"file":"fetch-url.tool.d.ts","sourceRoot":"","sources":["../../../src/tools/handlers/fetch-url.tool.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAEV,aAAa,EAEb,gBAAgB,EACjB,MAAM,uBAAuB,CAAC;AAmB/B,eAAO,MAAM,mBAAmB,cAAc,CAAC;AAC/C,eAAO,MAAM,0BAA0B,6JACqH,CAAC;AAoC7J,wBAAsB,mBAAmB,CACvC,KAAK,EAAE,aAAa,GACnB,OAAO,CAAC,gBAAgB,CAAC,CA+D3B"}
|