@j0hanz/superfetch 1.1.9 → 1.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +179 -469
- package/dist/config/constants.d.ts +19 -0
- package/dist/config/constants.d.ts.map +1 -0
- package/dist/config/constants.js +24 -0
- package/dist/config/constants.js.map +1 -0
- package/dist/config/formatting.d.ts +0 -2
- package/dist/config/formatting.d.ts.map +1 -1
- package/dist/config/formatting.js +1 -3
- package/dist/config/formatting.js.map +1 -1
- package/dist/config/index.d.ts +9 -3
- package/dist/config/index.d.ts.map +1 -1
- package/dist/config/index.js +19 -16
- package/dist/config/index.js.map +1 -1
- package/dist/config/types/content.d.ts +1 -20
- package/dist/config/types/content.d.ts.map +1 -1
- package/dist/config/types/content.js +0 -1
- package/dist/config/types/runtime.d.ts +7 -5
- package/dist/config/types/runtime.d.ts.map +1 -1
- package/dist/config/types/runtime.js +0 -1
- package/dist/config/types/tools.d.ts +5 -50
- package/dist/config/types/tools.d.ts.map +1 -1
- package/dist/config/types/tools.js +0 -1
- package/dist/errors/app-error.d.ts +0 -1
- package/dist/errors/app-error.js +0 -1
- package/dist/http/auth.d.ts +0 -1
- package/dist/http/auth.d.ts.map +1 -1
- package/dist/http/auth.js +17 -13
- package/dist/http/auth.js.map +1 -1
- package/dist/http/cors.d.ts +0 -1
- package/dist/http/cors.js +4 -1
- package/dist/http/cors.js.map +1 -1
- package/dist/http/download-routes.d.ts +14 -0
- package/dist/http/download-routes.d.ts.map +1 -0
- package/dist/http/download-routes.js +131 -0
- package/dist/http/download-routes.js.map +1 -0
- package/dist/http/mcp-routes.d.ts +1 -2
- package/dist/http/mcp-routes.d.ts.map +1 -1
- package/dist/http/mcp-routes.js +1 -2
- package/dist/http/mcp-routes.js.map +1 -1
- package/dist/http/mcp-session-helpers.d.ts +13 -0
- package/dist/http/mcp-session-helpers.d.ts.map +1 -0
- package/dist/http/mcp-session-helpers.js +64 -0
- package/dist/http/mcp-session-helpers.js.map +1 -0
- package/dist/http/mcp-session.d.ts +1 -3
- package/dist/http/mcp-session.d.ts.map +1 -1
- package/dist/http/mcp-session.js +7 -71
- package/dist/http/mcp-session.js.map +1 -1
- package/dist/http/mcp-validation.d.ts +1 -2
- package/dist/http/mcp-validation.d.ts.map +1 -1
- package/dist/http/mcp-validation.js +6 -27
- package/dist/http/mcp-validation.js.map +1 -1
- package/dist/http/rate-limit.d.ts +1 -2
- package/dist/http/rate-limit.d.ts.map +1 -1
- package/dist/http/rate-limit.js +0 -1
- package/dist/http/rate-limit.js.map +1 -1
- package/dist/http/server-middleware.d.ts +9 -0
- package/dist/http/server-middleware.d.ts.map +1 -0
- package/dist/http/server-middleware.js +111 -0
- package/dist/http/server-middleware.js.map +1 -0
- package/dist/http/server.d.ts +0 -1
- package/dist/http/server.d.ts.map +1 -1
- package/dist/http/server.js +20 -99
- package/dist/http/server.js.map +1 -1
- package/dist/http/session-cleanup.d.ts +2 -0
- package/dist/http/session-cleanup.d.ts.map +1 -0
- package/dist/http/session-cleanup.js +37 -0
- package/dist/http/session-cleanup.js.map +1 -0
- package/dist/http/sessions.d.ts +1 -2
- package/dist/http/sessions.d.ts.map +1 -1
- package/dist/http/sessions.js +0 -1
- package/dist/index.d.ts +0 -1
- package/dist/index.js +13 -6
- package/dist/index.js.map +1 -1
- package/dist/middleware/error-handler.d.ts +0 -1
- package/dist/middleware/error-handler.js +0 -1
- package/dist/resources/cached-content.d.ts +0 -1
- package/dist/resources/cached-content.d.ts.map +1 -1
- package/dist/resources/cached-content.js +76 -12
- package/dist/resources/cached-content.js.map +1 -1
- package/dist/resources/index.d.ts +0 -1
- package/dist/resources/index.js +0 -1
- package/dist/server.d.ts +0 -1
- package/dist/server.d.ts.map +1 -1
- package/dist/server.js +8 -3
- package/dist/server.js.map +1 -1
- package/dist/services/cache.d.ts +7 -4
- package/dist/services/cache.d.ts.map +1 -1
- package/dist/services/cache.js +86 -26
- package/dist/services/cache.js.map +1 -1
- package/dist/services/context.d.ts +2 -2
- package/dist/services/context.d.ts.map +1 -1
- package/dist/services/context.js +0 -1
- package/dist/services/extractor.d.ts +1 -2
- package/dist/services/extractor.d.ts.map +1 -1
- package/dist/services/extractor.js +45 -18
- package/dist/services/extractor.js.map +1 -1
- package/dist/services/fetcher/agents.d.ts +0 -1
- package/dist/services/fetcher/agents.d.ts.map +1 -1
- package/dist/services/fetcher/agents.js +3 -7
- package/dist/services/fetcher/agents.js.map +1 -1
- package/dist/services/fetcher/errors.d.ts +0 -1
- package/dist/services/fetcher/errors.js +0 -1
- package/dist/services/fetcher/headers.d.ts.map +1 -1
- package/dist/services/fetcher/headers.js +2 -24
- package/dist/services/fetcher/headers.js.map +1 -1
- package/dist/services/fetcher/interceptors.d.ts +2 -2
- package/dist/services/fetcher/interceptors.d.ts.map +1 -1
- package/dist/services/fetcher/interceptors.js +30 -21
- package/dist/services/fetcher/interceptors.js.map +1 -1
- package/dist/services/fetcher/redirects.d.ts +0 -2
- package/dist/services/fetcher/redirects.d.ts.map +1 -1
- package/dist/services/fetcher/redirects.js +20 -18
- package/dist/services/fetcher/redirects.js.map +1 -1
- package/dist/services/fetcher/response.d.ts +0 -1
- package/dist/services/fetcher/response.js +4 -5
- package/dist/services/fetcher/retry-policy.d.ts +1 -28
- package/dist/services/fetcher/retry-policy.d.ts.map +1 -1
- package/dist/services/fetcher/retry-policy.js +119 -126
- package/dist/services/fetcher/retry-policy.js.map +1 -1
- package/dist/services/fetcher.d.ts +1 -2
- package/dist/services/fetcher.d.ts.map +1 -1
- package/dist/services/fetcher.js +18 -13
- package/dist/services/fetcher.js.map +1 -1
- package/dist/services/logger.d.ts +1 -2
- package/dist/services/logger.d.ts.map +1 -1
- package/dist/services/logger.js +0 -1
- package/dist/services/parser.d.ts +1 -3
- package/dist/services/parser.d.ts.map +1 -1
- package/dist/services/parser.js +5 -39
- package/dist/services/parser.js.map +1 -1
- package/dist/tools/handlers/fetch-links/link-extractor.d.ts.map +1 -1
- package/dist/tools/handlers/fetch-links/link-extractor.js +15 -19
- package/dist/tools/handlers/fetch-links/link-extractor.js.map +1 -1
- package/dist/tools/handlers/fetch-links.tool.d.ts.map +1 -1
- package/dist/tools/handlers/fetch-links.tool.js +0 -2
- package/dist/tools/handlers/fetch-links.tool.js.map +1 -1
- package/dist/tools/handlers/fetch-markdown.tool.d.ts +1 -2
- package/dist/tools/handlers/fetch-markdown.tool.d.ts.map +1 -1
- package/dist/tools/handlers/fetch-markdown.tool.js +50 -20
- package/dist/tools/handlers/fetch-markdown.tool.js.map +1 -1
- package/dist/tools/handlers/fetch-single.shared.d.ts +14 -3
- package/dist/tools/handlers/fetch-single.shared.d.ts.map +1 -1
- package/dist/tools/handlers/fetch-single.shared.js +66 -3
- package/dist/tools/handlers/fetch-single.shared.js.map +1 -1
- package/dist/tools/handlers/fetch-url.tool.d.ts +1 -2
- package/dist/tools/handlers/fetch-url.tool.d.ts.map +1 -1
- package/dist/tools/handlers/fetch-url.tool.js +39 -17
- package/dist/tools/handlers/fetch-url.tool.js.map +1 -1
- package/dist/tools/handlers/fetch-urls/validation.d.ts +0 -1
- package/dist/tools/handlers/fetch-urls/validation.d.ts.map +1 -1
- package/dist/tools/handlers/fetch-urls/validation.js +1 -1
- package/dist/tools/handlers/fetch-urls/validation.js.map +1 -1
- package/dist/tools/index.d.ts +0 -1
- package/dist/tools/index.d.ts.map +1 -1
- package/dist/tools/index.js +1 -20
- package/dist/tools/index.js.map +1 -1
- package/dist/tools/schemas.d.ts +57 -250
- package/dist/tools/schemas.d.ts.map +1 -1
- package/dist/tools/schemas.js +38 -198
- package/dist/tools/schemas.js.map +1 -1
- package/dist/tools/utils/cache-vary.d.ts +0 -2
- package/dist/tools/utils/cache-vary.d.ts.map +1 -1
- package/dist/tools/utils/cache-vary.js +8 -40
- package/dist/tools/utils/cache-vary.js.map +1 -1
- package/dist/tools/utils/common.d.ts +2 -4
- package/dist/tools/utils/common.d.ts.map +1 -1
- package/dist/tools/utils/common.js +6 -7
- package/dist/tools/utils/common.js.map +1 -1
- package/dist/tools/utils/content-transform.d.ts +1 -3
- package/dist/tools/utils/content-transform.d.ts.map +1 -1
- package/dist/tools/utils/content-transform.js +65 -14
- package/dist/tools/utils/content-transform.js.map +1 -1
- package/dist/tools/utils/fetch-pipeline.d.ts +1 -2
- package/dist/tools/utils/fetch-pipeline.d.ts.map +1 -1
- package/dist/tools/utils/fetch-pipeline.js +25 -21
- package/dist/tools/utils/fetch-pipeline.js.map +1 -1
- package/dist/tools/utils/inline-content.d.ts +3 -3
- package/dist/tools/utils/inline-content.d.ts.map +1 -1
- package/dist/tools/utils/inline-content.js +0 -1
- package/dist/transformers/jsonl.transformer.d.ts +1 -2
- package/dist/transformers/jsonl.transformer.d.ts.map +1 -1
- package/dist/transformers/jsonl.transformer.js +0 -1
- package/dist/transformers/jsonl.transformer.js.map +1 -1
- package/dist/transformers/markdown.transformer.d.ts +1 -2
- package/dist/transformers/markdown.transformer.d.ts.map +1 -1
- package/dist/transformers/markdown.transformer.js +11 -7
- package/dist/transformers/markdown.transformer.js.map +1 -1
- package/dist/utils/code-language.d.ts +2 -0
- package/dist/utils/code-language.d.ts.map +1 -0
- package/dist/utils/code-language.js +56 -0
- package/dist/utils/code-language.js.map +1 -0
- package/dist/utils/content-cleaner.d.ts +0 -2
- package/dist/utils/content-cleaner.d.ts.map +1 -1
- package/dist/utils/content-cleaner.js +0 -4
- package/dist/utils/content-cleaner.js.map +1 -1
- package/dist/utils/crypto.d.ts +2 -0
- package/dist/utils/crypto.d.ts.map +1 -0
- package/dist/utils/crypto.js +32 -0
- package/dist/utils/crypto.js.map +1 -0
- package/dist/utils/download-url.d.ts +8 -0
- package/dist/utils/download-url.d.ts.map +1 -0
- package/dist/utils/download-url.js +27 -0
- package/dist/utils/download-url.js.map +1 -0
- package/dist/utils/error-utils.d.ts +3 -0
- package/dist/utils/error-utils.d.ts.map +1 -0
- package/dist/utils/error-utils.js +12 -0
- package/dist/utils/error-utils.js.map +1 -0
- package/dist/utils/filename-generator.d.ts +1 -0
- package/dist/utils/filename-generator.d.ts.map +1 -0
- package/dist/utils/filename-generator.js +59 -0
- package/dist/utils/filename-generator.js.map +1 -0
- package/dist/utils/header-normalizer.d.ts +7 -4
- package/dist/utils/header-normalizer.d.ts.map +1 -1
- package/dist/utils/header-normalizer.js +23 -17
- package/dist/utils/header-normalizer.js.map +1 -1
- package/dist/utils/html-truncator.d.ts +0 -1
- package/dist/utils/html-truncator.js +0 -1
- package/dist/utils/sanitizer.d.ts +0 -1
- package/dist/utils/sanitizer.js +0 -1
- package/dist/utils/tool-error-handler.d.ts +1 -3
- package/dist/utils/tool-error-handler.d.ts.map +1 -1
- package/dist/utils/tool-error-handler.js +11 -6
- package/dist/utils/tool-error-handler.js.map +1 -1
- package/dist/utils/url-sanitizer.d.ts +2 -0
- package/dist/utils/url-sanitizer.d.ts.map +1 -0
- package/dist/utils/url-sanitizer.js +12 -0
- package/dist/utils/url-sanitizer.js.map +1 -0
- package/dist/utils/url-validator.d.ts +1 -3
- package/dist/utils/url-validator.d.ts.map +1 -1
- package/dist/utils/url-validator.js +89 -53
- package/dist/utils/url-validator.js.map +1 -1
- package/package.json +7 -9
|
@@ -1,4 +1,7 @@
|
|
|
1
1
|
import { config } from '../../config/index.js';
|
|
2
|
+
import { buildFileDownloadInfo } from '../../utils/download-url.js';
|
|
3
|
+
import { generateSafeFilename } from '../../utils/filename-generator.js';
|
|
4
|
+
import { createToolErrorResponse } from '../../utils/tool-error-handler.js';
|
|
2
5
|
import { appendHeaderVary } from '../utils/cache-vary.js';
|
|
3
6
|
import { executeFetchPipeline } from '../utils/fetch-pipeline.js';
|
|
4
7
|
import { applyInlineContentLimit } from '../utils/inline-content.js';
|
|
@@ -9,6 +12,7 @@ export async function performSharedFetch(options) {
|
|
|
9
12
|
extractMainContent: options.extractMainContent,
|
|
10
13
|
includeMetadata: options.includeMetadata,
|
|
11
14
|
maxContentLength: options.maxContentLength,
|
|
15
|
+
...(options.cacheVariant ? { variant: options.cacheVariant } : {}),
|
|
12
16
|
...(options.format === 'markdown' ? {} : { contentBlocks: true }),
|
|
13
17
|
}, options.customHeaders);
|
|
14
18
|
const pipeline = await executeFetchPipeline({
|
|
@@ -19,10 +23,36 @@ export async function performSharedFetch(options) {
|
|
|
19
23
|
timeout: options.timeout,
|
|
20
24
|
cacheVary,
|
|
21
25
|
transform: options.transform,
|
|
26
|
+
serialize: options.serialize,
|
|
27
|
+
deserialize: options.deserialize,
|
|
22
28
|
});
|
|
23
29
|
const inlineResult = applyInlineContentLimit(pipeline.data.content, pipeline.cacheKey ?? null, options.format);
|
|
24
30
|
return { pipeline, inlineResult };
|
|
25
31
|
}
|
|
32
|
+
export function getFileDownloadInfo(context) {
|
|
33
|
+
return buildFileDownloadInfo({
|
|
34
|
+
cacheKey: context.cacheKey,
|
|
35
|
+
url: context.url,
|
|
36
|
+
title: context.title,
|
|
37
|
+
});
|
|
38
|
+
}
|
|
39
|
+
export function getInlineErrorResponse(inlineResult, url) {
|
|
40
|
+
if (!inlineResult.error)
|
|
41
|
+
return null;
|
|
42
|
+
return createToolErrorResponse(inlineResult.error, url, 'INTERNAL_ERROR');
|
|
43
|
+
}
|
|
44
|
+
export function applyInlineResultToStructuredContent(structuredContent, inlineResult, contentKey) {
|
|
45
|
+
if (inlineResult.truncated) {
|
|
46
|
+
structuredContent.truncated = true;
|
|
47
|
+
}
|
|
48
|
+
if (typeof inlineResult.content === 'string') {
|
|
49
|
+
structuredContent[contentKey] = inlineResult.content;
|
|
50
|
+
}
|
|
51
|
+
if (inlineResult.resourceUri) {
|
|
52
|
+
structuredContent.resourceUri = inlineResult.resourceUri;
|
|
53
|
+
structuredContent.resourceMimeType = inlineResult.resourceMimeType;
|
|
54
|
+
}
|
|
55
|
+
}
|
|
26
56
|
function serializeStructuredContent(structuredContent, fromCache) {
|
|
27
57
|
return JSON.stringify(structuredContent, fromCache ? undefined : null, fromCache ? undefined : 2);
|
|
28
58
|
}
|
|
@@ -38,12 +68,45 @@ function buildResourceLink(inlineResult, name) {
|
|
|
38
68
|
description: `Content exceeds inline limit (${config.constants.maxInlineContentChars} chars)`,
|
|
39
69
|
};
|
|
40
70
|
}
|
|
41
|
-
|
|
71
|
+
function buildEmbeddedResource(content, mimeType, url, title) {
|
|
72
|
+
if (!content) {
|
|
73
|
+
return null;
|
|
74
|
+
}
|
|
75
|
+
// Generate a proper filename with extension
|
|
76
|
+
const extension = mimeType === 'text/markdown' ? '.md' : '.jsonl';
|
|
77
|
+
const filename = generateSafeFilename(url, title, undefined, extension);
|
|
78
|
+
// Use file: URI scheme with filename for better VS Code integration
|
|
79
|
+
const uri = `file:///${filename}`;
|
|
80
|
+
return {
|
|
81
|
+
type: 'resource',
|
|
82
|
+
resource: {
|
|
83
|
+
uri,
|
|
84
|
+
mimeType,
|
|
85
|
+
text: content,
|
|
86
|
+
},
|
|
87
|
+
};
|
|
88
|
+
}
|
|
89
|
+
export function buildToolContentBlocks(structuredContent, fromCache, inlineResult, resourceName, cacheKey, fullContent, format, url, title) {
|
|
42
90
|
const textBlock = {
|
|
43
91
|
type: 'text',
|
|
44
92
|
text: serializeStructuredContent(structuredContent, fromCache),
|
|
45
93
|
};
|
|
94
|
+
const blocks = [textBlock];
|
|
95
|
+
// Embed full content in stdio mode; HTTP mode relies on inline content or links.
|
|
96
|
+
const mimeType = format === 'markdown' ? 'text/markdown' : 'application/jsonl';
|
|
97
|
+
const contentToEmbed = config.runtime.httpMode
|
|
98
|
+
? inlineResult.content
|
|
99
|
+
: (fullContent ?? inlineResult.content);
|
|
100
|
+
if (typeof contentToEmbed === 'string' && url) {
|
|
101
|
+
const embeddedResource = buildEmbeddedResource(contentToEmbed, mimeType, url, title);
|
|
102
|
+
if (embeddedResource) {
|
|
103
|
+
blocks.push(embeddedResource);
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
// Add resource link for HTTP mode downloads (only when truncated)
|
|
46
107
|
const resourceLink = buildResourceLink(inlineResult, resourceName);
|
|
47
|
-
|
|
108
|
+
if (resourceLink) {
|
|
109
|
+
blocks.push(resourceLink);
|
|
110
|
+
}
|
|
111
|
+
return blocks;
|
|
48
112
|
}
|
|
49
|
-
//# sourceMappingURL=fetch-single.shared.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"fetch-single.shared.js","sourceRoot":"","sources":["../../../src/tools/handlers/fetch-single.shared.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,uBAAuB,CAAC;
|
|
1
|
+
{"version":3,"file":"fetch-single.shared.js","sourceRoot":"","sources":["../../../src/tools/handlers/fetch-single.shared.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,uBAAuB,CAAC;AAU/C,OAAO,EAAE,qBAAqB,EAAE,MAAM,6BAA6B,CAAC;AACpE,OAAO,EAAE,oBAAoB,EAAE,MAAM,mCAAmC,CAAC;AACzE,OAAO,EAAE,uBAAuB,EAAE,MAAM,mCAAmC,CAAC;AAC5E,OAAO,EAAE,gBAAgB,EAAE,MAAM,wBAAwB,CAAC;AAC1D,OAAO,EAAE,oBAAoB,EAAE,MAAM,4BAA4B,CAAC;AAClE,OAAO,EAAE,uBAAuB,EAAE,MAAM,4BAA4B,CAAC;AAgBrE,MAAM,CAAC,KAAK,UAAU,kBAAkB,CACtC,OAA8B;IAK9B,MAAM,cAAc,GAAG,OAAO,CAAC,MAAM,KAAK,UAAU,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,KAAK,CAAC;IAC1E,MAAM,SAAS,GAAG,gBAAgB,CAChC;QACE,MAAM,EAAE,OAAO,CAAC,MAAM;QACtB,kBAAkB,EAAE,OAAO,CAAC,kBAAkB;QAC9C,eAAe,EAAE,OAAO,CAAC,eAAe;QACxC,gBAAgB,EAAE,OAAO,CAAC,gBAAgB;QAC1C,GAAG,CAAC,OAAO,CAAC,MAAM,KAAK,UAAU,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC;KAClE,EACD,OAAO,CAAC,aAAa,CACtB,CAAC;IAEF,MAAM,QAAQ,GAAG,MAAM,oBAAoB,CAAI;QAC7C,GAAG,EAAE,OAAO,CAAC,GAAG;QAChB,cAAc;QACd,aAAa,EAAE,OAAO,CAAC,aAAa;QACpC,OAAO,EAAE,OAAO,CAAC,OAAO;QACxB,OAAO,EAAE,OAAO,CAAC,OAAO;QACxB,SAAS;QACT,SAAS,EAAE,OAAO,CAAC,SAAS;KAC7B,CAAC,CAAC;IAEH,MAAM,YAAY,GAAG,uBAAuB,CAC1C,QAAQ,CAAC,IAAI,CAAC,OAAO,EACrB,QAAQ,CAAC,QAAQ,IAAI,IAAI,EACzB,OAAO,CAAC,MAAM,CACf,CAAC;IAEF,OAAO,EAAE,QAAQ,EAAE,YAAY,EAAE,CAAC;AACpC,CAAC;AAUD,MAAM,UAAU,mBAAmB,CACjC,OAAwB;IAExB,OAAO,qBAAqB,CAAC;QAC3B,QAAQ,EAAE,OAAO,CAAC,QAAQ;QAC1B,GAAG,EAAE,OAAO,CAAC,GAAG;QAChB,KAAK,EAAE,OAAO,CAAC,KAAK;KACrB,CAAC,CAAC;AACL,CAAC;AAED,MAAM,UAAU,sBAAsB,CACpC,YAA0B,EAC1B,GAAW;IAEX,IAAI,CAAC,YAAY,CAAC,KAAK;QAAE,OAAO,IAAI,CAAC;IACrC,OAAO,uBAAuB,CAAC,YAAY,CAAC,KAAK,EAAE,GAAG,EAAE,gBAAgB,CAAC,CAAC;AAC5E,CAAC;AAED,MAAM,UAAU,oCAAoC,CAClD,iBAA0C,EAC1C,YAA0B,EAC1B,UAAkB;IAElB,IAAI,YAAY,CAAC,SAAS,EAAE,CAAC;QAC3B,iBAAiB,CAAC,SAAS,GAAG,IAAI,CAAC;IACrC,CAAC;IAED,IAAI,OAAO,YAAY,CAAC,OAAO,KAAK,QAAQ,EAAE,CAAC;QAC7C,iBAAiB,CAAC,UAAU,CAAC,GAAG,YAAY,CAAC,OAAO,CAAC;IACvD,CAAC;IAED,IAAI,YAAY,CAAC,WAAW,EAAE,CAAC;QAC7B,iBAAiB,CAAC,WAAW,GAAG,YAAY,CAAC,WAAW,CAAC;QACzD,iBAAiB,CAAC,gBAAgB,GAAG,YAAY,CAAC,gBAAgB,CAAC;IACrE,CAAC;AACH,CAAC;AAED,SAAS,0BAA0B,CACjC,iBAA0C,EAC1C,SAAkB;IAElB,OAAO,IAAI,CAAC,SAAS,CACnB,iBAAiB,EACjB,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,EAC5B,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAC1B,CAAC;AACJ,CAAC;AAED,SAAS,iBAAiB,CACxB,YAA0B,EAC1B,IAAY;IAEZ,IAAI,CAAC,YAAY,CAAC,WAAW,EAAE,CAAC;QAC9B,OAAO,IAAI,CAAC;IACd,CAAC;IAED,OAAO;QACL,IAAI,EAAE,eAAe;QACrB,GAAG,EAAE,YAAY,CAAC,WAAW;QAC7B,IAAI;QACJ,QAAQ,EAAE,YAAY,CAAC,gBAAgB;QACvC,WAAW,EAAE,iCAAiC,MAAM,CAAC,SAAS,CAAC,qBAAqB,SAAS;KAC9F,CAAC;AACJ,CAAC;AAED,SAAS,qBAAqB,CAC5B,OAAe,EACf,QAAgB,EAChB,GAAW,EACX,KAAc;IAEd,IAAI,CAAC,OAAO,EAAE,CAAC;QACb,OAAO,IAAI,CAAC;IACd,CAAC;IAED,4CAA4C;IAC5C,MAAM,SAAS,GAAG,QAAQ,KAAK,eAAe,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,QAAQ,CAAC;IAClE,MAAM,QAAQ,GAAG,oBAAoB,CAAC,GAAG,EAAE,KAAK,EAAE,SAAS,EAAE,SAAS,CAAC,CAAC;IAExE,oEAAoE;IACpE,MAAM,GAAG,GAAG,WAAW,QAAQ,EAAE,CAAC;IAElC,OAAO;QACL,IAAI,EAAE,UAAU;QAChB,QAAQ,EAAE;YACR,GAAG;YACH,QAAQ;YACR,IAAI,EAAE,OAAO;SACd;KACF,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,sBAAsB,CACpC,iBAA0C,EAC1C,SAAkB,EAClB,YAA0B,EAC1B,YAAoB,EACpB,QAAwB,EACxB,WAAoB,EACpB,MAA0B,EAC1B,GAAY,EACZ,KAAc;IAEd,MAAM,SAAS,GAAqB;QAClC,IAAI,EAAE,MAAM;QACZ,IAAI,EAAE,0BAA0B,CAAC,iBAAiB,EAAE,SAAS,CAAC;KAC/D,CAAC;IAEF,MAAM,MAAM,GAAuB,CAAC,SAAS,CAAC,CAAC;IAE/C,0EAA0E;IAC1E,MAAM,QAAQ,GACZ,MAAM,KAAK,UAAU,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,mBAAmB,CAAC;IAChE,MAAM,cAAc,GAAG,WAAW,IAAI,YAAY,CAAC,OAAO,CAAC;IAC3D,IAAI,cAAc,IAAI,GAAG,EAAE,CAAC;QAC1B,MAAM,gBAAgB,GAAG,qBAAqB,CAC5C,cAAc,EACd,QAAQ,EACR,GAAG,EACH,KAAK,CACN,CAAC;QACF,IAAI,gBAAgB,EAAE,CAAC;YACrB,MAAM,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC;QAChC,CAAC;IACH,CAAC;IAED,kEAAkE;IAClE,MAAM,YAAY,GAAG,iBAAiB,CAAC,YAAY,EAAE,YAAY,CAAC,CAAC;IACnE,IAAI,YAAY,EAAE,CAAC;QACjB,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;IAC5B,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC"}
|
|
@@ -1,5 +1,4 @@
|
|
|
1
|
-
import type { FetchUrlInput, ToolResponseBase } from '../../config/types.js';
|
|
1
|
+
import type { FetchUrlInput, ToolResponseBase } from '../../config/types/tools.js';
|
|
2
2
|
export declare const FETCH_URL_TOOL_NAME = "fetch-url";
|
|
3
3
|
export declare const FETCH_URL_TOOL_DESCRIPTION = "Fetches a webpage and converts it to AI-readable JSONL format with semantic content blocks. Supports custom headers, retries, and content length limits.";
|
|
4
4
|
export declare function fetchUrlToolHandler(input: FetchUrlInput): Promise<ToolResponseBase>;
|
|
5
|
-
//# sourceMappingURL=fetch-url.tool.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"fetch-url.tool.d.ts","sourceRoot":"","sources":["../../../src/tools/handlers/fetch-url.tool.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"fetch-url.tool.d.ts","sourceRoot":"","sources":["../../../src/tools/handlers/fetch-url.tool.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EACV,aAAa,EACb,gBAAgB,EACjB,MAAM,6BAA6B,CAAC;AAqBrC,eAAO,MAAM,mBAAmB,cAAc,CAAC;AAC/C,eAAO,MAAM,0BAA0B,6JACqH,CAAC;AAoH7J,wBAAsB,mBAAmB,CACvC,KAAK,EAAE,aAAa,GACnB,OAAO,CAAC,gBAAgB,CAAC,CAU3B"}
|
|
@@ -1,13 +1,45 @@
|
|
|
1
|
+
import { config } from '../../config/index.js';
|
|
1
2
|
import { logDebug, logError } from '../../services/logger.js';
|
|
2
3
|
import { createToolErrorResponse, handleToolError, } from '../../utils/tool-error-handler.js';
|
|
3
4
|
import { transformHtmlToJsonl, transformHtmlToMarkdownWithBlocks, } from '../utils/content-transform.js';
|
|
4
|
-
import { buildToolContentBlocks, performSharedFetch, } from './fetch-single.shared.js';
|
|
5
|
+
import { applyInlineResultToStructuredContent, buildToolContentBlocks, getInlineErrorResponse, performSharedFetch, } from './fetch-single.shared.js';
|
|
5
6
|
export const FETCH_URL_TOOL_NAME = 'fetch-url';
|
|
6
7
|
export const FETCH_URL_TOOL_DESCRIPTION = 'Fetches a webpage and converts it to AI-readable JSONL format with semantic content blocks. Supports custom headers, retries, and content length limits.';
|
|
8
|
+
function isRecord(value) {
|
|
9
|
+
return value !== null && typeof value === 'object';
|
|
10
|
+
}
|
|
11
|
+
function deserializeJsonlTransformResult(cached) {
|
|
12
|
+
try {
|
|
13
|
+
const parsed = JSON.parse(cached);
|
|
14
|
+
if (!isRecord(parsed))
|
|
15
|
+
return undefined;
|
|
16
|
+
const { content, contentBlocks, title, truncated } = parsed;
|
|
17
|
+
if (typeof content !== 'string')
|
|
18
|
+
return undefined;
|
|
19
|
+
if (typeof contentBlocks !== 'number' || !Number.isFinite(contentBlocks)) {
|
|
20
|
+
return undefined;
|
|
21
|
+
}
|
|
22
|
+
if (title !== undefined && typeof title !== 'string')
|
|
23
|
+
return undefined;
|
|
24
|
+
if (truncated !== undefined && typeof truncated !== 'boolean') {
|
|
25
|
+
return undefined;
|
|
26
|
+
}
|
|
27
|
+
const resolvedTitle = typeof title === 'string' ? title : undefined;
|
|
28
|
+
return {
|
|
29
|
+
content,
|
|
30
|
+
contentBlocks,
|
|
31
|
+
title: resolvedTitle,
|
|
32
|
+
...(truncated !== undefined ? { truncated } : {}),
|
|
33
|
+
};
|
|
34
|
+
}
|
|
35
|
+
catch {
|
|
36
|
+
return undefined;
|
|
37
|
+
}
|
|
38
|
+
}
|
|
7
39
|
function resolveFetchUrlOptions(input) {
|
|
8
40
|
return {
|
|
9
|
-
extractMainContent: input.extractMainContent ??
|
|
10
|
-
includeMetadata: input.includeMetadata ??
|
|
41
|
+
extractMainContent: input.extractMainContent ?? config.extraction.extractMainContent,
|
|
42
|
+
includeMetadata: input.includeMetadata ?? config.extraction.includeMetadata,
|
|
11
43
|
maxContentLength: input.maxContentLength,
|
|
12
44
|
format: input.format ?? 'jsonl',
|
|
13
45
|
};
|
|
@@ -33,20 +65,9 @@ function buildFetchUrlStructuredContent(format, pipeline, inlineResult) {
|
|
|
33
65
|
if (inlineResult.truncated) {
|
|
34
66
|
structuredContent.truncated = true;
|
|
35
67
|
}
|
|
36
|
-
|
|
37
|
-
structuredContent.content = inlineResult.content;
|
|
38
|
-
}
|
|
39
|
-
if (inlineResult.resourceUri) {
|
|
40
|
-
structuredContent.resourceUri = inlineResult.resourceUri;
|
|
41
|
-
structuredContent.resourceMimeType = inlineResult.resourceMimeType;
|
|
42
|
-
}
|
|
68
|
+
applyInlineResultToStructuredContent(structuredContent, inlineResult, 'content');
|
|
43
69
|
return structuredContent;
|
|
44
70
|
}
|
|
45
|
-
function getInlineErrorResponse(inlineResult, url) {
|
|
46
|
-
if (!inlineResult.error)
|
|
47
|
-
return null;
|
|
48
|
-
return createToolErrorResponse(inlineResult.error, url, 'INTERNAL_ERROR');
|
|
49
|
-
}
|
|
50
71
|
function logFetchUrlStart(url, options) {
|
|
51
72
|
logDebug('Fetching URL', {
|
|
52
73
|
url,
|
|
@@ -65,13 +86,15 @@ async function fetchUrlPipeline(url, input, options) {
|
|
|
65
86
|
customHeaders: input.customHeaders,
|
|
66
87
|
retries: input.retries,
|
|
67
88
|
timeout: input.timeout,
|
|
89
|
+
cacheVariant: options.format === 'markdown' ? 'markdown-with-blocks' : undefined,
|
|
68
90
|
transform: buildFetchUrlTransform(options),
|
|
91
|
+
deserialize: deserializeJsonlTransformResult,
|
|
69
92
|
});
|
|
70
93
|
}
|
|
71
94
|
function buildFetchUrlResponse(pipeline, inlineResult, format) {
|
|
72
95
|
const structuredContent = buildFetchUrlStructuredContent(format, pipeline, inlineResult);
|
|
73
96
|
return {
|
|
74
|
-
content: buildToolContentBlocks(structuredContent, pipeline.fromCache, inlineResult, 'Fetched content'),
|
|
97
|
+
content: buildToolContentBlocks(structuredContent, pipeline.fromCache, inlineResult, 'Fetched content', pipeline.cacheKey, pipeline.data.content, format, pipeline.url, pipeline.data.title),
|
|
75
98
|
structuredContent,
|
|
76
99
|
};
|
|
77
100
|
}
|
|
@@ -97,4 +120,3 @@ async function executeFetchUrl(input) {
|
|
|
97
120
|
return inlineError;
|
|
98
121
|
return buildFetchUrlResponse(pipeline, inlineResult, options.format);
|
|
99
122
|
}
|
|
100
|
-
//# sourceMappingURL=fetch-url.tool.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"fetch-url.tool.js","sourceRoot":"","sources":["../../../src/tools/handlers/fetch-url.tool.ts"],"names":[],"mappings":"AAOA,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,0BAA0B,CAAC;AAE9D,OAAO,EACL,uBAAuB,EACvB,eAAe,GAChB,MAAM,mCAAmC,CAAC;AAC3C,OAAO,EACL,oBAAoB,EACpB,iCAAiC,GAClC,MAAM,+BAA+B,CAAC;AAEvC,OAAO,EACL,sBAAsB,EAEtB,kBAAkB,GACnB,MAAM,0BAA0B,CAAC;AAElC,MAAM,CAAC,MAAM,mBAAmB,GAAG,WAAW,CAAC;AAC/C,MAAM,CAAC,MAAM,0BAA0B,GACrC,0JAA0J,CAAC;AAW7J,SAAS,sBAAsB,CAAC,KAAoB;IAClD,OAAO;QACL,kBAAkB,EAAE,KAAK,CAAC,kBAAkB,IAAI,IAAI;QACpD,eAAe,EAAE,KAAK,CAAC,eAAe,IAAI,IAAI;QAC9C,gBAAgB,EAAE,KAAK,CAAC,gBAAgB;QACxC,MAAM,EAAE,KAAK,CAAC,MAAM,IAAI,OAAO;KAChC,CAAC;AACJ,CAAC;AAED,SAAS,sBAAsB,CAAC,OAAwB;IACtD,OAAO,CAAC,IAAY,EAAE,GAAW,EAAE,EAAE,CACnC,OAAO,CAAC,MAAM,KAAK,UAAU;QAC3B,CAAC,CAAC,iCAAiC,CAAC,IAAI,EAAE,GAAG,EAAE,OAAO,CAAC;QACvD,CAAC,CAAC,oBAAoB,CAAC,IAAI,EAAE,GAAG,EAAE,OAAO,CAAC,CAAC;AACjD,CAAC;AAED,SAAS,8BAA8B,CACrC,MAAc,EACd,QAA8C,EAC9C,YAA0B;IAE1B,MAAM,iBAAiB,GAA4B;QACjD,GAAG,EAAE,QAAQ,CAAC,GAAG;QACjB,KAAK,EAAE,QAAQ,CAAC,IAAI,CAAC,KAAK;QAC1B,aAAa,EAAE,QAAQ,CAAC,IAAI,CAAC,aAAa;QAC1C,SAAS,EAAE,QAAQ,CAAC,SAAS;QAC7B,MAAM;QACN,WAAW,EAAE,YAAY,CAAC,WAAW;QACrC,MAAM,EAAE,QAAQ,CAAC,SAAS;KAC3B,CAAC;IAEF,IAAI,QAAQ,CAAC,IAAI,CAAC,SAAS,EAAE,CAAC;QAC5B,iBAAiB,CAAC,SAAS,GAAG,IAAI,CAAC;IACrC,CAAC;IAED,IAAI,YAAY,CAAC,SAAS,EAAE,CAAC;QAC3B,iBAAiB,CAAC,SAAS,GAAG,IAAI,CAAC;IACrC,CAAC;IAED,
|
|
1
|
+
{"version":3,"file":"fetch-url.tool.js","sourceRoot":"","sources":["../../../src/tools/handlers/fetch-url.tool.ts"],"names":[],"mappings":"AAOA,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,0BAA0B,CAAC;AAE9D,OAAO,EACL,uBAAuB,EACvB,eAAe,GAChB,MAAM,mCAAmC,CAAC;AAC3C,OAAO,EACL,oBAAoB,EACpB,iCAAiC,GAClC,MAAM,+BAA+B,CAAC;AAEvC,OAAO,EACL,oCAAoC,EACpC,sBAAsB,EACtB,sBAAsB,EAEtB,kBAAkB,GACnB,MAAM,0BAA0B,CAAC;AAElC,MAAM,CAAC,MAAM,mBAAmB,GAAG,WAAW,CAAC;AAC/C,MAAM,CAAC,MAAM,0BAA0B,GACrC,0JAA0J,CAAC;AAW7J,SAAS,sBAAsB,CAAC,KAAoB;IAClD,OAAO;QACL,kBAAkB,EAAE,KAAK,CAAC,kBAAkB,IAAI,IAAI;QACpD,eAAe,EAAE,KAAK,CAAC,eAAe,IAAI,IAAI;QAC9C,gBAAgB,EAAE,KAAK,CAAC,gBAAgB;QACxC,MAAM,EAAE,KAAK,CAAC,MAAM,IAAI,OAAO;KAChC,CAAC;AACJ,CAAC;AAED,SAAS,sBAAsB,CAAC,OAAwB;IACtD,OAAO,CAAC,IAAY,EAAE,GAAW,EAAE,EAAE,CACnC,OAAO,CAAC,MAAM,KAAK,UAAU;QAC3B,CAAC,CAAC,iCAAiC,CAAC,IAAI,EAAE,GAAG,EAAE,OAAO,CAAC;QACvD,CAAC,CAAC,oBAAoB,CAAC,IAAI,EAAE,GAAG,EAAE,OAAO,CAAC,CAAC;AACjD,CAAC;AAED,SAAS,8BAA8B,CACrC,MAAc,EACd,QAA8C,EAC9C,YAA0B;IAE1B,MAAM,iBAAiB,GAA4B;QACjD,GAAG,EAAE,QAAQ,CAAC,GAAG;QACjB,KAAK,EAAE,QAAQ,CAAC,IAAI,CAAC,KAAK;QAC1B,aAAa,EAAE,QAAQ,CAAC,IAAI,CAAC,aAAa;QAC1C,SAAS,EAAE,QAAQ,CAAC,SAAS;QAC7B,MAAM;QACN,WAAW,EAAE,YAAY,CAAC,WAAW;QACrC,MAAM,EAAE,QAAQ,CAAC,SAAS;KAC3B,CAAC;IAEF,IAAI,QAAQ,CAAC,IAAI,CAAC,SAAS,EAAE,CAAC;QAC5B,iBAAiB,CAAC,SAAS,GAAG,IAAI,CAAC;IACrC,CAAC;IAED,IAAI,YAAY,CAAC,SAAS,EAAE,CAAC;QAC3B,iBAAiB,CAAC,SAAS,GAAG,IAAI,CAAC;IACrC,CAAC;IAED,oCAAoC,CAClC,iBAAiB,EACjB,YAAY,EACZ,SAAS,CACV,CAAC;IAEF,OAAO,iBAAiB,CAAC;AAC3B,CAAC;AAED,SAAS,gBAAgB,CAAC,GAAW,EAAE,OAAwB;IAC7D,QAAQ,CAAC,cAAc,EAAE;QACvB,GAAG;QACH,kBAAkB,EAAE,OAAO,CAAC,kBAAkB;QAC9C,eAAe,EAAE,OAAO,CAAC,eAAe;QACxC,MAAM,EAAE,OAAO,CAAC,MAAM;KACvB,CAAC,CAAC;AACL,CAAC;AAED,KAAK,UAAU,gBAAgB,CAC7B,GAAW,EACX,KAAoB,EACpB,OAAwB;IAKxB,OAAO,kBAAkB,CAAuB;QAC9C,GAAG;QACH,MAAM,EAAE,OAAO,CAAC,MAAM;QACtB,kBAAkB,EAAE,OAAO,CAAC,kBAAkB;QAC9C,eAAe,EAAE,OAAO,CAAC,eAAe;QACxC,gBAAgB,EAAE,OAAO,CAAC,gBAAgB;QAC1C,aAAa,EAAE,KAAK,CAAC,aAAa;QAClC,OAAO,EAAE,KAAK,CAAC,OAAO;QACtB,OAAO,EAAE,KAAK,CAAC,OAAO;QACtB,SAAS,EAAE,sBAAsB,CAAC,OAAO,CAAC;KAC3C,CAAC,CAAC;AACL,CAAC;AAED,SAAS,qBAAqB,CAC5B,QAA8C,EAC9C,YAA0B,EAC1B,MAAc;IAEd,MAAM,iBAAiB,GAAG,8BAA8B,CACtD,MAAM,EACN,QAAQ,EACR,YAAY,CACb,CAAC;IAEF,OAAO;QACL,OAAO,EAAE,sBAAsB,CAC7B,iBAAiB,EACjB,QAAQ,CAAC,SAAS,EAClB,YAAY,EACZ,iBAAiB,EACjB,QAAQ,CAAC,QAAQ,EACjB,QAAQ,CAAC,IAAI,CAAC,OAAO,EACrB,MAAM,EACN,QAAQ,CAAC,GAAG,EACZ,QAAQ,CAAC,IAAI,CAAC,KAAK,CACpB;QACD,iBAAiB;KAClB,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,mBAAmB,CACvC,KAAoB;IAEpB,IAAI,CAAC;QACH,OAAO,MAAM,eAAe,CAAC,KAAK,CAAC,CAAC;IACtC,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,QAAQ,CACN,sBAAsB,EACtB,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS,CAC3C,CAAC;QACF,OAAO,eAAe,CAAC,KAAK,EAAE,KAAK,CAAC,GAAG,EAAE,qBAAqB,CAAC,CAAC;IAClE,CAAC;AACH,CAAC;AAED,KAAK,UAAU,eAAe,CAC5B,KAAoB;IAEpB,MAAM,EAAE,GAAG,EAAE,GAAG,KAAK,CAAC;IACtB,IAAI,CAAC,GAAG,EAAE,CAAC;QACT,OAAO,uBAAuB,CAAC,iBAAiB,EAAE,EAAE,EAAE,kBAAkB,CAAC,CAAC;IAC5E,CAAC;IAED,MAAM,OAAO,GAAG,sBAAsB,CAAC,KAAK,CAAC,CAAC;IAC9C,gBAAgB,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;IAE/B,MAAM,EAAE,QAAQ,EAAE,YAAY,EAAE,GAAG,MAAM,gBAAgB,CACvD,GAAG,EACH,KAAK,EACL,OAAO,CACR,CAAC;IAEF,MAAM,WAAW,GAAG,sBAAsB,CAAC,YAAY,EAAE,GAAG,CAAC,CAAC;IAC9D,IAAI,WAAW;QAAE,OAAO,WAAW,CAAC;IACpC,OAAO,qBAAqB,CAAC,QAAQ,EAAE,YAAY,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;AACvE,CAAC"}
|
|
@@ -1,5 +1,4 @@
|
|
|
1
1
|
import type { FetchUrlsInput, ToolResponseBase } from '../../../config/types.js';
|
|
2
|
-
export declare const MAX_URLS_PER_BATCH = 10;
|
|
3
2
|
export declare const DEFAULT_CONCURRENCY = 3;
|
|
4
3
|
export declare const MAX_CONCURRENCY = 5;
|
|
5
4
|
export declare function validateBatchInput(input: FetchUrlsInput): string[] | ToolResponseBase;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"validation.d.ts","sourceRoot":"","sources":["../../../../src/tools/handlers/fetch-urls/validation.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,cAAc,EACd,gBAAgB,EACjB,MAAM,0BAA0B,CAAC;
|
|
1
|
+
{"version":3,"file":"validation.d.ts","sourceRoot":"","sources":["../../../../src/tools/handlers/fetch-urls/validation.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,cAAc,EACd,gBAAgB,EACjB,MAAM,0BAA0B,CAAC;AAKlC,eAAO,MAAM,mBAAmB,IAAI,CAAC;AACrC,eAAO,MAAM,eAAe,IAAI,CAAC;AAEjC,wBAAgB,kBAAkB,CAChC,KAAK,EAAE,cAAc,GACpB,MAAM,EAAE,GAAG,gBAAgB,CA8B7B"}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { createToolErrorResponse } from '../../../utils/tool-error-handler.js';
|
|
2
|
-
|
|
2
|
+
const MAX_URLS_PER_BATCH = 10;
|
|
3
3
|
export const DEFAULT_CONCURRENCY = 3;
|
|
4
4
|
export const MAX_CONCURRENCY = 5;
|
|
5
5
|
export function validateBatchInput(input) {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"validation.js","sourceRoot":"","sources":["../../../../src/tools/handlers/fetch-urls/validation.ts"],"names":[],"mappings":"AAKA,OAAO,EAAE,uBAAuB,EAAE,MAAM,sCAAsC,CAAC;AAE/E,MAAM,
|
|
1
|
+
{"version":3,"file":"validation.js","sourceRoot":"","sources":["../../../../src/tools/handlers/fetch-urls/validation.ts"],"names":[],"mappings":"AAKA,OAAO,EAAE,uBAAuB,EAAE,MAAM,sCAAsC,CAAC;AAE/E,MAAM,kBAAkB,GAAG,EAAE,CAAC;AAC9B,MAAM,CAAC,MAAM,mBAAmB,GAAG,CAAC,CAAC;AACrC,MAAM,CAAC,MAAM,eAAe,GAAG,CAAC,CAAC;AAEjC,MAAM,UAAU,kBAAkB,CAChC,KAAqB;IAErB,IAAI,KAAK,CAAC,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC5B,OAAO,uBAAuB,CAC5B,8BAA8B,EAC9B,EAAE,EACF,kBAAkB,CACnB,CAAC;IACJ,CAAC;IAED,IAAI,KAAK,CAAC,IAAI,CAAC,MAAM,GAAG,kBAAkB,EAAE,CAAC;QAC3C,OAAO,uBAAuB,CAC5B,WAAW,kBAAkB,yBAAyB,EACtD,EAAE,EACF,kBAAkB,CACnB,CAAC;IACJ,CAAC;IAED,MAAM,SAAS,GAAG,KAAK,CAAC,IAAI,CAAC,MAAM,CACjC,CAAC,GAAG,EAAE,EAAE,CAAC,OAAO,GAAG,KAAK,QAAQ,IAAI,GAAG,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,CAC1D,CAAC;IAEF,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC3B,OAAO,uBAAuB,CAC5B,wBAAwB,EACxB,EAAE,EACF,kBAAkB,CACnB,CAAC;IACJ,CAAC;IAED,OAAO,SAAS,CAAC;AACnB,CAAC"}
|
package/dist/tools/index.d.ts
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/tools/index.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,yCAAyC,CAAC;
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/tools/index.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,yCAAyC,CAAC;AAsCzE,wBAAgB,aAAa,CAAC,MAAM,EAAE,SAAS,GAAG,IAAI,CAarD"}
|
package/dist/tools/index.js
CHANGED
|
@@ -1,8 +1,6 @@
|
|
|
1
|
-
import { FETCH_LINKS_TOOL_DESCRIPTION, FETCH_LINKS_TOOL_NAME, fetchLinksToolHandler, } from './handlers/fetch-links.tool.js';
|
|
2
1
|
import { FETCH_MARKDOWN_TOOL_DESCRIPTION, FETCH_MARKDOWN_TOOL_NAME, fetchMarkdownToolHandler, } from './handlers/fetch-markdown.tool.js';
|
|
3
2
|
import { FETCH_URL_TOOL_DESCRIPTION, FETCH_URL_TOOL_NAME, fetchUrlToolHandler, } from './handlers/fetch-url.tool.js';
|
|
4
|
-
import {
|
|
5
|
-
import { fetchLinksInputSchema, fetchLinksOutputSchema, fetchMarkdownInputSchema, fetchMarkdownOutputSchema, fetchUrlInputSchema, fetchUrlOutputSchema, fetchUrlsInputSchema, fetchUrlsOutputSchema, } from './schemas.js';
|
|
3
|
+
import { fetchMarkdownInputSchema, fetchMarkdownOutputSchema, fetchUrlInputSchema, fetchUrlOutputSchema, } from './schemas.js';
|
|
6
4
|
const TOOL_DEFINITIONS = [
|
|
7
5
|
{
|
|
8
6
|
name: FETCH_URL_TOOL_NAME,
|
|
@@ -12,14 +10,6 @@ const TOOL_DEFINITIONS = [
|
|
|
12
10
|
outputSchema: fetchUrlOutputSchema,
|
|
13
11
|
handler: fetchUrlToolHandler,
|
|
14
12
|
},
|
|
15
|
-
{
|
|
16
|
-
name: FETCH_LINKS_TOOL_NAME,
|
|
17
|
-
title: 'Fetch Links',
|
|
18
|
-
description: FETCH_LINKS_TOOL_DESCRIPTION,
|
|
19
|
-
inputSchema: fetchLinksInputSchema,
|
|
20
|
-
outputSchema: fetchLinksOutputSchema,
|
|
21
|
-
handler: fetchLinksToolHandler,
|
|
22
|
-
},
|
|
23
13
|
{
|
|
24
14
|
name: FETCH_MARKDOWN_TOOL_NAME,
|
|
25
15
|
title: 'Fetch Markdown',
|
|
@@ -28,14 +18,6 @@ const TOOL_DEFINITIONS = [
|
|
|
28
18
|
outputSchema: fetchMarkdownOutputSchema,
|
|
29
19
|
handler: fetchMarkdownToolHandler,
|
|
30
20
|
},
|
|
31
|
-
{
|
|
32
|
-
name: FETCH_URLS_TOOL_NAME,
|
|
33
|
-
title: 'Fetch URLs (Batch)',
|
|
34
|
-
description: FETCH_URLS_TOOL_DESCRIPTION,
|
|
35
|
-
inputSchema: fetchUrlsInputSchema,
|
|
36
|
-
outputSchema: fetchUrlsOutputSchema,
|
|
37
|
-
handler: fetchUrlsToolHandler,
|
|
38
|
-
},
|
|
39
21
|
];
|
|
40
22
|
export function registerTools(server) {
|
|
41
23
|
for (const tool of TOOL_DEFINITIONS) {
|
|
@@ -47,4 +29,3 @@ export function registerTools(server) {
|
|
|
47
29
|
}, tool.handler);
|
|
48
30
|
}
|
|
49
31
|
}
|
|
50
|
-
//# sourceMappingURL=index.js.map
|
package/dist/tools/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/tools/index.ts"],"names":[],"mappings":"AAEA,OAAO,EACL
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/tools/index.ts"],"names":[],"mappings":"AAEA,OAAO,EACL,+BAA+B,EAC/B,wBAAwB,EACxB,wBAAwB,GACzB,MAAM,mCAAmC,CAAC;AAC3C,OAAO,EACL,0BAA0B,EAC1B,mBAAmB,EACnB,mBAAmB,GACpB,MAAM,8BAA8B,CAAC;AACtC,OAAO,EACL,wBAAwB,EACxB,yBAAyB,EACzB,mBAAmB,EACnB,oBAAoB,GACrB,MAAM,cAAc,CAAC;AAEtB,MAAM,gBAAgB,GAAG;IACvB;QACE,IAAI,EAAE,mBAAmB;QACzB,KAAK,EAAE,WAAW;QAClB,WAAW,EAAE,0BAA0B;QACvC,WAAW,EAAE,mBAAmB;QAChC,YAAY,EAAE,oBAAoB;QAClC,OAAO,EAAE,mBAAmB;KAC7B;IACD;QACE,IAAI,EAAE,wBAAwB;QAC9B,KAAK,EAAE,gBAAgB;QACvB,WAAW,EAAE,+BAA+B;QAC5C,WAAW,EAAE,wBAAwB;QACrC,YAAY,EAAE,yBAAyB;QACvC,OAAO,EAAE,wBAAwB;KAClC;CACO,CAAC;AAEX,MAAM,UAAU,aAAa,CAAC,MAAiB;IAC7C,KAAK,MAAM,IAAI,IAAI,gBAAgB,EAAE,CAAC;QACpC,MAAM,CAAC,YAAY,CACjB,IAAI,CAAC,IAAI,EACT;YACE,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,WAAW,EAAE,IAAI,CAAC,WAAW;YAC7B,WAAW,EAAE,IAAI,CAAC,WAAW;YAC7B,YAAY,EAAE,IAAI,CAAC,YAAY;SAChC,EACD,IAAI,CAAC,OAAO,CACb,CAAC;IACJ,CAAC;AACH,CAAC"}
|