@j0hanz/superfetch 1.0.0 → 1.0.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/errors/app-error.d.ts +0 -25
- package/dist/errors/app-error.d.ts.map +1 -1
- package/dist/errors/app-error.js +0 -34
- package/dist/errors/app-error.js.map +1 -1
- package/dist/index.js +86 -21
- package/dist/index.js.map +1 -1
- package/dist/middleware/error-handler.d.ts.map +1 -1
- package/dist/middleware/error-handler.js.map +1 -1
- package/dist/middleware/rate-limiter.d.ts +1 -0
- package/dist/middleware/rate-limiter.d.ts.map +1 -1
- package/dist/middleware/rate-limiter.js +29 -10
- package/dist/middleware/rate-limiter.js.map +1 -1
- package/dist/services/cache.d.ts +9 -5
- package/dist/services/cache.d.ts.map +1 -1
- package/dist/services/cache.js +67 -28
- package/dist/services/cache.js.map +1 -1
- package/dist/services/extractor.d.ts +8 -18
- package/dist/services/extractor.d.ts.map +1 -1
- package/dist/services/extractor.js +28 -31
- package/dist/services/extractor.js.map +1 -1
- package/dist/services/fetcher.d.ts +6 -2
- package/dist/services/fetcher.d.ts.map +1 -1
- package/dist/services/fetcher.js +86 -17
- package/dist/services/fetcher.js.map +1 -1
- package/dist/services/logger.d.ts +1 -1
- package/dist/services/logger.d.ts.map +1 -1
- package/dist/services/logger.js +14 -4
- package/dist/services/logger.js.map +1 -1
- package/dist/services/parser.d.ts +2 -0
- package/dist/services/parser.d.ts.map +1 -1
- package/dist/services/parser.js +42 -11
- package/dist/services/parser.js.map +1 -1
- package/dist/tools/handlers/fetch-links.tool.d.ts +8 -4
- package/dist/tools/handlers/fetch-links.tool.d.ts.map +1 -1
- package/dist/tools/handlers/fetch-links.tool.js +40 -21
- package/dist/tools/handlers/fetch-links.tool.js.map +1 -1
- package/dist/tools/handlers/fetch-markdown.tool.d.ts +7 -8
- package/dist/tools/handlers/fetch-markdown.tool.d.ts.map +1 -1
- package/dist/tools/handlers/fetch-markdown.tool.js +48 -36
- package/dist/tools/handlers/fetch-markdown.tool.js.map +1 -1
- package/dist/tools/handlers/fetch-url.tool.d.ts +9 -8
- package/dist/tools/handlers/fetch-url.tool.d.ts.map +1 -1
- package/dist/tools/handlers/fetch-url.tool.js +55 -39
- package/dist/tools/handlers/fetch-url.tool.js.map +1 -1
- package/dist/tools/index.d.ts.map +1 -1
- package/dist/tools/index.js +42 -0
- package/dist/tools/index.js.map +1 -1
- package/dist/transformers/jsonl.transformer.d.ts +0 -1
- package/dist/transformers/jsonl.transformer.d.ts.map +1 -1
- package/dist/transformers/jsonl.transformer.js +27 -22
- package/dist/transformers/jsonl.transformer.js.map +1 -1
- package/dist/transformers/markdown.transformer.d.ts +1 -2
- package/dist/transformers/markdown.transformer.d.ts.map +1 -1
- package/dist/transformers/markdown.transformer.js +20 -63
- package/dist/transformers/markdown.transformer.js.map +1 -1
- package/dist/types/content.types.d.ts +1 -1
- package/dist/types/content.types.d.ts.map +1 -1
- package/dist/utils/sanitizer.d.ts +6 -2
- package/dist/utils/sanitizer.d.ts.map +1 -1
- package/dist/utils/sanitizer.js +13 -5
- package/dist/utils/sanitizer.js.map +1 -1
- package/dist/utils/tool-error-handler.d.ts +18 -0
- package/dist/utils/tool-error-handler.d.ts.map +1 -0
- package/dist/utils/tool-error-handler.js +27 -0
- package/dist/utils/tool-error-handler.js.map +1 -0
- package/dist/utils/url-validator.d.ts +1 -0
- package/dist/utils/url-validator.d.ts.map +1 -1
- package/dist/utils/url-validator.js +39 -6
- package/dist/utils/url-validator.js.map +1 -1
- package/package.json +79 -80
|
@@ -2,6 +2,8 @@ import { validateAndNormalizeUrl, isInternalUrl, } from '../../utils/url-validat
|
|
|
2
2
|
import { fetchUrlWithRetry } from '../../services/fetcher.js';
|
|
3
3
|
import * as cache from '../../services/cache.js';
|
|
4
4
|
import * as cheerio from 'cheerio';
|
|
5
|
+
import { logError } from '../../services/logger.js';
|
|
6
|
+
import { createToolErrorResponse, handleToolError, } from '../../utils/tool-error-handler.js';
|
|
5
7
|
export const FETCH_LINKS_TOOL_NAME = 'fetch-links';
|
|
6
8
|
export const FETCH_LINKS_TOOL_DESCRIPTION = 'Extracts all hyperlinks from a webpage with anchor text and type classification';
|
|
7
9
|
/**
|
|
@@ -25,7 +27,9 @@ function extractLinksFromHtml(html, baseUrl, options) {
|
|
|
25
27
|
return;
|
|
26
28
|
}
|
|
27
29
|
seenUrls.add(absoluteUrl);
|
|
28
|
-
const type = isInternalUrl(absoluteUrl, baseUrl)
|
|
30
|
+
const type = isInternalUrl(absoluteUrl, baseUrl)
|
|
31
|
+
? 'internal'
|
|
32
|
+
: 'external';
|
|
29
33
|
// Filter based on options
|
|
30
34
|
if (type === 'internal' && !options.includeInternal)
|
|
31
35
|
return;
|
|
@@ -48,44 +52,59 @@ function extractLinksFromHtml(html, baseUrl, options) {
|
|
|
48
52
|
*/
|
|
49
53
|
export async function fetchLinksToolHandler(input) {
|
|
50
54
|
try {
|
|
55
|
+
// Validate URL input
|
|
56
|
+
if (!input.url) {
|
|
57
|
+
return createToolErrorResponse('URL is required', '', 'VALIDATION_ERROR');
|
|
58
|
+
}
|
|
51
59
|
const url = validateAndNormalizeUrl(input.url);
|
|
52
60
|
const cacheKey = cache.createCacheKey('links', url);
|
|
53
|
-
|
|
54
|
-
if (
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
61
|
+
// Check cache first
|
|
62
|
+
if (cacheKey) {
|
|
63
|
+
const cached = cache.get(cacheKey);
|
|
64
|
+
if (cached) {
|
|
65
|
+
// Parse the cached content to return as structuredContent
|
|
66
|
+
try {
|
|
67
|
+
const structuredContent = JSON.parse(cached.content);
|
|
68
|
+
return {
|
|
69
|
+
content: [{ type: 'text', text: cached.content }],
|
|
70
|
+
structuredContent,
|
|
71
|
+
};
|
|
72
|
+
}
|
|
73
|
+
catch {
|
|
74
|
+
return {
|
|
75
|
+
content: [{ type: 'text', text: cached.content }],
|
|
76
|
+
};
|
|
77
|
+
}
|
|
78
|
+
}
|
|
58
79
|
}
|
|
59
80
|
const html = await fetchUrlWithRetry(url);
|
|
81
|
+
// Validate HTML content was received
|
|
82
|
+
if (!html) {
|
|
83
|
+
return createToolErrorResponse('No content received from URL', url, 'EMPTY_CONTENT');
|
|
84
|
+
}
|
|
60
85
|
// Extract links
|
|
61
86
|
const links = extractLinksFromHtml(html, url, {
|
|
62
87
|
includeInternal: input.includeInternal ?? true,
|
|
63
88
|
includeExternal: input.includeExternal ?? true,
|
|
64
89
|
});
|
|
65
|
-
const
|
|
90
|
+
const structuredContent = {
|
|
66
91
|
url,
|
|
67
92
|
linkCount: links.length,
|
|
68
93
|
links,
|
|
69
94
|
};
|
|
70
|
-
const outputText = JSON.stringify(
|
|
71
|
-
|
|
95
|
+
const outputText = JSON.stringify(structuredContent, null, 2);
|
|
96
|
+
// Cache the result
|
|
97
|
+
if (cacheKey) {
|
|
98
|
+
cache.set(cacheKey, outputText);
|
|
99
|
+
}
|
|
72
100
|
return {
|
|
73
101
|
content: [{ type: 'text', text: outputText }],
|
|
102
|
+
structuredContent,
|
|
74
103
|
};
|
|
75
104
|
}
|
|
76
105
|
catch (error) {
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
{
|
|
80
|
-
type: 'text',
|
|
81
|
-
text: JSON.stringify({
|
|
82
|
-
error: `Failed to extract links: ${error instanceof Error ? error.message : 'Unknown error'}`,
|
|
83
|
-
url: input.url,
|
|
84
|
-
}),
|
|
85
|
-
},
|
|
86
|
-
],
|
|
87
|
-
isError: true,
|
|
88
|
-
};
|
|
106
|
+
logError('fetch-links tool error', error instanceof Error ? error : undefined);
|
|
107
|
+
return handleToolError(error, input.url, 'Failed to extract links');
|
|
89
108
|
}
|
|
90
109
|
}
|
|
91
110
|
//# sourceMappingURL=fetch-links.tool.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"fetch-links.tool.js","sourceRoot":"","sources":["../../../src/tools/handlers/fetch-links.tool.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,uBAAuB,EACvB,aAAa,GACd,MAAM,8BAA8B,CAAC;AACtC,OAAO,EAAE,iBAAiB,EAAE,MAAM,2BAA2B,CAAC;AAC9D,OAAO,KAAK,KAAK,MAAM,yBAAyB,CAAC;AACjD,OAAO,KAAK,OAAO,MAAM,SAAS,CAAC;
|
|
1
|
+
{"version":3,"file":"fetch-links.tool.js","sourceRoot":"","sources":["../../../src/tools/handlers/fetch-links.tool.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,uBAAuB,EACvB,aAAa,GACd,MAAM,8BAA8B,CAAC;AACtC,OAAO,EAAE,iBAAiB,EAAE,MAAM,2BAA2B,CAAC;AAC9D,OAAO,KAAK,KAAK,MAAM,yBAAyB,CAAC;AACjD,OAAO,KAAK,OAAO,MAAM,SAAS,CAAC;AACnC,OAAO,EAAE,QAAQ,EAAE,MAAM,0BAA0B,CAAC;AACpD,OAAO,EACL,uBAAuB,EACvB,eAAe,GAChB,MAAM,mCAAmC,CAAC;AAG3C,MAAM,CAAC,MAAM,qBAAqB,GAAG,aAAa,CAAC;AACnD,MAAM,CAAC,MAAM,4BAA4B,GACvC,iFAAiF,CAAC;AAEpF;;GAEG;AACH,SAAS,oBAAoB,CAC3B,IAAY,EACZ,OAAe,EACf,OAA+D;IAE/D,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC7B,MAAM,KAAK,GAAoB,EAAE,CAAC;IAClC,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAU,CAAC;IAEnC,CAAC,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,EAAE;QAC/B,MAAM,IAAI,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACrC,MAAM,IAAI,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;QAEtC,qBAAqB;QACrB,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,aAAa,CAAC,EAAE,CAAC;YACpE,OAAO;QACT,CAAC;QAED,IAAI,CAAC;YACH,MAAM,WAAW,GAAG,IAAI,GAAG,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC,IAAI,CAAC;YAEhD,kBAAkB;YAClB,IAAI,QAAQ,CAAC,GAAG,CAAC,WAAW,CAAC,EAAE,CAAC;gBAC9B,OAAO;YACT,CAAC;YACD,QAAQ,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC;YAE1B,MAAM,IAAI,GAAG,aAAa,CAAC,WAAW,EAAE,OAAO,CAAC;gBAC9C,CAAC,CAAC,UAAU;gBACZ,CAAC,CAAC,UAAU,CAAC;YAEf,0BAA0B;YAC1B,IAAI,IAAI,KAAK,UAAU,IAAI,CAAC,OAAO,CAAC,eAAe;gBAAE,OAAO;YAC5D,IAAI,IAAI,KAAK,UAAU,IAAI,CAAC,OAAO,CAAC,eAAe;gBAAE,OAAO;YAE5D,KAAK,CAAC,IAAI,CAAC;gBACT,IAAI,EAAE,WAAW;gBACjB,IAAI,EAAE,IAAI,IAAI,WAAW;gBACzB,IAAI;aACL,CAAC,CAAC;QACL,CAAC;QAAC,MAAM,CAAC;YACP,6BAA6B;QAC/B,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,qBAAqB,CAAC,KAAsB;IAChE,IAAI,CAAC;QACH,qBAAqB;QACrB,IAAI,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC;YACf,OAAO,uBAAuB,CAAC,iBAAiB,EAAE,EAAE,EAAE,kBAAkB,CAAC,CAAC;QAC5E,CAAC;QAED,MAAM,GAAG,GAAG,uBAAuB,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QAC/C,MAAM,QAAQ,GAAG,KAAK,CAAC,cAAc,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;QAEpD,oBAAoB;QACpB,IAAI,QAAQ,EAAE,CAAC;YACb,MAAM,MAAM,GAAG,KAAK,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;YACnC,IAAI,MAAM,EAAE,CAAC;gBACX,0DAA0D;gBAC1D,IAAI,CAAC;oBACH,MAAM,iBAAiB,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,OAAO,CAIlD,CAAC;oBACF,OAAO;wBACL,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAe,EAAE,IAAI,EAAE,MAAM,CAAC,OAAO,EAAE,CAAC;wBAC1D,iBAAiB;qBAClB,CAAC;gBACJ,CAAC;gBAAC,MAAM,CAAC;oBACP,OAAO;wBACL,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAe,EAAE,IAAI,EAAE,MAAM,CAAC,OAAO,EAAE,CAAC;qBAC3D,CAAC;gBACJ,CAAC;YACH,CAAC;QACH,CAAC;QAED,MAAM,IAAI,GAAG,MAAM,iBAAiB,CAAC,GAAG,CAAC,CAAC;QAE1C,qCAAqC;QACrC,IAAI,CAAC,IAAI,EAAE,CAAC;YACV,OAAO,uBAAuB,CAC5B,8BAA8B,EAC9B,GAAG,EACH,eAAe,CAChB,CAAC;QACJ,CAAC;QAED,gBAAgB;QAChB,MAAM,KAAK,GAAG,oBAAoB,CAAC,IAAI,EAAE,GAAG,EAAE;YAC5C,eAAe,EAAE,KAAK,CAAC,eAAe,IAAI,IAAI;YAC9C,eAAe,EAAE,KAAK,CAAC,eAAe,IAAI,IAAI;SAC/C,CAAC,CAAC;QAEH,MAAM,iBAAiB,GAAG;YACxB,GAAG;YACH,SAAS,EAAE,KAAK,CAAC,MAAM;YACvB,KAAK;SACN,CAAC;QAEF,MAAM,UAAU,GAAG,IAAI,CAAC,SAAS,CAAC,iBAAiB,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;QAE9D,mBAAmB;QACnB,IAAI,QAAQ,EAAE,CAAC;YACb,KAAK,CAAC,GAAG,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC;QAClC,CAAC;QAED,OAAO;YACL,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAe,EAAE,IAAI,EAAE,UAAU,EAAE,CAAC;YACtD,iBAAiB;SAClB,CAAC;IACJ,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,QAAQ,CACN,wBAAwB,EACxB,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS,CAC3C,CAAC;QACF,OAAO,eAAe,CAAC,KAAK,EAAE,KAAK,CAAC,GAAG,EAAE,yBAAyB,CAAC,CAAC;IACtE,CAAC;AACH,CAAC"}
|
|
@@ -1,17 +1,16 @@
|
|
|
1
1
|
import type { FetchMarkdownInput } from '../../types/index.js';
|
|
2
2
|
export declare const FETCH_MARKDOWN_TOOL_NAME = "fetch-markdown";
|
|
3
3
|
export declare const FETCH_MARKDOWN_TOOL_DESCRIPTION = "Fetches a webpage and converts it to clean Markdown format with optional frontmatter";
|
|
4
|
-
export declare function fetchMarkdownToolHandler(input: FetchMarkdownInput): Promise<{
|
|
4
|
+
export declare function fetchMarkdownToolHandler(input: FetchMarkdownInput): Promise<import("../../utils/tool-error-handler.js").ToolErrorResponse | {
|
|
5
5
|
content: {
|
|
6
6
|
type: "text";
|
|
7
7
|
text: string;
|
|
8
8
|
}[];
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
}
|
|
15
|
-
isError: boolean;
|
|
9
|
+
structuredContent: {
|
|
10
|
+
url: string;
|
|
11
|
+
cached: boolean;
|
|
12
|
+
fetchedAt: string;
|
|
13
|
+
markdown: string;
|
|
14
|
+
};
|
|
16
15
|
}>;
|
|
17
16
|
//# sourceMappingURL=fetch-markdown.tool.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"fetch-markdown.tool.d.ts","sourceRoot":"","sources":["../../../src/tools/handlers/fetch-markdown.tool.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"fetch-markdown.tool.d.ts","sourceRoot":"","sources":["../../../src/tools/handlers/fetch-markdown.tool.ts"],"names":[],"mappings":"AAWA,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,sBAAsB,CAAC;AAE/D,eAAO,MAAM,wBAAwB,mBAAmB,CAAC;AACzD,eAAO,MAAM,+BAA+B,yFAC4C,CAAC;AAmDzF,wBAAsB,wBAAwB,CAAC,KAAK,EAAE,kBAAkB;;;;;;;;;;;GA6EvE"}
|
|
@@ -4,12 +4,16 @@ import { extractContent } from '../../services/extractor.js';
|
|
|
4
4
|
import { htmlToMarkdown } from '../../transformers/markdown.transformer.js';
|
|
5
5
|
import * as cache from '../../services/cache.js';
|
|
6
6
|
import { config } from '../../config/index.js';
|
|
7
|
+
import { logError } from '../../services/logger.js';
|
|
8
|
+
import { createToolErrorResponse, handleToolError, } from '../../utils/tool-error-handler.js';
|
|
7
9
|
export const FETCH_MARKDOWN_TOOL_NAME = 'fetch-markdown';
|
|
8
10
|
export const FETCH_MARKDOWN_TOOL_DESCRIPTION = 'Fetches a webpage and converts it to clean Markdown format with optional frontmatter';
|
|
9
11
|
function extractAndConvertToMarkdown(html, url, options) {
|
|
10
12
|
// Use the optimized extractContent that parses JSDOM only once
|
|
11
13
|
const { article, metadata: extractedMeta } = extractContent(html, url);
|
|
12
|
-
if (options.extractMainContent &&
|
|
14
|
+
if (options.extractMainContent &&
|
|
15
|
+
config.extraction.extractMainContent &&
|
|
16
|
+
article) {
|
|
13
17
|
const metadata = options.includeMetadata && config.extraction.includeMetadata
|
|
14
18
|
? {
|
|
15
19
|
type: 'metadata',
|
|
@@ -42,58 +46,66 @@ function extractAndConvertToMarkdown(html, url, options) {
|
|
|
42
46
|
}
|
|
43
47
|
export async function fetchMarkdownToolHandler(input) {
|
|
44
48
|
try {
|
|
49
|
+
// Validate URL input
|
|
50
|
+
if (!input.url) {
|
|
51
|
+
return createToolErrorResponse('URL is required', '', 'VALIDATION_ERROR');
|
|
52
|
+
}
|
|
45
53
|
const url = validateAndNormalizeUrl(input.url);
|
|
46
54
|
const cacheKey = cache.createCacheKey('markdown', url);
|
|
47
|
-
|
|
48
|
-
if (
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
55
|
+
// Check cache first
|
|
56
|
+
if (cacheKey) {
|
|
57
|
+
const cached = cache.get(cacheKey);
|
|
58
|
+
if (cached) {
|
|
59
|
+
const structuredContent = {
|
|
60
|
+
url,
|
|
61
|
+
cached: true,
|
|
62
|
+
fetchedAt: cached.fetchedAt,
|
|
63
|
+
markdown: cached.content,
|
|
64
|
+
};
|
|
65
|
+
return {
|
|
66
|
+
content: [
|
|
67
|
+
{
|
|
68
|
+
type: 'text',
|
|
69
|
+
text: JSON.stringify(structuredContent),
|
|
70
|
+
},
|
|
71
|
+
],
|
|
72
|
+
structuredContent,
|
|
73
|
+
};
|
|
74
|
+
}
|
|
62
75
|
}
|
|
63
76
|
const html = await fetchUrlWithRetry(url);
|
|
77
|
+
// Validate HTML content was received
|
|
78
|
+
if (!html) {
|
|
79
|
+
return createToolErrorResponse('No content received from URL', url, 'EMPTY_CONTENT');
|
|
80
|
+
}
|
|
64
81
|
const { markdown, title } = extractAndConvertToMarkdown(html, url, {
|
|
65
82
|
extractMainContent: input.extractMainContent ?? true,
|
|
66
83
|
includeMetadata: input.includeMetadata ?? true,
|
|
67
84
|
});
|
|
68
|
-
|
|
85
|
+
// Cache the result
|
|
86
|
+
if (cacheKey) {
|
|
87
|
+
cache.set(cacheKey, markdown);
|
|
88
|
+
}
|
|
89
|
+
const structuredContent = {
|
|
90
|
+
url,
|
|
91
|
+
title,
|
|
92
|
+
fetchedAt: new Date().toISOString(),
|
|
93
|
+
markdown,
|
|
94
|
+
cached: false,
|
|
95
|
+
};
|
|
69
96
|
return {
|
|
70
97
|
content: [
|
|
71
98
|
{
|
|
72
99
|
type: 'text',
|
|
73
|
-
text: JSON.stringify(
|
|
74
|
-
url,
|
|
75
|
-
title,
|
|
76
|
-
fetchedAt: new Date().toISOString(),
|
|
77
|
-
markdown,
|
|
78
|
-
cached: false,
|
|
79
|
-
}, null, 2),
|
|
100
|
+
text: JSON.stringify(structuredContent, null, 2),
|
|
80
101
|
},
|
|
81
102
|
],
|
|
103
|
+
structuredContent,
|
|
82
104
|
};
|
|
83
105
|
}
|
|
84
106
|
catch (error) {
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
{
|
|
88
|
-
type: 'text',
|
|
89
|
-
text: JSON.stringify({
|
|
90
|
-
error: `Failed to fetch markdown: ${error instanceof Error ? error.message : 'Unknown error'}`,
|
|
91
|
-
url: input.url,
|
|
92
|
-
}),
|
|
93
|
-
},
|
|
94
|
-
],
|
|
95
|
-
isError: true,
|
|
96
|
-
};
|
|
107
|
+
logError('fetch-markdown tool error', error instanceof Error ? error : undefined);
|
|
108
|
+
return handleToolError(error, input.url, 'Failed to fetch markdown');
|
|
97
109
|
}
|
|
98
110
|
}
|
|
99
111
|
//# sourceMappingURL=fetch-markdown.tool.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"fetch-markdown.tool.js","sourceRoot":"","sources":["../../../src/tools/handlers/fetch-markdown.tool.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,uBAAuB,EAAE,MAAM,8BAA8B,CAAC;AACvE,OAAO,EAAE,iBAAiB,EAAE,MAAM,2BAA2B,CAAC;AAC9D,OAAO,EAAE,cAAc,EAAE,MAAM,6BAA6B,CAAC;AAC7D,OAAO,EAAE,cAAc,EAAE,MAAM,4CAA4C,CAAC;AAC5E,OAAO,KAAK,KAAK,MAAM,yBAAyB,CAAC;AACjD,OAAO,EAAE,MAAM,EAAE,MAAM,uBAAuB,CAAC;
|
|
1
|
+
{"version":3,"file":"fetch-markdown.tool.js","sourceRoot":"","sources":["../../../src/tools/handlers/fetch-markdown.tool.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,uBAAuB,EAAE,MAAM,8BAA8B,CAAC;AACvE,OAAO,EAAE,iBAAiB,EAAE,MAAM,2BAA2B,CAAC;AAC9D,OAAO,EAAE,cAAc,EAAE,MAAM,6BAA6B,CAAC;AAC7D,OAAO,EAAE,cAAc,EAAE,MAAM,4CAA4C,CAAC;AAC5E,OAAO,KAAK,KAAK,MAAM,yBAAyB,CAAC;AACjD,OAAO,EAAE,MAAM,EAAE,MAAM,uBAAuB,CAAC;AAC/C,OAAO,EAAE,QAAQ,EAAE,MAAM,0BAA0B,CAAC;AACpD,OAAO,EACL,uBAAuB,EACvB,eAAe,GAChB,MAAM,mCAAmC,CAAC;AAG3C,MAAM,CAAC,MAAM,wBAAwB,GAAG,gBAAgB,CAAC;AACzD,MAAM,CAAC,MAAM,+BAA+B,GAC1C,sFAAsF,CAAC;AAEzF,SAAS,2BAA2B,CAClC,IAAY,EACZ,GAAW,EACX,OAAkE;IAElE,+DAA+D;IAC/D,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,aAAa,EAAE,GAAG,cAAc,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;IAEvE,IACE,OAAO,CAAC,kBAAkB;QAC1B,MAAM,CAAC,UAAU,CAAC,kBAAkB;QACpC,OAAO,EACP,CAAC;QACD,MAAM,QAAQ,GACZ,OAAO,CAAC,eAAe,IAAI,MAAM,CAAC,UAAU,CAAC,eAAe;YAC1D,CAAC,CAAC;gBACE,IAAI,EAAE,UAAmB;gBACzB,KAAK,EAAE,OAAO,CAAC,KAAK;gBACpB,MAAM,EAAE,OAAO,CAAC,MAAM;gBACtB,GAAG;gBACH,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;aACpC;YACH,CAAC,CAAC,SAAS,CAAC;QAEhB,OAAO;YACL,QAAQ,EAAE,cAAc,CAAC,OAAO,CAAC,OAAO,EAAE,QAAQ,CAAC;YACnD,KAAK,EAAE,OAAO,CAAC,KAAK;SACrB,CAAC;IACJ,CAAC;IAED,8BAA8B;IAC9B,MAAM,QAAQ,GACZ,OAAO,CAAC,eAAe,IAAI,MAAM,CAAC,UAAU,CAAC,eAAe;QAC1D,CAAC,CAAC;YACE,IAAI,EAAE,UAAmB;YACzB,KAAK,EAAE,aAAa,CAAC,KAAK;YAC1B,WAAW,EAAE,aAAa,CAAC,WAAW;YACtC,MAAM,EAAE,aAAa,CAAC,MAAM;YAC5B,GAAG;YACH,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;SACpC;QACH,CAAC,CAAC,SAAS,CAAC;IAEhB,OAAO;QACL,QAAQ,EAAE,cAAc,CAAC,IAAI,EAAE,QAAQ,CAAC;QACxC,KAAK,EAAE,aAAa,CAAC,KAAK;KAC3B,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,wBAAwB,CAAC,KAAyB;IACtE,IAAI,CAAC;QACH,qBAAqB;QACrB,IAAI,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC;YACf,OAAO,uBAAuB,CAAC,iBAAiB,EAAE,EAAE,EAAE,kBAAkB,CAAC,CAAC;QAC5E,CAAC;QAED,MAAM,GAAG,GAAG,uBAAuB,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QAC/C,MAAM,QAAQ,GAAG,KAAK,CAAC,cAAc,CAAC,UAAU,EAAE,GAAG,CAAC,CAAC;QAEvD,oBAAoB;QACpB,IAAI,QAAQ,EAAE,CAAC;YACb,MAAM,MAAM,GAAG,KAAK,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;YACnC,IAAI,MAAM,EAAE,CAAC;gBACX,MAAM,iBAAiB,GAAG;oBACxB,GAAG;oBACH,MAAM,EAAE,IAAI;oBACZ,SAAS,EAAE,MAAM,CAAC,SAAS;oBAC3B,QAAQ,EAAE,MAAM,CAAC,OAAO;iBACzB,CAAC;gBACF,OAAO;oBACL,OAAO,EAAE;wBACP;4BACE,IAAI,EAAE,MAAe;4BACrB,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,iBAAiB,CAAC;yBACxC;qBACF;oBACD,iBAAiB;iBAClB,CAAC;YACJ,CAAC;QACH,CAAC;QAED,MAAM,IAAI,GAAG,MAAM,iBAAiB,CAAC,GAAG,CAAC,CAAC;QAE1C,qCAAqC;QACrC,IAAI,CAAC,IAAI,EAAE,CAAC;YACV,OAAO,uBAAuB,CAC5B,8BAA8B,EAC9B,GAAG,EACH,eAAe,CAChB,CAAC;QACJ,CAAC;QAED,MAAM,EAAE,QAAQ,EAAE,KAAK,EAAE,GAAG,2BAA2B,CAAC,IAAI,EAAE,GAAG,EAAE;YACjE,kBAAkB,EAAE,KAAK,CAAC,kBAAkB,IAAI,IAAI;YACpD,eAAe,EAAE,KAAK,CAAC,eAAe,IAAI,IAAI;SAC/C,CAAC,CAAC;QAEH,mBAAmB;QACnB,IAAI,QAAQ,EAAE,CAAC;YACb,KAAK,CAAC,GAAG,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;QAChC,CAAC;QAED,MAAM,iBAAiB,GAAG;YACxB,GAAG;YACH,KAAK;YACL,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;YACnC,QAAQ;YACR,MAAM,EAAE,KAAK;SACd,CAAC;QAEF,OAAO;YACL,OAAO,EAAE;gBACP;oBACE,IAAI,EAAE,MAAe;oBACrB,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,iBAAiB,EAAE,IAAI,EAAE,CAAC,CAAC;iBACjD;aACF;YACD,iBAAiB;SAClB,CAAC;IACJ,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,QAAQ,CACN,2BAA2B,EAC3B,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS,CAC3C,CAAC;QACF,OAAO,eAAe,CAAC,KAAK,EAAE,KAAK,CAAC,GAAG,EAAE,0BAA0B,CAAC,CAAC;IACvE,CAAC;AACH,CAAC"}
|
|
@@ -1,17 +1,18 @@
|
|
|
1
1
|
import type { FetchUrlInput } from '../../types/index.js';
|
|
2
2
|
export declare const FETCH_URL_TOOL_NAME = "fetch-url";
|
|
3
3
|
export declare const FETCH_URL_TOOL_DESCRIPTION = "Fetches a webpage and converts it to AI-readable JSONL format with semantic content blocks";
|
|
4
|
-
export declare function fetchUrlToolHandler(input: FetchUrlInput): Promise<{
|
|
4
|
+
export declare function fetchUrlToolHandler(input: FetchUrlInput): Promise<import("../../utils/tool-error-handler.js").ToolErrorResponse | {
|
|
5
5
|
content: {
|
|
6
6
|
type: "text";
|
|
7
7
|
text: string;
|
|
8
8
|
}[];
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
9
|
+
structuredContent: {
|
|
10
|
+
url: string;
|
|
11
|
+
cached: boolean;
|
|
12
|
+
fetchedAt: string;
|
|
13
|
+
content: string;
|
|
14
|
+
format: "jsonl";
|
|
15
|
+
contentBlocks: number;
|
|
16
|
+
};
|
|
16
17
|
}>;
|
|
17
18
|
//# sourceMappingURL=fetch-url.tool.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"fetch-url.tool.d.ts","sourceRoot":"","sources":["../../../src/tools/handlers/fetch-url.tool.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"fetch-url.tool.d.ts","sourceRoot":"","sources":["../../../src/tools/handlers/fetch-url.tool.ts"],"names":[],"mappings":"AAYA,OAAO,KAAK,EACV,aAAa,EAGd,MAAM,sBAAsB,CAAC;AAE9B,eAAO,MAAM,mBAAmB,cAAc,CAAC;AAC/C,eAAO,MAAM,0BAA0B,+FACuD,CAAC;AAsD/F,wBAAsB,mBAAmB,CAAC,KAAK,EAAE,aAAa;;;;;;;;;;;;;GAgG7D"}
|
|
@@ -5,12 +5,16 @@ import { parseHtml } from '../../services/parser.js';
|
|
|
5
5
|
import { toJsonl } from '../../transformers/jsonl.transformer.js';
|
|
6
6
|
import * as cache from '../../services/cache.js';
|
|
7
7
|
import { config } from '../../config/index.js';
|
|
8
|
+
import { logError } from '../../services/logger.js';
|
|
9
|
+
import { createToolErrorResponse, handleToolError, } from '../../utils/tool-error-handler.js';
|
|
8
10
|
export const FETCH_URL_TOOL_NAME = 'fetch-url';
|
|
9
11
|
export const FETCH_URL_TOOL_DESCRIPTION = 'Fetches a webpage and converts it to AI-readable JSONL format with semantic content blocks';
|
|
10
12
|
function extractContentFromHtml(html, url, options) {
|
|
11
13
|
// Use the optimized extractContent that parses JSDOM only once
|
|
12
14
|
const { article, metadata: extractedMeta } = extractContent(html, url);
|
|
13
|
-
if (options.extractMainContent &&
|
|
15
|
+
if (options.extractMainContent &&
|
|
16
|
+
config.extraction.extractMainContent &&
|
|
17
|
+
article) {
|
|
14
18
|
const contentBlocks = parseHtml(article.content);
|
|
15
19
|
const metadata = options.includeMetadata && config.extraction.includeMetadata
|
|
16
20
|
? {
|
|
@@ -39,65 +43,77 @@ function extractContentFromHtml(html, url, options) {
|
|
|
39
43
|
}
|
|
40
44
|
export async function fetchUrlToolHandler(input) {
|
|
41
45
|
try {
|
|
46
|
+
// Validate URL input
|
|
47
|
+
if (!input.url) {
|
|
48
|
+
return createToolErrorResponse('URL is required', '', 'VALIDATION_ERROR');
|
|
49
|
+
}
|
|
42
50
|
const url = validateAndNormalizeUrl(input.url);
|
|
43
51
|
const cacheKey = cache.createCacheKey('url', url);
|
|
44
|
-
|
|
45
|
-
if (
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
52
|
+
// Check cache first
|
|
53
|
+
if (cacheKey) {
|
|
54
|
+
const cached = cache.get(cacheKey);
|
|
55
|
+
if (cached) {
|
|
56
|
+
const structuredContent = {
|
|
57
|
+
url,
|
|
58
|
+
cached: true,
|
|
59
|
+
fetchedAt: cached.fetchedAt,
|
|
60
|
+
content: cached.content,
|
|
61
|
+
format: 'jsonl',
|
|
62
|
+
contentBlocks: 0, // Unknown from cache
|
|
63
|
+
};
|
|
64
|
+
return {
|
|
65
|
+
content: [
|
|
66
|
+
{
|
|
67
|
+
type: 'text',
|
|
68
|
+
text: JSON.stringify(structuredContent),
|
|
69
|
+
},
|
|
70
|
+
],
|
|
71
|
+
structuredContent,
|
|
72
|
+
};
|
|
73
|
+
}
|
|
59
74
|
}
|
|
60
75
|
const html = await fetchUrlWithRetry(url, input.customHeaders);
|
|
76
|
+
// Validate HTML content was received
|
|
77
|
+
if (!html) {
|
|
78
|
+
return createToolErrorResponse('No content received from URL', url, 'EMPTY_CONTENT');
|
|
79
|
+
}
|
|
61
80
|
const { contentBlocks, metadata, title } = extractContentFromHtml(html, url, {
|
|
62
81
|
extractMainContent: input.extractMainContent ?? true,
|
|
63
82
|
includeMetadata: input.includeMetadata ?? true,
|
|
64
83
|
});
|
|
65
84
|
let jsonlContent = toJsonl(contentBlocks, metadata);
|
|
66
|
-
if (input.maxContentLength &&
|
|
85
|
+
if (input.maxContentLength &&
|
|
86
|
+
input.maxContentLength > 0 &&
|
|
87
|
+
jsonlContent.length > input.maxContentLength) {
|
|
67
88
|
jsonlContent =
|
|
68
89
|
jsonlContent.substring(0, input.maxContentLength) + '\n...[truncated]';
|
|
69
90
|
}
|
|
70
|
-
|
|
91
|
+
// Cache the result
|
|
92
|
+
if (cacheKey) {
|
|
93
|
+
cache.set(cacheKey, jsonlContent);
|
|
94
|
+
}
|
|
95
|
+
const structuredContent = {
|
|
96
|
+
url,
|
|
97
|
+
title,
|
|
98
|
+
contentBlocks: contentBlocks.length,
|
|
99
|
+
fetchedAt: new Date().toISOString(),
|
|
100
|
+
format: 'jsonl',
|
|
101
|
+
content: jsonlContent,
|
|
102
|
+
cached: false,
|
|
103
|
+
};
|
|
71
104
|
return {
|
|
72
105
|
content: [
|
|
73
106
|
{
|
|
74
107
|
type: 'text',
|
|
75
|
-
text: JSON.stringify(
|
|
76
|
-
url,
|
|
77
|
-
title,
|
|
78
|
-
contentBlocks: contentBlocks.length,
|
|
79
|
-
fetchedAt: new Date().toISOString(),
|
|
80
|
-
format: 'jsonl',
|
|
81
|
-
content: jsonlContent,
|
|
82
|
-
cached: false,
|
|
83
|
-
}, null, 2),
|
|
108
|
+
text: JSON.stringify(structuredContent, null, 2),
|
|
84
109
|
},
|
|
85
110
|
],
|
|
111
|
+
structuredContent,
|
|
86
112
|
};
|
|
87
113
|
}
|
|
88
114
|
catch (error) {
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
{
|
|
92
|
-
type: 'text',
|
|
93
|
-
text: JSON.stringify({
|
|
94
|
-
error: `Failed to fetch URL: ${error instanceof Error ? error.message : 'Unknown error'}`,
|
|
95
|
-
url: input.url,
|
|
96
|
-
}),
|
|
97
|
-
},
|
|
98
|
-
],
|
|
99
|
-
isError: true,
|
|
100
|
-
};
|
|
115
|
+
logError('fetch-url tool error', error instanceof Error ? error : undefined);
|
|
116
|
+
return handleToolError(error, input.url, 'Failed to fetch URL');
|
|
101
117
|
}
|
|
102
118
|
}
|
|
103
119
|
//# sourceMappingURL=fetch-url.tool.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"fetch-url.tool.js","sourceRoot":"","sources":["../../../src/tools/handlers/fetch-url.tool.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,uBAAuB,EAAE,MAAM,8BAA8B,CAAC;AACvE,OAAO,EAAE,iBAAiB,EAAE,MAAM,2BAA2B,CAAC;AAC9D,OAAO,EAAE,cAAc,EAAE,MAAM,6BAA6B,CAAC;AAC7D,OAAO,EAAE,SAAS,EAAE,MAAM,0BAA0B,CAAC;AACrD,OAAO,EAAE,OAAO,EAAE,MAAM,yCAAyC,CAAC;AAClE,OAAO,KAAK,KAAK,MAAM,yBAAyB,CAAC;AACjD,OAAO,EAAE,MAAM,EAAE,MAAM,uBAAuB,CAAC;
|
|
1
|
+
{"version":3,"file":"fetch-url.tool.js","sourceRoot":"","sources":["../../../src/tools/handlers/fetch-url.tool.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,uBAAuB,EAAE,MAAM,8BAA8B,CAAC;AACvE,OAAO,EAAE,iBAAiB,EAAE,MAAM,2BAA2B,CAAC;AAC9D,OAAO,EAAE,cAAc,EAAE,MAAM,6BAA6B,CAAC;AAC7D,OAAO,EAAE,SAAS,EAAE,MAAM,0BAA0B,CAAC;AACrD,OAAO,EAAE,OAAO,EAAE,MAAM,yCAAyC,CAAC;AAClE,OAAO,KAAK,KAAK,MAAM,yBAAyB,CAAC;AACjD,OAAO,EAAE,MAAM,EAAE,MAAM,uBAAuB,CAAC;AAC/C,OAAO,EAAE,QAAQ,EAAE,MAAM,0BAA0B,CAAC;AACpD,OAAO,EACL,uBAAuB,EACvB,eAAe,GAChB,MAAM,mCAAmC,CAAC;AAO3C,MAAM,CAAC,MAAM,mBAAmB,GAAG,WAAW,CAAC;AAC/C,MAAM,CAAC,MAAM,0BAA0B,GACrC,4FAA4F,CAAC;AAQ/F,SAAS,sBAAsB,CAC7B,IAAY,EACZ,GAAW,EACX,OAAkE;IAElE,+DAA+D;IAC/D,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,aAAa,EAAE,GAAG,cAAc,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;IAEvE,IACE,OAAO,CAAC,kBAAkB;QAC1B,MAAM,CAAC,UAAU,CAAC,kBAAkB;QACpC,OAAO,EACP,CAAC;QACD,MAAM,aAAa,GAAG,SAAS,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;QACjD,MAAM,QAAQ,GACZ,OAAO,CAAC,eAAe,IAAI,MAAM,CAAC,UAAU,CAAC,eAAe;YAC1D,CAAC,CAAC;gBACE,IAAI,EAAE,UAAmB;gBACzB,KAAK,EAAE,OAAO,CAAC,KAAK;gBACpB,MAAM,EAAE,OAAO,CAAC,MAAM;gBACtB,GAAG;gBACH,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;aACpC;YACH,CAAC,CAAC,SAAS,CAAC;QAEhB,OAAO,EAAE,aAAa,EAAE,QAAQ,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,CAAC;IAC3D,CAAC;IAED,qCAAqC;IACrC,MAAM,aAAa,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC;IAEtC,MAAM,QAAQ,GACZ,OAAO,CAAC,eAAe,IAAI,MAAM,CAAC,UAAU,CAAC,eAAe;QAC1D,CAAC,CAAC;YACE,IAAI,EAAE,UAAmB;YACzB,KAAK,EAAE,aAAa,CAAC,KAAK;YAC1B,WAAW,EAAE,aAAa,CAAC,WAAW;YACtC,MAAM,EAAE,aAAa,CAAC,MAAM;YAC5B,GAAG;YACH,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;SACpC;QACH,CAAC,CAAC,SAAS,CAAC;IAEhB,OAAO,EAAE,aAAa,EAAE,QAAQ,EAAE,KAAK,EAAE,aAAa,CAAC,KAAK,EAAE,CAAC;AACjE,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,mBAAmB,CAAC,KAAoB;IAC5D,IAAI,CAAC;QACH,qBAAqB;QACrB,IAAI,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC;YACf,OAAO,uBAAuB,CAAC,iBAAiB,EAAE,EAAE,EAAE,kBAAkB,CAAC,CAAC;QAC5E,CAAC;QAED,MAAM,GAAG,GAAG,uBAAuB,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QAC/C,MAAM,QAAQ,GAAG,KAAK,CAAC,cAAc,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;QAElD,oBAAoB;QACpB,IAAI,QAAQ,EAAE,CAAC;YACb,MAAM,MAAM,GAAG,KAAK,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;YACnC,IAAI,MAAM,EAAE,CAAC;gBACX,MAAM,iBAAiB,GAAG;oBACxB,GAAG;oBACH,MAAM,EAAE,IAAI;oBACZ,SAAS,EAAE,MAAM,CAAC,SAAS;oBAC3B,OAAO,EAAE,MAAM,CAAC,OAAO;oBACvB,MAAM,EAAE,OAAgB;oBACxB,aAAa,EAAE,CAAC,EAAE,qBAAqB;iBACxC,CAAC;gBACF,OAAO;oBACL,OAAO,EAAE;wBACP;4BACE,IAAI,EAAE,MAAe;4BACrB,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,iBAAiB,CAAC;yBACxC;qBACF;oBACD,iBAAiB;iBAClB,CAAC;YACJ,CAAC;QACH,CAAC;QAED,MAAM,IAAI,GAAG,MAAM,iBAAiB,CAAC,GAAG,EAAE,KAAK,CAAC,aAAa,CAAC,CAAC;QAE/D,qCAAqC;QACrC,IAAI,CAAC,IAAI,EAAE,CAAC;YACV,OAAO,uBAAuB,CAC5B,8BAA8B,EAC9B,GAAG,EACH,eAAe,CAChB,CAAC;QACJ,CAAC;QAED,MAAM,EAAE,aAAa,EAAE,QAAQ,EAAE,KAAK,EAAE,GAAG,sBAAsB,CAC/D,IAAI,EACJ,GAAG,EACH;YACE,kBAAkB,EAAE,KAAK,CAAC,kBAAkB,IAAI,IAAI;YACpD,eAAe,EAAE,KAAK,CAAC,eAAe,IAAI,IAAI;SAC/C,CACF,CAAC;QAEF,IAAI,YAAY,GAAG,OAAO,CAAC,aAAa,EAAE,QAAQ,CAAC,CAAC;QAEpD,IACE,KAAK,CAAC,gBAAgB;YACtB,KAAK,CAAC,gBAAgB,GAAG,CAAC;YAC1B,YAAY,CAAC,MAAM,GAAG,KAAK,CAAC,gBAAgB,EAC5C,CAAC;YACD,YAAY;gBACV,YAAY,CAAC,SAAS,CAAC,CAAC,EAAE,KAAK,CAAC,gBAAgB,CAAC,GAAG,kBAAkB,CAAC;QAC3E,CAAC;QAED,mBAAmB;QACnB,IAAI,QAAQ,EAAE,CAAC;YACb,KAAK,CAAC,GAAG,CAAC,QAAQ,EAAE,YAAY,CAAC,CAAC;QACpC,CAAC;QAED,MAAM,iBAAiB,GAAG;YACxB,GAAG;YACH,KAAK;YACL,aAAa,EAAE,aAAa,CAAC,MAAM;YACnC,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;YACnC,MAAM,EAAE,OAAgB;YACxB,OAAO,EAAE,YAAY;YACrB,MAAM,EAAE,KAAK;SACd,CAAC;QAEF,OAAO;YACL,OAAO,EAAE;gBACP;oBACE,IAAI,EAAE,MAAe;oBACrB,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,iBAAiB,EAAE,IAAI,EAAE,CAAC,CAAC;iBACjD;aACF;YACD,iBAAiB;SAClB,CAAC;IACJ,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,QAAQ,CACN,sBAAsB,EACtB,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS,CAC3C,CAAC;QACF,OAAO,eAAe,CAAC,KAAK,EAAE,KAAK,CAAC,GAAG,EAAE,qBAAqB,CAAC,CAAC;IAClE,CAAC;AACH,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/tools/index.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,yCAAyC,CAAC;
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/tools/index.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,yCAAyC,CAAC;AAwHzE;;;GAGG;AACH,wBAAgB,aAAa,CAAC,MAAM,EAAE,SAAS,GAAG,IAAI,CAoCrD"}
|
package/dist/tools/index.js
CHANGED
|
@@ -3,6 +3,7 @@ import { fetchUrlToolHandler, FETCH_URL_TOOL_NAME, FETCH_URL_TOOL_DESCRIPTION, }
|
|
|
3
3
|
import { fetchLinksToolHandler, FETCH_LINKS_TOOL_NAME, FETCH_LINKS_TOOL_DESCRIPTION, } from './handlers/fetch-links.tool.js';
|
|
4
4
|
import { fetchMarkdownToolHandler, FETCH_MARKDOWN_TOOL_NAME, FETCH_MARKDOWN_TOOL_DESCRIPTION, } from './handlers/fetch-markdown.tool.js';
|
|
5
5
|
// Zod schemas for runtime validation - single source of truth
|
|
6
|
+
// Input schemas
|
|
6
7
|
const FetchUrlInputSchema = {
|
|
7
8
|
url: z.string().min(1).describe('The URL to fetch'),
|
|
8
9
|
extractMainContent: z
|
|
@@ -56,6 +57,44 @@ const FetchMarkdownInputSchema = {
|
|
|
56
57
|
.default(true)
|
|
57
58
|
.describe('Include YAML frontmatter metadata'),
|
|
58
59
|
};
|
|
60
|
+
// Output schemas for structured content validation
|
|
61
|
+
const FetchUrlOutputSchema = {
|
|
62
|
+
url: z.string().describe('The fetched URL'),
|
|
63
|
+
title: z.string().optional().describe('Page title'),
|
|
64
|
+
contentBlocks: z.number().describe('Number of content blocks extracted'),
|
|
65
|
+
fetchedAt: z
|
|
66
|
+
.string()
|
|
67
|
+
.describe('ISO timestamp of when the content was fetched'),
|
|
68
|
+
format: z.enum(['jsonl', 'markdown']).describe('Output format used'),
|
|
69
|
+
content: z.string().describe('The extracted content in JSONL format'),
|
|
70
|
+
cached: z.boolean().describe('Whether the result was served from cache'),
|
|
71
|
+
error: z.string().optional().describe('Error message if the request failed'),
|
|
72
|
+
errorCode: z.string().optional().describe('Error code if the request failed'),
|
|
73
|
+
};
|
|
74
|
+
const FetchLinksOutputSchema = {
|
|
75
|
+
url: z.string().describe('The source URL'),
|
|
76
|
+
linkCount: z.number().describe('Total number of links extracted'),
|
|
77
|
+
links: z
|
|
78
|
+
.array(z.object({
|
|
79
|
+
href: z.string().describe('The link URL'),
|
|
80
|
+
text: z.string().describe('The link anchor text'),
|
|
81
|
+
type: z.enum(['internal', 'external']).describe('Link type'),
|
|
82
|
+
}))
|
|
83
|
+
.describe('Array of extracted links'),
|
|
84
|
+
error: z.string().optional().describe('Error message if the request failed'),
|
|
85
|
+
errorCode: z.string().optional().describe('Error code if the request failed'),
|
|
86
|
+
};
|
|
87
|
+
const FetchMarkdownOutputSchema = {
|
|
88
|
+
url: z.string().describe('The fetched URL'),
|
|
89
|
+
title: z.string().optional().describe('Page title'),
|
|
90
|
+
fetchedAt: z
|
|
91
|
+
.string()
|
|
92
|
+
.describe('ISO timestamp of when the content was fetched'),
|
|
93
|
+
markdown: z.string().describe('The extracted content in Markdown format'),
|
|
94
|
+
cached: z.boolean().describe('Whether the result was served from cache'),
|
|
95
|
+
error: z.string().optional().describe('Error message if the request failed'),
|
|
96
|
+
errorCode: z.string().optional().describe('Error code if the request failed'),
|
|
97
|
+
};
|
|
59
98
|
/**
|
|
60
99
|
* Registers all tools with the MCP server using the modern McpServer API
|
|
61
100
|
* Tools are registered with Zod schemas for automatic validation
|
|
@@ -66,18 +105,21 @@ export function registerTools(server) {
|
|
|
66
105
|
title: 'Fetch URL',
|
|
67
106
|
description: FETCH_URL_TOOL_DESCRIPTION,
|
|
68
107
|
inputSchema: FetchUrlInputSchema,
|
|
108
|
+
outputSchema: FetchUrlOutputSchema,
|
|
69
109
|
}, async (args) => fetchUrlToolHandler(args));
|
|
70
110
|
// Register fetch-links tool
|
|
71
111
|
server.registerTool(FETCH_LINKS_TOOL_NAME, {
|
|
72
112
|
title: 'Fetch Links',
|
|
73
113
|
description: FETCH_LINKS_TOOL_DESCRIPTION,
|
|
74
114
|
inputSchema: FetchLinksInputSchema,
|
|
115
|
+
outputSchema: FetchLinksOutputSchema,
|
|
75
116
|
}, async (args) => fetchLinksToolHandler(args));
|
|
76
117
|
// Register fetch-markdown tool
|
|
77
118
|
server.registerTool(FETCH_MARKDOWN_TOOL_NAME, {
|
|
78
119
|
title: 'Fetch Markdown',
|
|
79
120
|
description: FETCH_MARKDOWN_TOOL_DESCRIPTION,
|
|
80
121
|
inputSchema: FetchMarkdownInputSchema,
|
|
122
|
+
outputSchema: FetchMarkdownOutputSchema,
|
|
81
123
|
}, async (args) => fetchMarkdownToolHandler(args));
|
|
82
124
|
}
|
|
83
125
|
//# sourceMappingURL=index.js.map
|
package/dist/tools/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/tools/index.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,EACL,mBAAmB,EACnB,mBAAmB,EACnB,0BAA0B,GAC3B,MAAM,8BAA8B,CAAC;AACtC,OAAO,EACL,qBAAqB,EACrB,qBAAqB,EACrB,4BAA4B,GAC7B,MAAM,gCAAgC,CAAC;AACxC,OAAO,EACL,wBAAwB,EACxB,wBAAwB,EACxB,+BAA+B,GAChC,MAAM,mCAAmC,CAAC;AAE3C,8DAA8D;
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/tools/index.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,EACL,mBAAmB,EACnB,mBAAmB,EACnB,0BAA0B,GAC3B,MAAM,8BAA8B,CAAC;AACtC,OAAO,EACL,qBAAqB,EACrB,qBAAqB,EACrB,4BAA4B,GAC7B,MAAM,gCAAgC,CAAC;AACxC,OAAO,EACL,wBAAwB,EACxB,wBAAwB,EACxB,+BAA+B,GAChC,MAAM,mCAAmC,CAAC;AAE3C,8DAA8D;AAE9D,gBAAgB;AAChB,MAAM,mBAAmB,GAAG;IAC1B,GAAG,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,kBAAkB,CAAC;IACnD,kBAAkB,EAAE,CAAC;SAClB,OAAO,EAAE;SACT,QAAQ,EAAE;SACV,OAAO,CAAC,IAAI,CAAC;SACb,QAAQ,CAAC,iDAAiD,CAAC;IAC9D,eAAe,EAAE,CAAC;SACf,OAAO,EAAE;SACT,QAAQ,EAAE;SACV,OAAO,CAAC,IAAI,CAAC;SACb,QAAQ,CAAC,kDAAkD,CAAC;IAC/D,gBAAgB,EAAE,CAAC;SAChB,MAAM,EAAE;SACR,QAAQ,EAAE;SACV,QAAQ,EAAE;SACV,QAAQ,CAAC,sCAAsC,CAAC;IACnD,MAAM,EAAE,CAAC;SACN,IAAI,CAAC,CAAC,OAAO,EAAE,UAAU,CAAC,CAAC;SAC3B,QAAQ,EAAE;SACV,OAAO,CAAC,OAAO,CAAC;SAChB,QAAQ,CAAC,eAAe,CAAC;IAC5B,aAAa,EAAE,CAAC;SACb,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;SAClB,QAAQ,EAAE;SACV,QAAQ,CAAC,qCAAqC,CAAC;CACnD,CAAC;AAEF,MAAM,qBAAqB,GAAG;IAC5B,GAAG,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,+BAA+B,CAAC;IAChE,eAAe,EAAE,CAAC;SACf,OAAO,EAAE;SACT,QAAQ,EAAE;SACV,OAAO,CAAC,IAAI,CAAC;SACb,QAAQ,CAAC,wBAAwB,CAAC;IACrC,eAAe,EAAE,CAAC;SACf,OAAO,EAAE;SACT,QAAQ,EAAE;SACV,OAAO,CAAC,IAAI,CAAC;SACb,QAAQ,CAAC,wBAAwB,CAAC;CACtC,CAAC;AAEF,MAAM,wBAAwB,GAAG;IAC/B,GAAG,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,kBAAkB,CAAC;IACnD,kBAAkB,EAAE,CAAC;SAClB,OAAO,EAAE;SACT,QAAQ,EAAE;SACV,OAAO,CAAC,IAAI,CAAC;SACb,QAAQ,CAAC,gDAAgD,CAAC;IAC7D,eAAe,EAAE,CAAC;SACf,OAAO,EAAE;SACT,QAAQ,EAAE;SACV,OAAO,CAAC,IAAI,CAAC;SACb,QAAQ,CAAC,mCAAmC,CAAC;CACjD,CAAC;AAEF,mDAAmD;AACnD,MAAM,oBAAoB,GAAG;IAC3B,GAAG,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,iBAAiB,CAAC;IAC3C,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,YAAY,CAAC;IACnD,aAAa,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,oCAAoC,CAAC;IACxE,SAAS,EAAE,CAAC;SACT,MAAM,EAAE;SACR,QAAQ,CAAC,+CAA+C,CAAC;IAC5D,MAAM,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,UAAU,CAAC,CAAC,CAAC,QAAQ,CAAC,oBAAoB,CAAC;IACpE,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,uCAAuC,CAAC;IACrE,MAAM,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,CAAC,0CAA0C,CAAC;IACxE,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,qCAAqC,CAAC;IAC5E,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,kCAAkC,CAAC;CAC9E,CAAC;AAEF,MAAM,sBAAsB,GAAG;IAC7B,GAAG,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,gBAAgB,CAAC;IAC1C,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,iCAAiC,CAAC;IACjE,KAAK,EAAE,CAAC;SACL,KAAK,CACJ,CAAC,CAAC,MAAM,CAAC;QACP,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,cAAc,CAAC;QACzC,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,sBAAsB,CAAC;QACjD,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,UAAU,EAAE,UAAU,CAAC,CAAC,CAAC,QAAQ,CAAC,WAAW,CAAC;KAC7D,CAAC,CACH;SACA,QAAQ,CAAC,0BAA0B,CAAC;IACvC,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,qCAAqC,CAAC;IAC5E,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,kCAAkC,CAAC;CAC9E,CAAC;AAEF,MAAM,yBAAyB,GAAG;IAChC,GAAG,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,iBAAiB,CAAC;IAC3C,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,YAAY,CAAC;IACnD,SAAS,EAAE,CAAC;SACT,MAAM,EAAE;SACR,QAAQ,CAAC,+CAA+C,CAAC;IAC5D,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,0CAA0C,CAAC;IACzE,MAAM,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,CAAC,0CAA0C,CAAC;IACxE,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,qCAAqC,CAAC;IAC5E,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,kCAAkC,CAAC;CAC9E,CAAC;AAEF;;;GAGG;AACH,MAAM,UAAU,aAAa,CAAC,MAAiB;IAC7C,0BAA0B;IAC1B,MAAM,CAAC,YAAY,CACjB,mBAAmB,EACnB;QACE,KAAK,EAAE,WAAW;QAClB,WAAW,EAAE,0BAA0B;QACvC,WAAW,EAAE,mBAAmB;QAChC,YAAY,EAAE,oBAAoB;KACnC,EACD,KAAK,EAAE,IAAI,EAAE,EAAE,CAAC,mBAAmB,CAAC,IAAI,CAAC,CAC1C,CAAC;IAEF,4BAA4B;IAC5B,MAAM,CAAC,YAAY,CACjB,qBAAqB,EACrB;QACE,KAAK,EAAE,aAAa;QACpB,WAAW,EAAE,4BAA4B;QACzC,WAAW,EAAE,qBAAqB;QAClC,YAAY,EAAE,sBAAsB;KACrC,EACD,KAAK,EAAE,IAAI,EAAE,EAAE,CAAC,qBAAqB,CAAC,IAAI,CAAC,CAC5C,CAAC;IAEF,+BAA+B;IAC/B,MAAM,CAAC,YAAY,CACjB,wBAAwB,EACxB;QACE,KAAK,EAAE,gBAAgB;QACvB,WAAW,EAAE,+BAA+B;QAC5C,WAAW,EAAE,wBAAwB;QACrC,YAAY,EAAE,yBAAyB;KACxC,EACD,KAAK,EAAE,IAAI,EAAE,EAAE,CAAC,wBAAwB,CAAC,IAAI,CAAC,CAC/C,CAAC;AACJ,CAAC"}
|
|
@@ -1,4 +1,3 @@
|
|
|
1
1
|
import type { ContentBlockUnion, MetadataBlock } from '../types/index.js';
|
|
2
2
|
export declare function toJsonl(blocks: ContentBlockUnion[], metadata?: MetadataBlock): string;
|
|
3
|
-
export declare function fromJsonl(jsonl: string): ContentBlockUnion[];
|
|
4
3
|
//# sourceMappingURL=jsonl.transformer.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"jsonl.transformer.d.ts","sourceRoot":"","sources":["../../src/transformers/jsonl.transformer.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,iBAAiB,EAAE,aAAa,EAAE,MAAM,mBAAmB,CAAC;
|
|
1
|
+
{"version":3,"file":"jsonl.transformer.d.ts","sourceRoot":"","sources":["../../src/transformers/jsonl.transformer.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,iBAAiB,EAAE,aAAa,EAAE,MAAM,mBAAmB,CAAC;AA4B1E,wBAAgB,OAAO,CACrB,MAAM,EAAE,iBAAiB,EAAE,EAC3B,QAAQ,CAAC,EAAE,aAAa,GACvB,MAAM,CA0BR"}
|