@j0hanz/superfetch 1.2.0 → 1.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +18 -204
- package/dist/config/constants.d.ts +0 -1
- package/dist/config/constants.js +0 -1
- package/dist/config/formatting.d.ts +0 -1
- package/dist/config/formatting.d.ts.map +1 -1
- package/dist/config/formatting.js +1 -3
- package/dist/config/formatting.js.map +1 -1
- package/dist/config/index.d.ts +1 -2
- package/dist/config/index.d.ts.map +1 -1
- package/dist/config/index.js +5 -11
- package/dist/config/index.js.map +1 -1
- package/dist/config/types/content.d.ts +0 -1
- package/dist/config/types/content.js +0 -1
- package/dist/config/types/runtime.d.ts +0 -1
- package/dist/config/types/runtime.js +0 -1
- package/dist/config/types/tools.d.ts +0 -1
- package/dist/config/types/tools.js +0 -1
- package/dist/errors/app-error.d.ts +0 -1
- package/dist/errors/app-error.js +0 -1
- package/dist/http/auth.d.ts +0 -1
- package/dist/http/auth.js +0 -1
- package/dist/http/cors.d.ts +0 -1
- package/dist/http/cors.js +0 -1
- package/dist/http/download-routes.d.ts +1 -2
- package/dist/http/download-routes.d.ts.map +1 -1
- package/dist/http/download-routes.js +0 -1
- package/dist/http/mcp-routes.d.ts +0 -1
- package/dist/http/mcp-routes.js +0 -1
- package/dist/http/mcp-session-helpers.d.ts +0 -1
- package/dist/http/mcp-session-helpers.js +0 -1
- package/dist/http/mcp-session.d.ts +1 -2
- package/dist/http/mcp-session.d.ts.map +1 -1
- package/dist/http/mcp-session.js +0 -1
- package/dist/http/mcp-validation.d.ts +1 -2
- package/dist/http/mcp-validation.d.ts.map +1 -1
- package/dist/http/mcp-validation.js +6 -27
- package/dist/http/mcp-validation.js.map +1 -1
- package/dist/http/rate-limit.d.ts +1 -2
- package/dist/http/rate-limit.d.ts.map +1 -1
- package/dist/http/rate-limit.js +0 -1
- package/dist/http/rate-limit.js.map +1 -1
- package/dist/http/server-middleware.d.ts +0 -1
- package/dist/http/server-middleware.d.ts.map +1 -1
- package/dist/http/server-middleware.js +56 -1
- package/dist/http/server-middleware.js.map +1 -1
- package/dist/http/server.d.ts +0 -1
- package/dist/http/server.js +0 -1
- package/dist/http/session-cleanup.d.ts +0 -1
- package/dist/http/session-cleanup.js +0 -1
- package/dist/http/sessions.d.ts +1 -2
- package/dist/http/sessions.d.ts.map +1 -1
- package/dist/http/sessions.js +0 -1
- package/dist/index.d.ts +0 -1
- package/dist/index.js +0 -1
- package/dist/middleware/error-handler.d.ts +0 -1
- package/dist/middleware/error-handler.js +0 -1
- package/dist/resources/cached-content.d.ts +0 -1
- package/dist/resources/cached-content.js +0 -1
- package/dist/resources/index.d.ts +0 -1
- package/dist/resources/index.js +0 -1
- package/dist/server.d.ts +0 -1
- package/dist/server.d.ts.map +1 -1
- package/dist/server.js +8 -3
- package/dist/server.js.map +1 -1
- package/dist/services/cache.d.ts +1 -2
- package/dist/services/cache.d.ts.map +1 -1
- package/dist/services/cache.js +1 -2
- package/dist/services/context.d.ts +0 -1
- package/dist/services/context.js +0 -1
- package/dist/services/extractor.d.ts +1 -2
- package/dist/services/extractor.d.ts.map +1 -1
- package/dist/services/extractor.js +0 -1
- package/dist/services/fetcher/agents.d.ts +0 -1
- package/dist/services/fetcher/agents.js +0 -1
- package/dist/services/fetcher/errors.d.ts +0 -1
- package/dist/services/fetcher/errors.js +0 -1
- package/dist/services/fetcher/interceptors.d.ts +0 -1
- package/dist/services/fetcher/interceptors.js +0 -1
- package/dist/services/fetcher/redirects.d.ts +0 -1
- package/dist/services/fetcher/redirects.js +2 -3
- package/dist/services/fetcher/response.d.ts +0 -1
- package/dist/services/fetcher/response.js +4 -5
- package/dist/services/fetcher/retry-policy.d.ts +0 -1
- package/dist/services/fetcher/retry-policy.js +0 -1
- package/dist/services/fetcher.d.ts +1 -2
- package/dist/services/fetcher.d.ts.map +1 -1
- package/dist/services/fetcher.js +3 -4
- package/dist/services/fetcher.js.map +1 -1
- package/dist/services/logger.d.ts +1 -2
- package/dist/services/logger.d.ts.map +1 -1
- package/dist/services/logger.js +0 -1
- package/dist/services/parser.d.ts +1 -2
- package/dist/services/parser.d.ts.map +1 -1
- package/dist/services/parser.js +0 -1
- package/dist/tools/handlers/fetch-markdown.tool.d.ts +1 -2
- package/dist/tools/handlers/fetch-markdown.tool.d.ts.map +1 -1
- package/dist/tools/handlers/fetch-markdown.tool.js +34 -3
- package/dist/tools/handlers/fetch-markdown.tool.js.map +1 -1
- package/dist/tools/handlers/fetch-single.shared.d.ts +5 -3
- package/dist/tools/handlers/fetch-single.shared.d.ts.map +1 -1
- package/dist/tools/handlers/fetch-single.shared.js +8 -4
- package/dist/tools/handlers/fetch-single.shared.js.map +1 -1
- package/dist/tools/handlers/fetch-url.tool.d.ts +1 -2
- package/dist/tools/handlers/fetch-url.tool.d.ts.map +1 -1
- package/dist/tools/handlers/fetch-url.tool.js +36 -3
- package/dist/tools/index.d.ts +0 -1
- package/dist/tools/index.js +0 -1
- package/dist/tools/schemas.d.ts +29 -30
- package/dist/tools/schemas.js +0 -1
- package/dist/tools/utils/cache-vary.d.ts +0 -1
- package/dist/tools/utils/cache-vary.d.ts.map +1 -1
- package/dist/tools/utils/cache-vary.js +7 -25
- package/dist/tools/utils/cache-vary.js.map +1 -1
- package/dist/tools/utils/common.d.ts +2 -3
- package/dist/tools/utils/common.d.ts.map +1 -1
- package/dist/tools/utils/common.js +6 -7
- package/dist/tools/utils/common.js.map +1 -1
- package/dist/tools/utils/content-transform.d.ts +1 -3
- package/dist/tools/utils/content-transform.d.ts.map +1 -1
- package/dist/tools/utils/content-transform.js +42 -6
- package/dist/tools/utils/content-transform.js.map +1 -1
- package/dist/tools/utils/fetch-pipeline.d.ts +1 -2
- package/dist/tools/utils/fetch-pipeline.d.ts.map +1 -1
- package/dist/tools/utils/fetch-pipeline.js +11 -18
- package/dist/tools/utils/inline-content.d.ts +0 -1
- package/dist/tools/utils/inline-content.js +0 -1
- package/dist/transformers/jsonl.transformer.d.ts +1 -2
- package/dist/transformers/jsonl.transformer.d.ts.map +1 -1
- package/dist/transformers/jsonl.transformer.js +0 -1
- package/dist/transformers/jsonl.transformer.js.map +1 -1
- package/dist/transformers/markdown.transformer.d.ts +1 -2
- package/dist/transformers/markdown.transformer.d.ts.map +1 -1
- package/dist/transformers/markdown.transformer.js +8 -1
- package/dist/utils/code-language.d.ts +0 -1
- package/dist/utils/code-language.js +0 -1
- package/dist/utils/content-cleaner.d.ts +0 -1
- package/dist/utils/content-cleaner.js +0 -1
- package/dist/utils/crypto.d.ts +0 -1
- package/dist/utils/crypto.js +0 -1
- package/dist/utils/download-url.d.ts +1 -2
- package/dist/utils/download-url.d.ts.map +1 -1
- package/dist/utils/download-url.js +0 -1
- package/dist/utils/error-utils.d.ts +0 -1
- package/dist/utils/error-utils.js +1 -3
- package/dist/utils/filename-generator.d.ts +0 -1
- package/dist/utils/filename-generator.js +0 -1
- package/dist/utils/header-normalizer.d.ts +0 -1
- package/dist/utils/header-normalizer.js +0 -1
- package/dist/utils/html-truncator.d.ts +0 -1
- package/dist/utils/html-truncator.js +0 -1
- package/dist/utils/sanitizer.d.ts +0 -1
- package/dist/utils/sanitizer.js +0 -1
- package/dist/utils/tool-error-handler.d.ts +1 -2
- package/dist/utils/tool-error-handler.d.ts.map +1 -1
- package/dist/utils/tool-error-handler.js +0 -1
- package/dist/utils/url-validator.d.ts +1 -3
- package/dist/utils/url-validator.d.ts.map +1 -1
- package/dist/utils/url-validator.js +44 -10
- package/dist/utils/url-validator.js.map +1 -1
- package/package.json +4 -4
|
@@ -1,10 +1,19 @@
|
|
|
1
1
|
import { TRUNCATION_MARKER } from '../../config/formatting.js';
|
|
2
2
|
import { extractContent } from '../../services/extractor.js';
|
|
3
3
|
import { parseHtml } from '../../services/parser.js';
|
|
4
|
+
import { sanitizeText } from '../../utils/sanitizer.js';
|
|
4
5
|
import { toJsonl } from '../../transformers/jsonl.transformer.js';
|
|
5
6
|
import { htmlToMarkdown } from '../../transformers/markdown.transformer.js';
|
|
6
7
|
import { createContentMetadataBlock, determineContentExtractionSource, truncateContent, } from './common.js';
|
|
8
|
+
const TITLE_PATTERN = /<title[^>]*>([\s\S]*?)<\/title>/i;
|
|
7
9
|
function resolveContentSource(html, url, options) {
|
|
10
|
+
if (!options.extractMainContent && !options.includeMetadata) {
|
|
11
|
+
return {
|
|
12
|
+
sourceHtml: html,
|
|
13
|
+
title: extractTitleFromHtml(html),
|
|
14
|
+
metadata: undefined,
|
|
15
|
+
};
|
|
16
|
+
}
|
|
8
17
|
const { article, metadata: extractedMeta } = extractContent(html, url, {
|
|
9
18
|
extractArticle: options.extractMainContent,
|
|
10
19
|
});
|
|
@@ -14,8 +23,36 @@ function resolveContentSource(html, url, options) {
|
|
|
14
23
|
const title = shouldExtractFromArticle ? article.title : extractedMeta.title;
|
|
15
24
|
return { sourceHtml, title, metadata };
|
|
16
25
|
}
|
|
17
|
-
function
|
|
18
|
-
|
|
26
|
+
function extractTitleFromHtml(html) {
|
|
27
|
+
const match = TITLE_PATTERN.exec(html);
|
|
28
|
+
if (!match?.[1])
|
|
29
|
+
return undefined;
|
|
30
|
+
const decoded = decodeHtmlEntities(match[1]);
|
|
31
|
+
const text = sanitizeText(decoded);
|
|
32
|
+
return text || undefined;
|
|
33
|
+
}
|
|
34
|
+
function decodeHtmlEntities(value) {
|
|
35
|
+
if (!value.includes('&'))
|
|
36
|
+
return value;
|
|
37
|
+
const basicDecoded = value
|
|
38
|
+
.replace(/&/g, '&')
|
|
39
|
+
.replace(/</g, '<')
|
|
40
|
+
.replace(/>/g, '>')
|
|
41
|
+
.replace(/"/g, '"')
|
|
42
|
+
.replace(/'/g, "'");
|
|
43
|
+
return basicDecoded
|
|
44
|
+
.replace(/&#(\d+);/g, (match, code) => {
|
|
45
|
+
const parsed = Number.parseInt(code, 10);
|
|
46
|
+
return Number.isFinite(parsed) && parsed >= 0 && parsed <= 0x10ffff
|
|
47
|
+
? String.fromCodePoint(parsed)
|
|
48
|
+
: match;
|
|
49
|
+
})
|
|
50
|
+
.replace(/&#x([0-9a-fA-F]+);/g, (match, code) => {
|
|
51
|
+
const parsed = Number.parseInt(code, 16);
|
|
52
|
+
return Number.isFinite(parsed) && parsed >= 0 && parsed <= 0x10ffff
|
|
53
|
+
? String.fromCodePoint(parsed)
|
|
54
|
+
: match;
|
|
55
|
+
});
|
|
19
56
|
}
|
|
20
57
|
function buildJsonlPayload(context, maxContentLength) {
|
|
21
58
|
const contentBlocks = parseHtml(context.sourceHtml);
|
|
@@ -32,7 +69,7 @@ function buildMarkdownPayload(context, maxContentLength) {
|
|
|
32
69
|
return { content, truncated };
|
|
33
70
|
}
|
|
34
71
|
export function transformHtmlToJsonl(html, url, options) {
|
|
35
|
-
const context =
|
|
72
|
+
const context = resolveContentSource(html, url, options);
|
|
36
73
|
const { content, contentBlocks, truncated } = buildJsonlPayload(context, options.maxContentLength);
|
|
37
74
|
return {
|
|
38
75
|
content,
|
|
@@ -42,7 +79,7 @@ export function transformHtmlToJsonl(html, url, options) {
|
|
|
42
79
|
};
|
|
43
80
|
}
|
|
44
81
|
export function transformHtmlToMarkdown(html, url, options) {
|
|
45
|
-
const context =
|
|
82
|
+
const context = resolveContentSource(html, url, options);
|
|
46
83
|
const { content, truncated } = buildMarkdownPayload(context, options.maxContentLength);
|
|
47
84
|
return {
|
|
48
85
|
markdown: content,
|
|
@@ -51,7 +88,7 @@ export function transformHtmlToMarkdown(html, url, options) {
|
|
|
51
88
|
};
|
|
52
89
|
}
|
|
53
90
|
export function transformHtmlToMarkdownWithBlocks(html, url, options) {
|
|
54
|
-
const context =
|
|
91
|
+
const context = resolveContentSource(html, url, options);
|
|
55
92
|
const contentBlocks = parseHtml(context.sourceHtml);
|
|
56
93
|
const { content, truncated } = buildMarkdownPayload(context, options.maxContentLength);
|
|
57
94
|
return {
|
|
@@ -61,4 +98,3 @@ export function transformHtmlToMarkdownWithBlocks(html, url, options) {
|
|
|
61
98
|
...(truncated && { truncated }),
|
|
62
99
|
};
|
|
63
100
|
}
|
|
64
|
-
//# sourceMappingURL=content-transform.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"content-transform.js","sourceRoot":"","sources":["../../../src/tools/utils/content-transform.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iBAAiB,EAAE,MAAM,4BAA4B,CAAC;AAM/D,OAAO,EAAE,cAAc,EAAE,MAAM,6BAA6B,CAAC;AAC7D,OAAO,EAAE,SAAS,EAAE,MAAM,0BAA0B,CAAC;AAErD,OAAO,EAAE,OAAO,EAAE,MAAM,yCAAyC,CAAC;AAClE,OAAO,EAAE,cAAc,EAAE,MAAM,4CAA4C,CAAC;AAE5E,OAAO,EACL,0BAA0B,EAC1B,gCAAgC,EAChC,eAAe,GAChB,MAAM,aAAa,CAAC;
|
|
1
|
+
{"version":3,"file":"content-transform.js","sourceRoot":"","sources":["../../../src/tools/utils/content-transform.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iBAAiB,EAAE,MAAM,4BAA4B,CAAC;AAM/D,OAAO,EAAE,cAAc,EAAE,MAAM,6BAA6B,CAAC;AAC7D,OAAO,EAAE,SAAS,EAAE,MAAM,0BAA0B,CAAC;AAErD,OAAO,EAAE,OAAO,EAAE,MAAM,yCAAyC,CAAC;AAClE,OAAO,EAAE,cAAc,EAAE,MAAM,4CAA4C,CAAC;AAE5E,OAAO,EACL,0BAA0B,EAC1B,gCAAgC,EAChC,eAAe,GAChB,MAAM,aAAa,CAAC;AAmBrB,SAAS,oBAAoB,CAC3B,IAAY,EACZ,GAAW,EACX,OAA0B;IAE1B,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,aAAa,EAAE,GAAG,cAAc,CAAC,IAAI,EAAE,GAAG,EAAE;QACrE,cAAc,EAAE,OAAO,CAAC,kBAAkB;KAC3C,CAAC,CAAC;IAEH,MAAM,wBAAwB,GAAG,gCAAgC,CAC/D,OAAO,CAAC,kBAAkB,EAC1B,OAAO,CACR,CAAC;IAEF,MAAM,UAAU,GAAG,wBAAwB,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC;IACrE,MAAM,QAAQ,GAAG,0BAA0B,CACzC,GAAG,EACH,OAAO,EACP,aAAa,EACb,wBAAwB,EACxB,OAAO,CAAC,eAAe,CACxB,CAAC;IACF,MAAM,KAAK,GAAG,wBAAwB,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,aAAa,CAAC,KAAK,CAAC;IAE7E,OAAO,EAAE,UAAU,EAAE,KAAK,EAAE,QAAQ,EAAE,CAAC;AACzC,CAAC;AAED,SAAS,iBAAiB,CACxB,OAAsB,EACtB,gBAAyB;IAEzB,MAAM,aAAa,GAAG,SAAS,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;IACpD,MAAM,EAAE,OAAO,EAAE,SAAS,EAAE,GAAG,eAAe,CAC5C,OAAO,CAAC,aAAa,EAAE,OAAO,CAAC,QAAQ,CAAC,EACxC,gBAAgB,CACjB,CAAC;IAEF,OAAO;QACL,OAAO;QACP,aAAa,EAAE,aAAa,CAAC,MAAM;QACnC,SAAS;KACV,CAAC;AACJ,CAAC;AAED,SAAS,oBAAoB,CAC3B,OAAsB,EACtB,gBAAyB;IAEzB,MAAM,QAAQ,GAAG,cAAc,CAAC,OAAO,CAAC,UAAU,EAAE,OAAO,CAAC,QAAQ,CAAC,CAAC;IACtE,MAAM,EAAE,OAAO,EAAE,SAAS,EAAE,GAAG,eAAe,CAC5C,QAAQ,EACR,gBAAgB,EAChB,iBAAiB,CAClB,CAAC;IAEF,OAAO,EAAE,OAAO,EAAE,SAAS,EAAE,CAAC;AAChC,CAAC;AAED,MAAM,UAAU,oBAAoB,CAClC,IAAY,EACZ,GAAW,EACX,OAAiD;IAEjD,MAAM,OAAO,GAAG,oBAAoB,CAAC,IAAI,EAAE,GAAG,EAAE,OAAO,CAAC,CAAC;IACzD,MAAM,EAAE,OAAO,EAAE,aAAa,EAAE,SAAS,EAAE,GAAG,iBAAiB,CAC7D,OAAO,EACP,OAAO,CAAC,gBAAgB,CACzB,CAAC;IAEF,OAAO;QACL,OAAO;QACP,aAAa;QACb,KAAK,EAAE,OAAO,CAAC,KAAK;QACpB,GAAG,CAAC,SAAS,IAAI,EAAE,SAAS,EAAE,CAAC;KAChC,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,uBAAuB,CACrC,IAAY,EACZ,GAAW,EACX,OAAwB;IAExB,MAAM,OAAO,GAAG,oBAAoB,CAAC,IAAI,EAAE,GAAG,EAAE,OAAO,CAAC,CAAC;IACzD,MAAM,EAAE,OAAO,EAAE,SAAS,EAAE,GAAG,oBAAoB,CACjD,OAAO,EACP,OAAO,CAAC,gBAAgB,CACzB,CAAC;IAEF,OAAO;QACL,QAAQ,EAAE,OAAO;QACjB,KAAK,EAAE,OAAO,CAAC,KAAK;QACpB,SAAS;KACV,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,iCAAiC,CAC/C,IAAY,EACZ,GAAW,EACX,OAAiD;IAEjD,MAAM,OAAO,GAAG,oBAAoB,CAAC,IAAI,EAAE,GAAG,EAAE,OAAO,CAAC,CAAC;IACzD,MAAM,aAAa,GAAG,SAAS,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;IACpD,MAAM,EAAE,OAAO,EAAE,SAAS,EAAE,GAAG,oBAAoB,CACjD,OAAO,EACP,OAAO,CAAC,gBAAgB,CACzB,CAAC;IAEF,OAAO;QACL,OAAO;QACP,aAAa,EAAE,aAAa,CAAC,MAAM;QACnC,KAAK,EAAE,OAAO,CAAC,KAAK;QACpB,GAAG,CAAC,SAAS,IAAI,EAAE,SAAS,EAAE,CAAC;KAChC,CAAC;AACJ,CAAC"}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type { FetchPipelineOptions, PipelineResult } from '../../config/types.js';
|
|
1
|
+
import type { FetchPipelineOptions, PipelineResult } from '../../config/types/runtime.js';
|
|
2
2
|
/**
|
|
3
3
|
* Unified fetch pipeline that handles caching, fetching, and transformation.
|
|
4
4
|
* Implements cache-first strategy with automatic serialization.
|
|
@@ -8,4 +8,3 @@ import type { FetchPipelineOptions, PipelineResult } from '../../config/types.js
|
|
|
8
8
|
* @returns Promise resolving to the pipeline result
|
|
9
9
|
*/
|
|
10
10
|
export declare function executeFetchPipeline<T>(options: FetchPipelineOptions<T>): Promise<PipelineResult<T>>;
|
|
11
|
-
//# sourceMappingURL=fetch-pipeline.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"fetch-pipeline.d.ts","sourceRoot":"","sources":["../../../src/tools/utils/fetch-pipeline.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAEV,oBAAoB,EACpB,cAAc,EACf,MAAM,
|
|
1
|
+
{"version":3,"file":"fetch-pipeline.d.ts","sourceRoot":"","sources":["../../../src/tools/utils/fetch-pipeline.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAEV,oBAAoB,EACpB,cAAc,EACf,MAAM,+BAA+B,CAAC;AAuDvC;;;;;;;GAOG;AACH,wBAAsB,oBAAoB,CAAC,CAAC,EAC1C,OAAO,EAAE,oBAAoB,CAAC,CAAC,CAAC,GAC/B,OAAO,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,CAwB5B"}
|
|
@@ -1,29 +1,22 @@
|
|
|
1
1
|
import * as cache from '../../services/cache.js';
|
|
2
2
|
import { fetchUrlWithRetry } from '../../services/fetcher.js';
|
|
3
|
-
import { logDebug
|
|
3
|
+
import { logDebug } from '../../services/logger.js';
|
|
4
4
|
import { validateAndNormalizeUrl } from '../../utils/url-validator.js';
|
|
5
5
|
import { appendHeaderVary } from './cache-vary.js';
|
|
6
|
-
function safeJsonParse(cached, cacheKey) {
|
|
7
|
-
try {
|
|
8
|
-
return JSON.parse(cached);
|
|
9
|
-
}
|
|
10
|
-
catch {
|
|
11
|
-
logWarn('Cache deserialize failed, treating as miss', {
|
|
12
|
-
key: cacheKey.substring(0, 100),
|
|
13
|
-
});
|
|
14
|
-
return undefined;
|
|
15
|
-
}
|
|
16
|
-
}
|
|
17
6
|
function attemptCacheRetrieval(cacheKey, deserialize, cacheNamespace, normalizedUrl) {
|
|
18
7
|
if (!cacheKey)
|
|
19
8
|
return null;
|
|
20
9
|
const cached = cache.get(cacheKey);
|
|
21
10
|
if (!cached)
|
|
22
11
|
return null;
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
12
|
+
if (!deserialize) {
|
|
13
|
+
logDebug('Cache miss due to missing deserializer', {
|
|
14
|
+
namespace: cacheNamespace,
|
|
15
|
+
url: normalizedUrl,
|
|
16
|
+
});
|
|
17
|
+
return null;
|
|
18
|
+
}
|
|
19
|
+
const data = deserialize(cached.content);
|
|
27
20
|
if (data === undefined) {
|
|
28
21
|
logDebug('Cache miss due to deserialize failure', {
|
|
29
22
|
namespace: cacheNamespace,
|
|
@@ -31,6 +24,7 @@ function attemptCacheRetrieval(cacheKey, deserialize, cacheNamespace, normalized
|
|
|
31
24
|
});
|
|
32
25
|
return null;
|
|
33
26
|
}
|
|
27
|
+
logDebug('Cache hit', { namespace: cacheNamespace, url: normalizedUrl });
|
|
34
28
|
return {
|
|
35
29
|
data,
|
|
36
30
|
fromCache: true,
|
|
@@ -48,7 +42,7 @@ function attemptCacheRetrieval(cacheKey, deserialize, cacheNamespace, normalized
|
|
|
48
42
|
* @returns Promise resolving to the pipeline result
|
|
49
43
|
*/
|
|
50
44
|
export async function executeFetchPipeline(options) {
|
|
51
|
-
const normalizedUrl = validateAndNormalizeUrl(options.url);
|
|
45
|
+
const normalizedUrl = await validateAndNormalizeUrl(options.url);
|
|
52
46
|
const cacheKey = resolveCacheKey(options, normalizedUrl);
|
|
53
47
|
const cachedResult = attemptCacheRetrieval(cacheKey, options.deserialize, options.cacheNamespace, normalizedUrl);
|
|
54
48
|
if (cachedResult)
|
|
@@ -97,4 +91,3 @@ function buildPipelineResult(url, data, cacheKey) {
|
|
|
97
91
|
cacheKey,
|
|
98
92
|
};
|
|
99
93
|
}
|
|
100
|
-
//# sourceMappingURL=fetch-pipeline.js.map
|
|
@@ -1,3 +1,2 @@
|
|
|
1
|
-
import type { ContentBlockUnion, MetadataBlock } from '../config/types.js';
|
|
1
|
+
import type { ContentBlockUnion, MetadataBlock } from '../config/types/content.js';
|
|
2
2
|
export declare function toJsonl(blocks: readonly ContentBlockUnion[], metadata?: MetadataBlock): string;
|
|
3
|
-
//# sourceMappingURL=jsonl.transformer.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"jsonl.transformer.d.ts","sourceRoot":"","sources":["../../src/transformers/jsonl.transformer.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,
|
|
1
|
+
{"version":3,"file":"jsonl.transformer.d.ts","sourceRoot":"","sources":["../../src/transformers/jsonl.transformer.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EACV,iBAAiB,EACjB,aAAa,EACd,MAAM,4BAA4B,CAAC;AAkEpC,wBAAgB,OAAO,CACrB,MAAM,EAAE,SAAS,iBAAiB,EAAE,EACpC,QAAQ,CAAC,EAAE,aAAa,GACvB,MAAM,CAGR"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"jsonl.transformer.js","sourceRoot":"","sources":["../../src/transformers/jsonl.transformer.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,yBAAyB,CAAC;AACpD,OAAO,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;
|
|
1
|
+
{"version":3,"file":"jsonl.transformer.js","sourceRoot":"","sources":["../../src/transformers/jsonl.transformer.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,yBAAyB,CAAC;AACpD,OAAO,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAM5C,OAAO,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAC;AAErD,MAAM,gBAAgB,GAAG,IAAI,GAAG,CAAC;IAC/B,WAAW;IACX,SAAS;IACT,MAAM;IACN,YAAY;CACb,CAAC,CAAC;AAEH,SAAS,WAAW,CAClB,KAAwB;IAExB,OAAO,MAAM,IAAI,KAAK,CAAC;AACzB,CAAC;AAED,SAAS,WAAW,CAClB,KAAwB;IAExB,OAAO,KAAK,CAAC,IAAI,KAAK,MAAM,CAAC;AAC/B,CAAC;AAED,SAAS,iBAAiB,CACxB,KAAmD,EACnD,SAAiB;IAEjB,MAAM,SAAS,GAAG,YAAY,CAAC,KAAK,CAAC,IAAI,EAAE,SAAS,CAAC,CAAC;IACtD,OAAO,SAAS,KAAK,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,GAAG,KAAK,EAAE,IAAI,EAAE,SAAS,EAAE,CAAC;AAC1E,CAAC;AAED,SAAS,iBAAiB,CACxB,KAAsD,EACtD,SAAiB;IAEjB,MAAM,cAAc,GAAG,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAC9C,YAAY,CAAC,IAAI,EAAE,SAAS,CAAC,CAC9B,CAAC;IACF,MAAM,UAAU,GAAG,cAAc,CAAC,IAAI,CACpC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,IAAI,KAAK,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,CAC7C,CAAC;IACF,OAAO,UAAU,CAAC,CAAC,CAAC,EAAE,GAAG,KAAK,EAAE,KAAK,EAAE,cAAc,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC;AAClE,CAAC;AAED,SAAS,aAAa,CAAC,KAAwB;IAC7C,MAAM,SAAS,GAAG,MAAM,CAAC,UAAU,CAAC,cAAc,CAAC;IAEnD,IAAI,gBAAgB,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,WAAW,CAAC,KAAK,CAAC,EAAE,CAAC;QAC3D,OAAO,iBAAiB,CAAC,KAAK,EAAE,SAAS,CAAC,CAAC;IAC7C,CAAC;IAED,IAAI,WAAW,CAAC,KAAK,CAAC,EAAE,CAAC;QACvB,OAAO,iBAAiB,CAAC,KAAK,EAAE,SAAS,CAAC,CAAC;IAC7C,CAAC;IAED,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAS,cAAc,CAAC,KAAwB;IAC9C,IAAI,CAAC;QACH,OAAO,IAAI,CAAC,SAAS,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC,CAAC;IAC9C,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED,MAAM,UAAU,OAAO,CACrB,MAAoC,EACpC,QAAwB;IAExB,MAAM,KAAK,GAAG,iBAAiB,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;IAClD,OAAO,SAAS,CAAC,KAAK,CAAC,CAAC;AAC1B,CAAC;AAED,SAAS,iBAAiB,CACxB,MAAoC,EACpC,QAAwB;IAExB,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,MAAM,MAAM,GAAG,iBAAiB,CAAC,QAAQ,CAAC,CAAC;IAC3C,IAAI,MAAM,EAAE,CAAC;QACX,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IACrB,CAAC;IAED,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,MAAM,UAAU,GAAG,cAAc,CAAC,KAAK,CAAC,CAAC;QACzC,IAAI,UAAU,EAAE,CAAC;YACf,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QACzB,CAAC;IACH,CAAC;IAED,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAS,iBAAiB,CAAC,QAAwB;IACjD,IAAI,CAAC,QAAQ;QAAE,OAAO,IAAI,CAAC;IAC3B,IAAI,CAAC;QACH,MAAM,eAAe,GAAG;YACtB,IAAI,EAAE,QAAQ,CAAC,IAAI;YACnB,KAAK,EAAE,QAAQ,CAAC,KAAK;YACrB,GAAG,EAAE,QAAQ,CAAC,GAAG;SAClB,CAAC;QACF,OAAO,IAAI,CAAC,SAAS,CAAC,eAAe,CAAC,CAAC;IACzC,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"markdown.transformer.d.ts","sourceRoot":"","sources":["../../src/transformers/markdown.transformer.ts"],"names":[],"mappings":"AAOA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,
|
|
1
|
+
{"version":3,"file":"markdown.transformer.d.ts","sourceRoot":"","sources":["../../src/transformers/markdown.transformer.ts"],"names":[],"mappings":"AAOA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,4BAA4B,CAAC;AA+HhE,wBAAgB,cAAc,CAAC,IAAI,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,aAAa,GAAG,MAAM,CAa7E"}
|
|
@@ -41,8 +41,16 @@ function isFencedCodeBlock(node, options) {
|
|
|
41
41
|
return false;
|
|
42
42
|
return firstChild.nodeName === 'CODE';
|
|
43
43
|
}
|
|
44
|
+
function isElement(node) {
|
|
45
|
+
return (node !== null &&
|
|
46
|
+
typeof node === 'object' &&
|
|
47
|
+
'getAttribute' in node &&
|
|
48
|
+
typeof node.getAttribute === 'function');
|
|
49
|
+
}
|
|
44
50
|
function formatFencedCodeBlock(node) {
|
|
45
51
|
const codeNode = node.firstChild;
|
|
52
|
+
if (!isElement(codeNode))
|
|
53
|
+
return '';
|
|
46
54
|
const code = codeNode.textContent || '';
|
|
47
55
|
const language = resolveCodeLanguage(codeNode, code);
|
|
48
56
|
return CODE_BLOCK.format(code, language);
|
|
@@ -116,4 +124,3 @@ export function htmlToMarkdown(html, metadata) {
|
|
|
116
124
|
function isValidHtmlInput(html) {
|
|
117
125
|
return Boolean(html && typeof html === 'string');
|
|
118
126
|
}
|
|
119
|
-
//# sourceMappingURL=markdown.transformer.js.map
|
|
@@ -3,4 +3,3 @@ export declare function cleanHeading(text: string): string | null;
|
|
|
3
3
|
export declare function cleanListItems(items: string[]): string[];
|
|
4
4
|
export declare function cleanCodeBlock(code: string): string | null;
|
|
5
5
|
export declare function removeInlineTimestamps(text: string): string;
|
|
6
|
-
//# sourceMappingURL=content-cleaner.d.ts.map
|
package/dist/utils/crypto.d.ts
CHANGED
package/dist/utils/crypto.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type { FileDownloadInfo } from '../config/types.js';
|
|
1
|
+
import type { FileDownloadInfo } from '../config/types/tools.js';
|
|
2
2
|
interface DownloadInfoOptions {
|
|
3
3
|
cacheKey: string | null;
|
|
4
4
|
url: string;
|
|
@@ -6,4 +6,3 @@ interface DownloadInfoOptions {
|
|
|
6
6
|
}
|
|
7
7
|
export declare function buildFileDownloadInfo(options: DownloadInfoOptions): FileDownloadInfo | null;
|
|
8
8
|
export {};
|
|
9
|
-
//# sourceMappingURL=download-url.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"download-url.d.ts","sourceRoot":"","sources":["../../src/utils/download-url.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,
|
|
1
|
+
{"version":3,"file":"download-url.d.ts","sourceRoot":"","sources":["../../src/utils/download-url.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,0BAA0B,CAAC;AAMjE,UAAU,mBAAmB;IAC3B,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IACxB,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,wBAAgB,qBAAqB,CACnC,OAAO,EAAE,mBAAmB,GAC3B,gBAAgB,GAAG,IAAI,CA0BzB"}
|
|
@@ -1,4 +1,3 @@
|
|
|
1
1
|
export declare function getErrorMessage(error: unknown): string;
|
|
2
2
|
export declare function createErrorWithCode(message: string, code: string): NodeJS.ErrnoException;
|
|
3
3
|
export declare function isSystemError(error: unknown): error is NodeJS.ErrnoException;
|
|
4
|
-
//# sourceMappingURL=error-utils.d.ts.map
|
|
@@ -3,12 +3,10 @@ export function getErrorMessage(error) {
|
|
|
3
3
|
}
|
|
4
4
|
export function createErrorWithCode(message, code) {
|
|
5
5
|
const error = new Error(message);
|
|
6
|
-
error
|
|
7
|
-
return error;
|
|
6
|
+
return Object.assign(error, { code });
|
|
8
7
|
}
|
|
9
8
|
export function isSystemError(error) {
|
|
10
9
|
return (error instanceof Error &&
|
|
11
10
|
'code' in error &&
|
|
12
11
|
typeof error.code === 'string');
|
|
13
12
|
}
|
|
14
|
-
//# sourceMappingURL=error-utils.js.map
|
|
@@ -6,4 +6,3 @@ export declare function normalizeHeaderEntries(headers: Record<string, string>,
|
|
|
6
6
|
export declare function hasHeaderEntries(headers: Headers): boolean;
|
|
7
7
|
export declare function headersToRecord(headers: Headers): Record<string, string>;
|
|
8
8
|
export {};
|
|
9
|
-
//# sourceMappingURL=header-normalizer.d.ts.map
|
package/dist/utils/sanitizer.js
CHANGED
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
import type { ToolErrorResponse } from '../config/types.js';
|
|
1
|
+
import type { ToolErrorResponse } from '../config/types/tools.js';
|
|
2
2
|
export declare function createToolErrorResponse(message: string, url: string, code: string): ToolErrorResponse;
|
|
3
3
|
export declare function handleToolError(error: unknown, url: string, fallbackMessage?: string): ToolErrorResponse;
|
|
4
|
-
//# sourceMappingURL=tool-error-handler.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"tool-error-handler.d.ts","sourceRoot":"","sources":["../../src/utils/tool-error-handler.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,
|
|
1
|
+
{"version":3,"file":"tool-error-handler.d.ts","sourceRoot":"","sources":["../../src/utils/tool-error-handler.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,0BAA0B,CAAC;AA6BlE,wBAAgB,uBAAuB,CACrC,OAAO,EAAE,MAAM,EACf,GAAG,EAAE,MAAM,EACX,IAAI,EAAE,MAAM,GACX,iBAAiB,CAanB;AAgBD,wBAAgB,eAAe,CAC7B,KAAK,EAAE,OAAO,EACd,GAAG,EAAE,MAAM,EACX,eAAe,SAAqB,GACnC,iBAAiB,CAoBnB"}
|
|
@@ -1,4 +1,2 @@
|
|
|
1
1
|
export declare function isBlockedIp(ip: string): boolean;
|
|
2
|
-
export declare function validateAndNormalizeUrl(urlString: string): string
|
|
3
|
-
export declare function isInternalUrl(url: string, baseUrl: string): boolean;
|
|
4
|
-
//# sourceMappingURL=url-validator.d.ts.map
|
|
2
|
+
export declare function validateAndNormalizeUrl(urlString: string): Promise<string>;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"url-validator.d.ts","sourceRoot":"","sources":["../../src/utils/url-validator.ts"],"names":[],"mappings":"AA8CA,wBAAgB,WAAW,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAS/C;AAcD,wBAAgB,uBAAuB,CAAC,SAAS,EAAE,MAAM,GAAG,MAAM,CAYjE
|
|
1
|
+
{"version":3,"file":"url-validator.d.ts","sourceRoot":"","sources":["../../src/utils/url-validator.ts"],"names":[],"mappings":"AA8CA,wBAAgB,WAAW,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAS/C;AAcD,wBAAgB,uBAAuB,CAAC,SAAS,EAAE,MAAM,GAAG,MAAM,CAYjE"}
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { lookup } from 'node:dns/promises';
|
|
1
2
|
import { BlockList, isIP } from 'node:net';
|
|
2
3
|
import { config } from '../config/index.js';
|
|
3
4
|
import { createErrorWithCode } from './error-utils.js';
|
|
@@ -30,6 +31,7 @@ for (const entry of BLOCKED_IPV4_SUBNETS) {
|
|
|
30
31
|
for (const entry of BLOCKED_IPV6_SUBNETS) {
|
|
31
32
|
BLOCK_LIST.addSubnet(entry.subnet, entry.prefix, 'ipv6');
|
|
32
33
|
}
|
|
34
|
+
const DNS_LOOKUP_TIMEOUT_MS = 5000;
|
|
33
35
|
function matchesBlockedIpPatterns(resolvedIp) {
|
|
34
36
|
for (const pattern of config.security.blockedIpPatterns) {
|
|
35
37
|
if (pattern.test(resolvedIp)) {
|
|
@@ -60,7 +62,47 @@ function isBlockedByList(ip, ipType) {
|
|
|
60
62
|
}
|
|
61
63
|
return BLOCK_LIST.check(ip, 'ipv6');
|
|
62
64
|
}
|
|
63
|
-
|
|
65
|
+
function lookupWithTimeout(hostname) {
|
|
66
|
+
return new Promise((resolve, reject) => {
|
|
67
|
+
const timer = setTimeout(() => {
|
|
68
|
+
reject(createValidationError(`DNS lookup timed out for ${hostname}`));
|
|
69
|
+
}, DNS_LOOKUP_TIMEOUT_MS);
|
|
70
|
+
lookup(hostname, { all: true })
|
|
71
|
+
.then((result) => {
|
|
72
|
+
clearTimeout(timer);
|
|
73
|
+
resolve(result);
|
|
74
|
+
})
|
|
75
|
+
.catch((error) => {
|
|
76
|
+
clearTimeout(timer);
|
|
77
|
+
reject(error instanceof Error ? error : createValidationError(String(error)));
|
|
78
|
+
});
|
|
79
|
+
});
|
|
80
|
+
}
|
|
81
|
+
async function assertResolvedAddressesAllowed(hostname) {
|
|
82
|
+
try {
|
|
83
|
+
const result = await lookupWithTimeout(hostname);
|
|
84
|
+
const addresses = Array.isArray(result) ? result : [result];
|
|
85
|
+
if (addresses.length === 0) {
|
|
86
|
+
throw createValidationError(`Unable to resolve hostname: ${hostname}`);
|
|
87
|
+
}
|
|
88
|
+
for (const { address } of addresses) {
|
|
89
|
+
if (isBlockedIp(address.toLowerCase())) {
|
|
90
|
+
throw createValidationError(`Blocked IP range resolved from hostname: ${hostname}`);
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
catch (error) {
|
|
95
|
+
const code = error?.code;
|
|
96
|
+
if (code === 'ENOTFOUND' || code === 'EAI_AGAIN') {
|
|
97
|
+
throw createValidationError(`Unable to resolve hostname: ${hostname}`);
|
|
98
|
+
}
|
|
99
|
+
if (error instanceof Error) {
|
|
100
|
+
throw error;
|
|
101
|
+
}
|
|
102
|
+
throw createValidationError(String(error));
|
|
103
|
+
}
|
|
104
|
+
}
|
|
105
|
+
export async function validateAndNormalizeUrl(urlString) {
|
|
64
106
|
const trimmedUrl = requireTrimmedUrl(urlString);
|
|
65
107
|
assertUrlLength(trimmedUrl);
|
|
66
108
|
const url = parseUrl(trimmedUrl);
|
|
@@ -68,6 +110,7 @@ export function validateAndNormalizeUrl(urlString) {
|
|
|
68
110
|
assertNoCredentials(url);
|
|
69
111
|
const hostname = normalizeHostname(url);
|
|
70
112
|
assertHostnameAllowed(hostname);
|
|
113
|
+
await assertResolvedAddressesAllowed(hostname);
|
|
71
114
|
return url.href;
|
|
72
115
|
}
|
|
73
116
|
const VALIDATION_ERROR_CODE = 'VALIDATION_ERROR';
|
|
@@ -127,12 +170,3 @@ function assertHostnameAllowed(hostname) {
|
|
|
127
170
|
function matchesBlockedSuffix(hostname) {
|
|
128
171
|
return BLOCKED_HOST_SUFFIXES.some((suffix) => hostname.endsWith(suffix));
|
|
129
172
|
}
|
|
130
|
-
export function isInternalUrl(url, baseUrl) {
|
|
131
|
-
if (!URL.canParse(baseUrl) || !URL.canParse(url, baseUrl)) {
|
|
132
|
-
return false;
|
|
133
|
-
}
|
|
134
|
-
const urlObj = new URL(url, baseUrl);
|
|
135
|
-
const baseUrlObj = new URL(baseUrl);
|
|
136
|
-
return urlObj.hostname === baseUrlObj.hostname;
|
|
137
|
-
}
|
|
138
|
-
//# sourceMappingURL=url-validator.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"url-validator.js","sourceRoot":"","sources":["../../src/utils/url-validator.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,IAAI,EAAE,MAAM,UAAU,CAAC;AAE3C,OAAO,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAE5C,OAAO,EAAE,mBAAmB,EAAE,MAAM,kBAAkB,CAAC;AAEvD,MAAM,UAAU,GAAG,IAAI,SAAS,EAAE,CAAC;AACnC,MAAM,oBAAoB,GAAG;IAC3B,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,CAAC,EAAE;IAChC,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,EAAE,CAAC,EAAE;IACjC,EAAE,MAAM,EAAE,YAAY,EAAE,MAAM,EAAE,EAAE,EAAE;IACpC,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,EAAE,CAAC,EAAE;IAClC,EAAE,MAAM,EAAE,aAAa,EAAE,MAAM,EAAE,EAAE,EAAE;IACrC,EAAE,MAAM,EAAE,YAAY,EAAE,MAAM,EAAE,EAAE,EAAE;IACpC,EAAE,MAAM,EAAE,aAAa,EAAE,MAAM,EAAE,EAAE,EAAE;IACrC,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,EAAE,CAAC,EAAE;IAClC,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,EAAE,CAAC,EAAE;CAC1B,CAAC;AACX,MAAM,oBAAoB,GAAG;IAC3B,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE;IAC7B,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE;IAC9B,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,EAAE,EAAE,EAAE;IACnC,EAAE,MAAM,EAAE,aAAa,EAAE,MAAM,EAAE,EAAE,EAAE;IACrC,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,EAAE,EAAE;IAChC,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,EAAE,EAAE;IAChC,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC,EAAE;IAC/B,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,EAAE,EAAE;IAChC,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC,EAAE;CACvB,CAAC;AAEX,KAAK,MAAM,KAAK,IAAI,oBAAoB,EAAE,CAAC;IACzC,UAAU,CAAC,SAAS,CAAC,KAAK,CAAC,MAAM,EAAE,KAAK,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;AAC3D,CAAC;AACD,KAAK,MAAM,KAAK,IAAI,oBAAoB,EAAE,CAAC;IACzC,UAAU,CAAC,SAAS,CAAC,KAAK,CAAC,MAAM,EAAE,KAAK,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;AAC3D,CAAC;AAED,SAAS,wBAAwB,CAAC,UAAkB;IAClD,KAAK,MAAM,OAAO,IAAI,MAAM,CAAC,QAAQ,CAAC,iBAAiB,EAAE,CAAC;QACxD,IAAI,OAAO,CAAC,IAAI,CAAC,UAAU,CAAC,EAAE,CAAC;YAC7B,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,MAAM,UAAU,WAAW,CAAC,EAAU;IACpC,IAAI,MAAM,CAAC,QAAQ,CAAC,YAAY,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,CAAC;QACzC,OAAO,IAAI,CAAC;IACd,CAAC;IACD,MAAM,MAAM,GAAG,aAAa,CAAC,EAAE,CAAC,CAAC;IACjC,IAAI,CAAC,MAAM;QAAE,OAAO,KAAK,CAAC;IAC1B,MAAM,YAAY,GAAG,EAAE,CAAC,WAAW,EAAE,CAAC;IACtC,IAAI,eAAe,CAAC,YAAY,EAAE,MAAM,CAAC;QAAE,OAAO,IAAI,CAAC;IACvD,OAAO,wBAAwB,CAAC,YAAY,CAAC,CAAC;AAChD,CAAC;AAED,SAAS,aAAa,CAAC,EAAU;IAC/B,MAAM,MAAM,GAAG,IAAI,CAAC,EAAE,CAAC,CAAC;IACxB,OAAO,MAAM,KAAK,CAAC,IAAI,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC;AACtD,CAAC;AAED,SAAS,eAAe,CAAC,EAAU,EAAE,MAAa;IAChD,IAAI,MAAM,KAAK,CAAC,EAAE,CAAC;QACjB,OAAO,UAAU,CAAC,KAAK,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC;IACtC,CAAC;IACD,OAAO,UAAU,CAAC,KAAK,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC;AACtC,CAAC;AAED,MAAM,UAAU,uBAAuB,CAAC,SAAiB;IACvD,MAAM,UAAU,GAAG,iBAAiB,CAAC,SAAS,CAAC,CAAC;IAChD,eAAe,CAAC,UAAU,CAAC,CAAC;IAE5B,MAAM,GAAG,GAAG,QAAQ,CAAC,UAAU,CAAC,CAAC;IACjC,kBAAkB,CAAC,GAAG,CAAC,CAAC;IACxB,mBAAmB,CAAC,GAAG,CAAC,CAAC;IAEzB,MAAM,QAAQ,GAAG,iBAAiB,CAAC,GAAG,CAAC,CAAC;IACxC,qBAAqB,CAAC,QAAQ,CAAC,CAAC;IAEhC,OAAO,GAAG,CAAC,IAAI,CAAC;AAClB,CAAC;AAED,MAAM,qBAAqB,GAAG,kBAAkB,CAAC;AAEjD,SAAS,qBAAqB,CAAC,OAAe;IAC5C,OAAO,mBAAmB,CAAC,OAAO,EAAE,qBAAqB,CAAC,CAAC;AAC7D,CAAC;AAED,SAAS,iBAAiB,CAAC,SAAiB;IAC1C,IAAI,CAAC,SAAS,IAAI,OAAO,SAAS,KAAK,QAAQ,EAAE,CAAC;QAChD,MAAM,qBAAqB,CAAC,iBAAiB,CAAC,CAAC;IACjD,CAAC;IAED,MAAM,UAAU,GAAG,SAAS,CAAC,IAAI,EAAE,CAAC;IACpC,IAAI,CAAC,UAAU,EAAE,CAAC;QAChB,MAAM,qBAAqB,CAAC,qBAAqB,CAAC,CAAC;IACrD,CAAC;IAED,OAAO,UAAU,CAAC;AACpB,CAAC;AAED,SAAS,eAAe,CAAC,GAAW;IAClC,IAAI,GAAG,CAAC,MAAM,IAAI,MAAM,CAAC,SAAS,CAAC,YAAY;QAAE,OAAO;IACxD,MAAM,qBAAqB,CACzB,iCAAiC,MAAM,CAAC,SAAS,CAAC,YAAY,aAAa,CAC5E,CAAC;AACJ,CAAC;AAED,SAAS,QAAQ,CAAC,SAAiB;IACjC,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE,CAAC;QAC7B,MAAM,qBAAqB,CAAC,oBAAoB,CAAC,CAAC;IACpD,CAAC;IACD,OAAO,IAAI,GAAG,CAAC,SAAS,CAAC,CAAC;AAC5B,CAAC;AAED,SAAS,kBAAkB,CAAC,GAAQ;IAClC,IAAI,GAAG,CAAC,QAAQ,KAAK,OAAO,IAAI,GAAG,CAAC,QAAQ,KAAK,QAAQ;QAAE,OAAO;IAClE,MAAM,qBAAqB,CACzB,qBAAqB,GAAG,CAAC,QAAQ,qCAAqC,CACvE,CAAC;AACJ,CAAC;AAED,SAAS,mBAAmB,CAAC,GAAQ;IACnC,IAAI,CAAC,GAAG,CAAC,QAAQ,IAAI,CAAC,GAAG,CAAC,QAAQ;QAAE,OAAO;IAC3C,MAAM,qBAAqB,CAAC,gDAAgD,CAAC,CAAC;AAChF,CAAC;AAED,SAAS,iBAAiB,CAAC,GAAQ;IACjC,MAAM,QAAQ,GAAG,GAAG,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC;IAC5C,IAAI,CAAC,QAAQ,EAAE,CAAC;QACd,MAAM,qBAAqB,CAAC,gCAAgC,CAAC,CAAC;IAChE,CAAC;IACD,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED,MAAM,qBAAqB,GAAG,CAAC,QAAQ,EAAE,WAAW,CAAU,CAAC;AAE/D,SAAS,qBAAqB,CAAC,QAAgB;IAC7C,IAAI,MAAM,CAAC,QAAQ,CAAC,YAAY,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC;QAC/C,MAAM,qBAAqB,CACzB,iBAAiB,QAAQ,kCAAkC,CAC5D,CAAC;IACJ,CAAC;IAED,IAAI,WAAW,CAAC,QAAQ,CAAC,EAAE,CAAC;QAC1B,MAAM,qBAAqB,CACzB,qBAAqB,QAAQ,+BAA+B,CAC7D,CAAC;IACJ,CAAC;IAED,IAAI,oBAAoB,CAAC,QAAQ,CAAC,EAAE,CAAC;QACnC,MAAM,qBAAqB,CACzB,6BAA6B,QAAQ,4CAA4C,CAClF,CAAC;IACJ,CAAC;AACH,CAAC;AAED,SAAS,oBAAoB,CAAC,QAAgB;IAC5C,OAAO,qBAAqB,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,QAAQ,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC;AAC3E,CAAC
|
|
1
|
+
{"version":3,"file":"url-validator.js","sourceRoot":"","sources":["../../src/utils/url-validator.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,IAAI,EAAE,MAAM,UAAU,CAAC;AAE3C,OAAO,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAE5C,OAAO,EAAE,mBAAmB,EAAE,MAAM,kBAAkB,CAAC;AAEvD,MAAM,UAAU,GAAG,IAAI,SAAS,EAAE,CAAC;AACnC,MAAM,oBAAoB,GAAG;IAC3B,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,CAAC,EAAE;IAChC,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,EAAE,CAAC,EAAE;IACjC,EAAE,MAAM,EAAE,YAAY,EAAE,MAAM,EAAE,EAAE,EAAE;IACpC,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,EAAE,CAAC,EAAE;IAClC,EAAE,MAAM,EAAE,aAAa,EAAE,MAAM,EAAE,EAAE,EAAE;IACrC,EAAE,MAAM,EAAE,YAAY,EAAE,MAAM,EAAE,EAAE,EAAE;IACpC,EAAE,MAAM,EAAE,aAAa,EAAE,MAAM,EAAE,EAAE,EAAE;IACrC,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,EAAE,CAAC,EAAE;IAClC,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,EAAE,CAAC,EAAE;CAC1B,CAAC;AACX,MAAM,oBAAoB,GAAG;IAC3B,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE;IAC7B,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE;IAC9B,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,EAAE,EAAE,EAAE;IACnC,EAAE,MAAM,EAAE,aAAa,EAAE,MAAM,EAAE,EAAE,EAAE;IACrC,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,EAAE,EAAE;IAChC,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,EAAE,EAAE;IAChC,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC,EAAE;IAC/B,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,EAAE,EAAE;IAChC,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC,EAAE;CACvB,CAAC;AAEX,KAAK,MAAM,KAAK,IAAI,oBAAoB,EAAE,CAAC;IACzC,UAAU,CAAC,SAAS,CAAC,KAAK,CAAC,MAAM,EAAE,KAAK,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;AAC3D,CAAC;AACD,KAAK,MAAM,KAAK,IAAI,oBAAoB,EAAE,CAAC;IACzC,UAAU,CAAC,SAAS,CAAC,KAAK,CAAC,MAAM,EAAE,KAAK,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;AAC3D,CAAC;AAED,SAAS,wBAAwB,CAAC,UAAkB;IAClD,KAAK,MAAM,OAAO,IAAI,MAAM,CAAC,QAAQ,CAAC,iBAAiB,EAAE,CAAC;QACxD,IAAI,OAAO,CAAC,IAAI,CAAC,UAAU,CAAC,EAAE,CAAC;YAC7B,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,MAAM,UAAU,WAAW,CAAC,EAAU;IACpC,IAAI,MAAM,CAAC,QAAQ,CAAC,YAAY,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,CAAC;QACzC,OAAO,IAAI,CAAC;IACd,CAAC;IACD,MAAM,MAAM,GAAG,aAAa,CAAC,EAAE,CAAC,CAAC;IACjC,IAAI,CAAC,MAAM;QAAE,OAAO,KAAK,CAAC;IAC1B,MAAM,YAAY,GAAG,EAAE,CAAC,WAAW,EAAE,CAAC;IACtC,IAAI,eAAe,CAAC,YAAY,EAAE,MAAM,CAAC;QAAE,OAAO,IAAI,CAAC;IACvD,OAAO,wBAAwB,CAAC,YAAY,CAAC,CAAC;AAChD,CAAC;AAED,SAAS,aAAa,CAAC,EAAU;IAC/B,MAAM,MAAM,GAAG,IAAI,CAAC,EAAE,CAAC,CAAC;IACxB,OAAO,MAAM,KAAK,CAAC,IAAI,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC;AACtD,CAAC;AAED,SAAS,eAAe,CAAC,EAAU,EAAE,MAAa;IAChD,IAAI,MAAM,KAAK,CAAC,EAAE,CAAC;QACjB,OAAO,UAAU,CAAC,KAAK,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC;IACtC,CAAC;IACD,OAAO,UAAU,CAAC,KAAK,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC;AACtC,CAAC;AAED,MAAM,UAAU,uBAAuB,CAAC,SAAiB;IACvD,MAAM,UAAU,GAAG,iBAAiB,CAAC,SAAS,CAAC,CAAC;IAChD,eAAe,CAAC,UAAU,CAAC,CAAC;IAE5B,MAAM,GAAG,GAAG,QAAQ,CAAC,UAAU,CAAC,CAAC;IACjC,kBAAkB,CAAC,GAAG,CAAC,CAAC;IACxB,mBAAmB,CAAC,GAAG,CAAC,CAAC;IAEzB,MAAM,QAAQ,GAAG,iBAAiB,CAAC,GAAG,CAAC,CAAC;IACxC,qBAAqB,CAAC,QAAQ,CAAC,CAAC;IAEhC,OAAO,GAAG,CAAC,IAAI,CAAC;AAClB,CAAC;AAED,MAAM,qBAAqB,GAAG,kBAAkB,CAAC;AAEjD,SAAS,qBAAqB,CAAC,OAAe;IAC5C,OAAO,mBAAmB,CAAC,OAAO,EAAE,qBAAqB,CAAC,CAAC;AAC7D,CAAC;AAED,SAAS,iBAAiB,CAAC,SAAiB;IAC1C,IAAI,CAAC,SAAS,IAAI,OAAO,SAAS,KAAK,QAAQ,EAAE,CAAC;QAChD,MAAM,qBAAqB,CAAC,iBAAiB,CAAC,CAAC;IACjD,CAAC;IAED,MAAM,UAAU,GAAG,SAAS,CAAC,IAAI,EAAE,CAAC;IACpC,IAAI,CAAC,UAAU,EAAE,CAAC;QAChB,MAAM,qBAAqB,CAAC,qBAAqB,CAAC,CAAC;IACrD,CAAC;IAED,OAAO,UAAU,CAAC;AACpB,CAAC;AAED,SAAS,eAAe,CAAC,GAAW;IAClC,IAAI,GAAG,CAAC,MAAM,IAAI,MAAM,CAAC,SAAS,CAAC,YAAY;QAAE,OAAO;IACxD,MAAM,qBAAqB,CACzB,iCAAiC,MAAM,CAAC,SAAS,CAAC,YAAY,aAAa,CAC5E,CAAC;AACJ,CAAC;AAED,SAAS,QAAQ,CAAC,SAAiB;IACjC,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE,CAAC;QAC7B,MAAM,qBAAqB,CAAC,oBAAoB,CAAC,CAAC;IACpD,CAAC;IACD,OAAO,IAAI,GAAG,CAAC,SAAS,CAAC,CAAC;AAC5B,CAAC;AAED,SAAS,kBAAkB,CAAC,GAAQ;IAClC,IAAI,GAAG,CAAC,QAAQ,KAAK,OAAO,IAAI,GAAG,CAAC,QAAQ,KAAK,QAAQ;QAAE,OAAO;IAClE,MAAM,qBAAqB,CACzB,qBAAqB,GAAG,CAAC,QAAQ,qCAAqC,CACvE,CAAC;AACJ,CAAC;AAED,SAAS,mBAAmB,CAAC,GAAQ;IACnC,IAAI,CAAC,GAAG,CAAC,QAAQ,IAAI,CAAC,GAAG,CAAC,QAAQ;QAAE,OAAO;IAC3C,MAAM,qBAAqB,CAAC,gDAAgD,CAAC,CAAC;AAChF,CAAC;AAED,SAAS,iBAAiB,CAAC,GAAQ;IACjC,MAAM,QAAQ,GAAG,GAAG,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC;IAC5C,IAAI,CAAC,QAAQ,EAAE,CAAC;QACd,MAAM,qBAAqB,CAAC,gCAAgC,CAAC,CAAC;IAChE,CAAC;IACD,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED,MAAM,qBAAqB,GAAG,CAAC,QAAQ,EAAE,WAAW,CAAU,CAAC;AAE/D,SAAS,qBAAqB,CAAC,QAAgB;IAC7C,IAAI,MAAM,CAAC,QAAQ,CAAC,YAAY,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC;QAC/C,MAAM,qBAAqB,CACzB,iBAAiB,QAAQ,kCAAkC,CAC5D,CAAC;IACJ,CAAC;IAED,IAAI,WAAW,CAAC,QAAQ,CAAC,EAAE,CAAC;QAC1B,MAAM,qBAAqB,CACzB,qBAAqB,QAAQ,+BAA+B,CAC7D,CAAC;IACJ,CAAC;IAED,IAAI,oBAAoB,CAAC,QAAQ,CAAC,EAAE,CAAC;QACnC,MAAM,qBAAqB,CACzB,6BAA6B,QAAQ,4CAA4C,CAClF,CAAC;IACJ,CAAC;AACH,CAAC;AAED,SAAS,oBAAoB,CAAC,QAAgB;IAC5C,OAAO,qBAAqB,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,QAAQ,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC;AAC3E,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@j0hanz/superfetch",
|
|
3
|
-
"version": "1.2.
|
|
3
|
+
"version": "1.2.1",
|
|
4
4
|
"mcpName": "io.github.j0hanz/superfetch",
|
|
5
5
|
"description": "Intelligent web content fetcher MCP server that converts HTML to clean, AI-readable JSONL format",
|
|
6
6
|
"type": "module",
|
|
@@ -35,7 +35,7 @@
|
|
|
35
35
|
],
|
|
36
36
|
"scripts": {
|
|
37
37
|
"dev": "tsx watch src/index.ts",
|
|
38
|
-
"build": "tsc && shx chmod +x dist/*.js",
|
|
38
|
+
"build": "tsc -p tsconfig.build.json && shx chmod +x dist/*.js",
|
|
39
39
|
"prepare": "npm run build",
|
|
40
40
|
"prepublishOnly": "npm run build && npm run lint",
|
|
41
41
|
"start": "node dist/index.js",
|
|
@@ -71,12 +71,12 @@
|
|
|
71
71
|
"eslint": "^9.23.2",
|
|
72
72
|
"eslint-config-prettier": "^10.1.8",
|
|
73
73
|
"eslint-plugin-unused-imports": "^4.3.0",
|
|
74
|
-
"knip": "^5.
|
|
74
|
+
"knip": "^5.78.0",
|
|
75
75
|
"prettier": "^3.7.4",
|
|
76
76
|
"shx": "^0.4.0",
|
|
77
77
|
"tsx": "^4.21.0",
|
|
78
78
|
"typescript": "^5.9.3",
|
|
79
|
-
"typescript-eslint": "^8.
|
|
79
|
+
"typescript-eslint": "^8.51.0",
|
|
80
80
|
"vitest": "^2.1.9"
|
|
81
81
|
},
|
|
82
82
|
"engines": {
|