@j0hanz/superfetch 1.0.2 → 1.0.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +590 -327
- package/dist/config/index.d.ts.map +1 -1
- package/dist/config/index.js +6 -10
- package/dist/config/index.js.map +1 -1
- package/dist/config/types.d.ts +251 -0
- package/dist/config/types.d.ts.map +1 -0
- package/dist/config/types.js +2 -0
- package/dist/config/types.js.map +1 -0
- package/dist/errors/app-error.d.ts +2 -20
- package/dist/errors/app-error.d.ts.map +1 -1
- package/dist/errors/app-error.js +0 -18
- package/dist/errors/app-error.js.map +1 -1
- package/dist/index.js +13 -47
- package/dist/index.js.map +1 -1
- package/dist/middleware/error-handler.d.ts +1 -5
- package/dist/middleware/error-handler.d.ts.map +1 -1
- package/dist/middleware/error-handler.js +1 -11
- package/dist/middleware/error-handler.js.map +1 -1
- package/dist/middleware/rate-limiter.d.ts +2 -20
- package/dist/middleware/rate-limiter.d.ts.map +1 -1
- package/dist/middleware/rate-limiter.js +11 -44
- package/dist/middleware/rate-limiter.js.map +1 -1
- package/dist/prompts/index.d.ts +0 -3
- package/dist/prompts/index.d.ts.map +1 -1
- package/dist/prompts/index.js +0 -3
- package/dist/prompts/index.js.map +1 -1
- package/dist/resources/index.d.ts +0 -3
- package/dist/resources/index.d.ts.map +1 -1
- package/dist/resources/index.js +1 -4
- package/dist/resources/index.js.map +1 -1
- package/dist/server.d.ts +0 -4
- package/dist/server.d.ts.map +1 -1
- package/dist/server.js +2 -6
- package/dist/server.js.map +1 -1
- package/dist/services/cache.d.ts +9 -6
- package/dist/services/cache.d.ts.map +1 -1
- package/dist/services/cache.js +71 -20
- package/dist/services/cache.js.map +1 -1
- package/dist/services/card-extractor.d.ts +10 -0
- package/dist/services/card-extractor.d.ts.map +1 -0
- package/dist/services/card-extractor.js +187 -0
- package/dist/services/card-extractor.js.map +1 -0
- package/dist/services/extractor.d.ts +6 -19
- package/dist/services/extractor.d.ts.map +1 -1
- package/dist/services/extractor.js +53 -46
- package/dist/services/extractor.js.map +1 -1
- package/dist/services/fetcher.d.ts +4 -11
- package/dist/services/fetcher.d.ts.map +1 -1
- package/dist/services/fetcher.js +30 -36
- package/dist/services/fetcher.js.map +1 -1
- package/dist/services/logger.d.ts.map +1 -1
- package/dist/services/logger.js +4 -6
- package/dist/services/logger.js.map +1 -1
- package/dist/services/parser.d.ts +1 -6
- package/dist/services/parser.d.ts.map +1 -1
- package/dist/services/parser.js +64 -47
- package/dist/services/parser.js.map +1 -1
- package/dist/tools/handlers/fetch-links.tool.d.ts +5 -12
- package/dist/tools/handlers/fetch-links.tool.d.ts.map +1 -1
- package/dist/tools/handlers/fetch-links.tool.js +104 -79
- package/dist/tools/handlers/fetch-links.tool.js.map +1 -1
- package/dist/tools/handlers/fetch-markdown.tool.d.ts +7 -4
- package/dist/tools/handlers/fetch-markdown.tool.d.ts.map +1 -1
- package/dist/tools/handlers/fetch-markdown.tool.js +84 -84
- package/dist/tools/handlers/fetch-markdown.tool.js.map +1 -1
- package/dist/tools/handlers/fetch-url.tool.d.ts +8 -6
- package/dist/tools/handlers/fetch-url.tool.d.ts.map +1 -1
- package/dist/tools/handlers/fetch-url.tool.js +51 -93
- package/dist/tools/handlers/fetch-url.tool.js.map +1 -1
- package/dist/tools/handlers/fetch-urls.tool.d.ts +5 -0
- package/dist/tools/handlers/fetch-urls.tool.d.ts.map +1 -0
- package/dist/tools/handlers/fetch-urls.tool.js +147 -0
- package/dist/tools/handlers/fetch-urls.tool.js.map +1 -0
- package/dist/tools/index.d.ts +0 -4
- package/dist/tools/index.d.ts.map +1 -1
- package/dist/tools/index.js +145 -15
- package/dist/tools/index.js.map +1 -1
- package/dist/tools/utils/common.d.ts +8 -0
- package/dist/tools/utils/common.d.ts.map +1 -0
- package/dist/tools/utils/common.js +35 -0
- package/dist/tools/utils/common.js.map +1 -0
- package/dist/tools/utils/fetch-pipeline.d.ts +3 -0
- package/dist/tools/utils/fetch-pipeline.d.ts.map +1 -0
- package/dist/tools/utils/fetch-pipeline.js +37 -0
- package/dist/tools/utils/fetch-pipeline.js.map +1 -0
- package/dist/tools/utils/index.d.ts +4 -0
- package/dist/tools/utils/index.d.ts.map +1 -0
- package/dist/tools/utils/index.js +3 -0
- package/dist/tools/utils/index.js.map +1 -0
- package/dist/tools/utils/response-builder.d.ts +3 -0
- package/dist/tools/utils/response-builder.d.ts.map +1 -0
- package/dist/tools/utils/response-builder.js +24 -0
- package/dist/tools/utils/response-builder.js.map +1 -0
- package/dist/transformers/jsonl.transformer.d.ts +1 -1
- package/dist/transformers/jsonl.transformer.d.ts.map +1 -1
- package/dist/transformers/jsonl.transformer.js +2 -1
- package/dist/transformers/jsonl.transformer.js.map +1 -1
- package/dist/transformers/markdown.transformer.d.ts +1 -1
- package/dist/transformers/markdown.transformer.d.ts.map +1 -1
- package/dist/transformers/markdown.transformer.js +116 -2
- package/dist/transformers/markdown.transformer.js.map +1 -1
- package/dist/types/content.types.d.ts +11 -11
- package/dist/types/content.types.d.ts.map +1 -1
- package/dist/types/index.d.ts +1 -2
- package/dist/types/index.d.ts.map +1 -1
- package/dist/types/index.js +1 -2
- package/dist/types/index.js.map +1 -1
- package/dist/types/schemas.d.ts +39 -12
- package/dist/types/schemas.d.ts.map +1 -1
- package/dist/utils/concurrency.d.ts +2 -0
- package/dist/utils/concurrency.d.ts.map +1 -0
- package/dist/utils/concurrency.js +25 -0
- package/dist/utils/concurrency.js.map +1 -0
- package/dist/utils/content-cleaner.d.ts +32 -0
- package/dist/utils/content-cleaner.d.ts.map +1 -0
- package/dist/utils/content-cleaner.js +240 -0
- package/dist/utils/content-cleaner.js.map +1 -0
- package/dist/utils/language-detector.d.ts +5 -0
- package/dist/utils/language-detector.d.ts.map +1 -0
- package/dist/utils/language-detector.js +50 -0
- package/dist/utils/language-detector.js.map +1 -0
- package/dist/utils/sanitizer.d.ts +0 -10
- package/dist/utils/sanitizer.d.ts.map +1 -1
- package/dist/utils/sanitizer.js +3 -11
- package/dist/utils/sanitizer.js.map +1 -1
- package/dist/utils/tool-error-handler.d.ts +1 -15
- package/dist/utils/tool-error-handler.d.ts.map +1 -1
- package/dist/utils/tool-error-handler.js +1 -1
- package/dist/utils/tool-error-handler.js.map +1 -1
- package/dist/utils/url-validator.d.ts +0 -8
- package/dist/utils/url-validator.d.ts.map +1 -1
- package/dist/utils/url-validator.js +17 -31
- package/dist/utils/url-validator.js.map +1 -1
- package/package.json +4 -3
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/config/index.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/config/index.ts"],"names":[],"mappings":"AAYA,eAAO,MAAM,MAAM;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAsCT,CAAC"}
|
package/dist/config/index.js
CHANGED
|
@@ -1,7 +1,3 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Safely parses an integer from environment variable with validation
|
|
3
|
-
* Environment variables can still be used for deployment overrides (e.g., PORT=8080 npm start)
|
|
4
|
-
*/
|
|
5
1
|
function parseIntEnv(value, defaultValue, min = 0, max = Number.MAX_SAFE_INTEGER) {
|
|
6
2
|
if (!value)
|
|
7
3
|
return defaultValue;
|
|
@@ -15,17 +11,17 @@ export const config = {
|
|
|
15
11
|
name: 'superFetch',
|
|
16
12
|
version: '1.0.0',
|
|
17
13
|
port: parseIntEnv(process.env.PORT, 3000, 1, 65535),
|
|
18
|
-
host: process.env.HOST
|
|
14
|
+
host: process.env.HOST ?? '127.0.0.1',
|
|
19
15
|
},
|
|
20
16
|
fetcher: {
|
|
21
|
-
timeout: parseIntEnv(process.env.FETCH_TIMEOUT, 30000, 1000, 120000),
|
|
17
|
+
timeout: parseIntEnv(process.env.FETCH_TIMEOUT, 30000, 1000, 120000),
|
|
22
18
|
maxRedirects: parseIntEnv(process.env.MAX_REDIRECTS, 5, 0, 20),
|
|
23
|
-
userAgent: process.env.USER_AGENT
|
|
24
|
-
maxContentLength: parseIntEnv(process.env.MAX_CONTENT_LENGTH, 10485760, 1024, 52428800),
|
|
19
|
+
userAgent: process.env.USER_AGENT ?? 'superFetch-MCP/1.0',
|
|
20
|
+
maxContentLength: parseIntEnv(process.env.MAX_CONTENT_LENGTH, 10485760, 1024, 52428800),
|
|
25
21
|
},
|
|
26
22
|
cache: {
|
|
27
23
|
enabled: process.env.CACHE_ENABLED !== 'false',
|
|
28
|
-
ttl: parseIntEnv(process.env.CACHE_TTL, 3600, 60, 86400),
|
|
24
|
+
ttl: parseIntEnv(process.env.CACHE_TTL, 3600, 60, 86400),
|
|
29
25
|
maxKeys: parseIntEnv(process.env.CACHE_MAX_KEYS, 100, 10, 10000),
|
|
30
26
|
},
|
|
31
27
|
extraction: {
|
|
@@ -35,7 +31,7 @@ export const config = {
|
|
|
35
31
|
minParagraphLength: parseIntEnv(process.env.MIN_PARAGRAPH_LENGTH, 10, 0, 1000),
|
|
36
32
|
},
|
|
37
33
|
logging: {
|
|
38
|
-
level: process.env.LOG_LEVEL
|
|
34
|
+
level: process.env.LOG_LEVEL ?? 'info',
|
|
39
35
|
enabled: process.env.ENABLE_LOGGING !== 'false',
|
|
40
36
|
},
|
|
41
37
|
};
|
package/dist/config/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/config/index.ts"],"names":[],"mappings":"AAAA
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/config/index.ts"],"names":[],"mappings":"AAAA,SAAS,WAAW,CAClB,KAAyB,EACzB,YAAoB,EACpB,GAAG,GAAG,CAAC,EACP,GAAG,GAAG,MAAM,CAAC,gBAAgB;IAE7B,IAAI,CAAC,KAAK;QAAE,OAAO,YAAY,CAAC;IAChC,MAAM,MAAM,GAAG,QAAQ,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;IACnC,IAAI,KAAK,CAAC,MAAM,CAAC;QAAE,OAAO,YAAY,CAAC;IACvC,OAAO,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC,CAAC;AAC9C,CAAC;AAED,MAAM,CAAC,MAAM,MAAM,GAAG;IACpB,MAAM,EAAE;QACN,IAAI,EAAE,YAAY;QAClB,OAAO,EAAE,OAAO;QAChB,IAAI,EAAE,WAAW,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC,EAAE,KAAK,CAAC;QACnD,IAAI,EAAE,OAAO,CAAC,GAAG,CAAC,IAAI,IAAI,WAAW;KACtC;IACD,OAAO,EAAE;QACP,OAAO,EAAE,WAAW,CAAC,OAAO,CAAC,GAAG,CAAC,aAAa,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,CAAC;QACpE,YAAY,EAAE,WAAW,CAAC,OAAO,CAAC,GAAG,CAAC,aAAa,EAAE,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;QAC9D,SAAS,EAAE,OAAO,CAAC,GAAG,CAAC,UAAU,IAAI,oBAAoB;QACzD,gBAAgB,EAAE,WAAW,CAC3B,OAAO,CAAC,GAAG,CAAC,kBAAkB,EAC9B,QAAQ,EACR,IAAI,EACJ,QAAQ,CACT;KACF;IACD,KAAK,EAAE;QACL,OAAO,EAAE,OAAO,CAAC,GAAG,CAAC,aAAa,KAAK,OAAO;QAC9C,GAAG,EAAE,WAAW,CAAC,OAAO,CAAC,GAAG,CAAC,SAAS,EAAE,IAAI,EAAE,EAAE,EAAE,KAAK,CAAC;QACxD,OAAO,EAAE,WAAW,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,EAAE,GAAG,EAAE,EAAE,EAAE,KAAK,CAAC;KACjE;IACD,UAAU,EAAE;QACV,kBAAkB,EAAE,OAAO,CAAC,GAAG,CAAC,oBAAoB,KAAK,OAAO;QAChE,eAAe,EAAE,OAAO,CAAC,GAAG,CAAC,gBAAgB,KAAK,OAAO;QACzD,cAAc,EAAE,WAAW,CAAC,OAAO,CAAC,GAAG,CAAC,gBAAgB,EAAE,IAAI,EAAE,GAAG,EAAE,KAAK,CAAC;QAC3E,kBAAkB,EAAE,WAAW,CAC7B,OAAO,CAAC,GAAG,CAAC,oBAAoB,EAChC,EAAE,EACF,CAAC,EACD,IAAI,CACL;KACF;IACD,OAAO,EAAE;QACP,KAAK,EAAE,OAAO,CAAC,GAAG,CAAC,SAAS,IAAI,MAAM;QACtC,OAAO,EAAE,OAAO,CAAC,GAAG,CAAC,cAAc,KAAK,OAAO;KAChD;CACO,CAAC"}
|
|
@@ -0,0 +1,251 @@
|
|
|
1
|
+
type ContentBlockType = 'metadata' | 'heading' | 'paragraph' | 'list' | 'code' | 'table' | 'image' | 'blockquote';
|
|
2
|
+
interface ContentBlock {
|
|
3
|
+
type: ContentBlockType;
|
|
4
|
+
}
|
|
5
|
+
export interface MetadataBlock extends ContentBlock {
|
|
6
|
+
type: 'metadata';
|
|
7
|
+
title?: string | undefined;
|
|
8
|
+
description?: string | undefined;
|
|
9
|
+
author?: string | undefined;
|
|
10
|
+
url: string;
|
|
11
|
+
fetchedAt: string;
|
|
12
|
+
}
|
|
13
|
+
export interface HeadingBlock extends ContentBlock {
|
|
14
|
+
type: 'heading';
|
|
15
|
+
level: number;
|
|
16
|
+
text: string;
|
|
17
|
+
}
|
|
18
|
+
export interface ParagraphBlock extends ContentBlock {
|
|
19
|
+
type: 'paragraph';
|
|
20
|
+
text: string;
|
|
21
|
+
}
|
|
22
|
+
export interface ListBlock extends ContentBlock {
|
|
23
|
+
type: 'list';
|
|
24
|
+
ordered: boolean;
|
|
25
|
+
items: string[];
|
|
26
|
+
}
|
|
27
|
+
export interface CodeBlock extends ContentBlock {
|
|
28
|
+
type: 'code';
|
|
29
|
+
language?: string | undefined;
|
|
30
|
+
text: string;
|
|
31
|
+
}
|
|
32
|
+
export interface TableBlock extends ContentBlock {
|
|
33
|
+
type: 'table';
|
|
34
|
+
headers?: string[] | undefined;
|
|
35
|
+
rows: string[][];
|
|
36
|
+
}
|
|
37
|
+
export interface ImageBlock extends ContentBlock {
|
|
38
|
+
type: 'image';
|
|
39
|
+
src: string;
|
|
40
|
+
alt?: string | undefined;
|
|
41
|
+
}
|
|
42
|
+
export interface BlockquoteBlock extends ContentBlock {
|
|
43
|
+
type: 'blockquote';
|
|
44
|
+
text: string;
|
|
45
|
+
}
|
|
46
|
+
export type ContentBlockUnion = MetadataBlock | HeadingBlock | ParagraphBlock | ListBlock | CodeBlock | TableBlock | ImageBlock | BlockquoteBlock;
|
|
47
|
+
export interface ExtractedArticle {
|
|
48
|
+
title?: string | undefined;
|
|
49
|
+
byline?: string | undefined;
|
|
50
|
+
content: string;
|
|
51
|
+
textContent: string;
|
|
52
|
+
excerpt?: string | undefined;
|
|
53
|
+
siteName?: string | undefined;
|
|
54
|
+
}
|
|
55
|
+
export interface CacheEntry {
|
|
56
|
+
url: string;
|
|
57
|
+
content: string;
|
|
58
|
+
fetchedAt: string;
|
|
59
|
+
expiresAt: string;
|
|
60
|
+
}
|
|
61
|
+
export interface ExtractedLink {
|
|
62
|
+
href: string;
|
|
63
|
+
text: string;
|
|
64
|
+
type: 'internal' | 'external' | 'image';
|
|
65
|
+
}
|
|
66
|
+
interface RequestOptions {
|
|
67
|
+
/** Custom HTTP headers for the request */
|
|
68
|
+
customHeaders?: Record<string, string> | undefined;
|
|
69
|
+
/** Request timeout in milliseconds (1000-60000) */
|
|
70
|
+
timeout?: number | undefined;
|
|
71
|
+
/** Number of retry attempts (1-10) */
|
|
72
|
+
retries?: number | undefined;
|
|
73
|
+
}
|
|
74
|
+
export interface FetchUrlInput extends RequestOptions {
|
|
75
|
+
url: string;
|
|
76
|
+
extractMainContent?: boolean | undefined;
|
|
77
|
+
includeMetadata?: boolean | undefined;
|
|
78
|
+
maxContentLength?: number | undefined;
|
|
79
|
+
format?: 'jsonl' | 'markdown' | undefined;
|
|
80
|
+
}
|
|
81
|
+
export interface FetchLinksInput extends RequestOptions {
|
|
82
|
+
url: string;
|
|
83
|
+
includeExternal?: boolean | undefined;
|
|
84
|
+
includeInternal?: boolean | undefined;
|
|
85
|
+
maxLinks?: number | undefined;
|
|
86
|
+
filterPattern?: string | undefined;
|
|
87
|
+
includeImages?: boolean | undefined;
|
|
88
|
+
}
|
|
89
|
+
export interface FetchMarkdownInput extends RequestOptions {
|
|
90
|
+
url: string;
|
|
91
|
+
extractMainContent?: boolean | undefined;
|
|
92
|
+
includeMetadata?: boolean | undefined;
|
|
93
|
+
maxContentLength?: number | undefined;
|
|
94
|
+
generateToc?: boolean | undefined;
|
|
95
|
+
}
|
|
96
|
+
export interface FetchUrlsInput extends RequestOptions {
|
|
97
|
+
urls: string[];
|
|
98
|
+
extractMainContent?: boolean | undefined;
|
|
99
|
+
includeMetadata?: boolean | undefined;
|
|
100
|
+
maxContentLength?: number | undefined;
|
|
101
|
+
format?: 'jsonl' | 'markdown' | undefined;
|
|
102
|
+
concurrency?: number | undefined;
|
|
103
|
+
continueOnError?: boolean | undefined;
|
|
104
|
+
}
|
|
105
|
+
export interface ErrorResponse {
|
|
106
|
+
error: {
|
|
107
|
+
message: string;
|
|
108
|
+
code: string;
|
|
109
|
+
statusCode: number;
|
|
110
|
+
details?: Record<string, unknown> | undefined;
|
|
111
|
+
stack?: string | undefined;
|
|
112
|
+
};
|
|
113
|
+
}
|
|
114
|
+
export interface RateLimitEntry {
|
|
115
|
+
count: number;
|
|
116
|
+
resetTime: number;
|
|
117
|
+
}
|
|
118
|
+
export interface RateLimiterOptions {
|
|
119
|
+
maxRequests: number;
|
|
120
|
+
windowMs: number;
|
|
121
|
+
cleanupIntervalMs: number;
|
|
122
|
+
}
|
|
123
|
+
export interface ExtractedMetadata {
|
|
124
|
+
title?: string | undefined;
|
|
125
|
+
description?: string | undefined;
|
|
126
|
+
author?: string | undefined;
|
|
127
|
+
}
|
|
128
|
+
export interface ExtractionResult {
|
|
129
|
+
article: ExtractedArticle | null;
|
|
130
|
+
metadata: ExtractedMetadata;
|
|
131
|
+
}
|
|
132
|
+
export type ParseableTagName = 'h1' | 'h2' | 'h3' | 'h4' | 'h5' | 'h6' | 'p' | 'ul' | 'ol' | 'pre' | 'code' | 'table' | 'img' | 'blockquote';
|
|
133
|
+
export interface LinksTransformResult {
|
|
134
|
+
links: ExtractedLink[];
|
|
135
|
+
linkCount: number;
|
|
136
|
+
filtered: number;
|
|
137
|
+
truncated: boolean;
|
|
138
|
+
}
|
|
139
|
+
export interface ExtractLinksOptions {
|
|
140
|
+
includeInternal: boolean;
|
|
141
|
+
includeExternal: boolean;
|
|
142
|
+
includeImages: boolean;
|
|
143
|
+
maxLinks?: number | undefined;
|
|
144
|
+
filterPattern?: RegExp | undefined;
|
|
145
|
+
}
|
|
146
|
+
export interface TocEntry {
|
|
147
|
+
level: number;
|
|
148
|
+
text: string;
|
|
149
|
+
slug: string;
|
|
150
|
+
}
|
|
151
|
+
export interface MarkdownTransformResult {
|
|
152
|
+
markdown: string;
|
|
153
|
+
title: string | undefined;
|
|
154
|
+
toc: TocEntry[] | undefined;
|
|
155
|
+
truncated: boolean;
|
|
156
|
+
}
|
|
157
|
+
export interface TransformOptions {
|
|
158
|
+
extractMainContent: boolean;
|
|
159
|
+
includeMetadata: boolean;
|
|
160
|
+
generateToc: boolean;
|
|
161
|
+
maxContentLength?: number | undefined;
|
|
162
|
+
}
|
|
163
|
+
export interface JsonlTransformResult {
|
|
164
|
+
content: string;
|
|
165
|
+
contentBlocks: number;
|
|
166
|
+
title: string | undefined;
|
|
167
|
+
}
|
|
168
|
+
export interface SingleUrlResult {
|
|
169
|
+
url: string;
|
|
170
|
+
success: boolean;
|
|
171
|
+
title?: string | undefined;
|
|
172
|
+
content?: string | undefined;
|
|
173
|
+
contentBlocks?: number | undefined;
|
|
174
|
+
cached: boolean;
|
|
175
|
+
error?: string | undefined;
|
|
176
|
+
errorCode?: string | undefined;
|
|
177
|
+
}
|
|
178
|
+
export interface FetchPipelineOptions<T> {
|
|
179
|
+
/** URL to fetch */
|
|
180
|
+
url: string;
|
|
181
|
+
/** Cache namespace (e.g., 'url', 'links', 'markdown') */
|
|
182
|
+
cacheNamespace: string;
|
|
183
|
+
/** Optional custom HTTP headers */
|
|
184
|
+
customHeaders?: Record<string, string> | undefined;
|
|
185
|
+
/** Optional: number of retry attempts (1-10, defaults to 3) */
|
|
186
|
+
retries?: number | undefined;
|
|
187
|
+
/** Transform function to process HTML into desired format */
|
|
188
|
+
transform: (html: string, url: string) => T;
|
|
189
|
+
/** Optional: serialize result for caching (defaults to JSON.stringify) */
|
|
190
|
+
serialize?: ((result: T) => string) | undefined;
|
|
191
|
+
/** Optional: deserialize cached content */
|
|
192
|
+
deserialize?: ((cached: string) => T) | undefined;
|
|
193
|
+
}
|
|
194
|
+
/** Result from the fetch pipeline */
|
|
195
|
+
export interface PipelineResult<T> {
|
|
196
|
+
/** The transformed data */
|
|
197
|
+
data: T;
|
|
198
|
+
/** Whether result came from cache */
|
|
199
|
+
fromCache: boolean;
|
|
200
|
+
/** The normalized URL that was fetched */
|
|
201
|
+
url: string;
|
|
202
|
+
/** Timestamp of when content was fetched/cached */
|
|
203
|
+
fetchedAt: string;
|
|
204
|
+
}
|
|
205
|
+
export interface ToolResponse<T = Record<string, unknown>> {
|
|
206
|
+
[x: string]: unknown;
|
|
207
|
+
content: {
|
|
208
|
+
type: 'text';
|
|
209
|
+
text: string;
|
|
210
|
+
}[];
|
|
211
|
+
structuredContent: T & Record<string, unknown>;
|
|
212
|
+
}
|
|
213
|
+
export interface BatchUrlResult {
|
|
214
|
+
url: string;
|
|
215
|
+
success: boolean;
|
|
216
|
+
title?: string | undefined;
|
|
217
|
+
content?: string | undefined;
|
|
218
|
+
contentBlocks?: number | undefined;
|
|
219
|
+
cached?: boolean | undefined;
|
|
220
|
+
error?: string | undefined;
|
|
221
|
+
errorCode?: string | undefined;
|
|
222
|
+
}
|
|
223
|
+
export interface BatchSummary {
|
|
224
|
+
total: number;
|
|
225
|
+
successful: number;
|
|
226
|
+
failed: number;
|
|
227
|
+
cached: number;
|
|
228
|
+
totalContentBlocks: number;
|
|
229
|
+
}
|
|
230
|
+
export interface BatchResponseContent {
|
|
231
|
+
[x: string]: unknown;
|
|
232
|
+
results: BatchUrlResult[];
|
|
233
|
+
summary: BatchSummary;
|
|
234
|
+
fetchedAt: string;
|
|
235
|
+
}
|
|
236
|
+
export interface ToolErrorResponse {
|
|
237
|
+
[x: string]: unknown;
|
|
238
|
+
content: {
|
|
239
|
+
type: 'text';
|
|
240
|
+
text: string;
|
|
241
|
+
}[];
|
|
242
|
+
structuredContent: {
|
|
243
|
+
[x: string]: unknown;
|
|
244
|
+
error: string;
|
|
245
|
+
url: string;
|
|
246
|
+
errorCode: string;
|
|
247
|
+
};
|
|
248
|
+
isError: true;
|
|
249
|
+
}
|
|
250
|
+
export {};
|
|
251
|
+
//# sourceMappingURL=types.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/config/types.ts"],"names":[],"mappings":"AAAA,KAAK,gBAAgB,GACjB,UAAU,GACV,SAAS,GACT,WAAW,GACX,MAAM,GACN,MAAM,GACN,OAAO,GACP,OAAO,GACP,YAAY,CAAC;AAEjB,UAAU,YAAY;IACpB,IAAI,EAAE,gBAAgB,CAAC;CACxB;AAED,MAAM,WAAW,aAAc,SAAQ,YAAY;IACjD,IAAI,EAAE,UAAU,CAAC;IACjB,KAAK,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC3B,WAAW,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IACjC,MAAM,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC5B,GAAG,EAAE,MAAM,CAAC;IACZ,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,YAAa,SAAQ,YAAY;IAChD,IAAI,EAAE,SAAS,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,MAAM,CAAC;CACd;AAED,MAAM,WAAW,cAAe,SAAQ,YAAY;IAClD,IAAI,EAAE,WAAW,CAAC;IAClB,IAAI,EAAE,MAAM,CAAC;CACd;AAED,MAAM,WAAW,SAAU,SAAQ,YAAY;IAC7C,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,OAAO,CAAC;IACjB,KAAK,EAAE,MAAM,EAAE,CAAC;CACjB;AAED,MAAM,WAAW,SAAU,SAAQ,YAAY;IAC7C,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC9B,IAAI,EAAE,MAAM,CAAC;CACd;AAED,MAAM,WAAW,UAAW,SAAQ,YAAY;IAC9C,IAAI,EAAE,OAAO,CAAC;IACd,OAAO,CAAC,EAAE,MAAM,EAAE,GAAG,SAAS,CAAC;IAC/B,IAAI,EAAE,MAAM,EAAE,EAAE,CAAC;CAClB;AAED,MAAM,WAAW,UAAW,SAAQ,YAAY;IAC9C,IAAI,EAAE,OAAO,CAAC;IACd,GAAG,EAAE,MAAM,CAAC;IACZ,GAAG,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;CAC1B;AAED,MAAM,WAAW,eAAgB,SAAQ,YAAY;IACnD,IAAI,EAAE,YAAY,CAAC;IACnB,IAAI,EAAE,MAAM,CAAC;CACd;AAED,MAAM,MAAM,iBAAiB,GACzB,aAAa,GACb,YAAY,GACZ,cAAc,GACd,SAAS,GACT,SAAS,GACT,UAAU,GACV,UAAU,GACV,eAAe,CAAC;AAEpB,MAAM,WAAW,gBAAgB;IAC/B,KAAK,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC3B,MAAM,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC5B,OAAO,EAAE,MAAM,CAAC;IAChB,WAAW,EAAE,MAAM,CAAC;IACpB,OAAO,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC7B,QAAQ,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;CAC/B;AAED,MAAM,WAAW,UAAU;IACzB,GAAG,EAAE,MAAM,CAAC;IACZ,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,aAAa;IAC5B,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,UAAU,GAAG,UAAU,GAAG,OAAO,CAAC;CACzC;AAED,UAAU,cAAc;IACtB,0CAA0C;IAC1C,aAAa,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,SAAS,CAAC;IACnD,mDAAmD;IACnD,OAAO,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC7B,sCAAsC;IACtC,OAAO,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;CAC9B;AAED,MAAM,WAAW,aAAc,SAAQ,cAAc;IACnD,GAAG,EAAE,MAAM,CAAC;IACZ,kBAAkB,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;IACzC,eAAe,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;IACtC,gBAAgB,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IACtC,MAAM,CAAC,EAAE,OAAO,GAAG,UAAU,GAAG,SAAS,CAAC;CAC3C;AAED,MAAM,WAAW,eAAgB,SAAQ,cAAc;IACrD,GAAG,EAAE,MAAM,CAAC;IACZ,eAAe,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;IACtC,eAAe,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;IACtC,QAAQ,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC9B,aAAa,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IACnC,aAAa,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;CACrC;AAED,MAAM,WAAW,kBAAmB,SAAQ,cAAc;IACxD,GAAG,EAAE,MAAM,CAAC;IACZ,kBAAkB,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;IACzC,eAAe,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;IACtC,gBAAgB,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IACtC,WAAW,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;CACnC;AAED,MAAM,WAAW,cAAe,SAAQ,cAAc;IACpD,IAAI,EAAE,MAAM,EAAE,CAAC;IACf,kBAAkB,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;IACzC,eAAe,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;IACtC,gBAAgB,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IACtC,MAAM,CAAC,EAAE,OAAO,GAAG,UAAU,GAAG,SAAS,CAAC;IAC1C,WAAW,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IACjC,eAAe,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;CACvC;AAED,MAAM,WAAW,aAAa;IAC5B,KAAK,EAAE;QACL,OAAO,EAAE,MAAM,CAAC;QAChB,IAAI,EAAE,MAAM,CAAC;QACb,UAAU,EAAE,MAAM,CAAC;QACnB,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,SAAS,CAAC;QAC9C,KAAK,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;KAC5B,CAAC;CACH;AAED,MAAM,WAAW,cAAc;IAC7B,KAAK,EAAE,MAAM,CAAC;IACd,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,kBAAkB;IACjC,WAAW,EAAE,MAAM,CAAC;IACpB,QAAQ,EAAE,MAAM,CAAC;IACjB,iBAAiB,EAAE,MAAM,CAAC;CAC3B;AAED,MAAM,WAAW,iBAAiB;IAChC,KAAK,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC3B,WAAW,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IACjC,MAAM,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;CAC7B;AAED,MAAM,WAAW,gBAAgB;IAC/B,OAAO,EAAE,gBAAgB,GAAG,IAAI,CAAC;IACjC,QAAQ,EAAE,iBAAiB,CAAC;CAC7B;AAED,MAAM,MAAM,gBAAgB,GACxB,IAAI,GACJ,IAAI,GACJ,IAAI,GACJ,IAAI,GACJ,IAAI,GACJ,IAAI,GACJ,GAAG,GACH,IAAI,GACJ,IAAI,GACJ,KAAK,GACL,MAAM,GACN,OAAO,GACP,KAAK,GACL,YAAY,CAAC;AAEjB,MAAM,WAAW,oBAAoB;IACnC,KAAK,EAAE,aAAa,EAAE,CAAC;IACvB,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,OAAO,CAAC;CACpB;AAED,MAAM,WAAW,mBAAmB;IAClC,eAAe,EAAE,OAAO,CAAC;IACzB,eAAe,EAAE,OAAO,CAAC;IACzB,aAAa,EAAE,OAAO,CAAC;IACvB,QAAQ,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC9B,aAAa,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;CACpC;AAED,MAAM,WAAW,QAAQ;IACvB,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;CACd;AAED,MAAM,WAAW,uBAAuB;IACtC,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,MAAM,GAAG,SAAS,CAAC;IAC1B,GAAG,EAAE,QAAQ,EAAE,GAAG,SAAS,CAAC;IAC5B,SAAS,EAAE,OAAO,CAAC;CACpB;AAED,MAAM,WAAW,gBAAgB;IAC/B,kBAAkB,EAAE,OAAO,CAAC;IAC5B,eAAe,EAAE,OAAO,CAAC;IACzB,WAAW,EAAE,OAAO,CAAC;IACrB,gBAAgB,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;CACvC;AAED,MAAM,WAAW,oBAAoB;IACnC,OAAO,EAAE,MAAM,CAAC;IAChB,aAAa,EAAE,MAAM,CAAC;IACtB,KAAK,EAAE,MAAM,GAAG,SAAS,CAAC;CAC3B;AAED,MAAM,WAAW,eAAe;IAC9B,GAAG,EAAE,MAAM,CAAC;IACZ,OAAO,EAAE,OAAO,CAAC;IACjB,KAAK,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC3B,OAAO,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC7B,aAAa,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IACnC,MAAM,EAAE,OAAO,CAAC;IAChB,KAAK,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC3B,SAAS,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;CAChC;AAED,MAAM,WAAW,oBAAoB,CAAC,CAAC;IACrC,mBAAmB;IACnB,GAAG,EAAE,MAAM,CAAC;IACZ,yDAAyD;IACzD,cAAc,EAAE,MAAM,CAAC;IACvB,mCAAmC;IACnC,aAAa,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,SAAS,CAAC;IACnD,+DAA+D;IAC/D,OAAO,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC7B,6DAA6D;IAC7D,SAAS,EAAE,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,KAAK,CAAC,CAAC;IAC5C,0EAA0E;IAC1E,SAAS,CAAC,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,KAAK,MAAM,CAAC,GAAG,SAAS,CAAC;IAChD,2CAA2C;IAC3C,WAAW,CAAC,EAAE,CAAC,CAAC,MAAM,EAAE,MAAM,KAAK,CAAC,CAAC,GAAG,SAAS,CAAC;CACnD;AAED,qCAAqC;AACrC,MAAM,WAAW,cAAc,CAAC,CAAC;IAC/B,2BAA2B;IAC3B,IAAI,EAAE,CAAC,CAAC;IACR,qCAAqC;IACrC,SAAS,EAAE,OAAO,CAAC;IACnB,0CAA0C;IAC1C,GAAG,EAAE,MAAM,CAAC;IACZ,mDAAmD;IACnD,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,YAAY,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC;IACvD,CAAC,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC;IACrB,OAAO,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC;IAC1C,iBAAiB,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CAChD;AAED,MAAM,WAAW,cAAc;IAC7B,GAAG,EAAE,MAAM,CAAC;IACZ,OAAO,EAAE,OAAO,CAAC;IACjB,KAAK,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC3B,OAAO,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC7B,aAAa,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IACnC,MAAM,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;IAC7B,KAAK,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC3B,SAAS,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;CAChC;AAED,MAAM,WAAW,YAAY;IAC3B,KAAK,EAAE,MAAM,CAAC;IACd,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,kBAAkB,EAAE,MAAM,CAAC;CAC5B;AAED,MAAM,WAAW,oBAAoB;IACnC,CAAC,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC;IACrB,OAAO,EAAE,cAAc,EAAE,CAAC;IAC1B,OAAO,EAAE,YAAY,CAAC;IACtB,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,iBAAiB;IAChC,CAAC,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC;IACrB,OAAO,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC;IAC1C,iBAAiB,EAAE;QACjB,CAAC,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC;QACrB,KAAK,EAAE,MAAM,CAAC;QACd,GAAG,EAAE,MAAM,CAAC;QACZ,SAAS,EAAE,MAAM,CAAC;KACnB,CAAC;IACF,OAAO,EAAE,IAAI,CAAC;CACf"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"types.js","sourceRoot":"","sources":["../../src/config/types.ts"],"names":[],"mappings":""}
|
|
@@ -1,44 +1,26 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Base application error class with status code support
|
|
3
|
-
*/
|
|
4
1
|
export declare class AppError extends Error {
|
|
5
2
|
readonly statusCode: number;
|
|
6
3
|
readonly isOperational: boolean;
|
|
7
4
|
readonly code: string;
|
|
8
5
|
constructor(message: string, statusCode?: number, code?: string, isOperational?: boolean);
|
|
9
6
|
}
|
|
10
|
-
/**
|
|
11
|
-
* Validation error (400)
|
|
12
|
-
*/
|
|
13
7
|
export declare class ValidationError extends AppError {
|
|
14
|
-
readonly details
|
|
8
|
+
readonly details: Record<string, unknown> | undefined;
|
|
15
9
|
constructor(message: string, details?: Record<string, unknown>);
|
|
16
10
|
}
|
|
17
|
-
/**
|
|
18
|
-
* URL validation error (400)
|
|
19
|
-
*/
|
|
20
11
|
export declare class UrlValidationError extends AppError {
|
|
21
12
|
readonly url: string;
|
|
22
13
|
constructor(message: string, url: string);
|
|
23
14
|
}
|
|
24
|
-
/**
|
|
25
|
-
* Fetch error - network/HTTP errors during URL fetching
|
|
26
|
-
*/
|
|
27
15
|
export declare class FetchError extends AppError {
|
|
28
16
|
readonly url: string;
|
|
29
|
-
readonly httpStatus
|
|
17
|
+
readonly httpStatus: number | undefined;
|
|
30
18
|
constructor(message: string, url: string, httpStatus?: number);
|
|
31
19
|
}
|
|
32
|
-
/**
|
|
33
|
-
* Rate limit error (429)
|
|
34
|
-
*/
|
|
35
20
|
export declare class RateLimitError extends AppError {
|
|
36
21
|
readonly retryAfter: number;
|
|
37
22
|
constructor(retryAfter: number);
|
|
38
23
|
}
|
|
39
|
-
/**
|
|
40
|
-
* Timeout error (408/504)
|
|
41
|
-
*/
|
|
42
24
|
export declare class TimeoutError extends AppError {
|
|
43
25
|
readonly timeoutMs: number;
|
|
44
26
|
constructor(timeoutMs: number, isGateway?: boolean);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"app-error.d.ts","sourceRoot":"","sources":["../../src/errors/app-error.ts"],"names":[],"mappings":"AAAA
|
|
1
|
+
{"version":3,"file":"app-error.d.ts","sourceRoot":"","sources":["../../src/errors/app-error.ts"],"names":[],"mappings":"AAAA,qBAAa,QAAS,SAAQ,KAAK;IACjC,SAAgB,UAAU,EAAE,MAAM,CAAC;IACnC,SAAgB,aAAa,EAAE,OAAO,CAAC;IACvC,SAAgB,IAAI,EAAE,MAAM,CAAC;gBAG3B,OAAO,EAAE,MAAM,EACf,UAAU,SAAM,EAChB,IAAI,SAAmB,EACvB,aAAa,UAAO;CASvB;AAED,qBAAa,eAAgB,SAAQ,QAAQ;IAC3C,SAAgB,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,SAAS,CAAC;gBAEjD,OAAO,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC;CAI/D;AAED,qBAAa,kBAAmB,SAAQ,QAAQ;IAC9C,SAAgB,GAAG,EAAE,MAAM,CAAC;gBAEhB,OAAO,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM;CAIzC;AAED,qBAAa,UAAW,SAAQ,QAAQ;IACtC,SAAgB,GAAG,EAAE,MAAM,CAAC;IAC5B,SAAgB,UAAU,EAAE,MAAM,GAAG,SAAS,CAAC;gBAEnC,OAAO,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,EAAE,UAAU,CAAC,EAAE,MAAM;CAK9D;AAED,qBAAa,cAAe,SAAQ,QAAQ;IAC1C,SAAgB,UAAU,EAAE,MAAM,CAAC;gBAEvB,UAAU,EAAE,MAAM;CAI/B;AAED,qBAAa,YAAa,SAAQ,QAAQ;IACxC,SAAgB,SAAS,EAAE,MAAM,CAAC;gBAEtB,SAAS,EAAE,MAAM,EAAE,SAAS,UAAQ;CAQjD"}
|
package/dist/errors/app-error.js
CHANGED
|
@@ -1,6 +1,3 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Base application error class with status code support
|
|
3
|
-
*/
|
|
4
1
|
export class AppError extends Error {
|
|
5
2
|
statusCode;
|
|
6
3
|
isOperational;
|
|
@@ -14,9 +11,6 @@ export class AppError extends Error {
|
|
|
14
11
|
Error.captureStackTrace(this, this.constructor);
|
|
15
12
|
}
|
|
16
13
|
}
|
|
17
|
-
/**
|
|
18
|
-
* Validation error (400)
|
|
19
|
-
*/
|
|
20
14
|
export class ValidationError extends AppError {
|
|
21
15
|
details;
|
|
22
16
|
constructor(message, details) {
|
|
@@ -24,9 +18,6 @@ export class ValidationError extends AppError {
|
|
|
24
18
|
this.details = details;
|
|
25
19
|
}
|
|
26
20
|
}
|
|
27
|
-
/**
|
|
28
|
-
* URL validation error (400)
|
|
29
|
-
*/
|
|
30
21
|
export class UrlValidationError extends AppError {
|
|
31
22
|
url;
|
|
32
23
|
constructor(message, url) {
|
|
@@ -34,9 +25,6 @@ export class UrlValidationError extends AppError {
|
|
|
34
25
|
this.url = url;
|
|
35
26
|
}
|
|
36
27
|
}
|
|
37
|
-
/**
|
|
38
|
-
* Fetch error - network/HTTP errors during URL fetching
|
|
39
|
-
*/
|
|
40
28
|
export class FetchError extends AppError {
|
|
41
29
|
url;
|
|
42
30
|
httpStatus;
|
|
@@ -46,9 +34,6 @@ export class FetchError extends AppError {
|
|
|
46
34
|
this.httpStatus = httpStatus;
|
|
47
35
|
}
|
|
48
36
|
}
|
|
49
|
-
/**
|
|
50
|
-
* Rate limit error (429)
|
|
51
|
-
*/
|
|
52
37
|
export class RateLimitError extends AppError {
|
|
53
38
|
retryAfter;
|
|
54
39
|
constructor(retryAfter) {
|
|
@@ -56,9 +41,6 @@ export class RateLimitError extends AppError {
|
|
|
56
41
|
this.retryAfter = retryAfter;
|
|
57
42
|
}
|
|
58
43
|
}
|
|
59
|
-
/**
|
|
60
|
-
* Timeout error (408/504)
|
|
61
|
-
*/
|
|
62
44
|
export class TimeoutError extends AppError {
|
|
63
45
|
timeoutMs;
|
|
64
46
|
constructor(timeoutMs, isGateway = false) {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"app-error.js","sourceRoot":"","sources":["../../src/errors/app-error.ts"],"names":[],"mappings":"AAAA
|
|
1
|
+
{"version":3,"file":"app-error.js","sourceRoot":"","sources":["../../src/errors/app-error.ts"],"names":[],"mappings":"AAAA,MAAM,OAAO,QAAS,SAAQ,KAAK;IACjB,UAAU,CAAS;IACnB,aAAa,CAAU;IACvB,IAAI,CAAS;IAE7B,YACE,OAAe,EACf,UAAU,GAAG,GAAG,EAChB,IAAI,GAAG,gBAAgB,EACvB,aAAa,GAAG,IAAI;QAEpB,KAAK,CAAC,OAAO,CAAC,CAAC;QACf,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC;QAC7B,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;QACjB,IAAI,CAAC,aAAa,GAAG,aAAa,CAAC;QACnC,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC;QAClC,KAAK,CAAC,iBAAiB,CAAC,IAAI,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;IAClD,CAAC;CACF;AAED,MAAM,OAAO,eAAgB,SAAQ,QAAQ;IAC3B,OAAO,CAAsC;IAE7D,YAAY,OAAe,EAAE,OAAiC;QAC5D,KAAK,CAAC,OAAO,EAAE,GAAG,EAAE,kBAAkB,CAAC,CAAC;QACxC,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;IACzB,CAAC;CACF;AAED,MAAM,OAAO,kBAAmB,SAAQ,QAAQ;IAC9B,GAAG,CAAS;IAE5B,YAAY,OAAe,EAAE,GAAW;QACtC,KAAK,CAAC,OAAO,EAAE,GAAG,EAAE,aAAa,CAAC,CAAC;QACnC,IAAI,CAAC,GAAG,GAAG,GAAG,CAAC;IACjB,CAAC;CACF;AAED,MAAM,OAAO,UAAW,SAAQ,QAAQ;IACtB,GAAG,CAAS;IACZ,UAAU,CAAqB;IAE/C,YAAY,OAAe,EAAE,GAAW,EAAE,UAAmB;QAC3D,KAAK,CAAC,OAAO,EAAE,UAAU,IAAI,GAAG,EAAE,aAAa,CAAC,CAAC;QACjD,IAAI,CAAC,GAAG,GAAG,GAAG,CAAC;QACf,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC;IAC/B,CAAC;CACF;AAED,MAAM,OAAO,cAAe,SAAQ,QAAQ;IAC1B,UAAU,CAAS;IAEnC,YAAY,UAAkB;QAC5B,KAAK,CAAC,mBAAmB,EAAE,GAAG,EAAE,cAAc,CAAC,CAAC;QAChD,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC;IAC/B,CAAC;CACF;AAED,MAAM,OAAO,YAAa,SAAQ,QAAQ;IACxB,SAAS,CAAS;IAElC,YAAY,SAAiB,EAAE,SAAS,GAAG,KAAK;QAC9C,KAAK,CACH,yBAAyB,SAAS,IAAI,EACtC,SAAS,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,EACrB,SAAS,CACV,CAAC;QACF,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;IAC7B,CAAC;CACF"}
|
package/dist/index.js
CHANGED
|
@@ -1,14 +1,13 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
|
-
import express from 'express';
|
|
2
|
+
import express, {} from 'express';
|
|
3
3
|
import { StreamableHTTPServerTransport } from '@modelcontextprotocol/sdk/server/streamableHttp.js';
|
|
4
4
|
import { isInitializeRequest } from '@modelcontextprotocol/sdk/types.js';
|
|
5
5
|
import { config } from './config/index.js';
|
|
6
|
-
import {
|
|
6
|
+
import { destroyAgents } from './services/fetcher.js';
|
|
7
|
+
import { logError, logInfo } from './services/logger.js';
|
|
7
8
|
import { errorHandler } from './middleware/error-handler.js';
|
|
8
9
|
import { rateLimiter } from './middleware/rate-limiter.js';
|
|
9
|
-
import {
|
|
10
|
-
import { destroyAgents } from './services/fetcher.js';
|
|
11
|
-
// Global error handlers for uncaught exceptions and rejections
|
|
10
|
+
import { createMcpServer } from './server.js';
|
|
12
11
|
process.on('uncaughtException', (error) => {
|
|
13
12
|
logError('Uncaught exception', error);
|
|
14
13
|
process.stderr.write(`Uncaught exception: ${error.message}\n`);
|
|
@@ -19,32 +18,24 @@ process.on('unhandledRejection', (reason) => {
|
|
|
19
18
|
logError('Unhandled rejection', error);
|
|
20
19
|
process.stderr.write(`Unhandled rejection: ${error.message}\n`);
|
|
21
20
|
});
|
|
22
|
-
// Check if running in stdio mode
|
|
23
21
|
const isStdioMode = process.argv.includes('--stdio');
|
|
24
|
-
// CORS allowlist - empty means allow all origins
|
|
25
|
-
// For production, configure this in the CORS middleware below
|
|
26
22
|
const ALLOWED_ORIGINS = [];
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
23
|
+
function getSessionId(req) {
|
|
24
|
+
const header = req.headers['mcp-session-id'];
|
|
25
|
+
return Array.isArray(header) ? header[0] : header;
|
|
26
|
+
}
|
|
31
27
|
const asyncHandler = (fn) => {
|
|
32
28
|
return (req, res, next) => {
|
|
33
29
|
Promise.resolve(fn(req, res, next)).catch(next);
|
|
34
30
|
};
|
|
35
31
|
};
|
|
36
32
|
if (isStdioMode) {
|
|
37
|
-
// Run in stdio mode for direct integration
|
|
38
33
|
const { startStdioServer } = await import('./server.js');
|
|
39
34
|
await startStdioServer();
|
|
40
35
|
}
|
|
41
36
|
else {
|
|
42
|
-
// Run HTTP server mode
|
|
43
37
|
const app = express();
|
|
44
|
-
// Middleware
|
|
45
|
-
// Limit request body size to prevent DoS
|
|
46
38
|
app.use(express.json({ limit: '1mb' }));
|
|
47
|
-
// Handle JSON parsing errors
|
|
48
39
|
app.use((err, _req, res, next) => {
|
|
49
40
|
if (err instanceof SyntaxError && 'body' in err) {
|
|
50
41
|
res.status(400).json({
|
|
@@ -59,30 +50,25 @@ else {
|
|
|
59
50
|
}
|
|
60
51
|
next(err);
|
|
61
52
|
});
|
|
62
|
-
// Rate limiting for HTTP mode
|
|
63
53
|
app.use(rateLimiter.middleware());
|
|
64
|
-
// CORS headers for MCP clients
|
|
65
54
|
app.use((req, res, next) => {
|
|
66
55
|
const origin = req.headers.origin;
|
|
67
|
-
// Validate origin format if provided
|
|
68
56
|
if (origin) {
|
|
69
57
|
try {
|
|
70
58
|
new URL(origin);
|
|
71
59
|
}
|
|
72
60
|
catch {
|
|
73
|
-
// Invalid origin format - skip CORS headers
|
|
74
61
|
next();
|
|
75
62
|
return;
|
|
76
63
|
}
|
|
77
64
|
}
|
|
78
|
-
// Allow if no origin (same-origin/non-browser), no allowlist configured, or origin in allowlist
|
|
79
65
|
if (!origin ||
|
|
80
66
|
ALLOWED_ORIGINS.length === 0 ||
|
|
81
67
|
ALLOWED_ORIGINS.includes(origin)) {
|
|
82
68
|
res.header('Access-Control-Allow-Origin', origin ?? '*');
|
|
83
69
|
res.header('Access-Control-Allow-Methods', 'GET, POST, DELETE, OPTIONS');
|
|
84
70
|
res.header('Access-Control-Allow-Headers', 'Content-Type, mcp-session-id');
|
|
85
|
-
res.header('Access-Control-Max-Age', '86400');
|
|
71
|
+
res.header('Access-Control-Max-Age', '86400');
|
|
86
72
|
}
|
|
87
73
|
if (req.method === 'OPTIONS') {
|
|
88
74
|
res.sendStatus(200);
|
|
@@ -90,7 +76,6 @@ else {
|
|
|
90
76
|
}
|
|
91
77
|
next();
|
|
92
78
|
});
|
|
93
|
-
// Health check endpoint
|
|
94
79
|
app.get('/health', (_req, res) => {
|
|
95
80
|
res.json({
|
|
96
81
|
status: 'healthy',
|
|
@@ -100,9 +85,7 @@ else {
|
|
|
100
85
|
});
|
|
101
86
|
});
|
|
102
87
|
const sessions = new Map();
|
|
103
|
-
// Session TTL: 30 minutes (sessions without activity will be cleaned up)
|
|
104
88
|
const SESSION_TTL_MS = 30 * 60 * 1000;
|
|
105
|
-
// Cleanup stale sessions periodically (every 5 minutes)
|
|
106
89
|
const sessionCleanupInterval = setInterval(() => {
|
|
107
90
|
const now = Date.now();
|
|
108
91
|
for (const [sessionId, entry] of sessions) {
|
|
@@ -114,11 +97,9 @@ else {
|
|
|
114
97
|
}
|
|
115
98
|
}, 5 * 60 * 1000);
|
|
116
99
|
sessionCleanupInterval.unref();
|
|
117
|
-
// MCP Streamable HTTP endpoint (modern replacement for SSE)
|
|
118
100
|
app.post('/mcp', asyncHandler(async (req, res) => {
|
|
119
|
-
const sessionId = req
|
|
101
|
+
const sessionId = getSessionId(req);
|
|
120
102
|
let transport;
|
|
121
|
-
// Debug logging
|
|
122
103
|
const body = req.body;
|
|
123
104
|
logInfo('[MCP POST]', {
|
|
124
105
|
method: body?.method,
|
|
@@ -129,12 +110,10 @@ else {
|
|
|
129
110
|
});
|
|
130
111
|
const existingSession = sessionId ? sessions.get(sessionId) : undefined;
|
|
131
112
|
if (existingSession) {
|
|
132
|
-
// Reuse existing session and update timestamp
|
|
133
113
|
transport = existingSession.transport;
|
|
134
|
-
existingSession.createdAt = Date.now();
|
|
114
|
+
existingSession.createdAt = Date.now();
|
|
135
115
|
}
|
|
136
116
|
else if (!sessionId && isInitializeRequest(req.body)) {
|
|
137
|
-
// New session initialization
|
|
138
117
|
transport = new StreamableHTTPServerTransport({
|
|
139
118
|
sessionIdGenerator: () => crypto.randomUUID(),
|
|
140
119
|
onsessioninitialized: (id) => {
|
|
@@ -155,7 +134,6 @@ else {
|
|
|
155
134
|
await mcpServer.connect(transport);
|
|
156
135
|
}
|
|
157
136
|
else {
|
|
158
|
-
// Invalid request - no session and not an initialize request
|
|
159
137
|
res.status(400).json({
|
|
160
138
|
jsonrpc: '2.0',
|
|
161
139
|
error: {
|
|
@@ -168,9 +146,8 @@ else {
|
|
|
168
146
|
}
|
|
169
147
|
await transport.handleRequest(req, res, req.body);
|
|
170
148
|
}));
|
|
171
|
-
// GET endpoint for SSE stream (for server-initiated messages)
|
|
172
149
|
app.get('/mcp', asyncHandler(async (req, res) => {
|
|
173
|
-
const sessionId = req
|
|
150
|
+
const sessionId = getSessionId(req);
|
|
174
151
|
if (!sessionId) {
|
|
175
152
|
res.status(400).json({ error: 'Missing mcp-session-id header' });
|
|
176
153
|
return;
|
|
@@ -180,14 +157,11 @@ else {
|
|
|
180
157
|
res.status(404).json({ error: 'Session not found' });
|
|
181
158
|
return;
|
|
182
159
|
}
|
|
183
|
-
// Refresh TTL on activity
|
|
184
160
|
session.createdAt = Date.now();
|
|
185
|
-
// Handle SSE stream for server-initiated messages
|
|
186
161
|
await session.transport.handleRequest(req, res);
|
|
187
162
|
}));
|
|
188
|
-
// DELETE endpoint for session cleanup
|
|
189
163
|
app.delete('/mcp', asyncHandler(async (req, res) => {
|
|
190
|
-
const sessionId = req
|
|
164
|
+
const sessionId = getSessionId(req);
|
|
191
165
|
const session = sessionId ? sessions.get(sessionId) : undefined;
|
|
192
166
|
if (session) {
|
|
193
167
|
await session.transport.handleRequest(req, res);
|
|
@@ -196,9 +170,7 @@ else {
|
|
|
196
170
|
res.status(204).end();
|
|
197
171
|
}
|
|
198
172
|
}));
|
|
199
|
-
// Error handling middleware (must be last)
|
|
200
173
|
app.use(errorHandler);
|
|
201
|
-
// Start server
|
|
202
174
|
const server = app
|
|
203
175
|
.listen(config.server.port, config.server.host, () => {
|
|
204
176
|
logInfo(`superFetch MCP server started`, {
|
|
@@ -214,25 +186,19 @@ else {
|
|
|
214
186
|
logError('Failed to start server', err);
|
|
215
187
|
process.exit(1);
|
|
216
188
|
});
|
|
217
|
-
// Graceful shutdown for HTTP mode
|
|
218
189
|
const shutdown = (signal) => {
|
|
219
190
|
process.stdout.write(`\n${signal} received, shutting down gracefully...\n`);
|
|
220
|
-
// Stop accepting new sessions
|
|
221
191
|
clearInterval(sessionCleanupInterval);
|
|
222
|
-
// Destroy rate limiter to stop cleanup interval
|
|
223
192
|
rateLimiter.destroy();
|
|
224
|
-
// Close all MCP transport sessions
|
|
225
193
|
for (const session of sessions.values()) {
|
|
226
194
|
void session.transport.close();
|
|
227
195
|
}
|
|
228
196
|
sessions.clear();
|
|
229
|
-
// Destroy HTTP agents to close all connections
|
|
230
197
|
destroyAgents();
|
|
231
198
|
server.close(() => {
|
|
232
199
|
logInfo('HTTP server closed');
|
|
233
200
|
process.exit(0);
|
|
234
201
|
});
|
|
235
|
-
// Force exit after timeout
|
|
236
202
|
setTimeout(() => {
|
|
237
203
|
logError('Forced shutdown after timeout');
|
|
238
204
|
process.exit(1);
|