@j0hanz/superfetch 1.0.3 → 1.0.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +615 -590
- package/dist/config/index.d.ts +5 -0
- package/dist/config/index.d.ts.map +1 -1
- package/dist/config/index.js +5 -0
- package/dist/config/index.js.map +1 -1
- package/dist/config/types.d.ts +5 -0
- package/dist/config/types.d.ts.map +1 -1
- package/dist/errors/app-error.d.ts +4 -0
- package/dist/errors/app-error.d.ts.map +1 -1
- package/dist/errors/app-error.js +7 -0
- package/dist/errors/app-error.js.map +1 -1
- package/dist/index.js +94 -17
- package/dist/index.js.map +1 -1
- package/dist/middleware/error-handler.d.ts.map +1 -1
- package/dist/middleware/error-handler.js +4 -2
- package/dist/middleware/error-handler.js.map +1 -1
- package/dist/middleware/rate-limiter.d.ts.map +1 -1
- package/dist/middleware/rate-limiter.js +46 -13
- package/dist/middleware/rate-limiter.js.map +1 -1
- package/dist/prompts/index.d.ts.map +1 -1
- package/dist/prompts/index.js +2 -7
- package/dist/prompts/index.js.map +1 -1
- package/dist/resources/cached-content.d.ts +4 -0
- package/dist/resources/cached-content.d.ts.map +1 -0
- package/dist/resources/cached-content.js +68 -0
- package/dist/resources/cached-content.js.map +1 -0
- package/dist/resources/index.d.ts.map +1 -1
- package/dist/resources/index.js +39 -1
- package/dist/resources/index.js.map +1 -1
- package/dist/server.d.ts.map +1 -1
- package/dist/server.js +10 -0
- package/dist/server.js.map +1 -1
- package/dist/services/cache.d.ts +11 -0
- package/dist/services/cache.d.ts.map +1 -1
- package/dist/services/cache.js +72 -8
- package/dist/services/cache.js.map +1 -1
- package/dist/services/card-extractor.d.ts +0 -4
- package/dist/services/card-extractor.d.ts.map +1 -1
- package/dist/services/card-extractor.js +17 -5
- package/dist/services/card-extractor.js.map +1 -1
- package/dist/services/extractor.d.ts +7 -1
- package/dist/services/extractor.d.ts.map +1 -1
- package/dist/services/extractor.js +16 -9
- package/dist/services/extractor.js.map +1 -1
- package/dist/services/fetcher.d.ts +10 -1
- package/dist/services/fetcher.d.ts.map +1 -1
- package/dist/services/fetcher.js +162 -36
- package/dist/services/fetcher.js.map +1 -1
- package/dist/services/parser.d.ts.map +1 -1
- package/dist/services/parser.js +41 -29
- package/dist/services/parser.js.map +1 -1
- package/dist/tools/handlers/fetch-links.tool.d.ts +5 -10
- package/dist/tools/handlers/fetch-links.tool.d.ts.map +1 -1
- package/dist/tools/handlers/fetch-links.tool.js +4 -0
- package/dist/tools/handlers/fetch-links.tool.js.map +1 -1
- package/dist/tools/handlers/fetch-markdown.tool.d.ts +5 -12
- package/dist/tools/handlers/fetch-markdown.tool.d.ts.map +1 -1
- package/dist/tools/handlers/fetch-markdown.tool.js +1 -2
- package/dist/tools/handlers/fetch-markdown.tool.js.map +1 -1
- package/dist/tools/handlers/fetch-url.tool.d.ts +4 -12
- package/dist/tools/handlers/fetch-url.tool.d.ts.map +1 -1
- package/dist/tools/handlers/fetch-url.tool.js.map +1 -1
- package/dist/tools/handlers/fetch-urls.tool.d.ts +8 -1
- package/dist/tools/handlers/fetch-urls.tool.d.ts.map +1 -1
- package/dist/tools/handlers/fetch-urls.tool.js +67 -16
- package/dist/tools/handlers/fetch-urls.tool.js.map +1 -1
- package/dist/tools/utils/common.js +1 -1
- package/dist/tools/utils/common.js.map +1 -1
- package/dist/tools/utils/fetch-pipeline.d.ts.map +1 -1
- package/dist/tools/utils/fetch-pipeline.js +90 -20
- package/dist/tools/utils/fetch-pipeline.js.map +1 -1
- package/dist/transformers/markdown.transformer.d.ts.map +1 -1
- package/dist/transformers/markdown.transformer.js +8 -28
- package/dist/transformers/markdown.transformer.js.map +1 -1
- package/dist/utils/concurrency.d.ts +5 -1
- package/dist/utils/concurrency.d.ts.map +1 -1
- package/dist/utils/concurrency.js +15 -2
- package/dist/utils/concurrency.js.map +1 -1
- package/dist/utils/content-cleaner.d.ts.map +1 -1
- package/dist/utils/content-cleaner.js +124 -108
- package/dist/utils/content-cleaner.js.map +1 -1
- package/dist/utils/language-detector.d.ts +1 -1
- package/dist/utils/language-detector.d.ts.map +1 -1
- package/dist/utils/sanitizer.js +1 -1
- package/dist/utils/sanitizer.js.map +1 -1
- package/dist/utils/tool-error-handler.d.ts.map +1 -1
- package/dist/utils/tool-error-handler.js +36 -6
- package/dist/utils/tool-error-handler.js.map +1 -1
- package/dist/utils/url-validator.d.ts +10 -0
- package/dist/utils/url-validator.d.ts.map +1 -1
- package/dist/utils/url-validator.js +43 -5
- package/dist/utils/url-validator.js.map +1 -1
- package/package.json +83 -80
|
@@ -2,93 +2,95 @@
|
|
|
2
2
|
* Post-processing content cleaner for removing noise artifacts
|
|
3
3
|
* that slip through Readability extraction.
|
|
4
4
|
*/
|
|
5
|
-
//
|
|
6
|
-
const
|
|
7
|
-
// Relative timestamps
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
// Share/action
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
// UI artifacts
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
// Empty
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
// Navigation
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
// Cookie/consent
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
//
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
];
|
|
5
|
+
// Pre-compiled combined pattern for optimal performance
|
|
6
|
+
const NOISE_PATTERN_COMBINED = new RegExp([
|
|
7
|
+
// Relative timestamps
|
|
8
|
+
'^\\d+\\s*(seconds?|minutes?|hours?|days?|weeks?|months?|years?)\\s*ago$',
|
|
9
|
+
'^(just now|recently|today|yesterday|last week|last month)$',
|
|
10
|
+
'^(updated|modified|edited|created|published)\\s*:?\\s*\\d+\\s*(seconds?|minutes?|hours?|days?|weeks?|months?|years?)\\s*ago$',
|
|
11
|
+
'^(jan|feb|mar|apr|may|jun|jul|aug|sep|oct|nov|dec)\\s+\\d{1,2},?\\s+\\d{4}$',
|
|
12
|
+
'^\\d{1,2}\\s+(jan|feb|mar|apr|may|jun|jul|aug|sep|oct|nov|dec)\\s+\\d{4}$',
|
|
13
|
+
'^\\d{4}-\\d{2}-\\d{2}$',
|
|
14
|
+
'^last\\s+updated\\s*:?',
|
|
15
|
+
// Share/action buttons
|
|
16
|
+
'^(share|copy|like|follow|subscribe|download|print|save|bookmark|tweet|pin it|email|export)$',
|
|
17
|
+
'^(copy to clipboard|copied!?|copy code|copy link)$',
|
|
18
|
+
'^(share on|share to|share via)\\s+(twitter|facebook|linkedin|reddit|x|email)$',
|
|
19
|
+
// UI artifacts
|
|
20
|
+
'^(click to copy|expand|collapse|show more|show less|load more|view more|read more|see more|see all|view all)$',
|
|
21
|
+
'^(toggle|switch|enable|disable|on|off)$',
|
|
22
|
+
'^(edit|delete|remove|add|new|create|update|cancel|confirm|submit|reset|clear)$',
|
|
23
|
+
'^(open in|view in|edit in)\\s+\\w+$',
|
|
24
|
+
'^(try it|run|execute|play|preview|demo|live demo|playground)$',
|
|
25
|
+
'^(source|view source|edit this page|edit on github|improve this doc)$',
|
|
26
|
+
// Empty/placeholder
|
|
27
|
+
'^(loading\\.{0,3}|please wait\\.{0,3}|\\.{2,})$',
|
|
28
|
+
'^(n\\/a|tbd|todo|coming soon|placeholder|untitled)$',
|
|
29
|
+
// Navigation
|
|
30
|
+
'^(next|previous|prev|back|forward|home|menu|close|open|skip to|jump to|go to)$',
|
|
31
|
+
'^(table of contents|toc|contents|on this page|in this article|in this section)$',
|
|
32
|
+
'^(scroll to top|back to top|top)$',
|
|
33
|
+
// Cookie/consent
|
|
34
|
+
'^(accept|reject|accept all|reject all|cookie settings|privacy settings|manage preferences)$',
|
|
35
|
+
'^(accept cookies|decline cookies|cookie policy|privacy policy|terms of service|terms & conditions)$',
|
|
36
|
+
// Counts
|
|
37
|
+
'^\\d+\\s*(comments?|replies?|reactions?|responses?)$',
|
|
38
|
+
'^\\d+\\s*(likes?|shares?|views?|followers?|retweets?|stars?|forks?|claps?|upvotes?|downvotes?)$',
|
|
39
|
+
'^(liked by|shared by|followed by)\\s+\\d+',
|
|
40
|
+
// Version badges
|
|
41
|
+
'^v?\\d+\\.\\d+(\\.\\d+)?(-\\w+)?$',
|
|
42
|
+
'^(stable|beta|alpha|rc|preview|experimental|deprecated|legacy|new|updated)$',
|
|
43
|
+
// Structural
|
|
44
|
+
'^(a|b|c|d|e|f|g|h|i|j|k|l|m|n|o|p|q|r|s|t|u|v|w|x|y|z)$',
|
|
45
|
+
'^panel\\s*[a-z]?$',
|
|
46
|
+
// API artifacts
|
|
47
|
+
'^(required|optional|default|type|example|description|parameters?|returns?|response|request)$',
|
|
48
|
+
'^(get|post|put|patch|delete|head|options)\\s*$',
|
|
49
|
+
// Interactive
|
|
50
|
+
'^(drag|drop|resize|zoom|scroll|swipe|tap|click|hover|focus)(\\s+to\\s+\\w+)?$',
|
|
51
|
+
'^(drag the|move the|resize the|drag to|click to)\\s+\\w+',
|
|
52
|
+
// Breadcrumbs
|
|
53
|
+
'^[/\\\\>→»›]+$',
|
|
54
|
+
// Ads
|
|
55
|
+
'^(ad|advertisement|sponsored|promoted|partner content)$',
|
|
56
|
+
].join('|'), 'i');
|
|
57
|
+
// Pre-compiled pattern for short text noise
|
|
58
|
+
const SHORT_TEXT_NOISE_PATTERN = new RegExp([
|
|
59
|
+
'^#\\w+$',
|
|
60
|
+
'^@\\w+$',
|
|
61
|
+
'^\\d+$',
|
|
62
|
+
'^[•·→←↑↓►▼▲◄▶◀■□●○★☆✓✗✔✘×]+$',
|
|
63
|
+
'^[,;:\\-–—]+$',
|
|
64
|
+
'^\\[\\d+\\]$',
|
|
65
|
+
'^\\(\\d+\\)$',
|
|
66
|
+
'^fig\\.?\\s*\\d+$',
|
|
67
|
+
'^table\\s*\\d+$',
|
|
68
|
+
'^step\\s*\\d+$',
|
|
69
|
+
'^note:?$',
|
|
70
|
+
'^tip:?$',
|
|
71
|
+
'^warning:?$',
|
|
72
|
+
'^info:?$',
|
|
73
|
+
'^caution:?$',
|
|
74
|
+
].join('|'), 'i');
|
|
75
|
+
// Pre-compiled pattern for UI chrome detection
|
|
76
|
+
const UI_CHROME_PATTERN = new RegExp([
|
|
77
|
+
'^(sign in|sign up|log in|log out|register|create account)$',
|
|
78
|
+
'^(search|search\\.\\.\\.|search docs|search documentation)$',
|
|
79
|
+
'^(dark mode|light mode|theme|language|locale)$',
|
|
80
|
+
'^(feedback|report issue|report a bug|file an issue|suggest edit)$',
|
|
81
|
+
'^(documentation|docs|api|reference|guide|tutorial|examples?)$',
|
|
82
|
+
"^(version|changelog|release notes|what's new)$",
|
|
83
|
+
].join('|'), 'i');
|
|
85
84
|
// Minimum lengths for different content types
|
|
86
85
|
const MIN_PARAGRAPH_LENGTH = 20;
|
|
87
86
|
const MIN_HEADING_LENGTH = 2;
|
|
88
87
|
const MIN_LIST_ITEM_LENGTH = 3;
|
|
89
88
|
const SHORT_TEXT_THRESHOLD = 25;
|
|
89
|
+
// Maximum text length to test against regex patterns (ReDoS protection)
|
|
90
|
+
const MAX_REGEX_INPUT_LENGTH = 500;
|
|
90
91
|
/**
|
|
91
92
|
* Check if text matches any noise pattern
|
|
93
|
+
* Protected against ReDoS by limiting input length
|
|
92
94
|
*/
|
|
93
95
|
function isNoiseText(text) {
|
|
94
96
|
const trimmed = text.trim();
|
|
@@ -96,50 +98,64 @@ function isNoiseText(text) {
|
|
|
96
98
|
if (!trimmed) {
|
|
97
99
|
return true;
|
|
98
100
|
}
|
|
99
|
-
//
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
101
|
+
// ReDoS protection: skip regex for very long strings
|
|
102
|
+
if (trimmed.length > MAX_REGEX_INPUT_LENGTH) {
|
|
103
|
+
return false;
|
|
104
|
+
}
|
|
105
|
+
// Check combined noise pattern (single regex test)
|
|
106
|
+
if (NOISE_PATTERN_COMBINED.test(trimmed)) {
|
|
107
|
+
return true;
|
|
104
108
|
}
|
|
105
109
|
// Check short text patterns for brief content
|
|
106
110
|
if (trimmed.length < SHORT_TEXT_THRESHOLD) {
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
return true;
|
|
110
|
-
}
|
|
111
|
+
if (SHORT_TEXT_NOISE_PATTERN.test(trimmed)) {
|
|
112
|
+
return true;
|
|
111
113
|
}
|
|
112
114
|
// Also check UI chrome patterns for short text
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
return true;
|
|
116
|
-
}
|
|
115
|
+
if (UI_CHROME_PATTERN.test(trimmed)) {
|
|
116
|
+
return true;
|
|
117
117
|
}
|
|
118
118
|
}
|
|
119
119
|
return false;
|
|
120
120
|
}
|
|
121
|
+
// Pre-compiled placeholder pattern (combined for performance)
|
|
122
|
+
const PLACEHOLDER_PATTERN = /^(lorem ipsum|sample text|placeholder|example (text|content|data)|test (text|content|data)|your (text|content|name|email) here|enter (your|a) |type (your|a|something) )/i;
|
|
123
|
+
const PLACEHOLDER_CACHE = new Map();
|
|
124
|
+
const PLACEHOLDER_CACHE_MAX_SIZE = 1000;
|
|
125
|
+
const PLACEHOLDER_CACHE_TTL_MS = 5 * 60 * 1000; // 5 minutes
|
|
121
126
|
/**
|
|
122
127
|
* Check if text looks like placeholder/demo content
|
|
128
|
+
* Uses caching with TTL for performance and memory safety
|
|
123
129
|
*/
|
|
124
130
|
function isPlaceholderContent(text) {
|
|
125
131
|
const trimmed = text.trim().toLowerCase();
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
/^your (text|content|name|email) here/i,
|
|
134
|
-
/^enter (your|a) /i,
|
|
135
|
-
/^type (your|a|something) /i,
|
|
136
|
-
];
|
|
137
|
-
for (const pattern of placeholders) {
|
|
138
|
-
if (pattern.test(trimmed)) {
|
|
139
|
-
return true;
|
|
132
|
+
const now = Date.now();
|
|
133
|
+
// Check cache first
|
|
134
|
+
const cached = PLACEHOLDER_CACHE.get(trimmed);
|
|
135
|
+
if (cached !== undefined) {
|
|
136
|
+
// Check if entry is still valid
|
|
137
|
+
if (now - cached.timestamp < PLACEHOLDER_CACHE_TTL_MS) {
|
|
138
|
+
return cached.value;
|
|
140
139
|
}
|
|
140
|
+
// Expired entry, remove it
|
|
141
|
+
PLACEHOLDER_CACHE.delete(trimmed);
|
|
141
142
|
}
|
|
142
|
-
|
|
143
|
+
// Single regex test (faster than array iteration)
|
|
144
|
+
const result = PLACEHOLDER_PATTERN.test(trimmed);
|
|
145
|
+
// Cache result with LRU eviction and timestamp
|
|
146
|
+
if (PLACEHOLDER_CACHE.size >= PLACEHOLDER_CACHE_MAX_SIZE) {
|
|
147
|
+
// Remove oldest entries (first 10% of cache)
|
|
148
|
+
const keysToDelete = Math.ceil(PLACEHOLDER_CACHE_MAX_SIZE * 0.1);
|
|
149
|
+
const iterator = PLACEHOLDER_CACHE.keys();
|
|
150
|
+
for (let i = 0; i < keysToDelete; i++) {
|
|
151
|
+
const key = iterator.next().value;
|
|
152
|
+
if (key !== undefined) {
|
|
153
|
+
PLACEHOLDER_CACHE.delete(key);
|
|
154
|
+
}
|
|
155
|
+
}
|
|
156
|
+
}
|
|
157
|
+
PLACEHOLDER_CACHE.set(trimmed, { value: result, timestamp: now });
|
|
158
|
+
return result;
|
|
143
159
|
}
|
|
144
160
|
/**
|
|
145
161
|
* Clean paragraph text by removing noise
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"content-cleaner.js","sourceRoot":"","sources":["../../src/utils/content-cleaner.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,
|
|
1
|
+
{"version":3,"file":"content-cleaner.js","sourceRoot":"","sources":["../../src/utils/content-cleaner.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,wDAAwD;AACxD,MAAM,sBAAsB,GAAG,IAAI,MAAM,CACvC;IACE,sBAAsB;IACtB,yEAAyE;IACzE,4DAA4D;IAC5D,8HAA8H;IAC9H,6EAA6E;IAC7E,2EAA2E;IAC3E,wBAAwB;IACxB,wBAAwB;IACxB,uBAAuB;IACvB,6FAA6F;IAC7F,oDAAoD;IACpD,+EAA+E;IAC/E,eAAe;IACf,+GAA+G;IAC/G,yCAAyC;IACzC,gFAAgF;IAChF,qCAAqC;IACrC,+DAA+D;IAC/D,uEAAuE;IACvE,oBAAoB;IACpB,iDAAiD;IACjD,qDAAqD;IACrD,aAAa;IACb,gFAAgF;IAChF,iFAAiF;IACjF,mCAAmC;IACnC,iBAAiB;IACjB,6FAA6F;IAC7F,qGAAqG;IACrG,SAAS;IACT,sDAAsD;IACtD,iGAAiG;IACjG,2CAA2C;IAC3C,iBAAiB;IACjB,mCAAmC;IACnC,6EAA6E;IAC7E,aAAa;IACb,yDAAyD;IACzD,mBAAmB;IACnB,gBAAgB;IAChB,8FAA8F;IAC9F,gDAAgD;IAChD,cAAc;IACd,+EAA+E;IAC/E,0DAA0D;IAC1D,cAAc;IACd,gBAAgB;IAChB,MAAM;IACN,yDAAyD;CAC1D,CAAC,IAAI,CAAC,GAAG,CAAC,EACX,GAAG,CACJ,CAAC;AAEF,4CAA4C;AAC5C,MAAM,wBAAwB,GAAG,IAAI,MAAM,CACzC;IACE,SAAS;IACT,SAAS;IACT,QAAQ;IACR,8BAA8B;IAC9B,eAAe;IACf,cAAc;IACd,cAAc;IACd,mBAAmB;IACnB,iBAAiB;IACjB,gBAAgB;IAChB,UAAU;IACV,SAAS;IACT,aAAa;IACb,UAAU;IACV,aAAa;CACd,CAAC,IAAI,CAAC,GAAG,CAAC,EACX,GAAG,CACJ,CAAC;AAEF,+CAA+C;AAC/C,MAAM,iBAAiB,GAAG,IAAI,MAAM,CAClC;IACE,4DAA4D;IAC5D,6DAA6D;IAC7D,gDAAgD;IAChD,mEAAmE;IACnE,+DAA+D;IAC/D,gDAAgD;CACjD,CAAC,IAAI,CAAC,GAAG,CAAC,EACX,GAAG,CACJ,CAAC;AAEF,8CAA8C;AAC9C,MAAM,oBAAoB,GAAG,EAAE,CAAC;AAChC,MAAM,kBAAkB,GAAG,CAAC,CAAC;AAC7B,MAAM,oBAAoB,GAAG,CAAC,CAAC;AAC/B,MAAM,oBAAoB,GAAG,EAAE,CAAC;AAEhC,wEAAwE;AACxE,MAAM,sBAAsB,GAAG,GAAG,CAAC;AAEnC;;;GAGG;AACH,SAAS,WAAW,CAAC,IAAY;IAC/B,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;IAE5B,2BAA2B;IAC3B,IAAI,CAAC,OAAO,EAAE,CAAC;QACb,OAAO,IAAI,CAAC;IACd,CAAC;IAED,qDAAqD;IACrD,IAAI,OAAO,CAAC,MAAM,GAAG,sBAAsB,EAAE,CAAC;QAC5C,OAAO,KAAK,CAAC;IACf,CAAC;IAED,mDAAmD;IACnD,IAAI,sBAAsB,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;QACzC,OAAO,IAAI,CAAC;IACd,CAAC;IAED,8CAA8C;IAC9C,IAAI,OAAO,CAAC,MAAM,GAAG,oBAAoB,EAAE,CAAC;QAC1C,IAAI,wBAAwB,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;YAC3C,OAAO,IAAI,CAAC;QACd,CAAC;QAED,+CAA+C;QAC/C,IAAI,iBAAiB,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;YACpC,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC;IAED,OAAO,KAAK,CAAC;AACf,CAAC;AAED,8DAA8D;AAC9D,MAAM,mBAAmB,GACvB,2KAA2K,CAAC;AAO9K,MAAM,iBAAiB,GAAG,IAAI,GAAG,EAAsB,CAAC;AACxD,MAAM,0BAA0B,GAAG,IAAI,CAAC;AACxC,MAAM,wBAAwB,GAAG,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,CAAC,YAAY;AAE5D;;;GAGG;AACH,SAAS,oBAAoB,CAAC,IAAY;IACxC,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IAC1C,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAEvB,oBAAoB;IACpB,MAAM,MAAM,GAAG,iBAAiB,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;IAC9C,IAAI,MAAM,KAAK,SAAS,EAAE,CAAC;QACzB,gCAAgC;QAChC,IAAI,GAAG,GAAG,MAAM,CAAC,SAAS,GAAG,wBAAwB,EAAE,CAAC;YACtD,OAAO,MAAM,CAAC,KAAK,CAAC;QACtB,CAAC;QACD,2BAA2B;QAC3B,iBAAiB,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;IACpC,CAAC;IAED,kDAAkD;IAClD,MAAM,MAAM,GAAG,mBAAmB,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IAEjD,+CAA+C;IAC/C,IAAI,iBAAiB,CAAC,IAAI,IAAI,0BAA0B,EAAE,CAAC;QACzD,6CAA6C;QAC7C,MAAM,YAAY,GAAG,IAAI,CAAC,IAAI,CAAC,0BAA0B,GAAG,GAAG,CAAC,CAAC;QACjE,MAAM,QAAQ,GAAG,iBAAiB,CAAC,IAAI,EAAE,CAAC;QAC1C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,YAAY,EAAE,CAAC,EAAE,EAAE,CAAC;YACtC,MAAM,GAAG,GAAG,QAAQ,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC;YAClC,IAAI,GAAG,KAAK,SAAS,EAAE,CAAC;gBACtB,iBAAiB,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YAChC,CAAC;QACH,CAAC;IACH,CAAC;IACD,iBAAiB,CAAC,GAAG,CAAC,OAAO,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,SAAS,EAAE,GAAG,EAAE,CAAC,CAAC;IAElE,OAAO,MAAM,CAAC;AAChB,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,cAAc,CAAC,IAAY;IACzC,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;IAE5B,6BAA6B;IAC7B,IAAI,OAAO,CAAC,MAAM,GAAG,oBAAoB,EAAE,CAAC;QAC1C,iFAAiF;QACjF,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;YAC5B,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC;IAED,mBAAmB;IACnB,IAAI,WAAW,CAAC,OAAO,CAAC,EAAE,CAAC;QACzB,OAAO,IAAI,CAAC;IACd,CAAC;IAED,0DAA0D;IAC1D,IAAI,oBAAoB,CAAC,OAAO,CAAC,EAAE,CAAC;QAClC,OAAO,IAAI,CAAC;IACd,CAAC;IAED,OAAO,OAAO,CAAC;AACjB,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,YAAY,CAAC,IAAY;IACvC,IAAI,OAAO,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;IAE1B,YAAY;IACZ,IAAI,OAAO,CAAC,MAAM,GAAG,kBAAkB,EAAE,CAAC;QACxC,OAAO,IAAI,CAAC;IACd,CAAC;IAED,uDAAuD;IACvD,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,wBAAwB,EAAE,IAAI,CAAC,CAAC;IAE1D,4DAA4D;IAC5D,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,yCAAyC,EAAE,EAAE,CAAC,CAAC;IAEzE,2DAA2D;IAC3D,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,WAAW,EAAE,EAAE,CAAC,CAAC;IAE3C,mBAAmB;IACnB,IAAI,WAAW,CAAC,OAAO,CAAC,EAAE,CAAC;QACzB,OAAO,IAAI,CAAC;IACd,CAAC;IAED,OAAO,OAAO,CAAC,IAAI,EAAE,CAAC;AACxB,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,cAAc,CAAC,KAAe;IAC5C,OAAO,KAAK;SACT,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;SAC1B,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE;QACf,IAAI,IAAI,CAAC,MAAM,GAAG,oBAAoB;YAAE,OAAO,KAAK,CAAC;QACrD,IAAI,WAAW,CAAC,IAAI,CAAC;YAAE,OAAO,KAAK,CAAC;QACpC,OAAO,IAAI,CAAC;IACd,CAAC,CAAC,CAAC;AACP,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,cAAc,CAAC,IAAY;IACzC,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;IAE5B,mBAAmB;IACnB,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACzB,OAAO,IAAI,CAAC;IACd,CAAC;IAED,qDAAqD;IACrD,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;QAC1D,OAAO,IAAI,CAAC;IACd,CAAC;IAED,OAAO,OAAO,CAAC;AACjB,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,kBAAkB,CAAC,IAAY;IAC7C,OAAO,IAAI,CAAC,OAAO,CAAC,wBAAwB,EAAE,IAAI,CAAC,CAAC;AACtD,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,sBAAsB,CAAC,IAAY;IACjD,OAAO,CACL,IAAI;QACF,yCAAyC;SACxC,OAAO,CACN,0EAA0E,EAC1E,EAAE,CACH;QACD,kCAAkC;SACjC,OAAO,CACN,6HAA6H,EAC7H,EAAE,CACH;QACD,0BAA0B;SACzB,OAAO,CACN,6EAA6E,EAC7E,EAAE,CACH;QACD,4BAA4B;SAC3B,OAAO,CAAC,SAAS,EAAE,GAAG,CAAC;SACvB,IAAI,EAAE,CACV,CAAC;AACJ,CAAC"}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Detect programming language from code content
|
|
3
3
|
*/
|
|
4
|
-
export declare function detectLanguage(code: string):
|
|
4
|
+
export declare function detectLanguage(code: string): string | undefined;
|
|
5
5
|
//# sourceMappingURL=language-detector.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"language-detector.d.ts","sourceRoot":"","sources":["../../src/utils/language-detector.ts"],"names":[],"mappings":"AA4CA;;GAEG;AACH,wBAAgB,cAAc,CAAC,IAAI,EAAE,MAAM,
|
|
1
|
+
{"version":3,"file":"language-detector.d.ts","sourceRoot":"","sources":["../../src/utils/language-detector.ts"],"names":[],"mappings":"AA4CA;;GAEG;AACH,wBAAgB,cAAc,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS,CAE/D"}
|
package/dist/utils/sanitizer.js
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"sanitizer.js","sourceRoot":"","sources":["../../src/utils/sanitizer.ts"],"names":[],"mappings":"AAAA,wDAAwD;AACxD,MAAM,gBAAgB,GAAG,MAAM,CAAC;AAEhC,MAAM,UAAU,YAAY,CAAC,IAA+B;IAC1D,IAAI,IAAI,IAAI,IAAI;QAAE,OAAO,EAAE,CAAC;IAC5B,IAAI,OAAO,IAAI,KAAK,QAAQ;QAAE,OAAO,MAAM,CAAC,IAAI,CAAC,CAAC;IAClD,OAAO,IAAI,CAAC,OAAO,CAAC,gBAAgB,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;AACpD,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,IAAY,EAAE,SAAiB;IAC1D,IAAI,SAAS,GAAG,CAAC,EAAE,CAAC;QAClB,OAAO,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IAC/C,CAAC;IACD,IAAI,IAAI,CAAC,MAAM,IAAI,SAAS,EAAE,CAAC;QAC7B,OAAO,IAAI,CAAC;IACd,CAAC;IACD,OAAO,IAAI,CAAC,SAAS,CAAC,CAAC,EAAE,SAAS,GAAG,CAAC,CAAC,
|
|
1
|
+
{"version":3,"file":"sanitizer.js","sourceRoot":"","sources":["../../src/utils/sanitizer.ts"],"names":[],"mappings":"AAAA,wDAAwD;AACxD,MAAM,gBAAgB,GAAG,MAAM,CAAC;AAEhC,MAAM,UAAU,YAAY,CAAC,IAA+B;IAC1D,IAAI,IAAI,IAAI,IAAI;QAAE,OAAO,EAAE,CAAC;IAC5B,IAAI,OAAO,IAAI,KAAK,QAAQ;QAAE,OAAO,MAAM,CAAC,IAAI,CAAC,CAAC;IAClD,OAAO,IAAI,CAAC,OAAO,CAAC,gBAAgB,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;AACpD,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,IAAY,EAAE,SAAiB;IAC1D,IAAI,SAAS,GAAG,CAAC,EAAE,CAAC;QAClB,OAAO,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IAC/C,CAAC;IACD,IAAI,IAAI,CAAC,MAAM,IAAI,SAAS,EAAE,CAAC;QAC7B,OAAO,IAAI,CAAC;IACd,CAAC;IACD,OAAO,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC,EAAE,SAAS,GAAG,CAAC,CAAC,KAAK,CAAC;AAClD,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"tool-error-handler.d.ts","sourceRoot":"","sources":["../../src/utils/tool-error-handler.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,oBAAoB,CAAC;
|
|
1
|
+
{"version":3,"file":"tool-error-handler.d.ts","sourceRoot":"","sources":["../../src/utils/tool-error-handler.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,oBAAoB,CAAC;AAgB5D,wBAAgB,uBAAuB,CACrC,OAAO,EAAE,MAAM,EACf,GAAG,EAAE,MAAM,EACX,IAAI,EAAE,MAAM,GACX,iBAAiB,CAOnB;AAED,wBAAgB,eAAe,CAC7B,KAAK,EAAE,OAAO,EACd,GAAG,EAAE,MAAM,EACX,eAAe,SAAqB,GACnC,iBAAiB,CA8CnB"}
|
|
@@ -1,4 +1,7 @@
|
|
|
1
|
-
import { AppError, FetchError, TimeoutError, UrlValidationError, } from '../errors/
|
|
1
|
+
import { AbortError, AppError, FetchError, RateLimitError, TimeoutError, UrlValidationError, } from '../errors/app-error.js';
|
|
2
|
+
// Stack traces only exposed when explicitly enabled in development
|
|
3
|
+
const isDevelopment = process.env.NODE_ENV === 'development' &&
|
|
4
|
+
process.env.EXPOSE_STACK_TRACES === 'true';
|
|
2
5
|
export function createToolErrorResponse(message, url, code) {
|
|
3
6
|
const structuredContent = { error: message, url, errorCode: code };
|
|
4
7
|
return {
|
|
@@ -9,19 +12,46 @@ export function createToolErrorResponse(message, url, code) {
|
|
|
9
12
|
}
|
|
10
13
|
export function handleToolError(error, url, fallbackMessage = 'Operation failed') {
|
|
11
14
|
if (error instanceof UrlValidationError) {
|
|
12
|
-
|
|
15
|
+
const message = isDevelopment
|
|
16
|
+
? `${error.message}\nURL: ${error.url}\nStack: ${error.stack ?? ''}`
|
|
17
|
+
: error.message;
|
|
18
|
+
return createToolErrorResponse(message, url, 'INVALID_URL');
|
|
19
|
+
}
|
|
20
|
+
if (error instanceof AbortError) {
|
|
21
|
+
const message = isDevelopment
|
|
22
|
+
? `Request aborted${error.reason ? `: ${error.reason}` : ''}\n${error.stack ?? ''}`
|
|
23
|
+
: `Request aborted${error.reason ? `: ${error.reason}` : ''}`;
|
|
24
|
+
return createToolErrorResponse(message, url, 'ABORTED');
|
|
13
25
|
}
|
|
14
26
|
if (error instanceof TimeoutError) {
|
|
15
|
-
|
|
27
|
+
const message = isDevelopment
|
|
28
|
+
? `Request timed out after ${error.timeoutMs}ms\n${error.stack ?? ''}`
|
|
29
|
+
: `Request timed out after ${error.timeoutMs}ms`;
|
|
30
|
+
return createToolErrorResponse(message, url, 'TIMEOUT');
|
|
31
|
+
}
|
|
32
|
+
if (error instanceof RateLimitError) {
|
|
33
|
+
const message = isDevelopment
|
|
34
|
+
? `Rate limited. Retry after ${error.retryAfter}s\n${error.stack ?? ''}`
|
|
35
|
+
: `Rate limited. Retry after ${error.retryAfter}s`;
|
|
36
|
+
return createToolErrorResponse(message, url, 'RATE_LIMITED');
|
|
16
37
|
}
|
|
17
38
|
if (error instanceof FetchError) {
|
|
18
39
|
const code = error.httpStatus ? `HTTP_${error.httpStatus}` : 'FETCH_ERROR';
|
|
19
|
-
|
|
40
|
+
const message = isDevelopment
|
|
41
|
+
? `${error.message}\n${error.stack ?? ''}`
|
|
42
|
+
: error.message;
|
|
43
|
+
return createToolErrorResponse(message, url, code);
|
|
20
44
|
}
|
|
21
45
|
if (error instanceof AppError) {
|
|
22
|
-
|
|
46
|
+
const message = isDevelopment
|
|
47
|
+
? `${error.message}\n${error.stack ?? ''}`
|
|
48
|
+
: error.message;
|
|
49
|
+
return createToolErrorResponse(message, url, error.code);
|
|
23
50
|
}
|
|
24
51
|
const message = error instanceof Error ? error.message : 'Unknown error';
|
|
25
|
-
|
|
52
|
+
const fullMessage = isDevelopment && error instanceof Error
|
|
53
|
+
? `${fallbackMessage}: ${message}\n${error.stack ?? ''}`
|
|
54
|
+
: `${fallbackMessage}: ${message}`;
|
|
55
|
+
return createToolErrorResponse(fullMessage, url, 'UNKNOWN_ERROR');
|
|
26
56
|
}
|
|
27
57
|
//# sourceMappingURL=tool-error-handler.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"tool-error-handler.js","sourceRoot":"","sources":["../../src/utils/tool-error-handler.ts"],"names":[],"mappings":"AAEA,OAAO,EACL,QAAQ,EACR,UAAU,EACV,YAAY,EACZ,kBAAkB,GACnB,MAAM,
|
|
1
|
+
{"version":3,"file":"tool-error-handler.js","sourceRoot":"","sources":["../../src/utils/tool-error-handler.ts"],"names":[],"mappings":"AAEA,OAAO,EACL,UAAU,EACV,QAAQ,EACR,UAAU,EACV,cAAc,EACd,YAAY,EACZ,kBAAkB,GACnB,MAAM,wBAAwB,CAAC;AAEhC,mEAAmE;AACnE,MAAM,aAAa,GACjB,OAAO,CAAC,GAAG,CAAC,QAAQ,KAAK,aAAa;IACtC,OAAO,CAAC,GAAG,CAAC,mBAAmB,KAAK,MAAM,CAAC;AAE7C,MAAM,UAAU,uBAAuB,CACrC,OAAe,EACf,GAAW,EACX,IAAY;IAEZ,MAAM,iBAAiB,GAAG,EAAE,KAAK,EAAE,OAAO,EAAE,GAAG,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC;IACnE,OAAO;QACL,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,iBAAiB,CAAC,EAAE,CAAC;QACpE,iBAAiB;QACjB,OAAO,EAAE,IAAI;KACd,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,eAAe,CAC7B,KAAc,EACd,GAAW,EACX,eAAe,GAAG,kBAAkB;IAEpC,IAAI,KAAK,YAAY,kBAAkB,EAAE,CAAC;QACxC,MAAM,OAAO,GAAG,aAAa;YAC3B,CAAC,CAAC,GAAG,KAAK,CAAC,OAAO,UAAU,KAAK,CAAC,GAAG,YAAY,KAAK,CAAC,KAAK,IAAI,EAAE,EAAE;YACpE,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC;QAClB,OAAO,uBAAuB,CAAC,OAAO,EAAE,GAAG,EAAE,aAAa,CAAC,CAAC;IAC9D,CAAC;IACD,IAAI,KAAK,YAAY,UAAU,EAAE,CAAC;QAChC,MAAM,OAAO,GAAG,aAAa;YAC3B,CAAC,CAAC,kBAAkB,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,KAAK,CAAC,KAAK,IAAI,EAAE,EAAE;YACnF,CAAC,CAAC,kBAAkB,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC;QAChE,OAAO,uBAAuB,CAAC,OAAO,EAAE,GAAG,EAAE,SAAS,CAAC,CAAC;IAC1D,CAAC;IACD,IAAI,KAAK,YAAY,YAAY,EAAE,CAAC;QAClC,MAAM,OAAO,GAAG,aAAa;YAC3B,CAAC,CAAC,2BAA2B,KAAK,CAAC,SAAS,OAAO,KAAK,CAAC,KAAK,IAAI,EAAE,EAAE;YACtE,CAAC,CAAC,2BAA2B,KAAK,CAAC,SAAS,IAAI,CAAC;QACnD,OAAO,uBAAuB,CAAC,OAAO,EAAE,GAAG,EAAE,SAAS,CAAC,CAAC;IAC1D,CAAC;IACD,IAAI,KAAK,YAAY,cAAc,EAAE,CAAC;QACpC,MAAM,OAAO,GAAG,aAAa;YAC3B,CAAC,CAAC,6BAA6B,KAAK,CAAC,UAAU,MAAM,KAAK,CAAC,KAAK,IAAI,EAAE,EAAE;YACxE,CAAC,CAAC,6BAA6B,KAAK,CAAC,UAAU,GAAG,CAAC;QACrD,OAAO,uBAAuB,CAAC,OAAO,EAAE,GAAG,EAAE,cAAc,CAAC,CAAC;IAC/D,CAAC;IACD,IAAI,KAAK,YAAY,UAAU,EAAE,CAAC;QAChC,MAAM,IAAI,GAAG,KAAK,CAAC,UAAU,CAAC,CAAC,CAAC,QAAQ,KAAK,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC,aAAa,CAAC;QAC3E,MAAM,OAAO,GAAG,aAAa;YAC3B,CAAC,CAAC,GAAG,KAAK,CAAC,OAAO,KAAK,KAAK,CAAC,KAAK,IAAI,EAAE,EAAE;YAC1C,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC;QAClB,OAAO,uBAAuB,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,CAAC,CAAC;IACrD,CAAC;IACD,IAAI,KAAK,YAAY,QAAQ,EAAE,CAAC;QAC9B,MAAM,OAAO,GAAG,aAAa;YAC3B,CAAC,CAAC,GAAG,KAAK,CAAC,OAAO,KAAK,KAAK,CAAC,KAAK,IAAI,EAAE,EAAE;YAC1C,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC;QAClB,OAAO,uBAAuB,CAAC,OAAO,EAAE,GAAG,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC;IAC3D,CAAC;IAED,MAAM,OAAO,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,CAAC;IACzE,MAAM,WAAW,GACf,aAAa,IAAI,KAAK,YAAY,KAAK;QACrC,CAAC,CAAC,GAAG,eAAe,KAAK,OAAO,KAAK,KAAK,CAAC,KAAK,IAAI,EAAE,EAAE;QACxD,CAAC,CAAC,GAAG,eAAe,KAAK,OAAO,EAAE,CAAC;IAEvC,OAAO,uBAAuB,CAAC,WAAW,EAAE,GAAG,EAAE,eAAe,CAAC,CAAC;AACpE,CAAC"}
|
|
@@ -1,3 +1,13 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Check if an IP address is in a blocked private range
|
|
3
|
+
*/
|
|
4
|
+
export declare function isBlockedIp(ip: string): boolean;
|
|
5
|
+
/**
|
|
6
|
+
* Validate resolved IP addresses to prevent DNS rebinding attacks.
|
|
7
|
+
* This should be called after DNS resolution to ensure the resolved
|
|
8
|
+
* IPs are not in blocked private ranges.
|
|
9
|
+
*/
|
|
10
|
+
export declare function validateResolvedIps(hostname: string): Promise<void>;
|
|
1
11
|
export declare function validateAndNormalizeUrl(urlString: string): string;
|
|
2
12
|
export declare function isInternalUrl(url: string, baseUrl: string): boolean;
|
|
3
13
|
//# sourceMappingURL=url-validator.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"url-validator.d.ts","sourceRoot":"","sources":["../../src/utils/url-validator.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"url-validator.d.ts","sourceRoot":"","sources":["../../src/utils/url-validator.ts"],"names":[],"mappings":"AAiCA;;GAEG;AACH,wBAAgB,WAAW,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAE/C;AAED;;;;GAIG;AACH,wBAAsB,mBAAmB,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAmCzE;AAED,wBAAgB,uBAAuB,CAAC,SAAS,EAAE,MAAM,GAAG,MAAM,CA2EjE;AAED,wBAAgB,aAAa,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,OAAO,CAQnE"}
|
|
@@ -1,5 +1,6 @@
|
|
|
1
|
+
import dns from 'dns/promises';
|
|
2
|
+
import { config } from '../config/index.js';
|
|
1
3
|
import { UrlValidationError, ValidationError } from '../errors/app-error.js';
|
|
2
|
-
const MAX_URL_LENGTH = 2048;
|
|
3
4
|
const BLOCKED_HOSTS = new Set([
|
|
4
5
|
'localhost',
|
|
5
6
|
'127.0.0.1',
|
|
@@ -25,8 +26,45 @@ const BLOCKED_IP_PATTERNS = [
|
|
|
25
26
|
/^::ffff:172\.(1[6-9]|2\d|3[01])\./,
|
|
26
27
|
/^::ffff:192\.168\./,
|
|
27
28
|
];
|
|
28
|
-
|
|
29
|
-
|
|
29
|
+
/**
|
|
30
|
+
* Check if an IP address is in a blocked private range
|
|
31
|
+
*/
|
|
32
|
+
export function isBlockedIp(ip) {
|
|
33
|
+
return BLOCKED_IP_PATTERNS.some((pattern) => pattern.test(ip));
|
|
34
|
+
}
|
|
35
|
+
/**
|
|
36
|
+
* Validate resolved IP addresses to prevent DNS rebinding attacks.
|
|
37
|
+
* This should be called after DNS resolution to ensure the resolved
|
|
38
|
+
* IPs are not in blocked private ranges.
|
|
39
|
+
*/
|
|
40
|
+
export async function validateResolvedIps(hostname) {
|
|
41
|
+
// Skip validation for direct IP addresses (already validated in validateAndNormalizeUrl)
|
|
42
|
+
if (/^[\d.]+$/.test(hostname) || hostname.includes(':')) {
|
|
43
|
+
return;
|
|
44
|
+
}
|
|
45
|
+
try {
|
|
46
|
+
// Resolve IPv4 addresses
|
|
47
|
+
const ipv4Addresses = await dns.resolve4(hostname).catch(() => []);
|
|
48
|
+
for (const ip of ipv4Addresses) {
|
|
49
|
+
if (isBlockedIp(ip) || BLOCKED_HOSTS.has(ip)) {
|
|
50
|
+
throw new UrlValidationError(`DNS rebinding detected: ${hostname} resolves to blocked IP ${ip}`, hostname);
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
// Resolve IPv6 addresses
|
|
54
|
+
const ipv6Addresses = await dns.resolve6(hostname).catch(() => []);
|
|
55
|
+
for (const ip of ipv6Addresses) {
|
|
56
|
+
if (isBlockedIp(ip) || BLOCKED_HOSTS.has(ip)) {
|
|
57
|
+
throw new UrlValidationError(`DNS rebinding detected: ${hostname} resolves to blocked IP ${ip}`, hostname);
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
catch (error) {
|
|
62
|
+
// Re-throw UrlValidationError, ignore DNS resolution errors
|
|
63
|
+
if (error instanceof UrlValidationError) {
|
|
64
|
+
throw error;
|
|
65
|
+
}
|
|
66
|
+
// DNS resolution failed - let the actual request handle the error
|
|
67
|
+
}
|
|
30
68
|
}
|
|
31
69
|
export function validateAndNormalizeUrl(urlString) {
|
|
32
70
|
// Check for empty or whitespace-only input
|
|
@@ -38,8 +76,8 @@ export function validateAndNormalizeUrl(urlString) {
|
|
|
38
76
|
throw new ValidationError('URL cannot be empty');
|
|
39
77
|
}
|
|
40
78
|
// Check URL length to prevent DoS
|
|
41
|
-
if (trimmedUrl.length >
|
|
42
|
-
throw new ValidationError(`URL exceeds maximum length of ${
|
|
79
|
+
if (trimmedUrl.length > config.constants.maxUrlLength) {
|
|
80
|
+
throw new ValidationError(`URL exceeds maximum length of ${config.constants.maxUrlLength} characters`, { length: trimmedUrl.length, maxLength: config.constants.maxUrlLength });
|
|
43
81
|
}
|
|
44
82
|
let url;
|
|
45
83
|
try {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"url-validator.js","sourceRoot":"","sources":["../../src/utils/url-validator.ts"],"names":[],"mappings":"AAAA,OAAO,
|
|
1
|
+
{"version":3,"file":"url-validator.js","sourceRoot":"","sources":["../../src/utils/url-validator.ts"],"names":[],"mappings":"AAAA,OAAO,GAAG,MAAM,cAAc,CAAC;AAE/B,OAAO,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAE5C,OAAO,EAAE,kBAAkB,EAAE,eAAe,EAAE,MAAM,wBAAwB,CAAC;AAE7E,MAAM,aAAa,GAAG,IAAI,GAAG,CAAC;IAC5B,WAAW;IACX,WAAW;IACX,SAAS;IACT,KAAK;IACL,iBAAiB;IACjB,0BAA0B;IAC1B,oBAAoB;IACpB,iBAAiB;IACjB,eAAe;CAChB,CAAC,CAAC;AAEH,MAAM,mBAAmB,GAAsB;IAC7C,OAAO;IACP,4BAA4B;IAC5B,aAAa;IACb,QAAQ;IACR,MAAM;IACN,aAAa;IACb,SAAS;IACT,SAAS;IACT,eAAe;IACf,cAAc;IACd,mCAAmC;IACnC,oBAAoB;CACrB,CAAC;AAEF;;GAEG;AACH,MAAM,UAAU,WAAW,CAAC,EAAU;IACpC,OAAO,mBAAmB,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC;AACjE,CAAC;AAED;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,mBAAmB,CAAC,QAAgB;IACxD,yFAAyF;IACzF,IAAI,UAAU,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,QAAQ,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;QACxD,OAAO;IACT,CAAC;IAED,IAAI,CAAC;QACH,yBAAyB;QACzB,MAAM,aAAa,GAAG,MAAM,GAAG,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC,CAAC;QACnE,KAAK,MAAM,EAAE,IAAI,aAAa,EAAE,CAAC;YAC/B,IAAI,WAAW,CAAC,EAAE,CAAC,IAAI,aAAa,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,CAAC;gBAC7C,MAAM,IAAI,kBAAkB,CAC1B,2BAA2B,QAAQ,2BAA2B,EAAE,EAAE,EAClE,QAAQ,CACT,CAAC;YACJ,CAAC;QACH,CAAC;QAED,yBAAyB;QACzB,MAAM,aAAa,GAAG,MAAM,GAAG,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC,CAAC;QACnE,KAAK,MAAM,EAAE,IAAI,aAAa,EAAE,CAAC;YAC/B,IAAI,WAAW,CAAC,EAAE,CAAC,IAAI,aAAa,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,CAAC;gBAC7C,MAAM,IAAI,kBAAkB,CAC1B,2BAA2B,QAAQ,2BAA2B,EAAE,EAAE,EAClE,QAAQ,CACT,CAAC;YACJ,CAAC;QACH,CAAC;IACH,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,4DAA4D;QAC5D,IAAI,KAAK,YAAY,kBAAkB,EAAE,CAAC;YACxC,MAAM,KAAK,CAAC;QACd,CAAC;QACD,kEAAkE;IACpE,CAAC;AACH,CAAC;AAED,MAAM,UAAU,uBAAuB,CAAC,SAAiB;IACvD,2CAA2C;IAC3C,IAAI,CAAC,SAAS,IAAI,OAAO,SAAS,KAAK,QAAQ,EAAE,CAAC;QAChD,MAAM,IAAI,eAAe,CAAC,iBAAiB,CAAC,CAAC;IAC/C,CAAC;IAED,MAAM,UAAU,GAAG,SAAS,CAAC,IAAI,EAAE,CAAC;IACpC,IAAI,CAAC,UAAU,EAAE,CAAC;QAChB,MAAM,IAAI,eAAe,CAAC,qBAAqB,CAAC,CAAC;IACnD,CAAC;IAED,kCAAkC;IAClC,IAAI,UAAU,CAAC,MAAM,GAAG,MAAM,CAAC,SAAS,CAAC,YAAY,EAAE,CAAC;QACtD,MAAM,IAAI,eAAe,CACvB,iCAAiC,MAAM,CAAC,SAAS,CAAC,YAAY,aAAa,EAC3E,EAAE,MAAM,EAAE,UAAU,CAAC,MAAM,EAAE,SAAS,EAAE,MAAM,CAAC,SAAS,CAAC,YAAY,EAAE,CACxE,CAAC;IACJ,CAAC;IAED,IAAI,GAAQ,CAAC;IAEb,IAAI,CAAC;QACH,GAAG,GAAG,IAAI,GAAG,CAAC,UAAU,CAAC,CAAC;IAC5B,CAAC;IAAC,MAAM,CAAC;QACP,MAAM,IAAI,kBAAkB,CAAC,oBAAoB,EAAE,UAAU,CAAC,CAAC;IACjE,CAAC;IAED,+BAA+B;IAC/B,IAAI,GAAG,CAAC,QAAQ,KAAK,OAAO,IAAI,GAAG,CAAC,QAAQ,KAAK,QAAQ,EAAE,CAAC;QAC1D,MAAM,IAAI,kBAAkB,CAC1B,qBAAqB,GAAG,CAAC,QAAQ,qCAAqC,EACtE,UAAU,CACX,CAAC;IACJ,CAAC;IAED,+CAA+C;IAC/C,IAAI,GAAG,CAAC,QAAQ,IAAI,GAAG,CAAC,QAAQ,EAAE,CAAC;QACjC,MAAM,IAAI,kBAAkB,CAC1B,gDAAgD,EAChD,UAAU,CACX,CAAC;IACJ,CAAC;IAED,MAAM,QAAQ,GAAG,GAAG,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC;IAE5C,uBAAuB;IACvB,IAAI,CAAC,QAAQ,EAAE,CAAC;QACd,MAAM,IAAI,kBAAkB,CAAC,gCAAgC,EAAE,UAAU,CAAC,CAAC;IAC7E,CAAC;IAED,sCAAsC;IACtC,IAAI,aAAa,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC;QAChC,MAAM,IAAI,kBAAkB,CAC1B,iBAAiB,QAAQ,kCAAkC,EAC3D,UAAU,CACX,CAAC;IACJ,CAAC;IAED,0BAA0B;IAC1B,IAAI,WAAW,CAAC,QAAQ,CAAC,EAAE,CAAC;QAC1B,MAAM,IAAI,kBAAkB,CAC1B,qBAAqB,QAAQ,+BAA+B,EAC5D,UAAU,CACX,CAAC;IACJ,CAAC;IAED,0EAA0E;IAC1E,IAAI,QAAQ,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,QAAQ,CAAC,QAAQ,CAAC,WAAW,CAAC,EAAE,CAAC;QAClE,MAAM,IAAI,kBAAkB,CAC1B,6BAA6B,QAAQ,4CAA4C,EACjF,UAAU,CACX,CAAC;IACJ,CAAC;IAED,OAAO,GAAG,CAAC,IAAI,CAAC;AAClB,CAAC;AAED,MAAM,UAAU,aAAa,CAAC,GAAW,EAAE,OAAe;IACxD,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;QACrC,MAAM,UAAU,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,CAAC;QACpC,OAAO,MAAM,CAAC,QAAQ,KAAK,UAAU,CAAC,QAAQ,CAAC;IACjD,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC"}
|