@page2ai/core 0.2.2 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/node/fetch-protected.d.ts.map +1 -1
- package/dist/node/fetch-protected.js +51 -15
- package/dist/node/fetch-protected.js.map +1 -1
- package/dist/node/ssrf-guard.d.ts +4 -0
- package/dist/node/ssrf-guard.d.ts.map +1 -1
- package/dist/node/ssrf-guard.js +42 -21
- package/dist/node/ssrf-guard.js.map +1 -1
- package/dist/shared/constants.d.ts +1 -1
- package/dist/shared/constants.js +1 -1
- package/package.json +5 -4
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"fetch-protected.d.ts","sourceRoot":"","sources":["../../src/node/fetch-protected.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"fetch-protected.d.ts","sourceRoot":"","sources":["../../src/node/fetch-protected.ts"],"names":[],"mappings":"AA+BA,MAAM,WAAW,qBAAqB;IACpC,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,kBAAkB,CAAC,EAAE,MAAM,CAAC;CAC7B;AAED,MAAM,WAAW,oBAAoB;IACnC,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;CACpB;AASD,eAAO,MAAM,cAAc,oEAAoE,CAAC;AAGhG,qBAAa,oBAAqB,SAAQ,KAAK;aACD,GAAG,EAAE,MAAM;gBAA3C,MAAM,EAAE,MAAM,EAAkB,GAAG,EAAE,MAAM;CAIxD;AAgCD,eAAO,MAAM,cAAc,GACzB,QAAQ,MAAM,EACd,OAAM,qBAA0B,KAC/B,OAAO,CAAC,oBAAoB,CAqH9B,CAAC"}
|
|
@@ -1,6 +1,8 @@
|
|
|
1
1
|
// Hardened fetch for the Node MCP server:
|
|
2
|
-
// - SSRF guard with DNS resolution
|
|
3
|
-
//
|
|
2
|
+
// - SSRF guard with DNS resolution + pinning: every hop is checked (string +
|
|
3
|
+
// DNS), and the connection is pinned to the validated addresses via an undici
|
|
4
|
+
// Agent so a DNS rebinding between the check and the connect cannot reach
|
|
5
|
+
// an internal address
|
|
4
6
|
// - Redirects followed MANUALLY (redirect: 'manual', max 10 hops) so an
|
|
5
7
|
// internal target is rejected before the request is made, not after
|
|
6
8
|
// - AbortController timeout (default 15s, covers the whole redirect chain)
|
|
@@ -19,7 +21,8 @@
|
|
|
19
21
|
//
|
|
20
22
|
// KNOWN LIMITATION: no charset detection for non-UTF-8 pages (Windows-1251 docs will
|
|
21
23
|
// mojibake); a later release may add it via `fetch-charset-detection`.
|
|
22
|
-
import {
|
|
24
|
+
import { Agent } from 'undici';
|
|
25
|
+
import { assertSafeUrl, resolveAndValidate, SsrfBlockedError, } from './ssrf-guard.js';
|
|
23
26
|
const DEFAULT_UA = 'page2ai-core/0.1 (+https://github.com/igorsaevets/page2ai-core)';
|
|
24
27
|
const DEFAULT_TIMEOUT = 15000;
|
|
25
28
|
const DEFAULT_MAX_BYTES = 10 * 1024 * 1024; // 10 MB
|
|
@@ -38,6 +41,25 @@ export class FetchProtectionError extends Error {
|
|
|
38
41
|
}
|
|
39
42
|
}
|
|
40
43
|
const stripBom = (s) => s.replace(/^/, '');
|
|
44
|
+
const normalizeHost = (hostname) => {
|
|
45
|
+
let h = hostname.toLowerCase().replace(/^\[|\]$/g, '');
|
|
46
|
+
if (h.length > 1 && h.endsWith('.'))
|
|
47
|
+
h = h.slice(0, -1);
|
|
48
|
+
return h;
|
|
49
|
+
};
|
|
50
|
+
const isIpv4Literal = (host) => /^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$/.test(host);
|
|
51
|
+
const createPinnedAgent = (addrs) => new Agent({
|
|
52
|
+
connect: {
|
|
53
|
+
lookup: (_hostname, options, callback) => {
|
|
54
|
+
if (options.all) {
|
|
55
|
+
callback(null, addrs);
|
|
56
|
+
}
|
|
57
|
+
else {
|
|
58
|
+
callback(null, addrs[0].address, addrs[0].family);
|
|
59
|
+
}
|
|
60
|
+
},
|
|
61
|
+
},
|
|
62
|
+
});
|
|
41
63
|
export const fetchProtected = async (rawUrl, opts = {}) => {
|
|
42
64
|
const timeoutMs = opts.timeoutMs ?? DEFAULT_TIMEOUT;
|
|
43
65
|
const maxBytes = opts.maxBytes ?? DEFAULT_MAX_BYTES;
|
|
@@ -47,22 +69,36 @@ export const fetchProtected = async (rawUrl, opts = {}) => {
|
|
|
47
69
|
const timer = setTimeout(() => controller.abort(), timeoutMs);
|
|
48
70
|
try {
|
|
49
71
|
// Follow redirects by hand: validate every hop (string checks + DNS)
|
|
50
|
-
// BEFORE connecting to it
|
|
51
|
-
//
|
|
52
|
-
//
|
|
72
|
+
// BEFORE connecting to it, AND pin the connection to the validated IPs
|
|
73
|
+
// so a DNS rebinding between our check and the connect cannot redirect
|
|
74
|
+
// the socket to an internal address.
|
|
53
75
|
let currentUrl = rawUrl;
|
|
54
76
|
let resp;
|
|
55
77
|
let hops = 0;
|
|
56
78
|
for (;;) {
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
79
|
+
const parsed = assertSafeUrl(currentUrl);
|
|
80
|
+
const host = normalizeHost(parsed.hostname);
|
|
81
|
+
let agent;
|
|
82
|
+
if (!isIpv4Literal(host) && !host.includes(':')) {
|
|
83
|
+
const addrs = await resolveAndValidate(host, currentUrl);
|
|
84
|
+
agent = createPinnedAgent(addrs);
|
|
85
|
+
}
|
|
86
|
+
try {
|
|
87
|
+
resp = await fetch(currentUrl, {
|
|
88
|
+
signal: controller.signal,
|
|
89
|
+
headers: {
|
|
90
|
+
'user-agent': ua,
|
|
91
|
+
accept: BROWSER_ACCEPT,
|
|
92
|
+
},
|
|
93
|
+
redirect: 'manual',
|
|
94
|
+
...(agent ? { dispatcher: agent } : {}),
|
|
95
|
+
});
|
|
96
|
+
}
|
|
97
|
+
finally {
|
|
98
|
+
if (agent) {
|
|
99
|
+
agent.close().catch(() => { });
|
|
100
|
+
}
|
|
101
|
+
}
|
|
66
102
|
if (resp.status >= 300 && resp.status < 400) {
|
|
67
103
|
const location = resp.headers.get('location');
|
|
68
104
|
if (!location) {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"fetch-protected.js","sourceRoot":"","sources":["../../src/node/fetch-protected.ts"],"names":[],"mappings":"AAAA,0CAA0C;AAC1C,
|
|
1
|
+
{"version":3,"file":"fetch-protected.js","sourceRoot":"","sources":["../../src/node/fetch-protected.ts"],"names":[],"mappings":"AAAA,0CAA0C;AAC1C,6EAA6E;AAC7E,gFAAgF;AAChF,4EAA4E;AAC5E,wBAAwB;AACxB,wEAAwE;AACxE,sEAAsE;AACtE,2EAA2E;AAC3E,6DAA6D;AAC7D,wCAAwC;AACxC,+FAA+F;AAC/F,gGAAgG;AAChG,iBAAiB;AACjB,kGAAkG;AAClG,kGAAkG;AAClG,iGAAiG;AACjG,mGAAmG;AACnG,wFAAwF;AACxF,kGAAkG;AAClG,qGAAqG;AACrG,EAAE;AACF,qFAAqF;AACrF,uEAAuE;AAEvE,OAAO,EAAE,KAAK,EAAE,MAAM,QAAQ,CAAC;AAC/B,OAAO,EACL,aAAa,EACb,kBAAkB,EAClB,gBAAgB,GACjB,MAAM,iBAAiB,CAAC;AAiBzB,MAAM,UAAU,GAAG,iEAAiE,CAAC;AACrF,MAAM,eAAe,GAAG,KAAK,CAAC;AAC9B,MAAM,iBAAiB,GAAG,EAAE,GAAG,IAAI,GAAG,IAAI,CAAC,CAAE,QAAQ;AACrD,MAAM,cAAc,GAAG,oFAAoF,CAAC;AAC5G,sGAAsG;AACtG,+GAA+G;AAC/G,4FAA4F;AAC5F,MAAM,CAAC,MAAM,cAAc,GAAG,iEAAiE,CAAC;AAChG,MAAM,aAAa,GAAG,EAAE,CAAC;AAEzB,MAAM,OAAO,oBAAqB,SAAQ,KAAK;IACD;IAA5C,YAAY,MAAc,EAAkB,GAAW;QACrD,KAAK,CAAC,oBAAoB,MAAM,UAAU,GAAG,GAAG,CAAC,CAAC;QADR,QAAG,GAAH,GAAG,CAAQ;QAErD,IAAI,CAAC,IAAI,GAAG,sBAAsB,CAAC;IACrC,CAAC;CACF;AAED,MAAM,QAAQ,GAAG,CAAC,CAAS,EAAU,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;AAE5D,MAAM,aAAa,GAAG,CAAC,QAAgB,EAAU,EAAE;IACjD,IAAI,CAAC,GAAG,QAAQ,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC;IACvD,IAAI,CAAC,CAAC,MAAM,GAAG,CAAC,IAAI,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC;QAAE,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IACxD,OAAO,CAAC,CAAC;AACX,CAAC,CAAC;AAEF,MAAM,aAAa,GAAG,CAAC,IAAY,EAAW,EAAE,CAC9C,sCAAsC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAEpD,MAAM,iBAAiB,GAAG,CACxB,KAA4C,EACrC,EAAE,CACT,IAAI,KAAK,CAAC;IACR,OAAO,EAAE;QACP,MAAM,EAAE,CACN,SAAiB,EACjB,OAA0B,EAC1B,QAAkB,EAClB,EAAE;YACF,IAAI,OAAO,CAAC,GAAG,EAAE,CAAC;gBAChB,QAAQ,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;YACxB,CAAC;iBAAM,CAAC;gBACN,QAAQ,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;YACpD,CAAC;QACH,CAAC;KACF;CACF,CAAC,CAAC;AAEL,MAAM,CAAC,MAAM,cAAc,GAAG,KAAK,EACjC,MAAc,EACd,OAA8B,EAAE,EACD,EAAE;IACjC,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,IAAI,eAAe,CAAC;IACpD,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,IAAI,iBAAiB,CAAC;IACpD,MAAM,EAAE,GAAG,IAAI,CAAC,SAAS,IAAI,UAAU,CAAC;IACxC,MAAM,MAAM,GAAG,IAAI,CAAC,kBAAkB,IAAI,cAAc,CAAC;IAEzD,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;IACzC,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,EAAE,SAAS,CAAC,CAAC;IAE9D,IAAI,CAAC;QACH,qEAAqE;QACrE,uEAAuE;QACvE,uEAAuE;QACvE,qCAAqC;QACrC,IAAI,UAAU,GAAG,MAAM,CAAC;QACxB,IAAI,IAAc,CAAC;QACnB,IAAI,IAAI,GAAG,CAAC,CAAC;QACb,SAAS,CAAC;YACR,MAAM,MAAM,GAAG,aAAa,CAAC,UAAU,CAAC,CAAC;YACzC,MAAM,IAAI,GAAG,aAAa,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;YAC5C,IAAI,KAAwB,CAAC;YAC7B,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;gBAChD,MAAM,KAAK,GAAG,MAAM,kBAAkB,CAAC,IAAI,EAAE,UAAU,CAAC,CAAC;gBACzD,KAAK,GAAG,iBAAiB,CAAC,KAAK,CAAC,CAAC;YACnC,CAAC;YACD,IAAI,CAAC;gBACH,IAAI,GAAG,MAAM,KAAK,CAAC,UAAU,EAAE;oBAC7B,MAAM,EAAE,UAAU,CAAC,MAAM;oBACzB,OAAO,EAAE;wBACP,YAAY,EAAE,EAAE;wBAChB,MAAM,EAAE,cAAc;qBACvB;oBACD,QAAQ,EAAE,QAAQ;oBAClB,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,KAAK,EAA6B,CAAC,CAAC,CAAC,EAAE,CAAC;iBACpD,CAAC,CAAC;YACpB,CAAC;oBAAS,CAAC;gBACT,IAAI,KAAK,EAAE,CAAC;oBACV,KAAK,CAAC,KAAK,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;gBAChC,CAAC;YACH,CAAC;YAED,IAAI,IAAI,CAAC,MAAM,IAAI,GAAG,IAAI,IAAI,CAAC,MAAM,GAAG,GAAG,EAAE,CAAC;gBAC5C,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;gBAC9C,IAAI,CAAC,QAAQ,EAAE,CAAC;oBACd,MAAM,IAAI,oBAAoB,CAC5B,QAAQ,IAAI,CAAC,MAAM,qCAAqC,EACxD,MAAM,CACP,CAAC;gBACJ,CAAC;gBACD,IAAI,CAAC;oBAAC,MAAM,IAAI,CAAC,IAAI,EAAE,MAAM,EAAE,CAAC;gBAAC,CAAC;gBAAC,MAAM,CAAC,CAAC,iBAAiB,CAAC,CAAC;gBAC9D,IAAI,EAAE,CAAC;gBACP,IAAI,IAAI,GAAG,aAAa,EAAE,CAAC;oBACzB,MAAM,IAAI,oBAAoB,CAAC,aAAa,aAAa,YAAY,EAAE,MAAM,CAAC,CAAC;gBACjF,CAAC;gBACD,IAAI,OAAe,CAAC;gBACpB,IAAI,CAAC;oBACH,OAAO,GAAG,IAAI,GAAG,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC,QAAQ,EAAE,CAAC;gBACrD,CAAC;gBAAC,MAAM,CAAC;oBACP,MAAM,IAAI,oBAAoB,CAAC,kCAAkC,QAAQ,GAAG,EAAE,MAAM,CAAC,CAAC;gBACxF,CAAC;gBACD,UAAU,GAAG,OAAO,CAAC;gBACrB,SAAS;YACX,CAAC;YACD,MAAM;QACR,CAAC;QAED,MAAM,QAAQ,GAAG,UAAU,CAAC;QAC5B,MAAM,WAAW,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,IAAI,EAAE,CAAC;QAC3D,IAAI,CAAC,IAAI,CAAC,EAAE,EAAE,CAAC;YACb,MAAM,IAAI,oBAAoB,CAC5B,QAAQ,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,UAAU,kBAAkB,WAAW,EAAE,EACrE,MAAM,CACP,CAAC;QACJ,CAAC;QACD,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,EAAE,CAAC;YAC9B,MAAM,IAAI,oBAAoB,CAC5B,iBAAiB,WAAW,uBAAuB,EACnD,MAAM,CACP,CAAC;QACJ,CAAC;QAED,wEAAwE;QACxE,4EAA4E;QAC5E,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,EAAE,SAAS,EAAE,CAAC;QACtC,IAAI,CAAC,MAAM,EAAE,CAAC;YACZ,MAAM,IAAI,oBAAoB,CAAC,uBAAuB,EAAE,MAAM,CAAC,CAAC;QAClE,CAAC;QACD,MAAM,MAAM,GAAiB,EAAE,CAAC;QAChC,IAAI,SAAS,GAAG,CAAC,CAAC;QAClB,OAAO,IAAI,EAAE,CAAC;YACZ,MAAM,EAAE,KAAK,EAAE,IAAI,EAAE,GAAG,MAAM,MAAM,CAAC,IAAI,EAAE,CAAC;YAC5C,IAAI,IAAI;gBAAE,MAAM;YAChB,IAAI,CAAC,KAAK;gBAAE,SAAS;YACrB,SAAS,IAAI,KAAK,CAAC,UAAU,CAAC;YAC9B,IAAI,SAAS,GAAG,QAAQ,EAAE,CAAC;gBACzB,IAAI,CAAC;oBAAC,MAAM,MAAM,CAAC,MAAM,EAAE,CAAC;gBAAC,CAAC;gBAAC,MAAM,CAAC,CAAC,iBAAiB,CAAC,CAAC;gBAC1D,MAAM,IAAI,oBAAoB,CAC5B,yBAAyB,QAAQ,YAAY,EAC7C,MAAM,CACP,CAAC;YACJ,CAAC;YACD,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACrB,CAAC;QAED,MAAM,GAAG,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,UAAU,EAAE,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;QAChG,MAAM,IAAI,GAAG,QAAQ,CAAC,GAAG,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC;QAE7C,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,SAAS,EAAE,UAAU,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC;IAC7E,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,2EAA2E;QAC3E,IAAI,CAAC,YAAY,gBAAgB,EAAE,CAAC;YAClC,MAAM,IAAI,oBAAoB,CAAC,0BAA0B,CAAC,CAAC,OAAO,EAAE,EAAE,MAAM,CAAC,CAAC;QAChF,CAAC;QACD,MAAM,CAAC,CAAC;IACV,CAAC;YAAS,CAAC;QACT,YAAY,CAAC,KAAK,CAAC,CAAC;IACtB,CAAC;AACH,CAAC,CAAC"}
|
|
@@ -4,5 +4,9 @@ export declare class SsrfBlockedError extends Error {
|
|
|
4
4
|
}
|
|
5
5
|
export declare const findIpBlockReason: (addr: string) => string | null;
|
|
6
6
|
export declare const assertSafeUrl: (rawUrl: string) => URL;
|
|
7
|
+
export declare const resolveAndValidate: (hostname: string, rawUrl: string) => Promise<{
|
|
8
|
+
address: string;
|
|
9
|
+
family: number;
|
|
10
|
+
}[]>;
|
|
7
11
|
export declare const assertSafeUrlResolved: (rawUrl: string) => Promise<URL>;
|
|
8
12
|
//# sourceMappingURL=ssrf-guard.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"ssrf-guard.d.ts","sourceRoot":"","sources":["../../src/node/ssrf-guard.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"ssrf-guard.d.ts","sourceRoot":"","sources":["../../src/node/ssrf-guard.ts"],"names":[],"mappings":"AA4DA,qBAAa,gBAAiB,SAAQ,KAAK;aACG,GAAG,EAAE,MAAM;gBAA3C,MAAM,EAAE,MAAM,EAAkB,GAAG,EAAE,MAAM;CAIxD;AAgBD,eAAO,MAAM,iBAAiB,GAAI,MAAM,MAAM,KAAG,MAAM,GAAG,IAqBzD,CAAC;AAEF,eAAO,MAAM,aAAa,GAAI,QAAQ,MAAM,KAAG,GAmB9C,CAAC;AAIF,eAAO,MAAM,kBAAkB,GAC7B,UAAU,MAAM,EAChB,QAAQ,MAAM,KACb,OAAO,CAAC;IAAE,OAAO,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,EAAE,CAoB/C,CAAC;AAIF,eAAO,MAAM,qBAAqB,GAAU,QAAQ,MAAM,KAAG,OAAO,CAAC,GAAG,CAQvE,CAAC"}
|
package/dist/node/ssrf-guard.js
CHANGED
|
@@ -11,21 +11,26 @@
|
|
|
11
11
|
// name resolving to 127.0.0.1 / 169.254.169.254
|
|
12
12
|
// sailed through the string-only check.
|
|
13
13
|
//
|
|
14
|
-
//
|
|
15
|
-
//
|
|
16
|
-
//
|
|
17
|
-
//
|
|
18
|
-
// for now to keep the dependency surface at linkedom + css.escape. Stated in
|
|
19
|
-
// SECURITY.md so the promise matches the code.
|
|
14
|
+
// DNS pinning (BL-22): fetchProtected now creates an undici Agent with a
|
|
15
|
+
// connect.lookup that returns only the addresses we validated, so the runtime
|
|
16
|
+
// cannot re-resolve to a different (internal) address between our check and
|
|
17
|
+
// the connect.
|
|
20
18
|
import { lookup } from 'node:dns/promises';
|
|
21
|
-
const
|
|
19
|
+
const BLOCKED_IPV4_PATTERNS = [
|
|
22
20
|
/^127\./, // 127.0.0.0/8 loopback
|
|
23
21
|
/^10\./, // 10.0.0.0/8 private
|
|
24
22
|
/^169\.254\./, // 169.254.0.0/16 link-local (AWS/GCP metadata)
|
|
25
23
|
/^172\.(1[6-9]|2[0-9]|3[0-1])\./, // 172.16.0.0/12 private
|
|
26
24
|
/^192\.168\./, // 192.168.0.0/16 private
|
|
27
25
|
/^0\./, // 0.0.0.0/8 current network
|
|
28
|
-
/^100\.(6[4-9]|[7-9][0-9]|1[01][0-9]|12[0-7])\./, // 100.64/10 CGNAT
|
|
26
|
+
/^100\.(6[4-9]|[7-9][0-9]|1[01][0-9]|12[0-7])\./, // 100.64.0.0/10 CGNAT
|
|
27
|
+
/^(22[4-9]|23\d)\./, // 224.0.0.0/4 multicast
|
|
28
|
+
/^(24\d|25[0-5])\./, // 240.0.0.0/4 reserved + 255.255.255.255 broadcast
|
|
29
|
+
/^192\.0\.[02]\./, // 192.0.0.0/24 IETF Protocol Assignments + 192.0.2.0/24 TEST-NET-1
|
|
30
|
+
/^192\.88\.99\./, // 192.88.99.0/24 deprecated 6to4 relay anycast (RFC 7526)
|
|
31
|
+
/^198\.51\.100\./, // 198.51.100.0/24 TEST-NET-2
|
|
32
|
+
/^203\.0\.113\./, // 203.0.113.0/24 TEST-NET-3
|
|
33
|
+
/^198\.1[89]\./, // 198.18.0.0/15 benchmarking
|
|
29
34
|
];
|
|
30
35
|
const BLOCKED_IPV6_PREFIXES = [
|
|
31
36
|
'::1', // loopback
|
|
@@ -35,6 +40,10 @@ const BLOCKED_IPV6_PREFIXES = [
|
|
|
35
40
|
'fe8', 'fe9', 'fea', 'feb',
|
|
36
41
|
'fc', // unique local (fc00::/7)
|
|
37
42
|
'fd', // unique local
|
|
43
|
+
'64:ff9b:', // NAT64 well-known (RFC 6146) + local-use (RFC 8215) — embeds IPv4
|
|
44
|
+
'2002:', // 6to4 (RFC 3056) — embeds IPv4 in bits 16-47
|
|
45
|
+
'2001:db8:', // documentation (RFC 3849)
|
|
46
|
+
'100:', // discard-only (RFC 6666); rest of 0100::/16 is unassigned
|
|
38
47
|
];
|
|
39
48
|
const BLOCKED_HOSTNAMES = new Set([
|
|
40
49
|
'localhost',
|
|
@@ -72,10 +81,16 @@ export const findIpBlockReason = (addr) => {
|
|
|
72
81
|
if (a.startsWith(prefix))
|
|
73
82
|
return `IPv6 in blocked range (${prefix})`;
|
|
74
83
|
}
|
|
84
|
+
// Teredo 2001::/32 — two canonical forms: 2001::x (all-zero groups compressed)
|
|
85
|
+
// and 2001:0:x (group 1 explicit zero). Must not match 2001:db8: (documentation,
|
|
86
|
+
// already caught) or 2001:4860: (Google DNS, legitimate).
|
|
87
|
+
if (a.startsWith('2001::') || a.startsWith('2001:0:')) {
|
|
88
|
+
return 'IPv6 in blocked range (Teredo 2001::/32)';
|
|
89
|
+
}
|
|
75
90
|
return null;
|
|
76
91
|
}
|
|
77
92
|
if (isIpv4Literal(a)) {
|
|
78
|
-
for (const p of
|
|
93
|
+
for (const p of BLOCKED_IPV4_PATTERNS) {
|
|
79
94
|
if (p.test(a))
|
|
80
95
|
return 'IPv4 in private range';
|
|
81
96
|
}
|
|
@@ -103,30 +118,36 @@ export const assertSafeUrl = (rawUrl) => {
|
|
|
103
118
|
}
|
|
104
119
|
return parsed;
|
|
105
120
|
};
|
|
106
|
-
//
|
|
107
|
-
//
|
|
108
|
-
export const
|
|
109
|
-
const parsed = assertSafeUrl(rawUrl);
|
|
110
|
-
const host = normalizeHost(parsed.hostname);
|
|
111
|
-
if (isIpv4Literal(host) || host.includes(':')) {
|
|
112
|
-
return parsed; // IP literal: already fully checked above.
|
|
113
|
-
}
|
|
121
|
+
// Resolve a hostname via DNS, check every returned address, return them.
|
|
122
|
+
// Exported so fetchProtected can pin the connection to these addresses.
|
|
123
|
+
export const resolveAndValidate = async (hostname, rawUrl) => {
|
|
114
124
|
let addrs;
|
|
115
125
|
try {
|
|
116
|
-
addrs = await lookup(
|
|
126
|
+
addrs = await lookup(hostname, { all: true, verbatim: true });
|
|
117
127
|
}
|
|
118
128
|
catch {
|
|
119
|
-
throw new SsrfBlockedError(`DNS resolution failed for "${
|
|
129
|
+
throw new SsrfBlockedError(`DNS resolution failed for "${hostname}"`, rawUrl);
|
|
120
130
|
}
|
|
121
131
|
if (!addrs || addrs.length === 0) {
|
|
122
|
-
throw new SsrfBlockedError(`DNS returned no addresses for "${
|
|
132
|
+
throw new SsrfBlockedError(`DNS returned no addresses for "${hostname}"`, rawUrl);
|
|
123
133
|
}
|
|
124
134
|
for (const { address } of addrs) {
|
|
125
135
|
const reason = findIpBlockReason(address);
|
|
126
136
|
if (reason) {
|
|
127
|
-
throw new SsrfBlockedError(`hostname "${
|
|
137
|
+
throw new SsrfBlockedError(`hostname "${hostname}" resolves to blocked address ${address} (${reason})`, rawUrl);
|
|
128
138
|
}
|
|
129
139
|
}
|
|
140
|
+
return addrs;
|
|
141
|
+
};
|
|
142
|
+
// String checks plus DNS: resolve the hostname and run every returned address
|
|
143
|
+
// through the same IP rules. Fail closed on resolution errors.
|
|
144
|
+
export const assertSafeUrlResolved = async (rawUrl) => {
|
|
145
|
+
const parsed = assertSafeUrl(rawUrl);
|
|
146
|
+
const host = normalizeHost(parsed.hostname);
|
|
147
|
+
if (isIpv4Literal(host) || host.includes(':')) {
|
|
148
|
+
return parsed;
|
|
149
|
+
}
|
|
150
|
+
await resolveAndValidate(host, rawUrl);
|
|
130
151
|
return parsed;
|
|
131
152
|
};
|
|
132
153
|
//# sourceMappingURL=ssrf-guard.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"ssrf-guard.js","sourceRoot":"","sources":["../../src/node/ssrf-guard.ts"],"names":[],"mappings":"AAAA,2EAA2E;AAC3E,mEAAmE;AACnE,EAAE;AACF,cAAc;AACd,2EAA2E;AAC3E,6EAA6E;AAC7E,qDAAqD;AACrD,gFAAgF;AAChF,4EAA4E;AAC5E,6EAA6E;AAC7E,gFAAgF;AAChF,wEAAwE;AACxE,EAAE;AACF
|
|
1
|
+
{"version":3,"file":"ssrf-guard.js","sourceRoot":"","sources":["../../src/node/ssrf-guard.ts"],"names":[],"mappings":"AAAA,2EAA2E;AAC3E,mEAAmE;AACnE,EAAE;AACF,cAAc;AACd,2EAA2E;AAC3E,6EAA6E;AAC7E,qDAAqD;AACrD,gFAAgF;AAChF,4EAA4E;AAC5E,6EAA6E;AAC7E,gFAAgF;AAChF,wEAAwE;AACxE,EAAE;AACF,yEAAyE;AACzE,8EAA8E;AAC9E,4EAA4E;AAC5E,eAAe;AAEf,OAAO,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;AAE3C,MAAM,qBAAqB,GAAa;IACtC,QAAQ,EAA8B,uBAAuB;IAC7D,OAAO,EAA+B,qBAAqB;IAC3D,aAAa,EAAyB,+CAA+C;IACrF,gCAAgC,EAAM,wBAAwB;IAC9D,aAAa,EAAyB,yBAAyB;IAC/D,MAAM,EAAgC,4BAA4B;IAClE,gDAAgD,EAAG,sBAAsB;IACzE,mBAAmB,EAAmB,wBAAwB;IAC9D,mBAAmB,EAAmB,mDAAmD;IACzF,iBAAiB,EAAqB,mEAAmE;IACzG,gBAAgB,EAAsB,0DAA0D;IAChG,iBAAiB,EAAqB,6BAA6B;IACnE,gBAAgB,EAAsB,4BAA4B;IAClE,eAAe,EAAuB,6BAA6B;CACpE,CAAC;AAEF,MAAM,qBAAqB,GAAa;IACtC,KAAK,EAAO,WAAW;IACvB,SAAS,EAAG,sDAAsD;IAClE,oGAAoG;IACpG,8GAA8G;IAC9G,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK;IAC1B,IAAI,EAAQ,0BAA0B;IACtC,IAAI,EAAQ,eAAe;IAC3B,UAAU,EAAE,mEAAmE;IAC/E,OAAO,EAAK,8CAA8C;IAC1D,WAAW,EAAC,2BAA2B;IACvC,MAAM,EAAM,2DAA2D;CACxE,CAAC;AAEF,MAAM,iBAAiB,GAAG,IAAI,GAAG,CAAC;IAChC,WAAW;IACX,SAAS;IACT,0BAA0B;IAC1B,UAAU;CACX,CAAC,CAAC;AAEH,MAAM,eAAe,GAAG,IAAI,GAAG,CAAC,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC,CAAC;AAErD,MAAM,OAAO,gBAAiB,SAAQ,KAAK;IACG;IAA5C,YAAY,MAAc,EAAkB,GAAW;QACrD,KAAK,CAAC,sBAAsB,GAAG,KAAK,MAAM,EAAE,CAAC,CAAC;QADJ,QAAG,GAAH,GAAG,CAAQ;QAErD,IAAI,CAAC,IAAI,GAAG,kBAAkB,CAAC;IACjC,CAAC;CACF;AAED,MAAM,aAAa,GAAG,CAAC,IAAY,EAAW,EAAE,CAC9C,sCAAsC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAEpD,gFAAgF;AAChF,0EAA0E;AAC1E,qCAAqC;AACrC,MAAM,aAAa,GAAG,CAAC,QAAgB,EAAU,EAAE;IACjD,IAAI,CAAC,GAAG,QAAQ,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC;IACvD,IAAI,CAAC,CAAC,MAAM,GAAG,CAAC,IAAI,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC;QAAE,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IACxD,OAAO,CAAC,CAAC;AACX,CAAC,CAAC;AAEF,mEAAmE;AACnE,0EAA0E;AAC1E,MAAM,CAAC,MAAM,iBAAiB,GAAG,CAAC,IAAY,EAAiB,EAAE;IAC/D,MAAM,CAAC,GAAG,IAAI,CAAC,WAAW,EAAE,CAAC;IAC7B,IAAI,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;QACpB,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,KAAK;YAAE,OAAO,2BAA2B,CAAC;QAClE,KAAK,MAAM,MAAM,IAAI,qBAAqB,EAAE,CAAC;YAC3C,IAAI,CAAC,CAAC,UAAU,CAAC,MAAM,CAAC;gBAAE,OAAO,0BAA0B,MAAM,GAAG,CAAC;QACvE,CAAC;QACD,+EAA+E;QAC/E,iFAAiF;QACjF,0DAA0D;QAC1D,IAAI,CAAC,CAAC,UAAU,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE,CAAC;YACtD,OAAO,0CAA0C,CAAC;QACpD,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC;IACD,IAAI,aAAa,CAAC,CAAC,CAAC,EAAE,CAAC;QACrB,KAAK,MAAM,CAAC,IAAI,qBAAqB,EAAE,CAAC;YACtC,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;gBAAE,OAAO,uBAAuB,CAAC;QAChD,CAAC;IACH,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,aAAa,GAAG,CAAC,MAAc,EAAO,EAAE;IACnD,IAAI,MAAW,CAAC;IAChB,IAAI,CAAC;QACH,MAAM,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,CAAC;IAC3B,CAAC;IAAC,MAAM,CAAC;QACP,MAAM,IAAI,gBAAgB,CAAC,eAAe,EAAE,MAAM,CAAC,CAAC;IACtD,CAAC;IACD,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC;QAC1C,MAAM,IAAI,gBAAgB,CAAC,WAAW,MAAM,CAAC,QAAQ,0BAA0B,EAAE,MAAM,CAAC,CAAC;IAC3F,CAAC;IACD,MAAM,IAAI,GAAG,aAAa,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;IAC5C,IAAI,iBAAiB,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;QAChC,MAAM,IAAI,gBAAgB,CAAC,aAAa,IAAI,WAAW,EAAE,MAAM,CAAC,CAAC;IACnE,CAAC;IACD,MAAM,MAAM,GAAG,iBAAiB,CAAC,IAAI,CAAC,CAAC;IACvC,IAAI,MAAM,EAAE,CAAC;QACX,MAAM,IAAI,gBAAgB,CAAC,GAAG,MAAM,MAAM,IAAI,GAAG,EAAE,MAAM,CAAC,CAAC;IAC7D,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC,CAAC;AAEF,yEAAyE;AACzE,wEAAwE;AACxE,MAAM,CAAC,MAAM,kBAAkB,GAAG,KAAK,EACrC,QAAgB,EAChB,MAAc,EACkC,EAAE;IAClD,IAAI,KAA4C,CAAC;IACjD,IAAI,CAAC;QACH,KAAK,GAAG,MAAM,MAAM,CAAC,QAAQ,EAAE,EAAE,GAAG,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC;IAChE,CAAC;IAAC,MAAM,CAAC;QACP,MAAM,IAAI,gBAAgB,CAAC,8BAA8B,QAAQ,GAAG,EAAE,MAAM,CAAC,CAAC;IAChF,CAAC;IACD,IAAI,CAAC,KAAK,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACjC,MAAM,IAAI,gBAAgB,CAAC,kCAAkC,QAAQ,GAAG,EAAE,MAAM,CAAC,CAAC;IACpF,CAAC;IACD,KAAK,MAAM,EAAE,OAAO,EAAE,IAAI,KAAK,EAAE,CAAC;QAChC,MAAM,MAAM,GAAG,iBAAiB,CAAC,OAAO,CAAC,CAAC;QAC1C,IAAI,MAAM,EAAE,CAAC;YACX,MAAM,IAAI,gBAAgB,CACxB,aAAa,QAAQ,iCAAiC,OAAO,KAAK,MAAM,GAAG,EAC3E,MAAM,CACP,CAAC;QACJ,CAAC;IACH,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC,CAAC;AAEF,8EAA8E;AAC9E,+DAA+D;AAC/D,MAAM,CAAC,MAAM,qBAAqB,GAAG,KAAK,EAAE,MAAc,EAAgB,EAAE;IAC1E,MAAM,MAAM,GAAG,aAAa,CAAC,MAAM,CAAC,CAAC;IACrC,MAAM,IAAI,GAAG,aAAa,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;IAC5C,IAAI,aAAa,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;QAC9C,OAAO,MAAM,CAAC;IAChB,CAAC;IACD,MAAM,kBAAkB,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;IACvC,OAAO,MAAM,CAAC;AAChB,CAAC,CAAC"}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import type { ExtractorConfig } from './types.js';
|
|
2
|
-
export declare const CORE_VERSION = "0.
|
|
2
|
+
export declare const CORE_VERSION = "0.3.0";
|
|
3
3
|
export declare const INDENT = " ";
|
|
4
4
|
export declare const DEFAULTS: ExtractorConfig;
|
|
5
5
|
export declare const BLOCK_TAGS: Set<string>;
|
package/dist/shared/constants.js
CHANGED
|
@@ -6,7 +6,7 @@
|
|
|
6
6
|
// in @page2ai/mcp, caught T51) — never add another literal. This copy itself drifted at 0.1.9
|
|
7
7
|
// (frontmatter said 0.1.8 in live ChatGPT output); tests/shared/version-sync.test.ts now fails
|
|
8
8
|
// the suite whenever the two copies disagree.
|
|
9
|
-
export const CORE_VERSION = '0.
|
|
9
|
+
export const CORE_VERSION = '0.3.0';
|
|
10
10
|
// Indent constant: 3 spaces for CommonMark-compatible nested lists.
|
|
11
11
|
export const INDENT = ' ';
|
|
12
12
|
export const DEFAULTS = {
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@page2ai/core",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.3.0",
|
|
4
4
|
"description": "Extract clean Markdown from any web page HTML. Works in Node.js (via linkedom) and browsers. Powers the @page2ai/mcp server.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"exports": {
|
|
@@ -33,7 +33,7 @@
|
|
|
33
33
|
"LICENSE"
|
|
34
34
|
],
|
|
35
35
|
"engines": {
|
|
36
|
-
"node": ">=18.
|
|
36
|
+
"node": ">=18.17.0"
|
|
37
37
|
},
|
|
38
38
|
"scripts": {
|
|
39
39
|
"build": "tsc -p tsconfig.build.json",
|
|
@@ -48,13 +48,14 @@
|
|
|
48
48
|
},
|
|
49
49
|
"dependencies": {
|
|
50
50
|
"css.escape": "^1.5.1",
|
|
51
|
-
"linkedom": "^0.18.11"
|
|
51
|
+
"linkedom": "^0.18.11",
|
|
52
|
+
"undici": "^6.29.1"
|
|
52
53
|
},
|
|
53
54
|
"devDependencies": {
|
|
54
55
|
"@types/css.escape": "^1.5.2",
|
|
55
56
|
"@types/node": "^26.2.0",
|
|
56
57
|
"typescript": "^5.7.2",
|
|
57
|
-
"vitest": "^
|
|
58
|
+
"vitest": "^5.0.5"
|
|
58
59
|
},
|
|
59
60
|
"keywords": [
|
|
60
61
|
"html-to-markdown",
|