domma-cms 1.7.1 → 1.7.2

package/server/services/wpImport/fetch.js

@@ -0,0 +1,264 @@
+/**
+ * Downloading media from the old WordPress site - without becoming a way to
+ * reach anything else (SSRF).
+ *
+ *  - Only the WordPress site's own host names (from its export) are fetched,
+ *    over http(s), on ports 80 and 443 only.
+ *  - The host name is resolved first, and EVERY address it resolves to must be
+ *    public: loopback, private, link-local (cloud metadata lives at
+ *    169.254.169.254), CGNAT, multicast, reserved and documentation ranges are
+ *    refused, IPv4-mapped and NAT64 IPv6 forms included. The connection then
+ *    goes to the address that was checked (a pinned lookup), so a second DNS
+ *    answer cannot swap in an internal one.
+ *  - Redirects are followed only to the same allowed hosts (each re-checked),
+ *    at most three.
+ *  - A size cap (refused early on Content-Length, cut off while streaming), a
+ *    time limit for the whole download, and only 200 answers.
+ *
+ * `allowPrivate` exists for tests and the local browser check, which serve
+ * files from 127.0.0.1; the runner only passes it when testHostsAllowed() says
+ * so - never in production.
+ *
+ * @module services/wpImport/fetch
+ */
+import dns from 'node:dns';
+import http from 'node:http';
+import https from 'node:https';
+import net from 'node:net';
+import os from 'node:os';
+
+export const FETCH_PORTS = Object.freeze([80, 443]);
+
+/** The port an address would be fetched on. */
+export function portOf(raw) {
+    try { const u = new URL(raw); return u.port ? Number(u.port) : (u.protocol === 'https:' ? 443 : 80); } catch { return 0; }
+}
+
+export const FETCH_LIMITS = Object.freeze({maxBytes: 50 * 1024 * 1024, timeoutMs: 60_000, maxRedirects: 3});
+
+export class FetchRefused extends Error {
+    constructor(message) { super(message); this.code = 'EREFUSED_SSRF'; }
+}
+
+/**
+ * The test-only switch: private addresses are allowed only outside
+ * production AND with WP_IMPORT_ALLOW_PRIVATE_HOSTS=1 set.
+ */
+export function testHostsAllowed(env = process.env) {
+    return env.NODE_ENV !== 'production' && env.WP_IMPORT_ALLOW_PRIVATE_HOSTS === '1';
+}
+
+const V4_BLOCKED = [
+    ['0.0.0.0', 8], ['10.0.0.0', 8], ['100.64.0.0', 10], ['127.0.0.0', 8], ['169.254.0.0', 16], ['172.16.0.0', 12],
+    ['192.0.0.0', 24], ['192.0.2.0', 24], ['192.88.99.0', 24], ['192.168.0.0', 16], ['198.18.0.0', 15],
+    ['198.51.100.0', 24], ['203.0.113.0', 24], ['224.0.0.0', 4], ['240.0.0.0', 4]
+];
+
+function v4ToInt(ip) {
+    const p = ip.split('.').map(Number);
+    if (p.length !== 4 || p.some(n => !Number.isInteger(n) || n < 0 || n > 255)) return null;
+    return ((p[0] << 24) >>> 0) + (p[1] << 16) + (p[2] << 8) + p[3];
+}
+
+function v4Blocked(ip) {
+    const n = v4ToInt(ip);
+    if (n === null) return true;
+    return V4_BLOCKED.some(([base, bits]) => {
+        const mask = bits === 0 ? 0 : (~0 << (32 - bits)) >>> 0;
+        return ((n & mask) >>> 0) === ((v4ToInt(base) & mask) >>> 0);
+    });
+}
+
+/** Expand an IPv6 address to 8 groups of numbers (null when it is not one). */
+function v6Groups(ip) {
+    let s = ip.toLowerCase().split('%')[0];
+    // A dotted IPv4 tail (::ffff:1.2.3.4).
+    const tail = /(\d+\.\d+\.\d+\.\d+)$/.exec(s);
+    if (tail) {
+        const n = v4ToInt(tail[1]);
+        if (n === null) return null;
+        s = s.slice(0, -tail[1].length) + `${(n >>> 16).toString(16)}:${(n & 0xffff).toString(16)}`;
+    }
+    const halves = s.split('::');
+    if (halves.length > 2) return null;
+    const left = halves[0] ? halves[0].split(':') : [];
+    const right = halves.length === 2 && halves[1] ? halves[1].split(':') : [];
+    const fill = halves.length === 2 ? 8 - left.length - right.length : 0;
+    const groups = [...left, ...Array(Math.max(0, fill)).fill('0'), ...right].map(g => Number.parseInt(g || '0', 16));
+    return groups.length === 8 && groups.every(g => Number.isInteger(g) && g >= 0 && g <= 0xffff) ? groups : null;
+}
+
+/**
+ * Is this address one a server-side fetch must never reach?
+ *
+ * @param {string} ip
+ * @returns {boolean}
+ */
+export function isBlockedAddress(ip) {
+    const kind = net.isIP(String(ip || ''));
+    if (kind === 4) return v4Blocked(ip);
+    if (kind !== 6) return true;
+    const g = v6Groups(ip);
+    if (!g) return true;
+    const embeddedV4 = () => `${g[6] >> 8}.${g[6] & 255}.${g[7] >> 8}.${g[7] & 255}`;
+    if (g.every(x => x === 0)) return true;                                   // ::
+    if (g.slice(0, 7).every(x => x === 0) && g[7] === 1) return true;         // ::1
+    if (g.slice(0, 5).every(x => x === 0) && g[5] === 0xffff) return v4Blocked(embeddedV4()); // ::ffff:a.b.c.d
+    if (g.slice(0, 4).every(x => x === 0) && g[4] === 0xffff && g[5] === 0) return true;      // ::ffff:0:0/96 (SIIT)
+    if (g[0] === 0x64 && g[1] === 0xff9b && g[2] === 1) return true;          // 64:ff9b:1::/48 (local NAT64)
+    if (g[0] === 0x100 && g[1] === 0 && g[2] === 0 && g[3] === 0) return true; // 100::/64 (discard)
+    if (g.slice(0, 6).every(x => x === 0)) return true;                       // ::a.b.c.d (deprecated compat)
+    if (g[0] === 0x64 && g[1] === 0xff9b && g.slice(2, 6).every(x => x === 0)) return v4Blocked(embeddedV4()); // NAT64
+    if ((g[0] & 0xfe00) === 0xfc00) return true;                              // fc00::/7 unique local
+    if ((g[0] & 0xffc0) === 0xfe80) return true;                              // fe80::/10 link-local
+    if ((g[0] & 0xffc0) === 0xfec0) return true;                              // fec0::/10 site-local
+    if ((g[0] & 0xff00) === 0xff00) return true;                              // multicast
+    if (g[0] === 0x2001 && g[1] === 0x0db8) return true;                      // documentation
+    if (g[0] === 0x2002) return v4Blocked(`${g[1] >> 8}.${g[1] & 255}.${g[2] >> 8}.${g[2] & 255}`); // 6to4
+    if (g[0] === 0x2001 && g[1] === 0) return true;                           // Teredo
+    return false;
+}
+
+/** Canonical text for an address, so the two spellings of one IPv6 address compare equal. */
+function canon(ip) {
+    const s = String(ip || '').toLowerCase().split('%')[0];
+    if (net.isIP(s) !== 6) return s;
+    const g = v6Groups(s);
+    if (g && g.slice(0, 5).every(x => x === 0) && g[5] === 0xffff) return `${g[6] >> 8}.${g[6] & 255}.${g[7] >> 8}.${g[7] & 255}`;
+    return g ? g.map(x => x.toString(16)).join(':') : s;
+}
+
+/**
+ * This machine's own addresses - every interface, public ones included, plus
+ * the unspecified ones. A server must never fetch from itself, whatever its
+ * public address resolves as.
+ */
+export function ownAddresses(interfaces = os.networkInterfaces()) {
+    const out = new Set(['0.0.0.0', canon('::')]);
+    for (const list of Object.values(interfaces || {})) {
+        for (const a of list || []) if (a?.address) out.add(canon(a.address));
+    }
+    return out;
+}
+
+/**
+ * Resolve a host and check every address.
+ *
+ * @returns {Promise<{address: string, family: number}>} the address to connect to
+ */
+export async function resolvePublic(hostname, {allowPrivate = false, lookup = dns.promises.lookup, own = ownAddresses()} = {}) {
+    const host = String(hostname || '').replace(/^\[|\]$/g, '');
+    let addrs;
+    if (net.isIP(host)) addrs = [{address: host, family: net.isIP(host)}];
+    else {
+        try { addrs = await lookup(host, {all: true, verbatim: true}); } catch (err) { throw new FetchRefused(`${host} could not be found (${err.code || err.message}).`); }
+    }
+    if (!addrs?.length) throw new FetchRefused(`${host} could not be found.`);
+    if (!allowPrivate) {
+        const bad = addrs.find(a => isBlockedAddress(a.address));
+        if (bad) throw new FetchRefused(`${host} points at a private or reserved address (${bad.address}) - refused.`);
+        const self = addrs.find(a => own.has(canon(a.address)));
+        if (self) throw new FetchRefused(`${host} points at this server itself (${self.address}) - refused.`);
+    }
+    return addrs[0];
+}
+
+/** May this URL be fetched at all (before any DNS)? Returns the URL, or throws. */
+export function checkUrl(raw, {hosts = [], allowPrivate = false} = {}) {
+    let u;
+    try { u = new URL(String(raw || '')); } catch { throw new FetchRefused('Not a web address.'); }
+    if (u.protocol !== 'http:' && u.protocol !== 'https:') throw new FetchRefused(`${u.protocol} addresses are not fetched.`);
+    if (u.username || u.password) throw new FetchRefused('Addresses with a user name or password are not fetched.');
+    const host = u.hostname.toLowerCase();
+    if (!hosts.includes(host)) throw new FetchRefused(`${host} is not the WordPress site's own host - not fetched.`);
+    const port = u.port ? Number(u.port) : (u.protocol === 'https:' ? 443 : 80);
+    // Web ports only. The tests' local server is the one exception (testHostsAllowed, never in production).
+    if (!allowPrivate && !FETCH_PORTS.includes(port)) throw new FetchRefused(`Port ${port} is not fetched - only 80 and 443.`);
+    return u;
+}
+
+/**
+ * Download one file.
+ *
+ * @param {string} url
+ * @param {object} opts
+ * @param {string[]} opts.hosts - host names allowed (the WordPress site's own)
+ * @param {number} [opts.maxBytes]
+ * @param {number} [opts.timeoutMs]
+ * @param {number} [opts.maxRedirects]
+ * @param {boolean} [opts.allowPrivate] - tests only (see testHostsAllowed)
+ * @param {Function} [opts.lookup] - dns.promises.lookup, swappable in tests
+ * @param {AbortSignal} [opts.signal]
+ * @returns {Promise<{buffer: Buffer, contentType: string, url: string}>}
+ */
+export async function fetchMedia(url, opts = {}) {
+    const lim = {...FETCH_LIMITS, ...opts};
+    const deadline = Date.now() + lim.timeoutMs;
+    let current = url;
+    for (let hop = 0; hop <= lim.maxRedirects; hop++) {
+        const u = checkUrl(current, opts);
+        const {address, family} = await resolvePublic(u.hostname, opts);
+        const res = await request(u, {address, family, deadline, maxBytes: lim.maxBytes, signal: opts.signal});
+        if (res.redirect) {
+            if (hop === lim.maxRedirects) throw new FetchRefused('Too many redirects.');
+            current = new URL(res.redirect, u).href;
+            continue;
+        }
+        return {buffer: res.buffer, contentType: res.contentType, url: u.href};
+    }
+    throw new FetchRefused('Too many redirects.');
+}
+
+function request(u, {address, family, deadline, maxBytes, signal}) {
+    return new Promise((resolve, reject) => {
+        const lib = u.protocol === 'https:' ? https : http;
+        let done = false;
+        const finish = (fn, v) => { if (!done) { done = true; clearTimeout(timer); fn(v); } };
+        const req = lib.request({
+            protocol: u.protocol,
+            hostname: u.hostname,
+            port: u.port || undefined,
+            path: `${u.pathname}${u.search}`,
+            method: 'GET',
+            headers: {'user-agent': 'DommaCMS-WordPress-Import/1', accept: '*/*'},
+            // Connect to the address that was checked - not to a fresh lookup.
+            lookup: (_host, options, cb) => {
+                if (options?.all) cb(null, [{address, family}]);
+                else cb(null, address, family);
+            },
+            agent: false,
+            signal
+        }, (res) => {
+            const status = res.statusCode || 0;
+            if (status >= 300 && status < 400 && res.headers.location) {
+                res.resume();
+                return finish(resolve, {redirect: String(res.headers.location)});
+            }
+            if (status !== 200) {
+                res.resume();
+                return finish(reject, Object.assign(new Error(`The old site answered ${status}.`), {code: 'EHTTP', status}));
+            }
+            const declared = Number(res.headers['content-length']);
+            if (Number.isFinite(declared) && declared > maxBytes) {
+                res.destroy();
+                return finish(reject, new FetchRefused(`The file is ${Math.round(declared / 1048576)} MB - over the ${Math.round(maxBytes / 1048576)} MB limit.`));
+            }
+            const chunks = [];
+            let size = 0;
+            res.on('data', (c) => {
+                size += c.length;
+                if (size > maxBytes) {
+                    res.destroy();
+                    finish(reject, new FetchRefused(`The file is over the ${Math.round(maxBytes / 1048576)} MB limit.`));
+                    return;
+                }
+                chunks.push(c);
+            });
+            res.on('end', () => finish(resolve, {buffer: Buffer.concat(chunks), contentType: String(res.headers['content-type'] || '')}));
+            res.on('error', (err) => finish(reject, err));
+        });
+        const timer = setTimeout(() => { req.destroy(); finish(reject, Object.assign(new Error('The download took too long.'), {code: 'ETIMEDOUT'})); }, Math.max(1, deadline - Date.now()));
+        req.on('error', (err) => finish(reject, err));
+        req.end();
+    });
+}