@xcanwin/manyoyo 8.3.3 → 8.5.0

package/lib/egress-proxy.js

@@ -0,0 +1,315 @@
+'use strict';
+
+const dns = require('dns').promises;
+const http = require('http');
+const net = require('net');
+const { domainAllowed, PRIVATE_V4, PRIVATE_V6 } = require('./network-policy');
+
+// 域名白名单的强制执行点：运行在 manyoyo 网络里专用 sidecar 容器中的 HTTP(S) 过滤代理（见 egress-sidecar.js）。
+// 开启 allowlist 的容器防火墙只放行到这个代理，直连被挡；这里按连接的来源 IP 认出是哪个容器（getClient），
+// 认不出的来源一律 403；按域名放行 CONNECT / 普通 HTTP，
+// 并且解析出的 IP 属于私有 / 环回 / 链路本地段一律拒绝（防借代理打宿主机 loopback 与 DNS 重绑定），
+// 除非该容器的 egress.rules 显式放行该 IP:端口。
+// 被拒绝时统一回 403（带 Proxy-Status，不带任何能暴露代理身份的内容），并通过 onDenied 记录。
+const CONNECT_TIMEOUT_MS = 15000;
+const IDLE_TIMEOUT_MS = 10 * 60 * 1000;
+const MAX_TUNNELS_PER_CONTAINER = 256;
+
+const blocked = new net.BlockList();
+[...PRIVATE_V4, '127.0.0.0/8', '224.0.0.0/4', '255.255.255.255/32'].forEach(cidr => {
+    const [addr, prefix] = cidr.split('/');
+    blocked.addSubnet(addr, Number(prefix), 'ipv4');
+});
+[...PRIVATE_V6, 'fe80::/10', 'ff00::/8', '::1/128', '::/128'].forEach(cidr => {
+    const [addr, prefix] = cidr.split('/');
+    blocked.addSubnet(addr, Number(prefix), 'ipv6');
+});
+
+// ::ffff:a.b.c.d 与 NAT64（64:ff9b::/96）里嵌的 IPv4 要按 IPv4 判断
+function embeddedV4(ip) {
+    const lower = ip.toLowerCase();
+    const mapped = /^(?:::ffff:|64:ff9b::)(\d+\.\d+\.\d+\.\d+)$/.exec(lower);
+    if (mapped) return mapped[1];
+    const hex = /^(?:::ffff:|64:ff9b::)([0-9a-f]{1,4}):([0-9a-f]{1,4})$/.exec(lower);
+    if (hex) {
+        const a = parseInt(hex[1], 16);
+        const b = parseInt(hex[2], 16);
+        return `${a >> 8}.${a & 255}.${b >> 8}.${b & 255}`;
+    }
+    return '';
+}
+
+function isRestrictedAddress(ip) {
+    const v4 = embeddedV4(ip);
+    if (v4) return blocked.check(v4, 'ipv4');
+    const family = net.isIP(ip);
+    if (!family) return true;
+    return blocked.check(ip, family === 4 ? 'ipv4' : 'ipv6');
+}
+
+function ruleCovers(rule, ip, port) {
+    if (rule.proto && rule.proto !== 'tcp') return false;
+    if (rule.ports) {
+        const ok = rule.ports.split(',').some(item => {
+            const [a, b] = item.split('-').map(Number);
+            return port >= a && port <= (b === undefined ? a : b);
+        });
+        if (!ok) return false;
+    }
+    const [addr, prefixText] = rule.cidr.split('/');
+    const family = net.isIP(addr);
+    if (!family || net.isIP(ip) !== family) return false;
+    const list = new net.BlockList();
+    list.addSubnet(addr, prefixText === undefined ? (family === 4 ? 32 : 128) : Number(prefixText), family === 4 ? 'ipv4' : 'ipv6');
+    return list.check(ip, family === 4 ? 'ipv4' : 'ipv6');
+}
+
+function parseTarget(text) {
+    const m = /^(?:\[([0-9a-fA-F:.]+)\]|([^:/\s]+)):(\d{1,5})$/.exec(text);
+    if (!m) return null;
+    const port = Number(m[3]);
+    if (port < 1 || port > 65535) return null;
+    return { host: (m[1] || m[2]).toLowerCase().replace(/\.$/, ''), port };
+}
+
+/**
+ * @param {object} options
+ * @param {(ip: string) => {id: string, preset: string, domains: string[], rules: object[]}|null} options.getClient 按来源 IP 认容器（认不出返回 null）
+ * @param {(event: {id: string, host: string, port: number, reason: string}) => void} [options.onDenied] 拒绝记录
+ * @param {string} [options.upstream] 上游 HTTP 代理 URL（串联）
+ * @param {(host: string) => Promise<string[]>} [options.lookup] 测试替身
+ */
+function createEgressProxy(options) {
+    const getClient = options.getClient;
+    const onDenied = options.onDenied || (() => {});
+    const lookup = options.lookup || (async host => (await dns.lookup(host, { all: true })).map(r => r.address));
+    // upstream 可以是字符串或返回字符串的函数（sidecar 里按文件改动热更新）；解析结果按字符串缓存
+    let parsedFor = null;
+    let parsedUpstream = null;
+    function currentUpstream() {
+        const text = typeof options.upstream === 'function' ? options.upstream() : options.upstream;
+        if (text !== parsedFor) {
+            parsedFor = text;
+            parsedUpstream = null;
+            if (text) {
+                try {
+                    const url = new URL(text);
+                    if (url.protocol === 'http:') {
+                        parsedUpstream = { host: url.hostname.replace(/^\[|\]$/g, ''), port: Number(url.port) || 80, auth: url.username ? `Basic ${Buffer.from(`${decodeURIComponent(url.username)}:${decodeURIComponent(url.password)}`).toString('base64')}` : '' };
+                    }
+                } catch (e) { /* 无效上游当作没有 */ }
+            }
+        }
+        return parsedUpstream;
+    }
+    const tunnels = new Map();
+    const sockets = new Set(); // CONNECT 劫持出来的 socket 不在 http.Server 的跟踪里，stop 时要自己关
+    let server = null;
+
+    function normalizeSource(address) {
+        const text = String(address || '');
+        return /^::ffff:\d+\.\d+\.\d+\.\d+$/i.test(text) ? text.slice(7) : text;
+    }
+
+    // 按来源 IP 认容器：返回 {id, policy}，认不出返回 null
+    function identify(socket) {
+        let client = null;
+        try { client = getClient(normalizeSource(socket.remoteAddress)); } catch (e) { client = null; }
+        return client && client.id ? { id: client.id, policy: client } : null;
+    }
+
+    // 返回 {ok:true, ip?} 或 {ok:false, status, message}
+    async function authorize(policy, host, port, upstream) {
+        if (!policy || policy.preset !== 'allowlist') return { ok: false, status: 403, message: 'domain' };
+        const isIp = net.isIP(host) !== 0;
+        const explicit = (ip, p) => (policy.rules || []).some(rule => ruleCovers(rule, ip, p));
+        if (isIp) {
+            if (isRestrictedAddress(host) && !explicit(host, port)) return { ok: false, status: 403, message: 'address' };
+            if (!explicit(host, port)) return { ok: false, status: 403, message: 'domain' };
+            return { ok: true, ip: host };
+        }
+        if (!domainAllowed(policy.domains, host)) return { ok: false, status: 403, message: 'domain' };
+        if (upstream) {
+            // 经上游代理时真正的连接由上游解析；这里仍做一次本地解析，解析得到私有 / 环回地址（且没有被显式放行）就拒绝。
+            // 本地解析不出来（宿主机不一定能解析）则放行给上游。残余风险：白名单里的通配域名若被控制了 DNS，
+            // 本地与上游可能得到不同答案——不要在白名单里放你不信任的通配域名。
+            try {
+                const local = await lookup(host);
+                if (local.length && local.every(ip => isRestrictedAddress(ip) && !explicit(ip, port))) return { ok: false, status: 403, message: 'address' };
+            } catch (e) { /* 本地解析失败：交给上游 */ }
+            return { ok: true, ip: '' };
+        }
+        let addresses;
+        try {
+            addresses = await lookup(host);
+        } catch (e) {
+            return { ok: false, status: 502, message: 'dns' };
+        }
+        const usable = addresses.filter(ip => !isRestrictedAddress(ip) || explicit(ip, port));
+        if (!usable.length) return { ok: false, status: 403, message: 'address' };
+        return { ok: true, ip: usable[0] };
+    }
+
+    // Proxy-Status（RFC 9209）只带通用错误类型；拒绝一律 403，正文只有 Forbidden
+    function statusHeader(message) {
+        return message === 'address' ? 'proxy; error=destination_ip_prohibited' : 'proxy; error=http_request_denied';
+    }
+
+    function reject(res, status, message) {
+        const body = { 400: 'Bad Request', 403: 'Forbidden', 429: 'Too Many Requests' }[status] || 'Bad Gateway';
+        const headers = { 'Content-Type': 'text/plain; charset=utf-8', 'Content-Length': Buffer.byteLength(body), Connection: 'close' };
+        if (status === 403) headers['Proxy-Status'] = statusHeader(message);
+        res.writeHead(status, headers);
+        res.end(body);
+    }
+
+    function deny(client, host, port, reason) {
+        try { onDenied({ id: client ? client.id : '', host, port, reason }); } catch (e) { /* 记录失败不影响拒绝 */ }
+    }
+
+    function acquire(id) {
+        const n = tunnels.get(id) || 0;
+        if (n >= MAX_TUNNELS_PER_CONTAINER) return false;
+        tunnels.set(id, n + 1);
+        return true;
+    }
+
+    function release(id) {
+        const n = (tunnels.get(id) || 1) - 1;
+        if (n <= 0) tunnels.delete(id); else tunnels.set(id, n);
+    }
+
+    function rawReject(socket, status, text, message) {
+        const extra = status === 403 ? `Proxy-Status: ${statusHeader(message)}\r\n` : '';
+        socket.end(`HTTP/1.1 ${status} ${text}\r\n${extra}Connection: close\r\nContent-Length: 0\r\n\r\n`);
+    }
+
+    function track(socket) {
+        sockets.add(socket);
+        socket.once('close', () => sockets.delete(socket));
+        return socket;
+    }
+
+    async function onConnect(req, clientSocket, head) {
+        track(clientSocket);
+        clientSocket.on('error', () => {});
+        const client = identify(clientSocket);
+        const target = parseTarget(String(req.url || ''));
+        if (!client) return rawReject(clientSocket, 403, 'Forbidden', 'domain');
+        if (!target) return rawReject(clientSocket, 400, 'Bad Request');
+        const id = client.id;
+        const upstream = currentUpstream();
+        const verdict = await authorize(client.policy, target.host, target.port, upstream);
+        if (!verdict.ok) {
+            if (verdict.status === 403) deny(client, target.host, target.port, verdict.message);
+            return rawReject(clientSocket, verdict.status, verdict.status === 403 ? 'Forbidden' : 'Bad Gateway', verdict.message);
+        }
+        if (!acquire(id)) return rawReject(clientSocket, 429, 'Too Many Requests');
+        let released = false;
+        const done = () => { if (!released) { released = true; release(id); } };
+        let upstreamSocket;
+        if (upstream) {
+            upstreamSocket = track(net.connect({ host: upstream.host, port: upstream.port }));
+            upstreamSocket.setTimeout(CONNECT_TIMEOUT_MS, () => upstreamSocket.destroy());
+            upstreamSocket.once('connect', () => {
+                upstreamSocket.setTimeout(IDLE_TIMEOUT_MS, () => upstreamSocket.destroy());
+                const authLine = upstream.auth ? `Proxy-Authorization: ${upstream.auth}\r\n` : '';
+                const hostPort = net.isIPv6(target.host) ? `[${target.host}]:${target.port}` : `${target.host}:${target.port}`;
+                upstreamSocket.write(`CONNECT ${hostPort} HTTP/1.1\r\nHost: ${hostPort}\r\n${authLine}\r\n`);
+                let buf = Buffer.alloc(0);
+                const onData = chunk => {
+                    buf = Buffer.concat([buf, chunk]);
+                    const end = buf.indexOf('\r\n\r\n');
+                    if (end < 0) { if (buf.length > 16384) upstreamSocket.destroy(); return; }
+                    upstreamSocket.removeListener('data', onData);
+                    const status = Number((/^HTTP\/1\.[01] (\d{3})/.exec(buf.toString('latin1', 0, end)) || [])[1]);
+                    if (status !== 200) { rawReject(clientSocket, 502, 'Bad Gateway'); upstreamSocket.destroy(); return; }
+                    clientSocket.write('HTTP/1.1 200 Connection Established\r\n\r\n');
+                    const rest = buf.subarray(end + 4);
+                    if (rest.length) clientSocket.write(rest);
+                    if (head && head.length) upstreamSocket.write(head);
+                    upstreamSocket.pipe(clientSocket);
+                    clientSocket.pipe(upstreamSocket);
+                };
+                upstreamSocket.on('data', onData);
+            });
+        } else {
+            upstreamSocket = track(net.connect({ host: verdict.ip, port: target.port }));
+            upstreamSocket.setTimeout(CONNECT_TIMEOUT_MS, () => upstreamSocket.destroy());
+            upstreamSocket.once('connect', () => {
+                upstreamSocket.setTimeout(IDLE_TIMEOUT_MS, () => upstreamSocket.destroy());
+                clientSocket.write('HTTP/1.1 200 Connection Established\r\n\r\n');
+                if (head && head.length) upstreamSocket.write(head);
+                upstreamSocket.pipe(clientSocket);
+                clientSocket.pipe(upstreamSocket);
+            });
+        }
+        upstreamSocket.on('error', () => { if (!clientSocket.destroyed) rawReject(clientSocket, 502, 'Bad Gateway'); });
+        upstreamSocket.on('close', () => { done(); clientSocket.destroy(); });
+        clientSocket.on('close', () => { done(); upstreamSocket.destroy(); });
+    }
+
+    // 普通 HTTP 代理（绝对 URI 请求）
+    async function onRequest(req, res) {
+        const client = identify(req.socket);
+        if (!client) return reject(res, 403, 'domain');
+        const id = client.id;
+        let url;
+        try { url = new URL(req.url); } catch (e) { return reject(res, 400, 'bad request'); }
+        if (url.protocol !== 'http:') return reject(res, 400, 'bad request');
+        const port = Number(url.port) || 80;
+        const host = url.hostname.replace(/^\[|\]$/g, '').toLowerCase().replace(/\.$/, '');
+        const upstream = currentUpstream();
+        const verdict = await authorize(client.policy, host, port, upstream);
+        if (!verdict.ok) {
+            if (verdict.status === 403) deny(client, host, port, verdict.message);
+            return reject(res, verdict.status, verdict.message);
+        }
+        if (!acquire(id)) return reject(res, 429, 'limit');
+        const headers = { ...req.headers, host: url.host }; // 放行判断看的是 URL 里的主机，Host 头必须与它一致（RFC 9112）
+        delete headers['proxy-authorization'];
+        delete headers['proxy-connection'];
+        const outbound = upstream
+            ? { host: upstream.host, port: upstream.port, path: req.url, headers: { ...headers, ...(upstream.auth ? { 'proxy-authorization': upstream.auth } : {}) } }
+            : { host: verdict.ip, port, path: `${url.pathname}${url.search}`, headers };
+        const proxyReq = http.request({ ...outbound, method: req.method, timeout: CONNECT_TIMEOUT_MS });
+        let released = false;
+        const done = () => { if (!released) { released = true; release(id); } };
+        proxyReq.on('response', proxyRes => {
+            res.writeHead(proxyRes.statusCode, proxyRes.headers);
+            proxyRes.pipe(res);
+        });
+        proxyReq.on('timeout', () => proxyReq.destroy());
+        proxyReq.on('error', () => { if (!res.headersSent) reject(res, 502, 'upstream'); else res.destroy(); });
+        res.on('close', () => { done(); proxyReq.destroy(); });
+        req.pipe(proxyReq);
+    }
+
+    function start(listen) {
+        return new Promise((resolve, fail) => {
+            server = http.createServer((req, res) => { onRequest(req, res).catch(() => { try { reject(res, 500, 'upstream'); } catch (e) { /* 连接已断 */ } }); });
+            server.on('connect', (req, socket, head) => { onConnect(req, socket, head).catch(() => socket.destroy()); });
+            server.headersTimeout = 20000;
+            server.on('clientError', (err, socket) => socket.destroy());
+            server.once('error', fail);
+            server.listen(listen.port, listen.host, () => {
+                server.removeListener('error', fail);
+                resolve(server.address().port);
+            });
+        });
+    }
+
+    function stop() {
+        return new Promise(resolve => {
+            if (!server) return resolve();
+            server.close(() => resolve());
+            server.closeAllConnections && server.closeAllConnections();
+            sockets.forEach(socket => socket.destroy());
+            server = null;
+        });
+    }
+
+    return { start, stop, authorize, isListening: () => Boolean(server && server.listening) };
+}
+
+module.exports = { createEgressProxy, isRestrictedAddress, ruleCovers, parseTarget };

package/lib/egress-sidecar-main.js

@@ -0,0 +1,112 @@
+'use strict';
+
+const fs = require('fs');
+const path = require('path');
+const { createEgressProxy } = require('./egress-proxy');
+const { createRecorder } = require('./egress-denied');
+
+// 过滤代理 sidecar 容器里的入口（宿主机把这几个文件只读挂进来，见 egress-sidecar.js）。
+// 数据目录（只读）：clients.json（来源 IP → 容器 id + 策略，宿主机在每次下发规则时原子替换）、upstream.txt（可选，上游代理）；
+// 拒绝记录目录（读写）：<容器 id>.jsonl。
+const CLIENTS_FILE = 'clients.json';
+const UPSTREAM_FILE = 'upstream.txt';
+const RELOAD_CHECK_MS = 500;
+
+function loadClients(file) {
+    try {
+        const parsed = JSON.parse(fs.readFileSync(file, 'utf-8'));
+        const clients = parsed && typeof parsed.clients === 'object' && parsed.clients ? parsed.clients : {};
+        return new Map(Object.entries(clients));
+    } catch (e) {
+        return new Map(); // 读不到 / 损坏：谁都不认识，全部 403（失败即关闭）
+    }
+}
+
+/**
+ * @param {object} options
+ * @param {string} options.dataDir
+ * @param {string} options.deniedDir
+ * @param {number} options.port
+ * @param {string} [options.host]
+ * @param {(host: string) => Promise<string[]>} [options.lookup]
+ */
+async function start(options) {
+    const clientsFile = path.join(options.dataDir, CLIENTS_FILE);
+    // 上游代理：upstream.txt 变了就热更新（CLI 与 serve 的环境可能不同，不能为此重建容器）
+    const upstreamFile = path.join(options.dataDir, UPSTREAM_FILE);
+    let upstream = '';
+    let upstreamSig = '';
+    let upstreamCheckedAt = 0;
+    const getUpstream = () => {
+        const nowMs = Date.now();
+        if (nowMs - upstreamCheckedAt < RELOAD_CHECK_MS) return upstream;
+        upstreamCheckedAt = nowMs;
+        try {
+            const stat = fs.statSync(upstreamFile);
+            const sig = `${stat.mtimeMs}:${stat.size}:${stat.ino}`;
+            if (sig !== upstreamSig) {
+                upstreamSig = sig;
+                upstream = fs.readFileSync(upstreamFile, 'utf-8').trim();
+            }
+        } catch (e) {
+            upstreamSig = '';
+            upstream = '';
+        }
+        return upstream;
+    };
+
+    let clients = loadClients(clientsFile);
+    let signature = '';
+    let checkedAt = 0;
+    const refresh = () => {
+        const nowMs = Date.now();
+        if (nowMs - checkedAt < RELOAD_CHECK_MS) return;
+        checkedAt = nowMs;
+        try {
+            const stat = fs.statSync(clientsFile);
+            const next = `${stat.mtimeMs}:${stat.size}:${stat.ino}`;
+            if (next !== signature) {
+                signature = next;
+                clients = loadClients(clientsFile);
+            }
+        } catch (e) {
+            signature = '';
+            clients = new Map();
+        }
+    };
+
+    const recorder = createRecorder({ dir: options.deniedDir });
+    const proxy = createEgressProxy({
+        upstream: getUpstream,
+        lookup: options.lookup,
+        getClient: ip => {
+            refresh();
+            const entry = clients.get(ip);
+            return entry ? { ...entry } : null;
+        },
+        onDenied: event => recorder.record(event)
+    });
+    const port = await proxy.start({ host: options.host || '0.0.0.0', port: options.port });
+    return {
+        port,
+        stop: async () => { recorder.stop(); await proxy.stop(); }
+    };
+}
+
+if (require.main === module) {
+    start({
+        dataDir: process.env.EGRESS_DATA_DIR || '/data',
+        deniedDir: process.env.EGRESS_DENIED_DIR || '/denied',
+        port: Number(process.env.EGRESS_PORT) || 3128
+    }).then(server => {
+        process.stdout.write(`egress-proxy ready ${server.port}\n`);
+        const shutdown = () => server.stop().finally(() => process.exit(0));
+        process.on('SIGTERM', shutdown);
+        process.on('SIGINT', shutdown);
+    }).catch(error => {
+        process.stderr.write(`egress-proxy failed: ${error.message}\n`);
+        process.exit(1);
+    });
+}
+
+module.exports = { start, CLIENTS_FILE, UPSTREAM_FILE };