@loadmill/droid-cua 2.40.0 → 2.42.0

package/build/src/network/local-proxy.js

@@ -0,0 +1,287 @@
+import { generateKeyPairSync, randomBytes, randomUUID } from "node:crypto";
+import http from "node:http";
+import https from "node:https";
+import net from "node:net";
+import forge from "node-forge";
+import { loadCertificateAuthority } from "./certificate-authority.js";
+import { BODY_CAPTURE_LIMIT } from "./network-capture.js";
+export const NETWORK_PROXY_HOST = "127.0.0.1";
+export const NETWORK_PROXY_PORT = 8765;
+function destination(authority) {
+    const url = new URL(`http://${authority}`);
+    const port = Number(url.port || 443);
+    if (!url.hostname || !Number.isInteger(port) || port < 1 || port > 65535) {
+        throw new Error("Invalid proxy destination.");
+    }
+    return { host: url.hostname.replace(/^\[|\]$/g, ""), port };
+}
+function leafCertificate(host, ca) {
+    const { privateKey, publicKey } = generateKeyPairSync("rsa", {
+        modulusLength: 2048,
+        publicKeyEncoding: { type: "spki", format: "pem" },
+        privateKeyEncoding: { type: "pkcs8", format: "pem" },
+    });
+    const root = forge.pki.certificateFromPem(ca.certificate);
+    const certificate = forge.pki.createCertificate();
+    certificate.publicKey = forge.pki.publicKeyFromPem(publicKey);
+    certificate.serialNumber = randomBytes(16).toString("hex").replace(/^./, "1");
+    certificate.validity.notBefore = new Date(Date.now() - 60_000);
+    certificate.validity.notAfter = new Date(Math.min(Date.now() + 24 * 60 * 60 * 1000, Date.parse(root.validity.notAfter)));
+    certificate.setSubject([{ name: "commonName", value: host }]);
+    certificate.setIssuer(root.subject.attributes);
+    certificate.setExtensions([
+        { name: "basicConstraints", cA: false, critical: true },
+        { name: "keyUsage", digitalSignature: true, keyEncipherment: true, critical: true },
+        { name: "extKeyUsage", serverAuth: true },
+        { name: "subjectAltName", altNames: net.isIP(host) ? [{ type: 7, ip: host }] : [{ type: 2, value: host }] },
+    ]);
+    certificate.sign(forge.pki.privateKeyFromPem(ca.privateKey), forge.md.sha256.create());
+    return { key: privateKey, cert: forge.pki.certificateToPem(certificate) };
+}
+function observeBody(stream, type, flowId, headers, onEvent) {
+    let bytes = 0;
+    let retained = 0;
+    let finished = false;
+    const chunks = [];
+    stream.on("data", (chunk) => {
+        bytes += chunk.length;
+        const length = Math.min(chunk.length, BODY_CAPTURE_LIMIT - retained);
+        if (length > 0) {
+            chunks.push(Buffer.from(chunk.subarray(0, length)));
+            retained += length;
+        }
+    });
+    function finish(complete) {
+        if (finished)
+            return;
+        finished = true;
+        onEvent({ type, flowId, headers, body: Buffer.concat(chunks), bytes, truncated: bytes > retained, complete });
+    }
+    stream.once("end", () => finish(true));
+    stream.once("aborted", () => finish(false));
+    stream.once("error", () => finish(false));
+    stream.once("close", () => finish(false));
+}
+function forward(request, response, target, onError, onEvent) {
+    const flowId = onEvent ? randomUUID() : null;
+    if (onEvent) {
+        onEvent({ type: "request", flowId, method: request.method, url: target.href, headers: request.headers });
+        observeBody(request, "request_body", flowId, request.headers, onEvent);
+    }
+    const headers = { ...request.headers };
+    delete headers["proxy-connection"];
+    delete headers["proxy-authorization"];
+    const outgoing = (target.protocol === "https:" ? https : http).request(target, {
+        method: request.method,
+        headers,
+        agent: false,
+    }, (upstream) => {
+        if (onEvent) {
+            onEvent({ type: "response", flowId, status: upstream.statusCode || 502, headers: upstream.headers });
+            observeBody(upstream, "response_body", flowId, upstream.headers, onEvent);
+        }
+        response.writeHead(upstream.statusCode || 502, upstream.headers);
+        upstream.once("aborted", () => response.destroy());
+        upstream.once("error", (error) => {
+            onError(error, { flowId, phase: "upstream" });
+            response.destroy();
+        });
+        upstream.pipe(response);
+    });
+    outgoing.on("error", (error) => {
+        onError(error, { flowId, phase: "upstream" });
+        if (!response.headersSent) {
+            if (onEvent) {
+                onEvent({ type: "response", flowId, status: 502, headers: {} });
+                onEvent({ type: "response_body", flowId, headers: {}, body: Buffer.alloc(0), bytes: 0, truncated: false, complete: true });
+            }
+            response.writeHead(502);
+        }
+        response.end();
+    });
+    request.on("error", () => outgoing.destroy());
+    response.on("close", () => outgoing.destroy());
+    request.pipe(outgoing);
+}
+export function createNetworkProxy({ host = NETWORK_PROXY_HOST, port = NETWORK_PROXY_PORT, onError = () => { }, onEvent = null, loadCa = loadCertificateAuthority } = {}) {
+    let server;
+    let ca;
+    let intercepting = false;
+    let activeEvents = null;
+    const leaves = new Map();
+    const sockets = new Set();
+    const interceptedSockets = new Set();
+    function handleError(error, { flowId, connectionId, phase = "proxy", emit = null } = {}) {
+        if (emit)
+            emit({ type: "error", flowId, connectionId, phase, code: error.code || null, message: error.message });
+        onError(error);
+    }
+    async function destroySockets(group) {
+        await Promise.all([...group].map((socket) => new Promise((resolve) => {
+            if (socket.destroyed)
+                return resolve();
+            socket.once("close", resolve);
+            socket.destroy();
+        })));
+    }
+    function rejectRequest(response, error, emit, status = 400) {
+        handleError(error, { emit });
+        if (response.headersSent) {
+            response.destroy();
+            return;
+        }
+        response.writeHead(status);
+        response.end();
+    }
+    function isSelf(target) {
+        return target.port === server?.address()?.port && [host, "localhost", "127.0.0.1"].includes(target.host);
+    }
+    function handleRequest(request, response) {
+        const emit = activeEvents;
+        let target;
+        try {
+            target = new URL(request.url.startsWith("http:") ? request.url : `http://${request.headers.host}${request.url}`);
+            if (isSelf({ host: target.hostname, port: Number(target.port || 80) }))
+                throw new Error("Proxy loop rejected.");
+        }
+        catch (error) {
+            rejectRequest(response, error, emit);
+            return;
+        }
+        try {
+            forward(request, response, target, (error, details) => handleError(error, { ...details, emit }), emit);
+        }
+        catch (error) {
+            rejectRequest(response, error, emit, 502);
+        }
+    }
+    function forwardTunnel(socket, head, target) {
+        const upstream = net.connect(target.port, target.host);
+        sockets.add(upstream);
+        upstream.once("connect", () => {
+            socket.write("HTTP/1.1 200 Connection Established\r\n\r\n");
+            if (head.length)
+                upstream.write(head);
+            socket.pipe(upstream).pipe(socket);
+        });
+        upstream.on("error", (error) => {
+            handleError(error);
+            if (!socket.destroyed)
+                socket.end("HTTP/1.1 502 Bad Gateway\r\n\r\n");
+        });
+        socket.on("error", () => upstream.destroy());
+        socket.on("close", () => upstream.destroy());
+        upstream.on("close", () => sockets.delete(upstream));
+    }
+    function interceptTls(request, socket, head, target, emit) {
+        const connectionId = randomUUID();
+        emit?.({ type: "connect", connectionId, host: target.host, port: target.port });
+        let leaf = leaves.get(target.host);
+        try {
+            if (!leaf) {
+                leaf = leafCertificate(target.host, ca);
+                leaves.set(target.host, leaf);
+            }
+        }
+        catch (error) {
+            socket.end("HTTP/1.1 502 Bad Gateway\r\n\r\n");
+            handleError(error, { connectionId, phase: "certificate", emit });
+            return;
+        }
+        let decryptedRequests = 0;
+        let tlsEstablished = false;
+        const secureServer = https.createServer(leaf, (decryptedRequest, decryptedResponse) => {
+            decryptedRequests++;
+            try {
+                const url = new URL(decryptedRequest.url, `https://${request.url}`);
+                forward(decryptedRequest, decryptedResponse, url, (error, details) => handleError(error, { ...details, connectionId, emit }), emit);
+            }
+            catch (error) {
+                rejectRequest(decryptedResponse, error, emit, error?.code === "ERR_INVALID_URL" ? 400 : 502);
+            }
+        });
+        secureServer.on("secureConnection", () => { tlsEstablished = true; });
+        secureServer.on("tlsClientError", (error) => handleError(error, { connectionId, phase: "tls", emit }));
+        secureServer.on("clientError", (error) => handleError(error, { connectionId, phase: "client", emit }));
+        interceptedSockets.add(socket);
+        socket.on("close", () => {
+            interceptedSockets.delete(socket);
+            emit?.({ type: "connect_end", connectionId, decryptedRequests, tlsEstablished });
+        });
+        socket.write("HTTP/1.1 200 Connection Established\r\n\r\n");
+        if (head.length)
+            socket.unshift(head);
+        secureServer.emit("connection", socket);
+        socket.resume();
+    }
+    function handleConnect(request, socket, head) {
+        let target;
+        try {
+            target = destination(request.url);
+            if (isSelf(target))
+                throw new Error("Proxy loop rejected.");
+        }
+        catch (error) {
+            socket.end("HTTP/1.1 400 Bad Request\r\n\r\n");
+            handleError(error);
+            return;
+        }
+        if (intercepting)
+            interceptTls(request, socket, head, target, activeEvents);
+        else
+            forwardTunnel(socket, head, target);
+    }
+    return {
+        get address() { return server?.address() || null; },
+        async start() {
+            if (server)
+                return server.address();
+            const next = http.createServer(handleRequest);
+            next.on("connect", handleConnect);
+            next.on("connection", (socket) => {
+                sockets.add(socket);
+                socket.on("close", () => sockets.delete(socket));
+            });
+            try {
+                await new Promise((resolve, reject) => {
+                    next.once("error", reject);
+                    next.listen(port, host, () => {
+                        next.off("error", reject);
+                        resolve();
+                    });
+                });
+            }
+            catch (error) {
+                next.close();
+                throw error;
+            }
+            server = next;
+            return server.address();
+        },
+        async beginInterception(emit = onEvent) {
+            if (intercepting)
+                throw new Error("Proxy interception is already active.");
+            ca = await loadCa();
+            leaves.clear();
+            activeEvents = emit;
+            intercepting = true;
+            await destroySockets(sockets);
+        },
+        async endInterception() {
+            intercepting = false;
+            activeEvents = null;
+            ca = null;
+            leaves.clear();
+            await destroySockets(interceptedSockets);
+        },
+        async stop() {
+            if (!server)
+                return;
+            const current = server;
+            server = null;
+            await this.endInterception();
+            await destroySockets(sockets);
+            await new Promise((resolve) => current.close(resolve));
+        },
+    };
+}

package/build/src/network/network-capture.js

@@ -0,0 +1,243 @@
+import { open, mkdir } from "node:fs/promises";
+import path from "node:path";
+import { brotliDecompressSync, gunzipSync, inflateSync } from "node:zlib";
+export const BODY_CAPTURE_LIMIT = 1024 * 1024;
+export const NETWORK_CAPTURE_LIMIT = 100 * 1024 * 1024;
+// Leave space to persist why recording stopped, even when the data budget is full.
+const CAPTURE_END_RESERVE = 1024;
+const REDACTED = "[REDACTED]";
+/**
+ * @typedef {{ type: "request", flowId: string, method: string, url: string, headers: object }
+ * | { type: "response", flowId: string, status: number, headers: object }
+ * | { type: "request_body" | "response_body", flowId: string, headers: object, body: Buffer, bytes: number, truncated: boolean, complete: boolean, decoded?: boolean, omitted?: string }
+ * | { type: "connect", connectionId: string, host: string, port: number }
+ * | { type: "connect_end", connectionId: string, decryptedRequests: number, tlsEstablished: boolean }
+ * | { type: "error", flowId?: string, connectionId?: string, phase: string, code: string | null, message: string }
+ * | { type: "capture_start", version: number }
+ * | { type: "capture_end", health: CaptureHealth }} NetworkEvent
+ */
+/**
+ * @typedef {object} CaptureRecord
+ * @property {string} timestamp
+ * @property {string} type
+ * @property {string} [flowId]
+ * @property {string} [method]
+ * @property {string} [url]
+ * @property {number} [status]
+ * @property {object} [headers]
+ * @property {string} [connectionId]
+ * @property {string} [host]
+ * @property {number} [port]
+ * @property {number} [decryptedRequests]
+ * @property {boolean} [tlsEstablished]
+ * @property {string} [phase]
+ * @property {string | null} [code]
+ * @property {string} [message]
+ * @property {number} [version]
+ * @property {CaptureHealth} [health]
+ * @property {{ bytes: number, truncated: boolean, complete: boolean, text?: string, omitted?: string }} [body]
+ */
+/**
+ * @typedef {object} CaptureHealth
+ * @property {number} requests
+ * @property {number} responses
+ * @property {number} httpsConnects
+ * @property {number} httpsConnectsWithoutRequests
+ * @property {number} decryptedHttpsRequests
+ * @property {number} tlsErrors
+ * @property {number} errors
+ */
+function sensitiveName(name) {
+    const normalized = name.replace(/([a-z0-9])([A-Z])/g, "$1-$2");
+    return /(?:^|[-_])(password|passwd|pwd|token|secret|authorization|auth|cookie|session|credential|api[-_]?key|access[-_]?key)(?:$|[-_])/i.test(normalized);
+}
+function redactHeaders(headers = {}) {
+    return Object.fromEntries(Object.entries(headers).map(([name, value]) => [
+        name,
+        sensitiveName(name) ? (Array.isArray(value) ? value.map(() => REDACTED) : REDACTED) : value,
+    ]));
+}
+function redactUrl(value) {
+    const url = new URL(value);
+    if (url.username)
+        url.username = REDACTED;
+    if (url.password)
+        url.password = REDACTED;
+    for (const name of url.searchParams.keys()) {
+        if (sensitiveName(name))
+            url.searchParams.set(name, REDACTED);
+    }
+    return url.href;
+}
+function redactObject(value) {
+    if (Array.isArray(value))
+        return value.map(redactObject);
+    if (value && typeof value === "object") {
+        return Object.fromEntries(Object.entries(value).map(([key, item]) => [key, sensitiveName(key) ? REDACTED : redactObject(item)]));
+    }
+    return value;
+}
+function redactText(value) {
+    return value
+        .replace(/\b(Bearer|Basic)\s+[A-Za-z0-9._~+/-]+={0,2}/gi, "$1 [REDACTED]")
+        .replace(/\b(password|passwd|pwd|token|secret|authorization|cookie|api[-_]?key)\b(\s*[:=]\s*)[^\s,;&]+/gi, "$1$2[REDACTED]");
+}
+function decodeBody(body, encoding) {
+    if (!encoding || encoding === "identity")
+        return body;
+    const options = { maxOutputLength: BODY_CAPTURE_LIMIT };
+    if (encoding === "gzip")
+        return gunzipSync(body, options);
+    if (encoding === "deflate")
+        return inflateSync(body, options);
+    if (encoding === "br")
+        return brotliDecompressSync(body, options);
+    return null;
+}
+function redactBody(event) {
+    const { body, headers = {}, bytes, truncated, complete } = event;
+    const result = { bytes, truncated, complete };
+    if (event.omitted)
+        return { ...result, omitted: event.omitted };
+    if (!body.length)
+        return { ...result, text: "" };
+    const contentType = String(headers["content-type"] || "").toLowerCase();
+    const encoding = event.decoded ? "identity" : String(headers["content-encoding"] || "").toLowerCase();
+    const textual = /(?:^text\/|json|xml|javascript|x-www-form-urlencoded)/.test(contentType);
+    if (!textual)
+        return { ...result, omitted: "binary-or-unknown-content-type" };
+    if (truncated && encoding && encoding !== "identity")
+        return { ...result, omitted: "truncated-compressed-body" };
+    try {
+        const decoded = decodeBody(body, encoding);
+        if (!decoded)
+            return { ...result, omitted: "unsupported-content-encoding" };
+        const charset = /charset\s*=\s*([^;\s]+)/.exec(contentType)?.[1] || "utf-8";
+        const text = new TextDecoder(charset.replaceAll('"', ""), { fatal: true }).decode(decoded);
+        if (contentType.includes("json"))
+            return { ...result, text: JSON.stringify(redactObject(JSON.parse(text))) };
+        if (contentType.includes("x-www-form-urlencoded")) {
+            const fields = new URLSearchParams(text);
+            for (const name of fields.keys())
+                if (sensitiveName(name))
+                    fields.set(name, REDACTED);
+            return { ...result, text: fields.toString() };
+        }
+        return { ...result, text: redactText(text) };
+    }
+    catch (error) {
+        if (error.code === "ERR_BUFFER_TOO_LARGE") {
+            return { ...result, truncated: true, omitted: "decoded-body-too-large" };
+        }
+        return { ...result, omitted: "unreadable-text-body" };
+    }
+}
+/** @param {NetworkEvent} event @returns {CaptureRecord} */
+function toCaptureRecord(event) {
+    const timestamp = new Date().toISOString();
+    if (event.type === "request_body" || event.type === "response_body") {
+        return { timestamp, type: event.type, flowId: event.flowId, body: redactBody(event) };
+    }
+    const record = { timestamp, ...event };
+    if (event.url)
+        record.url = redactUrl(event.url);
+    if (event.message)
+        record.message = redactText(event.message);
+    if (event.headers)
+        record.headers = redactHeaders(event.headers);
+    return record;
+}
+/** @param {string} outputPath */
+export async function createNetworkCapture(outputPath, { maxBytes = NETWORK_CAPTURE_LIMIT } = {}) {
+    const resolvedPath = path.resolve(outputPath);
+    await mkdir(path.dirname(resolvedPath), { recursive: true });
+    const file = await open(resolvedPath, "wx", 0o600);
+    const health = { requests: 0, responses: 0, httpsConnects: 0, httpsConnectsWithoutRequests: 0, decryptedHttpsRequests: 0, tlsErrors: 0, errors: 0 };
+    let pending = Promise.resolve();
+    let failure = null;
+    let closed = false;
+    let bytesWritten = 0;
+    let bytesQueued = 0;
+    /** @param {NetworkEvent} event */
+    function record(event) {
+        if (closed || failure)
+            return pending;
+        if (event.type === "request") {
+            health.requests++;
+            if (event.url.startsWith("https:"))
+                health.decryptedHttpsRequests++;
+        }
+        if (event.type === "response")
+            health.responses++;
+        if (event.type === "connect")
+            health.httpsConnects++;
+        if (event.type === "connect_end" && event.decryptedRequests === 0)
+            health.httpsConnectsWithoutRequests++;
+        if (event.type === "error") {
+            health.errors++;
+            if (event.phase === "tls")
+                health.tlsErrors++;
+        }
+        let line;
+        try {
+            line = `${JSON.stringify(toCaptureRecord(event))}\n`;
+        }
+        catch (error) {
+            failure = error;
+            return pending;
+        }
+        if (bytesQueued + Buffer.byteLength(line) > maxBytes - CAPTURE_END_RESERVE) {
+            failure = new Error(`Capture size limit reached (${maxBytes} bytes). Network evidence is incomplete.`);
+            health.errors++;
+            line = `${JSON.stringify(toCaptureRecord({
+                type: "error", phase: "capture", code: "CAPTURE_LIMIT", message: failure.message,
+            }))}\n`;
+        }
+        // Count queued writes too: concurrent traffic must not exceed the limit.
+        bytesQueued += Buffer.byteLength(line);
+        pending = pending.then(async () => {
+            await file.writeFile(line);
+            bytesWritten += Buffer.byteLength(line);
+        }).catch((error) => {
+            if (!failure) {
+                failure = error;
+            }
+        });
+        return pending;
+    }
+    await record({ type: "capture_start", version: 1 });
+    if (failure) {
+        await file.close();
+        throw failure;
+    }
+    return {
+        path: resolvedPath,
+        record,
+        get health() { return { ...health }; },
+        get failure() { return failure; },
+        get closed() { return closed; },
+        snapshot() {
+            const timestamp = new Date().toISOString();
+            return pending.then(() => {
+                if (failure)
+                    throw failure;
+                return { bytes: bytesWritten, timestamp };
+            });
+        },
+        async close() {
+            if (closed)
+                return;
+            record({ type: "capture_end", health: { ...health } });
+            closed = true;
+            await pending;
+            try {
+                await file.close();
+            }
+            catch (error) {
+                failure ||= error;
+            }
+            if (failure)
+                throw failure;
+        },
+    };
+}