@citeark/agent 0.3.20

package/src/network/autodl-routing-proxy.mjs

@@ -0,0 +1,462 @@
+import http from "node:http";
+import https from "node:https";
+import net from "node:net";
+import tls from "node:tls";
+import { pathToFileURL } from "node:url";
+
+const GOOGLE_HOSTS = [
+  "google.com",
+  ".google.com",
+  "googleapis.com",
+  ".googleapis.com",
+  "googlesource.com",
+  ".googlesource.com",
+  "gstatic.com",
+  ".gstatic.com",
+  "googleusercontent.com",
+  ".googleusercontent.com",
+  "research.google",
+];
+
+const ALWAYS_LIGUSE_HOSTS = [
+  ...GOOGLE_HOSTS,
+  "openai.com",
+  ".openai.com",
+  "chatgpt.com",
+  ".chatgpt.com",
+  "oaistatic.com",
+  ".oaistatic.com",
+  "oaiusercontent.com",
+  ".oaiusercontent.com",
+  "anthropic.com",
+  ".anthropic.com",
+  "openrouter.ai",
+  ".openrouter.ai",
+];
+
+const TURBO_HOSTS = [
+  "github.com",
+  ".github.com",
+  "githubusercontent.com",
+  ".githubusercontent.com",
+  "githubassets.com",
+  ".githubassets.com",
+  "huggingface.co",
+  ".huggingface.co",
+  "hf.co",
+  ".hf.co",
+  "xethub.hf.co",
+  ".xethub.hf.co",
+  "pypi.org",
+  ".pypi.org",
+  "pythonhosted.org",
+  ".pythonhosted.org",
+  "pytorch.org",
+  ".pytorch.org",
+];
+
+export function routeAutoDlHost(hostname, { alwaysProxyHosts = [] } = {}) {
+  const normalized = normalizeHost(hostname);
+  if (matchesHost(normalized, [...ALWAYS_LIGUSE_HOSTS, ...alwaysProxyHosts])) {
+    return "liguse-sg";
+  }
+  if (matchesHost(normalized, TURBO_HOSTS)) return "autodl-turbo";
+  return "direct";
+}
+
+export async function startAutoDlRoutingProxy({
+  listenHost = "127.0.0.1",
+  listenPort = 0,
+  turboProxyUrl,
+  liguseProxyUrl,
+  alwaysProxyHosts = [],
+  connectTimeoutMs = 12_000,
+  directConnector = connectDirect,
+  logger = () => {},
+} = {}) {
+  const upstreams = {
+    "autodl-turbo": parseProxyUrl(turboProxyUrl, "AutoDL 学术加速代理"),
+    "liguse-sg": parseProxyUrl(liguseProxyUrl, "LiguseSG 受限公网代理"),
+  };
+  const routing = { alwaysProxyHosts, connectTimeoutMs, directConnector };
+  const sockets = new Set();
+  const server = http.createServer((request, response) => {
+    if (request.url === "/__citeark_health") {
+      response.writeHead(204).end();
+      return;
+    }
+    void forwardHttp(request, response, upstreams, routing, logger);
+  });
+  server.on("connect", (request, clientSocket, head) => {
+    void forwardConnect(request, clientSocket, head, upstreams, routing, sockets, logger);
+  });
+  server.on("connection", (socket) => trackSocket(socket, sockets));
+  await new Promise((resolve, reject) => {
+    server.once("error", reject);
+    server.listen(listenPort, listenHost, resolve);
+  });
+  const address = server.address();
+  const port = typeof address === "object" && address ? address.port : null;
+  return {
+    host: listenHost,
+    port,
+    proxyUrl: `http://${listenHost}:${port}`,
+    close: async () => {
+      for (const socket of sockets) socket.destroy();
+      await new Promise((resolve) => server.close(resolve));
+    },
+  };
+}
+
+async function forwardHttp(request, response, upstreams, routing, logger) {
+  let target;
+  try {
+    target = new URL(request.url);
+  } catch {
+    response.writeHead(400).end("Absolute proxy URL required\n");
+    return;
+  }
+  if (!["http:", "https:"].includes(target.protocol)) {
+    response.writeHead(403).end("Proxy protocol is not allowed\n");
+    return;
+  }
+  const route = routeAutoDlHost(target.hostname, routing);
+  logger({ type: "http", route, host: target.hostname });
+  const headers = sanitizedHeaders(request.headers, target.host);
+  let body;
+  try {
+    body = await readRequestBody(request);
+  } catch {
+    response.writeHead(413).end("Proxy request body is too large\n");
+    return;
+  }
+  const routes = route === "liguse-sg" ? [route] : [route, "liguse-sg"];
+  for (let index = 0; index < routes.length; index += 1) {
+    const candidate = routes[index];
+    try {
+      const incoming = await httpAttempt({
+        request,
+        target,
+        headers,
+        body,
+        route: candidate,
+        upstreams,
+        timeoutMs: routing.connectTimeoutMs,
+      });
+      const canRetry = index + 1 < routes.length
+        && isIdempotent(request.method)
+        && retryableStatus(incoming.statusCode);
+      if (canRetry) {
+        incoming.destroy();
+        logger({
+          type: "http-fallback",
+          from: candidate,
+          route: "liguse-sg",
+          host: target.hostname,
+          status: incoming.statusCode,
+        });
+        continue;
+      }
+      response.writeHead(incoming.statusCode ?? 502, incoming.headers);
+      incoming.pipe(response);
+      return;
+    } catch {
+      if (index + 1 < routes.length) {
+        logger({
+          type: "http-fallback",
+          from: candidate,
+          route: "liguse-sg",
+          host: target.hostname,
+        });
+        continue;
+      }
+    }
+  }
+  if (!response.headersSent) response.writeHead(502);
+  response.end("CiteArk routing proxy upstream error\n");
+}
+
+function httpAttempt({ request, target, headers, body, route, upstreams, timeoutMs }) {
+  return new Promise((resolve, reject) => {
+    const upstream = route === "direct" ? null : upstreams[route];
+    const transport = upstream?.protocol === "https:"
+      ? https
+      : upstream
+        ? http
+        : target.protocol === "https:"
+          ? https
+          : http;
+    const options = upstream
+      ? {
+          hostname: upstream.hostname,
+          port: upstream.port,
+          method: request.method,
+          path: target.href,
+          headers: withProxyAuthorization(headers, upstream),
+        }
+      : {
+          hostname: target.hostname,
+          port: target.port || (target.protocol === "https:" ? 443 : 80),
+          method: request.method,
+          path: `${target.pathname}${target.search}`,
+          headers,
+        };
+    const outgoing = transport.request(options, resolve);
+    outgoing.setTimeout(timeoutMs, () => {
+      outgoing.destroy(new Error("HTTP upstream timed out"));
+    });
+    outgoing.once("error", reject);
+    outgoing.end(body);
+  });
+}
+
+async function readRequestBody(request, maximumBytes = 16 * 1024 * 1024) {
+  const chunks = [];
+  let bytes = 0;
+  for await (const chunk of request) {
+    bytes += chunk.length;
+    if (bytes > maximumBytes) throw new Error("Request body is too large");
+    chunks.push(chunk);
+  }
+  return Buffer.concat(chunks);
+}
+
+function isIdempotent(method) {
+  return new Set(["GET", "HEAD", "OPTIONS"]).has(String(method ?? "GET").toUpperCase());
+}
+
+function retryableStatus(status) {
+  return status === 408 || status === 425 || status === 429 || status >= 500;
+}
+
+async function forwardConnect(
+  request,
+  clientSocket,
+  head,
+  upstreams,
+  routing,
+  sockets,
+  logger,
+) {
+  let authority;
+  try {
+    authority = parseAuthority(request.url);
+  } catch {
+    clientSocket.end("HTTP/1.1 400 Bad Request\r\n\r\n");
+    return;
+  }
+  if (![80, 443].includes(authority.port)) {
+    clientSocket.end("HTTP/1.1 403 Forbidden\r\n\r\n");
+    return;
+  }
+  const route = routeAutoDlHost(authority.hostname, routing);
+  logger({ type: "connect", route, host: authority.hostname });
+  try {
+    let upstream;
+    try {
+      upstream = route === "direct"
+        ? await routing.directConnector(authority, sockets, routing.connectTimeoutMs)
+        : await connectThroughProxy(
+            authority,
+            upstreams[route],
+            sockets,
+            routing.connectTimeoutMs,
+          );
+    } catch (error) {
+      if (route === "liguse-sg") throw error;
+      logger({
+        type: "connect-fallback",
+        from: route,
+        route: "liguse-sg",
+        host: authority.hostname,
+      });
+      upstream = await connectThroughProxy(
+        authority,
+        upstreams["liguse-sg"],
+        sockets,
+        routing.connectTimeoutMs,
+      );
+    }
+    if (clientSocket.destroyed) {
+      upstream.destroy();
+      return;
+    }
+    clientSocket.write("HTTP/1.1 200 Connection Established\r\n\r\n");
+    if (head.length) upstream.write(head);
+    upstream.pipe(clientSocket);
+    clientSocket.pipe(upstream);
+    clientSocket.once("error", () => upstream.destroy());
+    upstream.once("error", () => clientSocket.destroy());
+  } catch {
+    if (!clientSocket.destroyed) {
+      clientSocket.end("HTTP/1.1 502 Bad Gateway\r\n\r\n");
+    }
+  }
+}
+
+function connectDirect(authority, sockets, timeoutMs) {
+  return new Promise((resolve, reject) => {
+    const socket = net.connect({ host: authority.hostname, port: authority.port });
+    trackSocket(socket, sockets);
+    const timer = setTimeout(() => {
+      socket.destroy();
+      reject(new Error("Direct connection timed out"));
+    }, timeoutMs);
+    timer.unref();
+    socket.once("connect", () => {
+      clearTimeout(timer);
+      resolve(socket);
+    });
+    socket.once("error", (error) => {
+      clearTimeout(timer);
+      reject(error);
+    });
+  });
+}
+
+function connectThroughProxy(authority, proxy, sockets, timeoutMs) {
+  return new Promise((resolve, reject) => {
+    const socket = proxy.protocol === "https:"
+      ? tls.connect({
+          host: proxy.hostname,
+          port: proxy.port,
+          servername: proxy.hostname,
+        })
+      : net.connect({ host: proxy.hostname, port: proxy.port });
+    trackSocket(socket, sockets);
+    let response = Buffer.alloc(0);
+    const timer = setTimeout(() => {
+      socket.destroy();
+      reject(new Error("Proxy connection timed out"));
+    }, timeoutMs);
+    timer.unref();
+    const connectedEvent = proxy.protocol === "https:" ? "secureConnect" : "connect";
+    socket.once(connectedEvent, () => {
+      const headers = [
+        `CONNECT ${authority.hostname}:${authority.port} HTTP/1.1`,
+        `Host: ${authority.hostname}:${authority.port}`,
+        "Proxy-Connection: keep-alive",
+      ];
+      const authorization = proxyAuthorization(proxy);
+      if (authorization) headers.push(`Proxy-Authorization: ${authorization}`);
+      socket.write(`${headers.join("\r\n")}\r\n\r\n`);
+    });
+    const onData = (chunk) => {
+      response = Buffer.concat([response, chunk]);
+      if (response.length > 64 * 1024) {
+        clearTimeout(timer);
+        socket.destroy();
+        reject(new Error("Proxy response header is too large"));
+        return;
+      }
+      const boundary = response.indexOf("\r\n\r\n");
+      if (boundary < 0) return;
+      socket.off("data", onData);
+      const statusLine = response.subarray(0, boundary).toString("latin1").split("\r\n")[0];
+      if (!/^HTTP\/1\.[01] 2\d\d\b/.test(statusLine)) {
+        clearTimeout(timer);
+        socket.destroy();
+        reject(new Error("Upstream proxy rejected CONNECT"));
+        return;
+      }
+      const remainder = response.subarray(boundary + 4);
+      if (remainder.length) socket.unshift(remainder);
+      clearTimeout(timer);
+      resolve(socket);
+    };
+    socket.on("data", onData);
+    socket.once("error", (error) => {
+      clearTimeout(timer);
+      reject(error);
+    });
+  });
+}
+
+function parseProxyUrl(value, label) {
+  let parsed;
+  try {
+    parsed = new URL(String(value ?? ""));
+  } catch {
+    throw new TypeError(`${label} URL 无效`);
+  }
+  if (!["http:", "https:"].includes(parsed.protocol) || !parsed.hostname) {
+    throw new TypeError(`${label} 必须是 http/https URL`);
+  }
+  return {
+    protocol: parsed.protocol,
+    hostname: parsed.hostname,
+    port: Number(parsed.port || (parsed.protocol === "https:" ? 443 : 80)),
+    username: decodeURIComponent(parsed.username),
+    password: decodeURIComponent(parsed.password),
+  };
+}
+
+function parseAuthority(value) {
+  const parsed = new URL(`http://${String(value ?? "")}`);
+  if (!parsed.hostname) throw new TypeError("CONNECT authority is invalid");
+  return {
+    hostname: normalizeHost(parsed.hostname),
+    port: Number(parsed.port || 443),
+  };
+}
+
+function sanitizedHeaders(input, host) {
+  const headers = { ...input, host };
+  delete headers["proxy-authorization"];
+  delete headers["proxy-connection"];
+  return headers;
+}
+
+function withProxyAuthorization(headers, proxy) {
+  const output = { ...headers };
+  const authorization = proxyAuthorization(proxy);
+  if (authorization) output["proxy-authorization"] = authorization;
+  return output;
+}
+
+function proxyAuthorization(proxy) {
+  if (!proxy.username) return null;
+  return `Basic ${Buffer.from(`${proxy.username}:${proxy.password}`).toString("base64")}`;
+}
+
+function matchesHost(hostname, patterns) {
+  return patterns.some((pattern) => pattern.startsWith(".")
+    ? hostname.endsWith(pattern)
+    : hostname === pattern);
+}
+
+function normalizeHost(value) {
+  return String(value ?? "").toLowerCase().replace(/^\[|\]$/g, "").replace(/\.$/, "");
+}
+
+function trackSocket(socket, sockets) {
+  sockets.add(socket);
+  socket.once("close", () => sockets.delete(socket));
+}
+
+async function main() {
+  const proxy = await startAutoDlRoutingProxy({
+    listenHost: process.env.CITEARK_ROUTING_PROXY_HOST ?? "127.0.0.1",
+    listenPort: Number(process.env.CITEARK_ROUTING_PROXY_PORT ?? 18080),
+    turboProxyUrl: process.env.CITEARK_AUTODL_TURBO_PROXY,
+    liguseProxyUrl: process.env.CITEARK_LIGUSE_SG_PROXY,
+    alwaysProxyHosts: commaList(process.env.CITEARK_ALWAYS_PROXY_HOSTS),
+    logger: (event) => process.stderr.write(`${JSON.stringify(event)}\n`),
+  });
+  process.stderr.write(`${JSON.stringify({
+    type: "citeark.routing_proxy.ready",
+    host: proxy.host,
+    port: proxy.port,
+  })}\n`);
+}
+
+function commaList(value) {
+  return String(value ?? "")
+    .split(",")
+    .map((entry) => normalizeHost(entry.trim()))
+    .filter(Boolean);
+}
+
+if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {
+  await main();
+}

package/src/network/egress-proxy.mjs

@@ -0,0 +1,158 @@
+import { randomBytes, timingSafeEqual } from "node:crypto";
+import http from "node:http";
+import https from "node:https";
+import net from "node:net";
+
+const DEFAULT_ALLOWED_HOSTS = [
+  "arxiv.org",
+  "export.arxiv.org",
+  "github.com",
+  ".githubusercontent.com",
+  ".githubassets.com",
+  "huggingface.co",
+  ".huggingface.co",
+  ".hf.co",
+  ".xethub.hf.co",
+  "pypi.org",
+  ".pythonhosted.org",
+  "download.pytorch.org",
+  ".pytorch.org",
+];
+
+export function previewEgressProxy() {
+  return {
+    mode: "host-allowlist-proxy",
+    containerProxyUrl: null,
+    allowedHosts: [...DEFAULT_ALLOWED_HOSTS],
+    close: async () => {},
+  };
+}
+
+export async function startEgressProxy({ allowedHosts = DEFAULT_ALLOWED_HOSTS } = {}) {
+  // 代理监听 0.0.0.0 以便容器经 host.docker.internal 访问；
+  // 使用每次执行随机生成的令牌做 Proxy-Authorization 鉴权，
+  // 避免同局域网的其他机器借用这条出口（与 provider relay 的令牌机制对齐）。
+  const token = randomBytes(24).toString("base64url");
+  const expectedAuth = Buffer.from(
+    `Basic ${Buffer.from(`citeark-agent:${token}`).toString("base64")}`,
+  );
+  const sockets = new Set();
+  const server = http.createServer((request, response) => {
+    if (!authorized(request.headers["proxy-authorization"], expectedAuth)) {
+      response
+        .writeHead(407, { "proxy-authenticate": 'Basic realm="citeark-egress"' })
+        .end("CiteArk egress proxy authentication required\n");
+      return;
+    }
+    void forwardHttpRequest(request, response, allowedHosts);
+  });
+  server.on("connect", (request, clientSocket, head) => {
+    if (!authorized(request.headers["proxy-authorization"], expectedAuth)) {
+      clientSocket.end(
+        'HTTP/1.1 407 Proxy Authentication Required\r\nProxy-Authenticate: Basic realm="citeark-egress"\r\n\r\n',
+      );
+      return;
+    }
+    forwardConnect(request, clientSocket, head, allowedHosts, sockets);
+  });
+  server.on("connection", (socket) => {
+    sockets.add(socket);
+    socket.once("close", () => sockets.delete(socket));
+  });
+  await new Promise((resolve, reject) => {
+    server.once("error", reject);
+    server.listen(0, "0.0.0.0", resolve);
+  });
+  const address = server.address();
+  const port = typeof address === "object" && address ? address.port : null;
+  return {
+    mode: "host-allowlist-proxy",
+    hostProxyUrl: `http://citeark-agent:${token}@127.0.0.1:${port}`,
+    containerProxyUrl: `http://citeark-agent:${token}@host.docker.internal:${port}`,
+    allowedHosts: [...allowedHosts],
+    close: async () => {
+      for (const socket of sockets) socket.destroy();
+      await new Promise((resolve) => server.close(resolve));
+    },
+  };
+}
+
+function authorized(header, expectedAuth) {
+  if (!header) return false;
+  const actual = Buffer.from(header);
+  return actual.length === expectedAuth.length && timingSafeEqual(actual, expectedAuth);
+}
+
+async function forwardHttpRequest(request, response, allowedHosts) {
+  let target;
+  try {
+    target = new URL(request.url);
+  } catch {
+    response.writeHead(400).end("Absolute proxy URL required\n");
+    return;
+  }
+  if (!allowed(target.hostname, allowedHosts)) {
+    response.writeHead(403).end("Host is not allowed by CiteArk egress policy\n");
+    return;
+  }
+  const transport = target.protocol === "https:" ? https : http;
+  const headers = { ...request.headers, host: target.host };
+  delete headers["proxy-connection"];
+  delete headers["proxy-authorization"];
+  const upstream = transport.request(target, {
+    method: request.method,
+    headers,
+  }, (upstreamResponse) => {
+    response.writeHead(upstreamResponse.statusCode ?? 502, upstreamResponse.headers);
+    upstreamResponse.pipe(response);
+  });
+  upstream.on("error", () => {
+    if (!response.headersSent) response.writeHead(502);
+    response.end("CiteArk egress upstream error\n");
+  });
+  request.pipe(upstream);
+}
+
+function forwardConnect(request, clientSocket, head, allowedHosts, sockets) {
+  const authority = request.url ?? "";
+  const separator = authority.lastIndexOf(":");
+  const hostname = separator > 0 ? authority.slice(0, separator) : authority;
+  const port = separator > 0 ? Number(authority.slice(separator + 1)) : 443;
+  if (!allowed(hostname, allowedHosts) || !new Set([80, 443]).has(port)) {
+    clientSocket.end("HTTP/1.1 403 Forbidden\r\n\r\n");
+    return;
+  }
+  const upstream = net.connect({ host: hostname, port });
+  sockets.add(upstream);
+  // A desktop task can detach its terminal while a long-running download is
+  // still in flight. Treat either half of the CONNECT tunnel disappearing as a
+  // normal cancellation; otherwise a late pipe write can raise an unhandled
+  // EPIPE and kill the whole pipeline coordinator.
+  clientSocket.once("error", () => upstream.destroy());
+  upstream.once("close", () => sockets.delete(upstream));
+  upstream.once("connect", () => {
+    if (clientSocket.destroyed) {
+      upstream.destroy();
+      return;
+    }
+    clientSocket.write("HTTP/1.1 200 Connection Established\r\n\r\n");
+    if (head.length) upstream.write(head);
+    upstream.pipe(clientSocket);
+    clientSocket.pipe(upstream);
+  });
+  upstream.once("error", () => {
+    if (!clientSocket.destroyed) {
+      clientSocket.end("HTTP/1.1 502 Bad Gateway\r\n\r\n");
+    }
+  });
+}
+
+function allowed(hostname, allowedHosts) {
+  const normalized = hostname.toLowerCase().replace(/^\[|\]$/g, "");
+  return allowedHosts.some((entry) => {
+    const expected = entry.toLowerCase();
+    return expected.startsWith(".")
+      ? normalized.endsWith(expected)
+      : normalized === expected;
+  });
+}