@citeark/agent 0.3.20

package/src/compute/autodl-pro-network.mjs

@@ -0,0 +1,295 @@
+import { readFile } from "node:fs/promises";
+
+import { GcpSecretManagerClient } from "../gcp/secret-manager.mjs";
+import { shellQuote } from "../util.mjs";
+
+const ROUTER_SOURCE = new URL("../network/autodl-routing-proxy.mjs", import.meta.url);
+
+export async function resolveAutoDlManagedNetwork({ config, secretManager }) {
+  const resource = config.liguseSgTunnelSecretResource;
+  if (!resource) return null;
+  const resolver = secretManager ?? new GcpSecretManagerClient();
+  const raw = await resolver.accessVersion(resource);
+  return parseLiguseSgTunnelSecret(raw);
+}
+
+export function parseLiguseSgTunnelSecret(raw) {
+  let value;
+  try {
+    value = JSON.parse(String(raw));
+  } catch {
+    throw new TypeError("LiguseSG 隧道 Secret 必须是 JSON");
+  }
+  const host = requiredText(value.host, "host", 255);
+  const username = requiredText(value.username, "username", 64);
+  const privateKey = requiredText(value.privateKey, "privateKey", 32 * 1024);
+  const knownHosts = requiredText(value.knownHosts, "knownHosts", 16 * 1024);
+  const port = positiveInteger(value.port, "port");
+  const remoteProxyPort = positiveInteger(value.remoteProxyPort ?? 18118, "remoteProxyPort");
+  if (!/^[a-z_][a-z0-9_-]*[$]?$/i.test(username)) {
+    throw new TypeError("LiguseSG 隧道 username 格式无效");
+  }
+  if (!privateKey.includes("BEGIN OPENSSH PRIVATE KEY")) {
+    throw new TypeError("LiguseSG 隧道 privateKey 不是 OpenSSH 私钥");
+  }
+  if (!/\bssh-(ed25519|rsa)\b/.test(knownHosts)) {
+    throw new TypeError("LiguseSG 隧道 knownHosts 缺少 SSH 主机公钥");
+  }
+  return { host, port, username, privateKey, knownHosts, remoteProxyPort };
+}
+
+export async function autoDlRoutingProxySource() {
+  return await readFile(ROUTER_SOURCE, "utf8");
+}
+
+export function buildAutoDlNetworkRunnerScript({
+  workerPath,
+  routerPath,
+  keyPath,
+  knownHostsPath,
+  turboScript,
+  tunnel,
+  providerBaseUrl = "",
+  alwaysProxyHosts = [],
+  routerPort = 18080,
+  tunnelPort = 18081,
+  executionDirectory = "/job/execution",
+}) {
+  const sshTarget = `${tunnel.username}@${tunnel.host}`;
+  return `#!/usr/bin/env bash
+set -euo pipefail
+
+WORKER=${shellQuote(workerPath)}
+ROUTER=${shellQuote(routerPath)}
+TUNNEL_KEY=${shellQuote(keyPath)}
+KNOWN_HOSTS=${shellQuote(knownHostsPath)}
+TURBO_SCRIPT=${shellQuote(turboScript)}
+ROUTER_PORT=${routerPort}
+TUNNEL_PORT=${tunnelPort}
+EXECUTION_DIRECTORY=${shellQuote(executionDirectory)}
+PROVIDER_BASE_URL=${shellQuote(providerBaseUrl)}
+ALWAYS_PROXY_HOSTS=${shellQuote(alwaysProxyHosts.join(","))}
+TUNNEL_PID=
+ROUTER_PID=
+
+cleanup_network() {
+  rm -f "$TUNNEL_KEY"
+  if [ -n "\${ROUTER_PID:-}" ] && kill -0 "$ROUTER_PID" 2>/dev/null; then
+    kill -TERM "$ROUTER_PID" 2>/dev/null || true
+    wait "$ROUTER_PID" 2>/dev/null || true
+  fi
+  if [ -n "\${TUNNEL_PID:-}" ] && kill -0 "$TUNNEL_PID" 2>/dev/null; then
+    kill -TERM "$TUNNEL_PID" 2>/dev/null || true
+    wait "$TUNNEL_PID" 2>/dev/null || true
+  fi
+}
+trap cleanup_network EXIT TERM INT
+
+mkdir -p "$EXECUTION_DIRECTORY"
+if [ ! -r "$TURBO_SCRIPT" ]; then
+  printf '%s\n' "AutoDL 学术加速脚本不存在：$TURBO_SCRIPT" >&2
+  exit 70
+fi
+set +u
+. "$TURBO_SCRIPT" >/dev/null
+set -u
+TURBO_PROXY="\${https_proxy:-\${HTTPS_PROXY:-\${http_proxy:-\${HTTP_PROXY:-}}}}"
+case "$TURBO_PROXY" in
+  http://*|https://*) ;;
+  *) printf '%s\n' "AutoDL 学术加速没有提供 http/https 代理" >&2; exit 70 ;;
+esac
+unset http_proxy https_proxy HTTP_PROXY HTTPS_PROXY all_proxy ALL_PROXY no_proxy NO_PROXY
+
+ssh -N -T \
+  -L "127.0.0.1:$TUNNEL_PORT:127.0.0.1:${tunnel.remoteProxyPort}" \
+  -i "$TUNNEL_KEY" \
+  -p ${tunnel.port} \
+  -o BatchMode=yes \
+  -o ConnectTimeout=20 \
+  -o ConnectionAttempts=3 \
+  -o ExitOnForwardFailure=yes \
+  -o IdentitiesOnly=yes \
+  -o ServerAliveInterval=15 \
+  -o ServerAliveCountMax=3 \
+  -o StrictHostKeyChecking=yes \
+  -o UserKnownHostsFile="$KNOWN_HOSTS" \
+  ${shellQuote(sshTarget)} >> "$EXECUTION_DIRECTORY/liguse-tunnel.log" 2>&1 &
+TUNNEL_PID=$!
+
+TUNNEL_READY=0
+TUNNEL_READY_DEADLINE=$((SECONDS + 60))
+while [ "$SECONDS" -lt "$TUNNEL_READY_DEADLINE" ]; do
+  kill -0 "$TUNNEL_PID" 2>/dev/null || {
+    printf '%s\n' "LiguseSG SSH 隧道启动失败" >&2
+    exit 70
+  }
+  if curl --proxy "http://127.0.0.1:$TUNNEL_PORT" --fail --silent \
+    --output /dev/null --connect-timeout 3 --max-time 10 \
+    https://www.google.com/generate_204; then
+    TUNNEL_READY=1
+    break
+  fi
+  sleep 0.5
+done
+if [ "$TUNNEL_READY" -ne 1 ]; then
+  printf '%s\n' "LiguseSG 受限公网出口在 60 秒内未就绪" >&2
+  exit 70
+fi
+rm -f "$TUNNEL_KEY"
+
+env \
+  CITEARK_ROUTING_PROXY_HOST=127.0.0.1 \
+  CITEARK_ROUTING_PROXY_PORT="$ROUTER_PORT" \
+  CITEARK_AUTODL_TURBO_PROXY="$TURBO_PROXY" \
+  CITEARK_LIGUSE_SG_PROXY="http://127.0.0.1:$TUNNEL_PORT" \
+  CITEARK_ALWAYS_PROXY_HOSTS="$ALWAYS_PROXY_HOSTS" \
+  node "$ROUTER" >> "$EXECUTION_DIRECTORY/network-router.log" 2>&1 &
+ROUTER_PID=$!
+unset TURBO_PROXY
+
+ROUTER_READY=0
+ROUTER_READY_DEADLINE=$((SECONDS + 30))
+while [ "$SECONDS" -lt "$ROUTER_READY_DEADLINE" ]; do
+  kill -0 "$ROUTER_PID" 2>/dev/null || {
+    printf '%s\n' "AutoDL 分域代理启动失败" >&2
+    exit 70
+  }
+  if curl --fail --silent --output /dev/null --connect-timeout 1 --max-time 2 \
+    "http://127.0.0.1:$ROUTER_PORT/__citeark_health"; then
+    ROUTER_READY=1
+    break
+  fi
+  sleep 0.25
+done
+if [ "$ROUTER_READY" -ne 1 ]; then
+  printf '%s\n' "AutoDL 分域代理在 30 秒内未就绪" >&2
+  exit 70
+fi
+
+PROXY_URL="http://127.0.0.1:$ROUTER_PORT"
+export HTTP_PROXY="$PROXY_URL" HTTPS_PROXY="$PROXY_URL"
+export http_proxy="$PROXY_URL" https_proxy="$PROXY_URL"
+export ALL_PROXY="$PROXY_URL" all_proxy="$PROXY_URL"
+export NO_PROXY="127.0.0.1,localhost,::1"
+export no_proxy="$NO_PROXY"
+export NODE_USE_ENV_PROXY=1
+install -d -o node -g node -m 0755 /opt/citeark/cache/cargo /opt/citeark/cache/pip /opt/citeark/cache/uv
+if [ -x /opt/citeark/venv/bin/python ]; then
+  if [ ! -e /opt/citeark/.baseline-readonly-v1 ]; then
+    chmod -R a-w /opt/citeark/venv
+    touch /opt/citeark/.baseline-readonly-v1
+  fi
+  TARGET_VIRTUAL_ENV=/job/runtime-venv
+  if [ ! -x /opt/citeark/create-runtime-venv.sh ]; then
+    printf '%s\n' "AutoDL 私有镜像缺少运行虚拟环境脚本" >&2
+    exit 70
+  fi
+  /opt/citeark/create-runtime-venv.sh "$TARGET_VIRTUAL_ENV"
+  export VIRTUAL_ENV="$TARGET_VIRTUAL_ENV"
+  export PATH="$TARGET_VIRTUAL_ENV/bin:/opt/citeark/venv/bin:$PATH"
+  export UV_PROJECT_ENVIRONMENT="$TARGET_VIRTUAL_ENV"
+fi
+export PIP_CACHE_DIR=/opt/citeark/cache/pip
+export UV_CACHE_DIR=/opt/citeark/cache/uv
+export UV_CONFIG_FILE=/dev/null
+PYTHON_PACKAGE_INDEX="\${CITEARK_PYTHON_PACKAGE_INDEX:-https://pypi.tuna.tsinghua.edu.cn/simple}"
+PYTHON_PACKAGE_FALLBACK_INDEX="\${CITEARK_PYTHON_PACKAGE_FALLBACK_INDEX:-https://mirrors.aliyun.com/pypi/simple}"
+HUGGINGFACE_ENDPOINT="\${CITEARK_HUGGINGFACE_ENDPOINT:-https://hf-mirror.com}"
+export PIP_INDEX_URL="$PYTHON_PACKAGE_INDEX"
+export PIP_DEFAULT_TIMEOUT=60
+export PIP_RETRIES=8
+export UV_INDEX="$PYTHON_PACKAGE_INDEX"
+export UV_DEFAULT_INDEX="$PYTHON_PACKAGE_FALLBACK_INDEX"
+export UV_INDEX_STRATEGY=first-index
+export UV_HTTP_TIMEOUT=60
+export UV_HTTP_RETRIES=8
+export PIP_FIND_LINKS=/opt/citeark/wheels
+export UV_FIND_LINKS=/opt/citeark/wheels
+export CARGO_HOME=/opt/citeark/cache/cargo
+export CARGO_NET_GIT_FETCH_WITH_CLI=true
+export HF_HOME=/job/runtime-cache/huggingface
+export HF_ENDPOINT="$HUGGINGFACE_ENDPOINT"
+export HF_HUB_ETAG_TIMEOUT=30
+export HF_HUB_DOWNLOAD_TIMEOUT=120
+export TORCH_HOME=/job/runtime-cache/torch
+install -d -o node -g node -m 0700 "$HF_HOME" "$TORCH_HOME"
+
+if ! node --help 2>&1 | grep -q -- '--use-env-proxy'; then
+  printf '%s\n' "AutoDL Node.js 运行时不支持全局环境代理" >&2
+  exit 70
+fi
+
+probe() {
+  local name="$1"
+  local url attempt
+  shift
+  for url in "$@"; do
+    attempt=1
+    while [ "$attempt" -le 5 ]; do
+      if curl --proxy "$PROXY_URL" --fail --silent --show-error --location \
+        --retry 2 --retry-all-errors --retry-delay 1 --connect-timeout 10 \
+        --output /dev/null --max-time 30 "$url"; then
+        return 0
+      fi
+      if [ "$attempt" -lt 5 ]; then sleep "$attempt"; fi
+      attempt=$((attempt + 1))
+    done
+    printf '%s\n' "AutoDL 网络预检候选失败：$name $url" >&2
+  done
+  printf '%s\n' "AutoDL 网络预检失败：$name（所有候选传输均不可达）" >&2
+  exit 70
+}
+probe github https://github.com/
+probe huggingface "$HUGGINGFACE_ENDPOINT/" https://huggingface.co/
+probe python-index "$PYTHON_PACKAGE_INDEX/pip/" "$PYTHON_PACKAGE_FALLBACK_INDEX/pip/"
+probe google https://www.google.com/generate_204
+probe citeark https://citeark.co/api/health
+if [ -n "$PROVIDER_BASE_URL" ]; then
+  if ! node --use-env-proxy --input-type=module -e '
+    import { setTimeout as delay } from "node:timers/promises";
+    for (let attempt = 1; attempt <= 5; attempt += 1) {
+      try {
+        const response = await fetch(process.argv[1], {
+          redirect: "manual",
+          signal: AbortSignal.timeout(20_000),
+        });
+        const reachable = response.status < 500;
+        await response.body?.cancel();
+        if (reachable) process.exit(0);
+      } catch {}
+      if (attempt < 5) await delay(attempt * 1_000);
+    }
+    process.exit(1);
+  ' "$PROVIDER_BASE_URL"; then
+    printf '%s\n' "AutoDL 网络预检失败：Node.js 全局代理未能访问模型 API" >&2
+    exit 70
+  fi
+fi
+
+jq -nc \
+  --arg checkedAt "$(date -u +%Y-%m-%dT%H:%M:%SZ)" \
+  '{schemaVersion:"0.1",status:"passed",checkedAt:$checkedAt,globalHttpProxy:true,nodeEnvProxy:true,routes:{github:"autodl-network-turbo-with-liguse-fallback",huggingface:"domestic-endpoint-with-official-turbo-fallback",google:"liguse-sg",openrouter:"liguse-sg",citeark:"direct-with-liguse-fallback",provider:"policy-routed",default:"direct-with-liguse-fallback"}}' \
+  > "$EXECUTION_DIRECTORY/network-preflight.json"
+
+set +e
+bash "$WORKER"
+WORKER_EXIT=$?
+set -e
+exit "$WORKER_EXIT"
+`;
+}
+
+function requiredText(value, field, maximumLength) {
+  if (typeof value !== "string" || !value.trim() || value.length > maximumLength) {
+    throw new TypeError(`LiguseSG 隧道 ${field} 无效`);
+  }
+  return value.trim();
+}
+
+function positiveInteger(value, field) {
+  const numeric = Number(value);
+  if (!Number.isInteger(numeric) || numeric <= 0 || numeric > 65_535) {
+    throw new TypeError(`LiguseSG 隧道 ${field} 必须是有效端口`);
+  }
+  return numeric;
+}