@sema-agent/server 1.185.0

package/deploy/sema-up/cluster-up.sh

@@ -0,0 +1,432 @@
+#!/usr/bin/env bash
+# cluster-up — sema-up 的 full-cluster 装配(Phase 1b as-built 的编码化)。
+# 由 sema-up.sh 在 PROFILE=full-cluster 时调用,也可单独跑(env 见下)。
+# 流程:机器互达检 → k3s HA(embedded etcd)→ VXLAN 守卫 → Longhorn → kata-deploy →
+#       CNPG operator → 镜像分发 → 渲染 sema-stack-cluster.yaml → apply → token 复制 →
+#       建桶 → smoke。全程幂等 check-then-apply;凭据不轮换;密码不入日志。
+#
+# 输入(sema-up.sh 解析 machines.yaml 后注入):
+#   MACHINE_COUNT + MACHINE_<i>_{HOST,USER,SSH_KEY,NAME,ROLE,PRIVATE_IP,PUBLIC_IFACE}
+#   WORK(状态目录,.env 在此)REGION PROJECT
+#   SEMA_SERVER_IMAGE(registry 引用)或 SEMA_SERVER_IMAGE_TAR(本地 docker save tar[.gz])二选一
+#   MODEL_GATEWAY_BASEURL MODEL_ID MODEL_API_KEY
+#   ALLOW_SMALL_CLUSTER=true 放行 <3 机(dev/CI 单节点验流程;生产必须 >=3 奇数)
+set -euo pipefail
+HERE="$(cd "$(dirname "$0")" && pwd)"
+log() { printf '[cluster-up] %s\n' "$*" >&2; }
+die() { log "⛔ $*"; exit 1; }
+
+: "${WORK:?WORK 未设置(状态目录)}"
+: "${MACHINE_COUNT:?MACHINE_COUNT 未设置(machines.yaml 未解析?)}"
+REGION="${REGION:-global}"
+K3S_CHANNEL="${K3S_CHANNEL:-stable}"
+CNPG_VERSION="${CNPG_VERSION:-1.30.0}"   # ⚠ in-tree barman 1.31 移除;升版前先迁 barman-cloud plugin
+LONGHORN_VERSION="${LONGHORN_VERSION:-1.10.0}"
+KATA_VERSION="${KATA_VERSION:-3.21.0}"
+NODEPORT=30890
+
+# <3 机=建议不强求(clay 2026-07-12:有建议的 3 节点配置,仍能处理;某台机器过小才拦——
+# 那个由 0/8 每机预检的 8G/40G 硬底线负责)。ALLOW_SMALL_CLUSTER 保留兼容,不再必需。
+if [ "$MACHINE_COUNT" -lt 3 ]; then
+  log "⚠ 建议:准生产=3 机(embedded etcd HA+PG 反亲和);当前 $MACHINE_COUNT 机能跑但 etcd 无 HA(单点/双机 quorum 脆),形态按机器数缩放"
+fi
+if [ $((MACHINE_COUNT % 2)) -eq 0 ]; then
+  log "⚠ 偶数台($MACHINE_COUNT)etcd 容错不增反险(quorum=$((MACHINE_COUNT/2+1))),建议奇数"
+fi
+
+# ── ssh 帮手(role 无关,i 从 0)────────────────────────────────────────────────
+mvar() { local n="MACHINE_$1_$2"; printf '%s' "${!n:-}"; }
+mssh() { # mssh <i> <cmd…>;远端跑 sudo bash -s 读 stdin 时用 msshs
+  local i="$1"; shift
+  local key; key="$(mvar "$i" SSH_KEY)"
+  ssh -o ConnectTimeout=15 -o ServerAliveInterval=15 -o ServerAliveCountMax=8 \
+    -o StrictHostKeyChecking=accept-new ${key:+-i "${key/#\~/$HOME}"} \
+    "$(mvar "$i" USER)@$(mvar "$i" HOST)" "$@"
+}
+mscp() { # mscp <i> <local> <remote>
+  local i="$1"; local key; key="$(mvar "$i" SSH_KEY)"
+  scp -o ConnectTimeout=15 -o StrictHostKeyChecking=accept-new ${key:+-i "${key/#\~/$HOME}"} \
+    "$2" "$(mvar "$i" USER)@$(mvar "$i" HOST):$3"
+}
+# 远端 root 执行:user=root 直跑,否则要求 passwordless sudo(预检里验)
+rsudo() { local i="$1"; shift; if [ "$(mvar "$i" USER)" = "root" ]; then mssh "$i" bash -s -- "$@"; else mssh "$i" sudo bash -s -- "$@"; fi; }
+# 远端单命令(stdin 归命令用,如 kubectl apply -f -)
+rrun() { local i="$1"; shift; if [ "$(mvar "$i" USER)" = "root" ]; then mssh "$i" "$@"; else mssh "$i" sudo "$@"; fi; }
+
+# ── 0. 互达 + sudo + 硬件预检(在目标机上,§3d-2:错误带原因+建议)+ hostname 告警 ──
+log "0/8 机器预检($MACHINE_COUNT 台)"
+declare -a SYS_HOSTNAMES=()
+for ((i=0; i<MACHINE_COUNT; i++)); do
+  out=$(rsudo "$i" <<'EOF'
+set -e
+command -v curl >/dev/null || { echo "FAIL curl 缺失——建议:apt-get install -y curl"; exit 1; }
+[ "$(uname -m)" = "x86_64" ] || [ "$(uname -m)" = "aarch64" ] || { echo "FAIL 架构 $(uname -m) 不支持——建议:x86_64/aarch64 机器"; exit 1; }
+ram=$(awk '/MemTotal/{printf "%d", $2/1024}' /proc/meminfo)
+[ "$ram" -ge 8192 ] || { echo "FAIL 内存 ${ram}MB < 8G——建议:full-cluster 节点至少 8G(数据面+沙箱同机)"; exit 1; }
+disk=$(df -m / | awk 'NR==2{print $4}')
+[ "$disk" -ge 40960 ] || { echo "FAIL 根盘可用 ${disk}MB < 40G——建议:Longhorn/镜像要空间,清盘或换机"; exit 1; }
+[ -e /dev/kvm ] || echo "WARN 无 /dev/kvm——kata VM 沙箱在本节点不可用(建议:开嵌套虚拟化/裸机;或全集群改 e2b lane)"
+hostname
+EOF
+  ) || die "机器 $i($(mvar "$i" HOST))预检失败:${out:-ssh/sudo 不通——需要 key 互信 + passwordless sudo}"
+  case "$out" in *WARN*) log "  ⚠ $(mvar "$i" HOST):${out%%$'\n'*}" ;; esac
+  SYS_HOSTNAMES+=("$(printf '%s\n' "$out" | tail -1)")
+done
+if [ "$(printf '%s\n' "${SYS_HOSTNAMES[@]}" | sort -u | wc -l)" -lt "$MACHINE_COUNT" ]; then
+  log "⚠ 系统 hostname 有重复(${SYS_HOSTNAMES[*]})——裸机批次常见;k3s 一律用唯一 --node-name 注册,不依赖系统名(Phase 1b etcd 撞名教训)"
+fi
+
+# region=auto:从目标机(不是操作员机)探测网络亲和
+if [ "$REGION" = "auto" ]; then
+  REGION=$(rsudo 0 <<'EOF'
+a=$(curl -m3 -so /dev/null -w '%{time_total}' https://mirrors.aliyun.com 2>/dev/null || echo 99)
+b=$(curl -m3 -so /dev/null -w '%{time_total}' https://registry.npmjs.org 2>/dev/null || echo 99)
+awk -v a="$a" -v b="$b" 'BEGIN{print (a<b)?"cn":"global"}'
+EOF
+  )
+  log "region=auto → $REGION(目标机探测)"
+fi
+
+# ── 1. k3s HA ────────────────────────────────────────────────────────────────────
+# 每台的 --node-ip:private_ip 优先(东西向走 VLAN);NodePort 限私网段;servicelb 关(入口=外部 LB)
+k3s_install_env() { # k3s 安装的 region 化(cn 走 rancher 国内镜像)
+  if [ "$REGION" = "cn" ]; then printf 'INSTALL_K3S_MIRROR=cn '; fi
+}
+node_flags() { # node_flags <i> → 该机的 k3s server 通用 flag(字段已在 parser 层严格校验)
+  local i="$1" ip cidr flags=""
+  ip="$(mvar "$i" PRIVATE_IP)"; [ -z "$ip" ] && ip="$(mvar "$i" HOST)"
+  flags="--node-name $(mvar "$i" NAME) --node-ip $ip --advertise-address $ip --disable servicelb"
+  # NodePort 限网段:private_cidr 显式优先;IPv4 才推断 /24;FQDN/IPv6 不加(全监听,由外层防火墙管)
+  cidr="$(mvar "$i" PRIVATE_CIDR)"
+  if [ -z "$cidr" ] && [[ "$ip" =~ ^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$ ]]; then cidr="${ip%.*}.0/24"; fi
+  [ -n "$cidr" ] && flags="$flags --kube-proxy-arg nodeport-addresses=$cidr"
+  printf '%s' "$flags"
+}
+K3S0_IP="$(mvar 0 PRIVATE_IP)"; [ -z "$K3S0_IP" ] && K3S0_IP="$(mvar 0 HOST)"
+
+if rsudo 0 <<<'test -f /usr/local/bin/k3s && systemctl is-active --quiet k3s' 2>/dev/null; then
+  log "1/8 k3s 首节点已在跑,跳过安装"
+else
+  log "1/8 k3s 首节点($(mvar 0 NAME) @ $(mvar 0 HOST))"
+  rsudo 0 <<EOF
+set -e
+curl -sfL https://get.k3s.io | $(k3s_install_env)INSTALL_K3S_CHANNEL=$K3S_CHANNEL \
+  INSTALL_K3S_EXEC="server --cluster-init $(node_flags 0)" sh -
+EOF
+fi
+TOKEN="$(rsudo 0 <<<'cat /var/lib/rancher/k3s/server/node-token')" || die "读不到 k3s node-token"
+for ((i=1; i<MACHINE_COUNT; i++)); do
+  if rsudo "$i" <<<'test -f /usr/local/bin/k3s && systemctl is-active --quiet k3s' 2>/dev/null; then
+    # 活着 ≠ 属于本集群:被复用的机器可能挂在别的集群上,静默跳过=幽灵成员
+    if rsudo 0 <<<"k3s kubectl get node $(mvar "$i" NAME) --no-headers >/dev/null 2>&1"; then
+      log "1/8 节点 $(mvar "$i" NAME) 已是本集群成员,跳过"
+      continue
+    fi
+    die "机器 $(mvar "$i" HOST) 上 k3s 在跑但不是本集群成员($(mvar "$i" NAME) 不在)——先在该机跑 k3s-uninstall.sh 再重来"
+  fi
+  log "1/8 加入节点 $(mvar "$i" NAME) @ $(mvar "$i" HOST)"
+  rsudo "$i" <<EOF
+set -e
+curl -sfL https://get.k3s.io | $(k3s_install_env)INSTALL_K3S_CHANNEL=$K3S_CHANNEL \
+  K3S_TOKEN='$TOKEN' INSTALL_K3S_EXEC="server --server https://$K3S0_IP:6443 $(node_flags "$i")" sh -
+EOF
+done
+# Ready 等待(embedded etcd 首次选主可能要 ~1min)
+rsudo 0 <<EOF || die "k3s 节点未全 Ready"
+set -e
+for t in \$(seq 1 60); do
+  n=\$(k3s kubectl get nodes --no-headers 2>/dev/null | grep -cw Ready || true)
+  [ "\$n" -ge "$MACHINE_COUNT" ] && { k3s kubectl get nodes -o wide; exit 0; }
+  sleep 5
+done
+k3s kubectl get nodes -o wide; exit 1
+EOF
+
+# ── 2. VXLAN 守卫(有公网口的机器:flannel 8472 只许私网,公网口 DROP;systemd 常驻,
+#      k3s 重装会重置 iptables——unit 常驻保证 reboot/重装后规则还在)────────────────
+log "2/8 VXLAN 公网口守卫"
+for ((i=0; i<MACHINE_COUNT; i++)); do
+  iface="$(mvar "$i" PUBLIC_IFACE)"; [ -z "$iface" ] && continue
+  rsudo "$i" <<EOF
+set -e
+cat > /etc/systemd/system/sema-vxlan-guard.service <<'UNIT'
+[Unit]
+Description=Drop flannel VXLAN on public iface (sema isolation invariant)
+After=network-online.target k3s.service
+[Service]
+Type=oneshot
+RemainAfterExit=yes
+ExecStart=/bin/sh -c 'iptables -C INPUT -i $iface -p udp --dport 8472 -j DROP 2>/dev/null || iptables -I INPUT -i $iface -p udp --dport 8472 -j DROP'
+[Install]
+WantedBy=multi-user.target
+UNIT
+systemctl daemon-reload && systemctl enable --now sema-vxlan-guard.service
+iptables -C INPUT -i $iface -p udp --dport 8472 -j DROP
+EOF
+  log "  $(mvar "$i" NAME):8472/udp DROP on $iface ✓"
+done
+
+# ── 3. helm(装配工具,仅首节点)────────────────────────────────────────────────
+log "3/8 helm + 基础组件"
+rsudo 0 <<'EOF'
+set -e
+command -v helm >/dev/null && exit 0
+curl -fsSL https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash >/dev/null
+EOF
+KC="--kubeconfig /etc/rancher/k3s/k3s.yaml"
+
+# Longhorn(分布式存储;replica 数=min(机器数,2)——RAID1 之上跨机冗余)
+LH_REPLICAS=$(( MACHINE_COUNT < 2 ? 1 : 2 ))
+rsudo 0 <<EOF
+set -e
+if ! k3s kubectl get storageclass longhorn >/dev/null 2>&1; then
+  helm $KC repo add longhorn https://charts.longhorn.io >/dev/null
+  helm $KC repo update >/dev/null
+  # ⚠ defaultSettings.* 必须 --set-string(chart 模板校验字符串型,int 直接 install 失败——真踩);
+  # upgrade --install:半途失败的 release 重跑可修复(裸 install 会撞已存在)
+  helm $KC upgrade --install longhorn longhorn/longhorn --namespace longhorn-system --create-namespace \
+    --version $LONGHORN_VERSION --set-string defaultSettings.defaultReplicaCount=$LH_REPLICAS \
+    --set persistence.defaultClassReplicaCount=$LH_REPLICAS --wait --timeout 10m >/dev/null
+fi
+# longhorn=默认 SC,local-path 摘掉 default 标(两个 default 会让 PVC 行为不定)
+k3s kubectl patch storageclass local-path -p '{"metadata":{"annotations":{"storageclass.kubernetes.io/is-default-class":"false"}}}' >/dev/null 2>&1 || true
+k3s kubectl patch storageclass longhorn -p '{"metadata":{"annotations":{"storageclass.kubernetes.io/is-default-class":"true"}}}' >/dev/null
+echo LONGHORN_OK
+EOF
+
+# kata-deploy(官方 helm chart;kustomize remote base 老路径已失效——真踩)
+log "4/8 kata-deploy(VM 沙箱运行时)"
+rsudo 0 <<EOF
+set -e
+k3s kubectl get runtimeclass kata-qemu >/dev/null 2>&1 && { echo KATA_ALREADY; exit 0; }
+helm $KC upgrade --install kata-deploy oci://ghcr.io/kata-containers/kata-deploy-charts/kata-deploy \
+  --version $KATA_VERSION --namespace kube-system --set k8sDistribution=k3s --wait --timeout 10m >/dev/null
+for t in \$(seq 1 36); do k3s kubectl get runtimeclass kata-qemu >/dev/null 2>&1 && break; sleep 5; done
+k3s kubectl get runtimeclass kata-qemu >/dev/null
+echo KATA_OK
+EOF
+
+# CNPG operator(钉 $CNPG_VERSION)
+log "5/8 CNPG operator $CNPG_VERSION"
+rsudo 0 <<EOF
+set -e
+if ! k3s kubectl get deployment -n cnpg-system cnpg-controller-manager >/dev/null 2>&1; then
+  # 组件刚装完 apiserver 聚合 openapi 可能未就绪(瞬时 "unable to handle the request")——重试非降级
+  ok=0
+  for t in 1 2 3 4 5; do
+    k3s kubectl apply --server-side -f \
+      "https://raw.githubusercontent.com/cloudnative-pg/cloudnative-pg/release-${CNPG_VERSION%.*}/releases/cnpg-$CNPG_VERSION.yaml" >/dev/null && { ok=1; break; }
+    sleep 15
+  done
+  [ "\$ok" = 1 ] || { echo "CNPG manifest apply 重试耗尽" >&2; exit 1; }
+fi
+k3s kubectl rollout status deployment/cnpg-controller-manager -n cnpg-system --timeout=300s >/dev/null
+echo CNPG_OK
+EOF
+
+# ── 6. server 镜像:registry 引用 or tar 预灌 ─────────────────────────────────────
+log "6/8 server 镜像"
+SERVER_IMAGE="" PULL_POLICY=""
+if [ -n "${SEMA_SERVER_IMAGE:-}" ]; then
+  SERVER_IMAGE="$SEMA_SERVER_IMAGE"; PULL_POLICY="IfNotPresent"
+  log "  registry 引用:$SERVER_IMAGE(节点自行拉取)"
+elif [ -n "${SEMA_SERVER_IMAGE_TAR:-}" ]; then
+  [ -r "$SEMA_SERVER_IMAGE_TAR" ] || die "SEMA_SERVER_IMAGE_TAR 不可读:$SEMA_SERVER_IMAGE_TAR"
+  SERVER_IMAGE="docker.io/library/sema-server:local"; PULL_POLICY="Never"
+  for ((i=0; i<MACHINE_COUNT; i++)); do
+    # ⚠ 每台都灌,包括"看起来已有"的——k3s-uninstall 重装过的节点镜像会被清(复审真抓)
+    log "  预灌 $(mvar "$i" NAME) …"
+    mscp "$i" "$SEMA_SERVER_IMAGE_TAR" /tmp/sema-server-image.tar.upload
+    rsudo "$i" <<'EOF'
+set -e
+trap 'rm -f /tmp/sema-server-image.tar.upload /tmp/sema-server-image.tar' EXIT
+f=/tmp/sema-server-image.tar.upload
+case "$(head -c2 "$f" | od -An -tx1 | tr -d ' ')" in 1f8b) gunzip -c "$f" > /tmp/sema-server-image.tar && f=/tmp/sema-server-image.tar ;; esac
+k3s ctr images import "$f" >/dev/null
+k3s ctr images ls | grep -q sema-server
+EOF
+  done
+else
+  die "需要 SEMA_SERVER_IMAGE(registry 引用)或 SEMA_SERVER_IMAGE_TAR(docker save 包)之一"
+fi
+
+# ── 6b. 私有 registry 拉取凭据(可选。SWR 已弃用 2026-07-13:现行镜像池=docker.io/claybobby 公开仓、
+#     sema-server=ghcr.io/clayboby/sema-server,均 public 匿名可拉,CN 兜底=内网 Gitea registry——
+#     公开池部署本步冗余无害,regcred 仅在镜像换成 private 仓时才需要;未设 env 即整段跳过)──
+# 凭据只走 env:IMAGE_PULL_USER + IMAGE_PULL_TOKEN(+可选 IMAGE_PULL_REGISTRY,默认 Docker Hub)。
+# 值不进仓、不进 argv(/proc cmdline 同机可见——渲染同款纪律):dockerconfigjson 本地拼好经 stdin apply。
+# secret 建在 sema + sema-sandbox 两个 ns 并 patch default SA 的 imagePullSecrets:
+#   - sema:server Deployment(default SA)拉 private 仓镜像(现行 GHCR/Hub 公开镜像无需凭据);
+#   - sema-sandbox:server 经 API 起的沙箱 pod(K8S_SANDBOX_IMAGE 换成 private 池镜像时)免改 adapter。
+# ⚠ SA 的 imagePullSecrets 在 pod 创建时快照——先建 secret/patch SA,再 apply 栈(顺序有意)。
+if [ -n "${IMAGE_PULL_USER:-}" ] && [ -n "${IMAGE_PULL_TOKEN:-}" ]; then
+  log "6b/8 imagePullSecret(registry=${IMAGE_PULL_REGISTRY:-https://index.docker.io/v1/})"
+  for ns in sema sema-sandbox; do
+    rrun 0 k3s kubectl create namespace "$ns" --dry-run=client -o yaml | rrun 0 k3s kubectl apply -f - >/dev/null
+    # helm adoption:chart 声明 Namespace/sema-sandbox——6b 先建了它,不打 helm 属主标记
+    # 的话 `helm upgrade --install` 拒绝接管(fresh helm 装直接失败)。render 路径不受影响(仅 helm 时打)。
+    if [ "$ns" = sema-sandbox ] && [ "${STACK_ENGINE:-render}" = "helm" ]; then
+      rrun 0 k3s kubectl label ns "$ns" app.kubernetes.io/managed-by=Helm --overwrite >/dev/null
+      rrun 0 k3s kubectl annotate ns "$ns" meta.helm.sh/release-name=sema-stack meta.helm.sh/release-namespace=sema --overwrite >/dev/null
+    fi
+    IPS_NS="$ns" IPS_REG="${IMAGE_PULL_REGISTRY:-https://index.docker.io/v1/}" \
+    IPS_USER="$IMAGE_PULL_USER" IPS_TOKEN="$IMAGE_PULL_TOKEN" python3 - <<'PY' | rrun 0 k3s kubectl apply -f - >/dev/null
+import base64, json, os
+u, p = os.environ["IPS_USER"], os.environ["IPS_TOKEN"]
+cfg = {"auths": {os.environ["IPS_REG"]: {"username": u, "password": p,
+       "auth": base64.b64encode(f"{u}:{p}".encode()).decode()}}}
+b64 = base64.b64encode(json.dumps(cfg).encode()).decode()
+print(json.dumps({"apiVersion": "v1", "kind": "Secret", "type": "kubernetes.io/dockerconfigjson",
+      "metadata": {"name": "sema-regcred", "namespace": os.environ["IPS_NS"]},
+      "data": {".dockerconfigjson": b64}}))
+PY
+    # default SA 由 ns 控制器异步建——等它出现再 patch(幂等)
+    rsudo 0 <<EOF
+set -e
+for t in \$(seq 1 12); do k3s kubectl get sa default -n $ns >/dev/null 2>&1 && break; sleep 5; done
+k3s kubectl patch sa default -n $ns -p '{"imagePullSecrets":[{"name":"sema-regcred"}]}' >/dev/null
+EOF
+  done
+elif [ -n "${IMAGE_PULL_USER:-}${IMAGE_PULL_TOKEN:-}" ]; then
+  die "IMAGE_PULL_USER/IMAGE_PULL_TOKEN 只给了一个——两个都要(或都不给=公共镜像)"
+fi
+
+# ── 7. sema-stack 装配:helm chart(STACK_ENGINE=helm)或渲染 apply(默认,as-built)──
+# chart/ 与 stack/sema-stack-cluster.yaml 同构(对象名一致,可从存量 adoption);helm 路径
+# 验收等价后再翻默认。
+log "7/8 sema-stack 渲染+apply(engine=${STACK_ENGINE:-render})"
+: "${MODEL_GATEWAY_BASEURL:?模型面必需(--gateway)}"; : "${MODEL_ID:?模型面必需(--model)}"
+ENVF="$WORK/.env"; touch "$ENVF"; chmod 600 "$ENVF"
+setk() { grep -q "^$1=" "$ENVF" || printf '%s=%s\n' "$1" "$2" >> "$ENVF"; }
+setk MINIO_PASSWORD "$(openssl rand -hex 24)"
+setk SERVICE_AUTH_TOKEN "$(openssl rand -hex 24)"
+# openssl 失败会被 $() 吞成空串写进 .env(set -e 不管参数位替换)——断言长度,fail-loud
+for k in MINIO_PASSWORD SERVICE_AUTH_TOKEN; do
+  grep -q "^$k=.\{32,\}$" "$ENVF" || die "$k 生成失败(openssl rand?)——检查 $ENVF"
+done
+. "$ENVF"
+[ -n "${MODEL_API_KEY:-}" ] || die "需要 MODEL_API_KEY(-f 配置或 --model-key)"
+# PG 实例数/反亲和/副本数按机器数缩放(<3 机=dev:preferred,required 会永久 Pending)
+PG_INSTANCES=$(( MACHINE_COUNT >= 3 ? 3 : MACHINE_COUNT ))
+PG_AA=$([ "$MACHINE_COUNT" -ge 3 ] && echo required || echo preferred)
+SERVER_REPLICAS=$(( MACHINE_COUNT >= 3 ? 3 : 2 ))
+if [ "${STACK_ENGINE:-render}" = "helm" ]; then
+  # ── helm 路径:chart tar 经 stdin 上载(不落敏感值;chart 本身无凭据),values 本地生成
+  #    (umask 077)经 stdin 给 helm -f -(凭据不进 argv、不落远端盘——与渲染路径同纪律)──
+  tar -C "$HERE" -czf - chart | mssh 0 'rm -rf /tmp/sema-stack-chart && mkdir -p /tmp/sema-stack-chart && tar xzf - -C /tmp/sema-stack-chart'
+  VALSF="$WORK/sema-stack-values.yaml"
+  ( umask 077
+    V_MINIO_PW="$MINIO_PASSWORD" V_SVC_TOKEN="$SERVICE_AUTH_TOKEN" V_MODEL_KEY="$MODEL_API_KEY" \
+    V_MODEL_GATEWAY="$MODEL_GATEWAY_BASEURL" V_MODEL_ID="$MODEL_ID" \
+    V_SERVER_IMAGE="$SERVER_IMAGE" V_PULL_POLICY="$PULL_POLICY" \
+    V_PKG_SOURCE="$([ "$REGION" = "global" ] && echo global || echo '')" \
+    V_PG_INSTANCES="$PG_INSTANCES" V_PG_AA="$PG_AA" V_REPLICAS="$SERVER_REPLICAS" \
+    python3 - > "$VALSF" <<'PY'
+import json, os
+e = os.environ
+# json 是合法 yaml;json.dumps 对任意凭据字符安全(引号/反斜杠/换行都正确转义)
+print(json.dumps({
+  "pg": {"instances": int(e["V_PG_INSTANCES"]), "antiAffinity": e["V_PG_AA"]},
+  "minio": {"rootPassword": e["V_MINIO_PW"]},
+  "server": {
+    "image": e["V_SERVER_IMAGE"], "pullPolicy": e["V_PULL_POLICY"],
+    "replicas": int(e["V_REPLICAS"]), "serviceAuthToken": e["V_SVC_TOKEN"],
+    "modelGatewayBaseurl": e["V_MODEL_GATEWAY"], "modelId": e["V_MODEL_ID"],
+    "modelApiKey": e["V_MODEL_KEY"], "sandboxPkgSource": e["V_PKG_SOURCE"],
+  },
+  # chart 0.3.0 起 registry.enabled 默认 true(单机档语义)——machines 多机路径的 registry/git
+  # 面还没接线,必须显式关,不然 helm 撞 registry.image required(2026-07-12 自查)
+  "registry": {"enabled": False},
+  "gitea": {"enabled": False},
+}))
+PY
+  )
+  [ -s "$VALSF" ] || die "helm values 生成失败"
+  # 失败也删 values 文件(die 在 rm 之前退出会把含 MODEL_API_KEY 的 0600 文件留在 WORK)
+  rrun 0 helm --kubeconfig /etc/rancher/k3s/k3s.yaml upgrade --install sema-stack \
+    /tmp/sema-stack-chart/chart -n sema --create-namespace -f - < "$VALSF" >/dev/null \
+    || { rm -f "$VALSF"; die "helm upgrade --install sema-stack 失败"; }
+  rm -f "$VALSF"
+  rrun 0 rm -rf /tmp/sema-stack-chart
+else
+RENDERED="$WORK/sema-stack-cluster.rendered.yaml"
+# python 逐字替换而非 sed:①密钥/URL 含 & | \ 时 sed 会注入/静默毁 YAML
+# ②值走 env 不走 argv(/proc/*/cmdline 对同机进程可见)。占位符缺值=fail-loud。
+( umask 077
+  R_MINIO_PW="$MINIO_PASSWORD" R_SVC_TOKEN="$SERVICE_AUTH_TOKEN" R_MODEL_KEY="$MODEL_API_KEY" \
+  R_MODEL_GATEWAY="$MODEL_GATEWAY_BASEURL" R_MODEL_ID="$MODEL_ID" \
+  R_SERVER_IMAGE="$SERVER_IMAGE" R_PULL_POLICY="$PULL_POLICY" \
+  R_PKG_SOURCE="$([ "$REGION" = "global" ] && echo global || echo '')" \
+  R_PG_INSTANCES="$PG_INSTANCES" R_PG_AA="$PG_AA" R_REPLICAS="$SERVER_REPLICAS" \
+  python3 - "$HERE/stack/sema-stack-cluster.yaml" > "$RENDERED" <<'PY'
+import os, sys
+MAP = {"__MINIO_PW__":"R_MINIO_PW", "__SVC_TOKEN__":"R_SVC_TOKEN", "__MODEL_KEY__":"R_MODEL_KEY",
+       "__MODEL_GATEWAY__":"R_MODEL_GATEWAY", "__MODEL_ID__":"R_MODEL_ID",
+       "__SERVER_IMAGE__":"R_SERVER_IMAGE", "__SERVER_PULL_POLICY__":"R_PULL_POLICY",
+       "__PKG_SOURCE__":"R_PKG_SOURCE", "__PG_INSTANCES__":"R_PG_INSTANCES",
+       "__PG_ANTI_AFFINITY__":"R_PG_AA", "__SERVER_REPLICAS__":"R_REPLICAS"}
+t = open(sys.argv[1], encoding="utf-8").read()
+for ph, env in MAP.items():
+    v = os.environ.get(env)
+    if v is None: raise SystemExit(f"渲染缺 {env}({ph})")
+    if any(c in v for c in "\n\"\\"): raise SystemExit(f"{env} 含 YAML 危险字符(换行/引号/反斜杠)")
+    t = t.replace(ph, v)
+import re
+left = sorted(set(re.findall(r"__[A-Z_]+__", t)))
+if left: raise SystemExit(f"未渲染占位符残留: {left}")
+sys.stdout.write(t)
+PY
+)
+[ -s "$RENDERED" ] || die "sema-stack 渲染失败"
+# 凭据清单经 stdin 直接 apply,不落远端盘(落 /tmp 失败时会残留)
+rrun 0 k3s kubectl apply -f - < "$RENDERED" >/dev/null
+fi
+rsudo 0 <<'EOF'
+set -e
+# runner token 跨 ns 复制要赶在 server 起 pod 前(晚了=CreateContainerConfigError 干等——真踩);
+# SA token secret 由控制器异步填充,先等它有值
+TOK=""
+for t in $(seq 1 24); do
+  TOK=$(k3s kubectl get secret sema-runner-token -n sema-sandbox -o jsonpath='{.data.token}' 2>/dev/null || true)
+  [ -n "$TOK" ] && break; sleep 5
+done
+[ -n "$TOK" ] || { echo "sema-runner-token 未被控制器填充" >&2; exit 1; }
+k3s kubectl create secret generic sema-runner-token-copy -n sema --from-literal=token="$(printf '%s' "$TOK" | base64 -d)" \
+  --dry-run=client -o yaml | k3s kubectl apply -f - >/dev/null
+# PG healthy(冷启拉镜像+init 可要几分钟)
+for t in $(seq 1 120); do
+  s=$(k3s kubectl get cluster sema-pg -n sema -o jsonpath='{.status.phase}' 2>/dev/null || true)
+  [ "$s" = "Cluster in healthy state" ] && break; sleep 5
+done
+[ "$s" = "Cluster in healthy state" ] || { k3s kubectl get cluster -n sema; echo "PG 未达 healthy" >&2; exit 1; }
+k3s kubectl rollout status deployment/sema-server -n sema --timeout=300s >/dev/null
+# MinIO 建桶(备份桶+对象桶)
+PW=$(k3s kubectl get secret sema-minio -n sema -o jsonpath='{.data.rootPassword}' | base64 -d)
+k3s kubectl exec deploy/sema-minio -n sema -- sh -c \
+  "mc alias set local http://localhost:9000 sema '$PW' >/dev/null && mc mb --ignore-existing local/sema-pg-backup local/workspaces local/blobs" >/dev/null
+echo STACK_OK
+EOF
+
+# ── 8. smoke(节点内经 NodePort;外网入口=LB 在云商侧配,打印指引)────────────────
+log "8/8 smoke"
+# NodePort 被 nodeport-addresses 钉在节点网段——loopback 不在其内,必须打 node-ip(真踩 curl exit 7)
+SMOKE=$(rsudo 0 <<EOF
+H=\$(curl -m5 -s -o /dev/null -w '%{http_code}' "http://$K3S0_IP:$NODEPORT/health" || echo 000)
+A=\$(curl -m5 -s -o /dev/null -w '%{http_code}' "http://$K3S0_IP:$NODEPORT/v1/runs" || echo 000)
+echo "health=\$H auth=\$A"
+EOF
+)
+case "$SMOKE" in *"health=200 auth=401"*) : ;; *) die "smoke 失败:$SMOKE(期望 health=200 auth=401)" ;; esac
+log "smoke ✓ $SMOKE"
+cat >&2 <<EOF
+
+══════════ cluster-up 完成($MACHINE_COUNT 节点)══════════
+ 入口   NodePort $NODEPORT(限私网段)。公网入口在云商 LB 上配:
+        listener TCP:8090 → pool(每节点 private_ip:$NODEPORT,member 必带 subnet_id——Gcore 真踩)
+ 数据   CNPG sema-pg($PG_INSTANCES 实例,反亲和 $PG_AA)+ MinIO;WAL 归档+每日基备→ s3://sema-pg-backup/
+        (⚠ 若手工毁重建 PG:app secret 会换,必须 kubectl rollout restart deploy/sema-server)
+ 沙箱   kata-qemu runtimeclass(ns sema-sandbox,SA sema-runner)
+ 凭据   $ENVF(0600;SERVICE_AUTH_TOKEN=API 门)
+ 管理   ssh 隧道 6443(kubeconfig 在首节点 /etc/rancher/k3s/k3s.yaml,server 改 127.0.0.1)
+═══════════════════════════════════════════════════════════
+EOF