grix-connector 4.0.2 → 4.2.0

package/scripts/upgrade-guardian.sh

@@ -1,14 +1,15 @@
 #!/bin/bash
 # upgrade-guardian.sh — post-upgrade health monitor
 # Started by daemon after npm install, runs as detached background process.
-# Monitors healthz for 90s, rolls back on crash_count >= 3.
+# Monitors healthz for 90s and rolls back on failure. The decision is
+# single-run: nothing relaunches this guardian after it exits, so any
+# crash-count threshold above one run could never be reached.
 # This file is NOT inside the npm package — installed once to ~/.grix/bin/.
 
 GRIX_HOME="${GRIX_CONNECTOR_HOME:-$HOME/.grix}"
 PENDING_FILE="$GRIX_HOME/data/upgrade-pending.json"
 LOCK_FILE="$GRIX_HOME/upgrade-guardian.lock"
 LOG_FILE="$GRIX_HOME/log/upgrade.log"
-MAX_CRASHES=3
 HEALTH_TIMEOUT=90
 HEALTH_PORT_FILE="$GRIX_HOME/data/health-port"
 HEALTH_PORT=$(cat "$HEALTH_PORT_FILE" 2>/dev/null || echo "19579")
@@ -38,22 +39,43 @@ if [ ! -f "$PENDING_FILE" ]; then
     exit 0
 fi
 
-# Extract fields from JSON (portable: no jq dependency)
-FROM_VERSION=$(cat "$PENDING_FILE" | grep -o '"from_version":"[^"]*"' | cut -d'"' -f4)
-TARGET_VERSION=$(cat "$PENDING_FILE" | grep -o '"target_version":"[^"]*"' | cut -d'"' -f4)
-UPGRADED_AT=$(cat "$PENDING_FILE" | grep -o '"upgraded_at":"[^"]*"' | cut -d'"' -f4)
-CRASH_COUNT=$(cat "$PENDING_FILE" | grep -o '"crash_count":[0-9]*' | cut -d: -f2)
-CRASH_COUNT=${CRASH_COUNT:-0}
-PACKAGE_DIR=$(cat "$PENDING_FILE" | grep -o '"package_dir":"[^"]*"' | cut -d'"' -f4)
-BACKUP_DIR=$(cat "$PENDING_FILE" | grep -o '"backup_dir":"[^"]*"' | cut -d'"' -f4)
+# 用 node 解析 pending：grep/cut 拼 JSON 对转义和 schema 演进都脆弱，而 guardian
+# 的激活与回滚本来就依赖 node，可用性等价。输出为单引号安全的 shell 赋值。
+PENDING_VARS=$(node -e '
+const f = JSON.parse(require("fs").readFileSync(process.argv[1], "utf8"));
+const q = (v) => "\x27" + String(v ?? "").replace(/\x27/g, "\x27\\\x27\x27") + "\x27";
+process.stdout.write([
+  "FROM_VERSION=" + q(f.from_version),
+  "TARGET_VERSION=" + q(f.target_version),
+  "CRASH_COUNT=" + q(f.crash_count ?? 0),
+  "PACKAGE_DIR=" + q(f.package_dir),
+  "BACKUP_DIR=" + q(f.backup_dir),
+].join("\n"));
+' "$PENDING_FILE" 2>> "$LOG_FILE") || PENDING_VARS=""
+if [ -z "$PENDING_VARS" ]; then
+    log "pending file corrupt or unreadable, removing"
+    rm -f "$PENDING_FILE"
+    exit 0
+fi
+eval "$PENDING_VARS"
 
 # Validate pending file
-if [ -z "$FROM_VERSION" ]; then
-    log "pending file corrupt, missing from_version, removing"
+if [ -z "$FROM_VERSION" ] || [ -z "$TARGET_VERSION" ]; then
+    log "pending file corrupt, missing from_version/target_version, removing"
     rm -f "$PENDING_FILE"
     exit 0
 fi
 
+# healthz 必须由目标版本的进程应答才算健康。只看 HTTP 200 会把仍在跑的旧进程
+# （或激活间隙被 launchd 拉起的旧 runtime）误判为升级成功。
+target_healthy() {
+    RESP=$(curl -sf --max-time 5 "$HEALTH_URL" 2>/dev/null) || return 1
+    case "$RESP" in
+        *"\"version\":\"$TARGET_VERSION\""*) return 0 ;;
+        *) return 1 ;;
+    esac
+}
+
 log "started: from=$FROM_VERSION target=$TARGET_VERSION crash_count=$CRASH_COUNT"
 
 # Wait for old daemon to exit (SIGTERM + shutdown + restart)
@@ -74,16 +96,16 @@ else
     log "forward activation failed for target $TARGET_VERSION; treating as startup failure"
 fi
 
-# Monitor loop: wait for healthz to return 200 (only after successful activation)
+# Monitor loop: wait for healthz to report the target version (only after successful activation)
 if [ $ACTIVATED -eq 1 ]; then
     ELAPSED=0
     while [ $ELAPSED -lt $HEALTH_TIMEOUT ]; do
-        if curl -sf "$HEALTH_URL" > /dev/null 2>&1; then
+        if target_healthy; then
             # healthz 通过=新版健康。这里不删 pending：删了会抢在新进程的
             # handlePendingOnStartup 之前,使其读不到 pending、success 回执发不出去
             # (后端只见 installed 不见 success)。交由新进程上报 success 后自行删 pending,
             # guardian 只负责崩溃回滚。
-            log "healthz passed, upgrade healthy; leaving pending for in-process success report"
+            log "healthz passed for $TARGET_VERSION, upgrade healthy; leaving pending for in-process success report"
             exit 0
         fi
         sleep 3
@@ -92,75 +114,78 @@ if [ $ACTIVATED -eq 1 ]; then
 
     # Extra grace period for slow systems
     sleep 15
-    if curl -sf "$HEALTH_URL" > /dev/null 2>&1; then
+    if target_healthy; then
         # 同上:不删 pending,交由进程内 handlePendingOnStartup 上报 success 并删除。
-        log "healthz passed (delayed), upgrade healthy; leaving pending for in-process success report"
+        log "healthz passed for $TARGET_VERSION (delayed), upgrade healthy; leaving pending for in-process success report"
         exit 0
     fi
 fi
 
-# Health check failed (or forward activation failed)
-CRASH_COUNT=$((CRASH_COUNT + 1))
-log "healthz timeout or forward activation failed, crash_count=$CRASH_COUNT"
-
-if [ $CRASH_COUNT -ge $MAX_CRASHES ]; then
-    # 新版升级器会把完整旧包保留在同文件系统的固定备份目录。优先原子恢复，
-    # 不依赖网络/npm，也不会再次制造全局 bin 缺失窗口。
-    case "$PACKAGE_DIR" in
-        */node_modules/*)
-            if [ -n "$BACKUP_DIR" ] && [ "$BACKUP_DIR" = "${PACKAGE_DIR}.grix-upgrade-backup" ] && [ -d "$BACKUP_DIR" ]; then
-                FAILED_DIR="${PACKAGE_DIR}.grix-upgrade-failed-$$"
-                log "atomic rollback: restoring $BACKUP_DIR"
+# Health check failed (or forward activation failed). This guardian runs exactly
+# once per upgrade — no supervisor relaunches it — so the verdict is made here:
+# roll back now instead of counting crashes toward a threshold that can never
+# be reached again.
+# 回滚前用 node 原位递增并持久化 crash_count：保留 pending 里的其余字段
+# （transaction_id、路径等），恢复后的旧运行时上报 rolled_back 时要带真实计数。
+CRASH_COUNT=$(node -e '
+const fs = require("fs");
+const file = process.argv[1];
+const f = JSON.parse(fs.readFileSync(file, "utf8"));
+f.crash_count = (Number(f.crash_count) || 0) + 1;
+const tmp = file + ".tmp";
+fs.writeFileSync(tmp, JSON.stringify(f));
+fs.renameSync(tmp, file);
+process.stdout.write(String(f.crash_count));
+' "$PENDING_FILE" 2>> "$LOG_FILE") || CRASH_COUNT=$((CRASH_COUNT + 1))
+log "healthz timeout or forward activation failed, crash_count=$CRASH_COUNT; rolling back"
+
+# 新版升级器会把完整旧包保留在同文件系统的固定备份目录。优先原子恢复，
+# 不依赖网络/npm，也不会再次制造全局 bin 缺失窗口。
+case "$PACKAGE_DIR" in
+    */node_modules/*)
+        if [ -n "$BACKUP_DIR" ] && [ "$BACKUP_DIR" = "${PACKAGE_DIR}.grix-upgrade-backup" ] && [ -d "$BACKUP_DIR" ]; then
+            FAILED_DIR="${PACKAGE_DIR}.grix-upgrade-failed-$$"
+            log "atomic rollback: restoring $BACKUP_DIR"
+            rm -rf "$FAILED_DIR"
+            if { [ ! -e "$PACKAGE_DIR" ] || mv "$PACKAGE_DIR" "$FAILED_DIR"; } \
+               && mv "$BACKUP_DIR" "$PACKAGE_DIR"; then
                 rm -rf "$FAILED_DIR"
-                if { [ ! -e "$PACKAGE_DIR" ] || mv "$PACKAGE_DIR" "$FAILED_DIR"; } \
-                   && mv "$BACKUP_DIR" "$PACKAGE_DIR"; then
-                    rm -rf "$FAILED_DIR"
-                    # 原子恢复只换回全局包；macOS 服务跑的是受管 runtime 副本，必须用
-                    # 旧包 CLI start 刷新副本并触发重启，激活成功才删 pending。
-                    NPM_ROOT="$(npm root -g 2>> "$LOG_FILE")"
-                    ROLLBACK_CLI="$NPM_ROOT/$NPM_PACKAGE/dist/grix.js"
-                    if [ -n "$NPM_ROOT" ] && [ -f "$ROLLBACK_CLI" ] \
-                       && node "$ROLLBACK_CLI" start >> "$LOG_FILE" 2>&1; then
-                        log "atomic rollback succeeded and managed runtime activated; leaving pending for rollback report"
-                        exit 0
-                    fi
-                    # 激活失败不退出：落到下面的 npm 回滚路径再试一次（npm install 旧版幂等）。
-                    log "atomic rollback package restored but managed runtime activation failed; falling through to npm rollback"
-                else
-                    log "atomic rollback failed; restoring current package before npm fallback"
-                    if [ ! -e "$PACKAGE_DIR" ] && [ -e "$FAILED_DIR" ]; then
-                        mv "$FAILED_DIR" "$PACKAGE_DIR" 2>/dev/null || true
-                    fi
+                # 原子恢复只换回全局包；macOS 服务跑的是受管 runtime 副本，必须用
+                # 旧包 CLI start 刷新副本并触发重启，激活成功才删 pending。
+                NPM_ROOT="$(npm root -g 2>> "$LOG_FILE")"
+                ROLLBACK_CLI="$NPM_ROOT/$NPM_PACKAGE/dist/grix.js"
+                if [ -n "$NPM_ROOT" ] && [ -f "$ROLLBACK_CLI" ] \
+                   && node "$ROLLBACK_CLI" start >> "$LOG_FILE" 2>&1; then
+                    log "atomic rollback succeeded and managed runtime activated; leaving pending for rollback report"
+                    exit 0
+                fi
+                # 激活失败不退出：落到下面的 npm 回滚路径再试一次（npm install 旧版幂等）。
+                log "atomic rollback package restored but managed runtime activation failed; falling through to npm rollback"
+            else
+                log "atomic rollback failed; restoring current package before npm fallback"
+                if [ ! -e "$PACKAGE_DIR" ] && [ -e "$FAILED_DIR" ]; then
+                    mv "$FAILED_DIR" "$PACKAGE_DIR" 2>/dev/null || true
                 fi
             fi
-            ;;
-    esac
-
-    log "rollback: installing ${NPM_PACKAGE}@${FROM_VERSION}"
-    # P5: 先用默认源,失败再显式回退官方 npm(默认源可能是淘宝镜像,旧版若没同步同样会失败)。
-    if npm install -g "${NPM_PACKAGE}@${FROM_VERSION}" --prefer-online --no-audit --no-fund >> "$LOG_FILE" 2>&1 \
-       || { log "rollback via default registry failed, retrying official npm"; \
-            npm install -g "${NPM_PACKAGE}@${FROM_VERSION}" --registry https://registry.npmjs.org --prefer-online --no-audit --no-fund >> "$LOG_FILE" 2>&1; }; then
-        log "rollback package installed; activating managed runtime"
-        NPM_ROOT="$(npm root -g 2>> "$LOG_FILE")"
-        ROLLBACK_CLI="$NPM_ROOT/$NPM_PACKAGE/dist/grix.js"
-        if [ -n "$NPM_ROOT" ] && [ -f "$ROLLBACK_CLI" ] \
-           && node "$ROLLBACK_CLI" start >> "$LOG_FILE" 2>&1; then
-            log "rollback succeeded and managed runtime activated; leaving pending for rollback report"
-        else
-            log "ROLLBACK ACTIVATION FAILED — package restored but managed runtime was not switched"
         fi
+        ;;
+esac
+
+log "rollback: installing ${NPM_PACKAGE}@${FROM_VERSION}"
+# P5: 先用默认源,失败再显式回退官方 npm(默认源可能是淘宝镜像,旧版若没同步同样会失败)。
+if npm install -g "${NPM_PACKAGE}@${FROM_VERSION}" --prefer-online --no-audit --no-fund >> "$LOG_FILE" 2>&1 \
+   || { log "rollback via default registry failed, retrying official npm"; \
+        npm install -g "${NPM_PACKAGE}@${FROM_VERSION}" --registry https://registry.npmjs.org --prefer-online --no-audit --no-fund >> "$LOG_FILE" 2>&1; }; then
+    log "rollback package installed; activating managed runtime"
+    NPM_ROOT="$(npm root -g 2>> "$LOG_FILE")"
+    ROLLBACK_CLI="$NPM_ROOT/$NPM_PACKAGE/dist/grix.js"
+    if [ -n "$NPM_ROOT" ] && [ -f "$ROLLBACK_CLI" ] \
+       && node "$ROLLBACK_CLI" start >> "$LOG_FILE" 2>&1; then
+        log "rollback succeeded and managed runtime activated; leaving pending for rollback report"
     else
-        log "ROLLBACK FAILED — manual intervention required"
+        log "ROLLBACK ACTIVATION FAILED — package restored but managed runtime was not switched"
     fi
-    exit 0
-fi
-
-# Under threshold — update crash_count and exit, let process manager restart
-PATH_FIELDS=""
-if [ -n "$PACKAGE_DIR" ] && [ -n "$BACKUP_DIR" ]; then
-    PATH_FIELDS=",\"package_dir\":\"$PACKAGE_DIR\",\"backup_dir\":\"$BACKUP_DIR\""
+else
+    log "ROLLBACK FAILED — manual intervention required"
 fi
-echo "{\"from_version\":\"$FROM_VERSION\",\"target_version\":\"$TARGET_VERSION\",\"upgraded_at\":\"$UPGRADED_AT\",\"crash_count\":$CRASH_COUNT$PATH_FIELDS}" > "$PENDING_FILE.tmp"
-mv "$PENDING_FILE.tmp" "$PENDING_FILE"
-log "updated crash_count to $CRASH_COUNT, exiting"
+exit 0