@keepur/hive 0.8.3 → 0.9.0

package/service/deploy.sh

@@ -127,10 +127,32 @@ notify() {
     > /dev/null
 }
 
+# log_size_before <log_file>
+# Byte size of the log right now, or 0 if it doesn't exist. Captured
+# immediately before `launchctl bootstrap` so health_check can scan only
+# bytes written by the new boot and ignore any stale "Hive is running"
+# from a prior run.
+log_size_before() {
+  local log_file="$1"
+  if [[ -f "$log_file" ]]; then
+    wc -c < "$log_file" | awk '{print $1}'
+  else
+    echo 0
+  fi
+}
+
+# health_check <log_file> [start_offset]
+# KPR-240: anchor the marker scan to the boot we just kicked off.
+# Reads bytes after $start_offset and succeeds iff "Hive is running"
+# appears after "Hive starting up" in that window. Avoids the tail -5
+# race on busy boots (12 agents + scheduler + memory lifecycle can push
+# the marker out of the last 5 lines within 1s) and refuses to match a
+# stale marker from a previous run.
 health_check() {
   local log_file="$1"
+  local start_offset="${2:-0}"
   if $DRY_RUN; then
-    echo "[DRY RUN] health_check: would check $log_file"
+    echo "[DRY RUN] health_check: would check $log_file (offset $start_offset)"
     return 0
   fi
   for attempt in $(seq 1 "$HEALTH_CHECK_RETRIES"); do
@@ -140,7 +162,7 @@ health_check() {
     fi
     for _ in $(seq 1 "$HEALTH_CHECK_WINDOW"); do
       sleep 1
-      if tail -5 "$log_file" 2>/dev/null | grep -q '"Hive is running"'; then
+      if _scan_new_boot "$log_file" "$start_offset"; then
         return 0
       fi
     done
@@ -149,6 +171,20 @@ health_check() {
   return 1
 }
 
+# _scan_new_boot <log_file> <start_offset>
+# True iff the bytes after $start_offset contain "Hive is running"
+# preceded by "Hive starting up". tail -c +N is 1-indexed: start at byte N.
+_scan_new_boot() {
+  local log_file="$1"
+  local start_offset="$2"
+  [[ -f "$log_file" ]] || return 1
+  tail -c "+$((start_offset + 1))" "$log_file" 2>/dev/null | awk '
+    /"Hive starting up"/ { started = 1; next }
+    started && /"Hive is running"/ { found = 1; exit }
+    END { if (found) exit 0; else exit 1 }
+  '
+}
+
 kill_ports() {
   local ports_str="$1"
   if $DRY_RUN; then
@@ -373,8 +409,10 @@ if $ROLLBACK; then
     notify "Rollback FAILED for \`$id\`: no previous engine (.hive.prev missing)."
     exit 1
   fi
+  health_log="$instance_root/$logs_dir/hive.log"
+  health_offset=$(log_size_before "$health_log")
   run_cmd launchctl bootstrap "gui/$(id -u)" "$plist_path"
-  if health_check "$instance_root/$logs_dir/hive.log"; then
+  if health_check "$health_log" "$health_offset"; then
     rollback_version=$(jq -r .version < "$instance_root/.hive/package.json" 2>/dev/null || echo "unknown")
     notify "Rollback succeeded for \`$id\` → \`$rollback_version\`."
     echo "Rollback complete."
@@ -529,11 +567,13 @@ for inst in "${INSTANCES[@]}"; do
   echo "  Swapping engine..."
   swap_engine "$instance_root"
 
+  health_log="$instance_root/$logs_dir/hive.log"
+  health_offset=$(log_size_before "$health_log")
   echo "  Restarting $label..."
   run_cmd launchctl bootstrap "gui/$(id -u)" "$plist_path"
 
   echo "  Checking health..."
-  if ! health_check "$instance_root/$logs_dir/hive.log"; then
+  if ! health_check "$health_log" "$health_offset"; then
     echo "  Health check FAILED for $id — rolling back"
     # New engine bound the port and failed health check — bootout it before swap.
     run_cmd launchctl bootout "gui/$(id -u)/$label" 2>/dev/null || true

package/service/deploy.test.sh

@@ -65,11 +65,15 @@ echo '{"name":"@keepur/hive","version":"0.2.0-dev"}' > "$BUILD_DIR/package.json"
 # function bodies via sed and source them in isolation. The inner `/!p` drops
 # the closing delimiter line so we don't capture the `if $ROLLBACK; then` line
 # (which would trip set -u on undefined ROLLBACK when sourced).
-sed -n '/^# --- Engine fetch\/swap\/rollback/,/^# --- Short-circuit:/{/^# --- Short-circuit:/!p;}' \
+sed -n '/^# --- Helpers ---/,/^# --- Short-circuit:/{/^# --- Short-circuit:/!p;}' \
   "$SCRIPT_DIR/deploy.sh" > "$TESTROOT/helpers.sh"
 # Helper bodies reference $DRY_RUN (added so --dry-run skips the destructive
 # ops); set it false here so the helpers actually execute under set -u.
 DRY_RUN=false
+# Keep health_check's retry/window/wait small so the new tests don't burn 90s.
+HEALTH_CHECK_RETRIES=1
+HEALTH_CHECK_WINDOW=2
+HEALTH_CHECK_WAIT_BETWEEN=0
 # shellcheck source=/dev/null
 source "$TESTROOT/helpers.sh"
 
@@ -205,4 +209,88 @@ if install_engine_deps "$DEPLOY_DIR" >/dev/null 2>&1; then
   exit 1
 fi
 
+# --- Health-check tests (KPR-240) ---
+# Anchor the marker scan to a captured byte offset so a busy boot logging
+# many lines after "Hive is running" can't be falsely flagged failed, and
+# so a stale marker from a previous run can't be falsely flagged healthy.
+LOG_DIR=$(mktemp -d -t hive-health-test.XXXXXX)
+trap 'rm -rf "$TESTROOT" "$LOG_DIR"' EXIT
+
+# --- Test 11: busy boot logs many lines after marker, still passes ---
+echo "test 11: busy boot passes despite marker scrolling out of tail -5"
+LOG_FILE="$LOG_DIR/hive-busy.log"
+: > "$LOG_FILE"
+OFFSET=$(log_size_before "$LOG_FILE")
+# Simulate the new boot: starting marker, running marker, then a flood of
+# log lines (12 agents + scheduler + memory lifecycle) that pushes the
+# marker far out of `tail -5`.
+{
+  echo '{"ts":"2026-05-25T00:00:00Z","level":"info","component":"hive","msg":"Hive starting up","instance":"test"}'
+  echo '{"ts":"2026-05-25T00:00:00Z","level":"info","component":"hive","msg":"Hive is running"}'
+  for i in $(seq 1 50); do
+    echo '{"ts":"2026-05-25T00:00:00Z","level":"info","component":"agent","msg":"agent-'"$i"' ready"}'
+  done
+} >> "$LOG_FILE"
+if ! health_check "$LOG_FILE" "$OFFSET" >/dev/null; then
+  echo "FAIL: healthy boot with flood after marker should pass"
+  exit 1
+fi
+# Sanity: with the legacy tail -5 strategy this would have failed.
+if tail -5 "$LOG_FILE" | grep -q '"Hive is running"'; then
+  echo "FAIL: tail -5 unexpectedly still contains the marker — test setup wrong"
+  exit 1
+fi
+
+# --- Test 12: genuine boot failure (never reaches marker) fails ---
+echo "test 12: genuine boot failure fails health_check"
+LOG_FILE="$LOG_DIR/hive-fail.log"
+: > "$LOG_FILE"
+OFFSET=$(log_size_before "$LOG_FILE")
+{
+  echo '{"ts":"2026-05-25T00:00:00Z","level":"info","component":"hive","msg":"Hive starting up","instance":"test"}'
+  echo '{"ts":"2026-05-25T00:00:00Z","level":"error","component":"hive","msg":"Mongo unreachable, exiting"}'
+} >> "$LOG_FILE"
+if health_check "$LOG_FILE" "$OFFSET" >/dev/null 2>&1; then
+  echo "FAIL: boot that never reached 'Hive is running' should fail"
+  exit 1
+fi
+
+# --- Test 13: stale "Hive is running" before offset is ignored ---
+echo "test 13: stale marker from previous boot is not matched"
+LOG_FILE="$LOG_DIR/hive-stale.log"
+: > "$LOG_FILE"
+# Previous boot: full happy-path markers land in the file.
+{
+  echo '{"ts":"2026-05-24T00:00:00Z","level":"info","component":"hive","msg":"Hive starting up","instance":"test"}'
+  echo '{"ts":"2026-05-24T00:00:00Z","level":"info","component":"hive","msg":"Hive is running"}'
+} >> "$LOG_FILE"
+# Capture offset AFTER the prior boot's markers — mimics the deploy.sh
+# call site that snapshots wc -c right before launchctl bootstrap.
+OFFSET=$(log_size_before "$LOG_FILE")
+# New boot crashes before "Hive is running". The stale marker is still
+# physically in the log, but past start_offset there is no marker.
+{
+  echo '{"ts":"2026-05-25T00:00:00Z","level":"info","component":"hive","msg":"Hive starting up","instance":"test"}'
+  echo '{"ts":"2026-05-25T00:00:00Z","level":"error","component":"hive","msg":"crashed"}'
+} >> "$LOG_FILE"
+if health_check "$LOG_FILE" "$OFFSET" >/dev/null 2>&1; then
+  echo "FAIL: stale marker from previous boot should not satisfy health_check"
+  exit 1
+fi
+
+# --- Test 14: log file absent at start (fresh install), then created ---
+echo "test 14: missing log at offset capture, populated by boot, passes"
+LOG_FILE="$LOG_DIR/hive-fresh.log"
+# No file yet — offset is 0 by contract.
+OFFSET=$(log_size_before "$LOG_FILE")
+[[ "$OFFSET" == "0" ]] || { echo "FAIL: log_size_before should return 0 for missing file (got '$OFFSET')"; exit 1; }
+{
+  echo '{"ts":"2026-05-25T00:00:00Z","level":"info","component":"hive","msg":"Hive starting up","instance":"test"}'
+  echo '{"ts":"2026-05-25T00:00:00Z","level":"info","component":"hive","msg":"Hive is running"}'
+} > "$LOG_FILE"
+if ! health_check "$LOG_FILE" "$OFFSET" >/dev/null; then
+  echo "FAIL: fresh-install boot should pass health_check"
+  exit 1
+fi
+
 echo "all tests passed."