@geekbeer/minion 3.42.2 → 3.43.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@geekbeer/minion",
-  "version": "3.42.2",
+  "version": "3.43.0",
   "description": "AI Agent runtime for Minion - manages status and skill deployment on VPS",
   "main": "linux/server.js",
   "bin": {

package/win/lib/process-manager.js

@@ -80,16 +80,29 @@ function buildGracefulStopBlock(agentPort, apiToken) {
 }
 
 /**
- * Generate a temporary PowerShell script for updating the agent:
- * 1. Graceful shutdown via HTTP API (offline heartbeat)
- * 2. Stop service via NSSM
- * 3. Run npm install -g
- * 4. Start service via NSSM
- * 5. Remove the temporary updater service (self-cleanup)
+ * Generate a temporary PowerShell script for updating the agent.
+ *
+ * Flow:
+ * 1. End MinionWSL scheduled task (holds Fastify/ajv file handles via
+ *    wsl-session-server.js in user session — previously caused EBUSY /
+ *    partial installs that left node_modules/ajv/dist/refs/data.json
+ *    missing and bricked the next startup).
+ * 2. Graceful shutdown via HTTP API (offline heartbeat).
+ * 3. Stop minion-agent via NSSM and WAIT until service is actually
+ *    Stopped (poll sc.exe, not a fixed Start-Sleep).
+ * 4. Kill stray node.exe processes that still reference the minion
+ *    package dir (workflow/routine runners, claude-code children).
+ * 5. Run npm install -g, then verify integrity by actually requiring
+ *    fastify + the package.json from the installed location. If the
+ *    install silently produced a broken tree, we detect it here
+ *    (instead of bouncing an unstartable service).
+ * 6. On persistent integrity failure, wipe the package dir to force a
+ *    clean reinstall on the next attempt.
+ * 7. Start minion-agent. Re-run the MinionWSL task if it existed.
+ * 8. Remove the temporary updater service (self-cleanup in finally).
  *
- * This script is registered as a temporary NSSM service ("minion-update")
- * so it runs independently of the minion-agent service and survives
- * the agent's stop/restart cycle.
+ * The script is registered as a temporary NSSM service ("minion-update")
+ * so it runs independently of the minion-agent service.
  *
  * @param {string} npmInstallCmd - The npm install command to run
  * @param {string} nssmPath - Absolute path to nssm.exe
@@ -111,6 +124,12 @@ function buildUpdateScript(npmInstallCmd, nssmPath, scriptName = 'update-agent',
     ? `${npmInstallCmd} --prefix "${npmPrefix}"`
     : npmInstallCmd
 
+  // Absolute path to the installed package dir (used for integrity check
+  // and, if integrity keeps failing, for a forced clean reinstall).
+  const pkgDir = npmPrefix
+    ? path.join(npmPrefix, 'node_modules', '@geekbeer', 'minion')
+    : ''
+
   const gracefulStop = buildGracefulStopBlock(agentPort, apiToken)
 
   const ps1 = [
@@ -121,38 +140,153 @@ function buildUpdateScript(npmInstallCmd, nssmPath, scriptName = 'update-agent',
     `# Use nssm.exe from data dir (copied there to avoid EBUSY on package files)`,
     `$nssm = '${path.join(dataDir, 'nssm.exe')}'`,
     `$logFile = '${logPath}'`,
+    `$pkgDir = '${pkgDir}'`,
     `function Log($msg) { "$(Get-Date -Format o) $msg" | Out-File -Append $logFile }`,
+    ``,
+    `# Poll until the service reaches Stopped (or timeout).`,
+    `function Wait-ServiceStopped($name, $timeoutSec) {`,
+    `  $deadline = (Get-Date).AddSeconds($timeoutSec)`,
+    `  while ((Get-Date) -lt $deadline) {`,
+    `    $svc = Get-Service -Name $name -ErrorAction SilentlyContinue`,
+    `    if (-not $svc -or $svc.Status -eq 'Stopped') { return $true }`,
+    `    Start-Sleep -Milliseconds 500`,
+    `  }`,
+    `  return $false`,
+    `}`,
+    ``,
+    `# Kill any node.exe whose command line references the minion package`,
+    `# (stale workflow/routine runners, wsl-session-entry children, etc.).`,
+    `# These hold file handles on node_modules and break npm install on Windows.`,
+    `function Kill-PackageProcesses {`,
+    `  try {`,
+    `    Get-CimInstance Win32_Process -Filter "Name='node.exe'" -ErrorAction SilentlyContinue |`,
+    `      Where-Object { $_.CommandLine -and ($_.CommandLine -match '@geekbeer.minion' -or $_.CommandLine -match 'wsl-session-entry') } |`,
+    `      ForEach-Object {`,
+    `        Log "Killing stray node.exe PID $($_.ProcessId): $($_.CommandLine)"`,
+    `        Stop-Process -Id $_.ProcessId -Force -ErrorAction SilentlyContinue`,
+    `      }`,
+    `  } catch { Log "Kill-PackageProcesses error: $_" }`,
+    `}`,
+    ``,
+    `# Load fastify + @geekbeer/minion/package.json from the installed tree.`,
+    `# Catches the "npm reports success but files are missing" failure mode`,
+    `# (the ajv/data.json crash we have been chasing).`,
+    `function Test-PackageIntegrity {`,
+    `  if (-not $pkgDir -or -not (Test-Path $pkgDir)) {`,
+    `    Log "Integrity check: pkgDir missing ($pkgDir)"`,
+    `    return $false`,
+    `  }`,
+    `  try {`,
+    `    Push-Location $pkgDir`,
+    `    $out = & node -e "require('fastify');require('./package.json');console.log('OK')" 2>&1`,
+    `    $code = $LASTEXITCODE`,
+    `    Pop-Location`,
+    `    if ($code -eq 0 -and "$out" -match 'OK') {`,
+    `      Log "Integrity check passed"`,
+    `      return $true`,
+    `    }`,
+    `    Log "Integrity check failed (exit $code): $out"`,
+    `    return $false`,
+    `  } catch {`,
+    `    try { Pop-Location } catch {}`,
+    `    Log "Integrity check error: $_"`,
+    `    return $false`,
+    `  }`,
+    `}`,
+    ``,
     `Log 'Update started'`,
+    `$wslTaskExists = $false`,
     `try {`,
+    `  # 1. Stop MinionWSL scheduled task so wsl-session-server.js releases`,
+    `  #    Fastify/ajv file handles before npm install.`,
+    `  try {`,
+    `    $null = Get-ScheduledTask -TaskName 'MinionWSL' -ErrorAction Stop`,
+    `    $wslTaskExists = $true`,
+    `    Log 'Ending MinionWSL scheduled task...'`,
+    `    schtasks /End /TN "MinionWSL" 2>&1 | ForEach-Object { Log "schtasks: $_" }`,
+    `  } catch {`,
+    `    Log 'MinionWSL task not registered (skipping)'`,
+    `  }`,
+    ``,
+    `  # 2. Ask the agent to flush an offline heartbeat before we yank it.`,
     `  Log 'Requesting graceful shutdown...'`,
     gracefulStop,
-    `  Log 'Stopping service via NSSM...'`,
-    `  & $nssm stop minion-agent`,
-    `  Start-Sleep -Seconds 3`,
-    `  # Retry npm install up to 5 times (Windows may hold file locks briefly after service stop)`,
+    ``,
+    `  # 3. Stop the agent service and WAIT for it to actually be Stopped.`,
+    `  Log 'Stopping minion-agent service via NSSM...'`,
+    `  & $nssm stop minion-agent 2>&1 | ForEach-Object { Log "nssm: $_" }`,
+    `  if (-not (Wait-ServiceStopped 'minion-agent' 30)) {`,
+    `    Log 'WARNING: minion-agent did not reach Stopped within 30s'`,
+    `  } else {`,
+    `    Log 'minion-agent stopped'`,
+    `  }`,
+    `  # Give Windows a moment to release kernel file handles after exit.`,
+    `  Start-Sleep -Seconds 2`,
+    ``,
+    `  # 4. Kill any straggler node.exe that still hold package files.`,
+    `  Kill-PackageProcesses`,
+    `  Start-Sleep -Seconds 1`,
+    ``,
+    `  # 5. Retry npm install + integrity check up to 5 times.`,
     `  Log 'Installing package...'`,
     `  $installed = $false`,
     `  for ($attempt = 1; $attempt -le 5; $attempt++) {`,
+    `    Log "npm install attempt $attempt..."`,
     `    $out = & cmd /c "${fullNpmCmd} 2>&1"`,
-    `    if ($LASTEXITCODE -eq 0) {`,
-    `      Log "npm output: $out"`,
-    `      $installed = $true`,
-    `      break`,
+    `    $npmExit = $LASTEXITCODE`,
+    `    if ($npmExit -ne 0) {`,
+    `      Log "npm install attempt $attempt failed (exit $npmExit): $out"`,
+    `    } else {`,
+    `      Log "npm install attempt $attempt exit 0"`,
+    `      if (Test-PackageIntegrity) {`,
+    `        $installed = $true`,
+    `        break`,
+    `      }`,
     `    }`,
-    `    Log "npm install attempt $attempt failed (exit $LASTEXITCODE): $out"`,
     `    if ($attempt -lt 5) {`,
+    `      # Before retrying, re-kill stragglers and (on later attempts) wipe`,
+    `      # the package dir so npm is forced to lay it down from scratch.`,
+    `      Kill-PackageProcesses`,
+    `      if ($attempt -ge 2 -and $pkgDir -and (Test-Path $pkgDir)) {`,
+    `        Log "Wiping package dir for clean reinstall: $pkgDir"`,
+    `        try { Remove-Item -Recurse -Force $pkgDir -ErrorAction Stop }`,
+    `        catch { Log "Remove-Item failed: $_" }`,
+    `      }`,
+    `      if ($attempt -ge 3) {`,
+    `        Log 'Running npm cache verify...'`,
+    `        & cmd /c 'npm cache verify 2>&1' | ForEach-Object { Log "npm cache: $_" }`,
+    `      }`,
     `      Log "Retrying in 10 seconds..."`,
     `      Start-Sleep -Seconds 10`,
     `    }`,
     `  }`,
-    `  if (-not $installed) { throw "npm install failed after 5 attempts" }`,
-    `  Log 'Starting service...'`,
-    `  & $nssm start minion-agent`,
+    `  if (-not $installed) { throw "npm install / integrity check failed after 5 attempts" }`,
+    ``,
+    `  # 6. Start the agent service.`,
+    `  Log 'Starting minion-agent service...'`,
+    `  & $nssm start minion-agent 2>&1 | ForEach-Object { Log "nssm: $_" }`,
+    ``,
+    `  # 7. Re-run MinionWSL if it was registered.`,
+    `  if ($wslTaskExists) {`,
+    `    Log 'Running MinionWSL scheduled task...'`,
+    `    schtasks /Run /TN "MinionWSL" 2>&1 | ForEach-Object { Log "schtasks: $_" }`,
+    `  }`,
+    ``,
     `  Log 'Update completed successfully'`,
     `} catch {`,
     `  Log "Update failed: $_"`,
-    `  Log 'Attempting to start service anyway...'`,
-    `  & $nssm start minion-agent`,
+    `  # Do NOT blindly start a possibly-broken agent. If the package is`,
+    `  # intact (verify passes), start it — otherwise leave it down so the`,
+    `  # failure surfaces via missed heartbeats instead of a crash loop.`,
+    `  if (Test-PackageIntegrity) {`,
+    `    Log 'Package looks intact — starting minion-agent anyway.'`,
+    `    & $nssm start minion-agent 2>&1 | ForEach-Object { Log "nssm: $_" }`,
+    `    if ($wslTaskExists) {`,
+    `      try { schtasks /Run /TN "MinionWSL" 2>&1 | ForEach-Object { Log "schtasks: $_" } } catch {}`,
+    `    }`,
+    `  } else {`,
+    `    Log 'Package integrity check failed — NOT starting minion-agent. Manual recovery required.'`,
+    `  }`,
     `} finally {`,
     `  Log 'Cleaning up updater service...'`,
     `  & $nssm stop minion-update confirm 2>$null`,

package/win/minion-cli.ps1

@@ -7,7 +7,7 @@
 #   minion-cli-win setup                    # Install software & register services (admin required)
 #   minion-cli-win configure --hq-url https://... --minion-id <UUID> --api-token <TOKEN>
 #   minion-cli-win uninstall [--keep-data]  # Remove agent (admin required)
-#   minion-cli-win start | stop | restart | status | health | daemons | diagnose | version | help
+#   minion-cli-win start | stop [--force] | restart | status | health | daemons | diagnose | version | help
 
 # Parse arguments manually to avoid issues with npm wrapper passing $args as array
 $Command = 'help'
@@ -16,6 +16,7 @@ $MinionId = ''
 $ApiToken = ''
 $SetupTunnel = $false
 $KeepData = $false
+$Force = $false
 
 $i = 0
 while ($i -lt $args.Count) {
@@ -28,6 +29,7 @@ while ($i -lt $args.Count) {
         '^--api-token$' { $i++; if ($i -lt $args.Count) { $ApiToken = [string]$args[$i] } }
         '^--setup-tunnel$' { $SetupTunnel = $true }
         '^--keep-data$' { $KeepData = $true }
+        '^--force$' { $Force = $true }
         '^(-h|--help)$' { $Command = 'help' }
     }
     $i++
@@ -40,10 +42,15 @@ $ErrorActionPreference = 'Stop'
 # Require Administrator for service management commands
 # ============================================================
 $adminRequired = @('setup', 'uninstall')
-if ($Command -in $adminRequired) {
+$needsAdmin = $Command -in $adminRequired
+# `stop --force` rewrites NSSM AppExit config and may need to kill processes
+# owned by the service account → requires Administrator.
+if ($Command -eq 'stop' -and $Force) { $needsAdmin = $true }
+if ($needsAdmin) {
     $currentPrincipal = New-Object Security.Principal.WindowsPrincipal([Security.Principal.WindowsIdentity]::GetCurrent())
     if (-not $currentPrincipal.IsInRole([Security.Principal.WindowsBuiltInRole]::Administrator)) {
-        Write-Host "ERROR: '$Command' requires Administrator privileges." -ForegroundColor Red
+        $label = if ($Command -eq 'stop' -and $Force) { 'stop --force' } else { $Command }
+        Write-Host "ERROR: '$label' requires Administrator privileges." -ForegroundColor Red
         Write-Host "  Right-click PowerShell and select 'Run as administrator'." -ForegroundColor Yellow
         exit 1
     }
@@ -440,6 +447,108 @@ function Stop-MinionService {
     Write-Host "minion-agent service stopped"
 }
 
+function Stop-MinionServiceForce {
+    # Force-stop path used when `stop` (graceful) is broken — typically after a
+    # corrupted update where .env is unreadable, the agent API is unresponsive,
+    # or NSSM keeps respawning node.exe and locking package files.
+    #
+    # Steps:
+    #   1. Disable NSSM auto-restart on every minion service (so sc.exe stop sticks)
+    #   2. sc.exe stop each service
+    #   3. Kill remaining helper processes (tvnserver/websockify/cloudflared)
+    #   4. Kill any node.exe whose CommandLine references a minion script
+    #   5. Restore NSSM AppExit Restart so subsequent `start` behaves normally
+    Write-Host ""
+    Write-Host "=========================================" -ForegroundColor Yellow
+    Write-Host " Stop --force: bypassing graceful shutdown" -ForegroundColor Yellow
+    Write-Host "=========================================" -ForegroundColor Yellow
+    Write-Host ""
+
+    $services = @('minion-agent', 'minion-websockify', 'minion-cloudflared', 'minion-vnc')
+    $nssmAvailable = $NssmPath -and (Test-Path $NssmPath)
+
+    # Step 1: Disable NSSM auto-restart so sc.exe stop is not undone.
+    if ($nssmAvailable) {
+        foreach ($svc in $services) {
+            $state = Get-ServiceState $svc
+            if (-not $state) { continue }
+            Invoke-Nssm set $svc AppExit Default Exit | Out-Null
+            Invoke-Nssm set $svc AppThrottle 0 | Out-Null
+            Write-Detail "Disabled auto-restart for $svc"
+        }
+    } else {
+        Write-Warn "NSSM not found — skipping AppExit override (services may auto-restart)"
+    }
+
+    # Step 2: Stop each service via sc.exe (works without admin if SDDL granted).
+    foreach ($svc in $services) {
+        $state = Get-ServiceState $svc
+        if (-not $state) { continue }
+        if ($state -eq 'STOPPED') {
+            Write-Detail "$svc already stopped"
+            continue
+        }
+        sc.exe stop $svc 2>&1 | Out-Null
+        for ($i = 0; $i -lt 10; $i++) {
+            $s = Get-ServiceState $svc
+            if ($s -eq 'STOPPED') { break }
+            Start-Sleep -Seconds 1
+        }
+        $final = Get-ServiceState $svc
+        if ($final -eq 'STOPPED') {
+            Write-Detail "$svc stopped"
+        } else {
+            Write-Warn "$svc state=$final (sc.exe stop did not complete in 10s)"
+        }
+    }
+
+    # Step 3: Kill helper processes that may have been spawned outside NSSM.
+    foreach ($name in @('tvnserver', 'websockify', 'cloudflared')) {
+        $procs = Get-Process -Name $name -ErrorAction SilentlyContinue
+        if ($procs) {
+            $procs | Stop-Process -Force -ErrorAction SilentlyContinue
+            Write-Detail "Killed $name ($($procs.Count) process(es))"
+        }
+    }
+
+    # Step 4: Kill node.exe processes that hold the package files. We match on
+    # CommandLine so we don't touch unrelated node processes the user is running.
+    $nodePattern = 'minion|@geekbeer\\minion|terminal-server|workflow-runner|routine-runner|wsl-session-server'
+    try {
+        $nodeProcs = Get-CimInstance Win32_Process -Filter "Name='node.exe'" -ErrorAction Stop |
+            Where-Object { $_.CommandLine -match $nodePattern }
+        foreach ($p in $nodeProcs) {
+            try {
+                Stop-Process -Id $p.ProcessId -Force -ErrorAction Stop
+                Write-Detail "Killed node.exe pid=$($p.ProcessId)"
+            } catch {
+                Write-Warn "Failed to kill node.exe pid=$($p.ProcessId): $($_.Exception.Message)"
+            }
+        }
+        if (-not $nodeProcs) {
+            Write-Detail "No matching node.exe processes"
+        }
+    } catch {
+        Write-Warn "Could not enumerate node processes: $($_.Exception.Message)"
+    }
+
+    # Step 5: Restore AppExit Restart so a subsequent `start` behaves normally.
+    if ($nssmAvailable) {
+        foreach ($svc in $services) {
+            $state = Get-ServiceState $svc
+            if (-not $state) { continue }
+            Invoke-Nssm set $svc AppExit Default Restart | Out-Null
+            Invoke-Nssm set $svc AppThrottle 1500 | Out-Null
+        }
+        Write-Detail "Restored NSSM auto-restart settings"
+    }
+
+    Write-Host ""
+    Write-Host "Force-stop complete. If files are still locked, run:" -ForegroundColor Cyan
+    Write-Host "  Get-Process | Where-Object { `$_.Modules.FileName -like '*\@geekbeer\minion\*' } | Select Id,ProcessName" -ForegroundColor Gray
+    Write-Host ""
+}
+
 function Restart-MinionService {
     Stop-MinionService
     Start-Sleep -Seconds 2
@@ -1673,7 +1782,9 @@ switch ($Command) {
     'reconfigure' { Invoke-Configure }  # alias for backwards compatibility
     'uninstall' { Invoke-Uninstall }
     'start' { Start-MinionService }
-    'stop' { Stop-MinionService }
+    'stop' {
+        if ($Force) { Stop-MinionServiceForce } else { Stop-MinionService }
+    }
     'restart' { Restart-MinionService }
     'status' { Show-Status }
     'health' { Show-Health }
@@ -1692,7 +1803,8 @@ switch ($Command) {
         Write-Host "Commands (no admin required):"
         Write-Host "  configure     Connect to HQ, deploy skills, start services"
         Write-Host "  start         Start the minion-agent service"
-        Write-Host "  stop          Stop the minion-agent service"
+        Write-Host "  stop          Stop the minion-agent service (graceful)"
+        Write-Host "  stop --force  Force-stop all minion services & processes (admin required)"
         Write-Host "  restart       Restart the minion-agent service"
         Write-Host "  status        Show agent service status"
         Write-Host "  health        Check agent health endpoint"
@@ -1709,5 +1821,11 @@ switch ($Command) {
         Write-Host ""
         Write-Host "Uninstall options:"
         Write-Host "  --keep-data          Preserve .env file"
+        Write-Host ""
+        Write-Host "Stop options:"
+        Write-Host "  --force              Disable NSSM auto-restart, sc.exe stop all services,"
+        Write-Host "                       kill remaining helpers (tvnserver/websockify/cloudflared)"
+        Write-Host "                       and node.exe processes that lock package files."
+        Write-Host "                       Use when graceful stop fails (e.g. corrupted update)."
     }
 }