@adhdev/daemon-core 1.0.18-rc.8 → 1.0.18-rc.9

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@adhdev/daemon-core",
-  "version": "1.0.18-rc.8",
+  "version": "1.0.18-rc.9",
   "description": "ADHDev daemon core — CDP, IDE detection, providers, command execution",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
@@ -47,8 +47,8 @@
   "author": "vilmire",
   "license": "AGPL-3.0-or-later",
   "dependencies": {
-    "@adhdev/mesh-shared": "1.0.18-rc.8",
-    "@adhdev/session-host-core": "1.0.18-rc.8",
+    "@adhdev/mesh-shared": "1.0.18-rc.9",
+    "@adhdev/session-host-core": "1.0.18-rc.9",
     "@agentclientprotocol/sdk": "^0.16.1",
     "ajv": "^8.20.0",
     "ajv-formats": "^3.0.1",

package/src/commands/upgrade-helper.ts

@@ -560,22 +560,36 @@ async function runDaemonUpgradeHelper(payload: DaemonUpgradeHelperPayload): Prom
       );
     }
 
-    await performWindowsAtomicUpgrade({
-      layout: windowsInstallerLayout,
-      packageName: payload.packageName,
-      targetVersion: payload.targetVersion,
-      portableNode,
-      excludePids: upgradePids,
-      hooks: createDefaultWindowsAtomicHooks({
+    try {
+      await performWindowsAtomicUpgrade({
+        layout: windowsInstallerLayout,
         packageName: payload.packageName,
         targetVersion: payload.targetVersion,
-        npmCliPath,
-        restartArgv,
-        cwd: payload.cwd || process.cwd(),
-        env: Object.fromEntries(Object.entries(process.env).filter(([key]) => key !== UPGRADE_HELPER_ENV)),
-        log: appendUpgradeLog,
-      }),
-    });
+        portableNode,
+        excludePids: upgradePids,
+        hooks: createDefaultWindowsAtomicHooks({
+          packageName: payload.packageName,
+          targetVersion: payload.targetVersion,
+          npmCliPath,
+          restartArgv,
+          cwd: payload.cwd || process.cwd(),
+          env: Object.fromEntries(Object.entries(process.env).filter(([key]) => key !== UPGRADE_HELPER_ENV)),
+          log: appendUpgradeLog,
+        }),
+      });
+    } catch (error: any) {
+      // performWindowsAtomicUpgrade already rolled the pointer/shims back to the
+      // prior version before rethrowing. Without a durable notice that rollback
+      // was silent — the daemon simply kept running the old version (the rc.6
+      // stuck-upgrade defect). Leave an actionable last-error file naming the
+      // target that failed its health/version gate.
+      emitUpgradeFailureNotice([
+        `adhdev ${payload.packageName}@${payload.targetVersion} upgrade failed and was rolled back: ${error?.message || String(error)}`,
+        `Previous version preserved (active prefix: ${windowsInstallerLayout.activePrefix}).`,
+        'See daemon-upgrade.log for the full install/health trace. The next daemon start will retry.',
+      ]);
+      throw error;
+    }
     try { fs.unlinkSync(getUpgradeFailureNoticePath()); } catch { /* no previous failure notice */ }
     appendUpgradeLog('Installer-managed Windows atomic upgrade completed');
     return;

package/src/commands/windows-atomic-upgrade.ts

@@ -8,6 +8,44 @@ const POINTER_NAME = '.adhdev-current';
 const STABLE_FILES = [POINTER_NAME, 'adhdev.cmd', 'adhdev.ps1', 'adhdev'] as const;
 const DEFAULT_HEALTH_PORT = 19222;
 
+function fetchLocalJson(port: number, pathname: string): Promise<{ ok: boolean; body: string }> {
+  return new Promise((resolve) => {
+    const req = http.get(`http://127.0.0.1:${port}${pathname}`, { timeout: 1500 }, (res) => {
+      let body = '';
+      res.setEncoding('utf8');
+      res.on('data', (chunk) => { body += chunk; });
+      res.on('end', () => resolve({ ok: res.statusCode === 200, body }));
+    });
+    req.on('timeout', () => req.destroy());
+    req.on('error', () => resolve({ ok: false, body: '' }));
+  });
+}
+
+// Liveness + pid identity: GET /health returns {ok, pid, wsPath, port}.
+async function fetchLocalHealth(port: number): Promise<{ ok: boolean; pid?: number }> {
+  const { ok, body } = await fetchLocalJson(port, '/health');
+  if (!ok) return { ok: false };
+  try {
+    const pid = Number(JSON.parse(body)?.pid);
+    return { ok: true, pid: Number.isFinite(pid) ? pid : undefined };
+  } catch {
+    return { ok: false };
+  }
+}
+
+// Running version: GET /api/v1/status exposes it at payload.status.version.
+// /health carries no version, so the upgrade version gate must read this.
+async function fetchLocalStatusVersion(port: number): Promise<string | undefined> {
+  const { ok, body } = await fetchLocalJson(port, '/api/v1/status');
+  if (!ok) return undefined;
+  try {
+    const version = (JSON.parse(body) as { status?: { version?: unknown } })?.status?.version;
+    return typeof version === 'string' ? version : undefined;
+  } catch {
+    return undefined;
+  }
+}
+
 // Markers that identify a Node process as ADHDev-owned when its command line
 // also lives under a versioned install prefix. This deliberately excludes
 // arbitrary user scripts that happen to be located under ~/.adhdev.
@@ -359,7 +397,13 @@ export function createDefaultWindowsAtomicHooks(options: {
   cwd: string;
   env: NodeJS.ProcessEnv;
   log: (message: string) => void;
+  /** Loopback IPC port to probe for health/version. Defaults to the daemon's local IPC port. */
+  healthPort?: number;
+  /** How long to poll for the replacement daemon to report the target version. */
+  healthTimeoutMs?: number;
 }): WindowsAtomicUpgradeHooks {
+  const healthPort = options.healthPort ?? DEFAULT_HEALTH_PORT;
+  const healthTimeoutMs = options.healthTimeoutMs ?? 30000;
   return {
     install: (stagedPrefix, portableNode) => {
       const env: NodeJS.ProcessEnv = {
@@ -370,7 +414,7 @@ export function createDefaultWindowsAtomicHooks(options: {
       };
       const pathKey = Object.keys(env).find((key) => key.toLowerCase() === 'path') || 'Path';
       env[pathKey] = `${path.dirname(portableNode)};${env[pathKey] || ''}`;
-      execFileSync(portableNode, [
+      const installOutput = String(execFileSync(portableNode, [
         options.npmCliPath,
         'install', '-g', `${options.packageName}@${options.targetVersion}`, '--force', '--prefer-online', '--prefix', stagedPrefix,
       ], {
@@ -379,7 +423,11 @@ export function createDefaultWindowsAtomicHooks(options: {
         maxBuffer: 20 * 1024 * 1024,
         windowsHide: true,
         env,
-      });
+      }));
+      // On failure execFileSync throws with stdout attached to the Error; on
+      // success it was previously discarded. Surface the npm output either way so
+      // a silently-succeeding-then-rolled-back upgrade leaves a diagnosable trail.
+      if (installOutput.trim()) options.log(installOutput.trim());
     },
     restart: (portableNode, stagedCliEntry) => {
       const restartArgv = options.restartArgv.map((arg, index) => index === 0 ? stagedCliEntry : arg);
@@ -406,23 +454,18 @@ export function createDefaultWindowsAtomicHooks(options: {
       child.unref();
     },
     waitForHealth: async (pid, targetVersion) => {
-      const deadline = Date.now() + 30000;
+      const deadline = Date.now() + healthTimeoutMs;
       while (Date.now() < deadline) {
-        const result = await new Promise<{ ok: boolean; pid?: number; body?: string }>((resolve) => {
-          const req = http.get(`http://127.0.0.1:${DEFAULT_HEALTH_PORT}/health`, { timeout: 1500 }, (res) => {
-            let body = '';
-            res.setEncoding('utf8');
-            res.on('data', (chunk) => { body += chunk; });
-            res.on('end', () => {
-              let responsePid: number | undefined;
-              try { responsePid = Number(JSON.parse(body)?.pid); } catch { /* noop */ }
-              resolve({ ok: res.statusCode === 200, pid: responsePid, body });
-            });
-          });
-          req.on('timeout', () => req.destroy());
-          req.on('error', () => resolve({ ok: false }));
-        });
-        if (result.ok && result.pid === pid && result.body?.includes(targetVersion)) return true;
+        // Liveness + pid identity come from GET /health, whose body is
+        // {ok, pid, wsPath, port} — it carries NO version. The running version
+        // lives only in GET /api/v1/status → payload.status.version, so the
+        // version gate must fetch that endpoint separately. Requiring the raw
+        // /health body to include targetVersion is unsatisfiable and silently
+        // rolls every upgrade back.
+        const liveness = await fetchLocalHealth(healthPort);
+        const alive = liveness.ok && liveness.pid === pid;
+        const version = alive ? await fetchLocalStatusVersion(healthPort) : undefined;
+        if (alive && version === targetVersion) return true;
         await new Promise((resolve) => setTimeout(resolve, 500));
       }
       return false;

package/src/mesh/mesh-queue-assignment.ts

@@ -6,7 +6,7 @@ import { getMesh } from '../config/mesh-config.js';
 import { detectCLI } from '../detection/cli-detector.js';
 import { LOG } from '../logging/logger.js';
 import { appendLedgerEntry } from './mesh-ledger.js';
-import { buildMeshNodeCapabilityTags, nodeSatisfiesRequiredTags, claimNextTask, updateTaskStatus, getQueue, recordTaskAutoLaunch, getActiveDirectDispatches, isTaskReadonly, taskDependenciesSatisfied, meshTaskNotBeforeReady, meshTaskPriorityRank } from './mesh-work-queue.js';
+import { buildMeshNodeCapabilityTags, nodeSatisfiesRequiredTags, claimNextTask, updateTaskStatus, getQueue, recordTaskAutoLaunch, getActiveDirectDispatches, isTaskReadonly, taskDependenciesSatisfied, meshTaskNotBeforeReady, meshTaskPriorityRank, requeueTask } from './mesh-work-queue.js';
 import type { MeshWorkQueueEntry } from './mesh-work-queue.js';
 import { fastForwardMeshNode } from './mesh-fast-forward.js';
 import { createSessionDelivery, updateSessionDeliveryStatus } from './mesh-delivery-policy.js';
@@ -931,6 +931,105 @@ function isTargetNodeTransientlyUnresolved(mesh: any, task: MeshWorkQueueEntry):
     return isWithinCloneBootstrapGrace(targetNodeId);
 }
 
+// ---------------------------------------------------------------------------
+// DEAD-TARGET-SELFHEAL: unpin a queue task hard-pinned to a session/node that has
+// died (absent from the live mesh) so a live idle session can claim it, instead of
+// leaving it stranded 'pending' forever behind the target_session_constraint skip.
+// ---------------------------------------------------------------------------
+
+// Conservative age gate before a pinned-but-dead target is unpinned. A target that is
+// merely briefly unassigned or momentarily reconnecting must not be reclaimed on the
+// tick it drops out of view; we require the task to have been idle (no updatedAt bump)
+// for at least this window first. Sized to comfortably outlast a transient P2P blip /
+// reconnect while staying well inside the reclaim cadence of the rest of the file
+// (AUTO_LAUNCH_AWAIT_CLAIM_MS is 90s; the stranded-reclaim watchdog fires on similar
+// scales), so a real reconnect wins the race and the self-heal only fires on a target
+// that is genuinely gone.
+const DEAD_TARGET_GRACE_MS = 60_000;
+
+interface DeadTargetVerdict {
+    /** The pinned target is confirmed dead and past the grace window → safe to unpin. */
+    dead: boolean;
+    /** True when the target NODE itself is absent from the live mesh (clear targetNodeId too). */
+    nodeDead: boolean;
+    /** Short reason string for the ledger/requeue. */
+    reason: string;
+}
+
+/**
+ * Decide whether a task's hard target pin (targetSessionId and/or targetNodeId) points at
+ * something that has DIED — i.e. is absent from the live mesh snapshot — and has been so
+ * long enough (DEAD_TARGET_GRACE_MS since the task's last update) that unpinning is safe.
+ *
+ * Two definitive death signals, deliberately conservative to never race a reconnecting node:
+ *
+ *  (1) NODE dead — the task pins a targetNodeId that matches NO node in the live mesh
+ *      (the same `meshNodeIdMatches`-over-mesh.nodes signal the targetPinUnmatched relabel
+ *      uses at the empty-candidate site). A pinned session on an absent node is unreachable
+ *      regardless, so the session pin is dead too. `nodeDead` ⇒ clear targetNodeId as well.
+ *      Excluded: a target that is only TRANSIENTLY unresolved (a freshly-cloned worktree
+ *      still propagating / bootstrapping) — isTargetNodeTransientlyUnresolved gates it out.
+ *
+ *  (2) SESSION dead on a LIVE LOCAL node — the target node IS present and is THIS daemon's
+ *      node, but the pinned session is absent from the local instance manager
+ *      (resolveSessionBusyVerdict === 'UNKNOWN'). Local session visibility is complete, so
+ *      absence here is genuine death, not a busy/generating flip. We KEEP targetNodeId (only
+ *      the session died; the node is healthy and can host a replacement claim).
+ *
+ * A live REMOTE node whose session is not in our idle view is NOT treated as dead: absence
+ * from the remote-idle mirror is explicitly UNKNOWN liveness (the session may be busy or its
+ * agent:ready pull merely lost), so unpinning it could race healthy in-flight work. Returns
+ * dead=false in that case, leaving the existing skip in place.
+ */
+function resolveDeadTargetVerdict(components: DaemonComponents, meshId: string, mesh: any, task: MeshWorkQueueEntry): DeadTargetVerdict {
+    const NOT_DEAD: DeadTargetVerdict = { dead: false, nodeDead: false, reason: '' };
+    const targetSessionId = readNonEmptyString(task.targetSessionId);
+    const targetNodeId = readNonEmptyString(task.targetNodeId);
+    if (!targetSessionId && !targetNodeId) return NOT_DEAD;
+
+    // Age gate: never reclaim a pin younger than the grace window (guards against a target
+    // that has only just dropped out of view for a momentary reconnect).
+    const lastUpdateMs = Date.parse(task.updatedAt || task.createdAt || '');
+    if (Number.isFinite(lastUpdateMs) && Date.now() - lastUpdateMs < DEAD_TARGET_GRACE_MS) return NOT_DEAD;
+
+    const nodes: any[] = Array.isArray(mesh?.nodes) ? mesh.nodes : [];
+
+    // (1) NODE-dead — a pinned node absent from the live mesh, and NOT merely transiently
+    // unresolved (a propagating/bootstrapping clone). This is a permanent routing miss.
+    if (targetNodeId) {
+        const nodePresent = nodes.some(n => meshNodeIdMatches(n, targetNodeId));
+        if (!nodePresent) {
+            if (isTargetNodeTransientlyUnresolved(mesh, task)) return NOT_DEAD;
+            return { dead: true, nodeDead: true, reason: 'dead_target_node_absent' };
+        }
+    }
+
+    // (2) SESSION-dead on a LIVE LOCAL node. Only meaningful when a session is pinned.
+    if (targetSessionId) {
+        // Resolve the pinned target's node (if any) to decide whether we can trust local
+        // absence. Without a targetNodeId, fall back to whichever live node hosts the session
+        // is unknowable here; treat that as a LOCAL check only (a session id we cannot see
+        // locally on a node we cannot resolve remotely stays UNKNOWN → not dead).
+        const node = targetNodeId
+            ? nodes.find(n => meshNodeIdMatches(n, targetNodeId))
+            : undefined;
+        // A pinned session on a REMOTE live node: absence from our view is UNKNOWN, not death.
+        // Only a LOCAL node (or no node pin at all — same-daemon assumption) lets us conclude
+        // death from local instance-manager absence.
+        const nodeIsLocal = node ? isLocalAutoLaunchNode(node) : true;
+        if (nodeIsLocal) {
+            const verdict = resolveSessionBusyVerdict(components, targetSessionId);
+            if (verdict === 'UNKNOWN') {
+                // Session absent from the complete local session view → genuinely gone.
+                return { dead: true, nodeDead: false, reason: 'dead_target_session_absent' };
+            }
+            // GENERATING / IDLE_CONFIRMED → the session is alive (possibly busy). Never disturb.
+        }
+    }
+
+    return NOT_DEAD;
+}
+
 /**
  * FALSE-BLOCKER-CLONE-QUEUE (stale-event clear): once a task whose actionable blocker we
  * previously paged either gets claimed or transitions to a self-resolving state, re-arm the
@@ -1929,6 +2028,32 @@ async function maybeAutoLaunchOneQueueSession(components: DaemonComponents, mesh
             continue;
         }
         if (task.targetSessionId) {
+            // DEAD-TARGET-SELFHEAL: before the unconditional target_session_constraint skip,
+            // check whether the pinned session/node has DIED (absent from the live mesh). A
+            // hard-pinned task whose target is gone can NEVER re-enter 'assigned' (the claim
+            // gate refuses every non-matching session) and this skip fires forever with no
+            // liveness check — the triple-walled stranded-pending defect. If the pin is
+            // confirmed dead past the grace window, requeue it (clearing the dead session
+            // pin, and the node pin too when the NODE itself is gone) so a live idle session
+            // can claim it. requeueTask counts toward maxTaskRetries → bounded self-heal that
+            // auto-fails past the cap (the desired terminal state, unblocking dependents).
+            const deadTarget = resolveDeadTargetVerdict(components, meshId, mesh, task);
+            if (deadTarget.dead) {
+                const requeued = requeueTask(meshId, task.id, {
+                    reason: deadTarget.reason,
+                    clearTargetSession: true,
+                    // Keep the node pin if only the SESSION died on a still-live node; clear it
+                    // when the NODE itself is absent (nothing to pin to).
+                    clearTargetNode: deadTarget.nodeDead,
+                });
+                if (requeued) {
+                    LOG.warn('MeshQueue', `DEAD-TARGET-SELFHEAL: task ${task.id} (mesh ${meshId}) was pinned to a dead target (${deadTarget.reason}); requeued${deadTarget.nodeDead ? ' and unpinned node' : ''} (requeueCount=${requeued.requeueCount ?? '?'}, status=${requeued.status}).`);
+                }
+                // Keep the skip for THIS tick (the requeue already flipped the row to
+                // pending/failed); a later tick assigns/launches the now-unpinned task.
+                markAutoLaunch(meshId, task.id, { status: 'skipped', reason: 'target_session_dead_requeued' });
+                continue;
+            }
             markAutoLaunch(meshId, task.id, { status: 'skipped', reason: 'target_session_constraint' });
             continue;
         }