@adhdev/daemon-core 0.9.77-rc.9 → 0.9.77

package/src/mesh/mesh-visualization.ts

@@ -0,0 +1,341 @@
+/**
+ * Mesh Visualization — Transform RepoMeshStatus into a graph structure
+ * for SVG/Canvas rendering without external dependencies.
+ */
+
+import type { RepoMeshStatus, RepoMeshNodeStatus, RepoMeshNodeHealth } from '../repo-mesh-types.js';
+import type { GitRepoStatus } from '../git/git-types.js';
+
+// ─── Graph Types ─────────────────────────────────
+
+export type MeshGraphNodeType = 'defaultBranchNode' | 'worktreeNode' | 'orphanNode';
+export type MeshGraphEdgeType = 'parentBranch' | 'worktreeLink' | 'sessionLink';
+
+export interface MeshGraphNode {
+  id: string;
+  type: MeshGraphNodeType;
+  label: string;
+  workspace: string;
+  branch: string | null;
+  health: RepoMeshNodeHealth;
+  ahead: number;
+  behind: number;
+  dirty: boolean;
+  dirtyFiles: number;
+  hasConflicts: boolean;
+  activeSessionCount: number;
+  activeSessions: string[];
+  providers: string[];
+  isOrphan: boolean;
+  orphanReasons: string[];
+  /** Next-step hint from convergence analysis */
+  nextStepHint?: string;
+  /** Original node status for drill-down */
+  source: RepoMeshNodeStatus;
+}
+
+export interface MeshGraphEdge {
+  id: string;
+  source: string;
+  target: string;
+  type: MeshGraphEdgeType;
+  label?: string;
+}
+
+export interface MeshGraph {
+  meshId: string;
+  meshName: string;
+  repoIdentity: string;
+  refreshedAt: string;
+  nodes: MeshGraphNode[];
+  edges: MeshGraphEdge[];
+  /** Summary statistics */
+  stats: {
+    totalNodes: number;
+    onlineNodes: number;
+    dirtyNodes: number;
+    orphanNodes: number;
+    errorNodes: number;
+    offlineNodes: number;
+    totalActiveSessions: number;
+  };
+  /** Global orphan / stale warnings */
+  warnings: string[];
+}
+
+// ─── Helpers ─────────────────────────────────────
+
+function isDirty(git?: GitRepoStatus): boolean {
+  if (!git) return false;
+  return (git.staged + git.modified + git.untracked + git.deleted + git.renamed) > 0;
+}
+
+function dirtyFileCount(git?: GitRepoStatus): number {
+  if (!git) return 0;
+  return git.staged + git.modified + git.untracked + git.deleted + git.renamed;
+}
+
+function detectOrphanReasons(node: RepoMeshNodeStatus, defaultBranch?: string | null): string[] {
+  const reasons: string[] = [];
+  const git = node.git;
+
+  if (!git) {
+    reasons.push('No git status available');
+    return reasons;
+  }
+
+  if (!git.isGitRepo) {
+    reasons.push('Not a git repository');
+    return reasons;
+  }
+
+  // Detached HEAD
+  if (git.branch === null && git.headCommit) {
+    reasons.push('Detached HEAD');
+  }
+
+  // No upstream tracking
+  if (git.branch && !git.upstream) {
+    // Local-only branch — orphan if not default
+    if (defaultBranch && git.branch !== defaultBranch) {
+      reasons.push(`No upstream: ${git.branch}`);
+    }
+  }
+
+  // Worktree branch with no upstream
+  if (git.branch && git.upstream === null && defaultBranch && git.branch !== defaultBranch) {
+    if (!reasons.includes(`No upstream: ${git.branch}`)) {
+      reasons.push(`No upstream: ${git.branch}`);
+    }
+  }
+
+  // Stale assigned tasks would be detected at queue level, not per node git status
+  // We surface node-level errors as orphan indicators
+  if (node.error) {
+    reasons.push(`Error: ${node.error}`);
+  }
+
+  return reasons;
+}
+
+function nodeHealthPriority(health: RepoMeshNodeHealth): number {
+  switch (health) {
+    case 'online': return 0;
+    case 'dirty': return 1;
+    case 'degraded': return 2;
+    case 'wrong_branch': return 3;
+    case 'offline': return 4;
+    case 'unknown': return 5;
+    default: return 5;
+  }
+}
+
+function pickDominantHealth(healths: RepoMeshNodeHealth[]): RepoMeshNodeHealth {
+  if (healths.length === 0) return 'unknown';
+  return healths.reduce((best, h) =>
+    nodeHealthPriority(h) > nodeHealthPriority(best) ? h : best
+  );
+}
+
+// ─── Graph Builder ─────────────────────────────────
+
+/**
+ * Build a visualization graph from RepoMeshStatus.
+ *
+ * Nodes:
+ *   - defaultBranchNode: the mesh's default branch (aggregated from nodes on that branch)
+ *   - worktreeNode: a node on a feature / worktree branch
+ *   - orphanNode: a node with orphanReasons (upstream missing, detached HEAD, etc.)
+ *
+ * Edges:
+ *   - parentBranch: default branch → worktree branch (when branch name differs)
+ *   - worktreeLink: links nodes that share the same branch (clustering hint)
+ *   - sessionLink: node → session (lightweight, optional; not rendered as primary edge)
+ */
+export function buildMeshGraph(status: RepoMeshStatus): MeshGraph {
+  const nodes: MeshGraphNode[] = [];
+  const edges: MeshGraphEdge[] = [];
+  const warnings: string[] = [];
+
+  // Track branch → nodes for edge creation
+  const branchToNodeIds = new Map<string, string[]>();
+
+  for (const nodeStatus of status.nodes) {
+    const git = nodeStatus.git;
+    const branch = git?.branch || null;
+    const orphanReasons = detectOrphanReasons(nodeStatus, null);
+    const dirty = isDirty(git);
+    const dfc = dirtyFileCount(git);
+
+    // Determine node type
+    let type: MeshGraphNodeType = 'worktreeNode';
+    if (orphanReasons.length > 0) {
+      type = 'orphanNode';
+    }
+
+    const graphNode: MeshGraphNode = {
+      id: nodeStatus.nodeId,
+      type,
+      label: nodeStatus.machineLabel || nodeStatus.nodeId.slice(0, 8),
+      workspace: nodeStatus.workspace,
+      branch,
+      health: nodeStatus.health,
+      ahead: git?.ahead ?? 0,
+      behind: git?.behind ?? 0,
+      dirty,
+      dirtyFiles: dfc,
+      hasConflicts: git?.hasConflicts ?? false,
+      activeSessionCount: nodeStatus.activeSessions?.length ?? 0,
+      activeSessions: nodeStatus.activeSessions ?? [],
+      providers: nodeStatus.providers ?? [],
+      isOrphan: orphanReasons.length > 0,
+      orphanReasons,
+      source: nodeStatus,
+    };
+
+    nodes.push(graphNode);
+
+    if (branch) {
+      const list = branchToNodeIds.get(branch) ?? [];
+      list.push(graphNode.id);
+      branchToNodeIds.set(branch, list);
+    }
+  }
+
+  // Create synthetic default branch node if we can infer one
+  // Heuristic: most common branch among nodes with upstream
+  const branchUpstreamCounts = new Map<string, number>();
+  for (const n of status.nodes) {
+    const b = n.git?.branch;
+    const upstream = n.git?.upstream;
+    if (b && upstream) {
+      branchUpstreamCounts.set(b, (branchUpstreamCounts.get(b) ?? 0) + 1);
+    }
+  }
+  let inferredDefaultBranch: string | null = null;
+  let bestCount = 0;
+  for (const [b, count] of branchUpstreamCounts) {
+    if (count > bestCount) {
+      bestCount = count;
+      inferredDefaultBranch = b;
+    }
+  }
+
+  // If we have a clear default branch, create a synthetic node for it
+  const defaultBranchNodeId = inferredDefaultBranch
+    ? `__branch_${inferredDefaultBranch}`
+    : null;
+
+  if (defaultBranchNodeId && inferredDefaultBranch) {
+    const branchNodes = branchToNodeIds.get(inferredDefaultBranch) ?? [];
+    const branchHealths = branchNodes.map(id =>
+      nodes.find(n => n.id === id)!.health
+    );
+
+    const defaultNode: MeshGraphNode = {
+      id: defaultBranchNodeId,
+      type: 'defaultBranchNode',
+      label: inferredDefaultBranch,
+      workspace: '',
+      branch: inferredDefaultBranch,
+      health: pickDominantHealth(branchHealths),
+      ahead: 0,
+      behind: 0,
+      dirty: false,
+      dirtyFiles: 0,
+      hasConflicts: false,
+      activeSessionCount: 0,
+      activeSessions: [],
+      providers: [],
+      isOrphan: false,
+      orphanReasons: [],
+      nextStepHint: branchNodes.length > 0 ? `${branchNodes.length} node(s) on default branch` : undefined,
+      source: {
+        nodeId: defaultBranchNodeId,
+        machineLabel: inferredDefaultBranch,
+        workspace: '',
+        health: 'online',
+        providers: [],
+        activeSessions: [],
+      },
+    };
+    nodes.push(defaultNode);
+
+    // Edge: default branch → each worktree branch that is NOT the default
+    const seenBranches = new Set<string>();
+    for (const n of nodes) {
+      if (n.type === 'defaultBranchNode') continue;
+      if (!n.branch) continue;
+      if (n.branch === inferredDefaultBranch) {
+        // Link worktree nodes on default branch directly to default branch node
+        edges.push({
+          id: `${defaultBranchNodeId}--${n.id}`,
+          source: defaultBranchNodeId,
+          target: n.id,
+          type: 'parentBranch',
+          label: 'default',
+        });
+        continue;
+      }
+      if (seenBranches.has(n.branch)) continue;
+      seenBranches.add(n.branch);
+      edges.push({
+        id: `${defaultBranchNodeId}--branch_${n.branch}`,
+        source: defaultBranchNodeId,
+        target: n.branch,
+        type: 'parentBranch',
+        label: n.branch,
+      });
+    }
+  }
+
+  // worktreeLink edges: connect nodes sharing the same branch
+  for (const [branch, ids] of branchToNodeIds) {
+    if (ids.length < 2) continue;
+    for (let i = 1; i < ids.length; i++) {
+      edges.push({
+        id: `wt_${ids[0]}--${ids[i]}`,
+        source: ids[0],
+        target: ids[i],
+        type: 'worktreeLink',
+        label: branch,
+      });
+    }
+  }
+
+  // Collect warnings
+  const orphanCount = nodes.filter(n => n.isOrphan).length;
+  if (orphanCount > 0) {
+    warnings.push(`${orphanCount} orphan node(s) detected`);
+  }
+  const conflictCount = nodes.filter(n => n.hasConflicts).length;
+  if (conflictCount > 0) {
+    warnings.push(`${conflictCount} node(s) with merge conflicts`);
+  }
+  const offlineCount = nodes.filter(n => n.health === 'offline').length;
+  if (offlineCount > 0) {
+    warnings.push(`${offlineCount} node(s) offline`);
+  }
+
+  // Stats
+  const stats = {
+    totalNodes: status.nodes.length,
+    onlineNodes: status.nodes.filter(n => n.health === 'online').length,
+    dirtyNodes: nodes.filter(n => n.dirty).length,
+    orphanNodes: orphanCount,
+    errorNodes: status.nodes.filter(n => !!n.error).length,
+    offlineNodes: offlineCount,
+    totalActiveSessions: status.nodes.reduce((sum, n) => sum + (n.activeSessions?.length ?? 0), 0),
+  };
+
+  return {
+    meshId: status.meshId,
+    meshName: status.meshName,
+    repoIdentity: status.repoIdentity,
+    refreshedAt: status.refreshedAt,
+    nodes,
+    edges,
+    stats,
+    warnings,
+  };
+}

package/src/mesh/mesh-work-queue.ts

@@ -3,7 +3,12 @@ import { join } from 'path';
 import { randomUUID } from 'crypto';
 import { getLedgerDir } from './mesh-ledger.js';
 
-export type MeshTaskStatus = 'pending' | 'assigned' | 'completed' | 'failed';
+export type MeshTaskStatus = 'pending' | 'assigned' | 'completed' | 'failed' | 'cancelled';
+export type MeshActiveTaskStatus = Extract<MeshTaskStatus, 'pending' | 'assigned'>;
+export type MeshHistoricalTaskStatus = Extract<MeshTaskStatus, 'completed' | 'failed' | 'cancelled'>;
+
+export const ACTIVE_MESH_QUEUE_STATUSES: MeshActiveTaskStatus[] = ['pending', 'assigned'];
+export const HISTORICAL_MESH_QUEUE_STATUSES: MeshHistoricalTaskStatus[] = ['completed', 'failed', 'cancelled'];
 
 export interface MeshWorkQueueEntry {
     id: string;
@@ -12,10 +17,30 @@ export interface MeshWorkQueueEntry {
     status: MeshTaskStatus;
     /** If specified, only this node can claim the task (used by legacy mesh_send_task) */
     targetNodeId?: string;
+    /** If specified, only this runtime session can claim the task */
+    targetSessionId?: string;
     /** The node that actually claimed and is executing the task */
     assignedNodeId?: string;
     /** The session currently executing the task */
     assignedSessionId?: string;
+    /** Human/operator reason for terminal cancellation. */
+    cancelReason?: string;
+    cancelledAt?: string;
+    /** Human/operator reason for manually requeueing a task. */
+    requeueReason?: string;
+    requeuedAt?: string;
+    requeueCount?: number;
+    /** Last automatic queue session spin-up attempt, for mesh_view_queue/debug visibility. */
+    autoLaunch?: {
+        status: 'skipped' | 'started' | 'failed' | 'completed';
+        reason?: string;
+        nodeId?: string;
+        providerType?: string;
+        sessionId?: string;
+        updatedAt: string;
+    };
+    /** ISO timestamp when the task was dispatched (assigned) to a node/session. Used for precise matching on completion. */
+    dispatchTimestamp?: string;
     createdAt: string;
     updatedAt: string;
 }
@@ -47,7 +72,7 @@ function writeQueue(meshId: string, queue: MeshWorkQueueEntry[]): void {
 export function enqueueTask(
     meshId: string,
     message: string,
-    opts?: { targetNodeId?: string }
+    opts?: { targetNodeId?: string; targetSessionId?: string }
 ): MeshWorkQueueEntry {
     const queue = readQueue(meshId);
     const entry: MeshWorkQueueEntry = {
@@ -56,6 +81,7 @@ export function enqueueTask(
         message,
         status: 'pending',
         targetNodeId: opts?.targetNodeId,
+        targetSessionId: opts?.targetSessionId,
         createdAt: new Date().toISOString(),
         updatedAt: new Date().toISOString(),
     };
@@ -81,13 +107,25 @@ export function getQueue(meshId: string, opts?: { status?: MeshTaskStatus[] }):
  */
 export function claimNextTask(meshId: string, nodeId: string, sessionId: string): MeshWorkQueueEntry | null {
     const queue = readQueue(meshId);
+
+    // A worker must finish or fail its current queued assignment before it can
+    // claim another one. maxParallelTasks limits total mesh concurrency; it is
+    // not permission for one node/session to accumulate multiple assigned items.
+    const hasActiveAssignment = queue.some(q => q.status === 'assigned' && (
+        q.assignedSessionId === sessionId || q.assignedNodeId === nodeId
+    ));
+    if (hasActiveAssignment) return null;
     
     // Find highest priority task:
-    // 1. Pending tasks explicitly targeted at this node
-    // 2. Pending tasks with no target node
-    let targetIdx = queue.findIndex(q => q.status === 'pending' && q.targetNodeId === nodeId);
+    // 1. Pending tasks explicitly targeted at this runtime session
+    // 2. Pending tasks explicitly targeted at this node (but not another session)
+    // 3. Pending tasks with no target node/session
+    let targetIdx = queue.findIndex(q => q.status === 'pending' && q.targetSessionId === sessionId);
     if (targetIdx === -1) {
-        targetIdx = queue.findIndex(q => q.status === 'pending' && !q.targetNodeId);
+        targetIdx = queue.findIndex(q => q.status === 'pending' && q.targetNodeId === nodeId && !q.targetSessionId);
+    }
+    if (targetIdx === -1) {
+        targetIdx = queue.findIndex(q => q.status === 'pending' && !q.targetNodeId && !q.targetSessionId);
     }
 
     if (targetIdx === -1) return null;
@@ -96,6 +134,7 @@ export function claimNextTask(meshId: string, nodeId: string, sessionId: string)
     entry.status = 'assigned';
     entry.assignedNodeId = nodeId;
     entry.assignedSessionId = sessionId;
+    entry.dispatchTimestamp = new Date().toISOString();
     entry.updatedAt = new Date().toISOString();
 
     writeQueue(meshId, queue);
@@ -121,6 +160,83 @@ export function updateTaskStatus(
     return queue[idx];
 }
 
+export function recordTaskAutoLaunch(
+    meshId: string,
+    taskId: string,
+    autoLaunch: Omit<NonNullable<MeshWorkQueueEntry['autoLaunch']>, 'updatedAt'>,
+): MeshWorkQueueEntry | null {
+    const queue = readQueue(meshId);
+    const idx = queue.findIndex(q => q.id === taskId);
+    if (idx === -1) return null;
+    const now = new Date().toISOString();
+    queue[idx].autoLaunch = {
+        ...autoLaunch,
+        updatedAt: now,
+    };
+    queue[idx].updatedAt = now;
+    writeQueue(meshId, queue);
+    return queue[idx];
+}
+
+/**
+ * Mark a queue task as manually cancelled without deleting audit history.
+ */
+export function cancelTask(
+    meshId: string,
+    taskId: string,
+    opts?: { reason?: string },
+): MeshWorkQueueEntry | null {
+    const queue = readQueue(meshId);
+    const idx = queue.findIndex(q => q.id === taskId);
+    if (idx === -1) return null;
+
+    const now = new Date().toISOString();
+    queue[idx].status = 'cancelled';
+    queue[idx].updatedAt = now;
+    queue[idx].cancelledAt = now;
+    if (opts?.reason) queue[idx].cancelReason = opts.reason;
+    writeQueue(meshId, queue);
+    return queue[idx];
+}
+
+/**
+ * Return a queue task to pending for retry. By default, dead session targeting
+ * and assigned ownership are cleared so stale assignments do not strand again.
+ */
+export function requeueTask(
+    meshId: string,
+    taskId: string,
+    opts?: {
+        reason?: string;
+        targetNodeId?: string;
+        targetSessionId?: string;
+        clearTargetNode?: boolean;
+        clearTargetSession?: boolean;
+    },
+): MeshWorkQueueEntry | null {
+    const queue = readQueue(meshId);
+    const idx = queue.findIndex(q => q.id === taskId);
+    if (idx === -1) return null;
+
+    const entry = queue[idx];
+    const now = new Date().toISOString();
+    entry.status = 'pending';
+    delete entry.assignedNodeId;
+    delete entry.assignedSessionId;
+    delete entry.cancelledAt;
+    delete entry.cancelReason;
+    if (opts?.clearTargetNode) delete entry.targetNodeId;
+    if (typeof opts?.targetNodeId === 'string') entry.targetNodeId = opts.targetNodeId;
+    if (opts?.clearTargetSession !== false) delete entry.targetSessionId;
+    if (typeof opts?.targetSessionId === 'string') entry.targetSessionId = opts.targetSessionId;
+    entry.updatedAt = now;
+    entry.requeuedAt = now;
+    entry.requeueCount = (entry.requeueCount || 0) + 1;
+    if (opts?.reason) entry.requeueReason = opts.reason;
+    writeQueue(meshId, queue);
+    return entry;
+}
+
 /**
  * Update the status of the task currently assigned to a specific session.
  */
@@ -130,24 +246,48 @@ export function updateSessionTaskStatus(
     status: MeshTaskStatus,
 ): MeshWorkQueueEntry | null {
     const queue = readQueue(meshId);
-    // Find the most recently assigned task for this session that isn't already terminal
-    // (In case multiple tasks were assigned to the same session over time, though rare)
+    // Collect all assigned tasks for this session, then pick the one with the
+    // most recent dispatchTimestamp (or updatedAt fallback for legacy entries).
+    // This prevents completing the wrong task when multiple tasks were assigned
+    // to the same session in rapid succession.
+    let bestIdx = -1;
+    let bestTime = 0;
     for (let i = queue.length - 1; i >= 0; i--) {
         if (queue[i].assignedSessionId === sessionId && queue[i].status === 'assigned') {
-            queue[i].status = status;
-            queue[i].updatedAt = new Date().toISOString();
-            writeQueue(meshId, queue);
-            return queue[i];
+            const time = new Date(queue[i].dispatchTimestamp || queue[i].updatedAt).getTime();
+            if (time > bestTime) {
+                bestTime = time;
+                bestIdx = i;
+            }
         }
     }
-    return null;
+    if (bestIdx === -1) return null;
+
+    queue[bestIdx].status = status;
+    queue[bestIdx].updatedAt = new Date().toISOString();
+    writeQueue(meshId, queue);
+    return queue[bestIdx];
 }
 
 export interface MeshWorkQueueStats {
+    total: number;
+    active: number;
+    historical: number;
     pending: number;
     assigned: number;
     completed: number;
     failed: number;
+    cancelled: number;
+    /** Source-of-truth active queue counters; only pending/assigned are live work. */
+    activeCounts: Record<MeshActiveTaskStatus, number>;
+    /** Terminal ledger records kept for audit/history; never count as active work. */
+    historicalCounts: Record<MeshHistoricalTaskStatus, number>;
+    activeAssignments: Array<{
+        id: string;
+        nodeId?: string;
+        sessionId?: string;
+        message: string;
+    }>;
 }
 
 /**
@@ -155,10 +295,36 @@ export interface MeshWorkQueueStats {
  */
 export function getMeshQueueStats(meshId: string): MeshWorkQueueStats {
     const queue = readQueue(meshId);
+    const pending = queue.filter(q => q.status === 'pending').length;
+    const assigned = queue.filter(q => q.status === 'assigned').length;
+    const completed = queue.filter(q => q.status === 'completed').length;
+    const failed = queue.filter(q => q.status === 'failed').length;
+    const cancelled = queue.filter(q => q.status === 'cancelled').length;
     return {
-        pending: queue.filter(q => q.status === 'pending').length,
-        assigned: queue.filter(q => q.status === 'assigned').length,
-        completed: queue.filter(q => q.status === 'completed').length,
-        failed: queue.filter(q => q.status === 'failed').length,
+        total: queue.length,
+        active: pending + assigned,
+        historical: completed + failed + cancelled,
+        pending,
+        assigned,
+        completed,
+        failed,
+        cancelled,
+        activeCounts: {
+            pending,
+            assigned,
+        },
+        historicalCounts: {
+            completed,
+            failed,
+            cancelled,
+        },
+        activeAssignments: queue
+            .filter(q => q.status === 'assigned')
+            .map(q => ({
+                id: q.id,
+                nodeId: q.assignedNodeId,
+                sessionId: q.assignedSessionId,
+                message: q.message,
+            })),
     };
 }