@adhdev/daemon-core 0.9.82-rc.34 → 0.9.82-rc.341

package/src/mesh/mesh-refine-batch.ts

@@ -0,0 +1,197 @@
+import { execFile } from 'node:child_process';
+import { promisify } from 'node:util';
+
+const execFileAsync = promisify(execFile);
+
+/**
+ * Change-area analysis for one worktree node, used to order sibling nodes for
+ * batch refinement so that nodes least likely to conflict merge first.
+ *
+ * The heuristic is deliberately git-only and side-effect-free: it inspects the
+ * commits the node's branch adds on top of the base (`base..branch`) and records
+ *   - whether any submodule gitlink path is touched (high-conflict signal: the
+ *     batch must rebase later siblings onto the advanced submodule main), and
+ *   - the set of changed top-level paths (so siblings touching disjoint trees can
+ *     be ordered ahead of ones that overlap).
+ */
+export interface MeshRefineBatchNodeChangeArea {
+    nodeId: string;
+    workspace: string;
+    branch: string;
+    /** Top-level path segments changed by the branch vs. base (e.g. 'oss', 'packages'). */
+    changedTopLevelPaths: string[];
+    /** Full changed file list (bounded) for overlap detection. */
+    changedFiles: string[];
+    /** Submodule gitlink paths touched by the branch (subset of changedTopLevelPaths). */
+    touchedSubmodulePaths: string[];
+    /** True when the branch touches at least one submodule gitlink. */
+    touchesSubmodule: boolean;
+    /** Number of commits the branch is ahead of base; 0 means nothing to merge. */
+    aheadCount: number;
+    /** Non-fatal analysis error (e.g. base/branch unresolved); ordering falls back to neutral. */
+    error?: string;
+}
+
+export interface MeshRefineBatchOrderingResult {
+    /** Node IDs in the order they should be refined. */
+    order: string[];
+    /** Per-node change areas, keyed by node id, for plan transparency. */
+    changeAreas: Record<string, MeshRefineBatchNodeChangeArea>;
+    /** Human-readable explanation of why the order was chosen. */
+    rationale: string[];
+}
+
+const MAX_CHANGED_FILES = 500;
+
+function topLevel(path: string): string {
+    const slash = path.indexOf('/');
+    return slash === -1 ? path : path.slice(0, slash);
+}
+
+/**
+ * Resolve the set of submodule gitlink paths declared in a repo's .gitmodules,
+ * relative to the repo root. Used to classify which changed paths are submodule
+ * pointer bumps vs. ordinary file edits.
+ */
+async function resolveSubmodulePaths(repoRoot: string): Promise<Set<string>> {
+    try {
+        const { stdout } = await execFileAsync(
+            'git',
+            ['config', '--file', '.gitmodules', '--get-regexp', 'path'],
+            { cwd: repoRoot, encoding: 'utf8' },
+        );
+        const paths = new Set<string>();
+        for (const line of stdout.split('\n')) {
+            const trimmed = line.trim();
+            if (!trimmed) continue;
+            // Format: "submodule.<name>.path <path>"
+            const spaceIdx = trimmed.indexOf(' ');
+            if (spaceIdx === -1) continue;
+            const value = trimmed.slice(spaceIdx + 1).trim();
+            if (value) paths.add(value);
+        }
+        return paths;
+    } catch {
+        // No .gitmodules (or git error) → repo has no submodules to classify.
+        return new Set();
+    }
+}
+
+/**
+ * Analyze one worktree node's change area against its merge base.
+ *
+ * @param baseRef  ref that the node will merge into (e.g. 'origin/main' or a SHA)
+ * @param branchRef the node's branch tip
+ * @param diffCwd  repo to run the diff in (the worktree itself is authoritative for
+ *                 `branch` resolution, but the diff `base..branch` is symmetric, so
+ *                 the worktree cwd works for both refs once base is a reachable SHA).
+ */
+export async function analyzeMeshRefineNodeChangeArea(args: {
+    nodeId: string;
+    workspace: string;
+    branch: string;
+    baseRef: string;
+    branchRef: string;
+    diffCwd: string;
+    submodulePaths: Set<string>;
+}): Promise<MeshRefineBatchNodeChangeArea> {
+    const { nodeId, workspace, branch, baseRef, branchRef, diffCwd, submodulePaths } = args;
+    const base: MeshRefineBatchNodeChangeArea = {
+        nodeId,
+        workspace,
+        branch,
+        changedTopLevelPaths: [],
+        changedFiles: [],
+        touchedSubmodulePaths: [],
+        touchesSubmodule: false,
+        aheadCount: 0,
+    };
+    try {
+        // Use the merge-base so we compare only the node's own commits, not changes
+        // that already landed on base via a sibling earlier in the batch.
+        let mergeBase = baseRef;
+        try {
+            const { stdout } = await execFileAsync('git', ['merge-base', baseRef, branchRef], { cwd: diffCwd, encoding: 'utf8' });
+            const resolved = stdout.trim();
+            if (resolved) mergeBase = resolved;
+        } catch { /* fall back to baseRef directly */ }
+
+        const { stdout: countStdout } = await execFileAsync(
+            'git',
+            ['rev-list', '--count', `${mergeBase}..${branchRef}`],
+            { cwd: diffCwd, encoding: 'utf8' },
+        );
+        base.aheadCount = Number.parseInt(countStdout.trim(), 10) || 0;
+
+        const { stdout: nameStdout } = await execFileAsync(
+            'git',
+            ['diff', '--name-only', `${mergeBase}..${branchRef}`],
+            { cwd: diffCwd, encoding: 'utf8' },
+        );
+        const files = nameStdout.split('\n').map(line => line.trim()).filter(Boolean).slice(0, MAX_CHANGED_FILES);
+        base.changedFiles = files;
+        const topSet = new Set<string>();
+        const submoduleSet = new Set<string>();
+        for (const file of files) {
+            const top = topLevel(file);
+            topSet.add(top);
+            // A changed path is a submodule touch if the file path IS a declared
+            // submodule path (gitlink bumps surface as the submodule path itself).
+            if (submodulePaths.has(file) || submodulePaths.has(top)) {
+                submoduleSet.add(submodulePaths.has(file) ? file : top);
+            }
+        }
+        base.changedTopLevelPaths = [...topSet].sort();
+        base.touchedSubmodulePaths = [...submoduleSet].sort();
+        base.touchesSubmodule = submoduleSet.size > 0;
+        return base;
+    } catch (e: any) {
+        base.error = e?.message || String(e);
+        return base;
+    }
+}
+
+/**
+ * Order nodes for batch refinement to minimize cross-sibling conflicts.
+ *
+ * Heuristic (deterministic, stable):
+ *   1. Nodes that do NOT touch any submodule come first — they cannot advance the
+ *      submodule main, so they never force a later submodule rebase.
+ *   2. Within each group, fewer touched top-level paths first (smaller blast radius).
+ *   3. Tie-break by node id for determinism.
+ *
+ * Submodule-touching siblings are intrinsically serial: each one that merges
+ * advances oss main, so the next must rebase. Ordering them last keeps the
+ * non-submodule merges (which never need a submodule rebase) clean and up front.
+ */
+export function orderMeshRefineBatchNodes(
+    changeAreas: MeshRefineBatchNodeChangeArea[],
+): MeshRefineBatchOrderingResult {
+    const areaById: Record<string, MeshRefineBatchNodeChangeArea> = {};
+    for (const area of changeAreas) areaById[area.nodeId] = area;
+
+    const ranked = [...changeAreas].sort((a, b) => {
+        const aSub = a.touchesSubmodule ? 1 : 0;
+        const bSub = b.touchesSubmodule ? 1 : 0;
+        if (aSub !== bSub) return aSub - bSub;
+        const aBreadth = a.changedTopLevelPaths.length;
+        const bBreadth = b.changedTopLevelPaths.length;
+        if (aBreadth !== bBreadth) return aBreadth - bBreadth;
+        return a.nodeId.localeCompare(b.nodeId);
+    });
+
+    const rationale: string[] = [];
+    const nonSub = ranked.filter(a => !a.touchesSubmodule).map(a => a.nodeId);
+    const sub = ranked.filter(a => a.touchesSubmodule).map(a => a.nodeId);
+    if (nonSub.length) {
+        rationale.push(`Non-submodule nodes first (no submodule-main advance, conflict-free ordering): ${nonSub.join(', ')}`);
+    }
+    if (sub.length) {
+        rationale.push(`Submodule-touching nodes last, serialized (each merge advances submodule main, forcing rebase of the next): ${sub.join(', ')}`);
+    }
+    for (const area of ranked) {
+        if (area.error) rationale.push(`Node ${area.nodeId}: change-area analysis degraded (${area.error}); placed with neutral priority.`);
+    }
+
+    return { order: ranked.map(a => a.nodeId), changeAreas: areaById, rationale };
+}

package/src/mesh/mesh-refine-status.ts

@@ -0,0 +1,231 @@
+import type { MeshLedgerEntry } from './mesh-ledger.js';
+import type { PendingMeshCoordinatorEvent } from './mesh-events.js';
+import type { MeshAsyncJobLifecycle } from '../repo-mesh-types.js';
+
+export type MeshAsyncRefineJobStatus = 'accepted' | 'running' | 'completed' | 'failed';
+
+export interface MeshAsyncRefineJobSummary extends MeshAsyncJobLifecycle {
+    jobId: string;
+    interactionId?: string;
+    status: MeshAsyncRefineJobStatus;
+    meshId?: string;
+    nodeId?: string;
+    targetNodeId?: string;
+    targetDaemonId?: string;
+    workspace?: string;
+    branch?: string;
+    into?: string;
+    retryOfJobId?: string;
+    lastEvent?: string;
+    lastLedgerKind?: string;
+    lastUpdatedAt?: string;
+    instruction: string;
+}
+
+function readString(value: unknown): string | undefined {
+    return typeof value === 'string' && value.trim() ? value.trim() : undefined;
+}
+
+function readRecord(value: unknown): Record<string, unknown> | undefined {
+    return value && typeof value === 'object' && !Array.isArray(value)
+        ? value as Record<string, unknown>
+        : undefined;
+}
+
+function eventStatus(event: string | undefined, fallback?: string): MeshAsyncRefineJobStatus | undefined {
+    if (event === 'refine:accepted') return 'accepted';
+    if (event === 'refine:completed') return 'completed';
+    if (event === 'refine:failed') return 'failed';
+    if (fallback === 'completed' || fallback === 'failed' || fallback === 'accepted') return fallback;
+    return undefined;
+}
+
+function ledgerStatus(kind: string, fallback?: string): MeshAsyncRefineJobStatus {
+    if (kind === 'task_completed') return 'completed';
+    if (kind === 'task_failed') return 'failed';
+    if (fallback === 'accepted') return 'accepted';
+    return 'running';
+}
+
+function instructionForStatus(status: MeshAsyncRefineJobStatus): string {
+    if (status === 'accepted') return 'Refine job is accepted; wait for asyncRefineJobs or pendingCoordinatorEvents to report running/completed/failed.';
+    if (status === 'running') return 'Refine job is running; do not poll the ledger repeatedly. Watch asyncRefineJobs or pendingCoordinatorEvents for the terminal result.';
+    if (status === 'completed') return 'Refine job completed; inspect branch convergence and cleanup evidence before reporting final merge state.';
+    return 'Refine job failed; inspect result/finalBranchConvergenceState in mesh_task_history, fix the blocker, then rerun mesh_refine_node when ready.';
+}
+
+function mergeJob(
+    jobs: Map<string, MeshAsyncRefineJobSummary>,
+    patch: Partial<MeshAsyncRefineJobSummary> & { jobId?: string },
+): void {
+    const jobId = readString(patch.jobId);
+    if (!jobId) return;
+    const previous = jobs.get(jobId);
+    const status = patch.status || previous?.status || 'running';
+    const definedPatch = Object.fromEntries(
+        Object.entries(patch).filter(([, value]) => value !== undefined),
+    ) as Partial<MeshAsyncRefineJobSummary>;
+    jobs.set(jobId, {
+        ...previous,
+        ...definedPatch,
+        jobId,
+        status,
+        instruction: instructionForStatus(status),
+    });
+}
+
+export function buildMeshAsyncRefineJobs(args: {
+    meshId?: string;
+    ledgerEntries?: MeshLedgerEntry[];
+    pendingEvents?: PendingMeshCoordinatorEvent[];
+}): MeshAsyncRefineJobSummary[] {
+    const jobs = new Map<string, MeshAsyncRefineJobSummary>();
+
+    for (const entry of args.ledgerEntries || []) {
+        const payload = readRecord(entry.payload);
+        if (payload?.source !== 'refine_mesh_node_async_job') continue;
+        const refineJob = readRecord(payload.refineJob);
+        const result = readRecord(payload.result);
+        const finalState = readRecord(payload.finalBranchConvergenceState) || readRecord(result?.finalBranchConvergenceState);
+        const jobId = readString(refineJob?.jobId);
+        if (!jobId) continue;
+        const status = ledgerStatus(entry.kind, readString(refineJob?.status));
+        mergeJob(jobs, {
+            jobId,
+            interactionId: readString(refineJob?.interactionId),
+            status,
+            meshId: readString(refineJob?.meshId) || args.meshId,
+            nodeId: readString(refineJob?.nodeId) || entry.nodeId,
+            targetNodeId: readString(refineJob?.nodeId) || entry.nodeId,
+            targetDaemonId: readString(refineJob?.targetDaemonId),
+            workspace: readString(refineJob?.workspace),
+            branch: readString(result?.branch) || readString(finalState?.branch),
+            into: readString(result?.into) || readString(finalState?.baseBranch),
+            startedAt: readString(refineJob?.startedAt),
+            completedAt: readString(refineJob?.completedAt),
+            retryOfJobId: readString(refineJob?.retryOfJobId) || readString(payload.retryOfJobId),
+            lastLedgerKind: entry.kind,
+            lastUpdatedAt: entry.timestamp,
+        });
+    }
+
+    for (const event of args.pendingEvents || []) {
+        const metadata = readRecord(event.metadataEvent);
+        if (metadata?.source !== 'refine_mesh_node_async_job') continue;
+        const result = readRecord(metadata.result);
+        const finalState = readRecord(result?.finalBranchConvergenceState);
+        const jobId = readString(metadata.jobId);
+        if (!jobId) continue;
+        const status = eventStatus(event.event, readString(metadata.status));
+        mergeJob(jobs, {
+            jobId,
+            interactionId: readString(metadata.interactionId),
+            ...(status ? { status } : {}),
+            meshId: readString(metadata.meshId) || event.meshId || args.meshId,
+            nodeId: readString(metadata.nodeId) || event.nodeId,
+            targetNodeId: readString(metadata.nodeId) || event.nodeId,
+            targetDaemonId: readString(metadata.targetDaemonId),
+            workspace: readString(metadata.workspace) || event.workspace,
+            branch: readString(result?.branch) || readString(finalState?.branch),
+            into: readString(result?.into) || readString(finalState?.baseBranch),
+            startedAt: readString(metadata.startedAt),
+            completedAt: readString(metadata.completedAt),
+            retryOfJobId: readString(metadata.retryOfJobId),
+            lastEvent: event.event,
+            lastUpdatedAt: new Date(event.queuedAt).toISOString(),
+        });
+    }
+
+    return Array.from(jobs.values()).sort((a, b) => {
+        const aTime = new Date(a.lastUpdatedAt || a.startedAt || '').getTime();
+        const bTime = new Date(b.lastUpdatedAt || b.startedAt || '').getTime();
+        return (Number.isFinite(bTime) ? bTime : 0) - (Number.isFinite(aTime) ? aTime : 0);
+    });
+}
+
+const TERMINAL_REFINE_STATUSES = new Set<MeshAsyncRefineJobStatus>(['completed', 'failed']);
+
+/** Terminal refine jobs older than this (relative to the newest job in the set) are
+ * "stale" — already-resolved historical refinery rejections/successes that should not
+ * keep inflating the status counts in mesh_status. 6h covers a long working session
+ * while still folding multi-day-old residue. */
+export const STALE_TERMINAL_REFINE_WINDOW_MS = 6 * 60 * 60 * 1000;
+
+/** Cap on how many recent terminal jobs are counted even if all fall inside the freshness
+ * window — prevents a burst of refines from dominating the summary. */
+export const RECENT_TERMINAL_REFINE_CAP = 8;
+
+export interface MeshAsyncRefineJobsSummary {
+    /** Count of jobs reflected in `byStatus` (active jobs + recent terminal jobs). */
+    total: number;
+    byStatus: Record<string, number>;
+    /** Terminal jobs dropped from the counts because they are stale residue. */
+    staleTerminal: number;
+    /** Non-terminal (accepted/running) jobs still in flight. */
+    activeJobs: MeshAsyncRefineJobSummary[];
+}
+
+function jobActivityTime(job: MeshAsyncRefineJobSummary): number {
+    const raw = job.lastUpdatedAt || job.completedAt || job.startedAt || '';
+    const t = new Date(raw).getTime();
+    return Number.isFinite(t) ? t : 0;
+}
+
+/**
+ * Build a compact summary of refine jobs that folds stale terminal jobs.
+ *
+ * The full job list (from `buildMeshAsyncRefineJobs`) is derived from a recent ledger
+ * window and deduped by jobId, but it still includes every terminal (completed/failed)
+ * job that happens to fall in that window — including multi-day-old refinery rejections
+ * that have long since been resolved. Those stale terminals inflate `byStatus.failed`
+ * and read as "current breakage" when they are historical noise.
+ *
+ * Active (accepted/running) jobs are always counted. Terminal jobs are counted only when
+ * they are recent: within `STALE_TERMINAL_REFINE_WINDOW_MS` of the newest job's activity
+ * time AND among the `RECENT_TERMINAL_REFINE_CAP` most-recent terminals. Everything else
+ * is folded into `staleTerminal` and excluded from `byStatus`.
+ *
+ * Freshness is measured relative to the newest job in the set (not wall-clock), so the
+ * result is deterministic for a given input — important for tests and for stale-clock
+ * environments.
+ */
+export function summarizeMeshAsyncRefineJobs(
+    jobs: MeshAsyncRefineJobSummary[],
+): MeshAsyncRefineJobsSummary {
+    const activeJobs: MeshAsyncRefineJobSummary[] = [];
+    const terminalJobs: MeshAsyncRefineJobSummary[] = [];
+    for (const job of jobs) {
+        if (TERMINAL_REFINE_STATUSES.has(job.status)) terminalJobs.push(job);
+        else activeJobs.push(job);
+    }
+
+    // Newest activity across ALL jobs anchors the freshness window.
+    let newest = 0;
+    for (const job of jobs) newest = Math.max(newest, jobActivityTime(job));
+    const cutoff = newest - STALE_TERMINAL_REFINE_WINDOW_MS;
+
+    const terminalByRecency = [...terminalJobs].sort(
+        (a, b) => jobActivityTime(b) - jobActivityTime(a),
+    );
+    const freshTerminal = terminalByRecency
+        .filter(job => jobActivityTime(job) >= cutoff)
+        .slice(0, RECENT_TERMINAL_REFINE_CAP);
+    const freshTerminalIds = new Set(freshTerminal.map(job => job.jobId));
+
+    const byStatus: Record<string, number> = {};
+    for (const job of activeJobs) {
+        byStatus[job.status] = (byStatus[job.status] ?? 0) + 1;
+    }
+    for (const job of freshTerminal) {
+        byStatus[job.status] = (byStatus[job.status] ?? 0) + 1;
+    }
+
+    const staleTerminal = terminalJobs.length - freshTerminalIds.size;
+
+    return {
+        total: activeJobs.length + freshTerminal.length,
+        byStatus,
+        staleTerminal,
+        activeJobs,
+    };
+}