@claude-flow/cli 3.27.4 → 3.28.0

package/dist/src/services/repo-supervisor.js

@@ -0,0 +1,228 @@
+/**
+ * #2661 root-fix — one repository-level supervisor.
+ *
+ * Ten worktree daemons of the same repository independently deciding "is it
+ * time to run audit/optimize/testgaps" every tick is the redundant-scheduler
+ * half of the original cardinality bug — the budget ledger and job dedup
+ * (global-ai-budget.ts, ai-job-dedup.ts) already bound and dedupe the actual
+ * `claude --print` spend, but every daemon still runs its own timer and its
+ * own "should I launch" decision. This module elects exactly ONE daemon per
+ * repositoryId to own the recurring AI-worker schedule; every other worktree
+ * of that repository stays a lease-only participant (workspace-lease.ts) —
+ * it keeps running its cheap, $0 local-only workers (map/consolidate/backup)
+ * on its own schedule, but does not attempt headless (`claude --print`)
+ * execution for its recurring ticks. An explicit `daemon trigger --headless`
+ * in a non-supervisor worktree is still honored (still budget/dedup-gated) —
+ * this module only governs the unattended recurring schedule.
+ *
+ * Election is a simple lock-protected takeover, same pattern as
+ * global-ai-budget.ts:
+ *   - No record, a dead PID, or a stale heartbeat (>SUPERVISOR_STALE_MS) →
+ *     the calling daemon takes over.
+ *   - A live supervisor with a fresh heartbeat → the calling daemon is not
+ *     elected; it does nothing (never overwrites a healthy supervisor).
+ *
+ * The registry lives under the user's home directory so every worktree's
+ * daemon can see it:
+ *
+ *   ~/.claude-flow/supervisors/<repositoryId>.json
+ */
+import * as fs from 'fs';
+import { join } from 'path';
+import { homedir } from 'os';
+// Longer than any plausible daemon heartbeat interval (daemons renew via the
+// existing 60s lifecycle-monitor tick) so a merely-slow tick never triggers
+// a false takeover, but short enough that a crashed supervisor's worktree
+// yields the schedule within a few minutes rather than indefinitely.
+export const SUPERVISOR_STALE_MS = 3 * 60 * 1000;
+const LOCK_STALE_MS = 10_000;
+function delay(ms) {
+    return new Promise((r) => setTimeout(r, ms));
+}
+function isProcessAlive(pid) {
+    try {
+        process.kill(pid, 0);
+        return true;
+    }
+    catch {
+        return false;
+    }
+}
+/** Invariant 9 (#2661): registry files must never be symlinks. */
+function assertNotSymlink(path) {
+    try {
+        const st = fs.lstatSync(path);
+        if (st.isSymbolicLink()) {
+            throw new Error(`Repo-supervisor file is a symlink (refusing): ${path}`);
+        }
+    }
+    catch (e) {
+        if (e.code === 'ENOENT')
+            return;
+        throw e;
+    }
+}
+export class RepoSupervisorRegistry {
+    dir;
+    constructor(options) {
+        this.dir = options?.baseDir
+            ?? process.env.RUFLO_AI_BUDGET_DIR
+            ?? join(homedir(), '.claude-flow');
+    }
+    fileFor(repositoryId) {
+        return join(this.dir, 'supervisors', `${repositoryId}.json`);
+    }
+    ensureDir() {
+        const dir = join(this.dir, 'supervisors');
+        if (!fs.existsSync(dir))
+            fs.mkdirSync(dir, { recursive: true, mode: 0o700 });
+    }
+    async withLock(repositoryId, fn) {
+        this.ensureDir();
+        const lockFile = `${this.fileFor(repositoryId)}.lock`;
+        const deadline = Date.now() + 2000;
+        for (;;) {
+            try {
+                const fd = fs.openSync(lockFile, fs.constants.O_CREAT | fs.constants.O_EXCL | fs.constants.O_WRONLY, 0o600);
+                fs.writeSync(fd, String(process.pid));
+                fs.closeSync(fd);
+                try {
+                    return fn();
+                }
+                finally {
+                    try {
+                        fs.unlinkSync(lockFile);
+                    }
+                    catch { /* already gone */ }
+                }
+            }
+            catch (e) {
+                if (e.code !== 'EEXIST')
+                    throw e;
+                try {
+                    const st = fs.lstatSync(lockFile);
+                    if (Date.now() - st.mtimeMs > LOCK_STALE_MS) {
+                        fs.unlinkSync(lockFile);
+                        continue;
+                    }
+                }
+                catch { /* raced — retry */ }
+                if (Date.now() > deadline)
+                    throw new Error('timed out acquiring repo-supervisor lock');
+                await delay(25);
+            }
+        }
+    }
+    readRecord(repositoryId) {
+        const file = this.fileFor(repositoryId);
+        assertNotSymlink(file);
+        if (!fs.existsSync(file))
+            return null;
+        try {
+            const raw = JSON.parse(fs.readFileSync(file, 'utf-8'));
+            if (raw && typeof raw.pid === 'number' && typeof raw.lastHeartbeat === 'number') {
+                return raw;
+            }
+        }
+        catch { /* corrupt — treat as absent */ }
+        return null;
+    }
+    writeRecord(repositoryId, record) {
+        const file = this.fileFor(repositoryId);
+        assertNotSymlink(file);
+        const tmp = `${file}.tmp.${process.pid}`;
+        fs.writeFileSync(tmp, JSON.stringify(record), { mode: 0o600 });
+        fs.renameSync(tmp, file);
+    }
+    isStale(record, now) {
+        return now - record.lastHeartbeat > SUPERVISOR_STALE_MS || !isProcessAlive(record.pid);
+    }
+    /**
+     * Attempt election (or renewal, if this process already holds it). Safe
+     * and cheap to call on every daemon tick — a no-op write when nothing
+     * needs to change would still cost a lock+read+write, so callers should
+     * still throttle to their own heartbeat cadence rather than calling this
+     * per-worker-tick.
+     */
+    async electOrRenew(repositoryId, worktreeRoot) {
+        try {
+            return await this.withLock(repositoryId, () => {
+                const now = Date.now();
+                const existing = this.readRecord(repositoryId);
+                if (existing && existing.pid === process.pid && existing.worktreeRoot === worktreeRoot) {
+                    const renewed = { ...existing, lastHeartbeat: now };
+                    this.writeRecord(repositoryId, renewed);
+                    return { isSupervisor: true, record: renewed };
+                }
+                if (!existing || this.isStale(existing, now)) {
+                    const record = { worktreeRoot, pid: process.pid, electedAt: now, lastHeartbeat: now };
+                    this.writeRecord(repositoryId, record);
+                    return { isSupervisor: true, record };
+                }
+                // A live supervisor already owns this repository — never overwrite it.
+                return { isSupervisor: false, record: existing };
+            });
+        }
+        catch {
+            // Fail closed on the SCHEDULE side too: if we can't safely coordinate,
+            // assume we are NOT the supervisor rather than risk two daemons both
+            // believing they own the schedule. The budget ledger is the actual
+            // hard invariant either way.
+            return { isSupervisor: false, record: null };
+        }
+    }
+    /** Cheap read-only check — does NOT renew or elect. */
+    isSupervisor(repositoryId, worktreeRoot) {
+        try {
+            const existing = this.readRecord(repositoryId);
+            if (!existing)
+                return false;
+            const now = Date.now();
+            if (this.isStale(existing, now))
+                return false;
+            return existing.pid === process.pid && existing.worktreeRoot === worktreeRoot;
+        }
+        catch {
+            return false;
+        }
+    }
+    /** Release supervisor status on graceful shutdown, so the next tick elsewhere can take over promptly. Best-effort. */
+    async release(repositoryId, worktreeRoot) {
+        try {
+            await this.withLock(repositoryId, () => {
+                const existing = this.readRecord(repositoryId);
+                if (existing && existing.pid === process.pid && existing.worktreeRoot === worktreeRoot) {
+                    const file = this.fileFor(repositoryId);
+                    try {
+                        fs.unlinkSync(file);
+                    }
+                    catch { /* already gone */ }
+                }
+            });
+        }
+        catch { /* best-effort */ }
+    }
+    /** Snapshot for `daemon status --all`. Read-only, never mutates. */
+    getRecord(repositoryId) {
+        try {
+            const existing = this.readRecord(repositoryId);
+            if (!existing || this.isStale(existing, Date.now()))
+                return null;
+            return existing;
+        }
+        catch {
+            return null;
+        }
+    }
+}
+let registryInstance = null;
+export function getRepoSupervisorRegistry() {
+    if (!registryInstance)
+        registryInstance = new RepoSupervisorRegistry();
+    return registryInstance;
+}
+/** Test hook: reset the singleton (e.g. after changing RUFLO_AI_BUDGET_DIR). */
+export function resetRepoSupervisorRegistryForTests() {
+    registryInstance = null;
+}
+//# sourceMappingURL=repo-supervisor.js.map

package/dist/src/services/worker-daemon.d.ts

@@ -12,6 +12,7 @@
  */
 import { EventEmitter } from 'events';
 import { HeadlessWorkerExecutor } from './headless-worker-executor.js';
+import { type SupervisorRecord } from './repo-supervisor.js';
 export type WorkerType = 'ultralearn' | 'optimize' | 'consolidate' | 'predict' | 'audit' | 'map' | 'preload' | 'deepdive' | 'document' | 'refactor' | 'benchmark' | 'testgaps' | 'backup' | 'harness';
 interface WorkerConfig {
     type: WorkerType;
@@ -45,6 +46,12 @@ interface DaemonStatus {
     startedAt?: Date;
     workers: Map<WorkerType, WorkerState>;
     config: DaemonConfig;
+    supervisor?: {
+        repositoryId: string;
+        isSupervisor: boolean;
+        record: SupervisorRecord | null;
+        activeLeases: number;
+    } | null;
 }
 export interface DaemonConfig {
     autoStart: boolean;
@@ -79,6 +86,8 @@ export declare class WorkerDaemon extends EventEmitter {
     private headlessAvailable;
     private headlessInitPromise;
     private originalConfig?;
+    private gitIdentity;
+    private lastSupervisorRenewalMs;
     constructor(projectRoot: string, config?: Partial<DaemonConfig>);
     /**
      * Initialize headless executor if Claude Code is available
@@ -264,6 +273,23 @@ export declare class WorkerDaemon extends EventEmitter {
      * service timers — leaving a zombie that reports stopped but never exits.
      */
     private selfShutdown;
+    /**
+     * #2661 root-fix — heartbeat this worktree's lease and attempt/renew
+     * repository-supervisor election. Best-effort: any failure here must
+     * never affect worker scheduling correctness (the budget ledger remains
+     * the hard invariant regardless of supervisor state) — a daemon that
+     * can't safely coordinate simply falls back to "not supervisor" and runs
+     * its cheap local-only workers, same as any other lease-only worktree.
+     */
+    private renewLeaseAndSupervisor;
+    /**
+     * Cheap read-only check: does THIS daemon currently hold repository
+     * supervisor status? Used to gate recurring headless (`claude --print`)
+     * execution — see repo-supervisor.ts's module doc comment for the full
+     * rationale. Never true when AI workers are disabled or the git identity
+     * couldn't be resolved (e.g. a non-git directory).
+     */
+    private isRepositorySupervisor;
     /**
      * Get daemon status
      */

package/dist/src/services/worker-daemon.js

@@ -22,6 +22,9 @@ import { HeadlessWorkerExecutor, isHeadlessWorker, } from './headless-worker-exe
 // quick_check-gated, so it's safe to call unconditionally on every tick.
 import { runDistillation, defaultMemoryDbPath } from './memory-distillation.js';
 import { backupMemoryDb } from './memory-backup.js';
+import { resolveGitWorkspaceIdentity } from './git-workspace-identity.js';
+import { getWorkspaceLeaseRegistry } from './workspace-lease.js';
+import { getRepoSupervisorRegistry } from './repo-supervisor.js';
 // Default worker configurations with improved intervals (P0 fix: map 5min -> 15min)
 const DEFAULT_WORKERS = [
     { type: 'map', intervalMs: 15 * 60 * 1000, offsetMs: 0, priority: 'normal', description: 'Codebase mapping', enabled: true },
@@ -112,6 +115,11 @@ export class WorkerDaemon extends EventEmitter {
     // Preserve the original constructor config so we can detect explicit overrides
     // during state restoration (R1: constructor config takes priority over stale state)
     originalConfig;
+    // #2661 root-fix — resolved once (git identity doesn't change at runtime)
+    // and reused for lease heartbeats + supervisor election, both gated on
+    // aiWorkersEnabled since they only matter for the recurring AI schedule.
+    gitIdentity = null;
+    lastSupervisorRenewalMs = 0;
     constructor(projectRoot, config) {
         super();
         this.projectRoot = projectRoot;
@@ -739,6 +747,14 @@ export class WorkerDaemon extends EventEmitter {
         this.startedAt = new Date();
         this.writePidFile();
         this.emit('started', { pid: process.pid, startedAt: this.startedAt });
+        // #2661 root-fix — resolve repository identity and register/attempt
+        // election immediately at start, not just on the first 60s lifecycle
+        // tick, so `daemon status` reflects supervisor state right away. Only
+        // meaningful when AI workers are enabled (see field doc comment).
+        if (this.config.aiWorkersEnabled) {
+            this.gitIdentity = resolveGitWorkspaceIdentity(this.projectRoot);
+            void this.renewLeaseAndSupervisor();
+        }
         // Schedule all enabled workers
         for (const workerConfig of this.config.workers) {
             if (workerConfig.enabled) {
@@ -867,6 +883,19 @@ export class WorkerDaemon extends EventEmitter {
             }
             catch { /* best-effort */ }
         }
+        // #2661 root-fix — release this worktree's lease and, if held,
+        // supervisor status, so a sibling worktree's daemon can take over the
+        // schedule within its next tick instead of waiting out the 3-minute
+        // supervisor staleness window. Best-effort — a graceful release is an
+        // optimization; the staleness timeout is what actually bounds a crash.
+        if (this.config.aiWorkersEnabled && this.gitIdentity) {
+            const { repositoryId, worktreeRoot } = this.gitIdentity;
+            try {
+                await getRepoSupervisorRegistry().release(repositoryId, worktreeRoot);
+                await getWorkspaceLeaseRegistry().release(repositoryId, worktreeRoot);
+            }
+            catch { /* best-effort */ }
+        }
         this.running = false;
         this.removePidFile();
         this.saveState();
@@ -896,6 +925,13 @@ export class WorkerDaemon extends EventEmitter {
             const reason = this.lifecycleShutdownReason(Date.now());
             if (reason) {
                 void this.selfShutdown(reason);
+                return;
+            }
+            // #2661 root-fix — renew this worktree's lease + supervisor status on
+            // the same cadence. Both windows (15min lease TTL, 3min supervisor
+            // staleness) comfortably outlive a single missed 60s tick.
+            if (this.config.aiWorkersEnabled) {
+                void this.renewLeaseAndSupervisor();
             }
         }, CHECK_INTERVAL_MS);
         if (typeof this.lifecycleTimer.unref === 'function') {
@@ -971,16 +1007,58 @@ export class WorkerDaemon extends EventEmitter {
         catch { /* best-effort — we are exiting regardless */ }
         process.exit(0);
     }
+    /**
+     * #2661 root-fix — heartbeat this worktree's lease and attempt/renew
+     * repository-supervisor election. Best-effort: any failure here must
+     * never affect worker scheduling correctness (the budget ledger remains
+     * the hard invariant regardless of supervisor state) — a daemon that
+     * can't safely coordinate simply falls back to "not supervisor" and runs
+     * its cheap local-only workers, same as any other lease-only worktree.
+     */
+    async renewLeaseAndSupervisor() {
+        if (!this.gitIdentity)
+            return;
+        const { repositoryId, worktreeRoot } = this.gitIdentity;
+        try {
+            await getWorkspaceLeaseRegistry().heartbeat(repositoryId, worktreeRoot);
+            await getRepoSupervisorRegistry().electOrRenew(repositoryId, worktreeRoot);
+            this.lastSupervisorRenewalMs = Date.now();
+        }
+        catch { /* best-effort — see docstring */ }
+    }
+    /**
+     * Cheap read-only check: does THIS daemon currently hold repository
+     * supervisor status? Used to gate recurring headless (`claude --print`)
+     * execution — see repo-supervisor.ts's module doc comment for the full
+     * rationale. Never true when AI workers are disabled or the git identity
+     * couldn't be resolved (e.g. a non-git directory).
+     */
+    isRepositorySupervisor() {
+        if (!this.config.aiWorkersEnabled || !this.gitIdentity)
+            return false;
+        return getRepoSupervisorRegistry().isSupervisor(this.gitIdentity.repositoryId, this.gitIdentity.worktreeRoot);
+    }
     /**
      * Get daemon status
      */
     getStatus() {
+        let supervisor = null;
+        if (this.config.aiWorkersEnabled && this.gitIdentity) {
+            const { repositoryId, worktreeRoot } = this.gitIdentity;
+            supervisor = {
+                repositoryId,
+                isSupervisor: getRepoSupervisorRegistry().isSupervisor(repositoryId, worktreeRoot),
+                record: getRepoSupervisorRegistry().getRecord(repositoryId),
+                activeLeases: getWorkspaceLeaseRegistry().listActive(repositoryId).length,
+            };
+        }
         return {
             running: this.running,
             pid: process.pid,
             startedAt: this.startedAt,
             workers: new Map(this.workers),
             config: this.config,
+            supervisor,
         };
     }
     /**
@@ -1038,7 +1116,7 @@ export class WorkerDaemon extends EventEmitter {
     /**
      * Execute a worker with timeout protection
      */
-    async executeWorker(workerConfig) {
+    async executeWorker(workerConfig, opts) {
         const state = this.workers.get(workerConfig.type);
         const workerId = `${workerConfig.type}_${Date.now()}`;
         const startTime = Date.now();
@@ -1052,7 +1130,7 @@ export class WorkerDaemon extends EventEmitter {
         try {
             // Execute worker logic with timeout (P1 fix)
             // Pass cleanup callback to kill orphan child processes on timeout (#1117)
-            const output = await this.runWithTimeout(() => this.runWorkerLogic(workerConfig), this.config.workerTimeoutMs, `Worker ${workerConfig.type} timed out after ${this.config.workerTimeoutMs / 1000}s`, () => {
+            const output = await this.runWithTimeout(() => this.runWorkerLogic(workerConfig, opts), this.config.workerTimeoutMs, `Worker ${workerConfig.type} timed out after ${this.config.workerTimeoutMs / 1000}s`, () => {
                 // On timeout, cancel any headless execution to prevent orphan processes
                 if (this.headlessExecutor) {
                     this.headlessExecutor.cancelAll();
@@ -1148,12 +1226,21 @@ export class WorkerDaemon extends EventEmitter {
     /**
      * Run the actual worker logic
      */
-    async runWorkerLogic(workerConfig) {
+    async runWorkerLogic(workerConfig, opts) {
         // Check if this is a headless worker type and headless execution is available.
         // #2661 — aiWorkersEnabled is re-checked here (not just at init) as
         // defence in depth: no code path may promote a worker to `claude --print`
         // without explicit consent.
-        if (this.config.aiWorkersEnabled && isHeadlessWorker(workerConfig.type) && this.headlessAvailable && this.headlessExecutor) {
+        //
+        // #2661 root-fix — the RECURRING schedule additionally requires this
+        // daemon to be the elected repository supervisor (see repo-supervisor.ts):
+        // ten worktree daemons of one repository must not each independently
+        // decide "is it time to run audit" every tick. An explicit
+        // `daemon trigger --headless` (opts.manualTrigger) is still honored
+        // regardless of supervisor status — it's a one-off, user-initiated
+        // action, still budget/dedup-gated the same as any other launch.
+        const supervisorGateOk = opts?.manualTrigger === true || this.isRepositorySupervisor();
+        if (this.config.aiWorkersEnabled && supervisorGateOk && isHeadlessWorker(workerConfig.type) && this.headlessAvailable && this.headlessExecutor) {
             try {
                 this.log('info', `Running ${workerConfig.type} in headless mode (Claude Code AI)`);
                 const result = await this.headlessExecutor.execute(workerConfig.type);
@@ -1671,7 +1758,10 @@ export class WorkerDaemon extends EventEmitter {
         // use headless correctly. Scheduled fires already wait long enough
         // (timer + offset) that this is a no-op for them.
         await this.headlessInitPromise;
-        return this.executeWorker(workerConfig);
+        // #2661 root-fix — an explicit manual trigger bypasses the repository-
+        // supervisor gate (still budget/dedup-gated) — see runWorkerLogic()'s
+        // doc comment.
+        return this.executeWorker(workerConfig, { manualTrigger: true });
     }
     /**
      * Enable/disable a worker

package/dist/src/services/workspace-lease.d.ts

@@ -0,0 +1,55 @@
+/**
+ * #2661 root-fix — worktree leases.
+ *
+ * Registers which worktrees of a repository are currently "alive" (have a
+ * running daemon actively heartbeating), independent of whether that
+ * worktree's daemon is the elected repository supervisor (see
+ * repo-supervisor.ts). A lease expires after 15 minutes without a heartbeat
+ * — a removed worktree, or a daemon that crashed without a graceful
+ * shutdown, becomes ineligible within one expiry window instead of lingering
+ * forever in the registry.
+ *
+ * The registry lives under the user's home directory (not the workspace) so
+ * it is visible to every worktree's daemon, keyed by repositoryId:
+ *
+ *   ~/.claude-flow/leases/<repositoryId>.json
+ *
+ * This is deliberately a thin, single-purpose registry — repo-supervisor.ts
+ * is the piece that actually elects one process to own the recurring AI
+ * worker schedule; leases only answer "which worktrees are currently live"
+ * for status reporting and future supervisor-dispatched work.
+ */
+export declare const LEASE_TTL_MS: number;
+export interface LeaseRecord {
+    worktreeRoot: string;
+    pid: number;
+    registeredAt: number;
+    lastHeartbeat: number;
+}
+export declare class WorkspaceLeaseRegistry {
+    private readonly dir;
+    constructor(options?: {
+        baseDir?: string;
+    });
+    private fileFor;
+    private ensureDir;
+    private withLock;
+    private readFile;
+    private writeFile;
+    /** Register or renew this process's lease on a worktree. Best-effort. */
+    heartbeat(repositoryId: string, worktreeRoot: string): Promise<void>;
+    /** Release this worktree's lease on graceful shutdown. Best-effort. */
+    release(repositoryId: string, worktreeRoot: string): Promise<void>;
+    /**
+     * Active leases for a repository — expired (>15 min stale) or dead-PID
+     * entries are excluded, not just filtered at read time, so callers never
+     * see a worktree that's actually gone.
+     */
+    listActive(repositoryId: string): LeaseRecord[];
+    /** True when the given worktree currently holds a live (non-expired) lease. */
+    isLeaseActive(repositoryId: string, worktreeRoot: string): boolean;
+}
+export declare function getWorkspaceLeaseRegistry(): WorkspaceLeaseRegistry;
+/** Test hook: reset the singleton (e.g. after changing RUFLO_AI_BUDGET_DIR). */
+export declare function resetWorkspaceLeaseRegistryForTests(): void;
+//# sourceMappingURL=workspace-lease.d.ts.map