@n-dx/core 0.1.0

package/packages/hench/dist/process/index.d.ts

@@ -0,0 +1,22 @@
+/**
+ * Process execution — centralized child-process management.
+ *
+ * @module hench/process
+ */
+export { exec, execStdout, execShellCmd, getCurrentHead, getCurrentBranch, isExecutableOnPath, } from "./exec.js";
+export type { ExecResult, ExecOptions, } from "./exec.js";
+export { execShell } from "./exec-shell.js";
+export type { ExecShellOptions } from "./exec-shell.js";
+export { ProcessLimiter, ProcessLimitReachedError } from "./limiter.js";
+export { ProcessLifecycleValidator, LifecycleAuditTrail, } from "./lifecycle.js";
+export type { LifecycleEvent, TerminationReport, OrphanReport, ResourceSnapshot, ResourceThresholds, ProcessLifecycleValidatorOptions, } from "./lifecycle.js";
+export { MemoryThrottle, MemoryThrottleRejectError, DEFAULT_MEMORY_THROTTLE_CONFIG, } from "./memory-throttle.js";
+export type { MemoryThrottleConfig, MemoryThrottleStatus, ThrottleDecision, SystemMemoryReader, } from "./memory-throttle.js";
+export { SystemMemoryMonitor, DEFAULT_MEMORY_MONITOR_CONFIG, } from "./memory-monitor.js";
+export type { MemoryMonitorConfig, SystemMemorySnapshot, SpawnMemoryCheck, MemoryMonitorOverrides, } from "./memory-monitor.js";
+export { RuntimePool, PoolExhaustedError, DEFAULT_RUNTIME_POOL_CONFIG, } from "./pool.js";
+export type { RuntimePoolConfig, WorkerHandle, WorkerFactory, WorkerState, PooledRuntime, PooledRuntimeInfo, RuntimePoolStatus, } from "./pool.js";
+export { ProcessMemoryTracker, DEFAULT_PROCESS_MEMORY_TRACKER_CONFIG, } from "./process-memory-tracker.js";
+export type { ProcessMemoryTrackerConfig, ProcessMemorySample, MemoryTrend, ProcessMemoryHistory, LeakReport, LeakDetectionSummary, } from "./process-memory-tracker.js";
+export { ConcurrentExecutionMetrics, DEFAULT_CONCURRENT_EXECUTION_METRICS_CONFIG, } from "./concurrent-execution-metrics.js";
+export type { ConcurrentExecutionMetricsConfig, ExecutionMetricsSnapshot, TaskResourceMetrics, UtilizationPatterns, ExecutionMetricsSummary, } from "./concurrent-execution-metrics.js";

package/packages/hench/dist/process/index.js

@@ -0,0 +1,15 @@
+/**
+ * Process execution — centralized child-process management.
+ *
+ * @module hench/process
+ */
+export { exec, execStdout, execShellCmd, getCurrentHead, getCurrentBranch, isExecutableOnPath, } from "./exec.js";
+export { execShell } from "./exec-shell.js";
+export { ProcessLimiter, ProcessLimitReachedError } from "./limiter.js";
+export { ProcessLifecycleValidator, LifecycleAuditTrail, } from "./lifecycle.js";
+export { MemoryThrottle, MemoryThrottleRejectError, DEFAULT_MEMORY_THROTTLE_CONFIG, } from "./memory-throttle.js";
+export { SystemMemoryMonitor, DEFAULT_MEMORY_MONITOR_CONFIG, } from "./memory-monitor.js";
+export { RuntimePool, PoolExhaustedError, DEFAULT_RUNTIME_POOL_CONFIG, } from "./pool.js";
+export { ProcessMemoryTracker, DEFAULT_PROCESS_MEMORY_TRACKER_CONFIG, } from "./process-memory-tracker.js";
+export { ConcurrentExecutionMetrics, DEFAULT_CONCURRENT_EXECUTION_METRICS_CONFIG, } from "./concurrent-execution-metrics.js";
+//# sourceMappingURL=index.js.map

package/packages/hench/dist/process/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/process/index.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EACL,IAAI,EACJ,UAAU,EACV,YAAY,EACZ,cAAc,EACd,gBAAgB,EAChB,kBAAkB,GACnB,MAAM,WAAW,CAAC;AAOnB,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAG5C,OAAO,EAAE,cAAc,EAAE,wBAAwB,EAAE,MAAM,cAAc,CAAC;AAExE,OAAO,EACL,yBAAyB,EACzB,mBAAmB,GACpB,MAAM,gBAAgB,CAAC;AAWxB,OAAO,EACL,cAAc,EACd,yBAAyB,EACzB,8BAA8B,GAC/B,MAAM,sBAAsB,CAAC;AAS9B,OAAO,EACL,mBAAmB,EACnB,6BAA6B,GAC9B,MAAM,qBAAqB,CAAC;AAS7B,OAAO,EACL,WAAW,EACX,kBAAkB,EAClB,2BAA2B,GAC5B,MAAM,WAAW,CAAC;AAYnB,OAAO,EACL,oBAAoB,EACpB,qCAAqC,GACtC,MAAM,6BAA6B,CAAC;AAWrC,OAAO,EACL,0BAA0B,EAC1B,2CAA2C,GAC5C,MAAM,mCAAmC,CAAC"}

package/packages/hench/dist/process/lifecycle.d.ts

@@ -0,0 +1,208 @@
+/**
+ * Process lifecycle validation, orphan detection, and resource monitoring.
+ *
+ * Ensures hench processes properly release resources on completion and
+ * detects resource leaks or orphaned processes that failed to clean up.
+ *
+ * Works alongside {@link ProcessLimiter} — the limiter controls concurrency
+ * at acquire/release time, while the lifecycle validator audits cleanup
+ * after the fact, detects orphans, and monitors resource usage.
+ *
+ * Key capabilities:
+ * - **Termination validation**: verify a process released its lock file
+ * - **Orphan detection**: find dead-PID lock files and stale "running" records
+ * - **Automatic cleanup**: remove orphaned locks and mark stale runs as failed
+ * - **Resource monitoring**: check heap/RSS against configurable thresholds
+ * - **Audit trail**: structured event log for debugging process lifecycle
+ *
+ * @module hench/process/lifecycle
+ */
+/**
+ * A lifecycle event recorded in the audit trail.
+ *
+ * Events track the full process lifecycle: start, lock acquisition,
+ * resource checks, orphan cleanup, termination, and threshold alerts.
+ */
+export interface LifecycleEvent {
+    /** Event type identifier. */
+    type: "process_started" | "process_terminated" | "lock_acquired" | "lock_released" | "resource_check" | "resource_threshold_exceeded" | "orphan_detected" | "orphan_cleaned" | "stale_run_detected" | "stale_run_cleaned";
+    /** PID of the process this event relates to. */
+    pid: number;
+    /** ISO timestamp of when the event occurred. */
+    timestamp: string;
+    /** Optional task ID associated with this process. */
+    taskId?: string;
+    /** Optional structured metadata. */
+    detail?: Record<string, unknown>;
+}
+/**
+ * Result of validating a process's termination cleanup.
+ */
+export interface TerminationReport {
+    /** Whether termination was clean (no orphaned resources). */
+    clean: boolean;
+    /** Whether the process's lock file was properly removed. */
+    lockFileRemoved: boolean;
+    /** List of issues found during validation. */
+    issues: Array<{
+        type: "orphaned_lock" | "stale_run";
+        description: string;
+    }>;
+}
+/**
+ * Result of scanning for orphaned processes and stale runs.
+ */
+export interface OrphanReport {
+    /** Lock files belonging to dead PIDs. */
+    orphanedLocks: Array<{
+        pid: number;
+        taskId?: string;
+        startedAt: string;
+    }>;
+    /** Run records in "running" status with no recent activity. */
+    staleRuns: Array<{
+        id: string;
+        taskId: string;
+        lastActivityAt?: string;
+    }>;
+    /** Number of resources cleaned up (locks removed + runs marked failed). */
+    cleanedUp: number;
+}
+/**
+ * Point-in-time memory snapshot of the current process.
+ */
+export interface ResourceSnapshot {
+    pid: number;
+    heapUsedMB: number;
+    heapTotalMB: number;
+    rssMB: number;
+    externalMB: number;
+    timestamp: string;
+}
+/**
+ * Configurable memory thresholds for resource leak detection.
+ */
+export interface ResourceThresholds {
+    /** Maximum heap used in MB before alerting. */
+    heapUsedMB: number;
+    /** Maximum RSS in MB before alerting. */
+    rssMB: number;
+}
+/**
+ * Records structured lifecycle events for a hench process and persists
+ * them to `.hench/lifecycle/` for post-mortem debugging.
+ *
+ * Events are accumulated in memory during the process run and flushed
+ * to disk on persist(). Each persist creates a new file keyed by the
+ * primary PID in the event list.
+ *
+ * @example
+ * ```ts
+ * const trail = new LifecycleAuditTrail(henchDir);
+ * trail.record({ type: "process_started", pid: process.pid, timestamp: new Date().toISOString() });
+ * // ... work ...
+ * trail.record({ type: "process_terminated", pid: process.pid, timestamp: new Date().toISOString() });
+ * await trail.persist();
+ * ```
+ */
+export declare class LifecycleAuditTrail {
+    private readonly _henchDir;
+    private _events;
+    constructor(henchDir: string);
+    /** Current events (not yet persisted). */
+    get events(): readonly LifecycleEvent[];
+    /** Record a lifecycle event. */
+    record(event: LifecycleEvent): void;
+    /**
+     * Persist accumulated events to disk and clear the buffer.
+     *
+     * Writes to `.hench/lifecycle/{pid}-{timestamp}.json`. No-ops when
+     * there are no events to persist.
+     */
+    persist(): Promise<void>;
+}
+export interface ProcessLifecycleValidatorOptions {
+    /** Memory thresholds for resource leak detection. */
+    thresholds?: ResourceThresholds;
+    /** Duration (ms) after which a "running" record with no activity is stale. */
+    staleRunThresholdMs?: number;
+    /** Audit trail instance for recording lifecycle events. */
+    auditTrail?: LifecycleAuditTrail;
+}
+/**
+ * Validates process lifecycle cleanup, detects orphaned processes,
+ * and monitors resource usage.
+ *
+ * Designed to be instantiated once per `hench run` (or on-demand for
+ * diagnostic scans) and used throughout the process lifecycle.
+ *
+ * @example
+ * ```ts
+ * const validator = new ProcessLifecycleValidator(henchDir, {
+ *   thresholds: { heapUsedMB: 512, rssMB: 1024 },
+ *   staleRunThresholdMs: 120_000,
+ * });
+ *
+ * // Periodic resource check
+ * const alerts = validator.checkResourceThresholds();
+ * if (alerts.length > 0) console.warn("Resource alerts:", alerts);
+ *
+ * // On shutdown: validate cleanup
+ * const report = await validator.validateTermination(process.pid);
+ * if (!report.clean) console.warn("Cleanup issues:", report.issues);
+ *
+ * // Maintenance: detect and clean orphans
+ * const orphans = await validator.cleanupOrphans();
+ * console.log(`Cleaned up ${orphans.cleanedUp} orphaned resources`);
+ * ```
+ */
+export declare class ProcessLifecycleValidator {
+    private readonly _henchDir;
+    private readonly _thresholds;
+    private readonly _staleRunThresholdMs;
+    private readonly _auditTrail;
+    constructor(henchDir: string, options?: ProcessLifecycleValidatorOptions);
+    /**
+     * Validate that a process properly released its resources.
+     *
+     * Checks:
+     * 1. Lock file for the given PID has been removed (or PID is alive)
+     * 2. No orphaned state left behind
+     *
+     * @param pid  The PID to validate termination for.
+     */
+    validateTermination(pid: number): Promise<TerminationReport>;
+    /**
+     * Scan for orphaned processes and stale run records.
+     *
+     * Orphaned locks: lock files whose PIDs are no longer alive.
+     * Stale runs: records in "running" status whose `lastActivityAt` exceeds
+     * the configured stale threshold.
+     *
+     * Does NOT clean up — use {@link cleanupOrphans} for that.
+     */
+    detectOrphans(): Promise<OrphanReport>;
+    /**
+     * Detect and clean up orphaned resources.
+     *
+     * - Removes orphaned lock files
+     * - Marks stale "running" records as "failed" with an orphan error message
+     * - Records audit events for each cleanup action
+     *
+     * @returns Report with details of what was found and cleaned.
+     */
+    cleanupOrphans(): Promise<OrphanReport>;
+    /**
+     * Take a point-in-time memory snapshot of the current process.
+     */
+    takeResourceSnapshot(): ResourceSnapshot;
+    /**
+     * Check current resource usage against configured thresholds.
+     *
+     * @returns Array of human-readable alert strings. Empty if all OK.
+     */
+    checkResourceThresholds(): string[];
+    private _lockFileExists;
+    private _findOrphanedLocks;
+    private _findStaleRuns;
+}

package/packages/hench/dist/process/lifecycle.js

@@ -0,0 +1,387 @@
+/**
+ * Process lifecycle validation, orphan detection, and resource monitoring.
+ *
+ * Ensures hench processes properly release resources on completion and
+ * detects resource leaks or orphaned processes that failed to clean up.
+ *
+ * Works alongside {@link ProcessLimiter} — the limiter controls concurrency
+ * at acquire/release time, while the lifecycle validator audits cleanup
+ * after the fact, detects orphans, and monitors resource usage.
+ *
+ * Key capabilities:
+ * - **Termination validation**: verify a process released its lock file
+ * - **Orphan detection**: find dead-PID lock files and stale "running" records
+ * - **Automatic cleanup**: remove orphaned locks and mark stale runs as failed
+ * - **Resource monitoring**: check heap/RSS against configurable thresholds
+ * - **Audit trail**: structured event log for debugging process lifecycle
+ *
+ * @module hench/process/lifecycle
+ */
+import { readdir, writeFile, unlink, mkdir, readFile } from "node:fs/promises";
+import { join } from "node:path";
+// ---------------------------------------------------------------------------
+// Constants
+// ---------------------------------------------------------------------------
+const LOCKS_DIR = "locks";
+const RUNS_DIR = "runs";
+const LIFECYCLE_DIR = "lifecycle";
+/** Default stale run threshold: 2 minutes without activity. */
+const DEFAULT_STALE_RUN_THRESHOLD_MS = 120_000;
+/** Default resource thresholds (generous — only flag serious leaks). */
+const DEFAULT_THRESHOLDS = {
+    heapUsedMB: 1024,
+    rssMB: 2048,
+};
+// ---------------------------------------------------------------------------
+// PID liveness check (matches limiter.ts implementation)
+// ---------------------------------------------------------------------------
+function isPidAlive(pid) {
+    try {
+        process.kill(pid, 0);
+        return true;
+    }
+    catch {
+        return false;
+    }
+}
+// ---------------------------------------------------------------------------
+// LifecycleAuditTrail
+// ---------------------------------------------------------------------------
+/**
+ * Records structured lifecycle events for a hench process and persists
+ * them to `.hench/lifecycle/` for post-mortem debugging.
+ *
+ * Events are accumulated in memory during the process run and flushed
+ * to disk on persist(). Each persist creates a new file keyed by the
+ * primary PID in the event list.
+ *
+ * @example
+ * ```ts
+ * const trail = new LifecycleAuditTrail(henchDir);
+ * trail.record({ type: "process_started", pid: process.pid, timestamp: new Date().toISOString() });
+ * // ... work ...
+ * trail.record({ type: "process_terminated", pid: process.pid, timestamp: new Date().toISOString() });
+ * await trail.persist();
+ * ```
+ */
+export class LifecycleAuditTrail {
+    _henchDir;
+    _events = [];
+    constructor(henchDir) {
+        this._henchDir = henchDir;
+    }
+    /** Current events (not yet persisted). */
+    get events() {
+        return this._events;
+    }
+    /** Record a lifecycle event. */
+    record(event) {
+        this._events.push(event);
+    }
+    /**
+     * Persist accumulated events to disk and clear the buffer.
+     *
+     * Writes to `.hench/lifecycle/{pid}-{timestamp}.json`. No-ops when
+     * there are no events to persist.
+     */
+    async persist() {
+        if (this._events.length === 0)
+            return;
+        const dir = join(this._henchDir, LIFECYCLE_DIR);
+        await mkdir(dir, { recursive: true });
+        // Use the PID from the first event as the primary identifier
+        const pid = this._events[0].pid;
+        const ts = new Date().toISOString().replace(/[:.]/g, "-");
+        const filename = `${pid}-${ts}.json`;
+        const data = {
+            pid,
+            recordedAt: new Date().toISOString(),
+            events: [...this._events],
+        };
+        await writeFile(join(dir, filename), JSON.stringify(data, null, 2), "utf-8");
+        this._events = [];
+    }
+}
+// ---------------------------------------------------------------------------
+// ProcessLifecycleValidator
+// ---------------------------------------------------------------------------
+/**
+ * Validates process lifecycle cleanup, detects orphaned processes,
+ * and monitors resource usage.
+ *
+ * Designed to be instantiated once per `hench run` (or on-demand for
+ * diagnostic scans) and used throughout the process lifecycle.
+ *
+ * @example
+ * ```ts
+ * const validator = new ProcessLifecycleValidator(henchDir, {
+ *   thresholds: { heapUsedMB: 512, rssMB: 1024 },
+ *   staleRunThresholdMs: 120_000,
+ * });
+ *
+ * // Periodic resource check
+ * const alerts = validator.checkResourceThresholds();
+ * if (alerts.length > 0) console.warn("Resource alerts:", alerts);
+ *
+ * // On shutdown: validate cleanup
+ * const report = await validator.validateTermination(process.pid);
+ * if (!report.clean) console.warn("Cleanup issues:", report.issues);
+ *
+ * // Maintenance: detect and clean orphans
+ * const orphans = await validator.cleanupOrphans();
+ * console.log(`Cleaned up ${orphans.cleanedUp} orphaned resources`);
+ * ```
+ */
+export class ProcessLifecycleValidator {
+    _henchDir;
+    _thresholds;
+    _staleRunThresholdMs;
+    _auditTrail;
+    constructor(henchDir, options) {
+        this._henchDir = henchDir;
+        this._thresholds = options?.thresholds ?? { ...DEFAULT_THRESHOLDS };
+        this._staleRunThresholdMs = options?.staleRunThresholdMs ?? DEFAULT_STALE_RUN_THRESHOLD_MS;
+        this._auditTrail = options?.auditTrail;
+    }
+    // -------------------------------------------------------------------------
+    // Termination validation
+    // -------------------------------------------------------------------------
+    /**
+     * Validate that a process properly released its resources.
+     *
+     * Checks:
+     * 1. Lock file for the given PID has been removed (or PID is alive)
+     * 2. No orphaned state left behind
+     *
+     * @param pid  The PID to validate termination for.
+     */
+    async validateTermination(pid) {
+        const issues = [];
+        // Check lock file
+        const lockExists = await this._lockFileExists(pid);
+        const alive = isPidAlive(pid);
+        if (lockExists && !alive) {
+            issues.push({
+                type: "orphaned_lock",
+                description: `Lock file for dead PID ${pid} still exists`,
+            });
+        }
+        const lockFileRemoved = !lockExists;
+        return {
+            clean: issues.length === 0,
+            lockFileRemoved,
+            issues,
+        };
+    }
+    // -------------------------------------------------------------------------
+    // Orphan detection
+    // -------------------------------------------------------------------------
+    /**
+     * Scan for orphaned processes and stale run records.
+     *
+     * Orphaned locks: lock files whose PIDs are no longer alive.
+     * Stale runs: records in "running" status whose `lastActivityAt` exceeds
+     * the configured stale threshold.
+     *
+     * Does NOT clean up — use {@link cleanupOrphans} for that.
+     */
+    async detectOrphans() {
+        const orphanedLocks = await this._findOrphanedLocks();
+        const staleRuns = await this._findStaleRuns();
+        return {
+            orphanedLocks,
+            staleRuns,
+            cleanedUp: 0,
+        };
+    }
+    /**
+     * Detect and clean up orphaned resources.
+     *
+     * - Removes orphaned lock files
+     * - Marks stale "running" records as "failed" with an orphan error message
+     * - Records audit events for each cleanup action
+     *
+     * @returns Report with details of what was found and cleaned.
+     */
+    async cleanupOrphans() {
+        const orphanedLocks = await this._findOrphanedLocks();
+        const staleRuns = await this._findStaleRuns();
+        let cleanedUp = 0;
+        // Clean orphaned lock files
+        for (const lock of orphanedLocks) {
+            const lockPath = join(this._henchDir, LOCKS_DIR, `${lock.pid}.lock`);
+            try {
+                await unlink(lockPath);
+                cleanedUp++;
+                this._auditTrail?.record({
+                    type: "orphan_cleaned",
+                    pid: lock.pid,
+                    timestamp: new Date().toISOString(),
+                    taskId: lock.taskId,
+                    detail: { resource: "lock_file", path: lockPath },
+                });
+            }
+            catch {
+                // Lock file already gone — fine
+            }
+        }
+        // Mark stale runs as failed
+        for (const staleRun of staleRuns) {
+            try {
+                const runPath = join(this._henchDir, RUNS_DIR, `${staleRun.id}.json`);
+                const raw = await readFile(runPath, "utf-8");
+                const record = JSON.parse(raw);
+                record.status = "failed";
+                record.error = `Process orphaned: no activity since ${staleRun.lastActivityAt ?? "unknown"}`;
+                record.finishedAt = new Date().toISOString();
+                await writeFile(runPath, JSON.stringify(record, null, 2), "utf-8");
+                cleanedUp++;
+                this._auditTrail?.record({
+                    type: "stale_run_cleaned",
+                    pid: 0,
+                    timestamp: new Date().toISOString(),
+                    taskId: staleRun.taskId,
+                    detail: { runId: staleRun.id, lastActivityAt: staleRun.lastActivityAt },
+                });
+            }
+            catch {
+                // Run file read/write failed — skip
+            }
+        }
+        return {
+            orphanedLocks,
+            staleRuns,
+            cleanedUp,
+        };
+    }
+    // -------------------------------------------------------------------------
+    // Resource monitoring
+    // -------------------------------------------------------------------------
+    /**
+     * Take a point-in-time memory snapshot of the current process.
+     */
+    takeResourceSnapshot() {
+        const mem = process.memoryUsage();
+        const toMB = (bytes) => Math.round((bytes / 1024 / 1024) * 100) / 100;
+        return {
+            pid: process.pid,
+            heapUsedMB: toMB(mem.heapUsed),
+            heapTotalMB: toMB(mem.heapTotal),
+            rssMB: toMB(mem.rss),
+            externalMB: toMB(mem.external),
+            timestamp: new Date().toISOString(),
+        };
+    }
+    /**
+     * Check current resource usage against configured thresholds.
+     *
+     * @returns Array of human-readable alert strings. Empty if all OK.
+     */
+    checkResourceThresholds() {
+        const snapshot = this.takeResourceSnapshot();
+        const alerts = [];
+        if (snapshot.heapUsedMB > this._thresholds.heapUsedMB) {
+            alerts.push(`heap used (${snapshot.heapUsedMB}MB) exceeds threshold (${this._thresholds.heapUsedMB}MB)`);
+        }
+        if (snapshot.rssMB > this._thresholds.rssMB) {
+            alerts.push(`RSS (${snapshot.rssMB}MB) exceeds threshold (${this._thresholds.rssMB}MB)`);
+        }
+        if (alerts.length > 0) {
+            this._auditTrail?.record({
+                type: "resource_threshold_exceeded",
+                pid: process.pid,
+                timestamp: snapshot.timestamp,
+                detail: {
+                    heapUsedMB: snapshot.heapUsedMB,
+                    rssMB: snapshot.rssMB,
+                    thresholds: { ...this._thresholds },
+                    alerts,
+                },
+            });
+        }
+        return alerts;
+    }
+    // -------------------------------------------------------------------------
+    // Private helpers
+    // -------------------------------------------------------------------------
+    async _lockFileExists(pid) {
+        try {
+            await readFile(join(this._henchDir, LOCKS_DIR, `${pid}.lock`), "utf-8");
+            return true;
+        }
+        catch {
+            return false;
+        }
+    }
+    async _findOrphanedLocks() {
+        const dir = join(this._henchDir, LOCKS_DIR);
+        let entries;
+        try {
+            entries = await readdir(dir);
+        }
+        catch {
+            return [];
+        }
+        const lockFiles = entries.filter((f) => f.endsWith(".lock"));
+        const orphaned = [];
+        for (const file of lockFiles) {
+            try {
+                const raw = await readFile(join(dir, file), "utf-8");
+                const data = JSON.parse(raw);
+                if (!isPidAlive(data.pid)) {
+                    orphaned.push({
+                        pid: data.pid,
+                        taskId: data.taskId,
+                        startedAt: data.startedAt,
+                    });
+                }
+            }
+            catch {
+                // Corrupted lock file — treat as orphaned with unknown PID
+                const pidMatch = file.match(/^(\d+)\.lock$/);
+                if (pidMatch) {
+                    orphaned.push({
+                        pid: parseInt(pidMatch[1], 10),
+                        startedAt: new Date().toISOString(),
+                    });
+                }
+            }
+        }
+        return orphaned;
+    }
+    async _findStaleRuns() {
+        const dir = join(this._henchDir, RUNS_DIR);
+        let entries;
+        try {
+            entries = await readdir(dir);
+        }
+        catch {
+            return [];
+        }
+        const jsonFiles = entries.filter((f) => f.endsWith(".json"));
+        const stale = [];
+        const now = Date.now();
+        for (const file of jsonFiles) {
+            try {
+                const raw = await readFile(join(dir, file), "utf-8");
+                const record = JSON.parse(raw);
+                if (record.status !== "running")
+                    continue;
+                const lastActivity = record.lastActivityAt ?? record.startedAt;
+                const age = now - new Date(lastActivity).getTime();
+                if (age > this._staleRunThresholdMs) {
+                    stale.push({
+                        id: record.id,
+                        taskId: record.taskId,
+                        lastActivityAt: record.lastActivityAt,
+                    });
+                }
+            }
+            catch {
+                // Skip unreadable run files
+            }
+        }
+        return stale;
+    }
+}
+//# sourceMappingURL=lifecycle.js.map

package/packages/hench/dist/process/lifecycle.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"lifecycle.js","sourceRoot":"","sources":["../../src/process/lifecycle.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;GAkBG;AAEH,OAAO,EAAE,OAAO,EAAE,SAAS,EAAE,MAAM,EAAE,KAAK,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAC/E,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAEjC,8EAA8E;AAC9E,YAAY;AACZ,8EAA8E;AAE9E,MAAM,SAAS,GAAG,OAAO,CAAC;AAC1B,MAAM,QAAQ,GAAG,MAAM,CAAC;AACxB,MAAM,aAAa,GAAG,WAAW,CAAC;AAElC,+DAA+D;AAC/D,MAAM,8BAA8B,GAAG,OAAO,CAAC;AAE/C,wEAAwE;AACxE,MAAM,kBAAkB,GAAuB;IAC7C,UAAU,EAAE,IAAI;IAChB,KAAK,EAAE,IAAI;CACZ,CAAC;AAoFF,8EAA8E;AAC9E,yDAAyD;AACzD,8EAA8E;AAE9E,SAAS,UAAU,CAAC,GAAW;IAC7B,IAAI,CAAC;QACH,OAAO,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC;QACrB,OAAO,IAAI,CAAC;IACd,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC;AAED,8EAA8E;AAC9E,sBAAsB;AACtB,8EAA8E;AAE9E;;;;;;;;;;;;;;;;GAgBG;AACH,MAAM,OAAO,mBAAmB;IACb,SAAS,CAAS;IAC3B,OAAO,GAAqB,EAAE,CAAC;IAEvC,YAAY,QAAgB;QAC1B,IAAI,CAAC,SAAS,GAAG,QAAQ,CAAC;IAC5B,CAAC;IAED,0CAA0C;IAC1C,IAAI,MAAM;QACR,OAAO,IAAI,CAAC,OAAO,CAAC;IACtB,CAAC;IAED,gCAAgC;IAChC,MAAM,CAAC,KAAqB;QAC1B,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAC3B,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,OAAO;QACX,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO;QAEtC,MAAM,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,aAAa,CAAC,CAAC;QAChD,MAAM,KAAK,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAEtC,6DAA6D;QAC7D,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,CAAC,CAAE,CAAC,GAAG,CAAC;QACjC,MAAM,EAAE,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;QAC1D,MAAM,QAAQ,GAAG,GAAG,GAAG,IAAI,EAAE,OAAO,CAAC;QAErC,MAAM,IAAI,GAAG;YACX,GAAG;YACH,UAAU,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;YACpC,MAAM,EAAE,CAAC,GAAG,IAAI,CAAC,OAAO,CAAC;SAC1B,CAAC;QAEF,MAAM,SAAS,CAAC,IAAI,CAAC,GAAG,EAAE,QAAQ,CAAC,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC;QAC7E,IAAI,CAAC,OAAO,GAAG,EAAE,CAAC;IACpB,CAAC;CACF;AAeD,8EAA8E;AAC9E,4BAA4B;AAC5B,8EAA8E;AAE9E;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;AACH,MAAM,OAAO,yBAAyB;IACnB,SAAS,CAAS;IAClB,WAAW,CAAqB;IAChC,oBAAoB,CAAS;IAC7B,WAAW,CAAkC;IAE9D,YAAY,QAAgB,EAAE,OAA0C;QACtE,IAAI,CAAC,SAAS,GAAG,QAAQ,CAAC;QAC1B,IAAI,CAAC,WAAW,GAAG,OAAO,EAAE,UAAU,IAAI,EAAE,GAAG,kBAAkB,EAAE,CAAC;QACpE,IAAI,CAAC,oBAAoB,GAAG,OAAO,EAAE,mBAAmB,IAAI,8BAA8B,CAAC;QAC3F,IAAI,CAAC,WAAW,GAAG,OAAO,EAAE,UAAU,CAAC;IACzC,CAAC;IAED,4EAA4E;IAC5E,yBAAyB;IACzB,4EAA4E;IAE5E;;;;;;;;OAQG;IACH,KAAK,CAAC,mBAAmB,CAAC,GAAW;QACnC,MAAM,MAAM,GAAgC,EAAE,CAAC;QAE/C,kBAAkB;QAClB,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,CAAC;QACnD,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC;QAE9B,IAAI,UAAU,IAAI,CAAC,KAAK,EAAE,CAAC;YACzB,MAAM,CAAC,IAAI,CAAC;gBACV,IAAI,EAAE,eAAe;gBACrB,WAAW,EAAE,0BAA0B,GAAG,eAAe;aAC1D,CAAC,CAAC;QACL,CAAC;QAED,MAAM,eAAe,GAAG,CAAC,UAAU,CAAC;QAEpC,OAAO;YACL,KAAK,EAAE,MAAM,CAAC,MAAM,KAAK,CAAC;YAC1B,eAAe;YACf,MAAM;SACP,CAAC;IACJ,CAAC;IAED,4EAA4E;IAC5E,mBAAmB;IACnB,4EAA4E;IAE5E;;;;;;;;OAQG;IACH,KAAK,CAAC,aAAa;QACjB,MAAM,aAAa,GAAG,MAAM,IAAI,CAAC,kBAAkB,EAAE,CAAC;QACtD,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,cAAc,EAAE,CAAC;QAE9C,OAAO;YACL,aAAa;YACb,SAAS;YACT,SAAS,EAAE,CAAC;SACb,CAAC;IACJ,CAAC;IAED;;;;;;;;OAQG;IACH,KAAK,CAAC,cAAc;QAClB,MAAM,aAAa,GAAG,MAAM,IAAI,CAAC,kBAAkB,EAAE,CAAC;QACtD,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,cAAc,EAAE,CAAC;QAC9C,IAAI,SAAS,GAAG,CAAC,CAAC;QAElB,4BAA4B;QAC5B,KAAK,MAAM,IAAI,IAAI,aAAa,EAAE,CAAC;YACjC,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,SAAS,EAAE,GAAG,IAAI,CAAC,GAAG,OAAO,CAAC,CAAC;YACrE,IAAI,CAAC;gBACH,MAAM,MAAM,CAAC,QAAQ,CAAC,CAAC;gBACvB,SAAS,EAAE,CAAC;gBAEZ,IAAI,CAAC,WAAW,EAAE,MAAM,CAAC;oBACvB,IAAI,EAAE,gBAAgB;oBACtB,GAAG,EAAE,IAAI,CAAC,GAAG;oBACb,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;oBACnC,MAAM,EAAE,IAAI,CAAC,MAAM;oBACnB,MAAM,EAAE,EAAE,QAAQ,EAAE,WAAW,EAAE,IAAI,EAAE,QAAQ,EAAE;iBAClD,CAAC,CAAC;YACL,CAAC;YAAC,MAAM,CAAC;gBACP,gCAAgC;YAClC,CAAC;QACH,CAAC;QAED,4BAA4B;QAC5B,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;YACjC,IAAI,CAAC;gBACH,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,QAAQ,EAAE,GAAG,QAAQ,CAAC,EAAE,OAAO,CAAC,CAAC;gBACtE,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;gBAC7C,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;gBAE/B,MAAM,CAAC,MAAM,GAAG,QAAQ,CAAC;gBACzB,MAAM,CAAC,KAAK,GAAG,uCAAuC,QAAQ,CAAC,cAAc,IAAI,SAAS,EAAE,CAAC;gBAC7F,MAAM,CAAC,UAAU,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;gBAE7C,MAAM,SAAS,CAAC,OAAO,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC;gBACnE,SAAS,EAAE,CAAC;gBAEZ,IAAI,CAAC,WAAW,EAAE,MAAM,CAAC;oBACvB,IAAI,EAAE,mBAAmB;oBACzB,GAAG,EAAE,CAAC;oBACN,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;oBACnC,MAAM,EAAE,QAAQ,CAAC,MAAM;oBACvB,MAAM,EAAE,EAAE,KAAK,EAAE,QAAQ,CAAC,EAAE,EAAE,cAAc,EAAE,QAAQ,CAAC,cAAc,EAAE;iBACxE,CAAC,CAAC;YACL,CAAC;YAAC,MAAM,CAAC;gBACP,oCAAoC;YACtC,CAAC;QACH,CAAC;QAED,OAAO;YACL,aAAa;YACb,SAAS;YACT,SAAS;SACV,CAAC;IACJ,CAAC;IAED,4EAA4E;IAC5E,sBAAsB;IACtB,4EAA4E;IAE5E;;OAEG;IACH,oBAAoB;QAClB,MAAM,GAAG,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC;QAClC,MAAM,IAAI,GAAG,CAAC,KAAa,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,KAAK,GAAG,IAAI,GAAG,IAAI,CAAC,GAAG,GAAG,CAAC,GAAG,GAAG,CAAC;QAE9E,OAAO;YACL,GAAG,EAAE,OAAO,CAAC,GAAG;YAChB,UAAU,EAAE,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC;YAC9B,WAAW,EAAE,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC;YAChC,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC;YACpB,UAAU,EAAE,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC;YAC9B,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;SACpC,CAAC;IACJ,CAAC;IAED;;;;OAIG;IACH,uBAAuB;QACrB,MAAM,QAAQ,GAAG,IAAI,CAAC,oBAAoB,EAAE,CAAC;QAC7C,MAAM,MAAM,GAAa,EAAE,CAAC;QAE5B,IAAI,QAAQ,CAAC,UAAU,GAAG,IAAI,CAAC,WAAW,CAAC,UAAU,EAAE,CAAC;YACtD,MAAM,CAAC,IAAI,CACT,cAAc,QAAQ,CAAC,UAAU,0BAA0B,IAAI,CAAC,WAAW,CAAC,UAAU,KAAK,CAC5F,CAAC;QACJ,CAAC;QAED,IAAI,QAAQ,CAAC,KAAK,GAAG,IAAI,CAAC,WAAW,CAAC,KAAK,EAAE,CAAC;YAC5C,MAAM,CAAC,IAAI,CACT,QAAQ,QAAQ,CAAC,KAAK,0BAA0B,IAAI,CAAC,WAAW,CAAC,KAAK,KAAK,CAC5E,CAAC;QACJ,CAAC;QAED,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACtB,IAAI,CAAC,WAAW,EAAE,MAAM,CAAC;gBACvB,IAAI,EAAE,6BAA6B;gBACnC,GAAG,EAAE,OAAO,CAAC,GAAG;gBAChB,SAAS,EAAE,QAAQ,CAAC,SAAS;gBAC7B,MAAM,EAAE;oBACN,UAAU,EAAE,QAAQ,CAAC,UAAU;oBAC/B,KAAK,EAAE,QAAQ,CAAC,KAAK;oBACrB,UAAU,EAAE,EAAE,GAAG,IAAI,CAAC,WAAW,EAAE;oBACnC,MAAM;iBACP;aACF,CAAC,CAAC;QACL,CAAC;QAED,OAAO,MAAM,CAAC;IAChB,CAAC;IAED,4EAA4E;IAC5E,kBAAkB;IAClB,4EAA4E;IAEpE,KAAK,CAAC,eAAe,CAAC,GAAW;QACvC,IAAI,CAAC;YACH,MAAM,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,SAAS,EAAE,GAAG,GAAG,OAAO,CAAC,EAAE,OAAO,CAAC,CAAC;YACxE,OAAO,IAAI,CAAC;QACd,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,KAAK,CAAC;QACf,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,kBAAkB;QAC9B,MAAM,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,SAAS,CAAC,CAAC;QAC5C,IAAI,OAAiB,CAAC;QACtB,IAAI,CAAC;YACH,OAAO,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,CAAC;QAC/B,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,EAAE,CAAC;QACZ,CAAC;QAED,MAAM,SAAS,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC;QAC7D,MAAM,QAAQ,GAAkC,EAAE,CAAC;QAEnD,KAAK,MAAM,IAAI,IAAI,SAAS,EAAE,CAAC;YAC7B,IAAI,CAAC;gBACH,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,IAAI,CAAC,GAAG,EAAE,IAAI,CAAC,EAAE,OAAO,CAAC,CAAC;gBACrD,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;gBAE7B,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;oBAC1B,QAAQ,CAAC,IAAI,CAAC;wBACZ,GAAG,EAAE,IAAI,CAAC,GAAG;wBACb,MAAM,EAAE,IAAI,CAAC,MAAM;wBACnB,SAAS,EAAE,IAAI,CAAC,SAAS;qBAC1B,CAAC,CAAC;gBACL,CAAC;YACH,CAAC;YAAC,MAAM,CAAC;gBACP,2DAA2D;gBAC3D,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,eAAe,CAAC,CAAC;gBAC7C,IAAI,QAAQ,EAAE,CAAC;oBACb,QAAQ,CAAC,IAAI,CAAC;wBACZ,GAAG,EAAE,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAE,EAAE,EAAE,CAAC;wBAC/B,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;qBACpC,CAAC,CAAC;gBACL,CAAC;YACH,CAAC;QACH,CAAC;QAED,OAAO,QAAQ,CAAC;IAClB,CAAC;IAEO,KAAK,CAAC,cAAc;QAC1B,MAAM,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC;QAC3C,IAAI,OAAiB,CAAC;QACtB,IAAI,CAAC;YACH,OAAO,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,CAAC;QAC/B,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,EAAE,CAAC;QACZ,CAAC;QAED,MAAM,SAAS,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC;QAC7D,MAAM,KAAK,GAA8B,EAAE,CAAC;QAC5C,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAEvB,KAAK,MAAM,IAAI,IAAI,SAAS,EAAE,CAAC;YAC7B,IAAI,CAAC;gBACH,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,IAAI,CAAC,GAAG,EAAE,IAAI,CAAC,EAAE,OAAO,CAAC,CAAC;gBACrD,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;gBAE/B,IAAI,MAAM,CAAC,MAAM,KAAK,SAAS;oBAAE,SAAS;gBAE1C,MAAM,YAAY,GAAG,MAAM,CAAC,cAAc,IAAI,MAAM,CAAC,SAAS,CAAC;gBAC/D,MAAM,GAAG,GAAG,GAAG,GAAG,IAAI,IAAI,CAAC,YAAY,CAAC,CAAC,OAAO,EAAE,CAAC;gBAEnD,IAAI,GAAG,GAAG,IAAI,CAAC,oBAAoB,EAAE,CAAC;oBACpC,KAAK,CAAC,IAAI,CAAC;wBACT,EAAE,EAAE,MAAM,CAAC,EAAE;wBACb,MAAM,EAAE,MAAM,CAAC,MAAM;wBACrB,cAAc,EAAE,MAAM,CAAC,cAAc;qBACtC,CAAC,CAAC;gBACL,CAAC;YACH,CAAC;YAAC,MAAM,CAAC;gBACP,4BAA4B;YAC9B,CAAC;QACH,CAAC;QAED,OAAO,KAAK,CAAC;IACf,CAAC;CACF"}