@pinet/broker-core 0.2.2 → 0.2.6

package/dist/hibernation-telemetry.js

@@ -0,0 +1,119 @@
+import { sanitizeOperatorReason } from "./hibernation-status.js";
+const ACCEPTED_OUTCOME = "accepted";
+/**
+ * Summarize a window of lifecycle events into an operable status rollup. Pass
+ * the newest events (any order); retention info is optional passthrough.
+ */
+export function summarizeHibernationTelemetry(events, retention) {
+    let hibernations = 0;
+    let wakeSuccesses = 0;
+    let failures = 0;
+    let maxQueueDepth = 0;
+    let maxOldestQueueAgeMs = 0;
+    let recoveredRssBytes = 0;
+    const wakeDurations = [];
+    const refusalCounts = new Map();
+    const agents = new Set();
+    for (const event of events) {
+        agents.add(event.agentId);
+        if (event.queueDepth != null && event.queueDepth > maxQueueDepth) {
+            maxQueueDepth = event.queueDepth;
+        }
+        if (event.oldestQueueAgeMs != null && event.oldestQueueAgeMs > maxOldestQueueAgeMs) {
+            maxOldestQueueAgeMs = event.oldestQueueAgeMs;
+        }
+        if (event.outcome !== ACCEPTED_OUTCOME) {
+            failures += 1;
+            // Defense-in-depth: reasons SHOULD be sanitized at write time, but a
+            // path-bearing reason must never reach this rendered aggregate even if an
+            // upstream writer regresses.
+            const reason = sanitizeOperatorReason(event.errorCode ?? event.reason) ?? "unspecified";
+            refusalCounts.set(reason, (refusalCounts.get(reason) ?? 0) + 1);
+            continue;
+        }
+        if (event.toState === "hibernated") {
+            hibernations += 1;
+            if (event.rssBytesBefore != null && event.rssBytesAfter != null) {
+                recoveredRssBytes += Math.max(event.rssBytesBefore - event.rssBytesAfter, 0);
+            }
+        }
+        if (event.fromState === "waking" && event.toState === "live") {
+            wakeSuccesses += 1;
+            if (event.durationMs != null)
+                wakeDurations.push(event.durationMs);
+        }
+    }
+    wakeDurations.sort((left, right) => left - right);
+    const meanWakeMs = wakeDurations.length === 0
+        ? null
+        : wakeDurations.reduce((sum, value) => sum + value, 0) / wakeDurations.length;
+    // Nearest-rank p95: rank = ceil(0.95 * N), clamped to [1, N].
+    const p95WakeMs = wakeDurations.length === 0
+        ? null
+        : wakeDurations[Math.min(Math.ceil(0.95 * wakeDurations.length), wakeDurations.length) - 1];
+    const refusalReasons = [...refusalCounts.entries()]
+        .map(([reason, count]) => ({ reason, count }))
+        .sort((left, right) => right.count - left.count || left.reason.localeCompare(right.reason));
+    return {
+        totalEvents: events.length,
+        hibernations,
+        wakeSuccesses,
+        failures,
+        meanWakeMs: meanWakeMs === null ? null : Math.round(meanWakeMs * 10) / 10,
+        p95WakeMs,
+        maxQueueDepth,
+        maxOldestQueueAgeMs,
+        recoveredRssBytes,
+        refusalReasons,
+        agentCount: agents.size,
+        retainedCount: retention?.retainedCount ?? null,
+        prunedCount: retention?.prunedCount ?? null,
+        lastPrunedAt: retention?.lastPrunedAt ?? null,
+    };
+}
+/**
+ * Render a compact, human-readable status block from a telemetry summary. Safe
+ * for CLI/Slack output: it contains only aggregate counters, never bodies.
+ */
+export function formatHibernationTelemetry(summary) {
+    if (summary.totalEvents === 0) {
+        return "Hibernation telemetry: no lifecycle events recorded.";
+    }
+    const lines = [];
+    lines.push(`Hibernation telemetry (${summary.totalEvents} events, ${summary.agentCount} agents):`);
+    lines.push(`  hibernations=${summary.hibernations}  wake_successes=${summary.wakeSuccesses}  failures=${summary.failures}`);
+    const wakeParts = [];
+    if (summary.meanWakeMs !== null)
+        wakeParts.push(`mean=${summary.meanWakeMs}ms`);
+    if (summary.p95WakeMs !== null)
+        wakeParts.push(`p95=${summary.p95WakeMs}ms`);
+    if (wakeParts.length > 0)
+        lines.push(`  wake latency: ${wakeParts.join("  ")}`);
+    if (summary.maxQueueDepth > 0 || summary.maxOldestQueueAgeMs > 0) {
+        lines.push(`  queue: max_depth=${summary.maxQueueDepth}  max_oldest_age=${summary.maxOldestQueueAgeMs}ms`);
+    }
+    if (summary.recoveredRssBytes > 0) {
+        const units = ["B", "KiB", "MiB", "GiB", "TiB"];
+        let value = summary.recoveredRssBytes;
+        let unitIndex = 0;
+        while (value >= 1024 && unitIndex < units.length - 1) {
+            value /= 1024;
+            unitIndex += 1;
+        }
+        const rounded = unitIndex === 0 ? Math.round(value) : Math.round(value * 10) / 10;
+        lines.push(`  recovered RSS (est.): ${rounded} ${units[unitIndex]}`);
+    }
+    if (summary.refusalReasons.length > 0) {
+        const top = summary.refusalReasons
+            .slice(0, 5)
+            .map((entry) => `${entry.reason} x${entry.count}`)
+            .join(", ");
+        lines.push(`  refusals: ${top}`);
+    }
+    if (summary.retainedCount !== null) {
+        const pruned = summary.prunedCount ?? 0;
+        const prunedSuffix = summary.lastPrunedAt ? ` (last pruned ${summary.lastPrunedAt})` : "";
+        lines.push(`  retention: retained=${summary.retainedCount}  pruned=${pruned}${prunedSuffix}`);
+    }
+    return lines.join("\n");
+}

package/dist/index.d.ts

@@ -1,6 +1,12 @@
 export * from "./agent-messaging.js";
 export * from "./auth.js";
 export * from "./leader.js";
+export * from "./lifecycle.js";
+export * from "./hibernation-commands.js";
+export * from "./hibernation-orchestrator.js";
+export * from "./hibernation-projection.js";
+export * from "./hibernation-status.js";
+export * from "./hibernation-telemetry.js";
 export * from "./maintenance.js";
 export * from "./mail-classification.js";
 export * from "./message-send.js";

package/dist/index.js

@@ -1,6 +1,12 @@
 export * from "./agent-messaging.js";
 export * from "./auth.js";
 export * from "./leader.js";
+export * from "./lifecycle.js";
+export * from "./hibernation-commands.js";
+export * from "./hibernation-orchestrator.js";
+export * from "./hibernation-projection.js";
+export * from "./hibernation-status.js";
+export * from "./hibernation-telemetry.js";
 export * from "./maintenance.js";
 export * from "./mail-classification.js";
 export * from "./message-send.js";

package/dist/leader.d.ts

@@ -1,19 +1,144 @@
 export declare function defaultLockPath(): string;
 /**
- * Leader election via PID lock file.
+ * Identity of the process recorded in the broker leader lock.
  *
- * Only one broker process should run at a time. The leader writes its
- * PID to the lock file. Stale locks (PID no longer running) are
- * automatically reclaimed.
+ * Legacy locks (written by older builds) contain only a PID; structured locks
+ * additionally record the owner's process start time, a per-acquisition
+ * instance id, hostname, and creation timestamp so a second session can tell
+ * a live owner apart from a reused PID.
+ */
+export interface BrokerLockOwner {
+    pid: number;
+    processStartTime: string | null;
+    instanceId: string | null;
+    hostname: string | null;
+    createdAt: string | null;
+    /** True when the lock file only contained a bare PID (older builds). */
+    legacy: boolean;
+}
+export type BrokerLockInspection = 
+/** No lock file exists. */
+{
+    state: "none";
+    owner: null;
+}
+/** Lock file exists but cannot be parsed — safe to reclaim. */
+ | {
+    state: "unreadable";
+    owner: null;
+}
+/** Recorded PID is no longer running — safe to reclaim. */
+ | {
+    state: "stale-dead";
+    owner: BrokerLockOwner;
+}
+/**
+ * Recorded PID is running but its process start time differs from the one
+ * recorded at lock creation — the PID was reused by an unrelated process,
+ * so the lock is stale and safe to reclaim.
+ */
+ | {
+    state: "stale-pid-reused";
+    owner: BrokerLockOwner;
+    currentStartTime: string;
+}
+/** Recorded PID is running and not provably stale. */
+ | {
+    state: "alive";
+    owner: BrokerLockOwner;
+};
+/** Injectable process probes (for tests). */
+export interface BrokerLockProbes {
+    isProcessRunning?: (pid: number) => boolean;
+    getProcessStartTime?: (pid: number) => string | null;
+}
+/**
+ * Deterministically capture a process's start time for PID-reuse detection.
+ *
+ * Uses `/proc/<pid>/stat` (field 22, clock ticks since boot) on Linux and
+ * `LC_ALL=C ps -p <pid> -o lstart=` elsewhere. Values are only ever compared
+ * for exact equality against a value captured by this same function, so the
+ * format does not need to be parseable — only stable for a given process.
+ *
+ * On Linux the tick count is only unique within a single boot, so it is
+ * scoped with the kernel boot id: a PID reused after a reboot can then never
+ * present the same identity as the pre-reboot owner. `ps lstart` has
+ * one-second resolution, so a PID reused within the same wall-clock second
+ * as the original process is indistinguishable — an accepted residual risk.
+ *
+ * Returns null when the start time cannot be determined; callers must treat
+ * null as "unknown" and never use it as evidence of staleness.
+ */
+export declare function getProcessStartTime(pid: number): string | null;
+/**
+ * Read the current broker lock owner, or null when no lock file exists or it
+ * cannot be parsed.
+ */
+export declare function readBrokerLockOwner(lockPath?: string): BrokerLockOwner | null;
+/**
+ * Inspect the broker leader lock and classify its owner.
+ *
+ * `stale-pid-reused` is only reported when both the recorded and current
+ * process start times are known and differ; unknown start times classify as
+ * `alive` so uncertainty never reclaims a live broker's lock.
+ */
+export declare function inspectBrokerLock(lockPath?: string, probes?: BrokerLockProbes): BrokerLockInspection;
+/**
+ * Leader election via lock file.
+ *
+ * Only one broker process should run at a time. The leader creates the lock
+ * file with an exclusive create (`O_CREAT | O_EXCL`), writing its PID on the
+ * first line (kept legacy-compatible so older builds still see a live owner)
+ * followed by a JSON metadata line recording process start time and a
+ * per-acquisition instance id.
+ *
+ * Exclusive creation is the only way the lock comes into existence, so
+ * simultaneous contenders on an empty path get exactly one winner from the
+ * kernel. Stale locks (dead PID, reused PID, unreadable content) are only
+ * ever unlinked while holding an exclusively-created reclaim mutex file,
+ * with staleness re-verified under that mutex — so a fresh lock can never be
+ * destroyed by a concurrent reclaimer, and the lock path never goes empty
+ * while a live owner's lock exists.
+ *
+ * Known mixed-version limitation: builds that predate the structured format
+ * replace a lock they consider stale with a plain rename over the lock path,
+ * which can overwrite a just-acquired v2 lock when an old and a new build
+ * race over the same stale lock. Exclusive acquisition is therefore only
+ * guaranteed among processes running this code; the window disappears once
+ * no pre-v2 sessions remain. A representation old builds cannot overwrite
+ * (such as a lock directory) would also break their ability to read the
+ * owner PID, which this format deliberately preserves.
  */
 export declare class LeaderLock {
     private readonly lockPath;
+    private readonly probes;
     private acquired;
-    constructor(lockPath?: string);
+    private instanceId;
+    constructor(lockPath?: string, probes?: BrokerLockProbes);
     /**
      * Try to acquire the lock. Returns true if this process is now the leader.
      */
     tryAcquire(): boolean;
+    /**
+     * Create the lock file exclusively. Returns true when this process now
+     * holds the lock; false when another lock file already exists.
+     */
+    private tryExclusiveCreate;
+    /**
+     * Remove a stale lock under an exclusive reclaim mutex.
+     *
+     * The mutex file (`<lockPath>.reclaim`) is created with `O_EXCL` and
+     * records the reclaimer's PID plus start identity, so at most one
+     * reclaimer proceeds at a time, and staleness is re-verified while holding
+     * it. Because stale locks are only ever unlinked under this mutex, the
+     * lock path cannot go empty while a live owner's lock exists — which is
+     * what makes the exclusive create in `tryAcquire` a sound arbiter. A mutex
+     * left behind by a crashed reclaimer (dead PID, or a PID provably reused
+     * by an unrelated process) is itself reclaimed.
+     *
+     * Returns true when the caller may retry an exclusive create.
+     */
+    private reclaimStaleLock;
     /**
      * Release the lock if we hold it.
      */
@@ -26,4 +151,8 @@ export declare class LeaderLock {
      * Get the lock file path (for testing).
      */
     getLockPath(): string;
+    /**
+     * Per-acquisition instance id, set while the lock is held.
+     */
+    getInstanceId(): string | null;
 }