@pinet/slack-bridge 0.2.2 → 0.2.6

package/dist/broker/lock-conflict.js

@@ -0,0 +1,409 @@
+import * as net from "node:net";
+import { inspectBrokerLock, } from "./leader.js";
+import { getDefaultSocketPath } from "./paths.js";
+import { RPC_METHOD_NOT_FOUND } from "./types.js";
+// ─── Constants ───────────────────────────────────────────
+export const DEFAULT_PROBE_TIMEOUT_MS = 2000;
+export const DEFAULT_SHUTDOWN_RPC_TIMEOUT_MS = 5000;
+export const DEFAULT_GRACEFUL_WAIT_MS = 8000;
+export const DEFAULT_TERMINATE_WAIT_MS = 8000;
+export const DEFAULT_REPLACE_POLL_INTERVAL_MS = 250;
+class BrokerSocketFailure extends Error {
+    kind;
+    constructor(kind, message) {
+        super(message);
+        this.name = "BrokerSocketFailure";
+        this.kind = kind;
+    }
+}
+/**
+ * Minimal sequential JSON-RPC client used for lock-conflict diagnostics.
+ * Deliberately independent from the full follower client: no reconnects, no
+ * heartbeats — one bounded connection for probe/shutdown calls.
+ */
+class OneShotBrokerRpcClient {
+    socket;
+    buffer = "";
+    nextId = 1;
+    waiter = null;
+    constructor(socket) {
+        this.socket = socket;
+        socket.on("data", (chunk) => {
+            this.buffer += chunk.toString("utf-8");
+            let newlineIndex = this.buffer.indexOf("\n");
+            while (newlineIndex !== -1) {
+                const line = this.buffer.slice(0, newlineIndex).trim();
+                this.buffer = this.buffer.slice(newlineIndex + 1);
+                if (line && this.waiter) {
+                    let raw = null;
+                    try {
+                        const parsed = JSON.parse(line);
+                        if (parsed && typeof parsed === "object" && !Array.isArray(parsed)) {
+                            raw = parsed;
+                        }
+                    }
+                    catch {
+                        raw = null;
+                    }
+                    if (raw && raw.id === this.waiter.id) {
+                        const rawError = raw.error;
+                        const response = {
+                            error: rawError && typeof rawError === "object" && typeof rawError.code === "number"
+                                ? {
+                                    code: rawError.code,
+                                    message: typeof rawError.message === "string" ? rawError.message : "",
+                                }
+                                : null,
+                        };
+                        const { resolve } = this.waiter;
+                        this.waiter = null;
+                        resolve(response);
+                    }
+                }
+                newlineIndex = this.buffer.indexOf("\n");
+            }
+        });
+        const failPending = (kind, message) => {
+            if (!this.waiter)
+                return;
+            const { reject } = this.waiter;
+            this.waiter = null;
+            reject(new BrokerSocketFailure(kind, message));
+        };
+        socket.on("error", (err) => failPending("unreachable", err.message));
+        socket.on("close", () => failPending("unresponsive", "Broker socket closed before responding."));
+    }
+    static connect(target, timeoutMs) {
+        return new Promise((resolve, reject) => {
+            const socket = target.type === "unix"
+                ? net.createConnection({ path: target.path })
+                : net.createConnection({ port: target.port, host: target.host });
+            const timer = setTimeout(() => {
+                socket.destroy();
+                reject(new BrokerSocketFailure("unresponsive", `Broker socket connect timed out after ${timeoutMs}ms.`));
+            }, timeoutMs);
+            timer.unref?.();
+            socket.once("connect", () => {
+                clearTimeout(timer);
+                resolve(new OneShotBrokerRpcClient(socket));
+            });
+            socket.once("error", (err) => {
+                clearTimeout(timer);
+                socket.destroy();
+                reject(new BrokerSocketFailure("unreachable", err.message));
+            });
+        });
+    }
+    call(method, params, timeoutMs) {
+        const id = this.nextId++;
+        return new Promise((resolve, reject) => {
+            const timer = setTimeout(() => {
+                if (this.waiter?.id === id)
+                    this.waiter = null;
+                reject(new BrokerSocketFailure("unresponsive", `Broker did not respond to ${method} within ${timeoutMs}ms.`));
+            }, timeoutMs);
+            timer.unref?.();
+            this.waiter = {
+                id,
+                resolve: (response) => {
+                    clearTimeout(timer);
+                    resolve(response);
+                },
+                reject: (error) => {
+                    clearTimeout(timer);
+                    reject(error);
+                },
+            };
+            try {
+                this.socket.write(JSON.stringify({ jsonrpc: "2.0", id, method, params }) + "\n");
+            }
+            catch (err) {
+                clearTimeout(timer);
+                this.waiter = null;
+                reject(new BrokerSocketFailure("unreachable", err instanceof Error ? err.message : String(err)));
+            }
+        });
+    }
+    close() {
+        this.socket.destroy();
+    }
+}
+function resolveTarget(options) {
+    if (options.target)
+        return options.target;
+    return { type: "unix", path: options.socketPath ?? getDefaultSocketPath() };
+}
+function failureToProbeResult(err) {
+    return err instanceof BrokerSocketFailure && err.kind === "unreachable"
+        ? "unreachable"
+        : "unresponsive";
+}
+/**
+ * Bounded liveness probe for a broker socket. Any well-formed JSON-RPC
+ * response — including an auth error — proves the broker event loop is
+ * serving, so it classifies as healthy.
+ */
+export async function probeBrokerSocket(options = {}) {
+    const target = resolveTarget(options);
+    const timeoutMs = options.timeoutMs ?? DEFAULT_PROBE_TIMEOUT_MS;
+    let client;
+    try {
+        client = await OneShotBrokerRpcClient.connect(target, timeoutMs);
+    }
+    catch (err) {
+        return failureToProbeResult(err instanceof Error ? err : new Error(String(err)));
+    }
+    try {
+        await client.call("auth", {}, timeoutMs);
+        return "healthy";
+    }
+    catch (err) {
+        return failureToProbeResult(err instanceof Error ? err : new Error(String(err)));
+    }
+    finally {
+        client.close();
+    }
+}
+/**
+ * Ask a running broker to shut down gracefully via the authenticated
+ * `admin.shutdown` RPC. Brokers that predate the RPC report `unsupported`.
+ */
+export async function requestBrokerShutdown(options = {}) {
+    const target = resolveTarget(options);
+    const timeoutMs = options.timeoutMs ?? DEFAULT_SHUTDOWN_RPC_TIMEOUT_MS;
+    let client;
+    try {
+        client = await OneShotBrokerRpcClient.connect(target, timeoutMs);
+    }
+    catch (err) {
+        return err instanceof BrokerSocketFailure && err.kind === "unreachable"
+            ? "unreachable"
+            : "failed";
+    }
+    try {
+        const secret = options.meshSecret?.trim();
+        const authResponse = await client.call("auth", secret ? { secret } : {}, timeoutMs);
+        if (authResponse.error)
+            return "rejected";
+        const shutdownResponse = await client.call("admin.shutdown", {}, timeoutMs);
+        if (shutdownResponse.error) {
+            return shutdownResponse.error.code === RPC_METHOD_NOT_FOUND ? "unsupported" : "rejected";
+        }
+        return "accepted";
+    }
+    catch {
+        return "failed";
+    }
+    finally {
+        client.close();
+    }
+}
+/**
+ * Combine lock-owner inspection with a bounded socket probe to classify a
+ * leader-lock conflict.
+ */
+export async function classifyBrokerLockConflict(options = {}) {
+    const inspection = inspectBrokerLock(options.lockPath, options.probes ?? {});
+    if (inspection.state !== "alive") {
+        return { kind: "reclaimable", inspection, probe: null };
+    }
+    const target = resolveTarget(options);
+    const timeoutMs = options.probeTimeoutMs ?? DEFAULT_PROBE_TIMEOUT_MS;
+    const probe = options.probeSocket
+        ? await options.probeSocket(target, timeoutMs)
+        : await probeBrokerSocket({ target, timeoutMs });
+    return {
+        kind: "conflict",
+        classification: probe === "healthy" ? "active-broker" : "unresponsive-broker",
+        owner: inspection.owner,
+        probe,
+    };
+}
+export function formatBrokerLockConflictMessage(input) {
+    const owner = input.owner;
+    const ownerBits = owner
+        ? ` (pid ${owner.pid}${owner.createdAt ? `, lock created ${owner.createdAt}` : ""})`
+        : "";
+    if (input.classification === "active-broker") {
+        return (`Another pinet broker is already running${ownerBits} and its socket is responding. ` +
+            `Use /pinet follow to join it, or /pinet start replace to take over.`);
+    }
+    const socketState = input.probe === "unreachable" ? "not reachable" : "not responding";
+    return (`Another process holds the pinet broker lock${ownerBits} but the broker socket is ` +
+        `${socketState} — the broker looks stranded. Use /pinet start replace to recover.`);
+}
+/**
+ * Thrown by `startBroker` when the leader lock is held by another live
+ * process, carrying enough context for callers to present a per-state
+ * recovery path instead of a generic failure.
+ */
+export class BrokerLockConflictError extends Error {
+    classification;
+    owner;
+    probe;
+    constructor(input) {
+        super(formatBrokerLockConflictMessage(input));
+        this.name = "BrokerLockConflictError";
+        this.classification = input.classification;
+        this.owner = input.owner;
+        this.probe = input.probe;
+    }
+}
+/**
+ * Conservatively replace the current broker lock owner:
+ *
+ * 1. No live owner → nothing to do (a normal start reclaims stale state).
+ * 2. Live owner → request graceful `admin.shutdown` and wait (bounded).
+ * 3. Fallback → re-verify the owner fence (pid + process start time +
+ *    instance id), SIGTERM the verified owner, and wait (bounded).
+ * 4. Never SIGKILL; abort if the owner identity changes mid-flight.
+ */
+export async function replaceBrokerOwner(options = {}) {
+    const steps = [];
+    const probes = options.probes ?? {};
+    const inspect = () => inspectBrokerLock(options.lockPath, probes);
+    // Deliberately not unref'd: the poll loop must keep the event loop alive
+    // until replacement resolves (caught by the cross-process E2E run).
+    const sleep = options.sleep ??
+        ((ms) => new Promise((resolve) => setTimeout(resolve, ms)));
+    const kill = options.kill ??
+        ((pid, signal) => {
+            process.kill(pid, signal);
+        });
+    const requestShutdown = options.requestShutdown ?? requestBrokerShutdown;
+    const pollIntervalMs = options.pollIntervalMs ?? DEFAULT_REPLACE_POLL_INTERVAL_MS;
+    const gracefulWaitMs = options.gracefulWaitMs ?? DEFAULT_GRACEFUL_WAIT_MS;
+    const terminateWaitMs = options.terminateWaitMs ?? DEFAULT_TERMINATE_WAIT_MS;
+    const initial = inspect();
+    if (initial.state !== "alive") {
+        steps.push(`Lock state: ${initial.state} — nothing to replace; a normal start reclaims it.`);
+        return { outcome: "no-conflict", owner: initial.owner ?? null, steps, error: null };
+    }
+    const owner = initial.owner;
+    if (owner.pid === process.pid) {
+        // Never signal ourselves: a same-process conflict means this session
+        // already hosts the broker (or another runtime within it does).
+        steps.push(`Lock is held by this session's own process (pid ${owner.pid}).`);
+        return {
+            outcome: "failed",
+            owner,
+            steps,
+            error: "The broker lock is held by this session's own process — nothing to replace. " +
+                "Use /pinet start to reload the current broker runtime instead.",
+        };
+    }
+    steps.push(`Lock held by live pid ${owner.pid}${owner.legacy ? " (legacy lock format — weaker identity fence)" : ""}.`);
+    const sameOwner = (current) => current.state === "alive" &&
+        current.owner.pid === owner.pid &&
+        current.owner.instanceId === owner.instanceId &&
+        current.owner.processStartTime === owner.processStartTime;
+    const ownerChangedResult = () => {
+        steps.push("Lock owner changed during replacement — aborting.");
+        return {
+            outcome: "owner-changed",
+            owner,
+            steps,
+            error: "The broker lock changed owners during replacement — another session may have recovered it. Check /pinet status and retry if needed.",
+        };
+    };
+    const waitForRelease = async (waitMs) => {
+        const deadline = Date.now() + waitMs;
+        for (;;) {
+            const current = inspect();
+            if (current.state !== "alive")
+                return "released";
+            if (!sameOwner(current))
+                return "owner-changed";
+            if (Date.now() >= deadline)
+                return "held";
+            await sleep(pollIntervalMs);
+        }
+    };
+    // ── Step 1: graceful shutdown over the socket ──
+    const shutdownResult = await requestShutdown({
+        ...(options.target ? { target: options.target } : {}),
+        ...(options.socketPath ? { socketPath: options.socketPath } : {}),
+        meshSecret: options.meshSecret ?? null,
+        timeoutMs: options.shutdownRpcTimeoutMs ?? DEFAULT_SHUTDOWN_RPC_TIMEOUT_MS,
+    });
+    steps.push(`Graceful shutdown request: ${shutdownResult}.`);
+    if (shutdownResult === "accepted") {
+        const wait = await waitForRelease(gracefulWaitMs);
+        if (wait === "released") {
+            steps.push("Broker released the lock after graceful shutdown.");
+            return { outcome: "replaced-graceful", owner, steps, error: null };
+        }
+        if (wait === "owner-changed")
+            return ownerChangedResult();
+        steps.push("Broker accepted shutdown but did not release the lock in time.");
+    }
+    if (shutdownResult === "rejected") {
+        // The broker is responsive and refused the request (typically a mesh
+        // secret mismatch). A responsive broker is not stranded — do not
+        // escalate to signals.
+        return {
+            outcome: "failed",
+            owner,
+            steps,
+            error: "The running broker rejected the shutdown request (often a mesh secret mismatch). " +
+                "It is responsive, so it was not terminated. Fix the mesh secret configuration, " +
+                "or stop that broker from its own session, then retry.",
+        };
+    }
+    // ── Step 2: fenced termination fallback ──
+    if (!owner.processStartTime) {
+        // Legacy locks (and structured locks with an unknown start time) carry no
+        // PID-reuse fence, so an automatic SIGTERM could hit an unrelated process
+        // that reused the PID. Require a manual, human-verified step instead.
+        steps.push("Lock owner has no recorded process start identity — refusing automatic termination.");
+        return {
+            outcome: "failed",
+            owner,
+            steps,
+            error: `The lock owner (pid ${owner.pid}) predates identity fencing, so automatic termination ` +
+                `cannot verify it is still the original broker. Inspect it manually (ps -p ${owner.pid}), ` +
+                `terminate it yourself if it is truly the stranded broker, then run /pinet start.`,
+        };
+    }
+    const preTerminate = inspect();
+    if (preTerminate.state !== "alive") {
+        steps.push("Lock was released before termination was needed.");
+        return {
+            outcome: shutdownResult === "accepted" ? "replaced-graceful" : "no-conflict",
+            owner,
+            steps,
+            error: null,
+        };
+    }
+    if (!sameOwner(preTerminate))
+        return ownerChangedResult();
+    // The fence re-check above and the signal below are not atomic: the
+    // verified owner could exit and its PID be reused in between. Closing that
+    // window needs a kernel handle bound to process identity (e.g. Linux
+    // pidfd_send_signal), which Node does not expose without native code — the
+    // immediately-preceding pid + start-time + instanceId fence keeps the
+    // residual window negligible.
+    steps.push(`Sending SIGTERM to verified lock owner pid ${owner.pid}.`);
+    try {
+        kill(owner.pid, "SIGTERM");
+    }
+    catch {
+        // ESRCH etc. — the process may have just exited; the release wait decides.
+        steps.push(`Signal delivery to pid ${owner.pid} failed — it may have already exited.`);
+    }
+    const wait = await waitForRelease(terminateWaitMs);
+    if (wait === "released") {
+        steps.push("Broker released the lock after SIGTERM.");
+        return { outcome: "replaced-terminated", owner, steps, error: null };
+    }
+    if (wait === "owner-changed")
+        return ownerChangedResult();
+    return {
+        outcome: "failed",
+        owner,
+        steps,
+        error: `Broker pid ${owner.pid} is still holding the lock after SIGTERM. ` +
+            `Not escalating to SIGKILL automatically — inspect the process (ps -p ${owner.pid}) ` +
+            `and terminate it manually if it is truly stranded, then run /pinet start.`,
+    };
+}

package/dist/broker/socket-server.d.ts

@@ -47,6 +47,7 @@ export declare class BrokerSocketServer {
     private agentStatusChangeCallback;
     private outboundMessageAdapters;
     private agentRegistrationResolver;
+    private adminShutdownHandler;
     constructor(db: BrokerDB, target?: ListenTarget | string, options?: BrokerSocketServerOptions);
     start(): Promise<void>;
     stop(): Promise<void>;
@@ -75,6 +76,15 @@ export declare class BrokerSocketServer {
      */
     onAgentStatusChange(cb: AgentStatusChangeCallback): void;
     setAgentRegistrationResolver(resolver: AgentRegistrationResolver | null): void;
+    disconnectAgentConnections(agentId: string): void;
+    /**
+     * Register a handler invoked when an authenticated local client requests a
+     * graceful broker shutdown via the `admin.shutdown` RPC (used by
+     * `/pinet start replace` to take over a broker whose controlling session
+     * was lost). When no handler is wired, the RPC reports method-not-found so
+     * callers treat the broker as not supporting remote shutdown.
+     */
+    setAdminShutdownHandler(handler: (() => void | Promise<void>) | null): void;
     setOutboundMessageAdapters(adapters: ReadonlyArray<Pick<MessageAdapter, "name" | "send" | "invokeCapability">>): void;
     private startPruning;
     private stopPruning;
@@ -88,6 +98,14 @@ export declare class BrokerSocketServer {
     private send;
     private handleRequest;
     private handleAuth;
+    /**
+     * Enforce the accepted-generation wake fence for durable hibernation
+     * identities. Returns a rejection response when the fence fails; on success
+     * for a hibernation identity it accepts the generation and binds this
+     * connection to the revived agent id. Ordinary registration returns no
+     * rejection and leaves state untouched.
+     */
+    private enforceWakeFence;
     private handleRegister;
     private handleUnregister;
     private handleHeartbeat;
@@ -114,8 +132,16 @@ export declare class BrokerSocketServer {
     private handlePortLeaseExpire;
     private handleScheduleCreate;
     private handleStatusUpdate;
+    /**
+     * Graceful remote shutdown. Requires authentication (enforced by the
+     * generic auth gate). Responds first, then invokes the wired handler on the
+     * next tick so the requester receives the acknowledgement before the broker
+     * tears down its socket.
+     */
+    private handleAdminShutdown;
     private handleAdapterCapability;
     private handleLegacySlackProxy;
     private dispatchAdapterCapability;
+    private checkAdapterCapabilityThreadOwnership;
     private applyAdapterCapabilityEffects;
 }