mono-pilot 0.2.9 → 0.2.12

package/dist/src/cluster_v2/rpc.js

@@ -0,0 +1,389 @@
+import { createClusterLogContext, logClusterEvent, RequestCounters, } from "./observability.js";
+/** Wire protocol for cluster_v2 IPC over Unix domain sockets. */
+export const CLUSTER_V2_PROTOCOL_VERSION = 3;
+export function isPush(msg) {
+    return "type" in msg && msg.type === "push";
+}
+export function isResponse(msg) {
+    return !isPush(msg) && "id" in msg && !("method" in msg);
+}
+// --- Framing ---
+/** Encode as [4-byte little-endian length][utf8 JSON payload]. */
+export function encodeMessage(msg) {
+    const json = Buffer.from(JSON.stringify(msg), "utf8");
+    const header = Buffer.alloc(4);
+    header.writeUInt32LE(json.length, 0);
+    return Buffer.concat([header, json]);
+}
+/** Incremental decoder for framed JSON stream. */
+export class MessageDecoder {
+    buf = Buffer.alloc(0);
+    feed(chunk) {
+        this.buf = Buffer.concat([this.buf, chunk]);
+        const messages = [];
+        while (this.buf.length >= 4) {
+            const len = this.buf.readUInt32LE(0);
+            if (this.buf.length < 4 + len) {
+                break;
+            }
+            const json = this.buf.subarray(4, 4 + len).toString("utf8");
+            this.buf = this.buf.subarray(4 + len);
+            messages.push(JSON.parse(json));
+        }
+        return messages;
+    }
+}
+export class ClusterRpcClient {
+    socket;
+    nextId = 1;
+    pending = new Map();
+    terminalStates = new Map();
+    counters = new RequestCounters();
+    decoder = new MessageDecoder();
+    closed = false;
+    from;
+    logContext;
+    disconnectHandlers = new Set();
+    pushHandlers = new Map();
+    constructor(socket, identity) {
+        this.socket = socket;
+        this.from = {
+            pid: process.pid,
+            agentId: identity?.agentId,
+            sessionId: identity?.sessionId,
+        };
+        this.logContext = createClusterLogContext({
+            agentId: identity?.agentId,
+            sessionId: identity?.sessionId,
+            scope: identity?.scope,
+            role: identity?.role ?? "rpc_client",
+        });
+        socket.on("data", (chunk) => {
+            const messages = this.decoder.feed(chunk);
+            for (const msg of messages) {
+                this.handleIncoming(msg);
+            }
+        });
+        socket.on("error", () => {
+            logClusterEvent("warn", "rpc_socket_error", this.logContext);
+            this.abortAll("socket error");
+        });
+        socket.on("close", () => {
+            logClusterEvent("info", "rpc_socket_closed", this.logContext);
+            this.abortAll("socket closed");
+            for (const handler of this.disconnectHandlers) {
+                handler();
+            }
+        });
+    }
+    onDisconnect(handler) {
+        this.disconnectHandlers.add(handler);
+        return () => {
+            this.disconnectHandlers.delete(handler);
+        };
+    }
+    /**
+     * Register push listener by method. Use "*" to receive all push methods.
+     */
+    onPush(method, handler) {
+        const current = this.pushHandlers.get(method) ?? new Set();
+        current.add(handler);
+        this.pushHandlers.set(method, current);
+        return () => {
+            const set = this.pushHandlers.get(method);
+            if (!set)
+                return;
+            set.delete(handler);
+            if (set.size === 0) {
+                this.pushHandlers.delete(method);
+            }
+        };
+    }
+    call(method, params, options) {
+        if (this.closed) {
+            return Promise.reject(new Error("client closed"));
+        }
+        const id = this.nextId++;
+        const timeoutMs = options?.timeoutMs ?? 30_000;
+        this.counters.start();
+        return new Promise((resolve, reject) => {
+            const timer = setTimeout(() => {
+                this.markTerminal(id, "timeout", method);
+                this.pending.delete(id);
+                this.assertClientCounters("timeout");
+                reject(new Error(`cluster_v2 RPC timeout: ${method}`));
+            }, timeoutMs);
+            let abortCleanup;
+            if (options?.signal) {
+                const onAbort = () => {
+                    this.markTerminal(id, "aborted", method);
+                    this.pending.delete(id);
+                    clearTimeout(timer);
+                    this.assertClientCounters("aborted");
+                    reject(new Error(`cluster_v2 RPC aborted: ${method}`));
+                };
+                if (options.signal.aborted) {
+                    onAbort();
+                    return;
+                }
+                options.signal.addEventListener("abort", onAbort, { once: true });
+                abortCleanup = () => options.signal?.removeEventListener("abort", onAbort);
+            }
+            this.pending.set(id, {
+                resolve: (value) => {
+                    clearTimeout(timer);
+                    abortCleanup?.();
+                    resolve(value);
+                },
+                reject: (err) => {
+                    clearTimeout(timer);
+                    abortCleanup?.();
+                    reject(err);
+                },
+                timer,
+                method,
+                abortCleanup,
+            });
+            const request = {
+                id,
+                version: CLUSTER_V2_PROTOCOL_VERSION,
+                method,
+                params,
+                from: this.from,
+            };
+            this.socket.write(encodeMessage(request));
+            this.assertClientCounters("request_started");
+        });
+    }
+    close() {
+        if (this.closed) {
+            return;
+        }
+        this.closed = true;
+        logClusterEvent("info", "rpc_client_close", this.logContext);
+        this.socket.destroy();
+        this.abortAll("client closed");
+    }
+    handleIncoming(msg) {
+        if (isPush(msg)) {
+            this.dispatchPush(msg.method, msg.payload);
+            return;
+        }
+        const response = msg;
+        const pending = this.pending.get(response.id);
+        if (!pending) {
+            if (!this.terminalStates.has(response.id)) {
+                logClusterEvent("warn", "response_without_pending", this.logContext, {
+                    requestId: response.id,
+                });
+            }
+            return;
+        }
+        this.pending.delete(response.id);
+        if (response.error) {
+            this.markTerminal(response.id, "error", pending.method);
+            this.assertClientCounters("response_error");
+            pending.reject(new Error(response.error));
+            return;
+        }
+        this.markTerminal(response.id, "ok", pending.method);
+        this.assertClientCounters("response_ok");
+        pending.resolve(response.result);
+    }
+    dispatchPush(method, payload) {
+        const direct = this.pushHandlers.get(method);
+        if (direct) {
+            for (const handler of direct) {
+                handler(payload);
+            }
+        }
+        const wildcard = this.pushHandlers.get("*");
+        if (wildcard) {
+            for (const handler of wildcard) {
+                handler(payload);
+            }
+        }
+    }
+    abortAll(reason) {
+        const terminalState = reason.includes("closed") ? "closed" : "error";
+        for (const [id, pending] of this.pending) {
+            this.markTerminal(id, terminalState, pending.method);
+            pending.reject(new Error(reason));
+        }
+        this.pending.clear();
+        this.assertClientCounters("abort_all");
+    }
+    markTerminal(id, state, method) {
+        if (id < 0) {
+            return;
+        }
+        const previous = this.terminalStates.get(id);
+        if (previous && previous !== state) {
+            logClusterEvent("warn", "request_terminalized_twice", this.logContext, {
+                requestId: id,
+                method,
+                fromState: previous,
+                toState: state,
+            });
+            return;
+        }
+        if (!previous) {
+            this.terminalStates.set(id, state);
+            this.counters.complete(state);
+        }
+        if (this.terminalStates.size > 2048) {
+            const oldest = this.terminalStates.keys().next().value;
+            if (typeof oldest === "number") {
+                this.terminalStates.delete(oldest);
+            }
+        }
+    }
+    assertClientCounters(source) {
+        this.counters.assertConsistency(this.pending.size, this.logContext, source);
+    }
+}
+export function bindRpcConnection(socket, handler, options) {
+    const maxInFlight = options?.backpressure?.maxInFlight ?? 64;
+    const maxQueue = options?.backpressure?.maxQueue ?? 256;
+    let inFlight = 0;
+    const queue = [];
+    const serverContext = createClusterLogContext({ role: "rpc_server" });
+    const counters = new RequestCounters();
+    const updateServerContext = (request) => {
+        if (request.from?.agentId) {
+            serverContext.agentId = request.from.agentId;
+        }
+        if (request.from?.sessionId) {
+            serverContext.sessionId = request.from.sessionId;
+        }
+    };
+    const assertRequestAccounting = (source, expectedOutstanding = inFlight + queue.length) => {
+        counters.assertConsistency(expectedOutstanding, serverContext, source);
+    };
+    const assertBackpressureBounds = () => {
+        if (inFlight > maxInFlight || queue.length > maxQueue) {
+            logClusterEvent("warn", "rpc_backpressure_bounds_exceeded", serverContext, {
+                inFlight,
+                maxInFlight,
+                queued: queue.length,
+                maxQueue,
+            });
+        }
+    };
+    const decoder = new MessageDecoder();
+    const state = new Map();
+    const connection = {
+        socket,
+        state,
+        sendPush: (method, payload) => {
+            if (socket.destroyed) {
+                return;
+            }
+            socket.write(encodeMessage({
+                type: "push",
+                method,
+                payload,
+            }));
+        },
+    };
+    const executeRequest = (request) => {
+        updateServerContext(request);
+        inFlight++;
+        assertBackpressureBounds();
+        assertRequestAccounting("request_start");
+        void (async () => {
+            let terminalState = "ok";
+            try {
+                const result = await handler(request, connection);
+                if (!socket.destroyed) {
+                    socket.write(encodeMessage({
+                        id: request.id,
+                        result,
+                    }));
+                }
+            }
+            catch (error) {
+                terminalState = "error";
+                if (!socket.destroyed) {
+                    socket.write(encodeMessage({
+                        id: request.id,
+                        error: error instanceof Error ? error.message : String(error),
+                    }));
+                }
+            }
+            finally {
+                counters.complete(terminalState);
+                inFlight = Math.max(0, inFlight - 1);
+                while (inFlight < maxInFlight && queue.length > 0) {
+                    const queued = queue.shift();
+                    if (!queued) {
+                        break;
+                    }
+                    executeRequest(queued);
+                }
+                assertBackpressureBounds();
+                assertRequestAccounting("request_end");
+            }
+        })();
+    };
+    const enqueueOrExecute = (request) => {
+        updateServerContext(request);
+        counters.start();
+        assertRequestAccounting("request_received", inFlight + queue.length + 1);
+        if (queue.length > 0 || inFlight >= maxInFlight) {
+            if (queue.length >= maxQueue) {
+                counters.complete("error");
+                socket.write(encodeMessage({
+                    id: request.id,
+                    error: `server overloaded: queue limit ${maxQueue} reached`,
+                }));
+                assertRequestAccounting("queue_overload");
+                return;
+            }
+            queue.push(request);
+            assertBackpressureBounds();
+            assertRequestAccounting("request_queued");
+            return;
+        }
+        executeRequest(request);
+    };
+    socket.on("data", (chunk) => {
+        const messages = decoder.feed(chunk);
+        for (const msg of messages) {
+            if (isPush(msg)) {
+                continue;
+            }
+            const request = msg;
+            if (request.version !== CLUSTER_V2_PROTOCOL_VERSION) {
+                const response = {
+                    id: request.id,
+                    error: `unsupported protocol version: ${request.version}`,
+                };
+                socket.write(encodeMessage(response));
+                continue;
+            }
+            enqueueOrExecute(request);
+        }
+    });
+    socket.on("error", (error) => {
+        logClusterEvent("warn", "rpc_server_socket_error", serverContext, {
+            error: error instanceof Error ? error.message : String(error),
+        });
+        options?.onError?.(error instanceof Error ? error : new Error(String(error)));
+    });
+    socket.on("close", () => {
+        if (queue.length > 0) {
+            const dropped = queue.length;
+            for (let i = 0; i < dropped; i++) {
+                counters.complete("closed");
+            }
+            queue.length = 0;
+            logClusterEvent("warn", "rpc_server_queue_dropped_on_close", serverContext, {
+                dropped,
+            });
+        }
+        assertRequestAccounting("socket_close");
+        options?.onClose?.(connection);
+    });
+    return connection;
+}

package/dist/src/cluster_v2/rpc.test.js

@@ -0,0 +1,110 @@
+import net from "node:net";
+import { mkdtemp, rm } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { afterEach, describe, expect, it } from "vitest";
+import { bindRpcConnection, ClusterRpcClient, encodeMessage, MessageDecoder, } from "./rpc.js";
+const activeServers = [];
+const activeClients = [];
+afterEach(async () => {
+    for (const client of activeClients.splice(0)) {
+        client.close();
+    }
+    for (const handle of activeServers.splice(0)) {
+        await new Promise((resolve) => handle.server.close(() => resolve()));
+        await rm(handle.dir, { recursive: true, force: true });
+    }
+});
+async function startRpcServer(handler, options) {
+    const dir = await mkdtemp(join(tmpdir(), "cluster-v2-rpc-test-"));
+    const socketPath = join(dir, "rpc.sock");
+    const server = net.createServer((socket) => {
+        bindRpcConnection(socket, handler, {
+            backpressure: {
+                maxInFlight: options?.maxInFlight,
+                maxQueue: options?.maxQueue,
+            },
+        });
+    });
+    await new Promise((resolve, reject) => {
+        server.once("error", reject);
+        server.listen(socketPath, () => resolve());
+    });
+    const handle = { dir, socketPath, server };
+    activeServers.push(handle);
+    return handle;
+}
+async function connectClient(socketPath) {
+    const socket = await new Promise((resolve, reject) => {
+        const s = net.createConnection(socketPath);
+        s.once("connect", () => resolve(s));
+        s.once("error", reject);
+    });
+    const client = new ClusterRpcClient(socket, {
+        agentId: "rpc-test",
+        sessionId: "session-test",
+        scope: "rpc-test",
+        role: "rpc-test-client",
+    });
+    activeClients.push(client);
+    return client;
+}
+function sleep(ms) {
+    return new Promise((resolve) => setTimeout(resolve, ms));
+}
+describe("cluster_v2 rpc", () => {
+    it("rejects timeout path deterministically", async () => {
+        const server = await startRpcServer(async () => {
+            await new Promise(() => {
+                // never resolves
+            });
+            return null;
+        });
+        const client = await connectClient(server.socketPath);
+        await expect(client.call("hang", {}, { timeoutMs: 40 })).rejects.toThrow("timeout");
+    });
+    it("ignores late duplicate response for same request id", async () => {
+        const dir = await mkdtemp(join(tmpdir(), "cluster-v2-rpc-dup-"));
+        const socketPath = join(dir, "rpc.sock");
+        const server = net.createServer((socket) => {
+            const decoder = new MessageDecoder();
+            socket.on("data", (chunk) => {
+                const messages = decoder.feed(chunk);
+                for (const msg of messages) {
+                    if (!("method" in msg))
+                        continue;
+                    const req = msg;
+                    const first = { id: req.id, result: { value: 1 } };
+                    const duplicate = { id: req.id, result: { value: 2 } };
+                    socket.write(encodeMessage(first));
+                    socket.write(encodeMessage(duplicate));
+                }
+            });
+        });
+        await new Promise((resolve, reject) => {
+            server.once("error", reject);
+            server.listen(socketPath, () => resolve());
+        });
+        activeServers.push({ dir, socketPath, server });
+        const client = await connectClient(socketPath);
+        const result = await client.call("dup", {}, { timeoutMs: 200 });
+        expect(result.value).toBe(1);
+    });
+    it("enforces backpressure bounds and returns overload error", async () => {
+        const server = await startRpcServer(async () => {
+            await sleep(80);
+            return { ok: true };
+        }, { maxInFlight: 1, maxQueue: 1 });
+        const client = await connectClient(server.socketPath);
+        const p1 = client.call("slow", { n: 1 }, { timeoutMs: 500 });
+        const p2 = client.call("slow", { n: 2 }, { timeoutMs: 500 });
+        const p3 = client.call("slow", { n: 3 }, { timeoutMs: 500 });
+        const [r1, r2, r3] = await Promise.allSettled([p1, p2, p3]);
+        expect(r1.status).toBe("fulfilled");
+        expect(r2.status).toBe("fulfilled");
+        expect(r3.status).toBe("rejected");
+        if (r3.status === "rejected") {
+            expect(String(r3.reason?.message ?? r3.reason)).toContain("queue limit 1 reached");
+        }
+    });
+});

package/dist/src/cluster_v2/runtime.failover.integration.test.js

@@ -0,0 +1,156 @@
+import { afterEach, describe, expect, it, vi } from "vitest";
+import { tryListen } from "./connection.js";
+import { bindRpcConnection, } from "./rpc.js";
+import { createEmbeddingHandlers } from "./services/embedding.js";
+import { createBusService } from "./services/bus.js";
+import { ServiceRegistry } from "./services/registry.js";
+import { closeClusterV2, initClusterV2 } from "./runtime.js";
+function createMockEmbeddingProvider(model) {
+    return {
+        id: "local",
+        model,
+        embedQuery: async (text) => [text.length],
+        embedBatch: async (texts) => texts.map((text) => [text.length]),
+        dispose: async () => {
+            // no-op in tests
+        },
+    };
+}
+vi.mock("../memory/embeddings/local.js", () => ({
+    createLocalEmbeddingProvider: async (params) => createMockEmbeddingProvider(params.modelPath ?? "mock-local"),
+}));
+let externalLeader = null;
+afterEach(async () => {
+    if (externalLeader) {
+        await externalLeader.close();
+        externalLeader = null;
+    }
+    await closeClusterV2();
+});
+function uniqueScope(prefix) {
+    return `${prefix}-${process.pid}-${Date.now()}-${Math.floor(Math.random() * 100000)}`;
+}
+async function sleep(ms) {
+    await new Promise((resolve) => setTimeout(resolve, ms));
+}
+async function waitForRole(params, role, timeoutMs) {
+    const deadline = Date.now() + timeoutMs;
+    let last = null;
+    while (Date.now() < deadline) {
+        last = await initClusterV2(params);
+        if (last.role === role) {
+            return last;
+        }
+        await sleep(25);
+    }
+    throw new Error(`timed out waiting for role=${role}, last=${last?.role ?? "none"}`);
+}
+function registerDefaultServices(registry) {
+    const services = [
+        { name: "embedding", version: "v2", capabilities: { methods: ["embedding.embedBatch"] } },
+        {
+            name: "bus",
+            version: "v2",
+            capabilities: {
+                methods: ["bus.register", "bus.subscribe", "bus.send", "bus.broadcast", "bus.roster"],
+            },
+        },
+    ];
+    for (const service of services) {
+        registry.register(service);
+    }
+}
+async function startExternalLeader(scope) {
+    const connection = await tryListen({ scope });
+    if (!connection) {
+        throw new Error("failed to start external leader for test");
+    }
+    const registry = new ServiceRegistry();
+    registerDefaultServices(registry);
+    const busService = createBusService();
+    const provider = createMockEmbeddingProvider("external-leader");
+    const embeddingHandlers = createEmbeddingHandlers(provider);
+    const handlers = {
+        "cluster.ping": async () => "pong",
+        "registry.list": async () => registry.snapshot(),
+        "registry.resolve": async (request) => {
+            const name = request.params?.name;
+            if (!name || typeof name !== "string") {
+                throw new Error("registry.resolve requires string name");
+            }
+            return {
+                revision: registry.getRevision(),
+                service: registry.resolve(name),
+            };
+        },
+        ...embeddingHandlers,
+        ...busService.handlers,
+    };
+    const sockets = new Set();
+    connection.server.on("connection", (socket) => {
+        sockets.add(socket);
+        socket.on("close", () => {
+            sockets.delete(socket);
+        });
+        bindRpcConnection(socket, async (request, rpcConnection) => {
+            const handler = handlers[request.method];
+            if (!handler) {
+                throw new Error(`unknown method: ${request.method}`);
+            }
+            return handler(request, rpcConnection);
+        }, {
+            onClose: (rpcConnection) => {
+                busService.onConnectionClosed(rpcConnection);
+            },
+        });
+    });
+    const bus = busService.createLeaderHandle("leader-harness", "Leader Harness");
+    return {
+        connection,
+        bus,
+        close: async () => {
+            for (const socket of sockets) {
+                socket.destroy();
+            }
+            await connection.close();
+            bus.close();
+        },
+    };
+}
+describe("cluster_v2 runtime failover integration", () => {
+    it("re-elects after leader crash and keeps embedding/bus usable", async () => {
+        const scope = uniqueScope("runtime-failover");
+        externalLeader = await startExternalLeader(scope);
+        const params = {
+            agentId: "follower-a",
+            displayName: "Follower A",
+            scope,
+            modelPath: "mock-local",
+            rpcTimeoutMs: 500,
+            getSessionId: () => "session-a",
+        };
+        const follower = await initClusterV2(params);
+        expect(follower.role).toBe("follower");
+        expect(follower.bus).not.toBeNull();
+        const beforeFailoverEmbedding = await follower.embedding.embedBatch(["x", "hello"]);
+        expect(beforeFailoverEmbedding).toEqual([[1], [5]]);
+        const followerMessages = [];
+        follower.bus?.onMessage((msg) => {
+            if (typeof msg.payload === "object" && msg.payload !== null && "text" in msg.payload) {
+                followerMessages.push(msg.payload);
+            }
+        });
+        await externalLeader.bus.send("follower-a", { text: "before-failover" });
+        await sleep(30);
+        expect(followerMessages.some((m) => m.text === "before-failover")).toBe(true);
+        await externalLeader.close();
+        externalLeader = null;
+        const reElected = await waitForRole(params, "leader", 4000);
+        expect(reElected.role).toBe("leader");
+        expect(reElected.bus).not.toBeNull();
+        const afterFailoverEmbedding = await reElected.embedding.embedQuery("world");
+        expect(afterFailoverEmbedding).toEqual([5]);
+        const broadcastResult = await reElected.bus?.broadcast({ text: "after-failover" }, "public");
+        expect(typeof broadcastResult?.seq).toBe("number");
+    });
+});