mono-pilot 0.2.9 → 0.2.12

package/dist/src/cluster_v2/runtime.js

@@ -0,0 +1,531 @@
+import { createLocalEmbeddingProvider } from "../memory/embeddings/local.js";
+import { tryConnect, tryListen } from "./connection.js";
+import { ConnectionLifecycleTracker } from "./connection-lifecycle.js";
+import { ClusterRpcClient, bindRpcConnection, } from "./rpc.js";
+import { createClusterLogContext, logClusterEvent } from "./observability.js";
+import { emitClusterV2LeaderOffline, emitClusterV2LeaderRecovered } from "./events.js";
+import { createEmbeddingClient, createEmbeddingHandlers } from "./services/embedding.js";
+import { connectBusClient, createBusService } from "./services/bus.js";
+import { maybeStartDiscordCollector } from "./services/discord/index.js";
+import { maybeStartTwitterCollector } from "./services/twitter/index.js";
+import { FollowerRegistryCache } from "./services/registry-cache.js";
+import { ServiceRegistry } from "./services/registry.js";
+let activeService = null;
+let cachedParams = null;
+let reElecting = null;
+let initializingService = null;
+const activeLeaderScopes = new Set();
+const DEFAULT_CLUSTER_V2_EMBEDDING_MAX_CONCURRENT_REQUESTS = 4;
+const DEFAULT_CLUSTER_V2_EMBEDDING_MAX_TEXTS_PER_REQUEST = 4;
+const DEFAULT_CLUSTER_V2_INIT_MAX_ATTEMPTS = 6;
+const DEFAULT_CLUSTER_V2_INIT_RETRY_DELAY_MS = 100;
+function parsePositiveIntegerEnv(name, fallback) {
+    const raw = process.env[name];
+    if (!raw) {
+        return fallback;
+    }
+    const parsed = Number.parseInt(raw, 10);
+    if (!Number.isFinite(parsed) || parsed <= 0) {
+        return fallback;
+    }
+    return parsed;
+}
+function sleep(ms) {
+    return new Promise((resolve) => setTimeout(resolve, ms));
+}
+export function getActiveClusterV2Service() {
+    return activeService;
+}
+function normalizeScope(scope) {
+    const raw = scope?.trim() || "default";
+    return raw.replace(/[^a-zA-Z0-9._-]/g, "_");
+}
+function transitionConnectionLifecycle(tracker, next, label, context) {
+    const previous = tracker.state();
+    tracker.transition(next, label);
+    if (previous !== next) {
+        logClusterEvent("info", "connection_lifecycle_transition", context, {
+            fromState: previous,
+            toState: next,
+            label,
+        });
+    }
+}
+export async function initClusterV2(params) {
+    if (activeService) {
+        return activeService;
+    }
+    if (initializingService) {
+        return initializingService;
+    }
+    initializingService = (async () => {
+        cachedParams = params;
+        const modelId = params.modelPath ?? "local";
+        const sessionId = params.getSessionId?.();
+        const scope = normalizeScope(params.scope);
+        const initLogContext = createClusterLogContext({
+            agentId: params.agentId,
+            sessionId,
+            scope,
+            role: "runtime",
+        });
+        for (let attempt = 1; attempt <= DEFAULT_CLUSTER_V2_INIT_MAX_ATTEMPTS; attempt += 1) {
+            const follower = await tryFollowAsClient(params, modelId, sessionId);
+            if (follower) {
+                activeService = follower;
+                logClusterEvent("info", "runtime_role_selected", initLogContext, { role: follower.role });
+                return follower;
+            }
+            const leader = await tryServeAsLeader(params);
+            if (leader) {
+                activeService = leader;
+                logClusterEvent("info", "runtime_role_selected", initLogContext, { role: leader.role });
+                return leader;
+            }
+            const retryFollower = await tryFollowAsClient(params, modelId, sessionId);
+            if (retryFollower) {
+                activeService = retryFollower;
+                logClusterEvent("info", "runtime_role_selected", initLogContext, { role: retryFollower.role });
+                return retryFollower;
+            }
+            if (attempt < DEFAULT_CLUSTER_V2_INIT_MAX_ATTEMPTS) {
+                logClusterEvent("warn", "runtime_role_resolution_retry", initLogContext, {
+                    attempt,
+                    maxAttempts: DEFAULT_CLUSTER_V2_INIT_MAX_ATTEMPTS,
+                    retryDelayMs: DEFAULT_CLUSTER_V2_INIT_RETRY_DELAY_MS,
+                });
+                await sleep(DEFAULT_CLUSTER_V2_INIT_RETRY_DELAY_MS);
+            }
+        }
+        const standaloneProvider = await createLocalEmbeddingProvider(params);
+        activeService = {
+            role: "standalone",
+            embedding: standaloneProvider,
+            bus: null,
+            async getServiceRegistrySnapshot() {
+                return {
+                    revision: 0,
+                    services: [
+                        {
+                            name: "embedding",
+                            version: "standalone",
+                            capabilities: { methods: ["embedding.embedBatch"] },
+                        },
+                    ],
+                };
+            },
+            async close() {
+                if (standaloneProvider.dispose) {
+                    await standaloneProvider.dispose();
+                }
+                activeService = null;
+            },
+        };
+        logClusterEvent("warn", "runtime_role_selected", initLogContext, { role: "standalone" });
+        return activeService;
+    })();
+    try {
+        return await initializingService;
+    }
+    finally {
+        initializingService = null;
+    }
+}
+export async function closeClusterV2() {
+    if (!activeService) {
+        return;
+    }
+    logClusterEvent("info", "runtime_close", createClusterLogContext({ role: "runtime" }), {
+        role: activeService.role,
+    });
+    await activeService.close();
+    activeService = null;
+}
+export async function reelectClusterV2() {
+    if (!cachedParams) {
+        throw new Error("cluster_v2 is not initialized");
+    }
+    await closeClusterV2();
+    return initClusterV2(cachedParams);
+}
+export async function stepdownClusterV2Leader() {
+    if (!activeService) {
+        throw new Error("cluster_v2 is not active");
+    }
+    if (activeService.role !== "leader") {
+        throw new Error(`stepdown requires leader role, current role: ${activeService.role}`);
+    }
+    if (!cachedParams) {
+        throw new Error("cluster_v2 is missing cached init params");
+    }
+    await closeClusterV2();
+    return initClusterV2(cachedParams);
+}
+async function tryFollowAsClient(params, modelId, sessionId) {
+    const scope = normalizeScope(params.scope);
+    const lifecycle = new ConnectionLifecycleTracker("disconnected");
+    const logContext = createClusterLogContext({
+        agentId: params.agentId,
+        sessionId,
+        scope,
+        role: "follower",
+    });
+    transitionConnectionLifecycle(lifecycle, "connecting", "tryFollowAsClient:start", logContext);
+    const socket = await tryConnect(params.scope);
+    if (!socket) {
+        transitionConnectionLifecycle(lifecycle, "disconnected", "tryFollowAsClient:no_socket", logContext);
+        return null;
+    }
+    transitionConnectionLifecycle(lifecycle, "connected", "tryFollowAsClient:connected", logContext);
+    lifecycle.assertOpenImpliesConnected(!socket.destroyed, "tryFollowAsClient:connected");
+    const client = new ClusterRpcClient(socket, {
+        agentId: params.agentId,
+        sessionId,
+        scope,
+        role: "rpc_client_follower",
+    });
+    try {
+        const pong = await client.call("cluster.ping", null, { timeoutMs: 3000 });
+        if (pong !== "pong") {
+            client.close();
+            transitionConnectionLifecycle(lifecycle, "disconnected", "tryFollowAsClient:ping_mismatch", logContext);
+            return null;
+        }
+    }
+    catch {
+        client.close();
+        transitionConnectionLifecycle(lifecycle, "disconnected", "tryFollowAsClient:ping_failed", logContext);
+        return null;
+    }
+    const registryCache = new FollowerRegistryCache(client, logContext);
+    try {
+        await registryCache.refresh();
+        await registryCache.requireService("embedding");
+    }
+    catch (error) {
+        logClusterEvent("warn", "registry_bootstrap_failed", logContext, {
+            error: error instanceof Error ? error.message : String(error),
+        });
+        registryCache.invalidate("bootstrap_failed");
+        client.close();
+        transitionConnectionLifecycle(lifecycle, "disconnected", "tryFollowAsClient:registry_failed", logContext);
+        return null;
+    }
+    const baseEmbedding = createEmbeddingClient(client, {
+        model: modelId,
+        timeoutMs: params.rpcTimeoutMs,
+    });
+    const embedding = {
+        id: baseEmbedding.id,
+        model: baseEmbedding.model,
+        embedQuery: (text) => withReconnect(baseEmbedding, (p) => p.embedQuery(text)),
+        embedBatch: (texts) => withReconnect(baseEmbedding, (p) => p.embedBatch(texts)),
+        dispose: async () => {
+            client.close();
+        },
+    };
+    let bus = null;
+    try {
+        await registryCache.requireService("bus");
+        bus = await connectBusClient(client, params.agentId, params.displayName);
+    }
+    catch (error) {
+        logClusterEvent("warn", "bus_service_unavailable", logContext, {
+            error: error instanceof Error ? error.message : String(error),
+            cacheRevision: registryCache.currentRevision(),
+        });
+        registryCache.invalidate("bus_unavailable");
+        client.close();
+        transitionConnectionLifecycle(lifecycle, "disconnected", "tryFollowAsClient:bus_unavailable", logContext);
+        return null;
+    }
+    const detachDisconnect = client.onDisconnect(() => {
+        if (reElecting || !activeService || activeService.role !== "follower" || !cachedParams) {
+            return;
+        }
+        registryCache.invalidate("leader_disconnect");
+        emitClusterV2LeaderOffline({
+            agentId: params.agentId,
+            sessionId,
+            scope,
+            reason: "follower_disconnect",
+        });
+        transitionConnectionLifecycle(lifecycle, "reconnecting", "follower_disconnect", logContext);
+        lifecycle.assertOpenImpliesConnected(!socket.destroyed, "follower_disconnect");
+        logClusterEvent("warn", "follower_disconnected_re_elect", logContext);
+        activeService = null;
+        reElecting = initClusterV2(cachedParams)
+            .then((next) => {
+            if (next.role === "leader" || next.role === "follower") {
+                emitClusterV2LeaderRecovered({
+                    agentId: params.agentId,
+                    sessionId,
+                    scope,
+                    role: next.role,
+                });
+            }
+            transitionConnectionLifecycle(lifecycle, "disconnected", "re_elect_complete", logContext);
+            reElecting = null;
+        })
+            .catch(() => {
+            transitionConnectionLifecycle(lifecycle, "disconnected", "re_elect_failed", logContext);
+            reElecting = null;
+        });
+    });
+    return {
+        role: "follower",
+        embedding,
+        bus,
+        async getServiceRegistrySnapshot() {
+            return client.call("registry.list", {});
+        },
+        async close() {
+            detachDisconnect();
+            if (bus) {
+                bus.close();
+            }
+            registryCache.invalidate("follower_close");
+            client.close();
+            transitionConnectionLifecycle(lifecycle, "closed", "follower_close", logContext);
+            lifecycle.assertOpenImpliesConnected(!socket.destroyed, "follower_close");
+            activeService = null;
+        },
+    };
+}
+async function withReconnect(inner, op) {
+    try {
+        return await op(inner);
+    }
+    catch (error) {
+        if (!isConnectionError(error) || !cachedParams) {
+            throw error;
+        }
+        if (reElecting) {
+            await reElecting;
+        }
+        else {
+            activeService = null;
+            await initClusterV2(cachedParams);
+        }
+        if (!activeService) {
+            throw error;
+        }
+        return op(activeService.embedding);
+    }
+}
+async function tryServeAsLeader(params) {
+    const leaderScope = normalizeScope(params.scope);
+    const lifecycle = new ConnectionLifecycleTracker("disconnected");
+    const sessionId = params.getSessionId?.();
+    const logContext = createClusterLogContext({
+        agentId: params.agentId,
+        sessionId,
+        scope: leaderScope,
+        role: "leader",
+    });
+    let leaseCompromised = false;
+    let leaseCompromisedError = null;
+    let handleLeaseCompromised = null;
+    let closingLeader = false;
+    if (activeLeaderScopes.has(leaderScope)) {
+        logClusterEvent("warn", "leader_guard_precheck_failed", logContext, {
+            leaderScope,
+        });
+        throw new Error(`[cluster_v2/runtime] leader guard violated for scope: ${leaderScope}`);
+    }
+    const provider = await createLocalEmbeddingProvider(params);
+    transitionConnectionLifecycle(lifecycle, "connecting", "tryServeAsLeader:start", logContext);
+    const connection = await tryListen({
+        scope: params.scope,
+        onLeaseCompromised: (error) => {
+            leaseCompromised = true;
+            leaseCompromisedError = error;
+            logClusterEvent("warn", "leader_lease_compromised", logContext, {
+                error: error.message,
+                leaderScope,
+            });
+            handleLeaseCompromised?.();
+        },
+    });
+    if (!connection) {
+        transitionConnectionLifecycle(lifecycle, "disconnected", "tryServeAsLeader:listen_failed", logContext);
+        if (provider.dispose) {
+            await provider.dispose();
+        }
+        return null;
+    }
+    transitionConnectionLifecycle(lifecycle, "connected", "tryServeAsLeader:listening", logContext);
+    lifecycle.assertOpenImpliesConnected(connection.server.listening, "tryServeAsLeader:listening");
+    if (leaseCompromised) {
+        await connection.close();
+        if (provider.dispose) {
+            await provider.dispose();
+        }
+        transitionConnectionLifecycle(lifecycle, "closed", "leader_lease_compromised_before_activate", logContext);
+        return null;
+    }
+    if (activeLeaderScopes.has(leaderScope)) {
+        await connection.close();
+        if (provider.dispose) {
+            await provider.dispose();
+        }
+        transitionConnectionLifecycle(lifecycle, "closed", "leader_guard_race_failed", logContext);
+        logClusterEvent("warn", "leader_guard_race_failed", logContext, {
+            leaderScope,
+        });
+        throw new Error(`[cluster_v2/runtime] leader guard violated for scope: ${leaderScope}`);
+    }
+    activeLeaderScopes.add(leaderScope);
+    logClusterEvent("info", "leader_scope_activated", logContext, { leaderScope });
+    const busService = createBusService();
+    const embeddingHandlers = createEmbeddingHandlers(provider, {
+        maxConcurrentRequests: parsePositiveIntegerEnv("MONO_PILOT_CLUSTER_V2_EMBEDDING_MAX_CONCURRENCY", DEFAULT_CLUSTER_V2_EMBEDDING_MAX_CONCURRENT_REQUESTS),
+        maxTextsPerRequest: parsePositiveIntegerEnv("MONO_PILOT_CLUSTER_V2_EMBEDDING_MAX_TEXTS_PER_REQUEST", DEFAULT_CLUSTER_V2_EMBEDDING_MAX_TEXTS_PER_REQUEST),
+    });
+    let discordCollector = null;
+    let twitterCollector = null;
+    try {
+        discordCollector = await maybeStartDiscordCollector(logContext);
+    }
+    catch (error) {
+        logClusterEvent("warn", "discord_collector_start_failed", logContext, {
+            error: error instanceof Error ? error.message : String(error),
+        });
+        discordCollector = null;
+    }
+    try {
+        twitterCollector = await maybeStartTwitterCollector(logContext);
+    }
+    catch (error) {
+        logClusterEvent("warn", "twitter_collector_start_failed", logContext, {
+            error: error instanceof Error ? error.message : String(error),
+        });
+        twitterCollector = null;
+    }
+    const extraServices = [];
+    if (discordCollector) {
+        extraServices.push(discordCollector.descriptor);
+    }
+    if (twitterCollector) {
+        extraServices.push(twitterCollector.descriptor);
+    }
+    const registry = new ServiceRegistry();
+    registerDefaultServices(registry, extraServices);
+    const handlers = {
+        "cluster.ping": async () => "pong",
+        "registry.list": async () => registry.snapshot(),
+        "registry.resolve": async (request) => {
+            const name = request.params?.name;
+            if (!name || typeof name !== "string") {
+                throw new Error("registry.resolve requires string name");
+            }
+            const service = registry.resolve(name);
+            return { revision: registry.getRevision(), service };
+        },
+        ...embeddingHandlers,
+        ...busService.handlers,
+    };
+    connection.server.on("connection", (socket) => {
+        bindRpcConnection(socket, async (request, rpcConnection) => {
+            const handler = handlers[request.method];
+            if (!handler) {
+                throw new Error(`unknown method: ${request.method}`);
+            }
+            return handler(request, rpcConnection);
+        }, {
+            onClose: (rpcConnection) => {
+                busService.onConnectionClosed(rpcConnection);
+            },
+        });
+    });
+    const bus = busService.createLeaderHandle(params.agentId, params.displayName);
+    const closeLeaderResources = async (reason) => {
+        if (closingLeader) {
+            return;
+        }
+        closingLeader = true;
+        activeLeaderScopes.delete(leaderScope);
+        if (discordCollector) {
+            await discordCollector.close();
+            discordCollector = null;
+        }
+        if (twitterCollector) {
+            await twitterCollector.close();
+            twitterCollector = null;
+        }
+        bus.close();
+        await connection.close();
+        transitionConnectionLifecycle(lifecycle, "closed", `leader_close:${reason}`, logContext);
+        lifecycle.assertOpenImpliesConnected(connection.server.listening, `leader_close:${reason}`);
+        logClusterEvent("info", "leader_scope_released", logContext, { leaderScope, reason });
+        if (provider.dispose) {
+            await provider.dispose();
+        }
+        activeService = null;
+    };
+    handleLeaseCompromised = () => {
+        if (closingLeader) {
+            return;
+        }
+        void (async () => {
+            await closeLeaderResources("lease_compromised");
+            if (!cachedParams || reElecting) {
+                return;
+            }
+            reElecting = initClusterV2(cachedParams)
+                .then(() => {
+                reElecting = null;
+            })
+                .catch((error) => {
+                logClusterEvent("warn", "leader_lease_compromise_re_elect_failed", logContext, {
+                    error: error instanceof Error ? error.message : String(error),
+                    leaseError: leaseCompromisedError?.message,
+                });
+                reElecting = null;
+            });
+        })();
+    };
+    if (leaseCompromised) {
+        await closeLeaderResources("lease_compromised_pre_activate");
+        return null;
+    }
+    return {
+        role: "leader",
+        embedding: provider,
+        bus,
+        async getServiceRegistrySnapshot() {
+            return registry.snapshot();
+        },
+        async close() {
+            await closeLeaderResources("explicit_close");
+        },
+    };
+}
+function registerDefaultServices(registry, extras) {
+    const services = [
+        { name: "embedding", version: "v2", capabilities: { methods: ["embedding.embedBatch"] } },
+        {
+            name: "bus",
+            version: "v2",
+            capabilities: {
+                methods: ["bus.register", "bus.subscribe", "bus.send", "bus.broadcast", "bus.roster"],
+            },
+        },
+        ...(extras ?? []),
+    ];
+    for (const service of services) {
+        registry.register(service);
+    }
+}
+function isConnectionError(error) {
+    if (!(error instanceof Error)) {
+        return false;
+    }
+    const message = error.message;
+    return (message.includes("socket closed") ||
+        message.includes("socket error") ||
+        message.includes("client closed") ||
+        message.includes("timeout") ||
+        message.includes("EPIPE") ||
+        message.includes("ECONNRESET"));
+}

package/dist/src/cluster_v2/runtime.lease-compromise.integration.test.js

@@ -0,0 +1,91 @@
+import net from "node:net";
+import { afterEach, describe, expect, it, vi } from "vitest";
+const harness = vi.hoisted(() => ({
+    listenCalls: 0,
+    compromiseTriggered: false,
+}));
+vi.mock("../memory/embeddings/local.js", () => ({
+    createLocalEmbeddingProvider: async (params) => ({
+        id: "local",
+        model: params.modelPath ?? "mock-local",
+        embedQuery: async (text) => [text.length],
+        embedBatch: async (texts) => texts.map((text) => [text.length]),
+        dispose: async () => {
+            // no-op in tests
+        },
+    }),
+}));
+vi.mock("./connection.js", () => ({
+    tryConnect: async () => null,
+    tryListen: async (options) => {
+        harness.listenCalls++;
+        const server = net.createServer();
+        await new Promise((resolve, reject) => {
+            server.once("error", reject);
+            server.listen(0, "127.0.0.1", () => resolve());
+        });
+        let closed = false;
+        const close = async () => {
+            if (closed) {
+                return;
+            }
+            closed = true;
+            await new Promise((resolve) => server.close(() => resolve()));
+        };
+        if (!harness.compromiseTriggered) {
+            harness.compromiseTriggered = true;
+            setTimeout(() => {
+                options?.onLeaseCompromised?.(new Error("lease compromised in test"));
+            }, 15);
+        }
+        return {
+            socketPath: `mock-socket-${harness.listenCalls}`,
+            server,
+            close,
+        };
+    },
+}));
+async function sleep(ms) {
+    await new Promise((resolve) => setTimeout(resolve, ms));
+}
+async function waitFor(condition, timeoutMs) {
+    const deadline = Date.now() + timeoutMs;
+    while (Date.now() < deadline) {
+        if (await condition()) {
+            return;
+        }
+        await sleep(20);
+    }
+    throw new Error("waitFor timeout");
+}
+afterEach(async () => {
+    const runtime = await import("./runtime.js");
+    await runtime.closeClusterV2();
+    harness.listenCalls = 0;
+    harness.compromiseTriggered = false;
+});
+describe("cluster_v2 runtime lease compromise integration", () => {
+    it("steps down and re-elects when lease is compromised", async () => {
+        const runtime = await import("./runtime.js");
+        const params = {
+            agentId: "lease-agent",
+            displayName: "Lease Agent",
+            scope: `lease-scope-${process.pid}-${Date.now()}`,
+            modelPath: "mock-local",
+            rpcTimeoutMs: 500,
+            getSessionId: () => "lease-session",
+        };
+        const first = await runtime.initClusterV2(params);
+        expect(first.role).toBe("leader");
+        await waitFor(async () => {
+            const current = await runtime.initClusterV2(params);
+            return current.role === "leader" && current !== first;
+        }, 3000);
+        const second = await runtime.initClusterV2(params);
+        expect(second.role).toBe("leader");
+        expect(second).not.toBe(first);
+        expect(harness.listenCalls).toBeGreaterThanOrEqual(2);
+        const embedding = await second.embedding.embedQuery("ok");
+        expect(embedding).toEqual([2]);
+    });
+});