@poolzin/pool-bot 2026.3.7 → 2026.3.10

package/dist/gateway/protocol/schema/swarm.js

@@ -0,0 +1,80 @@
+import { Type } from "@sinclair/typebox";
+export const SwarmStrategySchema = Type.Union([
+    Type.Literal("round_robin"),
+    Type.Literal("least_busy"),
+    Type.Literal("capability_match"),
+    Type.Literal("priority_queue"),
+]);
+export const SwarmMemberSchema = Type.Object({
+    agentId: Type.String(),
+    sessionKey: Type.String(),
+    status: Type.Union([Type.Literal("idle"), Type.Literal("working"), Type.Literal("offline")]),
+    capabilities: Type.Array(Type.String()),
+    joinedAt: Type.Number(),
+    lastHeartbeatAt: Type.Number(),
+    completedTasks: Type.Number(),
+    failedTasks: Type.Number(),
+    currentTaskId: Type.Optional(Type.String()),
+});
+export const SwarmTaskSchema = Type.Object({
+    id: Type.String(),
+    description: Type.String(),
+    status: Type.Union([
+        Type.Literal("pending"),
+        Type.Literal("in_progress"),
+        Type.Literal("completed"),
+        Type.Literal("failed"),
+    ]),
+    priority: Type.Number(),
+    createdAt: Type.Number(),
+    startedAt: Type.Optional(Type.Number()),
+    completedAt: Type.Optional(Type.Number()),
+    assignedTo: Type.Optional(Type.String()),
+    requiredCapabilities: Type.Array(Type.String()),
+});
+export const SwarmStateSchema = Type.Object({
+    id: Type.String(),
+    name: Type.String(),
+    description: Type.Optional(Type.String()),
+    createdAt: Type.Number(),
+    orchestratorAgentId: Type.String(),
+    members: Type.Array(SwarmMemberSchema),
+    tasks: Type.Array(SwarmTaskSchema),
+    strategy: SwarmStrategySchema,
+    status: Type.Union([Type.Literal("active"), Type.Literal("paused"), Type.Literal("shutdown")]),
+});
+export const SwarmListParamsSchema = Type.Object({});
+export const SwarmStatusParamsSchema = Type.Object({
+    swarmId: Type.Optional(Type.String()),
+});
+export const SwarmCreateParamsSchema = Type.Object({
+    name: Type.String(),
+    description: Type.Optional(Type.String()),
+    strategy: SwarmStrategySchema,
+    orchestrator: Type.Optional(Type.String()),
+});
+export const SwarmListResultSchema = Type.Object({
+    swarms: Type.Array(Type.Object({
+        id: Type.String(),
+        name: Type.String(),
+        status: Type.Union([
+            Type.Literal("active"),
+            Type.Literal("paused"),
+            Type.Literal("shutdown"),
+        ]),
+        members: Type.Number(),
+        tasks: Type.Number(),
+    })),
+});
+export const SwarmStatusResultSchema = Type.Object({
+    swarm: Type.Union([SwarmStateSchema, Type.Null()]),
+});
+export const SwarmCreateResultSchema = Type.Object({
+    id: Type.String(),
+    name: Type.String(),
+    description: Type.Optional(Type.String()),
+    strategy: SwarmStrategySchema,
+    orchestratorAgentId: Type.String(),
+    createdAt: Type.Number(),
+    status: Type.Union([Type.Literal("active"), Type.Literal("paused"), Type.Literal("shutdown")]),
+});

package/dist/gateway/protocol/schema.js

@@ -13,5 +13,6 @@ export * from "./schema/push.js";
 export * from "./schema/protocol-schemas.js";
 export * from "./schema/sessions.js";
 export * from "./schema/snapshot.js";
+export * from "./schema/swarm.js";
 export * from "./schema/types.js";
 export * from "./schema/wizard.js";

package/dist/gateway/server-close.js

@@ -54,6 +54,7 @@ export function createGatewayCloseHandler(params) {
         clearInterval(params.tickInterval);
         clearInterval(params.healthInterval);
         clearInterval(params.dedupeCleanup);
+        clearInterval(params.telemetryInterval);
         if (params.agentUnsub) {
             try {
                 params.agentUnsub();
@@ -84,6 +85,9 @@ export function createGatewayCloseHandler(params) {
         if (params.browserControl) {
             await params.browserControl.stop().catch(() => { });
         }
+        if (params.telemetryService) {
+            await params.telemetryService.shutdown().catch(() => { });
+        }
         await new Promise((resolve) => params.wss.close(() => resolve()));
         const servers = params.httpServers && params.httpServers.length > 0
             ? params.httpServers

package/dist/gateway/server-constants.js

@@ -29,5 +29,6 @@ export const getHandshakeTimeoutMs = () => {
 };
 export const TICK_INTERVAL_MS = 30_000;
 export const HEALTH_REFRESH_INTERVAL_MS = 60_000;
+export const TELEMETRY_METRICS_INTERVAL_MS = 30_000; // Broadcast telemetry metrics every 30s
 export const DEDUPE_TTL_MS = 5 * 60_000;
 export const DEDUPE_MAX = 1000;

package/dist/gateway/server-cron.js

@@ -338,6 +338,35 @@ export function buildGatewayCronService(params) {
                 });
             }
         },
+        onJobMetrics: (metrics) => {
+            // Import telemetry service dynamically to avoid circular dependencies
+            void import("../telemetry/service.js")
+                .then(({ getGlobalTelemetryService }) => {
+                const telemetry = getGlobalTelemetryService();
+                if (telemetry) {
+                    telemetry.recordCounter("poolbot.cron.jobs_executed", 1, {
+                        job_id: metrics.jobId,
+                        job_name: metrics.jobName,
+                        schedule: metrics.schedule,
+                        status: metrics.success ? "success" : "failure",
+                    });
+                    telemetry.recordHistogram("poolbot.cron.job_duration_ms", metrics.durationMs, {
+                        job_id: metrics.jobId,
+                        job_name: metrics.jobName,
+                    });
+                    if (!metrics.success) {
+                        telemetry.recordCounter("poolbot.cron.job_failures", 1, {
+                            job_id: metrics.jobId,
+                            job_name: metrics.jobName,
+                            error_type: metrics.errorType ?? "unknown",
+                        });
+                    }
+                }
+            })
+                .catch(() => {
+                // Silently ignore telemetry errors
+            });
+        },
     });
     return { cron, storePath, cronEnabled };
 }

package/dist/gateway/server-maintenance.js

@@ -1,7 +1,11 @@
+import { createAlertEngine } from "../telemetry/alert-engine.js";
+import { getGlobalTelemetryService } from "../telemetry/service.js";
 import { abortChatRunById } from "./chat-abort.js";
-import { DEDUPE_MAX, DEDUPE_TTL_MS, HEALTH_REFRESH_INTERVAL_MS, TICK_INTERVAL_MS, } from "./server-constants.js";
+import { DEDUPE_MAX, DEDUPE_TTL_MS, HEALTH_REFRESH_INTERVAL_MS, TELEMETRY_METRICS_INTERVAL_MS, TICK_INTERVAL_MS, } from "./server-constants.js";
 import { formatError } from "./server-utils.js";
 import { setBroadcastHealthUpdate } from "./server/health-state.js";
+import { createSubsystemLogger } from "../logging/subsystem.js";
+const logTelemetry = createSubsystemLogger("telemetry:broadcast");
 export function startGatewayMaintenanceTimers(params) {
     setBroadcastHealthUpdate((snap) => {
         params.broadcast("health", snap, {
@@ -28,6 +32,35 @@ export function startGatewayMaintenanceTimers(params) {
     void params
         .refreshGatewayHealthSnapshot({ probe: true })
         .catch((err) => params.logHealth.error(`initial refresh failed: ${formatError(err)}`));
+    // Setup alert engine
+    const telemetry = getGlobalTelemetryService();
+    if (telemetry?.isEnabled()) {
+        const alertEngine = createAlertEngine();
+        telemetry.setAlertEngine(alertEngine);
+        logTelemetry.info(`Alert engine initialized with ${alertEngine.getRules().length} rules`);
+    }
+    // periodic telemetry metrics broadcast
+    const telemetryInterval = setInterval(() => {
+        const telemetry = getGlobalTelemetryService();
+        if (!telemetry?.isEnabled()) {
+            return;
+        }
+        const snapshot = telemetry.getSnapshot();
+        if (snapshot) {
+            const payload = { metrics: snapshot, ts: Date.now() };
+            params.broadcast("telemetry.metrics", payload, { dropIfSlow: true });
+            params.nodeSendToAllSubscribed("telemetry.metrics", payload);
+            logTelemetry.debug(`broadcasted ${snapshot.metrics.length} metrics, ${snapshot.spans.length} spans`);
+            // Evaluate alerts
+            const alerts = telemetry.evaluateAlerts(snapshot);
+            for (const alert of alerts) {
+                const alertPayload = { alert, ts: Date.now() };
+                params.broadcast("telemetry.alert", alertPayload, { dropIfSlow: false });
+                params.nodeSendToAllSubscribed("telemetry.alert", alertPayload);
+                logTelemetry.warn(`Alert triggered: ${alert.name} (${alert.severity}) - ${alert.message}`);
+            }
+        }
+    }, TELEMETRY_METRICS_INTERVAL_MS);
     // dedupe cache cleanup
     const dedupeCleanup = setInterval(() => {
         const AGENT_RUN_SEQ_MAX = 10_000;
@@ -79,5 +112,5 @@ export function startGatewayMaintenanceTimers(params) {
             params.chatDeltaSentAt.delete(runId);
         }
     }, 60_000);
-    return { tickInterval, healthInterval, dedupeCleanup };
+    return { tickInterval, healthInterval, dedupeCleanup, telemetryInterval };
 }

package/dist/gateway/server-methods/swarm.js

@@ -0,0 +1,58 @@
+import { ErrorCodes, errorShape, formatValidationErrors, validateSwarmCreateParams, validateSwarmListParams, validateSwarmStatusParams, } from "../protocol/index.js";
+import { swarmService } from "../../swarm/service.js";
+export const swarmHandlers = {
+    "swarm.list": async ({ params, respond }) => {
+        if (!validateSwarmListParams(params)) {
+            respond(false, undefined, errorShape(ErrorCodes.INVALID_REQUEST, `invalid swarm.list params: ${formatValidationErrors(validateSwarmListParams.errors)}`));
+            return;
+        }
+        try {
+            const result = await swarmService.list();
+            respond(true, result, undefined);
+        }
+        catch (err) {
+            respond(false, undefined, errorShape(ErrorCodes.INTERNAL_ERROR, `swarm.list failed: ${err}`));
+        }
+    },
+    "swarm.status": async ({ params, respond }) => {
+        if (!validateSwarmStatusParams(params)) {
+            respond(false, undefined, errorShape(ErrorCodes.INVALID_REQUEST, `invalid swarm.status params: ${formatValidationErrors(validateSwarmStatusParams.errors)}`));
+            return;
+        }
+        try {
+            const p = params;
+            // If no swarmId provided, return default swarm
+            const swarmId = p.swarmId ?? "swarm-default";
+            let result = await swarmService.getStatus(swarmId);
+            // If swarm doesn't exist, create default one
+            if (!result.swarm) {
+                const defaultSwarm = await swarmService.getOrCreateDefaultSwarm();
+                result = { swarm: defaultSwarm };
+            }
+            respond(true, result, undefined);
+        }
+        catch (err) {
+            respond(false, undefined, errorShape(ErrorCodes.INTERNAL_ERROR, `swarm.status failed: ${err}`));
+        }
+    },
+    "swarm.create": async ({ params, respond }) => {
+        if (!validateSwarmCreateParams(params)) {
+            respond(false, undefined, errorShape(ErrorCodes.INVALID_REQUEST, `invalid swarm.create params: ${formatValidationErrors(validateSwarmCreateParams.errors)}`));
+            return;
+        }
+        try {
+            const p = params;
+            const result = await swarmService.create({
+                name: p.name,
+                description: p.description,
+                strategy: p.strategy ??
+                    "capability_match",
+                orchestratorAgentId: p.orchestrator ?? "main",
+            });
+            respond(true, result, undefined);
+        }
+        catch (err) {
+            respond(false, undefined, errorShape(ErrorCodes.INTERNAL_ERROR, `swarm.create failed: ${err}`));
+        }
+    },
+};

package/dist/gateway/server-methods/telemetry.js

@@ -0,0 +1,71 @@
+import { getGlobalTelemetryService } from "../../telemetry/service.js";
+import { ErrorCodes, errorShape } from "../protocol/index.js";
+export const telemetryHandlers = {
+    "telemetry.status": ({ respond }) => {
+        const telemetry = getGlobalTelemetryService();
+        if (!telemetry) {
+            respond(true, { enabled: false, initialized: false }, undefined);
+            return;
+        }
+        const config = telemetry.getConfig();
+        const snapshot = telemetry.getSnapshot();
+        respond(true, {
+            enabled: config.enabled,
+            initialized: true,
+            serviceName: config.serviceName,
+            exporter: config.tracing.exporter,
+            endpoint: config.tracing.otlpEndpoint,
+            tracing: config.tracing,
+            metrics: config.metrics,
+            sampleRate: config.tracing.sampleRate,
+            metricsSnapshot: snapshot,
+        }, undefined);
+    },
+    "telemetry.metrics": ({ respond }) => {
+        const telemetry = getGlobalTelemetryService();
+        if (!telemetry) {
+            respond(false, undefined, errorShape(ErrorCodes.UNAVAILABLE, "Telemetry service not initialized"));
+            return;
+        }
+        const snapshot = telemetry.getSnapshot();
+        respond(true, { metrics: snapshot }, undefined);
+    },
+    "telemetry.config": ({ params, respond }) => {
+        const telemetry = getGlobalTelemetryService();
+        if (!telemetry) {
+            respond(false, undefined, errorShape(ErrorCodes.UNAVAILABLE, "Telemetry service not initialized"));
+            return;
+        }
+        // Allow runtime updates to sample rate
+        if (typeof params.sampleRate === "number") {
+            // Note: This would require restarting the service with new config
+            // For now, we just return the current config
+        }
+        const config = telemetry.getConfig();
+        respond(true, {
+            enabled: config.enabled,
+            serviceName: config.serviceName,
+            tracing: config.tracing,
+            metrics: config.metrics,
+        }, undefined);
+    },
+    "telemetry.alerts.list": ({ respond }) => {
+        const telemetry = getGlobalTelemetryService();
+        if (!telemetry) {
+            respond(false, undefined, errorShape(ErrorCodes.UNAVAILABLE, "Telemetry service not initialized"));
+            return;
+        }
+        const alerts = telemetry.evaluateAlerts();
+        respond(true, { alerts }, undefined);
+    },
+    "telemetry.alerts.test": ({ respond }) => {
+        const telemetry = getGlobalTelemetryService();
+        if (!telemetry) {
+            respond(false, undefined, errorShape(ErrorCodes.UNAVAILABLE, "Telemetry service not initialized"));
+            return;
+        }
+        // Force alert evaluation and return any triggered alerts
+        const alerts = telemetry.evaluateAlerts();
+        respond(true, { triggered: alerts.length > 0, alerts }, undefined);
+    },
+};

package/dist/gateway/server-methods-list.js

@@ -89,6 +89,12 @@ const BASE_METHODS = [
     "chat.history",
     "chat.abort",
     "chat.send",
+    // Telemetry methods
+    "telemetry.status",
+    "telemetry.metrics",
+    "telemetry.config",
+    "telemetry.alerts.list",
+    "telemetry.alerts.test",
 ];
 export function listGatewayMethods() {
     const channelMethods = listChannelPlugins().flatMap((plugin) => plugin.gatewayMethods ?? []);
@@ -113,4 +119,6 @@ export const GATEWAY_EVENTS = [
     "voicewake.changed",
     "exec.approval.requested",
     "exec.approval.resolved",
+    "telemetry.metrics",
+    "telemetry.alert",
 ];

package/dist/gateway/server-methods.js

@@ -8,7 +8,6 @@ import { configHandlers } from "./server-methods/config.js";
 import { connectHandlers } from "./server-methods/connect.js";
 import { cronHandlers } from "./server-methods/cron.js";
 import { deviceHandlers } from "./server-methods/devices.js";
-import { execApprovalsHandlers } from "./server-methods/exec-approvals.js";
 import { healthHandlers } from "./server-methods/health.js";
 import { logsHandlers } from "./server-methods/logs.js";
 import { modelsHandlers } from "./server-methods/models.js";
@@ -16,7 +15,9 @@ import { nodeHandlers } from "./server-methods/nodes.js";
 import { sendHandlers } from "./server-methods/send.js";
 import { sessionsHandlers } from "./server-methods/sessions.js";
 import { skillsHandlers } from "./server-methods/skills.js";
+import { swarmHandlers } from "./server-methods/swarm.js";
 import { systemHandlers } from "./server-methods/system.js";
+import { telemetryHandlers } from "./server-methods/telemetry.js";
 import { talkHandlers } from "./server-methods/talk.js";
 import { ttsHandlers } from "./server-methods/tts.js";
 import { updateHandlers } from "./server-methods/update.js";
@@ -68,6 +69,8 @@ const READ_METHODS = new Set([
     "cron.list",
     "cron.status",
     "cron.runs",
+    "swarm.list",
+    "swarm.status",
     "system-presence",
     "last-heartbeat",
     "node.list",
@@ -75,6 +78,9 @@ const READ_METHODS = new Set([
     "chat.history",
     "config.get",
     "talk.config",
+    "telemetry.status",
+    "telemetry.metrics",
+    "telemetry.config",
 ]);
 const WRITE_METHODS = new Set([
     "send",
@@ -170,7 +176,7 @@ export const coreGatewayHandlers = {
     ...chatHandlers,
     ...cronHandlers,
     ...deviceHandlers,
-    ...execApprovalsHandlers,
+    ...telemetryHandlers,
     ...webHandlers,
     ...modelsHandlers,
     ...configHandlers,
@@ -187,6 +193,7 @@ export const coreGatewayHandlers = {
     ...agentHandlers,
     ...agentsHandlers,
     ...browserHandlers,
+    ...swarmHandlers,
 };
 export async function handleGatewayRequest(opts) {
     const { req, respond, client, isWebchatConnect, context } = opts;

package/dist/gateway/server.impl.js

@@ -27,6 +27,7 @@ import { createSubsystemLogger, runtimeForLogger } from "../logging/subsystem.js
 import { getGlobalHookRunner, runGlobalGatewayStopSafely } from "../plugins/hook-runner-global.js";
 import { createEmptyPluginRegistry } from "../plugins/registry.js";
 import { getTotalQueueSize } from "../process/command-queue.js";
+import { createTelemetryService, setGlobalTelemetryService, telemetryConfigFromOtelConfig, } from "../telemetry/service.js";
 import { runOnboardingWizard } from "../wizard/onboarding.js";
 import { createAuthRateLimiter } from "./auth-rate-limit.js";
 import { startChannelHealthMonitor } from "./channel-health-monitor.js";
@@ -148,6 +149,18 @@ export async function startGatewayServer(port = 18789, opts = {}) {
     if (diagnosticsEnabled) {
         startDiagnosticHeartbeat();
     }
+    // Initialize telemetry service if enabled
+    let telemetryService = null;
+    const otelConfig = cfgAtStart.diagnostics?.otel;
+    if (otelConfig?.enabled) {
+        const telemetryConfig = telemetryConfigFromOtelConfig(otelConfig, {
+            defaultServiceName: "poolbot-gateway",
+        });
+        telemetryService = createTelemetryService(telemetryConfig);
+        setGlobalTelemetryService(telemetryService);
+        await telemetryService.start();
+        log.info(`telemetry: initialized with ${telemetryConfig.tracing.exporter} exporter`);
+    }
     setGatewaySigusr1RestartPolicy({ allowExternal: isRestartEnabled(cfgAtStart) });
     setPreRestartDeferralCheck(() => getTotalQueueSize() + getTotalPendingReplies() + getActiveEmbeddedRunCount());
     initSubagentRegistry();
@@ -326,23 +339,25 @@ export async function startGatewayServer(port = 18789, opts = {}) {
     let tickInterval = noopInterval();
     let healthInterval = noopInterval();
     let dedupeCleanup = noopInterval();
+    let telemetryInterval = noopInterval();
     if (!minimalTestGateway) {
-        ({ tickInterval, healthInterval, dedupeCleanup } = startGatewayMaintenanceTimers({
-            broadcast,
-            nodeSendToAllSubscribed,
-            getPresenceVersion,
-            getHealthVersion,
-            refreshGatewayHealthSnapshot,
-            logHealth,
-            dedupe,
-            chatAbortControllers,
-            chatRunState,
-            chatRunBuffers,
-            chatDeltaSentAt,
-            removeChatRun,
-            agentRunSeq,
-            nodeSendToSession,
-        }));
+        ({ tickInterval, healthInterval, dedupeCleanup, telemetryInterval } =
+            startGatewayMaintenanceTimers({
+                broadcast,
+                nodeSendToAllSubscribed,
+                getPresenceVersion,
+                getHealthVersion,
+                refreshGatewayHealthSnapshot,
+                logHealth,
+                dedupe,
+                chatAbortControllers,
+                chatRunState,
+                chatRunBuffers,
+                chatDeltaSentAt,
+                removeChatRun,
+                agentRunSeq,
+                nodeSendToSession,
+            }));
     }
     const agentUnsub = minimalTestGateway
         ? null
@@ -564,12 +579,14 @@ export async function startGatewayServer(port = 18789, opts = {}) {
         tickInterval,
         healthInterval,
         dedupeCleanup,
+        telemetryInterval,
         agentUnsub,
         heartbeatUnsub,
         chatRunState,
         clients,
         configReloader,
         browserControl,
+        telemetryService,
         wss,
         httpServer,
         httpServers,

package/dist/infra/abort-pattern.js

@@ -0,0 +1,106 @@
+/**
+ * Abort Pattern Utilities
+ *
+ * Provides memory-leak-free abort signal handling.
+ *
+ * CRITICAL FIX: Uses `.bind()` instead of closures to prevent memory leaks.
+ * Issue #7174: Closure-based abort handlers capture scope and leak memory.
+ *
+ * @example
+ * ```typescript
+ * // BAD: Captures closure scope (leaks memory)
+ * signal.addEventListener('abort', () => controller.abort());
+ *
+ * // GOOD: No closure capture
+ * signal.addEventListener('abort', relayAbort.bind(controller));
+ * ```
+ */
+/**
+ * Relay abort signal without closure capture
+ * Prevents memory leak by using bind instead of arrow function
+ */
+export function relayAbort() {
+    this.abort();
+}
+/**
+ * Create an abort relay that doesn't capture closure scope
+ */
+export function createAbortRelay(controller) {
+    return relayAbort.bind(controller);
+}
+/**
+ * Link an abort signal to a controller without memory leaks
+ */
+export function linkAbortSignal(source, target) {
+    const handler = createAbortRelay(target);
+    source.addEventListener("abort", handler, { once: true });
+    // Return cleanup function
+    return () => {
+        source.removeEventListener("abort", handler);
+    };
+}
+/**
+ * Create a timeout-based abort controller
+ */
+export function createTimeoutAbortController(timeoutMs) {
+    const controller = new AbortController();
+    if (timeoutMs > 0) {
+        const timeout = setTimeout(() => {
+            controller.abort();
+        }, timeoutMs);
+        return {
+            controller,
+            cleanup: () => clearTimeout(timeout),
+        };
+    }
+    return {
+        controller,
+        cleanup: () => { },
+    };
+}
+/**
+ * Race multiple abort signals
+ */
+export function raceAbortSignals(signals, controller) {
+    const handlers = [];
+    let alreadyAborted = false;
+    for (const signal of signals) {
+        if (signal.aborted && !alreadyAborted) {
+            alreadyAborted = true;
+            controller.abort();
+        }
+        else if (!alreadyAborted) {
+            const handler = createAbortRelay(controller);
+            signal.addEventListener("abort", handler, { once: true });
+            handlers.push(() => signal.removeEventListener("abort", handler));
+        }
+    }
+    return () => {
+        for (const cleanup of handlers) {
+            cleanup();
+        }
+    };
+}
+/**
+ * Fetch with timeout and proper abort handling
+ */
+export async function fetchWithTimeout(url, options = {}) {
+    const { timeoutMs = 30000, ...fetchOptions } = options;
+    const { controller, cleanup } = createTimeoutAbortController(timeoutMs);
+    // Link existing signal if provided
+    let unlink;
+    if (fetchOptions.signal) {
+        unlink = linkAbortSignal(fetchOptions.signal, controller);
+    }
+    try {
+        const response = await fetch(url, {
+            ...fetchOptions,
+            signal: controller.signal,
+        });
+        return response;
+    }
+    finally {
+        unlink?.();
+        cleanup();
+    }
+}