@naisys/hub 3.0.0-beta.6

package/dist/handlers/hubCostService.js

@@ -0,0 +1,279 @@
+import { AgentConfigFileSchema, calculatePeriodBoundaries, } from "@naisys/common";
+import { CostWriteRequestSchema, HubEvents, } from "@naisys/hub-protocol";
+const SPEND_LIMIT_CHECK_INTERVAL_MS = 10_000;
+/** Handles cost_write events from NAISYS instances (fire-and-forget) */
+export function createHubCostService(naisysServer, { hubDb }, logService, heartbeatService, configService) {
+    // Track which users have been suspended due to spend limit overrun
+    const suspendedByGlobal = new Set();
+    const suspendedByAgent = new Set();
+    naisysServer.registerEvent(HubEvents.COST_WRITE, async (hostId, data, ack) => {
+        try {
+            const parsed = CostWriteRequestSchema.parse(data);
+            // Roll up cost deltas by user/run/session for supervisor push,
+            // and per-user totals for budget_left decrement
+            const costPushMap = new Map();
+            const userCostTotals = new Map();
+            for (const entry of parsed.entries) {
+                await hubDb.costs.create({
+                    data: {
+                        user_id: entry.userId,
+                        run_id: entry.runId,
+                        session_id: entry.sessionId,
+                        host_id: hostId,
+                        source: entry.source,
+                        model: entry.model,
+                        cost: entry.cost,
+                        input_tokens: entry.inputTokens,
+                        output_tokens: entry.outputTokens,
+                        cache_write_tokens: entry.cacheWriteTokens,
+                        cache_read_tokens: entry.cacheReadTokens,
+                    },
+                });
+                // Update run_session total_cost
+                await hubDb.run_session.updateMany({
+                    where: {
+                        user_id: entry.userId,
+                        run_id: entry.runId,
+                        session_id: entry.sessionId,
+                    },
+                    data: {
+                        total_cost: { increment: entry.cost },
+                    },
+                });
+                const key = `${entry.userId}:${entry.runId}:${entry.sessionId}`;
+                const existing = costPushMap.get(key);
+                if (existing) {
+                    existing.costDelta += entry.cost;
+                }
+                else {
+                    costPushMap.set(key, {
+                        userId: entry.userId,
+                        runId: entry.runId,
+                        sessionId: entry.sessionId,
+                        costDelta: entry.cost,
+                    });
+                }
+                userCostTotals.set(entry.userId, (userCostTotals.get(entry.userId) ?? 0) + entry.cost);
+            }
+            // Push rolled-up cost deltas to supervisor connections
+            if (costPushMap.size > 0) {
+                naisysServer.broadcastToSupervisors(HubEvents.COST_PUSH, {
+                    entries: Array.from(costPushMap.values()),
+                });
+            }
+            // Re-send cost_control to any suspended users still writing costs
+            for (const userId of userCostTotals.keys()) {
+                if (suspendedByGlobal.has(userId) || suspendedByAgent.has(userId)) {
+                    sendCostControl(userId, false, "Spend limit exceeded");
+                }
+            }
+            // Decrement budget_left and return updated values
+            const budgets = await Promise.all(Array.from(userCostTotals.entries()).map(([userId, batchCost]) => decrementBudgetLeft(hubDb, userId, batchCost)));
+            ack({ budgets });
+        }
+        catch (error) {
+            logService.error(`[Hub:Costs] Error processing cost_write from host ${hostId}: ${error}`);
+            ack({ budgets: [] });
+        }
+    });
+    // Periodic spend limit checking
+    const spendLimitCheckInterval = setInterval(() => void checkSpendLimits().catch((error) => {
+        logService.error(`[Hub:Costs] Error in spend limit check: ${error}`);
+    }), SPEND_LIMIT_CHECK_INTERVAL_MS);
+    async function checkSpendLimits(candidateUserIds) {
+        const activeUserIds = heartbeatService.getActiveUserIds();
+        const usersToCheck = new Set(activeUserIds);
+        for (const userId of suspendedByGlobal)
+            usersToCheck.add(userId);
+        for (const userId of suspendedByAgent)
+            usersToCheck.add(userId);
+        if (candidateUserIds) {
+            for (const userId of candidateUserIds)
+                usersToCheck.add(userId);
+        }
+        if (usersToCheck.size === 0)
+            return;
+        const config = configService.getConfig().config;
+        const spendLimitDollars = config?.spendLimitDollars;
+        const spendLimitHours = config?.spendLimitHours;
+        // Query user configs (needed by both global and per-agent checks)
+        const users = await hubDb.users.findMany({
+            where: { id: { in: Array.from(usersToCheck) } },
+            select: {
+                id: true,
+                config: true,
+                user_notifications: {
+                    select: { spend_limit_reset_at: true },
+                },
+            },
+        });
+        // Identify which users have per-agent spend limits (exempt from global)
+        const usersWithAgentLimit = new Set();
+        for (const user of users) {
+            try {
+                const parsed = AgentConfigFileSchema.safeParse(JSON.parse(user.config));
+                if (parsed.success && parsed.data.spendLimitDollars !== undefined) {
+                    usersWithAgentLimit.add(user.id);
+                }
+            }
+            catch {
+                /* ignore parse errors */
+            }
+        }
+        // 1. Global spend limit — only applies to agents WITHOUT a per-agent limit
+        if (spendLimitDollars !== undefined) {
+            await checkGlobalSpendLimit(hubDb, usersToCheck, spendLimitDollars, spendLimitHours, usersWithAgentLimit);
+        }
+        // 2. Per-agent spend limit checks
+        for (const user of users) {
+            try {
+                const parsed = AgentConfigFileSchema.safeParse(JSON.parse(user.config));
+                if (!parsed.success)
+                    continue;
+                const config = parsed.data;
+                if (config.spendLimitDollars === undefined)
+                    continue;
+                await checkAgentSpendLimit(hubDb, user.id, config.spendLimitDollars, config.spendLimitHours, user.user_notifications?.spend_limit_reset_at ?? undefined);
+            }
+            catch (userError) {
+                logService.error(`[Hub:Costs] Error checking spend limit for user ${user.id}: ${userError}`);
+            }
+        }
+    }
+    /**
+     * Finds the total cost over the period, if this period is used up, we wait until the next period to resume.
+     * We don't use a sliding window as that would cause the LLM to get stuck in a cycle of sending off a query,
+     * only for the window to close again, and the LLM cache to *expire* creating constant cache misses.
+     */
+    async function queryCostSum(hubDb, spendLimitHours, userIdFilter, spendLimitResetAt) {
+        const where = {};
+        if (userIdFilter) {
+            where.user_id = userIdFilter;
+        }
+        let effectiveStart;
+        if (spendLimitHours !== undefined) {
+            const { periodStart } = calculatePeriodBoundaries(spendLimitHours);
+            effectiveStart = periodStart;
+        }
+        if (spendLimitResetAt &&
+            (!effectiveStart || spendLimitResetAt > effectiveStart)) {
+            effectiveStart = spendLimitResetAt;
+        }
+        if (effectiveStart) {
+            where.created_at = { gte: effectiveStart };
+        }
+        const result = await hubDb.costs.aggregate({
+            where,
+            _sum: { cost: true },
+        });
+        return result._sum.cost ?? 0;
+    }
+    function sendCostControl(userId, enabled, reason) {
+        const hostIds = heartbeatService.findHostsForAgent(userId);
+        for (const hostId of hostIds) {
+            naisysServer.sendMessage(hostId, HubEvents.COST_CONTROL, {
+                userId,
+                enabled,
+                reason,
+            });
+        }
+    }
+    async function setCostSuspendedReason(hubDb, userId, reason) {
+        await hubDb.user_notifications.updateMany({
+            where: { user_id: userId },
+            data: { cost_suspended_reason: reason },
+        });
+    }
+    /** Check the global spend limit — only applies to agents without a per-agent limit */
+    async function checkGlobalSpendLimit(hubDb, usersToCheck, spendLimit, spendLimitHours, usersWithAgentLimit) {
+        const totalCost = await queryCostSum(hubDb, spendLimitHours);
+        const isOverLimit = totalCost >= spendLimit;
+        async function resumeFromGlobal(userId, reason) {
+            logService.log(`[Hub:Costs] Resuming user ${userId} (global limit): ${reason}`);
+            sendCostControl(userId, true, reason);
+            suspendedByGlobal.delete(userId);
+            if (!suspendedByAgent.has(userId)) {
+                await setCostSuspendedReason(hubDb, userId, null);
+            }
+        }
+        for (const userId of usersToCheck) {
+            // Agents with their own spend limit are exempt from the global limit
+            if (usersWithAgentLimit.has(userId)) {
+                if (suspendedByGlobal.has(userId)) {
+                    await resumeFromGlobal(userId, "Agent has per-agent spend limit");
+                }
+                continue;
+            }
+            const wasSuspended = suspendedByGlobal.has(userId);
+            if (isOverLimit && !wasSuspended) {
+                const reason = `Global spend limit of $${spendLimit} reached (total: $${totalCost.toFixed(2)})`;
+                logService.log(`[Hub:Costs] Suspending user ${userId} (global limit): ${reason}`);
+                sendCostControl(userId, false, reason);
+                suspendedByGlobal.add(userId);
+                await setCostSuspendedReason(hubDb, userId, reason);
+            }
+            else if (!isOverLimit && wasSuspended) {
+                await resumeFromGlobal(userId, `Global spend limit period reset (total: $${totalCost.toFixed(2)}, limit: $${spendLimit})`);
+            }
+        }
+    }
+    /** Check a per-agent spend limit */
+    async function checkAgentSpendLimit(hubDb, userId, spendLimit, spendLimitHours, spendLimitResetAt) {
+        const periodCost = await queryCostSum(hubDb, spendLimitHours, userId, spendLimitResetAt);
+        const isOverLimit = periodCost >= spendLimit;
+        const wasSuspended = suspendedByAgent.has(userId);
+        // Persist budget_left for supervisor display
+        const budgetLeft = Math.max(0, spendLimit - periodCost);
+        await hubDb.user_notifications.updateMany({
+            where: { user_id: userId },
+            data: { budget_left: budgetLeft },
+        });
+        if (isOverLimit && !wasSuspended) {
+            const reason = `Spend limit of $${spendLimit} reached (current: $${periodCost.toFixed(2)})`;
+            logService.log(`[Hub:Costs] Suspending user ${userId}: ${reason}`);
+            sendCostControl(userId, false, reason);
+            suspendedByAgent.add(userId);
+            await setCostSuspendedReason(hubDb, userId, reason);
+        }
+        else if (!isOverLimit && wasSuspended) {
+            const reason = `Spend limit period reset (current: $${periodCost.toFixed(2)}, limit: $${spendLimit})`;
+            logService.log(`[Hub:Costs] Resuming user ${userId}: ${reason}`);
+            sendCostControl(userId, true, reason);
+            suspendedByAgent.delete(userId);
+            await setCostSuspendedReason(hubDb, userId, null);
+        }
+    }
+    /** Decrement budget_left by the batch cost and return the updated value */
+    async function decrementBudgetLeft(hubDb, userId, batchCost) {
+        try {
+            const notification = await hubDb.user_notifications.findUnique({
+                where: { user_id: userId },
+                select: { budget_left: true },
+            });
+            if (notification?.budget_left == null) {
+                return { userId, budgetLeft: null };
+            }
+            const budgetLeft = Math.max(0, Number(notification.budget_left) - batchCost);
+            await hubDb.user_notifications.update({
+                where: { user_id: userId },
+                data: { budget_left: budgetLeft },
+            });
+            return { userId, budgetLeft };
+        }
+        catch {
+            return { userId, budgetLeft: null };
+        }
+    }
+    function isUserSpendSuspended(userId) {
+        return suspendedByGlobal.has(userId) || suspendedByAgent.has(userId);
+    }
+    function cleanup() {
+        clearInterval(spendLimitCheckInterval);
+    }
+    return {
+        cleanup,
+        checkSpendLimits,
+        isUserSpendSuspended,
+    };
+}
+//# sourceMappingURL=hubCostService.js.map

package/dist/handlers/hubHeartbeatService.js

@@ -0,0 +1,132 @@
+import { HeartbeatSchema, HUB_HEARTBEAT_INTERVAL_MS, HubEvents, } from "@naisys/hub-protocol";
+/** Tracks NAISYS instance heartbeats and pushes aggregate active user status to all instances */
+export function createHubHeartbeatService(naisysServer, { hubDb }, logService) {
+    // Track active agent user IDs per host from heartbeat data
+    const hostActiveAgents = new Map();
+    // Track per-agent notification IDs (latestLogId, latestMailId)
+    const agentNotifications = new Map();
+    /** Update a single notification field for an agent */
+    function updateAgentNotification(userId, field, value) {
+        let entry = agentNotifications.get(userId);
+        if (!entry) {
+            entry = { latestLogId: 0, latestMailId: 0, latestChatId: 0 };
+            agentNotifications.set(userId, entry);
+        }
+        entry[field] = value;
+    }
+    // Handle heartbeat from NAISYS instances
+    naisysServer.registerEvent(HubEvents.HEARTBEAT, async (hostId, data) => {
+        const parsed = HeartbeatSchema.parse(data);
+        // Update in-memory per-host active agent IDs
+        hostActiveAgents.set(hostId, parsed.activeUserIds);
+        try {
+            const now = new Date().toISOString();
+            // Update host last_active
+            await hubDb.hosts.updateMany({
+                where: { id: hostId },
+                data: { last_active: now },
+            });
+            // Update user_notifications.last_active for each active user
+            if (parsed.activeUserIds.length > 0) {
+                await hubDb.user_notifications.updateMany({
+                    where: { user_id: { in: parsed.activeUserIds } },
+                    data: { last_active: now, latest_host_id: hostId },
+                });
+            }
+        }
+        catch (error) {
+            logService.error(`[Hub:Heartbeat] Error updating heartbeat for host ${hostId}: ${error}`);
+        }
+    });
+    // Clean up tracking when a host disconnects
+    naisysServer.registerEvent(HubEvents.CLIENT_DISCONNECTED, (hostId) => {
+        hostActiveAgents.delete(hostId);
+        throttledPushAgentsStatus();
+    });
+    /** Push aggregate agent status to all connected NAISYS instances */
+    let lastPushedJson = "";
+    function pushAgentsStatus() {
+        const payload = {
+            hostActiveAgents: Object.fromEntries(hostActiveAgents),
+            agentNotifications: Object.fromEntries(agentNotifications),
+        };
+        const json = JSON.stringify(payload);
+        if (json === lastPushedJson)
+            return;
+        lastPushedJson = json;
+        naisysServer.broadcastToAll(HubEvents.AGENTS_STATUS, payload);
+    }
+    /** Throttled push for agent start/stop changes — at most once per 500ms */
+    let throttleTimer = null;
+    function throttledPushAgentsStatus() {
+        if (throttleTimer)
+            return;
+        pushAgentsStatus();
+        throttleTimer = setTimeout(() => {
+            throttleTimer = null;
+        }, 500);
+    }
+    // Periodically push aggregate active user status to all NAISYS instances
+    const pushInterval = setInterval(pushAgentsStatus, HUB_HEARTBEAT_INTERVAL_MS);
+    function getHostActiveAgentCount(hostId) {
+        return hostActiveAgents.get(hostId)?.length ?? 0;
+    }
+    /** Find which hosts a given agent is currently running on */
+    function findHostsForAgent(userId) {
+        const hostIds = [];
+        for (const [hostId, userIds] of hostActiveAgents) {
+            if (userIds.includes(userId)) {
+                hostIds.push(hostId);
+            }
+        }
+        return hostIds;
+    }
+    /** Add a userId to a host's active list after a successful start */
+    function addStartedAgent(hostId, userId) {
+        const userIds = hostActiveAgents.get(hostId);
+        if (userIds) {
+            if (!userIds.includes(userId)) {
+                userIds.push(userId);
+            }
+        }
+        else {
+            hostActiveAgents.set(hostId, [userId]);
+        }
+        throttledPushAgentsStatus();
+    }
+    /** Remove a userId from a host's active list after a successful stop */
+    function removeStoppedAgent(hostId, userId) {
+        const userIds = hostActiveAgents.get(hostId);
+        if (userIds) {
+            const index = userIds.indexOf(userId);
+            if (index !== -1) {
+                userIds.splice(index, 1);
+            }
+        }
+        throttledPushAgentsStatus();
+    }
+    function cleanup() {
+        clearInterval(pushInterval);
+    }
+    /** Get all active user IDs across all connected hosts */
+    function getActiveUserIds() {
+        const allActiveUserIds = new Set();
+        for (const userIds of hostActiveAgents.values()) {
+            for (const id of userIds) {
+                allActiveUserIds.add(id);
+            }
+        }
+        return allActiveUserIds;
+    }
+    return {
+        cleanup,
+        getActiveUserIds,
+        getHostActiveAgentCount,
+        findHostsForAgent,
+        addStartedAgent,
+        removeStoppedAgent,
+        updateAgentNotification,
+        throttledPushAgentsStatus,
+    };
+}
+//# sourceMappingURL=hubHeartbeatService.js.map

package/dist/handlers/hubHostService.js

@@ -0,0 +1,35 @@
+import { HubEvents } from "@naisys/hub-protocol";
+/** Pushes the host list to all connections when connected hosts change */
+export function createHubHostService(naisysServer, hostRegistrar, logService) {
+    let cachedHostListJson = "";
+    function broadcastHostList(newConnection) {
+        const connectedHostIds = new Set(naisysServer.getConnectedClients().map((c) => c.getHostId()));
+        const hosts = hostRegistrar.getAllHosts().map((h) => ({
+            ...h,
+            online: connectedHostIds.has(h.hostId),
+        }));
+        const payload = { hosts };
+        const json = JSON.stringify(payload);
+        // Send to the newly connecting client directly
+        if (newConnection) {
+            newConnection.sendMessage(HubEvents.HOSTS_UPDATED, payload);
+        }
+        // Broadcast to all connections only if the list changed
+        // (new connection may get a harmless duplicate — HOSTS_UPDATED is idempotent)
+        if (json !== cachedHostListJson) {
+            cachedHostListJson = json;
+            logService.log(`[Hub:Hosts] Broadcasting host list (${hosts.length} hosts)`);
+            naisysServer.broadcastToAll(HubEvents.HOSTS_UPDATED, payload);
+        }
+    }
+    naisysServer.registerEvent(HubEvents.CLIENT_CONNECTED, (_hostId, connection) => {
+        broadcastHostList(connection);
+    });
+    naisysServer.registerEvent(HubEvents.HOSTS_CHANGED, async () => {
+        logService.log("[Hub:Hosts] Received HOSTS_CHANGED, refreshing cache...");
+        await hostRegistrar.refreshHosts();
+        cachedHostListJson = ""; // Force broadcast
+        broadcastHostList();
+    });
+}
+//# sourceMappingURL=hubHostService.js.map

package/dist/handlers/hubLogService.js

@@ -0,0 +1,121 @@
+import { HubEvents, LogWriteRequestSchema, } from "@naisys/hub-protocol";
+/** Handles log_write events from NAISYS instances (fire-and-forget) */
+export function createHubLogService(naisysServer, { hubDb }, logService, heartbeatService) {
+    // Track last pushed log ID per session for gap detection
+    const lastPushedLogId = new Map();
+    naisysServer.registerEvent(HubEvents.LOG_WRITE, async (hostId, data) => {
+        try {
+            const parsed = LogWriteRequestSchema.parse(data);
+            // Collect push entries and session deltas
+            const pushEntries = [];
+            const sessionUpdates = new Map();
+            for (const entry of parsed.entries) {
+                const now = new Date().toISOString();
+                const lineCount = entry.message.split("\n").length;
+                const log = await hubDb.context_log.create({
+                    data: {
+                        user_id: entry.userId,
+                        run_id: entry.runId,
+                        session_id: entry.sessionId,
+                        host_id: hostId,
+                        role: entry.role,
+                        source: entry.source ?? null,
+                        type: entry.type ?? null,
+                        message: entry.message,
+                        created_at: entry.createdAt,
+                        attachment_id: entry.attachmentId ?? null,
+                    },
+                });
+                // Update session table with total lines and last active
+                await hubDb.run_session.updateMany({
+                    where: {
+                        user_id: entry.userId,
+                        run_id: entry.runId,
+                        session_id: entry.sessionId,
+                    },
+                    data: {
+                        last_active: now,
+                        latest_log_id: log.id,
+                        total_lines: {
+                            increment: lineCount,
+                        },
+                    },
+                });
+                // Update user_notifications with latest_log_id and last_active
+                await hubDb.user_notifications.updateMany({
+                    where: {
+                        user_id: entry.userId,
+                    },
+                    data: {
+                        latest_log_id: log.id,
+                        last_active: now,
+                    },
+                });
+                // Push notification ID update via heartbeat
+                heartbeatService.updateAgentNotification(entry.userId, "latestLogId", log.id);
+                // Look up attachment metadata if present
+                let attachmentPublicId;
+                let attachmentFilename;
+                let attachmentFileSize;
+                if (entry.attachmentId) {
+                    const att = await hubDb.attachments.findUnique({
+                        where: { id: entry.attachmentId },
+                        select: { public_id: true, filename: true, file_size: true },
+                    });
+                    if (att) {
+                        attachmentPublicId = att.public_id;
+                        attachmentFilename = att.filename;
+                        attachmentFileSize = att.file_size;
+                    }
+                }
+                // Collect push entry with DB-assigned ID
+                const sessionKey = `${entry.userId}-${entry.runId}-${entry.sessionId}`;
+                const previousId = lastPushedLogId.get(sessionKey) ?? null;
+                pushEntries.push({
+                    id: log.id,
+                    previousId,
+                    userId: entry.userId,
+                    runId: entry.runId,
+                    sessionId: entry.sessionId,
+                    role: entry.role,
+                    source: entry.source,
+                    type: entry.type,
+                    message: entry.message,
+                    createdAt: entry.createdAt,
+                    attachmentId: attachmentPublicId,
+                    attachmentFilename,
+                    attachmentFileSize,
+                });
+                lastPushedLogId.set(sessionKey, log.id);
+                // Track session delta (accumulate totalLinesDelta, keep latest logId)
+                const existing = sessionUpdates.get(sessionKey);
+                if (existing) {
+                    existing.latestLogId = Math.max(existing.latestLogId, log.id);
+                    existing.lastActive = now;
+                    existing.totalLinesDelta += lineCount;
+                }
+                else {
+                    sessionUpdates.set(sessionKey, {
+                        userId: entry.userId,
+                        runId: entry.runId,
+                        sessionId: entry.sessionId,
+                        lastActive: now,
+                        latestLogId: log.id,
+                        totalLinesDelta: lineCount,
+                    });
+                }
+            }
+            // Push full log data to supervisor connections
+            naisysServer.broadcastToSupervisors(HubEvents.LOG_PUSH, {
+                entries: pushEntries,
+                sessionUpdates: Array.from(sessionUpdates.values()),
+            });
+            // Trigger throttled push after all entries processed
+            heartbeatService.throttledPushAgentsStatus();
+        }
+        catch (error) {
+            logService.error(`[Hub:Logs] Error processing log_write from host ${hostId}: ${error}`);
+        }
+    });
+}
+//# sourceMappingURL=hubLogService.js.map