@naisys/hub 3.0.0-beta.38 → 3.0.0-beta.40

package/README.md

@@ -46,7 +46,9 @@ The hub owns mail, context logs and [attachments](../../docs/011-mail-attachment
 
 - `Authorization: Bearer` header auth with a rotatable access key
 - Hardened spawning: no shell interpretation, timeouts on `execFileSync`
-- API keys read from headers, not query params
+- API keys stored as SHA-256 hashes; per-user keys read from headers, not query params
+- Dynamic runtime API keys minted per agent and re-issued on hub restart
+- Redaction service scrubs sensitive variables and runtime keys from logs and mail before they hit the DB
 - Hub socket served at `/hub` for reverse-proxy friendliness (TLS terminated at the proxy)
 
 ### Deployment

package/dist/handlers/hubAgentService.js

@@ -1,29 +1,7 @@
-import { hashToken } from "@naisys/common-node";
 import { AgentPeekRequestSchema, AgentRunCommandRequestSchema, AgentRunPauseRequestSchema, AgentStartInboundSchema, AgentStopRequestSchema, HubEvents, } from "@naisys/hub-protocol";
-import { randomBytes } from "crypto";
 /** Handles agent_start requests by routing them to the least-loaded eligible host */
-export function createHubAgentService(naisysServer, { hubDb }, logService, heartbeatService, sendMailService, hostRegistrar) {
-    /**
-     * Mint a fresh runtime API key for a user, rotating any prior key. Plaintext
-     * is returned only here and travels once over the AGENT_START message; the
-     * DB only stores the hash. Keys persist across hub restarts/crashes/updates;
-     * revocation lives on the user disable/archive/delete paths and on graceful
-     * AGENT_STOP.
-     */
-    async function issueRuntimeApiKey(userId) {
-        const token = randomBytes(32).toString("hex");
-        await hubDb.users.update({
-            where: { id: userId },
-            data: { api_key_hash: hashToken(token) },
-        });
-        return token;
-    }
-    async function revokeRuntimeApiKey(userId) {
-        await hubDb.users.update({
-            where: { id: userId },
-            data: { api_key_hash: null },
-        });
-    }
+export function createHubAgentService(naisysServer, { hubDb }, logService, heartbeatService, sendMailService, hostRegistrar, runtimeKeyService) {
+    const { issueRuntimeApiKey, revokeRuntimeApiKey } = runtimeKeyService;
     /** Find the least-loaded eligible host for a given user */
     async function findBestHost(startUserId) {
         // Look up which hosts this user is assigned to
@@ -93,22 +71,102 @@ export function createHubAgentService(naisysServer, { hubDb }, logService, heart
         return { kind: "go", bestHostId };
     }
     /**
-     * Issue a runtime key and send AGENT_START. Stranded keys from a failed
-     * send or failed response stay in the DB until the next AGENT_START for
-     * that user rotates them, or the user is disabled/archived/deleted — the
-     * agent process that would have used the key never started, so there's
-     * nobody to authenticate with it in the meantime.
+     * Mint and ship a key with AGENT_START so the agent's authenticated from
+     * spawn time, avoiding the one-RTT window heartbeat-only would create.
+     * Heartbeat-driven reissue covers later hash mismatches.
+     *
+     * The run_session row is written up front with model_name="" so the
+     * agent's command loop (which starts before the host acks) can FK-safely
+     * write logs/costs. model_name is patched in from the ack; the session
+     * is pushed to supervisors and runId/sessionId returned to the caller
+     * only on success. Any failure rolls back the placeholder.
      */
     async function dispatchAgentStart(args) {
         const { bestHostId, payload, onResponse } = args;
         const startUserId = payload.startUserId;
         const runtimeApiKey = await issueRuntimeApiKey(startUserId);
-        const sent = naisysServer.sendMessage(bestHostId, HubEvents.AGENT_START, { ...payload, runtimeApiKey }, (response) => {
-            if (response.success) {
-                heartbeatService.addStartedAgent(bestHostId, startUserId);
+        const lastRun = await hubDb.run_session.findFirst({
+            select: { run_id: true },
+            orderBy: { run_id: "desc" },
+        });
+        const runId = lastRun ? lastRun.run_id + 1 : 1;
+        const sessionId = 1;
+        const subagentId = 0;
+        const now = new Date().toISOString();
+        const rowWhere = {
+            user_id: startUserId,
+            run_id: runId,
+            subagent_id: subagentId,
+            session_id: sessionId,
+        };
+        await hubDb.run_session.create({
+            data: {
+                ...rowWhere,
+                host_id: bestHostId,
+                model_name: "",
+                created_at: now,
+                last_active: now,
+            },
+        });
+        async function rollbackPlaceholder(reason) {
+            try {
+                await hubDb.run_session.deleteMany({ where: rowWhere });
+            }
+            catch (err) {
+                logService.error(`[Hub:Agents] Failed to roll back run_session row for run ${runId} (${reason}): ${err}`);
+            }
+        }
+        const sent = naisysServer.sendMessage(bestHostId, HubEvents.AGENT_START, { ...payload, runtimeApiKey, runId, sessionId }, async (response) => {
+            if (response.success && response.modelName) {
+                const modelName = response.modelName;
+                try {
+                    await hubDb.run_session.updateMany({
+                        where: rowWhere,
+                        data: { model_name: modelName },
+                    });
+                    heartbeatService.addStartedAgent(bestHostId, startUserId);
+                    naisysServer.broadcastToSupervisors(HubEvents.SESSION_PUSH, {
+                        session: {
+                            userId: startUserId,
+                            runId,
+                            sessionId,
+                            modelName,
+                            createdAt: now,
+                            lastActive: now,
+                            latestLogId: 0,
+                            totalLines: 0,
+                            totalCost: 0,
+                        },
+                    });
+                    onResponse({ ...response, runId, sessionId });
+                }
+                catch (err) {
+                    logService.error(`[Hub:Agents] Failed to finalize run_session row for run ${runId}: ${err}`);
+                    await rollbackPlaceholder("update failed");
+                    onResponse({
+                        success: false,
+                        error: `Failed to finalize run_session row: ${err}`,
+                        hostname: response.hostname,
+                    });
+                }
+            }
+            else {
+                if (response.success && !response.modelName) {
+                    logService.error(`[Hub:Agents] Host ${bestHostId} acked agent_start without modelName for user ${startUserId}; treating as failure`);
+                }
+                await rollbackPlaceholder(response.success ? "missing modelName" : "host failure");
+                onResponse(response.success
+                    ? {
+                        success: false,
+                        error: "Host did not return modelName",
+                        hostname: response.hostname,
+                    }
+                    : response);
             }
-            onResponse(response);
         });
+        if (!sent) {
+            await rollbackPlaceholder("send failed");
+        }
         return { sent };
     }
     /** Try to start an agent on the best available host (fire-and-forget) */
@@ -256,6 +314,7 @@ export function createHubAgentService(naisysServer, { hubDb }, logService, heart
                     const sent = naisysServer.sendMessage(targetHostId, event, {
                         userId: parsed.userId,
                         runId: parsed.runId,
+                        subagentId: parsed.subagentId,
                         sessionId: parsed.sessionId,
                         sourceHostId: hostId,
                     }, (response) => {
@@ -300,6 +359,7 @@ export function createHubAgentService(naisysServer, { hubDb }, logService, heart
                 const sent = naisysServer.sendMessage(targetHostId, HubEvents.AGENT_RUN_COMMAND, {
                     userId: parsed.userId,
                     runId: parsed.runId,
+                    subagentId: parsed.subagentId,
                     sessionId: parsed.sessionId,
                     command: parsed.command,
                     sourceHostId: hostId,

package/dist/handlers/hubConfigService.js

@@ -1,4 +1,4 @@
-import { buildClientConfig, builtInImageModels, builtInLlmModels, } from "@naisys/common";
+import { buildClientConfig, builtInImageModels, builtInLlmModels, OPENAI_CODEX_ACCESS_TOKEN_VAR, OPENAI_CODEX_REFRESH_TOKEN_VAR, } from "@naisys/common";
 import { HubEvents } from "@naisys/hub-protocol";
 import dotenv from "dotenv";
 /** Pushes the global config to NAISYS instances when they connect or when variables change */
@@ -7,10 +7,13 @@ export async function createHubConfigService(naisysServer, { hubDb }, logService
         success: false,
         error: "Not yet loaded",
     };
-    // API key variable names referenced by built-in models — always sensitive
+    // API key variable names referenced by built-in models — always sensitive.
+    // EXPIRES_AT is a timestamp, not a credential — keep it out so nearby
+    // millisecond values in logs don't get rewritten as redactions.
     const sensitiveKeys = new Set([...builtInLlmModels, ...builtInImageModels]
         .map((m) => m.apiKeyVar)
-        .filter(Boolean));
+        .filter(Boolean)
+        .concat([OPENAI_CODEX_ACCESS_TOKEN_VAR, OPENAI_CODEX_REFRESH_TOKEN_VAR]));
     // Seed DB from .env on first run
     const existing = await hubDb.variables.findMany();
     if (existing.length > 0) {
@@ -40,6 +43,7 @@ export async function createHubConfigService(naisysServer, { hubDb }, logService
         { key: "SPEND_LIMIT_DOLLARS" },
         { key: "SPEND_LIMIT_HOURS" },
         { key: "TARGET_VERSION" },
+        { key: "MAIL_ENABLED" },
     ]);
     /** Read variables from DB and build a ConfigResponse */
     async function buildConfigPayload() {
@@ -92,6 +96,7 @@ export async function createHubConfigService(naisysServer, { hubDb }, logService
     await buildConfigPayload();
     return {
         getConfig: () => cachedConfig,
+        broadcastConfig,
     };
 }
 /** Create variable placeholders if they don't already exist.

package/dist/handlers/hubCostService.js

@@ -3,9 +3,10 @@ import { CostWriteRequestSchema, HubEvents, } from "@naisys/hub-protocol";
 const SPEND_LIMIT_CHECK_INTERVAL_MS = 10_000;
 /** Handles cost_write events from NAISYS instances (fire-and-forget) */
 export function createHubCostService(naisysServer, { hubDb }, logService, heartbeatService, configService) {
-    // Track which users have been suspended due to spend limit overrun
-    const suspendedByGlobal = new Set();
-    const suspendedByAgent = new Set();
+    // Suspended users → original suspension reason. Stored so a defensive
+    // re-send (when a suspended user keeps writing costs) can use the same text.
+    const suspendedByGlobal = new Map();
+    const suspendedByAgent = new Map();
     naisysServer.registerEvent(HubEvents.COST_WRITE, async (hostId, data, ack) => {
         try {
             const parsed = CostWriteRequestSchema.parse(data);
@@ -14,10 +15,14 @@ export function createHubCostService(naisysServer, { hubDb }, logService, heartb
             const costPushMap = new Map();
             const userCostTotals = new Map();
             for (const entry of parsed.entries) {
+                const subagentId = entry.subagentId ?? 0;
+                // Wire format: undefined for parent (subagent_id 0 in DB), number otherwise
+                const wireSubagentId = subagentId === 0 ? undefined : subagentId;
                 await hubDb.costs.create({
                     data: {
                         user_id: entry.userId,
                         run_id: entry.runId,
+                        subagent_id: subagentId,
                         session_id: entry.sessionId,
                         host_id: hostId,
                         source: entry.source,
@@ -34,13 +39,14 @@ export function createHubCostService(naisysServer, { hubDb }, logService, heartb
                     where: {
                         user_id: entry.userId,
                         run_id: entry.runId,
+                        subagent_id: subagentId,
                         session_id: entry.sessionId,
                     },
                     data: {
                         total_cost: { increment: entry.cost },
                     },
                 });
-                const key = `${entry.userId}:${entry.runId}:${entry.sessionId}`;
+                const key = `${entry.userId}:${entry.runId}:${subagentId}:${entry.sessionId}`;
                 const existing = costPushMap.get(key);
                 if (existing) {
                     existing.costDelta += entry.cost;
@@ -49,6 +55,7 @@ export function createHubCostService(naisysServer, { hubDb }, logService, heartb
                     costPushMap.set(key, {
                         userId: entry.userId,
                         runId: entry.runId,
+                        subagentId: wireSubagentId,
                         sessionId: entry.sessionId,
                         costDelta: entry.cost,
                     });
@@ -61,10 +68,12 @@ export function createHubCostService(naisysServer, { hubDb }, logService, heartb
                     entries: Array.from(costPushMap.values()),
                 });
             }
-            // Re-send cost_control to any suspended users still writing costs
+            // Re-send cost_control to suspended users still writing costs.
+            // Per-agent reason wins: a per-agent limit overrides the global one.
             for (const userId of userCostTotals.keys()) {
-                if (suspendedByGlobal.has(userId) || suspendedByAgent.has(userId)) {
-                    sendCostControl(userId, false, "Spend limit exceeded");
+                const reason = suspendedByAgent.get(userId) ?? suspendedByGlobal.get(userId);
+                if (reason !== undefined) {
+                    sendCostControl(userId, false, reason);
                 }
             }
             // Decrement budget_left and return updated values
@@ -83,9 +92,9 @@ export function createHubCostService(naisysServer, { hubDb }, logService, heartb
     async function checkSpendLimits(candidateUserIds) {
         const activeUserIds = heartbeatService.getActiveUserIds();
         const usersToCheck = new Set(activeUserIds);
-        for (const userId of suspendedByGlobal)
+        for (const userId of suspendedByGlobal.keys())
             usersToCheck.add(userId);
-        for (const userId of suspendedByAgent)
+        for (const userId of suspendedByAgent.keys())
             usersToCheck.add(userId);
         if (candidateUserIds) {
             for (const userId of candidateUserIds)
@@ -209,7 +218,7 @@ export function createHubCostService(naisysServer, { hubDb }, logService, heartb
                 const reason = `Global spend limit of $${spendLimit} reached (total: $${totalCost.toFixed(2)})`;
                 logService.log(`[Hub:Costs] Suspending user ${userId} (global limit): ${reason}`);
                 sendCostControl(userId, false, reason);
-                suspendedByGlobal.add(userId);
+                suspendedByGlobal.set(userId, reason);
                 await setCostSuspendedReason(hubDb, userId, reason);
             }
             else if (!isOverLimit && wasSuspended) {
@@ -232,7 +241,7 @@ export function createHubCostService(naisysServer, { hubDb }, logService, heartb
             const reason = `Spend limit of $${spendLimit} reached (current: $${periodCost.toFixed(2)})`;
             logService.log(`[Hub:Costs] Suspending user ${userId}: ${reason}`);
             sendCostControl(userId, false, reason);
-            suspendedByAgent.add(userId);
+            suspendedByAgent.set(userId, reason);
             await setCostSuspendedReason(hubDb, userId, reason);
         }
         else if (!isOverLimit && wasSuspended) {

package/dist/handlers/hubCostService.test.js

@@ -0,0 +1,263 @@
+import { buildDefaultAgentConfig } from "@naisys/common";
+import { HubEvents } from "@naisys/hub-protocol";
+import { describe, expect, test, vi } from "vitest";
+import { createHubCostService } from "./hubCostService.js";
+function createServerHarness() {
+    const handlers = new Map();
+    const server = {
+        registerEvent: vi.fn((event, handler) => {
+            handlers.set(event, handler);
+        }),
+        broadcastToSupervisors: vi.fn(),
+        sendMessage: vi.fn(() => true),
+    };
+    async function emitCostWrite(hostId, data) {
+        const handler = handlers.get(HubEvents.COST_WRITE);
+        if (!handler)
+            throw new Error("COST_WRITE handler was not registered");
+        let ackResponse;
+        await handler(hostId, data, (response) => {
+            ackResponse = response;
+        });
+        return ackResponse;
+    }
+    return { server, emitCostWrite };
+}
+function createHubDb() {
+    const budgetLeft = new Map([[1, 5]]);
+    const hubDb = {
+        costs: {
+            create: vi.fn(() => Promise.resolve({})),
+            aggregate: vi.fn(() => Promise.resolve({ _sum: { cost: 0 } })),
+        },
+        run_session: {
+            updateMany: vi.fn(() => Promise.resolve({})),
+        },
+        users: {
+            findMany: vi.fn(() => Promise.resolve([])),
+        },
+        user_notifications: {
+            findUnique: vi.fn(({ where }) => {
+                const value = budgetLeft.get(where.user_id);
+                return Promise.resolve(value === undefined ? null : { budget_left: value });
+            }),
+            update: vi.fn(({ where, data, }) => {
+                budgetLeft.set(where.user_id, data.budget_left);
+                return Promise.resolve({ budget_left: data.budget_left });
+            }),
+            updateMany: vi.fn(() => Promise.resolve({})),
+        },
+    };
+    return { hubDb, budgetLeft };
+}
+function createLogger() {
+    return {
+        log: vi.fn(),
+        error: vi.fn(),
+        disableConsole: vi.fn(),
+    };
+}
+function createHeartbeatService(activeUserIds, hostIdsByUser = new Map([[1, [101]]])) {
+    return {
+        getActiveUserIds: vi.fn(() => activeUserIds),
+        findHostsForAgent: vi.fn((userId) => hostIdsByUser.get(userId) ?? []),
+    };
+}
+function createConfigService(config) {
+    return {
+        getConfig: vi.fn(() => ({ config })),
+    };
+}
+function userRow(id, configOverrides, spendLimitResetAt) {
+    return {
+        id,
+        config: JSON.stringify({
+            ...buildDefaultAgentConfig(`agent-${id}`),
+            ...configOverrides,
+        }),
+        user_notifications: {
+            spend_limit_reset_at: spendLimitResetAt,
+        },
+    };
+}
+describe("hubCostService", () => {
+    test("persists subagent cost entries, pushes scoped deltas, and decrements budget", async () => {
+        const { server, emitCostWrite } = createServerHarness();
+        const { hubDb } = createHubDb();
+        const logger = createLogger();
+        const heartbeatService = createHeartbeatService([]);
+        const configService = createConfigService({});
+        const service = createHubCostService(server, { hubDb }, logger, heartbeatService, configService);
+        const ack = await emitCostWrite(42, {
+            entries: [
+                {
+                    userId: 1,
+                    runId: 7,
+                    subagentId: -1,
+                    sessionId: 1,
+                    source: "genimg",
+                    model: "mock-image",
+                    cost: 0.5,
+                    inputTokens: 0,
+                    outputTokens: 0,
+                    cacheWriteTokens: 0,
+                    cacheReadTokens: 0,
+                },
+                {
+                    userId: 1,
+                    runId: 7,
+                    subagentId: -1,
+                    sessionId: 1,
+                    source: "genimg",
+                    model: "mock-image",
+                    cost: 0.25,
+                    inputTokens: 0,
+                    outputTokens: 0,
+                    cacheWriteTokens: 0,
+                    cacheReadTokens: 0,
+                },
+            ],
+        });
+        expect(hubDb.costs.create).toHaveBeenCalledWith({
+            data: expect.objectContaining({
+                user_id: 1,
+                run_id: 7,
+                subagent_id: -1,
+                session_id: 1,
+                host_id: 42,
+                cost: 0.5,
+            }),
+        });
+        expect(hubDb.run_session.updateMany).toHaveBeenCalledWith({
+            where: {
+                user_id: 1,
+                run_id: 7,
+                subagent_id: -1,
+                session_id: 1,
+            },
+            data: {
+                total_cost: { increment: 0.5 },
+            },
+        });
+        expect(server.broadcastToSupervisors).toHaveBeenCalledWith(HubEvents.COST_PUSH, {
+            entries: [
+                {
+                    userId: 1,
+                    runId: 7,
+                    subagentId: -1,
+                    sessionId: 1,
+                    costDelta: 0.75,
+                },
+            ],
+        });
+        expect(hubDb.user_notifications.update).toHaveBeenCalledWith({
+            where: { user_id: 1 },
+            data: { budget_left: 4.25 },
+        });
+        expect(ack).toEqual({ budgets: [{ userId: 1, budgetLeft: 4.25 }] });
+        service.cleanup();
+    });
+    test("normalizes parent-agent subagentId to undefined on the wire (DB row stays 0)", async () => {
+        const { server, emitCostWrite } = createServerHarness();
+        const { hubDb } = createHubDb();
+        const logger = createLogger();
+        const heartbeatService = createHeartbeatService([]);
+        const configService = createConfigService({});
+        const service = createHubCostService(server, { hubDb }, logger, heartbeatService, configService);
+        await emitCostWrite(42, {
+            entries: [
+                {
+                    userId: 1,
+                    runId: 7,
+                    // subagentId omitted — represents the parent agent
+                    sessionId: 1,
+                    source: "console",
+                    model: "mock",
+                    cost: 0.1,
+                    inputTokens: 0,
+                    outputTokens: 0,
+                    cacheWriteTokens: 0,
+                    cacheReadTokens: 0,
+                },
+            ],
+        });
+        expect(hubDb.costs.create).toHaveBeenCalledWith({
+            data: expect.objectContaining({ subagent_id: 0 }),
+        });
+        expect(server.broadcastToSupervisors).toHaveBeenCalledWith(HubEvents.COST_PUSH, {
+            entries: [
+                {
+                    userId: 1,
+                    runId: 7,
+                    subagentId: undefined,
+                    sessionId: 1,
+                    costDelta: 0.1,
+                },
+            ],
+        });
+        service.cleanup();
+    });
+    test("suspends, re-sends, and resumes per-agent cost control", async () => {
+        const { server, emitCostWrite } = createServerHarness();
+        const { hubDb } = createHubDb();
+        const logger = createLogger();
+        const heartbeatService = createHeartbeatService([1]);
+        const configService = createConfigService({});
+        vi.mocked(hubDb.users.findMany).mockResolvedValue([
+            userRow(1, { spendLimitDollars: 1 }),
+        ]);
+        vi.mocked(hubDb.costs.aggregate)
+            .mockResolvedValueOnce({ _sum: { cost: 1.25 } })
+            .mockResolvedValueOnce({ _sum: { cost: 0.25 } });
+        const service = createHubCostService(server, { hubDb }, logger, heartbeatService, configService);
+        await service.checkSpendLimits();
+        const suspendReason = "Spend limit of $1 reached (current: $1.25)";
+        expect(server.sendMessage).toHaveBeenCalledWith(101, HubEvents.COST_CONTROL, {
+            userId: 1,
+            enabled: false,
+            reason: suspendReason,
+        });
+        expect(service.isUserSpendSuspended(1)).toBe(true);
+        expect(hubDb.user_notifications.updateMany).toHaveBeenCalledWith({
+            where: { user_id: 1 },
+            data: { cost_suspended_reason: suspendReason },
+        });
+        vi.mocked(server.sendMessage).mockClear();
+        await emitCostWrite(42, {
+            entries: [
+                {
+                    userId: 1,
+                    runId: 7,
+                    subagentId: -1,
+                    sessionId: 1,
+                    source: "genimg",
+                    model: "mock-image",
+                    cost: 0.1,
+                    inputTokens: 0,
+                    outputTokens: 0,
+                    cacheWriteTokens: 0,
+                    cacheReadTokens: 0,
+                },
+            ],
+        });
+        expect(server.sendMessage).toHaveBeenCalledWith(101, HubEvents.COST_CONTROL, {
+            userId: 1,
+            enabled: false,
+            reason: suspendReason,
+        });
+        vi.mocked(server.sendMessage).mockClear();
+        await service.checkSpendLimits();
+        expect(server.sendMessage).toHaveBeenCalledWith(101, HubEvents.COST_CONTROL, {
+            userId: 1,
+            enabled: true,
+            reason: "Spend limit period reset (current: $0.25, limit: $1)",
+        });
+        expect(service.isUserSpendSuspended(1)).toBe(false);
+        expect(hubDb.user_notifications.updateMany).toHaveBeenCalledWith({
+            where: { user_id: 1 },
+            data: { cost_suspended_reason: null },
+        });
+        service.cleanup();
+    });
+});
+//# sourceMappingURL=hubCostService.test.js.map