@camstack/server 1.2.564 → 1.2.566

package/dist/agent/main.js

@@ -50,6 +50,9 @@ const agent_config_js_1 = require("./agent-config.js");
 const agent_mesh_enroll_js_1 = require("./agent-mesh-enroll.js");
 const agent_group_runner_js_1 = require("./agent-group-runner.js");
 const agent_http_js_1 = require("./agent-http.js");
+const agent_node_certificates_js_1 = require("./agent-node-certificates.js");
+const agent_reconnect_js_1 = require("./agent-reconnect.js");
+const serial_queue_js_1 = require("./serial-queue.js");
 const agent_infra_capabilities_js_1 = require("./agent-infra-capabilities.js");
 const agent_service_js_1 = require("./agent-service.js");
 const agent_hub_addon_update_js_1 = require("./agent-hub-addon-update.js");
@@ -65,6 +68,8 @@ const register_agent_cap_dispatch_js_1 = require("./register-agent-cap-dispatch.
 // entries flow through it (console output + hub forwarding).
 // ---------------------------------------------------------------------------
 const system_2 = require("@camstack/system");
+/** A reconnect (stop + rebuild + start) that takes longer than this is abandoned. */
+const RECONNECT_DEADLINE_MS = 90_000;
 const agentLogManager = new system_2.LogManager(5000);
 /**
  * Caps whose `nodeId` arg is provider DATA (`nodeIdMode: 'data'` —
@@ -205,6 +210,8 @@ async function startAgent(configPath) {
     // Mesh mTLS (Release A): with a certificate the broker dials the hub's TLS
     // port; without one it keeps dialing clear 6000.
     let currentMeshTls = config.meshTls ?? null;
+    // The HTTP/HTTPS listener on the agent port (D856); set once it is up.
+    let agentHttpListener = null;
     console.log(`[Agent] Mesh transport: ${currentMeshTls !== null ? 'tls' : 'clear'}`);
     let broker = (0, system_1.createBroker)({
         nodeID: config.nodeId,
@@ -220,45 +227,74 @@ async function startAgent(configPath) {
      * create a new broker with the updated hub/secret, and restart.
      * The HTTP server stays alive throughout.
      */
-    const reconnect = async () => {
-        console.log('[Agent] Reconnecting with updated config...');
-        try {
-            await broker.stop();
-        }
-        catch {
-            /* already stopped */
-        }
-        // Reload config from file (UI may have written new hubAddress/secret)
-        const fresh = (0, agent_config_js_1.loadAgentConfig)(undefined, config.dataDir);
-        console.log(`[Agent] New config: hub=${fresh.hubAddress ?? 'none'}, secret=${fresh.secret ? 'yes' : 'none'}, ` +
-            `mesh=${fresh.meshTls !== undefined ? 'tls' : 'clear'}`);
-        // Re-derive CAMSTACK_HUB_URL when the dashboard has just written a hubAddress. Explicit operator values
-        // still win via the boot-captured `hubUrlWasExplicit` flag.
-        // KNOWN LIMITATION (Stage-0): runners already spawned before this reconnect
-        // keep the env they inherited; picking up a changed hub address in live
-        // children would need a runner respawn. Boot-time — the actual trap — is
-        // fully covered.
-        const freshHubUrl = (0, derive_hub_url_js_1.deriveHubUrlForExport)(hubUrlWasExplicit, fresh.hubAddress);
-        if (freshHubUrl !== undefined && process.env['CAMSTACK_HUB_URL'] !== freshHubUrl) {
-            process.env['CAMSTACK_HUB_URL'] = freshHubUrl;
-            console.log(`[Agent] Derived CAMSTACK_HUB_URL=${freshHubUrl} from hub address "${fresh.hubAddress}"`);
-        }
-        currentSecret = fresh.secret ?? null;
-        currentMeshTls = fresh.meshTls ?? null;
-        broker = (0, system_1.createBroker)({
-            nodeID: config.nodeId,
-            mode: 'agent',
-            hubAddress: fresh.hubAddress,
-            logLevel: fresh.logLevel,
-            middlewares: meshJoin.middlewares,
-            ...(currentMeshTls !== null ? { meshTls: currentMeshTls } : {}),
-        });
-        meshJoin.attach(broker);
+    // ONE reconnect at a time (serial-queue.ts): the UI, a mesh leaf swap and its rollback can
+    // all ask for one, and two interleaved stop/rebuild cycles would leave two brokers or none.
+    const reconnect = (0, serial_queue_js_1.createSerialQueue)(() => reconnectNow(), {
+        deadlineMs: RECONNECT_DEADLINE_MS,
+        onTimeout: () => {
+            // A broker whose start() never settles must not hold every later reconnect (including
+            // the rollback of a failed leaf swap): tear down what was half-built and move on.
+            consoleLogger.error('Reconnect did not finish in time - destroying the half-built broker', {
+                meta: { deadlineMs: RECONNECT_DEADLINE_MS },
+            });
+            void broker.stop().catch(() => undefined);
+        },
+    });
+    // The cycle itself runs through a generation guard: a run the queue abandoned at its deadline
+    // that resumes later builds nothing (agent-reconnect.ts), so two brokers never share a node id.
+    let pendingConfig = null;
+    const reconnectNow = (0, agent_reconnect_js_1.createReconnectRunner)({
+        current: () => broker,
+        assign: (next) => {
+            broker = next;
+        },
+        stop: (target) => target.stop(),
+        prepare: () => {
+            console.log('[Agent] Reconnecting with updated config...');
+            // Reload config from file (UI may have written new hubAddress/secret)
+            const fresh = (0, agent_config_js_1.loadAgentConfig)(undefined, config.dataDir);
+            pendingConfig = fresh;
+            console.log(`[Agent] New config: hub=${fresh.hubAddress ?? 'none'}, secret=${fresh.secret ? 'yes' : 'none'}, ` +
+                `mesh=${fresh.meshTls !== undefined ? 'tls' : 'clear'}`);
+            // Re-derive CAMSTACK_HUB_URL when the dashboard has just written a hubAddress. Explicit operator values
+            // still win via the boot-captured `hubUrlWasExplicit` flag.
+            // KNOWN LIMITATION (Stage-0): runners already spawned before this reconnect
+            // keep the env they inherited; picking up a changed hub address in live
+            // children would need a runner respawn. Boot-time — the actual trap — is
+            // fully covered.
+            const freshHubUrl = (0, derive_hub_url_js_1.deriveHubUrlForExport)(hubUrlWasExplicit, fresh.hubAddress);
+            if (freshHubUrl !== undefined && process.env['CAMSTACK_HUB_URL'] !== freshHubUrl) {
+                process.env['CAMSTACK_HUB_URL'] = freshHubUrl;
+                console.log(`[Agent] Derived CAMSTACK_HUB_URL=${freshHubUrl} from hub address "${fresh.hubAddress}"`);
+            }
+            currentSecret = fresh.secret ?? null;
+            currentMeshTls = fresh.meshTls ?? null;
+        },
+        build: () => {
+            const fresh = pendingConfig;
+            if (fresh === null)
+                throw new Error('reconnect: no configuration was prepared');
+            const next = (0, system_1.createBroker)({
+                nodeID: config.nodeId,
+                mode: 'agent',
+                hubAddress: fresh.hubAddress,
+                logLevel: fresh.logLevel,
+                middlewares: meshJoin.middlewares,
+                ...(currentMeshTls !== null ? { meshTls: currentMeshTls } : {}),
+            });
+            meshJoin.attach(next);
+            return next;
+        },
         // The new broker's hub connection runs a join, and its settle re-sends
         // the manifest (`registration.onJoinSettled`).
-        await broker.start();
-        console.log('[Agent] Reconnected successfully');
-    };
+        start: async (next) => {
+            await next.start();
+            console.log('[Agent] Reconnected successfully');
+        },
+        onSuperseded: (stage) => consoleLogger.warn('A superseded reconnect stopped without building a second broker', {
+            meta: { stage },
+        }),
+    });
     const loadedAddons = new Map();
     // D3 subtree registry: accumulates manifests from every group-runner child
     // that calls `$agent.registerNode`. The agent re-registers the UNION of its
@@ -444,6 +480,9 @@ async function startAgent(configPath) {
         types_1.audioAnalyzerCapability,
         types_1.platformProbeCapability,
         types_1.serverManagementCapability,
+        // Certificates of THIS node, called by the hub (D856): mesh leaf renewal and the
+        // hub-issued HTTPS leaf. Provided by the runtime, like `server-management`.
+        types_1.nodeCertificatesCapability,
     ];
     for (const cap of agentCapabilities) {
         capabilityRegistry.declareCapability(cap);
@@ -460,6 +499,22 @@ async function startAgent(configPath) {
         dataDir: config.dataDir,
     });
     capabilityRegistry.registerProvider('server-management', agent_update_service_js_1.AGENT_RUNTIME_ADDON_ID, (0, server_management_provider_js_1.buildServerManagementProvider)(agentUpdateService));
+    // The agent's certificates (D856): mesh leaf swap with automatic rollback, and the
+    // hub-issued HTTPS leaf hot-swapped into the sniffing listener. The listener is
+    // created later in boot, so it is read through a getter.
+    capabilityRegistry.registerProvider('node-certificates', agent_update_service_js_1.AGENT_RUNTIME_ADDON_ID, (0, agent_node_certificates_js_1.createAgentNodeCertificates)({
+        nodeId: config.nodeId,
+        dataDir: config.dataDir,
+        configPath: config.configPath,
+        reconnect,
+        getHttpsListener: () => agentHttpListener,
+        hostFacts: (0, system_2.systemHostFacts)(),
+        log: {
+            info: (message, extras) => consoleLogger.info(message, extras),
+            warn: (message, extras) => consoleLogger.warn(message, extras),
+            error: (message, extras) => consoleLogger.error(message, extras),
+        },
+    }));
     // Boot-health confirmation: on the agent's FIRST acked `$hub.registerNode`,
     // run the single-copy GC sweep once. Under the single-copy model there is NO
     // probation boot, NO N-1 promotion, and NO auto-rollback — the starter
@@ -838,6 +893,11 @@ async function startAgent(configPath) {
         },
     };
     void (0, agent_http_js_1.startAgentHttpServer)(getBrokerFn, {
+        log: {
+            info: (message, extras) => consoleLogger.info(message, extras),
+            warn: (message, extras) => consoleLogger.warn(message, extras),
+            error: (message, extras) => consoleLogger.error(message, extras),
+        },
         port: config.statusPort ?? 4444,
         nodeId: config.nodeId,
         dataDir: config.dataDir,
@@ -846,6 +906,8 @@ async function startAgent(configPath) {
         getHubConnectionState,
         getMeshTransport: () => (currentMeshTls !== null ? 'tls' : 'clear'),
         updateControls,
+    }).then((listener) => {
+        agentHttpListener = listener;
     });
     // ── Phase 1: Load core infrastructure addons (in-process) ──
     // storage + settings + metrics + hub-forwarder (log destination)
@@ -922,6 +984,8 @@ async function startAgent(configPath) {
                 }
             }
         }
+        // Stop accepting new HTTP/HTTPS connections before the broker goes away.
+        await agentHttpListener?.close().catch(() => undefined);
         await broker.stop();
         process.exit(0);
     };

package/dist/agent/serial-queue.js

@@ -0,0 +1,49 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.SerialQueueTimeoutError = void 0;
+exports.createSerialQueue = createSerialQueue;
+class SerialQueueTimeoutError extends Error {
+    deadlineMs;
+    constructor(deadlineMs) {
+        super(`the job did not finish within ${deadlineMs} ms and was abandoned`);
+        this.deadlineMs = deadlineMs;
+        this.name = 'SerialQueueTimeoutError';
+    }
+}
+exports.SerialQueueTimeoutError = SerialQueueTimeoutError;
+function defaultSetTimer(fn, ms) {
+    const timer = setTimeout(fn, ms);
+    timer.unref?.();
+    return { cancel: () => clearTimeout(timer) };
+}
+function createSerialQueue(job, options = {}) {
+    let tail = Promise.resolve();
+    const setTimer = options.setTimer ?? defaultSetTimer;
+    const guarded = () => {
+        const deadlineMs = options.deadlineMs;
+        if (deadlineMs === undefined)
+            return job();
+        return new Promise((resolve, reject) => {
+            const timer = setTimer(() => {
+                try {
+                    options.onTimeout?.();
+                }
+                finally {
+                    reject(new SerialQueueTimeoutError(deadlineMs));
+                }
+            }, deadlineMs);
+            job().then((value) => {
+                timer.cancel();
+                resolve(value);
+            }, (error) => {
+                timer.cancel();
+                reject(error);
+            });
+        });
+    };
+    return () => {
+        const run = tail.then(guarded, guarded);
+        tail = run.catch(() => undefined);
+        return run;
+    };
+}