@andersbakken/fisk 5.0.14 → 5.0.17

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@andersbakken/fisk",
-    "version": "5.0.14",
+    "version": "5.0.17",
     "description": "Fisk, a distributed compile system",
     "scripts": {
         "lint": "eslint . --ext .ts",

package/scheduler/fisk-scheduler.js

@@ -4426,10 +4426,9 @@ class Client extends require$$0__default$2["default"] {
         this.npmVersion = "";
         this.user = "";
         this.port = 0;
-        this.ws.on("pong", () => {
-            // console.log("got pong", this.name);
-            this.pingSent = undefined;
-        });
+    }
+    notePong() {
+        this.pingSent = undefined;
     }
     send(type, msg) {
         try {
@@ -4472,8 +4471,12 @@ class Client extends require$$0__default$2["default"] {
     error(message) {
         try {
             this.ws.send(`{"error": "${message}"}`);
-            this.ws.close();
+            // Emit before closing. A real websocket closes asynchronously, but a
+            // DaemonJobSocket does it synchronously and tears the job down -- and
+            // removes these listeners -- so closing first means nobody ever hears
+            // this, including whoever releases the job's counters.
             this.emit("error", message);
+            this.ws.close();
         }
         catch (err) {
             /* */
@@ -4506,6 +4509,115 @@ class Compile extends Client {
         this.environment = environment;
         this.sourcePath = sourcePath;
         this.sha1 = sha1;
+        // A job relayed by a daemon cannot stream an environment tarball to us: it is
+        // one multiplexed stream on a connection shared with every other compile on
+        // that host. Such a client reconnects directly to upload instead.
+        this.canUploadEnvironment = true;
+    }
+}
+
+// One fiskc job on a daemon's shared connection. Everything the scheduler sends
+// to "its client" is framed onto that connection by relay(), and closing it (a
+// version mismatch, Client.error()) has to be reported to the daemon rather than
+// dropping a TCP connection, or the fiskc process would wait for its watchdog.
+class DaemonJobSocket {
+    constructor(relay, closed) {
+        this.relay = relay;
+        this.closed = closed;
+        this.open = true;
+    }
+    send(data) {
+        if (!this.open) {
+            return;
+        }
+        if (typeof data === "string") {
+            this.relay(data);
+        }
+        else if (data instanceof Buffer) {
+            this.relay(data.toString("utf8"));
+        }
+        else {
+            console.error("Unexpected payload for a daemon job", typeof data);
+        }
+    }
+    close(code, reason) {
+        if (!this.open) {
+            return;
+        }
+        this.open = false;
+        this.closed(String(reason || `closed with ${code === undefined ? 1000 : code}`));
+    }
+    // eslint-disable-next-line class-methods-use-this
+    ping() {
+        // Liveness is a property of the daemon's connection, not of a single job.
+    }
+    terminate() {
+        this.close(1006, "terminated");
+    }
+}
+
+function decode$3(message) {
+    try {
+        return JSON.parse(message);
+    }
+    catch (err) {
+        return { error: `Unserializable scheduler message: ${message}` };
+    }
+}
+class DaemonConnection extends Client {
+    constructor(ws, ip, option) {
+        super(5 /* ClientType.Daemon */, ws, ip, option);
+        this.jobs = new Map();
+    }
+    get activeJobs() {
+        return this.jobs.size;
+    }
+    createJob(request) {
+        var _a;
+        if (this.jobs.has(request.id)) {
+            return undefined;
+        }
+        const socket = new DaemonJobSocket((message) => {
+            this.send({ type: "jobMessage", id: request.id, message: decode$3(message) });
+        }, (reason) => {
+            this.finishJob(request.id, reason, true);
+        });
+        const compile = new Compile(socket, this.ip, request.environment, request.sourcePath, request.sha1, this.option);
+        compile.canUploadEnvironment = false;
+        compile.npmVersion = request.npmVersion || "";
+        compile.name = request.name || this.name;
+        compile.hostname = request.hostname || this.hostname;
+        compile.user = request.user || "";
+        if (request.builder) {
+            compile.builder = request.builder;
+        }
+        if ((_a = request.labels) === null || _a === void 0 ? void 0 : _a.length) {
+            compile.labels = request.labels;
+        }
+        this.jobs.set(request.id, compile);
+        return compile;
+    }
+    // The end of a job is a message now, not a socket close, so every exit has to
+    // funnel through here: the daemon saying the fiskc process is gone, the
+    // scheduler closing the job itself, or this whole connection dropping. Missing
+    // one leaks a builder's activeClients slot and the scheduler's activeJobs
+    // count for as long as the scheduler runs.
+    finishJob(id, reason, notifyDaemon) {
+        const compile = this.jobs.get(id);
+        if (!compile) {
+            return;
+        }
+        this.jobs.delete(id);
+        if (notifyDaemon) {
+            this.send({ type: "jobClosed", id, reason });
+        }
+        compile.emit("close", { code: 1000, reason });
+        compile.removeAllListeners();
+    }
+    finishAllJobs(reason) {
+        for (const id of Array.from(this.jobs.keys())) {
+            this.finishJob(id, reason, false);
+        }
     }
 }
 
@@ -56206,17 +56318,20 @@ class Server extends require$$0__default$2["default"] {
         }
         this.emit("compile", client);
         const remaining = {};
-        client.ws.on("error", (err) => client.emit("error", err));
-        client.ws.on("close", (code, reason) => {
+        ws.on("pong", () => {
+            client.notePong();
+        });
+        ws.on("error", (err) => client.emit("error", err));
+        ws.on("close", (code, reason) => {
             if (remaining.bytes) {
                 client.emit("error", "Got close while reading a binary message");
             }
             if (client) {
                 client.emit("close", { code: code, reason: reason });
             }
-            client.ws.removeAllListeners();
+            ws.removeAllListeners();
         });
-        client.ws.on("message", (msg) => {
+        ws.on("message", (msg) => {
             switch (typeof msg) {
                 case "string": {
                     if (remaining.bytes) {
@@ -56284,14 +56399,18 @@ class Server extends require$$0__default$2["default"] {
             }
         });
     }
-    _handleBuilder(req, client) {
-        client.ws.on("close", (code, reason) => {
+    _handleBuilder(req, ws, ip) {
+        const client = new Builder(ws, ip);
+        ws.on("pong", () => {
+            client.notePong();
+        });
+        ws.on("close", (code, reason) => {
             client.emit("close", { code: code, reason: reason });
-            client.ws.removeAllListeners();
+            ws.removeAllListeners();
         });
-        client.ws.on("error", () => {
+        ws.on("error", () => {
             client.emit("close", { code: 1005, reason: "unknown" });
-            client.ws.removeAllListeners();
+            ws.removeAllListeners();
         });
         if (!("x-fisk-port" in req.headers)) {
             client.error("No x-fisk-port header");
@@ -56328,7 +56447,7 @@ class Server extends require$$0__default$2["default"] {
                 client.environments[env] = true;
             }
         });
-        client.ws.on("message", (msg) => {
+        ws.on("message", (msg) => {
             // console.log("Got message from builder", typeof msg, msg.length);
             switch (typeof msg) {
                 case "string": {
@@ -56364,28 +56483,128 @@ class Server extends require$$0__default$2["default"] {
         // console.log("Got dude", client);
         this.emit("builder", client);
     }
-    _handleMonitor(req, client) {
+    _handleMonitor(req, ws, ip) {
+        const client = new Client(3 /* ClientType.Monitor */, ws, ip);
         client.nonce = this.nonces.get(req);
         // console.log("Got nonce", req.nonce);
-        client.ws.on("message", (message) => client.emit("message", message));
+        ws.on("pong", () => {
+            client.notePong();
+        });
+        ws.on("message", (message) => client.emit("message", message));
         this.emit("monitor", client);
-        client.ws.on("close", (code, reason) => {
-            client.ws.removeAllListeners();
+        ws.on("close", (code, reason) => {
+            ws.removeAllListeners();
             client.emit("close", { code: code, reason: reason });
         });
-        client.ws.on("error", (err) => client.emit("error", err));
+        ws.on("error", (err) => client.emit("error", err));
     }
-    _handleClientVerify(req, client) {
+    _handleClientVerify(req, ws, ip) {
+        const client = new Client(4 /* ClientType.ClientVerify */, ws, ip);
         Object.assign(client, { npmVersion: header(req, "x-fisk-npm-version") });
         this.emit("clientVerify", client);
-        client.ws.on("close", (code, reason) => {
-            client.ws.removeAllListeners();
+        ws.on("pong", () => {
+            client.notePong();
+        });
+        ws.on("close", (code, reason) => {
+            ws.removeAllListeners();
             client.emit("close", { code: code, reason: reason });
         });
-        client.ws.on("error", (err) => client.emit("error", err));
+        ws.on("error", (err) => client.emit("error", err));
+    }
+    // One connection per host, shared by every fiskc process on it. fiskc used to
+    // open its own websocket per translation unit, which is a TCP connect plus an
+    // HTTP upgrade per compile; when this scheduler's event loop stalls the listen
+    // backlog fills, SYNs get dropped and those clients time out and compile
+    // locally instead. Jobs are multiplexed here as messages tagged with the
+    // daemon's request id, and each one gets a Compile that behaves like any other
+    // client so the scheduling and accounting code below stays untouched.
+    _handleDaemon(req, ws, ip) {
+        const configVersion = parseInt(header(req, "x-fisk-config-version") || "");
+        if (configVersion !== this.configVersion) {
+            ws.send(`{"error": "Bad config version, expected ${this.configVersion}, got ${configVersion}"}`);
+            ws.close();
+            return;
+        }
+        const connection = new DaemonConnection(ws, ip, this.option);
+        connection.name = header(req, "x-fisk-daemon-name") || "";
+        connection.hostname = header(req, "x-fisk-daemon-hostname") || "";
+        connection.npmVersion = header(req, "x-fisk-npm-version") || "";
+        ws.on("pong", () => {
+            connection.notePong();
+        });
+        ws.on("close", (code, reason) => {
+            connection.finishAllJobs(`daemon ${ip} disconnected: ${code} ${String(reason)}`);
+            ws.removeAllListeners();
+            connection.emit("close", { code: code, reason: reason });
+        });
+        ws.on("error", (err) => {
+            connection.finishAllJobs(`daemon ${ip} error: ${err.message}`);
+            connection.emit("error", err);
+        });
+        ws.on("message", (msg) => {
+            if (typeof msg !== "string") {
+                console.error("Unexpected binary message from daemon", ip);
+                return;
+            }
+            let json;
+            try {
+                json = JSON.parse(msg);
+            }
+            catch (err) {
+                console.error(`Unable to parse message from daemon ${ip} as JSON`, err);
+                return;
+            }
+            if (!json || typeof json.id !== "number") {
+                console.error("Message from daemon without a job id", ip, json);
+                return;
+            }
+            const id = json.id;
+            switch (json.type) {
+                case "compileRequest": {
+                    if (typeof json.environment !== "string" || !json.environment) {
+                        connection.send({ type: "jobMessage", id, message: { error: "No environment" } });
+                        return;
+                    }
+                    if (typeof json.sourceFile !== "string" || !json.sourceFile) {
+                        connection.send({ type: "jobMessage", id, message: { error: "No sourceFile" } });
+                        return;
+                    }
+                    const sha1 = typeof json.sha1 === "string" ? json.sha1 : undefined;
+                    if (sha1 && sha1.length !== 40) {
+                        connection.send({ type: "jobMessage", id, message: { error: `Bad sha1 sum: ${sha1}` } });
+                        return;
+                    }
+                    const labels = typeof json.labels === "string" ? json.labels.split(/ +/).filter((x) => x) : undefined;
+                    const compile = connection.createJob({
+                        id,
+                        environment: json.environment,
+                        sourcePath: json.sourceFile,
+                        sha1,
+                        name: typeof json.name === "string" ? json.name : undefined,
+                        user: typeof json.user === "string" ? json.user : undefined,
+                        hostname: typeof json.hostname === "string" ? json.hostname : undefined,
+                        builder: typeof json.builder === "string" ? json.builder : undefined,
+                        labels,
+                        npmVersion: typeof json.npmVersion === "string" ? json.npmVersion : undefined
+                    });
+                    if (!compile) {
+                        connection.send({ type: "jobMessage", id, message: { error: `Duplicate job id ${id}` } });
+                        return;
+                    }
+                    this.emit("compile", compile);
+                    break;
+                }
+                case "compileDone":
+                    connection.finishJob(id, typeof json.reason === "string" ? json.reason : "done", false);
+                    break;
+                default:
+                    console.error("Unexpected message type from daemon", ip, json.type);
+                    break;
+            }
+        });
+        this.emit("daemon", connection);
     }
     _handleConnection(ws, req) {
-        let client = undefined;
         let ip = req.connection.remoteAddress;
         // console.log("_handleConnection", ip);
         if (!ip) {
@@ -56402,16 +56621,16 @@ class Server extends require$$0__default$2["default"] {
                 this._handleCompile(req, ws, ip);
                 break;
             case "/builder":
-                client = new Builder(ws, ip);
-                this._handleBuilder(req, client);
+                this._handleBuilder(req, ws, ip);
                 break;
             case "/monitor":
-                client = new Client(3 /* ClientType.Monitor */, ws, ip);
-                this._handleMonitor(req, client);
+                this._handleMonitor(req, ws, ip);
                 break;
             case "/client_verify":
-                client = new Client(4 /* ClientType.ClientVerify */, ws, ip);
-                this._handleClientVerify(req, client);
+                this._handleClientVerify(req, ws, ip);
+                break;
+            case "/daemon":
+                this._handleDaemon(req, ws, ip);
                 break;
             default:
                 console.error(`Invalid pathname ${url.pathname} from: ${ip}`);
@@ -58095,9 +58314,9 @@ catch (err) {
     console.log("Couldn't parse package json", err);
     process.exit();
 }
-const builders = {};
+const builders = new Set();
+const daemons = new Set();
 const monitors = [];
-let builderCount = 0;
 let activeJobs = 0;
 let capacity = 0;
 let jobsFailed = 0;
@@ -58245,12 +58464,6 @@ function jobFinished(builder, job) {
         });
     }
 }
-function builderKey(ip, port) {
-    if (typeof ip === "object") {
-        return ip.ip + " " + ip.port;
-    }
-    return ip + " " + port;
-}
 function builderToMonitorInfo(builder, type) {
     return {
         type: type,
@@ -58270,8 +58483,7 @@ function builderToMonitorInfo(builder, type) {
     };
 }
 function insertBuilder(builder) {
-    builders[builderKey(builder)] = builder;
-    ++builderCount;
+    builders.add(builder);
     assert__default["default"](typeof builder.slots === "number");
     capacity += builder.slots;
     if (monitors.length) {
@@ -58285,8 +58497,8 @@ function insertBuilder(builder) {
     }
 }
 function forEachBuilder(cb) {
-    for (const key in builders) {
-        cb(builders[key]);
+    for (const builder of builders) {
+        cb(builder);
     }
 }
 function onObjectCacheCleared() {
@@ -58319,10 +58531,9 @@ if (option("object-cache")) {
     setObjectCacheEnabled(true);
 }
 function removeBuilder(builder) {
-    --builderCount;
     assert__default["default"](typeof builder.slots === "number");
     capacity -= builder.slots;
-    delete builders[builderKey(builder)];
+    builders.delete(builder);
     if (monitors.length) {
         const info = builderToMonitorInfo(builder, "builderRemoved");
         if (monitorsLog) {
@@ -58468,27 +58679,44 @@ server.on("listen", (app) => {
     app.get("/builders", (req, res) => {
         const ret = [];
         const now = Date.now();
-        for (const bKey in builders) {
-            const s = builders[bKey];
+        for (const builder of builders) {
+            ret.push({
+                ip: builder.ip,
+                name: builder.name,
+                labels: builder.labels,
+                slots: builder.slots,
+                port: builder.port,
+                activeClients: builder.activeClients,
+                jobsScheduled: builder.jobsScheduled,
+                lastJob: builder.lastJob ? new Date(builder.lastJob).toString() : "",
+                jobsPerformed: builder.jobsPerformed,
+                compileSpeed: builder.jobsPerformed / builder.totalCompileSpeed || 0,
+                uploadSpeed: builder.jobsPerformed / builder.totalUploadSpeed || 0,
+                hostname: builder.hostname,
+                system: builder.system,
+                created: builder.created,
+                load: builder.load,
+                uptime: now - builder.created.valueOf(),
+                npmVersion: builder.npmVersion,
+                environments: Object.keys(builder.environments)
+            });
+        }
+        const pretty = req.query && req.query.unpretty ? undefined : 4;
+        res.send(JSON.stringify(ret, null, pretty) + "\n");
+    });
+    app.get("/daemons", (req, res) => {
+        const ret = [];
+        const now = Date.now();
+        for (const daemon of daemons) {
             ret.push({
-                ip: s.ip,
-                name: s.name,
-                labels: s.labels,
-                slots: s.slots,
-                port: s.port,
-                activeClients: s.activeClients,
-                jobsScheduled: s.jobsScheduled,
-                lastJob: s.lastJob ? new Date(s.lastJob).toString() : "",
-                jobsPerformed: s.jobsPerformed,
-                compileSpeed: s.jobsPerformed / s.totalCompileSpeed || 0,
-                uploadSpeed: s.jobsPerformed / s.totalUploadSpeed || 0,
-                hostname: s.hostname,
-                system: s.system,
-                created: s.created,
-                load: s.load,
-                uptime: now - s.created.valueOf(),
-                npmVersion: s.npmVersion,
-                environments: Object.keys(s.environments)
+                ip: daemon.ip,
+                name: daemon.name,
+                labels: daemon.labels,
+                port: daemon.port,
+                hostname: daemon.hostname,
+                created: daemon.created,
+                uptime: now - daemon.created.valueOf(),
+                npmVersion: daemon.npmVersion
             });
         }
         const pretty = req.query && req.query.unpretty ? undefined : 4;
@@ -58501,17 +58729,18 @@ server.on("listen", (app) => {
             return { count: count, percentage: (count ? (count * 100) / jobs : 0).toFixed(1) + "%" };
         }
         const obj = {
-            builderCount: Object.keys(builders).length,
+            builderCount: builders.size,
+            daemonCount: daemons.size,
             npmVersion: schedulerNpmVersion,
             environments: environmentsInfo(),
             configVersion: common.Version,
-            capacity: capacity,
-            activeJobs: activeJobs,
+            capacity,
+            activeJobs,
             peaks: peakData(),
             jobsFailed: percentage(jobsFailed),
-            jobsStarted: jobsStarted,
-            jobs: jobs,
-            jobsScheduled: jobsScheduled,
+            jobsStarted,
+            jobs,
+            jobsScheduled,
             jobsFinished: percentage(jobsFinished),
             cacheHits: percentage(objectCache ? objectCache.hits : 0),
             uptimeMS: now - serverStartTime,
@@ -58560,9 +58789,9 @@ server.on("listen", (app) => {
             code: req.query.code || 0,
             purgeEnvironments: "purge_environments" in req.query
         };
-        console.log("Sending quit message to builders", msg, Object.keys(builders));
-        for (const ip in builders) {
-            builders[ip].send(msg);
+        console.log("Sending quit message to builders", msg, builders.size);
+        for (const builder of builders) {
+            builder.send(msg);
         }
     });
     app.get("/environment/*", (req, res) => {
@@ -58606,8 +58835,7 @@ server.on("listen", (app) => {
             return;
         }
         let found;
-        for (const key in builders) {
-            const builder = builders[key];
+        for (const builder of builders) {
             console.log(builder.ip, builder.name, builder.hostname, req.body.builder);
             if (builder.ip === req.body.builder ||
                 builder.name === req.body.builder ||
@@ -58742,7 +58970,7 @@ server.on("builder", (builder) => {
     }
     builder.activeClients = 0;
     insertBuilder(builder);
-    console.log("builder connected", builder.npmVersion, builder.ip, builder.name || "", builder.hostname || "", Object.keys(builder.environments), "builderCount is", builderCount);
+    console.log("builder connected", builder.npmVersion, builder.ip, builder.name || "", builder.hostname || "", Object.keys(builder.environments), "builderCount is", builders.size);
     syncEnvironments(builder);
     builder.on("environments", (message) => {
         builder.environments = {};
@@ -58775,7 +59003,7 @@ server.on("builder", (builder) => {
         if (objectCache) {
             objectCache.removeNode(builder);
         }
-        console.log(`builder disconnected ${builder.ip}:${builder.port} ${builder.name} ${builder.hostname} builderCount is ${builderCount}`);
+        console.log(`builder disconnected ${builder.ip}:${builder.port} ${builder.name} ${builder.hostname} builderCount is ${builders.size}`);
         builder.removeAllListeners();
     });
     builder.on("load", (message) => {
@@ -58810,6 +59038,20 @@ function requestEnvironment(compile) {
     if (compile.environment in pendingEnvironments) {
         return false;
     }
+    if (!compile.canUploadEnvironment) {
+        // Tell it what we are missing but leave the pending slot open: this client
+        // is going to reconnect with a websocket of its own to do the upload, and
+        // that connection is the one that has to claim the slot and get the
+        // listeners below.
+        console.log(`Asking ${compile.name} ${compile.ip} to upload ${compile.environment} over a direct connection`);
+        compile.send({ type: "needsEnvironment" });
+        // Nothing further will ever happen on this job -- the upload arrives on a
+        // new connection -- so release it now rather than holding it for the whole
+        // local compile fiskc does in the meantime. The direct branch below must
+        // NOT do this: that client uploads over this very socket.
+        compile.close();
+        return true;
+    }
     pendingEnvironments[compile.environment] = true;
     console.log(`Asking ${compile.name} ${compile.ip} to upload ${compile.environment}`);
     compile.send({ type: "needsEnvironment" });
@@ -58893,13 +59135,35 @@ server.on("clientVerify", (clientVerify) => {
         clientVerify.send("version_verified", { minimum_version: `${clientMinimumVersion}` });
     }
 });
+// Also gives the connection an "error" listener: EventEmitter throws on an
+// unhandled "error" event, and this one would take the scheduler down with it.
+server.on("daemon", (daemon) => {
+    console.log(`daemon connected ${daemon.ip} ${daemon.name} ${daemon.hostname} version ${daemon.npmVersion}`);
+    daemon.on("error", (err) => {
+        console.error(`daemon error ${daemon.ip}: ${typeof err === "string" ? err : err.message}`);
+        // ### do I get a close if I get an error?
+        daemons.delete(daemon);
+    });
+    daemon.on("close", () => {
+        console.log(`daemon disconnected ${daemon.ip} ${daemon.name}`);
+        daemon.removeAllListeners();
+        daemons.delete(daemon);
+    });
+    daemons.add(daemon);
+});
 server.on("compile", (compile) => {
     compile.on("log", (event) => {
         addLogFile({ source: "client", ip: compile.ip, contents: event.message });
     });
+    // Every dead end below closes the client. On a real websocket fiskc did that
+    // for us by disconnecting; a job relayed by a daemon has no socket to drop, so
+    // without this it sits in DaemonConnection.jobs holding a Compile until the
+    // daemon notices fiskc exit -- which for these paths means all the way through
+    // the local compile fiskc falls back to.
     if (clientTooOld(compile.npmVersion)) {
         ++jobsFailed;
         compile.send("version_mismatch", { minimum_version: `${clientMinimumVersion}` });
+        compile.close();
         return;
     }
     // console.log("request", compile.hostname, compile.ip, compile.environment);
@@ -58912,6 +59176,7 @@ server.on("compile", (compile) => {
     if (!usableEnvs.length) {
         console.log(`We're already waiting for ${compile.environment} and we don't have any compatible ones`);
         compile.send("builder", {});
+        compile.close();
         ++jobsFailed;
         return;
     }
@@ -58994,17 +59259,20 @@ server.on("compile", (compile) => {
             ++jobsFailed;
             console.log(`Specific builder "${compile.builder}" was requested and we couldn't find a builder with that ${compile.environment}`);
             compile.send("builder", {});
+            compile.close();
             return;
         }
         if (compile.labels) {
             ++jobsFailed;
             console.log(`Specific labels "${compile.labels}" were specified we couldn't match ${compile.environment} with any builder with those labels`);
             compile.send("builder", {});
+            compile.close();
             return;
         }
         ++jobsFailed;
         console.log("No builder for you", compile.ip);
         compile.send("builder", {});
+        compile.close();
         return;
     }
     const data = {};
@@ -59437,10 +59705,15 @@ Environments.instance
     .then(() => {
     setInterval(() => {
         // console.log("sending pings");
-        for (const key in builders) {
-            const builder = builders[key];
+        for (const builder of builders) {
             builder.ping();
         }
+        // A daemon holds every job on its host, so a silently dead one pins
+        // activeJobs and activeClients until the kernel gives up on the TCP
+        // connection. Pinging is what makes Client.ping close it instead.
+        for (const daemon of daemons) {
+            daemon.ping();
+        }
     }, option.int("ping-interval", 20000));
 })
     .catch((e) => {