@andersbakken/fisk 5.0.18 → 5.0.19

package/builder/fisk-builder.js

@@ -47,6 +47,36 @@ var require$$8__default = /*#__PURE__*/_interopDefaultLegacy(require$$8);
 var child_process__default = /*#__PURE__*/_interopDefaultLegacy(child_process);
 var require$$1__default$4 = /*#__PURE__*/_interopDefaultLegacy(require$$1$5);
 
+/******************************************************************************
+Copyright (c) Microsoft Corporation.
+
+Permission to use, copy, modify, and/or distribute this software for any
+purpose with or without fee is hereby granted.
+
+THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
+REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
+AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
+INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
+LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR
+OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
+PERFORMANCE OF THIS SOFTWARE.
+***************************************************************************** */
+
+function __awaiter(thisArg, _arguments, P, generator) {
+  function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+  return new (P || (P = Promise))(function (resolve, reject) {
+      function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+      function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+      function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+      step((generator = generator.apply(thisArg, _arguments || [])).next());
+  });
+}
+
+typeof SuppressedError === "function" ? SuppressedError : function (error, suppressed, message) {
+  var e = new Error(message);
+  return e.name = "SuppressedError", e.error = error, e.suppressed = suppressed, e;
+};
+
 var commonjsGlobal = typeof globalThis !== 'undefined' ? globalThis : typeof window !== 'undefined' ? window : typeof global !== 'undefined' ? global : typeof self !== 'undefined' ? self : {};
 
 function getDefaultExportFromCjs (x) {
@@ -8211,6 +8241,97 @@ class Job extends EventEmitter__default["default"] {
     }
 }
 
+const Version = 5;
+// 6: objects now carry the client's real source path and compilation dir,
+// baked in at compile time instead of the builder's /compiles paths, and the
+// stored response no longer keeps the paths the client used to patch with.
+const ObjectCacheFormatVersion = 6;
+function cacheDir(option) {
+    let dir = option("cache-dir");
+    if (!dir) {
+        dir = path__default["default"].join(os__default["default"].homedir(), ".cache", "fisk", path__default["default"].basename(option.prefix || ""));
+    }
+    return dir;
+}
+function validateCache(option) {
+    const dir = cacheDir(option);
+    const file = path__default["default"].join(dir, "version");
+    // console.log(dir);
+    let version;
+    try {
+        version = fs$3.readFileSync(file);
+        if (version.readUInt32BE() === Version) {
+            return;
+        }
+    }
+    catch (err) {
+        /* */
+    }
+    if (version) {
+        console.log(`Wrong version. Destroying cache ${dir}`);
+    }
+    fs$3.removeSync(dir);
+    fs$3.mkdirpSync(dir);
+    const buf = Buffer.allocUnsafe(4);
+    buf.writeUInt32BE(Version);
+    fs$3.writeFileSync(file, buf);
+}
+function validateObjectCache(option) {
+    const dir = cacheDir(option);
+    const objectCacheDir = option.string("object-cache-dir") || path__default["default"].join(dir, "objectcache");
+    const file = path__default["default"].join(objectCacheDir, "version");
+    let version;
+    try {
+        version = fs$3.readFileSync(file);
+        if (version.readUInt32BE() === ObjectCacheFormatVersion) {
+            return;
+        }
+    }
+    catch (err) {
+        /* */
+    }
+    if (version) {
+        console.log(`Wrong object cache version. Destroying object cache ${objectCacheDir}`);
+    }
+    fs$3.removeSync(objectCacheDir);
+    fs$3.mkdirpSync(objectCacheDir);
+    const buf = Buffer.allocUnsafe(4);
+    buf.writeUInt32BE(ObjectCacheFormatVersion);
+    fs$3.writeFileSync(file, buf);
+}
+// The listen backlog is the depth of the kernel's accept queue. When it fills
+// -- which is what happens whenever the event loop stalls long enough to stop
+// calling accept() -- Linux does not refuse the connection, it silently drops
+// the SYN, and the client's first retransmit is a second later. A client with a
+// sub-second handshake budget sees that as a timeout with no server-side trace.
+// listen(2) clamps to net.core.somaxconn, so following somaxconn is both the
+// largest useful value and the one an operator can actually tune.
+function defaultBacklog() {
+    try {
+        return parseInt(fs$3.readFileSync("/proc/sys/net/core/somaxconn", "utf8")) || 511;
+    }
+    catch (err) {
+        // Not Linux, or procfs isn't mounted. 511 is node's own default.
+        return 511;
+    }
+}
+// Only the builder keeps an object cache on disk. The scheduler tracks which
+// builder holds which sha1 in memory, and the daemon uses cacheDir purely for
+// the default socket path -- validating an object cache for either created a
+// directory they never read and, on a format bump, tried to destroy one they do
+// not necessarily own.
+function common$2(option, hasObjectCache = false) {
+    validateCache(option);
+    if (hasObjectCache) {
+        validateObjectCache(option);
+    }
+    return {
+        cacheDir: cacheDir.bind(undefined, option),
+        Version,
+        ObjectCacheFormatVersion
+    };
+}
+
 /**
  * Check if we're required to add a port number.
  *
@@ -54881,8 +55002,14 @@ class Server extends EventEmitter__default["default"] {
         this.emit("listen", this.app);
         this.port = this.option.int("port", 8096);
         this.server = require$$1__default$1["default"].createServer(this.app);
-        this.ws = new ws.Server({ noServer: true, backlog: this.option.int("backlog", 50) });
-        this.server.listen({ port: this.port, backlog: this.option.int("backlog", 50), host: "0.0.0.0" });
+        // No backlog here on purpose: ws only uses that option when it creates
+        // its own http server, which noServer mode explicitly does not do.
+        this.ws = new ws.Server({ noServer: true });
+        this.server.listen({
+            port: this.port,
+            backlog: this.option.int("backlog", defaultBacklog()),
+            host: "0.0.0.0"
+        });
         this.server.on("upgrade", (req, socket, head) => {
             assert__default["default"](this.ws, "Must have ws");
             this.ws.handleUpgrade(req, socket, head, (ws) => {
@@ -55061,47 +55188,59 @@ class CompileJob extends EventEmitter__default["default"] {
         this.dir = path__default["default"].join(vm.root, "compiles", String(this.id));
         this.vmDir = path__default["default"].join("/", "compiles", String(this.id));
         this.sourceFileName = sourcePath ? path__default["default"].basename(sourcePath) : "sourcefile";
-        fs$3.mkdirpSync(this.dir);
-        this.fd = fs$3.openSync(path__default["default"].join(this.dir, this.sourceFileName), "w");
         this.cppSize = 0;
         this.startCompile = undefined;
+        this.opened = fs$3.mkdirp(this.dir).then(() => fs$3.open(path__default["default"].join(this.dir, this.sourceFileName), "w"));
+        // Nothing awaits this until feed(), and an unhandled rejection in the
+        // meantime would take the builder down.
+        this.opened.catch(() => {
+            /* reported by feed */
+        });
     }
     sendCallback(error) {
         if (error) {
             console.error("Got send error for", this.vmDir, this.id, this.commandLine);
-            const compileFinished = {
-                type: "compileFinished",
-                success: false,
-                id: this.id,
-                files: [],
-                exitCode: -1,
-                sourcePath: "",
-                error: error.toString()
-            };
-            this.vm.compileFinished(compileFinished);
+            this.fail(error);
         }
     }
     feed(data) {
-        assert__default["default"](this.fd !== undefined, "Must have fd");
-        fs$3.writeSync(this.fd, data);
-        this.cppSize += data.length;
-        this.startCompile = Date.now();
-        fs$3.closeSync(this.fd);
-        this.fd = undefined;
-        this.vm.child.send({
-            type: "compile",
-            commandLine: this.commandLine,
-            argv0: this.argv0,
-            id: this.id,
-            dir: this.vmDir,
-            sourceFileName: this.sourceFileName,
-            clientSourcePath: this.sourcePath,
-            clientCwd: this.clientCwd
-        }, this.sendCallback.bind(this));
+        this.opened
+            .then((fd) => __awaiter(this, void 0, void 0, function* () {
+            yield fs$3.write(fd, data);
+            this.cppSize += data.length;
+            this.startCompile = Date.now();
+            yield fs$3.close(fd);
+            this.vm.child.send({
+                type: "compile",
+                commandLine: this.commandLine,
+                argv0: this.argv0,
+                id: this.id,
+                dir: this.vmDir,
+                sourceFileName: this.sourceFileName,
+                clientSourcePath: this.sourcePath,
+                clientCwd: this.clientCwd
+            }, this.sendCallback.bind(this));
+        }))
+            .catch((err) => {
+            console.error("Failed to write source file for", this.vmDir, this.id, err);
+            this.fail(err);
+        });
     }
     cancel() {
         this.vm.child.send({ type: "cancel", id: this.id }, this.sendCallback.bind(this));
     }
+    fail(error) {
+        const compileFinished = {
+            type: "compileFinished",
+            success: false,
+            id: this.id,
+            files: [],
+            exitCode: -1,
+            sourcePath: "",
+            error: error.toString()
+        };
+        this.vm.compileFinished(compileFinished);
+    }
 }
 
 class VM extends EventEmitter__default["default"] {
@@ -55179,6 +55318,28 @@ class VM extends EventEmitter__default["default"] {
             console.error("Got some error", msg.error);
         }
         const now = Date.now();
+        const dir = this.compiles[msg.id].dir;
+        let released = false;
+        const release = () => {
+            if (released) {
+                return;
+            }
+            released = true;
+            clearTimeout(leakTimer);
+            if (!this.keepCompiles) {
+                fs$3.remove(dir).catch((err) => {
+                    console.error(`Failed to remove compile directory ${dir}`, err);
+                });
+            }
+        };
+        // A handler that throws before releasing would keep the directory
+        // until the builder restarts and cleared its whole compiles root. The
+        // reads it is holding the directory for take milliseconds, so anything
+        // still outstanding this much later is a bug, not slow disk.
+        const leakTimer = setTimeout(() => {
+            console.error(`Compile directory ${dir} was never released, removing it anyway`);
+            release();
+        }, 5 * 60000).unref();
         const finishedEvent = {
             cppSize: compile.cppSize,
             compileDuration: now - (compile.startCompile || 0),
@@ -55191,13 +55352,19 @@ class VM extends EventEmitter__default["default"] {
                     path: file.path,
                     absolute: path__default["default"].join(this.root, file.mapped ? file.mapped : file.path)
                 };
-            })
+            }),
+            release
         };
-        compile.emit("finished", finishedEvent);
-        if (!this.keepCompiles) {
-            fs$3.remove(this.compiles[msg.id].dir);
-        }
         delete this.compiles[msg.id];
+        // The directory used to be removed right here, which forced the
+        // handler to read and compress every output file synchronously to beat
+        // the cleanup. It owns the directory now and releases it when done.
+        if (compile.listenerCount("finished")) {
+            compile.emit("finished", finishedEvent);
+        }
+        else {
+            release();
+        }
     }
     destroy() {
         this.destroying = true;
@@ -55219,81 +55386,6 @@ class VM extends EventEmitter__default["default"] {
     }
 }
 
-const Version = 5;
-// 6: objects now carry the client's real source path and compilation dir,
-// baked in at compile time instead of the builder's /compiles paths, and the
-// stored response no longer keeps the paths the client used to patch with.
-const ObjectCacheFormatVersion = 6;
-function cacheDir(option) {
-    let dir = option("cache-dir");
-    if (!dir) {
-        dir = path__default["default"].join(os__default["default"].homedir(), ".cache", "fisk", path__default["default"].basename(option.prefix || ""));
-    }
-    return dir;
-}
-function validateCache(option) {
-    const dir = cacheDir(option);
-    const file = path__default["default"].join(dir, "version");
-    // console.log(dir);
-    let version;
-    try {
-        version = fs$3.readFileSync(file);
-        if (version.readUInt32BE() === Version) {
-            return;
-        }
-    }
-    catch (err) {
-        /* */
-    }
-    if (version) {
-        console.log(`Wrong version. Destroying cache ${dir}`);
-    }
-    fs$3.removeSync(dir);
-    fs$3.mkdirpSync(dir);
-    const buf = Buffer.allocUnsafe(4);
-    buf.writeUInt32BE(Version);
-    fs$3.writeFileSync(file, buf);
-}
-function validateObjectCache(option) {
-    const dir = cacheDir(option);
-    const objectCacheDir = option.string("object-cache-dir") || path__default["default"].join(dir, "objectcache");
-    const file = path__default["default"].join(objectCacheDir, "version");
-    let version;
-    try {
-        version = fs$3.readFileSync(file);
-        if (version.readUInt32BE() === ObjectCacheFormatVersion) {
-            return;
-        }
-    }
-    catch (err) {
-        /* */
-    }
-    if (version) {
-        console.log(`Wrong object cache version. Destroying object cache ${objectCacheDir}`);
-    }
-    fs$3.removeSync(objectCacheDir);
-    fs$3.mkdirpSync(objectCacheDir);
-    const buf = Buffer.allocUnsafe(4);
-    buf.writeUInt32BE(ObjectCacheFormatVersion);
-    fs$3.writeFileSync(file, buf);
-}
-// Only the builder keeps an object cache on disk. The scheduler tracks which
-// builder holds which sha1 in memory, and the daemon uses cacheDir purely for
-// the default socket path -- validating an object cache for either created a
-// directory they never read and, on a format bump, tried to destroy one they do
-// not necessarily own.
-function common$2(option, hasObjectCache = false) {
-    validateCache(option);
-    if (hasObjectCache) {
-        validateObjectCache(option);
-    }
-    return {
-        cacheDir: cacheDir.bind(undefined, option),
-        Version,
-        ObjectCacheFormatVersion
-    };
-}
-
 var resolve;
 var hasRequiredResolve;
 
@@ -60174,6 +60266,14 @@ const option = createOptions({
     additionalFiles: ["fisk/builder.conf.override"]
 });
 const common = common$2(option, true);
+// Reads, writes, gzip and gunzip all run on libuv's threadpool, and now that
+// the job and object cache paths use the asynchronous forms they share it.
+// Four threads is not enough to keep as many jobs as we have slots moving.
+// libuv reads this when it first creates the pool, so it has to be set before
+// anything asynchronous has been submitted, which is why it lives up here.
+if (!process.env.UV_THREADPOOL_SIZE) {
+    process.env.UV_THREADPOOL_SIZE = String(Math.min(128, Math.max(8, os__default["default"].cpus().length)));
+}
 if (process.getuid() !== 0) {
     console.error("fisk builder needs to run as root to be able to chroot");
     process.exit(1);
@@ -60271,26 +60371,34 @@ function getFromCache(job, cb) {
                 else {
                     // console.log("got good response from file", file);
                     // console.log("sending some data", buffer.length, fileIdx, item.response.index.length);
-                    let sendBuffer = buffer;
-                    // Decompress if client doesn't support compressed responses from cache
-                    if (!sendCompressed && buffer.byteLength > 0) {
-                        try {
-                            sendBuffer = zlib__default["default"].gunzipSync(buffer);
+                    const sendAndContinue = (sendBuffer) => {
+                        job.send(sendBuffer);
+                        pos += read;
+                        if (++fileIdx < item.response.index.length) {
+                            work();
                         }
-                        catch (gunzipErr) {
-                            assert__default["default"](objectCache, "Must have objectCache");
-                            console.error(`Failed to gunzip ${path__default["default"].join(objectCache.dir, item.response.sha1)} for file index ${fileIdx}:`, gunzipErr);
-                            finish(gunzipErr);
-                            return;
+                        else {
+                            finish();
                         }
-                    }
-                    job.send(sendBuffer);
-                    pos += read;
-                    if (++fileIdx < item.response.index.length) {
-                        work();
+                    };
+                    // Decompress if client doesn't support compressed responses
+                    // from cache. Asynchronously: a cache hit is the fast path
+                    // and used to spend it inflating megabytes on the event
+                    // loop, once per file. Files are still sent in index order
+                    // because the next read only starts from sendAndContinue.
+                    if (!sendCompressed && buffer.byteLength > 0) {
+                        zlib__default["default"].gunzip(buffer, (gunzipErr, inflated) => {
+                            if (gunzipErr) {
+                                assert__default["default"](objectCache, "Must have objectCache");
+                                console.error(`Failed to gunzip ${path__default["default"].join(objectCache.dir, item.response.sha1)} for file index ${fileIdx}:`, gunzipErr);
+                                finish(gunzipErr);
+                                return;
+                            }
+                            sendAndContinue(inflated);
+                        });
                     }
                     else {
-                        finish();
+                        sendAndContinue(buffer);
                     }
                 }
             });
@@ -60848,6 +60956,19 @@ server.on("listen", (app) => {
         }
     });
 });
+const gzip = require$$1$3.promisify(zlib__default["default"].gzip);
+// Always compressed, because that is the form the object cache stores. The
+// uncompressed buffer is kept alongside it since the response index reports
+// both sizes and a client that cannot take compressed data gets that one.
+function readAndCompress(files) {
+    return __awaiter(this, void 0, void 0, function* () {
+        return Promise.all(files.map((f) => __awaiter(this, void 0, void 0, function* () {
+            const uncompressed = yield fs$3.promises.readFile(f.absolute);
+            const contents = uncompressed.byteLength > 0 ? yield gzip(uncompressed) : uncompressed;
+            return { contents, uncompressed, path: f.path };
+        })));
+    });
+}
 function startPending() {
     // console.log(`startPending called ${jobQueue.length}`);
     for (let idx = 0; idx < jobQueue.length; ++idx) {
@@ -60958,6 +61079,7 @@ server.on("job", (job) => {
             j.op.on("finished", (event) => {
                 j.done = true;
                 if (j.aborted) {
+                    event.release();
                     return;
                 }
                 const end = Date.now();
@@ -60968,108 +61090,131 @@ server.on("job", (job) => {
                 }
                 else {
                     console.error("Can't find j?");
+                    event.release();
                     return;
                 }
-                // this can't be async, the directory is removed after the event is fired
-                // Always compress for cache storage, but send based on client preference
-                const contents = event.files.map((f) => {
-                    const uncompressed = fs$3.readFileSync(f.absolute);
-                    const compressed = uncompressed.byteLength > 0 ? zlib__default["default"].gzipSync(uncompressed) : uncompressed;
-                    return {
-                        contents: compressed,
-                        uncompressed,
-                        path: f.path
-                    };
-                });
-                // Prepare data to send to client (compressed or uncompressed based on preference and capability)
-                // Only send compressed if client wants it AND supports compressed responses
-                const sendCompressed = j.job.compressed && j.job.supportsCompressedResponse;
-                const toSend = contents.map((item) => {
-                    assert__default["default"](item.uncompressed, "Must have uncompressed data");
-                    return {
-                        contents: sendCompressed ? item.contents : item.uncompressed,
-                        path: item.path
-                    };
+                // Reading and gzipping every output file synchronously blocked
+                // the event loop once per finished job, which on a full builder
+                // is once per core per compile wave, and that is what stopped
+                // handshakes from being answered. The compile directory is ours
+                // until we release it, so this can take as long as it needs to.
+                readAndCompress(event.files)
+                    .then((contents) => {
+                    if (j.aborted) {
+                        return;
+                    }
+                    respond(contents);
+                })
+                    .catch((err) => {
+                    console.error(`Failed to collect output for job ${j.id}`, err);
+                    if (!j.aborted) {
+                        jobJob.send({
+                            type: "response",
+                            index: [],
+                            success: false,
+                            exitCode: 1,
+                            error: `Failed to collect output: ${err.message}`,
+                            sha1: jobJob.sha1,
+                            sourcePath: j.op.sourceFileName,
+                            stderr: j.stderr,
+                            stdout: j.stdout
+                        });
+                    }
+                })
+                    .finally(() => {
+                    event.release();
+                    startPending();
                 });
-                const response = {
-                    type: "response",
-                    index: toSend.map((item) => {
-                        const original = contents.find((c) => c.path === item.path);
-                        assert__default["default"](original, "Must have original contents");
-                        assert__default["default"](original.uncompressed, "Must have uncompressed data");
-                        const ret = {
-                            path: item.path,
-                            bytes: item.contents.length,
-                            uncompressedSize: original.uncompressed.byteLength
+                function respond(contents) {
+                    // Prepare data to send to client (compressed or uncompressed based on preference and capability)
+                    // Only send compressed if client wants it AND supports compressed responses
+                    const sendCompressed = j.job.compressed && j.job.supportsCompressedResponse;
+                    const toSend = contents.map((item) => {
+                        assert__default["default"](item.uncompressed, "Must have uncompressed data");
+                        return {
+                            contents: sendCompressed ? item.contents : item.uncompressed,
+                            path: item.path
                         };
-                        return ret;
-                    }),
-                    success: event.success,
-                    exitCode: event.exitCode,
-                    sha1: jobJob.sha1,
-                    // Just the basename. This goes into the object cache and is
-                    // replayed on every hit, so it has to still mean something
-                    // later: the /compiles/<id> directory belongs to this one
-                    // job's chroot, and the requesting client's own path belongs
-                    // to whichever client happened to compile it first. Only the
-                    // file name survives being shared.
-                    sourcePath: j.op.sourceFileName,
-                    stderr: j.stderr,
-                    stdout: j.stdout
-                };
-                if (event.error) {
-                    response.error = event.error;
-                }
-                if (debug) {
-                    console.log("Sending response", jobJob.ip, jobJob.hostname, response);
-                }
-                jobJob.send(response);
-                if (response.exitCode === 0 &&
-                    event.success &&
-                    objectCache &&
-                    response.sha1 &&
-                    objectCache.state(response.sha1) === "none") {
-                    // Cache metadata needs to reflect compressed sizes since we store compressed
-                    const cacheResponse = Object.assign(Object.assign({}, response), { index: contents.map((item) => {
-                            assert__default["default"](item.uncompressed, "Must have uncompressed data");
-                            return {
+                    });
+                    const response = {
+                        type: "response",
+                        index: toSend.map((item) => {
+                            const original = contents.find((c) => c.path === item.path);
+                            assert__default["default"](original, "Must have original contents");
+                            assert__default["default"](original.uncompressed, "Must have uncompressed data");
+                            const ret = {
                                 path: item.path,
                                 bytes: item.contents.length,
-                                uncompressedSize: item.uncompressed.byteLength
+                                uncompressedSize: original.uncompressed.byteLength
                             };
-                        }) });
-                    cacheResponse.commandLine = jobJob.commandLine;
-                    cacheResponse.environment = jobJob.hash;
-                    objectCache.add(cacheResponse, contents);
-                }
-                toSend.forEach((x) => {
-                    if (x.contents && x.contents.byteLength) {
-                        jobJob.send(x.contents);
+                            return ret;
+                        }),
+                        success: event.success,
+                        exitCode: event.exitCode,
+                        sha1: jobJob.sha1,
+                        // Just the basename. This goes into the object cache and is
+                        // replayed on every hit, so it has to still mean something
+                        // later: the /compiles/<id> directory belongs to this one
+                        // job's chroot, and the requesting client's own path belongs
+                        // to whichever client happened to compile it first. Only the
+                        // file name survives being shared.
+                        sourcePath: j.op.sourceFileName,
+                        stderr: j.stderr,
+                        stdout: j.stdout
+                    };
+                    if (event.error) {
+                        response.error = event.error;
                     }
-                });
-                // console.log("GOT ID", j);
-                assert__default["default"](uploadDuration !== undefined, "Must have uploadDuration");
-                if (event.success) {
-                    client.send("jobFinished", {
-                        id: j.id,
-                        cppSize: event.cppSize,
-                        compileDuration: event.compileDuration,
-                        compileSpeed: event.cppSize / event.compileDuration,
-                        uploadDuration: uploadDuration,
-                        uploadSpeed: event.cppSize / uploadDuration
-                    });
-                }
-                else {
-                    client.send("jobAborted", {
-                        id: j.id,
-                        cppSize: event.cppSize,
-                        compileDuration: event.compileDuration,
-                        compileSpeed: event.cppSize / event.compileDuration,
-                        uploadDuration: uploadDuration,
-                        uploadSpeed: event.cppSize / uploadDuration
+                    if (debug) {
+                        console.log("Sending response", jobJob.ip, jobJob.hostname, response);
+                    }
+                    jobJob.send(response);
+                    if (response.exitCode === 0 &&
+                        event.success &&
+                        objectCache &&
+                        response.sha1 &&
+                        objectCache.state(response.sha1) === "none") {
+                        // Cache metadata needs to reflect compressed sizes since we store compressed
+                        const cacheResponse = Object.assign(Object.assign({}, response), { index: contents.map((item) => {
+                                assert__default["default"](item.uncompressed, "Must have uncompressed data");
+                                return {
+                                    path: item.path,
+                                    bytes: item.contents.length,
+                                    uncompressedSize: item.uncompressed.byteLength
+                                };
+                            }) });
+                        cacheResponse.commandLine = jobJob.commandLine;
+                        cacheResponse.environment = jobJob.hash;
+                        objectCache.add(cacheResponse, contents);
+                    }
+                    toSend.forEach((x) => {
+                        if (x.contents && x.contents.byteLength) {
+                            jobJob.send(x.contents);
+                        }
                     });
+                    // console.log("GOT ID", j);
+                    assert__default["default"](uploadDuration !== undefined, "Must have uploadDuration");
+                    if (event.success) {
+                        client.send("jobFinished", {
+                            id: j.id,
+                            cppSize: event.cppSize,
+                            compileDuration: event.compileDuration,
+                            compileSpeed: event.cppSize / event.compileDuration,
+                            uploadDuration: uploadDuration,
+                            uploadSpeed: event.cppSize / uploadDuration
+                        });
+                    }
+                    else {
+                        client.send("jobAborted", {
+                            id: j.id,
+                            cppSize: event.cppSize,
+                            compileDuration: event.compileDuration,
+                            compileSpeed: event.cppSize / event.compileDuration,
+                            uploadDuration: uploadDuration,
+                            uploadSpeed: event.cppSize / uploadDuration
+                        });
+                    }
                 }
-                startPending();
             });
         },
         cancel: function () {

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@andersbakken/fisk",
-    "version": "5.0.18",
+    "version": "5.0.19",
     "description": "Fisk, a distributed compile system",
     "scripts": {
         "lint": "eslint . --ext .ts",

package/scheduler/fisk-scheduler.js

@@ -3843,7 +3843,7 @@ class SHA1Data {
     constructor(fileSize, node) {
         this.fileSize = fileSize;
         this.fileSize = fileSize;
-        this.nodes = [node];
+        this.nodes = new Set([node]);
     }
 }
 
@@ -3974,10 +3974,8 @@ function prettySize(bytes) {
 function addToSHA1Map(bySHA1, sha1, fileSize, node) {
     const data = bySHA1.get(sha1);
     if (data) {
-        if (data.nodes.indexOf(node) === -1) {
-            data.nodes.push(node);
-        }
-        return data.nodes.length;
+        data.nodes.add(node);
+        return data.nodes.size;
     }
     bySHA1.set(sha1, new SHA1Data(fileSize, node));
     return 1;
@@ -3985,10 +3983,8 @@ function addToSHA1Map(bySHA1, sha1, fileSize, node) {
 function removeFromSHA1Map(bySHA1, sha1, node) {
     const data = bySHA1.get(sha1);
     if (data) {
-        const idx = data.nodes.indexOf(node);
-        if (idx !== -1) {
-            data.nodes.splice(idx, 1);
-            if (data.nodes.length === 0) {
+        if (data.nodes.delete(node)) {
+            if (data.nodes.size === 0) {
                 bySHA1.delete(sha1);
             }
         }
@@ -4009,7 +4005,8 @@ class ObjectCacheManager extends require$$0__default$2["default"] {
         this.hits = 0;
         this.bySHA1 = new Map();
         this.byNode = new Map();
-        this.pendingTransfers = new Set();
+        this.pendingBySHA1 = new Map();
+        this.pendingByNode = new Map();
         this.pendingTransferTimers = new Map();
         this.redundancy = option.int("object-cache-redundancy", 1);
         if (this.redundancy <= 0) {
@@ -4024,7 +4021,8 @@ class ObjectCacheManager extends require$$0__default$2["default"] {
         for (const timer of this.pendingTransferTimers.values()) {
             clearTimeout(timer);
         }
-        this.pendingTransfers.clear();
+        this.pendingBySHA1.clear();
+        this.pendingByNode.clear();
         this.pendingTransferTimers.clear();
         this.emit("cleared");
     }
@@ -4131,7 +4129,7 @@ class ObjectCacheManager extends require$$0__default$2["default"] {
                 // console.log(key, value);
                 sha1[key] = {
                     fileSize: prettySize(value.fileSize),
-                    nodes: value.nodes.map((node) => node.ip + ":" + node.port)
+                    nodes: Array.from(value.nodes, (node) => node.ip + ":" + node.port)
                 };
             });
             ret.sha1 = sha1;
@@ -4171,11 +4169,15 @@ class ObjectCacheManager extends require$$0__default$2["default"] {
                     return;
                 }
                 const pendingCount = this.pendingCountForSha1(sha);
-                const totalCopies = value.nodes.length + pendingCount;
+                const totalCopies = value.nodes.size + pendingCount;
                 const needed = Math.min(redundancy + 1 - totalCopies, this.byNode.size - 1);
                 if (needed > 0) {
                     let firstIdx;
                     let found = 0;
+                    // Only materialised if we actually pick a source, since
+                    // most objects already have enough copies and never get
+                    // this far.
+                    let sources;
                     while (found < needed) {
                         if (++nodeIdx === nodes.length) {
                             nodeIdx = 0;
@@ -4187,7 +4189,7 @@ class ObjectCacheManager extends require$$0__default$2["default"] {
                             break;
                         }
                         const node = nodes[nodeIdx];
-                        if (value.nodes.indexOf(node) !== -1) {
+                        if (value.nodes.has(node)) {
                             continue;
                         }
                         if (this.hasPendingTransfer(sha, node)) {
@@ -4215,7 +4217,10 @@ class ObjectCacheManager extends require$$0__default$2["default"] {
                         }
                         ++found;
                         data.available -= value.fileSize;
-                        const src = value.nodes[roundRobinIndex++ % value.nodes.length];
+                        if (!sources) {
+                            sources = Array.from(value.nodes);
+                        }
+                        const src = sources[roundRobinIndex++ % sources.length];
                         data.objects.push({ source: src.ip + ":" + src.port, sha1: sha });
                         if (max !== undefined && !--max) {
                             break;
@@ -4259,48 +4264,64 @@ class ObjectCacheManager extends require$$0__default$2["default"] {
         }
     }
     addPendingTransfer(sha1, node) {
+        let nodes = this.pendingBySHA1.get(sha1);
+        if (!nodes) {
+            nodes = new Set();
+            this.pendingBySHA1.set(sha1, nodes);
+        }
+        nodes.add(node);
+        let sha1s = this.pendingByNode.get(node);
+        if (!sha1s) {
+            sha1s = new Set();
+            this.pendingByNode.set(node, sha1s);
+        }
+        sha1s.add(sha1);
         const key = pendingKey(sha1, node);
-        this.pendingTransfers.add(key);
+        const existing = this.pendingTransferTimers.get(key);
+        if (existing) {
+            clearTimeout(existing);
+        }
         const timer = setTimeout(() => {
-            this.pendingTransfers.delete(key);
-            this.pendingTransferTimers.delete(key);
+            this.forgetPendingTransfer(sha1, node);
         }, this.pendingTransferTimeoutMs);
         timer.unref();
         this.pendingTransferTimers.set(key, timer);
     }
     clearPendingTransfer(sha1, node) {
-        const key = pendingKey(sha1, node);
-        this.pendingTransfers.delete(key);
-        const timer = this.pendingTransferTimers.get(key);
+        const timer = this.pendingTransferTimers.get(pendingKey(sha1, node));
         if (timer) {
             clearTimeout(timer);
-            this.pendingTransferTimers.delete(key);
         }
+        this.forgetPendingTransfer(sha1, node);
+    }
+    forgetPendingTransfer(sha1, node) {
+        const nodes = this.pendingBySHA1.get(sha1);
+        if (nodes && nodes.delete(node) && nodes.size === 0) {
+            this.pendingBySHA1.delete(sha1);
+        }
+        const sha1s = this.pendingByNode.get(node);
+        if (sha1s && sha1s.delete(sha1) && sha1s.size === 0) {
+            this.pendingByNode.delete(node);
+        }
+        this.pendingTransferTimers.delete(pendingKey(sha1, node));
     }
     clearAllPendingForNode(node) {
-        const suffix = ":" + node.ip + ":" + node.port;
-        for (const key of this.pendingTransfers) {
-            if (key.endsWith(suffix)) {
-                this.pendingTransfers.delete(key);
-                const timer = this.pendingTransferTimers.get(key);
-                if (timer) {
-                    clearTimeout(timer);
-                    this.pendingTransferTimers.delete(key);
-                }
-            }
+        const sha1s = this.pendingByNode.get(node);
+        if (!sha1s) {
+            return;
+        }
+        for (const sha1 of Array.from(sha1s)) {
+            this.clearPendingTransfer(sha1, node);
         }
+        this.pendingByNode.delete(node);
     }
     pendingCountForSha1(sha1) {
-        let count = 0;
-        for (const key of this.pendingTransfers) {
-            if (key.startsWith(sha1 + ":")) {
-                ++count;
-            }
-        }
-        return count;
+        var _a, _b;
+        return (_b = (_a = this.pendingBySHA1.get(sha1)) === null || _a === void 0 ? void 0 : _a.size) !== null && _b !== void 0 ? _b : 0;
     }
     hasPendingTransfer(sha1, node) {
-        return this.pendingTransfers.has(pendingKey(sha1, node));
+        var _a, _b;
+        return (_b = (_a = this.pendingBySHA1.get(sha1)) === null || _a === void 0 ? void 0 : _a.has(node)) !== null && _b !== void 0 ? _b : false;
     }
 }
 
@@ -4621,6 +4642,97 @@ class DaemonConnection extends Client {
     }
 }
 
+const Version = 5;
+// 6: objects now carry the client's real source path and compilation dir,
+// baked in at compile time instead of the builder's /compiles paths, and the
+// stored response no longer keeps the paths the client used to patch with.
+const ObjectCacheFormatVersion = 6;
+function cacheDir(option) {
+    let dir = option("cache-dir");
+    if (!dir) {
+        dir = path__default["default"].join(require$$1__default$1["default"].homedir(), ".cache", "fisk", path__default["default"].basename(option.prefix || ""));
+    }
+    return dir;
+}
+function validateCache(option) {
+    const dir = cacheDir(option);
+    const file = path__default["default"].join(dir, "version");
+    // console.log(dir);
+    let version;
+    try {
+        version = fs$4.readFileSync(file);
+        if (version.readUInt32BE() === Version) {
+            return;
+        }
+    }
+    catch (err) {
+        /* */
+    }
+    if (version) {
+        console.log(`Wrong version. Destroying cache ${dir}`);
+    }
+    fs$4.removeSync(dir);
+    fs$4.mkdirpSync(dir);
+    const buf = Buffer.allocUnsafe(4);
+    buf.writeUInt32BE(Version);
+    fs$4.writeFileSync(file, buf);
+}
+function validateObjectCache(option) {
+    const dir = cacheDir(option);
+    const objectCacheDir = option.string("object-cache-dir") || path__default["default"].join(dir, "objectcache");
+    const file = path__default["default"].join(objectCacheDir, "version");
+    let version;
+    try {
+        version = fs$4.readFileSync(file);
+        if (version.readUInt32BE() === ObjectCacheFormatVersion) {
+            return;
+        }
+    }
+    catch (err) {
+        /* */
+    }
+    if (version) {
+        console.log(`Wrong object cache version. Destroying object cache ${objectCacheDir}`);
+    }
+    fs$4.removeSync(objectCacheDir);
+    fs$4.mkdirpSync(objectCacheDir);
+    const buf = Buffer.allocUnsafe(4);
+    buf.writeUInt32BE(ObjectCacheFormatVersion);
+    fs$4.writeFileSync(file, buf);
+}
+// The listen backlog is the depth of the kernel's accept queue. When it fills
+// -- which is what happens whenever the event loop stalls long enough to stop
+// calling accept() -- Linux does not refuse the connection, it silently drops
+// the SYN, and the client's first retransmit is a second later. A client with a
+// sub-second handshake budget sees that as a timeout with no server-side trace.
+// listen(2) clamps to net.core.somaxconn, so following somaxconn is both the
+// largest useful value and the one an operator can actually tune.
+function defaultBacklog() {
+    try {
+        return parseInt(fs$4.readFileSync("/proc/sys/net/core/somaxconn", "utf8")) || 511;
+    }
+    catch (err) {
+        // Not Linux, or procfs isn't mounted. 511 is node's own default.
+        return 511;
+    }
+}
+// Only the builder keeps an object cache on disk. The scheduler tracks which
+// builder holds which sha1 in memory, and the daemon uses cacheDir purely for
+// the default socket path -- validating an object cache for either created a
+// directory they never read and, on a format bump, tried to destroy one they do
+// not necessarily own.
+function common$1(option, hasObjectCache = false) {
+    validateCache(option);
+    if (hasObjectCache) {
+        validateObjectCache(option);
+    }
+    return {
+        cacheDir: cacheDir.bind(undefined, option),
+        Version,
+        ObjectCacheFormatVersion
+    };
+}
+
 /**
  * Check if we're required to add a port number.
  *
@@ -56177,14 +56289,7 @@ class Server extends require$$0__default$2["default"] {
                     return;
                 }
             }
-            let defaultBacklog = 128;
-            try {
-                defaultBacklog = parseInt(fs__default["default"].readFileSync("/proc/sys/net/core/somaxconn", "utf8")) || 128;
-            }
-            catch (err) {
-                /* */
-            }
-            const backlog = this.option.int("backlog", defaultBacklog);
+            const backlog = this.option.int("backlog", defaultBacklog());
             this.ws = new ws.Server({ noServer: true });
             let waitingServers = 1;
             const port = this.option.int("port", 8097);
@@ -56639,81 +56744,6 @@ class Server extends require$$0__default$2["default"] {
     }
 }
 
-const Version = 5;
-// 6: objects now carry the client's real source path and compilation dir,
-// baked in at compile time instead of the builder's /compiles paths, and the
-// stored response no longer keeps the paths the client used to patch with.
-const ObjectCacheFormatVersion = 6;
-function cacheDir(option) {
-    let dir = option("cache-dir");
-    if (!dir) {
-        dir = path__default["default"].join(require$$1__default$1["default"].homedir(), ".cache", "fisk", path__default["default"].basename(option.prefix || ""));
-    }
-    return dir;
-}
-function validateCache(option) {
-    const dir = cacheDir(option);
-    const file = path__default["default"].join(dir, "version");
-    // console.log(dir);
-    let version;
-    try {
-        version = fs$4.readFileSync(file);
-        if (version.readUInt32BE() === Version) {
-            return;
-        }
-    }
-    catch (err) {
-        /* */
-    }
-    if (version) {
-        console.log(`Wrong version. Destroying cache ${dir}`);
-    }
-    fs$4.removeSync(dir);
-    fs$4.mkdirpSync(dir);
-    const buf = Buffer.allocUnsafe(4);
-    buf.writeUInt32BE(Version);
-    fs$4.writeFileSync(file, buf);
-}
-function validateObjectCache(option) {
-    const dir = cacheDir(option);
-    const objectCacheDir = option.string("object-cache-dir") || path__default["default"].join(dir, "objectcache");
-    const file = path__default["default"].join(objectCacheDir, "version");
-    let version;
-    try {
-        version = fs$4.readFileSync(file);
-        if (version.readUInt32BE() === ObjectCacheFormatVersion) {
-            return;
-        }
-    }
-    catch (err) {
-        /* */
-    }
-    if (version) {
-        console.log(`Wrong object cache version. Destroying object cache ${objectCacheDir}`);
-    }
-    fs$4.removeSync(objectCacheDir);
-    fs$4.mkdirpSync(objectCacheDir);
-    const buf = Buffer.allocUnsafe(4);
-    buf.writeUInt32BE(ObjectCacheFormatVersion);
-    fs$4.writeFileSync(file, buf);
-}
-// Only the builder keeps an object cache on disk. The scheduler tracks which
-// builder holds which sha1 in memory, and the daemon uses cacheDir purely for
-// the default socket path -- validating an object cache for either created a
-// directory they never read and, on a format bump, tried to destroy one they do
-// not necessarily own.
-function common$1(option, hasObjectCache = false) {
-    validateCache(option);
-    if (hasObjectCache) {
-        validateObjectCache(option);
-    }
-    return {
-        cacheDir: cacheDir.bind(undefined, option),
-        Version,
-        ObjectCacheFormatVersion
-    };
-}
-
 var bytesExports = requireBytes();
 var bytes = /*@__PURE__*/getDefaultExportFromCjs(bytesExports);