@andersbakken/fisk 5.0.17 → 5.0.19

package/package.json

@@ -1,12 +1,14 @@
 {
     "name": "@andersbakken/fisk",
-    "version": "5.0.17",
+    "version": "5.0.19",
     "description": "Fisk, a distributed compile system",
     "scripts": {
         "lint": "eslint . --ext .ts",
         "rollup": "rollup -c",
         "tsc": "tsc -p .",
         "build": "npm run clean:dist && mkdir -p dist && run-p lint rollup",
+        "verify-package-files": "node scripts/verify-package-files.js",
+        "prepublishOnly": "npm run build && npm run verify-package-files",
         "clean:dist": "del-cli dist/*",
         "clean:cache": "del-cli .cache/*",
         "clean": "run-p clean:dist clean:cache"

package/scheduler/fisk-scheduler.js

@@ -3843,7 +3843,7 @@ class SHA1Data {
     constructor(fileSize, node) {
         this.fileSize = fileSize;
         this.fileSize = fileSize;
-        this.nodes = [node];
+        this.nodes = new Set([node]);
     }
 }
 
@@ -3974,10 +3974,8 @@ function prettySize(bytes) {
 function addToSHA1Map(bySHA1, sha1, fileSize, node) {
     const data = bySHA1.get(sha1);
     if (data) {
-        if (data.nodes.indexOf(node) === -1) {
-            data.nodes.push(node);
-        }
-        return data.nodes.length;
+        data.nodes.add(node);
+        return data.nodes.size;
     }
     bySHA1.set(sha1, new SHA1Data(fileSize, node));
     return 1;
@@ -3985,10 +3983,8 @@ function addToSHA1Map(bySHA1, sha1, fileSize, node) {
 function removeFromSHA1Map(bySHA1, sha1, node) {
     const data = bySHA1.get(sha1);
     if (data) {
-        const idx = data.nodes.indexOf(node);
-        if (idx !== -1) {
-            data.nodes.splice(idx, 1);
-            if (data.nodes.length === 0) {
+        if (data.nodes.delete(node)) {
+            if (data.nodes.size === 0) {
                 bySHA1.delete(sha1);
             }
         }
@@ -4009,7 +4005,8 @@ class ObjectCacheManager extends require$$0__default$2["default"] {
         this.hits = 0;
         this.bySHA1 = new Map();
         this.byNode = new Map();
-        this.pendingTransfers = new Set();
+        this.pendingBySHA1 = new Map();
+        this.pendingByNode = new Map();
         this.pendingTransferTimers = new Map();
         this.redundancy = option.int("object-cache-redundancy", 1);
         if (this.redundancy <= 0) {
@@ -4024,7 +4021,8 @@ class ObjectCacheManager extends require$$0__default$2["default"] {
         for (const timer of this.pendingTransferTimers.values()) {
             clearTimeout(timer);
         }
-        this.pendingTransfers.clear();
+        this.pendingBySHA1.clear();
+        this.pendingByNode.clear();
         this.pendingTransferTimers.clear();
         this.emit("cleared");
     }
@@ -4131,7 +4129,7 @@ class ObjectCacheManager extends require$$0__default$2["default"] {
                 // console.log(key, value);
                 sha1[key] = {
                     fileSize: prettySize(value.fileSize),
-                    nodes: value.nodes.map((node) => node.ip + ":" + node.port)
+                    nodes: Array.from(value.nodes, (node) => node.ip + ":" + node.port)
                 };
             });
             ret.sha1 = sha1;
@@ -4171,11 +4169,15 @@ class ObjectCacheManager extends require$$0__default$2["default"] {
                     return;
                 }
                 const pendingCount = this.pendingCountForSha1(sha);
-                const totalCopies = value.nodes.length + pendingCount;
+                const totalCopies = value.nodes.size + pendingCount;
                 const needed = Math.min(redundancy + 1 - totalCopies, this.byNode.size - 1);
                 if (needed > 0) {
                     let firstIdx;
                     let found = 0;
+                    // Only materialised if we actually pick a source, since
+                    // most objects already have enough copies and never get
+                    // this far.
+                    let sources;
                     while (found < needed) {
                         if (++nodeIdx === nodes.length) {
                             nodeIdx = 0;
@@ -4187,7 +4189,7 @@ class ObjectCacheManager extends require$$0__default$2["default"] {
                             break;
                         }
                         const node = nodes[nodeIdx];
-                        if (value.nodes.indexOf(node) !== -1) {
+                        if (value.nodes.has(node)) {
                             continue;
                         }
                         if (this.hasPendingTransfer(sha, node)) {
@@ -4215,7 +4217,10 @@ class ObjectCacheManager extends require$$0__default$2["default"] {
                         }
                         ++found;
                         data.available -= value.fileSize;
-                        const src = value.nodes[roundRobinIndex++ % value.nodes.length];
+                        if (!sources) {
+                            sources = Array.from(value.nodes);
+                        }
+                        const src = sources[roundRobinIndex++ % sources.length];
                         data.objects.push({ source: src.ip + ":" + src.port, sha1: sha });
                         if (max !== undefined && !--max) {
                             break;
@@ -4259,48 +4264,64 @@ class ObjectCacheManager extends require$$0__default$2["default"] {
         }
     }
     addPendingTransfer(sha1, node) {
+        let nodes = this.pendingBySHA1.get(sha1);
+        if (!nodes) {
+            nodes = new Set();
+            this.pendingBySHA1.set(sha1, nodes);
+        }
+        nodes.add(node);
+        let sha1s = this.pendingByNode.get(node);
+        if (!sha1s) {
+            sha1s = new Set();
+            this.pendingByNode.set(node, sha1s);
+        }
+        sha1s.add(sha1);
         const key = pendingKey(sha1, node);
-        this.pendingTransfers.add(key);
+        const existing = this.pendingTransferTimers.get(key);
+        if (existing) {
+            clearTimeout(existing);
+        }
         const timer = setTimeout(() => {
-            this.pendingTransfers.delete(key);
-            this.pendingTransferTimers.delete(key);
+            this.forgetPendingTransfer(sha1, node);
         }, this.pendingTransferTimeoutMs);
         timer.unref();
         this.pendingTransferTimers.set(key, timer);
     }
     clearPendingTransfer(sha1, node) {
-        const key = pendingKey(sha1, node);
-        this.pendingTransfers.delete(key);
-        const timer = this.pendingTransferTimers.get(key);
+        const timer = this.pendingTransferTimers.get(pendingKey(sha1, node));
         if (timer) {
             clearTimeout(timer);
-            this.pendingTransferTimers.delete(key);
         }
+        this.forgetPendingTransfer(sha1, node);
+    }
+    forgetPendingTransfer(sha1, node) {
+        const nodes = this.pendingBySHA1.get(sha1);
+        if (nodes && nodes.delete(node) && nodes.size === 0) {
+            this.pendingBySHA1.delete(sha1);
+        }
+        const sha1s = this.pendingByNode.get(node);
+        if (sha1s && sha1s.delete(sha1) && sha1s.size === 0) {
+            this.pendingByNode.delete(node);
+        }
+        this.pendingTransferTimers.delete(pendingKey(sha1, node));
     }
     clearAllPendingForNode(node) {
-        const suffix = ":" + node.ip + ":" + node.port;
-        for (const key of this.pendingTransfers) {
-            if (key.endsWith(suffix)) {
-                this.pendingTransfers.delete(key);
-                const timer = this.pendingTransferTimers.get(key);
-                if (timer) {
-                    clearTimeout(timer);
-                    this.pendingTransferTimers.delete(key);
-                }
-            }
+        const sha1s = this.pendingByNode.get(node);
+        if (!sha1s) {
+            return;
+        }
+        for (const sha1 of Array.from(sha1s)) {
+            this.clearPendingTransfer(sha1, node);
         }
+        this.pendingByNode.delete(node);
     }
     pendingCountForSha1(sha1) {
-        let count = 0;
-        for (const key of this.pendingTransfers) {
-            if (key.startsWith(sha1 + ":")) {
-                ++count;
-            }
-        }
-        return count;
+        var _a, _b;
+        return (_b = (_a = this.pendingBySHA1.get(sha1)) === null || _a === void 0 ? void 0 : _a.size) !== null && _b !== void 0 ? _b : 0;
     }
     hasPendingTransfer(sha1, node) {
-        return this.pendingTransfers.has(pendingKey(sha1, node));
+        var _a, _b;
+        return (_b = (_a = this.pendingBySHA1.get(sha1)) === null || _a === void 0 ? void 0 : _a.has(node)) !== null && _b !== void 0 ? _b : false;
     }
 }
 
@@ -4621,6 +4642,97 @@ class DaemonConnection extends Client {
     }
 }
 
+const Version = 5;
+// 6: objects now carry the client's real source path and compilation dir,
+// baked in at compile time instead of the builder's /compiles paths, and the
+// stored response no longer keeps the paths the client used to patch with.
+const ObjectCacheFormatVersion = 6;
+function cacheDir(option) {
+    let dir = option("cache-dir");
+    if (!dir) {
+        dir = path__default["default"].join(require$$1__default$1["default"].homedir(), ".cache", "fisk", path__default["default"].basename(option.prefix || ""));
+    }
+    return dir;
+}
+function validateCache(option) {
+    const dir = cacheDir(option);
+    const file = path__default["default"].join(dir, "version");
+    // console.log(dir);
+    let version;
+    try {
+        version = fs$4.readFileSync(file);
+        if (version.readUInt32BE() === Version) {
+            return;
+        }
+    }
+    catch (err) {
+        /* */
+    }
+    if (version) {
+        console.log(`Wrong version. Destroying cache ${dir}`);
+    }
+    fs$4.removeSync(dir);
+    fs$4.mkdirpSync(dir);
+    const buf = Buffer.allocUnsafe(4);
+    buf.writeUInt32BE(Version);
+    fs$4.writeFileSync(file, buf);
+}
+function validateObjectCache(option) {
+    const dir = cacheDir(option);
+    const objectCacheDir = option.string("object-cache-dir") || path__default["default"].join(dir, "objectcache");
+    const file = path__default["default"].join(objectCacheDir, "version");
+    let version;
+    try {
+        version = fs$4.readFileSync(file);
+        if (version.readUInt32BE() === ObjectCacheFormatVersion) {
+            return;
+        }
+    }
+    catch (err) {
+        /* */
+    }
+    if (version) {
+        console.log(`Wrong object cache version. Destroying object cache ${objectCacheDir}`);
+    }
+    fs$4.removeSync(objectCacheDir);
+    fs$4.mkdirpSync(objectCacheDir);
+    const buf = Buffer.allocUnsafe(4);
+    buf.writeUInt32BE(ObjectCacheFormatVersion);
+    fs$4.writeFileSync(file, buf);
+}
+// The listen backlog is the depth of the kernel's accept queue. When it fills
+// -- which is what happens whenever the event loop stalls long enough to stop
+// calling accept() -- Linux does not refuse the connection, it silently drops
+// the SYN, and the client's first retransmit is a second later. A client with a
+// sub-second handshake budget sees that as a timeout with no server-side trace.
+// listen(2) clamps to net.core.somaxconn, so following somaxconn is both the
+// largest useful value and the one an operator can actually tune.
+function defaultBacklog() {
+    try {
+        return parseInt(fs$4.readFileSync("/proc/sys/net/core/somaxconn", "utf8")) || 511;
+    }
+    catch (err) {
+        // Not Linux, or procfs isn't mounted. 511 is node's own default.
+        return 511;
+    }
+}
+// Only the builder keeps an object cache on disk. The scheduler tracks which
+// builder holds which sha1 in memory, and the daemon uses cacheDir purely for
+// the default socket path -- validating an object cache for either created a
+// directory they never read and, on a format bump, tried to destroy one they do
+// not necessarily own.
+function common$1(option, hasObjectCache = false) {
+    validateCache(option);
+    if (hasObjectCache) {
+        validateObjectCache(option);
+    }
+    return {
+        cacheDir: cacheDir.bind(undefined, option),
+        Version,
+        ObjectCacheFormatVersion
+    };
+}
+
 /**
  * Check if we're required to add a port number.
  *
@@ -56177,14 +56289,7 @@ class Server extends require$$0__default$2["default"] {
                     return;
                 }
             }
-            let defaultBacklog = 128;
-            try {
-                defaultBacklog = parseInt(fs__default["default"].readFileSync("/proc/sys/net/core/somaxconn", "utf8")) || 128;
-            }
-            catch (err) {
-                /* */
-            }
-            const backlog = this.option.int("backlog", defaultBacklog);
+            const backlog = this.option.int("backlog", defaultBacklog());
             this.ws = new ws.Server({ noServer: true });
             let waitingServers = 1;
             const port = this.option.int("port", 8097);
@@ -56639,81 +56744,6 @@ class Server extends require$$0__default$2["default"] {
     }
 }
 
-const Version = 5;
-// 6: objects now carry the client's real source path and compilation dir,
-// baked in at compile time instead of the builder's /compiles paths, and the
-// stored response no longer keeps the paths the client used to patch with.
-const ObjectCacheFormatVersion = 6;
-function cacheDir(option) {
-    let dir = option("cache-dir");
-    if (!dir) {
-        dir = path__default["default"].join(require$$1__default$1["default"].homedir(), ".cache", "fisk", path__default["default"].basename(option.prefix || ""));
-    }
-    return dir;
-}
-function validateCache(option) {
-    const dir = cacheDir(option);
-    const file = path__default["default"].join(dir, "version");
-    // console.log(dir);
-    let version;
-    try {
-        version = fs$4.readFileSync(file);
-        if (version.readUInt32BE() === Version) {
-            return;
-        }
-    }
-    catch (err) {
-        /* */
-    }
-    if (version) {
-        console.log(`Wrong version. Destroying cache ${dir}`);
-    }
-    fs$4.removeSync(dir);
-    fs$4.mkdirpSync(dir);
-    const buf = Buffer.allocUnsafe(4);
-    buf.writeUInt32BE(Version);
-    fs$4.writeFileSync(file, buf);
-}
-function validateObjectCache(option) {
-    const dir = cacheDir(option);
-    const objectCacheDir = option.string("object-cache-dir") || path__default["default"].join(dir, "objectcache");
-    const file = path__default["default"].join(objectCacheDir, "version");
-    let version;
-    try {
-        version = fs$4.readFileSync(file);
-        if (version.readUInt32BE() === ObjectCacheFormatVersion) {
-            return;
-        }
-    }
-    catch (err) {
-        /* */
-    }
-    if (version) {
-        console.log(`Wrong object cache version. Destroying object cache ${objectCacheDir}`);
-    }
-    fs$4.removeSync(objectCacheDir);
-    fs$4.mkdirpSync(objectCacheDir);
-    const buf = Buffer.allocUnsafe(4);
-    buf.writeUInt32BE(ObjectCacheFormatVersion);
-    fs$4.writeFileSync(file, buf);
-}
-// Only the builder keeps an object cache on disk. The scheduler tracks which
-// builder holds which sha1 in memory, and the daemon uses cacheDir purely for
-// the default socket path -- validating an object cache for either created a
-// directory they never read and, on a format bump, tried to destroy one they do
-// not necessarily own.
-function common$1(option, hasObjectCache = false) {
-    validateCache(option);
-    if (hasObjectCache) {
-        validateObjectCache(option);
-    }
-    return {
-        cacheDir: cacheDir.bind(undefined, option),
-        Version,
-        ObjectCacheFormatVersion
-    };
-}
-
 var bytesExports = requireBytes();
 var bytes = /*@__PURE__*/getDefaultExportFromCjs(bytesExports);