@andersbakken/fisk 3.5.6 → 3.6.0

package/scheduler/objectcachemanager.js

@@ -1,306 +0,0 @@
-const EventEmitter = require("events");
-
-function prettysize(bytes)
-{
-    const prettysize = require("prettysize");
-    return prettysize(bytes, bytes >= 1024); // don't want 0Bytes
-}
-
-class NodeData
-{
-    constructor(size, maxSize, sha1s)
-    {
-        this.sha1s = sha1s;
-        this.size = size;
-        this.maxSize = maxSize;
-    }
-};
-
-class SHA1Data
-{
-    constructor(fileSize, node)
-    {
-        this.fileSize = fileSize;
-        this.nodes = [ node ];
-    }
-};
-
-function addToSHA1Map(bySHA1, sha1, fileSize, node)
-{
-    let data = bySHA1.get(sha1);
-    if (data) {
-        data.nodes.push(node);
-        return data.nodes.length;
-    } else {
-        bySHA1.set(sha1, new SHA1Data(fileSize, node));
-        return 1;
-    }
-}
-
-function removeFromSHA1Map(bySHA1, sha1, node)
-{
-    let data = bySHA1.get(sha1);
-    if (data) {
-        let idx = data.nodes.indexOf(node);
-        if (idx != -1) {
-            data.nodes.splice(idx, 1);
-            if (data.nodes.length == 0) {
-                bySHA1.delete(sha1);
-            }
-        } else {
-            console.error("We don't have", node.ip + ":" + node.port, "for", sha1);
-        }
-    } else {
-        console.error("We don't have", sha1);
-    }
-}
-
-class ObjectCacheManager extends EventEmitter
-{
-    constructor(option)
-    {
-        super();
-        this.hits = 0;
-        this.bySHA1 = new Map();
-        this.byNode = new Map();
-        this.redundancy = option.int("object-cache-redundancy", 1);
-        if (this.redundancy <= 0)
-            this.redundancy = 1;
-        this.distributeOnInsertion = option("distribute-object-cache-on-insertion") || false;
-        this.distributeOnCacheHit = option("distribute-object-cache-on-cache-hit") || false;
-    }
-
-    clear()
-    {
-        this.hits = 0;
-        this.emit("cleared");
-    }
-
-    hit(sha1)
-    {
-        ++this.hits;
-        if (this.distributeOnCacheHit) {
-            this.distribute({ sha1: sha1, redundancy: this.redundancy });
-        }
-    }
-
-    get(sha1)
-    {
-        // console.log("looking for", sha1, [ this.bySHA1.keys() ]);
-        return this.bySHA1.get(sha1);
-    }
-
-    insert(msg, node)
-    {
-        let nodeData = this.byNode.get(node);
-        console.log("adding", msg.sourceFile, msg.sha1, "for", node.ip + ":" + node.port, nodeData ? nodeData.sha1s.length : -1);
-        if (nodeData) {
-            nodeData.sha1s.push(msg.sha1);
-            nodeData.size = msg.cacheSize;
-            const count = addToSHA1Map(this.bySHA1, msg.sha1, msg.fileSize, node);
-            if (this.distributeOnInsertion && count - 1 < this.redundancy) {
-                this.distribute({ sha1: msg.sha1, redundancy: this.redundancy });
-            }
-        } else {
-            console.error("insert: We don't seem to have this node", node.ip + ":" + node.port);
-        }
-    }
-
-    remove(msg, node)
-    {
-        let nodeData = this.byNode.get(node);
-        console.log("removing", msg.sourceFile, msg.sha1, "for", node.ip + ":" + node.port, nodeData ? nodeData.sha1s.length : -1);
-        if (nodeData) {
-            let idx = nodeData.sha1s.indexOf(msg.sha1);
-            if (idx != -1) {
-                nodeData.sha1s.splice(idx, 1);
-            } else {
-                console.error("We don't have", msg.sha1, "on", node.ip + ":" + node.port);
-            }
-            removeFromSHA1Map(this.bySHA1, msg.sha1, node);
-            nodeData.size = msg.cacheSize;
-        } else {
-            console.error("remove: We don't seem to have this node", node.ip + ":" + node.port);
-        }
-    }
-
-    addNode(node, data)
-    {
-        console.log("adding object cache node",
-                    node.ip + ":" + node.port,
-                    node.name, node.hostname,
-                    "maxSize", prettysize(data.maxSize),
-                    "cacheSize", prettysize(data.cacheSize),
-                    "sha1s", data.sha1s.length);
-        if (this.byNode.get(node)) {
-            console.log("We already have", node.ip + ":" + node.port);
-            return;
-        }
-        let sha1s = data.sha1s.map(item => item.sha1);
-        this.byNode.set(node, new NodeData(data.cacheSize, data.maxSize, sha1s));
-        data.sha1s.forEach(item => {
-            addToSHA1Map(this.bySHA1, item.sha1, item.fileSize, node);
-        });
-    }
-
-    removeNode(node)
-    {
-        console.log("removing node", node.ip + ":" + node.port);
-        let nodeData = this.byNode.get(node);
-        if (!nodeData) {
-            console.error("We don't have", node.ip + ":" + node.port);
-            return;
-        }
-        this.byNode.delete(node);
-        nodeData.sha1s.forEach(sha1 => removeFromSHA1Map(this.bySHA1, sha1, node));
-    }
-
-    dump(query)
-    {
-        if ("clear" in query) {
-            this.clear();
-        }
-        let ret = {
-            hits: this.hits,
-        };
-
-        if ("nodes" in query) {
-            ret.nodes = {};
-            const verbose = "verbose" in query;
-            this.byNode.forEach((value, key) => {
-                let data =  {
-                    sha1s: verbose ? value.sha1s : value.sha1s.length,
-                    maxSize: prettysize(value.maxSize),
-                    size: prettysize(value.size)
-                };
-                if (key.name)
-                    data.name = key.name;
-                if (key.hostname)
-                    data.name = key.hostname;
-                ret.nodes[key.ip + ":" + key.port] = data;
-            });
-        }
-
-        if ("objects" in query) {
-            ret.sha1 = {};
-            this.bySHA1.forEach((value, key) => {
-                // console.log(key, value);
-                ret.sha1[key] = { fileSize: prettysize(value.fileSize), nodes: value.nodes.map(node => node.ip + ":" + node.port) };
-            });
-        }
-
-        return ret;
-    }
-
-    distribute(query, res)
-    {
-        const dry = "dry" in query;
-        let redundancy = parseInt(query.redundancy);
-        if (isNaN(redundancy) || redundancy <= 0)
-            redundancy = 1;
-        let max = parseInt(query.max);
-        if (isNaN(max) || max <= 0)
-            max = undefined;
-        const sha1 = query.sha1;
-
-        let ret;
-        if (res) {
-            ret = { type: "fetch_cache_objects", "dry": dry, commands: {} };
-            console.log("distribute called with redundancy of", redundancy, "and max of", max, "dry", dry, "sha1", sha1);
-        }
-
-        let that = this;
-        let nodes = Array.from(this.byNode.keys());
-        let nodeIdx = 0;
-        let commands = new Map();
-        if (!sha1) {
-            this.byNode.forEach((value, key) => {
-                commands.set(key, { objects: [], available: value.maxSize - value.size });
-            });
-        }
-        let nodeRestriction = query.node;
-        let count = 0;
-        // console.log(commands);
-        if (this.byNode.size >= 2) {
-            // let max = 1;
-            let roundRobinIndex = 0;
-            function processObject(sha1, value)
-            {
-                if (max != undefined && max <= 0)
-                    return;
-                let needed = Math.min(redundancy + 1 - value.nodes.length, that.byNode.size - 1);
-                if (needed > 0) {
-                    let needed = redundancy + 1 - value.nodes.length;
-                    // console.log("should distribute", key, "to", needed, "nodes");
-
-                    let firstIdx;
-                    let found = 0;
-                    while (found < needed) {
-                        if (++nodeIdx == nodes.length)
-                            nodeIdx = 0;
-                        if (firstIdx === undefined) {
-                            firstIdx = nodeIdx;
-                        } else if (firstIdx === nodeIdx) {
-                            break;
-                        }
-                        let node = nodes[nodeIdx];
-                        if (value.nodes.indexOf(node) != -1) {
-                            continue;
-                        }
-                        let data = commands.get(node);
-                        let available;
-                        if (data) {
-                            available = data.available;
-                        } else {
-                            let nodeData = this.byNode[node];
-                            available = nodeData.maxSize - nodeData.size;
-                        }
-                        if (available < value.fileSize) {
-                            continue;
-                        }
-                        if (!data) {
-                            data = { objects: [], available: available };
-                            commands.set(node, data);
-                        }
-                        ++found;
-                        data.available -= value.fileSize;
-                        const src = value.nodes[roundRobinIndex++ % value.nodes.length];
-                        data.objects.push({ source: src.ip + ":" + src.port, sha1: sha1 });
-                        if (max != undefined && !--max)
-                            break;
-                    }
-                    // console.log("found candidates", candidates.map(node => node.ip + ":" + node.port));
-                }
-            }
-            if (sha1) {
-                let val = this.bySHA1.get(sha1);
-                if (val) {
-                    processObject(sha1, val);
-                } else {
-                    console.error("Couldn't find sha1", sha1);
-                }
-            } else {
-                this.bySHA1.forEach((value, key) => processObject(key, value));
-            }
-        }
-        commands.forEach((value, key) => {
-            // console.log(key.ip + ": " + key.port, "will receive", value.objects);
-            if (value.objects.length && (!nodeRestriction || (key.ip + ":" + key.port) == nodeRestriction)) {
-                console.log(`sending ${value.objects.length}/${this.bySHA1.size} fetch_cache_objects to ${key.ip}:${key.port}`);
-                if (ret)
-                    ret[`${key.ip}:${key.port}`] = value.objects;
-                count += value.objects.length;
-                if (!dry)
-                    key.send({ type: "fetch_cache_objects", objects: value.objects });
-            }
-        });
-        if (ret) {
-            ret["count"] = count;
-            res.send(JSON.stringify(ret, null, 4));
-        } else if (res) {
-
-        }
-    }
-};
-
-module.exports = ObjectCacheManager;

package/scheduler/peak.js

@@ -1,112 +0,0 @@
-class Peak {
-    constructor(interval, name)
-    {
-        this.interval = interval;
-        this.name = name;
-        this.peakActiveJobs = 0;
-        this.peakActiveJobsTime = Date.now();
-        this.peakUtilization = 0;
-        this.peakUtilizationTime = Date.now();
-        if (interval) {
-            this.actives = [];
-            this.utilizations = [];
-        }
-    }
-
-    record(now, activeJobs, utilization)
-    {
-        let ret = false;
-        if (!this.interval) {
-            if (activeJobs > this.peakActiveJobs) {
-                this.peakActiveJobs = activeJobs;
-                this.peakActiveJobsTime = now;
-                ret = true;
-            }
-            if (utilization > this.peakUtilization) {
-                this.peakUtilization = utilization;
-                this.peakUtilizationTime = now;
-                ret = true;
-            }
-        } else {
-            const cutoff = now - this.interval;
-            {
-                let idx = 0;
-                while (idx < this.actives.length) {
-                    if (this.actives[idx][0] > cutoff && this.actives[idx][1] > activeJobs) {
-                        break;
-                    }
-                    ++idx;
-                }
-                if (idx == this.actives.length) {
-                    ret = true;
-                    this.peakActiveJobs = activeJobs;
-                    this.peakActiveJobsTime = now;
-                    this.actives = [];
-                } else if (idx) {
-                    this.actives.splice(0, idx);
-                }
-                this.actives.push([now, activeJobs]);
-            }
-            {
-                let idx = 0;
-                while (idx < this.utilizations.length) {
-                    if (this.utilizations[idx][0] > cutoff && this.utilizations[idx][1] > utilization) {
-                        break;
-                    }
-                    ++idx;
-                }
-                if (idx == this.utilizations.length) {
-                    ret = true;
-                    this.peakUtilization = utilization;
-                    this.peakUtilizationTime = now;
-                    this.utilization = [];
-                } else if (idx) {
-                    this.utilizations.splice(0, idx);
-                }
-                this.utilizations.push([now, utilization]);
-            }
-        }
-        return ret;
-    }
-
-    toObject()
-    {
-        if (this.interval) {
-            const cutoff = Date.now() - this.interval;
-            let peakActiveJobs = 0;
-            {
-                let splice = 0;
-                for (let idx=0; idx<this.actives.length; ++idx) {
-                    if (this.actives[idx][0] < cutoff) {
-                        splice = idx + 1;
-                    } else {
-                        peakActiveJobs = Math.max(peakActiveJobs, this.actives[idx][1]);
-                    }
-                }
-                if (splice)
-                    this.actives.splice(0, splice);
-            }
-            let peakUtilization = 0;
-            {
-                let splice = 0;
-                for (let idx=0; idx<this.utilizations.length; ++idx) {
-                    if (this.utilizations[idx][0] < cutoff) {
-                        splice = idx + 1;
-                    } else {
-                        peakUtilization = Math.max(peakUtilization, this.utilizations[idx][1]);
-                    }
-                }
-                if (splice)
-                    this.utilizations.splice(0, splice);
-            }
-            return { activeJobs: peakActiveJobs, utilizations: peakUtilization };
-        }
-
-        return {
-            activeJobs: this.peakActiveJobs,
-            utilization: this.peakUtilization
-        };
-    }
-};
-
-module.exports = Peak;

package/scheduler/public/index.html

@@ -1,37 +0,0 @@
-<html>
-  <head>
-    <title>Fisk!</title>
-    <script language="javascript">
-      function doshit() {
-         var div = document.getElementById("output");
-         div.style = "overflow:auto; height:100%";
-         var connection = new WebSocket("ws://" +
-         window.location.origin.substr(7) + "/monitor");
-
-         connection.onerror = function (error) {
-             console.log('WebSocket Error ' + error);
-         };
-         connection.onmessage = function (e) {
-             console.log('Server: ' + e.data);
-             var text;
-             try {
-                 var msg = JSON.parse(e.data);
-                 text = (new Date().toLocaleTimeString()) + ": " + msg.builder.name + ": " + msg.builder.port + " (" + msg.builder.ip + ") compiled " + msg.sourceFile + " for " + msg.client.name + " (" + msg.client.ip + ") in " + msg.compileDuration + "ms";
-             } catch (err) {
-                 text = e.data;
-             }
-             var atBottom = div.scrollTop + 20 >= div.scrollHeight - div.clientHeight;
-             div.appendChild(document.createTextNode(text));
-             div.appendChild(document.createElement("br"));
-             if (atBottom) {
-                 div.scrollTop = div.scrollHeight - div.clientHeight;
-             }
-         };
-      }
-      window.onload = doshit;
-    </script>
-  </head>
-  <body>
-    <div id="output"></div>
-  </body>
-</html>