@andersbakken/fisk 3.5.7 → 3.6.1

package/builder/load.js

@@ -1,85 +0,0 @@
-const EventEmitter = require("events");
-const os = require("os");
-
-// some code taken from https://gist.github.com/bag-man/5570809
-
-//Create function to get CPU information
-function cpuAverage() {
-
-    //Initialise sum of idle and time of cores and fetch CPU info
-    let totalIdle = 0, totalTick = 0;
-    const cpus = os.cpus();
-
-    if (!(cpus instanceof Array))
-        return undefined;
-
-    //Loop through CPU cores
-    for(let i = 0, len = cpus.length; i < len; i++) {
-
-        //Select CPU core
-        let cpu = cpus[i];
-
-        //Total up the time in the cores tick
-        for(let type in cpu.times) {
-            totalTick += cpu.times[type];
-        }
-
-        //Total up the idle time of the core
-        totalIdle += cpu.times.idle;
-    }
-
-    //Return the average Idle and Tick times
-    return {idle: totalIdle / cpus.length,  total: totalTick / cpus.length};
-}
-
-//Grab first CPU Measure
-let startMeasure = cpuAverage();
-
-function measure() {
-    //Grab second Measure
-    const endMeasure = cpuAverage();
-
-    let percentageCPU;
-    if (endMeasure && startMeasure) {
-        //Calculate the difference in idle and total time between the measures
-        const idleDifference = endMeasure.idle - startMeasure.idle;
-        const totalDifference = endMeasure.total - startMeasure.total;
-        //Calculate the average percentage CPU usage
-        percentageCPU = 100 - ~~(100 * idleDifference / totalDifference);
-    }
-
-    //Next measure will diff against this measure
-    startMeasure = endMeasure;
-
-    return percentageCPU ? percentageCPU / 100 : undefined;
-};
-
-class Load extends EventEmitter {
-    constructor() {
-        super();
-        this._interval = undefined;
-    }
-
-    get running() {
-        return this._interval !== undefined;
-    }
-
-    start(interval) {
-        if (this._interval !== undefined)
-            throw new Error("Interval already running");
-        this._interval = setInterval(() => {
-            let m = measure();
-            if (m)
-                this.emit("data", m);
-        }, interval);
-    }
-
-    stop() {
-        if (!this._interval)
-            throw new Error("No interval running");
-        clearInterval(this._interval);
-        this._interval = undefined;
-    }
-}
-
-module.exports = new Load();

package/builder/objectcache.js

@@ -1,301 +0,0 @@
-const fs = require("fs-extra");
-const path = require("path");
-const EventEmitter = require("events");
-
-function prettysize(bytes)
-{
-    const prettysize = require("prettysize");
-    return prettysize(bytes, bytes >= 1024); // don't want 0Bytes
-}
-
-class ObjectCacheItem
-{
-    constructor(response, headerSize)
-    {
-        this.headerSize = headerSize;
-        this.response = response;
-        this.cacheHits = 0;
-    }
-
-    get contentsSize() { return this.response.index.reduce((total, item) => { return total + item.bytes; }, 0); }
-    get fileSize() { return 4 + this.headerSize + this.contentsSize; }
-    // get headerSize
-};
-
-class PendingItem
-{
-    constructor(response, path, dataBytes) // not including the metadata
-    {
-        this.response = response;
-        this.path = path;
-        this.remaining = dataBytes;
-        this.endCB = undefined;
-        this.file = fs.createWriteStream(path);
-        this.file.on("drain", () => {
-            // console.log("Got drain", this.buffer.length);
-            let buf = this.buffer;
-            this.buffer = undefined;
-            buf.forEach(b => this.write(b));
-            if (!this.buffer && this.endCB) {
-                this.file.end();
-                this.endCB();
-            }
-        });
-        this.buffer = undefined;
-
-        const json = Buffer.from(JSON.stringify(response));
-        const headerSizeBuffer = Buffer.allocUnsafe(4);
-        headerSizeBuffer.writeUInt32LE(json.length);
-        this.write(headerSizeBuffer);
-        this.write(json);
-        this.jsonLength = json.length;
-    }
-
-    write(data) {
-        // console.log("GOT DATA", this.path, data.length, this.buffer);
-        if (this.buffer) {
-            this.buffer.push(data);
-        } else if (!this.file.write(data)) {
-            // console.log("Failed to write", data.length);
-            this.buffer = [];
-        }
-    }
-    end(cb)
-    {
-        if (this.buffer) {
-            this.endCB = cb;
-        } else {
-            this.file.end(cb);
-        }
-    }
-};
-
-class ObjectCache extends EventEmitter
-{
-    constructor(dir, maxSize, purgeSize)
-    {
-        super();
-        this.dir = dir;
-        fs.mkdirpSync(dir);
-        this.maxSize = maxSize;
-        this.purgeSize = purgeSize;
-        this.cache = {};
-        this.pending = {};
-        this.size = 0;
-        // console.log(fs.readdirSync(this.dir, { withFileTypes: true }));
-        try {
-            fs.readdirSync(this.dir).map(fileName => {
-                let ret = { path: path.join(this.dir, fileName) };
-                if (fileName.length == 32) {
-                    try {
-                        let stat = fs.statSync(ret.path);
-                        if (stat.isFile()) {
-                            ret.size = stat.size;
-                            ret.atime = stat.atimeMs;
-                        }
-                    } catch (err) {
-                        console.error("Got error stating", ret.path, err);
-                    }
-                }
-                return ret;
-            }).sort((a, b) => a.atime - b.atime).forEach(item => {
-                this.loadFile(item.path, item.size);
-            });
-        } catch (err) {
-            console.error(`Got error reading directory ${dir}:`, err);
-        }
-        console.log("initializing object cache with", this.dir, "maxSize", prettysize(maxSize), "size", prettysize(this.size));
-    }
-
-    loadFile(filePath, fileSize)
-    {
-        let fileName = path.basename(filePath);
-        // console.log("got file", file);
-        let fd;
-        let jsonBuffer;
-        try {
-            if (fileName.length == 32) {
-                const headerSizeBuffer = Buffer.allocUnsafe(4);
-                fd = fs.openSync(filePath, "r");
-                const stat = fs.statSync(filePath);
-                fs.readSync(fd, headerSizeBuffer, 0, 4);
-                const headerSize = headerSizeBuffer.readUInt32LE(0);
-                // console.log("got headerSize", headerSize);
-                if (headerSize < 10 || headerSize > 1024 * 16)
-                    throw new Error(`Got bad header size for ${fileName}: ${headerSize}`);
-                jsonBuffer = Buffer.allocUnsafe(headerSize);
-                fs.readSync(fd, jsonBuffer, 0, headerSize);
-                const response = JSON.parse(jsonBuffer.toString());
-                if (response.sha1 != fileName)
-                    throw new Error(`Got bad filename: ${fileName} vs ${response.sha1}`);
-                let item = new ObjectCacheItem(response, headerSize);
-                if (item.fileSize != fileSize)
-                    throw new Error(`Got bad size for ${fileName} expected ${item.fileSize} got ${fileSize}`);
-                fs.closeSync(fd);
-                this.size += item.fileSize;
-                this.cache[fileName] = item;
-                this.emit("added", { sha1: response.sha1, sourceFile: response.sourceFile, fileSize: stat.size });
-            } else {
-                throw new Error("Unexpected file " + fileName);
-            }
-        } catch (err) {
-            if (fd)
-                fs.closeSync(fd);
-            console.error("got failure", filePath, err, jsonBuffer ? jsonBuffer.toString().substr(0, 100) : undefined);
-            try {
-                fs.removeSync(filePath);
-            } catch (doubleError) {
-                console.error("Can't even delete this one", doubleError);
-            }
-        }
-        return undefined;
-    }
-
-    state(sha1)
-    {
-        if (sha1 in this.cache) {
-            return "exists";
-        } else if (sha1 in this.pending) {
-            return "pending";
-        }
-        return "none";
-    }
-
-    get keys()
-    {
-        return Object.keys(this.cache);
-    }
-
-    clear() {
-        this.purge(0);
-    }
-
-    add(response, contents) {
-        if (response.sha1 in this.pending) {
-            console.log("Already writing this, I suppose this is possible", response);
-            return;
-        } else if (response.sha1 in this.cache) {
-            throw new Error("This should not happen. We already have " + response.sha1 + " in the cache");
-        }
-        let absolutePath = path.join(this.dir, response.sha1);
-        try {
-            fs.mkdirpSync(this.dir);
-        } catch (err) {
-        }
-
-        let remaining = 0;
-        response.index.forEach(file => { remaining += file.bytes; });
-
-        let pendingItem = new PendingItem(response, absolutePath, remaining);
-        pendingItem.file.on("error", err => {
-            console.error("Failed to write pendingItem", response, err);
-            delete this.pending[response.sha1];
-        });
-        this.pending[response.sha1] = pendingItem;
-        contents.forEach(c => pendingItem.write(c.contents));
-        pendingItem.end(() => {
-            if (this.pending[response.sha1] == pendingItem) {
-                let cacheItem = new ObjectCacheItem(response, pendingItem.jsonLength);
-                try {
-                    let stat = fs.statSync(path.join(this.dir, response.sha1));
-                    // console.log("stat is", stat.size, "for", path.join(this.dir, response.sha1));
-                    // console.log("shit", cacheItem);
-                    // console.log("ass", pendingItem);
-                    if (cacheItem.fileSize != stat.size) {
-                        throw new Error(`Wrong file size for ${path.join(this.dir, response.sha1)}, should have been ${cacheItem.fileSize} but ended up being ${stat.size}`);
-                    }
-                    this.cache[response.sha1] = cacheItem;
-                    // console.log(response);
-                    this.emit("added", { sha1: response.sha1, sourceFile: response.sourceFile, fileSize: cacheItem.fileSize });
-
-                    this.size += cacheItem.fileSize;
-                    if (this.size > this.maxSize)
-                        this.purge(this.purgeSize);
-                    console.log("Finished writing", response.sha1);
-                } catch (err) {
-                    console.error("Something wrong", err);
-                    try {
-                        fs.unlinkSync(pendingItem.path);
-                    } catch (err) {
-                        if (err.code != "ENOENT")
-                            console.error(`Failed to unlink ${pendingItem.path} ${err}`);
-                    }
-                }
-
-                delete this.pending[response.sha1];
-            }
-        });
-    }
-
-    get cacheHits()
-    {
-        let ret = 0;
-        for (let sha1 in this.cache) {
-            ret += this.cache[sha1].cacheHits;
-        }
-        return ret;
-    }
-
-    info(query)
-    {
-        if (!query)
-            query = {};
-        const ret = Object.assign({ cacheHits: this.cacheHits, usage: ((this.size / this.maxSize) * 100).toFixed(1) }, this);
-        ret.count = Object.keys(ret.cache).length;
-        delete ret._events;
-        delete ret._eventsCount;
-        if (!("objects" in query))
-            delete ret.cache;
-        if (!("pending" in query))
-            delete ret.pending;
-        [ "maxSize", "size", "purgeSize" ].forEach(key => {
-            ret[key] = prettysize(ret[key]);
-        });
-        return ret;
-    }
-
-    remove(sha1)
-    {
-        try {
-            const info = this.cache[sha1];
-            this.size -= info.fileSize;
-            delete this.cache[sha1];
-            this.emit("removed", { sha1: sha1, sourceFile: info.response.sourceFile, fileSize: info.fileSize });
-            fs.unlinkSync(path.join(this.dir, sha1));
-        } catch (err) {
-            console.error("Can't remove file", path.join(this.dir, sha1), err.toString());
-        }
-    }
-
-    purge(targetSize)
-    {
-        for (let sha1 in this.cache) {
-            if (this.size <= targetSize) {
-                break;
-            }
-            console.log(`purging ${sha1} because ${this.size} >= ${targetSize}`);
-            this.remove(sha1);
-        }
-    }
-
-    get(sha1, dontTouch)
-    {
-        let ret = this.cache[sha1];
-        if (!dontTouch && ret) {
-            delete this.cache[sha1];
-            this.cache[sha1] = ret;
-        }
-        return ret;
-    }
-
-    syncData()
-    {
-        let ret = [];
-        for (let key in this.cache) {
-            ret.push({ sha1: key, fileSize: this.cache[key].fileSize });
-        }
-        return ret;
-    }
-};
-
-module.exports = ObjectCache;

package/builder/quit-on-error.js

@@ -1,9 +0,0 @@
-module.exports = function(option) {
-    return function() {
-        let quitInterval = option.int("quit-on-error-delay", 2000);
-        if (quitInterval >= 0) {
-            console.log(`Quitting in ${quitInterval}ms`);
-            setTimeout(process.exit.bind(process), quitInterval);
-        }
-    };
-}

package/builder/server.js

@@ -1,214 +0,0 @@
-const EventEmitter = require("events");
-const WebSocket = require("ws");
-const Url = require("url");
-const http = require("http");
-const express = require("express");
-const zlib = require("zlib");
-
-class Job extends EventEmitter {
-    constructor(data) {
-        super();
-        Object.assign(this, data);
-    }
-
-    send(type, msg) {
-        if (this.ws.readyState !== WebSocket.OPEN)
-            return;
-        try {
-            if (msg === undefined) {
-                if (type instanceof Buffer) {
-                    this.ws.send(type);
-                } else {
-                    this.ws.send(JSON.stringify(type));
-                }
-            } else {
-                let tosend;
-                if (typeof msg === "object") {
-                    tosend = msg;
-                    tosend.type = type;
-                } else {
-                    tosend = { type: type, message: msg };
-                }
-                this.ws.send(JSON.stringify(tosend));
-            }
-        } catch (err) {
-            console.error("got send error", this.id, type, err);
-        }
-    }
-
-    get readyState() {
-        return this.ws.readyState;
-    }
-
-    close() {
-        this.closed = true;
-        this.ws.close();
-    }
-};
-
-class Server extends EventEmitter {
-    constructor(option, configVersion) {
-        super();
-        this.option = option;
-        this.id = 0;
-        this.configVersion = configVersion;
-        this.app = undefined;
-    }
-
-    listen() {
-        this.app = express();
-        this.emit("listen", this.app);
-        this.port = this.option.int("port", 8096);
-
-        this.server = http.createServer(this.app);
-        this.ws = new WebSocket.Server({ noServer: true, backlog: this.option.int("backlog", 50) });
-        this.server.listen({ port: this.port, backlog: this.option.int("backlog", 50), host: "0.0.0.0" });
-
-        this.server.on("upgrade", (req, socket, head) => {
-            this.ws.handleUpgrade(req, socket, head, (ws) => {
-                this._handleConnection(ws, req);
-            });
-        });
-
-        console.log("listening on", this.port);
-        this.ws.on("headers", (headers, request) => {
-            this.emit("headers", headers, request);
-        });
-    }
-
-    _handleConnection(ws, req) {
-        const connectTime = Date.now();
-        let client = undefined;
-        let bytes = undefined;
-        let ip = req.connection.remoteAddress;
-        let clientEmitted = false;
-        const error = msg => {
-            ws.send(`{"error": "${msg}"}`);
-            ws.close();
-            if (client && clientEmitted) {
-                client.emit("error", msg);
-            } else {
-                this.emit("error", { ip: ip, message: msg });
-            }
-        };
-
-        if (!ip) { // already closed
-            // console.log(req.connection, ws.readyState);
-            return;
-        }
-        if (ip.substr(0, 7) == "::ffff:") {
-            ip = ip.substr(7);
-        }
-
-        const url = Url.parse(req.url);
-        switch (url.pathname) {
-        case "/compile":
-            const hash = req.headers["x-fisk-environments"];
-            if (!hash) {
-                error("Bad ws request, no environments");
-                return;
-            }
-            const name = req.headers["x-fisk-client-name"];
-            const configVersion = req.headers["x-fisk-config-version"];
-            if (configVersion != this.configVersion) {
-                error(`Bad config version, expected ${this.configVersion}, got ${configVersion}`);
-                return;
-            }
-
-            // console.log("GOT HEADERS", req.headers);
-            client = new Job({ ws: ws,
-                               ip: ip,
-                               hash: hash,
-                               name: name,
-                               hostname: req.headers["x-fisk-client-hostname"],
-                               user: req.headers["x-fisk-user"],
-                               sourceFile: req.headers["x-fisk-sourcefile"],
-                               priority: parseInt(req.headers["x-fisk-priority"]) || 0,
-                               sha1: req.headers["x-fisk-sha1"],
-                               id: parseInt(req.headers["x-fisk-job-id"]),
-                               builderIp: req.headers["x-fisk-builder-ip"] });
-
-            break;
-        default:
-            error(`Invalid pathname ${url.pathname}`);
-            return;
-        }
-
-        ws.on("message", msg => {
-            switch (typeof msg) {
-            case "string":
-                // console.log("Got message", msg);
-                if (bytes) {
-                    // bad, client have to send all the data in a binary message before sending JSON
-                    error(`Got JSON message while ${bytes} bytes remained of a binary message`);
-                    return;
-                }
-                // assume JSON
-                let json;
-                try {
-                    json = JSON.parse(msg);
-                } catch (e) {
-                }
-                if (json === undefined) {
-                    error("Unable to parse string message as JSON");
-                    return;
-                }
-                bytes = json.bytes;
-                client.compressed = json.compressed;
-                client.commandLine = json.commandLine;
-                client.argv0 = json.argv0;
-                client.connectTime = connectTime;
-                client.wait = json.wait;
-                this.emit("job", client);
-                clientEmitted = true;
-                break;
-            case "object":
-                if (msg instanceof Buffer) {
-                    // console.log("Got binary", msg.length, bytes);
-                    if (!msg.length) {
-                        // no data?
-                        error("No data in buffer");
-                        return;
-                    }
-                    if (!bytes) {
-                        error("Got binary message without a preceeding json message describing the data");
-                        return;
-                    }
-                    if (msg.length !== bytes) {
-                        // woops
-                        error(`length ${msg.length} !== ${bytes}`);
-                        return;
-                    }
-                    bytes = 0;
-                    // console.log("GOT DATA", client.compressed, msg.length);
-                    if (client.compressed) {
-                        zlib.gunzip(msg, (err, data) => {
-                            if (err) {
-                                error(`Got error inflating data ${err}`);
-                            } else {
-                                client.emit("data", { data });
-                            }
-                        });
-                    } else {
-                        client.emit("data", { data: msg });
-                    }
-                }
-                break;
-            }
-        });
-        ws.on("close", () => {
-            if (client && clientEmitted) {
-                // console.error("GOT WS CLOSE", bytes, client.objectcache);
-                client.emit("close");
-            }
-            ws.removeAllListeners();
-        });
-        ws.on("error", (error) => {
-            console.log("GOT WS ERROR", error);
-            if (client && clientEmitted)
-                client.emit("error", error);
-        });
-    }
-}
-
-module.exports = Server;

package/common/index.js

@@ -1,45 +0,0 @@
-const fs = require('fs-extra');
-const path = require('path');
-const os = require('os');
-
-const Version = 5;
-
-function cacheDir(option)
-{
-    let dir = option("cache-dir");
-    if (!dir) {
-        dir = path.join(os.homedir(), ".cache", "fisk", path.basename(option.prefix));
-    }
-    return dir;
-}
-
-function validateCache(option)
-{
-    const dir = cacheDir(option);
-    const file = path.join(dir, 'version');
-    // console.log(dir);
-    let version;
-    try {
-        version = fs.readFileSync(file);
-        if (version.readUInt32BE() == Version) {
-            return;
-        }
-    } catch (err) {
-    }
-    if (version)
-        console.log(`Wrong version. Destroying cache ${dir}`);
-    fs.removeSync(dir);
-    fs.mkdirpSync(dir);
-    let buf = Buffer.allocUnsafe(4);
-    buf.writeUInt32BE(Version);
-    fs.writeFileSync(file, buf);
-}
-
-
-module.exports = function(option) {
-    validateCache(option);
-    return {
-        cacheDir: cacheDir.bind(this, option),
-        Version: Version
-    };
-};