@andersbakken/fisk 3.5.6 → 3.6.0

package/package.json

@@ -1,36 +1,70 @@
 {
-  "name": "@andersbakken/fisk",
-  "version": "3.5.6",
-  "description": "Fisk, a distributed compile system",
-  "scripts": {
-    "test": "echo \"Error: no test specified\" && exit 1"
-  },
-  "repository": {
-    "type": "git",
-    "url": "git+https://github.com/Andersbakken/fisk.git"
-  },
-  "author": "",
-  "license": "ISC",
-  "bugs": {
-    "url": "https://github.com/Andersbakken/fisk/issues"
-  },
-  "homepage": "https://github.com/Andersbakken/fisk#readme",
-  "dependencies": {
-    "@jhanssen/options": "^1.0.25",
-    "axios": "^0.21.1",
-    "blessed": "^0.1.81",
-    "bytes": "^3.0.0",
-    "compare-versions": "^3.4.0",
-    "express": "^4.16.3",
-    "fs-extra": "^6.0.1",
-    "humanize-duration": "^3.17.0",
-    "lowdb": "^1.0.0",
-    "minimist": "^1.2.5",
-    "mktemp": "^0.4.0",
-    "parse-duration": "^0.1.1",
-    "posix": "^4.2.0",
-    "prettysize": "^2.0.0",
-    "wake_on_lan": "^1.0.0",
-    "ws": "^7.4.6"
-  }
+    "name": "@andersbakken/fisk",
+    "version": "3.6.0",
+    "description": "Fisk, a distributed compile system",
+    "scripts": {
+        "lint": "eslint . --ext .ts",
+        "rollup": "rollup -c",
+        "tsc": "tsc -p .",
+        "build": "npm run clean:dist && mkdir -p dist && run-p lint rollup",
+        "clean:dist": "del-cli dist/*",
+        "clean:cache": "del-cli .cache/*",
+        "clean": "run-p clean:dist clean:cache",
+        "watch": "rollup -c -w"
+    },
+    "repository": {
+        "type": "git",
+        "url": "git+https://github.com/Andersbakken/fisk.git"
+    },
+    "author": "",
+    "license": "ISC",
+    "bugs": {
+        "url": "https://github.com/Andersbakken/fisk/issues"
+    },
+    "homepage": "https://github.com/Andersbakken/fisk#readme",
+    "files": ["dist/*.js", "dist/*.js.map"],
+    "dependencies": {
+        "@jhanssen/options": "^1.0.34",
+        "@josh_stern/mktemp": "^1.0.5",
+        "axios": "^0.21.1",
+        "blessed": "^0.1.81",
+        "bufferutil": "^4.0.7",
+        "bytes": "^3.0.0",
+        "chroot": "^1.0.11",
+        "compare-versions": "^3.4.0",
+        "express": "^4.18.2",
+        "fs-extra": "^6.0.1",
+        "humanize-duration": "^3.17.0",
+        "lowdb": "^1.0.0",
+        "minimist": "^1.2.5",
+        "parse-duration": "^1.0.2",
+        "posix": "^4.2.0",
+        "pretty-bytes": "^6.0.0",
+        "typescript": "^4.3.5",
+        "utf-8-validate": "^5.0.10",
+        "ws": "^7.4.6"
+    },
+    "devDependencies": {
+        "@rollup/plugin-commonjs": "^24.0.1",
+        "@rollup/plugin-json": "^6.0.0",
+        "@rollup/plugin-node-resolve": "^13.0.6",
+        "@rollup/plugin-replace": "^2.3.4",
+        "@types/bytes": "^3.1.1",
+        "@types/express": "^4.17.13",
+        "@types/fs-extra": "^9.0.13",
+        "@types/humanize-duration": "^3.27.1",
+        "@types/node": "^14.14.25",
+        "@types/posix": "^4.2.0",
+        "@types/ws": "^8.5.3",
+        "@typescript-eslint/eslint-plugin": "^4.14.2",
+        "@typescript-eslint/parser": "^4.14.2",
+        "del-cli": "^3.0.1",
+        "eslint": "^7.19.0",
+        "eslint-plugin-sort-imports-es6-autofix": "^0.5.0",
+        "npm-run-all": "^4.1.5",
+        "prettier": "^2.2.1",
+        "rollup": "^2.38.4",
+        "rollup-plugin-hashbang": "^3.0.0",
+        "rollup-plugin-typescript2": "^0.27.0"
+    }
 }

package/builder/VM.js

@@ -1,156 +0,0 @@
-/* global quitOnError */
-const EventEmitter = require('events');
-const child_process = require('child_process');
-const fs = require('fs-extra');
-const path = require('path');
-let quitOnError;
-
-class CompileJob extends EventEmitter
-{
-    constructor(commandLine, argv0, id, vm) {
-        super();
-        this.vm = vm;
-        this.commandLine = commandLine;
-        this.argv0 = argv0;
-        this.id = id;
-        this.dir = path.join(vm.root, 'compiles', "" + this.id);
-        this.vmDir = path.join('/', 'compiles', "" + this.id);
-        fs.mkdirpSync(this.dir);
-        this.fd = fs.openSync(path.join(this.dir, 'sourcefile'), "w");
-        this.cppSize = 0;
-        this.startCompile = undefined;
-    }
-
-    sendCallback(error) {
-        if (error) {
-            console.error("Got send error for", this.vmDir, this.id, this.commandLine);
-            this.vm.compileFinished({type: 'compileFinished', success: false, id: this.id, files: [], exitCode: -1, error: error.toString() });
-        }
-    }
-
-    feed(data) {
-        fs.writeSync(this.fd, data);
-        this.cppSize += data.length;
-        this.startCompile = Date.now();
-        fs.close(this.fd);
-        this.fd = undefined;
-        this.vm.child.send({ type: "compile", commandLine: this.commandLine, argv0: this.argv0, id: this.id, dir: this.vmDir}, this.sendCallback.bind(this));
-    }
-
-    cancel() {
-        this.vm.child.send({ type: "cancel", id: this.id}, this.sendCallback.bind(this));
-    }
-};
-
-class VM extends EventEmitter
-{
-    constructor(root, hash, option) {
-        super();
-        quitOnError = require('./quit-on-error')(option);
-        this.root = root;
-        this.hash = hash;
-        this.compiles = {};
-        this.destroying = false;
-        this.keepCompiles = option("keep-compiles") || false;
-
-        fs.remove(path.join(root, 'compiles'));
-
-        let args = [ `--root=${root}`, `--hash=${hash}` ];
-        let user = option("vm-user");
-        if (user)
-            args.push(`--user=${user}`);
-        if (option("debug"))
-            args.push("--debug");
-
-        this.child = child_process.fork(path.join(__dirname, "VM_runtime.js"), args);
-        let gotReady = false;
-        this.child.on('message', msg => {
-            // console.log("Got message", msg);
-            switch (msg.type) {
-            case 'ready':
-                gotReady = true;
-                this.emit('ready', { success: true });
-                break;
-            case 'error':
-                console.error("Got error from runtime", this.hash, msg.message);
-                break;
-            case 'compileStdOut': {
-                let compile = this.compiles[msg.id];
-                if (compile)
-                    compile.emit('stdout', msg.data);
-                break; }
-            case 'compileStdErr': {
-                let compile = this.compiles[msg.id];
-                if (compile)
-                    compile.emit('stderr', msg.data);
-                break; }
-            case 'compileFinished':
-                this.compileFinished(msg);
-                break;
-            }
-        });
-        this.child.on('error', err => {
-            if (!gotReady) {
-                this.emit('ready', { success: false, error: err });
-            } else {
-                console.error("Got error", err);
-            }
-        });
-        this.child.on('exit', evt => {
-            console.log("Child going down", evt, this.destroying);
-            if (this.destroying)
-                fs.remove(root);
-            // ### need to handle the helper accidentally going down maybe?
-            this.emit("exit");
-        });
-    }
-
-    compileFinished(msg) {
-        let compile = this.compiles[msg.id];
-        if (!compile)
-            return;
-        if (msg.error)
-            console.error("Got some error", msg.error);
-        const now = Date.now();
-        compile.emit('finished', {
-            cppSize: compile.cppSize,
-            compileDuration: (now - compile.startCompile),
-            exitCode: msg.exitCode,
-            success: msg.success,
-            error: msg.error,
-            sourceFile: msg.sourceFile,
-            files: msg.files.map(file => {
-                file.absolute = path.join(this.root, file.mapped ? file.mapped : file.path);
-                delete file.mapped;
-                return file;
-            })
-        });
-
-        if (!this.keepCompiles)
-            fs.remove(this.compiles[msg.id].dir);
-        delete this.compiles[msg.id];
-    }
-
-    destroy() {
-        this.destroying = true;
-        this.child.send({type: 'destroy'}, err => {
-            if (err) {
-                console.error("Failed to send destroy message to child", this.hash, err);
-                this.child.kill();
-            }
-        });
-    }
-
-    startCompile(commandLine, argv0, id) {
-        let compile = new CompileJob(commandLine, argv0, id, this);
-        this.compiles[compile.id] = compile;
-        // console.log("startCompile " + compile.id);
-        return compile;
-    }
-
-    setDebug(debug) {
-        this.child.send({type: "setDebug", debug: debug});
-    }
-};
-
-module.exports = VM;

package/builder/VM_runtime.js

@@ -1,171 +0,0 @@
-const path = require('path');
-const fs = require('fs');
-const os = require('os');
-const posix = require('posix');
-const Compile = require('./compile');
-
-const argv = require('minimist')(process.argv.slice(2));
-
-function send(message) {
-    try {
-        process.send(message);
-    } catch (err) {
-        console.error(`Couldn't send message ${message.type}. Going down`);
-        process.exit();
-    }
-}
-
-process.on('unhandledRejection', (reason, p) => {
-    send({type: "error", message: `Unhandled rejection at: Promise ${p} reason: ${reason.stack}`});
-    console.error('Unhandled rejection at: Promise', p, 'reason:', reason.stack);
-});
-
-process.on('uncaughtException', err => {
-    send({type: "error", message: `Uncaught exception ${err.stack} ${err.toString()}`});
-    console.error("Uncaught exception", err);
-});
-
-let pwd;
-if (argv.user) {
-    try {
-        pwd = posix.getpwnam(argv.user);
-    } catch(err) {
-        console.error("Couldn't find user", argv.user);
-        throw err;
-    }
-
-    try {
-        process.initgroups(argv.user, pwd.gid);
-    } catch(err) {
-        throw new Error('Changing groups failed: ' + err.message);
-    }
-}
-
-try {
-    console.log("Chrooting to", argv.root);
-    posix.chroot(argv.root);
-} catch (err) {
-    console.error('Changing root or user failed', err);
-    process.exit(1);
-}
-
-if (pwd) {
-    process.setgid(pwd.gid);
-    process.setuid(pwd.uid);
-}
-
-process.on("error", error => {
-    console.error(`Got process error ${error} ${JSON.stringify(argv)}. Going down`);
-    process.exit();
-});
-
-let libDirs = [];
-const mac = os.type() === "Darwin";
-
-function isLibrary(file)
-{
-    if (file == "ld.so.conf || file == ld.so.cache")
-        return false;
-    const suffix = path.extname(file);
-    if (mac)
-        return suffix == ".dylib";
-
-    // console.log("got file", suffix, file);
-    if (suffix == ".so") {
-        return true;
-    }
-    return file.indexOf(".so.") != -1;
-}
-
-function findLibraries(dir)
-{
-    const files = fs.readdirSync(dir);
-    // console.log("findLibraries", dir, files.length);
-    let found = false;
-    files.forEach(file => {
-        let stat;
-        try {
-            stat = fs.statSync(path.join(dir, file));
-        } catch (err) {
-            console.error("Got error", err);
-            return;
-        }
-
-
-        if (stat.isDirectory()) {
-            findLibraries(path.join(dir, file));
-        } else if (!found && stat.isFile() && isLibrary(file)) {
-            found = true;
-            libDirs.push(dir);
-        }
-    });
-}
-
-findLibraries("/");
-if (argv.debug)
-    console.log("Got lib directories", argv.root, libDirs);
-if (libDirs.length) {
-    process.env[mac ? "DYLD_LIBRARY_PATH" : "LD_LIBRARY_PATH"] = libDirs.join(":");
-}
-setTimeout(() => { // hack
-    try {
-        send({type: "ready"});
-    } catch (err) {
-        console.error("Couldn't send ready. Going down");
-        process.exit();
-    }
-}, 1000);
-
-let compiles = {};
-let destroying = false;
-
-process.on('message', msg => {
-    switch (msg.type) {
-    case 'destroy':
-        if (!compiles.length) {
-            process.exit();
-        } else {
-            destroying = true;
-        }
-        break;
-    case 'setDebug':
-        if (msg.debug) {
-            argv.debug = true;
-        } else {
-            delete argv["debug"];
-        }
-        console.log("set debug to", msg.debug, "for", argv.root);
-        break;
-    case 'cancel':
-        let c = compiles[msg.id];
-        if (c)
-            c.kill();
-        break;
-    case 'compile':
-        try {
-            // console.log("compiling for );
-            if (argv.debug) {
-                console.log("Creating new compile", msg.commandLine, msg.argv0, msg.dir);
-            }
-            let compile = new Compile(msg.commandLine, msg.argv0, msg.dir, argv.debug);
-            // console.log("running thing", msg.commandLine);
-            compile.on('stdout', data => send({ type: 'compileStdOut', id: msg.id, data: data }));
-            compile.on('stderr', data => send({ type: 'compileStdErr', id: msg.id, data: data }));
-            compile.on('exit', event => {
-                delete compiles[msg.id];
-                if ("error" in event) {
-                    send({type: 'compileFinished', success: false, error: event.error, id: msg.id, files: event.files, exitCode: event.exitCode, sourceFile: event.sourceFile });
-                } else {
-                    send({type: 'compileFinished', success: true, id: msg.id, files: event.files, exitCode: event.exitCode, sourceFile: event.sourceFile });
-                }
-                if (destroying && !compiles.length)
-                    process.exit();
-            });
-            compiles[msg.id] = compile;
-        } catch (err) {
-            delete compiles[msg.id];
-            send({type: 'compileFinished', success: false, id: msg.id, files: [], exitCode: -1, error: err.toString() });
-        }
-        break;
-    }
-});

package/builder/client.js

@@ -1,181 +0,0 @@
-const EventEmitter = require("events");
-const WebSocket = require("ws");
-const os = require('os');
-const path = require('path');
-const fs = require('fs');
-
-class Client extends EventEmitter {
-    constructor(option, configVersion) {
-        super();
-
-        this.configVersion = configVersion;
-        this.scheduler = option("scheduler", "ws://localhost:8097");
-        if (this.scheduler.indexOf('://') == -1)
-            this.scheduler = "ws://" + this.scheduler;
-        if (!/:[0-9]+$/.exec(this.scheduler))
-            this.scheduler += ":8097";
-        this.serverPort = option.int("port", 8096);
-        this.hostname = option("hostname");
-        this.name = option("name");
-        this.slots = option.int("slots", os.cpus().length);
-        this.labels = option("labels");
-        try {
-            this.npmVersion = JSON.parse(fs.readFileSync(path.join(__dirname, "../package.json"))).version;
-        } catch (err) {
-        }
-        console.log("this is our npm version", this.npmVersion);
-        if (!this.name) {
-            if (this.hostname) {
-                this.name = this.hostname;
-            } else  {
-                this.name = os.hostname();
-            }
-        }
-    }
-
-    connect(environments) {
-        const url = `${this.scheduler}/builder`;
-        console.log("connecting to", url);
-
-        let remaining = 0;
-        let system;
-        switch (os.platform()) {
-        case 'darwin': system = "Darwin"; break;
-        case 'linux': system = "Linux"; break;
-        default: console.error("Unknown platform", os.platform()); break;
-        }
-        switch (os.arch()) {
-        case 'ia32': system += " i686"; break;
-        case 'x64': system += " x86_64"; break;
-        default: console.error("Unknown architecture", os.arch()); break;
-        }
-        let headers = {
-            "x-fisk-port": this.serverPort,
-            "x-fisk-environments": environments.join(";"),
-            "x-fisk-config-version": this.configVersion,
-            "x-fisk-builder-name": this.name,
-            "x-fisk-system": system,
-            "x-fisk-slots": this.slots,
-            "x-fisk-npm-version": this.npmVersion
-        };
-
-        if (this.labels) {
-            headers["x-fisk-builder-labels"] = this.labels;
-        }
-
-        if (this.hostname) {
-            headers["x-fisk-builder-hostname"] = this.hostname;
-        }
-
-        this.ws = new WebSocket(url, { headers: headers });
-        this.ws.on("open", () => {
-            this.emit("connect");
-        });
-        this.ws.on("error", err => {
-            console.error("client websocket error", err.message);
-        });
-        this.ws.on("upgrade", res => {
-            this.emit("objectCache", res.headers["x-fisk-object-cache"] == "true");
-        });
-
-        this.ws.on("message", msg => {
-            const error = msg => {
-                this.ws.send(`{"error": "${msg}"}`);
-                this.ws.close();
-                this.emit("error", msg);
-            };
-
-            switch (typeof msg) {
-            case "string":
-                if (remaining) {
-                    // bad, client have to send all the data in a binary message before sending JSON
-                    error(`Got JSON message while ${remaining.bytes} bytes remained of a binary message`);
-                    return;
-                }
-                // assume JSON
-                let json;
-                try {
-                    json = JSON.parse(msg);
-                } catch (e) {
-                }
-                if (json === undefined) {
-                    error("Unable to parse string message as JSON");
-                    return;
-                }
-                if (!json.type) {
-                    error("Bad message, no type");
-                    return;
-                }
-
-                console.log("got message from scheduler", json.type);
-
-                if (json.bytes) {
-                    remaining = json.bytes;
-                }
-                this.emit(json.type, json);
-                break;
-            case "object":
-                if (msg instanceof Buffer) {
-                    if (!msg.length) {
-                        // no data?
-                        error("No data in buffer");
-                        return;
-                    }
-                    if (!remaining) {
-                        error(`Unexpected binary message of length: ${msg.length}`);
-                        return;
-                    } else if (msg.length !== remaining) {
-                        // woops
-                        error(`length ${msg.length} !== ${remaining}`);
-                        return;
-                    }
-                    remaining = 0;
-                    this.emit("data", { data: msg });
-                } else {
-                    error("Unexpected object");
-                }
-                break;
-            }
-        });
-        this.ws.on("close", () => {
-            if (remaining.bytes)
-                this.emit("error", "Got close while reading a binary message");
-            this.emit("close");
-            if (this.ws)
-                this.ws.removeAllListeners();
-            this.ws = undefined;
-        });
-    }
-
-    sendBinary(blob) {
-        try {
-            this.ws.send(blob);
-        } catch (err) {
-            this.emit("err", err.toString());
-        }
-    }
-    send(type, msg) {
-        if (!this.ws) {
-            this.emit("error", "No connected websocket");
-            return;
-        }
-        try {
-            if (msg === undefined) {
-                this.ws.send(JSON.stringify(type));
-            } else {
-                let tosend;
-                if (typeof msg === "object") {
-                    tosend = msg;
-                    tosend.type = type;
-                } else {
-                    tosend = { type: type, message: msg };
-                }
-                this.ws.send(JSON.stringify(tosend));
-            }
-        } catch (err) {
-            this.emit("err", err.toString());
-        }
-    }
-}
-
-module.exports = Client;