@andersbakken/fisk 3.4.103 → 3.5.0

package/builder/VM.js

@@ -28,15 +28,13 @@ class CompileJob extends EventEmitter
         }
     }
 
-    feed(data, last) {
+    feed(data) {
         fs.writeSync(this.fd, data);
         this.cppSize += data.length;
-        if (last) {
-            this.startCompile = Date.now();
-            fs.close(this.fd);
-            this.fd = undefined;
-            this.vm.child.send({ type: "compile", commandLine: this.commandLine, argv0: this.argv0, id: this.id, dir: this.vmDir}, this.sendCallback.bind(this));
-        }
+        this.startCompile = Date.now();
+        fs.close(this.fd);
+        this.fd = undefined;
+        this.vm.child.send({ type: "compile", commandLine: this.commandLine, argv0: this.argv0, id: this.id, dir: this.vmDir}, this.sendCallback.bind(this));
     }
 
     cancel() {

package/builder/client.js

@@ -121,18 +121,16 @@ class Client extends EventEmitter {
                         error("No data in buffer");
                         return;
                     }
-                    if (remaining) {
-                        // more data
-                        if (msg.length > remaining) {
-                            // woops
-                            error(`length ${msg.length} > ${remaining}`);
-                            return;
-                        }
-                        remaining -= msg.length;
-                        this.emit("data", { data: msg, last: !remaining });
-                    } else {
+                    if (!remaining) {
                         error(`Unexpected binary message of length: ${msg.length}`);
+                        return;
+                    } else if (msg.length !== remaining) {
+                        // woops
+                        error(`length ${msg.length} !== ${remaining}`);
+                        return;
                     }
+                    remaining = 0;
+                    this.emit("data", { data: msg });
                 } else {
                     error("Unexpected object");
                 }

package/builder/fisk-builder.js

@@ -21,6 +21,7 @@ const VM = require("./VM");
 const load = require("./load");
 const ObjectCache = require("./objectcache");
 const quitOnError = require("./quit-on-error")(option);
+const zlib = require("zlib");
 
 if (process.getuid() !== 0) {
     console.error("fisk builder needs to run as root to be able to chroot");
@@ -652,7 +653,7 @@ server.on("job", job => {
         aborted: false,
         started: false,
         heartbeatTimer: undefined,
-        buffers: [],
+        buffer: undefined,
         stdout: "",
         stderr: "",
         start: function() {
@@ -713,11 +714,13 @@ server.on("job", job => {
 
             console.log("Starting job", j.id, job.sourceFile, "for", job.ip, job.name, "wait", job.wait);
             j.op = vm.startCompile(job.commandLine, job.argv0, job.id);
-            j.buffers.forEach(data => j.op.feed(data.data, data.last));
+            if (j.buffer) {
+                j.op.feed(j.buffer);
+                j.buffer = undefined;
+            }
             if (job.wait) {
                 job.send("resume", {});
             }
-            delete j.buffers;
             j.op.on("stdout", data => { j.stdout += data; }); // ### is there ever any stdout? If there is, does the order matter for stdout vs stderr?
             j.op.on("stderr", data => { j.stderr += data; });
             j.op.on("finished", event => {
@@ -735,7 +738,11 @@ server.on("job", job => {
                 }
 
                 // this can't be async, the directory is removed after the event is fired
-                let contents = event.files.map(f => { return { contents: fs.readFileSync(f.absolute), path: f.path }; });
+                const forCache = event.files.map(f => ({ contents: fs.readFileSync(f.absolute), path: f.path }));
+                const contents = !j.job.compressed ? forCache : forCache.map(x => ({
+                    path: x.path,
+                    contents: x.contents.byteLength ? zlib.gzipSync(x.contents) : x.contents
+                }));
                 let response = {
                     type: "response",
                     index: contents.map(item => { return { path: item.path, bytes: item.contents.length }; }),
@@ -755,13 +762,14 @@ server.on("job", job => {
                     response.sourceFile = job.sourceFile;
                     response.commandLine = job.commandLine;
                     response.environment = job.hash;
-                    objectCache.add(response, contents);
+                    objectCache.add(response, forCache);
                 }
 
-                for (let i=0; i<contents.length; ++i) {
-                    job.send(contents[i].contents);
-                }
-                // job.close();
+                contents.forEach(x => {
+                    if (x.contents.byteLength) {
+                        job.send(x.contents);
+                    }
+                });
                 // console.log("GOT ID", j);
                 if (event.success) {
                     client.send("jobFinished", {
@@ -822,14 +830,13 @@ server.on("job", job => {
     });
 
     job.on("data", data => {
-        // console.log("got data", this.id, data.last, typeof j.op);
-        if (data.last)
-            uploadDuration = Date.now() - jobStartTime;
+        // console.log("got data", this.id, typeof j.op);
+        uploadDuration = Date.now() - jobStartTime;
         if (!j.op) {
-            j.buffers.push(data);
-            console.log("buffering...", j.buffers.length);
+            j.buffer = data.data;
+            console.log("buffering...", j.buffer.byteLength);
         } else {
-            j.op.feed(data.data, data.last);
+            j.op.feed(data.data);
         }
     });
 

package/builder/server.js

@@ -1,15 +1,14 @@
-
 const EventEmitter = require("events");
 const WebSocket = require("ws");
 const Url = require("url");
 const http = require("http");
 const express = require("express");
+const zlib = require("zlib");
 
 class Job extends EventEmitter {
     constructor(data) {
         super();
-        for (let key in data)
-            this[key] = data[key];
+        Object.assign(this, data);
     }
 
     send(type, msg) {
@@ -154,6 +153,7 @@ class Server extends EventEmitter {
                     return;
                 }
                 bytes = json.bytes;
+                client.compressed = json.compressed;
                 client.commandLine = json.commandLine;
                 client.argv0 = json.argv0;
                 client.connectTime = connectTime;
@@ -166,21 +166,31 @@ class Server extends EventEmitter {
                     // console.log("Got binary", msg.length, bytes);
                     if (!msg.length) {
                         // no data?
-                        console.error("No data in buffer");
+                        error("No data in buffer");
                         return;
                     }
                     if (!bytes) {
                         error("Got binary message without a preceeding json message describing the data");
                         return;
                     }
-                    if (msg.length > bytes) {
+                    if (msg.length !== bytes) {
                         // woops
-                        error(`length ${msg.length} > ${bytes}`);
+                        error(`length ${msg.length} !== ${bytes}`);
                         return;
                     }
-                    bytes -= msg.length;
-                    // console.log("Emitting", "data", { data: msg.length, last: !bytes });
-                    client.emit("data", { data: msg, last: !bytes });
+                    bytes = 0;
+                    // console.log("GOT DATA", client.compressed, msg.length);
+                    if (client.compressed) {
+                        zlib.gunzip(msg, (err, data) => {
+                            if (err) {
+                                error(`Got error inflating data ${err}`);
+                            } else {
+                                client.emit("data", { data });
+                            }
+                        });
+                    } else {
+                        client.emit("data", { data: msg });
+                    }
                 }
                 break;
             }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@andersbakken/fisk",
-  "version": "3.4.103",
+  "version": "3.5.0",
   "description": "Fisk, a distributed compile system",
   "scripts": {
     "test": "echo \"Error: no test specified\" && exit 1"

package/scheduler/server.js

@@ -10,7 +10,7 @@ class Client extends EventEmitter {
     constructor(object) {
         super();
         this.created = new Date();
-        this.assign(object);
+        Object.assign(this, object);
         this.pingSent = undefined;
         this.ws.on("pong", () => {
             // console.log("got pong", this.name);
@@ -18,12 +18,6 @@ class Client extends EventEmitter {
         });
     }
 
-    assign(object) {
-        for (let key in object) {
-            this[key] = object[key];
-        }
-    }
-
     send(type, msg) {
         try {
             if (msg === undefined) {
@@ -188,7 +182,7 @@ class Server extends EventEmitter {
         const clientHostname = req.headers["x-fisk-client-hostname"];
         if (clientHostname)
             data.hostname = clientHostname;
-        client.assign(data);
+        Object.assign(client, data);
         this.emit("compile", client);
         let remaining = { bytes: undefined, type: undefined };
         client.ws.on("close", (status, reason) => client.emit("close", status, reason));
@@ -315,20 +309,22 @@ class Server extends EventEmitter {
             if (env)
                 environments[env] = true;
         });
-        client.assign({ port: port,
-                        name: name,
-                        labels: labels,
-                        slots: slots,
-                        jobsPerformed: 0,
-                        jobsScheduled: 0,
-                        totalCompileSpeed: 0,
-                        totalUploadSpeed: 0,
-                        lastJob: 0,
-                        load: 0,
-                        npmVersion: npmVersion,
-                        hostname: hostname,
-                        environments: environments,
-                        system: system });
+        Object.assign(client, {
+            port: port,
+            name: name,
+            labels: labels,
+            slots: slots,
+            jobsPerformed: 0,
+            jobsScheduled: 0,
+            totalCompileSpeed: 0,
+            totalUploadSpeed: 0,
+            lastJob: 0,
+            load: 0,
+            npmVersion: npmVersion,
+            hostname: hostname,
+            environments: environments,
+            system: system
+        });
         client.ws.on("message", msg => {
             // console.log("Got message from builder", typeof msg, msg.length);
             switch (typeof msg) {
@@ -375,7 +371,7 @@ class Server extends EventEmitter {
     }
 
     _handleClientVerify(req, client) {
-        client.assign({npmVersion: req.headers["x-fisk-npm-version"] });
+        Object.assign(client, {npmVersion: req.headers["x-fisk-npm-version"] });
         this.emit("clientVerify", client);
         client.ws.on("close", (code, reason) => {
             client.ws.removeAllListeners();