@andersbakken/fisk 4.0.40 → 4.0.42

package/builder/VM_runtime.js

@@ -3205,7 +3205,7 @@ class Compile extends EventEmitter__default["default"] {
         if (!fs.existsSync("/usr/bin/as")) {
             this.emit("stderr", "as doesn't exist");
         }
-        console.log("going for it", compiler, args);
+        console.log(`Compiling source file: ${sourceFile}\n${[compiler, ...args].join(" ")}`);
         // const env = Object.assign({ TMPDIR: dir, TEMPDIR: dir, TEMP: dir }, process.env);
         const proc = child_process__default["default"].spawn(compiler, args, {
             /*env: env, */ cwd: dir // , maxBuffer: 1024 * 1024 * 16
@@ -3263,7 +3263,7 @@ class Compile extends EventEmitter__default["default"] {
                             }
                         }
                         catch (err) {
-                            /* */
+                            console.error("Got an error file", path__default["default"].join(directory, file), err);
                         }
                     });
                 }

package/builder/fisk-builder.js

@@ -4322,7 +4322,7 @@ class ObjectCacheItem {
     }
     get contentsSize() {
         return this.response.index.reduce((total, item) => {
-            return total + item.uncompressedSize;
+            return total + item.bytes; // Cache stores compressed, so use compressed size
         }, 0);
     }
     get fileSize() {
@@ -7641,7 +7641,7 @@ class ObjectCache extends EventEmitter__default["default"] {
         }
         let remaining = 0;
         response.index.forEach((file) => {
-            remaining += file.uncompressedSize;
+            remaining += file.bytes; // Cache stores compressed, so use compressed size
         });
         const pendingItem = new ObjectCachePendingItem(response, absolutePath, remaining);
         pendingItem.file.on("error", (err) => {
@@ -7650,8 +7650,7 @@ class ObjectCache extends EventEmitter__default["default"] {
         });
         this.pending[response.sha1] = pendingItem;
         contents.forEach((c) => {
-            var _a;
-            pendingItem.write((_a = c.uncompressed) !== null && _a !== void 0 ? _a : c.contents);
+            pendingItem.write(c.contents); // Cache stores compressed data
         });
         pendingItem.end().then(() => {
             if (this.pending[response.sha1] === pendingItem) {
@@ -7764,6 +7763,7 @@ class Job extends EventEmitter__default["default"] {
         this.sha1 = data.sha1;
         this.id = data.id;
         this.builderIp = data.builderIp;
+        this.supportsCompressedResponse = data.supportsCompressedResponse;
     }
     get readyState() {
         return this.ws.readyState;
@@ -51229,6 +51229,7 @@ class Server extends EventEmitter__default["default"] {
                     sha1: String(req.headers["x-fisk-sha1"]),
                     sourceFile: String(req.headers["x-fisk-sourcefile"]),
                     user: String(req.headers["x-fisk-user"]),
+                    supportsCompressedResponse: req.headers["x-fisk-supports-compressed-response"] === "true",
                     ws
                 });
                 break;
@@ -55255,7 +55256,15 @@ function getFromCache(job, cb) {
         if (!item) {
             throw new Error("Couldn't find item " + job.sha1);
         }
-        job.send(Object.assign({ objectCache: true }, item === null || item === void 0 ? void 0 : item.response));
+        // Cache stores compressed data.
+        // Only send compressed if client both wants compression AND supports compressed responses from cache
+        const sendCompressed = job.compressed && job.supportsCompressedResponse;
+        const response = Object.assign({ objectCache: true }, item === null || item === void 0 ? void 0 : item.response);
+        if (!sendCompressed) {
+            // Client wants uncompressed (or doesn't support compressed cache), update metadata
+            response.index = response.index.map((f) => (Object.assign(Object.assign({}, f), { bytes: f.uncompressedSize })));
+        }
+        job.send(response);
         job.objectcache = true;
         pointOfNoReturn = true;
         fd = fs$3.openSync(path__default["default"].join(objectCache.dir, item.response.sha1), "r");
@@ -55283,6 +55292,7 @@ function getFromCache(job, cb) {
                 finish();
                 return;
             }
+            // Cache stores compressed data (f.bytes is compressed size)
             const buffer = Buffer.allocUnsafe(f.bytes);
             // console.log("reading from", file, path.join(objectCache.dir, item.response.sha1), pos);
             assert__default["default"](fd !== undefined, "Must have fd");
@@ -55299,7 +55309,12 @@ function getFromCache(job, cb) {
                 else {
                     // console.log("got good response from file", file);
                     // console.log("sending some data", buffer.length, fileIdx, item.response.index.length);
-                    job.send(buffer);
+                    let sendBuffer = buffer;
+                    // Decompress if client doesn't support compressed responses from cache
+                    if (!sendCompressed && buffer.byteLength > 0) {
+                        sendBuffer = zlib__default["default"].gunzipSync(buffer);
+                    }
+                    job.send(sendBuffer);
                     pos += read;
                     if (++fileIdx < item.response.index.length) {
                         work();
@@ -55947,27 +55962,36 @@ server.on("job", (job) => {
                     return;
                 }
                 // this can't be async, the directory is removed after the event is fired
+                // Always compress for cache storage, but send based on client preference
                 const contents = event.files.map((f) => {
-                    let fileContents = fs$3.readFileSync(f.absolute);
-                    let uncompressed;
-                    if (j.job.compressed) {
-                        uncompressed = fileContents;
-                        fileContents = zlib__default["default"].gzipSync(fileContents);
-                    }
+                    const uncompressed = fs$3.readFileSync(f.absolute);
+                    const compressed = uncompressed.byteLength > 0 ? zlib__default["default"].gzipSync(uncompressed) : uncompressed;
                     return {
-                        contents: fileContents,
+                        contents: compressed,
                         uncompressed,
                         path: f.path
                     };
                 });
+                // Prepare data to send to client (compressed or uncompressed based on preference and capability)
+                // Only send compressed if client wants it AND supports compressed responses
+                const sendCompressed = j.job.compressed && j.job.supportsCompressedResponse;
+                const toSend = contents.map((item) => {
+                    assert__default["default"](item.uncompressed, "Must have uncompressed data");
+                    return {
+                        contents: sendCompressed ? item.contents : item.uncompressed,
+                        path: item.path
+                    };
+                });
                 const response = {
                     type: "response",
-                    index: contents.map((item) => {
-                        var _a, _b;
+                    index: toSend.map((item) => {
+                        const original = contents.find((c) => c.path === item.path);
+                        assert__default["default"](original, "Must have original contents");
+                        assert__default["default"](original.uncompressed, "Must have uncompressed data");
                         return {
                             path: item.path,
                             bytes: item.contents.length,
-                            uncompressedSize: (_b = (_a = item.uncompressed) === null || _a === void 0 ? void 0 : _a.byteLength) !== null && _b !== void 0 ? _b : item.contents.byteLength
+                            uncompressedSize: original.uncompressed.byteLength
                         };
                     }),
                     success: event.success,
@@ -55988,13 +56012,22 @@ server.on("job", (job) => {
                     objectCache &&
                     response.sha1 &&
                     objectCache.state(response.sha1) === "none") {
-                    response.sourceFile = jobJob.sourceFile;
-                    response.commandLine = jobJob.commandLine;
-                    response.environment = jobJob.hash;
-                    objectCache.add(response, contents);
+                    // Cache metadata needs to reflect compressed sizes since we store compressed
+                    const cacheResponse = Object.assign(Object.assign({}, response), { index: contents.map((item) => {
+                            assert__default["default"](item.uncompressed, "Must have uncompressed data");
+                            return {
+                                path: item.path,
+                                bytes: item.contents.length,
+                                uncompressedSize: item.uncompressed.byteLength
+                            };
+                        }) });
+                    cacheResponse.sourceFile = jobJob.sourceFile;
+                    cacheResponse.commandLine = jobJob.commandLine;
+                    cacheResponse.environment = jobJob.hash;
+                    objectCache.add(cacheResponse, contents);
                 }
-                contents.forEach((x) => {
-                    if (x.contents.byteLength) {
+                toSend.forEach((x) => {
+                    if (x.contents && x.contents.byteLength) {
                         jobJob.send(x.contents);
                     }
                 });

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@andersbakken/fisk",
-    "version": "4.0.40",
+    "version": "4.0.42",
     "description": "Fisk, a distributed compile system",
     "scripts": {
         "lint": "eslint . --ext .ts",
@@ -30,7 +30,7 @@
     ],
     "dependencies": {
         "@andersbakken/blessed": "^0.1.82",
-        "@andersbakken/fisk-native": "^16.0.0",
+        "@andersbakken/fisk-native": "^17.0.0",
         "@jhanssen/options": "^10.0.0",
         "axios": "^0.21.1",
         "bufferutil": "^4.0.7",