containerify 2.0.1 → 2.2.0

package/README.md

@@ -27,6 +27,7 @@ Options:
   --fromImage <name:tag>         Required: Image name of base image - [path/]image:tag
   --toImage <name:tag>           Required: Image name of target image - [path/]image:tag
   --folder <full path>           Required: Base folder of node application (contains package.json)
+  --file <path>                  Optional: Name of configuration file (defaults to containerify.json if found on path)
   --fromRegistry <registry url>  Optional: URL of registry to pull base image from - Default: https://registry-1.docker.io/v2/
   --fromToken <token>            Optional: Authentication token for from registry
   --toRegistry <registry url>    Optional: URL of registry to push base image to - Default: https://registry-1.docker.io/v2/
@@ -39,17 +40,20 @@ Options:
   --workdir <directory>          Optional: Workdir where node app will be added and run from - default: /app
   --entrypoint <entrypoint>      Optional: Entrypoint when starting container - default: npm start
   --labels <labels>              Optional: Comma-separated list of key value pairs to use as labels
-  --label <label>                Optional: Single label (name=value). This option can be used multiple times. Wrap in double quotes if value has spaces or other characters that can cause arugment parsing issues.
-  --envs <envs>                  Optional: Comma-separated list of key value paris to use av environment variables.
+  --label <label>                Optional: Single label (name=value). This option can be used multiple times.
+  --envs <envs>                  Optional: Comma-separated list of key value pairs to use av environment variables.
   --env <env>                    Optional: Single environment variable (name=value). This option can be used multiple times.
-  --setTimeStamp <timestamp>     Optional: Set a specific ISO 8601 timestamp on all entries (e.g. git commit hash). Default: 1970 in tar files, and current time on manifest/config
+  --setTimeStamp <timestamp>     Optional: Set a specific ISO 8601 timestamp on all entries (e.g. git commit hash). Default: 1970 in tar files, and current time on
+                                 manifest/config
   --verbose                      Verbose logging
   --allowInsecureRegistries      Allow insecure registries (with self-signed/untrusted cert)
   --customContent <dirs/files>   Optional: Skip normal node_modules and applayer and include specified root folder files/directories instead
   --extraContent <dirs/files>    Optional: Add specific content. Specify as local-path:absolute-container-path,local-path2:absolute-container-path2 etc
   --layerOwner <gid:uid>         Optional: Set specific gid and uid on files in the added layers
   --buildFolder <path>           Optional: Use a specific build folder when creating the image
-  -h, --help                     output usage information
+  --layerCacheFolder <path>      Optional: Folder to cache base layers between builds
+  --version                      Get containerify version
+  -h, --help                     display help for command
 ```
 
 ## Detailed info

package/lib/appLayerCreator.js

@@ -166,7 +166,7 @@ function parseCommandLineToParts(entrypoint) {
 }
 function addAppLayers(options, config, todir, manifest, tmpdir) {
     return __awaiter(this, void 0, void 0, function* () {
-        if (options.customContent) {
+        if (options.customContent.length > 0) {
             yield addEnvsLayer(options, config);
             yield addLabelsLayer(options, config);
             yield addDataLayer(tmpdir, todir, options, config, manifest, options.customContent, "custom");
@@ -187,10 +187,8 @@ function addAppLayers(options, config, todir, manifest, tmpdir) {
             yield addDataLayer(tmpdir, todir, options, config, manifest, depLayerContent, "dependencies");
             yield addDataLayer(tmpdir, todir, options, config, manifest, appLayerContent, "app");
         }
-        if (options.extraContent) {
-            for (const i in options.extraContent) {
-                yield addDataLayer(tmpdir, todir, options, config, manifest, [options.extraContent[i]], "extra");
-            }
+        for (const extraContent of Object.entries(options.extraContent)) {
+            yield addDataLayer(tmpdir, todir, options, config, manifest, [extraContent], "extra");
         }
     });
 }

package/lib/cli.js

@@ -9,7 +9,7 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
         step((generator = generator.apply(thisArg, _arguments || [])).next());
     });
 };
-var _a, _b, _c, _d, _e, _f, _g, _h, _j;
+var _a, _b, _c, _d, _e, _f, _g, _h, _j, _k;
 Object.defineProperty(exports, "__esModule", { value: true });
 const os = require("os");
 const commander_1 = require("commander");
@@ -50,10 +50,11 @@ const possibleArgs = {
     "--extraContent <dirs/files>": "Optional: Add specific content. Specify as local-path:absolute-container-path,local-path2:absolute-container-path2 etc",
     "--layerOwner <gid:uid>": "Optional: Set specific gid and uid on files in the added layers",
     "--buildFolder <path>": "Optional: Use a specific build folder when creating the image",
+    "--layerCacheFolder <path>": "Optional: Folder to cache base layers between builds",
     "--version": "Get containerify version",
 };
-function setKeyValue(target, keyValue) {
-    const [k, v] = keyValue.split("=", 2);
+function setKeyValue(target, keyValue, separator = "=") {
+    const [k, v] = keyValue.split(separator, 2);
     target[k.trim()] = v.trim();
 }
 const cliLabels = {};
@@ -110,7 +111,13 @@ Object.keys(envOpt)
     .forEach((l) => {
     exitWithErrorIf(true, `Env ${l} specified both with --envs and --env`);
 });
-const envs = Object.assign(Object.assign(Object.assign({}, configFromFile.envs), envOpt), cliEnv); //Let cli arguments overide file
+const envs = Object.assign(Object.assign(Object.assign({}, configFromFile.envs), envOpt), cliEnv); //Let cli arguments override file
+const customContent = [];
+(_e = configFromFile.customContent) === null || _e === void 0 ? void 0 : _e.forEach((c) => customContent.push(c));
+(_f = cliOptions.customContent) === null || _f === void 0 ? void 0 : _f.split(",").forEach((c) => customContent.push(c));
+const cliExtraContent = {};
+(_g = cliOptions.extraContent) === null || _g === void 0 ? void 0 : _g.split(",").forEach((x) => setKeyValue(cliExtraContent, x, ":"));
+const extraContent = Object.assign(Object.assign({}, configFromFile.extraContent), cliExtraContent);
 const cliParams = (0, utils_1.omit)(cliOptions, [
     "label",
     "labels",
@@ -119,9 +126,9 @@ const cliParams = (0, utils_1.omit)(cliOptions, [
     "customContent",
     "extraContent",
 ]);
-cliParams.customContent = (_e = cliOptions.customContent) === null || _e === void 0 ? void 0 : _e.split(",");
-cliParams.extraContent = (_f = cliOptions.extraContent) === null || _f === void 0 ? void 0 : _f.split(",").map((x) => x.split(":"));
-const options = Object.assign(Object.assign(Object.assign(Object.assign({}, defaultOptions), configFromFile), cliParams), { labels, envs: Object.entries(envs).map(([k, v]) => `${k}=${v}`) });
+const options = Object.assign(Object.assign(Object.assign(Object.assign({}, defaultOptions), configFromFile), cliParams), { customContent,
+    extraContent,
+    labels, envs: Object.entries(envs).map(([k, v]) => `${k}=${v}`) });
 function exitWithErrorIf(check, error) {
     if (check) {
         logger_1.default.error("ERROR: " + error);
@@ -167,20 +174,29 @@ if (options.toRegistry && !options.toRegistry.endsWith("/"))
     options.toRegistry += "/";
 if (options.fromRegistry && !options.fromRegistry.endsWith("/"))
     options.fromRegistry += "/";
-if (!options.fromRegistry && !((_j = (_h = (_g = options.fromImage) === null || _g === void 0 ? void 0 : _g.split(":")) === null || _h === void 0 ? void 0 : _h[0]) === null || _j === void 0 ? void 0 : _j.includes("/"))) {
+if (!options.fromRegistry && !((_k = (_j = (_h = options.fromImage) === null || _h === void 0 ? void 0 : _h.split(":")) === null || _j === void 0 ? void 0 : _j[0]) === null || _k === void 0 ? void 0 : _k.includes("/"))) {
     options.fromImage = "library/" + options.fromImage;
 }
-if (options.customContent) {
-    options.customContent.forEach((p) => {
-        exitWithErrorIf(!fs.existsSync(p), "Could not find " + p + " in the base folder " + options.folder);
-    });
-}
-if (options.extraContent) {
-    options.extraContent.forEach((p) => {
-        exitWithErrorIf(p.length != 2, "Invalid extraContent - use comma between files/dirs, and : to separate local path and container path");
-        exitWithErrorIf(!fs.existsSync(options.folder + p[0]), "Could not find `" + p[0] + "` in the folder " + options.folder);
-    });
+options.customContent.forEach((p) => {
+    exitWithErrorIf(!fs.existsSync(p), "Could not find " + p + " in the base folder " + options.folder);
+});
+if (options.layerCacheFolder) {
+    if (!fs.existsSync(options.layerCacheFolder)) {
+        try {
+            logger_1.default.info(`Layer cache folder does not exist. Creating ${options.layerCacheFolder} ...`);
+            fs.mkdirSync(options.layerCacheFolder, { recursive: true });
+        }
+        catch (e) {
+            exitWithErrorIf(true, "Failed to create layer cache folder");
+        }
+    }
+    if (!options.layerCacheFolder.endsWith("/")) {
+        options.layerCacheFolder += "/";
+    }
 }
+Object.keys(options.extraContent).forEach(k => {
+    exitWithErrorIf(!fs.existsSync(options.folder + k), "Could not find `" + k + "` in the folder " + options.folder);
+});
 function run(options) {
     var _a, _b;
     return __awaiter(this, void 0, void 0, function* () {
@@ -193,7 +209,7 @@ function run(options) {
         const fromRegistry = options.fromRegistry
             ? (0, registry_1.createRegistry)(options.fromRegistry, (_a = options.fromToken) !== null && _a !== void 0 ? _a : "")
             : (0, registry_1.createDockerRegistry)(options.fromToken);
-        yield fromRegistry.download(options.fromImage, fromdir, (0, utils_1.getPreferredPlatform)(options.platform));
+        yield fromRegistry.download(options.fromImage, fromdir, (0, utils_1.getPreferredPlatform)(options.platform), options.layerCacheFolder);
         yield appLayerCreator_1.default.addLayers(tmpdir, fromdir, todir, options);
         if (options.toTar) {
             yield tarExporter_1.default.saveToTar(todir, tmpdir, options.toTar, [options.toImage], options);

package/lib/registry.js

@@ -68,8 +68,23 @@ function dlJson(uri, headers) {
         return JSON.parse(Buffer.from(data).toString("utf-8"));
     });
 }
-function dlToFile(uri, file, headers) {
+function dlToFile(uri, file, headers, cacheFolder, skipCache = false) {
     return new Promise((resolve, reject) => {
+        const [filename] = file.split("/").slice(-1);
+        if (cacheFolder && !skipCache) {
+            fss
+                .createReadStream(cacheFolder + filename)
+                .on("error", () => {
+                logger_1.default.debug("Not found in layer cache " + cacheFolder + filename + " - Downloading...");
+                dlToFile(uri, file, headers, cacheFolder, true).then(() => resolve());
+            })
+                .pipe(fss.createWriteStream(file))
+                .on("finish", () => {
+                logger_1.default.debug("Found in layer cache " + cacheFolder + filename);
+                resolve();
+            });
+            return;
+        }
         followRedirects(uri, headers, (result) => {
             var _a;
             if ("error" in result)
@@ -80,6 +95,10 @@ function dlToFile(uri, file, headers) {
                 return reject(toError(res));
             res.pipe(fss.createWriteStream(file)).on("finish", () => {
                 logger_1.default.debug("Done " + file + " - " + res.headers["content-length"] + " bytes ");
+                if (cacheFolder) {
+                    logger_1.default.debug(`Writing ${file} to cache ${cacheFolder + filename}`);
+                    fss.createReadStream(file).pipe(fss.createWriteStream(cacheFolder + filename));
+                }
                 resolve();
             });
         });
@@ -245,11 +264,10 @@ function createRegistry(registryBaseUrl, token) {
             return yield dlJson(`${registryBaseUrl}${image.path}/blobs/${config.digest}`, buildHeaders("*/*", auth));
         });
     }
-    function dlLayer(image, layer, folder) {
+    function dlLayer(image, layer, folder, cacheFolder) {
         return __awaiter(this, void 0, void 0, function* () {
-            logger_1.default.info(layer.digest);
             const file = getHash(layer.digest) + (0, utils_1.getLayerTypeFileEnding)(layer);
-            yield dlToFile(`${registryBaseUrl}${image.path}/blobs/${layer.digest}`, path.join(folder, file), buildHeaders(layer.mediaType, auth));
+            yield dlToFile(`${registryBaseUrl}${image.path}/blobs/${layer.digest}`, path.join(folder, file), buildHeaders(layer.mediaType, auth), cacheFolder);
             return file;
         });
     }
@@ -278,7 +296,7 @@ function createRegistry(registryBaseUrl, token) {
             yield uploadContent(`${registryBaseUrl}${image.path}/manifests/${image.tag}`, manifestFile, { mediaType: manifest.mediaType, size: manifestSize }, auth);
         });
     }
-    function download(imageStr, folder, preferredPlatform) {
+    function download(imageStr, folder, preferredPlatform, cacheFolder) {
         return __awaiter(this, void 0, void 0, function* () {
             const image = parseImage(imageStr);
             logger_1.default.info("Downloading manifest...");
@@ -294,7 +312,7 @@ function createRegistry(registryBaseUrl, token) {
             }
             yield fs_1.promises.writeFile(path.join(folder, "config.json"), JSON.stringify(config));
             logger_1.default.info("Downloading layers...");
-            yield Promise.all(manifest.layers.map((layer) => dlLayer(image, layer, folder)));
+            yield Promise.all(manifest.layers.map((layer) => dlLayer(image, layer, folder, cacheFolder)));
             logger_1.default.info("Image downloaded.");
         });
     }
@@ -312,12 +330,12 @@ function createDockerRegistry(auth) {
             return resp.token;
         });
     }
-    function download(imageStr, folder, platform) {
+    function download(imageStr, folder, platform, cacheFolder) {
         return __awaiter(this, void 0, void 0, function* () {
             const image = parseImage(imageStr);
             if (!auth)
                 auth = yield getToken(image);
-            yield createRegistry(registryBaseUrl, auth).download(imageStr, folder, platform);
+            yield createRegistry(registryBaseUrl, auth).download(imageStr, folder, platform, cacheFolder);
         });
     }
     function upload(imageStr, folder) {

package/lib/version.js

@@ -1,4 +1,4 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.VERSION = void 0;
-exports.VERSION = "2.0.1";
+exports.VERSION = "2.2.0";

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "containerify",
-	"version": "2.0.1",
+	"version": "2.2.0",
 	"description": "Build node.js docker images without docker",
 	"main": "./lib/cli.js",
 	"scripts": {