containerify 2.6.0 → 3.0.0

package/README.md

@@ -21,15 +21,31 @@ containerify --fromImage node:13-slim --folder src/ --toImage myapp:latest --toR
 ### customContent - Adding compiled code to non-node container
 
 If you want to build a non-node container (e.g. add compiled frontend code to an nginx container), you can use `--customContent`. When doing this
-the normal `node_modules` etc layers will not be added, and workdir, user and entrypoint will not be overridden (allthough they can be explicitely modified
-if needed).
+the normal `node_modules` etc layers will not be added. By default it does _NOT_ modify then entrypoint, user or workdir, so the base image settings are still used when running. You can still override with `--entrypoint` etc. if needed.
 
 ```
-npm run build
-containerify --fromImage nginx:alpine --folder . --toImage frontend:latest --customContent dist:/var/www/html --toRegistry https://registry.example.com/v2/
+npm run build  # or some other build command
+containerify --fromImage nginx:alpine --folder . --toImage frontend:latest --customContent dist:/usr/share/nginx/html --toRegistry https://registry.example.com/v2/
 ```
 
-This will take nginx:alpine and copy the files in `./dist/` into `/var/www/html`.
+This will take the `nginx:alpine` image, and copy the files from `./dist/` into `/usr/share/nginx/html`.
+
+### Using with GitHub Container Registry (ghcr.io)
+
+1. Create a token from https://github.com/settings/tokens/new?scopes=write:packages or use GITHUB_TOKEN from Github Actions
+2. Example: `containerify --registry https://ghcr.io/v2/ --token "$GITHUB_TOKEN" --fromImage docker-mirror/node:alpine --toImage <some image name>:<some tag> --folder . `
+
+### Using with AWS ECR
+
+1. Create the repository in AWS from the console or through using the CLI
+2. Create token using `aws ecr get-authorization-token --output text --query 'authorizationData[].authorizationToken'`
+3. Example: `containerify  --toToken "Basic $TOKEN" --toRegistry https://<AWS ACCOUNT ID>.dkr.ecr.<AWS region for repository>.amazonaws.com/v2/ --fromImage node:alpine --toImage <name of repository>:<some tag> --folder .`
+
+### Using with GitLab Container Registry (registry.gitlab.com)
+
+1. Create the repository in GitLab
+2. Login using your username and password, [CI-credentials](https://docs.gitlab.com/ee/ci/jobs/ci_job_token.html), or [obtain a token from GitLab](https://docs.gitlab.com/ee/api/container_registry.html#obtain-token-from-gitlab)
+3. Example using CI-credentials `containerify --toToken "Basic $(echo -n '${CI_REGISTRY_USER}:${CI_REGISTRY_PASSWORD}' | base64)" --to registry.gitlab.com/<Gitlab organisation>/<repository>:<tag>`
 
 ### Command line options
 
@@ -37,30 +53,35 @@ This will take nginx:alpine and copy the files in `./dist/` into `/var/www/html`
 Usage: containerify [options]
 
 Options:
+  --from <registry/image:tag>    Optional: Shorthand to specify fromRegistry and fromImage in one argument
+  --to <registry/image:tag>      Optional: Shorthand to specify toRegistry and toImage in one argument
   --fromImage <name:tag>         Required: Image name of base image - [path/]image:tag
   --toImage <name:tag>           Required: Image name of target image - [path/]image:tag
   --folder <full path>           Required: Base folder of node application (contains package.json)
   --file <path>                  Optional: Name of configuration file (defaults to containerify.json if found on path)
+  --doCrossMount                 Optional: Cross mount image layers from the base image (only works if fromImage and toImage are in the same registry) (default: false)
   --fromRegistry <registry url>  Optional: URL of registry to pull base image from - Default: https://registry-1.docker.io/v2/
   --fromToken <token>            Optional: Authentication token for from registry
   --toRegistry <registry url>    Optional: URL of registry to push base image to - Default: https://registry-1.docker.io/v2/
+  --optimisticToRegistryCheck    Treat redirects as layer existing in remote registry. Potentially unsafe, but can save bandwidth.
   --toToken <token>              Optional: Authentication token for target registry
   --toTar <path>                 Optional: Export to tar file
+  --toDocker                     Optional: Export to local docker registry
   --registry <path>              Optional: Convenience argument for setting both from and to registry
   --platform <platform>          Optional: Preferred platform, e.g. linux/amd64 or arm64
   --token <path>                 Optional: Convenience argument for setting token for both from and to registry
-  --user <user>                  Optional: User account to run process in container - default: 1000
-  --workdir <directory>          Optional: Workdir where node app will be added and run from - default: /app
-  --entrypoint <entrypoint>      Optional: Entrypoint when starting container - default: npm start
+  --user <user>                  Optional: User account to run process in container - default: 1000 (empty for customContent)
+  --workdir <directory>          Optional: Workdir where node app will be added and run from - default: /app (empty for customContent)
+  --entrypoint <entrypoint>      Optional: Entrypoint when starting container - default: npm start (empty for customContent)
   --labels <labels>              Optional: Comma-separated list of key value pairs to use as labels
   --label <label>                Optional: Single label (name=value). This option can be used multiple times.
   --envs <envs>                  Optional: Comma-separated list of key value pairs to use av environment variables.
   --env <env>                    Optional: Single environment variable (name=value). This option can be used multiple times.
-  --setTimeStamp <timestamp>     Optional: Set a specific ISO 8601 timestamp on all entries (e.g. git commit hash). Default: 1970 in tar files, and current time on
-                                 manifest/config
+  --setTimeStamp <timestamp>     Optional: Set a specific ISO 8601 timestamp on all entries (e.g. git commit hash). Default: 1970 in tar files, and current time on manifest/config
   --verbose                      Verbose logging
   --allowInsecureRegistries      Allow insecure registries (with self-signed/untrusted cert)
-  --customContent <dirs/files>   Optional: Skip normal node_modules and applayer and include specified root folder files/directories instead
+  --customContent <dirs/files>   Optional: Skip normal node_modules and applayer and include specified root folder files/directories instead. You can specify as
+                                 local-path:absolute-container-path if you want to place it in a specific location
   --extraContent <dirs/files>    Optional: Add specific content. Specify as local-path:absolute-container-path,local-path2:absolute-container-path2 etc
   --layerOwner <gid:uid>         Optional: Set specific gid and uid on files in the added layers
   --buildFolder <path>           Optional: Use a specific build folder when creating the image

package/lib/cli.js

@@ -21,17 +21,22 @@ const appLayerCreator_1 = require("./appLayerCreator");
 const dockerExporter_1 = require("./dockerExporter");
 const tarExporter_1 = require("./tarExporter");
 const logger_1 = require("./logger");
+const types_1 = require("./types");
 const utils_1 = require("./utils");
 const fileutil_1 = require("./fileutil");
 const version_1 = require("./version");
 const possibleArgs = {
+    "--from <registry/image:tag>": "Optional: Shorthand to specify fromRegistry and fromImage in one argument",
+    "--to <registry/image:tag>": "Optional: Shorthand to specify toRegistry and toImage in one argument",
     "--fromImage <name:tag>": "Required: Image name of base image - [path/]image:tag",
     "--toImage <name:tag>": "Required: Image name of target image - [path/]image:tag",
     "--folder <full path>": "Required: Base folder of node application (contains package.json)",
     "--file <path>": "Optional: Name of configuration file (defaults to containerify.json if found on path)",
+    "--doCrossMount": "Optional: Cross mount image layers from the base image (only works if fromImage and toImage are in the same registry) (default: false)",
     "--fromRegistry <registry url>": "Optional: URL of registry to pull base image from - Default: https://registry-1.docker.io/v2/",
     "--fromToken <token>": "Optional: Authentication token for from registry",
     "--toRegistry <registry url>": "Optional: URL of registry to push base image to - Default: https://registry-1.docker.io/v2/",
+    "--optimisticToRegistryCheck": "Treat redirects as layer existing in remote registry. Potentially unsafe, but can save bandwidth.",
     "--toToken <token>": "Optional: Authentication token for target registry",
     "--toTar <path>": "Optional: Export to tar file",
     "--toDocker": "Optional: Export to local docker registry",
@@ -84,6 +89,7 @@ const defaultOptions = {
     workdir: "/app",
     user: "1000",
     entrypoint: "npm start",
+    doCrossMount: false,
 };
 if (cliOptions.file && !fs.existsSync(cliOptions.file)) {
     logger_1.default.error(`Config file '${cliOptions.file}' not found`);
@@ -145,12 +151,25 @@ function exitWithErrorIf(check, error) {
 }
 if (options.verbose)
     logger_1.default.enableDebug();
-if (options.allowInsecureRegistries)
-    process.env["NODE_TLS_REJECT_UNAUTHORIZED"] = "0";
 exitWithErrorIf(!!options.registry && !!options.fromRegistry, "Do not set both --registry and --fromRegistry");
+exitWithErrorIf(!!options.from && !!options.fromRegistry, "Do not set both --from and --fromRegistry");
+exitWithErrorIf(!!options.registry && !!options.from, "Do not set both --registry and --from");
 exitWithErrorIf(!!options.registry && !!options.toRegistry, "Do not set both --registry and --toRegistry");
+exitWithErrorIf(!!options.to && !!options.toRegistry, "Do not set both --toRegistry and --to");
+exitWithErrorIf(!!options.registry && !!options.to, "Do not set both --registry and --to");
+if (options.from) {
+    const { registry, image } = (0, registry_1.parseFullImageUrl)(options.from);
+    options.fromRegistry = registry;
+    options.fromImage = image;
+}
+if (options.to) {
+    const { registry, image } = (0, registry_1.parseFullImageUrl)(options.to);
+    options.toRegistry = registry;
+    options.toImage = image;
+}
 exitWithErrorIf(!!options.token && !!options.fromToken, "Do not set both --token and --fromToken");
 exitWithErrorIf(!!options.token && !!options.toToken, "Do not set both --token and --toToken");
+exitWithErrorIf(!!options.doCrossMount && options.toRegistry != options.fromRegistry, `Cross mounting only works if fromRegistry and toRegistry are the same (${options.fromRegistry} != ${options.toRegistry})`);
 if (options.setTimeStamp) {
     try {
         options.setTimeStamp = new Date(options.setTimeStamp).toISOString();
@@ -206,7 +225,7 @@ Object.keys(options.extraContent).forEach((k) => {
     exitWithErrorIf(!fs.existsSync(options.folder + k), "Could not find `" + k + "` in the folder " + options.folder);
 });
 function run(options) {
-    var _a, _b;
+    var _a;
     return __awaiter(this, void 0, void 0, function* () {
         if (!(yield fse.pathExists(options.folder)))
             throw new Error("No such folder: " + options.folder);
@@ -214,10 +233,10 @@ function run(options) {
         logger_1.default.debug("Using " + tmpdir);
         const fromdir = yield (0, fileutil_1.ensureEmptyDir)(path.join(tmpdir, "from"));
         const todir = yield (0, fileutil_1.ensureEmptyDir)(path.join(tmpdir, "to"));
-        const fromRegistry = options.fromRegistry
-            ? (0, registry_1.createRegistry)(options.fromRegistry, (_a = options.fromToken) !== null && _a !== void 0 ? _a : "")
-            : (0, registry_1.createDockerRegistry)(options.fromToken);
-        yield fromRegistry.download(options.fromImage, fromdir, (0, utils_1.getPreferredPlatform)(options.platform), options.layerCacheFolder);
+        const allowInsecure = options.allowInsecureRegistries ? types_1.InsecureRegistrySupport.YES : types_1.InsecureRegistrySupport.NO;
+        const fromRegistryUrl = (_a = options.fromRegistry) !== null && _a !== void 0 ? _a : registry_1.DEFAULT_DOCKER_REGISTRY;
+        const fromRegistry = (0, registry_1.createRegistry)(fromRegistryUrl, yield (0, registry_1.processToken)(fromRegistryUrl, allowInsecure, options.fromImage, options.fromToken), allowInsecure);
+        const originalManifest = yield fromRegistry.download(options.fromImage, fromdir, (0, utils_1.getPreferredPlatform)(options.platform), options.layerCacheFolder);
         yield appLayerCreator_1.default.addLayers(tmpdir, fromdir, todir, options);
         if (options.toDocker) {
             if (!(yield dockerExporter_1.default.isAvailable())) {
@@ -231,8 +250,8 @@ function run(options) {
             yield tarExporter_1.default.saveToTar(todir, tmpdir, options.toTar, [options.toImage], options);
         }
         if (options.toRegistry) {
-            const toRegistry = (0, registry_1.createRegistry)(options.toRegistry, (_b = options.toToken) !== null && _b !== void 0 ? _b : "");
-            yield toRegistry.upload(options.toImage, todir);
+            const toRegistry = (0, registry_1.createRegistry)(options.toRegistry, yield (0, registry_1.processToken)(options.toRegistry, allowInsecure, options.toImage, options.toToken), allowInsecure, options.optimisticToRegistryCheck);
+            yield toRegistry.upload(options.toImage, todir, options.doCrossMount, originalManifest, options.fromImage);
         }
         logger_1.default.debug("Deleting " + tmpdir + " ...");
         yield fse.remove(tmpdir);

package/lib/httpRequest.d.ts

@@ -0,0 +1,23 @@
+/// <reference types="node" />
+/// <reference types="node" />
+/// <reference types="node" />
+import * as https from "https";
+import * as http from "http";
+import { InsecureRegistrySupport } from "./types";
+import { OutgoingHttpHeaders } from "http";
+export declare const redirectCodes: number[];
+export declare function isOk(httpStatus: number): boolean;
+type HttpMethod = "GET" | "POST" | "PUT" | "HEAD";
+export declare function createHttpOptions(method: HttpMethod, url: string, headers: OutgoingHttpHeaders): https.RequestOptions;
+export declare function buildHeaders(accept: string, auth: string): OutgoingHttpHeaders;
+export declare function request(options: https.RequestOptions, allowInsecure: InsecureRegistrySupport, callback: (res: http.IncomingMessage) => void): http.ClientRequest;
+export declare function toError(res: http.IncomingMessage): string;
+export declare function waitForResponseEnd(res: http.IncomingMessage, cb: (data: Buffer) => void): void;
+export declare function dlJson<T>(uri: string, headers: OutgoingHttpHeaders, allowInsecure: InsecureRegistrySupport): Promise<T>;
+type Callback = (result: {
+    error: string;
+} | {
+    res: http.IncomingMessage;
+}) => void;
+export declare function followRedirects(uri: string, headers: OutgoingHttpHeaders, allowInsecure: InsecureRegistrySupport, cb: Callback, count?: number): void;
+export {};

package/lib/httpRequest.js

@@ -0,0 +1,98 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.followRedirects = exports.dlJson = exports.waitForResponseEnd = exports.toError = exports.request = exports.buildHeaders = exports.createHttpOptions = exports.isOk = exports.redirectCodes = void 0;
+const https = require("https");
+const http = require("http");
+const URL = require("url");
+const logger_1 = require("./logger");
+const types_1 = require("./types");
+exports.redirectCodes = [308, 307, 303, 302, 301];
+function isOk(httpStatus) {
+    return httpStatus >= 200 && httpStatus < 300;
+}
+exports.isOk = isOk;
+function createHttpOptions(method, url, headers) {
+    const options = Object.assign({}, URL.parse(url));
+    options.headers = headers;
+    options.method = method;
+    return options;
+}
+exports.createHttpOptions = createHttpOptions;
+function buildHeaders(accept, auth) {
+    const headers = { accept: accept };
+    if (auth)
+        headers.authorization = auth;
+    return headers;
+}
+exports.buildHeaders = buildHeaders;
+function request(options, allowInsecure, callback) {
+    if (allowInsecure == types_1.InsecureRegistrySupport.YES)
+        options.rejectUnauthorized = false;
+    const req = (options.protocol == "https:" ? https : http).request(options, (res) => {
+        callback(res);
+    });
+    req.on("error", (e) => {
+        logger_1.default.error("ERROR: " + e, options.method, options.path);
+        throw e;
+    });
+    return req;
+}
+exports.request = request;
+function toError(res) {
+    return `Unexpected HTTP status ${res.statusCode} : ${res.statusMessage}`;
+}
+exports.toError = toError;
+function waitForResponseEnd(res, cb) {
+    const data = [];
+    res.on("data", (d) => data.push(d));
+    res.on("end", () => cb(Buffer.concat(data)));
+}
+exports.waitForResponseEnd = waitForResponseEnd;
+function dl(uri, headers, allowInsecure) {
+    logger_1.default.debug("dl", uri);
+    return new Promise((resolve, reject) => {
+        followRedirects(uri, headers, allowInsecure, (result) => {
+            var _a;
+            if ("error" in result)
+                return reject(result.error);
+            const { res } = result;
+            logger_1.default.debug(res.statusCode, res.statusMessage, res.headers["content-type"], res.headers["content-length"]);
+            if (!isOk((_a = res.statusCode) !== null && _a !== void 0 ? _a : 0))
+                return reject(toError(res));
+            waitForResponseEnd(res, (data) => resolve(data.toString()));
+        });
+    });
+}
+function dlJson(uri, headers, allowInsecure) {
+    return __awaiter(this, void 0, void 0, function* () {
+        const data = yield dl(uri, headers, allowInsecure);
+        return JSON.parse(Buffer.from(data).toString("utf-8"));
+    });
+}
+exports.dlJson = dlJson;
+function followRedirects(uri, headers, allowInsecure, cb, count = 0) {
+    logger_1.default.debug("rc", uri);
+    const options = createHttpOptions("GET", uri, headers);
+    request(options, allowInsecure, (res) => {
+        var _a;
+        if (exports.redirectCodes.includes((_a = res.statusCode) !== null && _a !== void 0 ? _a : 0)) {
+            if (count > 10)
+                return cb({ error: "Too many redirects for " + uri });
+            const location = res.headers.location;
+            if (!location)
+                return cb({ error: "Redirect, but missing location header" });
+            return followRedirects(location, headers, allowInsecure, cb, count + 1);
+        }
+        cb({ res });
+    }).end();
+}
+exports.followRedirects = followRedirects;

package/lib/registry.d.ts

@@ -1,9 +1,8 @@
-import { Platform } from "./types";
-export declare function createRegistry(registryBaseUrl: string, token: string): {
-    download: (imageStr: string, folder: string, preferredPlatform: Platform, cacheFolder?: string) => Promise<void>;
-    upload: (imageStr: string, folder: string) => Promise<void>;
-};
-export declare function createDockerRegistry(auth?: string): {
-    download: (imageStr: string, folder: string, platform: Platform, cacheFolder?: string) => Promise<void>;
-    upload: (imageStr: string, folder: string) => Promise<void>;
+import { InsecureRegistrySupport, Registry } from "./types";
+export declare function processToken(registryBaseUrl: string, allowInsecure: InsecureRegistrySupport, imagePath: string, token?: string): Promise<string>;
+export declare function createRegistry(registryBaseUrl: string, auth: string, allowInsecure: InsecureRegistrySupport, optimisticToRegistryCheck?: boolean): Registry;
+export declare const DEFAULT_DOCKER_REGISTRY = "https://registry-1.docker.io/v2/";
+export declare function parseFullImageUrl(imageStr: string): {
+    registry: string;
+    image: string;
 };

package/lib/registry.js

@@ -9,66 +9,18 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
     });
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.createDockerRegistry = exports.createRegistry = void 0;
-const https = require("https");
-const http = require("http");
+exports.parseFullImageUrl = exports.DEFAULT_DOCKER_REGISTRY = exports.createRegistry = exports.processToken = void 0;
 const URL = require("url");
+const fss = require("fs");
 const fs_1 = require("fs");
 const path = require("path");
 const fse = require("fs-extra");
-const fss = require("fs");
 const fileutil = require("./fileutil");
 const logger_1 = require("./logger");
 const MIMETypes_1 = require("./MIMETypes");
 const utils_1 = require("./utils");
-const redirectCodes = [307, 303, 302];
-function request(options, callback) {
-    return (options.protocol == "https:" ? https : http).request(options, (res) => {
-        callback(res);
-    });
-}
-function isOk(httpStatus) {
-    return httpStatus >= 200 && httpStatus < 300;
-}
-function getHash(digest) {
-    return digest.split(":")[1];
-}
-function parseImage(imageStr) {
-    const ar = imageStr.split(":");
-    const tag = ar[1] || "latest";
-    const ipath = ar[0];
-    return { path: ipath, tag: tag };
-}
-function toError(res) {
-    return `Unexpected HTTP status ${res.statusCode} : ${res.statusMessage}`;
-}
-function dl(uri, headers) {
-    logger_1.default.debug("dl", uri);
-    return new Promise((resolve, reject) => {
-        followRedirects(uri, headers, (result) => {
-            var _a;
-            if ("error" in result)
-                return reject(result.error);
-            const { res } = result;
-            logger_1.default.debug(res.statusCode, res.statusMessage, res.headers["content-type"], res.headers["content-length"]);
-            if (!isOk((_a = res.statusCode) !== null && _a !== void 0 ? _a : 0))
-                return reject(toError(res));
-            const data = [];
-            res
-                .on("data", (chunk) => data.push(chunk.toString()))
-                .on("end", () => {
-                resolve(data.reduce((a, b) => a.concat(b)));
-            });
-        });
-    });
-}
-function dlJson(uri, headers) {
-    return __awaiter(this, void 0, void 0, function* () {
-        const data = yield dl(uri, headers);
-        return JSON.parse(Buffer.from(data).toString("utf-8"));
-    });
-}
-function dlToFile(uri, file, headers, cacheFolder, skipCache = false) {
+const httpRequest_1 = require("./httpRequest");
+function dlToFile(uri, file, headers, allowInsecure, cacheFolder, skipCache = false) {
     return new Promise((resolve, reject) => {
         const [filename] = file.split("/").slice(-1);
         if (cacheFolder && !skipCache) {
@@ -76,7 +28,7 @@ function dlToFile(uri, file, headers, cacheFolder, skipCache = false) {
                 .createReadStream(cacheFolder + filename)
                 .on("error", () => {
                 logger_1.default.debug("Not found in layer cache " + cacheFolder + filename + " - Downloading...");
-                dlToFile(uri, file, headers, cacheFolder, true).then(() => resolve());
+                dlToFile(uri, file, headers, allowInsecure, cacheFolder, true).then(() => resolve());
             })
                 .pipe(fss.createWriteStream(file))
                 .on("finish", () => {
@@ -85,14 +37,14 @@ function dlToFile(uri, file, headers, cacheFolder, skipCache = false) {
             });
             return;
         }
-        followRedirects(uri, headers, (result) => {
+        (0, httpRequest_1.followRedirects)(uri, headers, allowInsecure, (result) => {
             var _a;
             if ("error" in result)
                 return reject(result.error);
             const { res } = result;
             logger_1.default.debug(res.statusCode, res.statusMessage, res.headers["content-type"], res.headers["content-length"]);
-            if (!isOk((_a = res.statusCode) !== null && _a !== void 0 ? _a : 0))
-                return reject(toError(res));
+            if (!(0, httpRequest_1.isOk)((_a = res.statusCode) !== null && _a !== void 0 ? _a : 0))
+                return reject((0, httpRequest_1.toError)(res));
             res.pipe(fss.createWriteStream(file)).on("finish", () => {
                 logger_1.default.debug("Done " + file + " - " + res.headers["content-length"] + " bytes ");
                 if (cacheFolder) {
@@ -104,128 +56,163 @@ function dlToFile(uri, file, headers, cacheFolder, skipCache = false) {
         });
     });
 }
-function followRedirects(uri, headers, cb, count = 0) {
-    logger_1.default.debug("rc", uri);
-    const options = Object.assign({}, URL.parse(uri));
-    options.headers = headers;
-    options.method = "GET";
-    request(options, (res) => {
-        var _a;
-        if (redirectCodes.includes((_a = res.statusCode) !== null && _a !== void 0 ? _a : 0)) {
-            if (count > 10)
-                return cb({ error: "Too many redirects for " + uri });
-            const location = res.headers.location;
-            if (!location)
-                return cb({ error: "Redirect, but missing location header" });
-            return followRedirects(location, headers, cb, count + 1);
-        }
-        cb({ res });
-    }).end();
-}
-function buildHeaders(accept, auth) {
-    const headers = { accept: accept };
-    if (auth)
-        headers.authorization = auth;
-    return headers;
-}
-function headOk(url, headers) {
+function checkIfLayerExists(url, headers, allowInsecure, optimisticCheck = false, depth = 0) {
+    if (depth >= 5) {
+        logger_1.default.info("Followed five redirects, assuming layer does not exist");
+        return new Promise((resolve) => resolve(false));
+    }
     return new Promise((resolve, reject) => {
         logger_1.default.debug(`HEAD ${url}`);
-        const options = URL.parse(url);
-        options.headers = headers;
-        options.method = "HEAD";
-        request(options, (res) => {
+        const options = (0, httpRequest_1.createHttpOptions)("HEAD", url, headers);
+        (0, httpRequest_1.request)(options, allowInsecure, (res) => {
             logger_1.default.debug(`HEAD ${url}`, res.statusCode);
-            if (res.statusCode == 404)
-                return resolve(false);
-            if (res.statusCode == 200)
-                return resolve(true);
-            reject(toError(res));
+            (0, httpRequest_1.waitForResponseEnd)(res, (data) => {
+                var _a;
+                // Not found
+                if (res.statusCode == 404)
+                    return resolve(false);
+                // OK
+                if (res.statusCode == 200)
+                    return resolve(true);
+                // Redirected
+                if (httpRequest_1.redirectCodes.includes((_a = res.statusCode) !== null && _a !== void 0 ? _a : 0) && res.headers.location) {
+                    if (optimisticCheck)
+                        return resolve(true);
+                    return resolve(checkIfLayerExists(res.headers.location, headers, allowInsecure, optimisticCheck, ++depth));
+                }
+                // Unauthorized
+                // Possibly related to https://gitlab.com/gitlab-org/gitlab/-/issues/23132
+                if (res.statusCode == 401) {
+                    return resolve(false);
+                }
+                // Other error
+                logger_1.default.error(`HEAD ${url}`, res.statusCode, res.statusMessage, data.toString());
+                reject((0, httpRequest_1.toError)(res));
+            });
         }).end();
     });
 }
-function uploadContent(uploadUrl, file, fileConfig, auth) {
+function uploadContent(uploadUrl, file, fileConfig, allowInsecure, auth, contentType = "application/octet-stream") {
     return new Promise((resolve, reject) => {
         logger_1.default.debug("Uploading: ", file);
         let url = uploadUrl;
         if (fileConfig.digest)
             url += (url.indexOf("?") == -1 ? "?" : "&") + "digest=" + fileConfig.digest;
-        const options = URL.parse(url);
-        options.method = "PUT";
-        options.headers = {
+        const options = (0, httpRequest_1.createHttpOptions)("PUT", url, {
             authorization: auth,
             "content-length": fileConfig.size,
-            "content-type": fileConfig.mediaType,
-        };
-        logger_1.default.debug("POST", url);
-        const req = request(options, (res) => {
+            "content-type": contentType,
+        });
+        logger_1.default.debug(options.method, url);
+        const req = (0, httpRequest_1.request)(options, allowInsecure, (res) => {
             var _a;
             logger_1.default.debug(res.statusCode, res.statusMessage, res.headers["content-type"], res.headers["content-length"]);
             if ([200, 201, 202, 203].includes((_a = res.statusCode) !== null && _a !== void 0 ? _a : 0)) {
                 resolve();
             }
             else {
-                const data = [];
-                res.on("data", (d) => data.push(d.toString()));
-                res.on("end", () => {
-                    reject(`Error uploading to ${uploadUrl}. Got ${res.statusCode} ${res.statusMessage}:\n${data.join("")}`);
+                (0, httpRequest_1.waitForResponseEnd)(res, (data) => {
+                    reject(`Error uploading to ${uploadUrl}. Got ${res.statusCode} ${res.statusMessage}:\n${data.toString()}`);
                 });
             }
         });
-        fss.createReadStream(file).pipe(req);
+        fss
+            .createReadStream(file)
+            .pipe(req)
+            .on("error", (e) => {
+            reject("Error reading file for upload: " + e);
+        });
     });
 }
-function createRegistry(registryBaseUrl, token) {
-    const auth = token.startsWith("Basic ") ? token : "Bearer " + token;
+function processToken(registryBaseUrl, allowInsecure, imagePath, token) {
+    return __awaiter(this, void 0, void 0, function* () {
+        const { hostname } = URL.parse(registryBaseUrl);
+        const image = (0, utils_1.parseImage)(imagePath);
+        if ((hostname === null || hostname === void 0 ? void 0 : hostname.endsWith(".docker.io")) && !token) {
+            const resp = yield (0, httpRequest_1.dlJson)(`https://auth.docker.io/token?service=registry.docker.io&scope=repository:${image.path}:pull`, {}, allowInsecure);
+            return `Bearer ${resp.token}`;
+        }
+        if ((hostname === null || hostname === void 0 ? void 0 : hostname.endsWith(".gitlab.com")) && (token === null || token === void 0 ? void 0 : token.startsWith("Basic"))) {
+            if (token === null || token === void 0 ? void 0 : token.includes(":")) {
+                token = "Basic " + Buffer.from(token === null || token === void 0 ? void 0 : token.replace("Basic ", "")).toString("base64");
+            }
+            const resp = yield (0, httpRequest_1.dlJson)(`https://gitlab.com/jwt/auth?service=container_registry&scope=repository:${image.path}:pull,push`, { Authorization: token }, allowInsecure);
+            return `Bearer ${resp.token}`;
+        }
+        if (!token)
+            throw new Error("Needs auth token to upload to " + registryBaseUrl);
+        if (token.startsWith("Basic "))
+            return token;
+        if (token.startsWith("ghp_"))
+            return "Bearer " + Buffer.from(token).toString("base64");
+        return "Bearer " + token;
+    });
+}
+exports.processToken = processToken;
+function createRegistry(registryBaseUrl, auth, allowInsecure, optimisticToRegistryCheck = false) {
     function exists(image, layer) {
         return __awaiter(this, void 0, void 0, function* () {
             const url = `${registryBaseUrl}${image.path}/blobs/${layer.digest}`;
-            return yield headOk(url, buildHeaders(layer.mediaType, auth));
+            return yield checkIfLayerExists(url, (0, httpRequest_1.buildHeaders)(layer.mediaType, auth), allowInsecure, optimisticToRegistryCheck, 0);
         });
     }
     function uploadLayerContent(uploadUrl, layer, dir) {
         return __awaiter(this, void 0, void 0, function* () {
             logger_1.default.info(layer.digest);
-            const file = path.join(dir, getHash(layer.digest) + (0, utils_1.getLayerTypeFileEnding)(layer));
-            yield uploadContent(uploadUrl, file, layer, auth);
+            const file = path.join(dir, (0, utils_1.getHash)(layer.digest) + (0, utils_1.getLayerTypeFileEnding)(layer));
+            yield uploadContent(uploadUrl, file, layer, allowInsecure, auth);
         });
     }
-    function getUploadUrl(image) {
+    function getUploadUrl(image, mountParameters = undefined) {
         return __awaiter(this, void 0, void 0, function* () {
             return new Promise((resolve, reject) => {
-                const url = `${registryBaseUrl}${image.path}/blobs/uploads/`;
+                const parameters = new URLSearchParams(mountParameters);
+                const url = `${registryBaseUrl}${image.path}/blobs/uploads/${parameters.size > 0 ? "?" + parameters : ""}`;
                 const options = URL.parse(url);
                 options.method = "POST";
                 options.headers = { authorization: auth };
-                request(options, (res) => {
+                (0, httpRequest_1.request)(options, allowInsecure, (res) => {
                     logger_1.default.debug("POST", `${url}`, res.statusCode);
                     if (res.statusCode == 202) {
                         const { location } = res.headers;
-                        if (location)
-                            resolve(location);
+                        if (location) {
+                            if (location.startsWith("http")) {
+                                resolve({ uploadUrl: location });
+                            }
+                            else {
+                                const regURL = URL.parse(registryBaseUrl);
+                                resolve({
+                                    uploadUrl: `${regURL.protocol}//${regURL.hostname}${regURL.port ? ":" + regURL.port : ""}${location}`,
+                                });
+                            }
+                        }
                         reject("Missing location for 202");
                     }
+                    else if (mountParameters && res.statusCode == 201) {
+                        const returnedDigest = res.headers["docker-content-digest"];
+                        if (returnedDigest && returnedDigest != mountParameters.mount) {
+                            reject(`ERROR: Layer mounted with wrong digest: Expected ${mountParameters.mount} but got ${returnedDigest}`);
+                        }
+                        resolve({ mountSuccess: true });
+                    }
                     else {
-                        const data = [];
-                        res
-                            .on("data", (c) => data.push(c.toString()))
-                            .on("end", () => {
-                            reject(`Error getting upload URL from ${url}. Got ${res.statusCode} ${res.statusMessage}:\n${data.join("")}`);
+                        (0, httpRequest_1.waitForResponseEnd)(res, (data) => {
+                            reject(`Error getting upload URL from ${url}. Got ${res.statusCode} ${res.statusMessage}:\n${data.toString()}`);
                         });
                     }
                 }).end();
             });
         });
     }
-    function dlManifest(image, preferredPlatform) {
+    function dlManifest(image, preferredPlatform, allowInsecure) {
         return __awaiter(this, void 0, void 0, function* () {
             // Accept both manifests and index/manifest lists
-            const res = yield dlJson(`${registryBaseUrl}${image.path}/manifests/${image.tag}`, buildHeaders(`${MIMETypes_1.OCI.index}, ${MIMETypes_1.OCI.manifest}, ${MIMETypes_1.DockerV2.index}, ${MIMETypes_1.DockerV2.manifest}`, auth));
+            const res = yield (0, httpRequest_1.dlJson)(`${registryBaseUrl}${image.path}/manifests/${image.tag}`, (0, httpRequest_1.buildHeaders)(`${MIMETypes_1.OCI.index}, ${MIMETypes_1.OCI.manifest}, ${MIMETypes_1.DockerV2.index}, ${MIMETypes_1.DockerV2.manifest}`, auth), allowInsecure);
             // We've received an OCI Index or Docker Manifest List and need to find which manifest we want
             if (res.mediaType === MIMETypes_1.OCI.index || res.mediaType === MIMETypes_1.DockerV2.index) {
                 const availableManifests = res.manifests;
                 const adequateManifest = pickManifest(availableManifests, preferredPlatform);
-                return dlManifest(Object.assign(Object.assign({}, image), { tag: adequateManifest.digest }), preferredPlatform);
+                return dlManifest(Object.assign(Object.assign({}, image), { tag: adequateManifest.digest }), preferredPlatform, allowInsecure);
             }
             return res;
         });
@@ -259,21 +246,21 @@ function createRegistry(registryBaseUrl, token) {
         logger_1.default.error("Available platforms:", JSON.stringify(manifests.map((m) => m.platform)));
         throw new Error("No image matching requested architecture");
     }
-    function dlConfig(image, config) {
+    function dlConfig(image, config, allowInsecure) {
         return __awaiter(this, void 0, void 0, function* () {
-            return yield dlJson(`${registryBaseUrl}${image.path}/blobs/${config.digest}`, buildHeaders("*/*", auth));
+            return yield (0, httpRequest_1.dlJson)(`${registryBaseUrl}${image.path}/blobs/${config.digest}`, (0, httpRequest_1.buildHeaders)("*/*", auth), allowInsecure);
         });
     }
-    function dlLayer(image, layer, folder, cacheFolder) {
+    function dlLayer(image, layer, folder, allowInsecure, cacheFolder) {
         return __awaiter(this, void 0, void 0, function* () {
-            const file = getHash(layer.digest) + (0, utils_1.getLayerTypeFileEnding)(layer);
-            yield dlToFile(`${registryBaseUrl}${image.path}/blobs/${layer.digest}`, path.join(folder, file), buildHeaders(layer.mediaType, auth), cacheFolder);
+            const file = (0, utils_1.getHash)(layer.digest) + (0, utils_1.getLayerTypeFileEnding)(layer);
+            yield dlToFile(`${registryBaseUrl}${image.path}/blobs/${layer.digest}`, path.join(folder, file), (0, httpRequest_1.buildHeaders)(layer.mediaType, auth), allowInsecure, cacheFolder);
             return file;
         });
     }
-    function upload(imageStr, folder) {
+    function upload(imageStr, folder, doCrossMount, originalManifest, originalRepository) {
         return __awaiter(this, void 0, void 0, function* () {
-            const image = parseImage(imageStr);
+            const image = (0, utils_1.parseImage)(imageStr);
             const manifestFile = path.join(folder, "manifest.json");
             const manifest = (yield fse.readJson(manifestFile));
             logger_1.default.info("Checking layer status...");
@@ -284,26 +271,40 @@ function createRegistry(registryBaseUrl, token) {
             logger_1.default.debug("Needs upload:", layersForUpload.map((l) => l.layer.digest));
             logger_1.default.info("Uploading layers...");
             yield Promise.all(layersForUpload.map((l) => __awaiter(this, void 0, void 0, function* () {
-                const url = yield getUploadUrl(image);
-                yield uploadLayerContent(url, l.layer, folder);
+                if (doCrossMount && originalManifest.layers.find((x) => x.digest == l.layer.digest)) {
+                    const mount = yield getUploadUrl(image, { mount: l.layer.digest, from: originalRepository });
+                    if ("mountSuccess" in mount) {
+                        logger_1.default.info(`Cross mounted layer ${l.layer.digest} from '${originalRepository}'`);
+                        return;
+                    }
+                    yield uploadLayerContent(mount.uploadUrl, l.layer, folder);
+                }
+                else {
+                    const url = yield getUploadUrl(image);
+                    if ("mountSuccess" in url)
+                        throw new Error("Mounting not supported for this upload");
+                    yield uploadLayerContent(url.uploadUrl, l.layer, folder);
+                }
             })));
             logger_1.default.info("Uploading config...");
             const configUploadUrl = yield getUploadUrl(image);
-            const configFile = path.join(folder, getHash(manifest.config.digest) + ".json");
-            yield uploadContent(configUploadUrl, configFile, manifest.config, auth);
+            if ("mountSuccess" in configUploadUrl)
+                throw new Error("Mounting not supported for config upload");
+            const configFile = path.join(folder, (0, utils_1.getHash)(manifest.config.digest) + ".json");
+            yield uploadContent(configUploadUrl.uploadUrl, configFile, manifest.config, allowInsecure, auth);
             logger_1.default.info("Uploading manifest...");
             const manifestSize = yield fileutil.sizeOf(manifestFile);
-            yield uploadContent(`${registryBaseUrl}${image.path}/manifests/${image.tag}`, manifestFile, { mediaType: manifest.mediaType, size: manifestSize }, auth);
+            yield uploadContent(`${registryBaseUrl}${image.path}/manifests/${image.tag}`, manifestFile, { mediaType: manifest.mediaType, size: manifestSize }, allowInsecure, auth, manifest.mediaType);
         });
     }
     function download(imageStr, folder, preferredPlatform, cacheFolder) {
         return __awaiter(this, void 0, void 0, function* () {
-            const image = parseImage(imageStr);
+            const image = (0, utils_1.parseImage)(imageStr);
             logger_1.default.info("Downloading manifest...");
-            const manifest = yield dlManifest(image, preferredPlatform);
+            const manifest = yield dlManifest(image, preferredPlatform, allowInsecure);
             yield fs_1.promises.writeFile(path.join(folder, "manifest.json"), JSON.stringify(manifest));
             logger_1.default.info("Downloading config...");
-            const config = yield dlConfig(image, manifest.config);
+            const config = yield dlConfig(image, manifest.config, allowInsecure);
             if (config.architecture != preferredPlatform.architecture) {
                 logger_1.default.info(`[WARN] Image architecture (${config.architecture}) does not match preferred architecture (${preferredPlatform.architecture}).`);
             }
@@ -312,42 +313,30 @@ function createRegistry(registryBaseUrl, token) {
             }
             yield fs_1.promises.writeFile(path.join(folder, "config.json"), JSON.stringify(config));
             logger_1.default.info("Downloading layers...");
-            yield Promise.all(manifest.layers.map((layer) => dlLayer(image, layer, folder, cacheFolder)));
+            yield Promise.all(manifest.layers.map((layer) => dlLayer(image, layer, folder, allowInsecure, cacheFolder)));
             logger_1.default.info("Image downloaded.");
+            return manifest;
         });
     }
     return {
         download: download,
         upload: upload,
+        registryBaseUrl,
     };
 }
 exports.createRegistry = createRegistry;
-function createDockerRegistry(auth) {
-    const registryBaseUrl = "https://registry-1.docker.io/v2/";
-    function getToken(image) {
-        return __awaiter(this, void 0, void 0, function* () {
-            const resp = yield dlJson(`https://auth.docker.io/token?service=registry.docker.io&scope=repository:${image.path}:pull`, {});
-            return resp.token;
-        });
-    }
-    function download(imageStr, folder, platform, cacheFolder) {
-        return __awaiter(this, void 0, void 0, function* () {
-            const image = parseImage(imageStr);
-            if (!auth)
-                auth = yield getToken(image);
-            yield createRegistry(registryBaseUrl, auth).download(imageStr, folder, platform, cacheFolder);
-        });
-    }
-    function upload(imageStr, folder) {
-        return __awaiter(this, void 0, void 0, function* () {
-            if (!auth)
-                throw new Error("Need auth token to upload to Docker");
-            yield createRegistry(registryBaseUrl, auth).upload(imageStr, folder);
-        });
+exports.DEFAULT_DOCKER_REGISTRY = "https://registry-1.docker.io/v2/";
+function parseFullImageUrl(imageStr) {
+    const [registry, ...rest] = imageStr.split("/");
+    if (registry == "docker.io") {
+        return {
+            registry: exports.DEFAULT_DOCKER_REGISTRY,
+            image: rest.join("/"),
+        };
     }
     return {
-        download: download,
-        upload: upload,
+        registry: `https://${registry}/v2/`,
+        image: rest.join("/"),
     };
 }
-exports.createDockerRegistry = createDockerRegistry;
+exports.parseFullImageUrl = parseFullImageUrl;

package/lib/types.d.ts

@@ -54,6 +54,8 @@ export type HistoryLine = {
     comment?: string;
 };
 export type Options = {
+    from?: string;
+    to?: string;
     fromImage: string;
     toImage: string;
     folder: string;
@@ -61,6 +63,8 @@ export type Options = {
     fromRegistry?: string;
     fromToken?: string;
     toRegistry?: string;
+    doCrossMount: boolean;
+    optimisticToRegistryCheck?: boolean;
     toToken?: string;
     toTar?: string;
     toDocker?: boolean;
@@ -86,4 +90,13 @@ export type Options = {
         entrypoint?: string;
     };
 };
+export declare enum InsecureRegistrySupport {
+    NO = 0,
+    YES = 1
+}
+export type Registry = {
+    download: (imageStr: string, folder: string, preferredPlatform: Platform, cacheFolder?: string) => Promise<Manifest>;
+    upload: (imageStr: string, folder: string, doCrossMount: boolean, originalManifest: Manifest, originalRepository: string) => Promise<void>;
+    registryBaseUrl: string;
+};
 export {};

package/lib/types.js

@@ -1,2 +1,8 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
+exports.InsecureRegistrySupport = void 0;
+var InsecureRegistrySupport;
+(function (InsecureRegistrySupport) {
+    InsecureRegistrySupport[InsecureRegistrySupport["NO"] = 0] = "NO";
+    InsecureRegistrySupport[InsecureRegistrySupport["YES"] = 1] = "YES";
+})(InsecureRegistrySupport || (exports.InsecureRegistrySupport = InsecureRegistrySupport = {}));

package/lib/utils.d.ts

@@ -4,3 +4,8 @@ export declare function omit<T>(obj: Record<string, T>, keys: string[]): Record<
 export declare function getPreferredPlatform(platform?: string): Platform;
 export declare function getManifestLayerType(manifest: Manifest): string;
 export declare function getLayerTypeFileEnding(layer: Layer): ".tar.gz" | ".tar";
+export declare function getHash(digest: string): string;
+export declare function parseImage(imageStr: string): {
+    path: string;
+    tag: string;
+};

package/lib/utils.js

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.getLayerTypeFileEnding = exports.getManifestLayerType = exports.getPreferredPlatform = exports.omit = exports.unique = void 0;
+exports.parseImage = exports.getHash = exports.getLayerTypeFileEnding = exports.getManifestLayerType = exports.getPreferredPlatform = exports.omit = exports.unique = void 0;
 const MIMETypes_1 = require("./MIMETypes");
 function unique(vals) {
     return [...new Set(vals)];
@@ -92,3 +92,14 @@ function getLayerTypeFileEnding(layer) {
     }
 }
 exports.getLayerTypeFileEnding = getLayerTypeFileEnding;
+function getHash(digest) {
+    return digest.split(":")[1];
+}
+exports.getHash = getHash;
+function parseImage(imageStr) {
+    const ar = imageStr.split(":");
+    const tag = ar[1] || "latest";
+    const ipath = ar[0];
+    return { path: ipath, tag: tag };
+}
+exports.parseImage = parseImage;

package/lib/version.d.ts

@@ -1 +1 @@
-export declare const VERSION = "2.6.0";
+export declare const VERSION = "3.0.0";

package/lib/version.js

@@ -1,4 +1,4 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.VERSION = void 0;
-exports.VERSION = "2.6.0";
+exports.VERSION = "3.0.0";

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "containerify",
-	"version": "2.6.0",
+	"version": "3.0.0",
 	"description": "Build node.js docker images without docker",
 	"main": "./lib/cli.js",
 	"scripts": {
@@ -8,6 +8,7 @@
 		"build": "tsc && chmod ugo+x lib/cli.js",
 		"lint": "eslint . --ext .ts --fix --ignore-path .gitignore",
 		"typecheck": "tsc --noEmit",
+		"watch": "tsc --watch",
 		"check": "npm run lint && npm run typecheck",
 		"dev": "tsc --watch",
 		"integrationTest": "cd tests/integration/ && ./test.sh",