keycloakify 10.0.0-rc.8 → 10.0.0-rc.80

package/src/bin/tools/downloadAndExtractArchive/downloadAndExtractArchive.ts

@@ -0,0 +1,262 @@
+import fetch from "make-fetch-happen";
+import { mkdir, unlink, writeFile, readdir, readFile } from "fs/promises";
+import { dirname as pathDirname, join as pathJoin } from "path";
+import { assert } from "tsafe/assert";
+import { extractArchive } from "../extractArchive";
+import { existsAsync } from "../fs.existsAsync";
+import { getProxyFetchOptions } from "./fetchProxyOptions";
+import * as crypto from "crypto";
+import { rm } from "../fs.rm";
+
+export async function downloadAndExtractArchive(params: {
+    url: string;
+    uniqueIdOfOnOnArchiveFile: string;
+    onArchiveFile: (params: {
+        fileRelativePath: string;
+        readFile: () => Promise<Buffer>;
+        writeFile: (params: {
+            fileRelativePath: string;
+            modifiedData?: Buffer;
+        }) => Promise<void>;
+    }) => Promise<void>;
+    cacheDirPath: string;
+    npmWorkspaceRootDirPath: string;
+}): Promise<{ extractedDirPath: string }> {
+    const {
+        url,
+        uniqueIdOfOnOnArchiveFile,
+        onArchiveFile,
+        cacheDirPath,
+        npmWorkspaceRootDirPath
+    } = params;
+
+    const archiveFileBasename = url.split("?")[0].split("/").reverse()[0];
+
+    const archiveFilePath = pathJoin(cacheDirPath, archiveFileBasename);
+
+    download: {
+        if (await existsAsync(archiveFilePath)) {
+            const isDownloaded = await SuccessTracker.getIsDownloaded({
+                cacheDirPath,
+                archiveFileBasename
+            });
+
+            if (isDownloaded) {
+                break download;
+            }
+
+            await unlink(archiveFilePath);
+
+            await SuccessTracker.removeFromDownloaded({
+                cacheDirPath,
+                archiveFileBasename
+            });
+        }
+
+        await mkdir(pathDirname(archiveFilePath), { recursive: true });
+
+        const response = await fetch(
+            url,
+            await getProxyFetchOptions({ npmWorkspaceRootDirPath })
+        );
+
+        response.body?.setMaxListeners(Number.MAX_VALUE);
+        assert(typeof response.body !== "undefined" && response.body != null);
+
+        await writeFile(archiveFilePath, response.body);
+
+        await SuccessTracker.markAsDownloaded({
+            cacheDirPath,
+            archiveFileBasename
+        });
+    }
+
+    const extractDirBasename = `${archiveFileBasename.split(".")[0]}_${uniqueIdOfOnOnArchiveFile}_${crypto
+        .createHash("sha256")
+        .update(onArchiveFile.toString())
+        .digest("hex")
+        .substring(0, 5)}`;
+
+    await Promise.all(
+        (await readdir(cacheDirPath))
+            .filter(
+                (() => {
+                    const prefix = extractDirBasename
+                        .split("_")
+                        .reverse()
+                        .slice(1)
+                        .reverse()
+                        .join("_");
+
+                    return basename =>
+                        basename !== extractDirBasename && basename.startsWith(prefix);
+                })()
+            )
+            .map(async extractDirBasename => {
+                await rm(pathJoin(cacheDirPath, extractDirBasename), { recursive: true });
+                await SuccessTracker.removeFromExtracted({
+                    cacheDirPath,
+                    extractDirBasename
+                });
+            })
+    );
+
+    const extractedDirPath = pathJoin(cacheDirPath, extractDirBasename);
+
+    extract_and_transform: {
+        if (await existsAsync(extractedDirPath)) {
+            const isExtracted = await SuccessTracker.getIsExtracted({
+                cacheDirPath,
+                extractDirBasename
+            });
+
+            if (isExtracted) {
+                break extract_and_transform;
+            }
+
+            await rm(extractedDirPath, { recursive: true });
+
+            await SuccessTracker.removeFromExtracted({
+                cacheDirPath,
+                extractDirBasename
+            });
+        }
+
+        await extractArchive({
+            archiveFilePath,
+            onArchiveFile: async ({ relativeFilePathInArchive, readFile, writeFile }) =>
+                onArchiveFile({
+                    fileRelativePath: relativeFilePathInArchive,
+                    readFile,
+                    writeFile: ({ fileRelativePath, modifiedData }) =>
+                        writeFile({
+                            filePath: pathJoin(extractedDirPath, fileRelativePath),
+                            modifiedData
+                        })
+                })
+        });
+
+        await SuccessTracker.markAsExtracted({
+            cacheDirPath,
+            extractDirBasename
+        });
+    }
+
+    return { extractedDirPath };
+}
+
+type SuccessTracker = {
+    archiveFileBasenames: string[];
+    extractDirBasenames: string[];
+};
+
+namespace SuccessTracker {
+    async function read(params: { cacheDirPath: string }): Promise<SuccessTracker> {
+        const { cacheDirPath } = params;
+
+        const filePath = pathJoin(cacheDirPath, "downloadAndExtractArchive.json");
+
+        if (!(await existsAsync(filePath))) {
+            return { archiveFileBasenames: [], extractDirBasenames: [] };
+        }
+
+        return JSON.parse((await readFile(filePath)).toString("utf8"));
+    }
+
+    async function write(params: {
+        cacheDirPath: string;
+        successTracker: SuccessTracker;
+    }) {
+        const { cacheDirPath, successTracker } = params;
+
+        const filePath = pathJoin(cacheDirPath, "downloadAndExtractArchive.json");
+
+        {
+            const dirPath = pathDirname(filePath);
+
+            if (!(await existsAsync(dirPath))) {
+                await mkdir(dirPath, { recursive: true });
+            }
+        }
+
+        await writeFile(filePath, JSON.stringify(successTracker));
+    }
+
+    export async function markAsDownloaded(params: {
+        cacheDirPath: string;
+        archiveFileBasename: string;
+    }) {
+        const { cacheDirPath, archiveFileBasename } = params;
+
+        const successTracker = await read({ cacheDirPath });
+
+        successTracker.archiveFileBasenames.push(archiveFileBasename);
+
+        await write({ cacheDirPath, successTracker });
+    }
+
+    export async function getIsDownloaded(params: {
+        cacheDirPath: string;
+        archiveFileBasename: string;
+    }): Promise<boolean> {
+        const { cacheDirPath, archiveFileBasename } = params;
+
+        const successTracker = await read({ cacheDirPath });
+
+        return successTracker.archiveFileBasenames.includes(archiveFileBasename);
+    }
+
+    export async function removeFromDownloaded(params: {
+        cacheDirPath: string;
+        archiveFileBasename: string;
+    }) {
+        const { cacheDirPath, archiveFileBasename } = params;
+
+        const successTracker = await read({ cacheDirPath });
+
+        successTracker.archiveFileBasenames = successTracker.archiveFileBasenames.filter(
+            basename => basename !== archiveFileBasename
+        );
+
+        await write({ cacheDirPath, successTracker });
+    }
+
+    export async function markAsExtracted(params: {
+        cacheDirPath: string;
+        extractDirBasename: string;
+    }) {
+        const { cacheDirPath, extractDirBasename } = params;
+
+        const successTracker = await read({ cacheDirPath });
+
+        successTracker.extractDirBasenames.push(extractDirBasename);
+
+        await write({ cacheDirPath, successTracker });
+    }
+
+    export async function getIsExtracted(params: {
+        cacheDirPath: string;
+        extractDirBasename: string;
+    }): Promise<boolean> {
+        const { cacheDirPath, extractDirBasename } = params;
+
+        const successTracker = await read({ cacheDirPath });
+
+        return successTracker.extractDirBasenames.includes(extractDirBasename);
+    }
+
+    export async function removeFromExtracted(params: {
+        cacheDirPath: string;
+        extractDirBasename: string;
+    }) {
+        const { cacheDirPath, extractDirBasename } = params;
+
+        const successTracker = await read({ cacheDirPath });
+
+        successTracker.extractDirBasenames = successTracker.extractDirBasenames.filter(
+            basename => basename !== extractDirBasename
+        );
+
+        await write({ cacheDirPath, successTracker });
+    }
+}

package/src/bin/tools/{fetchProxyOptions.ts → downloadAndExtractArchive/fetchProxyOptions.ts}

@@ -11,7 +11,10 @@ function ensureSingleOrNone<T>(arg0: T | T[]) {
     if (!Array.isArray(arg0)) return arg0;
     if (arg0.length === 0) return undefined;
     if (arg0.length === 1) return arg0[0];
-    throw new Error("Illegal configuration, expected a single value but found multiple: " + arg0.map(String).join(", "));
+    throw new Error(
+        "Illegal configuration, expected a single value but found multiple: " +
+            arg0.map(String).join(", ")
+    );
 }
 
 type NPMConfig = Record<string, string | string[]>;
@@ -24,10 +27,15 @@ async function getNmpConfig(params: { npmWorkspaceRootDirPath: string }) {
 
     const exec = promisify(execCallback);
 
-    const stdout = await exec("npm config get", { "encoding": "utf8", "cwd": npmWorkspaceRootDirPath }).then(({ stdout }) => stdout);
+    const stdout = await exec("npm config get", {
+        encoding: "utf8",
+        cwd: npmWorkspaceRootDirPath
+    }).then(({ stdout }) => stdout);
 
     const npmConfigReducer = (cfg: NPMConfig, [key, value]: [string, string]) =>
-        key in cfg ? { ...cfg, [key]: [...ensureArray(cfg[key]), value] } : { ...cfg, [key]: value };
+        key in cfg
+            ? { ...cfg, [key]: [...ensureArray(cfg[key]), value] }
+            : { ...cfg, [key]: value };
 
     return stdout
         .split("\n")
@@ -37,9 +45,14 @@ async function getNmpConfig(params: { npmWorkspaceRootDirPath: string }) {
         .reduce(npmConfigReducer, {} as NPMConfig);
 }
 
-export type ProxyFetchOptions = Pick<FetchOptions, "proxy" | "noProxy" | "strictSSL" | "cert" | "ca">;
+export type ProxyFetchOptions = Pick<
+    FetchOptions,
+    "proxy" | "noProxy" | "strictSSL" | "cert" | "ca"
+>;
 
-export async function getProxyFetchOptions(params: { npmWorkspaceRootDirPath: string }): Promise<ProxyFetchOptions> {
+export async function getProxyFetchOptions(params: {
+    npmWorkspaceRootDirPath: string;
+}): Promise<ProxyFetchOptions> {
     const { npmWorkspaceRootDirPath } = params;
 
     const cfg = await getNmpConfig({ npmWorkspaceRootDirPath });
@@ -60,14 +73,32 @@ export async function getProxyFetchOptions(params: { npmWorkspaceRootDirPath: st
         ca.push(
             ...(await (async () => {
                 function chunks<T>(arr: T[], size: number = 2) {
-                    return arr.map((_, i) => i % size == 0 && arr.slice(i, i + size)).filter(Boolean) as T[][];
+                    return arr
+                        .map((_, i) => i % size == 0 && arr.slice(i, i + size))
+                        .filter(Boolean) as T[][];
                 }
 
                 const cafileContent = await readFile(cafile, "utf-8");
-                return chunks(cafileContent.split(/(-----END CERTIFICATE-----)/), 2).map(ca => ca.join("").replace(/^\n/, "").replace(/\n/g, "\\n"));
+
+                const newLinePlaceholder = "NEW_LINE_PLACEHOLDER_xIsPsK23svt";
+
+                return chunks(cafileContent.split(/(-----END CERTIFICATE-----)/), 2).map(
+                    ca =>
+                        ca
+                            .join("")
+                            .replace(/\r?\n/g, newLinePlaceholder)
+                            .replace(new RegExp(`^${newLinePlaceholder}`), "")
+                            .replace(new RegExp(newLinePlaceholder, "g"), "\\n")
+                );
             })())
         );
     }
 
-    return { proxy, noProxy, strictSSL, cert, "ca": ca.length === 0 ? undefined : ca };
+    return {
+        proxy,
+        noProxy,
+        strictSSL,
+        cert,
+        ca: ca.length === 0 ? undefined : ca
+    };
 }

package/src/bin/tools/downloadAndExtractArchive/index.ts

@@ -0,0 +1 @@
+export * from "./downloadAndExtractArchive";

package/src/bin/tools/escapeStringForPropertiesFile.ts

@@ -0,0 +1,64 @@
+// Convert a JavaScript string to UTF-16 encoding
+function toUTF16(codePoint: number): string {
+    if (codePoint <= 0xffff) {
+        // BMP character
+        return "\\u" + codePoint.toString(16).padStart(4, "0");
+    } else {
+        // Non-BMP character
+        codePoint -= 0x10000;
+        let highSurrogate = (codePoint >> 10) + 0xd800;
+        let lowSurrogate = (codePoint % 0x400) + 0xdc00;
+        return (
+            "\\u" +
+            highSurrogate.toString(16).padStart(4, "0") +
+            "\\u" +
+            lowSurrogate.toString(16).padStart(4, "0")
+        );
+    }
+}
+
+// Escapes special characters for use in a .properties file
+export function escapeStringForPropertiesFile(str: string): string {
+    let escapedStr = "";
+    for (const char of [...str]) {
+        const codePoint = char.codePointAt(0);
+        if (!codePoint) continue;
+
+        switch (char) {
+            case "\n":
+                escapedStr += "\\n";
+                break;
+            case "\r":
+                escapedStr += "\\r";
+                break;
+            case "\t":
+                escapedStr += "\\t";
+                break;
+            case "\\":
+                escapedStr += "\\\\";
+                break;
+            case ":":
+                escapedStr += "\\:";
+                break;
+            case "=":
+                escapedStr += "\\=";
+                break;
+            case "#":
+                escapedStr += "\\#";
+                break;
+            case "!":
+                escapedStr += "\\!";
+                break;
+            case "'":
+                escapedStr += "''";
+                break;
+            default:
+                if (codePoint > 0x7f) {
+                    escapedStr += toUTF16(codePoint); // Non-ASCII characters
+                } else {
+                    escapedStr += char; // ASCII character needs no escape
+                }
+        }
+    }
+    return escapedStr;
+}

package/src/bin/tools/extractArchive.ts

@@ -0,0 +1,140 @@
+import fs from "fs/promises";
+import fsSync from "fs";
+import yauzl from "yauzl";
+import stream from "stream";
+import { Deferred } from "evt/tools/Deferred";
+import { dirname as pathDirname, sep as pathSep } from "path";
+import { existsAsync } from "./fs.existsAsync";
+
+export async function extractArchive(params: {
+    archiveFilePath: string;
+    onArchiveFile: (params: {
+        relativeFilePathInArchive: string;
+        readFile: () => Promise<Buffer>;
+        /** NOTE: Will create the directory if it does not exist */
+        writeFile: (params: { filePath: string; modifiedData?: Buffer }) => Promise<void>;
+        earlyExit: () => void;
+    }) => Promise<void>;
+}) {
+    const { archiveFilePath, onArchiveFile } = params;
+
+    const zipFile = await new Promise<yauzl.ZipFile>((resolve, reject) => {
+        yauzl.open(archiveFilePath, { lazyEntries: true }, async (error, zipFile) => {
+            if (error) {
+                reject(error);
+                return;
+            }
+            resolve(zipFile);
+        });
+    });
+
+    const dDone = new Deferred<void>();
+
+    zipFile.once("end", () => {
+        zipFile.close();
+        dDone.resolve();
+    });
+
+    const writeFile = async (
+        entry: yauzl.Entry,
+        params: {
+            filePath: string;
+            modifiedData?: Buffer;
+        }
+    ): Promise<void> => {
+        const { filePath, modifiedData } = params;
+
+        {
+            const dirPath = pathDirname(filePath);
+
+            if (!(await existsAsync(dirPath))) {
+                await fs.mkdir(dirPath, { recursive: true });
+            }
+        }
+
+        if (modifiedData !== undefined) {
+            await fs.writeFile(filePath, modifiedData);
+            return;
+        }
+
+        const readStream = await new Promise<stream.Readable>(resolve =>
+            zipFile.openReadStream(entry, async (error, readStream) => {
+                if (error) {
+                    dDone.reject(error);
+                    return;
+                }
+
+                resolve(readStream);
+            })
+        );
+
+        const dDoneWithFile = new Deferred<void>();
+
+        stream.pipeline(readStream, fsSync.createWriteStream(filePath), error => {
+            if (error) {
+                dDone.reject(error);
+                return;
+            }
+
+            dDoneWithFile.resolve();
+        });
+
+        await dDoneWithFile.pr;
+    };
+
+    const readFile = (entry: yauzl.Entry) =>
+        new Promise<Buffer>(resolve =>
+            zipFile.openReadStream(entry, async (error, readStream) => {
+                if (error) {
+                    dDone.reject(error);
+                    return;
+                }
+
+                const chunks: Buffer[] = [];
+
+                readStream.on("data", chunk => {
+                    chunks.push(chunk);
+                });
+
+                readStream.on("end", () => {
+                    resolve(Buffer.concat(chunks));
+                });
+
+                readStream.on("error", error => {
+                    dDone.reject(error);
+                });
+            })
+        );
+
+    zipFile.on("entry", async (entry: yauzl.Entry) => {
+        handle_file: {
+            // NOTE: Skip directories
+            if (entry.fileName.endsWith("/")) {
+                break handle_file;
+            }
+
+            let hasEarlyExitBeenCalled = false;
+
+            await onArchiveFile({
+                relativeFilePathInArchive: entry.fileName.split("/").join(pathSep),
+                readFile: () => readFile(entry),
+                writeFile: params => writeFile(entry, params),
+                earlyExit: () => {
+                    hasEarlyExitBeenCalled = true;
+                }
+            });
+
+            if (hasEarlyExitBeenCalled) {
+                zipFile.close();
+                dDone.resolve();
+                return;
+            }
+        }
+
+        zipFile.readEntry();
+    });
+
+    zipFile.readEntry();
+
+    await dDone.pr;
+}

package/src/bin/tools/getAbsoluteAndInOsFormatPath.ts

@@ -1,7 +1,15 @@
-import { isAbsolute as pathIsAbsolute, sep as pathSep, join as pathJoin, resolve as pathResolve } from "path";
+import {
+    isAbsolute as pathIsAbsolute,
+    sep as pathSep,
+    join as pathJoin,
+    resolve as pathResolve
+} from "path";
 import * as os from "os";
 
-export function getAbsoluteAndInOsFormatPath(params: { pathIsh: string; cwd: string }): string {
+export function getAbsoluteAndInOsFormatPath(params: {
+    pathIsh: string;
+    cwd: string;
+}): string {
     const { pathIsh, cwd } = params;
 
     let pathOut = pathIsh;

package/src/bin/tools/getNpmWorkspaceRootDirPath.ts

@@ -1,25 +1,71 @@
 import * as child_process from "child_process";
 import { join as pathJoin, resolve as pathResolve, sep as pathSep } from "path";
 import { assert } from "tsafe/assert";
+import * as fs from "fs";
 
-export function getNpmWorkspaceRootDirPath(params: { reactAppRootDirPath: string }) {
-    const { reactAppRootDirPath } = params;
+export function getNpmWorkspaceRootDirPath(params: {
+    projectDirPath: string;
+    dependencyExpected: string;
+}) {
+    const { projectDirPath, dependencyExpected } = params;
 
     const npmWorkspaceRootDirPath = (function callee(depth: number): string {
-        const cwd = pathResolve(pathJoin(...[reactAppRootDirPath, ...Array(depth).fill("..")]));
+        const cwd = pathResolve(
+            pathJoin(...[projectDirPath, ...Array(depth).fill("..")])
+        );
+
+        assert(cwd !== pathSep, "NPM workspace not found");
 
         try {
-            child_process.execSync("npm config get", { cwd, "stdio": "ignore" });
+            child_process.execSync("npm config get", {
+                cwd,
+                stdio: "ignore"
+            });
         } catch (error) {
             if (String(error).includes("ENOWORKSPACES")) {
-                assert(cwd !== pathSep, "NPM workspace not found");
-
                 return callee(depth + 1);
             }
 
             throw error;
         }
 
+        const packageJsonFilePath = pathJoin(cwd, "package.json");
+
+        if (!fs.existsSync(packageJsonFilePath)) {
+            return callee(depth + 1);
+        }
+
+        assert(fs.existsSync(packageJsonFilePath));
+
+        const parsedPackageJson = JSON.parse(
+            fs.readFileSync(packageJsonFilePath).toString("utf8")
+        );
+
+        let isExpectedDependencyFound = false;
+
+        for (const dependenciesOrDevDependencies of [
+            "dependencies",
+            "devDependencies"
+        ] as const) {
+            const dependencies = parsedPackageJson[dependenciesOrDevDependencies];
+
+            if (dependencies === undefined) {
+                continue;
+            }
+
+            assert(dependencies instanceof Object);
+
+            if (dependencies[dependencyExpected] === undefined) {
+                continue;
+            }
+
+            isExpectedDependencyFound = true;
+        }
+
+        if (!isExpectedDependencyFound && parsedPackageJson.name !== dependencyExpected) {
+            return callee(depth + 1);
+        }
+
         return cwd;
     })(0);
 

package/src/bin/tools/octokit-addons/getLatestsSemVersionedTag.ts

@@ -5,7 +5,11 @@ import { SemVer } from "../SemVer";
 export function getLatestsSemVersionedTagFactory(params: { octokit: Octokit }) {
     const { octokit } = params;
 
-    async function getLatestsSemVersionedTag(params: { owner: string; repo: string; count: number }): Promise<
+    async function getLatestsSemVersionedTag(params: {
+        owner: string;
+        repo: string;
+        count: number;
+    }): Promise<
         {
             tag: string;
             version: SemVer;
@@ -33,7 +37,9 @@ export function getLatestsSemVersionedTagFactory(params: { octokit: Octokit }) {
             semVersionedTags.push({ tag, version });
         }
 
-        return semVersionedTags.sort(({ version: vX }, { version: vY }) => SemVer.compare(vY, vX)).slice(0, count);
+        return semVersionedTags
+            .sort(({ version: vX }, { version: vY }) => SemVer.compare(vY, vX))
+            .slice(0, count);
     }
 
     return { getLatestsSemVersionedTag };