keycloakify 6.9.1 → 6.10.1

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "keycloakify",
-    "version": "6.9.1",
+    "version": "6.10.1",
     "description": "Keycloak theme generator for Reacts app",
     "repository": {
         "type": "git",
@@ -1294,11 +1294,11 @@
         "minimal-polyfills": "^2.2.2",
         "minimist": "^1.2.6",
         "path-browserify": "^1.0.1",
-        "powerhooks": "^0.22.0",
+        "powerhooks": "^0.22.1",
         "react-markdown": "^5.0.3",
         "rfc4648": "^1.5.2",
         "scripting-tools": "^0.19.13",
-        "tsafe": "^1.4.1",
+        "tsafe": "^1.4.2",
         "tss-react": "4.4.1-rc.0",
         "zod": "^3.17.10"
     }

package/src/bin/download-builtin-keycloak-theme.ts

@@ -7,11 +7,11 @@ import { promptKeycloakVersion } from "./promptKeycloakVersion";
 import { getCliOptions } from "./tools/cliOptions";
 import { getLogger } from "./tools/logger";
 
-export function downloadBuiltinKeycloakTheme(params: { keycloakVersion: string; destDirPath: string; isSilent: boolean }) {
+export async function downloadBuiltinKeycloakTheme(params: { keycloakVersion: string; destDirPath: string; isSilent: boolean }) {
     const { keycloakVersion, destDirPath, isSilent } = params;
 
     for (const ext of ["", "-community"]) {
-        downloadAndUnzip({
+        await downloadAndUnzip({
             "destDirPath": destDirPath,
             "url": `https://github.com/keycloak/keycloak/archive/refs/tags/${keycloakVersion}.zip`,
             "pathOfDirToExtractInArchive": `keycloak-${keycloakVersion}/themes/src/main/resources${ext}/theme`,
@@ -31,7 +31,7 @@ if (require.main === module) {
 
         logger.log(`Downloading builtins theme of Keycloak ${keycloakVersion} here ${destDirPath}`);
 
-        downloadBuiltinKeycloakTheme({
+        await downloadBuiltinKeycloakTheme({
             keycloakVersion,
             destDirPath,
             isSilent

package/src/bin/keycloakify/generateKeycloakThemeResources.ts

@@ -5,7 +5,6 @@ import { replaceImportsFromStaticInJsCode } from "./replacers/replaceImportsFrom
 import { replaceImportsInCssCode } from "./replacers/replaceImportsInCssCode";
 import { generateFtlFilesCodeFactory, pageIds } from "./generateFtl";
 import { downloadBuiltinKeycloakTheme } from "../download-builtin-keycloak-theme";
-import * as child_process from "child_process";
 import { mockTestingResourcesCommonPath, mockTestingResourcesPath, mockTestingSubDirOfPublicDirBasename } from "../mockTestingResourcesPath";
 import { isInside } from "../tools/isInside";
 import type { BuildOptions } from "./BuildOptions";
@@ -53,13 +52,13 @@ export namespace BuildOptionsLike {
     assert<typeof buildOptions extends BuildOptionsLike ? true : false>();
 }
 
-export function generateKeycloakThemeResources(params: {
+export async function generateKeycloakThemeResources(params: {
     reactAppBuildDirPath: string;
     keycloakThemeBuildingDirPath: string;
     keycloakThemeEmailDirPath: string;
     keycloakVersion: string;
     buildOptions: BuildOptionsLike;
-}): { doBundlesEmailTemplate: boolean } {
+}): Promise<{ doBundlesEmailTemplate: boolean }> {
     const { reactAppBuildDirPath, keycloakThemeBuildingDirPath, keycloakThemeEmailDirPath, keycloakVersion, buildOptions } = params;
 
     const logger = getLogger({ isSilent: buildOptions.isSilent });
@@ -155,7 +154,7 @@ export function generateKeycloakThemeResources(params: {
     {
         const tmpDirPath = pathJoin(themeDirPath, "..", "tmp_xxKdLpdIdLd");
 
-        downloadBuiltinKeycloakTheme({
+        await downloadBuiltinKeycloakTheme({
             keycloakVersion,
             "destDirPath": tmpDirPath,
             isSilent: buildOptions.isSilent
@@ -190,8 +189,7 @@ export function generateKeycloakThemeResources(params: {
         );
 
         fs.writeFileSync(pathJoin(keycloakResourcesWithinPublicDirPath, ".gitignore"), Buffer.from("*", "utf8"));
-
-        child_process.execSync(`rm -r ${tmpDirPath}`);
+        fs.rmSync(tmpDirPath, { recursive: true, force: true });
     }
 
     fs.writeFileSync(

package/src/bin/keycloakify/keycloakify.ts

@@ -13,7 +13,7 @@ const reactProjectDirPath = process.cwd();
 export const keycloakThemeBuildingDirPath = pathJoin(reactProjectDirPath, "build_keycloak");
 export const keycloakThemeEmailDirPath = pathJoin(keycloakThemeBuildingDirPath, "..", "keycloak_email");
 
-export function main() {
+export async function main() {
     const { isSilent, hasExternalAssets } = getCliOptions(process.argv.slice(2));
     const logger = getLogger({ isSilent });
     logger.log("🔏 Building the keycloak theme...⌚");
@@ -33,7 +33,7 @@ export function main() {
         "isSilent": isSilent
     });
 
-    const { doBundlesEmailTemplate } = generateKeycloakThemeResources({
+    const { doBundlesEmailTemplate } = await generateKeycloakThemeResources({
         keycloakThemeBuildingDirPath,
         keycloakThemeEmailDirPath,
         "reactAppBuildDirPath": pathJoin(reactProjectDirPath, "build"),

package/src/bin/link_in_test_app.ts

@@ -15,7 +15,8 @@ fs.writeFileSync(
                 return {
                     ...packageJsonParsed,
                     "main": packageJsonParsed["main"].replace(/^dist\//, ""),
-                    "types": packageJsonParsed["types"].replace(/^dist\//, "")
+                    "types": packageJsonParsed["types"].replace(/^dist\//, ""),
+                    "bin": Object.fromEntries(Object.entries<string>(packageJsonParsed["bin"]).map(([k, v]) => [k, v.replace(/^dist\//, "")]))
                 };
             })(),
             null,

package/src/bin/tools/downloadAndUnzip.ts

@@ -1,80 +1,289 @@
-import { basename as pathBasename, join as pathJoin } from "path";
-import { execSync } from "child_process";
-import * as fs from "fs";
+import { dirname as pathDirname, basename as pathBasename, join as pathJoin } from "path";
+import { createReadStream, createWriteStream, unlinkSync } from "fs";
+import { stat, mkdir, unlink, readFile, writeFile } from "fs/promises";
 import { transformCodebase } from "./transformCodebase";
-import * as crypto from "crypto";
+import { createHash } from "crypto";
+import http from "http";
+import https from "https";
+import { createInflateRaw } from "zlib";
 
-/** assert url ends with .zip */
-export function downloadAndUnzip(params: {
-    isSilent: boolean;
-    url: string;
-    destDirPath: string;
-    pathOfDirToExtractInArchive?: string;
-    cacheDirPath: string;
-}) {
-    const { url, destDirPath, pathOfDirToExtractInArchive, cacheDirPath } = params;
-
-    const extractDirPath = pathJoin(
-        cacheDirPath,
-        `_${crypto.createHash("sha256").update(JSON.stringify({ url, pathOfDirToExtractInArchive })).digest("hex").substring(0, 15)}`
-    );
+import type { Readable } from "stream";
 
-    fs.mkdirSync(cacheDirPath, { "recursive": true });
+function hash(s: string) {
+    return createHash("sha256").update(s).digest("hex");
+}
 
-    const { readIsSuccessByExtractDirPath, writeIsSuccessByExtractDirPath } = (() => {
-        const filePath = pathJoin(cacheDirPath, "isSuccessByExtractDirPath.json");
+async function maybeReadFile(path: string) {
+    try {
+        return await readFile(path, "utf-8");
+    } catch (error) {
+        if ((error as Error & { code: string }).code === "ENOENT") return undefined;
+        throw error;
+    }
+}
 
-        type IsSuccessByExtractDirPath = Record<string, boolean | undefined>;
+async function maybeStat(path: string) {
+    try {
+        return await stat(path);
+    } catch (error) {
+        if ((error as Error & { code: string }).code === "ENOENT") return undefined;
+        throw error;
+    }
+}
 
-        function readIsSuccessByExtractDirPath(): IsSuccessByExtractDirPath {
-            if (!fs.existsSync(filePath)) {
-                return {};
-            }
+/**
+ * Download a file from `url` to `dir`. Will try to avoid downloading existing
+ * files by using an `{hash(url)}.etag` file. If this file exists, we add an
+ * etag headear, so server can tell us if file changed and we should re-download
+ * or if our file is up-to-date.
+ *
+ * Warning, this method assumes that the target filename can be extracted from
+ * url, content-disposition headers are ignored.
+ *
+ * If the target directory does not exist, it will be created.
+ *
+ * If the target file exists and is out of date, it will be overwritten.
+ * If the target file exists and there is no etag file, the target file will
+ * be overwritten.
+ *
+ * @param url download url
+ * @param dir target directory
+ * @returns promise for the full path of the downloaded file
+ */
+async function download(url: string, dir: string): Promise<string> {
+    await mkdir(dir, { recursive: true });
+    const filename = pathBasename(url);
+    const filepath = pathJoin(dir, filename);
+    // If downloaded file exists already and has an `.etag` companion file,
+    // read the etag from that file. This will avoid re-downloading the file
+    // if it is up to date.
+    const exists = await maybeStat(filepath);
+    const etagFilepath = pathJoin(dir, "_" + hash(url).substring(0, 15) + ".etag");
+    const etag = !exists ? undefined : await maybeReadFile(etagFilepath);
 
-            return JSON.parse(fs.readFileSync(filePath).toString("utf8"));
+    return new Promise((resolve, reject) => {
+        // use inner method to allow following redirects
+        function request(url1: URL) {
+            const headers: Record<string, string> = {};
+            if (etag) headers["If-None-Match"] = etag;
+            (url1.protocol === "https:" ? https : http).get(url1, { headers }, response => {
+                if (response.statusCode === 301 || response.statusCode === 302) {
+                    // follow redirects
+                    request(new URL(response.headers.location!!));
+                } else if (response.statusCode === 304) {
+                    // up-to-date, resolve now
+                    resolve(filepath);
+                } else if (response.statusCode !== 200) {
+                    reject(new Error(`Request to ${url1} returned status ${response.statusCode}.`));
+                } else {
+                    const fp = createWriteStream(filepath, { autoClose: true });
+                    fp.on("err", e => {
+                        fp.close();
+                        unlinkSync(filepath);
+                        reject(e);
+                    });
+                    fp.on("finish", async () => {
+                        // when targetfile has been written, write etag file so that
+                        // next time around we don't need to re-download
+                        const responseEtag = response.headers.etag;
+                        if (responseEtag) await writeFile(etagFilepath, responseEtag, "utf-8");
+                        resolve(filepath);
+                    });
+                    response.pipe(fp);
+                }
+            });
         }
+        request(new URL(url));
+    });
+}
 
-        function writeIsSuccessByExtractDirPath(isSuccessByExtractDirPath: IsSuccessByExtractDirPath): void {
-            fs.writeFileSync(filePath, Buffer.from(JSON.stringify(isSuccessByExtractDirPath, null, 2), "utf8"));
-        }
+/**
+ * @typedef
+ * @type MultiError = Error & { cause: Error[] }
+ */
 
-        return { readIsSuccessByExtractDirPath, writeIsSuccessByExtractDirPath };
-    })();
+/**
+ * Extract the archive `zipFile` into the directory `dir`. If `archiveDir` is given,
+ * only that directory will be extracted, stripping the given path components.
+ *
+ * If dir does not exist, it will be created.
+ *
+ * If any archive file exists, it will be overwritten.
+ *
+ * Will unzip using all available nodejs worker threads.
+ *
+ * Will try to clean up extracted files on failure.
+ *
+ * If unpacking fails, will either throw an regular error, or
+ * possibly an `MultiError`, which contains a `cause` field with
+ * a number of root cause errors.
+ *
+ * Warning this method is not optimized for continuous reading of the zip
+ * archive, but is a trade-off between simplicity and allowing extraction
+ * of a single directory from the archive.
+ *
+ * @param zipFile the file to unzip
+ * @param dir the target directory
+ * @param archiveDir if given, unpack only files from this archive directory
+ * @throws {MultiError} error
+ * @returns Promise for a list of full file paths pointing to actually extracted files
+ */
+async function unzip(zipFile: string, dir: string, archiveDir?: string): Promise<string[]> {
+    await mkdir(dir, { recursive: true });
+    const promises: Promise<string>[] = [];
 
-    downloadAndUnzip: {
-        const isSuccessByExtractDirPath = readIsSuccessByExtractDirPath();
+    // Iterate over all files in the zip, skip files which are not in archiveDir,
+    // if given.
+    for await (const record of iterateZipArchive(zipFile)) {
+        const { path: recordPath, createReadStream: createRecordReadStream } = record;
+        const filePath = pathJoin(dir, recordPath);
+        const parent = pathDirname(filePath);
+        if (archiveDir && !recordPath.startsWith(archiveDir)) continue;
+        promises.push(
+            new Promise<string>(async (resolve, reject) => {
+                await mkdir(parent, { recursive: true });
+                // Pull the file out of the archive, write it to the target directory
+                const input = createRecordReadStream();
+                const output = createWriteStream(filePath);
+                output.on("error", e => reject(Object.assign(e, { filePath })));
+                output.on("finish", () => resolve(filePath));
+                input.pipe(output);
+            })
+        );
+    }
 
-        if (isSuccessByExtractDirPath[extractDirPath]) {
-            break downloadAndUnzip;
-        }
+    // Wait until _all_ files are either extracted or failed
+    const results = await Promise.allSettled(promises);
+    const success = results.filter(r => r.status === "fulfilled").map(r => (r as PromiseFulfilledResult<string>).value);
+    const failure = results.filter(r => r.status === "rejected").map(r => (r as PromiseRejectedResult).reason);
 
-        writeIsSuccessByExtractDirPath({
-            ...isSuccessByExtractDirPath,
-            [extractDirPath]: false
-        });
+    // If any extraction failed, try to clean up, then throw a MultiError,
+    // which has a `cause` field, containing a list of root cause errors.
+    if (failure.length) {
+        await Promise.all(success.map(path => unlink(path)));
+        await Promise.all(failure.map(e => e && e.path && unlink(e.path as string)));
+        const e = new Error("Failed to extract: " + failure.map(e => e.message).join(";"));
+        (e as any).cause = failure;
+        throw e;
+    }
 
-        fs.rmSync(extractDirPath, { "recursive": true, "force": true });
+    return success;
+}
 
-        fs.mkdirSync(extractDirPath);
+/**
+ *
+ * @param file file to read
+ * @param start first byte to read
+ * @param end last byte to read
+ * @returns Promise of a buffer of read bytes
+ */
+async function readFileChunk(file: string, start: number, end: number): Promise<Buffer> {
+    const chunks: Buffer[] = [];
+    return new Promise((resolve, reject) => {
+        const stream = createReadStream(file, { start, end });
+        stream.on("error", e => reject(e));
+        stream.on("end", () => resolve(Buffer.concat(chunks)));
+        stream.on("data", chunk => chunks.push(chunk as Buffer));
+    });
+}
 
-        const zipFileBasename = pathBasename(url);
+type ZipRecord = {
+    path: string;
+    createReadStream: () => Readable;
+    compressionMethod: "deflate" | undefined;
+};
 
-        execSync(`curl -L ${url} -o ${zipFileBasename} ${params.isSilent ? "-s" : ""}`, { "cwd": extractDirPath });
+type ZipRecordGenerator = AsyncGenerator<ZipRecord, void, unknown>;
 
-        execSync(`unzip -o ${zipFileBasename}${pathOfDirToExtractInArchive === undefined ? "" : ` "${pathOfDirToExtractInArchive}/**/*"`}`, {
-            "cwd": extractDirPath
-        });
+/**
+ * Iterate over all records of a zipfile, and yield a ZipRecord.
+ * Use `record.createReadStream()` to actually read the file.
+ *
+ * Warning this method will only work with single-disk zip files.
+ * Warning this method may fail if the zip archive has an crazy amount
+ * of files and the central directory is not fully contained within the
+ * last 65k bytes of the zip file.
+ *
+ * @param zipFile
+ * @returns AsyncGenerator which will yield ZipRecords
+ */
+async function* iterateZipArchive(zipFile: string): ZipRecordGenerator {
+    // Need to know zip file size before we can do anything  else
+    const { size } = await stat(zipFile);
+    const chunkSize = 65_535 + 22 + 1; // max comment size + end header size + wiggle
+    // Read last ~65k bytes. Zip files have an comment up to 65_535 bytes at the very end,
+    // before that comes the zip central directory end header.
+    let chunk = await readFileChunk(zipFile, size - chunkSize, size);
+    const unread = size - chunk.length;
+    let i = chunk.length - 4;
+    let found = false;
+    // Find central directory end header, reading backwards from the end
+    while (!found && i-- > 0) if (chunk[i] === 0x50 && chunk.readUInt32LE(i) === 0x06054b50) found = true;
+    if (!found) throw new Error("Not a zip file");
+    // This method will fail on a multi-disk zip, so bail early.
+    if (chunk.readUInt16LE(i + 4) !== 0) throw new Error("Multi-disk zip not supported");
+    let nFiles = chunk.readUint16LE(i + 10);
+    // Get the position of the central directory
+    const directorySize = chunk.readUint32LE(i + 12);
+    const directoryOffset = chunk.readUint32LE(i + 16);
+    if (directoryOffset === 0xffff_ffff) throw new Error("zip64 not supported");
+    if (directoryOffset > size) throw new Error(`Central directory offset ${directoryOffset} is outside file`);
+    i = directoryOffset - unread;
+    // If i < 0, it means that the central directory is not contained within `chunk`
+    if (i < 0) {
+        chunk = await readFileChunk(zipFile, directoryOffset, directoryOffset + directorySize);
+        i = 0;
+    }
+    // Now iterate the central directory records, yield an `ZipRecord` for every entry
+    while (nFiles-- > 0) {
+        // Check for marker bytes
+        if (chunk.readUInt32LE(i) !== 0x02014b50) throw new Error("No central directory record at position " + (unread + i));
+        const compressionMethod = ({ 8: "deflate" } as const)[chunk.readUint16LE(i + 10)];
+        const compressedFileSize = chunk.readUint32LE(i + 20);
+        const filenameLength = chunk.readUint16LE(i + 28);
+        const extraLength = chunk.readUint16LE(i + 30);
+        const commentLength = chunk.readUint16LE(i + 32);
+        // Start of thea actual content byte stream is after the 'local' record header,
+        // which is  30 bytes long plus filename and extra field
+        const start = chunk.readUint32LE(i + 42) + 30 + filenameLength + extraLength;
+        const end = start + compressedFileSize;
+        const filename = chunk.slice(i + 46, i + 46 + filenameLength).toString("utf-8");
+        const createRecordReadStream = () => {
+            const input = createReadStream(zipFile, { start, end });
+            if (compressionMethod === "deflate") {
+                const inflate = createInflateRaw();
+                input.pipe(inflate);
+                return inflate;
+            }
+            return input;
+        };
+        if (end > start) yield { path: filename, createReadStream: createRecordReadStream, compressionMethod };
+        // advance pointer to next central directory entry
+        i += 46 + filenameLength + extraLength + commentLength;
+    }
+}
 
-        fs.rmSync(pathJoin(extractDirPath, zipFileBasename), { "recursive": true, "force": true });
+export async function downloadAndUnzip({
+    url,
+    destDirPath,
+    pathOfDirToExtractInArchive,
+    cacheDirPath
+}: {
+    isSilent: boolean;
+    url: string;
+    destDirPath: string;
+    pathOfDirToExtractInArchive?: string;
+    cacheDirPath: string;
+}) {
+    const downloadHash = hash(JSON.stringify({ url, pathOfDirToExtractInArchive })).substring(0, 15);
+    const extractDirPath = pathJoin(cacheDirPath, `_${downloadHash}`);
 
-        writeIsSuccessByExtractDirPath({
-            ...isSuccessByExtractDirPath,
-            [extractDirPath]: true
-        });
-    }
+    const zipFilepath = await download(url, cacheDirPath);
+    const zipMtime = (await stat(zipFilepath)).mtimeMs;
+    const unzipMtime = (await maybeStat(extractDirPath))?.mtimeMs;
 
-    transformCodebase({
-        "srcDirPath": pathOfDirToExtractInArchive === undefined ? extractDirPath : pathJoin(extractDirPath, pathOfDirToExtractInArchive),
-        destDirPath
-    });
+    if (!unzipMtime || zipMtime > unzipMtime) await unzip(zipFilepath, extractDirPath, pathOfDirToExtractInArchive);
+
+    const srcDirPath = pathOfDirToExtractInArchive === undefined ? extractDirPath : pathJoin(extractDirPath, pathOfDirToExtractInArchive);
+    transformCodebase({ srcDirPath, destDirPath });
 }

package/src/bin/tools/grant-exec-perms.ts

@@ -1,10 +1,17 @@
 import { getProjectRoot } from "./getProjectRoot";
 import { join as pathJoin } from "path";
-import * as child_process from "child_process";
-import * as fs from "fs";
+import { constants } from "fs";
+import { chmod, stat } from "fs/promises";
 
-Object.entries<string>(JSON.parse(fs.readFileSync(pathJoin(getProjectRoot(), "package.json")).toString("utf8"))["bin"]).forEach(([, scriptPath]) =>
-    child_process.execSync(`chmod +x ${scriptPath}`, {
-        "cwd": getProjectRoot()
-    })
-);
+async () => {
+    var { bin } = await import(pathJoin(getProjectRoot(), "package.json"));
+
+    var promises = Object.values<string>(bin).map(async scriptPath => {
+        const fullPath = pathJoin(getProjectRoot(), scriptPath);
+        const oldMode = (await stat(fullPath)).mode;
+        const newMode = oldMode | constants.S_IXUSR | constants.S_IXGRP | constants.S_IXOTH;
+        await chmod(fullPath, newMode);
+    });
+
+    await Promise.all(promises);
+};