keycloakify 6.9.0 → 6.9.1

package/src/bin/tools/downloadAndUnzip.ts

@@ -1,289 +1,80 @@
-import { dirname as pathDirname, basename as pathBasename, join as pathJoin } from "path";
-import { createReadStream, createWriteStream, unlinkSync } from "fs";
-import { stat, mkdir, unlink, readFile, writeFile } from "fs/promises";
+import { basename as pathBasename, join as pathJoin } from "path";
+import { execSync } from "child_process";
+import * as fs from "fs";
 import { transformCodebase } from "./transformCodebase";
-import { createHash } from "crypto";
-import http from "http";
-import https from "https";
-import { createInflateRaw } from "zlib";
+import * as crypto from "crypto";
 
-import type { Readable } from "stream";
+/** assert url ends with .zip */
+export function downloadAndUnzip(params: {
+    isSilent: boolean;
+    url: string;
+    destDirPath: string;
+    pathOfDirToExtractInArchive?: string;
+    cacheDirPath: string;
+}) {
+    const { url, destDirPath, pathOfDirToExtractInArchive, cacheDirPath } = params;
 
-function hash(s: string) {
-    return createHash("sha256").update(s).digest("hex");
-}
+    const extractDirPath = pathJoin(
+        cacheDirPath,
+        `_${crypto.createHash("sha256").update(JSON.stringify({ url, pathOfDirToExtractInArchive })).digest("hex").substring(0, 15)}`
+    );
 
-async function maybeReadFile(path: string) {
-    try {
-        return await readFile(path, "utf-8");
-    } catch (error) {
-        if ((error as Error & { code: string }).code === "ENOENT") return undefined;
-        throw error;
-    }
-}
+    fs.mkdirSync(cacheDirPath, { "recursive": true });
 
-async function maybeStat(path: string) {
-    try {
-        return await stat(path);
-    } catch (error) {
-        if ((error as Error & { code: string }).code === "ENOENT") return undefined;
-        throw error;
-    }
-}
+    const { readIsSuccessByExtractDirPath, writeIsSuccessByExtractDirPath } = (() => {
+        const filePath = pathJoin(cacheDirPath, "isSuccessByExtractDirPath.json");
 
-/**
- * Download a file from `url` to `dir`. Will try to avoid downloading existing
- * files by using an `{hash(url)}.etag` file. If this file exists, we add an
- * etag headear, so server can tell us if file changed and we should re-download
- * or if our file is up-to-date.
- *
- * Warning, this method assumes that the target filename can be extracted from
- * url, content-disposition headers are ignored.
- *
- * If the target directory does not exist, it will be created.
- *
- * If the target file exists and is out of date, it will be overwritten.
- * If the target file exists and there is no etag file, the target file will
- * be overwritten.
- *
- * @param url download url
- * @param dir target directory
- * @returns promise for the full path of the downloaded file
- */
-async function download(url: string, dir: string): Promise<string> {
-    await mkdir(dir, { recursive: true });
-    const filename = pathBasename(url);
-    const filepath = pathJoin(dir, filename);
-    // If downloaded file exists already and has an `.etag` companion file,
-    // read the etag from that file. This will avoid re-downloading the file
-    // if it is up to date.
-    const exists = await maybeStat(filepath);
-    const etagFilepath = pathJoin(dir, "_" + hash(url).substring(0, 15) + ".etag");
-    const etag = !exists ? undefined : await maybeReadFile(etagFilepath);
+        type IsSuccessByExtractDirPath = Record<string, boolean | undefined>;
 
-    return new Promise((resolve, reject) => {
-        // use inner method to allow following redirects
-        function request(url1: URL) {
-            const headers: Record<string, string> = {};
-            if (etag) headers["If-None-Match"] = etag;
-            (url1.protocol === "https:" ? https : http).get(url1, { headers }, response => {
-                if (response.statusCode === 301 || response.statusCode === 302) {
-                    // follow redirects
-                    request(new URL(response.headers.location!!));
-                } else if (response.statusCode === 304) {
-                    // up-to-date, resolve now
-                    resolve(filepath);
-                } else if (response.statusCode !== 200) {
-                    reject(new Error(`Request to ${url1} returned status ${response.statusCode}.`));
-                } else {
-                    const fp = createWriteStream(filepath, { autoClose: true });
-                    fp.on("err", e => {
-                        fp.close();
-                        unlinkSync(filepath);
-                        reject(e);
-                    });
-                    fp.on("finish", async () => {
-                        // when targetfile has been written, write etag file so that
-                        // next time around we don't need to re-download
-                        const responseEtag = response.headers.etag;
-                        if (responseEtag) await writeFile(etagFilepath, responseEtag, "utf-8");
-                        resolve(filepath);
-                    });
-                    response.pipe(fp);
-                }
-            });
-        }
-        request(new URL(url));
-    });
-}
+        function readIsSuccessByExtractDirPath(): IsSuccessByExtractDirPath {
+            if (!fs.existsSync(filePath)) {
+                return {};
+            }
 
-/**
- * @typedef
- * @type MultiError = Error & { cause: Error[] }
- */
+            return JSON.parse(fs.readFileSync(filePath).toString("utf8"));
+        }
 
-/**
- * Extract the archive `zipFile` into the directory `dir`. If `archiveDir` is given,
- * only that directory will be extracted, stripping the given path components.
- *
- * If dir does not exist, it will be created.
- *
- * If any archive file exists, it will be overwritten.
- *
- * Will unzip using all available nodejs worker threads.
- *
- * Will try to clean up extracted files on failure.
- *
- * If unpacking fails, will either throw an regular error, or
- * possibly an `MultiError`, which contains a `cause` field with
- * a number of root cause errors.
- *
- * Warning this method is not optimized for continuous reading of the zip
- * archive, but is a trade-off between simplicity and allowing extraction
- * of a single directory from the archive.
- *
- * @param zipFile the file to unzip
- * @param dir the target directory
- * @param archiveDir if given, unpack only files from this archive directory
- * @throws {MultiError} error
- * @returns Promise for a list of full file paths pointing to actually extracted files
- */
-async function unzip(zipFile: string, dir: string, archiveDir?: string): Promise<string[]> {
-    await mkdir(dir, { recursive: true });
-    const promises: Promise<string>[] = [];
+        function writeIsSuccessByExtractDirPath(isSuccessByExtractDirPath: IsSuccessByExtractDirPath): void {
+            fs.writeFileSync(filePath, Buffer.from(JSON.stringify(isSuccessByExtractDirPath, null, 2), "utf8"));
+        }
 
-    // Iterate over all files in the zip, skip files which are not in archiveDir,
-    // if given.
-    for await (const record of iterateZipArchive(zipFile)) {
-        const { path: recordPath, createReadStream: createRecordReadStream } = record;
-        const filePath = pathJoin(dir, recordPath);
-        const parent = pathDirname(filePath);
-        if (archiveDir && !recordPath.startsWith(archiveDir)) continue;
-        promises.push(
-            new Promise<string>(async (resolve, reject) => {
-                await mkdir(parent, { recursive: true });
-                // Pull the file out of the archive, write it to the target directory
-                const input = createRecordReadStream();
-                const output = createWriteStream(filePath);
-                output.on("error", e => reject(Object.assign(e, { filePath })));
-                output.on("finish", () => resolve(filePath));
-                input.pipe(output);
-            })
-        );
-    }
+        return { readIsSuccessByExtractDirPath, writeIsSuccessByExtractDirPath };
+    })();
 
-    // Wait until _all_ files are either extracted or failed
-    const results = await Promise.allSettled(promises);
-    const success = results.filter(r => r.status === "fulfilled").map(r => (r as PromiseFulfilledResult<string>).value);
-    const failure = results.filter(r => r.status === "rejected").map(r => (r as PromiseRejectedResult).reason);
+    downloadAndUnzip: {
+        const isSuccessByExtractDirPath = readIsSuccessByExtractDirPath();
 
-    // If any extraction failed, try to clean up, then throw a MultiError,
-    // which has a `cause` field, containing a list of root cause errors.
-    if (failure.length) {
-        await Promise.all(success.map(path => unlink(path)));
-        await Promise.all(failure.map(e => e && e.path && unlink(e.path as string)));
-        const e = new Error("Failed to extract: " + failure.map(e => e.message).join(";"));
-        (e as any).cause = failure;
-        throw e;
-    }
+        if (isSuccessByExtractDirPath[extractDirPath]) {
+            break downloadAndUnzip;
+        }
 
-    return success;
-}
+        writeIsSuccessByExtractDirPath({
+            ...isSuccessByExtractDirPath,
+            [extractDirPath]: false
+        });
 
-/**
- *
- * @param file file to read
- * @param start first byte to read
- * @param end last byte to read
- * @returns Promise of a buffer of read bytes
- */
-async function readFileChunk(file: string, start: number, end: number): Promise<Buffer> {
-    const chunks: Buffer[] = [];
-    return new Promise((resolve, reject) => {
-        const stream = createReadStream(file, { start, end });
-        stream.on("error", e => reject(e));
-        stream.on("end", () => resolve(Buffer.concat(chunks)));
-        stream.on("data", chunk => chunks.push(chunk as Buffer));
-    });
-}
+        fs.rmSync(extractDirPath, { "recursive": true, "force": true });
 
-type ZipRecord = {
-    path: string;
-    createReadStream: () => Readable;
-    compressionMethod: "deflate" | undefined;
-};
+        fs.mkdirSync(extractDirPath);
 
-type ZipRecordGenerator = AsyncGenerator<ZipRecord, void, unknown>;
+        const zipFileBasename = pathBasename(url);
 
-/**
- * Iterate over all records of a zipfile, and yield a ZipRecord.
- * Use `record.createReadStream()` to actually read the file.
- *
- * Warning this method will only work with single-disk zip files.
- * Warning this method may fail if the zip archive has an crazy amount
- * of files and the central directory is not fully contained within the
- * last 65k bytes of the zip file.
- *
- * @param zipFile
- * @returns AsyncGenerator which will yield ZipRecords
- */
-async function* iterateZipArchive(zipFile: string): ZipRecordGenerator {
-    // Need to know zip file size before we can do anything  else
-    const { size } = await stat(zipFile);
-    const chunkSize = 65_535 + 22 + 1; // max comment size + end header size + wiggle
-    // Read last ~65k bytes. Zip files have an comment up to 65_535 bytes at the very end,
-    // before that comes the zip central directory end header.
-    let chunk = await readFileChunk(zipFile, size - chunkSize, size);
-    const unread = size - chunk.length;
-    let i = chunk.length - 4;
-    let found = false;
-    // Find central directory end header, reading backwards from the end
-    while (!found && i-- > 0) if (chunk[i] === 0x50 && chunk.readUInt32LE(i) === 0x06054b50) found = true;
-    if (!found) throw new Error("Not a zip file");
-    // This method will fail on a multi-disk zip, so bail early.
-    if (chunk.readUInt16LE(i + 4) !== 0) throw new Error("Multi-disk zip not supported");
-    let nFiles = chunk.readUint16LE(i + 10);
-    // Get the position of the central directory
-    const directorySize = chunk.readUint32LE(i + 12);
-    const directoryOffset = chunk.readUint32LE(i + 16);
-    if (directoryOffset === 0xffff_ffff) throw new Error("zip64 not supported");
-    if (directoryOffset > size) throw new Error(`Central directory offset ${directoryOffset} is outside file`);
-    i = directoryOffset - unread;
-    // If i < 0, it means that the central directory is not contained within `chunk`
-    if (i < 0) {
-        chunk = await readFileChunk(zipFile, directoryOffset, directoryOffset + directorySize);
-        i = 0;
-    }
-    // Now iterate the central directory records, yield an `ZipRecord` for every entry
-    while (nFiles-- > 0) {
-        // Check for marker bytes
-        if (chunk.readUInt32LE(i) !== 0x02014b50) throw new Error("No central directory record at position " + (unread + i));
-        const compressionMethod = ({ 8: "deflate" } as const)[chunk.readUint16LE(i + 10)];
-        const compressedFileSize = chunk.readUint32LE(i + 20);
-        const filenameLength = chunk.readUint16LE(i + 28);
-        const extraLength = chunk.readUint16LE(i + 30);
-        const commentLength = chunk.readUint16LE(i + 32);
-        // Start of thea actual content byte stream is after the 'local' record header,
-        // which is  30 bytes long plus filename and extra field
-        const start = chunk.readUint32LE(i + 42) + 30 + filenameLength + extraLength;
-        const end = start + compressedFileSize;
-        const filename = chunk.slice(i + 46, i + 46 + filenameLength).toString("utf-8");
-        const createRecordReadStream = () => {
-            const input = createReadStream(zipFile, { start, end });
-            if (compressionMethod === "deflate") {
-                const inflate = createInflateRaw();
-                input.pipe(inflate);
-                return inflate;
-            }
-            return input;
-        };
-        if (end > start) yield { path: filename, createReadStream: createRecordReadStream, compressionMethod };
-        // advance pointer to next central directory entry
-        i += 46 + filenameLength + extraLength + commentLength;
-    }
-}
+        execSync(`curl -L ${url} -o ${zipFileBasename} ${params.isSilent ? "-s" : ""}`, { "cwd": extractDirPath });
 
-export async function downloadAndUnzip({
-    url,
-    destDirPath,
-    pathOfDirToExtractInArchive,
-    cacheDirPath
-}: {
-    isSilent: boolean;
-    url: string;
-    destDirPath: string;
-    pathOfDirToExtractInArchive?: string;
-    cacheDirPath: string;
-}) {
-    const downloadHash = hash(JSON.stringify({ url, pathOfDirToExtractInArchive })).substring(0, 15);
-    const extractDirPath = pathJoin(cacheDirPath, `_${downloadHash}`);
+        execSync(`unzip -o ${zipFileBasename}${pathOfDirToExtractInArchive === undefined ? "" : ` "${pathOfDirToExtractInArchive}/**/*"`}`, {
+            "cwd": extractDirPath
+        });
 
-    const zipFilepath = await download(url, cacheDirPath);
-    const zipMtime = (await stat(zipFilepath)).mtimeMs;
-    const unzipMtime = (await maybeStat(extractDirPath))?.mtimeMs;
+        fs.rmSync(pathJoin(extractDirPath, zipFileBasename), { "recursive": true, "force": true });
 
-    if (!unzipMtime || zipMtime > unzipMtime) await unzip(zipFilepath, extractDirPath, pathOfDirToExtractInArchive);
+        writeIsSuccessByExtractDirPath({
+            ...isSuccessByExtractDirPath,
+            [extractDirPath]: true
+        });
+    }
 
-    const srcDirPath = pathOfDirToExtractInArchive === undefined ? extractDirPath : pathJoin(extractDirPath, pathOfDirToExtractInArchive);
-    transformCodebase({ srcDirPath, destDirPath });
+    transformCodebase({
+        "srcDirPath": pathOfDirToExtractInArchive === undefined ? extractDirPath : pathJoin(extractDirPath, pathOfDirToExtractInArchive),
+        destDirPath
+    });
 }

package/src/bin/tools/grant-exec-perms.ts

@@ -1,17 +1,10 @@
 import { getProjectRoot } from "./getProjectRoot";
 import { join as pathJoin } from "path";
-import { constants } from "fs";
-import { chmod, stat } from "fs/promises";
+import * as child_process from "child_process";
+import * as fs from "fs";
 
-async () => {
-    var { bin } = await import(pathJoin(getProjectRoot(), "package.json"));
-
-    var promises = Object.values<string>(bin).map(async scriptPath => {
-        const fullPath = pathJoin(getProjectRoot(), scriptPath);
-        const oldMode = (await stat(fullPath)).mode;
-        const newMode = oldMode | constants.S_IXUSR | constants.S_IXGRP | constants.S_IXOTH;
-        await chmod(fullPath, newMode);
-    });
-
-    await Promise.all(promises);
-};
+Object.entries<string>(JSON.parse(fs.readFileSync(pathJoin(getProjectRoot(), "package.json")).toString("utf8"))["bin"]).forEach(([, scriptPath]) =>
+    child_process.execSync(`chmod +x ${scriptPath}`, {
+        "cwd": getProjectRoot()
+    })
+);