jopi-toolkit 3.0.5 → 3.0.10

package/src/jk_fs/jBundler_ifServer.ts

@@ -1,13 +1,15 @@
 // noinspection JSUnusedGlobalSymbols
 
 import fs from "node:fs/promises";
-import fss, {createReadStream} from "node:fs";
+import fss, {createReadStream, createWriteStream} from "node:fs";
 import {fileURLToPath as n_fileURLToPath, pathToFileURL as n_pathToFileURL } from "node:url";
 import {lookup} from "mime-types";
 import {Readable} from "node:stream";
 import path from "node:path";
+import {createHash} from "node:crypto";
 import {isBunJS} from "jopi-toolkit/jk_what";
 import type {DirItem, FileState} from "./common.ts";
+import AdmZip from 'adm-zip';
 
 class WebToNodeReadableStreamAdapter extends Readable {
     private webStreamReader: ReadableStreamDefaultReader<any>;
@@ -170,7 +172,7 @@ export async function writeTextToFile(filePath: string, text: string, createDir:
     await fs.writeFile(filePath, text, {encoding: 'utf8', flag: 'w'});
 }
 
-export function writeTextSyncToFile(filePath: string, text: string, createDir: boolean = true): void {
+export function writeTextToFileSync(filePath: string, text: string, createDir: boolean = true): void {
     if (createDir) {
         try {
             fss.mkdirSync(path.dirname(filePath), {recursive: true});
@@ -183,7 +185,12 @@ export function readTextFromFile(filePath: string): Promise<string> {
     return fs.readFile(filePath, 'utf8');
 }
 
-export function readTextSyncFromFile(filePath: string): string {
+export async function readJsonFromFile<T = any>(filePath: string): Promise<T> {
+    let txt = await fs.readFile(filePath, 'utf8');
+    return JSON.parse(txt) as T;
+}
+
+export function readTextFromFileSync(filePath: string): string {
     return fss.readFileSync(filePath, 'utf8');
 }
 
@@ -287,6 +294,56 @@ export async function listDir(dirPath: string): Promise<DirItem[]> {
     return result;
 }
 
+const MEGA = 1024 * 1024;
+
+async function calcFileHash_bun(filePath: string): Promise<string|undefined> {
+    const file = Bun.file(filePath);
+    if (!await file.exists()) return undefined;
+
+    if (file.size>10 * MEGA) {
+        return calcFileHash_bun_streamed(file)
+    }
+
+    return Bun.hash(await file.arrayBuffer(), 12346).toString();
+}
+
+async function calcFileHash_bun_streamed(file: Bun.BunFile): Promise<string|undefined> {
+    const stream = file.stream();
+    const reader = stream.getReader();
+    const hasher = new Bun.CryptoHasher("sha256");
+
+    try {
+        while (true) {
+            const { done, value } = await reader.read();
+            if (done) break;
+            hasher.update(value);
+        }
+
+        return hasher.digest("hex");
+    } finally {
+        reader.releaseLock();
+    }
+}
+
+function calcFileHash_node(filePath: string): Promise<string|undefined> {
+    if (!isFile(filePath)) return Promise.resolve(undefined);
+
+    return new Promise((resolve, reject) => {
+        const hash = createHash('sha256');
+        const stream = createReadStream(filePath);
+        stream.on('data', (data) => hash.update(data));
+        stream.on('end', () => resolve(hash.digest('hex')));
+        stream.on('error', (error) => reject(error));
+    });
+}
+
+/**
+ * Calculate the hash of a file.
+ * Allows using it for HTTP ETag or another change proof.
+ * This version is optimized to use streams and avoid loading the whole file in memory.
+ */
+export const calcFileHash = isBunJS ? calcFileHash_bun : calcFileHash_node;
+
 /**
  * Convert a simple win32 path to a linux path.
  */
@@ -294,6 +351,114 @@ export function win32ToLinuxPath(filePath: string): string {
     return filePath.replace(/\\/g, '/');
 }
 
+/**
+ * Copy a directory recursively.
+ * Is optimized for large files.
+ */
+export async function copyDirectory(srcDir: string, destDir: string): Promise<void> {
+    if (!await isDirectory(srcDir)) {
+        throw new Error(`Directory doesn't exist : ${srcDir}`);
+    }
+
+    await mkDir(destDir);
+    const entries = await fs.readdir(srcDir, { withFileTypes: true });
+
+    await Promise.all(entries.map(async (entry) => {
+        const srcPath = path.join(srcDir, entry.name);
+        const destPath = path.join(destDir, entry.name);
+
+        if (entry.isDirectory()) {
+            await copyDirectory(srcPath, destPath);
+        } else {
+            try {
+                await copyFile(srcPath, destPath);
+            } catch {
+                console.warn(`jk_fs.copyDirectory - Failed to copy file ${srcPath}`);
+            }
+        }
+    }));
+}
+
+/**
+ * Copy of a file.
+ * Is optimized for large files.
+ */
+export async function copyFile(srcPath: string, destPath: string): Promise<void> {
+    let stat = await getFileStat(srcPath);
+    if (!stat) return;
+
+    // Assert the symlink exist.
+    if (stat.isSymbolicLink()) {
+        const symStat = await fs.lstat(srcPath);
+        if (!symStat) return;
+        if (!symStat.isFile()) return;
+    }
+
+    return new Promise<void>((resolve, reject) => {
+        const readStream = createReadStream(srcPath);
+        const writeStream = createWriteStream(destPath);
+
+        readStream.on('error', reject);
+        writeStream.on('error', reject);
+        writeStream.on('finish', resolve);
+
+        readStream.pipe(writeStream);
+    });
+}
+
+/**
+ * Unzip a .zip file in an optimized way.
+ * Note was using : "@types/unzipper": "^0.10.11"
+ * which has some bug, files/folders was forgottens.
+ * import * as unzipper from "unzipper";
+ */
+/*export async function unzipFile_old(zipFilePath: string, outputDir: string): Promise<void> {
+    if (!await isFile(zipFilePath)) {
+        throw new Error(`File doesn't exist : ${zipFilePath}`);
+    }
+
+    if (!await isDirectory(outputDir)) {
+        await mkDir(outputDir);
+    }
+
+    await createReadStream(zipFilePath)
+        .pipe(unzipper.Extract({path: outputDir}))
+        .promise();
+}*/
+
+/**
+ * Unzip a .zip file in an optimized way.
+ */
+export async function unzipFile(zipFilePath: string, outputDir: string): Promise<void> {
+    if (!await isFile(zipFilePath)) {
+        throw new Error(`File doesn't exist : ${zipFilePath}`);
+    }
+
+    if (!await isDirectory(outputDir)) {
+        await mkDir(outputDir);
+    }
+
+    const zip = new AdmZip(zipFilePath);
+    zip.extractAllTo(outputDir, true);
+}
+
+/**
+ * Create a temporary directory.
+ * Return an object containing the directory path and a cleanup function.
+ */
+export async function createTempDir(prefix: string): Promise<{path: string, remove: ()=>Promise<void>}> {
+    const dirPath = await fs.mkdtemp(prefix);
+
+    return {
+        path: dirPath,
+        remove: async () => {
+            return fs.rm(dirPath, {recursive: true, force: true});
+        }
+    }
+}
+
+//region Node.js functions
+
 export const join = path.join;
 export const resolve = path.resolve;
 export const dirname = path.dirname;
@@ -305,4 +470,6 @@ export const normalize = path.normalize;
 export const basename = path.basename;
 
 export const symlink = fs.symlink;
-export const rename = fs.rename;
+export const rename = fs.rename;
+
+//endregion

package/src/jk_logs/index.ts

@@ -0,0 +1,444 @@
+// noinspection JSUnusedGlobalSymbols
+
+import * as jk_terms from "jopi-toolkit/jk_term";
+import {init} from "./jBundler_ifServer.ts";
+
+//region Common
+
+export interface LogEntry {
+    level: LogLevel;
+    logger: string;
+
+    date: number;
+    title?: string;
+    data?: any;
+
+    timeDif?: number;
+}
+
+export type LogEntryFormater = (entry: LogEntry) => string;
+
+export enum LogLevel {
+    SPAM = 1,
+    INFO = 2,
+    WARN = 3,
+    ERROR = 4,
+    NONE = 10
+}
+
+export type LogCall = string | ((w: LogLevelHandler)=>void);
+
+//endregion
+
+//region Formater
+
+const RED = jk_terms.C_RED;
+const ORANGE = jk_terms.C_ORANGE;
+const GREY = jk_terms.C_GREY;
+const LIGHT_BLUE = jk_terms.C_LIGHT_BLUE;
+const RESET = jk_terms.T_RESET;
+
+export function formatDate1(timeStamp: number): string {
+    const date = new Date(timeStamp);
+    return date.toISOString();
+}
+
+export const formater_simpleJson: LogEntryFormater = (entry: LogEntry) => {
+    return JSON.stringify(entry);
+};
+
+export const formater_dateTypeTitleSourceData: LogEntryFormater = (entry: LogEntry) => {
+    const date = formatDate1(entry.date);
+
+    let json = entry.data ? JSON.stringify(entry.data) : "";
+    const title = (entry.title || "").padEnd(50, " ");
+
+    json = entry.logger + " |>" + json;
+
+    switch (entry.level) {
+        case LogLevel.ERROR:
+            return `${date} - ERROR - ${title}${json}`;
+        case LogLevel.WARN:
+            return `${date} - WARN  - ${title}${json}`;
+        case LogLevel.INFO:
+            return `${date} - INFO  - ${title}${json}`;
+        case LogLevel.SPAM:
+            return `${date} - SPAM  - ${title}${json}`;
+        default:
+            return "";
+    }
+}
+
+export const formater_typeTitleSourceData_colored: LogEntryFormater = (entry: LogEntry) => {
+    let json = entry.data ? JSON.stringify(entry.data) : "";
+    const title = (entry.title || "").padEnd(50, " ");
+
+    json = entry.timeDif === undefined
+        ? `${entry.logger} ${json}` : `${entry.logger} (${entry.timeDif} ms) ${json}`;
+
+    switch (entry.level) {
+        case LogLevel.ERROR:
+            return `${RED}error${RESET} - ${title}${GREY}${json}${RESET}`;
+        case LogLevel.WARN:
+            return `${ORANGE}warn ${RESET} - ${title}${GREY}${json}${RESET}`;
+        case LogLevel.INFO:
+            return `${LIGHT_BLUE}info ${RESET} - ${title}${GREY}${json}${RESET}`;
+        case LogLevel.SPAM:
+            return `${GREY}spam ${RESET} - ${title}${GREY}${json}${RESET}`;
+        default:
+            return "";
+    }
+}
+
+//endregion
+
+//region LogWriter
+
+export interface LogWriter {
+    addEntry(entry: LogEntry): void;
+    addBatch(entries: LogEntry[]): void;
+}
+
+class ConsoleLogWriter implements LogWriter {
+    constructor(private readonly formater: LogEntryFormater = gDefaultFormater) {
+    }
+
+    addEntry(entry: LogEntry): void {
+        console.log(this.formater(entry));
+    }
+
+    addBatch(entries: LogEntry[]) {
+        entries.forEach(e => this.addEntry(e));
+    }
+}
+
+export class VoidLogWriter implements LogWriter {
+    addBatch(_entries: LogEntry[]): void {
+    }
+
+    addEntry(_entry: LogEntry): void {
+    }
+}
+
+export function setDefaultWriter(writer: LogWriter) {
+    gDefaultWriter = writer;
+}
+
+export function getDefaultWriter(): LogWriter {
+    return gDefaultWriter;
+}
+
+export function setDefaultFormater(formater: LogEntryFormater) {
+    gDefaultFormater = formater;
+}
+
+export function getDefaultFormater(): LogEntryFormater {
+    return gDefaultFormater;
+}
+
+let gDefaultFormater: LogEntryFormater = formater_typeTitleSourceData_colored;
+let gDefaultWriter: LogWriter = new ConsoleLogWriter();
+
+//endregion
+
+//region JopiLogger
+
+export interface Logger {
+    get fullName(): string;
+
+    spam(_l?: LogCall): boolean;
+    info(_l?: LogCall): boolean;
+    warn(_l?: LogCall): boolean;
+    error(_l?: LogCall): boolean
+
+    beginSpam(l: LogCall): LoggerGroupCallback;
+    beginInfo(l: LogCall): LoggerGroupCallback;
+}
+
+export function getLogger(name: string, parent?: Logger): Logger {
+    let fullName = name;
+    if (parent) fullName = parent.fullName + '.' + name;
+
+    let level = getLogLevelFor(fullName);
+
+    switch (level) {
+        case LogLevel.SPAM:
+            return new Logger_Spam(parent as JopiLogger, name);
+        case LogLevel.INFO:
+            return new Logger_Info(parent as JopiLogger, name);
+        case LogLevel.WARN:
+            return new Logger_Warn(parent as JopiLogger, name);
+        case LogLevel.ERROR:
+            return new Logger_Error(parent as JopiLogger, name);
+    }
+
+    return new Logger_None(parent as JopiLogger, name);
+}
+
+abstract class JopiLogger implements Logger {
+    public readonly fullName: string;
+    private _onLog: LogWriter = gDefaultWriter;
+
+    protected readonly hSpam: LogLevelHandler;
+    protected readonly hInfo: LogLevelHandler;
+    protected readonly hWarn: LogLevelHandler;
+    protected readonly hError: LogLevelHandler;
+
+    private timeDif?: number;
+    private extraData?: any;
+
+    constructor(parent: JopiLogger|null, public readonly name: string) {
+        this.fullName = parent ? parent.fullName + '.' + name : name;
+
+        if (parent) {
+            this._onLog = parent._onLog;
+        }
+
+        const me = this;
+
+        this.hSpam = (title?: string, data?: any) => {
+            let td = this.timeDif;
+            this.timeDif = undefined;
+            data = this.mergeData(data);
+
+            me._onLog.addEntry({
+                level: LogLevel.SPAM,
+                logger: me.fullName, date: Date.now(), title, data, timeDif: td });
+        };
+
+        this.hInfo = (title?: string, data?: any) => {
+            let td = this.timeDif;
+            this.timeDif = undefined;
+            data = this.mergeData(data);
+
+            me._onLog.addEntry({
+                level: LogLevel.INFO,
+                logger: me.fullName, date: Date.now(), title, data, timeDif: td });
+        };
+
+        this.hWarn = (title?: string, data?: any) => {
+            let td = this.timeDif;
+            this.timeDif = undefined;
+            data = this.mergeData(data);
+
+            me._onLog.addEntry({
+                level: LogLevel.WARN,
+                logger: me.fullName, date: Date.now(), title, data, timeDif: td });
+        };
+
+        this.hError = (title?: string, data?: any) => {
+            let td = this.timeDif;
+            this.timeDif = undefined;
+            data = this.mergeData(data);
+
+            me._onLog.addEntry({
+                level: LogLevel.ERROR,
+                logger: me.fullName, date: Date.now(), title, data, timeDif: td });
+        };
+    }
+
+    private mergeData(data?: any): any|undefined {
+        if (!this.extraData) return data;
+        const extraData = this.extraData;
+        this.extraData = undefined;
+
+        if (!data) return extraData;
+        for (let p in extraData) data[p] = extraData[p];
+
+        return data;
+    }
+
+    setLogWriter(callback: LogWriter) {
+        if (!callback) callback = gDefaultWriter;
+        this._onLog = callback;
+    }
+
+    spam(_l?: (w: LogLevelHandler)=>void): boolean {
+        return false;
+    }
+
+    info(_l?: (w: LogLevelHandler)=>void): boolean {
+        return false;
+    }
+
+    warn(_l?: (w: LogLevelHandler)=>void) {
+        return false;
+    }
+
+    error(_l?: (w: LogLevelHandler)=>void) {
+        return false;
+    }
+
+    beginSpam(l: (w: LogLevelHandler)=>void): LoggerGroupCallback {
+        return gVoidLoggerGroupCallback;
+    }
+
+    beginInfo(l: (w: LogLevelHandler)=>void): LoggerGroupCallback {
+        return gVoidLoggerGroupCallback;
+    }
+
+    protected doBegin(l: LogCall, w: LogLevelHandler): LoggerGroupCallback {
+        const startTime = Date.now();
+
+        return (data?: any) => {
+            this.timeDif = Date.now() - startTime;
+            this.doCall(l, w, data);
+        }
+    }
+
+    protected doCall(l: LogCall|undefined, w: LogLevelHandler, data?: any) {
+        this.extraData = data;
+
+        if (l) {
+            if (l instanceof Function) {
+                l(w);
+            }
+            else {
+                w(l as string);
+            }
+        }
+
+        return true;
+    }
+}
+
+class Logger_None extends JopiLogger {
+}
+
+class Logger_Spam extends JopiLogger {
+    override spam(l?: LogCall) {
+        return this.doCall(l, this.hSpam);
+    }
+
+    override info(l?: LogCall) {
+        return this.doCall(l, this.hInfo);
+    }
+
+    override warn(l?: LogCall) {
+        return this.doCall(l, this.hWarn);
+    }
+
+    override error(l?: LogCall) {
+        return this.doCall(l, this.hError);
+    }
+
+    override beginSpam(l: LogCall) {
+        return this.doBegin(l, this.hSpam);
+    }
+
+    override beginInfo(l: LogCall): LoggerGroupCallback {
+        return this.doBegin(l, this.hInfo);
+    }
+}
+
+class Logger_Info extends JopiLogger {
+    override info(l?: LogCall) {
+        return this.doCall(l, this.hInfo);
+    }
+
+    override warn(l?: LogCall) {
+        return this.doCall(l, this.hWarn);
+    }
+
+    override error(l?: LogCall) {
+        return this.doCall(l, this.hError);
+    }
+
+    override beginInfo(l: LogCall): LoggerGroupCallback {
+        return this.doBegin(l, this.hInfo);
+    }
+}
+
+class Logger_Warn extends JopiLogger {
+    override warn(l?: LogCall) {
+        return this.doCall(l, this.hWarn);
+    }
+
+    override error(l?: LogCall) {
+        return this.doCall(l, this.hError);
+    }
+}
+
+class Logger_Error extends JopiLogger {
+    override error(l?: LogCall) {
+        return this.doCall(l, this.hError);
+    }
+}
+
+export type LoggerGroupCallback = (data?: any) => void;
+const gVoidLoggerGroupCallback = () => {};
+
+//endregion
+
+//region Log levels
+
+type LogLevelHandler = (title?: string, data?: any|undefined)=>void;
+
+function getLogLevelName(level: LogLevel) {
+    switch (level) {
+        case LogLevel.SPAM:
+            return "SPAM";
+        case LogLevel.ERROR:
+            return "ERROR";
+        case LogLevel.INFO:
+            return "INFO";
+        case LogLevel.WARN:
+            return "WARN";
+        case LogLevel.NONE:
+            return "NONE";
+    }
+}
+
+function getLogLevelByName(name: string): LogLevel | undefined {
+    switch (name) {
+        case "NONE": return LogLevel.NONE;
+        case "SPAM": return LogLevel.SPAM;
+        case "INFO": return LogLevel.INFO;
+        case "WARN": return LogLevel.WARN;
+        case "ERROR": return LogLevel.ERROR;
+    }
+
+    return undefined;
+}
+
+//endregion
+
+//region Registry
+
+export interface LogConfig {
+    level?: string;
+    writer?: string;
+    formater?: string;
+}
+
+export interface LogInitializer {
+    setLogLevel(name: string, config: LogConfig): void;
+}
+
+class Initializer implements LogInitializer {
+    setLogLevel(name: string, config: LogConfig): void {
+        let logLevel = getLogLevelByName(config.level || "NONE");
+        if (!logLevel) logLevel = LogLevel.NONE;
+        gRegistry[name] = logLevel;
+    }
+}
+
+function getLogLevelFor(name: string) {
+    let entry = gRegistry[name];
+    if (entry!==undefined) return entry;
+
+    for (let prefix in gRegistry) {
+        if (name.startsWith(prefix + ".")) {
+            return gRegistry[prefix];
+        }
+    }
+
+    return gDefaultLogLevel;
+}
+
+const gDefaultLogLevel: LogLevel = LogLevel.WARN;
+const gRegistry: Record<string, LogLevel> = {};
+
+//endregion
+
+init(new Initializer());

package/src/jk_logs/jBundler_ifBrowser.ts

@@ -0,0 +1,2 @@
+export function init() {
+}

package/src/jk_logs/jBundler_ifServer.ts

@@ -0,0 +1,27 @@
+import * as jk_app from "jopi-toolkit/jk_app";
+import * as jk_fs from "jopi-toolkit/jk_fs";
+import type {LogInitializer} from "./index.ts";
+
+export function init(init: LogInitializer) {
+    const mainDir = jk_app.findPackageJsonDir();
+    const filePath = jk_fs.join(mainDir, "logConfig.json");
+
+    if (!jk_fs.isFileSync(filePath)) return;
+    let text = jk_fs.readTextFromFileSync(jk_fs.join(mainDir, "logConfig.json"));
+
+    if (!text) return;
+    let logJson = JSON.parse(text)
+    if (!logJson.config) return;
+
+    for (let logName in logJson.config) {
+        let logConfig = logJson.config[logName];
+
+        if (typeof(logConfig)==="string") {
+            logConfig = {level: logConfig.toUpperCase().trim()};
+        } else if (!logConfig) {
+            logConfig = {level: "none" };
+        }
+
+        init.setLogLevel(logName, logConfig);
+    }
+}

package/src/jk_thread/common.js

@@ -0,0 +1,7 @@
+export var gIsUsingWorkers = false;
+export function declareUsingWorker() {
+    gIsUsingWorkers = true;
+}
+export function isUsingWorker() {
+    return gIsUsingWorkers;
+}