@entity-access/server-pages 1.1.620 → 1.1.623

package/src/cache/BaseDiskCache.ts

@@ -0,0 +1,330 @@
+/* eslint-disable no-console */
+import fsp, { opendir, rm, rmdir, stat, unlink } from "node:fs/promises";
+import { existsSync } from "node:fs";
+import { join, parse } from "node:path";
+import { randomUUID } from "node:crypto";
+import EntityAccessError from "@entity-access/entity-access/dist/common/EntityAccessError.js";
+import ensureDir from "../core/FileApi.js";
+import TempFolder from "../core/TempFolder.js";
+import { LocalFile } from "../core/LocalFile.js";
+import { spawnPromise } from "./spawnPromise.js";
+import LockFile from "./LockFile.js";
+import sleep from "../sleep.js";
+import RunOnce from "../core/RunOnce.js";
+import { toKMBString } from "../core/NumberFormats.js";
+
+const doNothing = () => void 0;
+
+export interface IDiskCacheContainer {
+    cache: BaseDiskCache;
+}
+
+export default class BaseDiskCache {
+
+    protected readonly root: string;
+    protected readonly keepTTLSeconds: number;
+    protected readonly minSize: number;
+    protected readonly updateAccessTime: boolean;
+    protected readonly maxAge: number;
+    protected readonly minAge: number;
+    constructor(
+        {
+            root,
+            keepTTLSeconds = 3600,
+            minSize = Number.MAX_SAFE_INTEGER,
+            updateAccessTime = true,
+            maxAge = 1,
+            minAge = 1
+        }: {
+            root: string;
+            keepTTLSeconds?: number;
+            minSize?: number;
+            updateAccessTime?: boolean;
+            maxAge?: number;
+            minAge?: number;
+        }
+    ) {
+        this.root = root;
+        this.keepTTLSeconds = keepTTLSeconds;
+        this.minSize = minSize;
+        this.updateAccessTime = updateAccessTime;
+        this.maxAge = maxAge;
+        this.minAge = minAge;
+        ensureDir(root);
+        // eslint-disable-next-line no-console
+        setTimeout(() => this.clean().catch(console.error), 1000);
+    }
+
+    newFolder(suffix = "") {
+        return new TempFolder(suffix, this.root);
+    }
+
+    async get(path: string) {
+        path = join(this.root, path);
+        if (existsSync(path)) {
+            if(this.updateAccessTime) {
+                const now = new Date();
+                await fsp.utimes(path, now, now);
+            }
+            return new LocalFile(path, void 0, void 0, doNothing);
+        }
+    }
+
+    async clear() {
+        try {
+            const path = this.root;
+            using _lock = await LockFile.lock(`df-clear:${path}`);
+            if (!existsSync(path)) {
+                return;
+            }
+            const stalePath = `${path}.old.${Date.now()}`;
+            await spawnPromise("mv",[path, stalePath]);
+            ensureDir(this.root);
+            rm(stalePath, { recursive: true, force: true}).catch(console.error);
+        } catch (error) {
+            console.error(error);
+        }
+    }
+
+    async clearFolder(path: string) {
+        try {
+            using _lock = await LockFile.lock(`df-clear:${path}`);
+            path = join(this.root, path);
+            if (!existsSync(path)) {
+                return;
+            }
+            const stalePath = `${path}.old.${Date.now()}`;
+            await spawnPromise("mv",[path, stalePath]);
+            rm(stalePath, { recursive: true, force: true}).catch(console.error);
+        } catch (error) {
+            console.error(error);
+        }
+    }
+
+    async getOrCreateJsonAsync<T>(path: string, factory: () => Promise<T>) {
+        const localFile = await this.getOrCreateAsync(path, async (lf) => {
+            const data = (await factory()) ?? null;
+            await lf.writeAllText(JSON.stringify(data));
+        });
+        const text = await localFile.readAsText();
+        return JSON.parse(text) as T;
+    }
+
+    deleteAt(path: string) {
+        path = join(this.root, path);
+        return unlink(path);
+    }
+
+    async getOrCreateAsync(path: string, factory: (fx: LocalFile) => Promise<void>, ext = ".dat") {
+        path = join(this.root, path);
+
+        const parsedPath = parse(path);
+        ensureDir(parsedPath.dir);
+
+        let error: Error;
+
+        for (let index = 0; index < 5; index++) {
+            if (existsSync(path)) {
+                if(this.updateAccessTime) {
+                    const now = new Date();
+                    await fsp.utimes(path, now, now);
+                }
+                return new LocalFile(path, void 0, void 0, doNothing);
+            }
+
+            using _lock = await LockFile.lock(`df:${path}`);
+
+            if (existsSync(path)) {
+                return new LocalFile(path, void 0, void 0, doNothing);
+            }
+
+            const tmpPath = join(this.root, randomUUID() + (parsedPath.ext || ext));
+
+            await factory(new LocalFile(tmpPath, void 0, void 0, doNothing));
+
+            try {
+                ensureDir(parsedPath.dir);
+                await fsp.rename(tmpPath, path);
+            } catch (e) {
+
+                if (existsSync(tmpPath)) {
+                    unlink(tmpPath).catch(doNothing);
+                }
+
+                error = e;
+                await sleep(1000);
+                continue;
+            }
+            return new LocalFile(path, void 0, void 0, doNothing);
+        }
+
+        throw new EntityAccessError(`Failed to write file due to error ${error.stack ?? error}`);
+    }
+
+    createTempFileDeleteOnExit(pathFragments: string[], name: string, contentType: string) {
+        const fileName = pathFragments.pop();
+        let folder = void 0;
+        if (pathFragments.length) {
+            folder = join(this.root, ... pathFragments);
+            ensureDir(folder);
+        }
+        const path = join(this.root, ... pathFragments, fileName);
+        return new LocalFile(path, name, contentType, () => unlink(path).then(() => folder ? rmdir(folder).catch(console.error) : void 0 , console.error));
+    }
+
+    protected async deleteFile(path: string) {
+        if (!path.startsWith(this.root)) {
+            return;
+        }
+        if (existsSync(path)) {
+            await unlink(path);
+        }
+        for(;;) {
+            const parsed = parse(path);
+            if (parsed.dir === this.root) {
+                break;
+            }
+            path = parsed.dir;
+            try {
+                if (existsSync(path)) {
+                    // check if folder is empty...
+                    if(await this.isEmptyDir(path)) {
+                        await rmdir(path);
+                    }
+                }
+            } catch (error) {
+                console.error(error);
+                return;
+            }
+        }
+    }
+
+    protected async isEmptyDir(path) {
+        try {
+            const directory = await opendir(path);
+            const entry = await directory.read();
+            await directory.close();
+            return entry === null; // It's empty if the first entry read is null
+        } catch (error) {
+            // Catches errors like 'ENOENT' (directory doesn't exist)
+            // and treats the path as effectively "empty" for the purpose of the check.
+            // Adjust error handling as needed for your specific use case.
+            return true;
+        }
+    }
+
+    protected async clean() {
+
+        if (!await RunOnce.canRun(this.root)) {
+            setTimeout(() => this.clean().catch(console.error), 60000);
+            return;
+        }
+
+        const start = Date.now();
+        let total = 0;
+
+        const min = this.minAge;
+
+        let all = null as { time, path, size  }[];
+        let freeSize = 0;
+        let deleted = 0;
+
+        for(let i=this.maxAge;i>= min;i--) {
+            const s = await fsp.statfs(this.root);
+            freeSize = s.bavail * s.bsize;
+
+            if (freeSize >= this.minSize) {
+                break;
+            }
+            all ??= await this.getFileStats();
+            try {
+                const keep = Date.now() - this.keepTTLSeconds * 1000 * i;
+                const pending = [];
+                for (const file of all) {
+                    if (file.time < keep) {
+                        await this.deleteFile(file.path);
+                        deleted++;
+                        total += file.size;
+                        continue;
+                    }
+                    pending.push(file);
+                }
+                all = pending;
+                if(!all.length) {
+                    break;
+                }
+            } catch (error) {
+                console.error(error);
+            }
+        }
+
+        if (total) {
+            console.log(`${this.root} (${deleted}/${all.length + deleted}) cleaned, ${toKMBString(total)} freed in ${Date.now()-start}ms.`);
+        } else {
+            if (all?.length) {
+                if (this.minSize === Number.MAX_SAFE_INTEGER) {
+                    console.log(`Cleaning ${this.root} with entries (${all.length}) for ${Date.now()-start}ms.`);
+                } else {
+                    console.log(`Cleaning ${this.root} with entries (${all.length}) for ${Date.now()-start}ms as ${toKMBString(freeSize)} < ${toKMBString(this.minSize)}.`);
+                }
+            }
+        }
+
+        setTimeout(() => this.clean().catch(console.error), 60000);
+
+    }
+
+    private async getFileStats() {
+        const min = Date.now() - this.minAge * this.keepTTLSeconds * 1000;
+        const files = [] as { path: string, size: number, time: number }[];
+        const dir = await fsp.opendir(this.root, { recursive: true });
+        for await (const entry of dir) {
+            if (!entry.isFile()) {
+                continue;
+            }
+            const f = join(entry.parentPath, entry.name);
+            const s = await stat(f);
+            const time = s.ctimeMs;
+            if (time > min) {
+                continue;
+            }
+            files.push({ path: f, size: s.size, time });
+        }
+        return files;
+    }
+
+    // private async getFilesToDelete(oldest: number) {
+    //     const dir = await fsp.opendir(this.root, { recursive: true });
+    //     const filesToDelete = [] as { path: string, statInfo: Stats }[];
+    //     try {
+    //         for await (const entry of dir) {
+    //             if (!entry.isFile()) {
+    //                 continue;
+    //             }
+    //             const path = join(entry.parentPath, entry.name);
+    //             try {
+    //                 const statInfo = await stat(path);
+    //                 if (statInfo.ctimeMs < oldest) {
+    //                     filesToDelete.push({ path, statInfo });
+    //                     if (filesToDelete.length === 1000) {
+    //                         break;
+    //                     }
+    //                 }
+    //             } catch (error) {
+    //                 // file may not exist anymore...
+    //             }
+    //         }
+    //     } catch (error) {
+    //         console.error(error);
+    //     } finally {
+    //         try {
+    //             await dir.close();
+    //         } catch {
+    //             // do nothing
+    //         }
+    //     }
+    //     return filesToDelete;
+    // }
+
+
+}

package/src/cache/LockFile.ts

@@ -0,0 +1,101 @@
+/* eslint-disable no-console */
+import { join } from "node:path";
+import { existsSync, mkdirSync, unlinkSync, statSync, utimesSync, writeFileSync } from "node:fs";
+import { createHash } from "node:crypto";
+import sleep from "../sleep.js";
+import TimeoutError from "../core/TimeoutError.js";
+
+const lockFolder = join("/tmp", "locks");
+
+if (!existsSync(lockFolder)) {
+    mkdirSync(lockFolder, { recursive: true });
+}
+
+const seconds = 1000;
+const minutes = 60*seconds;
+const hours = 60*minutes;
+
+export default class LockFile implements Disposable {
+
+
+    static async lock(someKey: string, timeout = 2 * hours, interval = 3000, throwOnFail = true) {
+        const sha256 = createHash("sha256");
+        const hash = sha256.update(someKey).digest("hex");
+
+        const lockFile = join(lockFolder, hash + ".lock");
+
+        let now = Date.now();
+        const till = now + timeout;
+
+        let lastError = null as Error;
+
+        do {
+
+            try {
+                // check if it exists..
+                if (!existsSync(lockFile)) {
+                    // create and return..
+                    return new LockFile(lockFile);
+                }
+
+                const stat = statSync(lockFile);
+                const past = now - 15000;
+                if(stat.mtimeMs < past) {
+                    // it is dead...
+                    unlinkSync(lockFile);
+                }
+            } catch (error){
+                // do nothing
+                lastError = error;
+                // console.error(error);
+            }
+            await sleep(interval);
+            now = Date.now();
+        }while(now < till);
+
+        if (throwOnFail) {
+            lastError ??= new TimeoutError("lock timed out");
+            throw new Error(`Could not acquire lock ${lockFile}, Reason: ${lastError?.stack ?? lastError?.toString()}`);
+        }
+
+        return {
+
+            locked: false,
+            timer: void 0,
+
+            [Symbol.dispose]() {
+                // do nothing..
+            }
+        };
+    }
+
+    timer: any;
+    public locked: any;
+
+    constructor(private readonly lockFile) {
+
+        this.locked = true;
+
+        writeFileSync(lockFile, "locked", {  flag :"wx", flush: true });
+
+        this.timer = setInterval(() => {
+            try {
+                const now = new Date();
+                utimesSync(lockFile, now, now);
+            } catch {
+                // do nothing
+            }
+        }, 1000);
+    }
+
+    [Symbol.dispose]() {
+        try {
+            clearInterval(this.timer);
+            unlinkSync(this.lockFile);
+        } catch (error) {
+            // ignore error...
+            console.error(error);
+        }
+    }
+
+}

package/src/cache/spawnPromise.ts

@@ -0,0 +1,69 @@
+/* eslint-disable no-console */
+import { SpawnOptionsWithoutStdio, spawn } from "node:child_process";
+
+import { color } from "console-log-colors";
+
+export const spawnPromise = (path, args?: string[], options?: SpawnOptionsWithoutStdio & { logCommand?: boolean, logData?: boolean, logError?: boolean, throwOnFail?: boolean }) => new Promise<{
+    get all(): string;
+    pid: number;
+    status: number;
+}>((resolve, reject) => {
+    const all = [];
+    const { logCommand = true, throwOnFail = false, logData = true, logError = true } = options ??= {};
+    options.timeout ??= 5*60*1000;
+    const cd = spawn(path, args, options);
+    const pid = cd.pid;
+
+    const clear = () => {
+        try {
+            cd.stderr.removeAllListeners();
+            cd.stdout.removeAllListeners();
+            cd.removeAllListeners();
+        } catch {
+            // do nothing...
+        }
+    };
+
+    cd.stdout.on("data", (data) => {
+        data = data.toString("utf-8");
+        all.push(data);
+    });
+    cd.stderr.on("data", (data) => {
+        data = data.toString("utf-8");
+        all.push(color.red(data));
+    });
+
+    cd.on("error", (error) => {
+        const errorText = color.red(error.stack ?? error.toString());
+        all.push(error.stack ?? error.toString());
+        if (logData || logError) {
+            console.error(errorText);
+        }
+        reject(error);
+    });
+    cd.on("close", (status) => {
+        if (status>0) {
+            if (logError) {
+                console.error(all);
+            }
+            if (throwOnFail) {
+                clear();
+                reject(new Error(all.join("\n")));
+                return;
+            }
+        }
+        if (logCommand) {
+            console.log(`Spawn: ${path} ${JSON.stringify(args, undefined, 2)}`);
+            if (logData) {
+                console.log(all.join("\n"));
+            }
+        }
+        clear();
+        resolve({
+            get all() {
+                return all.join("\n");
+            },
+            pid,
+            status });
+    });
+});

package/src/core/FileApi.ts

@@ -2,6 +2,7 @@ import { existsSync, unlinkSync, mkdirSync } from "fs";
 
 import { dirname } from "path";
 import { AsyncFileStream } from "./AsyncStream.js";
+import { stat } from "fs/promises";
 
 export function ensureParentFolder(filePath: string) {
     ensureDir(dirname(filePath));
@@ -23,6 +24,17 @@ export default function ensureDir(folder: string) {
 
 }
 
+export const fileInfo = async (filePath: string) => {
+    try {
+        return await stat(filePath);
+    } catch (error) {
+        if (error.code === "ENOENT") {
+            return null;
+        }
+        throw error;
+    }
+};
+
 export const deleteIfExists = (path) => existsSync(path) ? unlinkSync(path) : void 0;
 
 export const areFilesEqual = async (file1: string, file2: string) => {

package/src/core/NumberFormats.ts

@@ -0,0 +1,78 @@
+const k = 1000;
+const m = k*k;
+const b = m*k;
+
+const kb = 1024;
+const mb = kb*kb;
+const gb = mb*kb;
+const tb = gb*kb;
+const pb = tb*kb;
+
+export const toCompactFixed = (n: number, decimals: number) => {
+    if (decimals > 0) {
+        if (n>0) {
+            return (+(n).toFixed(decimals)).toString();
+        }
+    }
+    return n.toFixed();
+};
+
+export const toKMBString = (n: number, decimals = 0,  def = "") => {
+    if (!n) {
+        return def + "";
+    }
+    if (n >= b) {
+        return toCompactFixed((n/b), decimals) + "B";
+    }
+    if (n >= m) {
+        return toCompactFixed((n / m), decimals) + "M";
+    }
+    if (n >= k) {
+        return toCompactFixed((n / k), decimals) + "K";
+    }
+    return toCompactFixed(n, 0);
+};
+
+export const toFileSize = (n: number, decimals = 0,  def = "") => {
+    n = Number(n);
+    if (n === 0) {
+        return "0 B";
+    }
+    if (!n) {
+        return def + "";
+    }
+    if (n >= pb) {
+        return toCompactFixed((n/pb), decimals) + " PB";
+    }
+    if (n >= tb) {
+        return toCompactFixed((n/tb), decimals) + " TB";
+    }
+    if (n >= gb) {
+        return toCompactFixed((n/gb), decimals) + " GB";
+    }
+    if (n >= mb) {
+        return toCompactFixed((n / mb), decimals) + " MB";
+    }
+    if (n >= kb) {
+        return toCompactFixed((n / kb), decimals) + " KB";
+    }
+    return toCompactFixed(n, 0);
+};
+
+export interface IFormat {
+    prefix: string;
+    suffix: string;
+    numberToText?: (n: number) => string;
+}
+
+export const formatNumber = (n: number, {
+    prefix = "",
+    suffix = "",
+    def = "",
+    numberToText = toKMBString} = {}) => {
+    if (!n) {
+        return def;
+    }
+    const text = numberToText(n);
+    return prefix + text + suffix;
+};

package/src/core/RunOnce.ts

@@ -0,0 +1,30 @@
+import { createHash } from "node:crypto";
+import { utimes, writeFile } from "node:fs/promises";
+import { join } from "node:path";
+import { fileInfo } from "./FileApi.js";
+
+export default class RunOnce {
+
+    static async canRun(key: string, maxAge = 60*1000) {
+
+        const sha256 = createHash("sha256");
+        const hash = sha256.update(`run-once-file-${key}`).digest("hex");
+        const lockFile = join("/tmp", hash + ".last-run");
+
+        const info = await fileInfo(lockFile);
+        if (!info) {
+            await writeFile(lockFile, "running");
+            return true;
+        }
+
+        const past = Date.now() - maxAge;
+        if (info.mtimeMs > past) {
+            return false;
+        }
+        await writeFile(lockFile, "running");
+        const now = new Date();
+        await utimes(lockFile, now, now);
+        return true;
+    }
+
+}

package/src/core/TimeoutError.ts

@@ -0,0 +1,3 @@
+export default class TimeoutError extends Error {
+
+}

package/src/services/EntityAccessServer.ts

@@ -163,6 +163,9 @@ export default class EntityAccessServer {
 
         let items;
 
+        if (trace) {
+            q = q.trace(console.log);
+        }
 
         if (count) {
             const total = await oq.slice().count();
@@ -182,9 +185,6 @@ export default class EntityAccessServer {
                 .some();
         }
 
-        if (trace) {
-            q = q.trace(console.log);
-        }
         items = await q.toArray();
         return DbJsonService.toJson(db, {
             total: 0,