datafolder 1.0.9 → 1.0.11

package/src/QyLogFileSaver_Worker.js

@@ -1,5 +1,5 @@
 import {
-    getDefaultOptions as s
+    getDefaultOptions as r
 } from "./QyDefaultOptions.js";
 
 import {
@@ -7,16 +7,17 @@ import {
 } from "./QyMessageWorker.js";
 
 import {
-    QyBinWriter as r
+    QyBinWriter as s
 } from "./QyBinWriter.js";
 
 export default class extends t {
+    qyBinWriter;
     constructor(t) {
         super(t = {
-            ...s("QyLogFileSaver_Worker"),
+            ...r("QyLogFileSaver_Worker"),
             ...t
         }), Object.assign(this, {
-            qyBinWriter: new r(t)
+            qyBinWriter: new s(t)
         });
     }
     async _op_start(t) {
@@ -28,7 +29,7 @@ export default class extends t {
     async switch(t) {
         await this.qyBinWriter.switch(t);
     }
-    save(t, s) {
-        this.qyBinWriter.save(t, s);
+    save(t, r) {
+        this.qyBinWriter.save(t, r);
     }
 }

package/src/QyLogger.d.ts

@@ -1,10 +1,21 @@
-export declare const logger: {
-    currentLogLevel: number;
-    info: (...args: any[]) => void;
-    log: (...args: any[]) => void;
-    debug: (...args: any[]) => void;
-    warn: (...args: any[]) => void;
-    error: (...args: any[]) => void;
-};
-export declare function closeFileConsole(): void;
-export declare function setFileConsoleDateDirPath(dateDirPath: any, clearLogAtStart: any): void;
+export declare const ThreadMark: string;
+declare class QyLogger {
+    _level: number;
+    fileConsole: any;
+    stdout: any;
+    stderr: any;
+    constructor();
+    get level(): string;
+    set level(val: string);
+    info(...args: any[]): void;
+    log(...args: any[]): void;
+    debug(...args: any[]): void;
+    warn(...args: any[]): void;
+    error(...args: any[]): void;
+    handleOutMsg(msg: any): void;
+    handleErrMsg(msg: any): void;
+    closeFileConsole(): void;
+    setFileConsoleDateDirPath(dateDirPath: string, clearLogAtStart?: boolean): void;
+}
+export declare const logger: QyLogger;
+export {};

package/src/QyLogger.js

@@ -1,87 +1,131 @@
 import o from "node:fs";
 
-import n from "node:path";
+import r from "node:path";
 
 import {
-    Console as t
+    Console as i
 } from "node:console";
 
 import {
-    isMainThread as e,
-    threadId as r
+    parentPort as n,
+    isMainThread as l,
+    threadId as e
 } from "node:worker_threads";
 
-let i = e ? "" : `W${r}:`, u = {
-    Info: 0,
-    Debug: 1,
-    Warn: 2,
-    Error: 3,
-    None: 4
-}, c = {
-    verbose: u.Info,
-    info: u.Info,
-    log: u.Debug,
-    debug: u.Debug,
-    warn: u.Warn,
-    error: u.Error,
-    none: u.None
-}, s = Object.fromEntries(Object.entries(c).map(([ e, r ]) => [ r, e ])), l = {
-    currentLogLevel: u.Debug,
-    info: function(...e) {
-        l.currentLogLevel == u.Info && b("info", e);
-    },
-    log: function(...e) {
-        l.currentLogLevel <= u.Debug && b("log", e);
-    },
-    debug: function(...e) {
-        l.currentLogLevel <= u.Debug && b("debug", e);
-    },
-    warn: function(...e) {
-        l.currentLogLevel <= u.Warn && b("warn", e);
-    },
-    error: function(...e) {
-        l.currentLogLevel <= u.Error && b("error", e);
+import {
+    Writable as t
+} from "node:stream";
+
+let a = l ? "" : `W${e}:`, d = {
+    info: 1,
+    debug: 2,
+    log: 3,
+    warn: 4,
+    error: 5,
+    none: 6
+}, s = Object.fromEntries(Object.entries(d).map(([ e, t ]) => [ t, e ]));
+
+class h {
+    _level = d.debug;
+    fileConsole;
+    stdout;
+    stderr;
+    constructor() {
+        var e, t;
+        l || (e = this.stdout = p("%o_"), t = this.stderr = p("%e_"), this.fileConsole = new i({
+            stdout: e,
+            stderr: t,
+            inspectOptions: {
+                depth: 3,
+                colors: !1
+            }
+        }));
+    }
+    get level() {
+        return s[this._level];
+    }
+    set level(e) {
+        this._level = d[e];
+    }
+    info(...e) {
+        c(this, "info", e);
+    }
+    log(...e) {
+        c(this, "log", e);
+    }
+    debug(...e) {
+        c(this, "debug", e);
+    }
+    warn(...e) {
+        c(this, "warn", e);
+    }
+    error(...e) {
+        c(this, "error", e);
     }
-}, a, g, f;
+    handleOutMsg(e) {
+        l ? u(this.stdout, e) : n?.postMessage(e);
+    }
+    handleErrMsg(e) {
+        l ? u(this.stderr, e) : n?.postMessage(e);
+    }
+    closeFileConsole() {
+        l && (this.fileConsole = void 0, this.stdout && (this.stdout.close(), this.stdout = void 0), 
+        this.stderr) && (this.stderr.close(), this.stderr = void 0);
+    }
+    setFileConsoleDateDirPath(e, t = !1) {
+        var s;
+        l && (this.closeFileConsole(), t = {
+            flags: t ? "w" : "a",
+            flush: !0,
+            autoClose: !1
+        }, s = this.stdout = o.createWriteStream(r.join(e, "stdout.txt"), t), e = this.stderr = o.createWriteStream(r.join(e, "stderr.txt"), t), 
+        this.fileConsole = new i({
+            stdout: s,
+            stderr: e,
+            inspectOptions: {
+                depth: 3,
+                colors: !1
+            }
+        }));
+    }
+}
 
-function d() {
-    a = void 0, g && (g.close(), g = void 0), f && (f.close(), f = void 0);
+function u(e, t) {
+    var s, o;
+    e && ({
+        chunks: t,
+        chunk: s,
+        encoding: o
+    } = t, t ? e.writev(t) : e.write(s, o));
 }
 
-function L(e, r) {
-    d();
-    r = {
-        flags: r ? "w" : "a",
-        flush: !0,
-        autoClose: !1
-    };
-    g = o.createWriteStream(n.join(e, "stdout.txt"), r), f = o.createWriteStream(n.join(e, "stderr.txt"), r), 
-    a = new t({
-        stdout: g,
-        stderr: f,
-        inspectOptions: {
-            depth: 3,
-            colors: !1
-        }
-    });
+function c(e, t, s) {
+    var o;
+    e._level <= d[t] && (o = ((o = (o = new Error().stack?.split("\n")) && /([^/\\()]+)\)?$/.exec(o[3] ?? "")) && o[1]) ?? "", 
+    o = `${new Date().toISOString()} ${a}${o}:${t}:`, console[t](o, ...s), e.fileConsole) && e.fileConsole[t](o, ...s);
 }
 
-function b(e, r) {
-    var o = new Error().stack?.split("\n"), o = o && /([^/\\()]+)\)?$/.exec(o[3] ?? ""), o = (o && o[1]) ?? "", o = `${new Date().toISOString()} ${i}${o}:${e}:`;
-    console[e](o, ...r), a && a[e](o, ...r);
+function p(o) {
+    return new t({
+        write(e, t, s) {
+            return n?.postMessage({
+                opName: o,
+                chunk: e,
+                encoding: t
+            }), s && s(), !0;
+        },
+        writev(e, t) {
+            return n?.postMessage({
+                opName: o,
+                chunks: e
+            }), t && t(), !0;
+        }
+    });
 }
 
-Object.defineProperty(l, "level", {
-    get() {
-        return s[l.currentLogLevel];
-    },
-    set(e) {
-        l.currentLogLevel = c[e];
-    }
-}), globalThis.logger = l;
+var f = new h();
 
 export {
-    l as logger,
-    d as closeFileConsole,
-    L as setFileConsoleDateDirPath
+    a as ThreadMark,
+    f as logger
 };

package/src/QyMessageWorker.d.ts

@@ -1,5 +1,5 @@
 import { QyOpRunner } from "./QyOpRunner.ts";
 export declare class QyMessageWorker extends QyOpRunner {
-    constructor(options: any, callOpNameList: any, castOpNameList: any);
+    constructor(options: any, callOpNameList?: any[], castOpNameList?: any[]);
     castParent(opName: any, ...args: any[]): void;
 }

package/src/QyMessageWorker.js

@@ -15,11 +15,11 @@ import {
 } from "./QyOpRunner.js";
 
 class e extends s {
-    constructor(s, e, r) {
+    constructor(s, e = [], r = []) {
         super(s = {
             ...a("QyMessageWorker"),
             ...s
-        }, e, r), o.on("message", async s => {
+        }, e, r), o?.on("message", async s => {
             var {
                 opName: e,
                 _cast: r
@@ -28,12 +28,12 @@ class e extends s {
                 var a = s.args || [];
                 try {
                     var t = await this[e](...a);
-                    r || o.postMessage({
+                    r || o?.postMessage({
                         ack: e,
                         result: t
                     }), "stop" == e && process.exit();
                 } catch (s) {
-                    n.error(`Running ${e} failed:`, s), r || o.postMessage({
+                    n.error(`Running ${e} failed:`, s), r || o?.postMessage({
                         ack: e,
                         exception: s
                     });
@@ -42,7 +42,7 @@ class e extends s {
         });
     }
     castParent(s, ...e) {
-        o.postMessage({
+        o?.postMessage({
             opName: s,
             args: e,
             _cast: !0

package/src/QyMessager.d.ts

@@ -1,5 +1,9 @@
 export declare class QyMessager {
-    constructor(parent: any, workerTypeFileName: any, initArgMap: any, options: any, callOpNameList?: never[], castOpNameList?: never[]);
+    started: any;
+    worker: any;
+    receiverQueueMap: any;
+    options: any;
+    constructor(parent: any, workerTypeFileName: any, initArgMap: any, options: any, callOpNameList?: any[], castOpNameList?: any[]);
     start(...args: any[]): any;
     stop(...args: any[]): any;
     castWorker(opName: any, args: any): void;

package/src/QyMessager.js

@@ -28,6 +28,10 @@ import {
 let k = e(import.meta.filename);
 
 class r {
+    started;
+    worker;
+    receiverQueueMap;
+    options;
     constructor(e, r, t, s, o = [], a = []) {
         s = {
             ...i("QyMessager"),
@@ -66,7 +70,8 @@ class r {
                     ack: t,
                     result: s
                 } = e;
-                r ? (e = e.args || [], (i.parent || i)[r](...e)) : o[t]?.shift().resolve(s);
+                r ? "%o_" == r ? n.handleOutMsg(e) : "%e_" == r ? n.handleErrMsg(e) : (e = e.args || [], 
+                (i.parent || i)[r](...e)) : o[t]?.shift().resolve(s);
             }), a.on("exit", e => {
                 for (var r in a.removeAllListeners(), 0 != e ? n.warn(`Worker ${s} stopped with exitCode ` + e) : n.info(`Worker ${s} stopped.`), 
                 o) for (var t = o[r]; !t.isEmpty; ) t.shift()?.resolve();

package/src/QyMessager_Run.js

@@ -14,11 +14,12 @@ import {
 } from "node:worker_threads";
 
 import {
-    logger as s
+    logger as s,
+    ThreadMark as n
 } from "./QyLogger.js";
 
 import {
-    __dirname as n
+    __dirname as m
 } from "./QyUtils.js";
 
 (async () => {
@@ -27,5 +28,5 @@ import {
         workerTypeFileName: e,
         initArgMap: r
     } = i;
-    new (await import(t("file://" + o(n, e)))).default(r);
+    new (await import(t("file://" + o(m, e)))).default(r), s.log("" + n + e);
 })();

package/src/QyMod.js

@@ -1,16 +1,17 @@
-import a from "node:module";
+import d from "node:module";
 
 import {
-    logger as m
+    logger as s
 } from "./QyLogger.js";
 
-function o(r, e, t, o) {
-    if (t) try {
-        var i = r + "_" + e, l = `_${i}_qydb`, d = (globalThis[l] = o, new a(e));
-        return d._compile(`(function(dataFolder){${t}})(${l});`, i + ".ts"), delete globalThis[l], 
-        d.exports;
+function o(e, r, l, o) {
+    if (l) try {
+        var t = e + "_" + r, g = `_${t}_qydb`, i = `_${t}_logger`, a = (globalThis[g] = o, 
+        globalThis[i] = s, new d(r));
+        return a._compile(`(function(dataFolder, logger){${l}})(${g},${i});`, t + ".ts"), 
+        delete globalThis[g], delete globalThis[i], a.exports;
     } catch (o) {
-        m.error("requireFromModCode failed:", r, e, t, o);
+        s.error("requireFromModCode failed:", e, r, l, o);
     }
 }
 

package/src/QyOpRunner.d.ts

@@ -1,6 +1,10 @@
 import EventEmitter from "node:events";
 export declare class QyOpRunner extends EventEmitter {
-    constructor(options: any, callOpNameList?: never[], castOpNameList?: never[]);
+    running: boolean;
+    opQueue: any;
+    options: any;
+    runningOp: any;
+    constructor(options: any, callOpNameList?: any[], castOpNameList?: any[]);
     get firstOp(): any;
     get lastOp(): any;
     finish(): any;

package/src/QyOpRunner.js

@@ -17,15 +17,15 @@ import {
 } from "./QyQueue.js";
 
 class e extends t {
+    running = !1;
+    opQueue;
+    options;
+    runningOp;
     constructor(t, s = [], e = []) {
-        super(), t = {
+        super(), t = this.options = {
             ...i("QyOpRunner"),
             ...t
-        }, Object.assign(this, {
-            options: t,
-            running: !1,
-            opQueue: new r(t)
-        }), n(this, s, !0), n(this, e), n(this, [ "start", "stop" ], !0);
+        }, this.opQueue = new r(t), n(this, s, !0), n(this, e), n(this, [ "start", "stop" ], !0);
     }
     get firstOp() {
         return this.opQueue.first;
@@ -70,7 +70,7 @@ class e extends t {
     }
 }
 
-function n(i, t, r) {
+function n(i, t, r = !1) {
     for (let e of t) {
         let s = "_op_" + e;
         if (i[e] || (i[e] = r ? (...t) => i.pushAndRunWithPromise({

package/src/QyQueue.d.ts

@@ -1,9 +1,13 @@
 export declare class QyQueue {
+    options: any;
+    queue: any[];
+    firstIdx: number;
+    _waterMark: number;
     constructor(options: any);
     get first(): any;
     get last(): any;
     get isEmpty(): boolean;
-    get waterMark(): any;
+    get waterMark(): number;
     pop(): any;
     reset(): void;
     push(...args: any[]): void;

package/src/QyQueue.js

@@ -7,16 +7,15 @@ import {
 } from "./QyDefaultOptions.js";
 
 class s {
+    options;
+    queue = [];
+    firstIdx = 0;
+    _waterMark = 0;
     constructor(t) {
-        t = {
+        this.options = {
             ...e("QyQueue"),
             ...t
-        }, Object.assign(this, {
-            options: t,
-            queue: [],
-            firstIdx: 0,
-            _waterMark: 0
-        });
+        };
     }
     get first() {
         return this.queue[this.firstIdx];

package/src/QySaver.d.ts

@@ -1,8 +1,11 @@
 export declare class QySaver {
+    dataFolder: any;
+    options: any;
     constructor(dataFolder: any, options: any);
     setDataFolder(dataFolder: any): this;
     processAfterLoad(data: any): any;
     processBeforeSave(data: any): Buffer<any>;
+    removeFile(fileName: any): Promise<void>;
     loadFromFile(fileName: any, defaultValue: any): Promise<any>;
     saveToFile(data: any, fileName: any, override?: boolean): Promise<boolean>;
     _saveTmpFileAndRename(contentBuffer: any, tmpFilePath: any, targetFilePath: any): Promise<boolean>;

package/src/QySaver.js

@@ -1,9 +1,10 @@
 import s, {
-    readFile as i
+    readFile as a,
+    rm as e
 } from "node:fs/promises";
 
 import {
-    join as c
+    join as f
 } from "node:path";
 
 import {
@@ -23,7 +24,9 @@ import {
     getDefaultOptions as t
 } from "./QyDefaultOptions.js";
 
-class e {
+class i {
+    dataFolder;
+    options;
     constructor(e, r) {
         Object.assign(this, {
             dataFolder: e,
@@ -42,32 +45,40 @@ class e {
     processBeforeSave(e) {
         return r(e);
     }
-    async loadFromFile(e, r) {
-        var t = c(this.dataFolder, e);
-        if (null != r && !await u(t)) return r;
+    async removeFile(r) {
+        r = f(this.dataFolder, r);
         try {
-            var a = await i(t);
-            return m.info(`Loading "${t}" succeeded. Total size:${o(a.length)}.`), 
-            this.processAfterLoad(a, e);
+            await e(r);
         } catch (e) {
-            if (null != r) return r;
-            m.error(`Loading "${t}" failed:`, e);
+            m.warn(`Deleting ${r} failed:`, e);
+        }
+    }
+    async loadFromFile(r, t) {
+        r = f(this.dataFolder, r);
+        if (null != t && !await u(r)) return t;
+        try {
+            var e = await a(r);
+            return m.info(`Loading "${r}" succeeded. Total size:${o(e.length)}.`), 
+            this.processAfterLoad(e);
+        } catch (e) {
+            if (null != t) return t;
+            m.error(`Loading "${r}" failed:`, e);
         }
     }
     async saveToFile(e, t, r = !1) {
         var {
             dataFolder: a,
             options: i
-        } = this, s = c(a, t);
+        } = this, s = f(a, t);
         if (!r && await u(s)) m.error(s + " already exists."); else {
             var {
                 retryTimes: o,
                 retryIntervalFactor: n,
                 maxRetryInterval: l
-            } = i, d = c(a, ".tmp", t), f = this.processBeforeSave(e, t);
+            } = i, d = f(a, ".tmp", t), c = this.processBeforeSave(e);
             let r;
             for (let e = 0; e < o; ++e) {
-                if (await this._saveTmpFileAndRename(f, d, s)) return !0;
+                if (await this._saveTmpFileAndRename(c, d, s)) return !0;
                 r = r ? Math.min(l, r * n) : 1, m.warn(`Retrying to create ${s} in ${r} seconds.`), 
                 await g(r);
             }
@@ -102,5 +113,5 @@ class e {
 }
 
 export {
-    e as QySaver
+    i as QySaver
 };

package/src/QySnapshotCompactor_Worker.d.ts

@@ -1,5 +1,12 @@
 import { QyMessageWorker } from "./QyMessageWorker.ts";
 export default class QySnapshotCompactor_Worker extends QyMessageWorker {
+    kvFolder: any;
+    snapshotFolder: any;
+    qySnapshots: any;
+    qyKVDataCleaner: any;
+    pMInfoMap: any;
+    vLInfoMap: any;
+    combinedSnapshotNumVerList: any[];
     constructor({ kvFolder, options }: {
         kvFolder: any;
         options: any;