datafolder 1.0.0

package/src/QyMessageWorker.js

@@ -0,0 +1,41 @@
+let parentPort = require("node:worker_threads").parentPort, getDefaultOptions = require("./QyDefaultOptions.js").getDefaultOptions, logger = require("./QyLogger.js").logger, QyOpRunner = require("./QyOpRunner.js").QyOpRunner;
+
+class QyMessageWorker extends QyOpRunner {
+    constructor(e, r, s) {
+        super(e = {
+            ...getDefaultOptions("QyMessageWorker"),
+            ...e
+        }, r, s), parentPort.on("message", async e => {
+            var {
+                opName: r,
+                _cast: s
+            } = e;
+            if (r) {
+                var t = e.args || [];
+                try {
+                    var a = await this[r](...t);
+                    s || parentPort.postMessage({
+                        ack: r,
+                        result: a
+                    }), "stop" == r && process.exit();
+                } catch (e) {
+                    logger.error(`Running ${r} failed:`, e), s || parentPort.postMessage({
+                        ack: r,
+                        exception: e
+                    });
+                }
+            } else logger.warn("Msg ignored:", e);
+        });
+    }
+    castParent(e, ...r) {
+        parentPort.postMessage({
+            opName: e,
+            args: r,
+            _cast: !0
+        });
+    }
+}
+
+Object.assign(module.exports, {
+    QyMessageWorker: QyMessageWorker
+});

package/src/QyMessager.js

@@ -0,0 +1,84 @@
+let path = require("node:path"), {
+    Worker,
+    setEnvironmentData
+} = require("node:worker_threads"), getDefaultOptions = require("./QyDefaultOptions.js").getDefaultOptions, logger = require("./QyLogger.js").logger, setPromise = require("./QyUtils.js").setPromise, QyQueue = require("./QyQueue.js").QyQueue;
+
+class QyMessager {
+    constructor(e, r, t, s, o = [], i = []) {
+        s = {
+            ...getDefaultOptions("QyMessager"),
+            ...s
+        }, Object.assign(this, {
+            parent: e,
+            workerTypeFileName: r,
+            initArgMap: t,
+            options: s,
+            receiverQueueMap: {}
+        });
+        for (let r of o) this[r] || (this[r] = (...e) => this.callWorker(r, e));
+        for (let r of i) this[r] || (this[r] = (...e) => this.castWorker(r, e));
+    }
+    start(...e) {
+        return this.started = !0, setEnvironmentData("logLevel", logger.level), 
+        this._startWorker(), this.callWorker("start", e);
+    }
+    stop(...e) {
+        if (this.started) return this.started = !1, this.callWorker("stop", e);
+    }
+    castWorker(e, r) {
+        this.worker.postMessage({
+            opName: e,
+            args: r,
+            _cast: !0
+        });
+    }
+    callWorker(e, r) {
+        var {
+            worker: t,
+            receiverQueueMap: s,
+            options: o
+        } = this;
+        t.postMessage({
+            opName: e,
+            args: r
+        });
+        let i = s[e];
+        i = i || (s[e] = new QyQueue(o));
+        t = setPromise({});
+        return i.push(t), t.promise;
+    }
+    _startWorker() {
+        if (!this.worker) {
+            let {
+                workerTypeFileName: s,
+                initArgMap: e,
+                receiverQueueMap: o
+            } = this;
+            var r = {
+                workerTypeFileName: s,
+                initArgMap: e
+            };
+            let i = this.worker = new Worker(path.join(__dirname, "QyMessager_Run.js"), {
+                workerData: r
+            });
+            i.on("error", e => {
+                logger.error(`Worker ${s} error:`, e);
+            }), i.on("message", e => {
+                var {
+                    opName: r,
+                    ack: t,
+                    result: s
+                } = e;
+                r ? (e = e.args || [], (this.parent || this)[r](...e)) : o[t]?.shift().resolve(s);
+            }), i.on("exit", e => {
+                for (var r in i.removeAllListeners(), 0 != e ? logger.warn(`Worker ${s} stopped with exitCode ` + e) : logger.info(`Worker ${s} stopped.`), 
+                o) for (var t = o[r]; !t.isEmpty; ) t.shift()?.resolve();
+                this.worker = void 0;
+            });
+        }
+    }
+}
+
+Object.assign(module.exports, {
+    QyMessager: QyMessager
+});

package/src/QyMessager_Run.js

@@ -0,0 +1,13 @@
+let path = require("node:path"), {
+    getEnvironmentData,
+    workerData
+} = require("node:worker_threads"), logger = require("./QyLogger.js").logger;
+
+(async () => {
+    logger.level = getEnvironmentData("logLevel");
+    var {
+        workerTypeFileName: e,
+        initArgMap: r
+    } = workerData;
+    new (require(path.join(__dirname, e)))(r);
+})();

package/src/QyMod.js

@@ -0,0 +1,15 @@
+let Module = require("node:module"), logger = require("./QyLogger.js").logger;
+
+function requireFromModCode(o, r, d, e) {
+    if (d) try {
+        var l = o + "_" + r, i = `_${l}_qydb`, u = (globalThis[i] = e, new Module(r));
+        return u._compile(`(function(qyDB){${d}})(${i});`, l + ".js"), delete globalThis[i], 
+        u.exports;
+    } catch (e) {
+        logger.error("requireFromModCode failed:", o, r, d, e);
+    }
+}
+
+Object.assign(module.exports, {
+    requireFromModCode: requireFromModCode
+});

package/src/QyOpRunner.js

@@ -0,0 +1,73 @@
+let EventEmitter = require("node:events"), setPromise = require("./QyUtils.js").setPromise, logger = require("./QyLogger.js").logger, getDefaultOptions = require("./QyDefaultOptions.js").getDefaultOptions, QyQueue = require("./QyQueue.js").QyQueue;
+
+class QyOpRunner extends EventEmitter {
+    constructor(e, s = [], t = []) {
+        super(), e = {
+            ...getDefaultOptions("QyOpRunner"),
+            ...e
+        }, Object.assign(this, {
+            options: e,
+            running: !1,
+            opQueue: new QyQueue(e)
+        }), this._setupOpNames(s, !0), this._setupOpNames(t);
+    }
+    _setupOpNames(e, i) {
+        for (let t of e) {
+            let s = "_op_" + t;
+            this[t] || (this[t] = i ? (...e) => this.pushAndRunWithPromise({
+                opName: t,
+                opFunName: s,
+                args: e
+            }) : (...e) => this.pushAndRun({
+                opName: t,
+                opFunName: s,
+                args: e
+            })), this[s] || (this[s] = () => logger.error(`Class method ${s} not implemented.`));
+        }
+    }
+    get firstOp() {
+        return this.opQueue.first;
+    }
+    get lastOp() {
+        return this.opQueue.last;
+    }
+    finish() {
+        return this.lastOp?.promise || this.runningOp?.promise;
+    }
+    pushAndRunWithPromise(e) {
+        return setPromise(e), this.pushAndRun(e), e.promise;
+    }
+    pushAndRun(...e) {
+        var s = this.opQueue, t = this.lastOp;
+        "_op_stop" === t?.opFunName ? (s.pop(), s.push(...e, t)) : s.push(...e), 
+        this.run();
+    }
+    async run() {
+        if (!this.running) {
+            this.running = !0;
+            for (var e = this.opQueue; !e.isEmpty; ) {
+                var s = e.shift(), {
+                    opFunName: t,
+                    eventName: i,
+                    resolve: n,
+                    args: r
+                } = s;
+                if (t) {
+                    this.runningOp = s;
+                    try {
+                        var u = await (r ? this[t](...r) : this[t]());
+                        n && n(u), i && this.emit(i, u);
+                    } catch (e) {
+                        logger.error(`Run ${t} failed:`, e), i && this.emit(i, e);
+                    }
+                    this.runningOp = void 0;
+                } else n && n(), i && this.emit(i);
+            }
+            this.running = !1;
+        }
+    }
+}
+
+Object.assign(module.exports, {
+    QyOpRunner: QyOpRunner
+});

package/src/QyQueue.js

@@ -0,0 +1,60 @@
+let arrayLast = require("./QyUtils.js").arrayLast, getDefaultOptions = require("./QyDefaultOptions.js").getDefaultOptions, logger = require("./QyLogger.js").logger;
+
+class QyQueue {
+    constructor(e) {
+        e = {
+            ...getDefaultOptions("QyQueue"),
+            ...e
+        }, Object.assign(this, {
+            options: e,
+            queue: [],
+            firstIdx: 0,
+            _waterMark: 0
+        });
+    }
+    get first() {
+        return this.queue[this.firstIdx];
+    }
+    get last() {
+        return this.isEmpty ? void 0 : arrayLast(this.queue);
+    }
+    get isEmpty() {
+        return this.firstIdx >= this.queue.length;
+    }
+    get waterMark() {
+        return this._waterMark;
+    }
+    pop() {
+        return this.queue.pop();
+    }
+    reset() {
+        this.firstIdx = 0, this.queue.length = 0;
+    }
+    push(...e) {
+        this.queue.push(...e);
+        e = this.queue.length;
+        e > this._waterMark && (this._waterMark = e);
+    }
+    shift() {
+        var e;
+        if (!this.isEmpty) return e = this.first, this._advanceIdx(), e;
+    }
+    _advanceIdx() {
+        var {
+            queue: t,
+            firstIdx: e,
+            options: s
+        } = this, r = t.length, i = e + 1;
+        if (r <= i) this.reset(); else {
+            var u = r - i;
+            if (i > s.maxLenBeforeRecycle || u <= i) {
+                for (let e = 0; e < u; ++e) t[e] = t[e + i];
+                this.firstIdx = 0, t.length = u;
+            } else this.firstIdx = i, t[e] = void 0;
+        }
+    }
+}
+
+Object.assign(module.exports, {
+    QyQueue: QyQueue
+});

package/src/QySaver.js

@@ -0,0 +1,83 @@
+let fsPromises = require("node:fs/promises"), path = require("node:path"), logger = require("./QyLogger.js").logger, qyUtils = require("./QyUtils.js"), getDefaultOptions = require("./QyDefaultOptions.js").getDefaultOptions;
+
+class QySaver {
+    constructor(e, r) {
+        Object.assign(this, {
+            dataFolder: e,
+            options: {
+                ...getDefaultOptions("QySaver"),
+                ...r
+            }
+        });
+    }
+    setDataFolder(e) {
+        return this.dataFolder = e, this;
+    }
+    processAfterLoad(e) {
+        return e;
+    }
+    processBeforeSave(e) {
+        return qyUtils.toBuffer(e);
+    }
+    async loadFromFile(e, r) {
+        var t = path.join(this.dataFolder, e);
+        if (null != r && !await qyUtils.folderOrFileExists(t)) return r;
+        try {
+            var a = await fsPromises.readFile(t);
+            return logger.info(`Loading "${t}" succeeded. Total size:${qyUtils.getSizeStr(a.length)}.`), 
+            this.processAfterLoad(a, e);
+        } catch (e) {
+            if (null != r) return r;
+            logger.error(`Loading "${t}" failed:`, e);
+        }
+    }
+    async saveToFile(e, t, r = !1) {
+        var {
+            dataFolder: a,
+            options: i
+        } = this, s = path.join(a, t);
+        if (!r && await qyUtils.folderOrFileExists(s)) logger.error(s + " already exists."); else {
+            var {
+                retryTimes: o,
+                retryIntervalFactor: l,
+                maxRetryInterval: n
+            } = i, g = path.join(a, ".tmp", t), d = this.processBeforeSave(e, t);
+            let r;
+            for (let e = 0; e < o; ++e) {
+                if (await this._saveTmpFileAndRename(d, g, s)) return !0;
+                r = r ? Math.min(n, r * l) : 1, logger.warn(`Retrying to create ${s} in ${r} seconds.`), 
+                await qyUtils.sleep(r);
+            }
+            logger.error(`Failed to create ${t} after retrying ${o} times.`);
+        }
+        return !1;
+    }
+    async _saveTmpFileAndRename(e, r, t) {
+        var a = e.length;
+        if (await this._saveContent(r, e) != a) logger.error(`Creating ${t} failed when trying to create ${r} first.`); else try {
+            if (await fsPromises.rename(r, t), await qyUtils.getSize(t) === a) return logger.info(`Renaming ${r} to ${t} succeeded.`), 
+            !0;
+            logger.error(t + " size is not correct.");
+        } catch (e) {
+            logger.error(`Renaming ${r} to ${t} failed：`, e);
+        }
+        return !1;
+    }
+    async _saveContent(r, e) {
+        var t = e.length;
+        try {
+            await qyUtils.ensureParentDir(r), await fsPromises.writeFile(r, e, {
+                flush: !0
+            });
+            var a = `Saving ${r} size ` + qyUtils.getSizeStr(t), i = await qyUtils.getSize(r);
+            return i === t ? logger.info(a + " succeeded.") : logger.warn(a + " failed."), 
+            i;
+        } catch (e) {
+            return logger.error(`Saving "${r}" failed:`, e), !1;
+        }
+    }
+}
+
+Object.assign(module.exports, {
+    QySaver: QySaver
+});

package/src/QySnapshotCompactor.js

@@ -0,0 +1,17 @@
+let QyMessager = require("./QyMessager.js").QyMessager, getDefaultOptions = require("./QyDefaultOptions.js").getDefaultOptions;
+
+class QySnapshotCompactor extends QyMessager {
+    constructor(s, o, t) {
+        super(s, "QySnapshotCompactor_Worker.js", {
+            kvFolder: o,
+            options: t = {
+                ...getDefaultOptions("QySnapshotCompactor"),
+                ...t
+            }
+        }, t, [], [ "compactSnapshots", "saveSnapshotInfos" ]);
+    }
+}
+
+Object.assign(module.exports, {
+    QySnapshotCompactor: QySnapshotCompactor
+});

package/src/QySnapshotCompactor_Worker.js

@@ -0,0 +1,151 @@
+let path = require("node:path"), fsPromises = require("node:fs/promises"), getDefaultOptions = require("./QyDefaultOptions.js").getDefaultOptions, QyMessageWorker = require("./QyMessageWorker.js").QyMessageWorker, QyKVDataCleaner = require("./QyKVDataCleaner.js").QyKVDataCleaner, QySnapshots = require("./QySnapshots.js").QySnapshots, {
+    isEmptyObj,
+    listToMap,
+    readBufferAsync
+} = require("./QyUtils.js");
+
+class QySnapshotCompactor_Worker extends QyMessageWorker {
+    constructor({
+        kvFolder: a,
+        options: s
+    }) {
+        super(s = {
+            ...getDefaultOptions("QySnapshotCompactor_Worker"),
+            ...s
+        }, [], [ "compactSnapshots", "saveSnapshotInfos" ]);
+        var e = path.join(a, "snapshot");
+        Object.assign(this, {
+            kvFolder: a,
+            snapshotFolder: e,
+            qySnapshots: new QySnapshots(e, s),
+            qyKVDataCleaner: new QyKVDataCleaner(this, a, s),
+            pMInfoMap: {},
+            vLInfoMap: {}
+        });
+    }
+    start() {
+        this.qyKVDataCleaner.start();
+    }
+    async stop() {
+        await this.qyKVDataCleaner.stop();
+    }
+    async _op_compactSnapshots(a, e, s) {
+        var o = {}, t = [];
+        if (0 < a.length) {
+            let s = o.pMUpdates = {};
+            t.push(...a.map(a => this._compactPosMap(a, s)));
+        }
+        if (0 < e.length) {
+            let s = o.vLUpdates = {};
+            t.push(...e.map(a => this._compactValueList(a, s)));
+        }
+        0 < s.length && t.push(this._combinePMLoadedSnapshots(s, o)), await Promise.all(t), 
+        this.castParent("onCompactUpdates", o);
+    }
+    async _op_saveSnapshotInfos(a) {
+        var s, e, {
+            qySnapshots: o,
+            pMInfoMap: t,
+            vLInfoMap: p,
+            qyKVDataCleaner: n
+        } = this, i = (Object.assign(this, {
+            pMInfoMap: {},
+            vLInfoMap: {}
+        }), listToMap(a)), r = [];
+        for (s in t) s in i && r.push(o.saveSnapshotInfo(s, t[s]));
+        for (e in p) e in i && r.push(o.saveSnapshotInfo(e, p[e]));
+        if (0 < r.length) {
+            await Promise.all(r);
+            var h, l, M = [];
+            for (h in t) {
+                var f = t[h];
+                M.push([ h, f.pMVersion ]);
+            }
+            for (l in p) {
+                var y = p[l];
+                M.push([ l, y.pMVersion ]), n.cleanVLFiles(l, y.vLVersion);
+            }
+            n.cleanPMFiles(M);
+        }
+    }
+    async _combinePMLoadedSnapshots(a, s) {
+        var e, {
+            qySnapshots: o,
+            vLInfoMap: t
+        } = this, p = o.getMaxCombinedSnapshot(a), {
+            snapshotNum: n,
+            info: i,
+            keyValueMap: r
+        } = (await this._loadKVMap(p), p), h = (Object.assign(p, {
+            posMap: void 0,
+            prefixPM: void 0
+        }), []);
+        for (e of a) e != p && (h.push(e.snapshotNum), await this._loadKVMap(e), 
+        Object.assign(r, e.keyValueMap), Object.assign(e, {
+            keyValueMap: void 0,
+            posMap: void 0,
+            prefixPM: void 0
+        }));
+        await o.saveKeyValueMap(p), t[n] = i, s.combineUpdate = {
+            snapshotNum: n,
+            combinedSnapshotNumList: h,
+            posMap: p.posMap
+        };
+    }
+    async _compactValueList(a, s) {
+        var e, o, {
+            qySnapshots: t,
+            vLInfoMap: p
+        } = this;
+        await this._loadPosMap(a) && (await this._loadKVMap(a), await t.saveKeyValueMap(a), 
+        {
+            snapshotNum: t,
+            info: a,
+            posMap: e,
+            prefixPM: o
+        } = a, p[t] = a, s[t] = {
+            posMap: e,
+            prefixPM: o
+        });
+    }
+    async _compactPosMap(a, s) {
+        var {
+            qySnapshots: e,
+            pMInfoMap: o
+        } = this, {
+            snapshotNum: t,
+            info: p,
+            prefixPM: n
+        } = a;
+        (null == n || isEmptyObj(n) || await this._loadPosMap(a)) && (await e.savePosMap(a), 
+        o[t] = p, s[t] = a.prefixPM);
+    }
+    async _loadPosMap(a) {
+        var s = this.qySnapshots;
+        let {
+            prefixPM: e,
+            posMap: o
+        } = a;
+        var t, p = s.getPMFilePath(a), n = await fsPromises.open(p);
+        for (t of Object.keys(e).sort()) {
+            var [ i, r ] = e[t], i = await readBufferAsync(p, n, i, r);
+            if (!i) return void n.close();
+            s.iterPMBuffer(i, (a, s) => o[a] = s);
+        }
+        return n.close(), !0;
+    }
+    async _loadKVMap(a) {
+        var s, e = this.qySnapshots, o = a.posMap, t = a.keyValueMap = {}, p = e.getVLFilePath(a), n = await fsPromises.open(p);
+        for (s of Object.keys(o).sort()) {
+            var [ i, r ] = o[s];
+            if (0 < r) {
+                i = await readBufferAsync(p, n, i, r);
+                if (!i) return void n.close();
+                t[s] = i;
+            } else t[s] = void 0;
+        }
+        return n.close(), !0;
+    }
+}
+
+module.exports = QySnapshotCompactor_Worker;

package/src/QySnapshotSaver.js

@@ -0,0 +1,17 @@
+let QyMessager = require("./QyMessager.js").QyMessager, getDefaultOptions = require("./QyDefaultOptions.js").getDefaultOptions;
+
+class QySnapshotSaver extends QyMessager {
+    constructor(e, s, a) {
+        super(e, "QySnapshotSaver_Worker.js", {
+            kvFolder: s,
+            options: a = {
+                ...getDefaultOptions("QySnapshotSaver"),
+                ...a
+            }
+        }, a, [], [ "saveSnapshot", "castSave", "releaseSnapshot" ]);
+    }
+}
+
+Object.assign(module.exports, {
+    QySnapshotSaver: QySnapshotSaver
+});