datafolder 1.0.0

package/src/QyCD.js

@@ -0,0 +1,61 @@
+let {
+    isString,
+    pathSplit
+} = require("./QyUtils.js");
+
+class QyCD {
+    constructor(r) {
+        this.qyCache = r;
+    }
+    cd(r, i = this.currentDir) {
+        return isString(r) && (r = pathSplit(r)), this.cdP(r, i);
+    }
+    cdP(r, i = this.currentDir) {
+        return this.currentDir = this.qyCache.getOrCreateDir(r, i), this;
+    }
+    dir(r, i = this.currentDir) {
+        return isString(r) && (r = pathSplit(r)), this.dirP(r, i);
+    }
+    dirP(r, i = this.currentDir) {
+        return this.qyCache.getDir(r, i);
+    }
+    indexDir(r, i, t = this.currentDir) {
+        return isString(r) && (r = pathSplit(r)), isString(i) && (i = pathSplit(i)), 
+        this.indexDirP(r, i, t);
+    }
+    indexDirP(r, i, t = this.currentDir) {
+        var e = this.qyCache, r = e.getDir(r, t);
+        if (r) {
+            t = e.getIndexDir(r);
+            if (t) return i && 0 != i.length ? e.getDir(i, t) : t;
+        }
+    }
+    file(r, i = this.currentDir) {
+        return isString(r) && (r = pathSplit(r)), this.fileP(r, i);
+    }
+    fileP(r, i = this.currentDir) {
+        return this.qyCache.getFile(r, i);
+    }
+    triggerFile(r) {
+        return this.qyCache.getTriggerFile(r);
+    }
+    view(r, i, t = this.currentDir) {
+        return isString(r) && (r = pathSplit(r)), this.viewP(r, i, t);
+    }
+    viewP(r, i, t = this.currentDir) {
+        return this.qyCache.getFile(r, t)?.view(i);
+    }
+    queryFiles(r, i, t, e, s = this.currentDir) {
+        return this.qyCache.queryFiles(r, i, t, e, s);
+    }
+    queryFilesMulti(r, i, t, e = this.currentDir) {
+        return this.qyCache.queryFilesMulti(r, i, t, e);
+    }
+    resetCurrentDir() {
+        this.currentDir = void 0;
+    }
+}
+
+Object.assign(module.exports, {
+    QyCD: QyCD
+});

package/src/QyCache.js

@@ -0,0 +1,212 @@
+let path = require("node:path"), EventEmitter = require("node:events"), QyDir = require("./QyDir.js").QyDir, QyFileLogger = require("./QyFileLogger.js").QyFileLogger, QyKVData = require("./QyKVData.js").QyKVData, {
+    QyTriggerNode,
+    updateTriggerNodes
+} = require("./QyTrigger.js"), {
+    isString,
+    lockExclusiveFilePath,
+    unlockExclusiveFilePath
+} = require("./QyUtils.js"), dumpStructure = require("./QyStructure.js").dumpStructure, {
+    getIndexPropPaths,
+    parseNamePathAndQuery
+} = require("./QyIndex.js"), getDefaultOptions = require("./QyDefaultOptions.js").getDefaultOptions, {
+    ensureDir,
+    SpecialFilePaths
+} = require("./QyUtils.js"), QueryCounter = require("./QyUnionNode.js").QueryCounter, QyKVDataLoader = require("./QyKVDataLoader.js").QyKVDataLoader, requireFromModCode = require("./QyMod.js").requireFromModCode, {
+    IndexDirPath,
+    TriggerDirPath,
+    RpcDirPath
+} = SpecialFilePaths;
+
+class QyCache extends EventEmitter {
+    constructor(e, r, t) {
+        super(), r = {
+            ...getDefaultOptions("QyCache"),
+            ...r
+        };
+        var i = path.join(e, "qykv");
+        Object.assign(this, {
+            qyDB: t,
+            rootFolder: e,
+            kvFolder: i,
+            rpcFolder: path.join(e, "rpc"),
+            options: r,
+            rootDir: new QyDir("", !0, void 0, this),
+            exclusiveFilePath: path.join(e, ".opened"),
+            qyKVData: new QyKVData(i, r, this),
+            qyFileLogger: new QyFileLogger(path.join(e, "qylog"), r),
+            rpcFunMap: {}
+        });
+    }
+    async start() {
+        var {
+            rootFolder: e,
+            kvFolder: r,
+            options: t,
+            qyKVData: i,
+            qyFileLogger: a,
+            exclusiveFilePath: o,
+            rootDir: s
+        } = this;
+        logger.log(`Starting at ${e}...`);
+        try {
+            await ensureDir(r), a.start();
+            var [ {
+                maxChangeId: g,
+                loadedMaxAclNum: n,
+                qySnapshots: l,
+                aclKeyValueMap: u
+            } ] = await Promise.all([ this._loadQyKVData(r, t), lockExclusiveFilePath(o) ]);
+            await i.start(g, n, l, u);
+        } catch (e) {
+            throw a.stop(), e;
+        }
+        return t.preloadFoldersAtStart && this._preloadFolders(s), this.indexRootDir = s.searchOrCreateSubdir(IndexDirPath), 
+        this._initTriggers(), this._initRpcs(), t.dumpStructureAtStart && await dumpStructure(path.join(e, "qystruct"), s), 
+        logger.log("Started."), this;
+    }
+    async stop() {
+        logger.log("Stopping...");
+        var {
+            qyKVData: e,
+            qyFileLogger: r,
+            exclusiveFilePath: t
+        } = this;
+        return await e.stop(), await r.stop(), unlockExclusiveFilePath(t), Object.assign(this, {
+            rootDir: void 0,
+            indexRootDir: void 0,
+            qyKVData: void 0,
+            rpcFunMap: void 0,
+            qyDB: void 0,
+            qyFileLogger: void 0,
+            triggerRootDir: void 0,
+            rpcRootDir: void 0
+        }), logger.log("Stopped."), this;
+    }
+    increaseChangeId() {
+        return this.qyKVData.increaseChangeId();
+    }
+    decreaseChangeId() {
+        this.qyKVData.decreaseChangeId();
+    }
+    save(e, r, t, i, a) {
+        var {
+            qyKVData: o,
+            qyFileLogger: s
+        } = this;
+        return t && s.save(e, t), o.save(e, r, i, a);
+    }
+    getFile(e, r = this.rootDir) {
+        return e.isFile ? e : isString(e) ? r.getFile(e) : r.searchFile(e);
+    }
+    getDir(e, r = this.rootDir) {
+        return e.isDir ? e : isString(e) ? r.getSubdir(e) : r.searchSubdir(e);
+    }
+    getOrCreateFile(e, r = this.rootDir) {
+        return e.isFile ? e : isString(e) ? r.getOrCreateFile(e) : r.searchOrCreateFile(e);
+    }
+    getOrCreateDir(e, r = this.rootDir) {
+        return e.isDir ? e : isString(e) ? r.getOrCreateSubdir(e) : r.searchOrCreateSubdir(e);
+    }
+    getIndexDir(e) {
+        return this.indexRootDir.getSubdir(e.fullPathHash);
+    }
+    getOrCreateIndexDir(e) {
+        return this.indexRootDir.getOrCreateSubdir(e.fullPathHash);
+    }
+    getTriggerFile(e) {
+        return this.triggerRootDir.getFile(e);
+    }
+    getOrCreateTriggerFile(e) {
+        return this.triggerRootDir.getOrCreateFile(e);
+    }
+    getRpcFile(e) {
+        return this.rpcRootDir.getFile(e);
+    }
+    getOrCreateRpcFile(e) {
+        return this.rpcRootDir.getOrCreateFile(e);
+    }
+    insertTriggerToTree(e, r, t, i) {
+        var {
+            rootDir: a,
+            qyDB: o
+        } = this, i = requireFromModCode("trigger", e, i, o);
+        a.triggerNodes[0].insertTrigger(e, r, t, i), updateTriggerNodes(a, r);
+    }
+    removeTriggerFromTree(e, r) {
+        var t = this.rootDir;
+        t.triggerNodes[0].removeTrigger(e, r) && updateTriggerNodes(t, r);
+    }
+    insertRpc(e, r) {
+        var {
+            rpcFunMap: t,
+            qyDB: i
+        } = this, r = requireFromModCode("rpc", e, r, i);
+        r && (t[e] = r);
+    }
+    removeRpc(e) {
+        delete this.rpcFunMap[e];
+    }
+    queryFiles(e, r, t, i, a = this.rootDir) {
+        var o = [], t = (isString(r) && null == t && (t = 1), new QueryCounter(t, i));
+        return this._queryFiles(e, r, o, t, a), o;
+    }
+    queryFilesMulti(e, r, t, i = this.rootDir) {
+        var a, o, s = [], g = new QueryCounter(r, t);
+        for ([ a, o ] of e) this._queryFiles(a, o, s, g, i);
+        return s;
+    }
+    _queryFiles(e, r, t, i, a) {
+        var o;
+        r && (o = this.options.queryAutoCreateIndex, o = o ? getIndexPropPaths(r) : void 0, 
+        parseNamePathAndQuery(e, a, o, r, t, i));
+    }
+    async callRpc(r, e, t) {
+        var i, a = this.rpcFunMap[r];
+        if (!a) return i = `RPC ${r} not implemented.`, logger.warn(i), new Error(i);
+        try {
+            return await a(e, t);
+        } catch (e) {
+            return logger.error(`Calling RPC ${r} failed:`, e), e;
+        }
+    }
+    _preloadFolders(e) {
+        var r, t = e.subdirMap;
+        for (r in t) {
+            var i = t[r];
+            i.created && this._preloadFolders(i);
+        }
+        e._ensureFileMapLoaded();
+    }
+    _initTriggers() {
+        var e, r = this.rootDir;
+        r.triggerNodes = [ new QyTriggerNode() ], this.triggerRootDir = r.searchOrCreateSubdir(TriggerDirPath);
+        for (e of this.triggerRootDir.fileList) {
+            var {
+                name: t,
+                fileContent: i
+            } = e, {
+                fileNamePath: i,
+                propNamePathList: a,
+                modCode: o
+            } = i;
+            this.insertTriggerToTree(t, i, a, o);
+        }
+    }
+    _initRpcs() {
+        this.rpcRootDir = this.rootDir.searchOrCreateSubdir(RpcDirPath);
+        for (var e of this.rpcRootDir.fileList) {
+            var {
+                name: e,
+                fileContent: r
+            } = e;
+            r && (r = r.modCode, r) && this.insertRpc(e, r);
+        }
+    }
+    _loadQyKVData(e, r) {
+        return new QyKVDataLoader(e, r).load();
+    }
+}
+
+Object.assign(module.exports, {
+    QyCache: QyCache
+});

package/src/QyCompressor.js

@@ -0,0 +1,60 @@
+let path = require("node:path"), pipeline = require("node:stream").pipeline, promisify = require("node:util").promisify, {
+    createGzip,
+    createUnzip
+} = require("node:zlib"), {
+    createReadStream,
+    createWriteStream
+} = require("node:fs"), fsPromises = require("node:fs/promises"), logger = require("./QyLogger.js").logger, {
+    isNotNullObj,
+    folderOrFileExists,
+    ensureParentDir
+} = require("./QyUtils.js"), runOnceWithWorker = require("./QyWorker.js").runOnceWithWorker, pipe = promisify(pipeline), Zip_File_Extension = ".gz", compressorWorkerJsFileName = "QyCompressor_Worker.js";
+
+function compressFile(e, i, r) {
+    return isNotNullObj(i) && (r = i, i = void 0), i = i || e + Zip_File_Extension, 
+    r = r || {}, e.endsWith(Zip_File_Extension) ? (logger.log(`${e} already ends with ${Zip_File_Extension}.`), 
+    !1) : i.endsWith(Zip_File_Extension) ? runOnceWithWorker(compressorWorkerJsFileName, {
+        operation: "compress",
+        srcFilePath: e,
+        destFilePath: i,
+        options: r
+    }).promise : (logger.log(`${i} doesn't end with ${Zip_File_Extension}.`), !1);
+}
+
+function decompressFile(e, i, r) {
+    return e.endsWith(Zip_File_Extension) ? (isNotNullObj(i) && (r = i, i = void 0), 
+    i = i || e.replace(new RegExp(Zip_File_Extension + "$"), ""), r = r || {}, path.extname(i) == Zip_File_Extension ? (logger.log(`${i} already ends with ${Zip_File_Extension}.`), 
+    !1) : runOnceWithWorker(compressorWorkerJsFileName, {
+        operation: "decompress",
+        srcFilePath: e,
+        destFilePath: i,
+        options: r
+    }).promise) : (logger.log(`${e} doesn't end with ${Zip_File_Extension}.`), !1);
+}
+
+async function runOperation({
+    operation: e,
+    srcFilePath: i,
+    destFilePath: r,
+    options: o
+}) {
+    var {
+        removeSrcAfterSuccess: o,
+        override: s
+    } = o;
+    if (!await folderOrFileExists(i)) throw new Error(i + " doesn't exist.");
+    if (!s && await folderOrFileExists(r)) throw new Error(r + " already exists.");
+    if (await ensureParentDir(r), await pipe(createReadStream(i), ("compress" == e ? createGzip : createUnzip)(), createWriteStream(r)), 
+    logger.info(e + `ed ${i} to ${r}.`), o) try {
+        await fsPromises.unlink(i), logger.info("Unlinked " + i);
+    } catch (e) {
+        logger.error(`Failed to unlink ${i}:`, e);
+    }
+}
+
+Object.assign(module.exports, {
+    Zip_File_Extension: Zip_File_Extension,
+    compressFile: compressFile,
+    decompressFile: decompressFile,
+    runOperation: runOperation
+});

package/src/QyCompressor_Worker.js

@@ -0,0 +1,13 @@
+let {
+    parentPort,
+    getEnvironmentData,
+    workerData
+} = require("node:worker_threads"), logger = require("./QyLogger.js").logger, runOperation = require("./QyCompressor.js").runOperation;
+
+logger.level = getEnvironmentData("logLevel"), (async () => {
+    try {
+        await runOperation(workerData), parentPort.postMessage(!0);
+    } catch (e) {
+        parentPort.postMessage(e);
+    }
+})();

package/src/QyDB.js

@@ -0,0 +1,122 @@
+let path = require("node:path"), QyFetch = require("./QyFetch.js").QyFetch, QyBatch = require("./QyBatch.js").QyBatch, QyCache = require("./QyCache.js").QyCache, getDefaultOptions = require("./QyDefaultOptions.js").getDefaultOptions, QyCD = require("./QyCD.js").QyCD, Default_RootFolder = path.join(__dirname, "../data/sampleDataFolder");
+
+class QyDB extends QyCD {
+    constructor(e = Default_RootFolder, t) {
+        super(), Object.assign(this, {
+            rootFolder: e,
+            options: {
+                ...getDefaultOptions("QyDB"),
+                ...t
+            }
+        });
+    }
+    async start(e) {
+        var t;
+        return this.qyCache || (e = {
+            ...this.options,
+            ...e
+        }, t = new QyCache(this.rootFolder, e, this), Object.assign(this, {
+            qyCache: t,
+            fetch: new QyFetch(t),
+            batch: new QyBatch(t, e),
+            _immediate: new QyBatch(t, e)
+        }), await t.start()), this;
+    }
+    async stop() {
+        var {
+            qyCache: e,
+            fetch: t,
+            batch: r,
+            _immediate: i
+        } = this;
+        e && (await e.stop(), t.qyCache = void 0, r.qyCache = void 0, i.qyCache = void 0, 
+        Object.assign(this, {
+            qyCache: void 0,
+            fetch: void 0,
+            batch: void 0,
+            _immediate: void 0
+        }));
+    }
+    createDir(e) {
+        return this._immediate.createDir(e).run(), this.dir(e);
+    }
+    createDirP(e) {
+        return this._immediate.createDirP(e).run(), this.dirP(e);
+    }
+    createFile(e) {
+        return this._immediate.createFile(e).run(), this.file(e);
+    }
+    createFileP(e) {
+        return this._immediate.createFileP(e).run(), this.fileP(e);
+    }
+    createIndex(e, t) {
+        return this._immediate.createIndex(e, t).run();
+    }
+    createIndexP(e, t) {
+        return this._immediate.createIndexP(e, t).run();
+    }
+    delIndex(e, t) {
+        return this._immediate.delIndex(e, t).run();
+    }
+    delIndexP(e, t) {
+        return this._immediate.delIndexP(e, t).run();
+    }
+    insert(e, t) {
+        return this._immediate.insert(e, t).run();
+    }
+    insertP(e, t) {
+        return this._immediate.insertP(e, t).run();
+    }
+    remove(e, t) {
+        return this._immediate.remove(e, t).run();
+    }
+    removeP(e, t) {
+        return this._immediate.removeP(e, t).run();
+    }
+    replace(e, t) {
+        return this._immediate.replace(e, t).run();
+    }
+    replaceP(e, t) {
+        return this._immediate.replaceP(e, t).run();
+    }
+    delDir(e) {
+        return this._immediate.delDir(e).run();
+    }
+    delDirP(e) {
+        return this._immediate.delDirP(e).run();
+    }
+    delFile(e) {
+        return this._immediate.delFile(e).run();
+    }
+    delFileP(e) {
+        return this._immediate.delFileP(e).run();
+    }
+    delContent(e) {
+        return this._immediate.delContent(e).run();
+    }
+    delContentP(e) {
+        return this._immediate.delContentP(e).run();
+    }
+    insertTrigger(e, t, r, i) {
+        return this._immediate.insertTrigger(e, t, r, i).run();
+    }
+    removeTrigger(e) {
+        return this._immediate.removeTrigger(e).run();
+    }
+    insertRpc(e, t) {
+        return this._immediate.insertRpc(e, t).run();
+    }
+    removeRpc(e) {
+        return this._immediate.removeRpc(e).run();
+    }
+    on(e, t) {
+        return this.qyCache.on(e, t);
+    }
+    callRpc(e, t, r) {
+        return this.qyCache.callRpc(e, t, r);
+    }
+}
+
+Object.assign(module.exports, {
+    QyDB: QyDB
+});

package/src/QyDefaultOptions.js

@@ -0,0 +1,59 @@
+let writerConfig = {
+    append: !1,
+    retryIntervalFactor: 2,
+    maxRetryInterval: 60,
+    emitSaveEvent: !1,
+    maxFileSize: 1 / 0
+}, Default_Options = {
+    QyBaseWriter: writerConfig,
+    QyBinWriter: writerConfig,
+    QyBatch: {
+        fileLogLevel: "delta",
+        maxHistoryCmdObjCount: 100
+    },
+    QyKVData: {
+        append: !0,
+        emitSaveEvent: !0,
+        saveSnapshotAtStop: !0,
+        autoRepairAclFile: !0,
+        snapshotMaxChangeCount: 1e6,
+        compactSnapshotsInterval: 6e4,
+        bgLoadPosMaps: !0,
+        unloadMemInterval: 12e4,
+        maxFileCountToUnload: 1e5
+    },
+    QyCache: {
+        dumpStructureAtStart: !1,
+        queryAutoCreateIndex: !0,
+        preloadFoldersAtStart: !1
+    },
+    QyFileLogger: {
+        clearLogAtStart: !1
+    },
+    QySaver: {
+        retryTimes: 10,
+        retryIntervalFactor: 2,
+        maxRetryInterval: 60
+    },
+    QyQueue: {
+        maxLenBeforeRecycle: 1e3
+    },
+    QySnapshots: {
+        pMOutdatedPercent: 30,
+        minSnapshotKeyCount: 3e3
+    },
+    QyKVDataCleaner_Worker: {
+        compressRedundantFile: !1
+    },
+    QySnapshotSaver_Worker: {
+        maxInMemSnapshotCount: 3
+    }
+};
+
+function getDefaultOptions(e) {
+    return e && Default_Options[e] || void 0;
+}
+
+Object.assign(module.exports, {
+    getDefaultOptions: getDefaultOptions
+});