datafolder 1.0.5 → 1.0.7

package/src/QySnapshots.js

@@ -1,26 +1,28 @@
 import {
-    close as o,
+    close as e,
     readSync as i,
     openSync as p
 } from "node:fs";
 
 import {
     readdir as l,
-    readFile as d
+    readFile as r
 } from "node:fs/promises";
 
 import {
     join as s
 } from "node:path";
 
+import m from "node:zlib";
+
 import {
     getDefaultOptions as n
 } from "./QyDefaultOptions.js";
 
 import {
     Hash_Key_Length as u,
-    folderOrFileExists as m,
-    listToMap as r,
+    folderOrFileExists as d,
+    listToMap as f,
     isEmptyObj as h,
     isString as v
 } from "./QyUtils.js";
@@ -32,19 +34,19 @@ import {
 import {
     bufferToNums as g,
     numsToBuffer as y,
-    applyAclChange as f,
+    applyAclChange as c,
     iterCmdKeys as S
 } from "./QyAcl.js";
 
 import {
-    QyJsonSaver as c
+    QyJsonSaver as C
 } from "./QyJsonSaver.js";
 
 import {
-    logger as C
+    logger as P
 } from "./QyLogger.js";
 
-let P = Array.isArray, x = Buffer.from("\n"), O = Buffer.from("'");
+let x = Array.isArray, O = Buffer.from("\n"), F = Buffer.from("'"), j = Buffer.from("z"), L = F[0], b = j[0];
 
 class t {
     constructor(t, a) {
@@ -52,12 +54,12 @@ class t {
             ...n("QySnapshots"),
             ...a
         }).vLOutdatedPercent = 2 * a.pMOutdatedPercent;
-        var e = s(t, "info");
+        var o = s(t, "info");
         Object.assign(this, {
             snapshotFolder: t,
-            snapshotInfoFolder: e,
+            snapshotInfoFolder: o,
             options: a,
-            qyJsonSaver: new c(e, a),
+            qyJsonSaver: new C(o, a),
             numToSnapshotMap: {},
             keyToSnapshotMap: {},
             fDMap: {},
@@ -70,73 +72,69 @@ class t {
     async loadSnapshotInfos() {
         let {
             snapshotInfoFolder: t,
-            qyJsonSaver: h,
-            numToSnapshotMap: f
+            qyJsonSaver: n,
+            numToSnapshotMap: r
         } = this;
-        if (await m(t)) {
+        if (await d(t)) {
             var a = (await l(t)).filter(t => /^[1-9]/.test(t)).map(t => parseInt(t)).sort((t, a) => t - a);
-            let p = 0, u = 0;
+            let e = 0, s = 0;
             await Promise.all(a.map(async t => {
-                var a = await h.loadFromFile(t + ".json"), e = f[t] = {
+                var a = await n.loadFromFile(t + ".json"), o = r[t] = {
                     snapshotNum: t,
                     info: a,
                     posMap: {},
                     outdatedCount: 0
-                };
-                ++this.snapshotCount, t > p && (p = t, u = a.maxChangeId);
-                var t = e, a = L(a = this, t), o = (C.info("Loading " + a), await d(a)), s = o.length, n = t.prefixPM = {};
-                for (let t = 0; t < s; ) {
-                    var r = t + V, i = o.toString("latin1", t, r), i = n[i] = [];
-                    t = g(o, r, i);
-                }
+                }, t = (++this.snapshotCount, t > e && (e = t, s = a.maxChangeId), 
+                this), a = o;
+                t = k(t, a), P.info("Loading " + t), a.prefixPM = await V(t);
             })), Object.assign(this, {
-                maxSnapshotNum: p,
-                snapshotMaxChangeId: u
+                maxSnapshotNum: e,
+                snapshotMaxChangeId: s
             });
             var {
-                numToSnapshotMap: e,
-                prefixMap: o
+                numToSnapshotMap: o,
+                prefixMap: i
             } = this;
-            for (r of a) {
-                var s, n = e[r], r = n.prefixPM;
-                for (s in r) {
-                    var i = o[s];
-                    i ? i.push(n) : o[s] = [ n ];
+            for (f of a) {
+                var p, u = o[f], f = u.prefixPM;
+                for (p in f) {
+                    var h = i[p];
+                    h ? h.push(u) : i[p] = [ u ];
                 }
             }
         }
     }
     closeAllFDs() {
         var t, a = this.fDMap;
-        for (t in a) o(a[t]), delete a[t];
+        for (t in a) e(a[t]), delete a[t];
     }
     saveKeyValueMap(t) {
-        var a, {
+        var a, o = this.options.minLenToCompress, {
             keyValueMap: e,
-            info: o
-        } = t, s = (++o.vLVersion, new M()), n = (s.start(this.getVLFilePath(t)), 
-        Object.keys(e).sort()), r = {};
-        let i = 0, p = 0, u;
-        for (a of n) {
-            var h, f = e[a];
+            info: s
+        } = t, n = (++s.vLVersion, new M()), r = (n.start(this.getVLFilePath(t)), 
+        Object.keys(e).sort()), i = {};
+        let p = 0, u = 0, f, h;
+        for (a of r) {
+            var l = e[a];
             let t;
-            null == f ? t = N : (u = Buffer.isBuffer(f) ? (s.save(f), f.length) : v(f) ? (h = Buffer.from(f), 
-            s.save(O), s.save(h), s.save(x), h.length + 2) : (h = Buffer.from(JSON.stringify(B(a) && !P(f) ? Object.keys(f) : f)), 
-            s.save(h), s.save(x), h.length + 1), t = [ i, u ], i += u), r[a] = t, 
-            ++p;
+            null == l ? t = K : (h = Buffer.isBuffer(l) ? (f = l, 0) : v(l) ? (f = l.length >= o ? (n.save(j), 
+            m.zstdCompressSync(l)) : (n.save(F), Buffer.from(l)), 1) : (f = Buffer.from(JSON.stringify(Q(a) && !x(l) ? Object.keys(l) : l)), 
+            0), n.save([ f, O ]), l = h + f.length, t = [ p, l ], p += l + 1), i[a] = t, 
+            ++u;
         }
         return Object.assign(t, {
-            posMap: r,
+            posMap: i,
             outdatedCount: 0
-        }), Object.assign(o, {
-            totalCount: p,
+        }), Object.assign(s, {
+            totalCount: u,
             outdatedCount: 0
-        }), Promise.all([ this.savePosMap(t, n), s.stop() ]);
+        }), Promise.all([ this.savePosMap(t, r), n.stop() ]);
     }
     addMaxSnapshot(t) {
         var a, {
-            numToSnapshotMap: e,
-            keyToSnapshotMap: o
+            numToSnapshotMap: o,
+            keyToSnapshotMap: e
         } = this, {
             posMap: s,
             keyValueMap: n,
@@ -146,77 +144,70 @@ class t {
             maxSnapshotNum: r,
             snapshotMaxChangeId: i
         }), n || s);
-        for (a in i) o[a] = t;
-        e[r] = t, ++this.snapshotCount;
+        for (a in i) e[a] = t;
+        o[r] = t, ++this.snapshotCount;
     }
     removeKey(t) {
-        var a = this.keyToSnapshotMap, e = a[t];
-        if (e) return this.removeKeyFromSnapshot(e, t), delete a[t], !0;
+        var a = this.keyToSnapshotMap, o = a[t];
+        if (o) return this.removeKeyFromSnapshot(o, t), delete a[t], !0;
     }
-    applyCmdObj(t, a, e) {
-        if (null != e) {
-            var o = e.length;
-            for (let t = 0; t < o; t += 2) {
-                var s = e[t], n = e[t + 1];
-                a[s] = null != n && B(s) ? r(n) : n;
+    applyCmdObj(t, a, o) {
+        if (null != o) {
+            var e = o.length;
+            for (let t = 0; t < e; t += 2) {
+                var s = o[t], n = o[t + 1];
+                a[s] = null != n && Q(s) ? f(n) : n;
             }
         }
         S(t, t => {
             t in a || (a[t] = this.getValueSync(t)), this.removeKey(t);
-        }), f(a, t);
+        }), c(a, t);
     }
     getValueSync(t) {
         var a = ((t, a) => {
             let {
-                keyToSnapshotMap: e,
-                prefixMap: o
-            } = t, s = e[a];
+                keyToSnapshotMap: o,
+                prefixMap: e
+            } = t, s = o[a];
             if (s) return s;
-            var n = b(a), r = o[n];
+            var n = I(a), r = e[n];
             if (r) for (;0 < r.length; ) {
                 s = r.pop();
-                var i = ((t, a, e) => {
-                    var o, {
+                var i = ((t, a, o) => {
+                    var e, {
                         posMap: s,
                         prefixPM: n
                     } = a;
-                    if (n = n[e]) return [ n, o ] = n, n = F(t, t.getPMFilePath(a), n, o), 
-                    t.loadPartialPosMapBuffer(a, e, n), s;
+                    if (n = n[o]) return [ n, e ] = n, n = T(t, t.getPMFilePath(a), n, e), 
+                    t.loadPartialPosMapBuffer(a, o, n), s;
                 })(t, s, n);
                 if (i && a in i) return s;
             }
         })(this, t);
         if (a) {
             var {
-                keyValueMap: e,
-                posMap: o
+                keyValueMap: o,
+                posMap: e
             } = a;
-            if (e) return e[t];
-            e = o[t];
-            if (e) {
-                var [ o, e ] = e;
-                if (0 != e) {
+            if (o) return o[t];
+            o = e[t];
+            if (o) {
+                var [ e, o ] = o;
+                if (0 != o) {
                     var s = this;
-                    if (a = s.getVLFilePath(a), s = F(s, a, o, e)) return s[0] == O[0] ? s.toString("utf8", 1, s.length - 1) : (a = JSON.parse(s), 
-                    B(t) ? r(a) : a);
+                    if (a = s.getVLFilePath(a), s = T(s, a, e, o)) return (a = s[0]) == L ? s.toString("utf8", 1) : a == b ? m.zstdDecompressSync(s.subarray(1)).toString() : (e = JSON.parse(s), 
+                    Q(t) ? f(e) : e);
                 }
             }
         }
     }
-    iterPMBuffer(a, e) {
-        var o = a.length;
-        for (let t = 0; t < o; ) {
-            var s = t + u, n = a.toString("latin1", t, s), r = [];
-            t = g(a, s, r), e(n, r);
-        }
-    }
-    loadPartialPosMapBuffer(e, t, a) {
+    loadPartialPosMapBuffer(o, t, a) {
         let {
-            posMap: o,
+            posMap: e,
             prefixPM: s
-        } = e, n = (delete s[t], this).keyToSnapshotMap;
-        this.iterPMBuffer(a, (t, a) => {
-            t in n ? ++e.outdatedCount : (o[t] = a, n[t] = e);
+        } = o, n = (delete s[t], this).keyToSnapshotMap;
+        N(a, (t, a) => {
+            t in n ? ++o.outdatedCount : (e[t] = a, n[t] = o);
         });
     }
     delSnapshot(t) {
@@ -224,7 +215,7 @@ class t {
         a(this, this.getPMFilePath(t));
     }
     calOutdates() {
-        var t, a = [], e = [], o = [], s = [], {
+        var t, a = [], o = [], e = [], s = [], {
             options: n,
             numToSnapshotMap: r
         } = this, {
@@ -232,24 +223,24 @@ class t {
             vLOutdatedPercent: p
         } = n;
         for (t in r) {
-            var u, h = r[t];
-            h.isOut || (100 == (u = T(h)) ? (this.delSnapshot(h), a.push(t)) : p < u ? (j(h) ? e : s).push(h) : Math.round(h.outdatedCount / h.info.totalCount * 100) > i ? o.push(h) : (u = this, 
-            h.info.totalCount < u.options.minSnapshotKeyCount && j(h) && e.push(h)));
+            var u, f = r[t];
+            f.isOut || (100 == (u = D(f)) ? (this.delSnapshot(f), a.push(t)) : p < u ? (B(f) ? o : s).push(f) : Math.round(f.outdatedCount / f.info.totalCount * 100) > i ? e.push(f) : (u = this, 
+            f.info.totalCount < u.options.minSnapshotKeyCount && B(f) && o.push(f)));
         }
-        return 1 == e.length && T(e[0]) < p && (e.length = 0), 0 < o.length && C.log("pMCompacts:" + o.map(t => t.snapshotNum)), 
-        0 < s.length && C.log("vLCompacts:" + s.map(t => t.snapshotNum)), 0 < e.length && C.log("combiningSnapshots:" + e.map(t => t.snapshotNum)), 
+        return 1 == o.length && D(o[0]) < p && (o.length = 0), 0 < e.length && P.log("pMCompacts:" + e.map(t => t.snapshotNum)), 
+        0 < s.length && P.log("vLCompacts:" + s.map(t => t.snapshotNum)), 0 < o.length && P.log("combiningSnapshots:" + o.map(t => t.snapshotNum)), 
         {
             emptySnapshotNumList: a,
-            combiningSnapshotList: e,
-            pMCompactSnapshotList: o,
+            combiningSnapshotList: o,
+            pMCompactSnapshotList: e,
             vLCompactSnapshotList: s
         };
     }
     getMaxCombinedSnapshot(t) {
-        let a, e;
-        for (var o of t) {
-            var s = o.snapshotNum;
-            (!e || e < s) && (a = o, e = s);
+        let a, o;
+        for (var e of t) {
+            var s = e.snapshotNum;
+            (!o || o < s) && (a = e, o = s);
         }
         return a;
     }
@@ -269,30 +260,30 @@ class t {
     }
     saveSnapshotInfo(t, a) {
         var {
-            outdatedCount: e,
-            totalCount: o
+            outdatedCount: o,
+            totalCount: e
         } = a;
-        return a.outdated = Math.round(100 * e / o) + "%", C.log("Saving snapshotInfo:" + t), 
+        return a.outdated = Math.round(100 * o / e) + "%", P.log("Saving snapshotInfo:" + t), 
         this.qyJsonSaver.saveToFile(a, t + ".json", !0);
     }
     savePosMap(t, a) {
-        var e, {
-            outdatedCount: o,
+        var o, {
+            outdatedCount: e,
             info: s,
             posMap: n
-        } = t, r = (s.outdatedCount += o, t.outdatedCount = 0, ++s.pMVersion, new M()), i = (r.start(this.getPMFilePath(t)), 
+        } = t, r = (s.outdatedCount += e, t.outdatedCount = 0, ++s.pMVersion, new M()), i = (r.start(this.getPMFilePath(t)), 
         t.prefixPM = {});
-        let p = 0, u = 0, h;
-        for (e of a = a || Object.keys(n).sort()) {
-            null == h ? h = b(e) : e.startsWith(h) || (i[h] = [ p, u ], h = b(e), 
+        let p = 0, u = 0, f;
+        for (o of a = a || Object.keys(n).sort()) {
+            null == f ? f = I(o) : o.startsWith(f) || (i[f] = [ p, u ], f = I(o), 
             p += u, u = 0);
-            var f = Buffer.from(e), l = y(n[e]);
-            r.save(f), r.save(l), u += f.length + l.length;
+            var h = Buffer.from(o), l = y(n[o]);
+            r.save([ h, l ]), u += h.length + l.length;
         }
-        return null != h && (i[h] = [ p, u ]), Promise.all([ r.stop(), ((t, a) => {
-            var e, o = new M(), s = (o.start(L(t, a)), a).prefixPM;
-            for (e in s) o.save(Buffer.from(e)), o.save(y(s[e]));
-            return o.stop();
+        return null != f && (i[f] = [ p, u ]), Promise.all([ r.stop(), ((t, a) => {
+            var o, e = new M(), s = (e.start(k(t, a)), a).prefixPM;
+            for (o in s) e.save([ Buffer.from(o), y(s[o]) ]);
+            return e.stop();
         })(this, t) ]);
     }
     isEmptySnapshot(t) {
@@ -304,73 +295,92 @@ class t {
     }
     removeKeyFromSnapshot(t, a) {
         var {
-            posMap: e,
-            keyValueMap: o
+            posMap: o,
+            keyValueMap: e
         } = t;
-        o && delete o[a], e && delete e[a], ++t.outdatedCount;
+        e && delete e[a], o && delete o[a], ++t.outdatedCount;
+    }
+}
+
+function N(a, o) {
+    var e = a.length;
+    for (let t = 0; t < e; ) {
+        var s = t + u, n = a.toString("latin1", t, s), r = [];
+        t = g(a, s, r), o(n, r);
+    }
+}
+
+async function V(t) {
+    var a = await r(t), o = a.length, e = {};
+    for (let t = 0; t < o; ) {
+        var s = t + w, n = e[a.toString("latin1", t, s)] = [];
+        t = g(a, s, n);
     }
+    return e;
 }
 
-function F(t, a, e, o) {
+function T(t, a, o, e) {
     var s = ((t, a) => {
-        let e = t.fDMap, o = e[a];
-        if (!o) {
-            C.info("Opening " + a);
+        let o = t.fDMap, e = o[a];
+        if (!e) {
+            P.info("Opening " + a);
             try {
-                o = e[a] = p(a, "r");
+                e = o[a] = p(a, "r");
             } catch (t) {
-                throw C.error(`Opening ${a} failed:`, t), t;
+                throw P.error(`Opening ${a} failed:`, t), t;
             }
         }
-        return o;
-    })(t, a), n = Buffer.allocUnsafe(o);
-    for (let t = 0; t < o; ) {
-        var r = i(s, n, t, o - t, e + t);
-        if (0 == r) return void C.error(`readSync ${a} at ${e + t} returning 0 byte`);
+        return e;
+    })(t, a), n = Buffer.allocUnsafe(e);
+    for (let t = 0; t < e; ) {
+        var r = i(s, n, t, e - t, o + t);
+        if (0 == r) return void P.error(`readSync ${a} at ${o + t} returning 0 byte`);
         t += r;
     }
     return n;
 }
 
-function j(t) {
+function B(t) {
     var {
         keyValueMap: a,
-        prefixPM: e
+        prefixPM: o
     } = t;
-    return a || null == e || h(t.prefixPM);
+    return a || null == o || h(t.prefixPM);
 }
 
-function L(t, a) {
+function k(t, a) {
     var {
         snapshotNum: a,
-        info: e
+        info: o
     } = a;
-    return s(t.snapshotFolder, "" + a, e.pMVersion + "_prefixPM.txt");
+    return s(t.snapshotFolder, "" + a, o.pMVersion + "_prefixPM.txt");
 }
 
 function a(t, a) {
-    var t = t.fDMap, e = t[a];
-    null != e && (o(e), delete t[a]);
+    var t = t.fDMap, o = t[a];
+    null != o && (e(o), delete t[a]);
 }
 
-let N = [ 0, 0 ], V = 3;
+let K = [ 0, 0 ], w = 3;
 
-function b(t) {
-    return t.substring(0, V);
+function I(t) {
+    return t.substring(0, w);
 }
 
-function B(t) {
+function Q(t) {
     return !t.startsWith("c");
 }
 
-function T(t) {
+function D(t) {
     var {
         outdatedCount: t,
         info: a
-    } = t, e = a.totalCount, t = t + a.outdatedCount;
-    return t == e ? 100 : Math.round(t / e * 100);
+    } = t, o = a.totalCount, t = t + a.outdatedCount;
+    return t == o ? 100 : Math.round(t / o * 100);
 }
 
 export {
-    t as QySnapshots
+    t as QySnapshots,
+    N as iterPMBuffer,
+    V as loadPrefixPMFile
 };

package/src/QyUnionNode.js

@@ -1,6 +1,6 @@
 class i {
-    constructor(i = 1 / 0, e = 0) {
-        Object.assign(this, {
+    constructor(i, e = 0) {
+        i = i || 1 / 0, Object.assign(this, {
             limit: i,
             cursor: e,
             added: 0,

package/src/rpc/evalCode.js

@@ -1,9 +1,9 @@
-module.exports = code => {
+module.exports = r => {
     try {
-        return eval(code);
-    } catch (exception) {
+        return new Function("dataFolder", `return (${r})();`)(dataFolder);
+    } catch (r) {
         return {
-            error: exception.message
+            error: r.message
         };
     }
 };

package/src/rpc/getFileContent.js

@@ -1,16 +1,16 @@
-let o = new Error("No such file");
+let a = new Error("No such file");
 
 module.exports = ({
     filePath: e,
     cChangeId: n
 }) => {
-    var t, l, i, e = qyDB.fileP(e);
+    var t, l, o, e = dataFolder.fileP(e);
     return e ? ({
         fileContent: e,
         cChangeId: t,
         fileContentKey: l
-    } = e, i = {}, n != t && (Object.assign(i, {
+    } = e, o = {}, n != t && (Object.assign(o, {
         fileContent: e,
         cChangeId: t
-    }), null == n) && (i.fileContentKey = l), i) : o;
+    }), null == n) && (o.fileContentKey = l), o) : a;
 };

package/src/rpc/getSubItems.js

@@ -1,16 +1,16 @@
-let h = new Error("No such dir");
+let s = new Error("No such dir");
 
 module.exports = ({
-    dirPath: e,
-    dChangeId: a,
-    fChangeId: d
+    dirPath: a,
+    dChangeId: d,
+    fChangeId: e
 }) => {
-    var s, e = qyDB.dirP(e);
-    return e ? (s = {}, a || d || (s.fullPathHash = e.fullPathHash), a != e.dChangeId && Object.assign(s, {
-        subdirList: e.sortedSubdirList.map(e => e.name),
-        dChangeId: e.dChangeId
-    }), d != e.fChangeId && Object.assign(s, {
-        fileList: e.sortedFileList.map(e => e.name),
-        fChangeId: e.fChangeId
-    }), s) : h;
+    var r, a = dataFolder.dirP(a);
+    return a ? (r = {}, d || e || (r.fullPathHash = a.fullPathHash), d != a.dChangeId && Object.assign(r, {
+        subdirList: a.sortedSubdirList.map(a => a.name),
+        dChangeId: a.dChangeId
+    }), e != a.fChangeId && Object.assign(r, {
+        fileList: a.sortedFileList.map(a => a.name),
+        fChangeId: a.fChangeId
+    }), r) : s;
 };

package/src/rpc/getTriggerFiles.js

@@ -0,0 +1 @@
+module.exports = () => dataFolder.fileListToTree(dataFolder.triggerFileList);

package/src/rpc/queryFiles.js

@@ -6,5 +6,5 @@ module.exports = queryText => {
         limit,
         cursor
     } = eval("((x)=>x)(" + queryText + ")");
-    return qyDB.fileListToTree(qyDB.queryFiles(folder, value, limit, cursor), view);
+    return dataFolder.queryTree(folder, value, view, limit, cursor);
 };

package/src/soc_pokec.js

@@ -33,21 +33,21 @@ let f = "./tmp/soc_pokec", w = new e(f);
     if (console.time("run"), await w.start(), e) {
         {
             console.time("readData");
-            var a, r = await c(m, "r"), s = {};
-            for await (o of r.readLines()) {
-                var [ o, i ] = o.split("\t"), n = s[o];
-                n ? n[i] = 1 : s[o] = {
+            var a, s = await c(m, "r"), r = {};
+            for await (o of s.readLines()) {
+                var [ o, i ] = o.split("\t"), n = r[o];
+                n ? n[i] = 1 : r[o] = {
                     [i]: 1
                 };
             }
             console.timeEnd("readData"), console.time("insertData");
             let e = w.batch, t = 0;
-            for (a in s) {
-                var l = s[a];
+            for (a in r) {
+                var l = r[a];
                 e.insertP([ "relationships", a ], l), 3e4 < ++t && (await e.run(!0), 
                 t = 0);
             }
-            await e.run(!0), console.timeEnd("insertData");
+            await e.run(!0), console.timeEnd("insertData"), s.close();
         }
         await 0;
     }