datafolder 1.0.5 → 1.0.7

package/src/QyKVData.js

@@ -7,7 +7,7 @@ import {
 } from "node:fs/promises";
 
 import {
-    getDefaultOptions as e
+    getDefaultOptions as s
 } from "./QyDefaultOptions.js";
 
 import {
@@ -24,116 +24,119 @@ import {
 
 import {
     arrayLast as y,
-    readBufferAsync as g,
+    readBufferAsync as C,
     sleep as c
 } from "./QyUtils.js";
 
-class t {
-    constructor(t, a, o) {
-        a = {
-            ...e("QyKVData"),
-            ...a
+class a {
+    constructor(a, t, o) {
+        t = {
+            ...s("QyKVData"),
+            ...t
         };
-        var s = n(t, "snapshot");
+        var e = n(a, "snapshot");
         Object.assign(this, {
             qyCache: o,
-            options: a,
-            kvFolder: t,
-            snapshotFolder: s,
-            qyKVDataSaver: new p(this, t, a),
-            qySnapshotCompactor: new i(this, t, a),
-            qyKVDataCleaner: new r(this, t, a),
+            options: t,
+            kvFolder: a,
+            snapshotFolder: e,
+            qyKVDataSaver: new p(this, a, t),
+            qySnapshotCompactor: new i(this, a, t),
+            qyKVDataCleaner: new r(this, a, t),
             compactState: 1,
             compactMaxChangeId: 0,
             snapshotSavedChangeId: 0,
-            snapshotInfoNumList: [],
             emptySnapshotNumList: []
         });
     }
-    start(t, a, o, s) {
+    start(a, t, o, e) {
         var {
             options: n,
-            qyKVDataSaver: e,
+            qyKVDataSaver: s,
             qySnapshotCompactor: p,
             qyKVDataCleaner: i
         } = this, {
             maxSnapshotNum: r,
             snapshotMaxChangeId: h
-        } = o, e = (e.start(t, r, a, s), p.start(), i.start(), setInterval(() => l(this), n.compactSnapshotsInterval)), r = setInterval(() => (async o => {
+        } = o, s = (s.start(a, r, t, e), p.start(), i.start(), setInterval(() => l(this), n.compactSnapshotsInterval)), r = setInterval(() => (async o => {
             if (!o.unloadingMemory) {
                 o.unloadingMemory = !0;
                 var {
-                    qyCache: s,
+                    qyCache: e,
                     options: n
                 } = o;
-                let t = 0, a = 0;
-                if (s) {
-                    var e, p = n.maxFileCountToUnload, n = s.rootDir;
-                    a = m(o, n), t = a;
-                    for (e of n.allLoadedSubdirs()) if ((a += m(o, e)) > p && (t += a, 
-                    a = 0, await c(0), o.isStopping)) break;
+                let a = 0, t = 0;
+                if (e) {
+                    var s, p = n.maxFileCountToUnload, n = e.rootDir;
+                    t = d(o, n), a = t;
+                    for (s of n.allLoadedSubdirs()) if ((t += d(o, s)) > p && (a += t, 
+                    t = 0, await c(0), o.isStopping)) break;
                 }
-                t += a, global.gc && global.gc(), o.unloadingMemory = !1;
+                a += t, global.gc && global.gc(), o.unloadingMemory = !1;
             }
         })(this), n.unloadMemInterval);
         return Object.assign(this, {
-            maxChangeId: t,
+            maxChangeId: a,
             snapshotSavedChangeId: h,
             qySnapshots: o,
-            aclKeyValueMap: s,
-            compactInterval: e,
+            aclKeyValueMap: e,
+            compactInterval: s,
             unloadInterval: r
-        }), n.bgLoadPosMaps ? (async t => {
-            for (var a of Object.keys(t.qySnapshots.numToSnapshotMap).sort((t, a) => a - t)) {
-                if (await async function e(p, i, r, h, c, l) {
+        }), n.bgLoadPosMaps ? (async a => {
+            var t, {
+                qySnapshots: o,
+                options: e
+            } = a;
+            for (t of Object.keys(o.numToSnapshotMap).sort((a, t) => t - a)) {
+                if (await async function s(p, i, r, h, c, l) {
                     if (p.isStopping) return void (l && l.close());
-                    let m = p.qySnapshots;
-                    let u = m.numToSnapshotMap;
-                    let d = u[i];
-                    if (!d) return void (l && l.close());
+                    let d = p.qySnapshots;
+                    let m = d.numToSnapshotMap;
+                    let u = m[i];
+                    if (!u) return void (l && l.close());
                     let {
                         info: S,
                         prefixPM: f
-                    } = d;
+                    } = u;
                     r = r || Object.keys(f).sort().reverse();
                     if (0 == r.length) return void (l && l.close());
-                    if (null == h || h != S.pMVersion) return l && l.close(), c = m.getPMFilePath(d), 
-                    e(p, i, r, S.pMVersion, c, await v(c));
+                    if (null == h || h != S.pMVersion) return l && l.close(), c = d.getPMFilePath(u), 
+                    s(p, i, r, S.pMVersion, c, await v(c));
                     for (;0 < r.length; ) {
-                        let s = y(r), n = f[s];
+                        let e = y(r), n = f[e];
                         if (n) {
-                            let [ t, a ] = n, o = await g(c, l, t, a);
-                            if (p.isStopping || !u[i]) return void l.close();
-                            if (h != S.pMVersion) return l.close(), e(p, i, r);
-                            s in f && m.loadPartialPosMapBuffer(d, s, o);
+                            let [ a, t ] = n, o = await C(c, l, a, t);
+                            if (p.isStopping || !m[i]) return void l.close();
+                            if (h != S.pMVersion) return l.close(), s(p, i, r);
+                            e in f && d.loadPartialPosMapBuffer(u, e, o);
                         }
                         r.pop();
                     }
                     l.close();
                     logger.info("bgLoadPosMap:" + i);
-                }(t, a), t.isStopping) break;
-                if (await c(0), t.isStopping) break;
+                }(a, t), a.isStopping) break;
+                if (await c(0), a.isStopping) break;
             }
-            !t.isStopping && t.options.compactAtStart && l(t);
+            !a.isStopping && e.compactAtStart && l(a);
         })(this) : n.compactAtStart && l(this), this;
     }
     async stop() {
         this.isStopping = !0;
         var {
-            qyKVDataSaver: t,
-            qySnapshotCompactor: a,
+            qyKVDataSaver: a,
+            qySnapshotCompactor: t,
             qySnapshots: o,
-            compactInterval: s,
+            compactInterval: e,
             unloadInterval: n,
-            qyKVDataCleaner: e
+            qyKVDataCleaner: s
         } = this;
-        clearInterval(s), clearInterval(n), await t.stop(), o && o.closeAllFDs(), 
-        await a.stop(), await e.stop();
+        clearInterval(e), clearInterval(n), this.onSnapshotSavedChangeId(await a.stop()), 
+        o && o.closeAllFDs(), await t.stop(), await s.stop();
     }
-    save(t, a, o, s) {
+    save(a, t, o, e) {
         var n = this.qyKVDataSaver;
-        if (s) return n.callSave(t, a, o);
-        n.castSave(t, a, o);
+        if (e) return n.callSave(a, t, o);
+        n.castSave(a, t, o);
     }
     increaseChangeId() {
         return ++this.maxChangeId;
@@ -141,183 +144,165 @@ class t {
     decreaseChangeId() {
         --this.maxChangeId;
     }
-    getValueSync(t) {
+    getValueSync(a) {
         var {
-            qySnapshots: a,
+            qySnapshots: t,
             aclKeyValueMap: o
         } = this;
-        return t in o ? o[t] : a.getValueSync(t);
+        return a in o ? o[a] : t.getValueSync(a);
     }
-    removeKey(t) {
+    removeKey(a) {
         var {
-            qySnapshots: a,
+            qySnapshots: t,
             aclKeyValueMap: o
         } = this;
-        if (!(t in o)) return a.removeKey(t);
-        o[t] = void 0;
+        if (!(a in o)) return t.removeKey(a);
+        o[a] = void 0;
     }
-    onSnapshotSavedChangeId(t, a) {
-        if (this.snapshotSavedChangeId = t, 3 == this.compactState && t >= this.compactMaxChangeId && (w(this), 
-        this.compactState = 1), a) {
-            var o, t = a, s = this.qySnapshots, n = s.keyToSnapshotMap;
-            for (o in t.posMap) {
-                var e = n[o];
-                e && s.removeKeyFromSnapshot(e, o);
+    onSnapshotSavedChangeId(a, t) {
+        this.snapshotSavedChangeId = a;
+        var {
+            compactState: o,
+            compactMaxChangeId: e,
+            qyKVDataSaver: n
+        } = this;
+        if (3 == o && e <= a && (K(this), this.compactState = 1), t) {
+            var s, o = t, p = this.qySnapshots, i = p.keyToSnapshotMap;
+            for (s in o.posMap) {
+                var r = i[s];
+                r && p.removeKeyFromSnapshot(r, s);
             }
-            s.addMaxSnapshot(t), logger.info("addPMOnlySnapshot: " + s.snapshotMaxChangeId), 
-            this.qyKVDataSaver.releaseSnapshot(a.snapshotNum);
+            p.addMaxSnapshot(o), logger.info("addPMOnlySnapshot: " + p.snapshotMaxChangeId), 
+            n.releaseSnapshot(t.snapshotNum);
         }
     }
-    onCompactUpdates(t) {
+    onCompactUpdates(a) {
         if (2 == this.compactState) {
-            var u = this, {
-                pMUpdates: t,
+            var l = this, {
+                pMUpdates: a,
                 vLUpdates: d,
-                combineUpdate: S
-            } = t = t;
-            if (t) {
-                var a, o = t, {
-                    qySnapshots: s,
-                    snapshotInfoNumList: n,
-                    emptySnapshotNumList: e
-                } = u, p = s.numToSnapshotMap;
-                for (a in o) {
-                    var i = p[a];
-                    if (s.isEmptySnapshot(i)) s.delSnapshot(i), e.push(a); else {
-                        var r, h = o[a], {
-                            prefixPM: c,
-                            info: l
-                        } = i;
-                        for (r in c) c[r] = h[r];
-                        ++l.pMVersion, (t => {
-                            var {
-                                info: a,
-                                compactOutedCount: o
-                            } = t;
-                            a.outdatedCount += o, t.outdatedCount -= o, t.compactOutedCount = 0;
-                        })(i), n.push(a);
-                    }
+                combineUpdate: m
+            } = a = a;
+            if (a) {
+                var t = a, o = l.qySnapshots.numToSnapshotMap;
+                for (s in t) {
+                    var e, n = t[s], s = o[s], {
+                        prefixPM: p,
+                        info: i
+                    } = s;
+                    for (e in p) p[e] = n[e];
+                    ++i.pMVersion, (a => {
+                        var {
+                            info: t,
+                            compactOutedCount: o
+                        } = a;
+                        t.outdatedCount += o, a.outdatedCount -= o, a.compactOutedCount = 0;
+                    })(s);
                 }
             }
             if (d) {
-                var m, f = d, {
-                    qySnapshots: v,
-                    snapshotInfoNumList: y,
-                    emptySnapshotNumList: g
-                } = t = u, C = v.numToSnapshotMap;
-                for (m in f) {
-                    var M = C[m];
-                    if (v.isEmptySnapshot(M)) v.delSnapshot(M), g.push(m); else {
-                        var I, q, {
-                            posMap: V,
-                            prefixPM: L
-                        } = f[m], {
-                            posMap: K,
-                            prefixPM: x,
-                            info: D
-                        } = M;
-                        for (I in K) K[I] = V[I];
-                        for (q in x) x[q] = L[q];
-                        ++D.pMVersion, ++D.vLVersion, j(M), y.push(m);
-                    }
+                var r = d, h = (a = l).qySnapshots.numToSnapshotMap;
+                for (v in r) {
+                    var c, u, {
+                        posMap: S,
+                        prefixPM: f
+                    } = r[v], v = h[v], {
+                        posMap: y,
+                        prefixPM: C,
+                        info: g
+                    } = v;
+                    for (c in y) y[c] = S[c];
+                    for (u in C) C[u] = f[u];
+                    ++g.pMVersion, ++g.vLVersion, V(v);
                 }
             }
-            if (S) {
-                var b, d = S;
-                let {
-                    qySnapshots: s,
-                    snapshotInfoNumList: t,
-                    emptySnapshotNumList: a
-                } = u, {
+            if (m) {
+                var M, d = m;
+                let e = l.qySnapshots, {
                     numToSnapshotMap: n,
-                    keyToSnapshotMap: e
-                } = s, {
-                    snapshotNum: o,
-                    combinedSnapshotNumList: p,
-                    posMap: i
-                } = d, r = n[o], {
-                    posMap: h,
-                    info: c
-                } = r;
-                for (b in h) h[b] = i[b];
-                j(r);
-                let l = c.totalCount, m = r.outdatedCount;
-                for (let o of p) {
-                    var N, O = n[o];
+                    keyToSnapshotMap: s
+                } = e, {
+                    snapshotNum: a,
+                    combinedSnapshotNumList: t,
+                    posMap: p
+                } = d, i = n[a], {
+                    posMap: r,
+                    info: o
+                } = i;
+                for (M in r) r[M] = p[M];
+                V(i);
+                let h = o.totalCount, c = i.outdatedCount;
+                for (let o of t) {
+                    var I, q = n[o];
                     let {
-                        posMap: t,
-                        info: a
-                    } = O;
-                    for (N in t) h[N] = i[N], e[N] = r;
-                    j(O), l += a.totalCount, m += O.outdatedCount, s.delSnapshot(O);
+                        posMap: a,
+                        info: t
+                    } = q;
+                    for (I in a) r[I] = p[I], s[I] = i;
+                    V(q), h += t.totalCount, c += q.outdatedCount, e.delSnapshot(q);
                 }
-                a.push(...p), (l == m ? a : (++c.pMVersion, ++c.vLVersion, c.totalCount = l, 
-                r.outdatedCount = m, t)).push(o);
+                ++o.pMVersion, ++o.vLVersion, o.totalCount = h, i.outdatedCount = c;
             }
-            this.snapshotSavedChangeId >= this.compactMaxChangeId ? (w(this), this.compactState = 1) : this.compactState = 3;
+            this.snapshotSavedChangeId >= this.compactMaxChangeId ? (K(this), this.compactState = 1) : this.compactState = 3;
         }
     }
 }
 
-function j(t) {
+function V(a) {
     var {
-        info: a,
+        info: t,
         compactOutedCount: o
-    } = t;
-    a.totalCount -= a.outdatedCount + o, a.outdatedCount = 0, t.outdatedCount -= o, 
-    t.compactOutedCount = 0;
+    } = a;
+    t.totalCount -= t.outdatedCount + o, t.outdatedCount = 0, a.outdatedCount -= o, 
+    a.compactOutedCount = 0;
 }
 
-function l(t) {
-    if (1 == t.compactState) {
+function l(a) {
+    if (1 == a.compactState) {
         var {
-            maxChangeId: a,
+            maxChangeId: t,
             qySnapshots: o,
-            qySnapshotCompactor: s
-        } = t, {
+            qySnapshotCompactor: e
+        } = a, {
             emptySnapshotNumList: o,
             pMCompactSnapshotList: n,
-            vLCompactSnapshotList: e,
+            vLCompactSnapshotList: s,
             combiningSnapshotList: p
         } = o.calOutdates();
-        if (0 < p.length || 0 < n.length || 0 < e.length) {
-            Object.assign(t, {
+        if (0 < p.length || 0 < n.length || 0 < s.length) {
+            Object.assign(a, {
                 compactState: 2,
-                compactMaxChangeId: a
+                compactMaxChangeId: t
             });
-            for (var i of [ n, e, p ]) for (var r of i) r.compactOutedCount = r.outdatedCount;
-            s.compactSnapshots(n, e, p);
+            for (var i of [ n, s, p ]) for (var r of i) r.compactOutedCount = r.outdatedCount;
+            e.compactSnapshots(n, s, p);
         }
-        0 < o.length && (t.emptySnapshotNumList.push(...o), t.snapshotSavedChangeId >= a ? h(t) : 1 == t.compactState && Object.assign(t, {
+        0 < o.length && (a.emptySnapshotNumList.push(...o), a.snapshotSavedChangeId >= t ? h(a) : 1 == a.compactState && Object.assign(a, {
             compactState: 3,
-            compactMaxChangeId: a
+            compactMaxChangeId: t
         }));
     }
 }
 
-function m(t, a) {
-    var o = t.qySnapshots.snapshotMaxChangeId;
-    let s = 0;
-    if (a.fileMapLoaded) for (var n of a.fileList) ++s, n.cChangeId <= o && (n.visited ? n.visited = !1 : n.unloadFileContent());
-    return s;
+function d(a, t) {
+    var o = a.qySnapshots.snapshotMaxChangeId;
+    let e = 0;
+    if (t.fileMapLoaded) for (var n of t.fileList) ++e, n.cChangeId <= o && (n.visited ? n.visited = !1 : n.unloadFileContent());
+    return e;
 }
 
-function h(t) {
+function h(a) {
     var {
-        qyKVDataCleaner: t,
-        emptySnapshotNumList: a
-    } = t;
-    0 < a.length && (t.removeSnapshotsFiles(a), a.length = 0);
+        qyKVDataCleaner: a,
+        emptySnapshotNumList: t
+    } = a;
+    0 < t.length && (a.removeSnapshotsFiles(t), t.length = 0);
 }
 
-function w(t) {
-    var {
-        qySnapshotCompactor: a,
-        snapshotInfoNumList: o
-    } = t;
-    0 < o.length && (a.saveSnapshotInfos(o), o.length = 0), h(t);
+function K(a) {
+    a.qySnapshotCompactor.saveSnapshotInfos(), h(a);
 }
 
 export {
-    t as QyKVData
+    a as QyKVData
 };

package/src/QyKVDataCleaner_Worker.js

@@ -5,7 +5,7 @@ import {
 import {
     rm as m,
     rename as r,
-    readdir as d
+    readdir as f
 } from "node:fs/promises";
 
 import {
@@ -17,8 +17,8 @@ import {
 } from "./QyMessageWorker.js";
 
 import {
-    folderOrFileExists as f,
-    ensureDir as h
+    folderOrFileExists as h,
+    ensureDir as d
 } from "./QyUtils.js";
 
 import {
@@ -34,7 +34,7 @@ class s extends e {
         super(s = {
             ...o("QyKVDataCleaner_Worker"),
             ...s
-        }, [ "start", "stop" ], [ "cleanAclFiles", "cleanPMFiles", "cleanVLFiles", "removeSnapshotsFiles" ]);
+        }, [], [ "cleanAclFiles", "cleanPMFiles", "cleanVLFiles", "removeSnapshotsFiles" ]);
         var s = c(e, "history"), a = c(e, "snapshot");
         Object.assign(this, {
             kvFolder: e,
@@ -49,14 +49,14 @@ class s extends e {
     _op_stop() {}
     async _op_cleanAclFiles(e) {
         var s = this, a = e, e = s.aclFolder;
-        if (await f(e)) {
+        if (await h(e)) {
             var o, r = [];
-            for (o of (await d(e)).filter(e => /^[1-9]/.test(e))) parseInt(o) <= a && r.push((async (e, s) => {
+            for (o of (await f(e)).filter(e => /^[1-9]/.test(e))) parseInt(o) <= a && r.push((async (e, s) => {
                 var {
                     aclFolder: a,
                     historyAclFolder: o
                 } = e;
-                return await h(o), F(e, s, a, o);
+                return await d(o), F(e, s, a, o);
             })(s, o));
             0 < r.length && await Promise.allSettled(r);
         }
@@ -72,11 +72,12 @@ class s extends e {
     }
     async _op_removeSnapshotsFiles(e) {
         let {
-            snapshotFolder: s,
-            snapshotInfoFolder: a
+            snapshotFolder: a,
+            snapshotInfoFolder: o
         } = this;
-        await Promise.all(e.map(async e => {
-            await m(c(a, e + ".json")), await m(c(s, "" + e), {
+        await Promise.all(e.map(async s => {
+            await m(c(o, s + ".json")), await Promise.all([ "pM", "prefixPM", "vL" ].map(e => l(this, s, 1 / 0, e))), 
+            await m(c(a, "" + s), {
                 recursive: !0
             });
         }));
@@ -96,13 +97,13 @@ async function l(s, a, o, r) {
         snapshotFolder: e,
         historySnapshotFolder: t
     } = s, i = c(e, "" + a);
-    if (await f(i)) {
+    if (await h(i)) {
         let e;
         var l, n = [];
-        for (l of await d(i)) {
+        for (l of await f(i)) {
             var p = new RegExp("^([0-9]+)_" + r).exec(l);
             p && (p = parseInt(p[1])) < o && (1 == p ? (e || (e = c(t, "" + a), 
-            await h(e)), n.push(F(s, l, i, e))) : (p = c(i, l), logger.info("Removing " + p), 
+            await d(e)), n.push(F(s, l, i, e))) : (p = c(i, l), logger.info("Removing " + p), 
             n.push(m(p))));
         }
         0 < n.length && await Promise.allSettled(n);

package/src/QyKVDataSaver_Worker.js

@@ -7,7 +7,7 @@ import {
 } from "./QyDefaultOptions.js";
 
 import {
-    QyAclSaver as n
+    QyAclSaver as o
 } from "./QyAclSaver.js";
 
 import {
@@ -15,11 +15,11 @@ import {
 } from "./QyMessageWorker.js";
 
 import {
-    QySnapshotSaver as o
+    QySnapshotSaver as n
 } from "./QySnapshotSaver.js";
 
 import {
-    delCmdKey as m
+    delCmdKey as u
 } from "./QyAcl.js";
 
 class s extends a {
@@ -33,32 +33,32 @@ class s extends a {
         }), Object.assign(this, {
             kvFolder: a,
             aclFolder: t(a, "acl"),
-            qyAclSaver: new n(this, s),
-            qySnapshotSaver: new o(this, a, s)
+            qyAclSaver: new o(this, s),
+            qySnapshotSaver: new n(this, a, s)
         });
     }
-    start(a, s, t, e) {
+    _op_start(a, s, t, e) {
         var {
-            qyAclSaver: n,
-            qySnapshotSaver: o
+            qyAclSaver: o,
+            qySnapshotSaver: n
         } = this;
         Object.assign(this, {
             maxChangeId: a,
             snapshotMaxChangeId: a,
             currentAclNum: (t || s) + 1
-        }), n.start(u(this)), o.start(a, t, e);
+        }), o.start(m(this)), n.start(a, t, e);
     }
-    async stop() {
+    async _op_stop() {
         var {
             qyAclSaver: a,
             options: s,
             qySnapshotSaver: t,
             currentAclNum: e,
-            maxChangeId: n,
-            snapshotMaxChangeId: o
+            maxChangeId: o,
+            snapshotMaxChangeId: n
         } = this;
-        this.isStopping = !0, await a.stop(), s.saveSnapshotAtStop && o < n && (this.isSavingSnapshot && await new Promise(a => this.saveSnapshotResolve = a), 
-        logger.log(`Saving snapshot ${e} at stop.`), t.saveSnapshot(e)), await t.stop();
+        return this.isStopping = !0, await a.stop(), s.saveSnapshotAtStop && n < o && (this.isSavingSnapshot && await new Promise(a => this.saveSnapshotResolve = a), 
+        logger.log(`Saving snapshot ${e} at stop.`), t.saveSnapshot(e)), t.stop();
     }
     callSave(a, s, t) {
         return r(this, a, s, t, !0);
@@ -74,9 +74,9 @@ class s extends a {
     }
 }
 
-function r(a, s, t, e, n) {
+function r(a, s, t, e, o) {
     var {
-        qyAclSaver: o,
+        qyAclSaver: n,
         qySnapshotSaver: r,
         options: h,
         snapshotMaxChangeId: i,
@@ -84,20 +84,20 @@ function r(a, s, t, e, n) {
         isSavingSnapshot: S
     } = a;
     let v;
-    n ? v = o.callSave(s, t) : o.castSave(s, t);
+    o ? v = n.callSave(s, t) : n.castSave(s, t);
     var l = t, c = e;
     if (c) {
         var g = c.length;
-        for (let a = 0; a < g; a += 2) m(l, c[a]);
+        for (let a = 0; a < g; a += 2) u(l, c[a]);
     }
     return r.castSave(s, t, e), a.maxChangeId = s, !S && s - i >= h.snapshotMaxChangeCount && (++a.currentAclNum, 
-    o.switch(u(a)), r.saveSnapshot(p), Object.assign(a, {
+    n.switch(m(a)), r.saveSnapshot(p), Object.assign(a, {
         snapshotMaxChangeId: s,
         isSavingSnapshot: !0
     })), v;
 }
 
-function u(a) {
+function m(a) {
     return t(a.aclFolder, a.currentAclNum + "_acl.txt");
 }