datafolder 1.0.9 → 1.0.10

package/src/QyKVData.js

@@ -3,7 +3,7 @@ import {
 } from "node:path";
 
 import {
-    open as v
+    open as f
 } from "node:fs/promises";
 
 import {
@@ -11,11 +11,11 @@ import {
 } from "./QyDefaultOptions.js";
 
 import {
-    QyKVDataSaver as p
+    QyKVDataSaver as i
 } from "./QyKVDataSaver.js";
 
 import {
-    QySnapshotCompactor as i
+    QySnapshotCompactor as p
 } from "./QySnapshotCompactor.js";
 
 import {
@@ -24,11 +24,32 @@ import {
 
 import {
     arrayLast as y,
-    readBufferAsync as g,
+    readBufferAsync as C,
     sleep as c
 } from "./QyUtils.js";
 
+import {
+    logger as g
+} from "./QyLogger.js";
+
 class a {
+    kvFolder;
+    snapshotFolder;
+    qyCache;
+    options;
+    qyKVDataSaver;
+    qySnapshotCompactor;
+    qyKVDataCleaner;
+    compactState = 1;
+    compactMaxChangeId = 0;
+    snapshotSavedChangeId = 0;
+    maxChangeId = 0;
+    emptySnapshotNumVersionList = [];
+    isStopping = !1;
+    qySnapshots;
+    aclKeyValueMap;
+    compactInterval;
+    unloadInterval;
     constructor(a, t, o) {
         t = {
             ...s("QyKVData"),
@@ -40,25 +61,21 @@ class a {
             options: t,
             kvFolder: a,
             snapshotFolder: e,
-            qyKVDataSaver: new p(this, a, t),
-            qySnapshotCompactor: new i(this, a, t),
-            qyKVDataCleaner: new r(this, a, t),
-            compactState: 1,
-            compactMaxChangeId: 0,
-            snapshotSavedChangeId: 0,
-            emptySnapshotNumList: []
+            qyKVDataSaver: new i(this, a, t),
+            qySnapshotCompactor: new p(this, a, t),
+            qyKVDataCleaner: new r(this, a, t)
         });
     }
     start(a, t, o, e) {
         var {
             options: n,
             qyKVDataSaver: s,
-            qySnapshotCompactor: p,
-            qyKVDataCleaner: i
+            qySnapshotCompactor: i,
+            qyKVDataCleaner: p
         } = this, {
             maxSnapshotNum: r,
             snapshotMaxChangeId: h
-        } = o, s = (s.start(a, r, t, e), p.start(), i.start(), setInterval(() => l(this), n.compactSnapshotsInterval)), r = setInterval(() => (async o => {
+        } = o, s = (s.start(a, r, t, e), i.start(), p.start(), setInterval(() => l(this), n.compactSnapshotsInterval)), r = setInterval(() => (async o => {
             if (!o.unloadingMemory) {
                 o.unloadingMemory = !0;
                 var {
@@ -67,9 +84,9 @@ class a {
                 } = o;
                 let a = 0, t = 0;
                 if (e) {
-                    var s, p = n.maxFileCountToUnload, n = e.rootDir;
+                    var s, i = n.maxFileCountToUnload, n = e.rootDir;
                     t = d(o, n), a = t;
-                    for (s of n.allLoadedSubdirs()) if ((t += d(o, s)) > p && (a += t, 
+                    for (s of n.allLoadedSubdirs()) if ((t += d(o, s)) > i && (a += t, 
                     t = 0, await c(0), o.isStopping)) break;
                 }
                 a += t, global.gc && global.gc(), o.unloadingMemory = !1;
@@ -89,32 +106,32 @@ class a {
             } = a;
             console.time("bgLoadPosMaps");
             for (t of Object.keys(o.numToSnapshotMap).sort((a, t) => t - a)) {
-                if (await async function s(p, i, r, h, c, l) {
-                    if (p.isStopping) return void (l && l.close());
-                    let d = p.qySnapshots;
+                if (await async function s(i, p, r = void 0, h = void 0, c = void 0, l = void 0) {
+                    if (i.isStopping) return void (l && l.close());
+                    let d = i.qySnapshots;
                     let m = d.numToSnapshotMap;
-                    let u = m[i];
-                    if (!u) return void (l && l.close());
+                    let S = m[p];
+                    if (!S) return void (l && l.close());
                     let {
-                        info: S,
-                        prefixPM: f
-                    } = u;
-                    r = r || Object.keys(f).sort().reverse();
+                        info: u,
+                        prefixPM: v
+                    } = S;
+                    r = r || Object.keys(v).sort().reverse();
                     if (0 == r.length) return void (l && l.close());
-                    if (null == h || h != S.pMVersion) return l && l.close(), c = d.getPMFilePath(u), 
-                    s(p, i, r, S.pMVersion, c, await v(c));
+                    if (null == h || h != u.pMVersion) return l && l.close(), c = d.getPMFilePath(S), 
+                    s(i, p, r, u.pMVersion, c, await f(c));
                     for (;0 < r.length; ) {
-                        let e = y(r), n = f[e];
+                        let e = y(r), n = v[e];
                         if (n) {
-                            let [ a, t ] = n, o = await g(c, l, a, t);
-                            if (p.isStopping || !m[i]) return void l.close();
-                            if (h != S.pMVersion) return l.close(), s(p, i, r);
-                            e in f && d.loadPartialPosMapBuffer(u, e, o);
+                            let [ a, t ] = n, o = await C(c, l, a, t);
+                            if (i.isStopping || !m[p]) return void l.close();
+                            if (h != u.pMVersion) return l.close(), s(i, p, r);
+                            e in v && d.loadPartialPosMapBuffer(S, e, o);
                         }
                         r.pop();
                     }
                     l.close();
-                    logger.info("bgLoadPosMap:" + i);
+                    g.info("bgLoadPosMap:" + p);
                 }(a, t), a.isStopping) break;
                 if (await c(0), a.isStopping) break;
             }
@@ -160,20 +177,20 @@ class a {
         if (!(a in o)) return t.removeKey(a);
         o[a] = void 0;
     }
-    onSnapshotSavedChangeId(a, t) {
+    onSnapshotSavedChangeId(a, t = void 0) {
         this.snapshotSavedChangeId = a;
         var {
             compactState: o,
             compactMaxChangeId: e,
             qyKVDataSaver: n
         } = this;
-        if (3 == o && e <= a && (x(this), this.compactState = 1), t) {
-            var s, o = t, p = this.qySnapshots, i = p.keyToSnapshotMap;
+        if (3 == o && e <= a && (L(this), this.compactState = 1), t) {
+            var s, o = t, i = this.qySnapshots, p = i.keyToSnapshotMap;
             for (s in o.posMap) {
-                var r = i[s];
-                r && p.removeKeyFromSnapshot(r, s);
+                var r = p[s];
+                r && i.removeKeyFromSnapshot(r, s);
             }
-            p.addMaxSnapshot(o), logger.info("addPMOnlySnapshot: " + p.snapshotMaxChangeId), 
+            i.addMaxSnapshot(o), g.info("addPMOnlySnapshot: " + i.snapshotMaxChangeId), 
             n.releaseSnapshot(t.snapshotNum);
         }
     }
@@ -182,17 +199,18 @@ class a {
             var l = this, {
                 pMUpdates: a,
                 vLUpdates: d,
-                combineUpdate: m
+                combinationUpdate: m
             } = a = a;
             if (a) {
                 var t = a, o = l.qySnapshots.numToSnapshotMap;
                 for (s in t) {
                     var e, n = t[s], s = o[s], {
-                        prefixPM: p,
-                        info: i
+                        prefixPM: i,
+                        info: p
                     } = s;
-                    for (e in p) p[e] = n[e];
-                    ++i.pMVersion, (a => {
+                    for (e in ++p.pMVersion, p.verList[0] = p.pMVersion + p.vLVersion, 
+                    i) i[e] = n[e];
+                    (a => {
                         var {
                             info: t,
                             compactOutedCount: o
@@ -203,18 +221,19 @@ class a {
             }
             if (d) {
                 var r = d, h = (a = l).qySnapshots.numToSnapshotMap;
-                for (v in r) {
-                    var c, u, {
-                        posMap: S,
-                        prefixPM: f
-                    } = r[v], v = h[v], {
+                for (f in r) {
+                    var c, S, {
+                        posMap: u,
+                        prefixPM: v
+                    } = r[f], f = h[f], {
                         posMap: y,
-                        prefixPM: g,
-                        info: C
-                    } = v;
-                    for (c in y) y[c] = S[c];
-                    for (u in g) g[u] = f[u];
-                    ++C.pMVersion, ++C.vLVersion, V(v);
+                        prefixPM: C,
+                        info: g
+                    } = f;
+                    for (c in ++g.pMVersion, ++g.vLVersion, g.verList[0] = g.pMVersion + g.vLVersion, 
+                    y) y[c] = u[c];
+                    for (S in C) C[S] = v[S];
+                    q(f);
                 }
             }
             if (m) {
@@ -224,32 +243,33 @@ class a {
                     keyToSnapshotMap: s
                 } = e, {
                     snapshotNum: a,
-                    combinedSnapshotNumList: t,
-                    posMap: p
-                } = d, i = n[a], {
+                    combinedSnapshotNumVersionList: t,
+                    posMap: i
+                } = d, p = n[a], {
                     posMap: r,
                     info: o
-                } = i;
-                for (M in r) r[M] = p[M];
-                V(i);
-                let h = o.totalCount, c = i.outdatedCount;
-                for (let o of t) {
-                    var I, q = n[o];
+                } = p;
+                for (M in ++o.pMVersion, ++o.vLVersion, o.verList[0] = o.pMVersion + o.vLVersion, 
+                r) r[M] = i[M];
+                q(p);
+                let h = o.totalCount, c = p.outdatedCount;
+                for (let [ o ] of t) {
+                    var V, I = n[o];
                     let {
                         posMap: a,
                         info: t
-                    } = q;
-                    for (I in a) r[I] = p[I], s[I] = i;
-                    V(q), h += t.totalCount, c += q.outdatedCount, e.delSnapshot(q);
+                    } = I;
+                    for (V in a) r[V] = i[V], s[V] = p;
+                    q(I), h += t.totalCount, c += I.outdatedCount, e.delSnapshot(I);
                 }
-                ++o.pMVersion, ++o.vLVersion, o.totalCount = h, i.outdatedCount = c;
+                o.totalCount = h, p.outdatedCount = c;
             }
-            this.snapshotSavedChangeId >= this.compactMaxChangeId ? (x(this), this.compactState = 1) : this.compactState = 3;
+            this.snapshotSavedChangeId >= this.compactMaxChangeId ? (L(this), this.compactState = 1) : this.compactState = 3;
         }
     }
 }
 
-function V(a) {
+function q(a) {
     var {
         info: t,
         compactOutedCount: o
@@ -265,20 +285,20 @@ function l(a) {
             qySnapshots: o,
             qySnapshotCompactor: e
         } = a, {
-            emptySnapshotNumList: o,
+            emptySnapshotNumVersionList: o,
             pMCompactSnapshotList: n,
             vLCompactSnapshotList: s,
-            combiningSnapshotList: p
+            combiningSnapshotList: i
         } = o.calOutdates();
-        if (0 < p.length || 0 < n.length || 0 < s.length) {
+        if (0 < i.length || 0 < n.length || 0 < s.length) {
             Object.assign(a, {
                 compactState: 2,
                 compactMaxChangeId: t
             });
-            for (var i of [ n, s, p ]) for (var r of i) r.compactOutedCount = r.outdatedCount;
-            e.compactSnapshots(n, s, p);
+            for (var p of [ n, s, i ]) for (var r of p) r.compactOutedCount = r.outdatedCount;
+            e.compactSnapshots(n, s, i);
         }
-        0 < o.length && (a.emptySnapshotNumList.push(...o), a.snapshotSavedChangeId >= t ? h(a) : 1 == a.compactState && Object.assign(a, {
+        0 < o.length && (a.emptySnapshotNumVersionList.push(...o), a.snapshotSavedChangeId >= t ? h(a) : 1 == a.compactState && Object.assign(a, {
             compactState: 3,
             compactMaxChangeId: t
         }));
@@ -295,12 +315,12 @@ function d(a, t) {
 function h(a) {
     var {
         qyKVDataCleaner: a,
-        emptySnapshotNumList: t
+        emptySnapshotNumVersionList: t
     } = a;
     0 < t.length && (a.removeSnapshotsFiles(t), t.length = 0);
 }
 
-function x(a) {
+function L(a) {
     a.qySnapshotCompactor.saveSnapshotInfos(), h(a);
 }
 

package/src/QyKVDataCleaner_Worker.d.ts

@@ -1,5 +1,11 @@
 import { QyMessageWorker } from "./QyMessageWorker.ts";
 export default class QyKVDataCleaner_Worker extends QyMessageWorker {
+    kvFolder: any;
+    snapshotFolder: any;
+    snapshotInfoFolder: any;
+    aclFolder: any;
+    historyAclFolder: any;
+    historySnapshotFolder: any;
     constructor({ kvFolder, options }: {
         kvFolder: any;
         options: any;
@@ -9,5 +15,5 @@ export default class QyKVDataCleaner_Worker extends QyMessageWorker {
     _op_cleanAclFiles(maxSnapshotNum: any): Promise<void>;
     _op_cleanPMFiles(pMChangeList: any): Promise<void>;
     _op_cleanVLFiles(snapshotNum: any, vLVersion: any): Promise<void>;
-    _op_removeSnapshotsFiles(snapshotNumList: any): Promise<void>;
+    _op_removeSnapshotsFiles(snapshotNumVersionList: any): Promise<void>;
 }

package/src/QyKVDataCleaner_Worker.js

@@ -1,24 +1,24 @@
 import {
-    join as c
+    join as m
 } from "node:path";
 
 import {
-    rm as m,
+    rm as c,
     rename as r,
-    readdir as f
+    readdir as h
 } from "node:fs/promises";
 
 import {
-    getDefaultOptions as o
+    getDefaultOptions as a
 } from "./QyDefaultOptions.js";
 
 import {
-    QyMessageWorker as e
+    QyMessageWorker as s
 } from "./QyMessageWorker.js";
 
 import {
-    folderOrFileExists as h,
-    ensureDir as d
+    folderOrFileExists as d,
+    ensureDir as F
 } from "./QyUtils.js";
 
 import {
@@ -26,85 +26,99 @@ import {
     Zip_File_Extension as i
 } from "./QyCompressor.js";
 
-export default class extends e {
+import {
+    logger as f
+} from "./QyLogger.js";
+
+import {
+    getSnapshotInfoFileName as l
+} from "./QySnapshots.js";
+
+export default class extends s {
+    kvFolder;
+    snapshotFolder;
+    snapshotInfoFolder;
+    aclFolder;
+    historyAclFolder;
+    historySnapshotFolder;
     constructor({
-        kvFolder: e,
-        options: s
+        kvFolder: s,
+        options: e
     }) {
-        super(s = {
-            ...o("QyKVDataCleaner_Worker"),
-            ...s
+        super(e = {
+            ...a("QyKVDataCleaner_Worker"),
+            ...e
         }, [], [ "cleanAclFiles", "cleanPMFiles", "cleanVLFiles", "removeSnapshotsFiles" ]);
-        var s = c(e, "history"), a = c(e, "snapshot");
+        var e = m(s, "history"), o = m(s, "snapshot");
         Object.assign(this, {
-            kvFolder: e,
-            snapshotFolder: a,
-            snapshotInfoFolder: c(a, "info"),
-            aclFolder: c(e, "acl"),
-            historyAclFolder: c(s, "acl"),
-            historySnapshotFolder: c(s, "snapshot")
+            kvFolder: s,
+            snapshotFolder: o,
+            snapshotInfoFolder: m(o, "info"),
+            aclFolder: m(s, "acl"),
+            historyAclFolder: m(e, "acl"),
+            historySnapshotFolder: m(e, "snapshot")
         });
     }
     _op_start() {}
     _op_stop() {}
-    async _op_cleanAclFiles(e) {
-        var s = this, a = e, e = s.aclFolder;
-        if (await h(e)) {
-            var o, r = [];
-            for (o of (await f(e)).filter(e => /^[1-9]/.test(e))) parseInt(o) <= a && r.push((async (e, s) => {
+    async _op_cleanAclFiles(s) {
+        var e = this, o = s, s = e.aclFolder;
+        if (await d(s)) {
+            var a, r = [];
+            for (a of (await h(s)).filter(s => /^[1-9]/.test(s))) parseInt(a) <= o && r.push((async (s, e) => {
                 var {
-                    aclFolder: a,
-                    historyAclFolder: o
-                } = e;
-                return await d(o), F(e, s, a, o);
-            })(s, o));
+                    aclFolder: o,
+                    historyAclFolder: a
+                } = s;
+                return await F(a), y(s, e, o, a);
+            })(e, a));
             0 < r.length && await Promise.allSettled(r);
         }
     }
-    async _op_cleanPMFiles(e) {
-        await Promise.allSettled(e.map(([ e, s ]) => {
-            return a = this, o = e, r = s, Promise.all([ "pM", "prefixPM" ].map(e => l(a, o, r, e)));
-            var a, o, r;
+    async _op_cleanPMFiles(s) {
+        await Promise.allSettled(s.map(([ s, e ]) => {
+            return o = this, a = s, r = e, Promise.all([ "pM", "prefixPM" ].map(s => n(o, a, r, s)));
+            var o, a, r;
         }));
     }
-    async _op_cleanVLFiles(e, s) {
-        await l(this, e, s, "vL");
+    async _op_cleanVLFiles(s, e) {
+        await n(this, s, e, "vL");
     }
-    async _op_removeSnapshotsFiles(e) {
+    async _op_removeSnapshotsFiles(s) {
         let {
-            snapshotFolder: a,
-            snapshotInfoFolder: o
+            snapshotFolder: o,
+            snapshotInfoFolder: a
         } = this;
-        await Promise.all(e.map(async s => {
-            await m(c(o, s + ".json")), await Promise.all([ "pM", "prefixPM", "vL" ].map(e => l(this, s, 1 / 0, e))), 
-            await m(c(a, "" + s), {
+        await Promise.all(s.map(async ([ e, s ]) => {
+            await c(m(a, l(e, s))), await Promise.all([ "pM", "prefixPM", "vL" ].map(s => n(this, e, 1 / 0, s))), 
+            await c(m(o, "" + e), {
                 recursive: !0
             });
         }));
     }
 }
 
-function F(e, s, a, o) {
-    a = c(a, s), o = c(o, s);
-    return e.options.compressRedundantFile ? t(a, o + i, {
+function y(s, e, o, a) {
+    o = m(o, e), a = m(a, e);
+    return s.options.compressRedundantFile ? t(o, a + i, {
         override: !0,
         removeSrcAfterSuccess: !0
-    }) : (logger.info(`rename ${a} to ` + o), r(a, o));
+    }) : (f.info(`rename ${o} to ` + a), r(o, a));
 }
 
-async function l(s, a, o, r) {
+async function n(e, o, a, r) {
     var {
-        snapshotFolder: e,
+        snapshotFolder: s,
         historySnapshotFolder: t
-    } = s, i = c(e, "" + a);
-    if (await h(i)) {
-        let e;
+    } = e, i = m(s, "" + o);
+    if (await d(i)) {
+        let s;
         var l, n = [];
-        for (l of await f(i)) {
+        for (l of await h(i)) {
             var p = new RegExp("^([0-9]+)_" + r).exec(l);
-            p && (p = parseInt(p[1])) < o && (1 == p ? (e || (e = c(t, "" + a), 
-            await d(e)), n.push(F(s, l, i, e))) : (p = c(i, l), logger.info("Removing " + p), 
-            n.push(m(p))));
+            p && (p = parseInt(p[1])) < a && (1 == p ? (s || (s = m(t, "" + o), 
+            await F(s)), n.push(y(e, l, i, s))) : (p = m(i, l), f.info("Removing " + p), 
+            n.push(c(p))));
         }
         0 < n.length && await Promise.allSettled(n);
     }

package/src/QyKVDataLoader.d.ts

@@ -1,8 +1,16 @@
 export declare class QyKVDataLoader {
+    options: any;
+    kvFolder: any;
+    aclFolder: any;
+    snapshotFolder: any;
+    qySnapshots: any;
+    loadedMaxAclNum: number;
+    aclKeyValueMap: any;
+    maxChangeId: number;
     constructor(kvFolder: any, options: any);
     load(): Promise<{
-        maxChangeId: any;
-        loadedMaxAclNum: any;
+        maxChangeId: number;
+        loadedMaxAclNum: number;
         qySnapshots: any;
         aclKeyValueMap: any;
     }>;

package/src/QyKVDataLoader.js

@@ -1,10 +1,10 @@
 import {
-    readFile as y,
+    readFile as u,
     readdir as x
 } from "node:fs/promises";
 
 import {
-    join as o
+    join as t
 } from "node:path";
 
 import {
@@ -12,7 +12,7 @@ import {
 } from "./QyUtils.js";
 
 import {
-    readAclBuffer as M
+    readAclBuffer as F
 } from "./QyAcl.js";
 
 import {
@@ -24,20 +24,26 @@ import {
 } from "./QyDefaultOptions.js";
 
 class a {
+    options;
+    kvFolder;
+    aclFolder;
+    snapshotFolder;
+    qySnapshots;
+    loadedMaxAclNum = 0;
+    aclKeyValueMap = {};
+    maxChangeId = 0;
     constructor(a, s) {
         s = {
             ...l("QyKVDataLoader"),
             ...s
         };
-        var t = o(a, "snapshot");
+        var o = t(a, "snapshot");
         Object.assign(this, {
             kvFolder: a,
             options: s,
-            aclFolder: o(a, "acl"),
-            snapshotFolder: t,
-            loadedMaxAclNum: 0,
-            aclKeyValueMap: {},
-            qySnapshots: new e(t, s)
+            aclFolder: t(a, "acl"),
+            snapshotFolder: o,
+            qySnapshots: new e(o, s)
         });
     }
     async load() {
@@ -45,19 +51,19 @@ class a {
             if (a = a.aclFolder, await g(a)) return (await x(a)).filter(a => /^[1-9]/.test(a)).map(a => parseInt(a));
         })(this), a.loadSnapshotInfos() ]);
         let {
-            maxSnapshotNum: t,
-            snapshotMaxChangeId: o
+            maxSnapshotNum: o,
+            snapshotMaxChangeId: t
         } = a;
-        if (this.maxChangeId = o, s) {
-            var e = this, l = s.filter(a => a > t), {
+        if (this.maxChangeId = t, s) {
+            var e = this, l = s.filter(a => a > o), {
                 aclKeyValueMap: r,
-                qySnapshots: i,
+                qySnapshots: n,
                 options: s
-            } = (l.sort((a, s) => a - s), e), n = s.autoRepairAclFile, p = await Promise.all(l.map(async a => y(F(e, a))));
+            } = (l.sort((a, s) => a - s), e), i = s.autoRepairAclFile, p = await Promise.all(l.map(async a => u(I(e, a))));
             for (let a = 0; a < l.length; ++a) {
-                var d = l[a], m = F(e, d), m = await M(m, p[a], n, e.maxChangeId);
+                var d = l[a], m = I(e, d), m = await F(m, p[a], i, e.maxChangeId);
                 if (0 < m.length) {
-                    for (var [ h, c ] of m) i.applyCmdObj(c, r), e.maxChangeId = h;
+                    for (var [ h, c ] of m) n.applyCmdObj(c, r), e.maxChangeId = h;
                     e.loadedMaxAclNum = d;
                 }
             }
@@ -66,19 +72,19 @@ class a {
         var {
             maxChangeId: s,
             loadedMaxAclNum: f,
-            aclKeyValueMap: u
+            aclKeyValueMap: y
         } = this;
         return {
             maxChangeId: s,
             loadedMaxAclNum: f,
             qySnapshots: a,
-            aclKeyValueMap: u
+            aclKeyValueMap: y
         };
     }
 }
 
-function F(a, s) {
-    return o(a.aclFolder, s + "_acl.txt");
+function I(a, s) {
+    return t(a.aclFolder, s + "_acl.txt");
 }
 
 export {

package/src/QyKVDataSaver_Worker.d.ts

@@ -1,5 +1,15 @@
 import { QyMessageWorker } from "./QyMessageWorker.ts";
 export default class QyKVDataSaver_Worker extends QyMessageWorker {
+    kvFolder: any;
+    aclFolder: any;
+    qyAclSaver: any;
+    qySnapshotSaver: any;
+    isStopping: any;
+    currentAclNum: any;
+    maxChangeId: any;
+    snapshotMaxChangeId: any;
+    isSavingSnapshot: any;
+    saveSnapshotResolve: any;
     constructor({ kvFolder, options }: {
         kvFolder: any;
         options: any;