datafolder 1.0.9 → 1.0.11

package/src/QyKVData.js

@@ -3,7 +3,7 @@ import {
 } from "node:path";
 
 import {
-    open as v
+    open as f
 } from "node:fs/promises";
 
 import {
@@ -24,11 +24,32 @@ import {
 
 import {
     arrayLast as y,
-    readBufferAsync as g,
+    readBufferAsync as C,
     sleep as c
 } from "./QyUtils.js";
 
+import {
+    logger as g
+} from "./QyLogger.js";
+
 class a {
+    kvFolder;
+    snapshotFolder;
+    qyCache;
+    options;
+    qyKVDataSaver;
+    qySnapshotCompactor;
+    qyKVDataCleaner;
+    compactState = 1;
+    compactMaxChangeId = 0;
+    snapshotSavedChangeId = 0;
+    maxChangeId = 0;
+    emptySnapshotNumVerList = [];
+    isStopping = !1;
+    qySnapshots;
+    aclKeyValueMap;
+    compactInterval;
+    unloadInterval;
     constructor(a, t, o) {
         t = {
             ...s("QyKVData"),
@@ -42,11 +63,7 @@ class a {
             snapshotFolder: e,
             qyKVDataSaver: new p(this, a, t),
             qySnapshotCompactor: new i(this, a, t),
-            qyKVDataCleaner: new r(this, a, t),
-            compactState: 1,
-            compactMaxChangeId: 0,
-            snapshotSavedChangeId: 0,
-            emptySnapshotNumList: []
+            qyKVDataCleaner: new r(this, a, t)
         });
     }
     start(a, t, o, e) {
@@ -89,32 +106,32 @@ class a {
             } = a;
             console.time("bgLoadPosMaps");
             for (t of Object.keys(o.numToSnapshotMap).sort((a, t) => t - a)) {
-                if (await async function s(p, i, r, h, c, l) {
+                if (await async function s(p, i, r = void 0, h = void 0, c = void 0, l = void 0) {
                     if (p.isStopping) return void (l && l.close());
                     let d = p.qySnapshots;
                     let m = d.numToSnapshotMap;
-                    let u = m[i];
-                    if (!u) return void (l && l.close());
+                    let S = m[i];
+                    if (!S) return void (l && l.close());
                     let {
-                        info: S,
-                        prefixPM: f
-                    } = u;
-                    r = r || Object.keys(f).sort().reverse();
+                        info: u,
+                        prefixPM: v
+                    } = S;
+                    r = r || Object.keys(v).sort().reverse();
                     if (0 == r.length) return void (l && l.close());
-                    if (null == h || h != S.pMVersion) return l && l.close(), c = d.getPMFilePath(u), 
-                    s(p, i, r, S.pMVersion, c, await v(c));
+                    if (null == h || h != u.pMVersion) return l && l.close(), c = d.getPMFilePath(S), 
+                    s(p, i, r, u.pMVersion, c, await f(c));
                     for (;0 < r.length; ) {
-                        let e = y(r), n = f[e];
+                        let e = y(r), n = v[e];
                         if (n) {
-                            let [ a, t ] = n, o = await g(c, l, a, t);
+                            let [ a, t ] = n, o = await C(c, l, a, t);
                             if (p.isStopping || !m[i]) return void l.close();
-                            if (h != S.pMVersion) return l.close(), s(p, i, r);
-                            e in f && d.loadPartialPosMapBuffer(u, e, o);
+                            if (h != u.pMVersion) return l.close(), s(p, i, r);
+                            e in v && d.loadPartialPosMapBuffer(S, e, o);
                         }
                         r.pop();
                     }
                     l.close();
-                    logger.info("bgLoadPosMap:" + i);
+                    g.info("bgLoadPosMap:" + i);
                 }(a, t), a.isStopping) break;
                 if (await c(0), a.isStopping) break;
             }
@@ -160,20 +177,20 @@ class a {
         if (!(a in o)) return t.removeKey(a);
         o[a] = void 0;
     }
-    onSnapshotSavedChangeId(a, t) {
+    onSnapshotSavedChangeId(a, t = void 0) {
         this.snapshotSavedChangeId = a;
         var {
             compactState: o,
             compactMaxChangeId: e,
             qyKVDataSaver: n
         } = this;
-        if (3 == o && e <= a && (x(this), this.compactState = 1), t) {
+        if (3 == o && e <= a && (L(this), this.compactState = 1), t) {
             var s, o = t, p = this.qySnapshots, i = p.keyToSnapshotMap;
             for (s in o.posMap) {
                 var r = i[s];
                 r && p.removeKeyFromSnapshot(r, s);
             }
-            p.addMaxSnapshot(o), logger.info("addPMOnlySnapshot: " + p.snapshotMaxChangeId), 
+            p.addMaxSnapshot(o), g.info("addPMOnlySnapshot: " + p.snapshotMaxChangeId), 
             n.releaseSnapshot(t.snapshotNum);
         }
     }
@@ -182,7 +199,7 @@ class a {
             var l = this, {
                 pMUpdates: a,
                 vLUpdates: d,
-                combineUpdate: m
+                combinationUpdate: m
             } = a = a;
             if (a) {
                 var t = a, o = l.qySnapshots.numToSnapshotMap;
@@ -191,8 +208,9 @@ class a {
                         prefixPM: p,
                         info: i
                     } = s;
-                    for (e in p) p[e] = n[e];
-                    ++i.pMVersion, (a => {
+                    for (e in ++i.pMVersion, i.verList[0] = i.pMVersion + i.vLVersion, 
+                    p) p[e] = n[e];
+                    (a => {
                         var {
                             info: t,
                             compactOutedCount: o
@@ -203,18 +221,19 @@ class a {
             }
             if (d) {
                 var r = d, h = (a = l).qySnapshots.numToSnapshotMap;
-                for (v in r) {
-                    var c, u, {
-                        posMap: S,
-                        prefixPM: f
-                    } = r[v], v = h[v], {
+                for (f in r) {
+                    var c, S, {
+                        posMap: u,
+                        prefixPM: v
+                    } = r[f], f = h[f], {
                         posMap: y,
-                        prefixPM: g,
-                        info: C
-                    } = v;
-                    for (c in y) y[c] = S[c];
-                    for (u in g) g[u] = f[u];
-                    ++C.pMVersion, ++C.vLVersion, V(v);
+                        prefixPM: C,
+                        info: g
+                    } = f;
+                    for (c in ++g.pMVersion, ++g.vLVersion, g.verList[0] = g.pMVersion + g.vLVersion, 
+                    y) y[c] = u[c];
+                    for (S in C) C[S] = v[S];
+                    q(f);
                 }
             }
             if (m) {
@@ -224,32 +243,33 @@ class a {
                     keyToSnapshotMap: s
                 } = e, {
                     snapshotNum: a,
-                    combinedSnapshotNumList: t,
+                    combinedSnapshotNumVerList: t,
                     posMap: p
                 } = d, i = n[a], {
                     posMap: r,
                     info: o
                 } = i;
-                for (M in r) r[M] = p[M];
-                V(i);
+                for (M in ++o.pMVersion, ++o.vLVersion, o.verList[0] = o.pMVersion + o.vLVersion, 
+                r) r[M] = p[M];
+                q(i);
                 let h = o.totalCount, c = i.outdatedCount;
-                for (let o of t) {
-                    var I, q = n[o];
+                for (let [ o ] of t) {
+                    var V, I = n[o];
                     let {
                         posMap: a,
                         info: t
-                    } = q;
-                    for (I in a) r[I] = p[I], s[I] = i;
-                    V(q), h += t.totalCount, c += q.outdatedCount, e.delSnapshot(q);
+                    } = I;
+                    for (V in a) r[V] = p[V], s[V] = i;
+                    q(I), h += t.totalCount, c += I.outdatedCount, e.delSnapshot(I);
                 }
-                ++o.pMVersion, ++o.vLVersion, o.totalCount = h, i.outdatedCount = c;
+                o.totalCount = h, i.outdatedCount = c;
             }
-            this.snapshotSavedChangeId >= this.compactMaxChangeId ? (x(this), this.compactState = 1) : this.compactState = 3;
+            this.snapshotSavedChangeId >= this.compactMaxChangeId ? (L(this), this.compactState = 1) : this.compactState = 3;
         }
     }
 }
 
-function V(a) {
+function q(a) {
     var {
         info: t,
         compactOutedCount: o
@@ -265,7 +285,7 @@ function l(a) {
             qySnapshots: o,
             qySnapshotCompactor: e
         } = a, {
-            emptySnapshotNumList: o,
+            emptySnapshotNumVerList: o,
             pMCompactSnapshotList: n,
             vLCompactSnapshotList: s,
             combiningSnapshotList: p
@@ -278,7 +298,7 @@ function l(a) {
             for (var i of [ n, s, p ]) for (var r of i) r.compactOutedCount = r.outdatedCount;
             e.compactSnapshots(n, s, p);
         }
-        0 < o.length && (a.emptySnapshotNumList.push(...o), a.snapshotSavedChangeId >= t ? h(a) : 1 == a.compactState && Object.assign(a, {
+        0 < o.length && (a.emptySnapshotNumVerList.push(...o), a.snapshotSavedChangeId >= t ? h(a) : 1 == a.compactState && Object.assign(a, {
             compactState: 3,
             compactMaxChangeId: t
         }));
@@ -295,12 +315,12 @@ function d(a, t) {
 function h(a) {
     var {
         qyKVDataCleaner: a,
-        emptySnapshotNumList: t
+        emptySnapshotNumVerList: t
     } = a;
     0 < t.length && (a.removeSnapshotsFiles(t), t.length = 0);
 }
 
-function x(a) {
+function L(a) {
     a.qySnapshotCompactor.saveSnapshotInfos(), h(a);
 }
 

package/src/QyKVDataCleaner_Worker.d.ts

@@ -1,5 +1,11 @@
 import { QyMessageWorker } from "./QyMessageWorker.ts";
 export default class QyKVDataCleaner_Worker extends QyMessageWorker {
+    kvFolder: any;
+    snapshotFolder: any;
+    snapshotInfoFolder: any;
+    aclFolder: any;
+    historyAclFolder: any;
+    historySnapshotFolder: any;
     constructor({ kvFolder, options }: {
         kvFolder: any;
         options: any;
@@ -9,5 +15,5 @@ export default class QyKVDataCleaner_Worker extends QyMessageWorker {
     _op_cleanAclFiles(maxSnapshotNum: any): Promise<void>;
     _op_cleanPMFiles(pMChangeList: any): Promise<void>;
     _op_cleanVLFiles(snapshotNum: any, vLVersion: any): Promise<void>;
-    _op_removeSnapshotsFiles(snapshotNumList: any): Promise<void>;
+    _op_removeSnapshotsFiles(snapshotNumVersionList: any): Promise<void>;
 }

package/src/QyKVDataCleaner_Worker.js

@@ -1,15 +1,15 @@
 import {
-    join as c
+    join as m
 } from "node:path";
 
 import {
-    rm as m,
+    rm as c,
     rename as r,
-    readdir as f
+    readdir as h
 } from "node:fs/promises";
 
 import {
-    getDefaultOptions as o
+    getDefaultOptions as a
 } from "./QyDefaultOptions.js";
 
 import {
@@ -17,8 +17,8 @@ import {
 } from "./QyMessageWorker.js";
 
 import {
-    folderOrFileExists as h,
-    ensureDir as d
+    folderOrFileExists as d,
+    ensureDir as F
 } from "./QyUtils.js";
 
 import {
@@ -26,85 +26,99 @@ import {
     Zip_File_Extension as i
 } from "./QyCompressor.js";
 
+import {
+    logger as f
+} from "./QyLogger.js";
+
+import {
+    getInfoFileName as l
+} from "./QySnapshots.js";
+
 export default class extends e {
+    kvFolder;
+    snapshotFolder;
+    snapshotInfoFolder;
+    aclFolder;
+    historyAclFolder;
+    historySnapshotFolder;
     constructor({
         kvFolder: e,
         options: s
     }) {
         super(s = {
-            ...o("QyKVDataCleaner_Worker"),
+            ...a("QyKVDataCleaner_Worker"),
             ...s
         }, [], [ "cleanAclFiles", "cleanPMFiles", "cleanVLFiles", "removeSnapshotsFiles" ]);
-        var s = c(e, "history"), a = c(e, "snapshot");
+        var s = m(e, "history"), o = m(e, "snapshot");
         Object.assign(this, {
             kvFolder: e,
-            snapshotFolder: a,
-            snapshotInfoFolder: c(a, "info"),
-            aclFolder: c(e, "acl"),
-            historyAclFolder: c(s, "acl"),
-            historySnapshotFolder: c(s, "snapshot")
+            snapshotFolder: o,
+            snapshotInfoFolder: m(o, "info"),
+            aclFolder: m(e, "acl"),
+            historyAclFolder: m(s, "acl"),
+            historySnapshotFolder: m(s, "snapshot")
         });
     }
     _op_start() {}
     _op_stop() {}
     async _op_cleanAclFiles(e) {
-        var s = this, a = e, e = s.aclFolder;
-        if (await h(e)) {
-            var o, r = [];
-            for (o of (await f(e)).filter(e => /^[1-9]/.test(e))) parseInt(o) <= a && r.push((async (e, s) => {
+        var s = this, o = e, e = s.aclFolder;
+        if (await d(e)) {
+            var a, r = [];
+            for (a of (await h(e)).filter(e => /^[1-9]/.test(e))) parseInt(a) <= o && r.push((async (e, s) => {
                 var {
-                    aclFolder: a,
-                    historyAclFolder: o
+                    aclFolder: o,
+                    historyAclFolder: a
                 } = e;
-                return await d(o), F(e, s, a, o);
-            })(s, o));
+                return await F(a), y(e, s, o, a);
+            })(s, a));
             0 < r.length && await Promise.allSettled(r);
         }
     }
     async _op_cleanPMFiles(e) {
         await Promise.allSettled(e.map(([ e, s ]) => {
-            return a = this, o = e, r = s, Promise.all([ "pM", "prefixPM" ].map(e => l(a, o, r, e)));
-            var a, o, r;
+            return o = this, a = e, r = s, Promise.all([ "pM", "prefixPM" ].map(e => n(o, a, r, e)));
+            var o, a, r;
         }));
     }
     async _op_cleanVLFiles(e, s) {
-        await l(this, e, s, "vL");
+        await n(this, e, s, "vL");
     }
     async _op_removeSnapshotsFiles(e) {
         let {
-            snapshotFolder: a,
-            snapshotInfoFolder: o
+            snapshotFolder: o,
+            snapshotInfoFolder: a
         } = this;
-        await Promise.all(e.map(async s => {
-            await m(c(o, s + ".json")), await Promise.all([ "pM", "prefixPM", "vL" ].map(e => l(this, s, 1 / 0, e))), 
-            await m(c(a, "" + s), {
+        await Promise.all(e.map(async ([ s, e ]) => {
+            await c(m(a, l(s, e))), await Promise.all([ "pM", "prefixPM", "vL" ].map(e => n(this, s, 1 / 0, e))), 
+            await c(m(o, "" + s), {
                 recursive: !0
             });
         }));
     }
 }
 
-function F(e, s, a, o) {
-    a = c(a, s), o = c(o, s);
-    return e.options.compressRedundantFile ? t(a, o + i, {
+function y(e, s, o, a) {
+    o = m(o, s), a = m(a, s);
+    return e.options.compressRedundantFile ? t(o, a + i, {
         override: !0,
         removeSrcAfterSuccess: !0
-    }) : (logger.info(`rename ${a} to ` + o), r(a, o));
+    }) : (f.info(`rename ${o} to ` + a), r(o, a));
 }
 
-async function l(s, a, o, r) {
+async function n(s, o, a, r) {
     var {
         snapshotFolder: e,
         historySnapshotFolder: t
-    } = s, i = c(e, "" + a);
-    if (await h(i)) {
+    } = s, i = m(e, "" + o);
+    if (await d(i)) {
         let e;
         var l, n = [];
-        for (l of await f(i)) {
+        for (l of await h(i)) {
             var p = new RegExp("^([0-9]+)_" + r).exec(l);
-            p && (p = parseInt(p[1])) < o && (1 == p ? (e || (e = c(t, "" + a), 
-            await d(e)), n.push(F(s, l, i, e))) : (p = c(i, l), logger.info("Removing " + p), 
-            n.push(m(p))));
+            p && (p = parseInt(p[1])) < a && (1 == p ? (e || (e = m(t, "" + o), 
+            await F(e)), n.push(y(s, l, i, e))) : (p = m(i, l), f.info("Removing " + p), 
+            n.push(c(p))));
         }
         0 < n.length && await Promise.allSettled(n);
     }

package/src/QyKVDataLoader.d.ts

@@ -1,8 +1,16 @@
 export declare class QyKVDataLoader {
+    options: any;
+    kvFolder: any;
+    aclFolder: any;
+    snapshotFolder: any;
+    qySnapshots: any;
+    loadedMaxAclNum: number;
+    aclKeyValueMap: any;
+    maxChangeId: number;
     constructor(kvFolder: any, options: any);
     load(): Promise<{
-        maxChangeId: any;
-        loadedMaxAclNum: any;
+        maxChangeId: number;
+        loadedMaxAclNum: number;
         qySnapshots: any;
         aclKeyValueMap: any;
     }>;

package/src/QyKVDataLoader.js

@@ -1,10 +1,10 @@
 import {
-    readFile as y,
+    readFile as u,
     readdir as x
 } from "node:fs/promises";
 
 import {
-    join as o
+    join as t
 } from "node:path";
 
 import {
@@ -12,7 +12,7 @@ import {
 } from "./QyUtils.js";
 
 import {
-    readAclBuffer as M
+    readAclBuffer as F
 } from "./QyAcl.js";
 
 import {
@@ -24,20 +24,26 @@ import {
 } from "./QyDefaultOptions.js";
 
 class a {
+    options;
+    kvFolder;
+    aclFolder;
+    snapshotFolder;
+    qySnapshots;
+    loadedMaxAclNum = 0;
+    aclKeyValueMap = {};
+    maxChangeId = 0;
     constructor(a, s) {
         s = {
             ...l("QyKVDataLoader"),
             ...s
         };
-        var t = o(a, "snapshot");
+        var o = t(a, "snapshot");
         Object.assign(this, {
             kvFolder: a,
             options: s,
-            aclFolder: o(a, "acl"),
-            snapshotFolder: t,
-            loadedMaxAclNum: 0,
-            aclKeyValueMap: {},
-            qySnapshots: new e(t, s)
+            aclFolder: t(a, "acl"),
+            snapshotFolder: o,
+            qySnapshots: new e(o, s)
         });
     }
     async load() {
@@ -45,19 +51,19 @@ class a {
             if (a = a.aclFolder, await g(a)) return (await x(a)).filter(a => /^[1-9]/.test(a)).map(a => parseInt(a));
         })(this), a.loadSnapshotInfos() ]);
         let {
-            maxSnapshotNum: t,
-            snapshotMaxChangeId: o
+            maxSnapshotNum: o,
+            snapshotMaxChangeId: t
         } = a;
-        if (this.maxChangeId = o, s) {
-            var e = this, l = s.filter(a => a > t), {
+        if (this.maxChangeId = t, s) {
+            var e = this, l = s.filter(a => a > o), {
                 aclKeyValueMap: r,
-                qySnapshots: i,
+                qySnapshots: n,
                 options: s
-            } = (l.sort((a, s) => a - s), e), n = s.autoRepairAclFile, p = await Promise.all(l.map(async a => y(F(e, a))));
+            } = (l.sort((a, s) => a - s), e), i = s.autoRepairAclFile, p = await Promise.all(l.map(async a => u(I(e, a))));
             for (let a = 0; a < l.length; ++a) {
-                var d = l[a], m = F(e, d), m = await M(m, p[a], n, e.maxChangeId);
+                var d = l[a], m = I(e, d), m = await F(m, p[a], i, e.maxChangeId);
                 if (0 < m.length) {
-                    for (var [ h, c ] of m) i.applyCmdObj(c, r), e.maxChangeId = h;
+                    for (var [ h, c ] of m) n.applyCmdObj(c, r), e.maxChangeId = h;
                     e.loadedMaxAclNum = d;
                 }
             }
@@ -66,19 +72,19 @@ class a {
         var {
             maxChangeId: s,
             loadedMaxAclNum: f,
-            aclKeyValueMap: u
+            aclKeyValueMap: y
         } = this;
         return {
             maxChangeId: s,
             loadedMaxAclNum: f,
             qySnapshots: a,
-            aclKeyValueMap: u
+            aclKeyValueMap: y
         };
     }
 }
 
-function F(a, s) {
-    return o(a.aclFolder, s + "_acl.txt");
+function I(a, s) {
+    return t(a.aclFolder, s + "_acl.txt");
 }
 
 export {

package/src/QyKVDataSaver_Worker.d.ts

@@ -1,5 +1,15 @@
 import { QyMessageWorker } from "./QyMessageWorker.ts";
 export default class QyKVDataSaver_Worker extends QyMessageWorker {
+    kvFolder: any;
+    aclFolder: any;
+    qyAclSaver: any;
+    qySnapshotSaver: any;
+    isStopping: any;
+    currentAclNum: any;
+    maxChangeId: any;
+    snapshotMaxChangeId: any;
+    isSavingSnapshot: any;
+    saveSnapshotResolve: any;
     constructor({ kvFolder, options }: {
         kvFolder: any;
         options: any;

package/src/QyKVDataSaver_Worker.js

@@ -19,10 +19,24 @@ import {
 } from "./QySnapshotSaver.js";
 
 import {
-    delCmdKey as u
+    delCmdKey as m
 } from "./QyAcl.js";
 
+import {
+    logger as r
+} from "./QyLogger.js";
+
 export default class extends a {
+    kvFolder;
+    aclFolder;
+    qyAclSaver;
+    qySnapshotSaver;
+    isStopping;
+    currentAclNum;
+    maxChangeId;
+    snapshotMaxChangeId;
+    isSavingSnapshot;
+    saveSnapshotResolve;
     constructor({
         kvFolder: a,
         options: s
@@ -46,7 +60,7 @@ export default class extends a {
             maxChangeId: a,
             snapshotMaxChangeId: a,
             currentAclNum: (t || s) + 1
-        }), o && o.start(m(this)), n.start(a, t, e);
+        }), o && o.start(d(this)), n.start(a, t, e);
     }
     async _op_stop() {
         var {
@@ -58,13 +72,13 @@ export default class extends a {
             snapshotMaxChangeId: n
         } = this;
         return this.isStopping = !0, a && await a.stop(), s.saveSnapshotAtStop && n < o && (this.isSavingSnapshot && await new Promise(a => this.saveSnapshotResolve = a), 
-        logger.log(`Saving snapshot ${e} at stop.`), t.saveSnapshot(e)), t.stop();
+        r.log(`Saving snapshot ${e} at stop.`), t.saveSnapshot(e)), t.stop();
     }
     callSave(a, s, t) {
-        return r(this, a, s, t, !0);
+        return h(this, a, s, t, !0);
     }
     castSave(a, s, t) {
-        return r(this, a, s, t);
+        return h(this, a, s, t);
     }
     releaseSnapshot(a) {
         this.qySnapshotSaver.releaseSnapshot(a);
@@ -74,7 +88,7 @@ export default class extends a {
     }
 }
 
-function r(a, s, t, e, o) {
+function h(a, s, t, e, o = !1) {
     var {
         qyAclSaver: n,
         qySnapshotSaver: r,
@@ -88,15 +102,15 @@ function r(a, s, t, e, o) {
     var l = t, c = e;
     if (c) {
         var g = c.length;
-        for (let a = 0; a < g; a += 2) u(l, c[a]);
+        for (let a = 0; a < g; a += 2) m(l, c[a]);
     }
     return r.castSave(s, t, e), a.maxChangeId = s, !S && s - i >= h.snapshotMaxChangeCount && (++a.currentAclNum, 
-    n && n.switch(m(a)), r.saveSnapshot(p), Object.assign(a, {
+    n && n.switch(d(a)), r.saveSnapshot(p), Object.assign(a, {
         snapshotMaxChangeId: s,
         isSavingSnapshot: !0
     })), v;
 }
 
-function m(a) {
+function d(a) {
     return t(a.aclFolder, a.currentAclNum + "_acl.txt");
 }

package/src/QyLogFileSaver_Worker.d.ts

@@ -1,5 +1,6 @@
 import { QyMessageWorker } from "./QyMessageWorker.ts";
 export default class QyLogFileSaver_Worker extends QyMessageWorker {
+    qyBinWriter: any;
     constructor(options: any);
     _op_start(logFilePath: any): Promise<void>;
     _op_stop(): Promise<void>;