datafolder 1.0.18 → 1.0.19

package/src/QySnapshots.js

@@ -1,32 +1,32 @@
 import {
     close as e,
-    readSync as i,
-    openSync as p
+    openSync as s
 } from "node:fs";
 
 import {
     readdir as l,
-    readFile as n
+    readFile as r
 } from "node:fs/promises";
 
 import {
-    join as s
+    join as n
 } from "node:path";
 
 import m from "node:zlib";
 
 import {
-    getDefaultOptions as r
+    getDefaultOptions as i
 } from "./QyDefaultOptions.js";
 
 import {
-    Hash_Key_Length as f,
+    Hash_Key_Length as p,
     folderOrFileExists as d,
-    listToMap as u,
-    isEmptyObj as h,
+    listToMap as f,
+    isEmptyObj as u,
     isString as v,
     isArray as M,
-    isFloat32Array as y
+    isUint8Array as y,
+    readBufferFromFD as h
 } from "./QyUtils.js";
 
 import {
@@ -48,7 +48,7 @@ import {
     logger as F
 } from "./QyLogger.js";
 
-let D = Buffer.from("\n"), L = Buffer.from("'"), O = Buffer.from("z"), b = Buffer.from("vect"), E = L[0], J = O[0], K = b[0];
+let J = Buffer.from("\n"), L = Buffer.from("'"), O = Buffer.from("z"), b = Buffer.from("vect"), K = L[0], _ = O[0], A = b[0];
 
 class t {
     snapshotFolder;
@@ -62,55 +62,55 @@ class t {
     snapshotMaxChangeId = 0;
     maxSnapshotNum = 0;
     snapshotCount = 0;
-    constructor(t, a = void 0) {
-        (a = {
-            ...r("QySnapshots"),
-            ...a
-        }).vLOutdatedPercent = 2 * a.pMOutdatedPercent;
-        var o = s(t, "info");
+    constructor(t, o = void 0) {
+        (o = {
+            ...i("QySnapshots"),
+            ...o
+        }).vLOutdatedPercent = 2 * o.pMOutdatedPercent;
+        var a = n(t, "info");
         Object.assign(this, {
             snapshotFolder: t,
-            snapshotInfoFolder: o,
-            options: a,
-            qyJsonSaver: new x(o, a)
+            snapshotInfoFolder: a,
+            options: o,
+            qyJsonSaver: new x(a, o)
         });
     }
     async loadSnapshotInfos() {
         let {
             snapshotInfoFolder: t,
-            qyJsonSaver: r,
-            numToSnapshotMap: n
+            qyJsonSaver: n,
+            numToSnapshotMap: r
         } = this;
         if (await d(t)) {
-            var a = await (async (t, a) => {
-                var o = {};
-                for (r of await l(a)) {
-                    var e, s, r = /^([1-9][0-9]*)(_([0-9]+))*\.json/.exec(r);
-                    r && (e = +r[1], r = r[3], r = r ? +r : 2, null == (s = o[e]) ? o[e] = r : (t.removeFile(_(e, Math.min(s, r))), 
-                    o[e] = Math.max(s, r)));
+            var o = await (async (t, o) => {
+                var a = {};
+                for (n of await l(o)) {
+                    var e, s, n = /^([1-9][0-9]*)(_([0-9]+))*\.json/.exec(n);
+                    n && (e = +n[1], n = n[3], n = n ? +n : 2, null == (s = a[e]) ? a[e] = n : (t.removeFile(Q(e, Math.min(s, n))), 
+                    a[e] = Math.max(s, n)));
                 }
-                return Object.entries(o).map(([ t, a ]) => [ +t, a ]).sort((t, a) => t[0] - a[0]);
-            })(r, t);
+                return Object.entries(a).map(([ t, o ]) => [ +t, o ]).sort((t, o) => t[0] - o[0]);
+            })(n, t);
             let e = 0, s = 0;
-            await Promise.all(a.map(async ([ t, a ]) => {
-                var o = await r.loadFromFile(_(t, a)), a = (o.verList = [ a ], n[t] = {
+            await Promise.all(o.map(async ([ t, o ]) => {
+                var a = await n.loadFromFile(Q(t, o)), o = (a.verList = [ o ], r[t] = {
                     snapshotNum: t,
-                    info: o,
+                    info: a,
                     posMap: {},
                     outdatedCount: 0
-                }), t = (++this.snapshotCount, t > e && (e = t, s = o.maxChangeId), 
-                this), o = a;
-                t = N(t, o), F.info("Loading " + t), o.prefixPM = await V(t);
+                }), t = (++this.snapshotCount, t > e && (e = t, s = a.maxChangeId), 
+                this), a = o;
+                t = T(t, a), F.info("Loading " + t), a.prefixPM = await V(t);
             })), Object.assign(this, {
                 maxSnapshotNum: e,
                 snapshotMaxChangeId: s
             });
             var {
-                numToSnapshotMap: o,
+                numToSnapshotMap: a,
                 prefixMap: i
             } = this;
-            for ([ u ] of a) {
-                var p, f = o[u], u = f.prefixPM;
+            for ([ u ] of o) {
+                var p, f = a[u], u = f.prefixPM;
                 for (p in u) {
                     var h = i[p];
                     h ? h.push(f) : i[p] = [ f ];
@@ -119,23 +119,23 @@ class t {
         }
     }
     closeAllFDs() {
-        var t, a = this.fDMap;
-        for (t in a) e(a[t]), delete a[t];
+        var t, o = this.fDMap;
+        for (t in o) e(o[t]), delete o[t];
     }
     saveKeyValueMap(t) {
-        var a, o = this.options.minLenToCompress, {
+        var o, a = this.options.minLenToCompress, {
             keyValueMap: e,
             info: s
-        } = t, r = (++s.vLVersion, new g()), n = (r.start(this.getVLFilePath(t)), 
+        } = t, n = (++s.vLVersion, new g()), r = (n.start(this.getVLFilePath(t)), 
         Object.keys(e).sort()), i = {};
         let p = 0, f = 0, u, h;
-        for (a of n) {
-            var l = e[a];
+        for (o of r) {
+            var l = e[o];
             let t;
-            null == l ? t = A : (h = Buffer.isBuffer(l) ? (u = l, 0) : v(l) ? (u = l.length >= o ? (r.save(O), 
-            m.zstdCompressSync(l)) : (r.save(L), Buffer.from(l)), 1) : y(l) ? (r.save(b), 
-            u = Buffer.from(l.buffer, l.byteOffset, l.byteLength), 4) : (u = Buffer.from(JSON.stringify(I(a) && !M(l) ? Object.keys(l) : l)), 
-            0), r.save([ u, D ]), l = h + u.length, t = [ p, l ], p += l + 1), i[a] = t, 
+            null == l ? t = q : (h = Buffer.isBuffer(l) ? (u = l, 0) : v(l) ? (u = l.length >= a ? (n.save(O), 
+            m.zstdCompressSync(l)) : (n.save(L), Buffer.from(l)), 1) : y(l) ? (n.save(b), 
+            u = Buffer.from(l.buffer, l.byteOffset, l.byteLength), 4) : (u = Buffer.from(JSON.stringify(I(o) && !M(l) ? Object.keys(l) : l)), 
+            0), n.save([ u, J ]), l = h + u.length, t = [ p, l ], p += l + 1), i[o] = t, 
             ++f;
         }
         return Object.assign(t, {
@@ -144,93 +144,93 @@ class t {
         }), Object.assign(s, {
             totalCount: f,
             outdatedCount: 0
-        }), Promise.all([ this.savePosMap(t, n), r.stop() ]);
+        }), Promise.all([ this.savePosMap(t, r), n.stop() ]);
     }
     addMaxSnapshot(t) {
-        var a, {
-            numToSnapshotMap: o,
+        var o, {
+            numToSnapshotMap: a,
             keyToSnapshotMap: e
         } = this, {
             posMap: s,
-            keyValueMap: r,
-            snapshotNum: n,
+            keyValueMap: n,
+            snapshotNum: r,
             info: i
         } = t, i = i.maxChangeId, i = (Object.assign(this, {
-            maxSnapshotNum: n,
+            maxSnapshotNum: r,
             snapshotMaxChangeId: i
-        }), r || s);
-        for (a in i) e[a] = t;
-        o[n] = t, ++this.snapshotCount;
+        }), n || s);
+        for (o in i) e[o] = t;
+        a[r] = t, ++this.snapshotCount;
     }
     removeKey(t) {
-        var a = this.keyToSnapshotMap, o = a[t];
-        if (o) return this.removeKeyFromSnapshot(o, t), delete a[t], !0;
+        var o = this.keyToSnapshotMap, a = o[t];
+        if (a) return this.removeKeyFromSnapshot(a, t), delete o[t], !0;
     }
-    applyCmdObj(t, a, o) {
-        if (null != o) {
-            var e = o.length;
+    applyCmdObj(t, o, a) {
+        if (null != a) {
+            var e = a.length;
             for (let t = 0; t < e; t += 2) {
-                var s = o[t], r = o[t + 1];
-                a[s] = null != r && I(s) ? u(r) : r;
+                var s = a[t], n = a[t + 1];
+                o[s] = null != n && I(s) ? f(n) : n;
             }
         }
         P(t, t => {
-            t in a || (a[t] = this.getValueSync(t)), this.removeKey(t);
-        }), C(a, t);
+            t in o || (o[t] = this.getValueSync(t)), this.removeKey(t);
+        }), C(o, t);
     }
     getValueSync(t) {
-        var a = ((t, a) => {
+        var o = ((t, o) => {
             let {
-                keyToSnapshotMap: o,
+                keyToSnapshotMap: a,
                 prefixMap: e
-            } = t, s = o[a];
+            } = t, s = a[o];
             if (s) return s;
-            var r = k(a), n = e[r];
-            if (n) for (;0 < n.length; ) {
-                s = n.pop();
-                var i = ((t, a, o) => {
-                    var e, {
-                        posMap: s,
+            var n = k(o), r = e[n];
+            if (r) for (;0 < r.length; ) {
+                s = r.pop();
+                var i = ((t, o, a) => {
+                    var e, s, {
+                        posMap: n,
                         prefixPM: r
-                    } = a;
-                    if (r = r[o]) return [ r, e ] = r, r = B(t, t.getPMFilePath(a), r, e), 
-                    t.loadPartialPosMapBuffer(a, o, r), s;
-                })(t, s, r);
-                if (i && a in i) return s;
+                    } = o;
+                    if (r = r[a]) return [ r, e ] = r, s = t.getPMFilePath(o), s = h(N(t, s), s, r, e), 
+                    t.loadPartialPosMapBuffer(o, a, s), n;
+                })(t, s, n);
+                if (i && o in i) return s;
             }
         })(this, t);
-        if (a) {
+        if (o) {
             var {
-                keyValueMap: o,
+                keyValueMap: a,
                 posMap: e
-            } = a;
-            if (o) return o[t];
-            o = e[t];
-            if (o) {
-                var [ e, o ] = o;
-                if (0 != o) {
+            } = o;
+            if (a) return a[t];
+            a = e[t];
+            if (a) {
+                var [ e, a ] = a;
+                if (0 != a) {
                     var s = this;
-                    if (a = s.getVLFilePath(a), s = B(s, a, e, o)) return (a = s[0]) == E ? s.toString("utf8", 1) : a == J ? m.zstdDecompressSync(s.subarray(1)).toString() : a == K ? new Float32Array(s.buffer, s.byteOffset + 4, s.byteLength / Float32Array.BYTES_PER_ELEMENT - 1) : (e = JSON.parse(s), 
-                    I(t) ? u(e) : e);
+                    if (o = s.getVLFilePath(o), s = h(N(s, o), o, e, a)) return (o = s[0]) == K ? s.toString("utf8", 1) : o == _ ? m.zstdDecompressSync(s.subarray(1)).toString() : o == A ? new Uint8Array(s.buffer, s.byteOffset + 4, s.byteLength - 4) : (e = JSON.parse(s), 
+                    I(t) ? f(e) : e);
                 }
             }
         }
     }
-    loadPartialPosMapBuffer(o, t, a) {
+    loadPartialPosMapBuffer(a, t, o) {
         let {
             posMap: e,
             prefixPM: s
-        } = o, r = (delete s[t], this).keyToSnapshotMap;
-        j(a, (t, a) => {
-            t in r ? ++o.outdatedCount : (e[t] = a, r[t] = o);
+        } = a, n = (delete s[t], this).keyToSnapshotMap;
+        j(o, (t, o) => {
+            t in n ? ++a.outdatedCount : (e[t] = o, n[t] = a);
         });
     }
     delSnapshot(t) {
-        delete this.numToSnapshotMap[t.snapshotNum], --this.snapshotCount, a(this, this.getVLFilePath(t)), 
-        a(this, this.getPMFilePath(t));
+        delete this.numToSnapshotMap[t.snapshotNum], --this.snapshotCount, o(this, this.getVLFilePath(t)), 
+        o(this, this.getPMFilePath(t));
     }
     calOutdates() {
-        var t, a, o, e = [], s = [], r = [], n = [], {
+        var t, o, a, e = [], s = [], n = [], r = [], {
             options: i,
             numToSnapshotMap: p
         } = this, {
@@ -239,150 +239,143 @@ class t {
         } = i;
         for (t in p) {
             var h, l = p[t];
-            l.isOut || (100 == (h = Q(l)) ? (this.delSnapshot(l), e.push([ t, l.info.verList[0] ])) : u < h ? (T(l) ? s : n).push(l) : Math.floor(l.outdatedCount / l.info.totalCount * 100) > f ? r.push(l) : (h = this, 
-            l.info.totalCount < h.options.minSnapshotKeyCount && T(l) && s.push(l)));
+            l.isOut || (100 == (h = D(l)) ? (this.delSnapshot(l), e.push([ t, l.info.verList[0] ])) : u < h ? (B(l) ? s : r).push(l) : Math.floor(l.outdatedCount / l.info.totalCount * 100) > f ? n.push(l) : (h = this, 
+            l.info.totalCount < h.options.minSnapshotKeyCount && B(l) && s.push(l)));
         }
-        1 == s.length && Q(s[0]) < u && (s.length = 0);
-        for ([ a, o ] of [ [ "pMCompacts", r ], [ "vLCompacts", n ], [ "combiningSnapshots", s ] ]) 0 < o.length && F.log(a + ":" + o.map(t => t.snapshotNum + "." + t.info.verList[0]));
+        1 == s.length && D(s[0]) < u && (s.length = 0);
+        for ([ o, a ] of [ [ "pMCompacts", n ], [ "vLCompacts", r ], [ "combiningSnapshots", s ] ]) 0 < a.length && F.log(o + ":" + a.map(t => t.snapshotNum + "." + t.info.verList[0]));
         return {
             emptySnapshotNumVerList: e,
             combiningSnapshotList: s,
-            pMCompactSnapshotList: r,
-            vLCompactSnapshotList: n
+            pMCompactSnapshotList: n,
+            vLCompactSnapshotList: r
         };
     }
     getMaxCombinedSnapshot(t) {
-        let a, o;
+        let o, a;
         for (var e of t) {
             var s = e.snapshotNum;
-            (!o || o < s) && (a = e, o = s);
+            (!a || a < s) && (o = e, a = s);
         }
-        return a;
+        return o;
     }
     getPMFilePath(t) {
         var {
             snapshotNum: t,
-            info: a
+            info: o
         } = t;
-        return s(this.snapshotFolder, "" + t, a.pMVersion + "_pM.txt");
+        return n(this.snapshotFolder, "" + t, o.pMVersion + "_pM.txt");
     }
     getVLFilePath(t) {
         var {
             snapshotNum: t,
-            info: a
+            info: o
         } = t;
-        return s(this.snapshotFolder, "" + t, a.vLVersion + "_vL.txt");
+        return n(this.snapshotFolder, "" + t, o.vLVersion + "_vL.txt");
     }
-    async saveSnapshotInfo(t, a) {
-        var o = this.qyJsonSaver, {
+    async saveSnapshotInfo(t, o) {
+        var a = this.qyJsonSaver, {
             pMVersion: e,
             vLVersion: s,
-            verList: r,
-            outdatedCount: n,
+            verList: n,
+            outdatedCount: r,
             totalCount: i
-        } = a, p = r[0], r = r[0] = e + s;
-        return a.outdated = Math.floor(100 * n / i) + "%", F.log(`Saving snapshotInfo:${p ? t + `.${p}->` : ""}${t}.` + r), 
-        !!await o.saveToFile({
-            ...a,
+        } = o, p = n[0], n = n[0] = e + s;
+        return o.outdated = Math.floor(100 * r / i) + "%", F.log(`Saving snapshotInfo:${p ? t + `.${p}->` : ""}${t}.` + n), 
+        !!await a.saveToFile({
+            ...o,
             verList: void 0
-        }, _(t, r), !0) && (null != p && o.removeFile(_(t, p)), !0);
+        }, Q(t, n), !0) && (null != p && a.removeFile(Q(t, p)), !0);
     }
-    savePosMap(t, a) {
-        var o, {
+    savePosMap(t, o) {
+        var a, {
             outdatedCount: e,
             info: s,
-            posMap: r
-        } = t, n = (s.outdatedCount += e, t.outdatedCount = 0, ++s.pMVersion, new g()), i = (n.start(this.getPMFilePath(t)), 
+            posMap: n
+        } = t, r = (s.outdatedCount += e, t.outdatedCount = 0, ++s.pMVersion, new g()), i = (r.start(this.getPMFilePath(t)), 
         t.prefixPM = {});
         let p = 0, f = 0, u;
-        for (o of a = a || Object.keys(r).sort()) {
-            null == u ? u = k(o) : o.startsWith(u) || (i[u] = [ p, f ], u = k(o), 
+        for (a of o = o || Object.keys(n).sort()) {
+            null == u ? u = k(a) : a.startsWith(u) || (i[u] = [ p, f ], u = k(a), 
             p += f, f = 0);
-            var h = Buffer.from(o), l = S(r[o]);
-            n.save([ h, l ]), f += h.length + l.length;
+            var h = Buffer.from(a), l = S(n[a]);
+            r.save([ h, l ]), f += h.length + l.length;
         }
-        return null != u && (i[u] = [ p, f ]), Promise.all([ n.stop(), ((t, a) => {
-            var o, e = new g(), s = (e.start(N(t, a)), a).prefixPM;
-            for (o in s) e.save([ Buffer.from(o), S(s[o]) ]);
+        return null != u && (i[u] = [ p, f ]), Promise.all([ r.stop(), ((t, o) => {
+            var a, e = new g(), s = (e.start(T(t, o)), o).prefixPM;
+            for (a in s) e.save([ Buffer.from(a), S(s[a]) ]);
             return e.stop();
         })(this, t) ]);
     }
     isEmptySnapshot(t) {
         var {
             info: t,
-            outdatedCount: a
+            outdatedCount: o
         } = t;
-        return a + t.outdatedCount == t.totalCount;
+        return o + t.outdatedCount == t.totalCount;
     }
-    removeKeyFromSnapshot(t, a) {
+    removeKeyFromSnapshot(t, o) {
         var {
-            posMap: o,
+            posMap: a,
             keyValueMap: e
         } = t;
-        e && delete e[a], o && delete o[a], ++t.outdatedCount;
+        e && delete e[o], a && delete a[o], ++t.outdatedCount;
     }
 }
 
-function j(a, o) {
-    var e = a.length;
+function j(o, a) {
+    var e = o.length;
     for (let t = 0; t < e; ) {
-        var s = t + f, r = a.toString("latin1", t, s), n = [];
-        t = c(a, s, n), o(r, n);
+        var s = t + p, n = o.toString("latin1", t, s), r = [];
+        t = c(o, s, r), a(n, r);
     }
 }
 
 async function V(t) {
-    var a = await n(t), o = a.length, e = {};
-    for (let t = 0; t < o; ) {
-        var s = t + w, r = e[a.toString("latin1", t, s)] = [];
-        t = c(a, s, r);
+    var o = await r(t), a = o.length, e = {};
+    for (let t = 0; t < a; ) {
+        var s = t + w, n = e[o.toString("latin1", t, s)] = [];
+        t = c(o, s, n);
     }
     return e;
 }
 
-function B(t, a, o, e) {
-    var s = ((t, a) => {
-        let o = t.fDMap, e = o[a];
-        if (!e) {
-            F.info("Opening " + a);
-            try {
-                e = o[a] = p(a, "r");
-            } catch (t) {
-                throw F.error(`Opening ${a} failed:`, t), t;
-            }
-        }
-        return e;
-    })(t, a), r = Buffer.allocUnsafe(e);
-    for (let t = 0; t < e; ) {
-        var n = i(s, r, t, e - t, o + t);
-        if (0 == n) return void F.error(`readSync ${a} at ${o + t} returning 0 byte`);
-        t += n;
-    }
-    return r;
-}
-
-function T(t) {
+function B(t) {
     var {
-        keyValueMap: a,
-        prefixPM: o
+        keyValueMap: o,
+        prefixPM: a
     } = t;
-    return a || null == o || h(t.prefixPM);
+    return o || null == a || u(t.prefixPM);
 }
 
-function N(t, a) {
+function T(t, o) {
     var {
-        snapshotNum: a,
-        info: o
-    } = a;
-    return s(t.snapshotFolder, "" + a, o.pMVersion + "_prefixPM.txt");
+        snapshotNum: o,
+        info: a
+    } = o;
+    return n(t.snapshotFolder, "" + o, a.pMVersion + "_prefixPM.txt");
 }
 
-function a(t, a) {
-    var t = t.fDMap, o = t[a];
-    null != o && (e(o), delete t[a]);
+function N(t, o) {
+    t = t.fDMap;
+    let a = t[o];
+    if (!a) {
+        F.info("Opening " + o);
+        try {
+            a = t[o] = s(o, "r");
+        } catch (t) {
+            throw F.error(`Opening ${o} failed:`, t), t;
+        }
+    }
+    return a;
 }
 
-let A = [ 0, 0 ], w = 3;
+function o(t, o) {
+    var t = t.fDMap, a = t[o];
+    null != a && (e(a), delete t[o]);
+}
+
+let q = [ 0, 0 ], w = 3;
 
 function k(t) {
     return t.substring(0, w);
@@ -392,21 +385,21 @@ function I(t) {
     return !t.startsWith("c");
 }
 
-function Q(t) {
+function D(t) {
     var {
         outdatedCount: t,
-        info: a
-    } = t, o = a.totalCount, t = t + a.outdatedCount;
-    return t == o ? 100 : Math.floor(t / o * 100);
+        info: o
+    } = t, a = o.totalCount, t = t + o.outdatedCount;
+    return t == a ? 100 : Math.floor(t / a * 100);
 }
 
-function _(t, a) {
-    return t + (2 < a ? "_" + a : "") + ".json";
+function Q(t, o) {
+    return t + (2 < o ? "_" + o : "") + ".json";
 }
 
 export {
     t as QySnapshots,
     j as iterPMBuffer,
     V as loadPrefixPMFile,
-    _ as getInfoFileName
+    Q as getInfoFileName
 };

package/src/QyUtils.d.ts

@@ -1,6 +1,6 @@
 import util from "node:util";
 export declare const isArray: (arg: any) => arg is any[];
-export declare const isDate: typeof util.types.isDate, isFloat32Array: typeof util.types.isFloat32Array;
+export declare const isDate: typeof util.types.isDate, isFloat32Array: typeof util.types.isFloat32Array, isUint8Array: typeof util.types.isUint8Array;
 export declare const __dirname: string;
 export declare const SpecialOperators: {
     [k: string]: string;
@@ -11,8 +11,9 @@ export declare const SpecialFilePaths: {
     RpcDirPath: string[];
     IndexDirPath: string[];
     ExpireDirPath: string[];
-    EmbeddingDirPath: string[];
-    H2FDirPath: string[];
+    HNSWDirPath: string[];
+    InfoDirPath: string[];
+    RecycleDirPath: string[];
     MetaFilePath: string[];
     IndexPropPathsFileName: string;
 };
@@ -58,4 +59,5 @@ export declare function loadFileList(fileList: any, preloadCount?: number): Asyn
 }, void, unknown>;
 export declare function getNowTime(now?: Date): number;
 export declare function getNowRounded(now?: Date): number;
+export declare function readBufferFromFD(fd: any, filePath: any, start: any, length: any): Buffer<ArrayBuffer> | undefined;
 export * as default from "./QyUtils.ts";