datafolder 1.0.9 → 1.0.11

package/src/QyUtils.js

@@ -1,6 +1,6 @@
-import t from "node:crypto";
+import n from "node:crypto";
 
-import n from "node:path";
+import t from "node:path";
 
 import e from "node:process";
 
@@ -14,114 +14,118 @@ import {
     fileURLToPath as r
 } from "node:url";
 
-var a = n.dirname(r(import.meta.url)), u = Object.fromEntries([ "$del", "$link", "$rpl", "$spec", "$raw", "$all", "$clone", "$ext", "$or", "$and", "$gt", "$lt", "$gte", "$lte" ].map(r => [ r, r ])), f = "%", f = {
-    HiddenFolderName: f,
-    TriggerDirPath: [ f, "trigger" ],
-    RpcDirPath: [ f, "rpc" ],
-    IndexDirPath: [ f, "index" ],
+import {
+    logger as u
+} from "../src/QyLogger.js";
+
+var a = t.dirname(r(import.meta.url)), f = Object.fromEntries([ "$del", "$link", "$rpl", "$spec", "$raw", "$all", "$clone", "$ext", "$or", "$and", "$gt", "$lt", "$gte", "$lte" ].map(r => [ r, r ])), c = "%", c = {
+    HiddenFolderName: c,
+    TriggerDirPath: [ c, "trigger" ],
+    RpcDirPath: [ c, "rpc" ],
+    IndexDirPath: [ c, "index" ],
     IndexPropPathsFileName: "propPaths"
 };
 
-function c(r) {
+function s(r) {
     return r.startsWith("{") && r.endsWith("}");
 }
 
-function s(r) {
-    return c(r) && r.substring(1, r.length - 1);
+function d(r) {
+    return s(r) && r.substring(1, r.length - 1);
 }
 
-function d(r) {
+function m(r) {
     return 1e12 < r ? Math.round(100 * r / 1e12) / 100 + "TB" : 1e9 < r ? Math.round(100 * r / 1e9) / 100 + "GB" : 1e6 < r ? Math.round(100 * r / 1e6) / 100 + "MB" : 1e3 < r ? Math.round(100 * r / 1e3) / 100 + "KB" : r.toString();
 }
 
-function m(r, n = 10) {
-    return t.createHash("sha256").update(r, "utf8").digest("base64url").slice(0, n);
+function p(r, t = 10) {
+    return n.createHash("sha256").update(r, "utf8").digest("base64url").slice(0, t);
 }
 
-var p = 11;
+var h = 11;
 
 function g(r) {
     return "d" + r;
 }
 
-function h(r) {
+function y(r) {
     return "f" + r;
 }
 
-function y(r) {
+function v(r) {
     return "c" + r;
 }
 
-function v(r = 8) {
-    return t.randomBytes(r).toString("hex");
+function b(r = 8) {
+    return n.randomBytes(r).toString("hex");
 }
 
-function b(r) {
+function $(r) {
     return r[r.length - 1];
 }
 
-function $(r) {
-    if (null != r) for (var n in r) if (null != r[n]) return !1;
+function w(r) {
+    if (null != r) for (var t in r) if (null != r[t]) return !1;
     return !0;
 }
 
-function w(r) {
+function P(r) {
     return null != r && "object" == typeof r && !(r instanceof Date);
 }
 
-function P(r) {
+function S(r) {
     return "string" == typeof r;
 }
 
-function S(r) {
+function j(r) {
     return "number" == typeof r;
 }
 
-function j(r) {
+function x(r) {
     return "boolean" == typeof r;
 }
 
-function x(r) {
+function B(r) {
     return "function" == typeof r;
 }
 
-function B(r) {
-    var n = typeof r;
-    return "string" == n || "number" == n || "boolean" == n || r instanceof Date;
+function M(r) {
+    var t = typeof r;
+    return "string" == t || "number" == t || "boolean" == t || r instanceof Date;
 }
 
-function M(r) {
-    let n = r.length;
-    for (;null == r[n - 1] && 0 < n; ) --n;
-    return r.length = n, r;
+function O(r) {
+    let t = r.length;
+    for (;null == r[t - 1] && 0 < t; ) --t;
+    return r.length = t, r;
 }
 
-function O(t) {
-    return t.promise || (t.promise = new Promise((r, n) => Object.assign(t, {
+function T(n) {
+    return n.promise || (n.promise = new Promise((r, t) => Object.assign(n, {
         resolve: r,
-        reject: n
-    }))), t;
+        reject: t
+    }))), n;
 }
 
 function k(r) {
     return Buffer.isBuffer(r) ? r : Buffer.from(r);
 }
 
-function D(r) {
+function A(r) {
     return 1 < r ? "s" : "";
 }
 
-function F(r) {
+function D(r) {
     return l.mkdir(r, {
         recursive: !0
     });
 }
 
-function I(r) {
-    return F(n.dirname(r));
+function F(r) {
+    return D(t.dirname(r));
 }
 
-async function T(r) {
+async function I(r) {
     try {
         return (await l.stat(r)).size;
     } catch (r) {
@@ -129,40 +133,41 @@ async function T(r) {
     }
 }
 
-async function A(r) {
-    return !1 !== await T(r);
+async function W(r) {
+    return !1 !== await I(r);
 }
 
-function W(r, n) {
-    return i.setTimeout(1e3 * r, !0, {
-        signal: n
-    });
+function E(r, t = void 0) {
+    r *= 1e3;
+    return t ? i.setTimeout(r, !0, {
+        signal: t
+    }) : i.setTimeout(r);
 }
 
-let N = void 0;
+let N;
 
-async function E(n) {
+async function L(t) {
     try {
-        await I(n);
+        await F(t);
         var r = new Date().toISOString() + ": " + e.argv.join(" ");
-        function t() {
+        function n() {
             for (var r of Object.keys(N)) U(r);
         }
-        await l.writeFile(n, r, {
+        await l.writeFile(t, r, {
             flag: "wx",
             flush: !0
-        }), logger.info("Locked " + n), N ? N[n] = 1 : (N = {
-            [n]: 1
-        }, e.on("exit", t), e.on("SIGINT", t));
+        }), u.info("Locked " + t), N ? N[t] = 1 : (N = {
+            [t]: 1
+        }, e.on("exit", n), e.on("SIGINT", n));
     } catch (r) {
-        throw "EEXIST" == r.code && logger.error(`Already started by ${(await l.readFile(n)).toString()}:`, r), 
+        throw "EEXIST" == r.code && u.error(`Already started by ${(await l.readFile(t)).toString()}:`, r), 
         r;
     }
 }
 
 function U(r) {
     try {
-        o.unlinkSync(r), logger.info("Unlocked " + r);
+        o.unlinkSync(r), u.info("Unlocked " + r);
     } catch (r) {}
 }
 
@@ -183,58 +188,58 @@ function J(r) {
     return r.split(".");
 }
 
-function L(r) {
+function Q(r) {
     return r.join(".");
 }
 
-function z(r, n = 0) {
-    return Math.floor(Math.random() * (r - n)) + n;
+function z(r, t = 0) {
+    return Math.floor(Math.random() * (r - t)) + t;
 }
 
 function K(r) {
-    var n, t = {};
-    for (n of r) t[n] = 1;
-    return t;
+    var t, n = {};
+    for (t of r) n[t] = 1;
+    return n;
 }
 
-async function Q(n, t, e, o) {
+async function X(t, n, e, o) {
     var i = Buffer.allocUnsafe(o);
     for (let r = 0; r < o; ) {
-        var a = (await t.read(i, r, o - r, e + r)).bytesRead;
-        if (0 == a) return void logger.error(`read ${n} at ${e + r} returning 0 byte`);
+        var a = (await n.read(i, r, o - r, e + r)).bytesRead;
+        if (0 == a) return void u.error(`read ${t} at ${e + r} returning 0 byte`);
         r += a;
     }
     return i;
 }
 
-function X() {
+function q() {
     var r = e.availableMemory();
     return Math.round(r / (e.memoryUsage.rss() + r) * 100);
 }
 
-async function* q(e, n = 3) {
-    var t = e.length, o = (t < n && (n = t), new Array(t));
-    for (let r = 0; r < n; ++r) {
+async function* C(e, t = 3) {
+    var n = e.length, o = (n < t && (t = n), new Array(n));
+    for (let r = 0; r < t; ++r) {
         var i = o[r] = {};
         s(r, i);
     }
-    for (let r = 0; r < t; ++r) {
-        var a = o[r % n], {
+    for (let r = 0; r < n; ++r) {
+        var a = o[r % t], {
             filePath: u,
             bin: f
-        } = (a.bin || await a.promise, a), c = r + n;
-        c < t && s(c, a), yield {
+        } = (a.bin || await a.promise, a), c = r + t;
+        c < n && s(c, a), yield {
             filePath: u,
             bin: f
         };
     }
-    async function s(r, n) {
-        var r = e[r], t = l.readFile(r);
-        Object.assign(n, {
+    async function s(r, t) {
+        var r = e[r], n = l.readFile(r);
+        Object.assign(t, {
             filePath: r,
-            promise: t,
+            promise: n,
             bin: void 0
-        }), n.bin = await t;
+        }), t.bin = await n;
     }
 }
 
@@ -242,44 +247,44 @@ export * as default from "./QyUtils.js";
 
 export {
     a as __dirname,
-    u as SpecialOperators,
-    f as SpecialFilePaths,
-    c as isMatchName,
-    s as getMatchParam,
-    d as getSizeStr,
-    m as calHash,
-    p as Hash_Key_Length,
+    f as SpecialOperators,
+    c as SpecialFilePaths,
+    s as isMatchName,
+    d as getMatchParam,
+    m as getSizeStr,
+    p as calHash,
+    h as Hash_Key_Length,
     g as getSubdirMapKey,
-    h as getFileMapKey,
-    y as getContentKey,
-    v as getRandomStr,
-    b as arrayLast,
-    $ as isEmptyObj,
-    w as isNotNullObj,
-    P as isString,
-    S as isNumber,
-    j as isBoolean,
-    x as isFunction,
-    B as isSimpleType,
-    M as trimArrayTail,
-    O as setPromise,
+    y as getFileMapKey,
+    v as getContentKey,
+    b as getRandomStr,
+    $ as arrayLast,
+    w as isEmptyObj,
+    P as isNotNullObj,
+    S as isString,
+    j as isNumber,
+    x as isBoolean,
+    B as isFunction,
+    M as isSimpleType,
+    O as trimArrayTail,
+    T as setPromise,
     k as toBuffer,
-    D as getSuffix,
-    F as ensureDir,
-    I as ensureParentDir,
-    T as getSize,
-    A as folderOrFileExists,
-    W as sleep,
-    E as lockExclusiveFilePath,
+    A as getSuffix,
+    D as ensureDir,
+    F as ensureParentDir,
+    I as getSize,
+    W as folderOrFileExists,
+    E as sleep,
+    L as lockExclusiveFilePath,
     U as unlockExclusiveFilePath,
     R as fromSingleName,
     G as toSingleName,
     H as pathSplit,
     J as propNameSplit,
-    L as propNameJoin,
+    Q as propNameJoin,
     z as randomInt,
     K as listToMap,
-    Q as readBufferAsync,
-    X as memoryPercent,
-    q as loadFileList
+    X as readBufferAsync,
+    q as memoryPercent,
+    C as loadFileList
 };

package/src/simulate.js

@@ -18,40 +18,40 @@ import {
 } from "./QyUtils.js";
 
 import {
-    runQueries as l
+    logger as l
+} from "./QyLogger.js";
+
+import {
+    runQueries as c
 } from "../scripts/airbnb2_queries.js";
 
-logger.level = "info";
+l.level = "info";
 
-let r = "../airbnb", c = "./tmp/simulate", m, p, f, v, d, g;
+let r = "../airbnb", m = "./tmp/simulate", p, f, v, d, g, y;
 
-async function y(e) {
+async function b(e) {
     return JSON.parse(await s(t(r, e)));
 }
 
-function b(e) {
+function h(e) {
     console.time("insertData");
     var t, s, r, o = e.batch;
-    for (t of v) o.insert([ "hosts", t ], m[t]);
-    for (s of d) {
-        var i = p[s], {
+    for (t of d) o.insert([ "hosts", t ], p[t]);
+    for (s of g) {
+        var i = f[s], {
             property_type: a,
             room_type: n,
             bed_type: l
         } = i;
         o.insert([ "airbnb", a, n, l, s ], i);
     }
-    for (r of g) {
-        var c = f[r];
+    for (r of y) {
+        var c = v[r];
         o.insert([ "reviews", c.listing_id, r ], c);
     }
     o.run(), console.timeEnd("insertData");
 }
 
-function h() {
-    return m[v[u(v.length)]];
-}
-
 function _() {
     return p[d[u(d.length)]];
 }
@@ -60,19 +60,23 @@ function j() {
     return f[g[u(g.length)]];
 }
 
-async function D(e, t) {
+function D() {
+    return v[y[u(y.length)]];
+}
+
+async function E(e, t) {
     for (;0 < t--; ) {
         switch (u(3)) {
           case 0:
             s = a = i = o = r = void 0;
-            var s, r = e, o = h(), i = h(), a = {};
+            var s, r = e, o = _(), i = _(), a = {};
             for (s in i) 1 == u(2) && (a[s] = i[s]);
             r.insert([ "hosts", o.host_id ], a);
             break;
 
           case 1:
             n = f = p = m = c = l = o = r = void 0;
-            var n, r = e, o = _(), l = _(), c = {};
+            var n, r = e, o = j(), l = j(), c = {};
             for (n in l) 1 == u(2) && (c[n] = l[n]);
             var {
                 property_type: o,
@@ -85,7 +89,7 @@ async function D(e, t) {
 
           case 2:
             v = g = d = p = m = void 0;
-            var v, m = e, p = j(), d = j(), g = {};
+            var v, m = e, p = D(), d = D(), g = {};
             for (v in d) 1 == u(2) && (g[v] = d[v]);
             m.insert([ "reviews", p.listing_id, p._id ], g);
         }
@@ -95,25 +99,24 @@ async function D(e, t) {
 
 (async () => {
     console.time("total");
-    let t = process.argv[2] || 1e3, s = process.argv[3] || 100, e;
+    let t = +(process.argv[2] || 1e3), s = +(process.argv[3] || 100), e;
     if ("true" == process.argv[4]) {
         try {
-            await i(c, {
+            await i(m, {
                 recursive: !0
             });
         } catch (e) {}
         e = !0;
-    } else await n(c) || (e = !0);
-    process.argv[5] && (logger.level = process.argv[5]), console.time("loadData"), 
-    [ m, p, f ] = await Promise.all([ y("hosts.json"), y("listings.json"), y("reviews.json") ]), 
-    v = Object.keys(m), d = Object.keys(p), g = Object.keys(f), await !console.timeEnd("loadData"), 
-    console.log(`hosts:${v.length}, listings:${d.length}, reviews:` + g.length);
-    var r = new a(c, {
+    } else await n(m) || (e = !0);
+    process.argv[5] && (l.level = process.argv[5]), console.time("loadData"), [ p, f, v ] = await Promise.all([ b("hosts.json"), b("listings.json"), b("reviews.json") ]), 
+    d = Object.keys(p), g = Object.keys(f), y = Object.keys(v), await !console.timeEnd("loadData"), 
+    console.log(`hosts:${d.length}, listings:${g.length}, reviews:` + y.length);
+    var r = new a(m, {
         fileLogLevel: "basic"
-    }), o = (e && (await r.start(), b(r), await r.stop()), console.time("simulate " + s), 
-    await r.start(), l(r), []);
-    for (let e = 0; e < t; ++e) o.push(D(r, s));
-    await Promise.all(o), b(r), await r.stop(), console.timeEnd("simulate " + s), 
-    console.time("start"), await r.start(), console.timeEnd("start"), l(r), await r.stop(), 
+    }), o = (e && (await r.start(), h(r), await r.stop()), console.time("simulate " + s), 
+    await r.start(), c(r), []);
+    for (let e = 0; e < t; ++e) o.push(E(r, s));
+    await Promise.all(o), h(r), await r.stop(), console.timeEnd("simulate " + s), 
+    console.time("start"), await r.start(), console.timeEnd("start"), c(r), await r.stop(), 
     console.timeEnd("total");
 })();

package/src/simulate2.js

@@ -1,5 +1,5 @@
 import {
-    join as t
+    join as e
 } from "node:path";
 
 import {
@@ -14,96 +14,99 @@ import {
 import {
     folderOrFileExists as l,
     randomInt as c,
-    sleep as g
+    sleep as m
 } from "./QyUtils.js";
 
 import {
-    runQueries as m
+    logger as f
+} from "./QyLogger.js";
+
+import {
+    runQueries as g
 } from "../scripts/flights2_queries.js";
 
-logger.level = "info";
+f.level = "info";
 
-let s = "../GraphCourse_DemoData_ArangoDB-2", f = "./tmp/simulate2", d, p;
+let s = "../GraphCourse_DemoData_ArangoDB-2", p = "C:\\tmp\\simulate2", d, h;
 
-async function h(e) {
-    return JSON.parse(await o(t(s, e)));
+async function u(t) {
+    return JSON.parse(await o(e(s, t)));
 }
 
-function u() {
+function v() {
     return d[c(d.length)];
 }
 
-function v(e) {
-    var t, o = c(p.length), {
+function w(t) {
+    var e, o = c(h.length), {
         Year: s,
         Month: r,
         Day: i
-    } = p[o], a = p[c(p.length)], n = {};
-    for (t in a) 1 == c(2) && (n[t] = a[t]);
-    e.insert([ "flights", s, r, i, o ], n);
+    } = h[o], a = h[c(h.length)], n = {};
+    for (e in a) 1 == c(2) && (n[e] = a[e]);
+    t.insert([ "flights", s, r, i, o ], n);
 }
 
-async function w(e, t) {
-    for (;0 < t--; ) {
+async function y(t, e) {
+    for (;0 < e--; ) {
         switch (c(2)) {
           case 0:
             o = a = i = r = s = void 0;
-            var o, s = e, r = u()._key, i = u(), a = {};
+            var o, s = t, r = v()._key, i = v(), a = {};
             for (o in i) 1 == c(2) && (a[o] = i[o]);
             s.insert([ "airports", r ], a);
             break;
 
           case 1:
-            v(e);
+            w(t);
         }
-        await g(1);
+        await m(1);
     }
 }
 
-function y(e) {
-    var t = e;
+function b(t) {
+    var e = t;
     console.time("insertAirports");
-    var o, s = t.batch;
+    var o, s = e.batch;
     for (o of d) s.insert("airports/" + o._key, o);
     s.run(), console.timeEnd("insertAirports");
-    t = e;
-    logger.log("insertFlights begin"), console.time("insertFlights");
-    var r, i = t.batch;
-    for (let e = 0; e < p.length; ++e) {
-        var a = p[e], {
+    e = t;
+    f.log("insertFlights begin"), console.time("insertFlights");
+    var r, i = e.batch;
+    for (let t = 0; t < h.length; ++t) {
+        var a = h[t], {
             Year: n,
             Month: l,
             Day: c
         } = a;
-        i.insert(`flights/${n}/${l}/${c}/` + e, a);
+        i.insert(`flights/${n}/${l}/${c}/` + t, a);
     }
-    i.run(), logger.log("insertFlights end"), console.timeEnd("insertFlights"), 
-    console.time("createFlightsIndex");
-    for (r of t.dir("flights").subdirList) for (var g of r.subdirList) for (var m of g.subdirList) i.createIndex(m, [ "_from", "_to" ]);
+    i.run(), f.log("insertFlights end"), console.timeEnd("insertFlights"), console.time("createFlightsIndex");
+    for (r of e.dir("flights").subdirList) for (var m of r.subdirList) for (var g of m.subdirList) i.createIndex(g, [ "_from", "_to" ]);
     i.run(), console.timeEnd("createFlightsIndex");
 }
 
 (async () => {
     console.time("total");
-    let t = process.argv[2] || 1e3, o = process.argv[3] || 100, e;
+    let e = +(process.argv[2] || 1e3), o = +(process.argv[3] || 100), t;
     if ("true" == process.argv[4]) {
         try {
-            await a(f, {
+            await a(p, {
                 recursive: !0
             });
-        } catch (e) {}
-        e = !0;
-    } else await l(f) || (e = !0);
-    process.argv[5] && (logger.level = process.argv[5]), console.time("loadData"), 
-    [ d, p ] = await Promise.all([ h("airports.json"), h("flights.json") ]), await !console.timeEnd("loadData"), 
-    console.log(`airports:${d.length}, flights:` + p.length);
-    var s = new n(f, {
-        fileLogLevel: "basic"
-    }), r = (e && (await s.start(), y(s), await s.stop()), await s.start(), m(s), 
-    `simulate ${t} ` + o), i = (console.time(r), console.time("simulateStop"), []);
-    for (let e = 0; e < t; ++e) i.push(w(s, o));
-    await Promise.all(i), console.timeEnd(r), logger.log(`Simulate ${t} for ${o} ended.`), 
-    console.time("del%"), s.delDir("%"), console.timeEnd("del%"), y(s), await s.stop(), 
+        } catch (t) {}
+        t = !0;
+    } else await l(p) || (t = !0);
+    process.argv[5] && (f.level = process.argv[5]), console.time("loadData"), [ d, h ] = await Promise.all([ u("airports.json"), u("flights.json") ]), 
+    await !console.timeEnd("loadData"), console.log(`airports:${d.length}, flights:` + h.length);
+    var s = new n(p, {
+        fileLogLevel: "basic",
+        maxInMemSnapshotCount: 3
+    }), r = (t && (await s.start(), b(s), await s.stop()), await s.start(), g(s), 
+    `simulate ${e} ` + o), i = (console.time(r), console.time("simulateStop"), []);
+    for (let t = 0; t < e; ++t) i.push(y(s, o));
+    await Promise.all(i), console.timeEnd(r), f.log(`Simulate ${e} for ${o} ended.`), 
+    console.time("del%"), s.delDir("%"), console.timeEnd("del%"), b(s), await s.stop(), 
     console.timeEnd("simulateStop"), console.time("start"), await s.start(), console.timeEnd("start"), 
-    m(s), await s.stop(), console.timeEnd("total"), global.gc && global.gc();
+    g(s), await s.stop(), console.timeEnd("total"), global.gc && global.gc();
 })();