datafolder 1.0.2 → 1.0.4

package/src/QyUtils.js

@@ -1,250 +1,262 @@
-let crypto = require("node:crypto"), path = require("node:path"), process = require("node:process"), fsPromises = require("node:fs/promises"), fs = require("node:fs"), timers = require("node:timers/promises"), SpecialOperators = Object.fromEntries([ "$del", "$link", "$rpl", "$spec", "$raw", "$all", "$clone", "$ext", "$or", "$and", "$gt", "$lt", "$gte", "$lte" ].map(e => [ e, e ])), HiddenFolderName = "%", SpecialFilePaths = {
-    HiddenFolderName: HiddenFolderName,
-    TriggerDirPath: [ HiddenFolderName, "trigger" ],
-    RpcDirPath: [ HiddenFolderName, "rpc" ],
-    IndexDirPath: [ HiddenFolderName, "index" ],
+import e from "node:crypto";
+
+import t from "node:path";
+
+import n from "node:process";
+
+import a from "node:fs/promises";
+
+import o from "node:fs";
+
+import i from "node:timers/promises";
+
+import {
+    fileURLToPath as r
+} from "node:url";
+
+var s = t.dirname(r(import.meta.url)), u = Object.fromEntries([ "$del", "$link", "$rpl", "$spec", "$raw", "$all", "$clone", "$ext", "$or", "$and", "$gt", "$lt", "$gte", "$lte" ].map(r => [ r, r ])), f = "%", f = {
+    HiddenFolderName: f,
+    TriggerDirPath: [ f, "trigger" ],
+    RpcDirPath: [ f, "rpc" ],
+    IndexDirPath: [ f, "index" ],
     IndexPropPathsFileName: "propPaths"
 };
 
-function isMatchName(e) {
-    return e.startsWith("{") && e.endsWith("}");
+function c(r) {
+    return r.startsWith("{") && r.endsWith("}");
 }
 
-function getMatchParam(e) {
-    return isMatchName(e) && e.substring(1, e.length - 1);
+function l(r) {
+    return c(r) && r.substring(1, r.length - 1);
 }
 
-function getSizeStr(e) {
-    return 1e12 < e ? Math.round(100 * e / 1e12) / 100 + "TB" : 1e9 < e ? Math.round(100 * e / 1e9) / 100 + "GB" : 1e6 < e ? Math.round(100 * e / 1e6) / 100 + "MB" : 1e3 < e ? Math.round(100 * e / 1e3) / 100 + "KB" : e;
+function m(r) {
+    return 1e12 < r ? Math.round(100 * r / 1e12) / 100 + "TB" : 1e9 < r ? Math.round(100 * r / 1e9) / 100 + "GB" : 1e6 < r ? Math.round(100 * r / 1e6) / 100 + "MB" : 1e3 < r ? Math.round(100 * r / 1e3) / 100 + "KB" : r;
 }
 
-let Hash_Length = 10;
+let p = 10;
 
-function calHash(e, t = Hash_Length) {
-    return crypto.createHash("sha256").update(e, "utf8").digest("base64url").slice(0, t);
+function d(r, t = 10) {
+    return e.createHash("sha256").update(r, "utf8").digest("base64url").slice(0, t);
 }
 
-let Hash_Key_Length = Hash_Length + 1;
+var g = 11;
 
-function getSubdirMapKey(e) {
-    return "d" + e;
+function h(r) {
+    return "d" + r;
 }
 
-function getFileMapKey(e) {
-    return "f" + e;
+function y(r) {
+    return "f" + r;
 }
 
-function getContentKey(e) {
-    return "c" + e;
+function S(r) {
+    return "c" + r;
 }
 
-function getRandomStr(e = 8) {
-    return crypto.randomBytes(e).toString("hex");
+function b(r = 8) {
+    return e.randomBytes(r).toString("hex");
 }
 
-function arrayLast(e) {
-    return e[e.length - 1];
+function v(r) {
+    return r[r.length - 1];
 }
 
-function isEmptyObj(e) {
-    if (null != e) for (var t in e) if (null != e[t]) return !1;
+function $(r) {
+    if (null != r) for (var t in r) if (null != r[t]) return !1;
     return !0;
 }
 
-function isNotNullObj(e) {
-    return null != e && "object" == typeof e && !(e instanceof Date);
+function P(r) {
+    return null != r && "object" == typeof r && !(r instanceof Date);
 }
 
-function isString(e) {
-    return "string" == typeof e;
+function M(r) {
+    return "string" == typeof r;
 }
 
-function isNumber(e) {
-    return "number" == typeof e;
+function x(r) {
+    return "number" == typeof r;
 }
 
-function isBoolean(e) {
-    return "boolean" == typeof e;
+function N(r) {
+    return "boolean" == typeof r;
 }
 
-function isFunction(e) {
-    return "function" == typeof e;
+function w(r) {
+    return "function" == typeof r;
 }
 
-function isSimpleType(e) {
-    var t = typeof e;
-    return "string" == t || "number" == t || "boolean" == t || e instanceof Date;
+function B(r) {
+    var t = typeof r;
+    return "string" == t || "number" == t || "boolean" == t || r instanceof Date;
 }
 
-function trimArrayTail(e) {
-    let t = e.length;
-    for (;null == e[t - 1] && 0 < t; ) --t;
-    return e.length = t, e;
+function j(r) {
+    let t = r.length;
+    for (;null == r[t - 1] && 0 < t; ) --t;
+    return r.length = t, r;
 }
 
-function setPromise(r) {
-    return r.promise || (r.promise = new Promise((e, t) => Object.assign(r, {
-        resolve: e,
+function F(e) {
+    return e.promise || (e.promise = new Promise((r, t) => Object.assign(e, {
+        resolve: r,
         reject: t
-    }))), r;
+    }))), e;
 }
 
-function toBuffer(e) {
-    return Buffer.isBuffer(e) ? e : Buffer.from(e);
+function A(r) {
+    return Buffer.isBuffer(r) ? r : Buffer.from(r);
 }
 
-function getSuffix(e) {
-    return 1 < e ? "s" : "";
+function H(r) {
+    return 1 < r ? "s" : "";
 }
 
-function ensureDir(e) {
-    return fsPromises.mkdir(e, {
+function O(r) {
+    return a.mkdir(r, {
         recursive: !0
     });
 }
 
-function ensureParentDir(e) {
-    return ensureDir(path.dirname(e));
+function T(r) {
+    return O(t.dirname(r));
 }
 
-async function getSize(e) {
+async function k(r) {
     try {
-        return (await fsPromises.stat(e)).size;
-    } catch (e) {
+        return (await a.stat(r)).size;
+    } catch (r) {
         return !1;
     }
 }
 
-async function folderOrFileExists(e) {
-    return !1 !== await getSize(e);
+async function K(r) {
+    return !1 !== await k(r);
 }
 
-function sleep(e, t) {
-    return timers.setTimeout(1e3 * e, !0, {
+function L(r, t) {
+    return i.setTimeout(1e3 * r, !0, {
         signal: t
     });
 }
 
-let exclusiveFilePathMap = void 0;
+let D = void 0;
 
-function unlockPathMap() {
-    for (var e of Object.keys(exclusiveFilePathMap)) unlockExclusiveFilePath(e);
+function E() {
+    for (var r of Object.keys(D)) I(r);
 }
 
-async function lockExclusiveFilePath(t) {
+async function U(t) {
     try {
-        await ensureParentDir(t);
-        var e = new Date().toISOString() + ": " + process.argv.join(" ");
-        await fsPromises.writeFile(t, e, {
+        await T(t);
+        var r = new Date().toISOString() + ": " + n.argv.join(" ");
+        await a.writeFile(t, r, {
             flag: "wx",
             flush: !0
-        }), logger.info("Locked " + t), exclusiveFilePathMap ? exclusiveFilePathMap[t] = 1 : (exclusiveFilePathMap = {
+        }), logger.info("Locked " + t), D ? D[t] = 1 : (D = {
             [t]: 1
-        }, process.on("exit", unlockPathMap), process.on("SIGINT", unlockPathMap));
-    } catch (e) {
-        throw "EEXIST" == e.code && logger.error(`Already started by ${(await fsPromises.readFile(t)).toString()}:`, e), 
-        e;
+        }, n.on("exit", E), n.on("SIGINT", E));
+    } catch (r) {
+        throw "EEXIST" == r.code && logger.error(`Already started by ${(await a.readFile(t)).toString()}:`, r), 
+        r;
     }
 }
 
-function unlockExclusiveFilePath(e) {
+function I(r) {
     try {
-        fs.unlinkSync(e), logger.info("Unlocked " + e);
-    } catch (e) {}
+        o.unlinkSync(r), logger.info("Unlocked " + r);
+    } catch (r) {}
 }
 
-function fromSingleName(e) {
-    return e.startsWith("[") && e.endsWith("]") ? JSON.parse(e) : e;
+function _(r) {
+    return r.startsWith("[") && r.endsWith("]") ? JSON.parse(r) : r;
 }
 
-function toSingleName(e) {
-    return Array.isArray(e) ? 1 == e.length ? e[0] : JSON.stringify(e) : e;
+function R(r) {
+    return Array.isArray(r) ? 1 == r.length ? r[0] : JSON.stringify(r) : r;
 }
 
-function pathSplit(e) {
-    e = e.split("/");
-    return e[0] || (e[0] = "/"), e;
+function W(r) {
+    r = r.split("/");
+    return r[0] || (r[0] = "/"), r;
 }
 
-function propNameSplit(e) {
-    return e.split(".");
+function z(r) {
+    return r.split(".");
 }
 
-function propNameJoin(e) {
-    return e.join(".");
+function J(r) {
+    return r.join(".");
 }
 
-function randomInt(e, t = 0) {
-    return Math.floor(Math.random() * (e - t)) + t;
+function G(r, t = 0) {
+    return Math.floor(Math.random() * (r - t)) + t;
 }
 
-function listToMap(e) {
-    var t, r = {};
-    for (t of e) r[t] = 1;
-    return r;
+function C(r) {
+    var t, e = {};
+    for (t of r) e[t] = 1;
+    return e;
 }
 
-async function readBufferAsync(t, r, n, i) {
-    var a = Buffer.allocUnsafe(i);
-    for (let e = 0; e < i; ) {
-        var o = (await r.read(a, e, i - e, n + e)).bytesRead;
-        if (0 == o) return void logger.error(`read ${t} at ${n + e} returning 0 byte`);
-        e += o;
+async function Q(t, e, n, a) {
+    var o = Buffer.allocUnsafe(a);
+    for (let r = 0; r < a; ) {
+        var i = (await e.read(o, r, a - r, n + r)).bytesRead;
+        if (0 == i) return void logger.error(`read ${t} at ${n + r} returning 0 byte`);
+        r += i;
     }
-    return a;
-}
-
-function memoryPercent() {
-    var e = process.availableMemory();
-    return Math.round(e / (process.memoryUsage.rss() + e) * 100);
-}
-
-function generatorToList(e) {
-    var t, r = [];
-    for (t of e) r.push(t);
-    return r;
-}
-
-Object.assign(module.exports, {
-    SpecialOperators: SpecialOperators,
-    calHash: calHash,
-    Hash_Length: Hash_Length,
-    Hash_Key_Length: Hash_Key_Length,
-    pathSplit: pathSplit,
-    isMatchName: isMatchName,
-    getMatchParam: getMatchParam,
-    isNumber: isNumber,
-    isString: isString,
-    isBoolean: isBoolean,
-    isFunction: isFunction,
-    isNotNullObj: isNotNullObj,
-    isSimpleType: isSimpleType,
-    isEmptyObj: isEmptyObj,
-    getSubdirMapKey: getSubdirMapKey,
-    getFileMapKey: getFileMapKey,
-    getContentKey: getContentKey,
-    arrayLast: arrayLast,
-    getRandomStr: getRandomStr,
-    getSizeStr: getSizeStr,
-    setPromise: setPromise,
-    trimArrayTail: trimArrayTail,
-    toBuffer: toBuffer,
-    getSuffix: getSuffix,
-    sleep: sleep,
-    ensureDir: ensureDir,
-    ensureParentDir: ensureParentDir,
-    getSize: getSize,
-    lockExclusiveFilePath: lockExclusiveFilePath,
-    unlockExclusiveFilePath: unlockExclusiveFilePath,
-    folderOrFileExists: folderOrFileExists,
-    SpecialFilePaths: SpecialFilePaths,
-    fromSingleName: fromSingleName,
-    toSingleName: toSingleName,
-    randomInt: randomInt,
-    propNameSplit: propNameSplit,
-    propNameJoin: propNameJoin,
-    listToMap: listToMap,
-    readBufferAsync: readBufferAsync,
-    memoryPercent: memoryPercent,
-    generatorToList: generatorToList
-});
+    return o;
+}
+
+function X() {
+    var r = n.availableMemory();
+    return Math.round(r / (n.memoryUsage.rss() + r) * 100);
+}
+
+export * as default from "./QyUtils.js";
+
+export {
+    u as SpecialOperators,
+    d as calHash,
+    p as Hash_Length,
+    g as Hash_Key_Length,
+    W as pathSplit,
+    c as isMatchName,
+    l as getMatchParam,
+    x as isNumber,
+    M as isString,
+    N as isBoolean,
+    w as isFunction,
+    P as isNotNullObj,
+    B as isSimpleType,
+    $ as isEmptyObj,
+    h as getSubdirMapKey,
+    y as getFileMapKey,
+    S as getContentKey,
+    v as arrayLast,
+    b as getRandomStr,
+    m as getSizeStr,
+    F as setPromise,
+    j as trimArrayTail,
+    A as toBuffer,
+    H as getSuffix,
+    L as sleep,
+    O as ensureDir,
+    T as ensureParentDir,
+    k as getSize,
+    U as lockExclusiveFilePath,
+    I as unlockExclusiveFilePath,
+    K as folderOrFileExists,
+    f as SpecialFilePaths,
+    _ as fromSingleName,
+    R as toSingleName,
+    G as randomInt,
+    z as propNameSplit,
+    J as propNameJoin,
+    C as listToMap,
+    Q as readBufferAsync,
+    X as memoryPercent,
+    s as __dirname
+};

package/src/QyWorker.js

@@ -1,33 +1,47 @@
-let path = require("node:path"), {
-    Worker,
-    setEnvironmentData
-} = require("node:worker_threads"), logger = require("./QyLogger.js").logger, setPromise = require("./QyUtils.js").setPromise;
+import {
+    isAbsolute as a,
+    join as m
+} from "node:path";
 
-function runOnceWithWorker(e, r) {
-    setEnvironmentData("logLevel", logger.level);
+import {
+    Worker as l,
+    setEnvironmentData as p
+} from "node:worker_threads";
+
+import {
+    logger as g
+} from "./QyLogger.js";
+
+import {
+    setPromise as k,
+    __dirname as d
+} from "./QyUtils.js";
+
+function e(e, r) {
+    p("logLevel", g.level);
     var o = r.operation;
     let t = e + " " + o, {
-        promise: i,
-        resolve: n
-    } = (logger.info("Starting worker " + t), setPromise({}));
-    path.isAbsolute(e) || (e = path.join(__dirname, e));
-    o = new Worker(e, {
+        promise: s,
+        resolve: i
+    } = (g.info("Starting worker " + t), k({}));
+    a(e) || (e = m(d, e));
+    o = new l(e, {
         workerData: r
     });
-    let s;
+    let n;
     return o.on("message", e => {
-        s = !0, n(e);
+        n = !0, i(e);
     }), o.on("error", e => {
-        s = !0, n(e);
+        n = !0, i(e);
     }), o.on("exit", e => {
-        s ? logger.info(`Worker ${t} exited.`) : (logger.log(`Worker ${t} stopped with exitCode ` + e), 
-        n());
+        n ? g.info(`Worker ${t} exited.`) : (g.log(`Worker ${t} stopped with exitCode ` + e), 
+        i());
     }), {
         worker: o,
-        promise: i
+        promise: s
     };
 }
 
-Object.assign(module.exports, {
-    runOnceWithWorker: runOnceWithWorker
-});
+export {
+    e as runOnceWithWorker
+};

package/src/hackernews.js

@@ -1,93 +1,85 @@
-let path = require("node:path"), fsPromises = require("node:fs/promises"), QyDB = require("./QyDB.js").QyDB, folderOrFileExists = require("./QyUtils.js").folderOrFileExists, shardRootFolder = (logger.level = "info", 
-"D:\\exp\\HackerBook\\static-shards_json"), dataFolder = "./tmp/hackernews", qyDB = new QyDB(dataFolder, {
+import {
+    join as o
+} from "node:path";
+
+import {
+    readdir as n,
+    readFile as i
+} from "node:fs/promises";
+
+import {
+    QyDB as e
+} from "./QyDB.js";
+
+logger.level = "info";
+
+let s = "D:\\exp\\HackerBook\\static-shards_json", w = new e({
     fileLogLevel: "basic",
     maxInMemSnapshotCount: 1,
     snapshotMaxChangeCount: 2e3,
     unloadMemInterval: 2e4
-}), batchSize = 100;
+}), C = 100;
 
-function insertShard(e, t) {
+function l(e, t) {
     var a, r = {}, o = {};
     for (a of e) {
         var {
             parent_id: n,
-            child_id: s
+            child_id: i
         } = a;
-        let e = o[s], t = (e = e || (o[s] = {}), o[n]), r = (t = t || (o[n] = {})).children;
-        (r = r || (t.children = {}))[s] = e;
+        let e = o[i], t = (e = e || (o[i] = {}), o[n]), r = (t = t || (o[n] = {})).children;
+        (r = r || (t.children = {}))[i] = e;
     }
-    var i, l, c, d = {};
-    let h = 0, f = 0;
-    for (i of t) {
+    var s, l, c, m = {};
+    let d = 0, f = 0;
+    for (s of t) {
         var {
-            id: m,
-            time: u,
+            id: h,
+            time: p,
             parent: g
-        } = i;
+        } = s;
         if (null == g) {
-            u = getDatePath(u);
-            r[m] = u;
-            let e = o[m];
-            e ? e.item = i : e = o[m] = {
-                item: i
-            }, d[m] = e, ++h;
+            p = (e => [ (e = new Date(1e3 * e)).getUTCFullYear(), e.getUTCMonth() + 1, e.getUTCDate() ])(p);
+            r[h] = p;
+            let e = o[h];
+            e ? e.item = s : e = o[h] = {
+                item: s
+            }, m[h] = e, ++d;
         } else {
-            u = o[g];
-            if (u) {
-                let e = u.children, t = (e = e || (u.children = {}))[m];
-                t ? t.item = i : t = e[m] = {
-                    item: i
+            p = o[g];
+            if (p) {
+                let e = p.children, t = (e = e || (p.children = {}))[h];
+                t ? t.item = s : t = e[h] = {
+                    item: s
                 }, ++f;
-            } else logger.warn(m + `'s parent ${g} not exist.`);
+            } else logger.warn(h + `'s parent ${g} not exist.`);
         }
     }
-    let p = 0;
-    for (l in d) {
-        var v = r[l];
-        qyDB.batch.insert([ "hackernews", ...v, l ], d[l]), ++p >= batchSize && (qyDB.batch.run(), 
-        p = 0);
-    }
-    for (c in 0 < p && qyDB.batch.run(), d) cleanContent(d[c]);
-}
-
-function cleanContent(e) {
-    delete e.item;
-    var t, r = e.children;
-    for (t in r) cleanContent(r[t]);
-}
-
-function getDatePath(e) {
-    e = new Date(1e3 * e);
-    return [ e.getUTCFullYear(), e.getUTCMonth() + 1, e.getUTCDate() ];
-}
-
-async function insertShards() {
-    var e;
-    for (e of (await fsPromises.readdir(shardRootFolder)).map(e => /([0-9]+).json/.exec(e)[1]).sort((e, t) => e - t)) {
-        var t = `shard_${e}.json`, {
-            edges: r,
-            items: a
-        } = JSON.parse(await fsPromises.readFile(path.join(shardRootFolder, t)));
-        logger.warn(t), insertShard(r, a);
+    let v = 0;
+    for (l in m) {
+        var u = r[l];
+        w.batch.insert([ "hackernews", ...u, l ], m[l]), ++v >= C && (w.batch.run(), 
+        v = 0);
     }
+    for (c in 0 < v && w.batch.run(), m) !function e(t) {
+        delete t.item;
+        let r = t.children;
+        for (var a in r) e(r[a]);
+    }(m[c]);
 }
 
-async function run() {
-    console.time("accu"), process.argv[3] && (logger.level = process.argv[3]);
-    let e = !0;
-    if ("true" == process.argv[2]) {
-        try {
-            await fsPromises.rm(dataFolder, {
-                recursive: !0
-            });
-        } catch (e) {
-            logger.warn(e);
+(async () => {
+    if (process.argv[2] && (logger.level = process.argv[2]), console.time("run"), 
+    console.time("start"), await w.start(), console.timeEnd("start"), !w.dir("hackernews")) {
+        var e;
+        for (e of (await n(s)).map(e => /([0-9]+).json/.exec(e)[1]).sort((e, t) => e - t)) {
+            var t = `shard_${e}.json`, {
+                edges: r,
+                items: a
+            } = JSON.parse(await i(o(s, t)));
+            logger.warn(t), l(r, a);
         }
-        e = !0;
-    } else await folderOrFileExists(dataFolder) || (e = !0);
-    console.time("run"), console.time("start"), await qyDB.start(), console.timeEnd("start"), 
-    e && await insertShards(), console.time("stop"), await qyDB.stop(), console.timeEnd("stop"), 
-    console.timeEnd("run"), console.timeEnd("accu");
-}
-
-run();
+        await 0;
+    }
+    console.time("stop"), await w.stop(), console.timeEnd("stop"), console.timeEnd("run");
+})();