datafolder 1.0.7 → 1.0.9

package/src/QyUtils.js

@@ -1,10 +1,10 @@
-import e from "node:crypto";
+import t from "node:crypto";
 
-import t from "node:path";
+import n from "node:path";
 
-import n from "node:process";
+import e from "node:process";
 
-import a from "node:fs/promises";
+import l from "node:fs/promises";
 
 import o from "node:fs";
 
@@ -14,7 +14,7 @@ import {
     fileURLToPath as r
 } from "node:url";
 
-var s = t.dirname(r(import.meta.url)), u = Object.fromEntries([ "$del", "$link", "$rpl", "$spec", "$raw", "$all", "$clone", "$ext", "$or", "$and", "$gt", "$lt", "$gte", "$lte" ].map(r => [ r, r ])), f = "%", f = {
+var a = n.dirname(r(import.meta.url)), u = Object.fromEntries([ "$del", "$link", "$rpl", "$spec", "$raw", "$all", "$clone", "$ext", "$or", "$and", "$gt", "$lt", "$gte", "$lte" ].map(r => [ r, r ])), f = "%", f = {
     HiddenFolderName: f,
     TriggerDirPath: [ f, "trigger" ],
     RpcDirPath: [ f, "rpc" ],
@@ -26,237 +26,260 @@ function c(r) {
     return r.startsWith("{") && r.endsWith("}");
 }
 
-function l(r) {
+function s(r) {
     return c(r) && r.substring(1, r.length - 1);
 }
 
-function m(r) {
-    return 1e12 < r ? Math.round(100 * r / 1e12) / 100 + "TB" : 1e9 < r ? Math.round(100 * r / 1e9) / 100 + "GB" : 1e6 < r ? Math.round(100 * r / 1e6) / 100 + "MB" : 1e3 < r ? Math.round(100 * r / 1e3) / 100 + "KB" : r;
+function d(r) {
+    return 1e12 < r ? Math.round(100 * r / 1e12) / 100 + "TB" : 1e9 < r ? Math.round(100 * r / 1e9) / 100 + "GB" : 1e6 < r ? Math.round(100 * r / 1e6) / 100 + "MB" : 1e3 < r ? Math.round(100 * r / 1e3) / 100 + "KB" : r.toString();
 }
 
-let p = 10;
-
-function d(r, t = 10) {
-    return e.createHash("sha256").update(r, "utf8").digest("base64url").slice(0, t);
+function m(r, n = 10) {
+    return t.createHash("sha256").update(r, "utf8").digest("base64url").slice(0, n);
 }
 
-var g = 11;
+var p = 11;
 
-function h(r) {
+function g(r) {
     return "d" + r;
 }
 
-function y(r) {
+function h(r) {
     return "f" + r;
 }
 
-function S(r) {
+function y(r) {
     return "c" + r;
 }
 
-function b(r = 8) {
-    return e.randomBytes(r).toString("hex");
+function v(r = 8) {
+    return t.randomBytes(r).toString("hex");
 }
 
-function v(r) {
+function b(r) {
     return r[r.length - 1];
 }
 
 function $(r) {
-    if (null != r) for (var t in r) if (null != r[t]) return !1;
+    if (null != r) for (var n in r) if (null != r[n]) return !1;
     return !0;
 }
 
-function P(r) {
+function w(r) {
     return null != r && "object" == typeof r && !(r instanceof Date);
 }
 
-function M(r) {
+function P(r) {
     return "string" == typeof r;
 }
 
-function x(r) {
+function S(r) {
     return "number" == typeof r;
 }
 
-function N(r) {
+function j(r) {
     return "boolean" == typeof r;
 }
 
-function w(r) {
+function x(r) {
     return "function" == typeof r;
 }
 
 function B(r) {
-    var t = typeof r;
-    return "string" == t || "number" == t || "boolean" == t || r instanceof Date;
+    var n = typeof r;
+    return "string" == n || "number" == n || "boolean" == n || r instanceof Date;
 }
 
-function j(r) {
-    let t = r.length;
-    for (;null == r[t - 1] && 0 < t; ) --t;
-    return r.length = t, r;
+function M(r) {
+    let n = r.length;
+    for (;null == r[n - 1] && 0 < n; ) --n;
+    return r.length = n, r;
 }
 
-function F(e) {
-    return e.promise || (e.promise = new Promise((r, t) => Object.assign(e, {
+function O(t) {
+    return t.promise || (t.promise = new Promise((r, n) => Object.assign(t, {
         resolve: r,
-        reject: t
-    }))), e;
+        reject: n
+    }))), t;
 }
 
-function A(r) {
+function k(r) {
     return Buffer.isBuffer(r) ? r : Buffer.from(r);
 }
 
-function H(r) {
+function D(r) {
     return 1 < r ? "s" : "";
 }
 
-function O(r) {
-    return a.mkdir(r, {
+function F(r) {
+    return l.mkdir(r, {
         recursive: !0
     });
 }
 
-function T(r) {
-    return O(t.dirname(r));
+function I(r) {
+    return F(n.dirname(r));
 }
 
-async function k(r) {
+async function T(r) {
     try {
-        return (await a.stat(r)).size;
+        return (await l.stat(r)).size;
     } catch (r) {
         return !1;
     }
 }
 
-async function K(r) {
-    return !1 !== await k(r);
+async function A(r) {
+    return !1 !== await T(r);
 }
 
-function L(r, t) {
+function W(r, n) {
     return i.setTimeout(1e3 * r, !0, {
-        signal: t
+        signal: n
     });
 }
 
-let D = void 0;
-
-function E() {
-    for (var r of Object.keys(D)) I(r);
-}
+let N = void 0;
 
-async function U(t) {
+async function E(n) {
     try {
-        await T(t);
-        var r = new Date().toISOString() + ": " + n.argv.join(" ");
-        await a.writeFile(t, r, {
+        await I(n);
+        var r = new Date().toISOString() + ": " + e.argv.join(" ");
+        function t() {
+            for (var r of Object.keys(N)) U(r);
+        }
+        await l.writeFile(n, r, {
             flag: "wx",
             flush: !0
-        }), logger.info("Locked " + t), D ? D[t] = 1 : (D = {
-            [t]: 1
-        }, n.on("exit", E), n.on("SIGINT", E));
+        }), logger.info("Locked " + n), N ? N[n] = 1 : (N = {
+            [n]: 1
+        }, e.on("exit", t), e.on("SIGINT", t));
     } catch (r) {
-        throw "EEXIST" == r.code && logger.error(`Already started by ${(await a.readFile(t)).toString()}:`, r), 
+        throw "EEXIST" == r.code && logger.error(`Already started by ${(await l.readFile(n)).toString()}:`, r), 
         r;
     }
 }
 
-function I(r) {
+function U(r) {
     try {
         o.unlinkSync(r), logger.info("Unlocked " + r);
     } catch (r) {}
 }
 
-function _(r) {
+function R(r) {
     return r.startsWith("[") && r.endsWith("]") ? JSON.parse(r) : r;
 }
 
-function R(r) {
+function G(r) {
     return Array.isArray(r) ? 1 == r.length ? r[0] : JSON.stringify(r) : r;
 }
 
-function W(r) {
+function H(r) {
     r = r.split("/");
     return r[0] || (r[0] = "/"), r;
 }
 
-function z(r) {
+function J(r) {
     return r.split(".");
 }
 
-function J(r) {
+function L(r) {
     return r.join(".");
 }
 
-function G(r, t = 0) {
-    return Math.floor(Math.random() * (r - t)) + t;
+function z(r, n = 0) {
+    return Math.floor(Math.random() * (r - n)) + n;
 }
 
-function C(r) {
-    var t, e = {};
-    for (t of r) e[t] = 1;
-    return e;
+function K(r) {
+    var n, t = {};
+    for (n of r) t[n] = 1;
+    return t;
 }
 
-async function Q(t, e, n, a) {
-    var o = Buffer.allocUnsafe(a);
-    for (let r = 0; r < a; ) {
-        var i = (await e.read(o, r, a - r, n + r)).bytesRead;
-        if (0 == i) return void logger.error(`read ${t} at ${n + r} returning 0 byte`);
-        r += i;
+async function Q(n, t, e, o) {
+    var i = Buffer.allocUnsafe(o);
+    for (let r = 0; r < o; ) {
+        var a = (await t.read(i, r, o - r, e + r)).bytesRead;
+        if (0 == a) return void logger.error(`read ${n} at ${e + r} returning 0 byte`);
+        r += a;
     }
-    return o;
+    return i;
 }
 
 function X() {
-    var r = n.availableMemory();
-    return Math.round(r / (n.memoryUsage.rss() + r) * 100);
+    var r = e.availableMemory();
+    return Math.round(r / (e.memoryUsage.rss() + r) * 100);
+}
+
+async function* q(e, n = 3) {
+    var t = e.length, o = (t < n && (n = t), new Array(t));
+    for (let r = 0; r < n; ++r) {
+        var i = o[r] = {};
+        s(r, i);
+    }
+    for (let r = 0; r < t; ++r) {
+        var a = o[r % n], {
+            filePath: u,
+            bin: f
+        } = (a.bin || await a.promise, a), c = r + n;
+        c < t && s(c, a), yield {
+            filePath: u,
+            bin: f
+        };
+    }
+    async function s(r, n) {
+        var r = e[r], t = l.readFile(r);
+        Object.assign(n, {
+            filePath: r,
+            promise: t,
+            bin: void 0
+        }), n.bin = await t;
+    }
 }
 
 export * as default from "./QyUtils.js";
 
 export {
+    a as __dirname,
     u as SpecialOperators,
-    d as calHash,
-    p as Hash_Length,
-    g as Hash_Key_Length,
-    W as pathSplit,
+    f as SpecialFilePaths,
     c as isMatchName,
-    l as getMatchParam,
-    x as isNumber,
-    M as isString,
-    N as isBoolean,
-    w as isFunction,
-    P as isNotNullObj,
-    B as isSimpleType,
+    s as getMatchParam,
+    d as getSizeStr,
+    m as calHash,
+    p as Hash_Key_Length,
+    g as getSubdirMapKey,
+    h as getFileMapKey,
+    y as getContentKey,
+    v as getRandomStr,
+    b as arrayLast,
     $ as isEmptyObj,
-    h as getSubdirMapKey,
-    y as getFileMapKey,
-    S as getContentKey,
-    v as arrayLast,
-    b as getRandomStr,
-    m as getSizeStr,
-    F as setPromise,
-    j as trimArrayTail,
-    A as toBuffer,
-    H as getSuffix,
-    L as sleep,
-    O as ensureDir,
-    T as ensureParentDir,
-    k as getSize,
-    U as lockExclusiveFilePath,
-    I as unlockExclusiveFilePath,
-    K as folderOrFileExists,
-    f as SpecialFilePaths,
-    _ as fromSingleName,
-    R as toSingleName,
-    G as randomInt,
-    z as propNameSplit,
-    J as propNameJoin,
-    C as listToMap,
+    w as isNotNullObj,
+    P as isString,
+    S as isNumber,
+    j as isBoolean,
+    x as isFunction,
+    B as isSimpleType,
+    M as trimArrayTail,
+    O as setPromise,
+    k as toBuffer,
+    D as getSuffix,
+    F as ensureDir,
+    I as ensureParentDir,
+    T as getSize,
+    A as folderOrFileExists,
+    W as sleep,
+    E as lockExclusiveFilePath,
+    U as unlockExclusiveFilePath,
+    R as fromSingleName,
+    G as toSingleName,
+    H as pathSplit,
+    J as propNameSplit,
+    L as propNameJoin,
+    z as randomInt,
+    K as listToMap,
     Q as readBufferAsync,
     X as memoryPercent,
-    s as __dirname
+    q as loadFileList
 };

package/src/QyWorker.d.ts

@@ -0,0 +1,5 @@
+import { Worker } from "node:worker_threads";
+export declare function runOnceWithWorker(workerJsFilePath: any, workerData: any): {
+    worker: Worker;
+    promise: any;
+};

package/src/QyWorker.js

@@ -1,5 +1,5 @@
 import {
-    isAbsolute as a,
+    isAbsolute as n,
     join as m
 } from "node:path";
 
@@ -13,31 +13,31 @@ import {
 } from "./QyLogger.js";
 
 import {
-    setPromise as k,
-    __dirname as d
+    setPromise as d,
+    __dirname as f
 } from "./QyUtils.js";
 
-function e(e, r) {
+function e(e, o) {
     p("logLevel", g.level);
-    var o = r.operation;
-    let t = e + " " + o, {
+    var r = o.operation;
+    let t = e + " " + r, {
         promise: s,
         resolve: i
-    } = (g.info("Starting worker " + t), k({}));
-    a(e) || (e = m(d, e));
-    o = new l(e, {
-        workerData: r
+    } = (g.info("Starting worker " + t), d({}));
+    n(e) || (e = m(f, e));
+    r = new l(e, {
+        workerData: o
     });
-    let n;
-    return o.on("message", e => {
-        n = !0, i(e);
-    }), o.on("error", e => {
-        n = !0, i(e);
-    }), o.on("exit", e => {
-        n ? g.info(`Worker ${t} exited.`) : (g.log(`Worker ${t} stopped with exitCode ` + e), 
+    let a;
+    return r.on("message", e => {
+        a = !0, i(e);
+    }), r.on("error", e => {
+        a = !0, i(e);
+    }), r.on("exit", e => {
+        a ? g.info(`Worker ${t} exited.`) : (g.log(`Worker ${t} stopped with exitCode ` + e), 
         i());
     }), {
-        worker: o,
+        worker: r,
         promise: s
     };
 }

package/src/hackernews.d.ts

@@ -0,0 +1 @@
+export {};

package/src/hackernews.js

@@ -1,85 +1,97 @@
 import {
-    join as o
+    join as n,
+    basename as i
 } from "node:path";
 
 import {
-    readdir as n,
-    readFile as i
+    readdir as s
 } from "node:fs/promises";
 
 import {
     QyDB as e
 } from "./QyDB.js";
 
-logger.level = "info";
+import {
+    loadFileList as l
+} from "./QyUtils.js";
+
+import {
+    logger as u
+} from "./QyLogger.js";
+
+u.level = "info";
 
-let s = "D:\\exp\\HackerBook\\static-shards_json", w = new e({
-    fileLogLevel: "basic",
+let m = "D:\\exp\\HackerBook\\static-shards_json", j = new e("C:\\tmp\\hackernews", {
+    fileLogLevel: "none",
     maxInMemSnapshotCount: 1,
     snapshotMaxChangeCount: 2e3,
-    unloadMemInterval: 2e4
+    unloadMemInterval: 2e4,
+    noAcl: !0
 }), C = 100;
 
-function l(e, t) {
-    var a, r = {}, o = {};
-    for (a of e) {
+function c(e, t) {
+    var o, a = {}, r = {};
+    for (o of e) {
         var {
             parent_id: n,
             child_id: i
-        } = a;
-        let e = o[i], t = (e = e || (o[i] = {}), o[n]), r = (t = t || (o[n] = {})).children;
-        (r = r || (t.children = {}))[i] = e;
+        } = o;
+        let e = r[i], t = (e = e || (r[i] = {}), r[n]), a = (t = t || (r[n] = {})).children;
+        (a = a || (t.children = {}))[i] = e;
     }
-    var s, l, c, m = {};
+    var s, l, m, c = {};
     let d = 0, f = 0;
     for (s of t) {
         var {
-            id: h,
-            time: p,
-            parent: g
+            id: p,
+            time: h,
+            parent: v
         } = s;
-        if (null == g) {
-            p = (e => [ (e = new Date(1e3 * e)).getUTCFullYear(), e.getUTCMonth() + 1, e.getUTCDate() ])(p);
-            r[h] = p;
-            let e = o[h];
-            e ? e.item = s : e = o[h] = {
+        if (null == v) {
+            h = (e => [ (e = new Date(1e3 * e)).getUTCFullYear(), e.getUTCMonth() + 1, e.getUTCDate() ])(h);
+            a[p] = h;
+            let e = r[p];
+            e ? e.item = s : e = r[p] = {
                 item: s
-            }, m[h] = e, ++d;
+            }, c[p] = e, ++d;
         } else {
-            p = o[g];
-            if (p) {
-                let e = p.children, t = (e = e || (p.children = {}))[h];
-                t ? t.item = s : t = e[h] = {
+            h = r[v];
+            if (h) {
+                let e = h.children, t = (e = e || (h.children = {}))[p];
+                t ? t.item = s : t = e[p] = {
                     item: s
                 }, ++f;
-            } else logger.warn(h + `'s parent ${g} not exist.`);
+            } else u.warn(p + `'s parent ${v} not exist.`);
         }
     }
-    let v = 0;
-    for (l in m) {
-        var u = r[l];
-        w.batch.insert([ "hackernews", ...u, l ], m[l]), ++v >= C && (w.batch.run(), 
-        v = 0);
+    let g = 0;
+    for (l in c) {
+        var w = a[l];
+        j.batch.insert([ "hackernews", ...w, l ], c[l]), ++g >= C && (j.batch.run(), 
+        g = 0);
     }
-    for (c in 0 < v && w.batch.run(), m) !function e(t) {
+    for (m in 0 < g && j.batch.run(), c) !function e(t) {
         delete t.item;
-        let r = t.children;
-        for (var a in r) e(r[a]);
-    }(m[c]);
+        let a = t.children;
+        for (var o in a) e(a[o]);
+    }(c[m]);
 }
 
 (async () => {
-    if (process.argv[2] && (logger.level = process.argv[2]), console.time("run"), 
-    console.time("start"), await w.start(), console.timeEnd("start"), !w.dir("hackernews")) {
-        var e;
-        for (e of (await n(s)).map(e => /([0-9]+).json/.exec(e)[1]).sort((e, t) => e - t)) {
-            var t = `shard_${e}.json`, {
-                edges: r,
-                items: a
-            } = JSON.parse(await i(o(s, t)));
-            logger.warn(t), l(r, a);
+    if (process.argv[2] && (u.level = process.argv[2]), console.time("run"), console.time("start"), 
+    await j.start(), console.timeEnd("start"), !j.dir("hackernews")) {
+        var e, t, a = (await s(m)).map(e => /([0-9]+).json/.exec(e)[1]).sort((e, t) => e - t).map(e => n(m, `shard_${e}.json`));
+        for await ({
+            filePath: e,
+            bin: t
+        } of l(a)) {
+            var {
+                edges: o,
+                items: r
+            } = JSON.parse(t);
+            u.warn(i(e)), c(o, r);
         }
         await 0;
     }
-    console.time("stop"), await w.stop(), console.timeEnd("stop"), console.timeEnd("run");
+    console.time("stop"), await j.stop(), console.timeEnd("stop"), console.timeEnd("run");
 })();

package/src/rpc/getRpcFiles.js

@@ -0,0 +1 @@
+module.exports = () => dataFolder.fileListToTree(dataFolder.rpcFileList);

package/src/simulate.d.ts

@@ -0,0 +1 @@
+export {};

package/src/simulate2.d.ts

@@ -0,0 +1 @@
+export {};

package/src/soc_pokec.d.ts

@@ -0,0 +1 @@
+export {};