datafolder 1.0.0

package/src/QyUnionNode.js

@@ -0,0 +1,105 @@
+class QueryCounter {
+    constructor(e = 1 / 0, i = 0) {
+        Object.assign(this, {
+            limit: e,
+            cursor: i,
+            added: 0,
+            count: 0,
+            encounteredMap: {}
+        });
+    }
+    get isFull() {
+        return this.limit <= this.added;
+    }
+    encounterOne(e) {
+        var {
+            encounteredMap: i,
+            isFull: r
+        } = this;
+        if (!r && !i[e]) return i[e] = 1, this.countOne();
+    }
+    countOne() {
+        if (this.count++ >= this.cursor) return ++this.added, !0;
+    }
+}
+
+class QyQueryNode {
+    constructor() {
+        Object.assign(this, {
+            fileCount: 0,
+            nodeList: []
+        });
+    }
+    encounterOne(e, i, r) {
+        if (r) {
+            if (r.encounterOne(i)) return e[i] = 1, r.isFull;
+        } else e[i] = 1;
+    }
+}
+
+class QyUnionNode extends QyQueryNode {
+    constructor(e) {
+        super(), Object.assign(this, {
+            dirMap: {},
+            checkedDirMap: {}
+        }), e && this.addDir(e);
+    }
+    containsFile(i) {
+        var {
+            fileCount: e,
+            dirMap: r,
+            nodeList: t
+        } = this;
+        if (0 < e) {
+            for (var n in r) if (null != r[n].getFile(i)) return !0;
+            for (let e = 0; e < t.length; ++e) if (t[e].containsFile(i)) return !0;
+        }
+        return !1;
+    }
+    getFileMap(e) {
+        var i, r, t, {
+            dirMap: n,
+            nodeList: o
+        } = this, s = {};
+        for (i in n) {
+            var u, d = n[i].fileMap;
+            for (u in d) if (d[u].created && !s[u] && this.encounterOne(s, u, e)) return s;
+        }
+        for (r of o) for (t in r.getFileMap()) if (!s[t] && this.encounterOne(s, t, e)) return s;
+        return s;
+    }
+    addNode(e) {
+        if (e) {
+            var {
+                dirMap: i,
+                checkedDirMap: r,
+                nodeList: t
+            } = this;
+            if (e instanceof QyUnionNode) {
+                for (var n in e.dirMap) i[n] = e.dirMap[n];
+                for (var o in t.push(...e.nodeList), e.checkedDirMap) r[o] = 1;
+            } else t.push(e);
+            this.fileCount += e.fileCount;
+        }
+    }
+    addDir(e) {
+        var {
+            dirMap: i,
+            checkedDirMap: r
+        } = this, {
+            fileCount: t,
+            subdirCount: n,
+            fullPathHash: o
+        } = e;
+        if (!i[o] && !r[o]) {
+            if (0 < n) for (var s of e.subdirList) this.addDir(s);
+            0 < t ? (i[o] = e, this.fileCount += t) : r[o] = 1;
+        }
+    }
+}
+
+Object.assign(module.exports, {
+    QyUnionNode: QyUnionNode,
+    QyQueryNode: QyQueryNode,
+    QueryCounter: QueryCounter
+});

package/src/QyUtils.js

@@ -0,0 +1,250 @@
+let crypto = require("node:crypto"), path = require("node:path"), process = require("node:process"), fsPromises = require("node:fs/promises"), fs = require("node:fs"), timers = require("node:timers/promises"), SpecialOperators = Object.fromEntries([ "$del", "$link", "$rpl", "$spec", "$raw", "$all", "$clone", "$ext", "$or", "$and", "$gt", "$lt", "$gte", "$lte" ].map(e => [ e, e ])), HiddenFolderName = "%", SpecialFilePaths = {
+    HiddenFolderName: HiddenFolderName,
+    TriggerDirPath: [ HiddenFolderName, "trigger" ],
+    RpcDirPath: [ HiddenFolderName, "rpc" ],
+    IndexDirPath: [ HiddenFolderName, "index" ],
+    IndexPropPathsFileName: "propPaths"
+};
+
+function isMatchName(e) {
+    return e.startsWith("{") && e.endsWith("}");
+}
+
+function getMatchParam(e) {
+    return isMatchName(e) && e.substring(1, e.length - 1);
+}
+
+function getSizeStr(e) {
+    return 1e12 < e ? Math.round(100 * e / 1e12) / 100 + "TB" : 1e9 < e ? Math.round(100 * e / 1e9) / 100 + "GB" : 1e6 < e ? Math.round(100 * e / 1e6) / 100 + "MB" : 1e3 < e ? Math.round(100 * e / 1e3) / 100 + "KB" : e;
+}
+
+let Hash_Length = 10;
+
+function calHash(e, t = Hash_Length) {
+    return crypto.createHash("sha256").update(e, "utf8").digest("base64url").slice(0, t);
+}
+
+let Hash_Key_Length = Hash_Length + 1;
+
+function getSubdirMapKey(e) {
+    return "d" + e;
+}
+
+function getFileMapKey(e) {
+    return "f" + e;
+}
+
+function getContentKey(e) {
+    return "c" + e;
+}
+
+function getRandomStr(e = 8) {
+    return crypto.randomBytes(e).toString("hex");
+}
+
+function arrayLast(e) {
+    return e[e.length - 1];
+}
+
+function isEmptyObj(e) {
+    if (null != e) for (var t in e) if (null != e[t]) return !1;
+    return !0;
+}
+
+function isNotNullObj(e) {
+    return null != e && "object" == typeof e && !(e instanceof Date);
+}
+
+function isString(e) {
+    return "string" == typeof e;
+}
+
+function isNumber(e) {
+    return "number" == typeof e;
+}
+
+function isBoolean(e) {
+    return "boolean" == typeof e;
+}
+
+function isFunction(e) {
+    return "function" == typeof e;
+}
+
+function isSimpleType(e) {
+    var t = typeof e;
+    return "string" == t || "number" == t || "boolean" == t || e instanceof Date;
+}
+
+function trimArrayTail(e) {
+    let t = e.length;
+    for (;null == e[t - 1] && 0 < t; ) --t;
+    return e.length = t, e;
+}
+
+function setPromise(r) {
+    return r.promise || (r.promise = new Promise((e, t) => Object.assign(r, {
+        resolve: e,
+        reject: t
+    }))), r;
+}
+
+function toBuffer(e) {
+    return Buffer.isBuffer(e) ? e : Buffer.from(e);
+}
+
+function getSuffix(e) {
+    return 1 < e ? "s" : "";
+}
+
+function ensureDir(e) {
+    return fsPromises.mkdir(e, {
+        recursive: !0
+    });
+}
+
+function ensureParentDir(e) {
+    return ensureDir(path.dirname(e));
+}
+
+async function getSize(e) {
+    try {
+        return (await fsPromises.stat(e)).size;
+    } catch (e) {
+        return !1;
+    }
+}
+
+async function folderOrFileExists(e) {
+    return !1 !== await getSize(e);
+}
+
+function sleep(e, t) {
+    return timers.setTimeout(1e3 * e, !0, {
+        signal: t
+    });
+}
+
+let exclusiveFilePathMap = void 0;
+
+function unlockPathMap() {
+    for (var e of Object.keys(exclusiveFilePathMap)) unlockExclusiveFilePath(e);
+}
+
+async function lockExclusiveFilePath(t) {
+    try {
+        await ensureParentDir(t);
+        var e = new Date().toISOString() + ": " + process.argv.join(" ");
+        await fsPromises.writeFile(t, e, {
+            flag: "wx",
+            flush: !0
+        }), logger.info("Locked " + t), exclusiveFilePathMap ? exclusiveFilePathMap[t] = 1 : (exclusiveFilePathMap = {
+            [t]: 1
+        }, process.on("exit", unlockPathMap), process.on("SIGINT", unlockPathMap));
+    } catch (e) {
+        throw "EEXIST" == e.code && logger.error(`Already started by ${(await fsPromises.readFile(t)).toString()}:`, e), 
+        e;
+    }
+}
+
+function unlockExclusiveFilePath(e) {
+    try {
+        fs.unlinkSync(e), logger.info("Unlocked " + e);
+    } catch (e) {}
+}
+
+function fromSingleName(e) {
+    return e.startsWith("[") && e.endsWith("]") ? JSON.parse(e) : e;
+}
+
+function toSingleName(e) {
+    return Array.isArray(e) ? 1 == e.length ? e[0] : JSON.stringify(e) : e;
+}
+
+function pathSplit(e) {
+    e = e.split("/");
+    return e[0] || (e[0] = "/"), e;
+}
+
+function propNameSplit(e) {
+    return e.split(".");
+}
+
+function propNameJoin(e) {
+    return e.join(".");
+}
+
+function randomInt(e, t = 0) {
+    return Math.floor(Math.random() * (e - t)) + t;
+}
+
+function listToMap(e) {
+    var t, r = {};
+    for (t of e) r[t] = 1;
+    return r;
+}
+
+async function readBufferAsync(t, r, n, i) {
+    var a = Buffer.allocUnsafe(i);
+    for (let e = 0; e < i; ) {
+        var o = (await r.read(a, e, i - e, n + e)).bytesRead;
+        if (0 == o) return void logger.error(`read ${t} at ${n + e} returning 0 byte`);
+        e += o;
+    }
+    return a;
+}
+
+function memoryPercent() {
+    var e = process.availableMemory();
+    return Math.round(e / (process.memoryUsage.rss() + e) * 100);
+}
+
+function generatorToList(e) {
+    var t, r = [];
+    for (t of e) r.push(t);
+    return r;
+}
+
+Object.assign(module.exports, {
+    SpecialOperators: SpecialOperators,
+    calHash: calHash,
+    Hash_Length: Hash_Length,
+    Hash_Key_Length: Hash_Key_Length,
+    pathSplit: pathSplit,
+    isMatchName: isMatchName,
+    getMatchParam: getMatchParam,
+    isNumber: isNumber,
+    isString: isString,
+    isBoolean: isBoolean,
+    isFunction: isFunction,
+    isNotNullObj: isNotNullObj,
+    isSimpleType: isSimpleType,
+    isEmptyObj: isEmptyObj,
+    getSubdirMapKey: getSubdirMapKey,
+    getFileMapKey: getFileMapKey,
+    getContentKey: getContentKey,
+    arrayLast: arrayLast,
+    getRandomStr: getRandomStr,
+    getSizeStr: getSizeStr,
+    setPromise: setPromise,
+    trimArrayTail: trimArrayTail,
+    toBuffer: toBuffer,
+    getSuffix: getSuffix,
+    sleep: sleep,
+    ensureDir: ensureDir,
+    ensureParentDir: ensureParentDir,
+    getSize: getSize,
+    lockExclusiveFilePath: lockExclusiveFilePath,
+    unlockExclusiveFilePath: unlockExclusiveFilePath,
+    folderOrFileExists: folderOrFileExists,
+    SpecialFilePaths: SpecialFilePaths,
+    fromSingleName: fromSingleName,
+    toSingleName: toSingleName,
+    randomInt: randomInt,
+    propNameSplit: propNameSplit,
+    propNameJoin: propNameJoin,
+    listToMap: listToMap,
+    readBufferAsync: readBufferAsync,
+    memoryPercent: memoryPercent,
+    generatorToList: generatorToList
+});

package/src/QyWorker.js

@@ -0,0 +1,33 @@
+let path = require("node:path"), {
+    Worker,
+    setEnvironmentData
+} = require("node:worker_threads"), logger = require("./QyLogger.js").logger, setPromise = require("./QyUtils.js").setPromise;
+
+function runOnceWithWorker(e, r) {
+    setEnvironmentData("logLevel", logger.level);
+    var o = r.operation;
+    let t = e + " " + o, {
+        promise: i,
+        resolve: n
+    } = (logger.info("Starting worker " + t), setPromise({}));
+    path.isAbsolute(e) || (e = path.join(__dirname, e));
+    o = new Worker(e, {
+        workerData: r
+    });
+    let s;
+    return o.on("message", e => {
+        s = !0, n(e);
+    }), o.on("error", e => {
+        s = !0, n(e);
+    }), o.on("exit", e => {
+        s ? logger.info(`Worker ${t} exited.`) : (logger.log(`Worker ${t} stopped with exitCode ` + e), 
+        n());
+    }), {
+        worker: o,
+        promise: i
+    };
+}
+
+Object.assign(module.exports, {
+    runOnceWithWorker: runOnceWithWorker
+});

package/src/hackernews.js

@@ -0,0 +1,93 @@
+let path = require("node:path"), fsPromises = require("node:fs/promises"), QyDB = require("./QyDB.js").QyDB, folderOrFileExists = require("./QyUtils.js").folderOrFileExists, shardRootFolder = (logger.level = "info", 
+"D:\\exp\\HackerBook\\static-shards_json"), dataFolder = "./tmp/hackernews", qyDB = new QyDB(dataFolder, {
+    fileLogLevel: "basic",
+    maxInMemSnapshotCount: 1,
+    snapshotMaxChangeCount: 2e3,
+    unloadMemInterval: 2e4
+}), batchSize = 100;
+
+function insertShard(e, t) {
+    var a, r = {}, o = {};
+    for (a of e) {
+        var {
+            parent_id: n,
+            child_id: s
+        } = a;
+        let e = o[s], t = (e = e || (o[s] = {}), o[n]), r = (t = t || (o[n] = {})).children;
+        (r = r || (t.children = {}))[s] = e;
+    }
+    var i, l, c, d = {};
+    let h = 0, f = 0;
+    for (i of t) {
+        var {
+            id: m,
+            time: u,
+            parent: g
+        } = i;
+        if (null == g) {
+            u = getDatePath(u);
+            r[m] = u;
+            let e = o[m];
+            e ? e.item = i : e = o[m] = {
+                item: i
+            }, d[m] = e, ++h;
+        } else {
+            u = o[g];
+            if (u) {
+                let e = u.children, t = (e = e || (u.children = {}))[m];
+                t ? t.item = i : t = e[m] = {
+                    item: i
+                }, ++f;
+            } else logger.warn(m + `'s parent ${g} not exist.`);
+        }
+    }
+    let p = 0;
+    for (l in d) {
+        var v = r[l];
+        qyDB.batch.insert([ "hackernews", ...v, l ], d[l]), ++p >= batchSize && (qyDB.batch.run(), 
+        p = 0);
+    }
+    for (c in 0 < p && qyDB.batch.run(), d) cleanContent(d[c]);
+}
+
+function cleanContent(e) {
+    delete e.item;
+    var t, r = e.children;
+    for (t in r) cleanContent(r[t]);
+}
+
+function getDatePath(e) {
+    e = new Date(1e3 * e);
+    return [ e.getUTCFullYear(), e.getUTCMonth() + 1, e.getUTCDate() ];
+}
+
+async function insertShards() {
+    var e;
+    for (e of (await fsPromises.readdir(shardRootFolder)).map(e => /([0-9]+).json/.exec(e)[1]).sort((e, t) => e - t)) {
+        var t = `shard_${e}.json`, {
+            edges: r,
+            items: a
+        } = JSON.parse(await fsPromises.readFile(path.join(shardRootFolder, t)));
+        logger.warn(t), insertShard(r, a);
+    }
+}
+
+async function run() {
+    console.time("accu"), process.argv[3] && (logger.level = process.argv[3]);
+    let e = !0;
+    if ("true" == process.argv[2]) {
+        try {
+            await fsPromises.rm(dataFolder, {
+                recursive: !0
+            });
+        } catch (e) {
+            logger.warn(e);
+        }
+        e = !0;
+    } else await folderOrFileExists(dataFolder) || (e = !0);
+    console.time("run"), console.time("start"), await qyDB.start(), console.timeEnd("start"), 
+    e && await insertShards(), console.time("stop"), await qyDB.stop(), console.timeEnd("stop"), 
+    console.timeEnd("run"), console.timeEnd("accu");
+}
+
+run();

package/src/rpc/evalCode.js

@@ -0,0 +1,9 @@
+module.exports = code => {
+    try {
+        return eval(code);
+    } catch (exception) {
+        return {
+            error: exception.message
+        };
+    }
+};

package/src/rpc/getFileContent.js

@@ -0,0 +1,16 @@
+let Error_No_Such_File = new Error("No such file");
+
+module.exports = ({
+    filePath: e,
+    cChangeId: n
+}) => {
+    var l, r, t, e = qyDB.fileP(e);
+    return e ? ({
+        fileContent: e,
+        cChangeId: l,
+        fileContentKey: r
+    } = e, t = {}, n != l && (Object.assign(t, {
+        fileContent: e,
+        cChangeId: l
+    }), null == n) && (t.fileContentKey = r), t) : Error_No_Such_File;
+};

package/src/rpc/getSubItems.js

@@ -0,0 +1,32 @@
+let Error_no_Such_Dir = new Error("No such dir");
+
+module.exports = ({
+    dirPath: r,
+    dChangeId: e,
+    fChangeId: a
+}) => {
+    r = qyDB.dirP(r);
+    if (!r) return Error_no_Such_Dir;
+    var d = {};
+    if (e || a || (d.fullPathHash = r.fullPathHash), e != r.dChangeId) {
+        var h, e = r.sortedSubdirList, i = [];
+        for ({
+            name: h
+        } of e) i.push(h);
+        Object.assign(d, {
+            subdirList: i,
+            dChangeId: r.dChangeId
+        });
+    }
+    if (a != r.fChangeId) {
+        var n, e = r.sortedFileList, s = [];
+        for ({
+            name: n
+        } of e) s.push(n);
+        Object.assign(d, {
+            fileList: s,
+            fChangeId: r.fChangeId
+        });
+    }
+    return d;
+};

package/src/rpc/queryFiles.js

@@ -0,0 +1,33 @@
+module.exports = queryText => {
+    let {
+        folder,
+        value,
+        view,
+        limit,
+        cursor
+    } = eval("((x)=>x)(" + queryText + ")"), fileList = qyDB.queryFiles(folder, value, limit, cursor), tree = {};
+    for (var file of fileList) {
+        let {
+            parentList
+        } = file, node = tree;
+        for (var dir of parentList) {
+            let {
+                name,
+                fullPathHash
+            } = dir, subdirMap = node.subdirMap;
+            subdirMap = subdirMap || (node.subdirMap = {}), node = subdirMap[name], 
+            node = node || (subdirMap[name] = {
+                fullPathHash: fullPathHash
+            });
+        }
+        let fileMap = node.fileMap;
+        fileMap = fileMap || (node.fileMap = {}), fileMap[file.name] = {
+            fileContent: file.view(view, !0),
+            fileContentKey: file.fileContentKey
+        };
+    }
+    return {
+        tree: tree,
+        count: fileList.length
+    };
+};

package/src/rpc/reloadRpc.js

@@ -0,0 +1 @@
+module.exports = () => {};

package/src/simulate.js

@@ -0,0 +1,114 @@
+let path = require("node:path"), fsPromises = require("node:fs/promises"), QyDB = require("./QyDB.js").QyDB, {
+    folderOrFileExists,
+    randomInt,
+    sleep
+} = require("./QyUtils.js"), runQueries = require("../scripts/airbnb2_queries.js").runQueries, airbnbDataFolder = (logger.level = "info", 
+"../airbnb"), dataFolder = "./tmp/simulate", hosts, listings, reviews, hostIdList, listingIdList, reviewIdList;
+
+async function loadJsonFile(t) {
+    return JSON.parse(await fsPromises.readFile(path.join(airbnbDataFolder, t)));
+}
+
+async function loadData() {
+    console.time("loadData"), [ hosts, listings, reviews ] = await Promise.all([ loadJsonFile("hosts.json"), loadJsonFile("listings.json"), loadJsonFile("reviews.json") ]), 
+    hostIdList = Object.keys(hosts), listingIdList = Object.keys(listings), reviewIdList = Object.keys(reviews), 
+    console.timeEnd("loadData");
+}
+
+function insertData(t) {
+    console.time("insertData");
+    var e, s, i, o = t.batch;
+    for (e of hostIdList) o.insert([ "hosts", e ], hosts[e]);
+    for (s of listingIdList) {
+        var n = listings[s], {
+            property_type: r,
+            room_type: a,
+            bed_type: l
+        } = n;
+        o.insert([ "airbnb", r, a, l, s ], n);
+    }
+    for (i of reviewIdList) {
+        var d = reviews[i];
+        o.insert([ "reviews", d.listing_id, i ], d);
+    }
+    o.run(), console.timeEnd("insertData");
+}
+
+function getRandomHost() {
+    return hosts[hostIdList[randomInt(hostIdList.length)]];
+}
+
+function getRandomListing() {
+    return listings[listingIdList[randomInt(listingIdList.length)]];
+}
+
+function getRandomReview() {
+    return reviews[reviewIdList[randomInt(reviewIdList.length)]];
+}
+
+function modifyHost(t) {
+    var e, s = getRandomHost(), i = getRandomHost(), o = {};
+    for (e in i) 1 == randomInt(2) && (o[e] = i[e]);
+    t.insert([ "hosts", s.host_id ], o);
+}
+
+function modifyListing(t) {
+    var e, s = getRandomListing(), i = getRandomListing(), o = {};
+    for (e in i) 1 == randomInt(2) && (o[e] = i[e]);
+    var {
+        property_type: s,
+        room_type: n,
+        bed_type: r,
+        _id: a
+    } = s;
+    t.insert([ "airbnb", s, n, r, a ], o);
+}
+
+function modifyReview(t) {
+    var e, s = getRandomReview(), i = getRandomReview(), o = {};
+    for (e in i) 1 == randomInt(2) && (o[e] = i[e]);
+    t.insert([ "reviews", s.listing_id, s._id ], o);
+}
+
+async function simulateOne(t, e) {
+    for (;0 < e--; ) {
+        switch (randomInt(3)) {
+          case 0:
+            modifyHost(t);
+            break;
+
+          case 1:
+            modifyListing(t);
+            break;
+
+          case 2:
+            modifyReview(t);
+        }
+        await sleep(randomInt(1e3) / 1e3);
+    }
+}
+
+async function run() {
+    console.time("total");
+    var e = process.argv[2] || 1e3, s = process.argv[3] || 100;
+    let t;
+    if ("true" == process.argv[4]) {
+        try {
+            await fsPromises.rm(dataFolder, {
+                recursive: !0
+            });
+        } catch (t) {}
+        t = !0;
+    } else await folderOrFileExists(dataFolder) || (t = !0);
+    process.argv[5] && (logger.level = process.argv[5]), await loadData(), console.log(`hosts:${hostIdList.length}, listings:${listingIdList.length}, reviews:` + reviewIdList.length);
+    var i = new QyDB(dataFolder, {
+        fileLogLevel: "basic"
+    }), o = (t && (await i.start(), insertData(i), await i.stop()), console.time("simulate " + s), 
+    await i.start(), runQueries(i), []);
+    for (let t = 0; t < e; ++t) o.push(simulateOne(i, s));
+    await Promise.all(o), insertData(i), await i.stop(), console.timeEnd("simulate " + s), 
+    console.time("start"), await i.start(), console.timeEnd("start"), runQueries(i), 
+    await i.stop(), console.timeEnd("total");
+}
+
+run();