datafolder 1.0.0

package/scripts/flights_queries.js

@@ -0,0 +1,403 @@
+const assert = require("node:assert/strict");
+
+// Return the names of all airports one can reach
+// directly (1 step) from Los Angeles International
+// (LAX) following the flights edges:
+
+function query1(qyDB) {
+    console.time("query1");
+    const resultMap = {};
+    for (const x of qyDB.queryFiles("flights", { _from: "airports/LAX" })) {
+        resultMap[x.fileContent._to] = 1;
+    }
+    assert.deepEqual(Object.keys(resultMap), [
+        "airports/CVG",
+        "airports/SMF",
+        "airports/ATL",
+        "airports/MSP",
+        "airports/DFW",
+        "airports/PHL",
+        "airports/IAH",
+        "airports/TUS",
+        "airports/ORD",
+        "airports/PDX",
+        "airports/MIA",
+        "airports/SFO",
+        "airports/DEN",
+        "airports/IAD",
+        "airports/SJC",
+        "airports/PHX",
+        "airports/SEA",
+        "airports/SLC",
+        "airports/JFK",
+        "airports/SAN",
+        "airports/ONT",
+        "airports/LAS",
+        "airports/MCI",
+        "airports/MDW",
+        "airports/FLL",
+        "airports/AUS",
+        "airports/EWR",
+        "airports/CLE",
+        "airports/SBP",
+        "airports/MEM",
+        "airports/SBA",
+        "airports/CLT",
+        "airports/COS",
+        "airports/ASE",
+        "airports/ICT",
+        "airports/HOU",
+        "airports/YUM",
+        "airports/STL",
+        "airports/FAT",
+        "airports/ABQ",
+        "airports/BOI",
+        "airports/BOS",
+        "airports/LIH",
+        "airports/HNL",
+        "airports/SMX",
+        "airports/MRY",
+        "airports/DTW",
+        "airports/RNO",
+        "airports/MCO",
+        "airports/OXR",
+        "airports/OAK",
+        "airports/OGG",
+        "airports/KOA",
+        "airports/BWI",
+        "airports/CLD",
+        "airports/IND",
+        "airports/PSP",
+        "airports/ELP",
+        "airports/TPA",
+        "airports/OKC",
+        "airports/MSY",
+        "airports/SNA",
+        "airports/IYK",
+        "airports/IPL",
+        "airports/SAT",
+        "airports/MOD",
+        "airports/BNA",
+        "airports/DCA",
+        "airports/SGU",
+        "airports/BFL",
+        "airports/EUG",
+        "airports/EGE",
+        "airports/GEG",
+        "airports/BDL",
+        "airports/JAX",
+        "airports/CMH",
+        "airports/PIT",
+        "airports/RDU",
+        "airports/SJU",
+        "airports/XNA",
+        "airports/MTJ",
+        "airports/JAC",
+    ]);
+    console.timeEnd("query1");
+}
+
+// Find all connections which depart from or
+// land at BIS on January 5th and 7th and
+// return the destination city and the arrival
+// time in universal time (UTC):
+// FOR airport, flight IN ANY
+// 'airports/BIS' flights
+// FILTER flight.Month == 1
+// AND flight.Day >= 5
+// AND flight.Day <= 7
+// RETURN { city: airport.city,
+// time: flight.ArrTimeUTC }
+
+function query2(qyDB) {
+    console.time("query2");
+    assert.deepEqual(
+        qyDB
+            .queryFiles("flights", {
+                $and: [
+                    { $or: [{ _from: "airports/BIS" }, { _to: "airports/BIS" }] },
+                    { Month: 1 },
+                    { Day: (x) => x >= 5 && x <= 7 },
+                ],
+            })
+            .map((x) => x.fileContent.FlightNum),
+        [
+            6635, 646, 6541, 5651, 6541, 5648, 687, 309, 6678, 6678, 6636, 439, 1567, 6635, 5647, 6541, 5660, 6541,
+            6553, 687, 6553, 309, 5649, 5650, 6636, 439, 6635, 574, 6541, 5651, 5648, 6541, 1705, 1535, 5649, 6678,
+            5650, 6636, 6678, 1087,
+        ],
+    );
+    console.timeEnd("query2");
+}
+
+function query3(qyDB) {
+    console.time("query3");
+    assert.deepEqual(
+        qyDB
+            .queryFiles("flights", {
+                $or: [
+                    { _from: "airports/BOI", _to: "airports/MSP" },
+                    { _from: "airports/MSP", _to: "airports/BOI" },
+                ],
+                Month: 1,
+                Day: (x) => x >= 1 && x <= 7,
+            })
+            .map((x) => x.view(["_from", "_to", "ArrTimeUTC"])),
+        [
+            {
+                _from: "airports/BOI",
+                _to: "airports/MSP",
+                ArrTimeUTC: "2008-01-01T15:46:00.000Z",
+            },
+            {
+                _from: "airports/BOI",
+                _to: "airports/MSP",
+                ArrTimeUTC: "2008-01-01T17:28:00.000Z",
+            },
+            {
+                _from: "airports/MSP",
+                _to: "airports/BOI",
+                ArrTimeUTC: "2008-01-01T19:59:00.000Z",
+            },
+            {
+                _from: "airports/BOI",
+                _to: "airports/MSP",
+                ArrTimeUTC: "2008-01-01T23:34:00.000Z",
+            },
+            {
+                _from: "airports/MSP",
+                _to: "airports/BOI",
+                ArrTimeUTC: "2008-01-02T02:37:00.000Z",
+            },
+            {
+                _from: "airports/MSP",
+                _to: "airports/BOI",
+                ArrTimeUTC: "2008-01-02T06:55:00.000Z",
+            },
+            {
+                _from: "airports/BOI",
+                _to: "airports/MSP",
+                ArrTimeUTC: "2008-01-02T15:25:00.000Z",
+            },
+            {
+                _from: "airports/BOI",
+                _to: "airports/MSP",
+                ArrTimeUTC: "2008-01-02T18:31:00.000Z",
+            },
+            {
+                _from: "airports/MSP",
+                _to: "airports/BOI",
+                ArrTimeUTC: "2008-01-02T18:25:00.000Z",
+            },
+            {
+                _from: "airports/BOI",
+                _to: "airports/MSP",
+                ArrTimeUTC: "2008-01-02T23:28:00.000Z",
+            },
+            {
+                _from: "airports/MSP",
+                _to: "airports/BOI",
+                ArrTimeUTC: "2008-01-03T02:42:00.000Z",
+            },
+            {
+                _from: "airports/MSP",
+                _to: "airports/BOI",
+                ArrTimeUTC: "2008-01-03T06:58:00.000Z",
+            },
+            {
+                _from: "airports/BOI",
+                _to: "airports/MSP",
+                ArrTimeUTC: "2008-01-03T15:36:00.000Z",
+            },
+            {
+                _from: "airports/BOI",
+                _to: "airports/MSP",
+                ArrTimeUTC: "2008-01-03T17:54:00.000Z",
+            },
+            {
+                _from: "airports/MSP",
+                _to: "airports/BOI",
+                ArrTimeUTC: "2008-01-03T19:46:00.000Z",
+            },
+            {
+                _from: "airports/BOI",
+                _to: "airports/MSP",
+                ArrTimeUTC: "2008-01-03T23:22:00.000Z",
+            },
+            {
+                _from: "airports/MSP",
+                _to: "airports/BOI",
+                ArrTimeUTC: "2008-01-04T02:26:00.000Z",
+            },
+            {
+                _from: "airports/MSP",
+                _to: "airports/BOI",
+                ArrTimeUTC: "2008-01-04T07:20:00.000Z",
+            },
+            {
+                _from: "airports/BOI",
+                _to: "airports/MSP",
+                ArrTimeUTC: "2008-01-04T15:23:00.000Z",
+            },
+            {
+                _from: "airports/BOI",
+                _to: "airports/MSP",
+                ArrTimeUTC: "2008-01-04T17:26:00.000Z",
+            },
+            {
+                _from: "airports/MSP",
+                _to: "airports/BOI",
+                ArrTimeUTC: "2008-01-04T18:37:00.000Z",
+            },
+            {
+                _from: "airports/BOI",
+                _to: "airports/MSP",
+                ArrTimeUTC: "2008-01-04T23:29:00.000Z",
+            },
+            {
+                _from: "airports/MSP",
+                _to: "airports/BOI",
+                ArrTimeUTC: "2008-01-05T03:52:00.000Z",
+            },
+            {
+                _from: "airports/MSP",
+                _to: "airports/BOI",
+                ArrTimeUTC: "2008-01-05T07:01:00.000Z",
+            },
+            {
+                _from: "airports/BOI",
+                _to: "airports/MSP",
+                ArrTimeUTC: "2008-01-05T15:31:00.000Z",
+            },
+            {
+                _from: "airports/BOI",
+                _to: "airports/MSP",
+                ArrTimeUTC: "2008-01-05T17:15:00.000Z",
+            },
+            {
+                _from: "airports/MSP",
+                _to: "airports/BOI",
+                ArrTimeUTC: "2008-01-05T18:36:00.000Z",
+            },
+            {
+                _from: "airports/BOI",
+                _to: "airports/MSP",
+                ArrTimeUTC: "2008-01-05T23:17:00.000Z",
+            },
+            {
+                _from: "airports/MSP",
+                _to: "airports/BOI",
+                ArrTimeUTC: "2008-01-06T02:38:00.000Z",
+            },
+            {
+                _from: "airports/MSP",
+                _to: "airports/BOI",
+                ArrTimeUTC: "2008-01-06T06:56:00.000Z",
+            },
+            {
+                _from: "airports/BOI",
+                _to: "airports/MSP",
+                ArrTimeUTC: "2008-01-06T15:15:00.000Z",
+            },
+            {
+                _from: "airports/BOI",
+                _to: "airports/MSP",
+                ArrTimeUTC: "2008-01-06T18:18:00.000Z",
+            },
+            {
+                _from: "airports/MSP",
+                _to: "airports/BOI",
+                ArrTimeUTC: "2008-01-06T18:48:00.000Z",
+            },
+            {
+                _from: "airports/BOI",
+                _to: "airports/MSP",
+                ArrTimeUTC: "2008-01-06T23:22:00.000Z",
+            },
+            {
+                _from: "airports/MSP",
+                _to: "airports/BOI",
+                ArrTimeUTC: "2008-01-07T06:49:00.000Z",
+            },
+            {
+                _from: "airports/BOI",
+                _to: "airports/MSP",
+                ArrTimeUTC: "2008-01-07T18:15:00.000Z",
+            },
+            {
+                _from: "airports/MSP",
+                _to: "airports/BOI",
+                ArrTimeUTC: "2008-01-07T18:33:00.000Z",
+            },
+            {
+                _from: "airports/BOI",
+                _to: "airports/MSP",
+                ArrTimeUTC: "2008-01-08T00:35:00.000Z",
+            },
+            {
+                _from: "airports/MSP",
+                _to: "airports/BOI",
+                ArrTimeUTC: "2008-01-08T06:29:00.000Z",
+            },
+        ],
+    );
+
+    console.timeEnd("query3");
+}
+
+// Find all flights with FlightNum 860 (number) on
+// January 5th and return the _from and _to
+// attributes only (you may use KEEP() for this).
+// FOR f IN flights
+// FILTER f.FlightNum == 860
+// AND f.Month == 1
+// AND f.Day == 5
+// RETURN KEEP(f, "_from", "_to")
+
+function query4(qyDB) {
+    console.time("query4");
+    const result = qyDB
+        .queryFiles("flights", { Month: 1, Day: 5, FlightNum: 860 })
+        .map((x) => x.view(["_from", "_to"]));
+    assert.deepEqual(result, [
+        { _from: "airports/IAD", _to: "airports/BOS" },
+        { _from: "airports/PBI", _to: "airports/JFK" },
+        { _from: "airports/ORD", _to: "airports/DTW" },
+        { _from: "airports/PHX", _to: "airports/DEN" },
+        { _from: "airports/SEA", _to: "airports/ATL" },
+    ]);
+    console.timeEnd("query4");
+}
+
+function query5(qyDB) {
+    logger.info("Query begins");
+    console.time("query5");
+    assert.deepEqual(
+        qyDB.queryFiles("airports", { vip: true }).map((x) => x.view(["name"]).name),
+        [
+            "Amarillo International",
+            "William B Hartsfield-Atlanta Intl",
+            "Dallas-Fort Worth International",
+            "John F Kennedy Intl",
+            "Los Angeles International",
+            "Chicago O'Hare International",
+            "San Francisco International",
+        ],
+    );
+    console.timeEnd("query5");
+}
+
+function runQueries(qyDB) {
+    query1(qyDB);
+    query1(qyDB);
+    query2(qyDB);
+    query2(qyDB);
+    query3(qyDB);
+    query3(qyDB);
+    query4(qyDB);
+    query4(qyDB);
+    query5(qyDB);
+    query5(qyDB);
+}
+
+Object.assign(module.exports, { runQueries });

package/src/QyAcl.js

@@ -0,0 +1,252 @@
+let fsPromises = require("node:fs/promises"), {
+    mergeContent,
+    mergeContentChanges,
+    setValByNamePath
+} = require("./QyFileContentUtils.js"), {
+    isEmptyObj,
+    arrayLast
+} = require("./QyUtils.js"), logger = require("./QyLogger.js").logger;
+
+function delCmdKey(e, r) {
+    var {
+        A: e,
+        R: t,
+        M: a,
+        D: n
+    } = e;
+    n && delete n[r], r.startsWith("c") ? a && delete a[r] : (e && delete e[r], 
+    t && delete t[r]);
+}
+
+function applyAclChange(e, r) {
+    var t, a, n, s, {
+        D: r,
+        A: i,
+        R: o,
+        M: l
+    } = r;
+    for (t in r) null != e[t] && (e[t] = void 0);
+    for (a in i) {
+        var f, u = e[a] || (e[a] = {});
+        for (f of i[a]) u[f] = 1;
+    }
+    for (n in o) {
+        var d = e[n];
+        if (d) {
+            var c = o[n];
+            if (c) {
+                for (var m of c) delete d[m];
+                isEmptyObj(d) && (e[n] = void 0);
+            }
+        }
+    }
+    for (s in l) e[s] = mergeContent(e[s], l[s]);
+}
+
+function iterCmdKeys(e, r) {
+    var t, {
+        D: e,
+        A: a,
+        R: n,
+        M: s
+    } = e;
+    for (t of [ e, a, n, s ]) for (var i in t) r(i);
+}
+
+class QyAclCmdGenerator {
+    constructor() {
+        Object.assign(this, {
+            cmdArgAsMapObj: {},
+            cmdCount: 0,
+            batchSize: 0
+        });
+    }
+    pushCmd(e, r, t) {
+        switch (++this.cmdCount, e) {
+          case "A":
+            this._setCmdByNamePath([ "A", r, t ]), this._delCmdByNamePath([ "R", r, t ]);
+            break;
+
+          case "R":
+            this._setCmdByNamePath([ "R", r, t ]), this._delCmdByNamePath([ "A", r, t ]);
+            break;
+
+          case "D":
+            if (this._setCmdByNamePath([ "D", r ]), r.startsWith("c")) this._delCmdByNamePath([ "M", r ]); else for (var a of [ "A", "R" ]) this._delCmdByNamePath([ a, r ]);
+            break;
+
+          case "M":
+            {
+                var n = this.cmdArgAsMapObj;
+                let e = n.M;
+                n = (e = null == e ? n.M = {} : e)[r];
+                e[r] = null != n ? mergeContentChanges(n, t) : t;
+                break;
+            }
+
+          default:
+            logger.error(`unsupported cmdName:${e}, called with ${r} ` + t), --this.cmdCount;
+        }
+    }
+    pushCmdArgAsListObj(e) {
+        if (e) {
+            ++this.batchSize;
+            var r, t, a, {
+                D: n,
+                M: s
+            } = e;
+            for (r in n) this.pushCmd("D", r);
+            for (t of [ "A", "R" ]) {
+                var i, o = e[t];
+                for (i in o) for (var l of o[i]) this.pushCmd(t, i, l);
+            }
+            for (a in s) this.pushCmd("M", a, s[a]);
+        }
+        return this;
+    }
+    takeCmdArgAsListObj() {
+        var e, r = this.cmdArgAsMapObj;
+        Object.assign(this, {
+            cmdArgAsMapObj: {},
+            cmdCount: 0,
+            batchSize: 0
+        });
+        for (e of [ "A", "R" ]) {
+            var t, a = r[e];
+            for (t in a) {
+                var n = Object.keys(a[t]);
+                0 < n.length ? a[t] = n : delete a[t];
+            }
+            isEmptyObj(a) && delete r[e];
+        }
+        var {
+            D: s,
+            M: i
+        } = r;
+        return s && isEmptyObj(s) && delete r.D, i && isEmptyObj(i) && delete r.M, 
+        r;
+    }
+    toAclBuffer(e) {
+        var r = this.batchSize;
+        if (0 < r) return _toAclBuffer(e, r, this.takeCmdArgAsListObj());
+    }
+    _setCmdByNamePath(e, r = 1) {
+        setValByNamePath(this.cmdArgAsMapObj, e, r);
+    }
+    _delCmdByNamePath(r) {
+        let t = this.cmdArgAsMapObj;
+        if (t && r && !(r.length <= 1)) {
+            for (let e = 0; e < r.length - 1; ++e) if (null == (t = t[r[e]])) return;
+            delete t[arrayLast(r)];
+        }
+    }
+}
+
+async function readAclFile(r, e = !1, t = 0) {
+    try {
+        return await readAclBuffer(r, await fsPromises.readFile(r), e, t);
+    } catch (e) {
+        throw logger.error(`Reading ${r} failed:`, e), e;
+    }
+}
+
+let ByteCountList = [ 1, 2, 4, 8 ];
+
+function numsToBuffer(r) {
+    var t = r.length, a = new Array(t), n = new Array(t);
+    let s = t << 6, i = 1;
+    for (let e = 0; e < t; ++e) {
+        var o = a[e] = _getByteCountType(r[e]), o = (s |= o << (e << 1), n[e] = ByteCountList[o]);
+        i += o;
+    }
+    var l = Buffer.allocUnsafe(i);
+    l.writeUInt8(s);
+    let f = 1;
+    for (let e = 0; e < t; ++e) _writeNum(l, r[e], n[e], f), f += n[e];
+    return l;
+}
+
+function bufferToNums(r, t, a) {
+    var n = r.readUInt8(t), s = (++t, n >> 6 & 3);
+    for (let e = 0; e < s; ++e) {
+        var i = ByteCountList[n >> (e << 1) & 3];
+        a[e] = _readNum(r, i, t), t += i;
+    }
+    return t;
+}
+
+function _toAclBuffer(e, r, t) {
+    t = Buffer.from(JSON.stringify(t) + "\n");
+    return Buffer.concat([ numsToBuffer([ e, r, t.length ]), t ]);
+}
+
+async function readAclBuffer(t, e, a, n) {
+    var s = [];
+    for (let r = 0; r < e.length; ) {
+        var i = [], o = bufferToNums(e, r, i), [ i, l, f ] = i;
+        if (i <= n) logger.warn(`changeId ${i} <= currentChangeId ${n} at ${t}:${r}. Ignored.`), 
+        r = o + f; else {
+            n + l != i && logger.warn(`currentChangeId ${n} + batchSize ${l} != changeId ` + i);
+            try {
+                var u = JSON.parse(e.subarray(o, o + f));
+                s.push([ i, u ]), n = i, r = o + f;
+            } catch (e) {
+                if (logger.error(`Parse ${t} failed at ${r}:`, e), a) return logger.warn(`Autorepair by truncating ${t} to ` + r), 
+                null == (l = await fsPromises.truncate(t, r)) ? logger.log("truncate success") : logger.error("truncate failed:", l), 
+                s;
+                throw e;
+            }
+        }
+    }
+    return s;
+}
+
+function _writeNum(e, r, t, a = 0) {
+    switch (t) {
+      case 1:
+        e.writeUInt8(r, a);
+        break;
+
+      case 2:
+        e.writeUInt16BE(r, a);
+        break;
+
+      case 4:
+        e.writeUInt32BE(r, a);
+        break;
+
+      default:
+        e.writeBigUInt64BE(BigInt(r), a);
+    }
+}
+
+function _readNum(e, r, t = 0) {
+    switch (r) {
+      case 1:
+        return e.readUInt8(t);
+
+      case 2:
+        return e.readUInt16BE(t);
+
+      case 4:
+        return e.readUInt32BE(t);
+
+      default:
+        return Number(e.readBigUInt64BE(t));
+    }
+}
+
+function _getByteCountType(e) {
+    return e <= 255 ? 0 : e <= 65535 ? 1 : e <= 4294967295 ? 2 : 3;
+}
+
+Object.assign(module.exports, {
+    QyAclCmdGenerator: QyAclCmdGenerator,
+    applyAclChange: applyAclChange,
+    iterCmdKeys: iterCmdKeys,
+    readAclFile: readAclFile,
+    readAclBuffer: readAclBuffer,
+    numsToBuffer: numsToBuffer,
+    bufferToNums: bufferToNums,
+    delCmdKey: delCmdKey
+});

package/src/QyAclSaver.js

@@ -0,0 +1,14 @@
+let QyMessager = require("./QyMessager.js").QyMessager, getDefaultOptions = require("./QyDefaultOptions.js").getDefaultOptions;
+
+class QyAclSaver extends QyMessager {
+    constructor(e, s) {
+        super(e, "QyAclSaver_Worker.js", s = {
+            ...getDefaultOptions("QyAclSaver"),
+            ...s
+        }, s, [ "callSave" ], [ "switch", "castSave" ]);
+    }
+}
+
+Object.assign(module.exports, {
+    QyAclSaver: QyAclSaver
+});

package/src/QyAclSaver_Worker.js

@@ -0,0 +1,29 @@
+let QyMessageWorker = require("./QyMessageWorker.js").QyMessageWorker, QyAclWriter = require("./QyAclWriter.js").QyAclWriter;
+
+class QyAclSaver_Worker extends QyMessageWorker {
+    constructor(r) {
+        super(r), Object.assign(this, {
+            options: r,
+            qyAclWriter: new QyAclWriter({
+                append: !0
+            })
+        });
+    }
+    async start(r) {
+        await this.qyAclWriter.start(r);
+    }
+    async stop() {
+        await this.qyAclWriter.stop();
+    }
+    switch(r) {
+        this.qyAclWriter.switch(r);
+    }
+    async callSave(r, e) {
+        await this.qyAclWriter.save(r, e, !0);
+    }
+    castSave(r, e) {
+        this.qyAclWriter.save(r, e);
+    }
+}
+
+module.exports = QyAclSaver_Worker;