datafolder 1.0.11 → 1.0.12

package/README.md

@@ -4,6 +4,8 @@ datafolder is a lightning fluid JSON database for high speed SSDs, featuring an
 
 ### Changelog
 
+1.0.12 Added expire support to expire a file or some properties of the JSON data in future.
+
 1.0.11 Added [SKILL.md](https://github.com/datafolderdev/datafolder/blob/main/SKILL.md) for feeding to AI
 
 1.0.10 Added document [datafolder for high speed SSDs](https://github.com/datafolderdev/datafolder/wiki/datafolder-for-high-speed-SSDs)
@@ -162,6 +164,16 @@ onPromptScoreChanged({ promptId: "student2", feedbackScore: { oldValue: undefine
 onPromptScoreChanged({ promptId: "student1", feedbackScore: { oldValue: 80, newValue: 32 } });
 ```
 
+### 5. Expire Support
+
+```javascript
+dataFolder.expireFile("aitrainingdata/tmp/prompt", 3600); // the file will be deleted after 1 hour
+dataFolder.expireContent("users/login", { sessionId: 300, token: 3600 });
+
+dataFolder.removeFileExpire("aitrainingdata/tmp/prompt", 3600);
+dataFolder.removeContentExpire("users/login", { sessionId: 1, token: 1 });
+```
+
 ## Example
 
 Once you install datafolder via

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "datafolder",
-	"version": "1.0.11",
+	"version": "1.0.12",
 	"author": {
 		"name": "lysh",
 		"email": "datafolderdev@outlook.com"

package/scripts/flights2.js

@@ -14,6 +14,7 @@ const qyDB = new QyDB(dataFolder, { preloadFoldersAtStart: false });
 run();
 function insertAirports() {
     const data = JSON.parse(readFileSync(join(arangoJsonFolderPath, "airports.json")));
+    console.log(`airports:${data.length}`);
     console.time("insertAirports");
     const { batch } = qyDB;
     for (const record of data) {
@@ -25,6 +26,7 @@ function insertAirports() {
 }
 function insertFlights() {
     const data = JSON.parse(readFileSync(join(arangoJsonFolderPath, "flights.json")));
+    console.log(`flights:${data.length}`);
     console.time("insertFlights");
     const { batch } = qyDB;
     for (let i = 0; i < data.length; ++i) {

package/src/QyAcl.d.ts

@@ -1,14 +1,20 @@
-export declare function delCmdKey(cmdArgAsListObj: any, key: string): void;
-export declare function applyAclChange(aclKeyValueMap: any, cmdArgAsListObj: any): void;
-export declare function iterCmdKeys(cmdArgAsListObj: any, fun: any): void;
+export declare const CmdNames: {
+    M: number;
+    A: number;
+    R: number;
+    D: number;
+};
+export declare function delCmdKey(cmdArgAsListChanges: any, key: string): void;
+export declare function applyAclChange(aclKeyValueMap: any, cmdArgAsListChanges: any): void;
+export declare function iterCmdKeys(cmdArgAsListChanges: any, fun: any): void;
 export declare class QyAclCmdGenerator {
-    cmdArgAsMapObj: {};
+    cmdArgAsMapChanges: any[];
     cmdCount: number;
     batchSize: number;
     constructor();
-    pushCmd(cmdName: string, cmdHashKey: string, cmdArg?: any): void;
-    pushCmdArgAsListObj(cmdArgAsListObj: any): this;
-    takeCmdArgAsListObj(): {};
+    pushCmd(cmdName: number, cmdHashKey: string, cmdArg?: any): void;
+    pushCmdArgAsListObj(cmdArgAsListChanges: any): this;
+    takeCmdArgAsListObj(): any[];
     toAclBuffer(changeId: any): Buffer<ArrayBuffer> | undefined;
 }
 export declare function readAclFile(aclFilePath: any, autoRepairAclFile?: boolean, currentChangeId?: number): Promise<any[]>;

package/src/QyAcl.js

@@ -1,250 +1,234 @@
-import u from "node:fs/promises";
+import l from "node:fs/promises";
 
 import {
     mergeContent as g,
-    mergeContentChanges as i,
+    mergeContentChanges as s,
     setValByNamePath as a
 } from "./QyFileContentUtils.js";
 
 import {
-    isEmptyObj as m,
-    arrayLast as n
+    isEmptyObj as m
 } from "./QyUtils.js";
 
 import {
-    logger as d
+    logger as u
 } from "./QyLogger.js";
 
-function r(r, t) {
-    var {
-        A: r,
-        R: e,
-        M: a,
-        D: n
-    } = r;
-    n && delete n[t], t.startsWith("c") ? a && delete a[t] : (r && delete r[t], 
-    e && delete e[t]);
+let f = {
+    M: 0,
+    A: 1,
+    R: 2,
+    D: 3
+};
+
+function r(r, e) {
+    var [ r, t, a, n ] = r;
+    n && delete n[e], e.startsWith("c") ? r && delete r[e] : (t && delete t[e], 
+    a && delete a[e]);
 }
 
-function t(r, t) {
-    var e, a, n, i, {
-        D: t,
-        A: s,
-        R: o,
-        M: f
-    } = t;
-    for (e in t) null != r[e] && (r[e] = void 0);
-    for (a in s) {
-        var c, l = r[a] || (r[a] = {});
-        for (c of s[a]) l[c] = 1;
+function e(r, e) {
+    var t, a, n, s, [ i, o, f, e ] = e;
+    for (t in e) null != r[t] && (r[t] = void 0);
+    for (a in o) {
+        var c, h = r[a] || (r[a] = {});
+        for (c of o[a]) h[c] = 1;
     }
-    for (n in o) {
-        var u = r[n];
-        if (u) {
-            var d = o[n];
-            if (d) {
-                for (var h of d) delete u[h];
-                m(u) && (r[n] = void 0);
+    for (n in f) {
+        var l = r[n];
+        if (l) {
+            var u = f[n];
+            if (u) {
+                for (var d of u) delete l[d];
+                m(l) && (r[n] = void 0);
             }
         }
     }
-    for (i in f) r[i] = g(r[i], f[i]);
+    for (s in i) r[s] = g(r[s], i[s]);
 }
 
-function e(r, t) {
-    var e, {
-        D: r,
-        A: a,
-        R: n,
-        M: i
-    } = r;
-    for (e of [ r, a, n, i ]) for (var s in e) t(s);
+function t(r, e) {
+    for (var t of r) for (var a in t) e(a);
 }
 
-class s {
-    cmdArgAsMapObj = {};
+class n {
+    cmdArgAsMapChanges = [];
     cmdCount = 0;
     batchSize = 0;
     constructor() {}
-    pushCmd(r, t, e = void 0) {
+    pushCmd(r, e, t = void 0) {
         switch (++this.cmdCount, r) {
-          case "A":
-            o(this, [ "A", t, e ]), f(this, [ "R", t, e ]);
+          case f.A:
+            i(this, [ f.A, e, t ]), o(this, [ f.R, e, t ]);
             break;
 
-          case "R":
-            o(this, [ "R", t, e ]), f(this, [ "A", t, e ]);
+          case f.R:
+            i(this, [ f.R, e, t ]), o(this, [ f.A, e, t ]);
             break;
 
-          case "D":
-            if (o(this, [ "D", t ]), t.startsWith("c")) f(this, [ "M", t ]); else for (var a of [ "A", "R" ]) f(this, [ a, t ]);
+          case f.D:
+            if (i(this, [ f.D, e ]), e.startsWith("c")) o(this, [ f.M, e ]); else for (var a of [ f.A, f.R ]) o(this, [ a, e ]);
             break;
 
-          case "M":
+          case f.M:
             {
-                var n = this.cmdArgAsMapObj;
-                let r = n.M;
-                n = (r = null == r ? n.M = {} : r)[t];
-                r[t] = null != n ? i(n, e) : e;
+                var n = this.cmdArgAsMapChanges;
+                let [ r ] = n;
+                n = (r = null == r ? n[f.M] = {} : r)[e];
+                r[e] = null != n ? s(n, t) : t;
                 break;
             }
 
           default:
-            d.error(`unsupported cmdName:${r}, called with ${t} ` + e), --this.cmdCount;
+            u.error(`unsupported cmdName:${r}, called with ${e} ` + t), --this.cmdCount;
         }
     }
     pushCmdArgAsListObj(r) {
         if (r) {
-            ++this.batchSize;
-            var t, e, a, {
-                D: n,
-                M: i
-            } = r;
-            for (t in n) this.pushCmd("D", t);
-            for (e of [ "A", "R" ]) {
-                var s, o = r[e];
-                for (s in o) for (var f of o[s]) this.pushCmd(e, s, f);
+            for (var e in ++this.batchSize, r[f.D]) this.pushCmd(f.D, e);
+            for (var t of [ f.A, f.R ]) {
+                var a, n = r[t];
+                for (a in n) for (var s of n[a]) this.pushCmd(t, a, s);
             }
-            for (a in i) this.pushCmd("M", a, i[a]);
+            var i, [ o ] = r;
+            for (i in o) this.pushCmd(f.M, i, o[i]);
         }
         return this;
     }
     takeCmdArgAsListObj() {
-        var r, t = this.cmdArgAsMapObj;
+        var r, e = this.cmdArgAsMapChanges;
         Object.assign(this, {
-            cmdArgAsMapObj: {},
+            cmdArgAsMapChanges: [],
             cmdCount: 0,
             batchSize: 0
         });
-        for (r of [ "A", "R" ]) {
-            var e, a = t[r];
-            for (e in a) {
-                var n = Object.keys(a[e]);
-                0 < n.length ? a[e] = n : delete a[e];
+        for (r of [ f.A, f.R ]) {
+            var t, a = e[r];
+            for (t in a) {
+                var n = Object.keys(a[t]);
+                0 < n.length ? a[t] = n : delete a[t];
             }
-            m(a) && delete t[r];
+            m(a) && delete e[r];
         }
-        var {
-            D: i,
-            M: s
-        } = t;
-        return i && m(i) && delete t.D, s && m(s) && delete t.M, t;
+        var s = e[f.D], s = (s && m(s) && delete e[f.D], e[f.M]);
+        return s && m(s) && delete e[f.M], e;
     }
     toAclBuffer(r) {
-        var t, e = this.batchSize;
-        if (0 < e) return r = r, e = e, t = this.takeCmdArgAsListObj(), t = Buffer.from(JSON.stringify(t)), 
-        Buffer.concat([ l([ r, e, t.length ]), t ]);
+        var e, t = this.batchSize;
+        if (0 < t) return r = r, t = t, e = this.takeCmdArgAsListObj(), e = Buffer.from(JSON.stringify(e)), 
+        Buffer.concat([ h([ r, t, e.length ]), e ]);
     }
 }
 
-function o(r, t, e = 1) {
-    a(r.cmdArgAsMapObj, t, e);
+function i(r, e, t = 1) {
+    a(r.cmdArgAsMapChanges, e, t);
 }
 
-function f(r, t) {
-    let e = r.cmdArgAsMapObj;
-    if (e && t && !(t.length <= 1)) {
-        for (let r = 0; r < t.length - 1; ++r) if (null == (e = e[t[r]])) return;
-        delete e[n(t)];
+function o(r, e) {
+    let t = r.cmdArgAsMapChanges;
+    if (t && e && !(e.length <= 1)) {
+        for (let r = 0; r < e.length - 1; ++r) if (null == (t = t[e[r]])) return;
+        delete t[e.at(-1)];
     }
 }
 
-async function c(t, r = !1, e = 0) {
+async function c(e, r = !1, t = 0) {
     try {
-        return await A(t, await u.readFile(t), r, e);
+        return await A(e, await l.readFile(e), r, t);
     } catch (r) {
-        throw d.error(`Reading ${t} failed:`, r), r;
+        throw u.error(`Reading ${e} failed:`, r), r;
     }
 }
 
 let v = [ 1, 2, 4, 8 ];
 
-function l(t) {
-    var e = t.length, a = new Array(e), n = new Array(e);
-    let i = e << 6, s = 1;
-    for (let r = 0; r < e; ++r) {
-        var o = a[r] = (o = t[r]) <= 255 ? 0 : o <= 65535 ? 1 : o <= 4294967295 ? 2 : 3, o = (i |= o << (r << 1), 
+function h(e) {
+    var t = e.length, a = new Array(t), n = new Array(t);
+    let s = t << 6, i = 1;
+    for (let r = 0; r < t; ++r) {
+        var o = a[r] = (o = e[r]) <= 255 ? 0 : o <= 65535 ? 1 : o <= 4294967295 ? 2 : 3, o = (s |= o << (r << 1), 
         n[r] = v[o]);
-        s += o;
+        i += o;
     }
-    var f = Buffer.allocUnsafe(s);
-    f.writeUInt8(i);
+    var f = Buffer.allocUnsafe(i);
+    f.writeUInt8(s);
     let c = 1;
-    for (let r = 0; r < e; ++r) {
-        h = d = u = l = void 0;
-        var [ l, u, d, h = 0 ] = [ f, t[r], n[r], c ];
-        switch (d) {
+    for (let r = 0; r < t; ++r) {
+        d = u = l = h = void 0;
+        var [ h, l, u, d = 0 ] = [ f, e[r], n[r], c ];
+        switch (u) {
           case 1:
-            l.writeUInt8(u, h);
+            h.writeUInt8(l, d);
             break;
 
           case 2:
-            l.writeUInt16BE(u, h);
+            h.writeUInt16BE(l, d);
             break;
 
           case 4:
-            l.writeUInt32BE(u, h);
+            h.writeUInt32BE(l, d);
             break;
 
           default:
-            l.writeBigUInt64BE(BigInt(u), h);
+            h.writeBigUInt64BE(BigInt(l), d);
         }
         c += n[r];
     }
     return f;
 }
 
-function h(t, e, a) {
-    var n = t.readUInt8(e), i = (++e, n >> 6 & 3);
-    for (let r = 0; r < i; ++r) {
-        var s = v[n >> (r << 1) & 3];
-        a[r] = ((r, t, e = 0) => {
-            switch (t) {
+function d(e, t, a) {
+    var n = e.readUInt8(t), s = (++t, n >> 6 & 3);
+    for (let r = 0; r < s; ++r) {
+        var i = v[n >> (r << 1) & 3];
+        a[r] = ((r, e, t = 0) => {
+            switch (e) {
               case 1:
-                return r.readUInt8(e);
+                return r.readUInt8(t);
 
               case 2:
-                return r.readUInt16BE(e);
+                return r.readUInt16BE(t);
 
               case 4:
-                return r.readUInt32BE(e);
+                return r.readUInt32BE(t);
 
               default:
-                return Number(r.readBigUInt64BE(e));
+                return Number(r.readBigUInt64BE(t));
             }
-        })(t, s, e), e += s;
+        })(e, i, t), t += i;
     }
-    return e;
+    return t;
 }
 
-async function A(e, r, a, n) {
-    var i = [];
-    for (let t = 0; t < r.length; ) {
-        var s = [], o = h(r, t, s), [ s, f, c ] = s;
-        if (s <= n) d.warn(`changeId ${s} <= currentChangeId ${n} at ${e}:${t}. Ignored.`), 
-        t = o + c; else {
-            n + f != s && d.warn(`currentChangeId ${n} + batchSize ${f} != changeId ` + s);
+async function A(t, r, a, n) {
+    var s = [];
+    for (let e = 0; e < r.length; ) {
+        var i = [], o = d(r, e, i), [ i, f, c ] = i;
+        if (i <= n) u.warn(`changeId ${i} <= currentChangeId ${n} at ${t}:${e}. Ignored.`), 
+        e = o + c; else {
+            n + f != i && u.warn(`currentChangeId ${n} + batchSize ${f} != changeId ` + i);
             try {
-                var l = JSON.parse(r.subarray(o, o + c));
-                i.push([ s, l ]), n = s, t = o + c;
+                var h = JSON.parse(r.subarray(o, o + c));
+                s.push([ i, h ]), n = i, e = o + c;
             } catch (r) {
-                if (d.error(`Parse ${e} failed at ${t}:`, r), a) return d.warn(`Autorepair by truncating ${e} to ` + t), 
-                null == (f = await u.truncate(e, t)) ? d.log("truncate success") : d.error("truncate failed:", f), 
-                i;
+                if (u.error(`Parse ${t} failed at ${e}:`, r), a) return u.warn(`Autorepair by truncating ${t} to ` + e), 
+                null == (f = await l.truncate(t, e)) ? u.log("truncate success") : u.error("truncate failed:", f), 
+                s;
                 throw r;
             }
         }
     }
-    return i;
+    return s;
 }
 
 export {
+    f as CmdNames,
     r as delCmdKey,
-    t as applyAclChange,
-    e as iterCmdKeys,
-    s as QyAclCmdGenerator,
+    e as applyAclChange,
+    t as iterCmdKeys,
+    n as QyAclCmdGenerator,
     c as readAclFile,
-    l as numsToBuffer,
-    h as bufferToNums,
+    h as numsToBuffer,
+    d as bufferToNums,
     A as readAclBuffer
 };

package/src/QyAclSaver.js

@@ -8,10 +8,10 @@ import {
 
 class e extends s {
     constructor(s, e) {
-        super(s, "QyAclSaver_Worker", e = {
+        super("QyAclSaver_Worker", e = {
             ...r("QyAclSaver"),
             ...e
-        }, e, [ "callSave" ], [ "switch", "castSave" ]);
+        }, e, [ "callSave" ], [ "switch", "castSave" ], s);
     }
 }
 

package/src/QyAclSaver_Worker.d.ts

@@ -5,6 +5,6 @@ export default class QyAclSaver_Worker extends QyMessageWorker {
     _op_start(aclFilePath: any): Promise<void>;
     _op_stop(): Promise<void>;
     switch(aclFilePath: any): void;
-    callSave(changeId: any, cmdArgAsListObj: any): Promise<void>;
-    castSave(changeId: any, cmdArgAsListObj: any): void;
+    callSave(changeId: any, cmdArgAsListChanges: any): Promise<void>;
+    castSave(changeId: any, cmdArgAsListChanges: any): void;
 }

package/src/QyBatch.d.ts

@@ -7,27 +7,22 @@ export declare class QyBatch extends QyCD {
     syncList: any;
     qyAclCmdGenerator: any;
     qyCmdObjHistory: any;
+    running: any;
     constructor(qyCache?: any, options?: any);
     createDir(namePath: any, parentDir?: any): any;
-    createDirP(namePath: any, parentDir?: any): any;
     delDir(namePath: any, parentDir?: any): any;
-    delDirP(namePath: any, parentDir?: any): any;
     createFile(namePath: any, parentDir?: any): any;
-    createFileP(namePath: any, parentDir?: any): any;
     delFile(namePath: any, parentDir?: any): any;
-    delFileP(namePath: any, parentDir?: any): any;
+    expireFile(namePath: any, expireSeconds: number, parentDir?: any): any;
+    expireContent(namePath: any, expireInfoObj: any, parentDir?: any): any;
+    removeFileExpire(namePath: any, parentDir?: any): any;
+    removeContentExpire(namePath: any, expireInfoObj: any, infoFileOnly?: any, parentDir?: any): any;
     createIndex(indexHostDirNamePath: any, indexPropPathList: any, parentDir?: any): any;
-    createIndexP(indexHostDirNamePath: any, indexPropPathList: any, parentDir?: any): any;
     delIndex(indexHostDirNamePath: any, indexPropPath: any, parentDir?: any): any;
-    delIndexP(indexHostDirNamePath: any, indexPropPath: any, parentDir?: any): any;
-    insert(namePath: any, content: any, parentDir?: any): any;
-    insertP(namePath: any, content: any, parentDir?: any): any;
+    insert(namePath: any, content: any, noClone?: boolean, parentDir?: any): any;
     remove(namePath: any, content: any, parentDir?: any): any;
-    removeP(namePath: any, content: any, parentDir?: any): any;
     replace(namePath: any, content: any, parentDir?: any): any;
-    replaceP(namePath: any, content: any, parentDir?: any): any;
     delContent(namePath: any, parentDir?: any): any;
-    delContentP(namePath: any, parentDir?: any): any;
     insertTrigger(triggerName: any, fileNamePath: any, propNamePathList: any, modCode: any): any;
     removeTrigger(triggerName: any): any;
     insertRpc(rpcFunName: any, modCode: any): any;
@@ -46,4 +41,6 @@ export declare class QyBatch extends QyCD {
     _action_insert(action: any): boolean;
     _action_createFile(action: any): boolean;
     _action_createDir(action: any): boolean;
+    _action_expireContent(action: any): boolean;
+    _action_removeContentExpire(action: any): boolean | undefined;
 }