datafolder 1.0.19 → 1.0.20

package/README.md

@@ -4,6 +4,8 @@ datafolder is a lightning fluid JSON database for high speed SSDs, featuring an
 
 ### Changelog
 
+1.0.20 Add HNSW support.
+
 1.0.19 Add DocumentDB test case (test/testQyDocumentDB.js)
 
 1.0.18 Add Embedding Storage Support. Related APIs: dataFolder.insertEmbedding() and dataFolder.delEmbedding(). TODO: Add HNSW support and vector query.

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "datafolder",
-	"version": "1.0.19",
+	"version": "1.0.20",
 	"author": {
 		"name": "lysh",
 		"email": "datafolderdev@outlook.com"

package/scripts/flights2.js

@@ -10,7 +10,7 @@ import { runQueries } from "./flights2_queries.js";
 logger.level = "info";
 const arangoJsonFolderPath = "D:\\scripts\\GraphCourse_DemoData_ArangoDB-2";
 const dataFolder = "./tmp/flights2";
-const qyDB = new QyDB(dataFolder, { preloadFoldersAtStart: false });
+const qyDB = new QyDB(dataFolder, { fileLogLevel: "basic" });
 run();
 function insertAirports() {
     const data = JSON.parse(readFileSync(join(arangoJsonFolderPath, "airports.json")));
@@ -19,7 +19,7 @@ function insertAirports() {
     const { batch } = qyDB;
     for (const record of data) {
         const { country, state, city } = record;
-        batch.insert(`airports/${country}/${state}/${city}/${record._key}`, record, true);
+        batch.put(`airports/${country}/${state}/${city}/${record._key}`, record);
     }
     batch.run();
     console.timeEnd("insertAirports");
@@ -32,7 +32,7 @@ function insertFlights() {
     for (let i = 0; i < data.length; ++i) {
         const record = data[i];
         const { Year, Month, Day } = record;
-        batch.insert(`flights/${Year}/${Month}/${Day}/${i}`, record, true);
+        batch.put(`flights/${Year}/${Month}/${Day}/${i}`, record);
         // batch.insertP(["flights", Year, Month, Day, i], record);
     }
     batch.run();

package/src/QyAcl.js

@@ -1,4 +1,4 @@
-import l from "node:fs/promises";
+import c from "node:fs/promises";
 
 import {
     cleanContent as s,
@@ -19,9 +19,8 @@ import {
 
 let {
     $del: f,
-    $rpl: u,
-    $u8: c
-} = r, d = {
+    $rpl: l
+} = r, u = {
     M: 0,
     A: 1,
     R: 2,
@@ -32,16 +31,16 @@ function t(r, t) {
     var e, a, n, i, [ s, o, f, t ] = t;
     for (e in t) null != r[e] && (r[e] = void 0);
     for (a in o) {
-        var u, c = r[a] || (r[a] = {});
-        for (u of o[a]) c[u] = 1;
+        var l, u = r[a] || (r[a] = {});
+        for (l of o[a]) u[l] = 1;
     }
     for (n in f) {
-        var l = r[n];
-        if (l) {
+        var c = r[n];
+        if (c) {
             var h = f[n];
             if (h) {
-                for (var d of h) delete l[d];
-                m(l) && (r[n] = void 0);
+                for (var d of h) delete c[d];
+                m(c) && (r[n] = void 0);
             }
         }
     }
@@ -59,35 +58,35 @@ class i {
     constructor() {}
     pushCmd(r, t, e = void 0) {
         switch (++this.cmdCount, r) {
-          case d.A:
-            v(this, [ d.A, t, e ]), A(this, [ d.R, t, e ]);
+          case u.A:
+            d(this, [ u.A, t, e ]), v(this, [ u.R, t, e ]);
             break;
 
-          case d.R:
-            v(this, [ d.R, t, e ]), A(this, [ d.A, t, e ]);
+          case u.R:
+            d(this, [ u.R, t, e ]), v(this, [ u.A, t, e ]);
             break;
 
-          case d.D:
-            if (v(this, [ d.D, t ]), t.startsWith("c")) A(this, [ d.M, t ]); else for (var a of [ d.A, d.R ]) A(this, [ a, t ]);
+          case u.D:
+            if (d(this, [ u.D, t ]), t.startsWith("c")) v(this, [ u.M, t ]); else for (var a of [ u.A, u.R ]) v(this, [ a, t ]);
             break;
 
-          case d.M:
+          case u.M:
             {
                 var n = this.cmdArgAsMapChanges;
                 let [ r ] = n;
-                n = (r = null == r ? n[d.M] = {} : r)[t];
+                n = (r = null == r ? n[u.M] = {} : r)[t];
                 r[t] = null != n ? function e(a, n) {
                     if (n == f) return f;
-                    let r = n[u];
+                    let r = n.$rpl;
                     if (null != r) return n;
-                    let t = a[u];
-                    if (null != t) return a[u] = g(t, n), a;
+                    let t = a.$rpl;
+                    if (null != t) return a[l] = g(t, n), a;
                     if (a == f) return {
-                        [u]: s(n)
+                        [l]: s(n)
                     };
                     if (o(a) || o(n)) return n;
-                    if (c in n) return n;
-                    for (var i in n) {
+                    if (n.$u8) return n;
+                    for (var i of Object.keys(n)) {
                         let r = n[i], t = a[i];
                         null == t ? a[i] = r : a[i] = e(t, r);
                     }
@@ -102,13 +101,13 @@ class i {
     }
     pushCmdArgAsListObj(r) {
         if (r) {
-            for (var t in ++this.batchSize, r[d.D]) this.pushCmd(d.D, t);
-            for (var e of [ d.A, d.R ]) {
+            for (var t in ++this.batchSize, r[u.D]) this.pushCmd(u.D, t);
+            for (var e of [ u.A, u.R ]) {
                 var a, n = r[e];
                 for (a in n) for (var i of n[a]) this.pushCmd(e, a, i);
             }
             var s, [ o ] = r;
-            for (s in o) this.pushCmd(d.M, s, o[s]);
+            for (s in o) this.pushCmd(u.M, s, o[s]);
         }
         return this;
     }
@@ -119,7 +118,7 @@ class i {
             cmdCount: 0,
             batchSize: 0
         });
-        for (r of [ d.A, d.R ]) {
+        for (r of [ u.A, u.R ]) {
             var e, a = t[r];
             for (e in a) {
                 var n = Object.keys(a[e]);
@@ -127,20 +126,20 @@ class i {
             }
             m(a) && delete t[r];
         }
-        var i = t[d.D], i = (i && m(i) && delete t[d.D], t[d.M]);
-        return i && m(i) && delete t[d.M], t;
+        var i = t[u.D], i = (i && m(i) && delete t[u.D], t[u.M]);
+        return i && m(i) && delete t[u.M], t;
     }
     toAclBuffer(r) {
         var t = this.batchSize;
-        if (0 < t) return y(r, t, JSON.stringify(this.takeCmdArgAsListObj()));
+        if (0 < t) return w(r, t, JSON.stringify(this.takeCmdArgAsListObj()));
     }
 }
 
-function v(r, t, e = 1) {
+function d(r, t, e = 1) {
     a(r.cmdArgAsMapChanges, t, e);
 }
 
-function A(r, t) {
+function v(r, t) {
     let e = r.cmdArgAsMapChanges;
     if (e && t && !(t.length <= 1)) {
         for (let r = 0; r < t.length - 1; ++r) if (null == (e = e[t[r]])) return;
@@ -150,53 +149,53 @@ function A(r, t) {
 
 async function p(t, r = !1, e = 0) {
     try {
-        return await I(t, await l.readFile(t), r, e);
+        return await y(t, await c.readFile(t), r, e);
     } catch (r) {
         throw h.error(`Reading ${t} failed:`, r), r;
     }
 }
 
-let C = [ 1, 2, 4, 8 ];
+let A = [ 1, 2, 4, 8 ];
 
 function b(t) {
     var e = t.length, a = new Array(e), n = new Array(e);
     let i = e << 6, s = 1;
     for (let r = 0; r < e; ++r) {
         var o = a[r] = (o = t[r]) <= 255 ? 0 : o <= 65535 ? 1 : o <= 4294967295 ? 2 : 3, o = (i |= o << (r << 1), 
-        n[r] = C[o]);
+        n[r] = A[o]);
         s += o;
     }
     var f = Buffer.allocUnsafe(s);
     f.writeUInt8(i);
-    let u = 1;
+    let l = 1;
     for (let r = 0; r < e; ++r) {
-        d = h = l = c = void 0;
-        var [ c, l, h, d = 0 ] = [ f, t[r], n[r], u ];
+        d = h = c = u = void 0;
+        var [ u, c, h, d = 0 ] = [ f, t[r], n[r], l ];
         switch (h) {
           case 1:
-            c.writeUInt8(l, d);
+            u.writeUInt8(c, d);
             break;
 
           case 2:
-            c.writeUInt16BE(l, d);
+            u.writeUInt16BE(c, d);
             break;
 
           case 4:
-            c.writeUInt32BE(l, d);
+            u.writeUInt32BE(c, d);
             break;
 
           default:
-            c.writeBigUInt64BE(BigInt(l), d);
+            u.writeBigUInt64BE(BigInt(c), d);
         }
-        u += n[r];
+        l += n[r];
     }
     return f;
 }
 
-function w(t, e, a) {
+function C(t, e, a) {
     var n = t.readUInt8(e), i = (++e, n >> 6 & 3);
     for (let r = 0; r < i; ++r) {
-        var s = C[n >> (r << 1) & 3];
+        var s = A[n >> (r << 1) & 3];
         a[r] = ((r, t, e = 0) => {
             switch (t) {
               case 1:
@@ -216,24 +215,24 @@ function w(t, e, a) {
     return e;
 }
 
-function y(r, t, e) {
+function w(r, t, e) {
     e = n(e);
     return Buffer.concat([ b([ r, t, e.length ]), e ]);
 }
 
-async function I(e, r, a, n) {
+async function y(e, r, a, n) {
     var i = [];
     for (let t = 0; t < r.length; ) {
-        var s = [], o = w(r, t, s), [ s, f, u ] = s;
+        var s = [], o = C(r, t, s), [ s, f, l ] = s;
         if (s <= n) h.warn(`changeId ${s} <= currentChangeId ${n} at ${e}:${t}. Ignored.`), 
-        t = o + u; else {
+        t = o + l; else {
             n + f != s && h.warn(`currentChangeId ${n} + batchSize ${f} != changeId ` + s);
             try {
-                var c = JSON.parse(r.subarray(o, o + u));
-                i.push([ s, c ]), n = s, t = o + u;
+                var u = JSON.parse(r.subarray(o, o + l));
+                i.push([ s, u ]), n = s, t = o + l;
             } catch (r) {
                 if (h.error(`Parse ${e} failed at ${t}:`, r), a) return h.warn(`Autorepair by truncating ${e} to ` + t), 
-                null == (f = await l.truncate(e, t)) ? h.log("truncate success") : h.error("truncate failed:", f), 
+                null == (f = await c.truncate(e, t)) ? h.log("truncate success") : h.error("truncate failed:", f), 
                 i;
                 throw r;
             }
@@ -243,13 +242,13 @@ async function I(e, r, a, n) {
 }
 
 export {
-    d as CmdNames,
+    u as CmdNames,
     t as applyAclChange,
     e as iterCmdKeys,
     i as QyAclCmdGenerator,
     p as readAclFile,
     b as numsToBuffer,
-    w as bufferToNums,
-    y as toAclBuffer,
-    I as readAclBuffer
+    C as bufferToNums,
+    w as toAclBuffer,
+    y as readAclBuffer
 };

package/src/QyBatch.d.ts

@@ -29,7 +29,7 @@ export declare class QyBatch extends QyCD {
     delEmbedding(namePath: any, parentDir?: any): any;
     insertWithEmbedding(namePath: any, content: any, embedding: any, noClone?: boolean, noCreate?: boolean, parentDir?: any): any;
     remove(namePath: any, content: any, parentDir?: any): any;
-    put(namePath: any, content: any, noClone?: boolean, parentDir?: any): any;
+    put(namePath: any, content: any, parentDir?: any): any;
     delContent(namePath: any, parentDir?: any): any;
     insertTrigger(triggerName: any, fileNamePath: any, propNamePathList: any, modCode: any): any;
     removeTrigger(triggerName: any): any;
@@ -48,6 +48,7 @@ export declare class QyBatch extends QyCD {
     _action_delContent(action: any): boolean;
     _action_insertEmbedding(action: any): boolean;
     _action_delEmbedding(action: any): true | undefined;
+    _action_put(action: any): boolean;
     _action_insert(action: any): boolean;
     _action_createFile(action: any): boolean;
     _action_createDir(action: any): boolean;