datafolder 1.0.17 → 1.0.19

package/README.md

@@ -4,6 +4,10 @@ datafolder is a lightning fluid JSON database for high speed SSDs, featuring an
 
 ### Changelog
 
+1.0.19 Add DocumentDB test case (test/testQyDocumentDB.js)
+
+1.0.18 Add Embedding Storage Support. Related APIs: dataFolder.insertEmbedding() and dataFolder.delEmbedding(). TODO: Add HNSW support and vector query.
+
 1.0.17 Add Northwind sample data test (test/testQyNorthwind.js)
 
 1.0.16 Add GraphQL sample data test (test/testQyGraphQL.js) to show how to write complex queries.
@@ -46,6 +50,7 @@ dataFolder.insert("traindata/2026/1/20/chat_0001", {
 });
 dataFolder.insert("traindata/2026/1/20/chat_0001/prompt", "What is async/await?");
 dataFolder.insert("traindata/2026/1/20/chat_0001/answer", "async/await is ...");
+dataFolder.insertEmbedding("traindata/2026/1/20/chat_0001/prompt", new Float32Array([0.36646, 332.8091, 2363.982]));
 const { email, password, address, feedbackScore } = dataFolder.view("traindata/2026/1/20/chat_0001");
 address.zipcode = 678;
 dataFolder.insert("traindata/2026/1/20/chat_0001", { feedbackScore: feedbackScore + 1, address });
@@ -80,6 +85,7 @@ dataFolder.insert(file, { feedbackScore: feedbackScore + 1, address: { city: "c"
 dataFolder.createFile("path/to/file");
 dataFolder.createDir("path/to/dir");
 dataFolder.insert("path/to/file", { email, address: { city, zipcode } });
+dataFolder.insertEmbedding("path/to/file", new Float32Array([235.235, 2949.8823]));
 ```
 
 #### Read
@@ -89,6 +95,7 @@ const dir = dataFolder.dir("path/to/dir");
 const { subdirList, fileList, sortedSubdirList } = dir;
 const file = dataFolder.file("path/to/file");
 const { email, address } = dataFolder.view("path/to/file", { email: 1, addres: 1 });
+const embedding = dataFolder.embedding("path/to/file");
 ```
 
 #### Update
@@ -104,6 +111,7 @@ dataFolder.insert(file, { feedbackScore: feedbackScore + 1 });
 dataFolder.delDir("path/to/dir");
 dataFolder.delFile("path/to/file");
 dataFolder.delContent("path/to/file");
+dataFolder.delEmbedding("path/to/file");
 dataFolder.remove("path/to/file", { email: 1, address: { zipcode: 1 } });
 dataFolder.remove(file, { email: 1, address: { zipcode: 1 } });
 ```
@@ -120,6 +128,7 @@ dataFolder.batch
   .insert("path/to/chat_details", { user: "user_0001", feedbackScore: 100 })
   .insert("path/to/prompt", "what is LLM?")
   .insert("path/to/answer", "LLM is AI...")
+  .insertEmbedding("path/to/prompt", [235.0235, 9923.2356])
   .remove("path/to/file", { address: { city: 1 } })
   .run();
 ```

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "datafolder",
-	"version": "1.0.17",
+	"version": "1.0.19",
 	"author": {
 		"name": "lysh",
 		"email": "datafolderdev@outlook.com"
@@ -17,28 +17,28 @@
 		"email": "datafolderdev@outlook.com"
 	},
 	"keywords": [
-		"JSON",
-		"database",
 		"AI",
-		"low-latency",
-		"tree-view",
-		"in-memory",
+		"HNSW",
+		"JSON",
+		"LSM",
+		"RAG",
 		"SSD",
-		"schema-free",
-		"atomic-writes",
-		"typescript",
-		"edge",
 		"backend",
-		"gaming",
-		"iot",
+		"database",
+		"easy",
+		"edge",
 		"embed",
 		"embedded",
-		"simple",
-		"easy",
-		"quick",
-		"millisecond",
+		"gaming",
+		"iot",
 		"lightweight",
-		"LSM"
+		"low-latency",
+		"millisecond",
+		"quick",
+		"schema-free",
+		"simple",
+		"tree-view",
+		"vector"
 	],
 	"dependencies": {},
 	"devDependencies": {

package/sample.js

@@ -13,6 +13,12 @@ async function run() {
 	});
 	dataFolder.insert("traindata/2026/1/20/chat_0001/prompt", "What is async/await?");
 	dataFolder.insert("traindata/2026/1/20/chat_0001/answer", "async/await is ...");
+	dataFolder.insertEmbedding("traindata/2026/1/20/chat_0001/prompt", [235.3536, 19.2358, 92.0929846]);
+	dataFolder.insertWithEmbedding(
+		"traindata/content",
+		{ score: 90, prompt: "hello" },
+		[235.3536, 19.2358, 92.0929846],
+	);
 	const { userid, email, address, feedbackScore } = dataFolder.view("traindata/2026/1/20/chat_0001");
 	logger.log(userid, email, feedbackScore, address);
 	address.zipcode = 678;
@@ -20,6 +26,9 @@ async function run() {
 	logger.log(dataFolder.view("traindata/2026/1/20/chat_0001", ["feedbackScore", "address"]));
 	logger.debug(dataFolder.view("traindata/2026/1/20/chat_0001/prompt"));
 	logger.warn(dataFolder.view("traindata/2026/1/20/chat_0001/answer"));
+	logger.log(dataFolder.embedding("traindata/2026/1/20/chat_0001/prompt"));
+	logger.log(dataFolder.view("traindata/content"));
+	logger.log(dataFolder.embedding("traindata/content"));
 	await dataFolder.stop();
 	await dataFolder.start();
 	{
@@ -32,6 +41,9 @@ async function run() {
 			dataFolder.view("traindata/2026/1/20/chat_0001/prompt"),
 			dataFolder.view("traindata/2026/1/20/chat_0001/answer"),
 		);
+		logger.log(dataFolder.embedding("traindata/2026/1/20/chat_0001/prompt"));
+		logger.log(dataFolder.view("traindata/content"));
+		logger.log(dataFolder.embedding("traindata/content"));
 	}
 	await dataFolder.stop();
 }

package/scripts/fastify.js

@@ -15,6 +15,7 @@ const config = {
 const { port, staticRoot } = config;
 const fastify = Fastify({ logger: false, forceCloseConnections: true });
 // const qyDB = new QyDB(join(__dirname, "../tmp/simulate2"));
+// const qyDB = new QyDB(join(__dirname, "../tmp/QyEmbeddings/1"));
 const qyDB = new QyDB();
 let sigintReceived = false;
 process.on("SIGINT", async () => {

package/scripts/hnsw.js

@@ -0,0 +1,39 @@
+"use strict";
+import { join, basename } from "node:path";
+import { readdir, readFile } from "node:fs/promises";
+import fastifyStatic from "@fastify/static";
+import fastifyCompress from "@fastify/compress";
+import { QyDB } from "../src/QyDB.js";
+import { folderOrFileExists, __dirname } from "../src/QyUtils.js";
+import { logger } from "../src/QyLogger.js";
+// logger.level = "warn";
+const baseVectPath = "D:\\syncsoft\\hnsw\\bench\\datasets\\siftsmall\\siftsmall_base.fvecs";
+const qyDB = new QyDB();
+run();
+async function run() {
+    await qyDB.start({ appendOnly: false });
+    const vectorList = await loadVects(baseVectPath);
+    const { length } = vectorList;
+    console.time(`insertEmbedding${length}`);
+    for (let idx = 0; idx < length; ++idx) {
+        const vect = vectorList[idx];
+        qyDB.insertEmbedding(["embedding", idx], vect);
+        // qyDB.insert(["prompt", idx], new Uint8Array(vect.buffer, vect.byteOffset, vect.byteLength));
+        // qyDB.delEmbedding(["embedding", idx]);
+        // qyDB.delFile(["prompt", idx]);
+    }
+    await qyDB.stop();
+    console.timeEnd(`insertEmbedding${length}`);
+}
+async function loadVects(filePath) {
+    const buffer = await readFile(filePath);
+    const { length } = buffer;
+    let offset = 0;
+    const vectorList = [];
+    while (offset < length) {
+        const dim = buffer.readUint32LE(offset);
+        vectorList.push(new Float32Array(buffer.buffer, offset + 4, dim));
+        offset += (dim + 1) * 4;
+    }
+    return vectorList;
+}

package/src/QyAcl.js

@@ -1,92 +1,114 @@
-import h from "node:fs/promises";
+import l from "node:fs/promises";
 
 import {
+    cleanContent as s,
     mergeContent as g,
-    mergeChanges as s,
     setValByNamePath as a
 } from "./QyFileContentUtils.js";
 
 import {
     isEmptyObj as m,
-    toBuffer as n
+    toBuffer as n,
+    SpecialOperators as r,
+    isSimpleType as o
 } from "./QyUtils.js";
 
 import {
-    logger as l
+    logger as h
 } from "./QyLogger.js";
 
-let f = {
+let {
+    $del: f,
+    $rpl: u,
+    $u8: c
+} = r, d = {
     M: 0,
     A: 1,
     R: 2,
     D: 3
 };
 
-function r(r, t) {
-    var e, a, n, s, [ i, o, f, t ] = t;
+function t(r, t) {
+    var e, a, n, i, [ s, o, f, t ] = t;
     for (e in t) null != r[e] && (r[e] = void 0);
     for (a in o) {
-        var c, u = r[a] || (r[a] = {});
-        for (c of o[a]) u[c] = 1;
+        var u, c = r[a] || (r[a] = {});
+        for (u of o[a]) c[u] = 1;
     }
     for (n in f) {
-        var h = r[n];
-        if (h) {
-            var l = f[n];
-            if (l) {
-                for (var d of l) delete h[d];
-                m(h) && (r[n] = void 0);
+        var l = r[n];
+        if (l) {
+            var h = f[n];
+            if (h) {
+                for (var d of h) delete l[d];
+                m(l) && (r[n] = void 0);
             }
         }
     }
-    for (s in i) r[s] = g(r[s], i[s]);
+    for (i in s) r[i] = g(r[i], s[i]);
 }
 
-function t(r, t) {
+function e(r, t) {
     for (var e of r) for (var a in e) t(a);
 }
 
-class e {
+class i {
     cmdArgAsMapChanges = [];
     cmdCount = 0;
     batchSize = 0;
     constructor() {}
     pushCmd(r, t, e = void 0) {
         switch (++this.cmdCount, r) {
-          case f.A:
-            i(this, [ f.A, t, e ]), o(this, [ f.R, t, e ]);
+          case d.A:
+            v(this, [ d.A, t, e ]), A(this, [ d.R, t, e ]);
             break;
 
-          case f.R:
-            i(this, [ f.R, t, e ]), o(this, [ f.A, t, e ]);
+          case d.R:
+            v(this, [ d.R, t, e ]), A(this, [ d.A, t, e ]);
             break;
 
-          case f.D:
-            if (i(this, [ f.D, t ]), t.startsWith("c")) o(this, [ f.M, t ]); else for (var a of [ f.A, f.R ]) o(this, [ a, t ]);
+          case d.D:
+            if (v(this, [ d.D, t ]), t.startsWith("c")) A(this, [ d.M, t ]); else for (var a of [ d.A, d.R ]) A(this, [ a, t ]);
             break;
 
-          case f.M:
+          case d.M:
             {
                 var n = this.cmdArgAsMapChanges;
                 let [ r ] = n;
-                n = (r = null == r ? n[f.M] = {} : r)[t];
-                r[t] = null != n ? s(n, e) : e;
+                n = (r = null == r ? n[d.M] = {} : r)[t];
+                r[t] = null != n ? function e(a, n) {
+                    if (n == f) return f;
+                    let r = n[u];
+                    if (null != r) return n;
+                    let t = a[u];
+                    if (null != t) return a[u] = g(t, n), a;
+                    if (a == f) return {
+                        [u]: s(n)
+                    };
+                    if (o(a) || o(n)) return n;
+                    if (c in n) return n;
+                    for (var i in n) {
+                        let r = n[i], t = a[i];
+                        null == t ? a[i] = r : a[i] = e(t, r);
+                    }
+                    return a;
+                }(n, e) : e;
                 break;
             }
 
           default:
-            l.error(`unsupported cmdName:${r}, called with ${t} ` + e), --this.cmdCount;
+            h.error(`unsupported cmdName:${r}, called with ${t} ` + e), --this.cmdCount;
         }
     }
     pushCmdArgAsListObj(r) {
         if (r) {
-            for (var t in ++this.batchSize, r[f.D]) this.pushCmd(f.D, t);
-            for (var e of [ f.A, f.R ]) {
+            for (var t in ++this.batchSize, r[d.D]) this.pushCmd(d.D, t);
+            for (var e of [ d.A, d.R ]) {
                 var a, n = r[e];
-                for (a in n) for (var s of n[a]) this.pushCmd(e, a, s);
+                for (a in n) for (var i of n[a]) this.pushCmd(e, a, i);
             }
-            var i, [ o ] = r;
-            for (i in o) this.pushCmd(f.M, i, o[i]);
+            var s, [ o ] = r;
+            for (s in o) this.pushCmd(d.M, s, o[s]);
         }
         return this;
     }
@@ -97,7 +119,7 @@ class e {
             cmdCount: 0,
             batchSize: 0
         });
-        for (r of [ f.A, f.R ]) {
+        for (r of [ d.A, d.R ]) {
             var e, a = t[r];
             for (e in a) {
                 var n = Object.keys(a[e]);
@@ -105,20 +127,20 @@ class e {
             }
             m(a) && delete t[r];
         }
-        var s = t[f.D], s = (s && m(s) && delete t[f.D], t[f.M]);
-        return s && m(s) && delete t[f.M], t;
+        var i = t[d.D], i = (i && m(i) && delete t[d.D], t[d.M]);
+        return i && m(i) && delete t[d.M], t;
     }
     toAclBuffer(r) {
         var t = this.batchSize;
-        if (0 < t) return A(r, t, JSON.stringify(this.takeCmdArgAsListObj()));
+        if (0 < t) return y(r, t, JSON.stringify(this.takeCmdArgAsListObj()));
     }
 }
 
-function i(r, t, e = 1) {
+function v(r, t, e = 1) {
     a(r.cmdArgAsMapChanges, t, e);
 }
 
-function o(r, t) {
+function A(r, t) {
     let e = r.cmdArgAsMapChanges;
     if (e && t && !(t.length <= 1)) {
         for (let r = 0; r < t.length - 1; ++r) if (null == (e = e[t[r]])) return;
@@ -126,55 +148,55 @@ function o(r, t) {
     }
 }
 
-async function c(t, r = !1, e = 0) {
+async function p(t, r = !1, e = 0) {
     try {
-        return await p(t, await h.readFile(t), r, e);
+        return await I(t, await l.readFile(t), r, e);
     } catch (r) {
-        throw l.error(`Reading ${t} failed:`, r), r;
+        throw h.error(`Reading ${t} failed:`, r), r;
     }
 }
 
-let v = [ 1, 2, 4, 8 ];
+let C = [ 1, 2, 4, 8 ];
 
-function u(t) {
+function b(t) {
     var e = t.length, a = new Array(e), n = new Array(e);
-    let s = e << 6, i = 1;
+    let i = e << 6, s = 1;
     for (let r = 0; r < e; ++r) {
-        var o = a[r] = (o = t[r]) <= 255 ? 0 : o <= 65535 ? 1 : o <= 4294967295 ? 2 : 3, o = (s |= o << (r << 1), 
-        n[r] = v[o]);
-        i += o;
+        var o = a[r] = (o = t[r]) <= 255 ? 0 : o <= 65535 ? 1 : o <= 4294967295 ? 2 : 3, o = (i |= o << (r << 1), 
+        n[r] = C[o]);
+        s += o;
     }
-    var f = Buffer.allocUnsafe(i);
-    f.writeUInt8(s);
-    let c = 1;
+    var f = Buffer.allocUnsafe(s);
+    f.writeUInt8(i);
+    let u = 1;
     for (let r = 0; r < e; ++r) {
-        d = l = h = u = void 0;
-        var [ u, h, l, d = 0 ] = [ f, t[r], n[r], c ];
-        switch (l) {
+        d = h = l = c = void 0;
+        var [ c, l, h, d = 0 ] = [ f, t[r], n[r], u ];
+        switch (h) {
           case 1:
-            u.writeUInt8(h, d);
+            c.writeUInt8(l, d);
             break;
 
           case 2:
-            u.writeUInt16BE(h, d);
+            c.writeUInt16BE(l, d);
             break;
 
           case 4:
-            u.writeUInt32BE(h, d);
+            c.writeUInt32BE(l, d);
             break;
 
           default:
-            u.writeBigUInt64BE(BigInt(h), d);
+            c.writeBigUInt64BE(BigInt(l), d);
         }
-        c += n[r];
+        u += n[r];
     }
     return f;
 }
 
-function d(t, e, a) {
-    var n = t.readUInt8(e), s = (++e, n >> 6 & 3);
-    for (let r = 0; r < s; ++r) {
-        var i = v[n >> (r << 1) & 3];
+function w(t, e, a) {
+    var n = t.readUInt8(e), i = (++e, n >> 6 & 3);
+    for (let r = 0; r < i; ++r) {
+        var s = C[n >> (r << 1) & 3];
         a[r] = ((r, t, e = 0) => {
             switch (t) {
               case 1:
@@ -189,45 +211,45 @@ function d(t, e, a) {
               default:
                 return Number(r.readBigUInt64BE(e));
             }
-        })(t, i, e), e += i;
+        })(t, s, e), e += s;
     }
     return e;
 }
 
-function A(r, t, e) {
+function y(r, t, e) {
     e = n(e);
-    return Buffer.concat([ u([ r, t, e.length ]), e ]);
+    return Buffer.concat([ b([ r, t, e.length ]), e ]);
 }
 
-async function p(e, r, a, n) {
-    var s = [];
+async function I(e, r, a, n) {
+    var i = [];
     for (let t = 0; t < r.length; ) {
-        var i = [], o = d(r, t, i), [ i, f, c ] = i;
-        if (i <= n) l.warn(`changeId ${i} <= currentChangeId ${n} at ${e}:${t}. Ignored.`), 
-        t = o + c; else {
-            n + f != i && l.warn(`currentChangeId ${n} + batchSize ${f} != changeId ` + i);
+        var s = [], o = w(r, t, s), [ s, f, u ] = s;
+        if (s <= n) h.warn(`changeId ${s} <= currentChangeId ${n} at ${e}:${t}. Ignored.`), 
+        t = o + u; else {
+            n + f != s && h.warn(`currentChangeId ${n} + batchSize ${f} != changeId ` + s);
             try {
-                var u = JSON.parse(r.subarray(o, o + c));
-                s.push([ i, u ]), n = i, t = o + c;
+                var c = JSON.parse(r.subarray(o, o + u));
+                i.push([ s, c ]), n = s, t = o + u;
             } catch (r) {
-                if (l.error(`Parse ${e} failed at ${t}:`, r), a) return l.warn(`Autorepair by truncating ${e} to ` + t), 
-                null == (f = await h.truncate(e, t)) ? l.log("truncate success") : l.error("truncate failed:", f), 
-                s;
+                if (h.error(`Parse ${e} failed at ${t}:`, r), a) return h.warn(`Autorepair by truncating ${e} to ` + t), 
+                null == (f = await l.truncate(e, t)) ? h.log("truncate success") : h.error("truncate failed:", f), 
+                i;
                 throw r;
             }
         }
     }
-    return s;
+    return i;
 }
 
 export {
-    f as CmdNames,
-    r as applyAclChange,
-    t as iterCmdKeys,
-    e as QyAclCmdGenerator,
-    c as readAclFile,
-    u as numsToBuffer,
-    d as bufferToNums,
-    A as toAclBuffer,
-    p as readAclBuffer
+    d as CmdNames,
+    t as applyAclChange,
+    e as iterCmdKeys,
+    i as QyAclCmdGenerator,
+    p as readAclFile,
+    b as numsToBuffer,
+    w as bufferToNums,
+    y as toAclBuffer,
+    I as readAclBuffer
 };

package/src/QyBatch.d.ts

@@ -25,6 +25,9 @@ export declare class QyBatch extends QyCD {
     createIndex(indexHostDirNamePath: any, indexPropPathList: any, parentDir?: any): any;
     delIndex(indexHostDirNamePath: any, indexPropPath: any, parentDir?: any): any;
     insert(namePath: any, content: any, noClone?: boolean, noCreate?: boolean, parentDir?: any): any;
+    insertEmbedding(namePath: any, embedding: any, parentDir?: any): any;
+    delEmbedding(namePath: any, parentDir?: any): any;
+    insertWithEmbedding(namePath: any, content: any, embedding: any, noClone?: boolean, noCreate?: boolean, parentDir?: any): any;
     remove(namePath: any, content: any, parentDir?: any): any;
     put(namePath: any, content: any, noClone?: boolean, parentDir?: any): any;
     delContent(namePath: any, parentDir?: any): any;
@@ -43,6 +46,8 @@ export declare class QyBatch extends QyCD {
     _action_delFile(action: any, deletingParentDir?: boolean): boolean;
     _action_delDir(action: any): boolean;
     _action_delContent(action: any): boolean;
+    _action_insertEmbedding(action: any): boolean;
+    _action_delEmbedding(action: any): true | undefined;
     _action_insert(action: any): boolean;
     _action_createFile(action: any): boolean;
     _action_createDir(action: any): boolean;