datafolder 1.0.3 → 1.0.5

package/README.md

@@ -14,30 +14,26 @@ datafolder is a lightweight JSON database, featuring an AI-friendly API for simp
 import DataFolder from "datafolder";
 const dataFolder = new DataFolder(); //Specify where to store the data. Default is ./data/sampleDataFolder
 await dataFolder.start(); //start() and stop() are the only two async places. All other code doesn't need await.
-dataFolder.insert("users/2026/user_0001", {
+dataFolder.insert("traindata/2026/1/20/chat_0001", {
   userid: "user_0001",
   email: "a@b.com",
-  score: 5,
+  feedbackScore: 5,
   address: { city: "a", zipcode: 123 },
 });
-dataFolder.insert("users/2026/user_0002", {
-  userid: "user_0002",
-  email: "c@b.com",
-  score: 3,
-  address: { city: "b", zipcode: 321 },
-});
-const { email, password, address, score } = dataFolder.view("users/2026/user_0002");
+dataFolder.insert("traindata/2026/1/20/chat_0001/prompt", "What is async/await?");
+dataFolder.insert("traindata/2026/1/20/chat_0001/answer", "async/await is ...");
+const { email, password, address, feedbackScore } = dataFolder.view("traindata/2026/1/20/chat_0001");
 address.zipcode = 678;
-dataFolder.insert("users/2026/user_0002", { score: score + 1, address });
+dataFolder.insert("traindata/2026/1/20/chat_0001", { feedbackScore: feedbackScore + 1, address });
 await dataFolder.stop();
 ```
 
 Or operate with dir/file objects:
 
 ```javascript
-const file = dataFolder.file("users/2026/user_0002");
-const { score, address } = file.view({ score: 1, address: 1 });
-dataFolder.insert(file, { score: score + 1, address: { city: "c", zipcode: 663 } });
+const file = dataFolder.file("traindata/2026/1/20/chat_0001");
+const { feedbackScore, address } = file.view({ feedbackScore: 1, address: 1 });
+dataFolder.insert(file, { feedbackScore: feedbackScore + 1, address: { city: "c", zipcode: 663 } });
 ```
 
 ## Features
@@ -75,7 +71,7 @@ const { email, address } = dataFolder.view("path/to/file", { email: 1, addres: 1
 
 ```javascript
 dataFolder.insert("path/to/file", { email: newEmail, address: { zipcode: newZipcode } });
-dataFolder.insert(file, { score: score + 1 });
+dataFolder.insert(file, { feedbackScore: feedbackScore + 1 });
 ```
 
 #### Delete
@@ -91,14 +87,15 @@ dataFolder.remove(file, { email: 1, address: { zipcode: 1 } });
 ### 2. Batch APIs
 
 ```javascript
-const [dir, file, { email, score }] = dataFolder.fetch
+const [dir, file, { email, feedbackScore }] = dataFolder.fetch
   .dir("path/to/dir")
   .file("path/to/file")
   .view("path/to/file")
   .run();
 dataFolder.batch
-  .insert("path/to/file", { email: "email1@a.b.com" })
-  .insert("path/to/file", { email: "email2", score: 100 })
+  .insert("path/to/chat_details", { user: "user_0001", feedbackScore: 100 })
+  .insert("path/to/prompt", "what is LLM?")
+  .insert("path/to/answer", "LLM is AI...")
   .remove("path/to/file", { address: { city: 1 } })
   .run();
 ```
@@ -106,6 +103,15 @@ dataFolder.batch
 ### 3. Powerful Query Support. Auto Create Needed Indexes
 
 ```javascript
+// get all the training data in year 2026 with feedbackScore between 60 and 85.
+dataFolder.queryFiles("traindata/2026/**", { feedbackScore: (x) => x >= 60 && x <= 85 });
+
+// get training data of user_001 from July to Oct in year 2025, 2026 with feedbackScore higher than 90.
+dataFolder.queryFiles(["traindata", { $in: [2025, 2026] }, { $gt: 6, $lte: 10 }], {
+  feedbackScore: { $gt: 90 },
+  userId: "user_0001",
+});
+
 dataFolder.queryFiles("flights/**", { _from: "airports/LAX" });
 dataFolder.queryFiles(["flights", "*", "jan", (x) => x >= 5 && x <= 7], {
   $or: [{ _from: "airports/BIS" }, { _to: "airports/BIS" }],
@@ -125,23 +131,23 @@ dataFolder.queryFilesMulti([
 ### 4. Flexible Trigger Support to Monitor Any Changes
 
 ```javascript
-function onEmailChanged({ studentId, email }) {
-  const { oldValue, newValue } = email;
-  // Logic to handle email change.
+function onPromptScoreChanged({ promptId, feedbackScore }) {
+  const { oldValue, newValue } = feedbackScore;
+  // Logic to handle prompt feedback score change.
 }
-dataFolder.insertTrigger("emailChanged", "students/{studentId}", "email");
-dataFolder.on("emailChanged", onEmailChanged);
-dataFolder.insert("students/student1", { email: "email1" });
-dataFolder.insert("students/student2", { email: "email2" });
-dataFolder.insert("students/student1", { email: "email1_changed" });
+dataFolder.insertTrigger("promptScoreChanged", "traindata/{promptId}", "feedbackScore");
+dataFolder.on("promptScoreChanged", onPromptScoreChanged);
+dataFolder.insert("traindata/prompt_1", { feedbackScore: 80 });
+dataFolder.insert("traindata/prompt_2", { feedbackScore: 63 });
+dataFolder.insert("traindata/prompt_1", { feedbackScore: 32 });
 ```
 
 Will trigger the following calls:
 
 ```javascript
-onEmailChanged({ studentId: "student1", email: { oldValue: undefined, newValue: "email1" } });
-onEmailChanged({ studentId: "student2", email: { oldValue: undefined, newValue: "email2" } });
-onEmailChanged({ studentId: "student1", email: { oldValue: "email1", newValue: "email1_changed" } });
+onPromptScoreChanged({ promptId: "student1", feedbackScore: { oldValue: undefined, newValue: 80 } });
+onPromptScoreChanged({ promptId: "student2", feedbackScore: { oldValue: undefined, newValue: 63 } });
+onPromptScoreChanged({ promptId: "student1", feedbackScore: { oldValue: 80, newValue: 32 } });
 ```
 
 ## Example

package/index.js

@@ -1,2 +1 @@
-import { QyDB } from "./src/QyDB.js";
-export default QyDB;
+export { QyDB as default } from "./src/QyDB.js";

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "datafolder",
-	"version": "1.0.3",
+	"version": "1.0.5",
 	"author": {
 		"name": "lysh",
 		"email": "datafolderdev@outlook.com"
@@ -37,7 +37,8 @@
 		"easy",
 		"quick",
 		"millisecond",
-		"lightweight"
+		"lightweight",
+		"LSM"
 	],
 	"dependencies": {},
 	"devDependencies": {

package/scripts/airbnb2_queries.js

@@ -7,7 +7,7 @@ function query1(qyDB) {
             .queryFiles(["airbnb", "Loft", "Entire home/apt", "Real Bed"], {
                 address: { street: "Porto, Porto, Portugal" },
             })
-            .map((x) => x.view(["name"]).name),
+            .map((x) => x.prop("name")),
         [
             "Porto city centre apartment H",
             "Romantic loft near historic center",
@@ -41,7 +41,7 @@ function query2(qyDB) {
                 },
                 9,
             )
-            .map((x) => x.view(["name"]).name),
+            .map((x) => x.prop("name")),
         [
             "Porto city centre apartment H",
             "Romantic loft near historic center",
@@ -66,7 +66,7 @@ function query3(qyDB) {
                 9,
                 9,
             )
-            .map((x) => x.view(["name"]).name),
+            .map((x) => x.prop("name")),
         [
             "Bright Top Floor Two Rooms Apartment w/ Terrace",
             "ReUrban Flats - Santa Catarina #3",
@@ -89,7 +89,7 @@ function query4(qyDB) {
                 ["airbnb/Loft/**", { minimum_nights: (x) => x >= 60 }],
             ])
             .run()[0]
-            .map((x) => x.view(["name"]).name)
+            .map((x) => x.prop("name"))
             .sort(),
         [
             "Porto city centre apartment H",
@@ -128,7 +128,7 @@ function query5(qyDB) {
                 ],
                 3,
             )
-            .map((x) => x.view(["name"]).name),
+            .map((x) => x.prop("name")),
         [
             "Porto city centre apartment H",
             "Romantic loft near historic center",
@@ -152,7 +152,7 @@ function query6(qyDB) {
                 20,
                 10,
             )
-            .map((x) => x.view(["name"]).name),
+            .map((x) => x.prop("name")),
         [
             "ReUrban Flats - Santa Catarina #3",
             "The Blue and White Porto House - Vintage",
@@ -169,7 +169,7 @@ function query6(qyDB) {
 function query7(qyDB) {
     console.time("query7");
     deepEqual(
-        qyDB.queryFiles(["airbnb", "Loft", "**"], { minimum_nights: (x) => x >= 60 }).map((x) => x.view(["name"]).name),
+        qyDB.queryFiles(["airbnb", "Loft", "**"], { minimum_nights: (x) => x >= 60 }).map((x) => x.prop("name")),
         ["Romantic loft near historic center", "Brand New Luxury Downtown Studio"],
     );
     console.timeEnd("query7");
@@ -177,9 +177,7 @@ function query7(qyDB) {
 function query8(qyDB) {
     console.time("query8");
     deepEqual(
-        qyDB
-            .queryFiles(["airbnb", "Loft", ""], { minimum_nights: (x) => x >= 60 }, 10, 10)
-            .map((x) => x.view(["name"]).name),
+        qyDB.queryFiles(["airbnb", "Loft", ""], { minimum_nights: (x) => x >= 60 }, 10, 10).map((x) => x.prop("name")),
         [],
     );
     console.timeEnd("query8");
@@ -198,7 +196,7 @@ function query9(qyDB) {
                 ],
                 3,
             )
-            .map((x) => x.view(["name"]).name),
+            .map((x) => x.prop("name")),
         [
             "Porto city centre apartment H",
             "Romantic loft near historic center",

package/scripts/airbnb_queries.js

@@ -10,7 +10,7 @@ function query1(qyDB) {
                 bed_type: "Real Bed",
                 address: { street: "Porto, Porto, Portugal" },
             })
-            .map((x) => x.view(["name"]).name),
+            .map((x) => x.prop("name")),
         [
             "Porto city centre apartment H",
             "Romantic loft near historic center",
@@ -47,7 +47,7 @@ function query2(qyDB) {
                 },
                 9,
             )
-            .map((x) => x.view(["name"]).name),
+            .map((x) => x.prop("name")),
         [
             "Porto city centre apartment H",
             "Romantic loft near historic center",
@@ -77,7 +77,7 @@ function query3(qyDB) {
                 9,
                 9,
             )
-            .map((x) => x.view(["name"]).name),
+            .map((x) => x.prop("name")),
         [
             "Bright Top Floor Two Rooms Apartment w/ Terrace",
             "ReUrban Flats - Santa Catarina #3",
@@ -106,7 +106,7 @@ function query4(qyDB) {
                     { property_type: (x) => x == "Loft", minimum_nights: (x) => x >= 60 },
                 ],
             })
-            .map((x) => x.view(["name"]).name)
+            .map((x) => x.prop("name"))
             .sort(),
         [
             "Porto city centre apartment H",
@@ -150,7 +150,7 @@ function query5(qyDB) {
                 },
                 3,
             )
-            .map((x) => x.view(["name"]).name),
+            .map((x) => x.prop("name")),
         [
             "Porto city centre apartment H",
             "Romantic loft near historic center",
@@ -179,7 +179,7 @@ function query6(qyDB) {
                 20,
                 10,
             )
-            .map((x) => x.view(["name"]).name),
+            .map((x) => x.prop("name")),
         [
             "ReUrban Flats - Santa Catarina #3",
             "The Blue and White Porto House - Vintage",
@@ -198,7 +198,7 @@ function query7(qyDB) {
     deepEqual(
         qyDB
             .queryFiles("airbnb1", { property_type: (x) => x == "Loft", minimum_nights: (x) => x >= 60 })
-            .map((x) => x.view(["name"]).name),
+            .map((x) => x.prop("name")),
         ["Romantic loft near historic center", "Brand New Luxury Downtown Studio"],
     );
     console.timeEnd("query7");
@@ -208,7 +208,7 @@ function query8(qyDB) {
     deepEqual(
         qyDB
             .queryFiles("airbnb1", { property_type: (x) => x == "Loft", minimum_nights: (x) => x >= 60 }, 10, 10)
-            .map((x) => x.view(["name"]).name),
+            .map((x) => x.prop("name")),
         [],
     );
     console.timeEnd("query8");
@@ -232,7 +232,7 @@ function query9(qyDB) {
                 },
                 3,
             )
-            .map((x) => x.view(["name"]).name),
+            .map((x) => x.prop("name")),
         [
             "Porto city centre apartment H",
             "Romantic loft near historic center",

package/scripts/express.js

@@ -1,5 +1,6 @@
 import { join } from "node:path";
 import { QyDB } from "../src/QyDB.js";
+import Express from "express";
 import { __dirname } from "../src/QyUtils.js";
 
 const config = {
@@ -10,7 +11,7 @@ const config = {
 };
 
 const { port, qyDBDataFolder } = config;
-const app = require("express")();
+const app = Express();
 const qyDB = new QyDB(qyDBDataFolder);
 
 let server;

package/scripts/flights.js

@@ -9,7 +9,7 @@ import { runQueries } from "./flights_queries.js";
 
 logger.level = "info";
 
-const arangoJsonFolderPath = "../GraphCourse_DemoData_ArangoDB-2";
+const arangoJsonFolderPath = "D:\\scripts\\GraphCourse_DemoData_ArangoDB-2";
 const dataFolder = "./tmp/flights";
 
 const qyDB = new QyDB(dataFolder, { preloadFoldersAtStart: false });

package/scripts/flights2_queries.js

@@ -6,94 +6,93 @@ import { deepEqual } from "node:assert/strict";
 
 function query1(qyDB) {
     console.time("query1");
-    const resultMap = {};
-    for (const x of qyDB.queryFiles("flights/**", { _from: "airports/LAX" })) {
-        resultMap[x.fileContent._to] = 1;
-    }
-    deepEqual(Object.keys(resultMap), [
-        "airports/CVG",
-        "airports/SMF",
-        "airports/ATL",
-        "airports/MSP",
-        "airports/DFW",
-        "airports/PHL",
-        "airports/IAH",
-        "airports/TUS",
-        "airports/ORD",
-        "airports/PDX",
-        "airports/MIA",
-        "airports/SFO",
-        "airports/DEN",
-        "airports/IAD",
-        "airports/SJC",
-        "airports/PHX",
-        "airports/SEA",
-        "airports/SLC",
-        "airports/JFK",
-        "airports/SAN",
-        "airports/ONT",
-        "airports/LAS",
-        "airports/MCI",
-        "airports/MDW",
-        "airports/FLL",
-        "airports/AUS",
-        "airports/EWR",
-        "airports/CLE",
-        "airports/SBP",
-        "airports/MEM",
-        "airports/SBA",
-        "airports/CLT",
-        "airports/COS",
-        "airports/ASE",
-        "airports/ICT",
-        "airports/HOU",
-        "airports/YUM",
-        "airports/STL",
-        "airports/FAT",
-        "airports/ABQ",
-        "airports/BOI",
-        "airports/BOS",
-        "airports/LIH",
-        "airports/HNL",
-        "airports/SMX",
-        "airports/MRY",
-        "airports/DTW",
-        "airports/RNO",
-        "airports/MCO",
-        "airports/OXR",
-        "airports/OAK",
-        "airports/OGG",
-        "airports/KOA",
-        "airports/BWI",
-        "airports/CLD",
-        "airports/IND",
-        "airports/PSP",
-        "airports/ELP",
-        "airports/TPA",
-        "airports/OKC",
-        "airports/MSY",
-        "airports/SNA",
-        "airports/IYK",
-        "airports/IPL",
-        "airports/SAT",
-        "airports/MOD",
-        "airports/BNA",
-        "airports/DCA",
-        "airports/SGU",
-        "airports/BFL",
-        "airports/EUG",
-        "airports/EGE",
-        "airports/GEG",
-        "airports/BDL",
-        "airports/JAX",
-        "airports/CMH",
-        "airports/PIT",
-        "airports/RDU",
-        "airports/SJU",
-        "airports/XNA",
-        "airports/MTJ",
-        "airports/JAC",
-    ]);
+    deepEqual(
+        [...new Set(qyDB.queryFiles("flights/**", { _from: "airports/LAX" }).map((x) => x.prop("_to"))).values()],
+        [
+            "airports/CVG",
+            "airports/SMF",
+            "airports/ATL",
+            "airports/MSP",
+            "airports/DFW",
+            "airports/PHL",
+            "airports/IAH",
+            "airports/TUS",
+            "airports/ORD",
+            "airports/PDX",
+            "airports/MIA",
+            "airports/SFO",
+            "airports/DEN",
+            "airports/IAD",
+            "airports/SJC",
+            "airports/PHX",
+            "airports/SEA",
+            "airports/SLC",
+            "airports/JFK",
+            "airports/SAN",
+            "airports/ONT",
+            "airports/LAS",
+            "airports/MCI",
+            "airports/MDW",
+            "airports/FLL",
+            "airports/AUS",
+            "airports/EWR",
+            "airports/CLE",
+            "airports/SBP",
+            "airports/MEM",
+            "airports/SBA",
+            "airports/CLT",
+            "airports/COS",
+            "airports/ASE",
+            "airports/ICT",
+            "airports/HOU",
+            "airports/YUM",
+            "airports/STL",
+            "airports/FAT",
+            "airports/ABQ",
+            "airports/BOI",
+            "airports/BOS",
+            "airports/LIH",
+            "airports/HNL",
+            "airports/SMX",
+            "airports/MRY",
+            "airports/DTW",
+            "airports/RNO",
+            "airports/MCO",
+            "airports/OXR",
+            "airports/OAK",
+            "airports/OGG",
+            "airports/KOA",
+            "airports/BWI",
+            "airports/CLD",
+            "airports/IND",
+            "airports/PSP",
+            "airports/ELP",
+            "airports/TPA",
+            "airports/OKC",
+            "airports/MSY",
+            "airports/SNA",
+            "airports/IYK",
+            "airports/IPL",
+            "airports/SAT",
+            "airports/MOD",
+            "airports/BNA",
+            "airports/DCA",
+            "airports/SGU",
+            "airports/BFL",
+            "airports/EUG",
+            "airports/EGE",
+            "airports/GEG",
+            "airports/BDL",
+            "airports/JAX",
+            "airports/CMH",
+            "airports/PIT",
+            "airports/RDU",
+            "airports/SJU",
+            "airports/XNA",
+            "airports/MTJ",
+            "airports/JAC",
+        ],
+    );
     console.timeEnd("query1");
 }
 
@@ -116,7 +115,7 @@ function query2(qyDB) {
             .queryFiles(["flights", "*", 1, (x) => x >= 5 && x <= 7], {
                 $or: [{ _from: "airports/BIS" }, { _to: "airports/BIS" }],
             })
-            .map((x) => x.fileContent.FlightNum)
+            .map((x) => x.prop("FlightNum"))
             .sort((x, y) => x - y),
         [
             6635, 646, 6541, 5651, 6541, 5648, 687, 309, 6678, 6678, 6636, 439, 1567, 6635, 5647, 6541, 5660, 6541,
@@ -370,7 +369,7 @@ function query5(qyDB) {
     deepEqual(
         qyDB
             .queryFiles("**", { vip: true })
-            .map((x) => x.fileContent.name)
+            .map((x) => x.prop("name"))
             .sort(),
         [
             "Amarillo International",
@@ -414,7 +413,7 @@ function query7(qyDB) {
             .queryFiles(["flights", "*", { $or: [1] }, [5, 6, 7]], {
                 $or: [{ _from: "airports/BIS" }, { _to: "airports/BIS" }],
             })
-            .map((x) => x.fileContent.FlightNum)
+            .map((x) => x.prop("FlightNum"))
             .sort((x, y) => x - y),
         [
             6635, 646, 6541, 5651, 6541, 5648, 687, 309, 6678, 6678, 6636, 439, 1567, 6635, 5647, 6541, 5660, 6541,
@@ -428,7 +427,7 @@ function query7(qyDB) {
 function query8(qyDB) {
     console.time("query8");
     deepEqual(
-        qyDB.queryFiles("**", { name: "Thigpen " }).map((x) => x.fileContent.name),
+        qyDB.queryFiles("**", { name: "Thigpen " }).map((x) => x.prop("name")),
         ["Thigpen "],
     );
     console.timeEnd("query8");