datafolder 1.0.2 → 1.0.3

package/README.md

@@ -11,9 +11,9 @@ datafolder is a lightweight JSON database, featuring an AI-friendly API for simp
 ## How to use
 
 ```javascript
-const DataFolder = require("datafolder");
+import DataFolder from "datafolder";
 const dataFolder = new DataFolder(); //Specify where to store the data. Default is ./data/sampleDataFolder
-await dataFolder.start();
+await dataFolder.start(); //start() and stop() are the only two async places. All other code doesn't need await.
 dataFolder.insert("users/2026/user_0001", {
   userid: "user_0001",
   email: "a@b.com",
@@ -107,7 +107,13 @@ dataFolder.batch
 
 ```javascript
 dataFolder.queryFiles("flights/**", { _from: "airports/LAX" });
-dataFolder.queryFiles(["flights", "*", 1, (x) => x >= 5 && x <= 7], {
+dataFolder.queryFiles(["flights", "*", "jan", (x) => x >= 5 && x <= 7], {
+  $or: [{ _from: "airports/BIS" }, { _to: "airports/BIS" }],
+});
+dataFolder.queryFiles(["flights", "*", "feb", { $or: [5, 6, 7] }], {
+  $or: [{ _from: "airports/BIS" }, { _to: "airports/BIS" }],
+});
+dataFolder.queryFiles(["flights", "*", "mar", { $gte: 5, lte: 7 }], {
   $or: [{ _from: "airports/BIS" }, { _to: "airports/BIS" }],
 });
 dataFolder.queryFilesMulti([

package/index.js

@@ -1,3 +1,2 @@
-const { QyDB } = require("./src/QyDB.js");
-module.exports = QyDB;
-module.exports.default = QyDB;
+import { QyDB } from "./src/QyDB.js";
+export default QyDB;

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "datafolder",
-	"version": "1.0.2",
+	"version": "1.0.3",
 	"author": {
 		"name": "lysh",
 		"email": "datafolderdev@outlook.com"
@@ -50,5 +50,6 @@
 	"scripts": {
 		"ui": "start http://localhost:3628/datafolderui/index.html && node ./scripts/fastify.js",
 		"airbnb": "node ./scripts/airbnb2.js && start http://localhost:3628/datafolderui/index.html && node ./scripts/fastify.js"
-	}
+	},
+	"type": "module"
 }

package/sample.js

@@ -1,37 +1,37 @@
-const DataFolder = require("datafolder");
-const dataFolder = new DataFolder();
-
-run();
-
-async function run() {
-	await dataFolder.start();
-	dataFolder.insert("users/2026/user_0001", {
-		userid: "user_0001",
-		email: "a@b.com",
-		password: "123",
-		score: 5,
-		address: { city: "a", zipcode: 123 },
-	});
-	dataFolder.insert("users/2026/user_0002", {
-		userid: "user_0002",
-		email: "c@b.com",
-		password: "333",
-		score: 3,
-		address: { city: "b", zipcode: 321 },
-	});
-	const { email, password, address, score } = dataFolder.view("users/2026/user_0002");
-	logger.log(email, password, score, address);
-	address.zipcode = 678;
-	dataFolder.insert("users/2026/user_0002", { score: score + 1, address });
-	logger.log(dataFolder.view("users/2026/user_0002", ["score", "address"]));
-	await dataFolder.stop();
-	await dataFolder.start();
-	{
-		const file = dataFolder.file("users/2026/user_0002");
-		const { email, password, score } = file.view();
-		logger.log(email, password, score);
-		dataFolder.insert(file, { score: score + 1 });
-		logger.log(file.view());
-	}
-	await dataFolder.stop();
-}
+import DataFolder from "datafolder";
+const dataFolder = new DataFolder();
+
+run();
+
+async function run() {
+	await dataFolder.start();
+	dataFolder.insert("users/2026/user_0001", {
+		userid: "user_0001",
+		email: "a@b.com",
+		password: "123",
+		score: 5,
+		address: { city: "a", zipcode: 123 },
+	});
+	dataFolder.insert("users/2026/user_0002", {
+		userid: "user_0002",
+		email: "c@b.com",
+		password: "333",
+		score: 3,
+		address: { city: "b", zipcode: 321 },
+	});
+	const { email, password, address, score } = dataFolder.view("users/2026/user_0002");
+	logger.log(email, password, score, address);
+	address.zipcode = 678;
+	dataFolder.insert("users/2026/user_0002", { score: score + 1, address });
+	logger.log(dataFolder.view("users/2026/user_0002", ["score", "address"]));
+	await dataFolder.stop();
+	await dataFolder.start();
+	{
+		const file = dataFolder.file("users/2026/user_0002");
+		const { email, password, score } = file.view();
+		logger.log(email, password, score);
+		dataFolder.insert(file, { score: score + 1 });
+		logger.log(file.view());
+	}
+	await dataFolder.stop();
+}

package/scripts/airbnb.js

@@ -1,13 +1,13 @@
-const path = require("node:path");
-const fsPromises = require("node:fs/promises");
-const { QyDB } = require("../src/QyDB.js");
-const { folderOrFileExists, isSimpleType } = require("../src/QyUtils.js");
-const { runQueries } = require("./airbnb_queries.js");
-const { downloadFile } = require("./downloadFile.js");
+import { join } from "node:path";
+import { open, rm } from "node:fs/promises";
+import { QyDB } from "../src/QyDB.js";
+import { folderOrFileExists, isSimpleType, __dirname } from "../src/QyUtils.js";
+import { runQueries } from "./airbnb_queries.js";
+import { downloadFile } from "./downloadFile.js";
 
 const fileName = "listingsAndReviews.json";
 const fileUrl = `https://raw.githubusercontent.com/neelabalan/mongodb-sample-dataset/refs/heads/main/sample_airbnb/${fileName}`;
-const airbnbJsonFilePath = path.join(__dirname, "../data/", fileName);
+const airbnbJsonFilePath = join(__dirname, "../data/", fileName);
 const qyDB = new QyDB();
 const { rootFolder } = qyDB;
 
@@ -15,7 +15,7 @@ run();
 async function insertData() {
     console.time("insertData");
     const { batch } = qyDB;
-    const fileHandle = await fsPromises.open(airbnbJsonFilePath, "r");
+    const fileHandle = await open(airbnbJsonFilePath, "r");
     for await (const line of fileHandle.readLines()) {
         const oneItem = convert(JSON.parse(line));
         batch.insert(["airbnb1", oneItem._id], oneItem);
@@ -39,7 +39,7 @@ async function run() {
     let shouldInsertData = true;
     if (process.argv[2] == "true") {
         try {
-            await fsPromises.rm(rootFolder, { recursive: true });
+            await rm(rootFolder, { recursive: true });
         } catch (exception) {}
         shouldInsertData = true;
     } else if (!(await folderOrFileExists(rootFolder))) shouldInsertData = true;
@@ -69,5 +69,4 @@ function convert(item) {
         item[prop] = convert(item[prop]);
     }
     return item;
-}
-Object.assign(module.exports, { runQueries });
+}

package/scripts/airbnb2.js

@@ -1,13 +1,13 @@
-const path = require("node:path");
-const fsPromises = require("node:fs/promises");
-const { QyDB } = require("../src/QyDB.js");
-const { folderOrFileExists, isSimpleType } = require("../src/QyUtils.js");
-const { runQueries } = require("./airbnb2_queries.js");
-const { downloadFile } = require("./downloadFile.js");
+import { join } from "node:path";
+import { open, rm } from "node:fs/promises";
+import { QyDB } from "../src/QyDB.js";
+import { folderOrFileExists, isSimpleType, __dirname } from "../src/QyUtils.js";
+import { runQueries } from "./airbnb2_queries.js";
+import { downloadFile } from "./downloadFile.js";
 
 const fileName = "listingsAndReviews.json";
 const fileUrl = `https://raw.githubusercontent.com/neelabalan/mongodb-sample-dataset/refs/heads/main/sample_airbnb/${fileName}`;
-const airbnbJsonFilePath = path.join(__dirname, "../data/", fileName);
+const airbnbJsonFilePath = join(__dirname, "../data/", fileName);
 const qyDB = new QyDB();
 const { rootFolder } = qyDB;
 
@@ -15,7 +15,7 @@ run();
 async function insertData() {
     console.time("insertData");
     const { batch } = qyDB;
-    const fileHandle = await fsPromises.open(airbnbJsonFilePath, "r");
+    const fileHandle = await open(airbnbJsonFilePath, "r");
     for await (const line of fileHandle.readLines()) {
         const oneItem = convert(JSON.parse(line));
         const { property_type, room_type, bed_type, _id } = oneItem;
@@ -40,7 +40,7 @@ async function run() {
     let shouldInsertData = true;
     if (process.argv[2] == "true") {
         try {
-            await fsPromises.rm(rootFolder, { recursive: true });
+            await rm(rootFolder, { recursive: true });
         } catch (exception) {}
         shouldInsertData = true;
     } else if (!(await folderOrFileExists(rootFolder))) shouldInsertData = true;
@@ -70,5 +70,4 @@ function convert(item) {
         item[prop] = convert(item[prop]);
     }
     return item;
-}
-Object.assign(module.exports, { runQueries });
+}

package/scripts/airbnb2_queries.js

@@ -1,8 +1,8 @@
-const assert = require("node:assert/strict");
+import { deepEqual } from "node:assert/strict";
 
 function query1(qyDB) {
     console.time("query1");
-    assert.deepEqual(
+    deepEqual(
         qyDB
             .queryFiles(["airbnb", "Loft", "Entire home/apt", "Real Bed"], {
                 address: { street: "Porto, Porto, Portugal" },
@@ -32,7 +32,7 @@ function query1(qyDB) {
 }
 function query2(qyDB) {
     console.time("query2");
-    assert.deepEqual(
+    deepEqual(
         qyDB
             .queryFiles(
                 ["airbnb", "Loft", "Entire home/apt", "Real Bed"],
@@ -58,7 +58,7 @@ function query2(qyDB) {
 }
 function query3(qyDB) {
     console.time("query3");
-    assert.deepEqual(
+    deepEqual(
         qyDB
             .queryFiles(
                 ["airbnb", "Loft", "Entire home/apt", "Real Bed"],
@@ -82,7 +82,7 @@ function query3(qyDB) {
 }
 function query4(qyDB) {
     console.time("query4");
-    assert.deepEqual(
+    deepEqual(
         qyDB.fetch
             .queryFilesMulti([
                 [["airbnb", "Loft", "Entire home/apt", "Real Bed"], { address: { street: "Porto, Porto, Portugal" } }],
@@ -116,7 +116,7 @@ function query4(qyDB) {
 }
 function query5(qyDB) {
     console.time("query5");
-    assert.deepEqual(
+    deepEqual(
         qyDB
             .queryFilesMulti(
                 [
@@ -139,7 +139,7 @@ function query5(qyDB) {
 }
 function query6(qyDB) {
     console.time("query6");
-    assert.deepEqual(
+    deepEqual(
         qyDB
             .queryFilesMulti(
                 [
@@ -168,7 +168,7 @@ function query6(qyDB) {
 }
 function query7(qyDB) {
     console.time("query7");
-    assert.deepEqual(
+    deepEqual(
         qyDB.queryFiles(["airbnb", "Loft", "**"], { minimum_nights: (x) => x >= 60 }).map((x) => x.view(["name"]).name),
         ["Romantic loft near historic center", "Brand New Luxury Downtown Studio"],
     );
@@ -176,7 +176,7 @@ function query7(qyDB) {
 }
 function query8(qyDB) {
     console.time("query8");
-    assert.deepEqual(
+    deepEqual(
         qyDB
             .queryFiles(["airbnb", "Loft", ""], { minimum_nights: (x) => x >= 60 }, 10, 10)
             .map((x) => x.view(["name"]).name),
@@ -186,7 +186,7 @@ function query8(qyDB) {
 }
 function query9(qyDB) {
     console.time("query9");
-    assert.deepEqual(
+    deepEqual(
         qyDB
             .queryFilesMulti(
                 [
@@ -228,4 +228,4 @@ function runQueries(qyDB) {
     query9(qyDB);
 }
 
-Object.assign(module.exports, { runQueries });
+export { runQueries };

package/scripts/airbnb_queries.js

@@ -1,8 +1,8 @@
-const assert = require("node:assert/strict");
+import { deepEqual } from "node:assert/strict";
 
 function query1(qyDB) {
     console.time("query1");
-    assert.deepEqual(
+    deepEqual(
         qyDB
             .queryFiles("airbnb1", {
                 property_type: "Loft",
@@ -35,7 +35,7 @@ function query1(qyDB) {
 }
 function query2(qyDB) {
     console.time("query2");
-    assert.deepEqual(
+    deepEqual(
         qyDB
             .queryFiles(
                 "airbnb1",
@@ -64,7 +64,7 @@ function query2(qyDB) {
 }
 function query3(qyDB) {
     console.time("query3");
-    assert.deepEqual(
+    deepEqual(
         qyDB
             .queryFiles(
                 "airbnb1",
@@ -93,7 +93,7 @@ function query3(qyDB) {
 }
 function query4(qyDB) {
     console.time("query4");
-    assert.deepEqual(
+    deepEqual(
         qyDB
             .queryFiles("airbnb1", {
                 $or: [
@@ -106,7 +106,8 @@ function query4(qyDB) {
                     { property_type: (x) => x == "Loft", minimum_nights: (x) => x >= 60 },
                 ],
             })
-            .map((x) => x.view(["name"]).name),
+            .map((x) => x.view(["name"]).name)
+            .sort(),
         [
             "Porto city centre apartment H",
             "Romantic loft near historic center",
@@ -126,13 +127,13 @@ function query4(qyDB) {
             "Your OWM Invicta",
             "Pury Apartments",
             "Live Porto & Douro - Breiner III",
-        ],
+        ].sort(),
     );
     console.timeEnd("query4");
 }
 function query5(qyDB) {
     console.time("query5");
-    assert.deepEqual(
+    deepEqual(
         qyDB
             .queryFiles(
                 "airbnb1",
@@ -160,7 +161,7 @@ function query5(qyDB) {
 }
 function query6(qyDB) {
     console.time("query6");
-    assert.deepEqual(
+    deepEqual(
         qyDB
             .queryFiles(
                 "airbnb1",
@@ -180,7 +181,6 @@ function query6(qyDB) {
             )
             .map((x) => x.view(["name"]).name),
         [
-            "Bright Top Floor Two Rooms Apartment w/ Terrace",
             "ReUrban Flats - Santa Catarina #3",
             "The Blue and White Porto House - Vintage",
             "4twoLoft at Great Location",
@@ -188,13 +188,14 @@ function query6(qyDB) {
             "Your OWM Invicta",
             "Pury Apartments",
             "Live Porto & Douro - Breiner III",
+            "Brand New Luxury Downtown Studio",
         ],
     );
     console.timeEnd("query6");
 }
 function query7(qyDB) {
     console.time("query7");
-    assert.deepEqual(
+    deepEqual(
         qyDB
             .queryFiles("airbnb1", { property_type: (x) => x == "Loft", minimum_nights: (x) => x >= 60 })
             .map((x) => x.view(["name"]).name),
@@ -204,7 +205,7 @@ function query7(qyDB) {
 }
 function query8(qyDB) {
     console.time("query8");
-    assert.deepEqual(
+    deepEqual(
         qyDB
             .queryFiles("airbnb1", { property_type: (x) => x == "Loft", minimum_nights: (x) => x >= 60 }, 10, 10)
             .map((x) => x.view(["name"]).name),
@@ -214,7 +215,7 @@ function query8(qyDB) {
 }
 function query9(qyDB) {
     console.time("query9");
-    assert.deepEqual(
+    deepEqual(
         qyDB
             .queryFiles(
                 "airbnb1",
@@ -261,4 +262,4 @@ function runQueries(qyDB) {
     query9(qyDB);
 }
 
-Object.assign(module.exports, { runQueries });
+export { runQueries };

package/scripts/downloadFile.js

@@ -1,7 +1,7 @@
-const fs = require("fs");
-const https = require("https");
-const { ensureParentDir, folderOrFileExists } = require("../src/QyUtils.js");
-const { logger } = require("../src/QyLogger.js");
+import { createWriteStream } from "fs";
+import { get } from "https";
+import { ensureParentDir, folderOrFileExists } from "../src/QyUtils.js";
+import { logger } from "../src/QyLogger.js";
 
 async function downloadFile(fileUrl, filePath) {
   if (await folderOrFileExists(filePath)) return;
@@ -11,32 +11,30 @@ async function downloadFile(fileUrl, filePath) {
     resolve = res;
     reject = rej;
   });
-  https
-    .get(fileUrl, (response) => {
-      if (response.statusCode !== 200) {
-        logger.error(`Download failed: Status Code ${response.statusCode}`);
-        reject(response.statusCode);
-      } else {
-        const fileStream = fs.createWriteStream(filePath);
-        let accu = 0;
-        response.on("data", (chunk) => {
-          fileStream.write(chunk);
-          accu += chunk.length;
-          if (accu >= 1000_000) {
-            process.stdout.write(".");
-            accu = 0;
-          }
-        });
-        response.on("end", () => {
-          fileStream.close();
-          resolve();
-        });
-      }
-    })
-    .on("error", (err) => {
-      logger.error(`Request error: ${err.message}`);
-      reject(err.message);
-    });
+  get(fileUrl, (response) => {
+    if (response.statusCode !== 200) {
+      logger.error(`Download failed: Status Code ${response.statusCode}`);
+      reject(response.statusCode);
+    } else {
+      const fileStream = createWriteStream(filePath);
+      let accu = 0;
+      response.on("data", (chunk) => {
+        fileStream.write(chunk);
+        accu += chunk.length;
+        if (accu >= 1000_000) {
+          process.stdout.write(".");
+          accu = 0;
+        }
+      });
+      response.on("end", () => {
+        fileStream.close();
+        resolve();
+      });
+    }
+  }).on("error", (err) => {
+    logger.error(`Request error: ${err.message}`);
+    reject(err.message);
+  });
   return promise;
 }
-Object.assign(module.exports, { downloadFile });
+export { downloadFile };

package/scripts/express.js

@@ -1,11 +1,12 @@
-const path = require("node:path");
-const { QyDB } = require("./QyDB.js");
+import { join } from "node:path";
+import { QyDB } from "../src/QyDB.js";
+import { __dirname } from "../src/QyUtils.js";
 
 const config = {
   port: 3629,
   qyDBDataFolder: "./tmp/QyExpress",
-  staticRoot: path.join(__dirname, "../static/"),
-  rpcFolder: path.join(__dirname, "rpc"),
+  staticRoot: join(__dirname, "../static/"),
+  rpcFolder: join(__dirname, "rpc"),
 };
 
 const { port, qyDBDataFolder } = config;

package/scripts/fastify.js

@@ -1,19 +1,20 @@
 "use strict";
-const path = require("node:path");
-const fsPromises = require("node:fs/promises");
-const fastifyStatic = require("@fastify/static");
-const fastifyCompress = require("@fastify/compress");
-const { QyDB } = require("../src/QyDB.js");
-const { folderOrFileExists } = require("../src/QyUtils.js");
+import { join, basename } from "node:path";
+import { readdir, readFile } from "node:fs/promises";
+import fastifyStatic from "@fastify/static";
+import fastifyCompress from "@fastify/compress";
+import { QyDB } from "../src/QyDB.js";
+import { folderOrFileExists, __dirname } from "../src/QyUtils.js";
+import Fastify from "fastify";
 
 const config = {
   port: 3628,
-  staticRoot: path.join(__dirname, "../static/"),
-  rpcFolder: path.join(__dirname, "../src/rpc"),
+  staticRoot: join(__dirname, "../static/"),
+  rpcFolder: join(__dirname, "../src/rpc"),
 };
 const { port, staticRoot } = config;
 
-const fastify = require("fastify")({ logger: false, forceCloseConnections: true });
+const fastify = Fastify({ logger: false, forceCloseConnections: true });
 
 const qyDB = new QyDB();
 
@@ -35,8 +36,8 @@ async function insertRpcs() {
   const { rpcFolder } = config;
   if (await folderOrFileExists(rpcFolder)) {
     await Promise.all(
-      (await fsPromises.readdir(rpcFolder)).map(async (x) => {
-        qyDB.insertRpc(path.basename(x, ".js"), (await fsPromises.readFile(path.join(rpcFolder, x))).toString());
+      (await readdir(rpcFolder)).map(async (x) => {
+        qyDB.insertRpc(basename(x, ".js"), (await readFile(join(rpcFolder, x))).toString());
       }),
     );
   }

package/scripts/flights.js

@@ -1,11 +1,11 @@
 // node --prof profile.js
 // node --prof-process isolate-0xnnnnnnnnnnnn-v8.log > processed.txt
-const path = require("node:path");
-const fs = require("node:fs");
-const { rm } = require("node:fs/promises");
-const { QyDB } = require("../src/QyDB.js");
-const { folderOrFileExists } = require("../src/QyUtils.js");
-const { runQueries } = require("./flights_queries.js");
+import { join } from "node:path";
+import { readFileSync } from "node:fs";
+import { rm } from "node:fs/promises";
+import { QyDB } from "../src/QyDB.js";
+import { folderOrFileExists } from "../src/QyUtils.js";
+import { runQueries } from "./flights_queries.js";
 
 logger.level = "info";
 
@@ -17,7 +17,7 @@ const qyDB = new QyDB(dataFolder, { preloadFoldersAtStart: false });
 run();
 
 function insertAirports() {
-    const data = JSON.parse(fs.readFileSync(path.join(arangoJsonFolderPath, "airports.json")));
+    const data = JSON.parse(readFileSync(join(arangoJsonFolderPath, "airports.json")));
     console.time("insertAirports");
     const { batch } = qyDB;
     for (const record of data) {
@@ -28,7 +28,7 @@ function insertAirports() {
 }
 
 function insertFlights() {
-    const data = JSON.parse(fs.readFileSync(path.join(arangoJsonFolderPath, "flights.json")));
+    const data = JSON.parse(readFileSync(join(arangoJsonFolderPath, "flights.json")));
     console.time("insertFlights");
     const { batch } = qyDB;
     for (let i = 0; i < data.length; ++i) {

package/scripts/flights2.js

@@ -1,22 +1,22 @@
 // node --prof profile.js
 // node --prof-process isolate-0xnnnnnnnnnnnn-v8.log > processed.txt
-const path = require("node:path");
-const fs = require("node:fs");
-const { rm } = require("node:fs/promises");
-const { QyDB } = require("../src/QyDB.js");
-const { folderOrFileExists } = require("../src/QyUtils.js");
-const { runQueries } = require("./flights2_queries.js");
+import { join } from "node:path";
+import { readFileSync } from "node:fs";
+import { rm } from "node:fs/promises";
+import { QyDB } from "../src/QyDB.js";
+import { folderOrFileExists } from "../src/QyUtils.js";
+import { runQueries } from "./flights2_queries.js";
 
 logger.level = "info";
 
-const arangoJsonFolderPath = "../GraphCourse_DemoData_ArangoDB-2";
+const arangoJsonFolderPath = "D:\\scripts\\GraphCourse_DemoData_ArangoDB-2";
 const dataFolder = "./tmp/flights2";
 const qyDB = new QyDB(dataFolder, { preloadFoldersAtStart: false });
 
 run();
 
 function insertAirports() {
-    const data = JSON.parse(fs.readFileSync(path.join(arangoJsonFolderPath, "airports.json")));
+    const data = JSON.parse(readFileSync(join(arangoJsonFolderPath, "airports.json")));
     console.time("insertAirports");
     const { batch } = qyDB;
     for (const record of data) {
@@ -28,7 +28,7 @@ function insertAirports() {
 }
 
 function insertFlights() {
-    const data = JSON.parse(fs.readFileSync(path.join(arangoJsonFolderPath, "flights.json")));
+    const data = JSON.parse(readFileSync(join(arangoJsonFolderPath, "flights.json")));
     console.time("insertFlights");
     const { batch } = qyDB;
     for (let i = 0; i < data.length; ++i) {