datafolder 1.0.0

package/README.md

@@ -0,0 +1,169 @@
+# datafolder
+
+datafolder is a highly efficient hierarchical virtual folder/file structured JSON database. It is implemented in pure native Node.js Javascript and has zero dependency, with full index and trigger features supported. The data can be organized into folders and sub folders, thus naturally partitioned. For example, airports, flights, orders can be stored as
+
+```
+airports/{country}/{state}/{airport_id} flights/{from}/{to}/{flight_id} orders/{year}/{month}/{day}/{order_id}
+```
+
+## Installation
+
+```javascript
+  npm install "datafolder"
+```
+
+## How to use
+
+```javascript
+const DataFolder = require("datafolder");
+const dataFolder = new DataFolder(); //Specify where to store the data. Default is ./data/sampleDataFolder
+await dataFolder.start();
+dataFolder.insert("users/2026/user_0001", {
+  userid: "user_0001",
+  email: "a@b.com",
+  score: 5,
+  address: { city: "a", zipcode: 123 },
+});
+dataFolder.insert("users/2026/user_0002", {
+  userid: "user_0002",
+  email: "c@b.com",
+  score: 3,
+  address: { city: "b", zipcode: 321 },
+});
+const { email, password, address, score } = dataFolder.view("users/2026/user_0002");
+address.zipcode = 678;
+dataFolder.insert("users/2026/user_0002", { score: score + 1, address });
+await dataFolder.stop();
+```
+
+Or operate with dir/file objects:
+
+```javascript
+const file = dataFolder.file("users/2026/user_0002");
+const { score, address } = file.view({ score: 1, address: 1 });
+dataFolder.insert(file, { score: score + 1, address: { city: "c", zipcode: 663 } });
+```
+
+## Features
+
+### 1. Simple CRUD APIs
+
+#### Create
+
+```javascript
+dataFolder.createFile("path/to/file");
+dataFolder.createDir("path/to/dir");
+dataFolder.insert("path/to/file", { email, address: { city, zipcode } });
+```
+
+#### Read
+
+```javascript
+const dir = dataFolder.dir("path/to/dir");
+const { subdirList, fileList, sortedSubdirList } = dir;
+const file = dataFolder.file("path/to/file");
+const { email, address } = dataFolder.view("path/to/file", { email: 1, addres: 1 });
+```
+
+#### Update
+
+```javascript
+dataFolder.insert("path/to/file", { email: newEmail, address: { zipcode: newZipcode } });
+dataFolder.insert(file, { score: score + 1 });
+```
+
+#### Delete
+
+```javascript
+dataFolder.delDir("path/to/dir");
+dataFolder.delFile("path/to/file");
+dataFolder.delContent("path/to/file");
+dataFolder.remove("path/to/file", { email: 1, address: { zipcode: 1 } });
+dataFolder.remove(file, { email: 1, address: { zipcode: 1 } });
+```
+
+### 2. Batch APIs
+
+```javascript
+const [dir, file, { email, score }] = dataFolder.fetch
+  .dir("path/to/dir")
+  .file("path/to/file")
+  .view("path/to/file")
+  .run();
+dataFolder.batch
+  .insert("path/to/file", { email: "email1@a.b.com" })
+  .insert("path/to/file", { email: "email2", score: 100 })
+  .remove("path/to/file", { address: { city: 1 } })
+  .run();
+```
+
+### 3. Powerful Query Support. Auto Create Needed Indexes
+
+```javascript
+dataFolder.queryFiles("flights/**", { _from: "airports/LAX" });
+dataFolder.queryFiles(["flights", "*", 1, (x) => x >= 5 && x <= 7], {
+  $or: [{ _from: "airports/BIS" }, { _to: "airports/BIS" }],
+});
+dataFolder.queryFilesMulti([
+  [["airbnb", "Loft", "Entire home/apt", "Real Bed"], { address: { street: "Porto, Porto, Portugal" } }],
+  ["airbnb/Loft/**", { minimum_nights: (x) => x >= 60 }],
+]);
+```
+
+### 4. Flexible Trigger Support to Monitor Any Changes
+
+```javascript
+function onEmailChanged({ studentId, email }) {
+  const { oldValue, newValue } = email;
+  // Logic to handle email change.
+}
+dataFolder.insertTrigger("emailChanged", "students/{studentId}", "email");
+dataFolder.on("emailChanged", onEmailChanged);
+dataFolder.insert("students/student1", { email: "email1" });
+dataFolder.insert("students/student2", { email: "email2" });
+dataFolder.insert("students/student1", { email: "email1_changed" });
+```
+
+Will trigger the following calls:
+
+```javascript
+onEmailChanged({ studentId: "student1", email: { oldValue: undefined, newValue: "email1" } });
+onEmailChanged({ studentId: "student2", email: { oldValue: undefined, newValue: "email2" } });
+onEmailChanged({ studentId: "student1", email: { oldValue: "email1", newValue: "email1_changed" } });
+```
+
+## Example
+
+Once you install datafolder via
+
+```bash
+npm insall datafolder
+```
+
+you can cd to node_modules/datafolder, then run
+
+```bash
+npm install
+```
+
+to install the dev dependency fastify. Then run
+
+```bash
+npm run airbnb
+```
+
+It will download the [AirBnB data](https://raw.githubusercontent.com/neelabalan/mongodb-sample-dataset/refs/heads/main/sample_airbnb/listingsAndReviews.json) (around 100MB) and insert into the following folder structure:
+
+```
+airbnb/{property_type}/{room_type}/{bed_type}/{_id}
+```
+
+Once the data is inserted, it will open the browser at [datafolderui](http://localhost:3628/datafolderui/index.html) and the whole folder structures can be navigated.
+
+Next time to bring up the UI:
+
+```bash
+npm run ui
+```
+
+Complex query examples can be found in the scripts folder source code.

package/index.js

@@ -0,0 +1,3 @@
+const { QyDB } = require("./src/QyDB.js");
+module.exports = QyDB;
+module.exports.default = QyDB;

package/package.json

@@ -0,0 +1,27 @@
+{
+	"name": "datafolder",
+	"version": "1.0.0",
+	"author": "lysh <1wopjdzn1@mozmail.com>",
+	"description": "datafolder is a highly efficient hierarchical virtual folder/file structured JSON database. It is implemented in pure native Node.js Javascript and has zero dependency, with full index and trigger features supported.",
+	"license": "MIT",
+	"keywords": [
+		"Hierarchical JSON Database",
+		"virtual folder/file structured",
+		"LSM Based Persistency",
+		"Pure Javascript",
+		"Zero Dependency",
+		"Highly Efficient"
+	],
+	"dependencies": {},
+	"devDependencies": {
+		"fastify": "^5.0.0",
+		"@fastify/compress": "^8.1.0",
+		"@fastify/express": "^4.0.2",
+		"@fastify/static": "^8.3.0"
+	},
+	"exports": "./index.js",
+	"scripts": {
+		"ui": "start http://localhost:3628/datafolderui/index.html && node ./scripts/fastify.js",
+		"airbnb": "node ./scripts/airbnb2.js && start http://localhost:3628/datafolderui/index.html && node ./scripts/fastify.js"
+	}
+}

package/sample.js

@@ -0,0 +1,37 @@
+const DataFolder = require("datafolder");
+const dataFolder = new DataFolder();
+
+run();
+
+async function run() {
+	await dataFolder.start();
+	dataFolder.insert("users/2026/user_0001", {
+		userid: "user_0001",
+		email: "a@b.com",
+		password: "123",
+		score: 5,
+		address: { city: "a", zipcode: 123 },
+	});
+	dataFolder.insert("users/2026/user_0002", {
+		userid: "user_0002",
+		email: "c@b.com",
+		password: "333",
+		score: 3,
+		address: { city: "b", zipcode: 321 },
+	});
+	const { email, password, address, score } = dataFolder.view("users/2026/user_0002");
+	logger.log(email, password, score, address);
+	address.zipcode = 678;
+	dataFolder.insert("users/2026/user_0002", { score: score + 1, address });
+	logger.log(dataFolder.view("users/2026/user_0002", ["score", "address"]));
+	await dataFolder.stop();
+	await dataFolder.start();
+	{
+		const file = dataFolder.file("users/2026/user_0002");
+		const { email, password, score } = file.view();
+		logger.log(email, password, score);
+		dataFolder.insert(file, { score: score + 1 });
+		logger.log(file.view());
+	}
+	await dataFolder.stop();
+}

package/scripts/airbnb.js

@@ -0,0 +1,73 @@
+const path = require("node:path");
+const fsPromises = require("node:fs/promises");
+const { QyDB } = require("../src/QyDB.js");
+const { folderOrFileExists, isSimpleType } = require("../src/QyUtils.js");
+const { runQueries } = require("./airbnb_queries.js");
+const { downloadFile } = require("./downloadFile.js");
+
+const fileName = "listingsAndReviews.json";
+const fileUrl = `https://raw.githubusercontent.com/neelabalan/mongodb-sample-dataset/refs/heads/main/sample_airbnb/${fileName}`;
+const airbnbJsonFilePath = path.join(__dirname, "../data/", fileName);
+const qyDB = new QyDB();
+const { rootFolder } = qyDB;
+
+run();
+async function insertData() {
+    console.time("insertData");
+    const { batch } = qyDB;
+    const fileHandle = await fsPromises.open(airbnbJsonFilePath, "r");
+    for await (const line of fileHandle.readLines()) {
+        const oneItem = convert(JSON.parse(line));
+        batch.insert(["airbnb1", oneItem._id], oneItem);
+    }
+    batch.run();
+    console.timeEnd("insertData");
+}
+async function run() {
+    try {
+        logger.log(`Downloading ${fileName}`);
+        await downloadFile(fileUrl, airbnbJsonFilePath);
+        console.log("done");
+    } catch (exception) {
+        logger.error(`Downloading ${fileUrl} failed:`, exception);
+        return;
+    }
+    console.time("total");
+    if (process.argv[3]) {
+        logger.level = process.argv[3];
+    }
+    let shouldInsertData = true;
+    if (process.argv[2] == "true") {
+        try {
+            await fsPromises.rm(rootFolder, { recursive: true });
+        } catch (exception) {}
+        shouldInsertData = true;
+    } else if (!(await folderOrFileExists(rootFolder))) shouldInsertData = true;
+    if (shouldInsertData) {
+        console.time("run");
+        await qyDB.start();
+        await insertData();
+        await qyDB.stop();
+        console.timeEnd("run");
+    }
+    console.time("queryWithStartEnd");
+    console.time("start");
+    await qyDB.start();
+    console.timeEnd("start");
+    runQueries(qyDB);
+    await qyDB.stop();
+    console.timeEnd("queryWithStartEnd");
+    console.timeEnd("total");
+}
+function convert(item) {
+    if (isSimpleType(item)) return item;
+    const { $date } = item;
+    if ($date != undefined) return new Date(+$date.$numberLong);
+    const number = item.$numberInt || item.$numberLong || item.$numberDecimal || item.$numberDouble;
+    if (number != undefined) return +number;
+    for (const prop in item) {
+        item[prop] = convert(item[prop]);
+    }
+    return item;
+}
+Object.assign(module.exports, { runQueries });

package/scripts/airbnb2.js

@@ -0,0 +1,74 @@
+const path = require("node:path");
+const fsPromises = require("node:fs/promises");
+const { QyDB } = require("../src/QyDB.js");
+const { folderOrFileExists, isSimpleType } = require("../src/QyUtils.js");
+const { runQueries } = require("./airbnb2_queries.js");
+const { downloadFile } = require("./downloadFile.js");
+
+const fileName = "listingsAndReviews.json";
+const fileUrl = `https://raw.githubusercontent.com/neelabalan/mongodb-sample-dataset/refs/heads/main/sample_airbnb/${fileName}`;
+const airbnbJsonFilePath = path.join(__dirname, "../data/", fileName);
+const qyDB = new QyDB();
+const { rootFolder } = qyDB;
+
+run();
+async function insertData() {
+    console.time("insertData");
+    const { batch } = qyDB;
+    const fileHandle = await fsPromises.open(airbnbJsonFilePath, "r");
+    for await (const line of fileHandle.readLines()) {
+        const oneItem = convert(JSON.parse(line));
+        const { property_type, room_type, bed_type, _id } = oneItem;
+        batch.insert(["airbnb", property_type, room_type, bed_type, _id], oneItem);
+    }
+    batch.run();
+    console.timeEnd("insertData");
+}
+async function run() {
+    try {
+        logger.log(`Downloading ${fileName}`);
+        await downloadFile(fileUrl, airbnbJsonFilePath);
+        console.log("done");
+    } catch (exception) {
+        logger.error(`Downloading ${fileUrl} failed:`, exception);
+        return;
+    }
+    console.time("total");
+    if (process.argv[3]) {
+        logger.level = process.argv[3];
+    }
+    let shouldInsertData = true;
+    if (process.argv[2] == "true") {
+        try {
+            await fsPromises.rm(rootFolder, { recursive: true });
+        } catch (exception) {}
+        shouldInsertData = true;
+    } else if (!(await folderOrFileExists(rootFolder))) shouldInsertData = true;
+    if (shouldInsertData) {
+        console.time("run");
+        await qyDB.start();
+        await insertData();
+        await qyDB.stop();
+        console.timeEnd("run");
+    }
+    console.time("queryWithStartEnd");
+    console.time("start");
+    await qyDB.start();
+    console.timeEnd("start");
+    runQueries(qyDB);
+    await qyDB.stop();
+    console.timeEnd("queryWithStartEnd");
+    console.timeEnd("total");
+}
+function convert(item) {
+    if (isSimpleType(item)) return item;
+    const { $date } = item;
+    if ($date != undefined) return new Date(+$date.$numberLong);
+    const number = item.$numberInt || item.$numberLong || item.$numberDecimal || item.$numberDouble;
+    if (number != undefined) return +number;
+    for (const prop in item) {
+        item[prop] = convert(item[prop]);
+    }
+    return item;
+}
+Object.assign(module.exports, { runQueries });

package/scripts/airbnb2_queries.js

@@ -0,0 +1,231 @@
+const assert = require("node:assert/strict");
+
+function query1(qyDB) {
+    console.time("query1");
+    assert.deepEqual(
+        qyDB
+            .queryFiles(["airbnb", "Loft", "Entire home/apt", "Real Bed"], {
+                address: { street: "Porto, Porto, Portugal" },
+            })
+            .map((x) => x.view(["name"]).name),
+        [
+            "Porto city centre apartment H",
+            "Romantic loft near historic center",
+            "Lovely sun light  in a stylish loft, Downtown Spot",
+            "Charming & Sunny Mezzanine Loft",
+            "Be Happy in Porto",
+            "Quarry Apartment",
+            "Loft with balcony and view of Casa da Música PORTO",
+            "For 4 or 6 - Wonderful Apartment downtown",
+            "Clérigos Apartments - Apt. 4E for 2/3 Pax",
+            "Bright Top Floor Two Rooms Apartment w/ Terrace",
+            "ReUrban Flats - Santa Catarina #3",
+            "The Blue and White Porto House - Vintage",
+            "4twoLoft at Great Location",
+            "Step In Porto -  Loft w/Mezzanine",
+            "Your OWM Invicta",
+            "Pury Apartments",
+            "Live Porto & Douro - Breiner III",
+        ],
+    );
+    console.timeEnd("query1");
+}
+function query2(qyDB) {
+    console.time("query2");
+    assert.deepEqual(
+        qyDB
+            .queryFiles(
+                ["airbnb", "Loft", "Entire home/apt", "Real Bed"],
+                {
+                    address: { street: "Porto, Porto, Portugal" },
+                },
+                9,
+            )
+            .map((x) => x.view(["name"]).name),
+        [
+            "Porto city centre apartment H",
+            "Romantic loft near historic center",
+            "Lovely sun light  in a stylish loft, Downtown Spot",
+            "Charming & Sunny Mezzanine Loft",
+            "Be Happy in Porto",
+            "Quarry Apartment",
+            "Loft with balcony and view of Casa da Música PORTO",
+            "For 4 or 6 - Wonderful Apartment downtown",
+            "Clérigos Apartments - Apt. 4E for 2/3 Pax",
+        ],
+    );
+    console.timeEnd("query2");
+}
+function query3(qyDB) {
+    console.time("query3");
+    assert.deepEqual(
+        qyDB
+            .queryFiles(
+                ["airbnb", "Loft", "Entire home/apt", "Real Bed"],
+                { address: { street: "Porto, Porto, Portugal" } },
+                9,
+                9,
+            )
+            .map((x) => x.view(["name"]).name),
+        [
+            "Bright Top Floor Two Rooms Apartment w/ Terrace",
+            "ReUrban Flats - Santa Catarina #3",
+            "The Blue and White Porto House - Vintage",
+            "4twoLoft at Great Location",
+            "Step In Porto -  Loft w/Mezzanine",
+            "Your OWM Invicta",
+            "Pury Apartments",
+            "Live Porto & Douro - Breiner III",
+        ],
+    );
+    console.timeEnd("query3");
+}
+function query4(qyDB) {
+    console.time("query4");
+    assert.deepEqual(
+        qyDB.fetch
+            .queryFilesMulti([
+                [["airbnb", "Loft", "Entire home/apt", "Real Bed"], { address: { street: "Porto, Porto, Portugal" } }],
+                ["airbnb/Loft/**", { minimum_nights: (x) => x >= 60 }],
+            ])
+            .run()[0]
+            .map((x) => x.view(["name"]).name)
+            .sort(),
+        [
+            "Porto city centre apartment H",
+            "Romantic loft near historic center",
+            "Lovely sun light  in a stylish loft, Downtown Spot",
+            "Charming & Sunny Mezzanine Loft",
+            "Brand New Luxury Downtown Studio",
+            "Be Happy in Porto",
+            "Quarry Apartment",
+            "Loft with balcony and view of Casa da Música PORTO",
+            "For 4 or 6 - Wonderful Apartment downtown",
+            "Clérigos Apartments - Apt. 4E for 2/3 Pax",
+            "Bright Top Floor Two Rooms Apartment w/ Terrace",
+            "ReUrban Flats - Santa Catarina #3",
+            "The Blue and White Porto House - Vintage",
+            "4twoLoft at Great Location",
+            "Step In Porto -  Loft w/Mezzanine",
+            "Your OWM Invicta",
+            "Pury Apartments",
+            "Live Porto & Douro - Breiner III",
+        ].sort(),
+    );
+    console.timeEnd("query4");
+}
+function query5(qyDB) {
+    console.time("query5");
+    assert.deepEqual(
+        qyDB
+            .queryFilesMulti(
+                [
+                    [
+                        ["airbnb", "Loft", "Entire home/apt", "Real Bed"],
+                        { address: { street: "Porto, Porto, Portugal" } },
+                    ],
+                    ["airbnb/Loft/**", { minimum_nights: (x) => x >= 60 }],
+                ],
+                3,
+            )
+            .map((x) => x.view(["name"]).name),
+        [
+            "Porto city centre apartment H",
+            "Romantic loft near historic center",
+            "Lovely sun light  in a stylish loft, Downtown Spot",
+        ],
+    );
+    console.timeEnd("query5");
+}
+function query6(qyDB) {
+    console.time("query6");
+    assert.deepEqual(
+        qyDB
+            .queryFilesMulti(
+                [
+                    [
+                        ["airbnb", "Loft", "Entire home/apt", "Real Bed"],
+                        { address: { street: "Porto, Porto, Portugal" } },
+                    ],
+                    ["airbnb/Loft/**", { minimum_nights: (x) => x >= 60 }],
+                ],
+                20,
+                10,
+            )
+            .map((x) => x.view(["name"]).name),
+        [
+            "ReUrban Flats - Santa Catarina #3",
+            "The Blue and White Porto House - Vintage",
+            "4twoLoft at Great Location",
+            "Step In Porto -  Loft w/Mezzanine",
+            "Your OWM Invicta",
+            "Pury Apartments",
+            "Live Porto & Douro - Breiner III",
+            "Brand New Luxury Downtown Studio",
+        ],
+    );
+    console.timeEnd("query6");
+}
+function query7(qyDB) {
+    console.time("query7");
+    assert.deepEqual(
+        qyDB.queryFiles(["airbnb", "Loft", "**"], { minimum_nights: (x) => x >= 60 }).map((x) => x.view(["name"]).name),
+        ["Romantic loft near historic center", "Brand New Luxury Downtown Studio"],
+    );
+    console.timeEnd("query7");
+}
+function query8(qyDB) {
+    console.time("query8");
+    assert.deepEqual(
+        qyDB
+            .queryFiles(["airbnb", "Loft", ""], { minimum_nights: (x) => x >= 60 }, 10, 10)
+            .map((x) => x.view(["name"]).name),
+        [],
+    );
+    console.timeEnd("query8");
+}
+function query9(qyDB) {
+    console.time("query9");
+    assert.deepEqual(
+        qyDB
+            .queryFilesMulti(
+                [
+                    [
+                        ["airbnb", "Loft", "Entire home/apt", "Real Bed"],
+                        { address: { street: "Porto, Porto, Portugal" } },
+                    ],
+                    ["airbnb/Loft/**", { minimum_nights: (x) => x >= 60 }],
+                ],
+                3,
+            )
+            .map((x) => x.view(["name"]).name),
+        [
+            "Porto city centre apartment H",
+            "Romantic loft near historic center",
+            "Lovely sun light  in a stylish loft, Downtown Spot",
+        ],
+    );
+    console.timeEnd("query9");
+}
+function runQueries(qyDB) {
+    query1(qyDB);
+    query1(qyDB);
+    query2(qyDB);
+    query2(qyDB);
+    query3(qyDB);
+    query3(qyDB);
+    query4(qyDB);
+    query4(qyDB);
+    query5(qyDB);
+    query5(qyDB);
+    query6(qyDB);
+    query6(qyDB);
+    query7(qyDB);
+    query7(qyDB);
+    query8(qyDB);
+    query8(qyDB);
+    query9(qyDB);
+    query9(qyDB);
+}
+
+Object.assign(module.exports, { runQueries });