datafolder 1.0.7 → 1.0.9

package/README.md

@@ -1,10 +1,13 @@
 # datafolder
 
-datafolder is a lightweight JSON database, featuring an AI-friendly API for simple and human maintainable code generation, millisecond data response, with full index and trigger features supported, and the unique ability to browse and manage data as a folder tree directly in your browser. It works perfectly for a wide range of scenarios, from game/application backends to embedded systems to IoT to edge computing.
+datafolder is a lightning fluid JSON database for SSD, featuring an AI-friendly API for simple and human maintainable code generation, millisecond data response, with full index and trigger features supported, and the unique ability to browse and manage data as a folder tree directly in your browser. It works perfectly for a wide range of scenarios, from game/application backends to embedded systems to IoT to edge computing.
 
 ### Changelog
 
+1.0.9 Added Typescript type declarations.
+
 1.0.7 Added trigger editor and [internal.md](https://github.com/datafolderdev/datafolder/wiki/Internals-of-datafolder)
+
 1.0.6 Added code mirror as the query editor.
 
 ## Installation

package/index.js

@@ -1 +1 @@
-export { QyDB as default } from "./src/QyDB.js";
+export { QyDB as default } from "./src/QyDB.js";

package/internal.md

@@ -24,7 +24,7 @@ Airports can be stored as
 airports/{country}/{state}/{airport_id}
 ```
 
-When you conceive the data model, you don't need to think about tables, schemas, indexes etc. All you need to know is the nature of the data and then goes with the most comfortable one for your mental mind.
+When you conceive the data model, you don't need to think about tables, schemas, indexes etc. All you need to know is the nature of the data and then go with the most comfortable one for your mental mind.
 
 ### In-memory Tree Structure
 
@@ -56,7 +56,7 @@ JSON changes are further separated into the following:
 2. Adding a property with value.
 3. Changing a property to new value.
 
-These deltas are appended to the acl log file in sequence. Once the log file reaches a certain size a new log file will be created and appended to. Storing these acl1, acl2, ...,acln files keep the change history of the whole database. The whole data can be reconstructed by replying these acl files in sequence.
+These deltas are appended to the acl log file in sequence. Once the log file reaches a certain size a new log file will be created and appended to. Storing these acl1, acl2, ...,acln files keep the change history of the whole database. The whole data can be reconstructed by re-applying these acl files in sequence.
 
 ### Snapshots
 
@@ -114,6 +114,8 @@ hackernews/{year}/{month}/{day}/{id}
 
 And each file contains the full story and the nested comments in a Javascript object. Even with 22GB contents, since they are partitioned naturally and each folder only contains a few thousands files they can be read via datafolder UI directly within the browser.
 
-### Snapshot Shadowing and Compaction
+<img width="100%" alt="hackernews" src="https://github.com/user-attachments/assets/c7788426-9079-4628-b82f-c1bbff70f6ed" />
+
+### Snapshot Shadowing and compactions
 
 The same key with updated values will be stored in multiple snapshots and the latest one shadows all previous ones. At some point, thes redundant key/values will take up too much disc space. datafolder can automatically detect the threshold, and save the snapshots into new compacted ones with these redundances removed. These compactions are done in different processes and won't use the main process's time.

package/package.json

@@ -1,11 +1,11 @@
 {
 	"name": "datafolder",
-	"version": "1.0.7",
+	"version": "1.0.9",
 	"author": {
 		"name": "lysh",
 		"email": "datafolderdev@outlook.com"
 	},
-	"description": "datafolder is a lightweight JSON database, featuring an AI-friendly API for simple and human maintainable code generation, millisecond data response, with full index and trigger features supported, and the unique ability to browse and manage data as a folder tree directly in your browser. It works perfectly for a wide range of scenarios, from game/application backends to embedded systems to IoT to edge computing.",
+	"description": "datafolder is a lightning fluid JSON database for SSD, featuring an AI-friendly API for simple and human maintainable code generation, millisecond data response, with full index and trigger features supported, and the unique ability to browse and manage data as a folder tree directly in your browser. It works perfectly for a wide range of scenarios, from game/application backends to embedded systems to IoT to edge computing.",
 	"license": "MIT",
 	"repository": {
 		"type": "git",
@@ -42,15 +42,17 @@
 	],
 	"dependencies": {},
 	"devDependencies": {
-		"fastify": "^5.0.0",
 		"@fastify/compress": "^8.1.0",
 		"@fastify/express": "^4.0.2",
-		"@fastify/static": "^8.3.0"
+		"@fastify/static": "^8.3.0",
+		"fastify": "^5.0.0"
 	},
 	"exports": "./index.js",
+	"types": "./src/QyDB.d.ts",
 	"scripts": {
 		"ui": "start http://localhost:3628/datafolderui/index.html && node ./scripts/fastify.js",
-		"airbnb": "node ./scripts/airbnb2.js && start http://localhost:3628/datafolderui/index.html && node ./scripts/fastify.js"
+		"airbnb": "node ./scripts/airbnb2.js && start http://localhost:3628/datafolderui/index.html && node ./scripts/fastify.js",
+		"types:check": "tsc --noEmit"
 	},
 	"type": "module"
 }

package/scripts/airbnb.js

@@ -1,72 +1,77 @@
-import { join } from "node:path";
-import { open, rm } from "node:fs/promises";
-import { QyDB } from "../src/QyDB.js";
-import { folderOrFileExists, isSimpleType, __dirname } from "../src/QyUtils.js";
-import { runQueries } from "./airbnb_queries.js";
-import { downloadFile } from "./downloadFile.js";
-
-const fileName = "listingsAndReviews.json";
-const fileUrl = `https://raw.githubusercontent.com/neelabalan/mongodb-sample-dataset/refs/heads/main/sample_airbnb/${fileName}`;
-const airbnbJsonFilePath = join(__dirname, "../data/", fileName);
-const qyDB = new QyDB();
-const { rootFolder } = qyDB;
-
-run();
-async function insertData() {
-    console.time("insertData");
-    const { batch } = qyDB;
-    const fileHandle = await open(airbnbJsonFilePath, "r");
-    for await (const line of fileHandle.readLines()) {
-        const oneItem = convert(JSON.parse(line));
-        batch.insert(["airbnb1", oneItem._id], oneItem);
-    }
-    batch.run();
-    console.timeEnd("insertData");
-}
-async function run() {
-    try {
-        logger.log(`Downloading ${fileName}`);
-        await downloadFile(fileUrl, airbnbJsonFilePath);
-        console.log("done");
-    } catch (exception) {
-        logger.error(`Downloading ${fileUrl} failed:`, exception);
-        return;
-    }
-    console.time("total");
-    if (process.argv[3]) {
-        logger.level = process.argv[3];
-    }
-    let shouldInsertData = true;
-    if (process.argv[2] == "true") {
-        try {
-            await rm(rootFolder, { recursive: true });
-        } catch (exception) {}
-        shouldInsertData = true;
-    } else if (!(await folderOrFileExists(rootFolder))) shouldInsertData = true;
-    if (shouldInsertData) {
-        console.time("run");
-        await qyDB.start();
-        await insertData();
-        await qyDB.stop();
-        console.timeEnd("run");
-    }
-    console.time("queryWithStartEnd");
-    console.time("start");
-    await qyDB.start();
-    console.timeEnd("start");
-    runQueries(qyDB);
-    await qyDB.stop();
-    console.timeEnd("queryWithStartEnd");
-    console.timeEnd("total");
-}
-function convert(item) {
-    if (isSimpleType(item)) return item;
-    const { $date } = item;
-    if ($date != undefined) return new Date(+$date.$numberLong);
-    const number = item.$numberInt || item.$numberLong || item.$numberDecimal || item.$numberDouble;
-    if (number != undefined) return +number;
-    for (const prop in item) {
-        item[prop] = convert(item[prop]);
-    }
-    return item;
-}
+import { join } from "node:path";
+import { open, rm } from "node:fs/promises";
+import { QyDB } from "../src/QyDB.js";
+import { folderOrFileExists, isSimpleType, __dirname } from "../src/QyUtils.js";
+import { runQueries } from "./airbnb_queries.js";
+import { downloadFile } from "./downloadFile.js";
+const fileName = "listingsAndReviews.json";
+const fileUrl = `https://raw.githubusercontent.com/neelabalan/mongodb-sample-dataset/refs/heads/main/sample_airbnb/${fileName}`;
+const airbnbJsonFilePath = join(__dirname, "../data/", fileName);
+const qyDB = new QyDB();
+const { rootFolder } = qyDB;
+run();
+async function insertData() {
+    console.time("insertData");
+    const { batch } = qyDB;
+    const fileHandle = await open(airbnbJsonFilePath, "r");
+    for await (const line of fileHandle.readLines()) {
+        const oneItem = convert(JSON.parse(line));
+        batch.insert(["airbnb1", oneItem._id], oneItem);
+    }
+    batch.run();
+    console.timeEnd("insertData");
+}
+async function run() {
+    try {
+        logger.log(`Downloading ${fileName}`);
+        await downloadFile(fileUrl, airbnbJsonFilePath);
+        console.log("done");
+    }
+    catch (exception) {
+        logger.error(`Downloading ${fileUrl} failed:`, exception);
+        return;
+    }
+    console.time("total");
+    if (process.argv[3]) {
+        logger.level = process.argv[3];
+    }
+    let shouldInsertData = true;
+    if (process.argv[2] == "true") {
+        try {
+            await rm(rootFolder, { recursive: true });
+        }
+        catch (exception) { }
+        shouldInsertData = true;
+    }
+    else if (!(await folderOrFileExists(rootFolder)))
+        shouldInsertData = true;
+    if (shouldInsertData) {
+        console.time("run");
+        await qyDB.start();
+        await insertData();
+        await qyDB.stop();
+        console.timeEnd("run");
+    }
+    console.time("queryWithStartEnd");
+    console.time("start");
+    await qyDB.start();
+    console.timeEnd("start");
+    runQueries(qyDB);
+    await qyDB.stop();
+    console.timeEnd("queryWithStartEnd");
+    console.timeEnd("total");
+}
+function convert(item) {
+    if (isSimpleType(item))
+        return item;
+    const { $date } = item;
+    if ($date != undefined)
+        return new Date(+$date.$numberLong);
+    const number = item.$numberInt || item.$numberLong || item.$numberDecimal || item.$numberDouble;
+    if (number != undefined)
+        return +number;
+    for (const prop in item) {
+        item[prop] = convert(item[prop]);
+    }
+    return item;
+}

package/scripts/airbnb2.js

@@ -1,73 +1,78 @@
-import { join } from "node:path";
-import { open, rm } from "node:fs/promises";
-import { QyDB } from "../src/QyDB.js";
-import { folderOrFileExists, isSimpleType, __dirname } from "../src/QyUtils.js";
-import { runQueries } from "./airbnb2_queries.js";
-import { downloadFile } from "./downloadFile.js";
-
-const fileName = "listingsAndReviews.json";
-const fileUrl = `https://raw.githubusercontent.com/neelabalan/mongodb-sample-dataset/refs/heads/main/sample_airbnb/${fileName}`;
-const airbnbJsonFilePath = join(__dirname, "../data/", fileName);
-const qyDB = new QyDB();
-const { rootFolder } = qyDB;
-
-run();
-async function insertData() {
-    console.time("insertData");
-    const { batch } = qyDB;
-    const fileHandle = await open(airbnbJsonFilePath, "r");
-    for await (const line of fileHandle.readLines()) {
-        const oneItem = convert(JSON.parse(line));
-        const { property_type, room_type, bed_type, _id } = oneItem;
-        batch.insert(["airbnb", property_type, room_type, bed_type, _id], oneItem);
-    }
-    batch.run();
-    console.timeEnd("insertData");
-}
-async function run() {
-    try {
-        logger.log(`Downloading ${fileName}`);
-        await downloadFile(fileUrl, airbnbJsonFilePath);
-        console.log("done");
-    } catch (exception) {
-        logger.error(`Downloading ${fileUrl} failed:`, exception);
-        return;
-    }
-    console.time("total");
-    if (process.argv[3]) {
-        logger.level = process.argv[3];
-    }
-    let shouldInsertData = true;
-    if (process.argv[2] == "true") {
-        try {
-            await rm(rootFolder, { recursive: true });
-        } catch (exception) {}
-        shouldInsertData = true;
-    } else if (!(await folderOrFileExists(rootFolder))) shouldInsertData = true;
-    if (shouldInsertData) {
-        console.time("run");
-        await qyDB.start();
-        await insertData();
-        await qyDB.stop();
-        console.timeEnd("run");
-    }
-    console.time("queryWithStartEnd");
-    console.time("start");
-    await qyDB.start();
-    console.timeEnd("start");
-    runQueries(qyDB);
-    await qyDB.stop();
-    console.timeEnd("queryWithStartEnd");
-    console.timeEnd("total");
-}
-function convert(item) {
-    if (isSimpleType(item)) return item;
-    const { $date } = item;
-    if ($date != undefined) return new Date(+$date.$numberLong);
-    const number = item.$numberInt || item.$numberLong || item.$numberDecimal || item.$numberDouble;
-    if (number != undefined) return +number;
-    for (const prop in item) {
-        item[prop] = convert(item[prop]);
-    }
-    return item;
-}
+import { join } from "node:path";
+import { open, rm } from "node:fs/promises";
+import { QyDB } from "../src/QyDB.js";
+import { folderOrFileExists, isSimpleType, __dirname } from "../src/QyUtils.js";
+import { runQueries } from "./airbnb2_queries.js";
+import { downloadFile } from "./downloadFile.js";
+const fileName = "listingsAndReviews.json";
+const fileUrl = `https://raw.githubusercontent.com/neelabalan/mongodb-sample-dataset/refs/heads/main/sample_airbnb/${fileName}`;
+const airbnbJsonFilePath = join(__dirname, "../data/", fileName);
+const qyDB = new QyDB();
+const { rootFolder } = qyDB;
+run();
+async function insertData() {
+    console.time("insertData");
+    const { batch } = qyDB;
+    const fileHandle = await open(airbnbJsonFilePath, "r");
+    for await (const line of fileHandle.readLines()) {
+        const oneItem = convert(JSON.parse(line));
+        const { property_type, room_type, bed_type, _id } = oneItem;
+        batch.insert(["airbnb", property_type, room_type, bed_type, _id], oneItem);
+    }
+    batch.run();
+    console.timeEnd("insertData");
+}
+async function run() {
+    try {
+        logger.log(`Downloading ${fileName}`);
+        await downloadFile(fileUrl, airbnbJsonFilePath);
+        console.log("done");
+    }
+    catch (exception) {
+        logger.error(`Downloading ${fileUrl} failed:`, exception);
+        return;
+    }
+    console.time("total");
+    if (process.argv[3]) {
+        logger.level = process.argv[3];
+    }
+    let shouldInsertData = true;
+    if (process.argv[2] == "true") {
+        try {
+            await rm(rootFolder, { recursive: true });
+        }
+        catch (exception) { }
+        shouldInsertData = true;
+    }
+    else if (!(await folderOrFileExists(rootFolder)))
+        shouldInsertData = true;
+    if (shouldInsertData) {
+        console.time("run");
+        await qyDB.start();
+        await insertData();
+        await qyDB.stop();
+        console.timeEnd("run");
+    }
+    console.time("queryWithStartEnd");
+    console.time("start");
+    await qyDB.start();
+    console.timeEnd("start");
+    runQueries(qyDB);
+    await qyDB.stop();
+    console.timeEnd("queryWithStartEnd");
+    console.timeEnd("total");
+}
+function convert(item) {
+    if (isSimpleType(item))
+        return item;
+    const { $date } = item;
+    if ($date != undefined)
+        return new Date(+$date.$numberLong);
+    const number = item.$numberInt || item.$numberLong || item.$numberDecimal || item.$numberDouble;
+    if (number != undefined)
+        return +number;
+    for (const prop in item) {
+        item[prop] = convert(item[prop]);
+    }
+    return item;
+}