datafolder 1.0.9 → 1.0.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/{internal.md → Internal.md} +8 -1
- package/README.md +6 -2
- package/SKILL.md +191 -0
- package/SSD.md +60 -0
- package/index.js +1 -0
- package/package.json +2 -2
- package/sample.js +4 -4
- package/scripts/airbnb.js +1 -0
- package/scripts/airbnb2.js +1 -0
- package/scripts/convertBinFiles.js +1 -0
- package/scripts/express.js +2 -1
- package/scripts/fastify.js +1 -0
- package/scripts/flights.js +1 -0
- package/scripts/flights2.js +1 -0
- package/scripts/flights_queries.js +1 -0
- package/src/QyAcl.d.ts +3 -3
- package/src/QyAcl.js +7 -7
- package/src/QyAclSaver_Worker.d.ts +1 -0
- package/src/QyAclSaver_Worker.js +1 -0
- package/src/QyAclWriter.d.ts +2 -1
- package/src/QyAclWriter.js +7 -6
- package/src/QyBatch.d.ts +29 -22
- package/src/QyBatch.js +85 -72
- package/src/QyBinWriter.d.ts +9 -4
- package/src/QyBinWriter.js +83 -70
- package/src/QyCD.d.ts +19 -19
- package/src/QyCD.js +2 -2
- package/src/QyCache.d.ts +22 -23
- package/src/QyCache.js +47 -36
- package/src/QyCompressor_Worker.js +2 -2
- package/src/QyDB.d.ts +6 -9
- package/src/QyDB.js +1 -1
- package/src/QyFile.d.ts +2 -0
- package/src/QyFile.js +42 -40
- package/src/QyFileContentUtils.d.ts +1 -4
- package/src/QyFileLogger.d.ts +4 -0
- package/src/QyFileLogger.js +30 -26
- package/src/QyIndex.d.ts +2 -2
- package/src/QyIndex.js +84 -80
- package/src/QyIndexPropTree.d.ts +4 -0
- package/src/QyIndexPropTree.js +19 -10
- package/src/QyIntersectNode.d.ts +1 -0
- package/src/QyIntersectNode.js +1 -0
- package/src/QyJsonSaver.d.ts +2 -2
- package/src/QyJsonSaver.js +4 -4
- package/src/QyKVData.d.ts +18 -1
- package/src/QyKVData.js +73 -53
- package/src/QyKVDataCleaner_Worker.d.ts +7 -1
- package/src/QyKVDataCleaner_Worker.js +54 -40
- package/src/QyKVDataLoader.d.ts +10 -2
- package/src/QyKVDataLoader.js +27 -21
- package/src/QyKVDataSaver_Worker.d.ts +10 -0
- package/src/QyKVDataSaver_Worker.js +23 -9
- package/src/QyLogFileSaver_Worker.d.ts +1 -0
- package/src/QyLogFileSaver_Worker.js +7 -6
- package/src/QyLogger.d.ts +21 -10
- package/src/QyLogger.js +111 -67
- package/src/QyMessageWorker.d.ts +1 -1
- package/src/QyMessageWorker.js +5 -5
- package/src/QyMessager.d.ts +5 -1
- package/src/QyMessager.js +6 -1
- package/src/QyMessager_Run.js +4 -3
- package/src/QyMod.js +9 -8
- package/src/QyOpRunner.d.ts +5 -1
- package/src/QyOpRunner.js +7 -7
- package/src/QyQueue.d.ts +5 -1
- package/src/QyQueue.js +6 -7
- package/src/QySaver.d.ts +3 -0
- package/src/QySaver.js +26 -15
- package/src/QySnapshotCompactor_Worker.d.ts +7 -0
- package/src/QySnapshotCompactor_Worker.js +37 -26
- package/src/QySnapshotSaver_Worker.d.ts +8 -1
- package/src/QySnapshotSaver_Worker.js +21 -14
- package/src/QySnapshots.d.ts +19 -7
- package/src/QySnapshots.js +204 -181
- package/src/QyTrigger.d.ts +5 -1
- package/src/QyTrigger.js +44 -34
- package/src/QyUnionNode.d.ts +10 -1
- package/src/QyUnionNode.js +34 -36
- package/src/QyUtils.d.ts +2 -2
- package/src/QyUtils.js +116 -111
- package/src/simulate.js +34 -31
- package/src/simulate2.js +49 -46
- package/src/soc_pokec.js +17 -11
- package/static/codemirror.bundle/editor.bundle.js +2 -0
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
The best mechanism to index and partition data is **by the nature of data**. Instead of forcing data to transform, it should seamlessly embrace and fully cater to the **uniqueness** of each and every data to accomplish nature-level efficiency, **mentally and mechanically**. Whatever data is, it follows like fluid.
|
|
4
4
|
|
|
5
|
-
This is the **design philosophy** behind datafolder, a
|
|
5
|
+
This is the **design philosophy** behind datafolder, a lightning fluid schemaless JSON database, featuring an AI-friendly API for simple and human maintainable code generation, millisecond data response, with full index and trigger features supported, and the unique ability to browse and manage data as a folder tree directly in your browser. Data can be organized naturally into folders, sub folders and files. It works perfectly for a wide range of scenarios, from game/application backends to embedded systems to IoT to edge computing.
|
|
6
6
|
|
|
7
7
|
### User Level Data Layout
|
|
8
8
|
|
|
@@ -24,6 +24,13 @@ Airports can be stored as
|
|
|
24
24
|
airports/{country}/{state}/{airport_id}
|
|
25
25
|
```
|
|
26
26
|
|
|
27
|
+
AI training data can be stored as
|
|
28
|
+
|
|
29
|
+
```
|
|
30
|
+
AITraining/{year}/{month}/{day}/chat_id/prompt
|
|
31
|
+
/answer
|
|
32
|
+
```
|
|
33
|
+
|
|
27
34
|
When you conceive the data model, you don't need to think about tables, schemas, indexes etc. All you need to know is the nature of the data and then go with the most comfortable one for your mental mind.
|
|
28
35
|
|
|
29
36
|
### In-memory Tree Structure
|
package/README.md
CHANGED
|
@@ -1,12 +1,16 @@
|
|
|
1
1
|
# datafolder
|
|
2
2
|
|
|
3
|
-
datafolder is a lightning fluid JSON database for
|
|
3
|
+
datafolder is a lightning fluid JSON database for high speed SSDs, featuring an AI-friendly API for simple and human maintainable code generation, millisecond data response, with full index and trigger features supported, and the unique ability to browse and manage data as a folder tree directly in your browser. It works perfectly for a wide range of scenarios, from game/application backends to embedded systems to IoT to edge computing.
|
|
4
4
|
|
|
5
5
|
### Changelog
|
|
6
6
|
|
|
7
|
+
1.0.11 Added [SKILL.md](https://github.com/datafolderdev/datafolder/blob/main/SKILL.md) for feeding to AI
|
|
8
|
+
|
|
9
|
+
1.0.10 Added document [datafolder for high speed SSDs](https://github.com/datafolderdev/datafolder/wiki/datafolder-for-high-speed-SSDs)
|
|
10
|
+
|
|
7
11
|
1.0.9 Added Typescript type declarations.
|
|
8
12
|
|
|
9
|
-
1.0.7 Added trigger editor and [
|
|
13
|
+
1.0.7 Added trigger editor and [Internals of datafolder](https://github.com/datafolderdev/datafolder/wiki/Internals-of-datafolder)
|
|
10
14
|
|
|
11
15
|
1.0.6 Added code mirror as the query editor.
|
|
12
16
|
|
package/SKILL.md
ADDED
|
@@ -0,0 +1,191 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: datafolder-builder
|
|
3
|
+
description: >
|
|
4
|
+
Use this skill when building applications on top of datafolder — a lightning fluid JSON database for high speed SSDs, featuring an AI-friendly API for simple and human maintainable code generation, millisecond data response, with full index and trigger features supported, and the unique ability to browse and manage data as a folder tree directly in your browser. Data can be organized naturally into folders, sub folders and files. It works perfectly for a wide range of scenarios, from game/application backends to embedded systems to IoT to edge computing. Covers installation, basic usages, CRUD, batch APIs, powerful query support which auto creates needed indexes, flexible trigger support to monitor any changes.
|
|
5
|
+
Trigger when a user asks to build, query, or operate with datafolder, write application code against it, or understand how to structure data and queries.
|
|
6
|
+
---
|
|
7
|
+
|
|
8
|
+
# datafolder Builder Skill
|
|
9
|
+
|
|
10
|
+
## What Is datafolder?
|
|
11
|
+
|
|
12
|
+
The best mechanism to index and partition data is **by the nature of data**. Instead of forcing data to transform, it should seamlessly embrace and fully cater to the **uniqueness** of each and every data to accomplish nature-level efficiency, **mentally and mechanically**. Whatever data is, it follows like fluid.
|
|
13
|
+
|
|
14
|
+
This is the **design philosophy** behind datafolder, a lightning fluid schemaless JSON database, featuring an AI-friendly API for simple and human maintainable code generation, millisecond data response, with full index and trigger features supported, and the unique ability to browse and manage data as a folder tree directly in your browser. Data can be organized naturally into folders, sub folders and files. It works perfectly for a wide range of scenarios, from game/application backends to embedded systems to IoT to edge computing.
|
|
15
|
+
|
|
16
|
+
## Features
|
|
17
|
+
|
|
18
|
+
🤖 AI-Friendly, Simple API: Designed for easy code generation by AI and long-term maintainability by humans.
|
|
19
|
+
|
|
20
|
+
⚡ Extreme Data Access Speed: Supports fully in-memory operation for millisecond response, with configurable persistence to high-speed SSD.
|
|
21
|
+
|
|
22
|
+
🌐 Built-in Browser Management: Visually browse and manage your data as a folder hierarchy directly within the browser, no separate tools needed.
|
|
23
|
+
|
|
24
|
+
🧩 Plain JavaScript: Built with plain JavaScript, offers full TypeScript support and safe atomic writes.
|
|
25
|
+
|
|
26
|
+
🔧 Designed for application/game backends, IoT, and edge computing scenarios, easily deployable on servers, in browsers, and even on embedded hardware.
|
|
27
|
+
|
|
28
|
+
## User Level Data Layout
|
|
29
|
+
|
|
30
|
+
Data can be organized naturally into folders, sub folders and files. For example, Orders can be stored in
|
|
31
|
+
|
|
32
|
+
```
|
|
33
|
+
orders/{year}/{month}/{day}/{order_id}
|
|
34
|
+
```
|
|
35
|
+
|
|
36
|
+
Flights can be stored as
|
|
37
|
+
|
|
38
|
+
```
|
|
39
|
+
flights/{from}/{to}/{flight_id}
|
|
40
|
+
```
|
|
41
|
+
|
|
42
|
+
Airports can be stored as
|
|
43
|
+
|
|
44
|
+
```
|
|
45
|
+
airports/{country}/{state}/{airport_id}
|
|
46
|
+
```
|
|
47
|
+
|
|
48
|
+
AI training data can be stored as
|
|
49
|
+
|
|
50
|
+
```
|
|
51
|
+
AITraining/{year}/{month}/{day}/chat_id/prompt
|
|
52
|
+
/answer
|
|
53
|
+
```
|
|
54
|
+
|
|
55
|
+
When you conceive the data model, you don't need to think about tables, schemas, indexes etc. All you need to know is the nature of the data and then go with the most comfortable one for your mental mind.
|
|
56
|
+
|
|
57
|
+
## Installation
|
|
58
|
+
|
|
59
|
+
```javascript
|
|
60
|
+
npm install "datafolder"
|
|
61
|
+
```
|
|
62
|
+
|
|
63
|
+
## How to use
|
|
64
|
+
|
|
65
|
+
```javascript
|
|
66
|
+
import DataFolder from "datafolder";
|
|
67
|
+
const dataFolder = new DataFolder(); //Specify where to store the data. Default is ./data/sampleDataFolder
|
|
68
|
+
await dataFolder.start(); //start() and stop() are the only two async places. All other code doesn't need await.
|
|
69
|
+
dataFolder.insert("traindata/2026/1/20/chat_0001", {
|
|
70
|
+
userid: "user_0001",
|
|
71
|
+
email: "a@b.com",
|
|
72
|
+
feedbackScore: 5,
|
|
73
|
+
address: { city: "a", zipcode: 123 },
|
|
74
|
+
});
|
|
75
|
+
dataFolder.insert("traindata/2026/1/20/chat_0001/prompt", "What is async/await?");
|
|
76
|
+
dataFolder.insert("traindata/2026/1/20/chat_0001/answer", "async/await is ...");
|
|
77
|
+
const { email, password, address, feedbackScore } = dataFolder.view("traindata/2026/1/20/chat_0001");
|
|
78
|
+
address.zipcode = 678;
|
|
79
|
+
dataFolder.insert("traindata/2026/1/20/chat_0001", { feedbackScore: feedbackScore + 1, address });
|
|
80
|
+
await dataFolder.stop();
|
|
81
|
+
```
|
|
82
|
+
|
|
83
|
+
Or operate with dir/file objects:
|
|
84
|
+
|
|
85
|
+
```javascript
|
|
86
|
+
const file = dataFolder.file("traindata/2026/1/20/chat_0001");
|
|
87
|
+
const { feedbackScore, address } = file.view({ feedbackScore: 1, address: 1 });
|
|
88
|
+
dataFolder.insert(file, { feedbackScore: feedbackScore + 1, address: { city: "c", zipcode: 663 } });
|
|
89
|
+
```
|
|
90
|
+
|
|
91
|
+
### Simple CRUD APIs
|
|
92
|
+
|
|
93
|
+
#### Create
|
|
94
|
+
|
|
95
|
+
```javascript
|
|
96
|
+
dataFolder.createFile("path/to/file");
|
|
97
|
+
dataFolder.createDir("path/to/dir");
|
|
98
|
+
dataFolder.insert("path/to/file", { email, address: { city, zipcode } });
|
|
99
|
+
```
|
|
100
|
+
|
|
101
|
+
#### Read
|
|
102
|
+
|
|
103
|
+
```javascript
|
|
104
|
+
const dir = dataFolder.dir("path/to/dir");
|
|
105
|
+
const { subdirList, fileList, sortedSubdirList } = dir;
|
|
106
|
+
const file = dataFolder.file("path/to/file");
|
|
107
|
+
const { email, address } = dataFolder.view("path/to/file", { email: 1, addres: 1 });
|
|
108
|
+
```
|
|
109
|
+
|
|
110
|
+
#### Update
|
|
111
|
+
|
|
112
|
+
```javascript
|
|
113
|
+
dataFolder.insert("path/to/file", { email: newEmail, address: { zipcode: newZipcode } });
|
|
114
|
+
dataFolder.insert(file, { feedbackScore: feedbackScore + 1 });
|
|
115
|
+
```
|
|
116
|
+
|
|
117
|
+
#### Delete
|
|
118
|
+
|
|
119
|
+
```javascript
|
|
120
|
+
dataFolder.delDir("path/to/dir");
|
|
121
|
+
dataFolder.delFile("path/to/file");
|
|
122
|
+
dataFolder.delContent("path/to/file");
|
|
123
|
+
dataFolder.remove("path/to/file", { email: 1, address: { zipcode: 1 } });
|
|
124
|
+
dataFolder.remove(file, { email: 1, address: { zipcode: 1 } });
|
|
125
|
+
```
|
|
126
|
+
|
|
127
|
+
### Batch APIs
|
|
128
|
+
|
|
129
|
+
```javascript
|
|
130
|
+
const [dir, file, { email, feedbackScore }] = dataFolder.fetch
|
|
131
|
+
.dir("path/to/dir")
|
|
132
|
+
.file("path/to/file")
|
|
133
|
+
.view("path/to/file")
|
|
134
|
+
.run();
|
|
135
|
+
dataFolder.batch
|
|
136
|
+
.insert("path/to/chat_details", { user: "user_0001", feedbackScore: 100 })
|
|
137
|
+
.insert("path/to/prompt", "what is LLM?")
|
|
138
|
+
.insert("path/to/answer", "LLM is AI...")
|
|
139
|
+
.remove("path/to/file", { address: { city: 1 } })
|
|
140
|
+
.run();
|
|
141
|
+
```
|
|
142
|
+
|
|
143
|
+
### Powerful Query Support. Auto Create Needed Indexes
|
|
144
|
+
|
|
145
|
+
```javascript
|
|
146
|
+
// get all the training data in year 2026 with feedbackScore between 60 and 85.
|
|
147
|
+
dataFolder.queryFiles("traindata/2026/**", { feedbackScore: (x) => x >= 60 && x <= 85 });
|
|
148
|
+
|
|
149
|
+
// get training data of user_001 from July to Oct in year 2025, 2026 with feedbackScore higher than 90.
|
|
150
|
+
dataFolder.queryFiles(["traindata", { $in: [2025, 2026] }, { $gt: 6, $lte: 10 }], {
|
|
151
|
+
feedbackScore: { $gt: 90 },
|
|
152
|
+
userId: "user_0001",
|
|
153
|
+
});
|
|
154
|
+
|
|
155
|
+
dataFolder.queryFiles("flights/**", { _from: "airports/LAX" });
|
|
156
|
+
dataFolder.queryFiles(["flights", "*", "jan", (x) => x >= 5 && x <= 7], {
|
|
157
|
+
$or: [{ _from: "airports/BIS" }, { _to: "airports/BIS" }],
|
|
158
|
+
});
|
|
159
|
+
dataFolder.queryFiles(["flights", "*", "feb", { $or: [5, 6, 7] }], {
|
|
160
|
+
$or: [{ _from: "airports/BIS" }, { _to: "airports/BIS" }],
|
|
161
|
+
});
|
|
162
|
+
dataFolder.queryFiles(["flights", "*", "mar", { $gte: 5, lte: 7 }], {
|
|
163
|
+
$or: [{ _from: "airports/BIS" }, { _to: "airports/BIS" }],
|
|
164
|
+
});
|
|
165
|
+
dataFolder.queryFilesMulti([
|
|
166
|
+
[["airbnb", "Loft", "Entire home/apt", "Real Bed"], { address: { street: "Porto, Porto, Portugal" } }],
|
|
167
|
+
["airbnb/Loft/**", { minimum_nights: (x) => x >= 60 }],
|
|
168
|
+
]);
|
|
169
|
+
```
|
|
170
|
+
|
|
171
|
+
### Flexible Trigger Support to Monitor Any Changes
|
|
172
|
+
|
|
173
|
+
```javascript
|
|
174
|
+
function onPromptScoreChanged({ promptId, feedbackScore }) {
|
|
175
|
+
const { oldValue, newValue } = feedbackScore;
|
|
176
|
+
// Logic to handle prompt feedback score change.
|
|
177
|
+
}
|
|
178
|
+
dataFolder.insertTrigger("promptScoreChanged", "traindata/{promptId}", "feedbackScore");
|
|
179
|
+
dataFolder.on("promptScoreChanged", onPromptScoreChanged);
|
|
180
|
+
dataFolder.insert("traindata/prompt_1", { feedbackScore: 80 });
|
|
181
|
+
dataFolder.insert("traindata/prompt_2", { feedbackScore: 63 });
|
|
182
|
+
dataFolder.insert("traindata/prompt_1", { feedbackScore: 32 });
|
|
183
|
+
```
|
|
184
|
+
|
|
185
|
+
Will trigger the following calls:
|
|
186
|
+
|
|
187
|
+
```javascript
|
|
188
|
+
onPromptScoreChanged({ promptId: "student1", feedbackScore: { oldValue: undefined, newValue: 80 } });
|
|
189
|
+
onPromptScoreChanged({ promptId: "student2", feedbackScore: { oldValue: undefined, newValue: 63 } });
|
|
190
|
+
onPromptScoreChanged({ promptId: "student1", feedbackScore: { oldValue: 80, newValue: 32 } });
|
|
191
|
+
```
|
package/SSD.md
ADDED
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
## datafolder, a lightning fluid JSON database to smooth SSD bumps and keep the high speed
|
|
2
|
+
|
|
3
|
+
### Solid-State Drives (SSDs)
|
|
4
|
+
|
|
5
|
+
[SSDs](https://www.kingston.com/en/blog/pc-performance/nand-flash-technology-and-ssd) are non-volatile, high-speed solid-state drives that store data on flash memory chips without requiring power, making them ideal for computers and mobile devices. Using NAND logic gates, they provide faster read/write speeds, better durability, and lower power consumption than traditional mechanical hard drives.
|
|
6
|
+
|
|
7
|
+
But they have physical constraints:
|
|
8
|
+
|
|
9
|
+
- NAND flash **cannot overwrite** data;
|
|
10
|
+
- it must **erase a whole block** before writing new data.
|
|
11
|
+
|
|
12
|
+
**High** Speed. But with **bumps**.
|
|
13
|
+
|
|
14
|
+
### Flash Translation Layer (FTL)
|
|
15
|
+
|
|
16
|
+
The Flash Translation Layer (FTL) is critical firmware running on an SSD controller that maps logical block addresses (LBAs) from the operating system to physical NAND flash locations. The FTL manages the "write amplification" issue by handling random write operations efficiently, typically using a mapping table cached in DRAM to maintain high speed. It enables SSDs to act like hard drives, managing data mapping, wear leveling, garbage collection, and bad block management to ensure performance and durability.
|
|
17
|
+
|
|
18
|
+
Key Functions of the FTL:
|
|
19
|
+
|
|
20
|
+
- Address Mapping: Translates Logical Block Addresses (LBAs) used by the OS to Physical Block Addresses (PBAs) on the NAND flash.
|
|
21
|
+
- Wear Leveling: Distributes write/erase cycles evenly across flash cells to prevent premature failure of specific sectors.
|
|
22
|
+
- Garbage Collection: Cleans up invalid data (data marked for deletion) and consolidates valid data into new blocks, freeing up space.
|
|
23
|
+
- Bad Block Management: Identifies and blocks memory locations that can no longer hold data reliably.
|
|
24
|
+
|
|
25
|
+
### Log-structured merge-trees (LSM trees)
|
|
26
|
+
|
|
27
|
+
Log-structured merge-trees (LSM trees) are write-optimized data structures designed for high-throughput, insert-heavy workloads by batching updates in memory and writing them sequentially to disk. They reduce random writes—ideal for SSDs—by avoiding in-place updates, using an in-memory memtable, and periodically merging immutable Sorted String Tables (SSTables).
|
|
28
|
+
|
|
29
|
+
Key Characteristics and Benefits of LSM trees to run on SSDs
|
|
30
|
+
|
|
31
|
+
- Sequential IO: LSM trees convert random write operations into sequential writes, which are much faster and more efficient on SSDs compared to traditional B-tree in-place updates, reducing SSD endurance issues.
|
|
32
|
+
- Write-Optimized Structure: By using a "log-structured" approach, incoming data is buffered in a memtable in memory and, when full, flushed as an ordered, immutable file (SSTable) to the SSD, minimizing write amplification.
|
|
33
|
+
- Compaction: To manage storage and improve read performance, files are periodically merged in the background, consolidating data and removing old versions, often referred to as compaction.
|
|
34
|
+
|
|
35
|
+
### datafolder, a lightning fluid JSON database, is a natural fit for high speed SSDs
|
|
36
|
+
|
|
37
|
+
datafolder is [based on LSM trees](https://github.com/datafolderdev/datafolder/wiki/Internals-of-datafolder), but taken to a higher level.
|
|
38
|
+
|
|
39
|
+
- Complex JSON data in a single virtual file for a record
|
|
40
|
+
|
|
41
|
+
The data is just a normal JSON [object](https://github.com/datafolderdev/datafolder/wiki/Internals-of-datafolder#in-memory-tree-structure). It can be as complex as keeping a customer's whole airbnb listings and comments. Such a big record greatly reduceds read/write amplifications.
|
|
42
|
+
|
|
43
|
+
- Changes are stored as deltas in append-only mode.
|
|
44
|
+
|
|
45
|
+
When changes are made to each virtual file, only [the deltas](https://github.com/datafolderdev/datafolder/wiki/Internals-of-datafolder#accumulated-change-list) are stored. And the delta changes from different virtual files/folders are merged and appended to ACL log files.
|
|
46
|
+
|
|
47
|
+
- Never overwrite. Always append or create new file.
|
|
48
|
+
|
|
49
|
+
datafolder never overwrites. It always appends to an ACL file, or create new [snapshot files](https://github.com/datafolderdev/datafolder/wiki/Internals-of-datafolder#snapshot-indexes). Each snapshot is stored in a prefix Position Mapping file, a Position Mapping file, and a Value List file.
|
|
50
|
+
|
|
51
|
+
- Multi-level Indexing makes it lightning fast.
|
|
52
|
+
|
|
53
|
+
- Memory level folder tree indexing
|
|
54
|
+
- datafolder uses a tree structure to store the folder/file relationship in memory. Each node has a subdir map and a file map. subdir map stores subdir nodes, which in turn has its subdir map. file map stores the files that are inside this folder. JSON data is stored as file content within each file node. Thanks to V8 engine's highly optimized object [property](https://v8.dev/blog/fast-properties) [access](https://v8.dev/docs/hidden-classes), getting the dir/file node by path and getting properties from JSON data are extremely fast.
|
|
55
|
+
- memory level JSON property indexing
|
|
56
|
+
- For example, we can create "emailAddress" property indexing on "users" folder, and use this index to query for users with a given email address. Arbitrary properties and comparison combinations are supported, like "address.city" + "department" etc.
|
|
57
|
+
- Snapshot Position Mapping maps a key to its position in the corresponding valueList file.
|
|
58
|
+
- Snapshot Prefix Position Mapping maps a key prefix to its range position in the corresponding Positing Mapping file.
|
|
59
|
+
|
|
60
|
+
### Future: Make datafolder runs directly on SSDs without the FTL, or Zoned Namespaces(ZNS) SSDs (formerly Open-Channel SSDs)
|
package/index.js
CHANGED
package/package.json
CHANGED
|
@@ -1,11 +1,11 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "datafolder",
|
|
3
|
-
"version": "1.0.
|
|
3
|
+
"version": "1.0.11",
|
|
4
4
|
"author": {
|
|
5
5
|
"name": "lysh",
|
|
6
6
|
"email": "datafolderdev@outlook.com"
|
|
7
7
|
},
|
|
8
|
-
"description": "datafolder is a lightning fluid JSON database for
|
|
8
|
+
"description": "datafolder is a lightning fluid JSON database for high speed SSDs, featuring an AI-friendly API for simple and human maintainable code generation, millisecond data response, with full index and trigger features supported, and the unique ability to browse and manage data as a folder tree directly in your browser. It works perfectly for a wide range of scenarios, from game/application backends to embedded systems to IoT to edge computing.",
|
|
9
9
|
"license": "MIT",
|
|
10
10
|
"repository": {
|
|
11
11
|
"type": "git",
|
package/sample.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import DataFolder from "datafolder";
|
|
1
|
+
import DataFolder, { logger } from "datafolder";
|
|
2
2
|
const dataFolder = new DataFolder();
|
|
3
3
|
|
|
4
4
|
run();
|
|
@@ -18,8 +18,8 @@ async function run() {
|
|
|
18
18
|
address.zipcode = 678;
|
|
19
19
|
dataFolder.insert("traindata/2026/1/20/chat_0001", { feedbackScore: feedbackScore + 1, address });
|
|
20
20
|
logger.log(dataFolder.view("traindata/2026/1/20/chat_0001", ["feedbackScore", "address"]));
|
|
21
|
-
logger.
|
|
22
|
-
logger.
|
|
21
|
+
logger.debug(dataFolder.view("traindata/2026/1/20/chat_0001/prompt"));
|
|
22
|
+
logger.warn(dataFolder.view("traindata/2026/1/20/chat_0001/answer"));
|
|
23
23
|
await dataFolder.stop();
|
|
24
24
|
await dataFolder.start();
|
|
25
25
|
{
|
|
@@ -27,7 +27,7 @@ async function run() {
|
|
|
27
27
|
const { email, feedbackScore } = file.view();
|
|
28
28
|
logger.log(email, feedbackScore);
|
|
29
29
|
dataFolder.insert(file, { feedbackScore: feedbackScore + 1 });
|
|
30
|
-
logger.
|
|
30
|
+
logger.error(
|
|
31
31
|
file.view(),
|
|
32
32
|
dataFolder.view("traindata/2026/1/20/chat_0001/prompt"),
|
|
33
33
|
dataFolder.view("traindata/2026/1/20/chat_0001/answer"),
|
package/scripts/airbnb.js
CHANGED
|
@@ -2,6 +2,7 @@ import { join } from "node:path";
|
|
|
2
2
|
import { open, rm } from "node:fs/promises";
|
|
3
3
|
import { QyDB } from "../src/QyDB.js";
|
|
4
4
|
import { folderOrFileExists, isSimpleType, __dirname } from "../src/QyUtils.js";
|
|
5
|
+
import { logger } from "../src/QyLogger.js";
|
|
5
6
|
import { runQueries } from "./airbnb_queries.js";
|
|
6
7
|
import { downloadFile } from "./downloadFile.js";
|
|
7
8
|
const fileName = "listingsAndReviews.json";
|
package/scripts/airbnb2.js
CHANGED
|
@@ -2,6 +2,7 @@ import { join } from "node:path";
|
|
|
2
2
|
import { open, rm } from "node:fs/promises";
|
|
3
3
|
import { QyDB } from "../src/QyDB.js";
|
|
4
4
|
import { folderOrFileExists, isSimpleType, __dirname } from "../src/QyUtils.js";
|
|
5
|
+
import { logger } from "../src/QyLogger.js";
|
|
5
6
|
import { runQueries } from "./airbnb2_queries.js";
|
|
6
7
|
import { downloadFile } from "./downloadFile.js";
|
|
7
8
|
const fileName = "listingsAndReviews.json";
|
|
@@ -2,6 +2,7 @@ import { join, basename } from "node:path";
|
|
|
2
2
|
import { readFile, readdir, writeFile } from "node:fs/promises";
|
|
3
3
|
import { iterPMBuffer, loadPrefixPMFile } from "../src/QySnapshots.js";
|
|
4
4
|
import { ensureDir, folderOrFileExists } from "../src/QyUtils.js";
|
|
5
|
+
import { logger } from "../src/QyLogger.js";
|
|
5
6
|
logger.level = "info";
|
|
6
7
|
run();
|
|
7
8
|
async function loadPosMapFile(posMapFilePath) {
|
package/scripts/express.js
CHANGED
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
import { join } from "node:path";
|
|
2
2
|
import { QyDB } from "../src/QyDB.js";
|
|
3
|
-
import Express from "express";
|
|
4
3
|
import { __dirname } from "../src/QyUtils.js";
|
|
4
|
+
import { logger } from "../src/QyLogger.js";
|
|
5
|
+
import Express from "express";
|
|
5
6
|
const config = {
|
|
6
7
|
port: 3629,
|
|
7
8
|
qyDBDataFolder: "./tmp/QyExpress",
|
package/scripts/fastify.js
CHANGED
|
@@ -5,6 +5,7 @@ import fastifyStatic from "@fastify/static";
|
|
|
5
5
|
import fastifyCompress from "@fastify/compress";
|
|
6
6
|
import { QyDB } from "../src/QyDB.js";
|
|
7
7
|
import { folderOrFileExists, __dirname } from "../src/QyUtils.js";
|
|
8
|
+
import { logger } from "../src/QyLogger.js";
|
|
8
9
|
import Fastify from "fastify";
|
|
9
10
|
const config = {
|
|
10
11
|
port: 3628,
|
package/scripts/flights.js
CHANGED
|
@@ -5,6 +5,7 @@ import { readFileSync } from "node:fs";
|
|
|
5
5
|
import { rm } from "node:fs/promises";
|
|
6
6
|
import { QyDB } from "../src/QyDB.js";
|
|
7
7
|
import { folderOrFileExists } from "../src/QyUtils.js";
|
|
8
|
+
import { logger } from "../src/QyLogger.js";
|
|
8
9
|
import { runQueries } from "./flights_queries.js";
|
|
9
10
|
logger.level = "info";
|
|
10
11
|
const arangoJsonFolderPath = "D:\\scripts\\GraphCourse_DemoData_ArangoDB-2";
|
package/scripts/flights2.js
CHANGED
|
@@ -5,6 +5,7 @@ import { readFileSync } from "node:fs";
|
|
|
5
5
|
import { rm } from "node:fs/promises";
|
|
6
6
|
import { QyDB } from "../src/QyDB.js";
|
|
7
7
|
import { folderOrFileExists } from "../src/QyUtils.js";
|
|
8
|
+
import { logger } from "../src/QyLogger.js";
|
|
8
9
|
import { runQueries } from "./flights2_queries.js";
|
|
9
10
|
logger.level = "info";
|
|
10
11
|
const arangoJsonFolderPath = "D:\\scripts\\GraphCourse_DemoData_ArangoDB-2";
|
package/src/QyAcl.d.ts
CHANGED
|
@@ -6,12 +6,12 @@ export declare class QyAclCmdGenerator {
|
|
|
6
6
|
cmdCount: number;
|
|
7
7
|
batchSize: number;
|
|
8
8
|
constructor();
|
|
9
|
-
pushCmd(cmdName: string, cmdHashKey: string, cmdArg
|
|
9
|
+
pushCmd(cmdName: string, cmdHashKey: string, cmdArg?: any): void;
|
|
10
10
|
pushCmdArgAsListObj(cmdArgAsListObj: any): this;
|
|
11
11
|
takeCmdArgAsListObj(): {};
|
|
12
12
|
toAclBuffer(changeId: any): Buffer<ArrayBuffer> | undefined;
|
|
13
13
|
}
|
|
14
|
-
export declare function readAclFile(aclFilePath: any, autoRepairAclFile?: boolean, currentChangeId?: number): Promise<
|
|
14
|
+
export declare function readAclFile(aclFilePath: any, autoRepairAclFile?: boolean, currentChangeId?: number): Promise<any[]>;
|
|
15
15
|
export declare function numsToBuffer(nums: any): Buffer<ArrayBuffer>;
|
|
16
16
|
export declare function bufferToNums(buffer: any, offset: any, nums: any): any;
|
|
17
|
-
export declare function readAclBuffer(aclFilePath: any, contentBuffer: any, autoRepairAclFile: any, currentChangeId: any): Promise<
|
|
17
|
+
export declare function readAclBuffer(aclFilePath: any, contentBuffer: any, autoRepairAclFile: any, currentChangeId: any): Promise<any[]>;
|
package/src/QyAcl.js
CHANGED
|
@@ -66,7 +66,7 @@ class s {
|
|
|
66
66
|
cmdCount = 0;
|
|
67
67
|
batchSize = 0;
|
|
68
68
|
constructor() {}
|
|
69
|
-
pushCmd(r, t, e) {
|
|
69
|
+
pushCmd(r, t, e = void 0) {
|
|
70
70
|
switch (++this.cmdCount, r) {
|
|
71
71
|
case "A":
|
|
72
72
|
o(this, [ "A", t, e ]), f(this, [ "R", t, e ]);
|
|
@@ -151,20 +151,20 @@ function f(r, t) {
|
|
|
151
151
|
|
|
152
152
|
async function c(t, r = !1, e = 0) {
|
|
153
153
|
try {
|
|
154
|
-
return await
|
|
154
|
+
return await A(t, await u.readFile(t), r, e);
|
|
155
155
|
} catch (r) {
|
|
156
156
|
throw d.error(`Reading ${t} failed:`, r), r;
|
|
157
157
|
}
|
|
158
158
|
}
|
|
159
159
|
|
|
160
|
-
let
|
|
160
|
+
let v = [ 1, 2, 4, 8 ];
|
|
161
161
|
|
|
162
162
|
function l(t) {
|
|
163
163
|
var e = t.length, a = new Array(e), n = new Array(e);
|
|
164
164
|
let i = e << 6, s = 1;
|
|
165
165
|
for (let r = 0; r < e; ++r) {
|
|
166
166
|
var o = a[r] = (o = t[r]) <= 255 ? 0 : o <= 65535 ? 1 : o <= 4294967295 ? 2 : 3, o = (i |= o << (r << 1),
|
|
167
|
-
n[r] =
|
|
167
|
+
n[r] = v[o]);
|
|
168
168
|
s += o;
|
|
169
169
|
}
|
|
170
170
|
var f = Buffer.allocUnsafe(s);
|
|
@@ -197,7 +197,7 @@ function l(t) {
|
|
|
197
197
|
function h(t, e, a) {
|
|
198
198
|
var n = t.readUInt8(e), i = (++e, n >> 6 & 3);
|
|
199
199
|
for (let r = 0; r < i; ++r) {
|
|
200
|
-
var s =
|
|
200
|
+
var s = v[n >> (r << 1) & 3];
|
|
201
201
|
a[r] = ((r, t, e = 0) => {
|
|
202
202
|
switch (t) {
|
|
203
203
|
case 1:
|
|
@@ -217,7 +217,7 @@ function h(t, e, a) {
|
|
|
217
217
|
return e;
|
|
218
218
|
}
|
|
219
219
|
|
|
220
|
-
async function
|
|
220
|
+
async function A(e, r, a, n) {
|
|
221
221
|
var i = [];
|
|
222
222
|
for (let t = 0; t < r.length; ) {
|
|
223
223
|
var s = [], o = h(r, t, s), [ s, f, c ] = s;
|
|
@@ -246,5 +246,5 @@ export {
|
|
|
246
246
|
c as readAclFile,
|
|
247
247
|
l as numsToBuffer,
|
|
248
248
|
h as bufferToNums,
|
|
249
|
-
|
|
249
|
+
A as readAclBuffer
|
|
250
250
|
};
|
package/src/QyAclSaver_Worker.js
CHANGED
package/src/QyAclWriter.d.ts
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { QyBinWriter } from "./QyBinWriter.ts";
|
|
2
2
|
export declare class QyAclWriter extends QyBinWriter {
|
|
3
|
-
|
|
3
|
+
qyAclCmdGenerator: any;
|
|
4
|
+
constructor(options: any, filePath?: any);
|
|
4
5
|
getBinData(dataQueue: any, changeId: any): any;
|
|
5
6
|
}
|
package/src/QyAclWriter.js
CHANGED
|
@@ -7,20 +7,21 @@ import {
|
|
|
7
7
|
} from "./QyAcl.js";
|
|
8
8
|
|
|
9
9
|
import {
|
|
10
|
-
getDefaultOptions as
|
|
10
|
+
getDefaultOptions as o
|
|
11
11
|
} from "./QyDefaultOptions.js";
|
|
12
12
|
|
|
13
13
|
class t extends r {
|
|
14
|
-
|
|
14
|
+
qyAclCmdGenerator;
|
|
15
|
+
constructor(r, t = void 0) {
|
|
15
16
|
super({
|
|
16
|
-
...
|
|
17
|
+
...o("QyAclWriter"),
|
|
17
18
|
...r
|
|
18
19
|
}, t), this.qyAclCmdGenerator = new e();
|
|
19
20
|
}
|
|
20
21
|
getBinData(r, t) {
|
|
21
|
-
var e,
|
|
22
|
-
for (e of r)
|
|
23
|
-
return
|
|
22
|
+
var e, o = this.qyAclCmdGenerator;
|
|
23
|
+
for (e of r) o.pushCmdArgAsListObj(e);
|
|
24
|
+
return o.toAclBuffer(t);
|
|
24
25
|
}
|
|
25
26
|
}
|
|
26
27
|
|
package/src/QyBatch.d.ts
CHANGED
|
@@ -1,31 +1,38 @@
|
|
|
1
1
|
import { QyCD } from "./QyCD.ts";
|
|
2
2
|
export declare class QyBatch extends QyCD {
|
|
3
|
+
fileLogLevel: any;
|
|
4
|
+
actionQueue: any;
|
|
5
|
+
changeId: any;
|
|
6
|
+
changeList: any;
|
|
7
|
+
syncList: any;
|
|
8
|
+
qyAclCmdGenerator: any;
|
|
9
|
+
qyCmdObjHistory: any;
|
|
3
10
|
constructor(qyCache?: any, options?: any);
|
|
4
|
-
createDir(namePath: any, parentDir?:
|
|
5
|
-
createDirP(namePath: any, parentDir?:
|
|
6
|
-
delDir(namePath: any, parentDir?:
|
|
7
|
-
delDirP(namePath: any, parentDir?:
|
|
8
|
-
createFile(namePath: any, parentDir?:
|
|
9
|
-
createFileP(namePath: any, parentDir?:
|
|
10
|
-
delFile(namePath: any, parentDir?:
|
|
11
|
-
delFileP(namePath: any, parentDir?:
|
|
12
|
-
createIndex(indexHostDirNamePath: any, indexPropPathList: any, parentDir?:
|
|
13
|
-
createIndexP(indexHostDirNamePath: any, indexPropPathList: any, parentDir?:
|
|
14
|
-
delIndex(indexHostDirNamePath: any, indexPropPath: any, parentDir?:
|
|
15
|
-
delIndexP(indexHostDirNamePath: any, indexPropPath: any, parentDir?:
|
|
16
|
-
insert(namePath: any, content: any, parentDir?:
|
|
17
|
-
insertP(namePath: any, content: any, parentDir?:
|
|
18
|
-
remove(namePath: any, content: any, parentDir?:
|
|
19
|
-
removeP(namePath: any, content: any, parentDir?:
|
|
20
|
-
replace(namePath: any, content: any, parentDir?:
|
|
21
|
-
replaceP(namePath: any, content: any, parentDir?:
|
|
22
|
-
delContent(namePath: any, parentDir?:
|
|
23
|
-
delContentP(namePath: any, parentDir?:
|
|
11
|
+
createDir(namePath: any, parentDir?: any): any;
|
|
12
|
+
createDirP(namePath: any, parentDir?: any): any;
|
|
13
|
+
delDir(namePath: any, parentDir?: any): any;
|
|
14
|
+
delDirP(namePath: any, parentDir?: any): any;
|
|
15
|
+
createFile(namePath: any, parentDir?: any): any;
|
|
16
|
+
createFileP(namePath: any, parentDir?: any): any;
|
|
17
|
+
delFile(namePath: any, parentDir?: any): any;
|
|
18
|
+
delFileP(namePath: any, parentDir?: any): any;
|
|
19
|
+
createIndex(indexHostDirNamePath: any, indexPropPathList: any, parentDir?: any): any;
|
|
20
|
+
createIndexP(indexHostDirNamePath: any, indexPropPathList: any, parentDir?: any): any;
|
|
21
|
+
delIndex(indexHostDirNamePath: any, indexPropPath: any, parentDir?: any): any;
|
|
22
|
+
delIndexP(indexHostDirNamePath: any, indexPropPath: any, parentDir?: any): any;
|
|
23
|
+
insert(namePath: any, content: any, parentDir?: any): any;
|
|
24
|
+
insertP(namePath: any, content: any, parentDir?: any): any;
|
|
25
|
+
remove(namePath: any, content: any, parentDir?: any): any;
|
|
26
|
+
removeP(namePath: any, content: any, parentDir?: any): any;
|
|
27
|
+
replace(namePath: any, content: any, parentDir?: any): any;
|
|
28
|
+
replaceP(namePath: any, content: any, parentDir?: any): any;
|
|
29
|
+
delContent(namePath: any, parentDir?: any): any;
|
|
30
|
+
delContentP(namePath: any, parentDir?: any): any;
|
|
24
31
|
insertTrigger(triggerName: any, fileNamePath: any, propNamePathList: any, modCode: any): any;
|
|
25
32
|
removeTrigger(triggerName: any): any;
|
|
26
33
|
insertRpc(rpcFunName: any, modCode: any): any;
|
|
27
34
|
removeRpc(rpcFunName: any): any;
|
|
28
|
-
run(needPromise
|
|
35
|
+
run(needPromise?: boolean): any;
|
|
29
36
|
getTopHistoryCmdObj(): any;
|
|
30
37
|
_action_insertTrigger(action: any): boolean;
|
|
31
38
|
_action_removeTrigger(action: any): boolean;
|
|
@@ -33,7 +40,7 @@ export declare class QyBatch extends QyCD {
|
|
|
33
40
|
_action_removeRpc(action: any): boolean;
|
|
34
41
|
_action_createIndex(action: any): void;
|
|
35
42
|
_action_delIndex(action: any): void;
|
|
36
|
-
_action_delFile(action: any, deletingParentDir
|
|
43
|
+
_action_delFile(action: any, deletingParentDir?: boolean): boolean;
|
|
37
44
|
_action_delDir(action: any): boolean;
|
|
38
45
|
_action_delContent(action: any): boolean;
|
|
39
46
|
_action_insert(action: any): boolean;
|