json-database-st 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/JSONDatabase.js +627 -0
- package/LICENSE +21 -0
- package/README.md +118 -0
- package/package.json +45 -0
package/JSONDatabase.js
ADDED
|
@@ -0,0 +1,627 @@
|
|
|
1
|
+
// File: JSONDatabase.js
|
|
2
|
+
|
|
3
|
+
const fs = require("fs").promises;
|
|
4
|
+
const path = require("path");
|
|
5
|
+
const _ = require("lodash"); // Ensure lodash is installed: npm install lodash
|
|
6
|
+
|
|
7
|
+
const jsonRegex = /\.json$/;
|
|
8
|
+
|
|
9
|
+
/**
|
|
10
|
+
* @typedef {object} BatchOperationSet
|
|
11
|
+
* @property {'set'} type
|
|
12
|
+
* @property {string | string[]} path
|
|
13
|
+
* @property {any} value
|
|
14
|
+
*/
|
|
15
|
+
|
|
16
|
+
/**
|
|
17
|
+
* @typedef {object} BatchOperationDelete
|
|
18
|
+
* @property {'delete'} type
|
|
19
|
+
* @property {string | string[]} path
|
|
20
|
+
*/
|
|
21
|
+
|
|
22
|
+
/**
|
|
23
|
+
* @typedef {object} BatchOperationPush
|
|
24
|
+
* @property {'push'} type
|
|
25
|
+
* @property {string | string[]} path
|
|
26
|
+
* @property {any[]} values - Items to push uniquely using deep comparison.
|
|
27
|
+
*/
|
|
28
|
+
|
|
29
|
+
/**
|
|
30
|
+
* @typedef {object} BatchOperationPull
|
|
31
|
+
* @property {'pull'} type
|
|
32
|
+
* @property {string | string[]} path
|
|
33
|
+
* @property {any[]} values - Items to remove using deep comparison.
|
|
34
|
+
*/
|
|
35
|
+
|
|
36
|
+
/**
|
|
37
|
+
* @typedef {BatchOperationSet | BatchOperationDelete | BatchOperationPush | BatchOperationPull} BatchOperation
|
|
38
|
+
*/
|
|
39
|
+
|
|
40
|
+
/**
|
|
41
|
+
* A simple, promise-based JSON file database with atomic operations.
|
|
42
|
+
* Uses lodash for object manipulation.
|
|
43
|
+
*
|
|
44
|
+
* @class JSONDatabase
|
|
45
|
+
*/
|
|
46
|
+
class JSONDatabase {
|
|
47
|
+
/**
|
|
48
|
+
* Creates a database instance.
|
|
49
|
+
*
|
|
50
|
+
* @param {string} filename - Database file path (e.g., 'db.json' or './data/myDb'). '.json' extension is added if missing.
|
|
51
|
+
* @param {object} [options] - Configuration options.
|
|
52
|
+
* @param {boolean} [options.prettyPrint=false] - Pretty-print JSON file output (adds indentation). Defaults to false for smaller file size.
|
|
53
|
+
*/
|
|
54
|
+
constructor(filename, options = {}) {
|
|
55
|
+
if (!jsonRegex.test(filename)) {
|
|
56
|
+
this.filename = path.resolve(`${filename}.json`);
|
|
57
|
+
} else {
|
|
58
|
+
this.filename = path.resolve(filename);
|
|
59
|
+
}
|
|
60
|
+
this.cache = null; // In-memory cache of the database content
|
|
61
|
+
this.writeLock = null; // Promise acting as a lock for atomic writes
|
|
62
|
+
this.config = {
|
|
63
|
+
prettyPrint: options.prettyPrint === true, // Explicit check
|
|
64
|
+
};
|
|
65
|
+
this.stats = { reads: 0, writes: 0, cacheHits: 0 };
|
|
66
|
+
this._initPromise = null; // Track initialization promise
|
|
67
|
+
|
|
68
|
+
// Initialize cache asynchronously, handle errors during init.
|
|
69
|
+
this._init().catch((err) => {
|
|
70
|
+
console.error(
|
|
71
|
+
`[JSONDatabase] FATAL: Initialization failed for ${this.filename}:`,
|
|
72
|
+
err
|
|
73
|
+
);
|
|
74
|
+
// Optionally, set a state indicating permanent failure
|
|
75
|
+
});
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
/** @private Initialize cache by reading the file */
|
|
79
|
+
async _init() {
|
|
80
|
+
// Prevent race conditions during initial load if multiple operations trigger it
|
|
81
|
+
if (this._initPromise) return this._initPromise;
|
|
82
|
+
|
|
83
|
+
// Create the promise and store it
|
|
84
|
+
this._initPromise = this._refreshCache();
|
|
85
|
+
|
|
86
|
+
try {
|
|
87
|
+
await this._initPromise;
|
|
88
|
+
} finally {
|
|
89
|
+
// Clear the promise variable once initialization is complete (success or failure)
|
|
90
|
+
// So subsequent calls to _init (if needed, e.g. after close/re-open) would trigger a new load
|
|
91
|
+
this._initPromise = null;
|
|
92
|
+
}
|
|
93
|
+
return this.cache; // Return cache state after init attempt
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
/** @private Reads the database file and populates the cache. Creates file if not exists. */
|
|
97
|
+
async _refreshCache() {
|
|
98
|
+
try {
|
|
99
|
+
const data = await fs.readFile(this.filename, "utf8");
|
|
100
|
+
// Handle case where file is empty or contains only whitespace
|
|
101
|
+
this.cache = data.trim() === "" ? {} : JSON.parse(data);
|
|
102
|
+
this.stats.reads++;
|
|
103
|
+
return this.cache;
|
|
104
|
+
} catch (err) {
|
|
105
|
+
if (err.code === "ENOENT") {
|
|
106
|
+
// File doesn't exist, create it with an empty object
|
|
107
|
+
console.warn(
|
|
108
|
+
`[JSONDatabase] File ${this.filename} not found. Creating.`
|
|
109
|
+
);
|
|
110
|
+
try {
|
|
111
|
+
await fs.writeFile(this.filename, "{}", "utf8");
|
|
112
|
+
this.cache = {};
|
|
113
|
+
this.stats.writes++; // Count file creation as a write
|
|
114
|
+
return this.cache;
|
|
115
|
+
} catch (writeErr) {
|
|
116
|
+
console.error(
|
|
117
|
+
`[JSONDatabase] Failed to create ${this.filename}:`,
|
|
118
|
+
writeErr
|
|
119
|
+
);
|
|
120
|
+
throw writeErr; // Re-throw creation error
|
|
121
|
+
}
|
|
122
|
+
} else if (err instanceof SyntaxError) {
|
|
123
|
+
// Handle corrupted JSON file
|
|
124
|
+
console.error(
|
|
125
|
+
`[JSONDatabase] Error parsing JSON from ${this.filename}. File might be corrupted. Returning empty object and logging error.`,
|
|
126
|
+
err
|
|
127
|
+
);
|
|
128
|
+
// Decide recovery strategy: Throw, reset to {}, load backup?
|
|
129
|
+
// For robustness, let's reset the cache to {} but throw the original error
|
|
130
|
+
this.cache = {}; // Prevent operations on potentially bad data
|
|
131
|
+
throw new Error(
|
|
132
|
+
`Failed to parse JSON file: ${this.filename}. ${err.message}`
|
|
133
|
+
);
|
|
134
|
+
}
|
|
135
|
+
// Log and re-throw other unexpected errors
|
|
136
|
+
console.error(
|
|
137
|
+
`[JSONDatabase] Error reading or parsing ${this.filename}:`,
|
|
138
|
+
err
|
|
139
|
+
);
|
|
140
|
+
throw err;
|
|
141
|
+
}
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
/**
|
|
145
|
+
* @private Ensures atomic write operations using a lock.
|
|
146
|
+
* Takes a function that receives the current data state and should return the modified state.
|
|
147
|
+
* @param {(data: object) => object | Promise<object>} operationFn - Function performing the modification.
|
|
148
|
+
* @returns {Promise<object>} The final data state after the write.
|
|
149
|
+
*/
|
|
150
|
+
async _atomicWrite(operationFn) {
|
|
151
|
+
// Ensure initialization is complete before attempting writes
|
|
152
|
+
// If _initPromise exists, await it. If not, call _init() which handles the lock itself.
|
|
153
|
+
await (this._initPromise || this._init());
|
|
154
|
+
|
|
155
|
+
// Chain onto the existing lock promise if it exists
|
|
156
|
+
const performWrite = async () => {
|
|
157
|
+
let dataToModify;
|
|
158
|
+
try {
|
|
159
|
+
// Get the most recent data, MUST be from cache as it reflects the state after the previous write lock released.
|
|
160
|
+
// If cache is somehow null here (edge case after close?), refresh.
|
|
161
|
+
dataToModify = _.cloneDeep(this.cache ?? (await this._refreshCache()));
|
|
162
|
+
|
|
163
|
+
// Execute the user-provided operation function
|
|
164
|
+
const result = await operationFn(dataToModify);
|
|
165
|
+
|
|
166
|
+
// The operationFn *must* return the data object to be written.
|
|
167
|
+
const dataToWrite = result;
|
|
168
|
+
if (dataToWrite === undefined) {
|
|
169
|
+
// This indicates a potential programming error in the calling function (e.g., transaction, batch)
|
|
170
|
+
console.error(
|
|
171
|
+
"[JSONDatabase] FATAL: Atomic operation function returned undefined. This should not happen. Aborting write to prevent data loss."
|
|
172
|
+
);
|
|
173
|
+
throw new Error(
|
|
174
|
+
"Atomic operation function returned undefined, cannot proceed with write."
|
|
175
|
+
);
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
// Optimization: Only write if data actually changed. Deep compare can be costly for large objects.
|
|
179
|
+
// Let's keep it simple and write every time for guaranteed consistency, unless performance becomes an issue.
|
|
180
|
+
// if (_.isEqual(dataToWrite, this.cache)) {
|
|
181
|
+
// return this.cache; // No change, return current cache state
|
|
182
|
+
// }
|
|
183
|
+
|
|
184
|
+
await fs.writeFile(
|
|
185
|
+
this.filename,
|
|
186
|
+
JSON.stringify(dataToWrite, null, this.config.prettyPrint ? 2 : 0),
|
|
187
|
+
"utf8"
|
|
188
|
+
);
|
|
189
|
+
|
|
190
|
+
this.cache = dataToWrite; // Update cache AFTER successful write
|
|
191
|
+
this.stats.writes++;
|
|
192
|
+
return this.cache; // Return the final state written to disk/cache
|
|
193
|
+
} catch (error) {
|
|
194
|
+
console.error(
|
|
195
|
+
"[JSONDatabase] Error during atomic write operation:",
|
|
196
|
+
error
|
|
197
|
+
);
|
|
198
|
+
// Don't update cache on error, state is uncertain relative to the disk.
|
|
199
|
+
// A subsequent read/write *should* re-sync if the error was temporary.
|
|
200
|
+
throw error; // Re-throw the error to the caller
|
|
201
|
+
}
|
|
202
|
+
};
|
|
203
|
+
|
|
204
|
+
// Assign the promise of the current write operation to the lock, ensuring sequential execution.
|
|
205
|
+
// The `then()` ensures we wait for the previous lock to finish before starting the new write.
|
|
206
|
+
this.writeLock = (this.writeLock || Promise.resolve()).then(
|
|
207
|
+
performWrite,
|
|
208
|
+
performWrite
|
|
209
|
+
);
|
|
210
|
+
// We return the promise assigned to this.writeLock so callers can await this specific operation's completion.
|
|
211
|
+
return this.writeLock;
|
|
212
|
+
}
|
|
213
|
+
|
|
214
|
+
/**
|
|
215
|
+
* Gets a value from the database using a lodash path.
|
|
216
|
+
* Returns undefined if the path does not exist, unless a defaultValue is provided.
|
|
217
|
+
*
|
|
218
|
+
* @param {string | string[]} path - The lodash path (e.g., 'users.john.age' or ['users', 'john', 'age']).
|
|
219
|
+
* @param {any} [defaultValue] - Value to return if path is not found.
|
|
220
|
+
* @returns {Promise<any>} The value found at the path or the default value.
|
|
221
|
+
*/
|
|
222
|
+
async get(path, defaultValue) {
|
|
223
|
+
// Ensure cache is loaded. _init() handles the initialization promise logic.
|
|
224
|
+
await this._init();
|
|
225
|
+
const data = this.cache; // Read directly from cache after init ensures it's loaded
|
|
226
|
+
this.stats.cacheHits++;
|
|
227
|
+
return _.get(data, path, defaultValue);
|
|
228
|
+
}
|
|
229
|
+
|
|
230
|
+
/**
|
|
231
|
+
* Sets a value in the database at a specific lodash path.
|
|
232
|
+
*
|
|
233
|
+
* @param {string | string[]} path - The lodash path to set.
|
|
234
|
+
* @param {any} value - The value to set. Can be any JSON-serializable type.
|
|
235
|
+
* @returns {Promise<void>} Resolves when the write operation is complete.
|
|
236
|
+
*/
|
|
237
|
+
async set(path, value) {
|
|
238
|
+
// The void return type means callers don't get the data back directly,
|
|
239
|
+
// but the promise resolution confirms the write completed.
|
|
240
|
+
await this._atomicWrite((data) => {
|
|
241
|
+
_.set(data, path, value);
|
|
242
|
+
return data; // Return the modified data object for the atomic write process
|
|
243
|
+
});
|
|
244
|
+
}
|
|
245
|
+
|
|
246
|
+
/**
|
|
247
|
+
* Pushes one or more items into an array at the specified path.
|
|
248
|
+
* If the path doesn't exist or isn't an array, it creates/replaces it with a new array containing the items.
|
|
249
|
+
* Ensures items are unique within the array using deep comparison (`lodash.isEqual`).
|
|
250
|
+
*
|
|
251
|
+
* @param {string | string[]} path - The lodash path to the array.
|
|
252
|
+
* @param {...any} items - The items to push into the array.
|
|
253
|
+
* @returns {Promise<void>} Resolves when the write operation is complete.
|
|
254
|
+
*/
|
|
255
|
+
async push(path, ...items) {
|
|
256
|
+
if (items.length === 0) return; // No-op if no items provided
|
|
257
|
+
await this._atomicWrite((data) => {
|
|
258
|
+
let arr = _.get(data, path);
|
|
259
|
+
// Initialize as array if path doesn't exist or holds a non-array value
|
|
260
|
+
if (!Array.isArray(arr)) {
|
|
261
|
+
arr = [];
|
|
262
|
+
// Ensure the path is set to an empty array before proceeding
|
|
263
|
+
_.set(data, path, arr);
|
|
264
|
+
}
|
|
265
|
+
|
|
266
|
+
let changed = false;
|
|
267
|
+
items.forEach((item) => {
|
|
268
|
+
// Use lodash isEqual for deep comparison to check existence
|
|
269
|
+
const itemExists = arr.some((existingItem) =>
|
|
270
|
+
_.isEqual(existingItem, item)
|
|
271
|
+
);
|
|
272
|
+
if (!itemExists) {
|
|
273
|
+
arr.push(item);
|
|
274
|
+
changed = true;
|
|
275
|
+
}
|
|
276
|
+
});
|
|
277
|
+
|
|
278
|
+
// No need to _.set again if arr reference was modified in place and already set
|
|
279
|
+
// However, if arr was initialized (arr = []), _.set was needed above.
|
|
280
|
+
// The atomic write will handle writing 'data' regardless if 'changed' is true.
|
|
281
|
+
// Returning 'data' is sufficient.
|
|
282
|
+
|
|
283
|
+
return data; // Return the data object (potentially modified)
|
|
284
|
+
});
|
|
285
|
+
}
|
|
286
|
+
|
|
287
|
+
/**
|
|
288
|
+
* Removes specified items from an array at the given path.
|
|
289
|
+
* Uses deep comparison (`lodash.isEqual`) via `lodash.pullAllWith`.
|
|
290
|
+
* If the target path doesn't exist or is not an array, the operation is ignored silently.
|
|
291
|
+
*
|
|
292
|
+
* @param {string | string[]} path - Dot notation path to the array.
|
|
293
|
+
* @param {...any} itemsToRemove - Items to remove from the array.
|
|
294
|
+
* @returns {Promise<void>} Resolves when the write operation is complete.
|
|
295
|
+
*/
|
|
296
|
+
async pull(path, ...itemsToRemove) {
|
|
297
|
+
if (itemsToRemove.length === 0) return; // No-op if no items provided
|
|
298
|
+
await this._atomicWrite((data) => {
|
|
299
|
+
let arr = _.get(data, path);
|
|
300
|
+
|
|
301
|
+
if (!Array.isArray(arr)) {
|
|
302
|
+
// Silently ignore if target is not an array
|
|
303
|
+
return data; // Return unmodified data
|
|
304
|
+
}
|
|
305
|
+
|
|
306
|
+
const initialLength = arr.length;
|
|
307
|
+
// Use pullAllWith and isEqual for robust object comparison during removal
|
|
308
|
+
_.pullAllWith(arr, itemsToRemove, _.isEqual);
|
|
309
|
+
|
|
310
|
+
// If the array length changed, the data object 'data' now contains the modified array.
|
|
311
|
+
// No explicit _.set is needed here as pullAllWith modifies the array in place,
|
|
312
|
+
// and 'arr' is a reference within 'data'.
|
|
313
|
+
|
|
314
|
+
// Only trigger a file write if something actually changed (handled by atomicWrite comparing result)
|
|
315
|
+
// if (arr.length !== initialLength) {
|
|
316
|
+
// _.set(data, path, arr); // No need to set again, arr is ref inside data
|
|
317
|
+
// }
|
|
318
|
+
return data; // Return the data object (potentially modified)
|
|
319
|
+
});
|
|
320
|
+
}
|
|
321
|
+
|
|
322
|
+
/**
|
|
323
|
+
* Deletes a value or property at the specified path.
|
|
324
|
+
*
|
|
325
|
+
* @param {string | string[]} path - The lodash path to delete.
|
|
326
|
+
* @returns {Promise<boolean>} True if the path existed and was deleted, false otherwise.
|
|
327
|
+
*/
|
|
328
|
+
async delete(path) {
|
|
329
|
+
let deleted = false;
|
|
330
|
+
await this._atomicWrite((data) => {
|
|
331
|
+
// _.unset modifies 'data' in place and returns boolean
|
|
332
|
+
deleted = _.unset(data, path);
|
|
333
|
+
return data; // Return the modified data object
|
|
334
|
+
});
|
|
335
|
+
// Return the result captured from _.unset
|
|
336
|
+
return deleted;
|
|
337
|
+
}
|
|
338
|
+
|
|
339
|
+
/**
|
|
340
|
+
* Checks if a path exists in the database.
|
|
341
|
+
* Note: An existing path with a value of `undefined` will return `true`.
|
|
342
|
+
*
|
|
343
|
+
* @param {string | string[]} path - The lodash path to check.
|
|
344
|
+
* @returns {Promise<boolean>} True if the path exists, false otherwise.
|
|
345
|
+
*/
|
|
346
|
+
async has(path) {
|
|
347
|
+
await this._init(); // Ensure cache is ready
|
|
348
|
+
const data = this.cache;
|
|
349
|
+
this.stats.cacheHits++;
|
|
350
|
+
return _.has(data, path);
|
|
351
|
+
}
|
|
352
|
+
|
|
353
|
+
/**
|
|
354
|
+
* Performs an atomic transaction.
|
|
355
|
+
* The provided asynchronous function receives the current database state (as a deep clone)
|
|
356
|
+
* and should return the modified state to be written.
|
|
357
|
+
* If the function throws an error, the transaction is aborted, and no changes are saved.
|
|
358
|
+
* If the function returns `undefined`, an error is thrown to prevent potential data loss.
|
|
359
|
+
*
|
|
360
|
+
* @param {(data: object) => Promise<object> | object} asyncFn - An async function that modifies the data. It MUST return the modified data object.
|
|
361
|
+
* @returns {Promise<object>} A promise that resolves with the final database state after the transaction (the value returned by `asyncFn`).
|
|
362
|
+
* @throws {Error} Throws if the transaction function `asyncFn` throws an error, or if `asyncFn` returns undefined.
|
|
363
|
+
* @example
|
|
364
|
+
* await db.transaction(async (data) => {
|
|
365
|
+
* const user = _.get(data, 'users.john');
|
|
366
|
+
* if (user) {
|
|
367
|
+
* user.visits = (user.visits || 0) + 1;
|
|
368
|
+
* _.set(data, 'users.john', user); // Optional if modifying user in place
|
|
369
|
+
* }
|
|
370
|
+
* return data; // MUST return the modified data
|
|
371
|
+
* });
|
|
372
|
+
*/
|
|
373
|
+
async transaction(asyncFn) {
|
|
374
|
+
// _atomicWrite handles the locking, cloning, writing, and cache update.
|
|
375
|
+
// It expects a function that receives data and returns the modified data.
|
|
376
|
+
// asyncFn fits this signature. We just pass it through.
|
|
377
|
+
// _atomicWrite also handles the 'undefined' return check now.
|
|
378
|
+
return this._atomicWrite(asyncFn);
|
|
379
|
+
}
|
|
380
|
+
|
|
381
|
+
/**
|
|
382
|
+
* Executes multiple operations atomically within a single write lock.
|
|
383
|
+
* Supported operations: 'set', 'delete', 'push', 'pull'.
|
|
384
|
+
* Operations are executed sequentially in the provided order.
|
|
385
|
+
* If any operation definition is invalid (e.g., missing path/value, invalid type) or
|
|
386
|
+
* encounters an error during its execution (this is less likely for simple types but possible),
|
|
387
|
+
* an error is logged, and the batch processing *continues* with the next operation by default.
|
|
388
|
+
* The entire final state is written to disk once all operations are attempted.
|
|
389
|
+
*
|
|
390
|
+
* @param {BatchOperation[]} ops - An array of operation objects.
|
|
391
|
+
* @returns {Promise<void>} Resolves when the batch write is complete.
|
|
392
|
+
* @example
|
|
393
|
+
* await db.batch([
|
|
394
|
+
* { type: 'set', path: 'users.jane', value: { age: 30 } },
|
|
395
|
+
* { type: 'push', path: 'users.jane.hobbies', values: ['reading', 'hiking'] }, // deep compares items in 'values'
|
|
396
|
+
* { type: 'pull', path: 'config.features', values: ['betaFeature'] }, // deep compares items in 'values'
|
|
397
|
+
* { type: 'delete', path: 'users.oldUser' }
|
|
398
|
+
* ]);
|
|
399
|
+
*/
|
|
400
|
+
async batch(ops) {
|
|
401
|
+
if (!Array.isArray(ops) || ops.length === 0) {
|
|
402
|
+
console.warn("[JSONDatabase] Batch called with no operations.");
|
|
403
|
+
return; // No operations to perform
|
|
404
|
+
}
|
|
405
|
+
|
|
406
|
+
await this._atomicWrite((data) => {
|
|
407
|
+
ops.forEach((op, index) => {
|
|
408
|
+
try {
|
|
409
|
+
// Validate base operation structure
|
|
410
|
+
if (!op || typeof op !== "object" || !op.type) {
|
|
411
|
+
throw new Error(
|
|
412
|
+
`Operation at index ${index} is invalid or missing 'type'.`
|
|
413
|
+
);
|
|
414
|
+
}
|
|
415
|
+
|
|
416
|
+
switch (op.type) {
|
|
417
|
+
case "set":
|
|
418
|
+
if (op.path === undefined || !op.hasOwnProperty("value"))
|
|
419
|
+
throw new Error(
|
|
420
|
+
`Batch 'set' op index ${index} missing 'path' or 'value'.`
|
|
421
|
+
);
|
|
422
|
+
_.set(data, op.path, op.value);
|
|
423
|
+
break;
|
|
424
|
+
case "delete":
|
|
425
|
+
if (op.path === undefined)
|
|
426
|
+
throw new Error(
|
|
427
|
+
`Batch 'delete' op index ${index} missing 'path'.`
|
|
428
|
+
);
|
|
429
|
+
_.unset(data, op.path);
|
|
430
|
+
break;
|
|
431
|
+
case "push": {
|
|
432
|
+
if (op.path === undefined || !Array.isArray(op.values))
|
|
433
|
+
throw new Error(
|
|
434
|
+
`Batch 'push' op index ${index} missing 'path' or 'values' array.`
|
|
435
|
+
);
|
|
436
|
+
if (op.values.length === 0) break; // Skip if no values to push
|
|
437
|
+
|
|
438
|
+
let arr = _.get(data, op.path);
|
|
439
|
+
if (!Array.isArray(arr)) {
|
|
440
|
+
arr = []; // Initialize if not array
|
|
441
|
+
_.set(data, op.path, arr); // Set the path to the new array
|
|
442
|
+
}
|
|
443
|
+
op.values.forEach((item) => {
|
|
444
|
+
const itemExists = arr.some((existing) =>
|
|
445
|
+
_.isEqual(existing, item)
|
|
446
|
+
);
|
|
447
|
+
if (!itemExists) {
|
|
448
|
+
arr.push(item);
|
|
449
|
+
} // Push modifies arr in place
|
|
450
|
+
});
|
|
451
|
+
break;
|
|
452
|
+
}
|
|
453
|
+
case "pull": {
|
|
454
|
+
if (op.path === undefined || !Array.isArray(op.values))
|
|
455
|
+
throw new Error(
|
|
456
|
+
`Batch 'pull' op index ${index} missing 'path' or 'values' array.`
|
|
457
|
+
);
|
|
458
|
+
if (op.values.length === 0) break; // Skip if no values to pull
|
|
459
|
+
|
|
460
|
+
let arr = _.get(data, op.path);
|
|
461
|
+
if (Array.isArray(arr)) {
|
|
462
|
+
_.pullAllWith(arr, op.values, _.isEqual); // pullAllWith modifies arr in place
|
|
463
|
+
} // else ignore pull from non-array silently
|
|
464
|
+
break;
|
|
465
|
+
}
|
|
466
|
+
default:
|
|
467
|
+
// Use Error for invalid type as it indicates a programming mistake
|
|
468
|
+
throw new Error(
|
|
469
|
+
`Invalid batch operation type: '${op.type}' at index ${index}.`
|
|
470
|
+
);
|
|
471
|
+
}
|
|
472
|
+
} catch (batchOpError) {
|
|
473
|
+
// Log the error but continue processing the rest of the batch
|
|
474
|
+
console.error(
|
|
475
|
+
`[JSONDatabase] Error during batch operation index ${index} (type: ${op?.type}, path: ${op?.path}):`,
|
|
476
|
+
batchOpError.message
|
|
477
|
+
);
|
|
478
|
+
// Optional: Could add an option to stop batch on first error if needed
|
|
479
|
+
}
|
|
480
|
+
});
|
|
481
|
+
// Return the final state after all batch operations attempted
|
|
482
|
+
return data;
|
|
483
|
+
});
|
|
484
|
+
}
|
|
485
|
+
|
|
486
|
+
/**
|
|
487
|
+
* Queries the database based on a predicate function.
|
|
488
|
+
* Iterates over the direct properties (key-value pairs) of an object specified by `options.basePath`,
|
|
489
|
+
* or the root object of the database if `basePath` is omitted.
|
|
490
|
+
* Returns an array of the **values** of the properties that satisfy the predicate.
|
|
491
|
+
*
|
|
492
|
+
* @param {(value: any, key: string) => boolean} predicate - A function returning true for items to include. Receives (value, key) of each property being iterated.
|
|
493
|
+
* @param {object} [options] - Query options.
|
|
494
|
+
* @param {string | string[]} [options.basePath] - A lodash path to the object whose properties should be queried.
|
|
495
|
+
* @param {number} [options.limit] - Maximum number of results to return.
|
|
496
|
+
* @returns {Promise<any[]>} An array of **values** that satisfy the predicate.
|
|
497
|
+
* @example
|
|
498
|
+
* // Find all users older than 30 within the 'users' object
|
|
499
|
+
* const oldUsers = await db.query(
|
|
500
|
+
* (userValue, userKey) => typeof userValue === 'object' && userValue.age > 30,
|
|
501
|
+
* { basePath: 'users' }
|
|
502
|
+
* );
|
|
503
|
+
* // -> [ { name: 'Alice', age: 35, ... }, { name: 'Charlie', age: 40, ... } ]
|
|
504
|
+
*
|
|
505
|
+
* // Find top-level properties whose key starts with 'config'
|
|
506
|
+
* const configValues = await db.query(
|
|
507
|
+
* (value, key) => key.startsWith('config')
|
|
508
|
+
* );
|
|
509
|
+
* // -> [ { theme: 'dark', ... }, { region: 'us-east', ... } ] (assuming config objects exist at root)
|
|
510
|
+
*/
|
|
511
|
+
async query(predicate, options = {}) {
|
|
512
|
+
await this._init(); // Ensure cache is ready
|
|
513
|
+
const data = this.cache;
|
|
514
|
+
this.stats.cacheHits++; // Count query as cache hit
|
|
515
|
+
|
|
516
|
+
const basePath = options.basePath;
|
|
517
|
+
// Use _.get to safely retrieve the base object/value
|
|
518
|
+
const baseData = basePath ? _.get(data, basePath) : data;
|
|
519
|
+
|
|
520
|
+
// Ensure baseData is an object we can iterate over
|
|
521
|
+
if (
|
|
522
|
+
typeof baseData !== "object" ||
|
|
523
|
+
baseData === null ||
|
|
524
|
+
Array.isArray(baseData)
|
|
525
|
+
) {
|
|
526
|
+
if (basePath) {
|
|
527
|
+
console.warn(
|
|
528
|
+
`[JSONDatabase] Query basePath "${basePath}" does not point to an iterable object (must be a plain object). Returning empty array.`
|
|
529
|
+
);
|
|
530
|
+
} else {
|
|
531
|
+
console.warn(
|
|
532
|
+
`[JSONDatabase] Query attempted on non-object root data type (${typeof baseData}). Returning empty array.`
|
|
533
|
+
);
|
|
534
|
+
}
|
|
535
|
+
return [];
|
|
536
|
+
}
|
|
537
|
+
|
|
538
|
+
const results = [];
|
|
539
|
+
const limit = options.limit ?? Infinity;
|
|
540
|
+
|
|
541
|
+
// Iterate over the properties of the baseData object
|
|
542
|
+
for (const key in baseData) {
|
|
543
|
+
// Ensure we only iterate own properties
|
|
544
|
+
if (Object.prototype.hasOwnProperty.call(baseData, key)) {
|
|
545
|
+
if (results.length >= limit) {
|
|
546
|
+
break; // Stop iteration if limit is reached
|
|
547
|
+
}
|
|
548
|
+
const value = baseData[key];
|
|
549
|
+
try {
|
|
550
|
+
if (predicate(value, key)) {
|
|
551
|
+
results.push(value); // Return the value that matched
|
|
552
|
+
}
|
|
553
|
+
} catch (predicateError) {
|
|
554
|
+
console.error(
|
|
555
|
+
`[JSONDatabase] Error executing query predicate for key "${key}":`,
|
|
556
|
+
predicateError
|
|
557
|
+
);
|
|
558
|
+
// Skip this item or handle error as needed (currently skipping)
|
|
559
|
+
}
|
|
560
|
+
}
|
|
561
|
+
}
|
|
562
|
+
|
|
563
|
+
// Slice again in case limit was exactly reached (though break should handle it)
|
|
564
|
+
return results.slice(0, limit);
|
|
565
|
+
}
|
|
566
|
+
|
|
567
|
+
/**
|
|
568
|
+
* Clears the entire database content, replacing it with an empty object `{}`.
|
|
569
|
+
* This is an atomic write operation. Use with caution!
|
|
570
|
+
* @returns {Promise<void>} Resolves when the database has been cleared.
|
|
571
|
+
*/
|
|
572
|
+
async clear() {
|
|
573
|
+
await this._atomicWrite(() => {
|
|
574
|
+
// Return an empty object to be written
|
|
575
|
+
return {};
|
|
576
|
+
});
|
|
577
|
+
console.warn(`[JSONDatabase] Cleared all data from ${this.filename}.`);
|
|
578
|
+
}
|
|
579
|
+
|
|
580
|
+
/**
|
|
581
|
+
* Returns the current operational statistics for this database instance.
|
|
582
|
+
* @returns {{reads: number, writes: number, cacheHits: number}}
|
|
583
|
+
*/
|
|
584
|
+
getStats() {
|
|
585
|
+
// Return a copy to prevent external modification of the internal stats object
|
|
586
|
+
return { ...this.stats };
|
|
587
|
+
}
|
|
588
|
+
|
|
589
|
+
/**
|
|
590
|
+
* Waits for any pending write operations queued by this instance to complete,
|
|
591
|
+
* then clears the internal cache and releases the write lock.
|
|
592
|
+
* Does not delete the database file.
|
|
593
|
+
* Call this before your application exits to ensure data integrity.
|
|
594
|
+
*
|
|
595
|
+
* @returns {Promise<void>} Resolves when the database instance is closed and pending writes are finished.
|
|
596
|
+
*/
|
|
597
|
+
async close() {
|
|
598
|
+
// Wait for the current write lock promise chain to complete, if any exists
|
|
599
|
+
try {
|
|
600
|
+
// If there's a lock, await it. If not, nothing to wait for.
|
|
601
|
+
if (this.writeLock) {
|
|
602
|
+
await this.writeLock;
|
|
603
|
+
}
|
|
604
|
+
} catch (err) {
|
|
605
|
+
// Log error during final write if it occurs, but proceed with closing
|
|
606
|
+
console.error(
|
|
607
|
+
"[JSONDatabase] Error during final write operation while closing:",
|
|
608
|
+
err
|
|
609
|
+
);
|
|
610
|
+
} finally {
|
|
611
|
+
// Clear the cache and the lock reference *after* waiting
|
|
612
|
+
this.cache = null;
|
|
613
|
+
this.writeLock = null;
|
|
614
|
+
this._initPromise = null; // Reset init promise tracking
|
|
615
|
+
const stats = this.getStats(); // Get stats before resetting
|
|
616
|
+
console.log(
|
|
617
|
+
`[JSONDatabase] Closed connection to ${
|
|
618
|
+
this.filename
|
|
619
|
+
}. Final Stats: ${JSON.stringify(stats)}`
|
|
620
|
+
);
|
|
621
|
+
// Optionally reset stats, or leave them for inspection after close
|
|
622
|
+
// this.stats = { reads: 0, writes: 0, cacheHits: 0 };
|
|
623
|
+
}
|
|
624
|
+
}
|
|
625
|
+
}
|
|
626
|
+
|
|
627
|
+
module.exports = JSONDatabase;
|
package/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2025 Sethunthunder
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
package/README.md
ADDED
|
@@ -0,0 +1,118 @@
|
|
|
1
|
+
# JSON Database By ST
|
|
2
|
+
|
|
3
|
+
[](https://badge.fury.io/js/json-database-st)
|
|
4
|
+
[](https://opensource.org/licenses/MIT)
|
|
5
|
+
|
|
6
|
+
A simple, promise-based JSON file database for Node.js applications. Features atomic file operations, lodash integration for easy data access, transactions, batching, and basic querying.
|
|
7
|
+
|
|
8
|
+
Ideal for small projects, prototypes, configuration management, or simple data persistence needs where a full database server is overkill.
|
|
9
|
+
|
|
10
|
+
## Features
|
|
11
|
+
|
|
12
|
+
* **Promise-based API:** Fully asynchronous methods.
|
|
13
|
+
* **Atomic Writes:** Prevents data corruption from concurrent writes using an internal lock.
|
|
14
|
+
* **Simple API:** `get`, `set`, `has`, `delete`, `push` (deep unique), `pull` (deep removal), `clear`.
|
|
15
|
+
* **Lodash Integration:** Use dot/bracket notation paths (e.g., `'users.john.age'`) via `lodash`.
|
|
16
|
+
* **Transactions:** Execute multiple reads/writes as a single atomic unit (`transaction`).
|
|
17
|
+
* **Batch Operations:** Efficiently perform multiple `set`, `delete`, `push`, or `pull` operations in one atomic write (`batch`).
|
|
18
|
+
* **Basic Querying:** Filter object properties based on a predicate function (`query`).
|
|
19
|
+
* **File Auto-Creation:** Creates the JSON file (with `{}`) if it doesn't exist.
|
|
20
|
+
* **Pretty Print Option:** Optionally format the JSON file for readability.
|
|
21
|
+
* **Dependencies:** Only requires `lodash`. Uses built-in `fs.promises`.
|
|
22
|
+
|
|
23
|
+
## Installation
|
|
24
|
+
|
|
25
|
+
```bash
|
|
26
|
+
# Make sure you have lodash installed as well
|
|
27
|
+
npm install json-database-st lodash
|
|
28
|
+
# or
|
|
29
|
+
yarn add json-database-st lodash
|
|
30
|
+
```
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
## Quick Start
|
|
34
|
+
|
|
35
|
+
```javascript
|
|
36
|
+
const JSONDatabase = require('json-database-st'); // Use your package name
|
|
37
|
+
const path = require('path');
|
|
38
|
+
|
|
39
|
+
// Initialize (creates 'mydata.json' if needed)
|
|
40
|
+
const db = new JSONDatabase(path.join(__dirname, 'mydata'), { prettyPrint: true });
|
|
41
|
+
|
|
42
|
+
async function run() {
|
|
43
|
+
try {
|
|
44
|
+
// Set data
|
|
45
|
+
await db.set('user.name', 'Bob');
|
|
46
|
+
await db.set('user.settings.theme', 'dark');
|
|
47
|
+
|
|
48
|
+
// Get data
|
|
49
|
+
const theme = await db.get('user.settings.theme', 'light'); // Default value
|
|
50
|
+
console.log(`Theme: ${theme}`); // -> Theme: dark
|
|
51
|
+
|
|
52
|
+
// Push unique items (uses deep compare)
|
|
53
|
+
await db.push('user.tags', 'vip', { type: 'beta' });
|
|
54
|
+
await db.push('user.tags', { type: 'beta' }); // Won't be added again
|
|
55
|
+
console.log('Tags:', await db.get('user.tags')); // -> ['vip', { type: 'beta' }]
|
|
56
|
+
|
|
57
|
+
// Check existence
|
|
58
|
+
console.log('Has user age?', await db.has('user.age')); // -> false
|
|
59
|
+
|
|
60
|
+
// Delete
|
|
61
|
+
await db.delete('user.settings');
|
|
62
|
+
console.log('User object:', await db.get('user')); // -> { name: 'Bob', tags: [...] }
|
|
63
|
+
|
|
64
|
+
// Get Stats
|
|
65
|
+
console.log('DB Stats:', db.getStats());
|
|
66
|
+
|
|
67
|
+
} catch (err) {
|
|
68
|
+
console.error('Database operation failed:', err);
|
|
69
|
+
} finally {
|
|
70
|
+
// IMPORTANT: Always close the DB when done
|
|
71
|
+
await db.close();
|
|
72
|
+
console.log('Database closed.');
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
run();
|
|
77
|
+
```
|
|
78
|
+
|
|
79
|
+
## Documentation
|
|
80
|
+
|
|
81
|
+
**Full API details and advanced usage examples are available in the hosted documentation:**
|
|
82
|
+
|
|
83
|
+
**[View Documentation](https://sethunthunder111.github.io/json-database-st/)**
|
|
84
|
+
|
|
85
|
+
## API Summary
|
|
86
|
+
|
|
87
|
+
* `new JSONDatabase(filename, [options])`
|
|
88
|
+
* `async get(path, [defaultValue])`
|
|
89
|
+
* `async set(path, value)`
|
|
90
|
+
* `async has(path)`
|
|
91
|
+
* `async delete(path)`
|
|
92
|
+
* `async push(path, ...items)`
|
|
93
|
+
* `async pull(path, ...itemsToRemove)`
|
|
94
|
+
* `async transaction(asyncFn)`
|
|
95
|
+
* `async batch(operations)`
|
|
96
|
+
* `async query(predicateFn, [options])`
|
|
97
|
+
* `async clear()`
|
|
98
|
+
* `getStats()`
|
|
99
|
+
* `async close()`
|
|
100
|
+
* Properties: `filename`, `config`
|
|
101
|
+
|
|
102
|
+
## Concurrency and Atomicity
|
|
103
|
+
|
|
104
|
+
Writes are queued and executed one after another for a given instance, ensuring file integrity. Reads use an in-memory cache for speed. See Core Concepts in the full documentation for details.
|
|
105
|
+
|
|
106
|
+
## Limitations
|
|
107
|
+
|
|
108
|
+
* Best suited for small to medium-sized JSON files. Performance degrades with very large files.
|
|
109
|
+
* Loads the entire database into memory.
|
|
110
|
+
* Designed for single-process access to a given file. Not suitable for distributed systems.
|
|
111
|
+
|
|
112
|
+
## Contributing
|
|
113
|
+
|
|
114
|
+
Contributions (issues, PRs) are welcome! Please open an issue to discuss significant changes.
|
|
115
|
+
|
|
116
|
+
## License
|
|
117
|
+
|
|
118
|
+
[MIT](LICENSE)
|
package/package.json
ADDED
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "json-database-st",
|
|
3
|
+
"version": "1.0.0",
|
|
4
|
+
"description": "A simple, promise-based JSON file database for Node.js with atomic operations and lodash integration.",
|
|
5
|
+
"main": "JSONDatabase.js",
|
|
6
|
+
"types": "JSONDatabase.d.ts",
|
|
7
|
+
"scripts": {
|
|
8
|
+
"test": "echo \"Error: no test specified\" && exit 1"
|
|
9
|
+
},
|
|
10
|
+
"repository": {
|
|
11
|
+
"type": "git",
|
|
12
|
+
"url": "git+https://github.com/sethunthunder111/json-database-st.git"
|
|
13
|
+
},
|
|
14
|
+
"keywords": [
|
|
15
|
+
"json",
|
|
16
|
+
"database",
|
|
17
|
+
"file",
|
|
18
|
+
"fs",
|
|
19
|
+
"atomic",
|
|
20
|
+
"lodash",
|
|
21
|
+
"nosql",
|
|
22
|
+
"simple",
|
|
23
|
+
"promise",
|
|
24
|
+
"db",
|
|
25
|
+
"node"
|
|
26
|
+
],
|
|
27
|
+
"author": "Sethunthunder",
|
|
28
|
+
"license": "MIT",
|
|
29
|
+
"bugs": {
|
|
30
|
+
"url": "https://github.com/sethunthunder111/json-database-st/issues"
|
|
31
|
+
},
|
|
32
|
+
"homepage": "https://github.com/sethunthunder111/json-database-st#readme",
|
|
33
|
+
"dependencies": {
|
|
34
|
+
"lodash": "^4.17.21"
|
|
35
|
+
},
|
|
36
|
+
"engines": {
|
|
37
|
+
"node": ">=14.0.0"
|
|
38
|
+
},
|
|
39
|
+
"files": [
|
|
40
|
+
"JSONDatabase.js",
|
|
41
|
+
"LICENSE",
|
|
42
|
+
"README.md",
|
|
43
|
+
"JSONDatabase.d.ts"
|
|
44
|
+
]
|
|
45
|
+
}
|