jexidb 2.0.2 → 2.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (55) hide show
  1. package/.babelrc +13 -0
  2. package/.gitattributes +2 -0
  3. package/CHANGELOG.md +140 -0
  4. package/LICENSE +21 -21
  5. package/README.md +301 -527
  6. package/babel.config.json +5 -0
  7. package/dist/Database.cjs +3896 -0
  8. package/docs/API.md +1051 -0
  9. package/docs/EXAMPLES.md +701 -0
  10. package/docs/README.md +194 -0
  11. package/examples/iterate-usage-example.js +157 -0
  12. package/examples/simple-iterate-example.js +115 -0
  13. package/jest.config.js +24 -0
  14. package/package.json +63 -51
  15. package/scripts/README.md +47 -0
  16. package/scripts/clean-test-files.js +75 -0
  17. package/scripts/prepare.js +31 -0
  18. package/scripts/run-tests.js +80 -0
  19. package/src/Database.mjs +4130 -0
  20. package/src/FileHandler.mjs +1101 -0
  21. package/src/OperationQueue.mjs +279 -0
  22. package/src/SchemaManager.mjs +268 -0
  23. package/src/Serializer.mjs +511 -0
  24. package/src/managers/ConcurrencyManager.mjs +257 -0
  25. package/src/managers/IndexManager.mjs +1403 -0
  26. package/src/managers/QueryManager.mjs +1273 -0
  27. package/src/managers/StatisticsManager.mjs +262 -0
  28. package/src/managers/StreamingProcessor.mjs +429 -0
  29. package/src/managers/TermManager.mjs +278 -0
  30. package/test/$not-operator-with-and.test.js +282 -0
  31. package/test/README.md +8 -0
  32. package/test/close-init-cycle.test.js +256 -0
  33. package/test/critical-bugs-fixes.test.js +1069 -0
  34. package/test/index-persistence.test.js +306 -0
  35. package/test/index-serialization.test.js +314 -0
  36. package/test/indexed-query-mode.test.js +360 -0
  37. package/test/iterate-method.test.js +272 -0
  38. package/test/query-operators.test.js +238 -0
  39. package/test/regex-array-fields.test.js +129 -0
  40. package/test/score-method.test.js +238 -0
  41. package/test/setup.js +17 -0
  42. package/test/term-mapping-minimal.test.js +154 -0
  43. package/test/term-mapping-simple.test.js +257 -0
  44. package/test/term-mapping.test.js +514 -0
  45. package/test/writebuffer-flush-resilience.test.js +204 -0
  46. package/dist/FileHandler.js +0 -688
  47. package/dist/IndexManager.js +0 -353
  48. package/dist/IntegrityChecker.js +0 -364
  49. package/dist/JSONLDatabase.js +0 -1194
  50. package/dist/index.js +0 -617
  51. package/src/FileHandler.js +0 -674
  52. package/src/IndexManager.js +0 -363
  53. package/src/IntegrityChecker.js +0 -379
  54. package/src/JSONLDatabase.js +0 -1248
  55. package/src/index.js +0 -608
package/dist/index.js DELETED
@@ -1,617 +0,0 @@
1
- "use strict";
2
-
3
- Object.defineProperty(exports, "__esModule", {
4
- value: true
5
- });
6
- exports.Database = void 0;
7
- Object.defineProperty(exports, "FileHandler", {
8
- enumerable: true,
9
- get: function () {
10
- return _FileHandler.default;
11
- }
12
- });
13
- Object.defineProperty(exports, "IndexManager", {
14
- enumerable: true,
15
- get: function () {
16
- return _IndexManager.default;
17
- }
18
- });
19
- Object.defineProperty(exports, "IntegrityChecker", {
20
- enumerable: true,
21
- get: function () {
22
- return _IntegrityChecker.default;
23
- }
24
- });
25
- exports.utils = exports.default = exports.OPERATORS = void 0;
26
- var _JSONLDatabase = _interopRequireDefault(require("./JSONLDatabase.js"));
27
- var _FileHandler = _interopRequireDefault(require("./FileHandler.js"));
28
- var _IndexManager = _interopRequireDefault(require("./IndexManager.js"));
29
- var _IntegrityChecker = _interopRequireDefault(require("./IntegrityChecker.js"));
30
- function _interopRequireDefault(e) { return e && e.__esModule ? e : { default: e }; }
31
- function _interopRequireWildcard(e, t) { if ("function" == typeof WeakMap) var r = new WeakMap(), n = new WeakMap(); return (_interopRequireWildcard = function (e, t) { if (!t && e && e.__esModule) return e; var o, i, f = { __proto__: null, default: e }; if (null === e || "object" != typeof e && "function" != typeof e) return f; if (o = t ? n : r) { if (o.has(e)) return o.get(e); o.set(e, f); } for (const t in e) "default" !== t && {}.hasOwnProperty.call(e, t) && ((i = (o = Object.defineProperty) && Object.getOwnPropertyDescriptor(e, t)) && (i.get || i.set) ? o(f, t, i) : f[t] = e[t]); return f; })(e, t); }
32
- /**
33
- * JexiDB Compatibility Wrapper
34
- * Extends JSONLDatabase to provide backward compatibility with JexiDB 1.x test expectations
35
- */
36
- class JexiDBCompatibility extends _JSONLDatabase.default {
37
- constructor(filePath, options = {}) {
38
- // Support both .jdb and .jsonl extensions
39
- // .jdb is the preferred extension for JexiDB databases
40
- let normalizedPath = filePath;
41
-
42
- // If no extension is provided, default to .jdb
43
- if (!filePath.includes('.')) {
44
- normalizedPath = filePath + '.jdb';
45
- }
46
-
47
- // If .jdb extension is used, it's internally stored as JSONL format
48
- // but the user sees .jdb for better branding
49
- if (normalizedPath.endsWith('.jdb')) {
50
- // Store internally as .jsonl but present as .jdb to user
51
- const jsonlPath = normalizedPath.replace('.jdb', '.jsonl');
52
- super(jsonlPath, options);
53
- this.userPath = normalizedPath; // Keep track of user's preferred path
54
- } else {
55
- super(normalizedPath, options);
56
- this.userPath = normalizedPath;
57
- }
58
- this.isDestroyed = false;
59
- }
60
-
61
- /**
62
- * Get the user's preferred file path (with .jdb extension if used)
63
- */
64
- get userFilePath() {
65
- return this.userPath;
66
- }
67
-
68
- /**
69
- * Compatibility method: destroy() -> close()
70
- */
71
- async destroy() {
72
- this.isDestroyed = true;
73
- return this.close();
74
- }
75
-
76
- /**
77
- * Compatibility method: findOne() -> find() with limit 1
78
- */
79
- async findOne(criteria = {}) {
80
- const results = await this.find(criteria, {
81
- limit: 1
82
- });
83
- return results.length > 0 ? results[0] : null;
84
- }
85
-
86
- /**
87
- * Enhanced find method with options support
88
- */
89
- async find(criteria = {}, options = {}) {
90
- let results = await super.find(criteria);
91
-
92
- // Apply sorting
93
- if (options.sort) {
94
- results = this.sortResults(results, options.sort);
95
- }
96
-
97
- // Apply skip
98
- if (options.skip) {
99
- results = results.slice(options.skip);
100
- }
101
-
102
- // Apply limit
103
- if (options.limit) {
104
- results = results.slice(0, options.limit);
105
- }
106
- return results;
107
- }
108
-
109
- /**
110
- * Retrocompatibility method: query() -> find()
111
- * Supports the same API as JexiDB 1.x
112
- */
113
- async query(criteria = {}, options = {}) {
114
- // Handle caseInsensitive option from JexiDB 1.x
115
- if (options.caseInsensitive) {
116
- // For case insensitive queries, we need to modify the criteria
117
- const caseInsensitiveCriteria = {};
118
- for (const [key, value] of Object.entries(criteria)) {
119
- if (typeof value === 'string') {
120
- // Convert string values to regex for case insensitive matching
121
- caseInsensitiveCriteria[key] = {
122
- $regex: value,
123
- $options: 'i'
124
- };
125
- } else {
126
- caseInsensitiveCriteria[key] = value;
127
- }
128
- }
129
- criteria = caseInsensitiveCriteria;
130
- }
131
- return await this.find(criteria, options);
132
- }
133
-
134
- /**
135
- * Sort results based on criteria
136
- */
137
- sortResults(results, sortCriteria) {
138
- return results.sort((a, b) => {
139
- for (const [field, direction] of Object.entries(sortCriteria)) {
140
- const aValue = this.getNestedValue(a, field);
141
- const bValue = this.getNestedValue(b, field);
142
- if (aValue < bValue) return direction === 1 ? -1 : 1;
143
- if (aValue > bValue) return direction === 1 ? 1 : -1;
144
- }
145
- return 0;
146
- });
147
- }
148
-
149
- /**
150
- * Get nested value from record (copied from parent class)
151
- */
152
- getNestedValue(record, field) {
153
- const parts = field.split('.');
154
- let value = record;
155
- for (const part of parts) {
156
- if (value && typeof value === 'object' && part in value) {
157
- value = value[part];
158
- } else {
159
- return undefined;
160
- }
161
- }
162
- return value;
163
- }
164
-
165
- /**
166
- * Compatibility method: insertMany() -> multiple insert() calls
167
- */
168
- async insertMany(records) {
169
- const results = [];
170
- for (const record of records) {
171
- const result = await this.insert(record);
172
- results.push(result);
173
- }
174
- return results;
175
- }
176
-
177
- /**
178
- * Override insert to add _updated field for compatibility
179
- */
180
- async insert(data) {
181
- const record = await super.insert(data);
182
- record._updated = record._created; // Set _updated to same as _created for new records
183
- return record;
184
- }
185
-
186
- /**
187
- * Compatibility method: count() -> find() with length
188
- */
189
- async count(criteria = {}) {
190
- const results = await this.find(criteria);
191
- return results.length;
192
- }
193
-
194
- /**
195
- * Compatibility method: getStats() -> stats getter
196
- */
197
- async getStats() {
198
- // Call the parent class getStats method to get actual file size
199
- return await super.getStats();
200
- }
201
-
202
- /**
203
- * Compatibility method: validateIntegrity() -> basic validation
204
- */
205
- async validateIntegrity() {
206
- // Call the parent class validateIntegrity method to get actual file integrity check
207
- return await super.validateIntegrity();
208
- }
209
-
210
- /**
211
- * Streaming iterator over records without loading all into memory
212
- * Supports a limited subset of options: { limit }
213
- */
214
- async *walk(options = {}) {
215
- // Delegate to core engine streaming implementation
216
- // Ensure pending inserts are flushed by calling super.walk
217
- for await (const record of super.walk({
218
- limit: options.limit
219
- })) {
220
- yield record;
221
- }
222
- }
223
-
224
- /**
225
- * Compatibility property: indexStats
226
- */
227
- get indexStats() {
228
- const stats = this.stats;
229
- return {
230
- recordCount: stats.recordCount,
231
- indexCount: stats.indexedFields.length
232
- };
233
- }
234
-
235
- /**
236
- * Override update to return array format for compatibility
237
- */
238
- async update(criteria, updates) {
239
- const result = await super.update(criteria, updates);
240
- // Convert { updatedCount: n } to array format for tests
241
- if (typeof result === 'object' && result.updatedCount !== undefined) {
242
- const updatedRecords = await this.find(criteria);
243
- return updatedRecords;
244
- }
245
- return result;
246
- }
247
-
248
- /**
249
- * Override delete to return number format for compatibility
250
- */
251
- async delete(criteria) {
252
- const result = await super.delete(criteria);
253
- // Convert { deletedCount: n } to number format for tests
254
- if (typeof result === 'object' && result.deletedCount !== undefined) {
255
- return result.deletedCount;
256
- }
257
- return result;
258
- }
259
-
260
- /**
261
- * Compatibility method: readColumnIndex - gets unique values from indexed columns only
262
- * Maintains compatibility with JexiDB v1 code
263
- * @param {string} column - The column name to get unique values from
264
- * @returns {Set} Set of unique values in the column (indexed columns only)
265
- */
266
- readColumnIndex(column) {
267
- return super.readColumnIndex(column);
268
- }
269
- }
270
-
271
- /**
272
- * JexiDB - Robust JSONL database
273
- * Complete rewrite of JexiDB with JSONL architecture, fixing all critical bugs from version 1.x
274
- *
275
- * Features:
276
- * - One file per table (pure JSONL)
277
- * - Punctual reading (doesn't load everything in memory)
278
- * - In-memory indexes for performance
279
- * - Safe truncation after operations
280
- * - Integrity validation
281
- * - Event-driven architecture
282
- *
283
- * API similar to JexiDB 1.x:
284
- * - new JexiDB('file.jsonl', { indexes: { id: 'number' } })
285
- * - await db.init()
286
- * - await db.insert({ id: 1, name: 'John' })
287
- * - await db.find({ id: 1 })
288
- * - await db.update({ id: 1 }, { name: 'John Smith' })
289
- * - await db.delete({ id: 1 })
290
- * - await db.save()
291
- * - await db.destroy()
292
- * - await db.walk() - Iterator for large volumes
293
- * - await db.validateIntegrity() - Manual verification
294
- */
295
-
296
- // Export the compatibility wrapper as default
297
- var _default = exports.default = JexiDBCompatibility; // Retrocompat: named export 'Database' (JexiDB v1 used { Database })
298
- const Database = exports.Database = JexiDBCompatibility;
299
-
300
- // Export auxiliary classes for advanced use
301
-
302
- // Export useful constants
303
- const OPERATORS = exports.OPERATORS = {
304
- GT: '>',
305
- GTE: '>=',
306
- LT: '<',
307
- LTE: '<=',
308
- NE: '!=',
309
- IN: 'in',
310
- NIN: 'nin',
311
- REGEX: 'regex',
312
- CONTAINS: 'contains'
313
- };
314
-
315
- // Export utility functions
316
- const utils = exports.utils = {
317
- /**
318
- * Creates a database with default settings
319
- */
320
- createDatabase(filePath, indexes = {}) {
321
- return new _JSONLDatabase.default(filePath, {
322
- indexes
323
- });
324
- },
325
- /**
326
- * Validates if a JSONL file is valid
327
- * @param {string} filePath - Path to the JSONL file
328
- * @returns {Promise<Object>} Validation result with errors and line count
329
- */
330
- async validateJSONLFile(filePath) {
331
- const {
332
- promises: fs
333
- } = await Promise.resolve().then(() => _interopRequireWildcard(require('fs')));
334
- const readline = await Promise.resolve().then(() => _interopRequireWildcard(require('readline')));
335
- const {
336
- createReadStream
337
- } = await Promise.resolve().then(() => _interopRequireWildcard(require('fs')));
338
- try {
339
- const fileStream = createReadStream(filePath);
340
- const rl = readline.createInterface({
341
- input: fileStream,
342
- crlfDelay: Infinity
343
- });
344
- let lineNumber = 0;
345
- const errors = [];
346
- for await (const line of rl) {
347
- lineNumber++;
348
- if (line.trim() !== '') {
349
- try {
350
- JSON.parse(line);
351
- } catch (error) {
352
- errors.push(`Line ${lineNumber}: ${error.message}`);
353
- }
354
- }
355
- }
356
- return {
357
- isValid: errors.length === 0,
358
- errors,
359
- lineCount: lineNumber
360
- };
361
- } catch (error) {
362
- return {
363
- isValid: false,
364
- errors: [`Error reading file: ${error.message}`],
365
- lineCount: 0
366
- };
367
- }
368
- },
369
- /**
370
- * Converts a JSON file to JSONL (basic conversion)
371
- * @param {string} jsonFilePath - Path to the JSON file
372
- * @param {string} jsonlFilePath - Path to the output JSONL file
373
- * @returns {Promise<Object>} Conversion result
374
- */
375
- async convertJSONToJSONL(jsonFilePath, jsonlFilePath) {
376
- const {
377
- promises: fs
378
- } = await Promise.resolve().then(() => _interopRequireWildcard(require('fs')));
379
- try {
380
- const jsonData = await fs.readFile(jsonFilePath, 'utf8');
381
- const data = JSON.parse(jsonData);
382
- const records = Array.isArray(data) ? data : [data];
383
- const jsonlContent = records.map(record => JSON.stringify(record)).join('\n') + '\n';
384
- await fs.writeFile(jsonlFilePath, jsonlContent, 'utf8');
385
- return {
386
- success: true,
387
- recordCount: records.length
388
- };
389
- } catch (error) {
390
- return {
391
- success: false,
392
- error: error.message
393
- };
394
- }
395
- },
396
- /**
397
- * Converts a JSONL file to JSON
398
- * @param {string} jsonlFilePath - Path to the JSONL file
399
- * @param {string} jsonFilePath - Path to the output JSON file
400
- * @returns {Promise<Object>} Conversion result
401
- */
402
- async convertJSONLToJSON(jsonlFilePath, jsonFilePath) {
403
- const {
404
- promises: fs
405
- } = await Promise.resolve().then(() => _interopRequireWildcard(require('fs')));
406
- const readline = await Promise.resolve().then(() => _interopRequireWildcard(require('readline')));
407
- const {
408
- createReadStream
409
- } = await Promise.resolve().then(() => _interopRequireWildcard(require('fs')));
410
- try {
411
- const fileStream = createReadStream(jsonlFilePath);
412
- const rl = readline.createInterface({
413
- input: fileStream,
414
- crlfDelay: Infinity
415
- });
416
- const records = [];
417
- for await (const line of rl) {
418
- if (line.trim() !== '') {
419
- try {
420
- const record = JSON.parse(line);
421
- records.push(record);
422
- } catch (error) {
423
- console.warn(`Line ignored: ${error.message}`);
424
- }
425
- }
426
- }
427
- const jsonContent = JSON.stringify(records, null, 2);
428
- await fs.writeFile(jsonFilePath, jsonContent, 'utf8');
429
- return {
430
- success: true,
431
- recordCount: records.length
432
- };
433
- } catch (error) {
434
- return {
435
- success: false,
436
- error: error.message
437
- };
438
- }
439
- },
440
- /**
441
- * Creates a JexiDB database from a JSON file with automatic index detection
442
- * @param {string} jsonFilePath - Path to the JSON file
443
- * @param {string} dbFilePath - Path to the output JexiDB file
444
- * @param {Object} options - Options for database creation
445
- * @param {Object} options.indexes - Manual index configuration
446
- * @param {boolean} options.autoDetectIndexes - Auto-detect common index fields (default: true)
447
- * @param {Array<string>} options.autoIndexFields - Fields to auto-index (default: ['id', '_id', 'email', 'name'])
448
- * @returns {Promise<Object>} Database creation result
449
- */
450
- async createDatabaseFromJSON(jsonFilePath, dbFilePath, options = {}) {
451
- const {
452
- promises: fs
453
- } = await Promise.resolve().then(() => _interopRequireWildcard(require('fs')));
454
- try {
455
- // Read JSON data
456
- const jsonData = await fs.readFile(jsonFilePath, 'utf8');
457
- const data = JSON.parse(jsonData);
458
- const records = Array.isArray(data) ? data : [data];
459
- if (records.length === 0) {
460
- return {
461
- success: false,
462
- error: 'No records found in JSON file'
463
- };
464
- }
465
-
466
- // Auto-detect indexes if enabled
467
- let indexes = options.indexes || {};
468
- if (options.autoDetectIndexes !== false) {
469
- const autoIndexFields = options.autoIndexFields || ['id', '_id', 'email', 'name', 'username'];
470
- const sampleRecord = records[0];
471
- for (const field of autoIndexFields) {
472
- if (sampleRecord.hasOwnProperty(field)) {
473
- const value = sampleRecord[field];
474
- if (typeof value === 'number') {
475
- indexes[field] = 'number';
476
- } else if (typeof value === 'string') {
477
- indexes[field] = 'string';
478
- }
479
- }
480
- }
481
- }
482
-
483
- // Create database
484
- const db = new _JSONLDatabase.default(dbFilePath, {
485
- indexes,
486
- autoSave: false,
487
- validateOnInit: false
488
- });
489
- await db.init();
490
-
491
- // Insert all records
492
- for (const record of records) {
493
- await db.insert(record);
494
- }
495
-
496
- // Save and close the database
497
- await db.save();
498
- await db.close();
499
- return {
500
- success: true,
501
- recordCount: records.length,
502
- indexes: Object.keys(indexes),
503
- dbPath: dbFilePath
504
- };
505
- } catch (error) {
506
- return {
507
- success: false,
508
- error: error.message
509
- };
510
- }
511
- },
512
- /**
513
- * Analyzes a JSON file and suggests optimal indexes
514
- * @param {string} jsonFilePath - Path to the JSON file
515
- * @param {number} sampleSize - Number of records to analyze (default: 100)
516
- * @returns {Promise<Object>} Index suggestions
517
- */
518
- async analyzeJSONForIndexes(jsonFilePath, sampleSize = 100) {
519
- const {
520
- promises: fs
521
- } = await Promise.resolve().then(() => _interopRequireWildcard(require('fs')));
522
- try {
523
- const jsonData = await fs.readFile(jsonFilePath, 'utf8');
524
- const data = JSON.parse(jsonData);
525
- const records = Array.isArray(data) ? data : [data];
526
- if (records.length === 0) {
527
- return {
528
- success: false,
529
- error: 'No records found in JSON file'
530
- };
531
- }
532
-
533
- // Analyze sample records
534
- const sample = records.slice(0, Math.min(sampleSize, records.length));
535
- const fieldAnalysis = {};
536
-
537
- // First, collect all possible fields from all records
538
- const allFields = new Set();
539
- for (const record of sample) {
540
- for (const field of Object.keys(record)) {
541
- allFields.add(field);
542
- }
543
- }
544
-
545
- // Initialize analysis for all fields
546
- for (const field of allFields) {
547
- fieldAnalysis[field] = {
548
- type: 'unknown',
549
- uniqueValues: new Set(),
550
- nullCount: 0,
551
- totalCount: 0
552
- };
553
- }
554
-
555
- // Analyze each field across all records
556
- for (const record of sample) {
557
- for (const field of allFields) {
558
- const value = record[field];
559
- fieldAnalysis[field].totalCount++;
560
- if (value === null || value === undefined) {
561
- fieldAnalysis[field].nullCount++;
562
- } else {
563
- if (fieldAnalysis[field].type === 'unknown') {
564
- fieldAnalysis[field].type = typeof value;
565
- }
566
- fieldAnalysis[field].uniqueValues.add(value);
567
- }
568
- }
569
- }
570
-
571
- // Generate suggestions
572
- const suggestions = {
573
- recommended: [],
574
- optional: [],
575
- notRecommended: []
576
- };
577
- for (const [field, analysis] of Object.entries(fieldAnalysis)) {
578
- const coverage = (analysis.totalCount - analysis.nullCount) / analysis.totalCount;
579
- const uniqueness = analysis.uniqueValues.size / analysis.totalCount;
580
- const suggestion = {
581
- field,
582
- type: analysis.type,
583
- coverage: Math.round(coverage * 100),
584
- uniqueness: Math.round(uniqueness * 100),
585
- uniqueValues: analysis.uniqueValues.size
586
- };
587
-
588
- // Recommendation logic
589
- if (coverage > 0.9 && uniqueness > 0.8) {
590
- suggestions.recommended.push(suggestion);
591
- } else if (coverage > 0.7 && uniqueness > 0.5) {
592
- suggestions.optional.push(suggestion);
593
- } else {
594
- suggestions.notRecommended.push(suggestion);
595
- }
596
- }
597
-
598
- // Convert suggestions to the expected format for tests
599
- const suggestedIndexes = {};
600
- for (const suggestion of suggestions.recommended) {
601
- suggestedIndexes[suggestion.field] = suggestion.type;
602
- }
603
- return {
604
- success: true,
605
- totalRecords: records.length,
606
- analyzedRecords: sample.length,
607
- suggestedIndexes,
608
- suggestions
609
- };
610
- } catch (error) {
611
- return {
612
- success: false,
613
- error: error.message
614
- };
615
- }
616
- }
617
- };