jexidb 2.1.0 → 2.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/Database.cjs +1642 -334
- package/docs/API.md +1057 -1051
- package/package.json +1 -1
- package/scripts/benchmark-array-serialization.js +108 -0
- package/scripts/score-mode-demo.js +45 -0
- package/src/Database.mjs +1362 -167
- package/src/FileHandler.mjs +83 -44
- package/src/OperationQueue.mjs +23 -23
- package/src/Serializer.mjs +214 -23
- package/src/managers/IndexManager.mjs +778 -87
- package/src/managers/QueryManager.mjs +266 -49
- package/src/managers/TermManager.mjs +7 -7
- package/src/utils/operatorNormalizer.mjs +116 -0
- package/test/coverage-method.test.js +93 -0
- package/test/deserialize-corruption-fixes.test.js +296 -0
- package/test/exists-method.test.js +318 -0
- package/test/explicit-indexes-comparison.test.js +219 -0
- package/test/filehandler-non-adjacent-ranges-bug.test.js +175 -0
- package/test/index-line-number-regression.test.js +100 -0
- package/test/index-missing-index-data.test.js +91 -0
- package/test/index-persistence.test.js +205 -20
- package/test/insert-session-auto-flush.test.js +353 -0
- package/test/legacy-operator-compat.test.js +154 -0
- package/test/score-method.test.js +60 -0
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
import { normalizeOperator } from '../utils/operatorNormalizer.mjs'
|
|
2
|
+
|
|
1
3
|
/**
|
|
2
4
|
* QueryManager - Handles all query operations and strategies
|
|
3
5
|
*
|
|
@@ -35,6 +37,9 @@ export class QueryManager {
|
|
|
35
37
|
if (this.database.destroyed) throw new Error('Database is destroyed')
|
|
36
38
|
if (!this.database.initialized) await this.database.init()
|
|
37
39
|
|
|
40
|
+
// Rebuild indexes if needed (when index was corrupted/missing)
|
|
41
|
+
await this.database._rebuildIndexesIfNeeded()
|
|
42
|
+
|
|
38
43
|
// Manual save is now the responsibility of the application
|
|
39
44
|
|
|
40
45
|
// Preprocess query to handle array field syntax automatically
|
|
@@ -44,8 +49,8 @@ export class QueryManager {
|
|
|
44
49
|
|
|
45
50
|
// Validate strict indexed mode before processing
|
|
46
51
|
if (this.opts.indexedQueryMode === 'strict') {
|
|
47
|
-
this.validateStrictQuery(finalCriteria);
|
|
48
|
-
}
|
|
52
|
+
this.validateStrictQuery(finalCriteria, options);
|
|
53
|
+
}
|
|
49
54
|
|
|
50
55
|
const startTime = Date.now();
|
|
51
56
|
this.usageStats.totalQueries++;
|
|
@@ -99,7 +104,7 @@ export class QueryManager {
|
|
|
99
104
|
|
|
100
105
|
// Validate strict indexed mode before processing
|
|
101
106
|
if (this.opts.indexedQueryMode === 'strict') {
|
|
102
|
-
this.validateStrictQuery(processedCriteria);
|
|
107
|
+
this.validateStrictQuery(processedCriteria, options);
|
|
103
108
|
}
|
|
104
109
|
|
|
105
110
|
const startTime = Date.now();
|
|
@@ -147,11 +152,15 @@ export class QueryManager {
|
|
|
147
152
|
async count(criteria, options = {}) {
|
|
148
153
|
if (this.database.destroyed) throw new Error('Database is destroyed')
|
|
149
154
|
if (!this.database.initialized) await this.database.init()
|
|
155
|
+
|
|
156
|
+
// Rebuild indexes if needed (when index was corrupted/missing)
|
|
157
|
+
await this.database._rebuildIndexesIfNeeded()
|
|
158
|
+
|
|
150
159
|
// Manual save is now the responsibility of the application
|
|
151
160
|
|
|
152
161
|
// Validate strict indexed mode before processing
|
|
153
162
|
if (this.opts.indexedQueryMode === 'strict') {
|
|
154
|
-
this.validateStrictQuery(criteria);
|
|
163
|
+
this.validateStrictQuery(criteria, options);
|
|
155
164
|
}
|
|
156
165
|
|
|
157
166
|
// Use the same strategy as find method
|
|
@@ -164,9 +173,50 @@ export class QueryManager {
|
|
|
164
173
|
const results = await this.findWithStreaming(criteria, options);
|
|
165
174
|
count = results.length;
|
|
166
175
|
} else {
|
|
167
|
-
//
|
|
168
|
-
|
|
169
|
-
|
|
176
|
+
// OPTIMIZATION: For indexed strategy, use indexManager.query().size directly
|
|
177
|
+
// This avoids reading actual records from the file - much faster!
|
|
178
|
+
const lineNumbers = this.indexManager.query(criteria, options);
|
|
179
|
+
|
|
180
|
+
if (lineNumbers.size === 0) {
|
|
181
|
+
const missingIndexedFields = this._getIndexedFieldsWithMissingData(criteria)
|
|
182
|
+
if (missingIndexedFields.length > 0 && this._hasAnyRecords()) {
|
|
183
|
+
// Try to rebuild index before falling back to streaming (only if allowIndexRebuild is true)
|
|
184
|
+
if (this.database.opts.allowIndexRebuild) {
|
|
185
|
+
if (this.opts.debugMode) {
|
|
186
|
+
console.log(`⚠️ Indexed count returned 0 because index data is missing for: ${missingIndexedFields.join(', ')}. Attempting index rebuild...`);
|
|
187
|
+
}
|
|
188
|
+
this.database._indexRebuildNeeded = true
|
|
189
|
+
await this.database._rebuildIndexesIfNeeded()
|
|
190
|
+
|
|
191
|
+
// Retry indexed query after rebuild
|
|
192
|
+
const retryLineNumbers = this.indexManager.query(criteria, options)
|
|
193
|
+
if (retryLineNumbers.size > 0) {
|
|
194
|
+
if (this.opts.debugMode) {
|
|
195
|
+
console.log(`✅ Index rebuild successful, using indexed strategy.`);
|
|
196
|
+
}
|
|
197
|
+
count = retryLineNumbers.size
|
|
198
|
+
} else {
|
|
199
|
+
// Still no results after rebuild, fall back to streaming
|
|
200
|
+
if (this.opts.debugMode) {
|
|
201
|
+
console.log(`⚠️ Index rebuild did not help, falling back to streaming count.`);
|
|
202
|
+
}
|
|
203
|
+
const streamingResults = await this.findWithStreaming(criteria, { ...options, forceFullScan: true })
|
|
204
|
+
count = streamingResults.length
|
|
205
|
+
}
|
|
206
|
+
} else {
|
|
207
|
+
// allowIndexRebuild is false, fall back to streaming
|
|
208
|
+
if (this.opts.debugMode) {
|
|
209
|
+
console.log(`⚠️ Indexed count returned 0 because index data is missing for: ${missingIndexedFields.join(', ')}. Falling back to streaming count.`);
|
|
210
|
+
}
|
|
211
|
+
const streamingResults = await this.findWithStreaming(criteria, { ...options, forceFullScan: true })
|
|
212
|
+
count = streamingResults.length
|
|
213
|
+
}
|
|
214
|
+
} else {
|
|
215
|
+
count = 0
|
|
216
|
+
}
|
|
217
|
+
} else {
|
|
218
|
+
count = lineNumbers.size;
|
|
219
|
+
}
|
|
170
220
|
}
|
|
171
221
|
|
|
172
222
|
return count;
|
|
@@ -189,33 +239,43 @@ export class QueryManager {
|
|
|
189
239
|
* @returns {Promise<Array>} - Query results
|
|
190
240
|
*/
|
|
191
241
|
async findWithStreaming(criteria, options = {}) {
|
|
242
|
+
const streamingOptions = { ...options }
|
|
243
|
+
const forceFullScan = streamingOptions.forceFullScan === true
|
|
244
|
+
delete streamingOptions.forceFullScan
|
|
245
|
+
|
|
192
246
|
if (this.opts.debugMode) {
|
|
193
|
-
|
|
247
|
+
if (forceFullScan) {
|
|
248
|
+
console.log('🌊 Using streaming strategy (forced full scan to bypass missing index data)');
|
|
249
|
+
} else {
|
|
250
|
+
console.log('🌊 Using streaming strategy');
|
|
251
|
+
}
|
|
194
252
|
}
|
|
195
253
|
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
if (
|
|
200
|
-
|
|
254
|
+
if (!forceFullScan) {
|
|
255
|
+
// OPTIMIZATION: Try to use indices for pre-filtering when possible
|
|
256
|
+
const indexableFields = this._getIndexableFields(criteria);
|
|
257
|
+
if (indexableFields.length > 0) {
|
|
258
|
+
if (this.opts.debugMode) {
|
|
259
|
+
console.log(`🌊 Using pre-filtered streaming with ${indexableFields.length} indexable fields`);
|
|
260
|
+
}
|
|
261
|
+
|
|
262
|
+
// Use indices to pre-filter and reduce streaming scope
|
|
263
|
+
const preFilteredLines = this.indexManager.query(
|
|
264
|
+
this._extractIndexableCriteria(criteria),
|
|
265
|
+
streamingOptions
|
|
266
|
+
);
|
|
267
|
+
|
|
268
|
+
// Stream only the pre-filtered records
|
|
269
|
+
return this._streamPreFilteredRecords(preFilteredLines, criteria, streamingOptions);
|
|
201
270
|
}
|
|
202
|
-
|
|
203
|
-
// Use indices to pre-filter and reduce streaming scope
|
|
204
|
-
const preFilteredLines = this.indexManager.query(
|
|
205
|
-
this._extractIndexableCriteria(criteria),
|
|
206
|
-
options
|
|
207
|
-
);
|
|
208
|
-
|
|
209
|
-
// Stream only the pre-filtered records
|
|
210
|
-
return this._streamPreFilteredRecords(preFilteredLines, criteria, options);
|
|
211
271
|
}
|
|
212
272
|
|
|
213
273
|
// Fallback to full streaming
|
|
214
274
|
if (this.opts.debugMode) {
|
|
215
|
-
console.log('🌊 Using full streaming (no indexable fields found)');
|
|
275
|
+
console.log('🌊 Using full streaming (no indexable fields found or forced)');
|
|
216
276
|
}
|
|
217
277
|
|
|
218
|
-
return this._streamAllRecords(criteria,
|
|
278
|
+
return this._streamAllRecords(criteria, streamingOptions);
|
|
219
279
|
}
|
|
220
280
|
|
|
221
281
|
/**
|
|
@@ -294,6 +354,96 @@ export class QueryManager {
|
|
|
294
354
|
return indexableCriteria;
|
|
295
355
|
}
|
|
296
356
|
|
|
357
|
+
/**
|
|
358
|
+
* Determine whether the database currently has any records (persisted or pending)
|
|
359
|
+
* @returns {boolean}
|
|
360
|
+
*/
|
|
361
|
+
_hasAnyRecords() {
|
|
362
|
+
if (!this.database) {
|
|
363
|
+
return false
|
|
364
|
+
}
|
|
365
|
+
|
|
366
|
+
if (Array.isArray(this.database.offsets) && this.database.offsets.length > 0) {
|
|
367
|
+
return true
|
|
368
|
+
}
|
|
369
|
+
|
|
370
|
+
if (Array.isArray(this.database.writeBuffer) && this.database.writeBuffer.length > 0) {
|
|
371
|
+
return true
|
|
372
|
+
}
|
|
373
|
+
|
|
374
|
+
if (typeof this.database.length === 'number' && this.database.length > 0) {
|
|
375
|
+
return true
|
|
376
|
+
}
|
|
377
|
+
|
|
378
|
+
return false
|
|
379
|
+
}
|
|
380
|
+
|
|
381
|
+
/**
|
|
382
|
+
* Extract all indexed fields referenced in the criteria
|
|
383
|
+
* @param {Object} criteria
|
|
384
|
+
* @param {Set<string>} accumulator
|
|
385
|
+
* @returns {Array<string>}
|
|
386
|
+
*/
|
|
387
|
+
_extractIndexedFields(criteria, accumulator = new Set()) {
|
|
388
|
+
if (!criteria) {
|
|
389
|
+
return Array.from(accumulator)
|
|
390
|
+
}
|
|
391
|
+
|
|
392
|
+
if (Array.isArray(criteria)) {
|
|
393
|
+
for (const item of criteria) {
|
|
394
|
+
this._extractIndexedFields(item, accumulator)
|
|
395
|
+
}
|
|
396
|
+
return Array.from(accumulator)
|
|
397
|
+
}
|
|
398
|
+
|
|
399
|
+
if (typeof criteria !== 'object') {
|
|
400
|
+
return Array.from(accumulator)
|
|
401
|
+
}
|
|
402
|
+
|
|
403
|
+
for (const [key, value] of Object.entries(criteria)) {
|
|
404
|
+
if (key.startsWith('$')) {
|
|
405
|
+
this._extractIndexedFields(value, accumulator)
|
|
406
|
+
continue
|
|
407
|
+
}
|
|
408
|
+
|
|
409
|
+
accumulator.add(key)
|
|
410
|
+
|
|
411
|
+
if (Array.isArray(value)) {
|
|
412
|
+
for (const nested of value) {
|
|
413
|
+
this._extractIndexedFields(nested, accumulator)
|
|
414
|
+
}
|
|
415
|
+
}
|
|
416
|
+
}
|
|
417
|
+
|
|
418
|
+
return Array.from(accumulator)
|
|
419
|
+
}
|
|
420
|
+
|
|
421
|
+
/**
|
|
422
|
+
* Identify indexed fields present in criteria whose index data is missing
|
|
423
|
+
* @param {Object} criteria
|
|
424
|
+
* @returns {Array<string>}
|
|
425
|
+
*/
|
|
426
|
+
_getIndexedFieldsWithMissingData(criteria) {
|
|
427
|
+
if (!this.indexManager || !criteria) {
|
|
428
|
+
return []
|
|
429
|
+
}
|
|
430
|
+
|
|
431
|
+
const indexedFields = this._extractIndexedFields(criteria)
|
|
432
|
+
const missing = []
|
|
433
|
+
|
|
434
|
+
for (const field of indexedFields) {
|
|
435
|
+
if (!this.indexManager.isFieldIndexed(field)) {
|
|
436
|
+
continue
|
|
437
|
+
}
|
|
438
|
+
|
|
439
|
+
if (!this.indexManager.hasUsableIndexData(field)) {
|
|
440
|
+
missing.push(field)
|
|
441
|
+
}
|
|
442
|
+
}
|
|
443
|
+
|
|
444
|
+
return missing
|
|
445
|
+
}
|
|
446
|
+
|
|
297
447
|
/**
|
|
298
448
|
* OPTIMIZATION 4: Stream pre-filtered records using line numbers from indices with partial index optimization
|
|
299
449
|
* @param {Set} preFilteredLines - Line numbers from index query
|
|
@@ -498,6 +648,45 @@ export class QueryManager {
|
|
|
498
648
|
console.log(`🔍 IndexManager returned ${lineNumbers.size} line numbers:`, Array.from(lineNumbers))
|
|
499
649
|
}
|
|
500
650
|
|
|
651
|
+
if (lineNumbers.size === 0) {
|
|
652
|
+
const missingIndexedFields = this._getIndexedFieldsWithMissingData(criteria)
|
|
653
|
+
if (missingIndexedFields.length > 0 && this._hasAnyRecords()) {
|
|
654
|
+
// Try to rebuild index before falling back to streaming (only if allowIndexRebuild is true)
|
|
655
|
+
if (this.database.opts.allowIndexRebuild) {
|
|
656
|
+
if (this.opts.debugMode) {
|
|
657
|
+
console.log(`⚠️ Indexed query returned no results because index data is missing for: ${missingIndexedFields.join(', ')}. Attempting index rebuild...`)
|
|
658
|
+
}
|
|
659
|
+
this.database._indexRebuildNeeded = true
|
|
660
|
+
await this.database._rebuildIndexesIfNeeded()
|
|
661
|
+
|
|
662
|
+
// Retry indexed query after rebuild
|
|
663
|
+
const retryLineNumbers = this.indexManager.query(criteria, options)
|
|
664
|
+
if (retryLineNumbers.size > 0) {
|
|
665
|
+
if (this.opts.debugMode) {
|
|
666
|
+
console.log(`✅ Index rebuild successful, using indexed strategy.`)
|
|
667
|
+
}
|
|
668
|
+
// Update lineNumbers to use rebuilt index results
|
|
669
|
+
lineNumbers.clear()
|
|
670
|
+
for (const lineNumber of retryLineNumbers) {
|
|
671
|
+
lineNumbers.add(lineNumber)
|
|
672
|
+
}
|
|
673
|
+
} else {
|
|
674
|
+
// Still no results after rebuild, fall back to streaming
|
|
675
|
+
if (this.opts.debugMode) {
|
|
676
|
+
console.log(`⚠️ Index rebuild did not help, falling back to streaming.`)
|
|
677
|
+
}
|
|
678
|
+
return this.findWithStreaming(criteria, { ...options, forceFullScan: true })
|
|
679
|
+
}
|
|
680
|
+
} else {
|
|
681
|
+
// allowIndexRebuild is false, fall back to streaming
|
|
682
|
+
if (this.opts.debugMode) {
|
|
683
|
+
console.log(`⚠️ Indexed query returned no results because index data is missing for: ${missingIndexedFields.join(', ')}. Falling back to streaming.`)
|
|
684
|
+
}
|
|
685
|
+
return this.findWithStreaming(criteria, { ...options, forceFullScan: true })
|
|
686
|
+
}
|
|
687
|
+
}
|
|
688
|
+
}
|
|
689
|
+
|
|
501
690
|
// Read specific records using the line numbers
|
|
502
691
|
if (lineNumbers.size > 0) {
|
|
503
692
|
const lineNumbersArray = Array.from(lineNumbers)
|
|
@@ -666,7 +855,8 @@ export class QueryManager {
|
|
|
666
855
|
// Handle object conditions (operators)
|
|
667
856
|
if (typeof condition === 'object' && !Array.isArray(condition)) {
|
|
668
857
|
for (const [operator, operatorValue] of Object.entries(condition)) {
|
|
669
|
-
|
|
858
|
+
const normalizedOperator = normalizeOperator(operator);
|
|
859
|
+
if (!this.matchesOperator(value, normalizedOperator, operatorValue, options)) {
|
|
670
860
|
return false;
|
|
671
861
|
}
|
|
672
862
|
}
|
|
@@ -697,6 +887,8 @@ export class QueryManager {
|
|
|
697
887
|
*/
|
|
698
888
|
matchesOperator(value, operator, operatorValue, options = {}) {
|
|
699
889
|
switch (operator) {
|
|
890
|
+
case '$eq':
|
|
891
|
+
return value === operatorValue;
|
|
700
892
|
case '$gt':
|
|
701
893
|
return value > operatorValue;
|
|
702
894
|
case '$gte':
|
|
@@ -757,6 +949,7 @@ export class QueryManager {
|
|
|
757
949
|
}
|
|
758
950
|
}
|
|
759
951
|
|
|
952
|
+
|
|
760
953
|
/**
|
|
761
954
|
* Preprocess query to handle array field syntax automatically
|
|
762
955
|
* @param {Object} criteria - Query criteria
|
|
@@ -913,16 +1106,23 @@ export class QueryManager {
|
|
|
913
1106
|
}
|
|
914
1107
|
|
|
915
1108
|
if (typeof condition === 'object' && !Array.isArray(condition)) {
|
|
916
|
-
const operators = Object.keys(condition);
|
|
1109
|
+
const operators = Object.keys(condition).map(op => normalizeOperator(op));
|
|
1110
|
+
const indexType = this.indexManager?.opts?.indexes?.[field]
|
|
1111
|
+
const isNumericIndex = indexType === 'number' || indexType === 'auto' || indexType === 'array:number'
|
|
1112
|
+
const disallowedForNumeric = ['$all', '$in', '$not', '$regex', '$contains', '$exists', '$size']
|
|
1113
|
+
const disallowedDefault = ['$all', '$in', '$gt', '$gte', '$lt', '$lte', '$ne', '$not', '$regex', '$contains', '$exists', '$size']
|
|
1114
|
+
|
|
1115
|
+
// Check if this is a term mapping field (array:string or string fields with term mapping)
|
|
1116
|
+
const isTermMappingField = this.database.termManager &&
|
|
1117
|
+
this.database.termManager.termMappingFields &&
|
|
1118
|
+
this.database.termManager.termMappingFields.includes(field)
|
|
917
1119
|
|
|
918
|
-
if (
|
|
919
|
-
|
|
920
|
-
|
|
921
|
-
});
|
|
1120
|
+
if (isTermMappingField) {
|
|
1121
|
+
const termMappingDisallowed = ['$gt', '$gte', '$lt', '$lte', '$ne', '$regex', '$contains', '$exists', '$size']
|
|
1122
|
+
return operators.every(op => !termMappingDisallowed.includes(op));
|
|
922
1123
|
} else {
|
|
923
|
-
|
|
924
|
-
|
|
925
|
-
});
|
|
1124
|
+
const disallowed = isNumericIndex ? disallowedForNumeric : disallowedDefault
|
|
1125
|
+
return operators.every(op => !disallowed.includes(op));
|
|
926
1126
|
}
|
|
927
1127
|
}
|
|
928
1128
|
return true;
|
|
@@ -960,23 +1160,34 @@ export class QueryManager {
|
|
|
960
1160
|
}
|
|
961
1161
|
|
|
962
1162
|
if (typeof condition === 'object' && !Array.isArray(condition)) {
|
|
963
|
-
const operators = Object.keys(condition);
|
|
1163
|
+
const operators = Object.keys(condition).map(op => normalizeOperator(op));
|
|
964
1164
|
if (this.opts.debugMode) {
|
|
965
1165
|
console.log(`🔍 Field '${field}' has operators:`, operators)
|
|
966
1166
|
}
|
|
967
1167
|
|
|
968
|
-
|
|
969
|
-
|
|
970
|
-
|
|
971
|
-
|
|
972
|
-
|
|
973
|
-
|
|
974
|
-
|
|
1168
|
+
const indexType = this.indexManager?.opts?.indexes?.[field]
|
|
1169
|
+
const isNumericIndex = indexType === 'number' || indexType === 'auto' || indexType === 'array:number'
|
|
1170
|
+
const isArrayStringIndex = indexType === 'array:string'
|
|
1171
|
+
const disallowedForNumeric = ['$all', '$in', '$not', '$regex', '$contains', '$exists', '$size']
|
|
1172
|
+
const disallowedDefault = ['$all', '$in', '$gt', '$gte', '$lt', '$lte', '$ne', '$not', '$regex', '$contains', '$exists', '$size']
|
|
1173
|
+
|
|
1174
|
+
// Check if this is a term mapping field (array:string or string fields with term mapping)
|
|
1175
|
+
const isTermMappingField = this.database.termManager &&
|
|
1176
|
+
this.database.termManager.termMappingFields &&
|
|
1177
|
+
this.database.termManager.termMappingFields.includes(field)
|
|
1178
|
+
|
|
1179
|
+
// With term mapping enabled on THIS FIELD, we can support complex operators via partial reads
|
|
1180
|
+
// Also support $all for array:string indexed fields (IndexManager.query supports it via Set intersection)
|
|
1181
|
+
if (isTermMappingField) {
|
|
1182
|
+
const termMappingDisallowed = ['$gt', '$gte', '$lt', '$lte', '$ne', '$regex', '$contains', '$exists', '$size']
|
|
1183
|
+
return operators.every(op => !termMappingDisallowed.includes(op));
|
|
975
1184
|
} else {
|
|
976
|
-
|
|
977
|
-
|
|
978
|
-
|
|
979
|
-
|
|
1185
|
+
let disallowed = isNumericIndex ? disallowedForNumeric : disallowedDefault
|
|
1186
|
+
// Remove $all from disallowed if field is array:string (IndexManager supports $all via Set intersection)
|
|
1187
|
+
if (isArrayStringIndex) {
|
|
1188
|
+
disallowed = disallowed.filter(op => op !== '$all')
|
|
1189
|
+
}
|
|
1190
|
+
return operators.every(op => !disallowed.includes(op));
|
|
980
1191
|
}
|
|
981
1192
|
}
|
|
982
1193
|
return true;
|
|
@@ -1150,28 +1361,34 @@ export class QueryManager {
|
|
|
1150
1361
|
/**
|
|
1151
1362
|
* Validate strict query mode
|
|
1152
1363
|
* @param {Object} criteria - Query criteria
|
|
1364
|
+
* @param {Object} options - Query options
|
|
1153
1365
|
*/
|
|
1154
|
-
validateStrictQuery(criteria) {
|
|
1366
|
+
validateStrictQuery(criteria, options = {}) {
|
|
1367
|
+
// Allow bypassing strict mode validation with allowNonIndexed option
|
|
1368
|
+
if (options.allowNonIndexed === true) {
|
|
1369
|
+
return; // Skip validation for this query
|
|
1370
|
+
}
|
|
1371
|
+
|
|
1155
1372
|
if (!criteria || Object.keys(criteria).length === 0) {
|
|
1156
1373
|
return; // Empty criteria are always allowed
|
|
1157
1374
|
}
|
|
1158
1375
|
|
|
1159
1376
|
// Handle logical operators at the top level
|
|
1160
1377
|
if (criteria.$not) {
|
|
1161
|
-
this.validateStrictQuery(criteria.$not);
|
|
1378
|
+
this.validateStrictQuery(criteria.$not, options);
|
|
1162
1379
|
return;
|
|
1163
1380
|
}
|
|
1164
1381
|
|
|
1165
1382
|
if (criteria.$or && Array.isArray(criteria.$or)) {
|
|
1166
1383
|
for (const orCondition of criteria.$or) {
|
|
1167
|
-
this.validateStrictQuery(orCondition);
|
|
1384
|
+
this.validateStrictQuery(orCondition, options);
|
|
1168
1385
|
}
|
|
1169
1386
|
return;
|
|
1170
1387
|
}
|
|
1171
1388
|
|
|
1172
1389
|
if (criteria.$and && Array.isArray(criteria.$and)) {
|
|
1173
1390
|
for (const andCondition of criteria.$and) {
|
|
1174
|
-
this.validateStrictQuery(andCondition);
|
|
1391
|
+
this.validateStrictQuery(andCondition, options);
|
|
1175
1392
|
}
|
|
1176
1393
|
return;
|
|
1177
1394
|
}
|
|
@@ -38,19 +38,19 @@ export default class TermManager {
|
|
|
38
38
|
/**
|
|
39
39
|
* Get term ID without incrementing count (for IndexManager use)
|
|
40
40
|
* @param {string} term - Term to get ID for
|
|
41
|
-
* @returns {number} - Numeric ID for the term
|
|
41
|
+
* @returns {number|undefined} - Numeric ID for the term, or undefined if not found
|
|
42
|
+
* CRITICAL: Does NOT create new IDs - only returns existing ones
|
|
43
|
+
* This prevents creating invalid term IDs during queries when terms haven't been loaded yet
|
|
42
44
|
*/
|
|
43
45
|
getTermIdWithoutIncrement(term) {
|
|
44
46
|
if (this.termToId.has(term)) {
|
|
45
47
|
return this.termToId.get(term)
|
|
46
48
|
}
|
|
47
49
|
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
return id
|
|
50
|
+
// CRITICAL FIX: Don't create new IDs during queries
|
|
51
|
+
// If term doesn't exist, return undefined
|
|
52
|
+
// This ensures queries only work with terms that were actually saved to the database
|
|
53
|
+
return undefined
|
|
54
54
|
}
|
|
55
55
|
|
|
56
56
|
/**
|
|
@@ -0,0 +1,116 @@
|
|
|
1
|
+
const aliasToCanonical = {
|
|
2
|
+
'>': '$gt',
|
|
3
|
+
'>=': '$gte',
|
|
4
|
+
'<': '$lt',
|
|
5
|
+
'<=': '$lte',
|
|
6
|
+
'!=': '$ne',
|
|
7
|
+
'=': '$eq',
|
|
8
|
+
'==': '$eq',
|
|
9
|
+
eq: '$eq',
|
|
10
|
+
equals: '$eq',
|
|
11
|
+
in: '$in',
|
|
12
|
+
nin: '$nin',
|
|
13
|
+
regex: '$regex',
|
|
14
|
+
contains: '$contains',
|
|
15
|
+
all: '$all',
|
|
16
|
+
exists: '$exists',
|
|
17
|
+
size: '$size',
|
|
18
|
+
not: '$not'
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
const canonicalToLegacy = {
|
|
22
|
+
'$gt': '>',
|
|
23
|
+
'$gte': '>=',
|
|
24
|
+
'$lt': '<',
|
|
25
|
+
'$lte': '<=',
|
|
26
|
+
'$ne': '!=',
|
|
27
|
+
'$eq': '=',
|
|
28
|
+
'$contains': 'contains',
|
|
29
|
+
'$regex': 'regex'
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
/**
|
|
33
|
+
* Normalize an operator to its canonical Mongo-style representation (prefixed with $)
|
|
34
|
+
* @param {string} operator
|
|
35
|
+
* @returns {string}
|
|
36
|
+
*/
|
|
37
|
+
export function normalizeOperator(operator) {
|
|
38
|
+
if (typeof operator !== 'string') {
|
|
39
|
+
return operator
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
if (operator.startsWith('$')) {
|
|
43
|
+
return operator
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
if (aliasToCanonical[operator] !== undefined) {
|
|
47
|
+
return aliasToCanonical[operator]
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
const lowerCase = operator.toLowerCase()
|
|
51
|
+
if (aliasToCanonical[lowerCase] !== undefined) {
|
|
52
|
+
return aliasToCanonical[lowerCase]
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
return operator
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
/**
|
|
59
|
+
* Convert an operator to its legacy (non-prefixed) alias when available
|
|
60
|
+
* @param {string} operator
|
|
61
|
+
* @returns {string}
|
|
62
|
+
*/
|
|
63
|
+
export function operatorToLegacy(operator) {
|
|
64
|
+
if (typeof operator !== 'string') {
|
|
65
|
+
return operator
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
const canonical = normalizeOperator(operator)
|
|
69
|
+
if (canonicalToLegacy[canonical]) {
|
|
70
|
+
return canonicalToLegacy[canonical]
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
return operator
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
/**
|
|
77
|
+
* Normalize operator keys in a criteria object
|
|
78
|
+
* @param {Object} criteriaValue
|
|
79
|
+
* @param {Object} options
|
|
80
|
+
* @param {'canonical'|'legacy'} options.target - Preferred operator style
|
|
81
|
+
* @param {boolean} [options.preserveOriginal=false] - Whether to keep the original keys alongside normalized ones
|
|
82
|
+
* @returns {Object}
|
|
83
|
+
*/
|
|
84
|
+
export function normalizeCriteriaOperators(criteriaValue, { target = 'canonical', preserveOriginal = false } = {}) {
|
|
85
|
+
if (!criteriaValue || typeof criteriaValue !== 'object' || Array.isArray(criteriaValue)) {
|
|
86
|
+
return criteriaValue
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
const normalized = preserveOriginal ? { ...criteriaValue } : {}
|
|
90
|
+
|
|
91
|
+
for (const [operator, value] of Object.entries(criteriaValue)) {
|
|
92
|
+
const canonical = normalizeOperator(operator)
|
|
93
|
+
|
|
94
|
+
if (target === 'canonical') {
|
|
95
|
+
normalized[canonical] = value
|
|
96
|
+
if (preserveOriginal && canonical !== operator) {
|
|
97
|
+
normalized[operator] = value
|
|
98
|
+
}
|
|
99
|
+
} else if (target === 'legacy') {
|
|
100
|
+
const legacy = operatorToLegacy(operator)
|
|
101
|
+
normalized[legacy] = value
|
|
102
|
+
|
|
103
|
+
if (preserveOriginal) {
|
|
104
|
+
if (legacy !== canonical) {
|
|
105
|
+
normalized[canonical] = value
|
|
106
|
+
}
|
|
107
|
+
if (operator !== legacy && operator !== canonical) {
|
|
108
|
+
normalized[operator] = value
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
return normalized
|
|
115
|
+
}
|
|
116
|
+
|
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
import fs from 'fs'
|
|
2
|
+
import path from 'path'
|
|
3
|
+
import { Database } from '../src/Database.mjs'
|
|
4
|
+
|
|
5
|
+
const cleanUp = (filePath) => {
|
|
6
|
+
try {
|
|
7
|
+
if (fs.existsSync(filePath)) {
|
|
8
|
+
fs.unlinkSync(filePath)
|
|
9
|
+
}
|
|
10
|
+
} catch (error) {
|
|
11
|
+
// Ignore cleanup errors
|
|
12
|
+
}
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
describe('Coverage Method Tests', () => {
|
|
16
|
+
let db
|
|
17
|
+
let testDbPath
|
|
18
|
+
let testIdxPath
|
|
19
|
+
|
|
20
|
+
beforeEach(async () => {
|
|
21
|
+
const testId = Math.random().toString(36).substring(2, 10)
|
|
22
|
+
testDbPath = path.join(process.cwd(), `test-coverage-${testId}.jdb`)
|
|
23
|
+
testIdxPath = path.join(process.cwd(), `test-coverage-${testId}.idx.jdb`)
|
|
24
|
+
|
|
25
|
+
cleanUp(testDbPath)
|
|
26
|
+
cleanUp(testIdxPath)
|
|
27
|
+
|
|
28
|
+
db = new Database(testDbPath, {
|
|
29
|
+
indexes: {
|
|
30
|
+
nameTerms: 'array:string',
|
|
31
|
+
genre: 'string'
|
|
32
|
+
}
|
|
33
|
+
})
|
|
34
|
+
await db.init()
|
|
35
|
+
})
|
|
36
|
+
|
|
37
|
+
afterEach(async () => {
|
|
38
|
+
if (db) {
|
|
39
|
+
await db.close()
|
|
40
|
+
db = null
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
cleanUp(testDbPath)
|
|
44
|
+
cleanUp(testIdxPath)
|
|
45
|
+
})
|
|
46
|
+
|
|
47
|
+
describe('Grouped Coverage', () => {
|
|
48
|
+
test('calculates percentage with include and exclude groups', async () => {
|
|
49
|
+
await db.insert({ id: 1, title: 'Rede Brasil', nameTerms: ['rede', 'brasil', 'sul'] })
|
|
50
|
+
await db.insert({ id: 2, title: 'Band SP', nameTerms: ['band', 'sp'] })
|
|
51
|
+
await db.save()
|
|
52
|
+
|
|
53
|
+
const coverage = await db.coverage('nameTerms', [
|
|
54
|
+
{ terms: ['sbt'], excludes: [] },
|
|
55
|
+
{ terms: ['rede', 'brasil'], excludes: ['norte'] },
|
|
56
|
+
{ terms: ['tv', 'sancouper'], excludes: [] },
|
|
57
|
+
{ terms: ['band'] }
|
|
58
|
+
])
|
|
59
|
+
|
|
60
|
+
expect(coverage).toBeCloseTo(50)
|
|
61
|
+
})
|
|
62
|
+
|
|
63
|
+
test('applies excludes before counting matches', async () => {
|
|
64
|
+
await db.insert({ id: 1, title: 'Rede Norte', nameTerms: ['rede', 'brasil', 'norte'] })
|
|
65
|
+
await db.insert({ id: 2, title: 'Rede Sul', nameTerms: ['rede', 'sul'] })
|
|
66
|
+
await db.save()
|
|
67
|
+
|
|
68
|
+
const coverage = await db.coverage('nameTerms', [
|
|
69
|
+
{ terms: ['rede', 'brasil'], excludes: ['norte'] },
|
|
70
|
+
{ terms: ['rede'] }
|
|
71
|
+
])
|
|
72
|
+
|
|
73
|
+
expect(coverage).toBeCloseTo(50)
|
|
74
|
+
})
|
|
75
|
+
|
|
76
|
+
test('works with string indexed field and optional excludes', async () => {
|
|
77
|
+
await db.insert({ id: 1, title: 'Song A', genre: 'samba' })
|
|
78
|
+
await db.insert({ id: 2, title: 'Song B', genre: 'pagode' })
|
|
79
|
+
await db.insert({ id: 3, title: 'Song C', genre: 'rock' })
|
|
80
|
+
await db.save()
|
|
81
|
+
|
|
82
|
+
const coverage = await db.coverage('genre', [
|
|
83
|
+
{ terms: ['samba'] },
|
|
84
|
+
{ terms: ['pagode'], excludes: [] },
|
|
85
|
+
{ terms: ['rock'], excludes: ['rock'] },
|
|
86
|
+
{ terms: ['funk'] }
|
|
87
|
+
])
|
|
88
|
+
|
|
89
|
+
expect(coverage).toBeCloseTo(50)
|
|
90
|
+
})
|
|
91
|
+
})
|
|
92
|
+
})
|
|
93
|
+
|