jexidb 2.2.3 → 2.2.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +50 -0
- package/dist/Database.cjs +745 -157
- package/package.json +1 -1
- package/src/Database.mjs +274 -28
- package/src/FileHandler.mjs +88 -1
- package/src/managers/IndexManager.mjs +55 -6
- package/src/managers/QueryManager.mjs +320 -132
- package/src/utils/ioTimeout.mjs +36 -0
|
@@ -1016,6 +1016,29 @@ export default class IndexManager {
|
|
|
1016
1016
|
return result;
|
|
1017
1017
|
}
|
|
1018
1018
|
|
|
1019
|
+
// Handle $or queries - union of the branch results
|
|
1020
|
+
if (criteria.$or && Array.isArray(criteria.$or)) {
|
|
1021
|
+
const orResult = new Set();
|
|
1022
|
+
|
|
1023
|
+
for (const orCondition of criteria.$or) {
|
|
1024
|
+
const branchLines = this.query(orCondition, options);
|
|
1025
|
+
for (const lineNumber of branchLines) {
|
|
1026
|
+
orResult.add(lineNumber);
|
|
1027
|
+
}
|
|
1028
|
+
}
|
|
1029
|
+
|
|
1030
|
+
// Other fields at the root level still narrow the union (AND semantics)
|
|
1031
|
+
const otherCriteria = { ...criteria };
|
|
1032
|
+
delete otherCriteria.$or;
|
|
1033
|
+
|
|
1034
|
+
if (Object.keys(otherCriteria).length > 0) {
|
|
1035
|
+
const otherResults = this.query(otherCriteria, options);
|
|
1036
|
+
return new Set([...orResult].filter(x => otherResults.has(x)));
|
|
1037
|
+
}
|
|
1038
|
+
|
|
1039
|
+
return orResult;
|
|
1040
|
+
}
|
|
1041
|
+
|
|
1019
1042
|
// Handle $and queries with parallel processing optimization
|
|
1020
1043
|
if (criteria.$and && Array.isArray(criteria.$and)) {
|
|
1021
1044
|
// OPTIMIZATION: Process conditions in parallel for better performance
|
|
@@ -1074,18 +1097,44 @@ export default class IndexManager {
|
|
|
1074
1097
|
if (typeof data[field] === 'undefined') continue;
|
|
1075
1098
|
|
|
1076
1099
|
const originalCriteriaValue = criteria[field];
|
|
1077
|
-
|
|
1100
|
+
// INTENTIONAL (CHANGELOG [2.2.5]): RegExp is not an operator map - never normalize it
|
|
1101
|
+
// (spreading a RegExp yields {} and would make the regex match every record).
|
|
1102
|
+
const criteriaValue = originalCriteriaValue instanceof RegExp
|
|
1103
|
+
? originalCriteriaValue
|
|
1104
|
+
: normalizeCriteriaOperators(originalCriteriaValue, { target: 'legacy', preserveOriginal: true });
|
|
1078
1105
|
let lineNumbersForField = new Set();
|
|
1079
1106
|
const isNumericField = this.opts.indexes[field] === 'number';
|
|
1080
1107
|
|
|
1081
|
-
//
|
|
1108
|
+
// INTENTIONAL OPTIMIZATION (see CHANGELOG [2.2.5]): raw RegExp on string/term
|
|
1109
|
+
// indexes is answered via the index keys instead of full-file streaming. For
|
|
1110
|
+
// term-mapped fields the keys are numeric term IDs, translated back to words with
|
|
1111
|
+
// termManager.getTerm() before testing the regex. DO NOT change this back to
|
|
1112
|
+
// "RegExp => streaming" - it regresses partial search to a full scan per database.
|
|
1082
1113
|
if (criteriaValue instanceof RegExp) {
|
|
1083
|
-
|
|
1084
|
-
|
|
1085
|
-
|
|
1114
|
+
const fieldIndex = data[field];
|
|
1115
|
+
const indexType = this.opts.indexes && this.opts.indexes[field];
|
|
1116
|
+
const isNumericIndexType = indexType === 'number' || indexType === 'array:number';
|
|
1117
|
+
if (!fieldIndex || isNumericIndexType) {
|
|
1118
|
+
// Not index-answerable (numeric index) - handled by streaming fallback
|
|
1119
|
+
continue;
|
|
1120
|
+
}
|
|
1121
|
+
const termManager = this.database && this.database.termManager;
|
|
1122
|
+
const isTermField = !!(termManager && termManager.termMappingFields &&
|
|
1123
|
+
termManager.termMappingFields.includes(field));
|
|
1124
|
+
const regex = criteriaValue;
|
|
1125
|
+
for (const key of Object.keys(fieldIndex)) {
|
|
1126
|
+
const word = isTermField ? termManager.getTerm(Number(key)) : key;
|
|
1127
|
+
if (word === null || word === undefined) continue;
|
|
1128
|
+
regex.lastIndex = 0;
|
|
1129
|
+
if (!regex.test(String(word))) continue;
|
|
1130
|
+
const numbers = this._getAllLineNumbers(fieldIndex[key]);
|
|
1131
|
+
for (const lineNumber of numbers) lineNumbersForField.add(lineNumber);
|
|
1132
|
+
}
|
|
1133
|
+
// Fall through to field consolidation below (RegExp is an object, so it is
|
|
1134
|
+
// excluded from the object/operator branch that follows).
|
|
1086
1135
|
}
|
|
1087
1136
|
|
|
1088
|
-
if (typeof criteriaValue === 'object' && !Array.isArray(criteriaValue) && criteriaValue !== null) {
|
|
1137
|
+
if (typeof criteriaValue === 'object' && !(criteriaValue instanceof RegExp) && !Array.isArray(criteriaValue) && criteriaValue !== null) {
|
|
1089
1138
|
const fieldIndex = data[field];
|
|
1090
1139
|
|
|
1091
1140
|
// Handle $in operator for array queries
|
|
@@ -286,13 +286,18 @@ export class QueryManager {
|
|
|
286
286
|
}
|
|
287
287
|
|
|
288
288
|
// Use indices to pre-filter and reduce streaming scope
|
|
289
|
-
const
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
289
|
+
const preFilterCriteria = this._extractIndexableCriteria(criteria);
|
|
290
|
+
const preFilteredLines = this.indexManager.query(preFilterCriteria, streamingOptions);
|
|
291
|
+
|
|
292
|
+
// Fields answered by the pre-filter already match, so the streaming matcher
|
|
293
|
+
// only evaluates the residual conditions (instead of re-checking everything).
|
|
294
|
+
const preFilteredFields = this._collectCriteriaFields(preFilterCriteria);
|
|
295
|
+
|
|
294
296
|
// Stream only the pre-filtered records
|
|
295
|
-
return this._streamPreFilteredRecords(preFilteredLines, criteria,
|
|
297
|
+
return this._streamPreFilteredRecords(preFilteredLines, criteria, {
|
|
298
|
+
...streamingOptions,
|
|
299
|
+
_prefilteredFields: preFilteredFields
|
|
300
|
+
});
|
|
296
301
|
}
|
|
297
302
|
}
|
|
298
303
|
|
|
@@ -304,6 +309,29 @@ export class QueryManager {
|
|
|
304
309
|
return this._streamAllRecords(criteria, streamingOptions);
|
|
305
310
|
}
|
|
306
311
|
|
|
312
|
+
/**
|
|
313
|
+
* Whether a single field condition can/should be answered by the index during
|
|
314
|
+
* pre-filtering (so the LEAST number of entries is streamed).
|
|
315
|
+
*
|
|
316
|
+
* - Raw RegExp is pre-filterable only on string/term indexes (IndexManager.query
|
|
317
|
+
* resolves it via the index keys). Regex on numeric indexes -> residual.
|
|
318
|
+
* - Complement operators ($ne/$nin) expand to nearly every record, so they are
|
|
319
|
+
* NOT pre-filtered: they are applied LAST, on the minimal candidate set, by the
|
|
320
|
+
* streaming matcher. (CHANGELOG [2.2.5])
|
|
321
|
+
*/
|
|
322
|
+
_canPrefilterFieldCondition(field, condition) {
|
|
323
|
+
if (!this.indexManager?.opts?.indexes?.[field]) return false
|
|
324
|
+
if (condition instanceof RegExp) {
|
|
325
|
+
const idxType = this.indexManager.opts.indexes[field]
|
|
326
|
+
return idxType !== 'number' && idxType !== 'array:number'
|
|
327
|
+
}
|
|
328
|
+
if (condition && typeof condition === 'object' && !Array.isArray(condition)) {
|
|
329
|
+
const ops = Object.keys(condition).map(op => normalizeOperator(op))
|
|
330
|
+
if (ops.some(op => op === '$ne' || op === '$nin')) return false
|
|
331
|
+
}
|
|
332
|
+
return true
|
|
333
|
+
}
|
|
334
|
+
|
|
307
335
|
/**
|
|
308
336
|
* Get indexable fields from criteria
|
|
309
337
|
* @param {Object} criteria - Query criteria
|
|
@@ -322,13 +350,25 @@ export class QueryManager {
|
|
|
322
350
|
indexableFields.push(...this._getIndexableFields(andCondition));
|
|
323
351
|
}
|
|
324
352
|
}
|
|
353
|
+
|
|
354
|
+
// Handle $or conditions - only when EVERY branch is pre-filterable, otherwise
|
|
355
|
+
// the union would narrow the candidate set and lose rows.
|
|
356
|
+
if (criteria.$or && Array.isArray(criteria.$or) && criteria.$or.length > 0) {
|
|
357
|
+
const orCriteria = criteria.$or.map(condition => this._extractIndexableCriteria(condition));
|
|
358
|
+
if (orCriteria.every(condition => Object.keys(condition).length > 0)) {
|
|
359
|
+
for (const condition of orCriteria) {
|
|
360
|
+
indexableFields.push(...this._getIndexableFields(condition));
|
|
361
|
+
}
|
|
362
|
+
}
|
|
363
|
+
}
|
|
325
364
|
|
|
326
365
|
// Handle regular field conditions
|
|
327
366
|
for (const [field, condition] of Object.entries(criteria)) {
|
|
328
367
|
if (field.startsWith('$')) continue; // Skip logical operators
|
|
329
368
|
|
|
330
|
-
//
|
|
331
|
-
|
|
369
|
+
// Pre-filter with the index only what it can answer minimally; everything
|
|
370
|
+
// else is applied LAST over the streamed candidates (filter last, stream least).
|
|
371
|
+
if (!this._canPrefilterFieldCondition(field, condition)) {
|
|
332
372
|
continue;
|
|
333
373
|
}
|
|
334
374
|
|
|
@@ -362,6 +402,17 @@ export class QueryManager {
|
|
|
362
402
|
indexableCriteria.$and = indexableAndConditions;
|
|
363
403
|
}
|
|
364
404
|
}
|
|
405
|
+
|
|
406
|
+
// Handle $or conditions - a branch that cannot be pre-filtered would make the
|
|
407
|
+
// union too narrow, so $or is only pre-filterable when every branch is.
|
|
408
|
+
if (criteria.$or && Array.isArray(criteria.$or) && criteria.$or.length > 0) {
|
|
409
|
+
const indexableOrConditions = criteria.$or
|
|
410
|
+
.map(orCondition => this._extractIndexableCriteria(orCondition));
|
|
411
|
+
|
|
412
|
+
if (indexableOrConditions.every(condition => Object.keys(condition).length > 0)) {
|
|
413
|
+
indexableCriteria.$or = indexableOrConditions;
|
|
414
|
+
}
|
|
415
|
+
}
|
|
365
416
|
|
|
366
417
|
// Handle $not operator - include it if it can be processed by IndexManager
|
|
367
418
|
if (criteria.$not && typeof criteria.$not === 'object') {
|
|
@@ -384,8 +435,9 @@ export class QueryManager {
|
|
|
384
435
|
for (const [field, condition] of Object.entries(criteria)) {
|
|
385
436
|
if (field.startsWith('$')) continue; // Skip logical operators (already handled above)
|
|
386
437
|
|
|
387
|
-
//
|
|
388
|
-
|
|
438
|
+
// Pre-filter with the index only what it can answer minimally; complement/
|
|
439
|
+
// numeric-regex conditions are left residual and applied LAST on the candidates.
|
|
440
|
+
if (!this._canPrefilterFieldCondition(field, condition)) {
|
|
389
441
|
continue;
|
|
390
442
|
}
|
|
391
443
|
|
|
@@ -397,6 +449,43 @@ export class QueryManager {
|
|
|
397
449
|
return indexableCriteria;
|
|
398
450
|
}
|
|
399
451
|
|
|
452
|
+
/**
|
|
453
|
+
* Collect every field referenced by criteria, including inside $and/$or/$not
|
|
454
|
+
* @param {Object} criteria - Criteria to inspect
|
|
455
|
+
* @param {Set<string>} accumulator - Fields collected so far
|
|
456
|
+
* @returns {Set<string>} - Field names
|
|
457
|
+
*/
|
|
458
|
+
_collectCriteriaFields(criteria, accumulator = new Set()) {
|
|
459
|
+
if (!criteria || typeof criteria !== 'object') {
|
|
460
|
+
return accumulator;
|
|
461
|
+
}
|
|
462
|
+
|
|
463
|
+
if (Array.isArray(criteria)) {
|
|
464
|
+
for (const condition of criteria) {
|
|
465
|
+
this._collectCriteriaFields(condition, accumulator);
|
|
466
|
+
}
|
|
467
|
+
return accumulator;
|
|
468
|
+
}
|
|
469
|
+
|
|
470
|
+
for (const [field, value] of Object.entries(criteria)) {
|
|
471
|
+
if (field === '$and' || field === '$or') {
|
|
472
|
+
this._collectCriteriaFields(value, accumulator);
|
|
473
|
+
continue;
|
|
474
|
+
}
|
|
475
|
+
|
|
476
|
+
if (field === '$not') {
|
|
477
|
+
// $not is negated, so its fields are not "already matched" by the pre-filter
|
|
478
|
+
continue;
|
|
479
|
+
}
|
|
480
|
+
|
|
481
|
+
if (field.startsWith('$')) continue;
|
|
482
|
+
|
|
483
|
+
accumulator.add(field);
|
|
484
|
+
}
|
|
485
|
+
|
|
486
|
+
return accumulator;
|
|
487
|
+
}
|
|
488
|
+
|
|
400
489
|
/**
|
|
401
490
|
* Determine whether the database currently has any records (persisted or pending)
|
|
402
491
|
* @returns {boolean}
|
|
@@ -613,8 +702,10 @@ export class QueryManager {
|
|
|
613
702
|
return !notResult;
|
|
614
703
|
}
|
|
615
704
|
|
|
616
|
-
// OPTIMIZATION 4: Skip
|
|
617
|
-
const
|
|
705
|
+
// OPTIMIZATION 4: Skip only the fields that the pre-filter actually answered
|
|
706
|
+
const prefilteredFields = options._prefilteredFields instanceof Set
|
|
707
|
+
? options._prefilteredFields
|
|
708
|
+
: new Set(this._getIndexableFields(criteria));
|
|
618
709
|
|
|
619
710
|
// Handle explicit logical operators at the top level
|
|
620
711
|
if (criteria.$or && Array.isArray(criteria.$or)) {
|
|
@@ -641,8 +732,8 @@ export class QueryManager {
|
|
|
641
732
|
for (const [field, condition] of Object.entries(criteria)) {
|
|
642
733
|
if (field.startsWith('$')) continue;
|
|
643
734
|
|
|
644
|
-
// OPTIMIZATION 4: Skip
|
|
645
|
-
if (
|
|
735
|
+
// OPTIMIZATION 4: Skip fields that were already pre-filtered
|
|
736
|
+
if (prefilteredFields.has(field)) {
|
|
646
737
|
continue;
|
|
647
738
|
}
|
|
648
739
|
|
|
@@ -728,6 +819,19 @@ export class QueryManager {
|
|
|
728
819
|
}
|
|
729
820
|
return this.findWithStreaming(criteria, { ...options, forceFullScan: true })
|
|
730
821
|
}
|
|
822
|
+
|
|
823
|
+
// Swap detection: if another process atomically replaced the data file, the
|
|
824
|
+
// in-memory offsets/index describe the previous file. Reload cleanly, or -
|
|
825
|
+
// when a writer sentinel is present / the clean reload fails - fall back to
|
|
826
|
+
// a full streaming scan so stale byte offsets are never used on the new file.
|
|
827
|
+
const refreshStatus = await this.database._refreshFromDiskIfChanged()
|
|
828
|
+
if (refreshStatus === 'busy') {
|
|
829
|
+
if (this.opts.debugMode) {
|
|
830
|
+
console.log('⚠️ findWithIndexed: file busy (writer active) or clean reload failed, falling back to full streaming')
|
|
831
|
+
}
|
|
832
|
+
return this.findWithStreaming(criteria, { ...options, forceFullScan: true })
|
|
833
|
+
}
|
|
834
|
+
|
|
731
835
|
let results = []
|
|
732
836
|
const limit = options.limit // No default limit - return all results unless explicitly limited
|
|
733
837
|
|
|
@@ -942,20 +1046,37 @@ export class QueryManager {
|
|
|
942
1046
|
* @returns {boolean} - True if matches
|
|
943
1047
|
*/
|
|
944
1048
|
matchesFieldCondition(record, field, condition, options = {}) {
|
|
945
|
-
|
|
1049
|
+
let value = record[field];
|
|
946
1050
|
|
|
947
|
-
// Debug logging for all field conditions
|
|
1051
|
+
// Debug logging for all field conditions - SAMPLED: only the first record of
|
|
1052
|
+
// each query logs per field, so a full streaming scan (which calls this for
|
|
1053
|
+
// every record) does not spam the console with one line per record.
|
|
948
1054
|
if (this.database.opts.debugMode) {
|
|
949
|
-
|
|
1055
|
+
if (!options._dbFieldLog) options._dbFieldLog = {};
|
|
1056
|
+
if (!options._dbFieldLog[field]) {
|
|
1057
|
+
options._dbFieldLog[field] = true;
|
|
1058
|
+
console.log(`🔍 Checking field '${field}':`, { value, condition, record: record.name || record.id });
|
|
1059
|
+
}
|
|
950
1060
|
}
|
|
951
1061
|
|
|
952
|
-
// Debug logging for term mapping fields
|
|
1062
|
+
// Debug logging for term mapping fields (sampled the same way)
|
|
953
1063
|
if (this.database.opts.termMapping && Object.keys(this.database.opts.indexes || {}).includes(field)) {
|
|
954
1064
|
if (this.database.opts.debugMode) {
|
|
955
|
-
|
|
1065
|
+
if (!options._dbFieldLog) options._dbFieldLog = {};
|
|
1066
|
+
if (!options._dbFieldLog[`tm:${field}`]) {
|
|
1067
|
+
options._dbFieldLog[`tm:${field}`] = true;
|
|
1068
|
+
console.log(`🔍 Checking term mapping field '${field}':`, { value, condition, record: record.name || record.id });
|
|
1069
|
+
}
|
|
956
1070
|
}
|
|
957
1071
|
}
|
|
958
1072
|
|
|
1073
|
+
// Term-mapped array fields store numeric term IDs, so both sides are translated
|
|
1074
|
+
// to terms here; otherwise string queries would never match on the streaming path.
|
|
1075
|
+
if (Array.isArray(value) && this._isTermMappingField(field)) {
|
|
1076
|
+
value = this._termIdsToTerms(value);
|
|
1077
|
+
condition = this._normalizeTermCondition(condition);
|
|
1078
|
+
}
|
|
1079
|
+
|
|
959
1080
|
// Handle null/undefined values
|
|
960
1081
|
if (value === null || value === undefined) {
|
|
961
1082
|
return condition === null || condition === undefined;
|
|
@@ -1079,19 +1200,100 @@ export class QueryManager {
|
|
|
1079
1200
|
}
|
|
1080
1201
|
|
|
1081
1202
|
|
|
1203
|
+
/**
|
|
1204
|
+
* Whether a field stores its values as numeric term IDs (term mapping)
|
|
1205
|
+
* @param {string} field
|
|
1206
|
+
* @returns {boolean}
|
|
1207
|
+
*/
|
|
1208
|
+
_isTermMappingField(field) {
|
|
1209
|
+
const termManager = this.database && this.database.termManager
|
|
1210
|
+
return !!(termManager &&
|
|
1211
|
+
Array.isArray(termManager.termMappingFields) &&
|
|
1212
|
+
termManager.termMappingFields.includes(field))
|
|
1213
|
+
}
|
|
1214
|
+
|
|
1215
|
+
/**
|
|
1216
|
+
* Translate stored term IDs back to terms for matching
|
|
1217
|
+
* @param {Array} ids
|
|
1218
|
+
* @returns {Array}
|
|
1219
|
+
*/
|
|
1220
|
+
_termIdsToTerms(ids) {
|
|
1221
|
+
const termManager = this.database.termManager
|
|
1222
|
+
return ids.map(id => {
|
|
1223
|
+
if (typeof id !== 'number') return id
|
|
1224
|
+
const term = termManager.getTerm(id)
|
|
1225
|
+
return term === null ? id : term
|
|
1226
|
+
})
|
|
1227
|
+
}
|
|
1228
|
+
|
|
1229
|
+
/**
|
|
1230
|
+
* Translate numeric term IDs used in a query condition to their terms
|
|
1231
|
+
* @param {*} condition
|
|
1232
|
+
* @returns {*}
|
|
1233
|
+
*/
|
|
1234
|
+
_normalizeTermCondition(condition) {
|
|
1235
|
+
const toTerm = value => {
|
|
1236
|
+
if (typeof value !== 'number') return value
|
|
1237
|
+
const term = this.database.termManager.getTerm(value)
|
|
1238
|
+
return term === null ? value : term
|
|
1239
|
+
}
|
|
1240
|
+
|
|
1241
|
+
if (typeof condition === 'number') {
|
|
1242
|
+
return toTerm(condition)
|
|
1243
|
+
}
|
|
1244
|
+
|
|
1245
|
+
if (condition instanceof RegExp || condition === null || typeof condition !== 'object') {
|
|
1246
|
+
return condition
|
|
1247
|
+
}
|
|
1248
|
+
|
|
1249
|
+
if (Array.isArray(condition)) {
|
|
1250
|
+
return condition.map(toTerm)
|
|
1251
|
+
}
|
|
1252
|
+
|
|
1253
|
+
const normalized = {}
|
|
1254
|
+
for (const [operator, operatorValue] of Object.entries(condition)) {
|
|
1255
|
+
const canonical = normalizeOperator(operator)
|
|
1256
|
+
if (canonical === '$in' || canonical === '$all' || canonical === '$nin') {
|
|
1257
|
+
const values = Array.isArray(operatorValue) ? operatorValue : [operatorValue]
|
|
1258
|
+
normalized[operator] = values.map(toTerm)
|
|
1259
|
+
} else if (canonical === '$eq') {
|
|
1260
|
+
normalized[operator] = toTerm(operatorValue)
|
|
1261
|
+
} else {
|
|
1262
|
+
normalized[operator] = operatorValue
|
|
1263
|
+
}
|
|
1264
|
+
}
|
|
1265
|
+
return normalized
|
|
1266
|
+
}
|
|
1267
|
+
|
|
1082
1268
|
/**
|
|
1083
1269
|
* Preprocess query to handle array field syntax automatically
|
|
1084
1270
|
* @param {Object} criteria - Query criteria
|
|
1085
1271
|
* @returns {Object} - Processed criteria
|
|
1086
1272
|
*/
|
|
1087
|
-
preprocessQuery(criteria) {
|
|
1088
|
-
if (!criteria || typeof criteria !== 'object') {
|
|
1273
|
+
preprocessQuery(criteria, nested = false) {
|
|
1274
|
+
if (!criteria || typeof criteria !== 'object' || Array.isArray(criteria)) {
|
|
1089
1275
|
return criteria
|
|
1090
1276
|
}
|
|
1091
1277
|
|
|
1092
1278
|
const processed = {}
|
|
1093
1279
|
|
|
1094
1280
|
for (const [field, value] of Object.entries(criteria)) {
|
|
1281
|
+
// $and / $or / $not hold nested criteria and must be processed recursively,
|
|
1282
|
+
// otherwise conditions inside logical operators never reach the $in form.
|
|
1283
|
+
if (field === '$and' || field === '$or') {
|
|
1284
|
+
processed[field] = Array.isArray(value)
|
|
1285
|
+
? value.map(condition => this.preprocessQuery(condition, true))
|
|
1286
|
+
: value
|
|
1287
|
+
continue
|
|
1288
|
+
}
|
|
1289
|
+
|
|
1290
|
+
if (field === '$not') {
|
|
1291
|
+
processed[field] = value && typeof value === 'object' && !Array.isArray(value)
|
|
1292
|
+
? this.preprocessQuery(value, true)
|
|
1293
|
+
: value
|
|
1294
|
+
continue
|
|
1295
|
+
}
|
|
1296
|
+
|
|
1095
1297
|
// Check if this is a term mapping field
|
|
1096
1298
|
const isTermMappingField = this.database.opts.termMapping &&
|
|
1097
1299
|
this.database.termManager &&
|
|
@@ -1115,6 +1317,10 @@ export class QueryManager {
|
|
|
1115
1317
|
} else {
|
|
1116
1318
|
processed[field] = value
|
|
1117
1319
|
}
|
|
1320
|
+
} else if (nested && (value === null || value === undefined)) {
|
|
1321
|
+
// Nested branches may test for a missing value (e.g. $not: {field: null});
|
|
1322
|
+
// only top-level null queries are rejected as invalid.
|
|
1323
|
+
processed[field] = value
|
|
1118
1324
|
} else {
|
|
1119
1325
|
// Invalid value for term mapping field
|
|
1120
1326
|
throw new Error(`Invalid query for array field '${field}'. Use { $in: [value] } syntax or direct value.`)
|
|
@@ -1206,123 +1412,20 @@ export class QueryManager {
|
|
|
1206
1412
|
}
|
|
1207
1413
|
}
|
|
1208
1414
|
|
|
1209
|
-
// Handle $and queries -
|
|
1210
|
-
|
|
1211
|
-
const
|
|
1212
|
-
|
|
1213
|
-
|
|
1214
|
-
|
|
1215
|
-
return notFields.every(field => {
|
|
1216
|
-
if (!this.indexManager.opts.indexes || !this.indexManager.opts.indexes[field]) {
|
|
1217
|
-
return false;
|
|
1218
|
-
}
|
|
1219
|
-
// For term mapping, $not can be processed with indices
|
|
1220
|
-
return this.opts.termMapping && Object.keys(this.opts.indexes || {}).includes(field);
|
|
1221
|
-
});
|
|
1222
|
-
}
|
|
1223
|
-
|
|
1224
|
-
// Handle regular field conditions
|
|
1225
|
-
return Object.keys(andCondition).every(field => {
|
|
1226
|
-
if (!this.indexManager.opts.indexes || !this.indexManager.opts.indexes[field]) {
|
|
1227
|
-
return false;
|
|
1228
|
-
}
|
|
1229
|
-
|
|
1230
|
-
const condition = andCondition[field];
|
|
1231
|
-
|
|
1232
|
-
// RegExp cannot be efficiently queried using indices - must use streaming
|
|
1233
|
-
if (condition instanceof RegExp) {
|
|
1234
|
-
return false;
|
|
1235
|
-
}
|
|
1236
|
-
|
|
1237
|
-
if (typeof condition === 'object' && !Array.isArray(condition)) {
|
|
1238
|
-
const operators = Object.keys(condition).map(op => normalizeOperator(op));
|
|
1239
|
-
const indexType = this.indexManager?.opts?.indexes?.[field]
|
|
1240
|
-
const isNumericIndex = indexType === 'number' || indexType === 'auto' || indexType === 'array:number'
|
|
1241
|
-
const disallowedForNumeric = ['$all', '$in', '$not', '$regex', '$contains', '$exists', '$size']
|
|
1242
|
-
const disallowedDefault = ['$all', '$in', '$gt', '$gte', '$lt', '$lte', '$ne', '$not', '$regex', '$contains', '$exists', '$size']
|
|
1243
|
-
|
|
1244
|
-
// Check if this is a term mapping field (array:string or string fields with term mapping)
|
|
1245
|
-
const isTermMappingField = this.database.termManager &&
|
|
1246
|
-
this.database.termManager.termMappingFields &&
|
|
1247
|
-
this.database.termManager.termMappingFields.includes(field)
|
|
1248
|
-
|
|
1249
|
-
if (isTermMappingField) {
|
|
1250
|
-
const termMappingDisallowed = ['$gt', '$gte', '$lt', '$lte', '$ne', '$regex', '$contains', '$exists', '$size']
|
|
1251
|
-
return operators.every(op => !termMappingDisallowed.includes(op));
|
|
1252
|
-
} else {
|
|
1253
|
-
const disallowed = isNumericIndex ? disallowedForNumeric : disallowedDefault
|
|
1254
|
-
return operators.every(op => !disallowed.includes(op));
|
|
1255
|
-
}
|
|
1256
|
-
}
|
|
1257
|
-
return true;
|
|
1258
|
-
});
|
|
1259
|
-
});
|
|
1260
|
-
|
|
1261
|
-
if (!allAndConditionsIndexed) {
|
|
1415
|
+
// Handle $and / $or queries - every branch must be indexable to use indices
|
|
1416
|
+
for (const logicalOperator of ['$and', '$or']) {
|
|
1417
|
+
const branches = criteria[logicalOperator];
|
|
1418
|
+
if (!Array.isArray(branches)) continue;
|
|
1419
|
+
|
|
1420
|
+
if (!branches.every(branch => this._isCriteriaIndexable(branch))) {
|
|
1262
1421
|
if (this.opts.debugMode) {
|
|
1263
|
-
console.log(
|
|
1422
|
+
console.log(`📊 QueryStrategy: STREAMING - Some ${logicalOperator} conditions not indexed or operators not supported`);
|
|
1264
1423
|
}
|
|
1265
1424
|
return 'streaming';
|
|
1266
1425
|
}
|
|
1267
1426
|
}
|
|
1268
|
-
|
|
1269
|
-
const allFieldsIndexed = Object.keys(criteria).every(field => {
|
|
1270
|
-
// Skip $and and $not as they're handled separately above
|
|
1271
|
-
if (field === '$and' || field === '$not') return true;
|
|
1272
|
-
|
|
1273
|
-
if (!this.opts.indexes || !this.opts.indexes[field]) {
|
|
1274
|
-
if (this.opts.debugMode) {
|
|
1275
|
-
console.log(`🔍 Field '${field}' not indexed. Available indexes:`, Object.keys(this.opts.indexes || {}))
|
|
1276
|
-
}
|
|
1277
|
-
return false;
|
|
1278
|
-
}
|
|
1279
|
-
|
|
1280
|
-
// Check if the field uses operators that are supported by IndexManager
|
|
1281
|
-
const condition = criteria[field];
|
|
1282
|
-
|
|
1283
|
-
// RegExp cannot be efficiently queried using indices - must use streaming
|
|
1284
|
-
if (condition instanceof RegExp) {
|
|
1285
|
-
if (this.opts.debugMode) {
|
|
1286
|
-
console.log(`🔍 Field '${field}' uses RegExp - requires streaming strategy`)
|
|
1287
|
-
}
|
|
1288
|
-
return false;
|
|
1289
|
-
}
|
|
1290
|
-
|
|
1291
|
-
if (typeof condition === 'object' && !Array.isArray(condition) && condition !== null) {
|
|
1292
|
-
const operators = Object.keys(condition).map(op => normalizeOperator(op));
|
|
1293
|
-
if (this.opts.debugMode) {
|
|
1294
|
-
console.log(`🔍 Field '${field}' has operators:`, operators)
|
|
1295
|
-
}
|
|
1296
|
-
|
|
1297
|
-
const indexType = this.indexManager?.opts?.indexes?.[field]
|
|
1298
|
-
const isNumericIndex = indexType === 'number' || indexType === 'auto' || indexType === 'array:number'
|
|
1299
|
-
const isArrayStringIndex = indexType === 'array:string'
|
|
1300
|
-
const disallowedForNumeric = ['$all', '$in', '$not', '$regex', '$contains', '$exists', '$size']
|
|
1301
|
-
const disallowedDefault = ['$all', '$in', '$gt', '$gte', '$lt', '$lte', '$ne', '$not', '$regex', '$contains', '$exists', '$size']
|
|
1302
|
-
|
|
1303
|
-
// Check if this is a term mapping field (array:string or string fields with term mapping)
|
|
1304
|
-
const isTermMappingField = this.database.termManager &&
|
|
1305
|
-
this.database.termManager.termMappingFields &&
|
|
1306
|
-
this.database.termManager.termMappingFields.includes(field)
|
|
1307
|
-
|
|
1308
|
-
// With term mapping enabled on THIS FIELD, we can support complex operators via partial reads
|
|
1309
|
-
// Also support $all for array:string indexed fields (IndexManager.query supports it via Set intersection)
|
|
1310
|
-
if (isTermMappingField) {
|
|
1311
|
-
const termMappingDisallowed = ['$gt', '$gte', '$lt', '$lte', '$ne', '$regex', '$contains', '$exists', '$size']
|
|
1312
|
-
return operators.every(op => !termMappingDisallowed.includes(op));
|
|
1313
|
-
} else {
|
|
1314
|
-
let disallowed = isNumericIndex ? disallowedForNumeric : disallowedDefault
|
|
1315
|
-
// Remove $all from disallowed if field is array:string (IndexManager supports $all via Set intersection)
|
|
1316
|
-
if (isArrayStringIndex) {
|
|
1317
|
-
disallowed = disallowed.filter(op => op !== '$all')
|
|
1318
|
-
}
|
|
1319
|
-
return operators.every(op => !disallowed.includes(op));
|
|
1320
|
-
}
|
|
1321
|
-
}
|
|
1322
|
-
return true;
|
|
1323
|
-
});
|
|
1324
1427
|
|
|
1325
|
-
if (!
|
|
1428
|
+
if (!this._isCriteriaIndexable(criteria)) {
|
|
1326
1429
|
if (this.opts.debugMode) {
|
|
1327
1430
|
console.log('📊 QueryStrategy: STREAMING - Some fields not indexed or operators not supported');
|
|
1328
1431
|
}
|
|
@@ -1362,6 +1465,91 @@ export class QueryManager {
|
|
|
1362
1465
|
return 'indexed';
|
|
1363
1466
|
}
|
|
1364
1467
|
|
|
1468
|
+
/**
|
|
1469
|
+
* Whether a criteria object can be fully resolved through indexes
|
|
1470
|
+
* @param {Object} criteria - Criteria or a single logical branch
|
|
1471
|
+
* @returns {boolean} - True if every condition is index-backed
|
|
1472
|
+
*/
|
|
1473
|
+
_isCriteriaIndexable(criteria) {
|
|
1474
|
+
if (!criteria || typeof criteria !== 'object' || Array.isArray(criteria)) {
|
|
1475
|
+
return false;
|
|
1476
|
+
}
|
|
1477
|
+
|
|
1478
|
+
const fields = Object.keys(criteria);
|
|
1479
|
+
if (fields.length === 0) {
|
|
1480
|
+
return false;
|
|
1481
|
+
}
|
|
1482
|
+
|
|
1483
|
+
return fields.every(field => {
|
|
1484
|
+
const value = criteria[field];
|
|
1485
|
+
|
|
1486
|
+
if (field === '$and' || field === '$or') {
|
|
1487
|
+
return Array.isArray(value) &&
|
|
1488
|
+
value.length > 0 &&
|
|
1489
|
+
value.every(branch => this._isCriteriaIndexable(branch));
|
|
1490
|
+
}
|
|
1491
|
+
|
|
1492
|
+
if (field === '$not') {
|
|
1493
|
+
// $not is only index-supported when term mapping is enabled
|
|
1494
|
+
if (!this.opts.termMapping || !value || typeof value !== 'object' || Array.isArray(value)) {
|
|
1495
|
+
return false;
|
|
1496
|
+
}
|
|
1497
|
+
const notFields = Object.keys(value);
|
|
1498
|
+
return notFields.length > 0 && notFields.every(notField =>
|
|
1499
|
+
!!(this.indexManager.opts.indexes && this.indexManager.opts.indexes[notField])
|
|
1500
|
+
);
|
|
1501
|
+
}
|
|
1502
|
+
|
|
1503
|
+
if (field.startsWith('$')) {
|
|
1504
|
+
return false;
|
|
1505
|
+
}
|
|
1506
|
+
|
|
1507
|
+
return this._isFieldConditionIndexable(field, value);
|
|
1508
|
+
});
|
|
1509
|
+
}
|
|
1510
|
+
|
|
1511
|
+
/**
|
|
1512
|
+
* Whether a single field condition is supported by that field's index
|
|
1513
|
+
* @param {string} field - Field name
|
|
1514
|
+
* @param {*} condition - Condition value
|
|
1515
|
+
* @returns {boolean} - True if the index can answer this condition
|
|
1516
|
+
*/
|
|
1517
|
+
_isFieldConditionIndexable(field, condition) {
|
|
1518
|
+
if (!this.indexManager.opts.indexes || !this.indexManager.opts.indexes[field]) {
|
|
1519
|
+
return false;
|
|
1520
|
+
}
|
|
1521
|
+
|
|
1522
|
+
const indexType = this.indexManager.opts.indexes[field];
|
|
1523
|
+
|
|
1524
|
+
// INTENTIONAL (CHANGELOG [2.2.5]): raw RegExp on string/term indexes is resolved
|
|
1525
|
+
// via index keys (not streaming). Do not revert to "regex => streaming".
|
|
1526
|
+
if (condition instanceof RegExp) {
|
|
1527
|
+
return indexType !== 'number' && indexType !== 'array:number';
|
|
1528
|
+
}
|
|
1529
|
+
|
|
1530
|
+
// Index lookups cannot express "is null/undefined" reliably
|
|
1531
|
+
if (condition === null || condition === undefined) {
|
|
1532
|
+
return false;
|
|
1533
|
+
}
|
|
1534
|
+
|
|
1535
|
+
if (typeof condition !== 'object' || Array.isArray(condition)) {
|
|
1536
|
+
return true;
|
|
1537
|
+
}
|
|
1538
|
+
|
|
1539
|
+
const operators = Object.keys(condition).map(op => normalizeOperator(op));
|
|
1540
|
+
const isNumericIndex = indexType === 'number' || indexType === 'auto' || indexType === 'array:number';
|
|
1541
|
+
const disallowedForNumeric = ['$all', '$in', '$not', '$regex', '$contains', '$exists', '$size'];
|
|
1542
|
+
const disallowedDefault = ['$all', '$in', '$gt', '$gte', '$lt', '$lte', '$ne', '$not', '$regex', '$contains', '$exists', '$size'];
|
|
1543
|
+
|
|
1544
|
+
if (this._isTermMappingField(field)) {
|
|
1545
|
+
const termMappingDisallowed = ['$gt', '$gte', '$lt', '$lte', '$ne', '$regex', '$contains', '$exists', '$size'];
|
|
1546
|
+
return operators.every(op => !termMappingDisallowed.includes(op));
|
|
1547
|
+
}
|
|
1548
|
+
|
|
1549
|
+
const disallowed = isNumericIndex ? disallowedForNumeric : disallowedDefault;
|
|
1550
|
+
return operators.every(op => !disallowed.includes(op));
|
|
1551
|
+
}
|
|
1552
|
+
|
|
1365
1553
|
/**
|
|
1366
1554
|
* Estimate number of results for a query
|
|
1367
1555
|
* @param {Object} criteria - Query criteria
|