jexidb 2.2.5 → 2.2.6

package/dist/Database.cjs

@@ -1406,6 +1406,28 @@ class IndexManager {
       return result;
     }
 
+    // Handle $or queries - union of the branch results
+    if (criteria.$or && Array.isArray(criteria.$or)) {
+      const orResult = new Set();
+      for (const orCondition of criteria.$or) {
+        const branchLines = this.query(orCondition, options);
+        for (const lineNumber of branchLines) {
+          orResult.add(lineNumber);
+        }
+      }
+
+      // Other fields at the root level still narrow the union (AND semantics)
+      const otherCriteria = {
+        ...criteria
+      };
+      delete otherCriteria.$or;
+      if (Object.keys(otherCriteria).length > 0) {
+        const otherResults = this.query(otherCriteria, options);
+        return new Set([...orResult].filter(x => otherResults.has(x)));
+      }
+      return orResult;
+    }
+
     // Handle $and queries with parallel processing optimization
     if (criteria.$and && Array.isArray(criteria.$and)) {
       // OPTIMIZATION: Process conditions in parallel for better performance
@@ -6349,10 +6371,18 @@ class QueryManager {
         }
 
         // Use indices to pre-filter and reduce streaming scope
-        const preFilteredLines = this.indexManager.query(this._extractIndexableCriteria(criteria), streamingOptions);
+        const preFilterCriteria = this._extractIndexableCriteria(criteria);
+        const preFilteredLines = this.indexManager.query(preFilterCriteria, streamingOptions);
+
+        // Fields answered by the pre-filter already match, so the streaming matcher
+        // only evaluates the residual conditions (instead of re-checking everything).
+        const preFilteredFields = this._collectCriteriaFields(preFilterCriteria);
 
         // Stream only the pre-filtered records
-        return this._streamPreFilteredRecords(preFilteredLines, criteria, streamingOptions);
+        return this._streamPreFilteredRecords(preFilteredLines, criteria, {
+          ...streamingOptions,
+          _prefilteredFields: preFilteredFields
+        });
       }
     }
 
@@ -6404,6 +6434,17 @@ class QueryManager {
       }
     }
 
+    // Handle $or conditions - only when EVERY branch is pre-filterable, otherwise
+    // the union would narrow the candidate set and lose rows.
+    if (criteria.$or && Array.isArray(criteria.$or) && criteria.$or.length > 0) {
+      const orCriteria = criteria.$or.map(condition => this._extractIndexableCriteria(condition));
+      if (orCriteria.every(condition => Object.keys(condition).length > 0)) {
+        for (const condition of orCriteria) {
+          indexableFields.push(...this._getIndexableFields(condition));
+        }
+      }
+    }
+
     // Handle regular field conditions
     for (const [field, condition] of Object.entries(criteria)) {
       if (field.startsWith('$')) continue; // Skip logical operators
@@ -6439,6 +6480,15 @@ class QueryManager {
       }
     }
 
+    // Handle $or conditions - a branch that cannot be pre-filtered would make the
+    // union too narrow, so $or is only pre-filterable when every branch is.
+    if (criteria.$or && Array.isArray(criteria.$or) && criteria.$or.length > 0) {
+      const indexableOrConditions = criteria.$or.map(orCondition => this._extractIndexableCriteria(orCondition));
+      if (indexableOrConditions.every(condition => Object.keys(condition).length > 0)) {
+        indexableCriteria.$or = indexableOrConditions;
+      }
+    }
+
     // Handle $not operator - include it if it can be processed by IndexManager
     if (criteria.$not && typeof criteria.$not === 'object') {
       // Check if $not condition contains only indexable fields
@@ -6469,6 +6519,37 @@ class QueryManager {
     return indexableCriteria;
   }
 
+  /**
+   * Collect every field referenced by criteria, including inside $and/$or/$not
+   * @param {Object} criteria - Criteria to inspect
+   * @param {Set<string>} accumulator - Fields collected so far
+   * @returns {Set<string>} - Field names
+   */
+  _collectCriteriaFields(criteria, accumulator = new Set()) {
+    if (!criteria || typeof criteria !== 'object') {
+      return accumulator;
+    }
+    if (Array.isArray(criteria)) {
+      for (const condition of criteria) {
+        this._collectCriteriaFields(condition, accumulator);
+      }
+      return accumulator;
+    }
+    for (const [field, value] of Object.entries(criteria)) {
+      if (field === '$and' || field === '$or') {
+        this._collectCriteriaFields(value, accumulator);
+        continue;
+      }
+      if (field === '$not') {
+        // $not is negated, so its fields are not "already matched" by the pre-filter
+        continue;
+      }
+      if (field.startsWith('$')) continue;
+      accumulator.add(field);
+    }
+    return accumulator;
+  }
+
   /**
    * Determine whether the database currently has any records (persisted or pending)
    * @returns {boolean}
@@ -6674,8 +6755,8 @@ class QueryManager {
       return !notResult;
     }
 
-    // OPTIMIZATION 4: Skip indexable fields since they were already pre-filtered
-    const indexableFields = this._getIndexableFields(criteria);
+    // OPTIMIZATION 4: Skip only the fields that the pre-filter actually answered
+    const prefilteredFields = options._prefilteredFields instanceof Set ? options._prefilteredFields : new Set(this._getIndexableFields(criteria));
 
     // Handle explicit logical operators at the top level
     if (criteria.$or && Array.isArray(criteria.$or)) {
@@ -6701,8 +6782,8 @@ class QueryManager {
     for (const [field, condition] of Object.entries(criteria)) {
       if (field.startsWith('$')) continue;
 
-      // OPTIMIZATION 4: Skip indexable fields that were already pre-filtered
-      if (indexableFields.includes(field)) {
+      // OPTIMIZATION 4: Skip fields that were already pre-filtered
+      if (prefilteredFields.has(field)) {
         continue;
       }
       if (!this.matchesFieldCondition(record, field, condition, options)) {
@@ -7029,7 +7110,7 @@ class QueryManager {
    * @returns {boolean} - True if matches
    */
   matchesFieldCondition(record, field, condition, options = {}) {
-    const value = record[field];
+    let value = record[field];
 
     // Debug logging for all field conditions - SAMPLED: only the first record of
     // each query logs per field, so a full streaming scan (which calls this for
@@ -7061,6 +7142,13 @@ class QueryManager {
       }
     }
 
+    // Term-mapped array fields store numeric term IDs, so both sides are translated
+    // to terms here; otherwise string queries would never match on the streaming path.
+    if (Array.isArray(value) && this._isTermMappingField(field)) {
+      value = this._termIdsToTerms(value);
+      condition = this._normalizeTermCondition(condition);
+    }
+
     // Handle null/undefined values
     if (value === null || value === undefined) {
       return condition === null || condition === undefined;
@@ -7183,17 +7271,87 @@ class QueryManager {
     }
   }
 
+  /**
+   * Whether a field stores its values as numeric term IDs (term mapping)
+   * @param {string} field
+   * @returns {boolean}
+   */
+  _isTermMappingField(field) {
+    const termManager = this.database && this.database.termManager;
+    return !!(termManager && Array.isArray(termManager.termMappingFields) && termManager.termMappingFields.includes(field));
+  }
+
+  /**
+   * Translate stored term IDs back to terms for matching
+   * @param {Array} ids
+   * @returns {Array}
+   */
+  _termIdsToTerms(ids) {
+    const termManager = this.database.termManager;
+    return ids.map(id => {
+      if (typeof id !== 'number') return id;
+      const term = termManager.getTerm(id);
+      return term === null ? id : term;
+    });
+  }
+
+  /**
+   * Translate numeric term IDs used in a query condition to their terms
+   * @param {*} condition
+   * @returns {*}
+   */
+  _normalizeTermCondition(condition) {
+    const toTerm = value => {
+      if (typeof value !== 'number') return value;
+      const term = this.database.termManager.getTerm(value);
+      return term === null ? value : term;
+    };
+    if (typeof condition === 'number') {
+      return toTerm(condition);
+    }
+    if (condition instanceof RegExp || condition === null || typeof condition !== 'object') {
+      return condition;
+    }
+    if (Array.isArray(condition)) {
+      return condition.map(toTerm);
+    }
+    const normalized = {};
+    for (const [operator, operatorValue] of Object.entries(condition)) {
+      const canonical = normalizeOperator(operator);
+      if (canonical === '$in' || canonical === '$all' || canonical === '$nin') {
+        const values = Array.isArray(operatorValue) ? operatorValue : [operatorValue];
+        normalized[operator] = values.map(toTerm);
+      } else if (canonical === '$eq') {
+        normalized[operator] = toTerm(operatorValue);
+      } else {
+        normalized[operator] = operatorValue;
+      }
+    }
+    return normalized;
+  }
+
   /**
    * Preprocess query to handle array field syntax automatically
    * @param {Object} criteria - Query criteria
    * @returns {Object} - Processed criteria
    */
-  preprocessQuery(criteria) {
-    if (!criteria || typeof criteria !== 'object') {
+  preprocessQuery(criteria, nested = false) {
+    if (!criteria || typeof criteria !== 'object' || Array.isArray(criteria)) {
       return criteria;
     }
     const processed = {};
     for (const [field, value] of Object.entries(criteria)) {
+      // $and / $or / $not hold nested criteria and must be processed recursively,
+      // otherwise conditions inside logical operators never reach the $in form.
+      if (field === '$and' || field === '$or') {
+        processed[field] = Array.isArray(value) ? value.map(condition => this.preprocessQuery(condition, true)) : value;
+        continue;
+      }
+      if (field === '$not') {
+        processed[field] = value && typeof value === 'object' && !Array.isArray(value) ? this.preprocessQuery(value, true) : value;
+        continue;
+      }
+
       // Check if this is a term mapping field
       const isTermMappingField = this.database.opts.termMapping && this.database.termManager && this.database.termManager.termMappingFields && this.database.termManager.termMappingFields.includes(field);
       if (isTermMappingField) {
@@ -7221,6 +7379,10 @@ class QueryManager {
           } else {
             processed[field] = value;
           }
+        } else if (nested && (value === null || value === undefined)) {
+          // Nested branches may test for a missing value (e.g. $not: {field: null});
+          // only top-level null queries are rejected as invalid.
+          processed[field] = value;
         } else {
           // Invalid value for term mapping field
           throw new Error(`Invalid query for array field '${field}'. Use { $in: [value] } syntax or direct value.`);
@@ -7308,117 +7470,18 @@ class QueryManager {
       }
     }
 
-    // Handle $and queries - check if all conditions in $and are indexable
-    if (criteria.$and && Array.isArray(criteria.$and)) {
-      const allAndConditionsIndexed = criteria.$and.every(andCondition => {
-        // Handle $not conditions within $and
-        if (andCondition.$not) {
-          const notFields = Object.keys(andCondition.$not);
-          return notFields.every(field => {
-            if (!this.indexManager.opts.indexes || !this.indexManager.opts.indexes[field]) {
-              return false;
-            }
-            // For term mapping, $not can be processed with indices
-            return this.opts.termMapping && Object.keys(this.opts.indexes || {}).includes(field);
-          });
-        }
-
-        // Handle regular field conditions
-        return Object.keys(andCondition).every(field => {
-          if (!this.indexManager.opts.indexes || !this.indexManager.opts.indexes[field]) {
-            return false;
-          }
-          const condition = andCondition[field];
-
-          // INTENTIONAL (CHANGELOG [2.2.5]): raw RegExp on string/term indexes is resolved
-          // via index keys (not streaming). Do not revert to "regex => streaming".
-          if (condition instanceof RegExp) {
-            const idxType = this.indexManager?.opts?.indexes?.[field];
-            return idxType !== 'number' && idxType !== 'array:number';
-          }
-          if (typeof condition === 'object' && !Array.isArray(condition)) {
-            const operators = Object.keys(condition).map(op => normalizeOperator(op));
-            const indexType = this.indexManager?.opts?.indexes?.[field];
-            const isNumericIndex = indexType === 'number' || indexType === 'auto' || indexType === 'array:number';
-            const disallowedForNumeric = ['$all', '$in', '$not', '$regex', '$contains', '$exists', '$size'];
-            const disallowedDefault = ['$all', '$in', '$gt', '$gte', '$lt', '$lte', '$ne', '$not', '$regex', '$contains', '$exists', '$size'];
-
-            // Check if this is a term mapping field (array:string or string fields with term mapping)
-            const isTermMappingField = this.database.termManager && this.database.termManager.termMappingFields && this.database.termManager.termMappingFields.includes(field);
-            if (isTermMappingField) {
-              const termMappingDisallowed = ['$gt', '$gte', '$lt', '$lte', '$ne', '$regex', '$contains', '$exists', '$size'];
-              return operators.every(op => !termMappingDisallowed.includes(op));
-            } else {
-              const disallowed = isNumericIndex ? disallowedForNumeric : disallowedDefault;
-              return operators.every(op => !disallowed.includes(op));
-            }
-          }
-          return true;
-        });
-      });
-      if (!allAndConditionsIndexed) {
+    // Handle $and / $or queries - every branch must be indexable to use indices
+    for (const logicalOperator of ['$and', '$or']) {
+      const branches = criteria[logicalOperator];
+      if (!Array.isArray(branches)) continue;
+      if (!branches.every(branch => this._isCriteriaIndexable(branch))) {
         if (this.opts.debugMode) {
-          console.log('📊 QueryStrategy: STREAMING - Some $and conditions not indexed or operators not supported');
+          console.log(`📊 QueryStrategy: STREAMING - Some ${logicalOperator} conditions not indexed or operators not supported`);
         }
         return 'streaming';
       }
     }
-    const allFieldsIndexed = Object.keys(criteria).every(field => {
-      // Skip $and and $not as they're handled separately above
-      if (field === '$and' || field === '$not') return true;
-      if (!this.opts.indexes || !this.opts.indexes[field]) {
-        if (this.opts.debugMode) {
-          console.log(`🔍 Field '${field}' not indexed. Available indexes:`, Object.keys(this.opts.indexes || {}));
-        }
-        return false;
-      }
-
-      // Check if the field uses operators that are supported by IndexManager
-      const condition = criteria[field];
-
-      // INTENTIONAL (CHANGELOG [2.2.5]): raw RegExp on string/term indexes is resolved
-      // via index keys (not streaming). Do not revert to "regex => streaming".
-      if (condition instanceof RegExp) {
-        const idxType = this.indexManager?.opts?.indexes?.[field];
-        if (idxType === 'number' || idxType === 'array:number') {
-          if (this.opts.debugMode) {
-            console.log(`🔍 Field '${field}' uses RegExp on a numeric index - requires streaming strategy`);
-          }
-          return false;
-        }
-        return true;
-      }
-      if (typeof condition === 'object' && !Array.isArray(condition) && condition !== null) {
-        const operators = Object.keys(condition).map(op => normalizeOperator(op));
-        if (this.opts.debugMode) {
-          console.log(`🔍 Field '${field}' has operators:`, operators);
-        }
-        const indexType = this.indexManager?.opts?.indexes?.[field];
-        const isNumericIndex = indexType === 'number' || indexType === 'auto' || indexType === 'array:number';
-        const isArrayStringIndex = indexType === 'array:string';
-        const disallowedForNumeric = ['$all', '$in', '$not', '$regex', '$contains', '$exists', '$size'];
-        const disallowedDefault = ['$all', '$in', '$gt', '$gte', '$lt', '$lte', '$ne', '$not', '$regex', '$contains', '$exists', '$size'];
-
-        // Check if this is a term mapping field (array:string or string fields with term mapping)
-        const isTermMappingField = this.database.termManager && this.database.termManager.termMappingFields && this.database.termManager.termMappingFields.includes(field);
-
-        // With term mapping enabled on THIS FIELD, we can support complex operators via partial reads
-        // Also support $all for array:string indexed fields (IndexManager.query supports it via Set intersection)
-        if (isTermMappingField) {
-          const termMappingDisallowed = ['$gt', '$gte', '$lt', '$lte', '$ne', '$regex', '$contains', '$exists', '$size'];
-          return operators.every(op => !termMappingDisallowed.includes(op));
-        } else {
-          let disallowed = isNumericIndex ? disallowedForNumeric : disallowedDefault;
-          // Remove $all from disallowed if field is array:string (IndexManager supports $all via Set intersection)
-          if (isArrayStringIndex) {
-            disallowed = disallowed.filter(op => op !== '$all');
-          }
-          return operators.every(op => !disallowed.includes(op));
-        }
-      }
-      return true;
-    });
-    if (!allFieldsIndexed) {
+    if (!this._isCriteriaIndexable(criteria)) {
       if (this.opts.debugMode) {
         console.log('📊 QueryStrategy: STREAMING - Some fields not indexed or operators not supported');
       }
@@ -7457,6 +7520,76 @@ class QueryManager {
     return 'indexed';
   }
 
+  /**
+   * Whether a criteria object can be fully resolved through indexes
+   * @param {Object} criteria - Criteria or a single logical branch
+   * @returns {boolean} - True if every condition is index-backed
+   */
+  _isCriteriaIndexable(criteria) {
+    if (!criteria || typeof criteria !== 'object' || Array.isArray(criteria)) {
+      return false;
+    }
+    const fields = Object.keys(criteria);
+    if (fields.length === 0) {
+      return false;
+    }
+    return fields.every(field => {
+      const value = criteria[field];
+      if (field === '$and' || field === '$or') {
+        return Array.isArray(value) && value.length > 0 && value.every(branch => this._isCriteriaIndexable(branch));
+      }
+      if (field === '$not') {
+        // $not is only index-supported when term mapping is enabled
+        if (!this.opts.termMapping || !value || typeof value !== 'object' || Array.isArray(value)) {
+          return false;
+        }
+        const notFields = Object.keys(value);
+        return notFields.length > 0 && notFields.every(notField => !!(this.indexManager.opts.indexes && this.indexManager.opts.indexes[notField]));
+      }
+      if (field.startsWith('$')) {
+        return false;
+      }
+      return this._isFieldConditionIndexable(field, value);
+    });
+  }
+
+  /**
+   * Whether a single field condition is supported by that field's index
+   * @param {string} field - Field name
+   * @param {*} condition - Condition value
+   * @returns {boolean} - True if the index can answer this condition
+   */
+  _isFieldConditionIndexable(field, condition) {
+    if (!this.indexManager.opts.indexes || !this.indexManager.opts.indexes[field]) {
+      return false;
+    }
+    const indexType = this.indexManager.opts.indexes[field];
+
+    // INTENTIONAL (CHANGELOG [2.2.5]): raw RegExp on string/term indexes is resolved
+    // via index keys (not streaming). Do not revert to "regex => streaming".
+    if (condition instanceof RegExp) {
+      return indexType !== 'number' && indexType !== 'array:number';
+    }
+
+    // Index lookups cannot express "is null/undefined" reliably
+    if (condition === null || condition === undefined) {
+      return false;
+    }
+    if (typeof condition !== 'object' || Array.isArray(condition)) {
+      return true;
+    }
+    const operators = Object.keys(condition).map(op => normalizeOperator(op));
+    const isNumericIndex = indexType === 'number' || indexType === 'auto' || indexType === 'array:number';
+    const disallowedForNumeric = ['$all', '$in', '$not', '$regex', '$contains', '$exists', '$size'];
+    const disallowedDefault = ['$all', '$in', '$gt', '$gte', '$lt', '$lte', '$ne', '$not', '$regex', '$contains', '$exists', '$size'];
+    if (this._isTermMappingField(field)) {
+      const termMappingDisallowed = ['$gt', '$gte', '$lt', '$lte', '$ne', '$regex', '$contains', '$exists', '$size'];
+      return operators.every(op => !termMappingDisallowed.includes(op));
+    }
+    const disallowed = isNumericIndex ? disallowedForNumeric : disallowedDefault;
+    return operators.every(op => !disallowed.includes(op));
+  }
+
   /**
    * Estimate number of results for a query
    * @param {Object} criteria - Query criteria

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "jexidb",
-  "version": "2.2.5",
+  "version": "2.2.6",
   "type": "module",
   "description": "JexiDB is a pure JS NPM library for managing data on disk efficiently, without the need for a server.",
   "main": "./dist/Database.cjs",

package/src/managers/IndexManager.mjs

@@ -1016,6 +1016,29 @@ export default class IndexManager {
       return result;
     }
     
+    // Handle $or queries - union of the branch results
+    if (criteria.$or && Array.isArray(criteria.$or)) {
+      const orResult = new Set();
+
+      for (const orCondition of criteria.$or) {
+        const branchLines = this.query(orCondition, options);
+        for (const lineNumber of branchLines) {
+          orResult.add(lineNumber);
+        }
+      }
+
+      // Other fields at the root level still narrow the union (AND semantics)
+      const otherCriteria = { ...criteria };
+      delete otherCriteria.$or;
+
+      if (Object.keys(otherCriteria).length > 0) {
+        const otherResults = this.query(otherCriteria, options);
+        return new Set([...orResult].filter(x => otherResults.has(x)));
+      }
+
+      return orResult;
+    }
+
     // Handle $and queries with parallel processing optimization
     if (criteria.$and && Array.isArray(criteria.$and)) {
       // OPTIMIZATION: Process conditions in parallel for better performance

package/src/managers/QueryManager.mjs

@@ -286,13 +286,18 @@ export class QueryManager {
         }
         
         // Use indices to pre-filter and reduce streaming scope
-        const preFilteredLines = this.indexManager.query(
-          this._extractIndexableCriteria(criteria), 
-          streamingOptions
-        );
-        
+        const preFilterCriteria = this._extractIndexableCriteria(criteria);
+        const preFilteredLines = this.indexManager.query(preFilterCriteria, streamingOptions);
+
+        // Fields answered by the pre-filter already match, so the streaming matcher
+        // only evaluates the residual conditions (instead of re-checking everything).
+        const preFilteredFields = this._collectCriteriaFields(preFilterCriteria);
+
         // Stream only the pre-filtered records
-        return this._streamPreFilteredRecords(preFilteredLines, criteria, streamingOptions);
+        return this._streamPreFilteredRecords(preFilteredLines, criteria, {
+          ...streamingOptions,
+          _prefilteredFields: preFilteredFields
+        });
       }
     }
     
@@ -345,6 +350,17 @@ export class QueryManager {
         indexableFields.push(...this._getIndexableFields(andCondition));
       }
     }
+
+    // Handle $or conditions - only when EVERY branch is pre-filterable, otherwise
+    // the union would narrow the candidate set and lose rows.
+    if (criteria.$or && Array.isArray(criteria.$or) && criteria.$or.length > 0) {
+      const orCriteria = criteria.$or.map(condition => this._extractIndexableCriteria(condition));
+      if (orCriteria.every(condition => Object.keys(condition).length > 0)) {
+        for (const condition of orCriteria) {
+          indexableFields.push(...this._getIndexableFields(condition));
+        }
+      }
+    }
     
     // Handle regular field conditions
     for (const [field, condition] of Object.entries(criteria)) {
@@ -386,6 +402,17 @@ export class QueryManager {
         indexableCriteria.$and = indexableAndConditions;
       }
     }
+
+    // Handle $or conditions - a branch that cannot be pre-filtered would make the
+    // union too narrow, so $or is only pre-filterable when every branch is.
+    if (criteria.$or && Array.isArray(criteria.$or) && criteria.$or.length > 0) {
+      const indexableOrConditions = criteria.$or
+        .map(orCondition => this._extractIndexableCriteria(orCondition));
+
+      if (indexableOrConditions.every(condition => Object.keys(condition).length > 0)) {
+        indexableCriteria.$or = indexableOrConditions;
+      }
+    }
     
     // Handle $not operator - include it if it can be processed by IndexManager
     if (criteria.$not && typeof criteria.$not === 'object') {
@@ -422,6 +449,43 @@ export class QueryManager {
     return indexableCriteria;
   }
 
+  /**
+   * Collect every field referenced by criteria, including inside $and/$or/$not
+   * @param {Object} criteria - Criteria to inspect
+   * @param {Set<string>} accumulator - Fields collected so far
+   * @returns {Set<string>} - Field names
+   */
+  _collectCriteriaFields(criteria, accumulator = new Set()) {
+    if (!criteria || typeof criteria !== 'object') {
+      return accumulator;
+    }
+
+    if (Array.isArray(criteria)) {
+      for (const condition of criteria) {
+        this._collectCriteriaFields(condition, accumulator);
+      }
+      return accumulator;
+    }
+
+    for (const [field, value] of Object.entries(criteria)) {
+      if (field === '$and' || field === '$or') {
+        this._collectCriteriaFields(value, accumulator);
+        continue;
+      }
+
+      if (field === '$not') {
+        // $not is negated, so its fields are not "already matched" by the pre-filter
+        continue;
+      }
+
+      if (field.startsWith('$')) continue;
+
+      accumulator.add(field);
+    }
+
+    return accumulator;
+  }
+
   /**
    * Determine whether the database currently has any records (persisted or pending)
    * @returns {boolean}
@@ -638,8 +702,10 @@ export class QueryManager {
       return !notResult;
     }
 
-    // OPTIMIZATION 4: Skip indexable fields since they were already pre-filtered
-    const indexableFields = this._getIndexableFields(criteria);
+    // OPTIMIZATION 4: Skip only the fields that the pre-filter actually answered
+    const prefilteredFields = options._prefilteredFields instanceof Set
+      ? options._prefilteredFields
+      : new Set(this._getIndexableFields(criteria));
     
     // Handle explicit logical operators at the top level
     if (criteria.$or && Array.isArray(criteria.$or)) {
@@ -666,8 +732,8 @@ export class QueryManager {
     for (const [field, condition] of Object.entries(criteria)) {
       if (field.startsWith('$')) continue;
       
-      // OPTIMIZATION 4: Skip indexable fields that were already pre-filtered
-      if (indexableFields.includes(field)) {
+      // OPTIMIZATION 4: Skip fields that were already pre-filtered
+      if (prefilteredFields.has(field)) {
         continue;
       }
       
@@ -980,7 +1046,7 @@ export class QueryManager {
    * @returns {boolean} - True if matches
    */
   matchesFieldCondition(record, field, condition, options = {}) {
-    const value = record[field];
+    let value = record[field];
 
     // Debug logging for all field conditions - SAMPLED: only the first record of
     // each query logs per field, so a full streaming scan (which calls this for
@@ -1004,6 +1070,13 @@ export class QueryManager {
       }
     }
 
+    // Term-mapped array fields store numeric term IDs, so both sides are translated
+    // to terms here; otherwise string queries would never match on the streaming path.
+    if (Array.isArray(value) && this._isTermMappingField(field)) {
+      value = this._termIdsToTerms(value);
+      condition = this._normalizeTermCondition(condition);
+    }
+
     // Handle null/undefined values
     if (value === null || value === undefined) {
       return condition === null || condition === undefined;
@@ -1127,19 +1200,100 @@ export class QueryManager {
   }
 
 
+  /**
+   * Whether a field stores its values as numeric term IDs (term mapping)
+   * @param {string} field
+   * @returns {boolean}
+   */
+  _isTermMappingField(field) {
+    const termManager = this.database && this.database.termManager
+    return !!(termManager &&
+      Array.isArray(termManager.termMappingFields) &&
+      termManager.termMappingFields.includes(field))
+  }
+
+  /**
+   * Translate stored term IDs back to terms for matching
+   * @param {Array} ids
+   * @returns {Array}
+   */
+  _termIdsToTerms(ids) {
+    const termManager = this.database.termManager
+    return ids.map(id => {
+      if (typeof id !== 'number') return id
+      const term = termManager.getTerm(id)
+      return term === null ? id : term
+    })
+  }
+
+  /**
+   * Translate numeric term IDs used in a query condition to their terms
+   * @param {*} condition
+   * @returns {*}
+   */
+  _normalizeTermCondition(condition) {
+    const toTerm = value => {
+      if (typeof value !== 'number') return value
+      const term = this.database.termManager.getTerm(value)
+      return term === null ? value : term
+    }
+
+    if (typeof condition === 'number') {
+      return toTerm(condition)
+    }
+
+    if (condition instanceof RegExp || condition === null || typeof condition !== 'object') {
+      return condition
+    }
+
+    if (Array.isArray(condition)) {
+      return condition.map(toTerm)
+    }
+
+    const normalized = {}
+    for (const [operator, operatorValue] of Object.entries(condition)) {
+      const canonical = normalizeOperator(operator)
+      if (canonical === '$in' || canonical === '$all' || canonical === '$nin') {
+        const values = Array.isArray(operatorValue) ? operatorValue : [operatorValue]
+        normalized[operator] = values.map(toTerm)
+      } else if (canonical === '$eq') {
+        normalized[operator] = toTerm(operatorValue)
+      } else {
+        normalized[operator] = operatorValue
+      }
+    }
+    return normalized
+  }
+
   /**
    * Preprocess query to handle array field syntax automatically
    * @param {Object} criteria - Query criteria
    * @returns {Object} - Processed criteria
    */
-  preprocessQuery(criteria) {
-    if (!criteria || typeof criteria !== 'object') {
+  preprocessQuery(criteria, nested = false) {
+    if (!criteria || typeof criteria !== 'object' || Array.isArray(criteria)) {
       return criteria
     }
 
     const processed = {}
     
     for (const [field, value] of Object.entries(criteria)) {
+      // $and / $or / $not hold nested criteria and must be processed recursively,
+      // otherwise conditions inside logical operators never reach the $in form.
+      if (field === '$and' || field === '$or') {
+        processed[field] = Array.isArray(value)
+          ? value.map(condition => this.preprocessQuery(condition, true))
+          : value
+        continue
+      }
+
+      if (field === '$not') {
+        processed[field] = value && typeof value === 'object' && !Array.isArray(value)
+          ? this.preprocessQuery(value, true)
+          : value
+        continue
+      }
+
       // Check if this is a term mapping field
       const isTermMappingField = this.database.opts.termMapping && 
         this.database.termManager && 
@@ -1163,6 +1317,10 @@ export class QueryManager {
           } else {
             processed[field] = value
           }
+        } else if (nested && (value === null || value === undefined)) {
+          // Nested branches may test for a missing value (e.g. $not: {field: null});
+          // only top-level null queries are rejected as invalid.
+          processed[field] = value
         } else {
           // Invalid value for term mapping field
           throw new Error(`Invalid query for array field '${field}'. Use { $in: [value] } syntax or direct value.`)
@@ -1254,130 +1412,20 @@ export class QueryManager {
       }
     }
     
-    // Handle $and queries - check if all conditions in $and are indexable
-    if (criteria.$and && Array.isArray(criteria.$and)) {
-      const allAndConditionsIndexed = criteria.$and.every(andCondition => {
-        // Handle $not conditions within $and
-        if (andCondition.$not) {
-          const notFields = Object.keys(andCondition.$not);
-          return notFields.every(field => {
-            if (!this.indexManager.opts.indexes || !this.indexManager.opts.indexes[field]) {
-              return false;
-            }
-            // For term mapping, $not can be processed with indices
-            return this.opts.termMapping && Object.keys(this.opts.indexes || {}).includes(field);
-          });
-        }
-        
-        // Handle regular field conditions
-        return Object.keys(andCondition).every(field => {
-          if (!this.indexManager.opts.indexes || !this.indexManager.opts.indexes[field]) {
-            return false;
-          }
-          
-          const condition = andCondition[field];
-          
-          // INTENTIONAL (CHANGELOG [2.2.5]): raw RegExp on string/term indexes is resolved
-          // via index keys (not streaming). Do not revert to "regex => streaming".
-          if (condition instanceof RegExp) {
-            const idxType = this.indexManager?.opts?.indexes?.[field];
-            return idxType !== 'number' && idxType !== 'array:number';
-          }
-          
-          if (typeof condition === 'object' && !Array.isArray(condition)) {
-            const operators = Object.keys(condition).map(op => normalizeOperator(op));
-            const indexType = this.indexManager?.opts?.indexes?.[field]
-            const isNumericIndex = indexType === 'number' || indexType === 'auto' || indexType === 'array:number'
-            const disallowedForNumeric = ['$all', '$in', '$not', '$regex', '$contains', '$exists', '$size']
-            const disallowedDefault = ['$all', '$in', '$gt', '$gte', '$lt', '$lte', '$ne', '$not', '$regex', '$contains', '$exists', '$size']
-            
-            // Check if this is a term mapping field (array:string or string fields with term mapping)
-            const isTermMappingField = this.database.termManager && 
-              this.database.termManager.termMappingFields && 
-              this.database.termManager.termMappingFields.includes(field)
-            
-            if (isTermMappingField) {
-              const termMappingDisallowed = ['$gt', '$gte', '$lt', '$lte', '$ne', '$regex', '$contains', '$exists', '$size']
-              return operators.every(op => !termMappingDisallowed.includes(op));
-            } else {
-              const disallowed = isNumericIndex ? disallowedForNumeric : disallowedDefault
-              return operators.every(op => !disallowed.includes(op));
-            }
-          }
-          return true;
-        });
-      });
-      
-      if (!allAndConditionsIndexed) {
+    // Handle $and / $or queries - every branch must be indexable to use indices
+    for (const logicalOperator of ['$and', '$or']) {
+      const branches = criteria[logicalOperator];
+      if (!Array.isArray(branches)) continue;
+
+      if (!branches.every(branch => this._isCriteriaIndexable(branch))) {
         if (this.opts.debugMode) {
-          console.log('📊 QueryStrategy: STREAMING - Some $and conditions not indexed or operators not supported');
+          console.log(`📊 QueryStrategy: STREAMING - Some ${logicalOperator} conditions not indexed or operators not supported`);
         }
         return 'streaming';
       }
     }
-    
-    const allFieldsIndexed = Object.keys(criteria).every(field => {
-      // Skip $and and $not as they're handled separately above
-      if (field === '$and' || field === '$not') return true;
-      
-      if (!this.opts.indexes || !this.opts.indexes[field]) {
-        if (this.opts.debugMode) {
-          console.log(`🔍 Field '${field}' not indexed. Available indexes:`, Object.keys(this.opts.indexes || {}))
-        }
-        return false;
-      }
-      
-      // Check if the field uses operators that are supported by IndexManager
-      const condition = criteria[field];
-      
-      // INTENTIONAL (CHANGELOG [2.2.5]): raw RegExp on string/term indexes is resolved
-      // via index keys (not streaming). Do not revert to "regex => streaming".
-      if (condition instanceof RegExp) {
-        const idxType = this.indexManager?.opts?.indexes?.[field];
-        if (idxType === 'number' || idxType === 'array:number') {
-          if (this.opts.debugMode) {
-            console.log(`🔍 Field '${field}' uses RegExp on a numeric index - requires streaming strategy`)
-          }
-          return false;
-        }
-        return true;
-      }
-      
-      if (typeof condition === 'object' && !Array.isArray(condition) && condition !== null) {
-        const operators = Object.keys(condition).map(op => normalizeOperator(op));
-        if (this.opts.debugMode) {
-          console.log(`🔍 Field '${field}' has operators:`, operators)
-        }
-        
-        const indexType = this.indexManager?.opts?.indexes?.[field]
-        const isNumericIndex = indexType === 'number' || indexType === 'auto' || indexType === 'array:number'
-        const isArrayStringIndex = indexType === 'array:string'
-        const disallowedForNumeric = ['$all', '$in', '$not', '$regex', '$contains', '$exists', '$size']
-        const disallowedDefault = ['$all', '$in', '$gt', '$gte', '$lt', '$lte', '$ne', '$not', '$regex', '$contains', '$exists', '$size']
-        
-        // Check if this is a term mapping field (array:string or string fields with term mapping)
-        const isTermMappingField = this.database.termManager && 
-          this.database.termManager.termMappingFields && 
-          this.database.termManager.termMappingFields.includes(field)
-        
-        // With term mapping enabled on THIS FIELD, we can support complex operators via partial reads
-        // Also support $all for array:string indexed fields (IndexManager.query supports it via Set intersection)
-        if (isTermMappingField) {
-          const termMappingDisallowed = ['$gt', '$gte', '$lt', '$lte', '$ne', '$regex', '$contains', '$exists', '$size']
-          return operators.every(op => !termMappingDisallowed.includes(op));
-        } else {
-          let disallowed = isNumericIndex ? disallowedForNumeric : disallowedDefault
-          // Remove $all from disallowed if field is array:string (IndexManager supports $all via Set intersection)
-          if (isArrayStringIndex) {
-            disallowed = disallowed.filter(op => op !== '$all')
-          }
-          return operators.every(op => !disallowed.includes(op));
-        }
-      }
-      return true;
-    });
 
-    if (!allFieldsIndexed) {
+    if (!this._isCriteriaIndexable(criteria)) {
       if (this.opts.debugMode) {
         console.log('📊 QueryStrategy: STREAMING - Some fields not indexed or operators not supported');
       }
@@ -1417,6 +1465,91 @@ export class QueryManager {
     return 'indexed';
   }
 
+  /**
+   * Whether a criteria object can be fully resolved through indexes
+   * @param {Object} criteria - Criteria or a single logical branch
+   * @returns {boolean} - True if every condition is index-backed
+   */
+  _isCriteriaIndexable(criteria) {
+    if (!criteria || typeof criteria !== 'object' || Array.isArray(criteria)) {
+      return false;
+    }
+
+    const fields = Object.keys(criteria);
+    if (fields.length === 0) {
+      return false;
+    }
+
+    return fields.every(field => {
+      const value = criteria[field];
+
+      if (field === '$and' || field === '$or') {
+        return Array.isArray(value) &&
+          value.length > 0 &&
+          value.every(branch => this._isCriteriaIndexable(branch));
+      }
+
+      if (field === '$not') {
+        // $not is only index-supported when term mapping is enabled
+        if (!this.opts.termMapping || !value || typeof value !== 'object' || Array.isArray(value)) {
+          return false;
+        }
+        const notFields = Object.keys(value);
+        return notFields.length > 0 && notFields.every(notField =>
+          !!(this.indexManager.opts.indexes && this.indexManager.opts.indexes[notField])
+        );
+      }
+
+      if (field.startsWith('$')) {
+        return false;
+      }
+
+      return this._isFieldConditionIndexable(field, value);
+    });
+  }
+
+  /**
+   * Whether a single field condition is supported by that field's index
+   * @param {string} field - Field name
+   * @param {*} condition - Condition value
+   * @returns {boolean} - True if the index can answer this condition
+   */
+  _isFieldConditionIndexable(field, condition) {
+    if (!this.indexManager.opts.indexes || !this.indexManager.opts.indexes[field]) {
+      return false;
+    }
+
+    const indexType = this.indexManager.opts.indexes[field];
+
+    // INTENTIONAL (CHANGELOG [2.2.5]): raw RegExp on string/term indexes is resolved
+    // via index keys (not streaming). Do not revert to "regex => streaming".
+    if (condition instanceof RegExp) {
+      return indexType !== 'number' && indexType !== 'array:number';
+    }
+
+    // Index lookups cannot express "is null/undefined" reliably
+    if (condition === null || condition === undefined) {
+      return false;
+    }
+
+    if (typeof condition !== 'object' || Array.isArray(condition)) {
+      return true;
+    }
+
+    const operators = Object.keys(condition).map(op => normalizeOperator(op));
+    const isNumericIndex = indexType === 'number' || indexType === 'auto' || indexType === 'array:number';
+    const disallowedForNumeric = ['$all', '$in', '$not', '$regex', '$contains', '$exists', '$size'];
+    const disallowedDefault = ['$all', '$in', '$gt', '$gte', '$lt', '$lte', '$ne', '$not', '$regex', '$contains', '$exists', '$size'];
+
+    if (this._isTermMappingField(field)) {
+      const termMappingDisallowed = ['$gt', '$gte', '$lt', '$lte', '$ne', '$regex', '$contains', '$exists', '$size'];
+      return operators.every(op => !termMappingDisallowed.includes(op));
+    }
+
+    const disallowed = isNumericIndex ? disallowedForNumeric : disallowedDefault;
+    return operators.every(op => !disallowed.includes(op));
+  }
+
   /**
    * Estimate number of results for a query
    * @param {Object} criteria - Query criteria