jexidb 2.2.2 → 2.2.5

package/src/managers/IndexManager.mjs

@@ -1074,18 +1074,44 @@ export default class IndexManager {
       if (typeof data[field] === 'undefined') continue;
       
       const originalCriteriaValue = criteria[field];
-      const criteriaValue = normalizeCriteriaOperators(originalCriteriaValue, { target: 'legacy', preserveOriginal: true });
+      // INTENTIONAL (CHANGELOG [2.2.5]): RegExp is not an operator map - never normalize it
+      // (spreading a RegExp yields {} and would make the regex match every record).
+      const criteriaValue = originalCriteriaValue instanceof RegExp
+        ? originalCriteriaValue
+        : normalizeCriteriaOperators(originalCriteriaValue, { target: 'legacy', preserveOriginal: true });
       let lineNumbersForField = new Set();
       const isNumericField = this.opts.indexes[field] === 'number';
   
-      // Handle RegExp values directly (MUST check before object check since RegExp is an object)
+      // INTENTIONAL OPTIMIZATION (see CHANGELOG [2.2.5]): raw RegExp on string/term
+      // indexes is answered via the index keys instead of full-file streaming. For
+      // term-mapped fields the keys are numeric term IDs, translated back to words with
+      // termManager.getTerm() before testing the regex. DO NOT change this back to
+      // "RegExp => streaming" - it regresses partial search to a full scan per database.
       if (criteriaValue instanceof RegExp) {
-        // RegExp cannot be efficiently queried using indices - fall back to streaming
-        // This will be handled by the QueryManager's streaming strategy
-        continue;
+        const fieldIndex = data[field];
+        const indexType = this.opts.indexes && this.opts.indexes[field];
+        const isNumericIndexType = indexType === 'number' || indexType === 'array:number';
+        if (!fieldIndex || isNumericIndexType) {
+          // Not index-answerable (numeric index) - handled by streaming fallback
+          continue;
+        }
+        const termManager = this.database && this.database.termManager;
+        const isTermField = !!(termManager && termManager.termMappingFields &&
+          termManager.termMappingFields.includes(field));
+        const regex = criteriaValue;
+        for (const key of Object.keys(fieldIndex)) {
+          const word = isTermField ? termManager.getTerm(Number(key)) : key;
+          if (word === null || word === undefined) continue;
+          regex.lastIndex = 0;
+          if (!regex.test(String(word))) continue;
+          const numbers = this._getAllLineNumbers(fieldIndex[key]);
+          for (const lineNumber of numbers) lineNumbersForField.add(lineNumber);
+        }
+        // Fall through to field consolidation below (RegExp is an object, so it is
+        // excluded from the object/operator branch that follows).
       }
 
-      if (typeof criteriaValue === 'object' && !Array.isArray(criteriaValue) && criteriaValue !== null) {
+      if (typeof criteriaValue === 'object' && !(criteriaValue instanceof RegExp) && !Array.isArray(criteriaValue) && criteriaValue !== null) {
         const fieldIndex = data[field];
 
         // Handle $in operator for array queries
@@ -2351,6 +2377,11 @@ export default class IndexManager {
    * on-disk index after an unload must first ensure it is loaded again, e.g.
    * `await database._ensureLazyIndexLoaded()`.
    *
+   * The return format is ALWAYS the actual term strings (words/values), never the
+   * internal numeric term IDs. For term-mapped columns the index stores term IDs as
+   * keys, so they are translated back to words through the TermManager before being
+   * returned. This keeps the API consistent across indexed fields.
+   *
    * @param {string} column - Indexed field name (e.g. 'nameTerms', 'groupTerms', 'category')
    * @returns {Set<string>} Terms currently indexed for the column (empty Set if the
    * column is not indexed or the index is not loaded)
@@ -2359,7 +2390,35 @@ export default class IndexManager {
     if (!this.index || !this.index.data || !this.index.data[column]) {
       return new Set()
     }
-    return new Set(Object.keys(this.index.data[column]))
+
+    const keys = Object.keys(this.index.data[column])
+
+    // Consistent contract: always return term strings, never numeric term IDs.
+    const termManager = this.database?.termManager
+    const isTermMappingField = Boolean(
+      termManager &&
+      termManager.termMappingFields &&
+      termManager.termMappingFields.includes(column)
+    )
+
+    if (!isTermMappingField) {
+      return new Set(keys)
+    }
+
+    // Term-mapped column: index keys are numeric term IDs -> translate back to words.
+    const terms = new Set()
+    for (const key of keys) {
+      if (/^\d+$/.test(key)) {
+        const word = termManager.getTerm?.(Number(key))
+        if (word !== null && word !== undefined) {
+          terms.add(word)
+        }
+      } else {
+        // Defensive: key is already a term string (legacy/in-memory data)
+        terms.add(key)
+      }
+    }
+    return terms
   }
 
   /**

package/src/managers/QueryManager.mjs

@@ -304,6 +304,29 @@ export class QueryManager {
     return this._streamAllRecords(criteria, streamingOptions);
   }
 
+  /**
+   * Whether a single field condition can/should be answered by the index during
+   * pre-filtering (so the LEAST number of entries is streamed).
+   *
+   * - Raw RegExp is pre-filterable only on string/term indexes (IndexManager.query
+   *   resolves it via the index keys). Regex on numeric indexes -> residual.
+   * - Complement operators ($ne/$nin) expand to nearly every record, so they are
+   *   NOT pre-filtered: they are applied LAST, on the minimal candidate set, by the
+   *   streaming matcher. (CHANGELOG [2.2.5])
+   */
+  _canPrefilterFieldCondition(field, condition) {
+    if (!this.indexManager?.opts?.indexes?.[field]) return false
+    if (condition instanceof RegExp) {
+      const idxType = this.indexManager.opts.indexes[field]
+      return idxType !== 'number' && idxType !== 'array:number'
+    }
+    if (condition && typeof condition === 'object' && !Array.isArray(condition)) {
+      const ops = Object.keys(condition).map(op => normalizeOperator(op))
+      if (ops.some(op => op === '$ne' || op === '$nin')) return false
+    }
+    return true
+  }
+
   /**
    * Get indexable fields from criteria
    * @param {Object} criteria - Query criteria
@@ -327,8 +350,9 @@ export class QueryManager {
     for (const [field, condition] of Object.entries(criteria)) {
       if (field.startsWith('$')) continue; // Skip logical operators
       
-      // RegExp conditions cannot be pre-filtered using indices
-      if (condition instanceof RegExp) {
+      // Pre-filter with the index only what it can answer minimally; everything
+      // else is applied LAST over the streamed candidates (filter last, stream least).
+      if (!this._canPrefilterFieldCondition(field, condition)) {
         continue;
       }
       
@@ -384,8 +408,9 @@ export class QueryManager {
     for (const [field, condition] of Object.entries(criteria)) {
       if (field.startsWith('$')) continue; // Skip logical operators (already handled above)
       
-      // RegExp conditions cannot be pre-filtered using indices
-      if (condition instanceof RegExp) {
+      // Pre-filter with the index only what it can answer minimally; complement/
+      // numeric-regex conditions are left residual and applied LAST on the candidates.
+      if (!this._canPrefilterFieldCondition(field, condition)) {
         continue;
       }
       
@@ -728,6 +753,19 @@ export class QueryManager {
       }
       return this.findWithStreaming(criteria, { ...options, forceFullScan: true })
     }
+
+    // Swap detection: if another process atomically replaced the data file, the
+    // in-memory offsets/index describe the previous file. Reload cleanly, or -
+    // when a writer sentinel is present / the clean reload fails - fall back to
+    // a full streaming scan so stale byte offsets are never used on the new file.
+    const refreshStatus = await this.database._refreshFromDiskIfChanged()
+    if (refreshStatus === 'busy') {
+      if (this.opts.debugMode) {
+        console.log('⚠️ findWithIndexed: file busy (writer active) or clean reload failed, falling back to full streaming')
+      }
+      return this.findWithStreaming(criteria, { ...options, forceFullScan: true })
+    }
+
     let results = []
     const limit = options.limit // No default limit - return all results unless explicitly limited
 
@@ -944,15 +982,25 @@ export class QueryManager {
   matchesFieldCondition(record, field, condition, options = {}) {
     const value = record[field];
 
-    // Debug logging for all field conditions
+    // Debug logging for all field conditions - SAMPLED: only the first record of
+    // each query logs per field, so a full streaming scan (which calls this for
+    // every record) does not spam the console with one line per record.
     if (this.database.opts.debugMode) {
-      console.log(`🔍 Checking field '${field}':`, { value, condition, record: record.name || record.id });
+      if (!options._dbFieldLog) options._dbFieldLog = {};
+      if (!options._dbFieldLog[field]) {
+        options._dbFieldLog[field] = true;
+        console.log(`🔍 Checking field '${field}':`, { value, condition, record: record.name || record.id });
+      }
     }
 
-    // Debug logging for term mapping fields
+    // Debug logging for term mapping fields (sampled the same way)
     if (this.database.opts.termMapping && Object.keys(this.database.opts.indexes || {}).includes(field)) {
       if (this.database.opts.debugMode) {
-        console.log(`🔍 Checking term mapping field '${field}':`, { value, condition, record: record.name || record.id });
+        if (!options._dbFieldLog) options._dbFieldLog = {};
+        if (!options._dbFieldLog[`tm:${field}`]) {
+          options._dbFieldLog[`tm:${field}`] = true;
+          console.log(`🔍 Checking term mapping field '${field}':`, { value, condition, record: record.name || record.id });
+        }
       }
     }
 
@@ -1229,9 +1277,11 @@ export class QueryManager {
           
           const condition = andCondition[field];
           
-          // RegExp cannot be efficiently queried using indices - must use streaming
+          // INTENTIONAL (CHANGELOG [2.2.5]): raw RegExp on string/term indexes is resolved
+          // via index keys (not streaming). Do not revert to "regex => streaming".
           if (condition instanceof RegExp) {
-            return false;
+            const idxType = this.indexManager?.opts?.indexes?.[field];
+            return idxType !== 'number' && idxType !== 'array:number';
           }
           
           if (typeof condition === 'object' && !Array.isArray(condition)) {
@@ -1280,12 +1330,17 @@ export class QueryManager {
       // Check if the field uses operators that are supported by IndexManager
       const condition = criteria[field];
       
-      // RegExp cannot be efficiently queried using indices - must use streaming
+      // INTENTIONAL (CHANGELOG [2.2.5]): raw RegExp on string/term indexes is resolved
+      // via index keys (not streaming). Do not revert to "regex => streaming".
       if (condition instanceof RegExp) {
-        if (this.opts.debugMode) {
-          console.log(`🔍 Field '${field}' uses RegExp - requires streaming strategy`)
+        const idxType = this.indexManager?.opts?.indexes?.[field];
+        if (idxType === 'number' || idxType === 'array:number') {
+          if (this.opts.debugMode) {
+            console.log(`🔍 Field '${field}' uses RegExp on a numeric index - requires streaming strategy`)
+          }
+          return false;
         }
-        return false;
+        return true;
       }
       
       if (typeof condition === 'object' && !Array.isArray(condition) && condition !== null) {

package/src/utils/ioTimeout.mjs

@@ -0,0 +1,36 @@
+/**
+ * Shared helpers to normalize and detect I/O timeout / abort errors.
+ *
+ * Node surfaces read timeouts in several inconsistent shapes depending on the
+ * path used (`error.name`, `error.code` or `error.message`). These helpers
+ * centralize creation and detection so every retry branch across Database.mjs
+ * and FileHandler.mjs agrees, and timeouts are always delivered as normal
+ * promise rejections - never as an `uncaughtException`.
+ */
+
+/**
+ * Create a normalized I/O timeout error that every retry branch recognizes.
+ * @param {number} timeoutMs - Timeout that elapsed, in milliseconds.
+ * @returns {Error}
+ */
+export function createIoTimeoutError(timeoutMs) {
+  const err = new Error(`I/O timeout after ${timeoutMs}ms`)
+  err.name = 'AbortError'
+  err.code = 'ETIMEDOUT'
+  return err
+}
+
+/**
+ * Detect whether an error represents an I/O timeout or abort, regardless of the
+ * shape it was produced in.
+ * @param {*} error - Error to inspect (may be null/undefined).
+ * @returns {boolean}
+ */
+export function isIoTimeoutError(error) {
+  if (!error) return false
+  return (
+    error.name === 'AbortError' ||
+    error.code === 'ETIMEDOUT' ||
+    error.message === 'AbortError'
+  )
+}