@yoch/frozenminisearch 1.2.0 → 1.2.2

package/dist/cjs/index.cjs

@@ -4,10 +4,67 @@ Object.defineProperty(exports, '__esModule', { value: true });
 
 var zlib = require('node:zlib');
 
+/**
+ * Internal AND / AND_NOT gate thresholds (not exported from the public package entry).
+ */
+const DEFAULT_POSTING_GATE_MIN_LENGTH = 2048;
+const DEFAULT_POSTING_GATE_RATIO_SHIFT = 2;
+const DEFAULT_POSTING_GATE_POLICY = {
+    minLength: DEFAULT_POSTING_GATE_MIN_LENGTH,
+    ratioShift: DEFAULT_POSTING_GATE_RATIO_SHIFT,
+};
+function passGateByPostingRatio(gateSize, postingListLength, policy = DEFAULT_POSTING_GATE_POLICY) {
+    if (postingListLength < policy.minLength)
+        return false;
+    return gateSize <= (postingListLength >>> policy.ratioShift);
+}
+const DEFAULT_AND_GATE_LIMITS = {
+    maxAbsolute: 5000,
+    maxFraction: 0.1,
+};
+function resolveGateMaxSize(documentCount, limits = DEFAULT_AND_GATE_LIMITS) {
+    return Math.min(limits.maxAbsolute, Math.max(100, Math.floor(documentCount * limits.maxFraction)));
+}
+function gateIsSelectiveEnough(gateSize, documentCount, limits = DEFAULT_AND_GATE_LIMITS, postingListLength, postingGatePolicy = DEFAULT_POSTING_GATE_POLICY) {
+    if (gateSize === 0)
+        return true;
+    if (gateSize <= resolveGateMaxSize(documentCount, limits))
+        return true;
+    if (postingListLength != null
+        && postingListLength > 0
+        && passGateByPostingRatio(gateSize, postingListLength, postingGatePolicy)) {
+        return true;
+    }
+    return false;
+}
+
 const MAX_FREQ = 65535;
 function readDocId(docIds, index) {
     return docIds[index];
 }
+/** Binary search for docId in a sorted segment; returns global index or -1. */
+function findDocIndexInSortedSegment(docIds, offset, length, docId) {
+    let lo = 0;
+    let hi = length - 1;
+    while (lo <= hi) {
+        const mid = (lo + hi) >>> 1;
+        const v = readDocId(docIds, offset + mid);
+        if (v < docId)
+            lo = mid + 1;
+        else if (v > docId)
+            hi = mid - 1;
+        else
+            return offset + mid;
+    }
+    return -1;
+}
+/**
+ * Scan vs binary search once `allowedDocs` is already in effect (scoring layer).
+ * Uses the same numeric policy as {@link passGateByPostingRatio} today; distinct decision point.
+ */
+function shouldSeekAllowedDocs(gateSize, listLength) {
+    return passGateByPostingRatio(gateSize, listLength);
+}
 function allocateFreqs(length, maxValue) {
     if (maxValue <= 0xff)
         return new Uint8Array(length);
@@ -69,10 +126,15 @@ function bm25FieldConstants(bm25params, avgFieldLength) {
     const { k, b, d } = bm25params;
     return { k, d, k1: k + 1, oneMinusB: 1 - b, bOverAvg: b / avgFieldLength };
 }
-function calcBM25ScoreWithConstants(termFreq, matchingCount, totalCount, fieldLength, constants) {
+function bm25Idf(matchingCount, totalCount) {
+    return Math.log(1 + (totalCount - matchingCount + 0.5) / (matchingCount + 0.5));
+}
+function calcBm25TfWithConstants(termFreq, fieldLength, constants, idf) {
     const { k, d, k1, oneMinusB, bOverAvg } = constants;
-    const invDocFreq = Math.log(1 + (totalCount - matchingCount + 0.5) / (matchingCount + 0.5));
-    return invDocFreq * (d + termFreq * k1 / (termFreq + k * (oneMinusB + bOverAvg * fieldLength)));
+    return idf * (d + termFreq * k1 / (termFreq + k * (oneMinusB + bOverAvg * fieldLength)));
+}
+function calcBM25ScoreWithConstants(termFreq, matchingCount, totalCount, fieldLength, constants) {
+    return calcBm25TfWithConstants(termFreq, fieldLength, constants, bm25Idf(matchingCount, totalCount));
 }
 const getOwnProperty = (object, property) => Object.prototype.hasOwnProperty.call(object, property) ? object[property] : undefined;
 function fieldBoostsForQuery(options, fields) {
@@ -101,7 +163,7 @@ function getDerivedTerm(derivedTerm, cache) {
         cache.value = derivedTerm.resolve();
     return cache.value;
 }
-function scorePostingDoc(sourceTerm, derivedTerm, field, fieldId, docId, termFreq, termWeight, termBoost, fieldBoost, matchingFields, context, boostDocumentFn, bm25, results, derivedTermCache) {
+function scorePostingDoc(sourceTerm, derivedTerm, field, fieldId, docId, termFreq, termWeight, termBoost, fieldBoost, matchingFields, context, boostDocumentFn, bm25, results, derivedTermCache, hoistedIdf) {
     const resolvedDerivedTerm = getDerivedTerm(derivedTerm, derivedTermCache);
     const docBoost = boostDocumentFn
         ? boostDocumentFn(context.getExternalId(docId), resolvedDerivedTerm, context.getStoredFields(docId))
@@ -109,7 +171,9 @@ function scorePostingDoc(sourceTerm, derivedTerm, field, fieldId, docId, termFre
     if (!docBoost)
         return;
     const fieldLength = context.getFieldLength(docId, fieldId);
-    const rawScore = calcBM25ScoreWithConstants(termFreq, matchingFields, context.documentCount, fieldLength, bm25);
+    const rawScore = hoistedIdf !== undefined
+        ? calcBm25TfWithConstants(termFreq, fieldLength, bm25, hoistedIdf)
+        : calcBM25ScoreWithConstants(termFreq, matchingFields, context.documentCount, fieldLength, bm25);
     const weightedScore = termWeight * termBoost * fieldBoost * docBoost * rawScore;
     const result = results.get(docId);
     if (result) {
@@ -132,22 +196,39 @@ function scorePostingDoc(sourceTerm, derivedTerm, field, fieldId, docId, termFre
     }
 }
 function aggregateSegmentPostingList(sourceTerm, derivedTerm, termWeight, termBoost, field, fieldId, fieldBoost, list, context, boostDocumentFn, bm25params, results, allowedDocs) {
-    var _a;
+    var _a, _b;
     let matchingFields = list.length;
     const bm25 = bm25FieldConstants(bm25params, context.avgFieldLength[fieldId]);
+    const hoistedIdf = context.isDocActive == null
+        ? bm25Idf(matchingFields, context.documentCount)
+        : undefined;
     const { docIds, freqs, offset, length } = list;
     const derivedTermCache = {};
+    if (allowedDocs != null && shouldSeekAllowedDocs(allowedDocs.size, length)) {
+        for (const docId of allowedDocs) {
+            if (context.isDocActive != null && !context.isDocActive(docId)) {
+                (_a = context.onInactiveDoc) === null || _a === void 0 ? void 0 : _a.call(context, docId, fieldId, getDerivedTerm(derivedTerm, derivedTermCache));
+                matchingFields -= 1;
+                continue;
+            }
+            const index = findDocIndexInSortedSegment(docIds, offset, length, docId);
+            if (index < 0)
+                continue;
+            scorePostingDoc(sourceTerm, derivedTerm, field, fieldId, docId, freqs[index], termWeight, termBoost, fieldBoost, matchingFields, context, boostDocumentFn, bm25, results, derivedTermCache, hoistedIdf);
+        }
+        return matchingFields;
+    }
     for (let i = 0; i < length; i++) {
         const docId = readDocId(docIds, offset + i);
         const termFreq = freqs[offset + i];
         if (context.isDocActive != null && !context.isDocActive(docId)) {
-            (_a = context.onInactiveDoc) === null || _a === void 0 ? void 0 : _a.call(context, docId, fieldId, getDerivedTerm(derivedTerm, derivedTermCache));
+            (_b = context.onInactiveDoc) === null || _b === void 0 ? void 0 : _b.call(context, docId, fieldId, getDerivedTerm(derivedTerm, derivedTermCache));
             matchingFields -= 1;
             continue;
         }
         if (allowedDocs != null && !allowedDocs.has(docId))
             continue;
-        scorePostingDoc(sourceTerm, derivedTerm, field, fieldId, docId, termFreq, termWeight, termBoost, fieldBoost, matchingFields, context, boostDocumentFn, bm25, results, derivedTermCache);
+        scorePostingDoc(sourceTerm, derivedTerm, field, fieldId, docId, termFreq, termWeight, termBoost, fieldBoost, matchingFields, context, boostDocumentFn, bm25, results, derivedTermCache, hoistedIdf);
     }
     return matchingFields;
 }
@@ -167,6 +248,9 @@ function aggregateTerm(sourceTerm, derivedTerm, termWeight, termBoost, fieldTerm
         }
         let matchingFields = postingList.size;
         const bm25 = bm25FieldConstants(bm25params, context.avgFieldLength[fieldId]);
+        const hoistedIdf = context.isDocActive == null
+            ? bm25Idf(matchingFields, context.documentCount)
+            : undefined;
         const derivedTermCache = {};
         postingList.forEachDoc((docId, termFreq) => {
             var _a;
@@ -177,7 +261,7 @@ function aggregateTerm(sourceTerm, derivedTerm, termWeight, termBoost, fieldTerm
             }
             if (allowedDocs != null && !allowedDocs.has(docId))
                 return;
-            scorePostingDoc(sourceTerm, derivedTerm, field, fieldId, docId, termFreq, termWeight, termBoost, fieldBoost, matchingFields, context, boostDocumentFn, bm25, results, derivedTermCache);
+            scorePostingDoc(sourceTerm, derivedTerm, field, fieldId, docId, termFreq, termWeight, termBoost, fieldBoost, matchingFields, context, boostDocumentFn, bm25, results, derivedTermCache, hoistedIdf);
         });
     }
     return results;
@@ -260,7 +344,9 @@ function finalizeSearchResults(params) {
             queryTerms: terms,
             match,
         };
-        Object.assign(result, getStoredFields(docId));
+        if (getStoredFields != null) {
+            Object.assign(result, getStoredFields(docId));
+        }
         if (filter == null || filter(result)) {
             results.push(result);
         }
@@ -1744,82 +1830,47 @@ function readFieldLengthMatrixSection(buf, flags, cellCount) {
 
 const DISCARDED_DOC_ID = 0xffffffff;
 function postingFreqValue(freq, clampFrequencies) {
-    return clampFrequencies ? clampFreq(freq) : freq;
-}
-function materializeFlatPostings(params) {
-    const { fieldCount, termCount, forEachPosting, remapDocId, clampFrequencies } = params;
-    const slotCount = termCount * fieldCount;
-    const postingsOffsets = new Uint32Array(slotCount);
-    const postingsLengths = new Uint32Array(slotCount);
-    let totalPostings = 0;
-    let maxFreq = 0;
-    for (let ti = 0; ti < termCount; ti++) {
-        for (let f = 0; f < fieldCount; f++) {
-            forEachPosting(ti, f, (rawDocId, freq) => {
-                const docId = remapDocId != null ? remapDocId(rawDocId) : rawDocId;
-                if (docId === DISCARDED_DOC_ID)
-                    return;
-                totalPostings++;
-                const v = postingFreqValue(freq, clampFrequencies);
-                if (v > maxFreq)
-                    maxFreq = v;
-            });
-        }
-    }
-    const useUint16 = params.nextId != null && params.nextId <= 65535;
-    const allDocIds = useUint16
-        ? new Uint16Array(totalPostings)
-        : new Uint32Array(totalPostings);
-    const allFreqs = allocateFreqs(totalPostings, maxFreq);
-    // Slots are visited in ascending fieldId (0..fieldCount-1) per term. Sparse layouts
-    // rely on this ordering so field ids per term stay sorted for binary lookup.
-    let write = 0;
-    for (let ti = 0; ti < termCount; ti++) {
-        const base = ti * fieldCount;
-        for (let f = 0; f < fieldCount; f++) {
-            const offset = write;
-            let count = 0;
-            forEachPosting(ti, f, (rawDocId, freq) => {
-                const docId = remapDocId != null ? remapDocId(rawDocId) : rawDocId;
-                if (docId === DISCARDED_DOC_ID)
-                    return;
-                if (useUint16) {
-                    allDocIds[write] = docId;
-                }
-                else {
-                    allDocIds[write] = docId;
-                }
-                allFreqs[write] = postingFreqValue(freq, clampFrequencies);
-                write++;
-                count++;
-            });
-            postingsOffsets[base + f] = offset;
-            postingsLengths[base + f] = count;
-        }
-    }
-    return {
-        postingsOffsets,
-        postingsLengths,
-        allDocIds,
-        allFreqs,
-    };
+    return clampFreq(freq) ;
 }
 
 function readFieldId(fieldIds, index) {
     return fieldIds[index];
 }
-function choosePostingsLayout(fieldCount) {
-    return fieldCount === 1 ? 'dense' : 'sparse';
-}
 function chooseSparseFieldIdWidth(fieldCount) {
     return fieldCount > 255 ? 16 : 8;
 }
-function materializeFrozenPostings(params) {
-    const { fieldCount, termCount, nextId } = params;
-    const layout = choosePostingsLayout(fieldCount);
+function choosePostingsLayout(fieldCount, termCount, nonEmptySlots) {
+    const denseBytes = termCount * fieldCount * 8;
+    const sparseFieldIdBytes = chooseSparseFieldIdWidth(fieldCount) === 16 ? 2 : 1;
+    const sparseBytes = (termCount + 1) * 4 + nonEmptySlots * (sparseFieldIdBytes + 8);
+    return denseBytes <= sparseBytes ? 'dense' : 'sparse';
+}
+/** Shared dense/sparse layout emission; callers supply per-slot length and copy. */
+function buildFrozenPostingsLayout(fieldCount, termCount, nextId, totalPostings, maxFreq, source) {
+    const layout = choosePostingsLayout(fieldCount, termCount, source.nonEmptySlots);
     const docIdWidth = nextId <= 65535 ? 16 : 32;
+    const allDocIds = docIdWidth === 16
+        ? new Uint16Array(totalPostings)
+        : new Uint32Array(totalPostings);
+    const allFreqs = allocateFreqs(totalPostings, maxFreq);
+    const targets = { allDocIds, allFreqs, docIdWidth };
     if (layout === 'dense') {
-        const flat = materializeFlatPostings({ ...params, nextId });
+        const slotCount = termCount * fieldCount;
+        const denseOffsets = new Uint32Array(slotCount);
+        const denseLengths = new Uint32Array(slotCount);
+        let write = 0;
+        for (let ti = 0; ti < termCount; ti++) {
+            const base = ti * fieldCount;
+            for (let f = 0; f < fieldCount; f++) {
+                const slot = base + f;
+                const len = source.slotLength(ti, f);
+                denseOffsets[slot] = write;
+                denseLengths[slot] = len;
+                if (len > 0) {
+                    write = source.writeSlot(ti, f, write, targets);
+                }
+            }
+        }
         return {
             fieldCount,
             termCount,
@@ -1827,10 +1878,10 @@ function materializeFrozenPostings(params) {
             layout,
             docIdWidth,
             sparseFieldIdWidth: null,
-            allDocIds: flat.allDocIds,
-            allFreqs: flat.allFreqs,
-            denseOffsets: flat.postingsOffsets,
-            denseLengths: flat.postingsLengths,
+            allDocIds,
+            allFreqs,
+            denseOffsets,
+            denseLengths,
             sparseTermStarts: null,
             sparseFieldIds: null,
             sparseOffsets: null,
@@ -1842,60 +1893,23 @@ function materializeFrozenPostings(params) {
     const sparseOffsets = [];
     const sparseLengths = [];
     const termStarts = new Array(termCount + 1).fill(0);
-    const { forEachPosting, remapDocId, clampFrequencies } = params;
-    // Non-empty slots per term are emitted with fieldId in ascending order (f loops 0..fieldCount-1).
-    let totalPostings = 0;
-    let maxFreq = 0;
+    let write = 0;
     for (let ti = 0; ti < termCount; ti++) {
         termStarts[ti] = sparseFieldIdsScratch.length;
         for (let f = 0; f < fieldCount; f++) {
-            let count = 0;
-            forEachPosting(ti, f, (rawDocId, freq) => {
-                const docId = remapDocId != null ? remapDocId(rawDocId) : rawDocId;
-                if (docId === DISCARDED_DOC_ID)
-                    return;
-                count++;
-                const v = postingFreqValue(freq, clampFrequencies);
-                if (v > maxFreq)
-                    maxFreq = v;
-            });
-            if (count === 0)
+            const len = source.slotLength(ti, f);
+            if (len === 0)
                 continue;
             sparseFieldIdsScratch.push(f);
-            sparseOffsets.push(totalPostings);
-            sparseLengths.push(count);
-            totalPostings += count;
+            sparseOffsets.push(write);
+            sparseLengths.push(len);
+            write = source.writeSlot(ti, f, write, targets);
         }
         termStarts[ti + 1] = sparseFieldIdsScratch.length;
     }
-    const allDocIds = docIdWidth === 16
-        ? new Uint16Array(totalPostings)
-        : new Uint32Array(totalPostings);
-    const allFreqs = allocateFreqs(totalPostings, maxFreq);
     const sparseFieldIds = sparseFieldIdWidth === 16
         ? new Uint16Array(sparseFieldIdsScratch)
         : new Uint8Array(sparseFieldIdsScratch);
-    let write = 0;
-    for (let ti = 0; ti < termCount; ti++) {
-        const start = termStarts[ti];
-        const end = termStarts[ti + 1];
-        for (let s = start; s < end; s++) {
-            const f = readFieldId(sparseFieldIds, s);
-            forEachPosting(ti, f, (rawDocId, freq) => {
-                const docId = remapDocId != null ? remapDocId(rawDocId) : rawDocId;
-                if (docId === DISCARDED_DOC_ID)
-                    return;
-                if (docIdWidth === 16) {
-                    allDocIds[write] = docId;
-                }
-                else {
-                    allDocIds[write] = docId;
-                }
-                allFreqs[write] = postingFreqValue(freq, clampFrequencies);
-                write++;
-            });
-        }
-    }
     return {
         fieldCount,
         termCount,
@@ -1913,6 +1927,58 @@ function materializeFrozenPostings(params) {
         sparseLengths: new Uint32Array(sparseLengths),
     };
 }
+function materializeFrozenPostings(params) {
+    const { fieldCount, termCount, nextId } = params;
+    const { forEachPosting, remapDocId} = params;
+    const slotCount = termCount * fieldCount;
+    const slotLengths = new Uint32Array(slotCount);
+    let totalPostings = 0;
+    let maxFreq = 0;
+    let nonEmptySlots = 0;
+    for (let ti = 0; ti < termCount; ti++) {
+        const base = ti * fieldCount;
+        for (let f = 0; f < fieldCount; f++) {
+            let count = 0;
+            forEachPosting(ti, f, (rawDocId, freq) => {
+                const docId = remapDocId != null ? remapDocId(rawDocId) : rawDocId;
+                if (docId === DISCARDED_DOC_ID)
+                    return;
+                count++;
+                const v = postingFreqValue(freq);
+                if (v > maxFreq)
+                    maxFreq = v;
+            });
+            if (count === 0)
+                continue;
+            slotLengths[base + f] = count;
+            totalPostings += count;
+            nonEmptySlots++;
+        }
+    }
+    return buildFrozenPostingsLayout(fieldCount, termCount, nextId, totalPostings, maxFreq, {
+        nonEmptySlots,
+        slotLength(ti, f) {
+            return slotLengths[ti * fieldCount + f];
+        },
+        writeSlot(ti, f, write, targets) {
+            const { allDocIds: outDocIds, allFreqs: outFreqs, docIdWidth: width } = targets;
+            forEachPosting(ti, f, (rawDocId, freq) => {
+                const docId = remapDocId != null ? remapDocId(rawDocId) : rawDocId;
+                if (docId === DISCARDED_DOC_ID)
+                    return;
+                if (width === 16) {
+                    outDocIds[write] = docId;
+                }
+                else {
+                    outDocIds[write] = docId;
+                }
+                outFreqs[write] = postingFreqValue(freq);
+                write++;
+            });
+            return write;
+        },
+    });
+}
 function postingsTypedBytes(layout) {
     const allDocIdsBytes = layout.allDocIds.byteLength;
     const allFreqsBytes = layout.allFreqs.byteLength;
@@ -2675,7 +2741,6 @@ function buildFlatPostingsFromSearchableMap(searchableMap, fieldCount, nextId, s
         fieldCount,
         termCount,
         nextId,
-        clampFrequencies: true,
         remapDocId,
         forEachPosting(ti, f, emit) {
             var _a;
@@ -2919,8 +2984,8 @@ async function zlibPayloadChoiceAsync(uncompressed) {
     return { payload: compressed, codec: CODEC_ZLIB, zstdLevel: 0 };
 }
 const autoSyncCompressors = {
-    zstd: (uncompressed) => zlib.zstdCompressSync(uncompressed, msv5ZstdCompressOptions(uncompressed)),
-    zlib: (uncompressed) => zlib.deflateSync(uncompressed),
+    zstd: uncompressed => zlib.zstdCompressSync(uncompressed, msv5ZstdCompressOptions(uncompressed)),
+    zlib: uncompressed => zlib.deflateSync(uncompressed),
 };
 const autoAsyncCompressors = {
     zstd: zstdCompressAsync,
@@ -3863,93 +3928,23 @@ class IncrementalPostingsAccumulator {
         const totalPostings = this._totalPostings;
         const maxFreq = this._maxFreq;
         const slots = this._slots;
-        const layout = choosePostingsLayout(fieldCount);
-        const docIdWidth = nextId <= 65535 ? 16 : 32;
-        const allDocIds = docIdWidth === 16
-            ? new Uint16Array(totalPostings)
-            : new Uint32Array(totalPostings);
-        const allFreqs = allocateFreqs(totalPostings, maxFreq);
-        if (layout === 'dense') {
-            const slotCount = termCount * fieldCount;
-            const denseOffsets = new Uint32Array(slotCount);
-            const denseLengths = new Uint32Array(slotCount);
-            let write = 0;
-            for (let ti = 0; ti < termCount; ti++) {
-                const base = ti * fieldCount;
-                for (let f = 0; f < fieldCount; f++) {
-                    const slot = base + f;
-                    const ranges = slots.get(slot);
-                    const len = ranges == null ? 0 : this.slotLength(ranges);
-                    denseOffsets[slot] = write;
-                    denseLengths[slot] = len;
-                    if (len > 0) {
-                        write = this.copySlot(ranges, allDocIds, allFreqs, write, docIdWidth);
-                        slots.delete(slot);
-                    }
-                }
-            }
-            slots.clear();
-            this.clear();
-            return {
-                fieldCount,
-                termCount,
-                nextId,
-                layout,
-                docIdWidth,
-                sparseFieldIdWidth: null,
-                allDocIds,
-                allFreqs,
-                denseOffsets,
-                denseLengths,
-                sparseTermStarts: null,
-                sparseFieldIds: null,
-                sparseOffsets: null,
-                sparseLengths: null,
-            };
-        }
-        const sparseFieldIdWidth = chooseSparseFieldIdWidth(fieldCount);
-        const sparseFieldIdsScratch = [];
-        const sparseOffsets = [];
-        const sparseLengths = [];
-        const termStarts = new Array(termCount + 1).fill(0);
-        let write = 0;
-        for (let ti = 0; ti < termCount; ti++) {
-            termStarts[ti] = sparseFieldIdsScratch.length;
-            for (let f = 0; f < fieldCount; f++) {
+        const layout = buildFrozenPostingsLayout(fieldCount, termCount, nextId, totalPostings, maxFreq, {
+            nonEmptySlots: slots.size,
+            slotLength: (ti, f) => {
+                const ranges = slots.get(ti * fieldCount + f);
+                return ranges == null ? 0 : this.slotLength(ranges);
+            },
+            writeSlot: (ti, f, write, targets) => {
                 const slot = ti * fieldCount + f;
                 const ranges = slots.get(slot);
-                const len = ranges == null ? 0 : this.slotLength(ranges);
-                if (len === 0)
-                    continue;
-                sparseFieldIdsScratch.push(f);
-                sparseOffsets.push(write);
-                sparseLengths.push(len);
-                write = this.copySlot(ranges, allDocIds, allFreqs, write, docIdWidth);
+                const next = this.copySlot(ranges, targets.allDocIds, targets.allFreqs, write, targets.docIdWidth);
                 slots.delete(slot);
-            }
-            termStarts[ti + 1] = sparseFieldIdsScratch.length;
-        }
+                return next;
+            },
+        });
         slots.clear();
         this.clear();
-        const sparseFieldIds = sparseFieldIdWidth === 16
-            ? new Uint16Array(sparseFieldIdsScratch)
-            : new Uint8Array(sparseFieldIdsScratch);
-        return {
-            fieldCount,
-            termCount,
-            nextId,
-            layout,
-            docIdWidth,
-            sparseFieldIdWidth,
-            allDocIds,
-            allFreqs,
-            denseOffsets: null,
-            denseLengths: null,
-            sparseTermStarts: new Uint32Array(termStarts),
-            sparseFieldIds,
-            sparseOffsets: new Uint32Array(sparseOffsets),
-            sparseLengths: new Uint32Array(sparseLengths),
-        };
+        return layout;
     }
 }
 
@@ -4129,22 +4124,6 @@ function buildFrozenParamsFromDocuments(documents, options) {
     return builder.freezeParams();
 }
 
-/**
- * Internal AND / AND_NOT gate thresholds (not exported from the public package entry).
- */
-const DEFAULT_AND_GATE_LIMITS = {
-    maxAbsolute: 5000,
-    maxFraction: 0.1,
-};
-function resolveGateMaxSize(documentCount, limits = DEFAULT_AND_GATE_LIMITS) {
-    return Math.min(limits.maxAbsolute, Math.max(100, Math.floor(documentCount * limits.maxFraction)));
-}
-function gateIsSelectiveEnough(gateSize, documentCount, limits = DEFAULT_AND_GATE_LIMITS) {
-    if (gateSize === 0)
-        return true;
-    return gateSize <= resolveGateMaxSize(documentCount, limits);
-}
-
 function useGatedEvaluation(run, branchCount, operator, hasWildcard) {
     return shouldUseGatedEvaluation(branchCount, operator, hasWildcard);
 }
@@ -4186,80 +4165,155 @@ function normalizeStringQuery(query, searchOptions, params) {
         ...params.globalSearchOptions,
         ...searchOptions,
     };
-    const terms = options.tokenize(query)
-        .flatMap((term) => options.processTerm(term))
-        .filter(term => !!term);
+    const tokens = options.tokenize(query);
+    const terms = [];
+    for (const token of tokens) {
+        const processed = options.processTerm(token);
+        if (Array.isArray(processed)) {
+            for (const term of processed) {
+                if (term)
+                    terms.push(term);
+            }
+        }
+        else if (processed) {
+            terms.push(processed);
+        }
+    }
+    const toSpec = termToQuerySpec(options);
+    const specs = new Array(terms.length);
+    for (let i = 0; i < terms.length; i++) {
+        specs[i] = toSpec(terms[i], i, terms);
+    }
+    const { fuzzy: fuzzyWeight, prefix: prefixWeight } = {
+        ...defaultSearchOptions.weights,
+        ...options.weights,
+    };
     return {
         options,
-        specs: terms.map(termToQuerySpec(options)),
+        specs,
         operator: options.combineWith,
+        fieldBoosts: fieldBoostsForQuery(options, params.fields),
+        fuzzyWeight,
+        prefixWeight,
     };
 }
 function lazyIndexedTerm(indexView, termIndex) {
     return { kind: 'lazy', resolve: () => indexView.resolveTermByIndex(termIndex) };
 }
-function visitQuerySpecForScoring(query, options, params, visit) {
+function forEachQuerySpecTermRef(query, normalized, params, visit) {
     const { indexView } = params;
-    const { weights, maxFuzzy } = options;
-    const { fuzzy: fuzzyWeight, prefix: prefixWeight } = { ...defaultSearchOptions.weights, ...weights };
-    const maxDistance = maxFuzzyDistance(query, maxFuzzy);
-    const exactTi = indexView.resolveTermIndex(query.term);
-    visit(exactTi == null ? undefined : indexView.fieldTermData(exactTi), query.term, 1);
-    const seenPrefix = new Set();
+    const { options } = normalized;
+    const maxDistance = maxFuzzyDistance(query, options.maxFuzzy);
+    visit({ kind: 'exact', termIndex: indexView.resolveTermIndex(query.term) });
+    const seenPrefix = query.prefix && maxDistance ? new Set() : undefined;
     if (query.prefix) {
         for (const { termIndex, length } of indexView.getPrefixMatchesByIndex(query.term)) {
             const distance = length - query.term.length;
             if (!distance)
                 continue;
-            seenPrefix.add(termIndex);
-            visit(indexView.fieldTermData(termIndex), lazyIndexedTerm(indexView, termIndex), prefixWeight * length / (length + 0.3 * distance));
+            seenPrefix === null || seenPrefix === void 0 ? void 0 : seenPrefix.add(termIndex);
+            visit({ kind: 'prefix', termIndex, length, distance });
         }
     }
     if (!maxDistance)
         return;
     for (const { termIndex, length, distance } of indexView.getFuzzyMatchesByIndex(query.term, maxDistance)) {
-        if (!distance || seenPrefix.has(termIndex))
+        if (!distance || (seenPrefix === null || seenPrefix === void 0 ? void 0 : seenPrefix.has(termIndex)))
             continue;
-        visit(indexView.fieldTermData(termIndex), lazyIndexedTerm(indexView, termIndex), fuzzyWeight * length / (length + distance));
+        visit({ kind: 'fuzzy', termIndex, length, distance });
     }
 }
-function executeQuerySpecInternal(query, searchOptions, params, allowedDocs) {
-    const options = { ...params.globalSearchOptions, ...searchOptions };
-    const fieldBoosts = fieldBoostsForQuery(options, params.fields);
+function visitQuerySpecForScoring(query, normalized, params, visit) {
+    const { indexView } = params;
+    const { fuzzyWeight, prefixWeight } = normalized;
+    forEachQuerySpecTermRef(query, normalized, params, (ref) => {
+        if (ref.kind === 'exact') {
+            visit(ref.termIndex == null ? undefined : indexView.fieldTermData(ref.termIndex), query.term, 1);
+            return;
+        }
+        if (ref.kind === 'prefix') {
+            visit(indexView.fieldTermData(ref.termIndex), lazyIndexedTerm(indexView, ref.termIndex), prefixWeight * ref.length / (ref.length + 0.3 * ref.distance));
+            return;
+        }
+        visit(indexView.fieldTermData(ref.termIndex), lazyIndexedTerm(indexView, ref.termIndex), fuzzyWeight * ref.length / (ref.length + ref.distance));
+    });
+}
+function executeQuerySpecInternal(query, normalized, params, allowedDocs) {
+    const { fieldBoosts, options } = normalized;
     const termOptions = allowedDocs == null ? undefined : { allowedDocs };
     const results = new Map();
-    visitQuerySpecForScoring(query, options, params, (data, derivedTerm, termWeight) => {
+    visitQuerySpecForScoring(query, normalized, params, (data, derivedTerm, termWeight) => {
         aggregateTerm(query.term, derivedTerm, termWeight, query.termBoost, data, fieldBoosts, params.aggregateContext, options.boostDocument, options.bm25, results, termOptions);
     });
     return results;
 }
-function collectDocIdsForQuerySpec(query, searchOptions, params, allowedDocs) {
-    const options = { ...params.globalSearchOptions, ...searchOptions };
-    const fieldBoosts = fieldBoostsForQuery(options, params.fields);
-    const docIds = new Set();
-    const { indexView, aggregateContext } = params;
-    const maxDistance = maxFuzzyDistance(query, options.maxFuzzy);
-    const exactTi = indexView.resolveTermIndex(query.term);
-    if (exactTi != null) {
-        indexView.collectDocIds(exactTi, fieldBoosts, aggregateContext, docIds, allowedDocs);
+function maxPostingLengthForFieldTermData(data, fieldBoosts, fieldIds) {
+    if (data == null)
+        return 0;
+    let maxLen = 0;
+    for (const field of fieldBoosts.names) {
+        const fieldId = fieldIds[field];
+        const postingList = data.get(fieldId);
+        if (postingList == null)
+            continue;
+        const len = postingList instanceof SegmentPostingList ? postingList.length : postingList.size;
+        if (len > maxLen)
+            maxLen = len;
     }
-    const seenPrefix = new Set();
-    if (query.prefix) {
-        for (const { termIndex, length } of indexView.getPrefixMatchesByIndex(query.term)) {
-            const distance = length - query.term.length;
-            if (!distance)
-                continue;
-            seenPrefix.add(termIndex);
-            indexView.collectDocIds(termIndex, fieldBoosts, aggregateContext, docIds, allowedDocs);
+    return maxLen;
+}
+function estimateMaxPostingLengthForQuerySpec(query, normalized, params) {
+    const { indexView, aggregateContext } = params;
+    const { fieldBoosts } = normalized;
+    const { fieldIds } = aggregateContext;
+    let maxLen = 0;
+    const consider = (data) => {
+        maxLen = Math.max(maxLen, maxPostingLengthForFieldTermData(data, fieldBoosts, fieldIds));
+    };
+    forEachQuerySpecTermRef(query, normalized, params, (ref) => {
+        if (ref.kind === 'exact') {
+            if (ref.termIndex != null)
+                consider(indexView.fieldTermData(ref.termIndex));
+            return;
         }
+        consider(indexView.fieldTermData(ref.termIndex));
+    });
+    return maxLen;
+}
+function estimateMaxPostingLengthForQuery(query, searchOptions, params) {
+    if (isWildcardQuery(query)) {
+        return params.aggregateContext.documentCount;
     }
-    if (maxDistance) {
-        for (const { termIndex, distance } of indexView.getFuzzyMatchesByIndex(query.term, maxDistance)) {
-            if (!distance || seenPrefix.has(termIndex))
-                continue;
-            indexView.collectDocIds(termIndex, fieldBoosts, aggregateContext, docIds, allowedDocs);
+    if (isQueryCombination(query)) {
+        const options = { ...searchOptions, ...query, queries: undefined };
+        let maxLen = 0;
+        for (const branch of query.queries) {
+            maxLen = Math.max(maxLen, estimateMaxPostingLengthForQuery(branch, options, params));
         }
+        return maxLen;
     }
+    if (typeof query !== 'string')
+        return 0;
+    const normalized = normalizeStringQuery(query, searchOptions, params);
+    let maxLen = 0;
+    for (const spec of normalized.specs) {
+        maxLen = Math.max(maxLen, estimateMaxPostingLengthForQuerySpec(spec, normalized, params));
+    }
+    return maxLen;
+}
+function collectDocIdsForQuerySpec(query, normalized, params, allowedDocs) {
+    const { fieldBoosts } = normalized;
+    const docIds = new Set();
+    const { indexView, aggregateContext } = params;
+    forEachQuerySpecTermRef(query, normalized, params, (ref) => {
+        if (ref.kind === 'exact') {
+            if (ref.termIndex != null) {
+                indexView.collectDocIds(ref.termIndex, fieldBoosts, aggregateContext, docIds, allowedDocs);
+            }
+            return;
+        }
+        indexView.collectDocIds(ref.termIndex, fieldBoosts, aggregateContext, docIds, allowedDocs);
+    });
     return docIds;
 }
 function intersectDocIdsInPlace(docIds, branchDocIds) {
@@ -4309,7 +4363,8 @@ function collectCombinedDocIds(branches, operator, collectBranch, allowedDocs) {
  * AND_NOT: score the positive branch only; negated branches are collected as docId sets and
  * subtracted without scoring (avoids term materialization on excluded branches).
  */
-function executeCombinedBranches(branches, operator, params, executeBranch, collectBranch, allowedDocs, run) {
+function executeCombinedBranches(branches, operator, params, executeBranch, collectBranch, allowedDocs, run, estimateBranchPostingLength) {
+    var _a;
     if (branches.length === 0)
         return new Map();
     const op = operator.toLowerCase();
@@ -4321,8 +4376,16 @@ function executeCombinedBranches(branches, operator, params, executeBranch, coll
     if (op === 'and') {
         const limits = void 0 ;
         const documentCount = params.aggregateContext.documentCount;
+        const postingGatePolicy = (_a = void 0 ) !== null && _a !== void 0 ? _a : DEFAULT_POSTING_GATE_POLICY;
+        const maxGateSize = resolveGateMaxSize(documentCount, limits);
         for (let i = 1; i < branches.length; i++) {
-            const selective = gateIsSelectiveEnough(gate.size, documentCount, limits);
+            if (gate.size === 0)
+                return result;
+            const ratioPath = gate.size > maxGateSize;
+            const postingListLength = ratioPath
+                ? estimateBranchPostingLength === null || estimateBranchPostingLength === void 0 ? void 0 : estimateBranchPostingLength(branches[i])
+                : undefined;
+            const selective = gateIsSelectiveEnough(gate.size, documentCount, limits, postingListLength, postingGatePolicy);
             const branchAllowed = selective ? gate : allowedDocs;
             result = combineResults([result, executeBranch(branches[i], branchAllowed)], AND);
             gate = docIdsFromResult(result);
@@ -4386,14 +4449,15 @@ function collectDocIdsForQueryInternal(query, searchOptions, params, allowedDocs
     if (typeof query !== 'string') {
         throw new Error('FrozenMiniSearch: invalid query');
     }
-    const { options, specs, operator } = normalizeStringQuery(query, searchOptions, params);
+    const normalized = normalizeStringQuery(query, searchOptions, params);
+    const { specs, operator } = normalized;
     const combineWith = (operator !== null && operator !== void 0 ? operator : params.globalSearchOptions.combineWith);
     if (specs.length <= 1) {
         return specs.length === 1
-            ? collectDocIdsForQuerySpec(specs[0], options, params, allowedDocs)
+            ? collectDocIdsForQuerySpec(specs[0], normalized, params, allowedDocs)
             : new Set();
     }
-    return collectCombinedDocIds(specs, combineWith, (spec, branchAllowed) => collectDocIdsForQuerySpec(spec, options, params, branchAllowed), allowedDocs);
+    return collectCombinedDocIds(specs, combineWith, (spec, branchAllowed) => collectDocIdsForQuerySpec(spec, normalized, params, branchAllowed), allowedDocs);
 }
 function executeWildcardQuery(searchOptions, params) {
     const results = new Map();
@@ -4415,7 +4479,7 @@ function executeQueryInternal(query, searchOptions, params, allowedDocs, run) {
         const options = { ...searchOptions, ...query, queries: undefined };
         const operator = ((_b = (_a = query.combineWith) !== null && _a !== void 0 ? _a : options.combineWith) !== null && _b !== void 0 ? _b : params.globalSearchOptions.combineWith);
         if (useGatedEvaluation(run, query.queries.length, operator, combinationHasWildcard(query))) {
-            return executeCombinedBranches(query.queries, operator, params, (branch, branchAllowed) => executeQueryInternal(branch, options, params, branchAllowed, run), (branch, branchAllowed) => collectDocIdsForQueryInternal(branch, options, params, branchAllowed), allowedDocs);
+            return executeCombinedBranches(query.queries, operator, params, (branch, branchAllowed) => executeQueryInternal(branch, options, params, branchAllowed, run), (branch, branchAllowed) => collectDocIdsForQueryInternal(branch, options, params, branchAllowed), allowedDocs, run, branch => estimateMaxPostingLengthForQuery(branch, options, params));
         }
         const results = query.queries.map(subquery => executeQueryInternal(subquery, options, params, allowedDocs, run));
         return combineResults(results, operator);
@@ -4423,12 +4487,13 @@ function executeQueryInternal(query, searchOptions, params, allowedDocs, run) {
     if (typeof query !== 'string') {
         throw new Error('FrozenMiniSearch: invalid query');
     }
-    const { options, specs, operator } = normalizeStringQuery(query, searchOptions, params);
+    const normalized = normalizeStringQuery(query, searchOptions, params);
+    const { specs, operator } = normalized;
     const combineWith = (operator !== null && operator !== void 0 ? operator : params.globalSearchOptions.combineWith);
     if (useGatedEvaluation(run, specs.length, combineWith, false)) {
-        return executeCombinedBranches(specs, combineWith, params, (spec, branchAllowed) => executeQuerySpecInternal(spec, options, params, branchAllowed), (spec, branchAllowed) => collectDocIdsForQuerySpec(spec, options, params, branchAllowed), allowedDocs);
+        return executeCombinedBranches(specs, combineWith, params, (spec, branchAllowed) => executeQuerySpecInternal(spec, normalized, params, branchAllowed), (spec, branchAllowed) => collectDocIdsForQuerySpec(spec, normalized, params, branchAllowed), allowedDocs, run, spec => estimateMaxPostingLengthForQuerySpec(spec, normalized, params));
     }
-    const results = specs.map(spec => executeQuerySpecInternal(spec, options, params, allowedDocs));
+    const results = specs.map(spec => executeQuerySpecInternal(spec, normalized, params, allowedDocs));
     return combineResults(results, combineWith);
 }
 function executeQuery(query, searchOptions, params) {
@@ -4619,6 +4684,7 @@ function materializeOwnedSnapshot(params, mode) {
 function frozenMemoryBreakdown(frozen) {
     return frozen.memoryBreakdown();
 }
+const noStoredFields = () => undefined;
 function assertFieldsMatchSnapshot(optionsFields, snapFieldIds) {
     const snapNames = Object.keys(snapFieldIds).sort();
     const optNames = [...optionsFields].sort();
@@ -4672,24 +4738,31 @@ class FrozenMiniSearch {
         this._termCount = params.termCount;
         this._postings = params.postings;
         this._fieldTermFlyweight = createFrozenFieldTermFlyweight(this._postings);
+        this._hasStoredFields = this._storedFields.kind !== 'none';
         this._aggregateContext = {
             documentCount: this._documentCount,
             avgFieldLength: this._avgFieldLength,
             fieldIds: this._fieldIds,
             getFieldLength: (docId, fieldId) => this.getFieldLength(docId, fieldId),
             getExternalId: docId => this._externalIds[docId],
-            getStoredFields: docId => readStoredFields(this._storedFields, docId),
+            getStoredFields: this._hasStoredFields
+                ? docId => readStoredFields(this._storedFields, docId)
+                : noStoredFields,
         };
         this._queryEngineParams = {
             fields: this._options.fields,
             globalSearchOptions: this._options.searchOptions,
             tokenize: this._options.tokenize,
             processTerm: this._options.processTerm,
-            indexView: createFrozenQueryIndexView(this._index, this._postings, this._fieldTermFlyweight, (callback) => {
-                forEachLiveShortId(this._nextId, this._externalIds, (shortId, id) => {
-                    callback(shortId, id, readStoredFields(this._storedFields, shortId));
-                });
-            }),
+            indexView: createFrozenQueryIndexView(this._index, this._postings, this._fieldTermFlyweight, this._hasStoredFields
+                ? (callback) => {
+                    forEachLiveShortId(this._nextId, this._externalIds, (shortId, id) => {
+                        callback(shortId, id, readStoredFields(this._storedFields, shortId));
+                    });
+                }
+                : (callback) => {
+                    forEachLiveShortId(this._nextId, this._externalIds, callback);
+                }),
             aggregateContext: this._aggregateContext,
         };
     }
@@ -4746,7 +4819,9 @@ class FrozenMiniSearch {
         return shortId == null ? undefined : readStoredFields(this._storedFields, shortId);
     }
     search(query, searchOptions = {}) {
-        return finalizeRawSearchResults(this.executeQuery(query, searchOptions), query, searchOptions, this._options.searchOptions, docId => this._externalIds[docId], docId => readStoredFields(this._storedFields, docId));
+        return finalizeRawSearchResults(this.executeQuery(query, searchOptions), query, searchOptions, this._options.searchOptions, docId => this._externalIds[docId], this._hasStoredFields
+            ? docId => readStoredFields(this._storedFields, docId)
+            : undefined);
     }
     autoSuggest(queryString, options = {}) {
         const merged = { ...this._options.autoSuggestOptions, ...options };