cry-search 1.0.7 → 2.0.0

package/dist/index.js

@@ -1,4 +1,5 @@
 // src/types/index.ts
+var DEFAULT_FIRST_TOKENS_N = 8;
 var DEFAULT_SEARCH_OPTS = {
   matchWords: "start",
   matchNumbers: "startEnd",
@@ -268,11 +269,23 @@ function prepareStringForSearchNumeric(input, tokensMap) {
   for (let i = 0;i < stringTokens.length; i++) {
     numericTokens[i] = getOrCreateTokenId(stringTokens[i], tokensMap);
   }
-  numericTokens.sort((a, b) => a - b);
-  return new Uint32Array(numericTokens);
+  const unsorted = Uint32Array.from(numericTokens);
+  const sortedArr = numericTokens.slice().sort((a, b) => a - b);
+  const sorted = Uint32Array.from(sortedArr);
+  return { sorted, unsorted };
 }
 
-// src/common/string/createSearchArrayMetadata.ts
+// src/common/numeric/createSearchArrayMetadata.ts
+function isDeletedOrBlocked(obj) {
+  return obj._deleted != null || obj._blocked != null;
+}
+var MATCH_MODE_PREFIXES = {
+  start: "<",
+  end: ">",
+  startEnd: "+",
+  anywhere: "*",
+  whole: "!"
+};
 var PREFIX_TO_MATCH_MODE = {
   "<": "start",
   ">": "end",
@@ -282,12 +295,6 @@ var PREFIX_TO_MATCH_MODE = {
   "=": "whole",
   "?": "anywhere"
 };
-var DEFAULT_EXCLUDED_FIELDS = new Set(["_id", "_deleted", "_blocked"]);
-function isDeletedOrBlocked(obj) {
-  return obj._deleted != null || obj._blocked != null;
-}
-
-// src/common/numeric/createSearchArrayMetadata.ts
 var MATCH_MODE_BITS = {
   start: 0,
   end: 1,
@@ -318,7 +325,7 @@ function decodeMatchMode(encoded) {
       return "start";
   }
 }
-var DEFAULT_EXCLUDED_FIELDS2 = new Set(["_id", "_deleted", "_blocked"]);
+var DEFAULT_EXCLUDED_FIELDS = new Set(["_id", "_deleted", "_blocked"]);
 function getValueByPath(obj, path) {
   const parts = path.split(".");
   let current = obj;
@@ -375,26 +382,29 @@ function prepareObjectForSearchNumeric(obj, spec, tokensMap) {
   if (isDeletedOrBlocked(obj)) {
     return;
   }
-  if (spec?.extractSearchableStringFn) {
-    const searchableString = spec.extractSearchableStringFn(obj);
-    if (searchableString === undefined) {
-      return;
-    }
-    return prepareStringForSearchNumeric(searchableString, tokensMap);
-  }
-  return prepareObjectFieldsForSearchNumeric(obj, spec, tokensMap);
-}
-function prepareObjectFieldsForSearchNumeric(obj, spec, tokensMap) {
   if (spec?.enrichSearchObjectFn) {
     spec.enrichSearchObjectFn(obj, spec.searchContext);
   }
+  const sorted = spec?.extractSearchableStringFn ? buildSortedFromExtractFn(obj, spec.extractSearchableStringFn, tokensMap) : buildSortedFromFields(obj, spec, tokensMap);
+  if (sorted === undefined)
+    return;
+  const firstN = spec?.keyFields && spec.keyFields.length > 0 ? buildFirstNFromKeyFields(obj, spec.keyFields, spec.firstTokensN ?? DEFAULT_FIRST_TOKENS_N, tokensMap) : undefined;
+  return { sorted, firstN };
+}
+function buildSortedFromExtractFn(obj, extractFn, tokensMap) {
+  const str = extractFn(obj);
+  if (str === undefined)
+    return;
+  return prepareStringForSearchNumeric(str, tokensMap).sorted;
+}
+function buildSortedFromFields(obj, spec, tokensMap) {
   const allTokens = [];
   const fieldMatchModes = spec?.fieldMatchModes;
   const fieldsToSearch = spec?.searchInFields ?? Object.keys(obj);
   const fieldsToExclude = new Set(spec?.dontSearchInFields ?? []);
   for (const key of fieldsToSearch) {
     const keyStr = key;
-    if (!spec?.searchInFields && DEFAULT_EXCLUDED_FIELDS2.has(keyStr))
+    if (!spec?.searchInFields && DEFAULT_EXCLUDED_FIELDS.has(keyStr))
       continue;
     if (fieldsToExclude.has(key))
       continue;
@@ -425,14 +435,42 @@ function prepareObjectFieldsForSearchNumeric(obj, spec, tokensMap) {
   const uniqueTokens = [...new Set(allTokens)].sort((a, b) => a - b);
   return new Uint32Array(uniqueTokens);
 }
+function buildFirstNFromKeyFields(obj, keyFields, n, tokensMap) {
+  const strings = [];
+  for (const field of keyFields) {
+    const keyStr = field;
+    const value = isDotNotation(keyStr) ? getValueByPath(obj, keyStr) : obj[field];
+    if (value == null)
+      continue;
+    extractStringsFromValue(value, strings);
+  }
+  if (strings.length === 0)
+    return;
+  const text = strings.join(" ");
+  const datesNormalized = normalizeDates(text);
+  const preprocessed = preprocessString(datesNormalized);
+  const sanitised = sanitiseString(preprocessed);
+  const stringTokens = tokenize(sanitised, DEFAULT_TOKENIZE_CONFIG);
+  if (stringTokens.length === 0)
+    return;
+  const sliced = stringTokens.slice(0, n);
+  const out = new Uint32Array(sliced.length);
+  for (let i = 0;i < sliced.length; i++) {
+    out[i] = getOrCreateTokenId(sliced[i], tokensMap);
+  }
+  return out;
+}
 function createSearchArrayMetadataNumeric(data, spec) {
   const tokensMap = createTokensMap();
   try {
-    const metadata = new Map;
+    const metadata = emptyNumericSearchMetadata();
     for (const [id, item] of data) {
-      const tokens2 = prepareObjectForSearchNumeric(item, spec, tokensMap);
-      if (tokens2 !== undefined) {
-        metadata.set(id, tokens2);
+      const result = prepareObjectForSearchNumeric(item, spec, tokensMap);
+      if (result !== undefined) {
+        metadata.sorted.set(id, result.sorted);
+        if (result.firstN !== undefined) {
+          metadata.unsorted.set(id, result.firstN);
+        }
       }
     }
     return metadata;
@@ -440,6 +478,9 @@ function createSearchArrayMetadataNumeric(data, spec) {
     clearTokensMap(tokensMap);
   }
 }
+function emptyNumericSearchMetadata() {
+  return { sorted: new Map, unsorted: new Map };
+}
 // src/common/numeric/createSearchLinkedMetadata.ts
 function createSearchLinkedMetadataNumeric(primaryData, primaryMetadata, linkedData, linkedMetadata, foreignKeyGetter) {
   const primaryToLinked = new Map;
@@ -461,14 +502,19 @@ function createSearchLinkedMetadataNumeric(primaryData, primaryMetadata, linkedD
 }
 // src/common/numeric/updateSearchMetadata.ts
 function updateSearchMetadataNumeric(metadata, objectId, obj, spec, tokensMap) {
-  const tokens2 = obj != null ? prepareObjectForSearchNumeric(obj, spec, tokensMap) : undefined;
-  if (tokens2 === undefined) {
-    metadata.delete(objectId);
+  const result = obj != null ? prepareObjectForSearchNumeric(obj, spec, tokensMap) : undefined;
+  if (result === undefined) {
+    metadata.sorted.delete(objectId);
+    metadata.unsorted.delete(objectId);
     return false;
+  }
+  metadata.sorted.set(objectId, result.sorted);
+  if (result.firstN !== undefined) {
+    metadata.unsorted.set(objectId, result.firstN);
   } else {
-    metadata.set(objectId, tokens2);
-    return true;
+    metadata.unsorted.delete(objectId);
   }
+  return true;
 }
 function updateSearchMetadataBatchNumeric(metadata, batch, spec) {
   if (batch.length === 0) {
@@ -492,7 +538,9 @@ function updateSearchMetadataBatchNumeric(metadata, batch, spec) {
   }
 }
 function removeFromSearchMetadataNumeric(metadata, objectId) {
-  return metadata.delete(objectId);
+  const had = metadata.sorted.delete(objectId);
+  metadata.unsorted.delete(objectId);
+  return had;
 }
 function removeFromSearchMetadataBatchNumeric(metadata, objectIds) {
   if (objectIds.length === 0) {
@@ -500,9 +548,10 @@ function removeFromSearchMetadataBatchNumeric(metadata, objectIds) {
   }
   let removed = 0;
   for (const id of objectIds) {
-    if (metadata.delete(id)) {
+    const had = metadata.sorted.delete(id);
+    metadata.unsorted.delete(id);
+    if (had)
       removed++;
-    }
   }
   return removed;
 }
@@ -692,6 +741,106 @@ function allTokensMatchNumeric(searchTokenStrs, targetTokens, opts) {
   return allPreparedTokensMatch(preparedTokens, targetTokens, opts);
 }
 
+// src/common/numeric/scoring.ts
+function orderedTokenize(input) {
+  return tokenize(sanitiseString(preprocessString(normalizeDates(input))), DEFAULT_TOKENIZE_CONFIG);
+}
+function scoreOne(query, qTokens, itemTokens, exactKeys) {
+  if (exactKeys && exactKeys.length > 0) {
+    for (let i = 0;i < exactKeys.length; i++) {
+      if (exactKeys[i] === query)
+        return 1e5;
+    }
+  }
+  if (qTokens.length === 0)
+    return 0;
+  if (itemTokens.length === 0)
+    return 1;
+  let qi = 0;
+  const positions = [];
+  for (let i = 0;i < itemTokens.length; i++) {
+    if (qi < qTokens.length && itemTokens[i].startsWith(qTokens[qi])) {
+      positions.push(i);
+      qi++;
+    }
+  }
+  if (qi === 0)
+    return 1;
+  let score = 50;
+  if (qi === qTokens.length)
+    score += 100;
+  if (positions[0] === 0)
+    score += 50;
+  let perfect = qi === qTokens.length;
+  if (perfect) {
+    for (let i = 0;i < positions.length; i++) {
+      if (positions[i] !== i) {
+        perfect = false;
+        break;
+      }
+    }
+  }
+  if (perfect)
+    score += 80;
+  if (positions.length >= 2) {
+    const span = positions[positions.length - 1] - positions[0] + 1;
+    score += positions.length / span * 30;
+  }
+  let totalLen = 0;
+  for (const t of itemTokens)
+    totalLen += t.length + 1;
+  score -= totalLen * 0.1;
+  return score;
+}
+function scoreBag(qTokens, itemTokens, perTokenWeight = 30) {
+  if (qTokens.length === 0 || itemTokens.length === 0)
+    return 0;
+  const usedItemIdx = new Set;
+  let score = 0;
+  for (const qt of qTokens) {
+    for (let i = 0;i < itemTokens.length; i++) {
+      if (usedItemIdx.has(i))
+        continue;
+      if (itemTokens[i].startsWith(qt)) {
+        score += perTokenWeight;
+        usedItemIdx.add(i);
+        break;
+      }
+    }
+  }
+  return score;
+}
+function scoreOrderAndPresence(qTokens, itemTokens, weights) {
+  const orderedWeight = weights?.ordered ?? 30;
+  const presenceWeight = weights?.presence ?? 10;
+  if (qTokens.length === 0 || itemTokens.length === 0)
+    return 0;
+  const matchedQueryIndices = new Set;
+  let qi = 0;
+  let score = 0;
+  for (let i = 0;i < itemTokens.length && qi < qTokens.length; i++) {
+    if (itemTokens[i].startsWith(qTokens[qi])) {
+      score += orderedWeight;
+      matchedQueryIndices.add(qi);
+      qi++;
+    }
+  }
+  if (matchedQueryIndices.size < qTokens.length) {
+    for (let qj = 0;qj < qTokens.length; qj++) {
+      if (matchedQueryIndices.has(qj))
+        continue;
+      const qt = qTokens[qj];
+      for (let i = 0;i < itemTokens.length; i++) {
+        if (itemTokens[i].startsWith(qt)) {
+          score += presenceWeight;
+          break;
+        }
+      }
+    }
+  }
+  return score;
+}
+
 // src/common/numeric/searchInData.ts
 function searchInDataReturnIdsNumeric(query, metadata, opts) {
   return searchInDataReturnIdsWithLimitNumeric(query, metadata, Infinity, opts);
@@ -709,20 +858,48 @@ function searchInDataReturnObjectsNumeric(query, metadata, data, opts) {
 }
 function searchInDataReturnIdsWithLimitNumeric(query, metadata, limit, opts) {
   const queryTokenStrs = prepareStringForSearch(query);
-  if (queryTokenStrs.length === 0) {
+  if (queryTokenStrs.length === 0)
     return [];
-  }
   const preparedTokens = prepareQueryTokens(queryTokenStrs);
-  const results = [];
-  for (const [id, targetTokens] of metadata) {
+  const candidates = [];
+  for (const [id, targetTokens] of metadata.sorted) {
     if (allPreparedTokensMatch(preparedTokens, targetTokens, opts)) {
-      results.push(id);
-      if (results.length >= limit) {
-        break;
-      }
+      candidates.push(id);
     }
   }
-  return results;
+  if (candidates.length === 0)
+    return [];
+  const qTokens = orderedTokenize(query);
+  const hasUnsorted = metadata.unsorted.size > 0;
+  if (!hasUnsorted) {
+    if (limit === Infinity || limit >= candidates.length)
+      return candidates;
+    return candidates.slice(0, limit);
+  }
+  const scored = new Array(candidates.length);
+  for (let i = 0;i < candidates.length; i++) {
+    const id = candidates[i];
+    const unsortedIds = metadata.unsorted.get(id);
+    if (unsortedIds === undefined) {
+      scored[i] = { id, score: 0, order: i };
+      continue;
+    }
+    const itemTokens = new Array(unsortedIds.length);
+    for (let j = 0;j < unsortedIds.length; j++) {
+      itemTokens[j] = tokens[unsortedIds[j]] ?? "";
+    }
+    scored[i] = { id, score: scoreOne(query, qTokens, itemTokens), order: i };
+  }
+  scored.sort((a, b) => {
+    if (a.score !== b.score)
+      return b.score - a.score;
+    return a.order - b.order;
+  });
+  const out = [];
+  const max = limit === Infinity ? scored.length : Math.min(limit, scored.length);
+  for (let i = 0;i < max; i++)
+    out.push(scored[i].id);
+  return out;
 }
 // src/common/numeric/findInLinkedArrays.ts
 function findInLinkedArraysNumeric(query, linked, limit = 10, opts) {
@@ -733,11 +910,10 @@ function findInLinkedArraysNumeric(query, linked, limit = 10, opts) {
   const preparedTokens = prepareQueryTokens(queryTokenStrs);
   const positiveTokens = preparedTokens.filter((t) => !t.negated);
   const negatedTokens = preparedTokens.filter((t) => t.negated);
-  const results = [];
+  const scored = [];
+  let orderCounter = 0;
   for (const [primaryId, primary] of linked.primaryData) {
-    if (results.length >= limit)
-      break;
-    const primaryTokens = linked.primaryMeta.get(primaryId);
+    const primaryTokens = linked.primaryMeta.sorted.get(primaryId);
     if (!primaryTokens)
       continue;
     const linkedIds = linked.primaryToLinked.get(primaryId) || [];
@@ -748,22 +924,20 @@ function findInLinkedArraysNumeric(query, linked, limit = 10, opts) {
         break;
       }
     }
-    if (negatedMatchesInPrimary) {
+    if (negatedMatchesInPrimary)
       continue;
-    }
     let totalLength = primaryTokens.length;
     for (const linkedId of linkedIds) {
-      const linkedTokens = linked.linkedMeta.get(linkedId);
-      if (linkedTokens) {
+      const linkedTokens = linked.linkedMeta.sorted.get(linkedId);
+      if (linkedTokens)
         totalLength += linkedTokens.length;
-      }
     }
     const allTokens = new Uint32Array(totalLength);
     let offset = 0;
     allTokens.set(primaryTokens, offset);
     offset += primaryTokens.length;
     for (const linkedId of linkedIds) {
-      const linkedTokens = linked.linkedMeta.get(linkedId);
+      const linkedTokens = linked.linkedMeta.sorted.get(linkedId);
       if (linkedTokens) {
         allTokens.set(linkedTokens, offset);
         offset += linkedTokens.length;
@@ -777,230 +951,289 @@ function findInLinkedArraysNumeric(query, linked, limit = 10, opts) {
         break;
       }
     }
-    if (allPositiveMatch) {
-      let anyMatchInPrimary = false;
-      let anyMatchInLinked = false;
-      const matchingLinkedIds = new Set;
-      const tokensMatchingPrimary = new Set;
-      for (const preparedToken of positiveTokens) {
-        if (matchPreparedTokenInSortedList(preparedToken, primaryTokens, opts)) {
-          anyMatchInPrimary = true;
-          tokensMatchingPrimary.add(preparedToken);
-        }
+    if (!allPositiveMatch)
+      continue;
+    let anyMatchInPrimary = false;
+    let anyMatchInLinked = false;
+    const matchingLinkedIds = new Set;
+    const tokensMatchingPrimary = new Set;
+    for (const preparedToken of positiveTokens) {
+      if (matchPreparedTokenInSortedList(preparedToken, primaryTokens, opts)) {
+        anyMatchInPrimary = true;
+        tokensMatchingPrimary.add(preparedToken);
       }
-      const tokensForLinkedMatching = positiveTokens.filter((t) => !tokensMatchingPrimary.has(t));
-      if (tokensForLinkedMatching.length > 0 || negatedTokens.length > 0) {
-        for (const linkedId of linkedIds) {
-          const linkedTokens = linked.linkedMeta.get(linkedId);
-          if (linkedTokens) {
-            let matchesNegatedToken = false;
-            for (const negatedToken of negatedTokens) {
-              if (matchPreparedTokenInSortedList(negatedToken, linkedTokens, opts)) {
-                matchesNegatedToken = true;
-                break;
-              }
-            }
-            if (matchesNegatedToken) {
-              continue;
-            }
-            if (tokensForLinkedMatching.length > 0) {
-              const matchesAllNonPrimaryTokens = tokensForLinkedMatching.every((preparedToken) => matchPreparedTokenInSortedList(preparedToken, linkedTokens, opts));
-              if (matchesAllNonPrimaryTokens) {
-                anyMatchInLinked = true;
-                matchingLinkedIds.add(linkedId);
-              }
-            } else {
-              matchingLinkedIds.add(linkedId);
-            }
+    }
+    const tokensForLinkedMatching = positiveTokens.filter((t) => !tokensMatchingPrimary.has(t));
+    if (tokensForLinkedMatching.length > 0 || negatedTokens.length > 0) {
+      for (const linkedId of linkedIds) {
+        const linkedTokens = linked.linkedMeta.sorted.get(linkedId);
+        if (!linkedTokens)
+          continue;
+        let matchesNegatedToken = false;
+        for (const negatedToken of negatedTokens) {
+          if (matchPreparedTokenInSortedList(negatedToken, linkedTokens, opts)) {
+            matchesNegatedToken = true;
+            break;
           }
         }
+        if (matchesNegatedToken)
+          continue;
+        if (tokensForLinkedMatching.length > 0) {
+          const matchesAllNonPrimaryTokens = tokensForLinkedMatching.every((preparedToken) => matchPreparedTokenInSortedList(preparedToken, linkedTokens, opts));
+          if (matchesAllNonPrimaryTokens) {
+            anyMatchInLinked = true;
+            matchingLinkedIds.add(linkedId);
+          }
+        } else {
+          matchingLinkedIds.add(linkedId);
+        }
       }
-      if (tokensForLinkedMatching.length > 0 && matchingLinkedIds.size === 0) {
+    }
+    if (tokensForLinkedMatching.length > 0 && matchingLinkedIds.size === 0) {
+      continue;
+    }
+    const matchedLinked = Array.from(matchingLinkedIds).map((id) => linked.linkedData.get(id)).filter((item) => item !== undefined);
+    const result = {
+      primary,
+      linked: matchedLinked,
+      matchedIn: anyMatchInPrimary && anyMatchInLinked ? "both" : anyMatchInPrimary ? "primary" : "linked"
+    };
+    scored.push({
+      result,
+      score: 0,
+      order: orderCounter++,
+      primaryId,
+      matchedLinkedIds: Array.from(matchingLinkedIds)
+    });
+  }
+  if (scored.length === 0) {
+    return { queryTokens: queryTokenStrs, results: [], hasNegation: negatedTokens.length > 0 };
+  }
+  const qTokens = orderedTokenize(query);
+  const hasUnsorted = linked.primaryMeta.unsorted.size > 0 || linked.linkedMeta.unsorted.size > 0;
+  if (hasUnsorted) {
+    for (const s of scored) {
+      const itemTokenIds = [];
+      const pu = linked.primaryMeta.unsorted.get(s.primaryId);
+      if (pu)
+        for (const tid of pu)
+          itemTokenIds.push(tid);
+      for (const lid of s.matchedLinkedIds) {
+        const lu = linked.linkedMeta.unsorted.get(lid);
+        if (lu)
+          for (const tid of lu)
+            itemTokenIds.push(tid);
+      }
+      if (itemTokenIds.length === 0) {
+        s.score = 0;
         continue;
       }
-      const matchedLinked = Array.from(matchingLinkedIds).map((id) => linked.linkedData.get(id)).filter((item) => item !== undefined);
-      results.push({
-        primary,
-        linked: matchedLinked,
-        matchedIn: anyMatchInPrimary && anyMatchInLinked ? "both" : anyMatchInPrimary ? "primary" : "linked"
-      });
+      const itemTokens = new Array(itemTokenIds.length);
+      for (let i = 0;i < itemTokenIds.length; i++) {
+        itemTokens[i] = tokens[itemTokenIds[i]] ?? "";
+      }
+      s.score = scoreOne(query, qTokens, itemTokens);
     }
   }
+  scored.sort((a, b) => {
+    if (a.score !== b.score)
+      return b.score - a.score;
+    return a.order - b.order;
+  });
+  const max = Math.min(limit, scored.length);
+  const results = new Array(max);
+  for (let i = 0;i < max; i++)
+    results[i] = scored[i].result;
   return { queryTokens: queryTokenStrs, results, hasNegation: negatedTokens.length > 0 };
 }
 function findInLinkedArraysSimpleNumeric(query, linked, limit = 10, opts) {
   return findInLinkedArraysNumeric(query, linked, limit, opts).results;
 }
 // src/common/numeric/AugmentedMetadata.ts
-class AugmentedMetadata {
-  baseMetadata;
-  augmentedTokens;
-  spec;
-  constructor(baseMetadata, spec, augmentedDataMap) {
-    this.baseMetadata = baseMetadata;
-    this.augmentedTokens = new Map;
-    this.spec = spec;
-    if (augmentedDataMap && augmentedDataMap.size > 0) {
-      this.updateAugmentedBatch(augmentedDataMap);
-    }
-  }
-  updateAugmented(id, augmentedData) {
-    if (!augmentedData) {
-      this.augmentedTokens.delete(id);
-      return;
-    }
-    const tokensMap = createTokensMap();
-    const tokens2 = prepareObjectForSearchNumeric(augmentedData, this.spec, tokensMap);
-    if (tokens2 && tokens2.length > 0) {
-      this.augmentedTokens.set(id, tokens2);
+function mergeSortedTokenArrays(a, b) {
+  const merged = new Uint32Array(a.length + b.length);
+  let i = 0, j = 0, k = 0;
+  let lastToken = -1;
+  while (i < a.length && j < b.length) {
+    const tokenA = a[i];
+    const tokenB = b[j];
+    if (tokenA < tokenB) {
+      if (tokenA !== lastToken) {
+        merged[k++] = tokenA;
+        lastToken = tokenA;
+      }
+      i++;
+    } else if (tokenA > tokenB) {
+      if (tokenB !== lastToken) {
+        merged[k++] = tokenB;
+        lastToken = tokenB;
+      }
+      j++;
     } else {
-      this.augmentedTokens.delete(id);
+      if (tokenA !== lastToken) {
+        merged[k++] = tokenA;
+        lastToken = tokenA;
+      }
+      i++;
+      j++;
     }
-    clearTokensMap(tokensMap);
   }
-  updateAugmentedBatch(augmentedDataMap) {
-    const tokensMap = createTokensMap();
-    for (const [id, augmentedData] of augmentedDataMap) {
-      if (!augmentedData) {
-        this.augmentedTokens.delete(id);
-        continue;
-      }
-      const tokens2 = prepareObjectForSearchNumeric(augmentedData, this.spec, tokensMap);
-      if (tokens2 && tokens2.length > 0) {
-        this.augmentedTokens.set(id, tokens2);
-      } else {
-        this.augmentedTokens.delete(id);
-      }
+  while (i < a.length) {
+    const tokenA = a[i];
+    if (tokenA !== lastToken) {
+      merged[k++] = tokenA;
+      lastToken = tokenA;
     }
-    clearTokensMap(tokensMap);
+    i++;
   }
-  clearAugmented(id) {
-    this.augmentedTokens.delete(id);
+  while (j < b.length) {
+    const tokenB = b[j];
+    if (tokenB !== lastToken) {
+      merged[k++] = tokenB;
+      lastToken = tokenB;
+    }
+    j++;
   }
-  clearAllAugmented() {
-    this.augmentedTokens.clear();
+  return merged.slice(0, k);
+}
+
+class AugmentedSortedMap extends Map {
+  base;
+  augTokens;
+  constructor(base, augTokens) {
+    super();
+    this.base = base;
+    this.augTokens = augTokens;
   }
   get(id) {
-    const baseTokens = this.baseMetadata.get(id);
-    if (!baseTokens)
-      return;
-    const augTokens = this.augmentedTokens.get(id);
-    if (!augTokens)
+    const baseTokens = this.base.get(id);
+    const augTokens = this.augTokens.get(id);
+    if (baseTokens === undefined)
+      return augTokens;
+    if (augTokens === undefined)
       return baseTokens;
-    return this.mergeSortedTokenArrays(baseTokens, augTokens);
+    return mergeSortedTokenArrays(baseTokens, augTokens);
   }
   has(id) {
-    return this.baseMetadata.has(id);
+    return this.base.has(id) || this.augTokens.has(id);
   }
   get size() {
-    return this.baseMetadata.size;
-  }
-  [Symbol.iterator]() {
-    const self = this;
-    const baseIterator = this.baseMetadata[Symbol.iterator]();
-    return {
-      next() {
-        const baseResult = baseIterator.next();
-        if (baseResult.done) {
-          return { done: true, value: undefined };
-        }
-        const [id, baseTokens] = baseResult.value;
-        const augTokens = self.augmentedTokens.get(id);
-        const tokens2 = augTokens ? self.mergeSortedTokenArrays(baseTokens, augTokens) : baseTokens;
-        return { done: false, value: [id, tokens2] };
-      },
-      [Symbol.iterator]() {
-        return this;
-      },
-      [Symbol.dispose]() {}
-    };
+    if (this.augTokens.size === 0)
+      return this.base.size;
+    let count = this.base.size;
+    for (const id of this.augTokens.keys()) {
+      if (!this.base.has(id))
+        count++;
+    }
+    return count;
   }
   keys() {
-    return this.baseMetadata.keys();
+    return this.unionIdIterator();
   }
   values() {
     const self = this;
-    const keysIterator = this.baseMetadata.keys();
-    return {
-      next() {
-        const keyResult = keysIterator.next();
-        if (keyResult.done) {
-          return { done: true, value: undefined };
-        }
-        const tokens2 = self.get(keyResult.value);
-        return tokens2 ? { done: false, value: tokens2 } : { done: true, value: undefined };
-      },
-      [Symbol.iterator]() {
-        return this;
-      },
-      [Symbol.dispose]() {}
-    };
+    function* gen() {
+      for (const id of self.unionIdIterator()) {
+        const tokens2 = self.get(id);
+        if (tokens2 !== undefined)
+          yield tokens2;
+      }
+    }
+    return gen();
   }
   entries() {
-    return this[Symbol.iterator]();
+    const self = this;
+    function* gen() {
+      for (const id of self.unionIdIterator()) {
+        const tokens2 = self.get(id);
+        if (tokens2 !== undefined)
+          yield [id, tokens2];
+      }
+    }
+    return gen();
+  }
+  [Symbol.iterator]() {
+    return this.entries();
   }
   forEach(callbackfn) {
-    for (const [id, tokens2] of this) {
+    for (const [id, tokens2] of this.entries()) {
       callbackfn(tokens2, id, this);
     }
   }
   set(_key, _value) {
-    throw new Error("AugmentedMetadata.set() is not supported. Use updateAugmented() instead.");
+    throw new Error("AugmentedMetadata.sorted is read-only. Use updateAugmented() instead.");
   }
   delete(_key) {
-    throw new Error("AugmentedMetadata.delete() is not supported. Use clearAugmented() instead.");
+    throw new Error("AugmentedMetadata.sorted is read-only. Use clearAugmented() instead.");
   }
   clear() {
-    throw new Error("AugmentedMetadata.clear() is not supported. Use clearAllAugmented() instead.");
+    throw new Error("AugmentedMetadata.sorted is read-only. Use clearAllAugmented() instead.");
   }
   get [Symbol.toStringTag]() {
-    return "AugmentedMetadata";
-  }
-  mergeSortedTokenArrays(a, b) {
-    const merged = new Uint32Array(a.length + b.length);
-    let i = 0, j = 0, k = 0;
-    let lastToken = -1;
-    while (i < a.length && j < b.length) {
-      const tokenA = a[i];
-      const tokenB = b[j];
-      if (tokenA < tokenB) {
-        if (tokenA !== lastToken) {
-          merged[k++] = tokenA;
-          lastToken = tokenA;
-        }
-        i++;
-      } else if (tokenA > tokenB) {
-        if (tokenB !== lastToken) {
-          merged[k++] = tokenB;
-          lastToken = tokenB;
-        }
-        j++;
-      } else {
-        if (tokenA !== lastToken) {
-          merged[k++] = tokenA;
-          lastToken = tokenA;
-        }
-        i++;
-        j++;
-      }
+    return "AugmentedSortedMap";
+  }
+  *unionIdIterator() {
+    const seen = new Set;
+    for (const id of this.base.keys()) {
+      seen.add(id);
+      yield id;
     }
-    while (i < a.length) {
-      const tokenA = a[i];
-      if (tokenA !== lastToken) {
-        merged[k++] = tokenA;
-        lastToken = tokenA;
-      }
-      i++;
+    for (const id of this.augTokens.keys()) {
+      if (!seen.has(id))
+        yield id;
     }
-    while (j < b.length) {
-      const tokenB = b[j];
-      if (tokenB !== lastToken) {
-        merged[k++] = tokenB;
-        lastToken = tokenB;
+  }
+}
+
+class AugmentedMetadata {
+  sorted;
+  unsorted;
+  augmentedTokens;
+  spec;
+  constructor(baseMetadata, spec, augmentedDataMap) {
+    this.augmentedTokens = new Map;
+    this.spec = spec;
+    this.sorted = new AugmentedSortedMap(baseMetadata.sorted, this.augmentedTokens);
+    this.unsorted = baseMetadata.unsorted;
+    if (augmentedDataMap && augmentedDataMap.size > 0) {
+      this.updateAugmentedBatch(augmentedDataMap);
+    }
+  }
+  updateAugmented(id, augmentedData) {
+    if (augmentedData == null) {
+      this.augmentedTokens.delete(id);
+      return;
+    }
+    const tokensMap = createTokensMap();
+    const prepared = prepareObjectForSearchNumeric(augmentedData, this.spec, tokensMap);
+    clearTokensMap(tokensMap);
+    if (prepared !== undefined && prepared.sorted.length > 0) {
+      this.augmentedTokens.set(id, prepared.sorted);
+    } else {
+      this.augmentedTokens.delete(id);
+    }
+  }
+  updateAugmentedBatch(augmentedDataMap) {
+    const tokensMap = createTokensMap();
+    for (const [id, augmentedData] of augmentedDataMap) {
+      if (augmentedData == null) {
+        this.augmentedTokens.delete(id);
+        continue;
+      }
+      const prepared = prepareObjectForSearchNumeric(augmentedData, this.spec, tokensMap);
+      if (prepared !== undefined && prepared.sorted.length > 0) {
+        this.augmentedTokens.set(id, prepared.sorted);
+      } else {
+        this.augmentedTokens.delete(id);
       }
-      j++;
     }
-    return merged.slice(0, k);
+    clearTokensMap(tokensMap);
+  }
+  clearAugmented(id) {
+    this.augmentedTokens.delete(id);
+  }
+  clearAllAugmented() {
+    this.augmentedTokens.clear();
   }
 }
 // src/common/numeric/syncSearchArrayMetadata.ts
@@ -1013,6 +1246,25 @@ function tokensEqual(a, b) {
   }
   return true;
 }
+function firstNEqual(a, b) {
+  if (a === undefined && b === undefined)
+    return true;
+  if (a === undefined || b === undefined)
+    return false;
+  return tokensEqual(a, b);
+}
+function applyPreparedTokens(metadata, id, result) {
+  metadata.sorted.set(id, result.sorted);
+  if (result.firstN !== undefined) {
+    metadata.unsorted.set(id, result.firstN);
+  } else {
+    metadata.unsorted.delete(id);
+  }
+}
+function deletePreparedTokens(metadata, id) {
+  metadata.sorted.delete(id);
+  metadata.unsorted.delete(id);
+}
 function syncSearchArrayMetadataNumeric(metadata, newData, spec) {
   const result = {
     added: 0,
@@ -1023,30 +1275,31 @@ function syncSearchArrayMetadataNumeric(metadata, newData, spec) {
     removedIds: []
   };
   const tokensMap = createTokensMap();
-  for (const id of metadata.keys()) {
+  for (const id of metadata.sorted.keys()) {
     if (!newData.has(id)) {
-      metadata.delete(id);
+      deletePreparedTokens(metadata, id);
       result.removed++;
       result.removedIds.push(id);
     }
   }
   for (const [id, item] of newData) {
-    const newTokens = prepareObjectForSearchNumeric(item, spec, tokensMap);
-    if (newTokens === undefined) {
-      if (metadata.has(id)) {
-        metadata.delete(id);
+    const prepared = prepareObjectForSearchNumeric(item, spec, tokensMap);
+    if (prepared === undefined) {
+      if (metadata.sorted.has(id)) {
+        deletePreparedTokens(metadata, id);
         result.removed++;
         result.removedIds.push(id);
       }
       continue;
     }
-    const existingTokens = metadata.get(id);
-    if (existingTokens === undefined) {
-      metadata.set(id, newTokens);
+    const existingSorted = metadata.sorted.get(id);
+    const existingFirstN = metadata.unsorted.get(id);
+    if (existingSorted === undefined) {
+      applyPreparedTokens(metadata, id, prepared);
       result.added++;
       result.addedIds.push(id);
-    } else if (!tokensEqual(existingTokens, newTokens)) {
-      metadata.set(id, newTokens);
+    } else if (!tokensEqual(existingSorted, prepared.sorted) || !firstNEqual(existingFirstN, prepared.firstN)) {
+      applyPreparedTokens(metadata, id, prepared);
       result.updated++;
       result.updatedIds.push(id);
     }
@@ -1066,7 +1319,7 @@ function syncSearchArrayMetadataNumericWithPrevious(metadata, previousData, newD
   const tokensMap = createTokensMap();
   for (const id of previousData.keys()) {
     if (!newData.has(id)) {
-      metadata.delete(id);
+      deletePreparedTokens(metadata, id);
       result.removed++;
       result.removedIds.push(id);
     }
@@ -1074,22 +1327,23 @@ function syncSearchArrayMetadataNumericWithPrevious(metadata, previousData, newD
   for (const [id, item] of newData) {
     const wasInPrevious = previousData.has(id);
     if (!wasInPrevious) {
-      const tokens2 = prepareObjectForSearchNumeric(item, spec, tokensMap);
-      if (tokens2 !== undefined) {
-        metadata.set(id, tokens2);
+      const prepared = prepareObjectForSearchNumeric(item, spec, tokensMap);
+      if (prepared !== undefined) {
+        applyPreparedTokens(metadata, id, prepared);
         result.added++;
         result.addedIds.push(id);
       }
     } else if (!options?.skipUpdateCheck) {
-      const newTokens = prepareObjectForSearchNumeric(item, spec, tokensMap);
-      if (newTokens === undefined) {
-        metadata.delete(id);
+      const prepared = prepareObjectForSearchNumeric(item, spec, tokensMap);
+      if (prepared === undefined) {
+        deletePreparedTokens(metadata, id);
         result.removed++;
         result.removedIds.push(id);
       } else {
-        const existingTokens = metadata.get(id);
-        if (existingTokens && !tokensEqual(existingTokens, newTokens)) {
-          metadata.set(id, newTokens);
+        const existingSorted = metadata.sorted.get(id);
+        const existingFirstN = metadata.unsorted.get(id);
+        if (existingSorted && (!tokensEqual(existingSorted, prepared.sorted) || !firstNEqual(existingFirstN, prepared.firstN))) {
+          applyPreparedTokens(metadata, id, prepared);
           result.updated++;
           result.updatedIds.push(id);
         }
@@ -1109,29 +1363,48 @@ function tokensEqual2(a, b) {
   }
   return true;
 }
+function firstNEqual2(a, b) {
+  if (a === undefined && b === undefined)
+    return true;
+  if (a === undefined || b === undefined)
+    return false;
+  return tokensEqual2(a, b);
+}
+function applyOrClear(meta, id, prepared) {
+  if (prepared === undefined) {
+    meta.sorted.delete(id);
+    meta.unsorted.delete(id);
+    return;
+  }
+  meta.sorted.set(id, prepared.sorted);
+  if (prepared.firstN !== undefined) {
+    meta.unsorted.set(id, prepared.firstN);
+  } else {
+    meta.unsorted.delete(id);
+  }
+}
+function clearTokens(meta, id) {
+  meta.sorted.delete(id);
+  meta.unsorted.delete(id);
+}
 function upsertPrimaryItemNumeric(linked, item, spec) {
   const id = item._id;
   const wasAdded = !linked.primaryData.has(id);
   linked.primaryData.set(id, item);
   const tokensMap = createTokensMap();
-  const tokens2 = prepareObjectForSearchNumeric(item, spec, tokensMap);
+  const prepared = prepareObjectForSearchNumeric(item, spec, tokensMap);
   clearTokensMap(tokensMap);
-  if (tokens2 !== undefined) {
-    linked.primaryMeta.set(id, tokens2);
-  } else {
-    linked.primaryMeta.delete(id);
-  }
+  applyOrClear(linked.primaryMeta, id, prepared);
   if (wasAdded && !linked.primaryToLinked.has(id)) {
     linked.primaryToLinked.set(id, []);
   }
   return { added: wasAdded };
 }
 function removePrimaryItemNumeric(linked, primaryId) {
-  if (!linked.primaryData.has(primaryId)) {
+  if (!linked.primaryData.has(primaryId))
     return false;
-  }
   linked.primaryData.delete(primaryId);
-  linked.primaryMeta.delete(primaryId);
+  clearTokens(linked.primaryMeta, primaryId);
   linked.primaryToLinked.delete(primaryId);
   return true;
 }
@@ -1140,35 +1413,28 @@ function upsertLinkedItemNumeric(linked, item, foreignKeyGetter, spec) {
   const newPrimaryId = foreignKeyGetter(item);
   let previousPrimaryId;
   const existingItem = linked.linkedData.get(id);
-  if (existingItem) {
+  if (existingItem)
     previousPrimaryId = foreignKeyGetter(existingItem);
-  }
   const wasAdded = !existingItem;
   linked.linkedData.set(id, item);
   const tokensMap = createTokensMap();
-  const tokens2 = prepareObjectForSearchNumeric(item, spec, tokensMap);
+  const prepared = prepareObjectForSearchNumeric(item, spec, tokensMap);
   clearTokensMap(tokensMap);
-  if (tokens2 !== undefined) {
-    linked.linkedMeta.set(id, tokens2);
-  } else {
-    linked.linkedMeta.delete(id);
-  }
+  applyOrClear(linked.linkedMeta, id, prepared);
   if (previousPrimaryId !== newPrimaryId) {
     if (previousPrimaryId) {
       const oldList = linked.primaryToLinked.get(previousPrimaryId);
       if (oldList) {
         const index = oldList.indexOf(id);
-        if (index !== -1) {
+        if (index !== -1)
           oldList.splice(index, 1);
-        }
       }
     }
     if (newPrimaryId) {
       const newList = linked.primaryToLinked.get(newPrimaryId);
       if (newList) {
-        if (!newList.includes(id)) {
+        if (!newList.includes(id))
           newList.push(id);
-        }
       } else {
         linked.primaryToLinked.set(newPrimaryId, [id]);
       }
@@ -1181,64 +1447,57 @@ function upsertLinkedItemNumeric(linked, item, foreignKeyGetter, spec) {
 }
 function removeLinkedItemNumeric(linked, linkedId, foreignKeyGetter) {
   const existingItem = linked.linkedData.get(linkedId);
-  if (!existingItem) {
+  if (!existingItem)
     return false;
-  }
   const primaryId = foreignKeyGetter(existingItem);
   linked.linkedData.delete(linkedId);
-  linked.linkedMeta.delete(linkedId);
+  clearTokens(linked.linkedMeta, linkedId);
   if (primaryId) {
     const list = linked.primaryToLinked.get(primaryId);
     if (list) {
       const index = list.indexOf(linkedId);
-      if (index !== -1) {
+      if (index !== -1)
         list.splice(index, 1);
-      }
     }
   }
   return true;
 }
 function syncLinkedItemsForPrimaryNumeric(linked, primaryId, newLinkedItems, foreignKeyGetter, spec) {
-  const result = {
-    added: 0,
-    updated: 0,
-    removed: 0
-  };
+  const result = { added: 0, updated: 0, removed: 0 };
   const tokensMap = createTokensMap();
   const currentLinkedIds = new Set(linked.primaryToLinked.get(primaryId) || []);
   const newLinkedIds = new Set;
   for (const [id, item] of newLinkedItems) {
-    if (foreignKeyGetter(item) === primaryId) {
+    if (foreignKeyGetter(item) === primaryId)
       newLinkedIds.add(id);
-    }
   }
   for (const id of currentLinkedIds) {
     if (!newLinkedIds.has(id)) {
-      const item = linked.linkedData.get(id);
-      if (item) {
+      if (linked.linkedData.has(id)) {
         linked.linkedData.delete(id);
-        linked.linkedMeta.delete(id);
+        clearTokens(linked.linkedMeta, id);
         result.removed++;
       }
     }
   }
   for (const id of newLinkedIds) {
     const item = newLinkedItems.get(id);
-    const tokens2 = prepareObjectForSearchNumeric(item, spec, tokensMap);
+    const prepared = prepareObjectForSearchNumeric(item, spec, tokensMap);
     if (currentLinkedIds.has(id)) {
-      const existingTokens = linked.linkedMeta.get(id);
-      if (tokens2 !== undefined) {
-        if (!existingTokens || !tokensEqual2(existingTokens, tokens2)) {
-          linked.linkedMeta.set(id, tokens2);
+      const existingSorted = linked.linkedMeta.sorted.get(id);
+      const existingFirstN = linked.linkedMeta.unsorted.get(id);
+      if (prepared !== undefined) {
+        if (!existingSorted || !tokensEqual2(existingSorted, prepared.sorted) || !firstNEqual2(existingFirstN, prepared.firstN)) {
+          applyOrClear(linked.linkedMeta, id, prepared);
           result.updated++;
         }
+      } else {
+        clearTokens(linked.linkedMeta, id);
       }
       linked.linkedData.set(id, item);
     } else {
       linked.linkedData.set(id, item);
-      if (tokens2 !== undefined) {
-        linked.linkedMeta.set(id, tokens2);
-      }
+      applyOrClear(linked.linkedMeta, id, prepared);
       result.added++;
     }
   }
@@ -1247,54 +1506,14 @@ function syncLinkedItemsForPrimaryNumeric(linked, primaryId, newLinkedItems, for
   return result;
 }
 // src/common/numeric/searchAndRank.ts
-function orderedTokenize(input) {
-  return tokenize(sanitiseString(preprocessString(normalizeDates(input))), DEFAULT_TOKENIZE_CONFIG);
-}
-function scoreOne(item, query, qTokens, primaryTextFn, exactKeys) {
-  if (exactKeys && exactKeys.length > 0) {
-    for (let i = 0;i < exactKeys.length; i++) {
-      if (exactKeys[i] === query)
-        return 1e5;
-    }
-  }
-  if (qTokens.length === 0)
-    return 0;
-  const primaryText = primaryTextFn(item) || "";
-  const itemTokens = orderedTokenize(primaryText);
-  if (itemTokens.length === 0)
-    return 1;
-  let qi = 0;
-  const positions = [];
-  for (let i = 0;i < itemTokens.length; i++) {
-    if (qi < qTokens.length && itemTokens[i].startsWith(qTokens[qi])) {
-      positions.push(i);
-      qi++;
-    }
-  }
-  if (qi === 0)
-    return 1;
-  let score = 50;
-  if (qi === qTokens.length)
-    score += 100;
-  if (positions[0] === 0)
-    score += 50;
-  let perfect = qi === qTokens.length;
-  if (perfect) {
-    for (let i = 0;i < positions.length; i++) {
-      if (positions[i] !== i) {
-        perfect = false;
-        break;
-      }
-    }
-  }
-  if (perfect)
-    score += 80;
-  if (positions.length >= 2) {
-    const span = positions[positions.length - 1] - positions[0] + 1;
-    score += positions.length / span * 30;
+function tokenIdsToStrings(tokenIds) {
+  if (tokenIds === undefined)
+    return;
+  const out = new Array(tokenIds.length);
+  for (let i = 0;i < tokenIds.length; i++) {
+    out[i] = tokens[tokenIds[i]] ?? "";
   }
-  score -= primaryText.length * 0.1;
-  return score;
+  return out;
 }
 function searchAndRankNumeric(query, data, metadata, options) {
   const { primaryTextFn, exactKeysFn, limit, opts } = options;
@@ -1307,8 +1526,9 @@ function searchAndRankNumeric(query, data, metadata, options) {
     const item = data.get(id);
     if (!item)
       continue;
+    const itemTokens = tokenIdsToStrings(metadata.unsorted.get(id)) ?? orderedTokenize(primaryTextFn(item) || "");
     const exactKeys = exactKeysFn?.(item);
-    const score = scoreOne(item, query, qTokens, primaryTextFn, exactKeys);
+    const score = scoreOne(query, qTokens, itemTokens, exactKeys);
     scored.push({ id, item, score });
   }
   scored.sort((x, y) => {
@@ -1332,6 +1552,13 @@ function tokensEqual3(a, b) {
   }
   return true;
 }
+function firstNEqual3(a, b) {
+  if (a === undefined && b === undefined)
+    return true;
+  if (a === undefined || b === undefined)
+    return false;
+  return tokensEqual3(a, b);
+}
 
 class SearchUniverse {
   collections = new Map;
@@ -1341,7 +1568,7 @@ class SearchUniverse {
     }
     const collection = {
       data: new Map,
-      metadata: new Map,
+      metadata: emptyNumericSearchMetadata(),
       config
     };
     if (config.linkedTo) {
@@ -1358,7 +1585,8 @@ class SearchUniverse {
   loadCollection(name, items) {
     const collection = this.getCollectionOrThrow(name);
     collection.data.clear();
-    collection.metadata.clear();
+    collection.metadata.sorted.clear();
+    collection.metadata.unsorted.clear();
     if (collection.primaryToLinked) {
       collection.primaryToLinked.clear();
     }
@@ -1366,8 +1594,11 @@ class SearchUniverse {
       collection.data.set(item._id, item);
     }
     const newMetadata = createSearchArrayMetadataNumeric(collection.data, collection.config.spec);
-    for (const [id, tokens2] of newMetadata) {
-      collection.metadata.set(id, tokens2);
+    for (const [id, tokens2] of newMetadata.sorted) {
+      collection.metadata.sorted.set(id, tokens2);
+    }
+    for (const [id, tokens2] of newMetadata.unsorted) {
+      collection.metadata.unsorted.set(id, tokens2);
     }
     if (collection.config.linkedTo && collection.primaryToLinked) {
       const { foreignKeyGetter } = collection.config.linkedTo;
@@ -1463,11 +1694,17 @@ class SearchUniverse {
     collection.data.set(id, item);
     const tokensMap = createTokensMap();
     try {
-      const tokens2 = prepareObjectForSearchNumeric(item, collection.config.spec, tokensMap);
-      if (tokens2 !== undefined) {
-        collection.metadata.set(id, tokens2);
+      const prepared = prepareObjectForSearchNumeric(item, collection.config.spec, tokensMap);
+      if (prepared !== undefined) {
+        collection.metadata.sorted.set(id, prepared.sorted);
+        if (prepared.firstN !== undefined) {
+          collection.metadata.unsorted.set(id, prepared.firstN);
+        } else {
+          collection.metadata.unsorted.delete(id);
+        }
       } else {
-        collection.metadata.delete(id);
+        collection.metadata.sorted.delete(id);
+        collection.metadata.unsorted.delete(id);
       }
     } finally {
       clearTokensMap(tokensMap);
@@ -1521,7 +1758,8 @@ class SearchUniverse {
       }
     }
     collection.data.delete(id);
-    collection.metadata.delete(id);
+    collection.metadata.sorted.delete(id);
+    collection.metadata.unsorted.delete(id);
     return true;
   }
   removeFromSearchMetadataBatch(collectionName, ids) {
@@ -1572,9 +1810,10 @@ class SearchUniverse {
           }
         }
       }
-      const hadMetadata = collection.metadata.has(id);
+      const hadMetadata = collection.metadata.sorted.has(id);
       collection.data.delete(id);
-      collection.metadata.delete(id);
+      collection.metadata.sorted.delete(id);
+      collection.metadata.unsorted.delete(id);
       if (hadMetadata) {
         result.removed++;
         result.removedIds.push(id);
@@ -1585,11 +1824,11 @@ class SearchUniverse {
       for (const item of newItems) {
         const id = item._id;
         const existing = collection.data.get(id);
-        const hadMetadata = collection.metadata.has(id);
+        const hadMetadata = collection.metadata.sorted.has(id);
         const shouldTokenize = !existing || !skipUpdateCheck;
-        let newTokens;
+        let prepared;
         if (shouldTokenize) {
-          newTokens = prepareObjectForSearchNumeric(item, collection.config.spec, tokensMap);
+          prepared = prepareObjectForSearchNumeric(item, collection.config.spec, tokensMap);
         }
         if (fkGetter && collection.primaryToLinked) {
           const newFk = fkGetter(item);
@@ -1619,20 +1858,30 @@ class SearchUniverse {
         collection.data.set(id, item);
         if (!shouldTokenize)
           continue;
-        if (newTokens === undefined) {
+        if (prepared === undefined) {
           if (hadMetadata) {
-            collection.metadata.delete(id);
+            collection.metadata.sorted.delete(id);
+            collection.metadata.unsorted.delete(id);
             result.removed++;
             result.removedIds.push(id);
           }
         } else if (!hadMetadata) {
-          collection.metadata.set(id, newTokens);
+          collection.metadata.sorted.set(id, prepared.sorted);
+          if (prepared.firstN !== undefined) {
+            collection.metadata.unsorted.set(id, prepared.firstN);
+          }
           result.added++;
           result.addedIds.push(id);
         } else {
-          const existingTokens = collection.metadata.get(id);
-          if (!tokensEqual3(existingTokens, newTokens)) {
-            collection.metadata.set(id, newTokens);
+          const existingSorted = collection.metadata.sorted.get(id);
+          const existingFirstN = collection.metadata.unsorted.get(id);
+          if (!tokensEqual3(existingSorted, prepared.sorted) || !firstNEqual3(existingFirstN, prepared.firstN)) {
+            collection.metadata.sorted.set(id, prepared.sorted);
+            if (prepared.firstN !== undefined) {
+              collection.metadata.unsorted.set(id, prepared.firstN);
+            } else {
+              collection.metadata.unsorted.delete(id);
+            }
             result.updated++;
             result.updatedIds.push(id);
           }
@@ -1711,7 +1960,8 @@ class SearchUniverse {
   clear() {
     for (const collection of this.collections.values()) {
       collection.data.clear();
-      collection.metadata.clear();
+      collection.metadata.sorted.clear();
+      collection.metadata.unsorted.clear();
       if (collection.primaryToLinked) {
         collection.primaryToLinked.clear();
       }
@@ -1736,7 +1986,9 @@ export {
   searchInDataReturnIdsNumeric as searchInData,
   searchAndRankNumeric,
   searchAndRankNumeric as searchAndRank,
+  scoreOrderAndPresence,
   scoreOne,
+  scoreBag,
   sanitiseString,
   resetTokenRegistry,
   removePrimaryItemNumeric as removePrimaryItem,
@@ -1772,6 +2024,8 @@ export {
   allPreparedTokensMatch,
   TOKEN_ID_MASK,
   SearchUniverse,
+  PREFIX_TO_MATCH_MODE,
+  MATCH_MODE_PREFIXES,
   MATCH_MODE_BITS,
   DEFAULT_SEARCH_OPTS,
   AugmentedMetadata