cry-search 1.0.6 → 1.0.8

package/CLAUDE.md

@@ -19,18 +19,13 @@ Documentation is spread across multiple files that must be kept in sync:
 
 ## Architecture
 
-The library has two implementations:
-1. **Numeric (default)** - Memory-optimized using Uint32Array for token storage
-   - Location: `src/common/numeric/`
-   - ~45% less memory than string implementation
-   - Global token registry with array-based lookups
-   - Tokens stored as numeric IDs in Uint32Array
-2. **String (legacy)** - Original string-based implementation
-   - Location: `src/common/string/`
-   - Uses string arrays for token storage
-   - Available with `*String` suffix (e.g., `createSearchArrayMetadataString`)
+The library has a single implementation: memory-optimized using `Uint32Array` for token storage.
+- Location: `src/common/numeric/`
+- Global token registry (`src/utils/tokenRegistry.ts`) with array-based lookups
+- Tokens stored as numeric IDs in `Uint32Array` per item; strings interned globally
+- Match modes encoded in high 3 bits of each Uint32 token ID
 
-### Token Registry (Numeric Implementation)
+### Token Registry
 
 The numeric implementation uses a global token registry (`src/utils/tokenRegistry.ts`):
 - `tokens: string[]` - Global array of all tokens (preallocated 10k)
@@ -176,10 +171,10 @@ The numeric implementation uses a global token registry (`src/utils/tokenRegistr
 
 1. Id: string
 2. NumericToken: number (index into global tokens array)
-3. NumericTokenSortedList: Uint32Array (sorted numeric token IDs)
-4. NumericSearchMetadata: Map<Id, NumericTokenSortedList> (default SearchMetadata)
-5. StringSearchMetadata: Map<Id, string[]> (legacy, string-based)
-6. LinkedSearchMetadata: { primaryMeta, linkedMeta, primaryToLinked: Map<Id, Id[]> }
+3. NumericTokenSortedList: Uint32Array (sorted ASC numeric token IDs — retrieval)
+4. NumericTokenUnsortedList: Uint32Array (first-N tokens in SOURCE order — positional scoring)
+5. NumericSearchMetadata: { sorted: Map<Id, NumericTokenSortedList>, unsorted: Map<Id, NumericTokenUnsortedList> }
+6. NumericLinkedSearchMetadataWithData: { primaryMeta, linkedMeta, primaryToLinked, primaryData, linkedData }
 7. SearchableObject: { _id: Id, _deleted?: Date, _blocked?: Date }
 8. SearchableData<T>: Map<Id, T> where T extends SearchableObject
 9. SearchableObjectSpec<T extends SearchableObject, C = any>:
@@ -189,6 +184,8 @@ The numeric implementation uses a global token registry (`src/utils/tokenRegistr
    4. extractSearchableStringFn?: (obj: SearchableObject) => string | undefined
    5. enrichSearchObjectFn?: (obj: SearchableObject, searchContext: C) => void
    6. fieldMatchModes?: Partial<Record<keyof T | string, MatchMode>> - per-field match modes
+   7. keyFields?: (keyof T | string)[] - fields whose tokens populate `metadata.unsorted` (source order) for positional ranking
+   8. firstTokensN?: number (default 8) - cap on per-item `metadata.unsorted` length
 10. MatchMode: 'start' | 'end' | 'startEnd' | 'anywhere' | 'whole'
     1. Field token prefixes: '<' (start), '>' (end), '+' (startEnd), '*' (anywhere), '!' (whole)
     2. Numeric: encoded in high 3 bits of Uint32 token ID
@@ -252,14 +249,20 @@ Override match behavior per token at search time. Priority: query prefix > field
    2. stores tokens as Uint32Array per item
    3. clears tokensMap after use (only global tokens[] remains)
    4. for _deleted or _blocked nothing is added
-3. prepareObjectForSearch(obj, spec?, tokensMap): NumericTokenSortedList | undefined
+3. prepareObjectForSearch(obj, spec?, tokensMap): { sorted, firstN } | undefined
    1. returns undefined if _deleted or _blocked
-   2. if custom extractSearchableStringFn provided, uses simple processing
-   3. otherwise processes fields individually to support per-field match modes
-   4. recursively extracts strings from nested objects/arrays
-4. searchInData / searchInDataReturnIds(query: string, metadata): Id[]
-5. searchInDataReturnObjects<T>(query, metadata, data): Map<Id, T>
-6. searchInDataWithLimit(query, metadata, limit): Id[]
+   2. sorted: full sorted token IDs (with per-field mode bits)
+   3. firstN: first N ordered tokens from spec.keyFields (undefined if keyFields not set)
+   4. if custom extractSearchableStringFn provided, uses simple processing
+   5. otherwise processes fields individually to support per-field match modes
+   6. recursively extracts strings from nested objects/arrays
+4. searchInData / searchInDataReturnIds(query, metadata, opts?): Id[]
+   1. returns RANKED IDs (by score DESC) when metadata.unsorted is populated
+   2. falls back to insertion order when no keyFields declared
+5. searchInDataReturnObjects<T>(query, metadata, data, opts?): Map<Id, T>
+   1. iteration order = relevance DESC
+6. searchInDataWithLimit(query, metadata, limit, opts?): Id[]
+   1. ranks all candidates first, then slices to limit (so top N are truly best)
 7. updateSearchMetadata(metadata, objectId, obj, spec)
    1. updates or deletes metadata for this object
    2. taking into account _blocked and _deleted
@@ -277,7 +280,7 @@ Override match behavior per token at search time. Priority: query prefix > field
    2. uses previousData to determine removals (no metadata.keys() iteration)
    3. skipUpdateCheck option - skip checking for updates (even faster)
    4. best for regular refresh cycles (e.g., every 30s from server)
-11. AugmentedMetadata class
+11. AugmentedMetadata class (implements NumericSearchMetadata)
    1. constructor(baseMetadata, spec?, augmentedDataMap?)
       1. baseMetadata: NumericSearchMetadata - base metadata to wrap (not modified)
       2. spec?: SearchableObjectSpec - controls tokenization of augmented objects
@@ -286,8 +289,10 @@ Override match behavior per token at search time. Priority: query prefix > field
    3. updateAugmentedBatch(map) - batch update (more efficient than multiple single updates)
    4. clearAugmented(id) - remove augmentation for one item
    5. clearAllAugmented() - remove all augmentations
-   6. implements NumericSearchMetadata interface (get, has, size, keys, values, entries, forEach)
-   7. Map mutation methods (set, delete, clear) throw errors - use update methods instead
+   6. fields:
+      - sorted: Map subclass that lazily merges base + augmented tokens on .get()
+      - unsorted: direct reference to base.unsorted (augmentation does NOT participate in positional ranking)
+   7. sorted.set / sorted.delete / sorted.clear throw — augmentation must go through update methods
 12. Linked metadata update functions (incremental updates to linked metadata)
    1. upsertPrimaryItem(linked, item, spec?)
       1. adds or updates a primary item in linked metadata
@@ -315,6 +320,40 @@ Override match behavior per token at search time. Priority: query prefix > field
       5. only processes items whose foreign key matches primaryId
       6. returns { added: number, updated: number, removed: number }
 
+13. findInLinkedArrays(query, linkedMeta, limit?, opts?): LinkedSearchResultWithMeta
+    1. returns RANKED results (by combined primary.unsorted + matched-linked.unsorted score)
+    2. ordering: score DESC, primary insertion-order ASC as deterministic tiebreaker
+    3. matchedIn: 'primary' | 'linked' | 'both' — where positive tokens matched
+    4. tokens that match in primary are excluded from linked matching to prevent FK redundancy
+14. searchAndRank(query, data, metadata, options): ScoredResult<T>[]
+    1. searchAndRank = searchAndRankNumeric (alias for the numeric implementation)
+    2. like searchInDataReturnIdsNumeric but adds:
+       - exactKeysFn boost (+100_000 when raw query equals any extracted value)
+       - locale-aware primary-text tiebreaker
+       - fallback to orderedTokenize(primaryTextFn(item)) when metadata.unsorted is empty
+    3. options: { primaryTextFn, exactKeysFn?, limit?, opts? }
+    4. results sorted by score desc, primaryTextFn(item) localeCompare asc as tiebreaker
+15. orderedTokenize(input: string): string[]
+    1. prepareStringForSearch pipeline minus the trailing .sort()
+    2. required for any positional / order-aware scoring
+    3. do NOT use these tokens for candidate retrieval (sorted lists required)
+16. Scoring primitives (in src/common/numeric/scoring.ts, re-exported from main)
+    1. scoreOne(query, qTokens, itemTokens, exactKeys?): number
+       - feature-rich scoring: +100_000 exact, +50 base, +100 all-matched, +50 first-word,
+         +80 perfect prefix, +30*(matched/span) density, -0.1*Σitemtoken.length penalty
+       - returns 1 when no primary token matched (secondary-field-only signal)
+       - DEFAULT used internally by searchInData* and findInLinkedArrays
+    2. scoreOrderAndPresence(qTokens, itemTokens, weights?): number
+       - additive bonus: weights.ordered (default 30) per in-order match,
+         weights.presence (default 10) per out-of-order presence
+       - each query token contributes at most once
+       - "izdaja recepta":"izdaja recepta"=60, ":izdaja"=30, ":recepta izdaja"=40
+    3. scoreBag(qTokens, itemTokens, perTokenWeight=30): number
+       - ORDER-AGNOSTIC: same score regardless of query/item token order
+       - use for person names, tags, breeds where word order is conventional but not semantic
+       - "Janez Novak" vs "Novak Janez" = SAME score (invariant verified by tests)
+       - each item token consumed by at most one query token (greedy left-to-right)
+
 ## Token Registry Functions
 
 1. tokens: string[] - global array (read-only)
@@ -327,74 +366,36 @@ Override match behavior per token at search time. Priority: query prefix > field
 8. clearTokensMap(tokensMap): void
 9. resetTokenRegistry(): void (for testing only)
 
-## String Implementation (Legacy)
-
-The string-based implementation is available as a separate entry point for backwards compatibility.
-
-**Import from `cry-search/string`:**
-```typescript
-import {
-  createSearchArrayMetadataString,
-  updateSearchMetadataString,
-  findInArray,
-  // ... other string-based functions
-} from 'cry-search/string';
-```
-
-**Available functions with `*String` suffix:**
-- createSearchArrayMetadataString
-- createSearchLinkedMetadataString
-- updateSearchMetadataString
-- updateSearchMetadataBatchString
-- syncSearchArrayMetadataString
-- syncSearchArrayMetadataStringWithPrevious
-- upsertPrimaryItemString
-- removePrimaryItemString
-- upsertLinkedItemString
-- removeLinkedItemString
-- syncLinkedItemsForPrimaryString
-- findInArray (uses string metadata)
-- findInArrayReturnDataAndMeta
-- findInLinkedArrays
-
-**Bundle sizes:**
-- Main bundle (`cry-search`): ~49 KB - includes only numeric implementation
-- String bundle (`cry-search/string`): ~26 KB - includes only string implementation
-
 ## File Structure
 
 ```
 src/
 ├── common/
-│   ├── numeric/           # Default numeric implementation
+│   ├── numeric/                       # Numeric implementation (only impl)
 │   │   ├── index.ts
 │   │   ├── createSearchArrayMetadata.ts
 │   │   ├── createSearchLinkedMetadata.ts
 │   │   ├── updateSearchMetadata.ts
 │   │   ├── updateSearchLinkedMetadata.ts
-│   │   ├── searchInData.ts
+│   │   ├── searchInData.ts            # Ranked search via metadata.unsorted
+│   │   ├── findInLinkedArrays.ts      # Ranked linked search
+│   │   ├── searchAndRank.ts           # Wrapper w/ exactKeysFn boost + fallback
+│   │   ├── scoring.ts                 # scoreOne, scoreOrderAndPresence, scoreBag
 │   │   ├── syncSearchArrayMetadata.ts
 │   │   └── AugmentedMetadata.ts
-│   ├── string/            # Legacy string implementation
-│   │   ├── index.ts
-│   │   ├── createSearchArrayMetadata.ts
-│   │   ├── createSearchLinkedMetadata.ts
-│   │   └── updateSearchMetadata.ts
-│   ├── findInArray.ts           # String-based search
-│   ├── findInLinkedArrays.ts    # String-based linked search
-│   ├── syncSearchArrayMetadata.ts
-│   ├── updateSearchLinkedMetadata.ts
-│   └── SearchUniverse.ts        # Multi-collection manager (uses numeric)
+│   └── SearchUniverse.ts              # Multi-collection manager
 ├── utils/
-│   ├── tokenRegistry.ts         # Global tokens array
-│   ├── prepareStringForSearch.ts
-│   ├── prepareStringForSearchNumeric.ts
-│   ├── matchToken.ts            # String-based matching
-│   ├── matchTokenNumeric.ts     # Numeric matching
-│   └── ...
+│   ├── tokenRegistry.ts               # Global tokens array
+│   ├── prepareStringForSearch.ts      # Shared query tokenization (returns sorted strings)
+│   ├── prepareStringForSearchNumeric.ts  # For metadata build (returns Uint32Array of token IDs)
+│   ├── matchTokenNumeric.ts
+│   ├── normalizeDates.ts
+│   ├── preprocessString.ts
+│   ├── sanitiseString.ts
+│   └── tokenize.ts
 ├── types/
 │   └── index.ts
-└── index.ts                     # Main exports (numeric as default)
+└── index.ts                           # Public API
 ```
 
 ## Coding rules

package/README.md

@@ -5,12 +5,13 @@ A fast, memory-efficient search library for large datasets with support for toke
 ## Features
 
 - **Fast search** - Pre-built metadata enables instant queries on large datasets
-- **Memory efficient** - Numeric token storage uses ~45% less memory than string-based approaches
+- **Memory efficient** - Numeric token storage (`Uint32Array`) keeps metadata compact
 - **Updatable** - Add, update, or remove items without rebuilding metadata
 - **Augmented metadata** - Add local/temporary searchable data without modifying global metadata
 - **Flexible matching** - Per-field match modes and query prefixes for prefix, suffix, anywhere, exact, and negation
 - **Search prefixes** - use `-word`, `--word`, or `~word` to exclude, `..word` for suffix, `=word` for exact match
 - **Automatic normalization** - Handles diacritics, case, dates, and number formatting
+- **Ranked search** - `searchAndRank` returns results scored by positional relevance (token order, contiguity, perfect-prefix, length)
 - **[Universal search](./UNIVERSE.md)** - Search and update across linked collections (e.g., customers with their pets)
 
 ## Table of Contents
@@ -24,13 +25,13 @@ A fast, memory-efficient search library for large datasets with support for toke
     - [SearchUniverse with Linked Collections](#searchuniverse-with-linked-collections)
     - [Updating Existing Data](#updating-existing-data)
     - [Augmented Metadata - Local Search Context](#augmented-metadata---local-search-context)
+    - [Ranked Search with Positional Scoring](#ranked-search-with-positional-scoring)
   - [Architecture](#architecture)
     - [Text Processing Pipeline](#text-processing-pipeline)
     - [Metadata Building](#metadata-building)
     - [Searching](#searching)
     - [Memory Optimization](#memory-optimization)
   - [Query Prefixes](#query-prefixes)
-  - [Legacy String Implementation](#legacy-string-implementation)
   - [Specification](#specification)
   - [License](#license)
 
@@ -278,6 +279,95 @@ augmented.clearAllAugmented();  // Reset to global state
 - Documents with review status or approval state
 - Items with computed scores or temporary categories
 
+### Ranked Search with Positional Scoring
+
+**Since 2.0.0** all search functions return ranked results by default. `searchInData*` and `findInLinkedArrays` automatically score candidates and sort by relevance DESC. To activate positional scoring, declare `spec.keyFields` — those fields are tokenized in source order at build time and stored in `metadata.unsorted` for fast query-time scoring.
+
+`searchAndRank` is a thin wrapper that additionally supports:
+- **`exactKeysFn`** — +100 000 boost when raw query equals a designated field value (e.g. barcode)
+- **`primaryTextFn`** — fallback when `keyFields` not declared, plus locale-aware tiebreaker sorting
+
+```typescript
+import {
+  arrayToSearchableData,
+  createSearchArrayMetadata,
+  searchAndRank,
+  type SearchableObject,
+} from "cry-search";
+
+interface Product extends SearchableObject {
+  _id: string;
+  name: string;
+  barcodes?: string[];
+}
+
+const products: Product[] = [
+  { _id: "1", name: "POTROŠNI MATERIAL" },
+  { _id: "2", name: "POTROŠNI MATERIAL-SUCRALAN 1ML" },
+  { _id: "3", name: "KONTROLNI PREGLED" },
+  { _id: "4", name: "DERMATOLOŠKI KONTROLNI PREGLED" },
+];
+
+const data = arrayToSearchableData(products);
+// Declare keyFields so positional ranking has pre-stored ordered tokens (faster).
+const meta = createSearchArrayMetadata(data, {
+  keyFields: ['name'],
+  firstTokensN: 8,
+});
+
+// Returns items sorted by relevance, NOT discovery order:
+const results = searchAndRank("pot mat", data, meta, {
+  primaryTextFn: (p) => p.name,
+  exactKeysFn: (p) => p.barcodes,  // optional: exact-key boost (e.g. barcode)
+  limit: 10,
+});
+
+// results[0].item.name === "POTROŠNI MATERIAL"        (perfect prefix → +80)
+// results[1].item.name === "POTROŠNI MATERIAL-SUCRALAN 1ML"  (longer naziv)
+
+// Each result: { id: Id, item: T, score: number }
+```
+
+**Performance optimization via `keyFields`**: When the spec declares `keyFields`, the first N tokens (`firstTokensN`, default 8) of those fields are stored in `metadata.unsorted` at build time. `searchAndRank` reads those at query time instead of re-tokenizing — typically **10-50× faster scoring**. Without `keyFields`, `searchAndRank` falls back to tokenizing `primaryTextFn(item)` per candidate at query time (same correctness, slower).
+
+**Scoring formula** (`scoreOne` is exported as a primitive — useful for composing your own scorer):
+
+| Component | Value | When |
+|---|---|---|
+| Exact-key match | **+100 000** | `query === any(exactKeys)` (e.g. barcode equality) |
+| Base | +50 | Any in-order subsequence-prefix match of query tokens in itemTokens |
+| All tokens matched | +100 | Every query token had a target |
+| First-word match | +50 | First match at position 0 |
+| Perfect prefix | +80 | Matches at positions `[0,1,…,k-1]` (contiguous from start) |
+| Density | +30 × (matched / span) | Rewards contiguous matches over gapped |
+| Length penalty | -0.1 × `Σ itemToken.length` | Weak tiebreaker — shorter wins |
+| Secondary-field-only | 1 | Item matched (via retrieval) but no query token appears in itemTokens |
+
+**Lower-level scoring primitives** for composing custom rankers:
+
+| Function | Use case | Order-sensitive? |
+|---|---|---|
+| `scoreOne(query, qTokens, itemTokens, exactKeys?)` | Full-featured (perfect prefix, density, length penalty, exact-key boost) — default used by `searchInData*` and `searchAndRank` | yes |
+| `scoreOrderAndPresence(qTokens, itemTokens, weights?)` | Simpler additive bonus: ordered match (+30) + presence (+10) per token. Tunable via weights | yes |
+| `scoreBag(qTokens, itemTokens, perTokenWeight?)` | Order-agnostic: each query token present in itemTokens contributes equally. Use for **person names**, tags, pet breeds where word order is conventional but not semantic — `scoreBag(["janez","novak"], item)` === `scoreBag(["novak","janez"], item)` | no |
+| `orderedTokenize(input)` | Tokenize without alphabetic sort (required for any positional scoring). Output is NOT compatible with `searchInData*` retrieval (which requires sorted lists from `prepareStringForSearch`) | n/a |
+
+```typescript
+import { scoreBag, scoreOne, scoreOrderAndPresence, orderedTokenize, tokens, findTokenId } from 'cry-search';
+
+// Combine signals per field in a custom ranker:
+const qTokens = orderedTokenize(query);
+
+// From metadata.unsorted (numeric IDs → strings via global tokens):
+const unsortedIds = meta.unsorted.get(itemId);
+const itemTokens = unsortedIds ? Array.from(unsortedIds, (id) => tokens[id]!) : [];
+
+const baseScore = scoreOne(query, qTokens, itemTokens, item.barcodes);
+const nameBoost = scoreBag(qTokens, nameTokens);                 // order-agnostic for "Janez Novak"
+const titleBoost = scoreOrderAndPresence(qTokens, titleTokens);  // order-sensitive for "RTG slika"
+const total = baseScore + nameBoost + titleBoost;
+```
+
 ## Architecture
 
 cry-search uses a two-phase approach: **build** metadata once, then **search** instantly.
@@ -307,7 +397,7 @@ Query tokens are matched against metadata using binary search. All query tokens
 
 ### Memory Optimization
 
-The numeric implementation stores tokens as `Uint32Array` indices instead of string arrays, reducing memory usage by ~45% compared to the string-based implementation.
+Tokens are stored as `Uint32Array` indices into a global `tokens: string[]` registry. Each metadata entry is a compact sorted Uint32Array per item — tokens themselves are interned once globally.
 
 ## Query Prefixes
 
@@ -383,20 +473,6 @@ This enables matching by either part:
 
 Priority: query prefix > field match mode > global SearchOpts default.
 
-## Legacy String Implementation
-
-A string-based implementation is available for backwards compatibility. Import from the `/string` subpath:
-
-```typescript
-import {
-  createSearchArrayMetadataString,
-  updateSearchMetadataString,
-  findInArray,  // String-based search function
-} from 'cry-search/string';
-```
-
-The string implementation is excluded from the main bundle. The default import (`cry-search`) only includes the modern numeric implementation, keeping bundle sizes smaller.
-
 ## Specification
 
 See [CLAUDE.md](./CLAUDE.md) for the complete technical specification including:

package/UNIVERSE.md

@@ -627,8 +627,10 @@ SearchUniverse provides methods to update data and metadata after initial load.
 │        universe.syncCollection(collectionName, newItems, options?)          │
 ├─────────────────────────────────────────────────────────────────────────────┤
 │                                                                             │
-│  Diffs a fresh full snapshot against the current collection state.          │
-│  Use for periodic refresh cycles when you don't have a delta.               │
+│  ⚠️  newItems MUST be the FULL snapshot, not a partial delta.               │
+│      Anything currently in the collection whose _id is missing from         │
+│      newItems will be REMOVED (data + metadata + linked index).             │
+│      Passing a partial list will wipe out everything else.                  │
 │                                                                             │
 │  newItems = [                                                               │
 │    { _id: "s1", name: "Krajnik", address: "Ljubljana" },  ← unchanged       │
@@ -657,6 +659,20 @@ SearchUniverse provides methods to update data and metadata after initial load.
 └─────────────────────────────────────────────────────────────────────────────┘
 ```
 
+#### When to use which update method
+
+| Scenario                                          | Method                                                       |
+|---------------------------------------------------|--------------------------------------------------------------|
+| Full snapshot from server (periodic refresh)      | `syncCollection(name, allItems)`                             |
+| Known delta — add/update some items               | `updateSearchMetadataBatch(name, changedItems)`              |
+| Known IDs to remove                               | `removeFromSearchMetadataBatch(name, ids)`                   |
+| Mark item as soft-deleted but keep in data        | `updateSearchMetadata(name, id, { ...item, _deleted: ... })` |
+| Initial load / full reset                         | `loadCollection(name, items)`                                |
+
+**Rule of thumb:** if you don't have the complete current state of the
+collection, do NOT use `syncCollection` — use the batch methods instead.
+They only touch the items you pass in and leave the rest alone.
+
 ### Clear Universe
 
 ```
@@ -778,36 +794,29 @@ For direct metadata manipulation without SearchUniverse:
 │                                                                             │
 │  ──────────────────────────────────────────────────────────────────────     │
 │                                                                             │
-│  STRING (Legacy) - import with *String suffix:                              │
-│                                                                             │
-│  createSearchArrayMetadataString(data, spec)                                │
-│  updateSearchMetadataString(metadata, id, obj, spec)                        │
-│  updateSearchMetadataBatchString(metadata, batch, spec)                     │
-│  removeFromSearchMetadataString(metadata, id)                               │
-│  findInArray(query, data, metadata)                                         │
-│                                                                             │
-│  ──────────────────────────────────────────────────────────────────────     │
-│                                                                             │
-│  SYNC (String-based):                                                       │
+│  SYNC:                                                                      │
 │                                                                             │
-│  syncSearchArrayMetadata(data, metadata, spec)                              │
-│    Syncs metadata with current data state                                   │
+│  syncSearchArrayMetadata(metadata, newData, spec)                           │
+│    Syncs metadata with new full data snapshot                               │
 │    Returns: { added, updated, removed, addedIds, updatedIds, removedIds }   │
 │                                                                             │
+│  syncSearchArrayMetadataWithPrevious(metadata, prev, newData, opts?, spec)  │
+│    Same but uses previous data to skip metadata.keys() iteration            │
+│                                                                             │
 │  ──────────────────────────────────────────────────────────────────────     │
 │                                                                             │
-│  For linked metadata (String-based):                                        │
+│  Linked metadata (incremental):                                             │
 │                                                                             │
-│  upsertPrimaryItem(linked, id, item, spec)                                  │
+│  upsertPrimaryItem(linked, item, spec)                                      │
 │    Adds/updates primary item in linked structure                            │
 │                                                                             │
-│  upsertLinkedItem(linked, id, item, fkGetter, spec)                         │
+│  upsertLinkedItem(linked, item, fkGetter, spec)                             │
 │    Adds/updates linked item, handles FK changes                             │
 │                                                                             │
-│  removePrimaryItem(linked, id)                                              │
+│  removePrimaryItem(linked, primaryId)                                       │
 │    Removes primary item                                                     │
 │                                                                             │
-│  removeLinkedItem(linked, id, primaryId)                                    │
+│  removeLinkedItem(linked, linkedId, fkGetter)                               │
 │    Removes linked item from structure and index                             │
 │                                                                             │
 │  syncLinkedItemsForPrimary(linked, primaryId, items, fkGetter, spec)        │

package/dist/common/SearchUniverse.d.ts

@@ -155,30 +155,45 @@ export declare class SearchUniverse {
      */
     removeFromSearchMetadataBatch(collectionName: string, ids: Id[]): number;
     /**
-     * Syncs a collection against a fresh snapshot of items.
-     *
-     * Compares the provided `newItems` to the current collection state and applies
-     * the diff: items not in `newItems` are removed, new items are added, and existing
-     * items are updated when their tokens change. Items marked `_deleted` or `_blocked`
-     * are removed from metadata (and from `primaryToLinked` for linked collections),
-     * but remain in `collection.data` for reference — same convention as
-     * {@link updateSearchMetadata}.
-     *
-     * Use this for full refresh cycles (e.g. periodic snapshot from server) when you
-     * don't have a delta. For incremental delta updates use
-     * {@link updateSearchMetadataBatch} + {@link removeFromSearchMetadataBatch}.
+     * Syncs a collection against a **full snapshot** of items.
+     *
+     * **IMPORTANT:** `newItems` MUST contain the entire desired state of the
+     * collection, not a partial delta. Any item currently in the collection
+     * whose `_id` is missing from `newItems` will be **completely removed**
+     * (data + metadata + linked indexes). Passing a partial list will wipe
+     * out everything else.
+     *
+     * Compares `newItems` to the current collection state and applies the diff:
+     * - items absent from `newItems` → removed
+     * - new IDs in `newItems` → added
+     * - existing IDs with changed tokens → updated
+     * - items marked `_deleted` or `_blocked` → removed from metadata
+     *   (kept in `collection.data` for reference — same convention as
+     *   {@link updateSearchMetadata})
+     *
+     * **When to use which method:**
+     * - Full snapshot from server (periodic refresh): `syncCollection`
+     * - Known delta of changes: {@link updateSearchMetadataBatch}
+     * - Known IDs to delete: {@link removeFromSearchMetadataBatch}
+     *
+     * For partial delta updates, prefer the batch methods — they leave untouched
+     * items unaffected.
      *
      * @typeParam T - Type of items in the collection
      * @param collectionName - Name of the collection
-     * @param newItems - Full new dataset (snapshot)
+     * @param newItems - **Full** new dataset (complete snapshot, not a delta)
      * @param options - Optional sync options (e.g. `skipUpdateCheck` to avoid re-tokenizing existing items)
      * @returns SyncResult with counts and IDs of added/updated/removed items
      *
      * @example
      * ```typescript
+     * // Full refresh from server — pass ALL current products
      * const fresh = await fetchAllProducts();
      * const result = universe.syncCollection('products', fresh);
      * // { added: 3, updated: 12, removed: 1, addedIds: [...], ... }
+     *
+     * // ⚠️ DO NOT do this — would delete all other products:
+     * // universe.syncCollection('products', [oneChangedProduct]);
      * ```
      */
     syncCollection<T extends SearchableObject>(collectionName: string, newItems: T[], options?: SyncOptions): SyncResult;

package/dist/common/SearchUniverse.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"SearchUniverse.d.ts","sourceRoot":"","sources":["../../src/common/SearchUniverse.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,EAAE,EACF,qBAAqB,EAErB,cAAc,EACd,gBAAgB,EAEhB,mBAAmB,EACnB,gBAAgB,EAChB,YAAY,EACZ,iBAAiB,EACjB,UAAU,EACV,WAAW,EACZ,MAAM,UAAU,CAAC;AAWlB,YAAY,EAAE,mBAAmB,EAAE,gBAAgB,EAAE,YAAY,EAAE,iBAAiB,EAAE,MAAM,UAAU,CAAC;AA0BvG;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4BG;AACH,qBAAa,cAAc;IACzB,OAAO,CAAC,WAAW,CAAiC;IAEpD;;;;;OAKG;IACH,kBAAkB,CAAC,CAAC,SAAS,gBAAgB,EAAE,CAAC,GAAG,OAAO,EACxD,IAAI,EAAE,MAAM,EACZ,MAAM,GAAE,gBAAgB,CAAC,CAAC,EAAE,CAAC,CAAM,GAClC,IAAI;IAmBP;;;;;OAKG;IACH,aAAa,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO;IAIpC;;;;OAIG;IACH,kBAAkB,IAAI,MAAM,EAAE;IAI9B;;;;;;OAMG;IACH,cAAc,CAAC,CAAC,SAAS,gBAAgB,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,IAAI;IAsC1E;;;;;;;OAOG;IACH,OAAO,CAAC,CAAC,SAAS,gBAAgB,EAAE,cAAc,EAAE,MAAM,EAAE,EAAE,EAAE,EAAE,GAAG,CAAC,GAAG,SAAS;IAKlF;;;;;;OAMG;IACH,QAAQ,CAAC,CAAC,SAAS,gBAAgB,EAAE,cAAc,EAAE,MAAM,GAAG,CAAC,EAAE;IAKjE;;;;;OAKG;IACH,iBAAiB,CAAC,cAAc,EAAE,MAAM,GAAG,MAAM;IAKjD;;;;;OAKG;IACH,WAAW,CAAC,cAAc,EAAE,MAAM,GAAG,qBAAqB;IAK1D;;;;;;OAMG;IACH,OAAO,CAAC,CAAC,SAAS,gBAAgB,EAAE,cAAc,EAAE,MAAM,GAAG,cAAc,CAAC,CAAC,CAAC;IAK9E;;;;;;;OAOG;IACH,YAAY,CAAC,oBAAoB,EAAE,MAAM,EAAE,SAAS,EAAE,EAAE,GAAG,EAAE,EAAE;IAQ/D;;;;;;;OAOG;IACH,cAAc,CAAC,CAAC,SAAS,gBAAgB,EACvC,oBAAoB,EAAE,MAAM,EAC5B,SAAS,EAAE,EAAE,GACZ,CAAC,EAAE;IAMN;;;;;;;;;;;;OAYG;IACH,oBAAoB,CAAC,CAAC,SAAS,gBAAgB,EAC7C,cAAc,EAAE,MAAM,EACtB,EAAE,EAAE,EAAE,EACN,IAAI,EAAE,CAAC,GACN,YAAY;IAoFf;;;;;;OAMG;IACH,yBAAyB,CAAC,CAAC,SAAS,gBAAgB,EAClD,cAAc,EAAE,MAAM,EACtB,KAAK,EAAE,CAAC,EAAE,GACT,iBAAiB;IA0BpB;;;;;;OAMG;IACH,wBAAwB,CAAC,cAAc,EAAE,MAAM,EAAE,EAAE,EAAE,EAAE,GAAG,OAAO;IAmCjE;;;;;;OAMG;IACH,6BAA6B,CAAC,cAAc,EAAE,MAAM,EAAE,GAAG,EAAE,EAAE,EAAE,GAAG,MAAM;IAcxE;;;;;;;;;;;;;;;;;;;;;;;;;;OA0BG;IACH,cAAc,CAAC,CAAC,SAAS,gBAAgB,EACvC,cAAc,EAAE,MAAM,EACtB,QAAQ,EAAE,CAAC,EAAE,EACb,OAAO,CAAC,EAAE,WAAW,GACpB,UAAU;IA8Hb;;;;;;;OAOG;IACH,OAAO,CAAC,oBAAoB;IAQ5B;;;;;;;;;;;;;;;;;;;;;;;;;;;OA2BG;IACH,iBAAiB,CACf,KAAK,EAAE,MAAM,EACb,KAAK,GAAE,MAAW,EAClB,IAAI,CAAC,EAAE,mBAAmB,GACzB,MAAM,CAAC,MAAM,EAAE,gBAAgB,EAAE,GAAG,KAAK,CAAC;QAAE,MAAM,EAAE,gBAAgB,EAAE,CAAC;QAAC,OAAO,EAAE,gBAAgB,CAAC;QAAC,SAAS,EAAE,SAAS,GAAG,QAAQ,GAAG,MAAM,CAAA;KAAE,CAAC,CAAC;IA6FlJ;;;;;;;;OAQG;IACH,KAAK,IAAI,IAAI;CAUd"}
+{"version":3,"file":"SearchUniverse.d.ts","sourceRoot":"","sources":["../../src/common/SearchUniverse.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,EAAE,EACF,qBAAqB,EAErB,cAAc,EACd,gBAAgB,EAEhB,mBAAmB,EACnB,gBAAgB,EAChB,YAAY,EACZ,iBAAiB,EACjB,UAAU,EACV,WAAW,EACZ,MAAM,UAAU,CAAC;AAYlB,YAAY,EAAE,mBAAmB,EAAE,gBAAgB,EAAE,YAAY,EAAE,iBAAiB,EAAE,MAAM,UAAU,CAAC;AAiCvG;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4BG;AACH,qBAAa,cAAc;IACzB,OAAO,CAAC,WAAW,CAAiC;IAEpD;;;;;OAKG;IACH,kBAAkB,CAAC,CAAC,SAAS,gBAAgB,EAAE,CAAC,GAAG,OAAO,EACxD,IAAI,EAAE,MAAM,EACZ,MAAM,GAAE,gBAAgB,CAAC,CAAC,EAAE,CAAC,CAAM,GAClC,IAAI;IAmBP;;;;;OAKG;IACH,aAAa,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO;IAIpC;;;;OAIG;IACH,kBAAkB,IAAI,MAAM,EAAE;IAI9B;;;;;;OAMG;IACH,cAAc,CAAC,CAAC,SAAS,gBAAgB,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,IAAI;IA0C1E;;;;;;;OAOG;IACH,OAAO,CAAC,CAAC,SAAS,gBAAgB,EAAE,cAAc,EAAE,MAAM,EAAE,EAAE,EAAE,EAAE,GAAG,CAAC,GAAG,SAAS;IAKlF;;;;;;OAMG;IACH,QAAQ,CAAC,CAAC,SAAS,gBAAgB,EAAE,cAAc,EAAE,MAAM,GAAG,CAAC,EAAE;IAKjE;;;;;OAKG;IACH,iBAAiB,CAAC,cAAc,EAAE,MAAM,GAAG,MAAM;IAKjD;;;;;OAKG;IACH,WAAW,CAAC,cAAc,EAAE,MAAM,GAAG,qBAAqB;IAK1D;;;;;;OAMG;IACH,OAAO,CAAC,CAAC,SAAS,gBAAgB,EAAE,cAAc,EAAE,MAAM,GAAG,cAAc,CAAC,CAAC,CAAC;IAK9E;;;;;;;OAOG;IACH,YAAY,CAAC,oBAAoB,EAAE,MAAM,EAAE,SAAS,EAAE,EAAE,GAAG,EAAE,EAAE;IAQ/D;;;;;;;OAOG;IACH,cAAc,CAAC,CAAC,SAAS,gBAAgB,EACvC,oBAAoB,EAAE,MAAM,EAC5B,SAAS,EAAE,EAAE,GACZ,CAAC,EAAE;IAMN;;;;;;;;;;;;OAYG;IACH,oBAAoB,CAAC,CAAC,SAAS,gBAAgB,EAC7C,cAAc,EAAE,MAAM,EACtB,EAAE,EAAE,EAAE,EACN,IAAI,EAAE,CAAC,GACN,YAAY;IA0Ff;;;;;;OAMG;IACH,yBAAyB,CAAC,CAAC,SAAS,gBAAgB,EAClD,cAAc,EAAE,MAAM,EACtB,KAAK,EAAE,CAAC,EAAE,GACT,iBAAiB;IA0BpB;;;;;;OAMG;IACH,wBAAwB,CAAC,cAAc,EAAE,MAAM,EAAE,EAAE,EAAE,EAAE,GAAG,OAAO;IAoCjE;;;;;;OAMG;IACH,6BAA6B,CAAC,cAAc,EAAE,MAAM,EAAE,GAAG,EAAE,EAAE,EAAE,GAAG,MAAM;IAcxE;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OAyCG;IACH,cAAc,CAAC,CAAC,SAAS,gBAAgB,EACvC,cAAc,EAAE,MAAM,EACtB,QAAQ,EAAE,CAAC,EAAE,EACb,OAAO,CAAC,EAAE,WAAW,GACpB,UAAU;IA4Ib;;;;;;;OAOG;IACH,OAAO,CAAC,oBAAoB;IAQ5B;;;;;;;;;;;;;;;;;;;;;;;;;;;OA2BG;IACH,iBAAiB,CACf,KAAK,EAAE,MAAM,EACb,KAAK,GAAE,MAAW,EAClB,IAAI,CAAC,EAAE,mBAAmB,GACzB,MAAM,CAAC,MAAM,EAAE,gBAAgB,EAAE,GAAG,KAAK,CAAC;QAAE,MAAM,EAAE,gBAAgB,EAAE,CAAC;QAAC,OAAO,EAAE,gBAAgB,CAAC;QAAC,SAAS,EAAE,SAAS,GAAG,QAAQ,GAAG,MAAM,CAAA;KAAE,CAAC,CAAC;IA6FlJ;;;;;;;;OAQG;IACH,KAAK,IAAI,IAAI;CAWd"}