cry-search 1.0.5 → 1.0.6

package/UNIVERSE.md

@@ -620,6 +620,43 @@ SearchUniverse provides methods to update data and metadata after initial load.
 └─────────────────────────────────────────────────────────────────────────────┘
 ```
 
+### Sync Collection (Snapshot Diff)
+
+```
+┌─────────────────────────────────────────────────────────────────────────────┐
+│        universe.syncCollection(collectionName, newItems, options?)          │
+├─────────────────────────────────────────────────────────────────────────────┤
+│                                                                             │
+│  Diffs a fresh full snapshot against the current collection state.          │
+│  Use for periodic refresh cycles when you don't have a delta.               │
+│                                                                             │
+│  newItems = [                                                               │
+│    { _id: "s1", name: "Krajnik", address: "Ljubljana" },  ← unchanged       │
+│    { _id: "s2", name: "Novak",   address: "Celje" },      ← updated         │
+│    { _id: "s3", name: "Horvat",  address: "Koper" },      ← added           │
+│    // s4 from previous state is missing → removed                           │
+│  ]                                                                          │
+│                                                                             │
+│  Behavior:                                                                  │
+│    1. Items in collection but absent from newItems → removed                │
+│       (data + metadata + primaryToLinked all cleaned)                       │
+│    2. Items in both → tokens compared; metadata.set only if changed         │
+│    3. New items → tokenized, added to metadata + primaryToLinked            │
+│    4. _deleted/_blocked items in newItems → removed from metadata           │
+│       (kept in data, same convention as updateSearchMetadata)               │
+│                                                                             │
+│  Options:                                                                   │
+│    skipUpdateCheck: true → don't re-tokenize existing items (faster)        │
+│                            data refs still get updated                      │
+│                                                                             │
+│  Returns: SyncResult                                                        │
+│    { added, updated, removed, addedIds[], updatedIds[], removedIds[] }      │
+│                                                                             │
+│  Single shared tokensMap is used across the whole snapshot for efficiency.  │
+│                                                                             │
+└─────────────────────────────────────────────────────────────────────────────┘
+```
+
 ### Clear Universe
 
 ```

package/dist/common/SearchUniverse.d.ts

@@ -1,4 +1,4 @@
-import type { Id, NumericSearchMetadata, SearchableData, SearchableObject, UniversalSearchOpts, CollectionConfig, UpdateResult, BatchUpdateResult } from '../types';
+import type { Id, NumericSearchMetadata, SearchableData, SearchableObject, UniversalSearchOpts, CollectionConfig, UpdateResult, BatchUpdateResult, SyncResult, SyncOptions } from '../types';
 export type { UniversalSearchOpts, CollectionConfig, UpdateResult, BatchUpdateResult } from '../types';
 /**
  * SearchUniverse manages multiple collections with their metadata and relationships.
@@ -154,6 +154,34 @@ export declare class SearchUniverse {
      * @returns Number of items actually removed
      */
     removeFromSearchMetadataBatch(collectionName: string, ids: Id[]): number;
+    /**
+     * Syncs a collection against a fresh snapshot of items.
+     *
+     * Compares the provided `newItems` to the current collection state and applies
+     * the diff: items not in `newItems` are removed, new items are added, and existing
+     * items are updated when their tokens change. Items marked `_deleted` or `_blocked`
+     * are removed from metadata (and from `primaryToLinked` for linked collections),
+     * but remain in `collection.data` for reference — same convention as
+     * {@link updateSearchMetadata}.
+     *
+     * Use this for full refresh cycles (e.g. periodic snapshot from server) when you
+     * don't have a delta. For incremental delta updates use
+     * {@link updateSearchMetadataBatch} + {@link removeFromSearchMetadataBatch}.
+     *
+     * @typeParam T - Type of items in the collection
+     * @param collectionName - Name of the collection
+     * @param newItems - Full new dataset (snapshot)
+     * @param options - Optional sync options (e.g. `skipUpdateCheck` to avoid re-tokenizing existing items)
+     * @returns SyncResult with counts and IDs of added/updated/removed items
+     *
+     * @example
+     * ```typescript
+     * const fresh = await fetchAllProducts();
+     * const result = universe.syncCollection('products', fresh);
+     * // { added: 3, updated: 12, removed: 1, addedIds: [...], ... }
+     * ```
+     */
+    syncCollection<T extends SearchableObject>(collectionName: string, newItems: T[], options?: SyncOptions): SyncResult;
     /**
      * Gets collection or throws an error if not found.
      *

package/dist/common/SearchUniverse.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"SearchUniverse.d.ts","sourceRoot":"","sources":["../../src/common/SearchUniverse.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,EAAE,EACF,qBAAqB,EACrB,cAAc,EACd,gBAAgB,EAEhB,mBAAmB,EACnB,gBAAgB,EAChB,YAAY,EACZ,iBAAiB,EAClB,MAAM,UAAU,CAAC;AAWlB,YAAY,EAAE,mBAAmB,EAAE,gBAAgB,EAAE,YAAY,EAAE,iBAAiB,EAAE,MAAM,UAAU,CAAC;AAcvG;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4BG;AACH,qBAAa,cAAc;IACzB,OAAO,CAAC,WAAW,CAAiC;IAEpD;;;;;OAKG;IACH,kBAAkB,CAAC,CAAC,SAAS,gBAAgB,EAAE,CAAC,GAAG,OAAO,EACxD,IAAI,EAAE,MAAM,EACZ,MAAM,GAAE,gBAAgB,CAAC,CAAC,EAAE,CAAC,CAAM,GAClC,IAAI;IAmBP;;;;;OAKG;IACH,aAAa,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO;IAIpC;;;;OAIG;IACH,kBAAkB,IAAI,MAAM,EAAE;IAI9B;;;;;;OAMG;IACH,cAAc,CAAC,CAAC,SAAS,gBAAgB,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,IAAI;IAsC1E;;;;;;;OAOG;IACH,OAAO,CAAC,CAAC,SAAS,gBAAgB,EAAE,cAAc,EAAE,MAAM,EAAE,EAAE,EAAE,EAAE,GAAG,CAAC,GAAG,SAAS;IAKlF;;;;;;OAMG;IACH,QAAQ,CAAC,CAAC,SAAS,gBAAgB,EAAE,cAAc,EAAE,MAAM,GAAG,CAAC,EAAE;IAKjE;;;;;OAKG;IACH,iBAAiB,CAAC,cAAc,EAAE,MAAM,GAAG,MAAM;IAKjD;;;;;OAKG;IACH,WAAW,CAAC,cAAc,EAAE,MAAM,GAAG,qBAAqB;IAK1D;;;;;;OAMG;IACH,OAAO,CAAC,CAAC,SAAS,gBAAgB,EAAE,cAAc,EAAE,MAAM,GAAG,cAAc,CAAC,CAAC,CAAC;IAK9E;;;;;;;OAOG;IACH,YAAY,CAAC,oBAAoB,EAAE,MAAM,EAAE,SAAS,EAAE,EAAE,GAAG,EAAE,EAAE;IAQ/D;;;;;;;OAOG;IACH,cAAc,CAAC,CAAC,SAAS,gBAAgB,EACvC,oBAAoB,EAAE,MAAM,EAC5B,SAAS,EAAE,EAAE,GACZ,CAAC,EAAE;IAMN;;;;;;;;;;;;OAYG;IACH,oBAAoB,CAAC,CAAC,SAAS,gBAAgB,EAC7C,cAAc,EAAE,MAAM,EACtB,EAAE,EAAE,EAAE,EACN,IAAI,EAAE,CAAC,GACN,YAAY;IAoFf;;;;;;OAMG;IACH,yBAAyB,CAAC,CAAC,SAAS,gBAAgB,EAClD,cAAc,EAAE,MAAM,EACtB,KAAK,EAAE,CAAC,EAAE,GACT,iBAAiB;IA0BpB;;;;;;OAMG;IACH,wBAAwB,CAAC,cAAc,EAAE,MAAM,EAAE,EAAE,EAAE,EAAE,GAAG,OAAO;IAmCjE;;;;;;OAMG;IACH,6BAA6B,CAAC,cAAc,EAAE,MAAM,EAAE,GAAG,EAAE,EAAE,EAAE,GAAG,MAAM;IAcxE;;;;;;;OAOG;IACH,OAAO,CAAC,oBAAoB;IAQ5B;;;;;;;;;;;;;;;;;;;;;;;;;;;OA2BG;IACH,iBAAiB,CACf,KAAK,EAAE,MAAM,EACb,KAAK,GAAE,MAAW,EAClB,IAAI,CAAC,EAAE,mBAAmB,GACzB,MAAM,CAAC,MAAM,EAAE,gBAAgB,EAAE,GAAG,KAAK,CAAC;QAAE,MAAM,EAAE,gBAAgB,EAAE,CAAC;QAAC,OAAO,EAAE,gBAAgB,CAAC;QAAC,SAAS,EAAE,SAAS,GAAG,QAAQ,GAAG,MAAM,CAAA;KAAE,CAAC,CAAC;IA6FlJ;;;;;;;;OAQG;IACH,KAAK,IAAI,IAAI;CAUd"}
+{"version":3,"file":"SearchUniverse.d.ts","sourceRoot":"","sources":["../../src/common/SearchUniverse.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,EAAE,EACF,qBAAqB,EAErB,cAAc,EACd,gBAAgB,EAEhB,mBAAmB,EACnB,gBAAgB,EAChB,YAAY,EACZ,iBAAiB,EACjB,UAAU,EACV,WAAW,EACZ,MAAM,UAAU,CAAC;AAWlB,YAAY,EAAE,mBAAmB,EAAE,gBAAgB,EAAE,YAAY,EAAE,iBAAiB,EAAE,MAAM,UAAU,CAAC;AA0BvG;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4BG;AACH,qBAAa,cAAc;IACzB,OAAO,CAAC,WAAW,CAAiC;IAEpD;;;;;OAKG;IACH,kBAAkB,CAAC,CAAC,SAAS,gBAAgB,EAAE,CAAC,GAAG,OAAO,EACxD,IAAI,EAAE,MAAM,EACZ,MAAM,GAAE,gBAAgB,CAAC,CAAC,EAAE,CAAC,CAAM,GAClC,IAAI;IAmBP;;;;;OAKG;IACH,aAAa,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO;IAIpC;;;;OAIG;IACH,kBAAkB,IAAI,MAAM,EAAE;IAI9B;;;;;;OAMG;IACH,cAAc,CAAC,CAAC,SAAS,gBAAgB,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,IAAI;IAsC1E;;;;;;;OAOG;IACH,OAAO,CAAC,CAAC,SAAS,gBAAgB,EAAE,cAAc,EAAE,MAAM,EAAE,EAAE,EAAE,EAAE,GAAG,CAAC,GAAG,SAAS;IAKlF;;;;;;OAMG;IACH,QAAQ,CAAC,CAAC,SAAS,gBAAgB,EAAE,cAAc,EAAE,MAAM,GAAG,CAAC,EAAE;IAKjE;;;;;OAKG;IACH,iBAAiB,CAAC,cAAc,EAAE,MAAM,GAAG,MAAM;IAKjD;;;;;OAKG;IACH,WAAW,CAAC,cAAc,EAAE,MAAM,GAAG,qBAAqB;IAK1D;;;;;;OAMG;IACH,OAAO,CAAC,CAAC,SAAS,gBAAgB,EAAE,cAAc,EAAE,MAAM,GAAG,cAAc,CAAC,CAAC,CAAC;IAK9E;;;;;;;OAOG;IACH,YAAY,CAAC,oBAAoB,EAAE,MAAM,EAAE,SAAS,EAAE,EAAE,GAAG,EAAE,EAAE;IAQ/D;;;;;;;OAOG;IACH,cAAc,CAAC,CAAC,SAAS,gBAAgB,EACvC,oBAAoB,EAAE,MAAM,EAC5B,SAAS,EAAE,EAAE,GACZ,CAAC,EAAE;IAMN;;;;;;;;;;;;OAYG;IACH,oBAAoB,CAAC,CAAC,SAAS,gBAAgB,EAC7C,cAAc,EAAE,MAAM,EACtB,EAAE,EAAE,EAAE,EACN,IAAI,EAAE,CAAC,GACN,YAAY;IAoFf;;;;;;OAMG;IACH,yBAAyB,CAAC,CAAC,SAAS,gBAAgB,EAClD,cAAc,EAAE,MAAM,EACtB,KAAK,EAAE,CAAC,EAAE,GACT,iBAAiB;IA0BpB;;;;;;OAMG;IACH,wBAAwB,CAAC,cAAc,EAAE,MAAM,EAAE,EAAE,EAAE,EAAE,GAAG,OAAO;IAmCjE;;;;;;OAMG;IACH,6BAA6B,CAAC,cAAc,EAAE,MAAM,EAAE,GAAG,EAAE,EAAE,EAAE,GAAG,MAAM;IAcxE;;;;;;;;;;;;;;;;;;;;;;;;;;OA0BG;IACH,cAAc,CAAC,CAAC,SAAS,gBAAgB,EACvC,cAAc,EAAE,MAAM,EACtB,QAAQ,EAAE,CAAC,EAAE,EACb,OAAO,CAAC,EAAE,WAAW,GACpB,UAAU;IA8Hb;;;;;;;OAOG;IACH,OAAO,CAAC,oBAAoB;IAQ5B;;;;;;;;;;;;;;;;;;;;;;;;;;;OA2BG;IACH,iBAAiB,CACf,KAAK,EAAE,MAAM,EACb,KAAK,GAAE,MAAW,EAClB,IAAI,CAAC,EAAE,mBAAmB,GACzB,MAAM,CAAC,MAAM,EAAE,gBAAgB,EAAE,GAAG,KAAK,CAAC;QAAE,MAAM,EAAE,gBAAgB,EAAE,CAAC;QAAC,OAAO,EAAE,gBAAgB,CAAC;QAAC,SAAS,EAAE,SAAS,GAAG,QAAQ,GAAG,MAAM,CAAA;KAAE,CAAC,CAAC;IA6FlJ;;;;;;;;OAQG;IACH,KAAK,IAAI,IAAI;CAUd"}

package/dist/index.cjs

@@ -1332,6 +1332,16 @@ function syncLinkedItemsForPrimaryNumeric(linked, primaryId, newLinkedItems, for
   return result;
 }
 // src/common/SearchUniverse.ts
+function tokensEqual3(a, b) {
+  if (a.length !== b.length)
+    return false;
+  for (let i = 0;i < a.length; i++) {
+    if (a[i] !== b[i])
+      return false;
+  }
+  return true;
+}
+
 class SearchUniverse {
   collections = new Map;
   registerCollection(name, config = {}) {
@@ -1535,6 +1545,113 @@ class SearchUniverse {
     }
     return removed;
   }
+  syncCollection(collectionName, newItems, options) {
+    const collection = this.getCollectionOrThrow(collectionName);
+    const result = {
+      added: 0,
+      updated: 0,
+      removed: 0,
+      addedIds: [],
+      updatedIds: [],
+      removedIds: []
+    };
+    const skipUpdateCheck = options?.skipUpdateCheck ?? false;
+    const fkGetter = collection.config.linkedTo ? collection.config.linkedTo.foreignKeyGetter : undefined;
+    const newIds = new Set;
+    for (const item of newItems)
+      newIds.add(item._id);
+    const toRemove = [];
+    for (const id of collection.data.keys()) {
+      if (!newIds.has(id))
+        toRemove.push(id);
+    }
+    for (const id of toRemove) {
+      const existing = collection.data.get(id);
+      if (fkGetter && collection.primaryToLinked) {
+        const fk = fkGetter(existing);
+        if (fk) {
+          const list = collection.primaryToLinked.get(fk);
+          if (list) {
+            const idx = list.indexOf(id);
+            if (idx !== -1) {
+              list.splice(idx, 1);
+              if (list.length === 0)
+                collection.primaryToLinked.delete(fk);
+            }
+          }
+        }
+      }
+      const hadMetadata = collection.metadata.has(id);
+      collection.data.delete(id);
+      collection.metadata.delete(id);
+      if (hadMetadata) {
+        result.removed++;
+        result.removedIds.push(id);
+      }
+    }
+    const tokensMap = createTokensMap();
+    try {
+      for (const item of newItems) {
+        const id = item._id;
+        const existing = collection.data.get(id);
+        const hadMetadata = collection.metadata.has(id);
+        const shouldTokenize = !existing || !skipUpdateCheck;
+        let newTokens;
+        if (shouldTokenize) {
+          newTokens = prepareObjectForSearchNumeric(item, collection.config.spec, tokensMap);
+        }
+        if (fkGetter && collection.primaryToLinked) {
+          const newFk = fkGetter(item);
+          const oldFk = existing ? fkGetter(existing) : undefined;
+          const wasInIndex = hadMetadata;
+          const shouldBeInIndex = !!newFk && !item._deleted && !item._blocked;
+          if (oldFk !== newFk || wasInIndex !== shouldBeInIndex) {
+            if (oldFk && wasInIndex) {
+              const list = collection.primaryToLinked.get(oldFk);
+              if (list) {
+                const idx = list.indexOf(id);
+                if (idx !== -1) {
+                  list.splice(idx, 1);
+                  if (list.length === 0)
+                    collection.primaryToLinked.delete(oldFk);
+                }
+              }
+            }
+            if (shouldBeInIndex && newFk) {
+              const list = collection.primaryToLinked.get(newFk) || [];
+              if (!list.includes(id))
+                list.push(id);
+              collection.primaryToLinked.set(newFk, list);
+            }
+          }
+        }
+        collection.data.set(id, item);
+        if (!shouldTokenize)
+          continue;
+        if (newTokens === undefined) {
+          if (hadMetadata) {
+            collection.metadata.delete(id);
+            result.removed++;
+            result.removedIds.push(id);
+          }
+        } else if (!hadMetadata) {
+          collection.metadata.set(id, newTokens);
+          result.added++;
+          result.addedIds.push(id);
+        } else {
+          const existingTokens = collection.metadata.get(id);
+          if (!tokensEqual3(existingTokens, newTokens)) {
+            collection.metadata.set(id, newTokens);
+            result.updated++;
+            result.updatedIds.push(id);
+          }
+        }
+      }
+    } finally {
+      clearTokensMap(tokensMap);
+    }
+    return result;
+  }
   getCollectionOrThrow(name) {
     const collection = this.collections.get(name);
     if (!collection) {

package/dist/index.js

@@ -1247,6 +1247,16 @@ function syncLinkedItemsForPrimaryNumeric(linked, primaryId, newLinkedItems, for
   return result;
 }
 // src/common/SearchUniverse.ts
+function tokensEqual3(a, b) {
+  if (a.length !== b.length)
+    return false;
+  for (let i = 0;i < a.length; i++) {
+    if (a[i] !== b[i])
+      return false;
+  }
+  return true;
+}
+
 class SearchUniverse {
   collections = new Map;
   registerCollection(name, config = {}) {
@@ -1450,6 +1460,113 @@ class SearchUniverse {
     }
     return removed;
   }
+  syncCollection(collectionName, newItems, options) {
+    const collection = this.getCollectionOrThrow(collectionName);
+    const result = {
+      added: 0,
+      updated: 0,
+      removed: 0,
+      addedIds: [],
+      updatedIds: [],
+      removedIds: []
+    };
+    const skipUpdateCheck = options?.skipUpdateCheck ?? false;
+    const fkGetter = collection.config.linkedTo ? collection.config.linkedTo.foreignKeyGetter : undefined;
+    const newIds = new Set;
+    for (const item of newItems)
+      newIds.add(item._id);
+    const toRemove = [];
+    for (const id of collection.data.keys()) {
+      if (!newIds.has(id))
+        toRemove.push(id);
+    }
+    for (const id of toRemove) {
+      const existing = collection.data.get(id);
+      if (fkGetter && collection.primaryToLinked) {
+        const fk = fkGetter(existing);
+        if (fk) {
+          const list = collection.primaryToLinked.get(fk);
+          if (list) {
+            const idx = list.indexOf(id);
+            if (idx !== -1) {
+              list.splice(idx, 1);
+              if (list.length === 0)
+                collection.primaryToLinked.delete(fk);
+            }
+          }
+        }
+      }
+      const hadMetadata = collection.metadata.has(id);
+      collection.data.delete(id);
+      collection.metadata.delete(id);
+      if (hadMetadata) {
+        result.removed++;
+        result.removedIds.push(id);
+      }
+    }
+    const tokensMap = createTokensMap();
+    try {
+      for (const item of newItems) {
+        const id = item._id;
+        const existing = collection.data.get(id);
+        const hadMetadata = collection.metadata.has(id);
+        const shouldTokenize = !existing || !skipUpdateCheck;
+        let newTokens;
+        if (shouldTokenize) {
+          newTokens = prepareObjectForSearchNumeric(item, collection.config.spec, tokensMap);
+        }
+        if (fkGetter && collection.primaryToLinked) {
+          const newFk = fkGetter(item);
+          const oldFk = existing ? fkGetter(existing) : undefined;
+          const wasInIndex = hadMetadata;
+          const shouldBeInIndex = !!newFk && !item._deleted && !item._blocked;
+          if (oldFk !== newFk || wasInIndex !== shouldBeInIndex) {
+            if (oldFk && wasInIndex) {
+              const list = collection.primaryToLinked.get(oldFk);
+              if (list) {
+                const idx = list.indexOf(id);
+                if (idx !== -1) {
+                  list.splice(idx, 1);
+                  if (list.length === 0)
+                    collection.primaryToLinked.delete(oldFk);
+                }
+              }
+            }
+            if (shouldBeInIndex && newFk) {
+              const list = collection.primaryToLinked.get(newFk) || [];
+              if (!list.includes(id))
+                list.push(id);
+              collection.primaryToLinked.set(newFk, list);
+            }
+          }
+        }
+        collection.data.set(id, item);
+        if (!shouldTokenize)
+          continue;
+        if (newTokens === undefined) {
+          if (hadMetadata) {
+            collection.metadata.delete(id);
+            result.removed++;
+            result.removedIds.push(id);
+          }
+        } else if (!hadMetadata) {
+          collection.metadata.set(id, newTokens);
+          result.added++;
+          result.addedIds.push(id);
+        } else {
+          const existingTokens = collection.metadata.get(id);
+          if (!tokensEqual3(existingTokens, newTokens)) {
+            collection.metadata.set(id, newTokens);
+            result.updated++;
+            result.updatedIds.push(id);
+          }
+        }
+      }
+    } finally {
+      clearTokensMap(tokensMap);
+    }
+    return result;
+  }
   getCollectionOrThrow(name) {
     const collection = this.collections.get(name);
     if (!collection) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "cry-search",
-  "version": "1.0.5",
+  "version": "1.0.6",
   "description": "A fast, memory-efficient search library for large datasets with support for tokenized matching, linked collections, and per-field match modes.",
   "type": "module",
   "main": "./dist/index.js",