cry-synced-db-client 0.1.143 → 0.1.145

package/CHANGELOG.md

@@ -2,6 +2,40 @@
 
 ## Unreleased
 
+### Fix: filtered-sync tombstone (scope-exit from other writers)
+
+When a collection has `syncConfig.query` (e.g. `{ status: { $ne: "obsolete" } }`)
+and another device mutates a record out of scope while this client is offline,
+the filtered delta feed (`findNewer` with the positive query) never returns the
+mutated record — the server filters it out. The local copy stayed stale
+indefinitely, even across reloads and full re-syncs (positive query still
+excludes it).
+
+`evictOutOfScopeRecords` now runs a second, server-assisted pass that fires the
+**negated** query `{ $and: [{ $nor: [syncConfig.query] }, { _id: { $in: knownIds } }] }`
+against the server (scoped by local IDs for both bandwidth and authz) and
+evicts any returned records locally. Purely client-side — no server protocol
+change. Sound only when `syncConfig.query` is the complete server-side
+predicate (no implicit server policy).
+
+- New `opts.serverAssisted` parameter on `evictOutOfScopeRecords(collection, opts?)`.
+  Defaults to `true` when online and not `writeOnly`. Set `false` for pure
+  local pass (previous behavior).
+- New `opts.outOfWindowLookbehindMs` parameter restricts the server-assisted
+  query to records changed in the last N ms (translates to `findNewer`'s
+  seconds-resolution timestamp filter). When unspecified, the scope-exit
+  scan uses the collection's current `meta.lastSyncTs` — the same cursor
+  `sync()` passes to `findNewer`, so it covers the same window as a delta
+  sync. Pass `Date.now()` to scan full history. Note: when eviction runs
+  post-sync, `lastSyncTs` has advanced past the window sync just covered;
+  set a lookbehind to re-scan that window.
+- Dirty records are still preserved across the combined eviction.
+- Auto-eviction (`evictStaleRecordsEveryHrs`) picks up the new behavior
+  transparently — no config change needed.
+- `matchesQuery` (`src/utils/localQuery.ts`) gained top-level `$and`, `$or`,
+  `$nor` support, required for the negated query to evaluate against the test
+  mock and for any client-side filtering that uses logical operators.
+
 ### `getDirtyMeta()` for lightweight dirty-state inspection
 
 - New `SyncedDb.getDirtyMeta()` returns dirty-entry meta (everything except the
@@ -74,6 +108,37 @@ Signature is identical. No other callback changes.
   - Noop when offline or on writeOnly collections.
   - Ignored on `find` / `findOne` (use `referToServer` there).
 
+## 0.1.145 (2026-04-25)
+
+### Fix: `onSyncProgress` back-track during initial sync
+
+`onSyncProgress` is fired from two distinct phases that can run **concurrently**
+during initial sync — Dexie → in-mem hydration (`SyncedDb.loadCollectionsToInMem`)
+and server → Dexie download (`SyncEngine.findNewerManyStream`). Each phase
+carries its own `loaded`/`total`, so consumers wiring the callback into a single
+progress bar saw the percentage back-track every time a tick from the other
+phase arrived (e.g. dexie 9/58 → server 1/62 → dexie 10/58 → server 2/62 …).
+
+The payload now carries a `phase: 'dexie' | 'server'` discriminator so consumers
+can attribute each tick to its source and either filter or render the two
+streams separately. **Non-breaking**: consumers that destructure
+`{ collection, loaded, total }` and ignore `phase` keep working unchanged.
+
+Type change in `I_SyncedDb.SyncedDbConfig` and internal `SyncEngineCallbacks`:
+```ts
+onSyncProgress?: (info: {
+  phase: 'dexie' | 'server';
+  collection: string;
+  loaded: number;
+  total: number;
+  items: number;
+}) => void;
+```
+
+The JSDoc on `onSyncProgress` previously claimed "Fires only during
+init/setSyncOnlyTheseCollections" — that was wrong; it also fires during server
+sync. Doc corrected.
+
 ## 0.1.136 (2026-04-20)
 
 - `DexieDb.saveMany` is now fail-safe:

package/dist/index.js

@@ -39,6 +39,27 @@ import { ObjectId as ObjectId2 } from "bson";
 // src/utils/localQuery.ts
 function matchesQuery(item, query) {
   for (const [key, condition] of Object.entries(query)) {
+    if (key === "$and") {
+      if (!Array.isArray(condition)) return false;
+      if (!condition.every((sub) => matchesQuery(item, sub))) {
+        return false;
+      }
+      continue;
+    }
+    if (key === "$or") {
+      if (!Array.isArray(condition) || condition.length === 0) return false;
+      if (!condition.some((sub) => matchesQuery(item, sub))) {
+        return false;
+      }
+      continue;
+    }
+    if (key === "$nor") {
+      if (!Array.isArray(condition) || condition.length === 0) return false;
+      if (condition.some((sub) => matchesQuery(item, sub))) {
+        return false;
+      }
+      continue;
+    }
     if (!matchesCondition(item, key, condition)) {
       return false;
     }
@@ -2481,6 +2502,7 @@ var _SyncEngine = class _SyncEngine {
               if (!completedCollections.has(collection)) {
                 completedCollections.add(collection);
                 this.callbackSafe(this.callbacks.onSyncProgress, {
+                  phase: "server",
                   collection,
                   loaded: completedCollections.size,
                   total: syncSpecs.length,
@@ -4662,9 +4684,42 @@ var _SyncedDb = class _SyncedDb {
    * Remove records from Dexie and in-mem that no longer match the
    * collection's current syncConfig.query. Does NOT delete from server.
    * Skips records with dirty changes (pending local writes).
+   *
+   * Runs two passes:
+   * 1. Local: evicts items whose locally-cached state fails the query.
+   *    Covers self-inflicted scope exits (the client moved an item out).
+   * 2. Server-assisted (default when online and not writeOnly): of the
+   *    items that still match locally, ask the server which now fail the
+   *    query server-side. Covers scope exits caused by other writers — the
+   *    filtered delta feed (`findNewer` with the positive query) never
+   *    ships those records, so the local cache would otherwise go stale
+   *    forever.
+   *
+   * The server-assisted query is `{ $and: [{ $nor: [query] }, { _id: { $in: chunk } }] }`
+   * with `project: { _id: 1 }`. Scoped to known IDs to preserve both
+   * bandwidth and authorization (server only reveals state of IDs the
+   * client already knew about). Sound only if `syncConfig.query` is the
+   * complete server-side predicate (no implicit server policy).
+   *
+   * @param opts.serverAssisted - Override auto-detection. Default: `true`
+   *   when online and not writeOnly. Set `false` for pure local eviction.
+   * @param opts.outOfWindowLookbehindMs - Restrict the server-assisted
+   *   query to records changed in the last N ms (adds `_ts > now-N` via
+   *   `findNewer`'s timestamp filter). When unspecified, the collection's
+   *   current `meta.lastSyncTs` is used — the same cursor `sync()` passes
+   *   to `findNewer`, so scope-exit scan covers the same time window as a
+   *   regular delta sync. Pass `Date.now()` (or any value larger than
+   *   elapsed time) to scan full history.
+   *
+   *   Note on post-sync timing: when this method runs AFTER `sync()`
+   *   (including via auto-eviction), `lastSyncTs` has already advanced
+   *   past the window sync just covered, so a record whose scope-exit
+   *   mutation landed in that window will not be re-examined on this
+   *   pass. Specify `outOfWindowLookbehindMs` (e.g., to cover since last
+   *   eviction) or call this before `sync()` to close that gap.
    */
-  async evictOutOfScopeRecords(collection) {
-    var _a;
+  async evictOutOfScopeRecords(collection, opts) {
+    var _a, _b, _c, _d;
     this.assertCollection(collection);
     const config = this.collections.get(collection);
     const syncQuery = (_a = config.syncConfig) == null ? void 0 : _a.query;
@@ -4675,9 +4730,11 @@ var _SyncedDb = class _SyncedDb {
     await this.pendingChanges.flushForCollection(collection);
     const dirtyItems = await this.dexieDb.getDirty(collection);
     const dirtyIds = new Set(dirtyItems.map((d) => String(d._id)));
+    const serverAssisted = (_b = opts == null ? void 0 : opts.serverAssisted) != null ? _b : !config.writeOnly && this.connectionManager.isOnline();
     let scannedCount = 0;
     let dirtySkipped = 0;
     const evictIds = [];
+    const serverCandidateIds = [];
     await this.dexieDb.forEachBatch(
       collection,
       2e3,
@@ -4691,10 +4748,38 @@ var _SyncedDb = class _SyncedDb {
           }
           if (!matchesQuery(item, query)) {
             evictIds.push(id);
+          } else if (serverAssisted) {
+            serverCandidateIds.push(id);
           }
         }
       }
     );
+    if (serverAssisted && serverCandidateIds.length > 0) {
+      let scopeExitTimestamp;
+      const lookbehindMs = opts == null ? void 0 : opts.outOfWindowLookbehindMs;
+      if (lookbehindMs != null && lookbehindMs > 0) {
+        scopeExitTimestamp = Math.max(
+          0,
+          Math.floor((Date.now() - lookbehindMs) / 1e3)
+        );
+      } else {
+        scopeExitTimestamp = (_d = (_c = this.syncMetaCache.get(collection)) == null ? void 0 : _c.lastSyncTs) != null ? _d : 0;
+      }
+      try {
+        const serverExits = await this.findServerSideScopeExits(
+          collection,
+          query,
+          serverCandidateIds,
+          scopeExitTimestamp
+        );
+        for (const id of serverExits) evictIds.push(id);
+      } catch (err) {
+        console.error(
+          `[evict] server-assisted pass failed for ${collection} (proceeding with local-only):`,
+          err
+        );
+      }
+    }
     if (evictIds.length > 0) {
       await this.dexieDb.deleteMany(collection, evictIds);
       if (!config.writeOnly) {
@@ -4708,6 +4793,49 @@ var _SyncedDb = class _SyncedDb {
     }
     return { collection, evictedCount: evictIds.length, dirtySkipped, scannedCount };
   }
+  /**
+   * Ask the server which of the given IDs no longer match the positive
+   * query server-side. Used by `evictOutOfScopeRecords` server-assisted
+   * pass to close the "filtered-delta tombstone" gap — records mutated
+   * out of scope by other writers that the filtered delta feed would
+   * never report.
+   *
+   * Chunks `candidateIds` into `$in`-sized batches to keep the request
+   * payload bounded. Uses `project: { _id: 1 }` to keep the response
+   * payload minimal.
+   *
+   * @param timestamp - `findNewer` timestamp cursor. Only records with
+   *   `_ts > timestamp` are examined. Caller decides the window (usually
+   *   the collection's `meta.lastSyncTs` to mirror the positive sync's
+   *   cursor, or a lookbehind-derived value for shorter windows).
+   * @returns IDs the server reports as out-of-scope (i.e. matching
+   *   `NOT query`). These are candidates for local-only eviction.
+   */
+  async findServerSideScopeExits(collection, positiveQuery, candidateIds, timestamp) {
+    const CHUNK_SIZE = 500;
+    const scopeExits = [];
+    const negated = { $nor: [positiveQuery] };
+    for (let i = 0; i < candidateIds.length; i += CHUNK_SIZE) {
+      const chunk = candidateIds.slice(i, i + CHUNK_SIZE);
+      const scopedNegation = {
+        $and: [negated, { _id: { $in: chunk } }]
+      };
+      const results = await this.connectionManager.withRestTimeout(
+        this.restInterface.findNewer(
+          collection,
+          timestamp,
+          scopedNegation,
+          { project: { _id: 1 } }
+        ),
+        "evictOutOfScopeRecords.serverAssisted"
+      );
+      for (const item of results) {
+        const id = item._id;
+        if (id !== void 0) scopeExits.push(String(id));
+      }
+    }
+    return scopeExits;
+  }
   /**
    * Evict out-of-scope records for all collections.
    * Skips writeOnly and collections without syncConfig.query.
@@ -4856,6 +4984,7 @@ var _SyncedDb = class _SyncedDb {
           totalItems += items;
           loaded++;
           this.safeCallback(this.onSyncProgress, {
+            phase: "dexie",
             collection: name,
             loaded,
             total: names.length,

package/dist/src/db/SyncedDb.d.ts

@@ -183,8 +183,63 @@ export declare class SyncedDb implements I_SyncedDb {
      * Remove records from Dexie and in-mem that no longer match the
      * collection's current syncConfig.query. Does NOT delete from server.
      * Skips records with dirty changes (pending local writes).
+     *
+     * Runs two passes:
+     * 1. Local: evicts items whose locally-cached state fails the query.
+     *    Covers self-inflicted scope exits (the client moved an item out).
+     * 2. Server-assisted (default when online and not writeOnly): of the
+     *    items that still match locally, ask the server which now fail the
+     *    query server-side. Covers scope exits caused by other writers — the
+     *    filtered delta feed (`findNewer` with the positive query) never
+     *    ships those records, so the local cache would otherwise go stale
+     *    forever.
+     *
+     * The server-assisted query is `{ $and: [{ $nor: [query] }, { _id: { $in: chunk } }] }`
+     * with `project: { _id: 1 }`. Scoped to known IDs to preserve both
+     * bandwidth and authorization (server only reveals state of IDs the
+     * client already knew about). Sound only if `syncConfig.query` is the
+     * complete server-side predicate (no implicit server policy).
+     *
+     * @param opts.serverAssisted - Override auto-detection. Default: `true`
+     *   when online and not writeOnly. Set `false` for pure local eviction.
+     * @param opts.outOfWindowLookbehindMs - Restrict the server-assisted
+     *   query to records changed in the last N ms (adds `_ts > now-N` via
+     *   `findNewer`'s timestamp filter). When unspecified, the collection's
+     *   current `meta.lastSyncTs` is used — the same cursor `sync()` passes
+     *   to `findNewer`, so scope-exit scan covers the same time window as a
+     *   regular delta sync. Pass `Date.now()` (or any value larger than
+     *   elapsed time) to scan full history.
+     *
+     *   Note on post-sync timing: when this method runs AFTER `sync()`
+     *   (including via auto-eviction), `lastSyncTs` has already advanced
+     *   past the window sync just covered, so a record whose scope-exit
+     *   mutation landed in that window will not be re-examined on this
+     *   pass. Specify `outOfWindowLookbehindMs` (e.g., to cover since last
+     *   eviction) or call this before `sync()` to close that gap.
+     */
+    evictOutOfScopeRecords(collection: string, opts?: {
+        serverAssisted?: boolean;
+        outOfWindowLookbehindMs?: number;
+    }): Promise<EvictionCollectionInfo>;
+    /**
+     * Ask the server which of the given IDs no longer match the positive
+     * query server-side. Used by `evictOutOfScopeRecords` server-assisted
+     * pass to close the "filtered-delta tombstone" gap — records mutated
+     * out of scope by other writers that the filtered delta feed would
+     * never report.
+     *
+     * Chunks `candidateIds` into `$in`-sized batches to keep the request
+     * payload bounded. Uses `project: { _id: 1 }` to keep the response
+     * payload minimal.
+     *
+     * @param timestamp - `findNewer` timestamp cursor. Only records with
+     *   `_ts > timestamp` are examined. Caller decides the window (usually
+     *   the collection's `meta.lastSyncTs` to mirror the positive sync's
+     *   cursor, or a lookbehind-derived value for shorter windows).
+     * @returns IDs the server reports as out-of-scope (i.e. matching
+     *   `NOT query`). These are candidates for local-only eviction.
      */
-    evictOutOfScopeRecords(collection: string): Promise<EvictionCollectionInfo>;
+    private findServerSideScopeExits;
     /**
      * Evict out-of-scope records for all collections.
      * Skips writeOnly and collections without syncConfig.query.

package/dist/src/db/types/managers.d.ts

@@ -235,6 +235,7 @@ export interface SyncEngineCallbacks {
     }) => void;
     onSyncEnd?: (info: SyncInfo) => void;
     onSyncProgress?: (info: {
+        phase: 'dexie' | 'server';
         collection: string;
         loaded: number;
         total: number;

package/dist/src/types/I_SyncedDb.d.ts

@@ -345,8 +345,15 @@ export interface SyncedDbConfig {
         totalItems: number;
         durationMs: number;
     }) => void;
-    /** Callback after each collection is loaded during full sync (Dexie→inMem). Fires only during init/setSyncOnlyTheseCollections. */
+    /**
+     * Callback after each collection completes loading. Fires from two distinct phases that can run
+     * concurrently — `phase` discriminates the source so consumers can render progress coherently:
+     *  - `'dexie'`  — Dexie → in-memory hydration during init/setSyncOnlyTheseCollections
+     *  - `'server'` — server → Dexie download during full/initial sync (findNewerManyStream)
+     * `loaded`/`total` are scoped to the phase that emitted the event.
+     */
     onSyncProgress?: (info: {
+        phase: 'dexie' | 'server';
         collection: string;
         loaded: number;
         total: number;
@@ -738,9 +745,25 @@ export interface I_SyncedDb {
      * Remove records from Dexie and in-mem that no longer match the
      * collection's current syncConfig.query. Does NOT delete from server.
      * Skips records with dirty changes (pending local writes).
-     * Returns per-collection eviction stats.
-     */
-    evictOutOfScopeRecords(collection: string): Promise<EvictionCollectionInfo>;
+     *
+     * When `serverAssisted` is true (default when online and not writeOnly),
+     * also asks the server which of the locally-matching records no longer
+     * match the query server-side. This catches scope-exits caused by other
+     * writers — records the filtered delta feed would never report because
+     * the server-side state no longer matches the positive query.
+     *
+     * @param opts.serverAssisted - Override default auto-detection. Set
+     *   `false` to skip the server round-trip (local-only pass).
+     * @param opts.outOfWindowLookbehindMs - Restrict the server-assisted
+     *   query to records changed in the last N ms. When unspecified, uses
+     *   the collection's current `meta.lastSyncTs` — the same cursor
+     *   `sync()` passes to `findNewer`. Pass `Date.now()` to scan full
+     *   history.
+     */
+    evictOutOfScopeRecords(collection: string, opts?: {
+        serverAssisted?: boolean;
+        outOfWindowLookbehindMs?: number;
+    }): Promise<EvictionCollectionInfo>;
     /**
      * Same as evictOutOfScopeRecords but for all configured collections.
      * Skips writeOnly collections and collections without syncConfig.query.

package/dist/src/utils/localQuery.d.ts

@@ -1,8 +1,10 @@
 import type { QuerySpec, QueryOpts, Projection } from "../types/I_RestInterface";
 import type { DbEntity } from "../types/DbEntity";
 /**
- * Preveri, ali objekt ustreza MongoDB-style query specifikaciji
- * Podpira osnovne operatorje: $eq, $ne, $gt, $gte, $lt, $lte, $in, $nin, $exists, $regex
+ * Preveri, ali objekt ustreza MongoDB-style query specifikaciji.
+ * Podpira polje-operatorje: $eq, $ne, $gt, $gte, $lt, $lte, $in, $nin,
+ * $exists, $regex, $elemMatch, $size, $all — in logične operatorje
+ * $and, $or, $nor na top-levelu.
  */
 export declare function matchesQuery<T extends DbEntity>(item: T, query: QuerySpec<T>): boolean;
 /**

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "cry-synced-db-client",
-  "version": "0.1.143",
+  "version": "0.1.145",
   "type": "module",
   "main": "./dist/index.js",
   "module": "./dist/index.js",