cry-synced-db-client 0.1.71 → 0.1.73

package/dist/index.js

@@ -2155,29 +2155,49 @@ class SyncEngine {
       }
       this.callOnFindNewerManyCall(syncSpecs, calledFrom);
       const findNewerManyStartTime = Date.now();
-      let allServerData;
+      const allUpdatedIds = {};
+      const collectionState = new Map;
+      for (const [name] of configMap) {
+        collectionState.set(name, {
+          maxTs: undefined,
+          conflicts: 0,
+          updatedIds: [],
+          receivedCount: 0
+        });
+      }
       try {
-        allServerData = await this.deps.withSyncTimeout(this.restInterface.findNewerMany(syncSpecs), "findNewerMany");
-        this.callOnFindNewerManyResult(syncSpecs, allServerData, findNewerManyStartTime, true, calledFrom);
+        await this.deps.withSyncTimeout(this.restInterface.findNewerManyStream(syncSpecs, async (collection, items) => {
+          const config = configMap.get(collection);
+          if (!config)
+            return;
+          const state = collectionState.get(collection);
+          state.receivedCount += items.length;
+          const stats = await this.processIncomingServerData(collection, config, items);
+          state.conflicts += stats.conflictsResolved;
+          if (stats.maxTs) {
+            if (!state.maxTs || this.compareTimestamps(stats.maxTs, state.maxTs) > 0) {
+              state.maxTs = stats.maxTs;
+            }
+          }
+          state.updatedIds.push(...stats.updatedIds);
+        }), "findNewerManyStream");
+        for (const [name, state] of collectionState) {
+          receivedCount += state.receivedCount;
+          conflictsResolved += state.conflicts;
+          collectionStats[name] = {
+            receivedCount: state.receivedCount,
+            sentCount: 0,
+            receivedItems: []
+          };
+          if (state.updatedIds.length > 0) {
+            allUpdatedIds[name] = state.updatedIds;
+          }
+        }
+        this.callOnFindNewerManyResult(syncSpecs, {}, findNewerManyStartTime, true, calledFrom);
       } catch (err) {
         this.callOnFindNewerManyResult(syncSpecs, {}, findNewerManyStartTime, false, calledFrom, err);
         throw err;
       }
-      const allUpdatedIds = {};
-      for (const [collectionName, config] of configMap) {
-        const serverData = allServerData[collectionName] || [];
-        receivedCount += serverData.length;
-        collectionStats[collectionName] = {
-          receivedCount: serverData.length,
-          sentCount: 0,
-          receivedItems: serverData
-        };
-        const stats = await this.processIncomingServerData(collectionName, config, serverData);
-        conflictsResolved += stats.conflictsResolved;
-        if (stats.updatedIds.length > 0) {
-          allUpdatedIds[collectionName] = stats.updatedIds;
-        }
-      }
       if (Object.keys(allUpdatedIds).length > 0) {
         this.deps.broadcastUpdates(allUpdatedIds);
       }
@@ -2420,60 +2440,69 @@ class SyncEngine {
     }
     return { updatedIds: result.updatedIds };
   }
+  static SYNC_BATCH_SIZE = 200;
   async processIncomingServerData(collectionName, config, serverData) {
     if (serverData.length === 0) {
       return { conflictsResolved: 0, maxTs: undefined, updatedIds: [] };
     }
     let maxTs;
     let conflictsResolved = 0;
-    const serverIds = serverData.map((item) => item._id);
-    const localItems = await this.dexieDb.getByIds(collectionName, serverIds);
-    const dirtyChangesMap = await this.dexieDb.getDirtyChangesBatch(collectionName, serverIds);
-    const dexieBatch = [];
-    const inMemSaveBatch = [];
-    const inMemDeleteIds = [];
-    for (let i = 0;i < serverData.length; i++) {
-      const serverItem = serverData[i];
-      const localItem = localItems[i];
-      const dirtyChange = dirtyChangesMap.get(String(serverItem._id));
-      if (serverItem._ts) {
-        if (!maxTs || this.compareTimestamps(serverItem._ts, maxTs) > 0) {
-          maxTs = serverItem._ts;
+    const allUpdatedIds = [];
+    const BATCH = SyncEngine.SYNC_BATCH_SIZE;
+    for (let offset = 0;offset < serverData.length; offset += BATCH) {
+      const chunk = serverData.slice(offset, offset + BATCH);
+      const chunkIds = chunk.map((item) => item._id);
+      const localItems = await this.dexieDb.getByIds(collectionName, chunkIds);
+      const dirtyChangesMap = await this.dexieDb.getDirtyChangesBatch(collectionName, chunkIds);
+      const dexieBatch = [];
+      const inMemSaveBatch = [];
+      const inMemDeleteIds = [];
+      for (let i = 0;i < chunk.length; i++) {
+        const serverItem = chunk[i];
+        const localItem = localItems[i];
+        const dirtyChange = dirtyChangesMap.get(String(serverItem._id));
+        if (serverItem._ts) {
+          if (!maxTs || this.compareTimestamps(serverItem._ts, maxTs) > 0) {
+            maxTs = serverItem._ts;
+          }
         }
-      }
-      if (localItem) {
-        if (dirtyChange) {
-          conflictsResolved++;
-          const resolved = this.resolveCollectionConflict(collectionName, config, localItem, serverItem, "sync");
-          dexieBatch.push(resolved);
-          if (!resolved._deleted && !resolved._archived) {
-            inMemSaveBatch.push(resolved);
+        if (localItem) {
+          if (dirtyChange) {
+            conflictsResolved++;
+            const resolved = this.resolveCollectionConflict(collectionName, config, localItem, serverItem, "sync");
+            dexieBatch.push(resolved);
+            if (!resolved._deleted && !resolved._archived) {
+              inMemSaveBatch.push(resolved);
+            } else {
+              inMemDeleteIds.push(serverItem._id);
+            }
           } else {
-            inMemDeleteIds.push(serverItem._id);
+            dexieBatch.push(serverItem);
+            if (!serverItem._deleted && !serverItem._archived) {
+              inMemSaveBatch.push(serverItem);
+            } else {
+              inMemDeleteIds.push(serverItem._id);
+            }
           }
         } else {
           dexieBatch.push(serverItem);
           if (!serverItem._deleted && !serverItem._archived) {
             inMemSaveBatch.push(serverItem);
-          } else {
-            inMemDeleteIds.push(serverItem._id);
           }
         }
-      } else {
-        dexieBatch.push(serverItem);
-        if (!serverItem._deleted && !serverItem._archived) {
-          inMemSaveBatch.push(serverItem);
-        }
       }
-    }
-    if (dexieBatch.length > 0) {
-      await this.dexieDb.saveMany(collectionName, dexieBatch);
-    }
-    if (inMemSaveBatch.length > 0) {
-      this.deps.writeToInMemBatch(collectionName, inMemSaveBatch, "upsert");
-    }
-    if (inMemDeleteIds.length > 0) {
-      this.deps.writeToInMemBatch(collectionName, inMemDeleteIds.map((id) => ({ _id: id })), "delete");
+      if (dexieBatch.length > 0) {
+        await this.dexieDb.saveMany(collectionName, dexieBatch);
+      }
+      if (inMemSaveBatch.length > 0) {
+        this.deps.writeToInMemBatch(collectionName, inMemSaveBatch, "upsert");
+      }
+      if (inMemDeleteIds.length > 0) {
+        this.deps.writeToInMemBatch(collectionName, inMemDeleteIds.map((id) => ({ _id: id })), "delete");
+      }
+      for (const id of chunkIds) {
+        allUpdatedIds.push(String(id));
+      }
     }
     if (maxTs) {
       await this.dexieDb.setSyncMeta(collectionName, maxTs);
@@ -2483,8 +2512,7 @@ class SyncEngine {
         lastSyncTs: maxTs
       });
     }
-    const updatedIds = serverIds.map((id) => String(id));
-    return { conflictsResolved, maxTs, updatedIds };
+    return { conflictsResolved, maxTs, updatedIds: allUpdatedIds };
   }
   compareTimestamps(a, b) {
     const aT = typeof a === "object" && "t" in a ? a.t : 0;
@@ -3251,8 +3279,15 @@ class SyncedDb {
     await this.pendingChanges.recoverPendingWrites();
     for (const [name] of this.collections) {
       const data = await this.dexieDb.getAll(name);
-      const activeData = data.filter((item) => !item._deleted && !item._archived);
-      this.inMemManager.initCollection(name, activeData);
+      let writeIdx = 0;
+      for (let i = 0;i < data.length; i++) {
+        const item = data[i];
+        if (!item._deleted && !item._archived) {
+          data[writeIdx++] = item;
+        }
+      }
+      data.length = writeIdx;
+      this.inMemManager.initCollection(name, data);
       const meta = await this.dexieDb.getSyncMeta(name);
       if (meta) {
         this.syncMetaCache.set(name, meta);
@@ -6210,6 +6245,12 @@ var pack2 = (x) => packr.pack(preprocessForPack(x));
 var unpack2 = (x) => unpackr.unpack(x);
 var DEFAULT_TIMEOUT = 5000;
 var DEFAULT_PROGRESS_CHUNK_SIZE = 16 * 1024;
+function concatUint8(a, b) {
+  const result = new Uint8Array(a.length + b.length);
+  result.set(a);
+  result.set(b, a.length);
+  return result;
+}
 
 class RestProxy {
   endpoint;
@@ -6370,6 +6411,129 @@ class RestProxy {
   async findNewerMany(spec) {
     return await this.restCall("findNewerMany", { spec });
   }
+  async findNewerManyStream(spec, onChunk, options) {
+    const connectTimeout = options?.timeoutMs ?? this.defaultTimeoutMs;
+    const activityTimeout = options?.activityTimeoutMs ?? 30000;
+    const externalSignal = options?.signal ?? this.globalSignal;
+    const startTime = this.timeRequests ? performance.now() : 0;
+    const data = {
+      payload: {
+        db: this.tenant,
+        operation: "findNewerMany",
+        spec
+      },
+      audit: {
+        tenant: this.tenant,
+        user: this.audit.user,
+        naprava: this.audit.device
+      }
+    };
+    const body = pack2(data);
+    const requestUrl = this.apiKey ? `${this.endpoint}?apikey=${this.apiKey}&stream=1` : `${this.endpoint}?stream=1`;
+    const controller = new AbortController;
+    let timeoutId = setTimeout(() => controller.abort(), connectTimeout);
+    const combinedSignal = externalSignal ? this.combineSignals(externalSignal, controller.signal) : controller.signal;
+    try {
+      const response = await fetch(requestUrl, {
+        method: "POST",
+        headers: { "Content-Type": "application/octet-stream" },
+        body,
+        signal: combinedSignal
+      });
+      clearTimeout(timeoutId);
+      timeoutId = undefined;
+      if (!response.ok) {
+        const errorText = await response.text();
+        throw new Error(`REST call failed: ${response.status} - ${errorText}`);
+      }
+      const resetActivity = () => {
+        if (timeoutId !== undefined)
+          clearTimeout(timeoutId);
+        timeoutId = setTimeout(() => controller.abort(), activityTimeout);
+      };
+      resetActivity();
+      await this.parseStreamingResponse(response, onChunk, resetActivity);
+      if (timeoutId !== undefined)
+        clearTimeout(timeoutId);
+      timeoutId = undefined;
+      if (this.timeRequests) {
+        const elapsed = performance.now() - startTime;
+        this._lastRequestMs = elapsed;
+        this._totalRequestMs += elapsed;
+        this._requestCount++;
+        if (this.timeRequestsPrint) {
+          console.log(`[RestProxy] findNewerManyStream: ${elapsed.toFixed(2)}ms (total: ${this._totalRequestMs.toFixed(2)}ms, count: ${this._requestCount})`);
+        }
+      }
+    } catch (err) {
+      if (timeoutId !== undefined)
+        clearTimeout(timeoutId);
+      if (err.name === "AbortError") {
+        if (controller.signal.aborted && !externalSignal?.aborted) {
+          throw new Error(`REST call timeout: findNewerManyStream`);
+        }
+        throw new Error("REST call aborted: findNewerManyStream");
+      }
+      throw err;
+    }
+  }
+  async parseStreamingResponse(response, onChunk, onActivity) {
+    const reader = response.body.getReader();
+    let buffer = new Uint8Array(0);
+    const readMore = async () => {
+      const { done, value } = await reader.read();
+      if (done)
+        return false;
+      onActivity();
+      buffer = concatUint8(buffer, value);
+      return true;
+    };
+    while (buffer.length < 1) {
+      if (!await readMore())
+        return;
+    }
+    const firstByte = buffer[0];
+    if (firstByte !== 0 && firstByte !== 1) {
+      while (await readMore()) {}
+      const result = unpack2(buffer);
+      for (const [collection, items] of Object.entries(result)) {
+        if (items.length > 0) {
+          await onChunk(collection, items);
+        }
+      }
+      return;
+    }
+    while (true) {
+      while (buffer.length < 1) {
+        if (!await readMore())
+          return;
+      }
+      if (buffer[0] === 0)
+        return;
+      while (buffer.length < 3) {
+        if (!await readMore())
+          throw new Error("Unexpected end of stream in chunk header");
+      }
+      const nameLen = buffer[1] << 8 | buffer[2];
+      const headerSize = 1 + 2 + nameLen + 4;
+      while (buffer.length < headerSize) {
+        if (!await readMore())
+          throw new Error("Unexpected end of stream in chunk header");
+      }
+      const collection = new TextDecoder().decode(buffer.slice(3, 3 + nameLen));
+      const dataOffset = 3 + nameLen;
+      const dataLen = buffer[dataOffset] << 24 | buffer[dataOffset + 1] << 16 | buffer[dataOffset + 2] << 8 | buffer[dataOffset + 3];
+      const totalChunkSize = headerSize + dataLen;
+      while (buffer.length < totalChunkSize) {
+        if (!await readMore())
+          throw new Error("Unexpected end of stream in chunk data");
+      }
+      const payloadBytes = buffer.slice(headerSize, totalChunkSize);
+      const items = unpack2(payloadBytes);
+      buffer = buffer.slice(totalChunkSize);
+      await onChunk(collection, items);
+    }
+  }
   async deleteOne(collection, query) {
     return await this.restCall("deleteOne", { collection, query });
   }

package/dist/src/db/RestProxy.d.ts

@@ -101,6 +101,28 @@ export declare class RestProxy implements I_RestInterface {
     findByIds<T>(collection: string, ids: Id[]): Promise<T[]>;
     findNewer<T>(collection: string, timestamp: Timestamp | number | string | Date, query?: QuerySpec<T>, opts?: QueryOpts): Promise<T[]>;
     findNewerMany<T>(spec?: GetNewerSpec<T>[]): Promise<Record<string, any[]>>;
+    /**
+     * Streaming variant of findNewerMany.
+     * Reads chunked binary response and calls onChunk for each batch.
+     * Peak memory = one chunk (~200 docs) instead of entire result set.
+     *
+     * Binary chunk format:
+     *   [type:1][nameLen:2][name:N][dataLen:4][msgpack(items[]):M]
+     *   type=0x01 for data, type=0x00 for end-of-stream.
+     */
+    findNewerManyStream<T>(spec: GetNewerSpec<T>[], onChunk: (collection: string, items: T[]) => Promise<void>, options?: {
+        timeoutMs?: number;
+        signal?: AbortSignal;
+        activityTimeoutMs?: number;
+    }): Promise<void>;
+    /**
+     * Parse streaming response. Auto-detects format:
+     * - Streaming: first byte is 0x00 (end) or 0x01 (data chunk)
+     * - Legacy msgpack: first byte is msgpack type marker (0x80+ for map, etc.)
+     *
+     * Streaming chunk format: [type:1][nameLen:2][name:N][dataLen:4][msgpack(items[]):M]
+     */
+    private parseStreamingResponse;
     deleteOne<T>(collection: string, query: QuerySpec<T>): Promise<T>;
     aggregate<T>(collection: string, pipeline: object[], opts?: AggregateOptions): Promise<T[]>;
     upsertBatch<T>(collection: string, batch: BatchSpec<T>): Promise<T[]>;

package/dist/src/db/sync/SyncEngine.d.ts

@@ -38,6 +38,8 @@ export declare class SyncEngine implements I_SyncEngine {
     processCollectionServerData(collectionName: string, serverData: LocalDbEntity[]): Promise<{
         updatedIds: string[];
     }>;
+    /** Max items to process per batch in processIncomingServerData */
+    private static readonly SYNC_BATCH_SIZE;
     private processIncomingServerData;
     private compareTimestamps;
     private resolveCollectionConflict;

package/dist/src/types/I_RestInterface.d.ts

@@ -67,6 +67,11 @@ export interface I_RestInterface {
     findByIds<T>(collection: string, ids: Id[]): Promise<T[]>;
     findNewer<T>(collection: string, timestamp: Timestamp | number | string | Date, query?: QuerySpec<T>, opts?: QueryOpts): Promise<T[]>;
     findNewerMany<T>(spec?: GetNewerSpec<T>[]): Promise<Record<string, any[]>>;
+    /** Streaming variant of findNewerMany. Calls onChunk for each batch of items as they arrive. */
+    findNewerManyStream<T>(spec: GetNewerSpec<T>[], onChunk: (collection: string, items: T[]) => Promise<void>, options?: {
+        timeoutMs?: number;
+        signal?: AbortSignal;
+    }): Promise<void>;
     deleteOne<T>(collection: string, query: QuerySpec<T>): Promise<T>;
     /** Izvede agregacijo na serverju */
     aggregate<T>(collection: string, pipeline: object[], opts?: AggregateOptions): Promise<T[]>;

package/dist/src/types/I_SyncedDb.d.ts

@@ -206,7 +206,7 @@ export interface CollectionSyncStats {
     receivedCount: number;
     /** Number of dirty items sent to server for this collection */
     sentCount: number;
-    /** The actual items received from server (for debugging/logging) */
+    /** @deprecated Use receivedCount instead. Will be empty array in future streaming mode. */
     receivedItems: LocalDbEntity[];
 }
 /**

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "cry-synced-db-client",
-  "version": "0.1.71",
+  "version": "0.1.73",
   "type": "module",
   "main": "./dist/index.js",
   "module": "./dist/index.js",