cry-synced-db-client 0.1.72 → 0.1.73

package/dist/index.js

@@ -2155,30 +2155,49 @@ class SyncEngine {
       }
       this.callOnFindNewerManyCall(syncSpecs, calledFrom);
       const findNewerManyStartTime = Date.now();
-      let allServerData;
+      const allUpdatedIds = {};
+      const collectionState = new Map;
+      for (const [name] of configMap) {
+        collectionState.set(name, {
+          maxTs: undefined,
+          conflicts: 0,
+          updatedIds: [],
+          receivedCount: 0
+        });
+      }
       try {
-        allServerData = await this.deps.withSyncTimeout(this.restInterface.findNewerMany(syncSpecs), "findNewerMany");
-        this.callOnFindNewerManyResult(syncSpecs, allServerData, findNewerManyStartTime, true, calledFrom);
+        await this.deps.withSyncTimeout(this.restInterface.findNewerManyStream(syncSpecs, async (collection, items) => {
+          const config = configMap.get(collection);
+          if (!config)
+            return;
+          const state = collectionState.get(collection);
+          state.receivedCount += items.length;
+          const stats = await this.processIncomingServerData(collection, config, items);
+          state.conflicts += stats.conflictsResolved;
+          if (stats.maxTs) {
+            if (!state.maxTs || this.compareTimestamps(stats.maxTs, state.maxTs) > 0) {
+              state.maxTs = stats.maxTs;
+            }
+          }
+          state.updatedIds.push(...stats.updatedIds);
+        }), "findNewerManyStream");
+        for (const [name, state] of collectionState) {
+          receivedCount += state.receivedCount;
+          conflictsResolved += state.conflicts;
+          collectionStats[name] = {
+            receivedCount: state.receivedCount,
+            sentCount: 0,
+            receivedItems: []
+          };
+          if (state.updatedIds.length > 0) {
+            allUpdatedIds[name] = state.updatedIds;
+          }
+        }
+        this.callOnFindNewerManyResult(syncSpecs, {}, findNewerManyStartTime, true, calledFrom);
       } catch (err) {
         this.callOnFindNewerManyResult(syncSpecs, {}, findNewerManyStartTime, false, calledFrom, err);
         throw err;
       }
-      const allUpdatedIds = {};
-      for (const [collectionName, config] of configMap) {
-        const serverData = allServerData[collectionName] || [];
-        delete allServerData[collectionName];
-        receivedCount += serverData.length;
-        collectionStats[collectionName] = {
-          receivedCount: serverData.length,
-          sentCount: 0,
-          receivedItems: []
-        };
-        const stats = await this.processIncomingServerData(collectionName, config, serverData);
-        conflictsResolved += stats.conflictsResolved;
-        if (stats.updatedIds.length > 0) {
-          allUpdatedIds[collectionName] = stats.updatedIds;
-        }
-      }
       if (Object.keys(allUpdatedIds).length > 0) {
         this.deps.broadcastUpdates(allUpdatedIds);
       }
@@ -6226,6 +6245,12 @@ var pack2 = (x) => packr.pack(preprocessForPack(x));
 var unpack2 = (x) => unpackr.unpack(x);
 var DEFAULT_TIMEOUT = 5000;
 var DEFAULT_PROGRESS_CHUNK_SIZE = 16 * 1024;
+function concatUint8(a, b) {
+  const result = new Uint8Array(a.length + b.length);
+  result.set(a);
+  result.set(b, a.length);
+  return result;
+}
 
 class RestProxy {
   endpoint;
@@ -6386,6 +6411,129 @@ class RestProxy {
   async findNewerMany(spec) {
     return await this.restCall("findNewerMany", { spec });
   }
+  async findNewerManyStream(spec, onChunk, options) {
+    const connectTimeout = options?.timeoutMs ?? this.defaultTimeoutMs;
+    const activityTimeout = options?.activityTimeoutMs ?? 30000;
+    const externalSignal = options?.signal ?? this.globalSignal;
+    const startTime = this.timeRequests ? performance.now() : 0;
+    const data = {
+      payload: {
+        db: this.tenant,
+        operation: "findNewerMany",
+        spec
+      },
+      audit: {
+        tenant: this.tenant,
+        user: this.audit.user,
+        naprava: this.audit.device
+      }
+    };
+    const body = pack2(data);
+    const requestUrl = this.apiKey ? `${this.endpoint}?apikey=${this.apiKey}&stream=1` : `${this.endpoint}?stream=1`;
+    const controller = new AbortController;
+    let timeoutId = setTimeout(() => controller.abort(), connectTimeout);
+    const combinedSignal = externalSignal ? this.combineSignals(externalSignal, controller.signal) : controller.signal;
+    try {
+      const response = await fetch(requestUrl, {
+        method: "POST",
+        headers: { "Content-Type": "application/octet-stream" },
+        body,
+        signal: combinedSignal
+      });
+      clearTimeout(timeoutId);
+      timeoutId = undefined;
+      if (!response.ok) {
+        const errorText = await response.text();
+        throw new Error(`REST call failed: ${response.status} - ${errorText}`);
+      }
+      const resetActivity = () => {
+        if (timeoutId !== undefined)
+          clearTimeout(timeoutId);
+        timeoutId = setTimeout(() => controller.abort(), activityTimeout);
+      };
+      resetActivity();
+      await this.parseStreamingResponse(response, onChunk, resetActivity);
+      if (timeoutId !== undefined)
+        clearTimeout(timeoutId);
+      timeoutId = undefined;
+      if (this.timeRequests) {
+        const elapsed = performance.now() - startTime;
+        this._lastRequestMs = elapsed;
+        this._totalRequestMs += elapsed;
+        this._requestCount++;
+        if (this.timeRequestsPrint) {
+          console.log(`[RestProxy] findNewerManyStream: ${elapsed.toFixed(2)}ms (total: ${this._totalRequestMs.toFixed(2)}ms, count: ${this._requestCount})`);
+        }
+      }
+    } catch (err) {
+      if (timeoutId !== undefined)
+        clearTimeout(timeoutId);
+      if (err.name === "AbortError") {
+        if (controller.signal.aborted && !externalSignal?.aborted) {
+          throw new Error(`REST call timeout: findNewerManyStream`);
+        }
+        throw new Error("REST call aborted: findNewerManyStream");
+      }
+      throw err;
+    }
+  }
+  async parseStreamingResponse(response, onChunk, onActivity) {
+    const reader = response.body.getReader();
+    let buffer = new Uint8Array(0);
+    const readMore = async () => {
+      const { done, value } = await reader.read();
+      if (done)
+        return false;
+      onActivity();
+      buffer = concatUint8(buffer, value);
+      return true;
+    };
+    while (buffer.length < 1) {
+      if (!await readMore())
+        return;
+    }
+    const firstByte = buffer[0];
+    if (firstByte !== 0 && firstByte !== 1) {
+      while (await readMore()) {}
+      const result = unpack2(buffer);
+      for (const [collection, items] of Object.entries(result)) {
+        if (items.length > 0) {
+          await onChunk(collection, items);
+        }
+      }
+      return;
+    }
+    while (true) {
+      while (buffer.length < 1) {
+        if (!await readMore())
+          return;
+      }
+      if (buffer[0] === 0)
+        return;
+      while (buffer.length < 3) {
+        if (!await readMore())
+          throw new Error("Unexpected end of stream in chunk header");
+      }
+      const nameLen = buffer[1] << 8 | buffer[2];
+      const headerSize = 1 + 2 + nameLen + 4;
+      while (buffer.length < headerSize) {
+        if (!await readMore())
+          throw new Error("Unexpected end of stream in chunk header");
+      }
+      const collection = new TextDecoder().decode(buffer.slice(3, 3 + nameLen));
+      const dataOffset = 3 + nameLen;
+      const dataLen = buffer[dataOffset] << 24 | buffer[dataOffset + 1] << 16 | buffer[dataOffset + 2] << 8 | buffer[dataOffset + 3];
+      const totalChunkSize = headerSize + dataLen;
+      while (buffer.length < totalChunkSize) {
+        if (!await readMore())
+          throw new Error("Unexpected end of stream in chunk data");
+      }
+      const payloadBytes = buffer.slice(headerSize, totalChunkSize);
+      const items = unpack2(payloadBytes);
+      buffer = buffer.slice(totalChunkSize);
+      await onChunk(collection, items);
+    }
+  }
   async deleteOne(collection, query) {
     return await this.restCall("deleteOne", { collection, query });
   }

package/dist/src/db/RestProxy.d.ts

@@ -101,6 +101,28 @@ export declare class RestProxy implements I_RestInterface {
     findByIds<T>(collection: string, ids: Id[]): Promise<T[]>;
     findNewer<T>(collection: string, timestamp: Timestamp | number | string | Date, query?: QuerySpec<T>, opts?: QueryOpts): Promise<T[]>;
     findNewerMany<T>(spec?: GetNewerSpec<T>[]): Promise<Record<string, any[]>>;
+    /**
+     * Streaming variant of findNewerMany.
+     * Reads chunked binary response and calls onChunk for each batch.
+     * Peak memory = one chunk (~200 docs) instead of entire result set.
+     *
+     * Binary chunk format:
+     *   [type:1][nameLen:2][name:N][dataLen:4][msgpack(items[]):M]
+     *   type=0x01 for data, type=0x00 for end-of-stream.
+     */
+    findNewerManyStream<T>(spec: GetNewerSpec<T>[], onChunk: (collection: string, items: T[]) => Promise<void>, options?: {
+        timeoutMs?: number;
+        signal?: AbortSignal;
+        activityTimeoutMs?: number;
+    }): Promise<void>;
+    /**
+     * Parse streaming response. Auto-detects format:
+     * - Streaming: first byte is 0x00 (end) or 0x01 (data chunk)
+     * - Legacy msgpack: first byte is msgpack type marker (0x80+ for map, etc.)
+     *
+     * Streaming chunk format: [type:1][nameLen:2][name:N][dataLen:4][msgpack(items[]):M]
+     */
+    private parseStreamingResponse;
     deleteOne<T>(collection: string, query: QuerySpec<T>): Promise<T>;
     aggregate<T>(collection: string, pipeline: object[], opts?: AggregateOptions): Promise<T[]>;
     upsertBatch<T>(collection: string, batch: BatchSpec<T>): Promise<T[]>;

package/dist/src/types/I_RestInterface.d.ts

@@ -67,6 +67,11 @@ export interface I_RestInterface {
     findByIds<T>(collection: string, ids: Id[]): Promise<T[]>;
     findNewer<T>(collection: string, timestamp: Timestamp | number | string | Date, query?: QuerySpec<T>, opts?: QueryOpts): Promise<T[]>;
     findNewerMany<T>(spec?: GetNewerSpec<T>[]): Promise<Record<string, any[]>>;
+    /** Streaming variant of findNewerMany. Calls onChunk for each batch of items as they arrive. */
+    findNewerManyStream<T>(spec: GetNewerSpec<T>[], onChunk: (collection: string, items: T[]) => Promise<void>, options?: {
+        timeoutMs?: number;
+        signal?: AbortSignal;
+    }): Promise<void>;
     deleteOne<T>(collection: string, query: QuerySpec<T>): Promise<T>;
     /** Izvede agregacijo na serverju */
     aggregate<T>(collection: string, pipeline: object[], opts?: AggregateOptions): Promise<T[]>;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "cry-synced-db-client",
-  "version": "0.1.72",
+  "version": "0.1.73",
   "type": "module",
   "main": "./dist/index.js",
   "module": "./dist/index.js",