@gudhub/core 1.2.4-beta.21 → 1.2.4-beta.23

package/GUDHUB/AppProcessor/AppProcessor.js

@@ -92,10 +92,6 @@ export class AppProcessor {
     return app && app.field_list.length ? app : null;
   }
 
-  getAppFromStorageWithoutChecks(id) {
-    return this.storage.getApp(id);
-  }
-
   addNewAppToStorage(app) {
     return this.storage.getAppsList().push(app);
   }
@@ -124,7 +120,7 @@ export class AppProcessor {
   }
 
   updateAppFieldsAndViews(app) {
-    const storageApp = this.getAppFromStorageWithoutChecks(app.app_id);
+    const storageApp = this.getAppFromStorage(app.app_id);
     app.items_list = storageApp.items_list;
     app.file_list = storageApp.file_list;
     app.from_apps_list = storageApp.from_apps_list;
@@ -239,7 +235,7 @@ export class AppProcessor {
       // Here we get default user's app
       // We do it here to do it only once
       // We do it because we need to initialize WebSockets, which we do in getApp method
-      this.getApp(this.storage.getUser().app_init);
+      // this.getApp(this.storage.getUser().app_init); // commented due to unfinished app_init development, its shape causes exceptions
     }
 
     return app_list;

package/GUDHUB/DataService/AppDataService.js

@@ -39,17 +39,30 @@ export class AppDataService extends DataService {
 //   }
 
             // let res = await this.chunkDataService.getChunks(id, sentData.chunks);
-            let res = await Promise.all(sentData.chunks.map((chunk_id) => this.chunkDataService.getChunk(id, chunk_id)));
-            if (!res) res = [];
+            // let res = await Promise.all(sentData.chunks.map((chunk_id) => this.chunkDataService.getChunk(id, chunk_id)));
 
+            let res = await this.chunkDataService.getMergedChunkForIDList(id, sentData.chunks);
 
 
+            if (
+                res
+            ) {
+                sentData.items_list = await this.gudhub.util.mergeChunks([
+                    res,
+                    sentData,
+                ]);
+            }
 
+            // if (!res) res = [];
 
-            sentData.items_list = await this.gudhub.util.mergeChunks([
-                ...res,
-                sentData,
-            ]);
+
+
+            //TODO should be in new thread too even for Node.js becauses this will block loop
+
+            // sentData.items_list = await this.gudhub.util.mergeChunks([
+            //     ...res,
+            //     sentData,
+            // ]);
 
 
             return sentData;
@@ -69,25 +82,33 @@ export class AppDataService extends DataService {
 
 
         try {
-            let cachedVersion = await this.getCached(id);
+
+            // я там собираюсь склеивать все данные в этом методе, надо посмотреть не будет ли проблем для ноды итп там тоже process вызывается
+            let cachedApp = await this.getCached(id); // this should be merged chunk + app from db
+
+
+            // let cachedMergedChunk = await this.chunkDataService.getCachedMergedChunk(id, cachedApp.chunks);
 
             if (
-                !cachedVersion
+                !cachedApp
+                // cachedMergedChunk ?
             ) {
 
                 //TODO maybe dont wait for chunks at first load
 
                 // let res = await this.chunkDataService.getChunks(id, sentData.chunks);
-                let res = await Promise.all(sentData.chunks.map((chunk_id) => this.chunkDataService.getChunk(id, chunk_id)));
-                if (!res) res = [];
-
-
+                // let res = await Promise.all(sentData.chunks.map((chunk_id) => this.chunkDataService.getChunk(id, chunk_id)));
+                let res = await this.chunkDataService.getMergedChunkForIDList(id, sentData.chunks);
 
-                sentData.items_list = await this.gudhub.util.mergeChunks([
-                    ...res,
-                    sentData,
-                ]);
 
+                if (
+                    res
+                ) {
+                    sentData.items_list = await this.gudhub.util.mergeChunks([
+                        res,
+                        sentData,
+                    ]);
+                }
 
                 // this.putInCache(id, sentData);
     
@@ -96,27 +117,43 @@ export class AppDataService extends DataService {
             }
 
 
-            let newHashesList = sentData.chunks.filter(c => !cachedVersion.chunks.includes(c));
+            // let newHashesList = sentData.chunks.filter(c => !cachedApp.chunks.includes(c));
 
 
             // let self = this;
 
 
             // let res = await this.chunkDataService.getChunks(id, newHashesList);
-            let res = await Promise.all(newHashesList.map((chunk_id) => this.chunkDataService.getChunk(id, chunk_id)));
-            if (!res) res = [];
-
+            // let res = await Promise.all(newHashesList.map((chunk_id) => this.chunkDataService.getChunk(id, chunk_id)));
 
+            let res = await this.chunkDataService.getCachedMergedChunk(id, cachedApp.chunks);
 
+            // if (!res) res = [];
 
-            // this.chunkDataService.getChunks(id, newHashesList).then((res) => {
 
 
+            if (
+                res
+            ) {
                 sentData.items_list = await this.gudhub.util.mergeChunks([
-                    cachedVersion,
-                    ...res,
+                    cachedApp,
+                    res,
                     sentData,
                 ]);
+            } else {
+                sentData.items_list = await this.gudhub.util.mergeChunks([
+                    cachedApp,
+                    sentData,
+                ]);
+            }
+
+
+
+
+            // this.chunkDataService.getChunks(id, newHashesList).then((res) => {
+
+
+
 
 
                 //TODO check this for AppHttpService, seems isnt necessary
@@ -134,18 +171,24 @@ export class AppDataService extends DataService {
             // return cachedVersion;
             
         } catch (error) {
-            // res = await this.chunkDataService.getChunks(id, sentData.chunks);
-            let res = await Promise.all(sentData.chunks.map((chunk_id) => this.chunkDataService.getChunk(id, chunk_id)));
-            if (!res) res = [];
 
+            // seems try catch here isnt handling almost anything, almost identical code here
 
+            // res = await this.chunkDataService.getChunks(id, sentData.chunks);
+            // let res = await Promise.all(sentData.chunks.map((chunk_id) => this.chunkDataService.getChunk(id, chunk_id)));
+            let res = await this.chunkDataService.getMergedChunkForIDList(sentData.chunks);
 
+            // if (!res) res = [];
 
-            sentData.items_list = await this.gudhub.util.mergeChunks([
-                ...res,
-                sentData,
-            ]);
 
+            if (
+                res
+            ) {
+                sentData.items_list = await this.gudhub.util.mergeChunks([
+                    res,
+                    sentData,
+                ]);
+            }
 
             // this.putInCache(id, sentData);
 

package/GUDHUB/DataService/ChunkDataService.js

@@ -2,4 +2,9 @@ import { DataService } from "./DataService.js";
 
 export class ChunkDataService extends DataService {
     //interface for indexeddb and http services
+
+
+    getMergedChunkForIDList(idList) {
+        
+    }
 }

package/GUDHUB/DataService/IndexedDB/IndexedDBAppService.js

@@ -9,6 +9,7 @@ import { chunksConf } from "./chunkDataConf.js";
 import { AppStoreManager } from "./StoreManager/managers.js";
 import { dbName, dbVersion } from "./consts.js";
 import { appsConf } from "./appDataConf.js";
+import { chunksMergeConf } from "./storeManagerConf/chunkCacheStoreManagerConf.js";
 
 
 //this should be global in project
@@ -174,7 +175,7 @@ export class IndexedDBAppServiceForWorker extends AppDataService {
       this.gudhub = gudhub;
 
 
-      this.chunkDataService = new IndexedDBChunkService(req, chunksConf, gudhub);
+      this.chunkDataService = new IndexedDBChunkService(req, chunksMergeConf, gudhub);
 
 
 
@@ -309,15 +310,18 @@ export class IndexedDBAppServiceForWorker extends AppDataService {
     async getApp(id) {
       // if (this.requestCache.has(id)) return this.requestCache.get(id);
 
+
+      // return at first time cached app request?????? no seems for worker its ok
+
       let self = this;
 
       // let dataServiceRequest = this.dataService.getAppWithoutProcessing(id);
       let data = await this.dataService.getAppWithoutProcessing(id);
 
-      let processedData = await self.processRequestedApp(id, data);//here
+      let processedData = await self.processRequestedApp(id, data);//here нужно пересмотреть
 
 
-      await self.putApp(id, processedData);
+      await self.putApp(id, data);
 
 
       return processedData;
@@ -407,7 +411,7 @@ export class IndexedDBAppServiceForWorker extends AppDataService {
 
     async getCached(id) {
 
-      return this.storeManager.getAppStoreManagerVersion(id); //TODO investigate method name
+      return this.storeManager.getApp(id); //TODO investigate method name
 
 
       let self = this;
@@ -547,7 +551,7 @@ export class IndexedDBAppService extends AppDataService {
       }
 
 
-
+      // todo check this for new code
       if (
         e.data.type == 'processData'
       ) {
@@ -621,7 +625,7 @@ export class IndexedDBAppService extends AppDataService {
   }
 
   async getApp(id) {//TODO -> get?
-    if (this.requestCache.has(id)) return this.requestCache.get(id);
+    if (this.requestCache.has(id)) return this.requestCache.get(id); // по этому коду есть вопрос - вызовы getApp  кешируются выше и тогда выходит возвращается старая версия?
 
 
     let self = this;
@@ -633,8 +637,27 @@ export class IndexedDBAppService extends AppDataService {
     if (
       hasId
     ) {
-      let getFromDBPrms = this.getCached(id);
 
+      // let promiseObject = Promise.
+
+      let cached = await this.getCached(id); // todo here should be merged with cached mergedChunk
+      
+      // repetitive operation again, seems should be in getCached
+      let cachedMergedChunk = await this.chunkDataService.getCachedMergedChunk(id, cached.chunks);
+
+
+      // let mergedData = this.gudhub
+
+
+      cached.items_list = await this.gudhub.util.mergeChunks([
+        cachedMergedChunk,
+        cached,
+      ]);
+
+
+      let getFromDBPrms = Promise.resolve(cached);
+      
+      // let getFromDBPrms = await this.getCached(id); // todo here should be merged with cached mergedChunk
       this.requestCache.set(id, getFromDBPrms);
 
       this.appRequestAndProcessWorker.postMessage({ type: 'processData', id: id });
@@ -713,7 +736,7 @@ export class IndexedDBAppService extends AppDataService {
   async getCached(id) {//TODO -> getFromStore?
 
 
-    return this.storeManager.getAppStoreManagerVersion(id); //TODO investigate method name
+    return this.storeManager.getApp(id); //TODO investigate method name
 
     let self = this;
 

package/GUDHUB/DataService/IndexedDB/IndexedDBChunkService.js

@@ -2,7 +2,7 @@ import { ChunkDataService } from "../ChunkDataService.js"; // removed "ChunkCach
 import { ChunkHttpService } from "../httpService/ChunkHttpService.js";
 import { objectAssignWithOverride } from "../utils.js";
 import { IndexedDBService } from "./IndexedDBService.js";
-import { ChunkCacheStoreManager, ChunksStoreManager } from "./StoreManager/managers.js";
+import { ChunksStoreManager, MergedChunksStoreManager } from "./StoreManager/managers.js";
 
 
 //this should be global in project
@@ -31,7 +31,11 @@ export class IndexedDBChunkService extends ChunkDataService {
       this.gudhub = gudhub;
 
 
-      this.storeManager = new ChunksStoreManager;
+      // this.storeManager = new ChunksStoreManager;
+      this.storeManager = new MergedChunksStoreManager;
+
+
+      // this.storeManager = new ChunksStoreManager;
 
 
       // objectAssignWithOverride(this, indexDBService);
@@ -53,9 +57,121 @@ export class IndexedDBChunkService extends ChunkDataService {
       }
     }
 
+    async getCachedMergedChunk(id, chunkHashesList) {
+      try {
+        let cachedData = await this.storeManager.get(id);
+
+        return cachedData.mergedData;
+      } catch (error) {
+        // let newHashes = chunkHashesList.filter(c => !cachedData.hashes.includes(c));
+
+
+        const requests = chunkHashesList.map(hash => this.dataService.getChunk(id, hash));
+    
+        const newChunks = await Promise.all(requests);
+
+
+        let mergedData = await this.gudhub.util.mergeChunks(newChunks);
+
+        this.putMergedChunks(id, {hashes: chunkHashesList, mergedData: {items_list: mergedData}});
+
+        return {items_list: mergedData};
+      }
+    }
+
+    async getMergedChunkForIDList(id, chunkHashesList) {
+
+
+      //TODO promise cache
+
+      // TODO also try catch maybe
+
+
+      try {
+        let cachedData = await this.storeManager.get(id);
+
+
+        if (
+          chunkHashesList.every(h1 => cachedData.hashes.includes(h1))
+        ) {
+          return cachedData.mergedData;
+        }
+
+          let newHashes = chunkHashesList.filter(c => !cachedData.hashes.includes(c));
+
+
+          const requests = newHashes.map(hash => this.dataService.getChunk(id, hash));
+      
+          const newChunks = await Promise.all(requests);
+
+
+          let mergedData = await this.gudhub.util.mergeChunks([
+            cachedData.mergedData,
+            ...newChunks,
+          ]);
+
+        this.putMergedChunks(id, {hashes: chunkHashesList, mergedData: {items_list: mergedData}});
+
+
+        return {items_list: mergedData};
+      } catch (error) {
+        // let newHashes = chunkHashesList.filter(c => !cachedData.hashes.includes(c));
+
+
+        const requests = chunkHashesList.map(hash => this.dataService.getChunk(id, hash));
+    
+        const newChunks = await Promise.all(requests);
+
+
+        let mergedData = await this.gudhub.util.mergeChunks(newChunks);
+
+        this.putMergedChunks(id, {hashes: chunkHashesList, mergedData: {items_list: mergedData}});
+
+        return {items_list: mergedData};
+      }
+    }
+
+
+
+    // async function fetchAndMergeArrays(urls) {
+    //   try {
+    //     // Create an array of fetch promises
+    //     const requests = urls.map(url => fetch(url).then(response => response.json()));
+    
+    //     // Wait for all requests to complete
+    //     const results = await Promise.all(requests);
+    
+    //     // Merge all arrays
+    //     const mergedArray = results.flat(); // or use reduce: results.reduce((acc, arr) => acc.concat(arr), []);
+    
+    //     return mergedArray;
+    //   } catch (error) {
+    //     console.error('Error fetching data:', error);
+    //     throw error;
+    //   }
+    // }
+    
+    // // Example usage:
+    // const urls = ['https://api.example.com/data1', 'https://api.example.com/data2'];
+    // fetchAndMergeArrays(urls)
+    //   .then(mergedArray => console.log(mergedArray))
+    //   .catch(error => console.error('Error:', error));
+    
+
     //todo also optimize chunks transactions etc
 
 
+
+    async putMergedChunks(id, data) {
+      return this.storeManager.put(id, data);
+    }
+
+
+    // async getMergedChunks(id, data) {
+    //   return this.storeManager.get(id, data);
+    // }
+
+
     async putChunk(id, data) {
       return this.storeManager.putChunk(id, data);
 
@@ -132,7 +248,7 @@ export class IndexedDBChunkService extends ChunkDataService {
 
       try {
 
-        let pr = this.storeManager.getChunkStoreManagerVersion(id);
+        let pr = this.storeManager.getChunk(id);
 
         // let self = this;
 

package/GUDHUB/DataService/IndexedDB/StoreManager/managers.js

@@ -2,6 +2,7 @@ import { appsConf } from "../appDataConf.js";
 import { chunksConf } from "../chunkDataConf.js";
 // import { dbName } from "../consts";
 import { gudhubIndexedDBManager } from "../IndexedDBService.js";
+import { chunksMergeConf } from "../storeManagerConf/chunkCacheStoreManagerConf.js";
 import { BaseStoreManager } from "./BaseStoreManager.js";
 
 export class AppStoreManager extends BaseStoreManager {
@@ -9,10 +10,7 @@ export class AppStoreManager extends BaseStoreManager {
     super(gudhubIndexedDBManager, appsConf.store);
   }
 
-  //use item methods from BaseStoreManager !!!!!!!!!!
-
-  //TODO getApp overrides getApp from IndexedDBAppService
-  async getAppStoreManagerVersion(id) {
+  async getApp(id) {
 
     return this.getItem(id);
 
@@ -50,7 +48,7 @@ export class ChunksStoreManager extends BaseStoreManager {
   }
 
 
-  async getChunkStoreManagerVersion(id) {
+  async getChunk(id) {
 
     return this.getItem(id);
 
@@ -82,10 +80,34 @@ export class ChunksStoreManager extends BaseStoreManager {
   // Additional methods specific to ChunksStoreManager if necessary
 }
 
-export class ChunkCacheStoreManager extends BaseStoreManager {
-  constructor(dbManager) {
-    super(dbManager, 'myDatabase', 'chunkCacheStore');
+export class MergedChunksStoreManager extends BaseStoreManager {
+  constructor() {
+    super(gudhubIndexedDBManager, chunksMergeConf.store);
+  }
+
+  async get(id) {
+
+    return this.getItem(id);
+
+    // return dbManager.enqueueOperation('apps', 'readonly', (store) => store.get(id));
+  }
+
+  async add(id, chunk) {
+    return this.addItem(id, chunk);
+
+    // return dbManager.enqueueOperation('apps', 'readwrite', (store) => store.add(app));
   }
 
-  // Additional methods specific to ChunkCacheStoreManager if necessary
+  //TODO update here ok?
+  async put(id, chunk) {
+    return this.putItem(id, chunk);
+
+    // return dbManager.enqueueOperation('apps', 'readwrite', (store) => store.add(app));
+  }
+
+  async delete(id) {
+    return this.deleteItem(id);
+
+    // return dbManager.enqueueOperation('apps', 'readwrite', (store) => store.delete(id));
+  }
 }

package/GUDHUB/DataService/IndexedDB/appDataConf.js

@@ -1,5 +1,8 @@
 // import { dbName } from "./consts.js";
 
+
+// move to folder also remove chunksConf
+
 export const appsConf = {
     // dbName,
     // dbVersion,

package/GUDHUB/DataService/IndexedDB/appRequestWorker.js

@@ -3,11 +3,15 @@ import { appDataServiceConf } from "../export.js";
 import { GudHub } from "../../gudhub.js";
 import { IndexedDBAppServiceForWorker } from "../IndexedDB/IndexedDBAppService.js";
 import { appsConf } from "../IndexedDB/appDataConf.js";
+import { IndexedDBChunkService } from "./IndexedDBChunkService.js";
+import { chunksConf } from "./chunkDataConf.js";
+import { chunksMergeConf } from "./storeManagerConf/chunkCacheStoreManagerConf.js";
 
 
 
 let gudhub = {};
 let appDataService = {};
+let chunkDataService = {};
 
 
 
@@ -17,8 +21,13 @@ let appDataService = {};
 self.onmessage = async function(e) {
   if (e.data.type === 'init') {
 
+
+    e.data.gudhubSerialized.options.initWebsocket = false;
+
     gudhub = new GudHub(e.data.gudhubSerialized.authKey, e.data.gudhubSerialized.options);
     appDataService = new IndexedDBAppServiceForWorker(gudhub.req, appsConf, gudhub);
+    chunkDataService = new IndexedDBChunkService(gudhub.req, chunksMergeConf, gudhub);
+
 
   }
 
@@ -28,9 +37,36 @@ self.onmessage = async function(e) {
 
         //todo cached could not exist !!!!!!!!!!!!!!!!!!!
 
-    let cached = await appDataService.getCached(e.data.id);
+    let cached = await appDataService.getCached(e.data.id); // merge ?
+
+
+
+    // TODO this is repetitive operation for getCached, maybe put all code in process or in getCached
+
+    let res = await chunkDataService.getCachedMergedChunk(e.data.id, cached.chunks);
+
+    // if (!res) res = [];
+
+
+    // TODO check that res always exists
+
+
+    if (
+        res
+    ) {
+        cached.items_list = await gudhub.util.mergeChunks([
+            // cachedApp,
+            res,
+            cached,
+        ]);
+    } else {
+        // sentData.items_list = await this.gudhub.util.mergeChunks([
+        //     cachedApp,
+        //     sentData,
+        // ]);
+    }
 
-    let nextVersion = await appDataService.getApp(e.data.id);
+    let nextVersion = await appDataService.getApp(e.data.id); //todo check that here most recent always
 
 
 
@@ -79,7 +115,7 @@ self.onmessage = async function(e) {
     e.data.type === 'requestApp'
   ) {
 
-    let data = await appDataService.getApp(e.data.id);
+    let data = await appDataService.getApp(e.data.id); //todo check process request method seems there i return cahced version not most recent data from server
 
 
     this.postMessage({id: e.data.id, type: e.data.type, data});

package/GUDHUB/DataService/httpService/ChunkHttpService.js

@@ -17,6 +17,22 @@ export class ChunkHttpService extends ChunkDataService {
     async getChunk(app_id, id) {
       return this.chunkApi.getChunk(app_id, id);
     }
+
+    async getChunksForId(app_id) {
+      // maybe use app get api for now and return app.chunks from there
+
+      return []; // list of chunks
+    }
+
+
+    getCachedMergedChunk(id, chunkHashesList) {
+
+    }
+
+
+    getMergedChunkForIDList(id, chunkHashesList) {
+      
+    }
   
   
     putChunk() {

package/GUDHUB/ItemProcessor/ItemProcessor.js

@@ -120,7 +120,7 @@ export class ItemProcessor {
 
   //here
 
-  ///
+  /////
   handleItemsChange(id, prevVersion, nextVersion) {
     let compareRes = this.util.compareItems(prevVersion.items_list, nextVersion.items_list);
 

package/GUDHUB/Utils/Utils.js

@@ -225,6 +225,7 @@ export class Utils {
     return compareObjects(obj1, obj2);
   }
   
+  //
   compareAppsItemsLists(items_list1, items_list2, callback) {
     const chunkWorkerBlob = new Blob([compare_items_lists_Worker()], {
       type: "application/javascript",

package/GUDHUB/Utils/merge_chunks/merge_chunks.js

@@ -5,6 +5,10 @@ import { mergeChunks as mergeChunksByMainThread, mergeChunksWorker } from "./mer
 
 
 export function mergeChunks(chunks) {
+  //dont launch new thread because already used in worker.
+  // Main reason is that renderer process allocates memory for structured clones for communication between trheads
+  return mergeChunksByMainThread(chunks);
+
   if (
     typeof Worker !== 'undefined'
   ) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@gudhub/core",
-  "version": "1.2.4-beta.21",
+  "version": "1.2.4-beta.23",
   "description": "",
   "main": "index.js",
   "scripts": {