@nu-art/thunderstorm-backend 0.400.5

package/modules/sync-env/ModuleBE_SyncEnv.js

@@ -0,0 +1,212 @@
+import { ApiException, arrayToMap, BadImplementationException, Dispatcher, Minute, Module, MUSTNeverHappenException, RuntimeModules } from '@nu-art/ts-common';
+import { ModuleBE_Firebase } from '@nu-art/firebase-backend';
+import { addRoutes } from '../ModuleBE_APIs.js';
+import { createBodyServerApi, createQueryServerApi } from '../../core/typed-api.js';
+import { ApiDef_SyncEnv, HeaderKey_Authorization, HttpMethod } from '@nu-art/thunderstorm-shared';
+import { AxiosHttpModule } from '../http/AxiosHttpModule.js';
+import { MemKey_HttpRequest } from '../server/consts.js';
+import { Storm } from '../../core/Storm.js';
+import { ModuleBE_BackupDocDB } from '../../_entity/backup-doc/index.js';
+import { Transform, Writable } from 'stream';
+import { HttpCodes } from '@nu-art/ts-common/core/exceptions/http-codes';
+const dispatch_OnSyncEnvCompleted = new Dispatcher('__onSyncEnvCompleted');
+class ModuleBE_SyncEnv_Class extends Module {
+    constructor() {
+        super();
+        this.setDefaultConfig({ maxBatch: 500 });
+    }
+    init() {
+        super.init();
+        addRoutes([
+            createBodyServerApi(ApiDef_SyncEnv.vv1.syncToEnv, this.pushToEnv),
+            createBodyServerApi(ApiDef_SyncEnv.vv1.syncFromEnvBackup, this.syncFromEnvBackup),
+            createQueryServerApi(ApiDef_SyncEnv.vv1.getLatestBackup, this.getLatestBackupId),
+            createQueryServerApi(ApiDef_SyncEnv.vv1.createBackup, this.createBackup),
+            createQueryServerApi(ApiDef_SyncEnv.vv1.fetchBackupMetadata, this.fetchBackupMetadata),
+            createQueryServerApi(ApiDef_SyncEnv.vv1.syncFirebaseFromBackup, this.syncFirebaseFromBackup),
+        ]);
+    }
+    fetchBackupMetadata = async (queryParams) => {
+        const backupInfo = await this.getBackupInfo(queryParams);
+        if (!backupInfo)
+            throw new ApiException(404, 'backup file not found');
+        if (!backupInfo.metadata)
+            throw new ApiException(404, 'No metadata found on this backup');
+        return {
+            ...backupInfo.metadata,
+            remoteCollectionNames: (RuntimeModules()
+                .filter((module) => !!module.dbDef?.dbKey)).map(_module => _module.dbDef.dbKey)
+        };
+    };
+    async pushToEnv(body) {
+        const remoteUrls = {
+            dev: 'https://us-central1-shopify-manager-tool-dev.cloudfunctions.net/api',
+            prod: 'https://mng.be.petitfawn.com'
+        };
+        const url = remoteUrls[body.env];
+        const sessionId = MemKey_HttpRequest.get().headers[HeaderKey_Authorization];
+        const module = RuntimeModules().find((module) => module.dbModule?.dbDef?.dbKey === body.moduleName);
+        const upsertAll = module.apiDef.v1.upsertAll;
+        const response = await AxiosHttpModule
+            .createRequest({ ...upsertAll, fullUrl: url + '/' + upsertAll.path, timeout: 5 * Minute })
+            .setBody(body.items)
+            .setUrlParams(body.items)
+            .addHeader(HeaderKey_Authorization, sessionId)
+            .executeSync(true);
+        console.log(response);
+    }
+    createBackup = async () => {
+        return ModuleBE_BackupDocDB.initiateBackup(true);
+    };
+    getLatestBackupId = async () => {
+        const backups = await ModuleBE_BackupDocDB.collection.query.custom({ orderBy: [{ key: "__created", order: "desc" }], limit: 1 });
+        const latestBackup = backups[0];
+        if (!latestBackup)
+            throw HttpCodes._4XX.ENTITY_DOESNT_EXISTS("No backup found");
+        const latestBackupId = latestBackup?._id;
+        return { latestBackupId: latestBackupId };
+    };
+    syncFromEnvBackup = async (body) => {
+        if (!this.config.allowSyncEnv)
+            throw new MUSTNeverHappenException(`SyncEnv is disabled on this env- to sync into this env, add 'allowSyncEnv: true'.`);
+        //CleanSync means deleting collections before syncing them
+        if (!this.config.allowCleanSync && body.cleanSync)
+            throw new MUSTNeverHappenException(`CleanSync is disabled on this env- to CleanSync into this env, add 'allowCleanSync: true'.`);
+        if (Storm.getInstance().getEnvironment().toLowerCase() === 'prod' && body.env.toLowerCase() !== 'prod')
+            throw new MUSTNeverHappenException('MUST NEVER SYNC ENV THAT IS NOT PROD TO PROD!!');
+        if (this.config.allowedEnvsToSyncFrom && !this.config.allowedEnvsToSyncFrom.includes(body.env))
+            throw new MUSTNeverHappenException(`Env ${Storm.getInstance().getEnvironment()
+                .toLowerCase()} doesn't have env ${body.env} in it's allowedEnvsToSyncFrom list.`);
+        this.logInfoBold('Received API call Fetch From Env!');
+        this.logInfo(`Origin env: ${body.env}, backupId: ${body.backupId}`);
+        let startTime = undefined; // required for log
+        let endTime = undefined; // required for log
+        if (this.config.shouldBackupBeforeSync) {
+            this.logInfo(`----  Creating Backup... ----`);
+            startTime = performance.now(); // required for log
+            await this.createBackup();
+            endTime = performance.now(); // required for log
+            this.logInfo(`Backup took ${((endTime - startTime) / 1000).toFixed(3)} seconds`);
+        }
+        if (body.cleanSync) {
+            this.logInfo(`----  Cleaning Collections From DB... ----`);
+            //Delete all modules specified for syncing
+            const modulesToDelete = RuntimeModules().filter((module) => body.selectedModules.includes(module.dbDef?.dbKey));
+            for (const module of modulesToDelete) {
+                await module.collection.delete.yes.iam.sure.iwant.todelete.the.collection.delete();
+                this.logInfo(`----  Cleaned Collection ${module.dbDef.dbKey} ----`);
+            }
+        }
+        //Prepare Syncing data
+        const backupInfo = await this.getBackupInfo(body);
+        const stream = await ModuleBE_BackupDocDB.createBackupReadStream(backupInfo);
+        const collectionFilter = new SyncCollectionFilter(body.selectedModules);
+        const collectionWriter = new CollectionBatchWriter(body.chunkSize);
+        this.logInfo(`----  Syncing Collections From Backup... ----`);
+        startTime = performance.now();
+        await new Promise((resolve, reject) => {
+            stream
+                .pipe(collectionFilter)
+                .pipe(collectionWriter)
+                .on('finish', () => resolve());
+        });
+        endTime = performance.now();
+        this.logInfo(`Syncing Collections took ${((endTime - startTime) / 1000).toFixed(3)} seconds`);
+        this.logInfo(`----  Syncing Other Modules... ----`);
+        await dispatch_OnSyncEnvCompleted.dispatchModuleAsync(body.env, this.config.urlMap[body.env], this.config.sessionMap[body.env]);
+        this.logInfo(`---- DONE Syncing Other Modules----`);
+        if (this.config.shouldBackupBeforeSync && endTime !== undefined && startTime !== undefined)
+            this.logInfo(`(Backup took ${((endTime - startTime) / 1000).toFixed(3)} seconds)`);
+    };
+    async getBackupInfo(queryParams) {
+        const { backupId, env } = queryParams;
+        if (!env)
+            throw new BadImplementationException(`Did not receive env in the fetch from env api call!`);
+        return ModuleBE_BackupDocDB.getBackupInfo(backupId, this.config.urlMap[env], this.config.sessionMap[env]);
+    }
+    syncFirebaseFromBackup = async (queryParams) => {
+        try {
+            this.logDebug('Getting the firebase backup file');
+            const firebaseSessionAdmin = ModuleBE_Firebase.createAdminSession();
+            const backupInfo = await this.getBackupInfo(queryParams);
+            const database = firebaseSessionAdmin.getDatabase();
+            this.logDebug('Reading the file from storage');
+            const signedUrlDef = {
+                method: HttpMethod.GET,
+                path: '',
+                fullUrl: backupInfo.firebaseSignedUrl
+            };
+            const firebaseFile = await AxiosHttpModule
+                .createRequest(signedUrlDef)
+                .executeSync();
+            this.logDebug('Setting the file in firebase database');
+            await database.set('/', firebaseFile);
+        }
+        catch (err) {
+            throw new ApiException(500, err);
+        }
+    };
+}
+export const ModuleBE_SyncEnv = new ModuleBE_SyncEnv_Class();
+class SyncCollectionFilter extends Transform {
+    allowedDbKeys;
+    constructor(allowedDbKeys) {
+        super({ objectMode: true });
+        this.allowedDbKeys = allowedDbKeys;
+    }
+    _transform(chunk, encoding, callback) {
+        if (this.allowedDbKeys.includes(chunk.dbKey)) {
+            this.push(chunk);
+        }
+        callback();
+    }
+}
+class CollectionBatchWriter extends Writable {
+    itemCount = 0;
+    paginationSize;
+    firestore;
+    batchWriter;
+    modules;
+    constructor(paginationSize) {
+        super({ objectMode: true });
+        this.paginationSize = paginationSize;
+        const firebaseSessionAdmin = ModuleBE_Firebase.createAdminSession();
+        this.firestore = firebaseSessionAdmin.getFirestoreV3().firestore;
+        this.batchWriter = this.firestore.batch();
+        this.modules = arrayToMap(RuntimeModules()
+            .filter((module) => !(!module || !module.dbDef)), module => module.dbDef.dbKey);
+    }
+    async _write(chunk, encoding, callback) {
+        try {
+            const module = this.modules[chunk.dbKey];
+            if (!module) {
+                ModuleBE_SyncEnv.logWarning(`Could not get module for chunk with dbKey ${chunk.dbKey}`);
+                callback();
+            }
+            const collectionName = module.dbDef.backend.name;
+            const docRef = this.firestore.doc(`${collectionName}/${chunk._id}`);
+            const data = JSON.parse(chunk.document);
+            this.batchWriter.set(docRef, data);
+            this.itemCount++;
+            if (this.itemCount === this.paginationSize) {
+                const prevBatchWriter = this.batchWriter;
+                this.batchWriter = this.firestore.batch();
+                this.itemCount = 0;
+                await prevBatchWriter.commit();
+            }
+            callback();
+        }
+        catch (error) {
+            callback(error instanceof Error ? error : new Error(String(error)));
+        }
+    }
+    async _final(callback) {
+        try {
+            await this.batchWriter.commit();
+            callback();
+        }
+        catch (err) {
+            callback(err);
+        }
+    }
+}

package/modules/sync-manager/ModuleBE_SyncManager.d.ts

@@ -0,0 +1,63 @@
+import { FirestoreQuery } from '@nu-art/firebase-shared';
+import { DB_Object, Module, ResolvableContent, TypedMap, UniqueId } from '@nu-art/ts-common';
+import { firestore } from 'firebase-admin';
+import { ModuleBE_BaseDB } from '../db-api-gen/ModuleBE_BaseDB.js';
+import { DeltaSyncModule, FullSyncModule, NoNeedToSyncModule, SyncDataFirebaseState } from '@nu-art/thunderstorm-shared/sync-manager/types';
+import { DBProto_DeletedDoc } from '@nu-art/thunderstorm-shared';
+import { OnSyncEnvCompleted } from '../sync-env/ModuleBE_SyncEnv.js';
+import { OnModuleCleanupV2 } from '../../_entity.js';
+import { FirestoreCollectionV3 } from '@nu-art/firebase-backend/firestore-v3/FirestoreCollectionV3';
+import Transaction = firestore.Transaction;
+type DeletedDBItem = DB_Object & {
+    __collectionName: string;
+    __docId: UniqueId;
+};
+type Config = {
+    retainDeletedCount: number;
+};
+/**
+ * # ModuleBE_SyncManager
+ *
+ * ## <ins>Description:</ins>
+ * This module manages all the {@link BaseDB_Module} updates and deleted items in order to allow incremental sync of items with clients
+ *
+ * ## <ins>Config:</ins>
+ *
+ * ```json
+ * "ModuleBE_SyncManager" : {
+ *   	retainDeletedCount: 100
+ * }
+ * ```
+ */
+export declare class ModuleBE_SyncManager_Class extends Module<Config> implements OnModuleCleanupV2, OnSyncEnvCompleted {
+    collection: FirestoreCollectionV3<DBProto_DeletedDoc>;
+    private database;
+    private dbModules;
+    smartSyncApi: import("../_imports.js")._ServerBodyApi<import("@nu-art/thunderstorm-shared").BodyApi<{
+        modules: (NoNeedToSyncModule | DeltaSyncModule | FullSyncModule)[];
+    }, {
+        modules: import("@nu-art/thunderstorm-shared/sync-manager/types").SyncDbData[];
+    }>>;
+    private resolvableFirebaseBasePath;
+    constructor();
+    __onSyncEnvCompleted(env: string, baseUrl: string, requiredHeaders: TypedMap<string>): Promise<void>;
+    init(): void;
+    private calculateSmartSync;
+    getOrCreateSyncData: () => Promise<SyncDataFirebaseState>;
+    private prepareItemToDelete;
+    onItemsDeleted(collectionName: string, items: DB_Object[], uniqueKeys?: string[], transaction?: Transaction): Promise<void>;
+    queryDeleted(collectionName: string, query: FirestoreQuery<DB_Object>): Promise<DeletedDBItem[]>;
+    __onCleanupInvokedV2: () => Promise<void>;
+    getFullSyncData: () => Promise<SyncDataFirebaseState>;
+    setLastUpdated(collectionName: string, lastUpdated: number): Promise<void>;
+    setOldestDeleted(collectionName: string, oldestDeleted: number): Promise<void>;
+    setModuleFilter: (filter: (modules: (ModuleBE_BaseDB<any>)[]) => Promise<(ModuleBE_BaseDB<any>)[]>) => void;
+    /**
+     * Set function that allows to set a custom resolver for the rtdb node path
+     * @param resolvablePath The resolver for the node path
+     */
+    setResolvablePath: (resolvablePath: ResolvableContent<string>) => void;
+    private filterModules;
+}
+export declare const ModuleBE_SyncManager: ModuleBE_SyncManager_Class;
+export {};

package/modules/sync-manager/ModuleBE_SyncManager.js

@@ -0,0 +1,254 @@
+/*
+ * Database API Generator is a utility library for Thunderstorm.
+ *
+ * Given proper configurations it will dynamically generate APIs to your Firestore
+ * collections, will assert uniqueness and restrict deletion... and more
+ *
+ * Copyright (C) 2020 Adam van der Kruk aka TacB0sS
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import { _EmptyQuery } from '@nu-art/firebase-shared';
+import { ModuleBE_Firebase } from '@nu-art/firebase-backend';
+import { __stringify, arrayToMap, currentTimeMillis, dispatch_onApplicationException, exists, filterDuplicates, filterInstances, LogLevel, Module, resolveContent, RuntimeModules } from '@nu-art/ts-common';
+import { createBodyServerApi } from '../../core/typed-api.js';
+import { addRoutes } from '../ModuleBE_APIs.js';
+import { SmartSync_DeltaSync, SmartSync_FullSync, SmartSync_UpToDateSync } from '@nu-art/thunderstorm-shared/sync-manager/types';
+import { DBDef_DeletedDoc } from '@nu-art/thunderstorm-shared';
+import { ApiDef_SyncManager } from '@nu-art/thunderstorm-shared/sync-manager/apis';
+/**
+ * # ModuleBE_SyncManager
+ *
+ * ## <ins>Description:</ins>
+ * This module manages all the {@link BaseDB_Module} updates and deleted items in order to allow incremental sync of items with clients
+ *
+ * ## <ins>Config:</ins>
+ *
+ * ```json
+ * "ModuleBE_SyncManager" : {
+ *   	retainDeletedCount: 100
+ * }
+ * ```
+ */
+export class ModuleBE_SyncManager_Class extends Module {
+    collection;
+    database;
+    dbModules;
+    smartSyncApi;
+    resolvableFirebaseBasePath = `/state/${this.getName()}`;
+    constructor() {
+        super();
+        this.setMinLevel(LogLevel.Debug);
+        this.smartSyncApi = createBodyServerApi(ApiDef_SyncManager.v1.smartSync, this.calculateSmartSync);
+        this.setDefaultConfig({ retainDeletedCount: 1000 });
+    }
+    async __onSyncEnvCompleted(env, baseUrl, requiredHeaders) {
+        await this.database.delete(resolveContent(this.resolvableFirebaseBasePath));
+    }
+    init() {
+        const firestore = ModuleBE_Firebase.createAdminSession().getFirestoreV3();
+        this.collection = firestore.getCollection(DBDef_DeletedDoc);
+        this.dbModules = RuntimeModules().filter(module => (module.ModuleBE_BaseDBV2));
+        this.database = ModuleBE_Firebase.createAdminSession().getDatabase();
+        addRoutes([this.smartSyncApi]);
+    }
+    calculateSmartSync = async (body) => {
+        const frontendCollectionNames = body.modules.map(item => item.dbKey);
+        this.logVerbose(`Modules wanted: ${__stringify(frontendCollectionNames)}`);
+        const permissibleModules = await this.filterModules(this.dbModules.filter(dbModule => frontendCollectionNames.includes(dbModule.dbDef.dbKey)));
+        const modulesAllowed = filterInstances(permissibleModules.map(_module => _module.dbDef.dbKey));
+        this.logVerbose(`Modules found: ${__stringify(modulesAllowed)}`);
+        this.logVerbose(`Modules not found: ${frontendCollectionNames.filter(collectionName => modulesAllowed.includes(collectionName))}`);
+        const dbNameToModuleMap = arrayToMap(permissibleModules, (item) => item.dbDef.dbKey);
+        const syncDataResponse = [];
+        const upToDateSyncData = await this.getOrCreateSyncData();
+        // For each module, create the response, which says what type of sync it needs: none, delta or full.
+        await Promise.all(body.modules.map(async (syncRequest) => {
+            const moduleToCheck = dbNameToModuleMap[syncRequest.dbKey];
+            if (!moduleToCheck)
+                return this.logError(`Calculating collections to sync, failing to find dbKey: ${syncRequest.dbKey}`);
+            const remoteSyncData = upToDateSyncData[syncRequest.dbKey] ?? { lastUpdated: 0, oldestDeleted: 0 };
+            // Local has no sync data, or it's too old - tell local to send a full sync request for this module
+            if (syncRequest.lastUpdated === 0 && remoteSyncData.lastUpdated > 0 || exists(remoteSyncData.oldestDeleted) && remoteSyncData.oldestDeleted > syncRequest.lastUpdated) {
+                // full sync
+                syncDataResponse.push({
+                    dbKey: syncRequest.dbKey,
+                    sync: SmartSync_FullSync,
+                    lastUpdated: remoteSyncData.lastUpdated,
+                });
+                return;
+            }
+            // Same lastUpdated timestamp in local and remote, no need to sync
+            if (syncRequest.lastUpdated === remoteSyncData.lastUpdated) {
+                // no sync
+                syncDataResponse.push({
+                    dbKey: syncRequest.dbKey,
+                    sync: SmartSync_UpToDateSync,
+                    lastUpdated: remoteSyncData.lastUpdated
+                });
+                return;
+            }
+            // Different lastUpdated timestamp in local and remote - tell local to send a delta sync request for this module
+            if (syncRequest.lastUpdated !== remoteSyncData.lastUpdated) {
+                // delta sync
+                let toUpdate = [];
+                try {
+                    toUpdate = await moduleToCheck.query.where({ __updated: { $gte: syncRequest.lastUpdated } });
+                }
+                catch (e) {
+                    this.logWarningBold(`Module assumed to be normal DB module: ${moduleToCheck.getName()}, collection:${moduleToCheck.dbDef.dbKey}`);
+                    throw e;
+                }
+                const itemsToReturn = {
+                    toUpdate: toUpdate,
+                    toDelete: await this.queryDeleted(syncRequest.dbKey, { where: { __updated: { $gte: syncRequest.lastUpdated } } })
+                };
+                syncDataResponse.push({
+                    dbKey: syncRequest.dbKey,
+                    sync: SmartSync_DeltaSync,
+                    lastUpdated: remoteSyncData.lastUpdated,
+                    items: itemsToReturn
+                });
+            }
+        }));
+        return {
+            modules: syncDataResponse
+        };
+    };
+    getOrCreateSyncData = async () => {
+        this.logVerbose('Current node path', `${resolveContent(this.resolvableFirebaseBasePath)}/syncData`);
+        const syncDataRef = this.database.ref(`${resolveContent(this.resolvableFirebaseBasePath)}/syncData`);
+        const rtdbSyncData = await syncDataRef.get({});
+        const dbModuleDbKeys = filterInstances(this.dbModules.map(module => module.dbDef.dbKey));
+        this.logVerbose(`BE DB Modules: ${__stringify(dbModuleDbKeys)}`);
+        const missingModules = this.dbModules.filter(dbModule => {
+            const dbBE_SyncData = rtdbSyncData[dbModule.dbDef.dbKey];
+            if (!dbBE_SyncData)
+                return true;
+            // const dbFE_SyncData = body.modules.find(module => module.dbName === dbModule.getCollectionName());
+            // if (!dbFE_SyncData)
+            return false;
+            // return dbFE_SyncData.lastUpdated > dbBE_SyncData.lastUpdated;
+        });
+        if (missingModules.length) {
+            this.logWarning(`Syncing missing modules: `, missingModules.map(module => module.dbDef.dbKey).sort());
+            const query = { limit: 1, orderBy: [{ key: '__updated', order: 'desc' }] };
+            const newestItems = (await Promise.all(missingModules.map(async (missingModule) => {
+                try {
+                    return (await missingModule.query.unManipulatedQuery(query))[0];
+                }
+                catch (e) {
+                    dispatch_onApplicationException.dispatchModule(e, this);
+                    this.logError(e);
+                }
+            })));
+            newestItems.forEach((item, index) => rtdbSyncData[missingModules[index].dbDef.dbKey] = { lastUpdated: item?.__updated || 0 });
+            await syncDataRef.set(rtdbSyncData);
+        }
+        return rtdbSyncData;
+    };
+    prepareItemToDelete = (collectionName, item, uniqueKeys = ['_id']) => {
+        const { _id, __updated, __created, _v } = item;
+        const deletedItem = {
+            __docId: _id,
+            __updated,
+            __created,
+            _v,
+            __collectionName: collectionName
+        };
+        uniqueKeys.forEach(key => {
+            //Don't replace the _id, some items in the system have a calculated _id and can be deleted and created over and over.
+            if (key === '_id')
+                return;
+            // @ts-ignore
+            deletedItem[key] = item[key] || '';
+        });
+        return deletedItem;
+    };
+    async onItemsDeleted(collectionName, items, uniqueKeys = ['_id'], transaction) {
+        const toInsert = items.map(item => this.prepareItemToDelete(collectionName, item, uniqueKeys));
+        const now = currentTimeMillis();
+        toInsert.forEach(item => item.__updated = now);
+        await this.collection.create.all(toInsert, transaction);
+        const deletedCountRef = this.database.ref(`${resolveContent(this.resolvableFirebaseBasePath)}/deletedCount`);
+        let deletedCount = await deletedCountRef.get(0);
+        deletedCount += items.length;
+        await deletedCountRef.set(deletedCount);
+    }
+    async queryDeleted(collectionName, query) {
+        const finalQuery = {
+            ...query,
+            where: { ...query.where, __collectionName: collectionName }
+        };
+        const deletedItems = await this.collection.query.custom(finalQuery);
+        deletedItems.forEach(_item => _item._id = _item.__docId || _item._id);
+        return deletedItems;
+    }
+    __onCleanupInvokedV2 = async () => {
+        if (!this.config.retainDeletedCount)
+            return this.logWarning('Will not run cleanup of deleted values:\n  No "retainDeletedCount" was specified in config..');
+        const deletedCountRef = this.database.ref(`${resolveContent(this.resolvableFirebaseBasePath)}/deletedCount`);
+        let deletedCount = await deletedCountRef.get();
+        if (deletedCount === undefined) {
+            deletedCount = (await this.collection.query.custom(_EmptyQuery)).length;
+            await deletedCountRef.set(deletedCount);
+        }
+        const toDeleteCount = deletedCount - this.config.retainDeletedCount;
+        if (toDeleteCount <= 0)
+            return;
+        this.logDebug('Docs to delete', deletedCount);
+        this.logDebug('Docs to retain', this.config.retainDeletedCount);
+        const deleted = await this.collection.delete.query({
+            limit: toDeleteCount,
+            orderBy: [{ key: '__updated', order: 'asc' }]
+        });
+        let newDeletedCount = deletedCount - deleted.length;
+        if (deleted.length !== toDeleteCount) {
+            this.logError(`Expected to delete ${toDeleteCount} but actually deleted ${deleted.length}`);
+            newDeletedCount = (await this.collection.query.custom(_EmptyQuery)).length;
+        }
+        await deletedCountRef.set(newDeletedCount);
+        const map = deleted.map(item => item.__collectionName);
+        const keys = filterDuplicates(map);
+        await Promise.all(keys.map(key => {
+            const newestDeletedItem = deleted.find(deletedItem => deletedItem.__collectionName === key);
+            this.logDebug(`setting oldest deleted timestamp ${key} = ${newestDeletedItem.__updated}`);
+            return this.setOldestDeleted(key, newestDeletedItem.__updated);
+        }));
+    };
+    getFullSyncData = async () => {
+        const syncDataRef = this.database.ref(`${resolveContent(this.resolvableFirebaseBasePath)}/syncData`);
+        return (await syncDataRef.get({}));
+    };
+    async setLastUpdated(collectionName, lastUpdated) {
+        return this.database.patch(`${resolveContent(this.resolvableFirebaseBasePath)}/syncData/${collectionName}`, { lastUpdated });
+    }
+    async setOldestDeleted(collectionName, oldestDeleted) {
+        return this.database.patch(`${resolveContent(this.resolvableFirebaseBasePath)}/deletedCount/${collectionName}`, { oldestDeleted });
+    }
+    setModuleFilter = (filter) => {
+        const previousFilter = this.filterModules;
+        this.filterModules = async (modules) => filter(await previousFilter(modules));
+    };
+    /**
+     * Set function that allows to set a custom resolver for the rtdb node path
+     * @param resolvablePath The resolver for the node path
+     */
+    setResolvablePath = (resolvablePath) => {
+        this.resolvableFirebaseBasePath = resolvablePath;
+    };
+    filterModules = async (modules) => {
+        return modules;
+    };
+}
+export const ModuleBE_SyncManager = new ModuleBE_SyncManager_Class();

package/package.json

@@ -0,0 +1,104 @@
+{
+  "name": "@nu-art/thunderstorm-backend",
+  "version": "0.400.5",
+  "description": "Thunderstorm Backend",
+  "keywords": [
+    "TacB0sS",
+    "express",
+    "infra",
+    "nu-art",
+    "thunderstorm",
+    "typescript"
+  ],
+  "homepage": "https://github.com/nu-art-js/thunderstorm",
+  "bugs": {
+    "url": "https://github.com/nu-art-js/thunderstorm/issues"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+ssh://git@github.com:nu-art-js/thunderstorm.git"
+  },
+  "license": "Apache-2.0",
+  "author": "TacB0sS",
+  "scripts": {
+    "build": "tsc"
+  },
+  "contributors": [
+    {
+      "name": "TacB0sS"
+    },
+    {
+      "name": "Cipher",
+      "url": "https://www.linkedin.com/in/itay-leybovich-470b87229/"
+    }
+  ],
+  "publishConfig": {
+    "directory": "dist",
+    "linkDirectory": true
+  },
+  "dependencies": {
+    "@nu-art/thunderstorm-shared": "0.400.5",
+    "@nu-art/firebase-backend": "0.400.5",
+    "@nu-art/firebase-shared": "0.400.5",
+    "@nu-art/google-services-backend": "0.400.5",
+    "@nu-art/ts-common": "0.400.5",
+    "@nu-art/ts-styles": "0.400.5",
+    "abort-controller": "^3.0.0",
+    "axios": "^1.13.1",
+    "body-parser": "^1.19.0",
+    "browserify-zlib": "^0.2.0",
+    "buffer": "^6.0.3",
+    "compression": "^1.7.4",
+    "cors": "^2.8.5",
+    "crypto-browserify": "^3.12.0",
+    "csstype": "^3.0.0",
+    "express": "^4.18.2",
+    "firebase": "^11.9.0",
+    "firebase-admin": "13.4.0",
+    "firebase-functions": "6.3.2",
+    "history": "^4.9.0",
+    "moment": "^2.29.4",
+    "pako": "^2.1.0",
+    "papaparse": "^5.4.1",
+    "qs": "^6.6.0",
+    "react": "^18.0.0",
+    "react-dom": "^18.0.0",
+    "react-router-dom": "^6.9.0",
+    "react-virtualized-auto-sizer": "^1.0.12",
+    "react-window": "^1.8.8",
+    "request": "^2.88.0",
+    "stream-browserify": "^3.0.0",
+    "util": "^0.12.4"
+  },
+  "devDependencies": {
+    "@types/papaparse": "^5.3.14",
+    "@types/pako": "^2.0.0",
+    "@types/cors": "^2.8.17",
+    "@types/react": "^18.0.0",
+    "@types/react-dom": "^18.0.0",
+    "@types/react-router": "^5.1.20",
+    "@types/react-router-dom": "^5.3.3",
+    "@types/compression": "^1.7.0",
+    "@types/express": "^4.17.17",
+    "@types/history": "^4.7.2",
+    "@types/request": "^2.48.1",
+    "@types/chrome": "^0.0.202",
+    "@types/react-window": "^1.8.5",
+    "@types/react-virtualized-auto-sizer": "^1.0.1"
+  },
+  "unitConfig": {
+    "type": "typescript-lib"
+  },
+  "type": "module",
+  "exports": {
+    ".": {
+      "types": "./index.d.ts",
+      "import": "./index.js"
+    },
+    "./*": {
+      "types": "./*.d.ts",
+      "import": "./*.js"
+    },
+    "./styles": "./styles.scss"
+  }
+}

package/shared.d.ts

@@ -0,0 +1 @@
+export * from '@nu-art/thunderstorm-shared';

package/shared.js

@@ -0,0 +1,21 @@
+/*
+ * Thunderstorm is a full web app framework!
+ *
+ * Typescript & Express backend infrastructure that natively runs on firebase function
+ * Typescript & React frontend infrastructure
+ *
+ * Copyright (C) 2020 Adam van der Kruk aka TacB0sS
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+export * from '@nu-art/thunderstorm-shared';