@nu-art/thunderstorm-backend 0.400.5

package/_entity/app-config/ModuleBE_AppConfigAPI.d.ts

@@ -0,0 +1,9 @@
+import { ModuleBE_BaseApi_Class } from '../../modules/db-api-gen/ModuleBE_BaseApi.js';
+import { DBProto_AppConfig } from '@nu-art/thunderstorm-shared';
+declare class ModuleBE_AppConfigAPI_Class extends ModuleBE_BaseApi_Class<DBProto_AppConfig> {
+    constructor();
+    init(): void;
+    private getConfigByKey;
+}
+export declare const ModuleBE_AppConfigAPI: ModuleBE_AppConfigAPI_Class;
+export {};

package/_entity/app-config/ModuleBE_AppConfigAPI.js

@@ -0,0 +1,20 @@
+import { createQueryServerApi } from '../../core/typed-api.js';
+import { addRoutes } from '../../modules/ModuleBE_APIs.js';
+import { ModuleBE_BaseApi_Class } from '../../modules/db-api-gen/ModuleBE_BaseApi.js';
+import { ApiDef_AppConfig } from '@nu-art/thunderstorm-shared/_entity/app-config/api-def';
+import { ModuleBE_AppConfigDB } from './ModuleBE_AppConfigDB.js';
+class ModuleBE_AppConfigAPI_Class extends ModuleBE_BaseApi_Class {
+    constructor() {
+        super(ModuleBE_AppConfigDB);
+    }
+    init() {
+        super.init();
+        addRoutes([
+            createQueryServerApi(ApiDef_AppConfig._v1.getConfigByKey, this.getConfigByKey)
+        ]);
+    }
+    getConfigByKey = async (request) => {
+        return ModuleBE_AppConfigDB.getResolverDataByKey(request.key);
+    };
+}
+export const ModuleBE_AppConfigAPI = new ModuleBE_AppConfigAPI_Class();

package/_entity/app-config/ModuleBE_AppConfigDB.d.ts

@@ -0,0 +1,27 @@
+import { DB_AppConfig, DBProto_AppConfig } from '@nu-art/thunderstorm-shared';
+import { DBApiConfigV3, ModuleBE_BaseDB } from '../../modules/db-api-gen/ModuleBE_BaseDB.js';
+import { Logger, PreDB, TypedKeyValue } from '@nu-art/ts-common';
+type InferType<T> = T extends AppConfigKey_BE<infer ValueType> ? ValueType : never;
+type Config = DBApiConfigV3<DBProto_AppConfig> & {};
+export declare class ModuleBE_AppConfigDB_Class extends ModuleBE_BaseDB<DBProto_AppConfig, Config> {
+    private keyMap;
+    constructor();
+    protected preWriteProcessing(dbInstance: PreDB<DB_AppConfig>, originalDbInstance: DBProto_AppConfig['dbType'], transaction?: FirebaseFirestore.Transaction): Promise<void>;
+    createDefaults: (logger?: Logger) => Promise<void>;
+    getResolverDataByKey: (key: string) => Promise<any>;
+    registerKey<K extends AppConfigKey_BE<any>>(appConfigKey: K): void;
+    getAppKey: <K extends AppConfigKey_BE<any>>(appConfigKey: K, logger?: Logger) => Promise<InferType<K>>;
+    setAppKey: <K extends AppConfigKey_BE<any>>(appConfigKey: K, data: InferType<K>) => Promise<DB_AppConfig<any>>;
+    _deleteAppKey: <K extends AppConfigKey_BE<any>>(appConfigKey: K) => Promise<void>;
+}
+export declare const ModuleBE_AppConfigDB: ModuleBE_AppConfigDB_Class;
+export declare class AppConfigKey_BE<Binder extends TypedKeyValue<string | number | object, any>> {
+    readonly key: Binder['key'];
+    readonly resolver: (logger: Logger) => Promise<Binder['value']>;
+    readonly dataManipulator: (data: Binder['value']) => Promise<Binder['value']>;
+    constructor(key: Binder['key'], resolver: (logger: Logger) => Promise<Binder['value']>, dataManipulator?: (data: Binder['value']) => Promise<Binder['value']>);
+    get(): Promise<Binder['value']>;
+    set(value: Binder['value']): Promise<void>;
+    delete(): Promise<void>;
+}
+export {};

package/_entity/app-config/ModuleBE_AppConfigDB.js

@@ -0,0 +1,91 @@
+import { DBDef_AppConfig } from '@nu-art/thunderstorm-shared';
+import { ModuleBE_BaseDB } from '../../modules/db-api-gen/ModuleBE_BaseDB.js';
+import { _keys, ApiException } from '@nu-art/ts-common';
+export class ModuleBE_AppConfigDB_Class extends ModuleBE_BaseDB {
+    keyMap = {};
+    // ######################## Lifecycle ########################
+    constructor() {
+        super(DBDef_AppConfig);
+    }
+    async preWriteProcessing(dbInstance, originalDbInstance, transaction) {
+        this.logVerbose('############## Pre Manipulation ##############');
+        this.logVerbose(dbInstance);
+        const appKey = this.keyMap[dbInstance.key];
+        dbInstance.data = await appKey.dataManipulator(dbInstance.data);
+        this.logVerbose('############## Post Manipulation ##############');
+        this.logVerbose(dbInstance);
+    }
+    createDefaults = async (logger = this) => {
+        const keys = _keys(this.keyMap);
+        for (const key of keys) {
+            try {
+                await this.getAppKey(this.keyMap[key]);
+                // const config = await this.getAppKey(this.keyMap[key]);
+                // this.logInfo(`Set App-Config default value for '${key}'`, config);
+            }
+            catch (err) {
+                logger.logError(`Failed to create app-config for key ${key}`, err);
+            }
+        }
+    };
+    // ######################## API ########################
+    getResolverDataByKey = async (key) => {
+        const appConfigKey = this.keyMap[key];
+        if (!appConfigKey)
+            throw new ApiException(404, `Could not find an app config with key ${key}`);
+        return this.getAppKey(appConfigKey);
+    };
+    // ######################## Logic ########################
+    registerKey(appConfigKey) {
+        this.keyMap[appConfigKey.key] = appConfigKey;
+    }
+    getAppKey = async (appConfigKey, logger = this) => {
+        try {
+            const config = await this.query.uniqueCustom({ where: { key: appConfigKey.key } });
+            return config?.data;
+        }
+        catch (e) {
+            const data = await appConfigKey.resolver(logger);
+            await this.setAppKey(appConfigKey, data);
+            return data;
+        }
+    };
+    setAppKey = async (appConfigKey, data) => {
+        let _config;
+        try {
+            _config = await this.query.uniqueCustom({ where: { key: appConfigKey.key } });
+        }
+        catch (e) {
+            _config = { key: appConfigKey.key };
+        }
+        _config.data = data;
+        return this.set.item(_config);
+    };
+    _deleteAppKey = async (appConfigKey) => {
+        await this.delete.query({ where: { key: appConfigKey.key } });
+    };
+}
+export const ModuleBE_AppConfigDB = new ModuleBE_AppConfigDB_Class();
+//TODO: Add validation by key
+export class AppConfigKey_BE {
+    key;
+    resolver;
+    dataManipulator = (data) => data;
+    constructor(key, resolver, dataManipulator) {
+        this.key = key;
+        this.resolver = resolver;
+        if (dataManipulator)
+            this.dataManipulator = dataManipulator;
+        ModuleBE_AppConfigDB.registerKey(this);
+    }
+    async get() {
+        return await ModuleBE_AppConfigDB.getAppKey(this);
+    }
+    async set(value) {
+        // @ts-ignore
+        await ModuleBE_AppConfig.setAppKey(this, value);
+    }
+    async delete() {
+        await ModuleBE_AppConfigDB._deleteAppKey(this);
+    }
+}

package/_entity/app-config/index.d.ts

@@ -0,0 +1,2 @@
+export * from './ModuleBE_AppConfigDB.js';
+export * from './module-pack.js';

package/_entity/app-config/index.js

@@ -0,0 +1,2 @@
+export * from './ModuleBE_AppConfigDB.js';
+export * from './module-pack.js';

package/_entity/app-config/module-pack.d.ts

@@ -0,0 +1,2 @@
+import { Module } from '@nu-art/ts-common';
+export declare const ModulePackBE_AppConfigDB: Module[];

package/_entity/app-config/module-pack.js

@@ -0,0 +1,3 @@
+import { ModuleBE_AppConfigDB } from './ModuleBE_AppConfigDB.js';
+import { ModuleBE_AppConfigAPI } from './ModuleBE_AppConfigAPI.js';
+export const ModulePackBE_AppConfigDB = [ModuleBE_AppConfigDB, ModuleBE_AppConfigAPI];

package/_entity/backup-doc/ModuleBE_BackupDocDB.d.ts

@@ -0,0 +1,52 @@
+import { Module, PreDB, TypedMap, UniqueId } from '@nu-art/ts-common';
+import { DBApiConfigV3, ModuleBE_BaseDB } from '../../modules/db-api-gen/ModuleBE_BaseDB.js';
+import { FirestoreQuery } from '@nu-art/firebase-shared';
+import { Readable } from 'stream';
+import { FirestoreCollectionV3 } from '@nu-art/firebase-backend/firestore-v3/FirestoreCollectionV3';
+import { DB_BackupDoc, DBProto_BackupDoc, FetchBackupDoc } from '@nu-art/thunderstorm-shared';
+import { Request_BackupId, Response_BackupDocs } from '@nu-art/thunderstorm-shared/_entity/backup-doc/api-def';
+export interface OnModuleCleanupV2 {
+    __onCleanupInvokedV2: () => Promise<void>;
+}
+type Config = DBApiConfigV3<DBProto_BackupDoc> & {
+    keepInterval: number;
+    minTimeThreshold: number;
+    excludedDbKeys?: string[];
+};
+type DBModules = ModuleBE_BaseDB<any>;
+/**
+ * This module is in charge of making a backup of the db in firebase storage,
+ * in order to test this module locally run firebase functions:shell command in a terminal
+ * when the firebase> shows up write the name of the function you want to run and call it "functionName()"
+ * **/
+export declare class ModuleBE_BackupDocDB_Class extends Module<Config> {
+    collection: FirestoreCollectionV3<DBProto_BackupDoc>;
+    constructor();
+    protected init(): void;
+    getBackupStatusCollection: () => FirestoreCollectionV3<DBProto_BackupDoc>;
+    /**
+     * Get metadata objects per each collection module that needs to be backed up.
+     */
+    getBackupDetails: () => DBModules[];
+    getBackupInfo(backupId: string, baseUrl: string, headers: TypedMap<string | string[]>): Promise<FetchBackupDoc>;
+    getBackupStreamFromId: (backupInfo: FetchBackupDoc) => Promise<Readable>;
+    query: (ourQuery: FirestoreQuery<DB_BackupDoc>) => Promise<DB_BackupDoc[]>;
+    queryUnique: (backupDocId: UniqueId) => Promise<DB_BackupDoc | undefined>;
+    upsert: (instance: PreDB<DB_BackupDoc>) => Promise<DB_BackupDoc>;
+    deleteItem: (instance: DB_BackupDoc) => Promise<DB_BackupDoc | undefined>;
+    private getDefaultPath;
+    initiateBackup: (force?: boolean, pathInBucket?: string) => Promise<{
+        pathToBackup: string;
+        backupId: string;
+    } | undefined>;
+    createBackupReadStream: (backupInfo: FetchBackupDoc) => Promise<Readable>;
+    createBackupReadStreamFromBucket: (pathInBucket: string) => Promise<Readable>;
+    /**
+     * @param body - needs to contain backupId with the key to fetch.
+     */
+    fetchBackupDocs: (body: Request_BackupId) => Promise<Response_BackupDocs>;
+    fetchLatestBackupDoc: () => Promise<Response_BackupDocs>;
+    private fetchDocImpl;
+}
+export declare const ModuleBE_BackupDocDB: ModuleBE_BackupDocDB_Class;
+export {};

package/_entity/backup-doc/ModuleBE_BackupDocDB.js

@@ -0,0 +1,350 @@
+import { __stringify, _logger_logException, ApiException, BadImplementationException, cloneObj, currentTimeMillis, Day, Dispatcher, filterInstances, Format_YYYYMMDD_HHmmss, formatTimestamp, LogLevel, Minute, Module, RuntimeModules, sortArray } from '@nu-art/ts-common';
+import { ModuleBE_Firebase } from '@nu-art/firebase-backend';
+import { _EmptyQuery } from '@nu-art/firebase-shared';
+import { Readable } from 'stream';
+import { HttpMethod } from '@nu-art/thunderstorm-shared';
+import { addRoutes } from '../../modules/ModuleBE_APIs.js';
+import { ApiDef_BackupDoc } from '@nu-art/thunderstorm-shared/_entity/backup-doc/api-def';
+import { createQueryServerApi } from '../../core/typed-api.js';
+import { DBDef_BackupDoc } from '@nu-art/thunderstorm-shared/_entity/backup-doc/db-def';
+import { HttpCodes } from '@nu-art/ts-common/core/exceptions/http-codes';
+import { MemKey_HttpRequestHeaders } from '../../modules/server/consts.js';
+import { AxiosHttpModule } from '../../index.js';
+import { CSVModuleV3 } from '@nu-art/ts-common/modules/CSVModuleV3';
+import { ModuleBE_CollectionActions } from '../../modules/collection-actions/ModuleBE_CollectionActions.js';
+const dispatch_onModuleCleanupV2 = new Dispatcher('__onCleanupInvokedV2');
+/**
+ * This module is in charge of making a backup of the db in firebase storage,
+ * in order to test this module locally run firebase functions:shell command in a terminal
+ * when the firebase> shows up write the name of the function you want to run and call it "functionName()"
+ * **/
+export class ModuleBE_BackupDocDB_Class extends Module {
+    collection;
+    constructor() {
+        super();
+        this.setMinLevel(LogLevel.Verbose);
+        this.setDefaultConfig({ minTimeThreshold: Day, keepInterval: 7 * Day });
+    }
+    init() {
+        super.init();
+        this.collection = this.getBackupStatusCollection();
+        addRoutes([
+            createQueryServerApi(ApiDef_BackupDoc._v1.initiateBackup, () => this.initiateBackup()),
+            createQueryServerApi(ApiDef_BackupDoc._v1.fetchBackupDocs, this.fetchBackupDocs),
+        ]);
+    }
+    getBackupStatusCollection = () => {
+        return ModuleBE_Firebase
+            .createAdminSession()
+            .getFirestoreV3()
+            .getCollection(DBDef_BackupDoc);
+    };
+    /**
+     * Get metadata objects per each collection module that needs to be backed up.
+     */
+    getBackupDetails = () => {
+        return RuntimeModules()
+            .filter((module) => {
+            if (!module || !module.dbDef)
+                return false;
+            if (this.config.excludedDbKeys?.includes(module.dbDef.dbKey)) {
+                this.logWarningBold(`Skipping module ${module.dbDef.dbKey} since it's in the exclusion list.`);
+                return false;
+            }
+            return true;
+        });
+    };
+    // private async getBackupInfo(queryParams: Request_GetMetadata) {
+    async getBackupInfo(backupId, baseUrl, headers) {
+        const url = `${baseUrl}/v1/fetch-backup-docs-v2`;
+        const outputDef = {
+            method: HttpMethod.GET,
+            path: '',
+            fullUrl: url
+        };
+        const requestBody = { backupId };
+        try {
+            let request = AxiosHttpModule
+                .createRequest(outputDef)
+                .setUrlParams(requestBody);
+            request = request.addHeaders(headers);
+            const response = await request.executeSync();
+            const backupInfo = response.backupInfo;
+            const wrongBackupIdDescriptor = backupInfo?._id !== backupId;
+            if (wrongBackupIdDescriptor)
+                throw new BadImplementationException(`Received backup descriptors with wrong backupId! provided id: ${backupId} received id: ${backupInfo?._id}`);
+            return backupInfo;
+        }
+        catch (err) {
+            throw new ApiException(500, err);
+        }
+    }
+    getBackupStreamFromId = async (backupInfo) => {
+        if (!backupInfo.backupFilePath)
+            throw new ApiException(404, 'Backup file path not found');
+        this.logInfo(`----  Fetching Backup Stream from: ${backupInfo.firestoreSignedUrl} ----`);
+        const signedUrlDef = {
+            method: HttpMethod.GET,
+            path: '',
+            fullUrl: backupInfo.firestoreSignedUrl
+        };
+        return (await AxiosHttpModule
+            .createRequest(signedUrlDef)
+            .setResponseType('stream')
+            .executeSync());
+    };
+    // ##################### Collection Interaction #####################
+    query = async (ourQuery) => {
+        return await this.collection.query.custom(ourQuery);
+    };
+    queryUnique = async (backupDocId) => {
+        return await this.collection.query.unique(backupDocId);
+    };
+    upsert = async (instance) => {
+        return await this.collection.create.item(instance);
+    };
+    deleteItem = async (instance) => {
+        return await this.collection.delete.unique(instance._id);
+    };
+    // ##################### Backup Actions #####################
+    getDefaultPath = () => {
+        const nowMs = currentTimeMillis();
+        const timeFormat = formatTimestamp(Format_YYYYMMDD_HHmmss, nowMs);
+        return `backup/${timeFormat}`;
+    };
+    initiateBackup = async (force = false, pathInBucket = this.getDefaultPath()) => {
+        const nowMs = currentTimeMillis();
+        const backupPath = `${pathInBucket}/firestore-backup.csv`;
+        const metadataPath = `${pathInBucket}/metadata.json`;
+        const configPath = `${pathInBucket}/firebase-backup.json`;
+        const query = {
+            where: {},
+            orderBy: [{ key: 'timestamp', order: 'asc' }],
+            limit: 1
+        };
+        const docs = await this.query(query);
+        const latestDoc = docs[0];
+        if (!force && latestDoc && latestDoc.timestamp + this.config.minTimeThreshold > nowMs)
+            return; // If the oldest doc is still in the keeping timeframe, don't delete any docs.
+        if (this.config.excludedDbKeys)
+            this.logInfo(`Found excluded modules list: ${this.config.excludedDbKeys}`);
+        try {
+            this.logInfo('Cleaning modules...');
+            await dispatch_onModuleCleanupV2.dispatchModuleAsync();
+            this.logInfo('Cleaned modules!');
+        }
+        catch (e) {
+            this.logWarning(`modules cleanup has failed with error`, e);
+            const errorMessage = `modules cleanup has failed with error\nError: ${_logger_logException(e)}`;
+            throw new ApiException(500, errorMessage, e);
+        }
+        const modules = filterInstances(this.getBackupDetails());
+        const backupsCounter = 0;
+        try {
+            this.logInfo('Upgrading Collections');
+            await ModuleBE_CollectionActions.upgrade_All({});
+            this.logInfo('Collections Upgraded');
+        }
+        catch (e) {
+            this.logError(e);
+            throw HttpCodes._5XX.INTERNAL_SERVER_ERROR('Could not upgrade collections', 'failed collection upgrade', e);
+        }
+        const firebaseSessionAdmin = ModuleBE_Firebase.createAdminSession();
+        const storage = firebaseSessionAdmin.getStorage();
+        const bucket = await storage.getMainBucket();
+        let metadata;
+        if (modules.length === 0)
+            throw new ApiException(404, 'No modules to backup');
+        try {
+            this.logDebug('Creating backup file...');
+            const file = await bucket.getFile(backupPath);
+            const reader = new DBModuleReader(modules);
+            const writer = file.createWriteStream({ gzip: true });
+            const formatter = CSVModuleV3.provideFormatter();
+            await new Promise((resolve, reject) => {
+                reader
+                    .pipe(formatter)
+                    .pipe(writer)
+                    .on('close', () => {
+                    metadata = { ...reader.getMetadata(), timestamp: nowMs };
+                    resolve();
+                })
+                    .on('error', err => reject(err));
+            });
+            this.logDebug('Backup file created');
+            this.logDebug('Backing up config db');
+            const database = firebaseSessionAdmin.getDatabase();
+            const configBackup = await database.ref('/').get();
+            const configFile = await bucket.getFile(configPath);
+            await configFile.write(configBackup);
+            this.logDebug('Config file created');
+            this.logDebug('Creating metadata file...');
+            const metadataFile = await bucket.getFile(metadataPath);
+            await metadataFile.write(metadata);
+            this.logDebug('Metadata file created');
+        }
+        catch (e) {
+            this.logWarning(`backup of ${modules[backupsCounter].dbDef.dbKey} has failed with error`, e);
+            const errorMessage = `Error backing up firestore collection config:\n ${__stringify(modules[backupsCounter].config, true)}\nError: ${_logger_logException(e)}`;
+            throw new ApiException(500, errorMessage, e);
+        }
+        const dbBackup = await this.upsert({
+            timestamp: nowMs,
+            backupPath,
+            metadataPath,
+            firebasePath: configPath,
+            metadata: metadata,
+        });
+        this.logWarning(dbBackup);
+        const oldBackupsToDelete = await this.query({ where: { timestamp: { $lt: nowMs - this.config.keepInterval } } });
+        if (oldBackupsToDelete.length === 0) {
+            this.logInfoBold('No older backups to delete');
+            return { pathToBackup: backupPath, backupId: dbBackup._id };
+        }
+        try {
+            this.logInfoBold('Received older backups to delete, count: ' + oldBackupsToDelete.length);
+            const backupDeleteOperations = oldBackupsToDelete
+                .map(doc => filterInstances([doc.metadataPath, doc.backupPath, doc.firebasePath]))
+                .flat()
+                .map(path => async () => {
+                try {
+                    const file = await bucket.getFile(path);
+                    file.delete();
+                }
+                catch (err) {
+                    this.logError(`Failed deleting file at path: ${path}`, err);
+                }
+            });
+            await Promise.all(backupDeleteOperations);
+            await this.collection.delete.all(oldBackupsToDelete);
+            this.logInfoBold('Successfully deleted old backups');
+        }
+        catch (err) {
+            this.logWarning(`Error while cleaning up older backups`, err);
+            throw new ApiException(500, err);
+        }
+        return { pathToBackup: backupPath, backupId: dbBackup._id };
+    };
+    createBackupReadStream = async (backupInfo) => {
+        const stream = await this.getBackupStreamFromId(backupInfo);
+        const transformer = CSVModuleV3.provideFormatterFromCsv();
+        return stream.pipe(transformer);
+    };
+    createBackupReadStreamFromBucket = async (pathInBucket) => {
+        const file = await ModuleBE_Firebase.createAdminSession().getStorage().getFile(pathInBucket);
+        const stream = file.createReadStream({ decompress: true });
+        const transformer = CSVModuleV3.provideFormatterFromCsv();
+        return stream.pipe(transformer);
+    };
+    // ##################### BackupDoc Fetching #####################
+    /**
+     * @param body - needs to contain backupId with the key to fetch.
+     */
+    fetchBackupDocs = async (body) => {
+        const backupDoc = await this.queryUnique(body.backupId);
+        if (!backupDoc)
+            throw new ApiException(500, `no backup doc found with this id ${body.backupId}`);
+        return await this.fetchDocImpl(backupDoc);
+    };
+    fetchLatestBackupDoc = async () => {
+        const docs = await this.query(_EmptyQuery);
+        if (!docs.length)
+            throw HttpCodes._4XX.NOT_FOUND('No backups to fetch', 'No backups in db to fetch');
+        const latest = sortArray(docs, doc => doc.__created, true)[0];
+        return await this.fetchDocImpl(latest);
+    };
+    fetchDocImpl = async (doc) => {
+        const bucket = await ModuleBE_Firebase.createAdminSession().getStorage().getMainBucket();
+        const contentType = MemKey_HttpRequestHeaders.get()['content-type'];
+        const firebaseDescriptor = await (await bucket.getFile(doc.firebasePath)).getReadSignedUrl(10 * Minute, contentType);
+        const firestoreDescriptor = await (await bucket.getFile(doc.backupPath)).getReadSignedUrl(10 * Minute, contentType);
+        return {
+            backupInfo: {
+                _id: doc._id,
+                backupFilePath: doc.backupPath,
+                metadataFilePath: doc.metadataPath,
+                firebaseFilePath: doc.firebasePath,
+                firebaseSignedUrl: firebaseDescriptor.signedUrl,
+                firestoreSignedUrl: firestoreDescriptor.signedUrl,
+                metadata: doc.metadata
+            }
+        };
+    };
+}
+export const ModuleBE_BackupDocDB = new ModuleBE_BackupDocDB_Class();
+class DBModuleReader extends Readable {
+    dbModules;
+    pageSize;
+    page = 0;
+    moduleIndex = 0;
+    done = false;
+    metadata;
+    constructor(dbModules, pageSize = 1000) {
+        super({ objectMode: true });
+        this.dbModules = dbModules;
+        this.pageSize = pageSize;
+        this.metadata = { collectionsData: [], timestamp: currentTimeMillis() };
+    }
+    // ##################### Metadata #####################
+    getMetadata = () => cloneObj(this.metadata);
+    updateMetadata = (module, items) => {
+        const dbKey = module.dbDef.dbKey;
+        const collectionData = this.metadata.collectionsData.find(data => data.dbKey === dbKey);
+        if (!collectionData)
+            return this.metadata.collectionsData.push({
+                dbKey: module.dbDef.dbKey,
+                numOfDocs: items.length,
+                version: module.dbDef.versions[0],
+            });
+        collectionData.numOfDocs += items.length;
+    };
+    // ##################### Logic #####################
+    getItems = async () => {
+        if (this.done)
+            return;
+        const module = this.dbModules[this.moduleIndex];
+        if (!module) {
+            this.done = true;
+            return;
+        }
+        const dbKey = module.dbDef.dbKey;
+        try {
+            const items = await module.query.unManipulatedQuery({
+                ..._EmptyQuery,
+                limit: { page: this.page, itemsCount: this.pageSize },
+            });
+            this.updateMetadata(module, items);
+            return items.map(item => ({
+                dbKey: dbKey,
+                _id: item._id,
+                document: JSON.stringify(item),
+            }));
+        }
+        catch (err) {
+            this.emit('error', err);
+        }
+    };
+    advanceModule = () => {
+        this.page = 0;
+        this.moduleIndex++;
+    };
+    async _read() {
+        const items = await this.getItems();
+        if (!items) { //Only getting undefined if no more items to get, end stream
+            this.push(null);
+            return;
+        }
+        //If got an array but it was empty, move on to next module
+        if (!items.length) {
+            this.advanceModule();
+            await this._read();
+            return;
+        }
+        items.forEach(item => this.push(item));
+        //If the amount of items was under the pageSize, there will be no more items to get from the module, move to the next
+        if (items.length < this.pageSize)
+            this.advanceModule();
+        else
+            this.page++;
+    }
+}

package/_entity/backup-doc/ModuleBE_BackupScheduler.d.ts

@@ -0,0 +1,7 @@
+import { ModuleBE_FirebaseScheduler } from '@nu-art/firebase-backend';
+declare class ModuleBE_BackupScheduler_Class extends ModuleBE_FirebaseScheduler {
+    constructor();
+    onScheduledEvent: () => Promise<any>;
+}
+export declare const ModuleBE_BackupScheduler: ModuleBE_BackupScheduler_Class;
+export {};

package/_entity/backup-doc/ModuleBE_BackupScheduler.js

@@ -0,0 +1,14 @@
+import { ModuleBE_FirebaseScheduler } from '@nu-art/firebase-backend';
+import { ModuleBE_BackupDocDB } from './ModuleBE_BackupDocDB.js';
+class ModuleBE_BackupScheduler_Class extends ModuleBE_FirebaseScheduler {
+    constructor() {
+        super();
+        this.setSchedule('every 24 hours');
+        this.setDefaultConfig({ timeoutSeconds: 540, maxInstances: 1, memory: '2GB' });
+    }
+    onScheduledEvent = async () => {
+        this.logInfoBold(`Running function ${this.getName()}`);
+        await ModuleBE_BackupDocDB.initiateBackup();
+    };
+}
+export const ModuleBE_BackupScheduler = new ModuleBE_BackupScheduler_Class();

package/_entity/backup-doc/index.d.ts

@@ -0,0 +1,3 @@
+export * from './ModuleBE_BackupDocDB.js';
+export * from './ModuleBE_BackupScheduler.js';
+export * from './module-pack.js';

package/_entity/backup-doc/index.js

@@ -0,0 +1,3 @@
+export * from './ModuleBE_BackupDocDB.js';
+export * from './ModuleBE_BackupScheduler.js';
+export * from './module-pack.js';

package/_entity/backup-doc/module-pack.d.ts

@@ -0,0 +1,2 @@
+import { Module } from '@nu-art/ts-common';
+export declare const ModulePackBE_BackupDocDB: Module[];

package/_entity/backup-doc/module-pack.js

@@ -0,0 +1,3 @@
+import { ModuleBE_BackupDocDB } from './ModuleBE_BackupDocDB.js';
+import { ModuleBE_BackupScheduler } from './ModuleBE_BackupScheduler.js';
+export const ModulePackBE_BackupDocDB = [ModuleBE_BackupDocDB, ModuleBE_BackupScheduler];

package/_entity/editable-test/ModuleBE_EditableTestDB.d.ts

@@ -0,0 +1,8 @@
+import { DBProto_EditableTest } from '@nu-art/thunderstorm-shared/_entity/editable-test/types';
+import { DBApiConfigV3, ModuleBE_BaseDB } from '../../modules/db-api-gen/ModuleBE_BaseDB.js';
+type Config = DBApiConfigV3<DBProto_EditableTest> & {};
+export declare class ModuleBE_EditableTestDB_Class extends ModuleBE_BaseDB<DBProto_EditableTest, Config> {
+    constructor();
+}
+export declare const ModuleBE_EditableTestDB: ModuleBE_EditableTestDB_Class;
+export {};

package/_entity/editable-test/ModuleBE_EditableTestDB.js

@@ -0,0 +1,8 @@
+import { ModuleBE_BaseDB } from '../../modules/db-api-gen/ModuleBE_BaseDB.js';
+import { DBDef_EditableTest } from '@nu-art/thunderstorm-shared/_entity/editable-test/db-def';
+export class ModuleBE_EditableTestDB_Class extends ModuleBE_BaseDB {
+    constructor() {
+        super(DBDef_EditableTest);
+    }
+}
+export const ModuleBE_EditableTestDB = new ModuleBE_EditableTestDB_Class();

package/_entity/editable-test/index.d.ts

@@ -0,0 +1 @@
+export * from './ModuleBE_EditableTestDB.js';

package/_entity/editable-test/index.js

@@ -0,0 +1 @@
+export * from './ModuleBE_EditableTestDB.js';

package/_entity/editable-test/module-pack.d.ts

@@ -0,0 +1 @@
+export declare const ModulePackBE_EditableTest: (import("./ModuleBE_EditableTestDB.js").ModuleBE_EditableTestDB_Class | import("../../index.js").ModuleBE_BaseApi_Class<import("@nu-art/thunderstorm-shared").DBProto_EditableTest>)[];

package/_entity/editable-test/module-pack.js

@@ -0,0 +1,3 @@
+import { ModuleBE_EditableTestDB } from './ModuleBE_EditableTestDB.js';
+import { createApisForDBModuleV3 } from '../../index.js';
+export const ModulePackBE_EditableTest = [ModuleBE_EditableTestDB, createApisForDBModuleV3(ModuleBE_EditableTestDB)];

package/_entity.d.ts

@@ -0,0 +1,3 @@
+export * from './_entity/editable-test/index.js';
+export * from './_entity/app-config/index.js';
+export * from './_entity/backup-doc/index.js';

package/_entity.js

@@ -0,0 +1,3 @@
+export * from './_entity/editable-test/index.js';
+export * from './_entity/app-config/index.js';
+export * from './_entity/backup-doc/index.js';