@nu-art/file-upload-backend 0.400.7

package/core/messages.d.ts

@@ -0,0 +1,3 @@
+import { PushKey_BE } from '@nu-art/push-pub-sub-backend';
+import { PushMessage_FileUploaded } from '@nu-art/file-upload-shared/assets/messages';
+export declare const PushMessageBE_FileUploadStatus: PushKey_BE<PushMessage_FileUploaded>;

package/core/messages.js

@@ -0,0 +1,2 @@
+import { PushKey_BE } from '@nu-art/push-pub-sub-backend';
+export const PushMessageBE_FileUploadStatus = new PushKey_BE('file-uploaded');

package/core/module-pack.d.ts

@@ -0,0 +1 @@
+export declare const ModulePackBE_FileUploader: (import("../modules/ModuleBE_AssetsTemp.js").ModuleBE_AssetsTemp_Class | import("../modules/ModuleBE_BucketListener.js").ModuleBE_BucketListener_Class | import("../modules/ModuleBE_AssetsStorage.js").ModuleBE_AssetsStorage_Class | import("../modules/ModuleBE_AssetsDB.js").ModuleBE_AssetsDB_Class | import("@nu-art/push-pub-sub-backend").ModuleBE_PushPubSub_Class | import("@nu-art/push-pub-sub-backend/modules/ModuleBE_PushSessionDB").ModuleBE_PushSessionDB_Class | import("@nu-art/push-pub-sub-backend/modules/ModuleBE_PushSubscriptionDB").ModuleBE_PushSubscriptionDB_Class | import("@nu-art/thunderstorm-backend").ModuleBE_BaseApi_Class<import("@nu-art/push-pub-sub-shared/push-subscription/types").DBProto_PushSubscription> | import("@nu-art/push-pub-sub-backend/modules/ModuleBE_PushMessagesHistoryDB").ModuleBE_PushMessagesHistoryDB_Class | import("../modules/ModuleBE_AssetsDeleted.js").ModuleBE_AssetsDeleted_Class | import("../modules/ModuleBE_AssetUploader.js").ModuleBE_AssetUploader_Class | import("../modules/ModuleBE_AssetsAPI.js").ModuleBE_AssetsAPI_Class | import("@nu-art/thunderstorm-backend").ModuleBE_BaseApi_Class<import("@nu-art/file-upload-shared").DBProto_AssetsTemp>)[];

package/core/module-pack.js

@@ -0,0 +1,36 @@
+/*
+ * Permissions management system, define access level for each of
+ * your server apis, and restrict users by giving them access levels
+ *
+ * Copyright (C) 2020 Adam van der Kruk aka TacB0sS
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import { ModuleBE_AssetsTemp } from '../modules/ModuleBE_AssetsTemp.js';
+import { ModuleBE_AssetsDB } from '../modules/ModuleBE_AssetsDB.js';
+import { ModuleBE_BucketListener } from '../modules/ModuleBE_BucketListener.js';
+import { ModulePackBE_PushPubSub } from '@nu-art/push-pub-sub-backend';
+import { ModuleBE_AssetUploader } from '../modules/ModuleBE_AssetUploader.js';
+import { ModuleBE_AssetsAPI } from '../modules/ModuleBE_AssetsAPI.js';
+import { createApisForDBModuleV3 } from '@nu-art/thunderstorm-backend';
+import { ModuleBE_AssetsStorage } from '../modules/ModuleBE_AssetsStorage.js';
+import { ModuleBE_AssetsDeleted } from '../modules/ModuleBE_AssetsDeleted.js';
+export const ModulePackBE_FileUploader = [
+    ...ModulePackBE_PushPubSub,
+    ModuleBE_AssetUploader,
+    ModuleBE_AssetsTemp, createApisForDBModuleV3(ModuleBE_AssetsTemp),
+    ModuleBE_AssetsDeleted,
+    ModuleBE_AssetsStorage,
+    ModuleBE_AssetsDB, ModuleBE_AssetsAPI,
+    ModuleBE_BucketListener
+];

package/core/permissions.d.ts

@@ -0,0 +1,6 @@
+import { DefaultDef_Group } from '@nu-art/permissions-shared';
+import { DefaultDef_Domain, DefaultDef_Package } from '@nu-art/permissions-backend/types';
+export declare const PermissionsDomain_AssetsManager: Readonly<DefaultDef_Domain>;
+export declare const PermissionsGroup_AssetsManager: Readonly<DefaultDef_Group>;
+export declare const PermissionsGroup_AssetsViewer: Readonly<DefaultDef_Group>;
+export declare const PermissionsPackage_AssetsManager: DefaultDef_Package;

package/core/permissions.js

@@ -0,0 +1,52 @@
+import { DefaultAccessLevel_Admin, DefaultAccessLevel_Delete, DefaultAccessLevel_Read, DefaultAccessLevel_Write } from '@nu-art/permissions-shared';
+import { ApiDef_Assets, ApiDef_AssetUploader } from '@nu-art/file-upload-shared';
+import { Domain_Developer } from '@nu-art/permissions-backend/permissions';
+import { ModuleBE_AssetsAPI } from '../modules/ModuleBE_AssetsAPI.js';
+const Domain_AssetsManager_ID = '993c496c6aaad9c67723034137d26c42';
+const _PermissionsDomain_AssetsManager = {
+    _id: Domain_AssetsManager_ID,
+    namespace: 'Assets',
+    dbNames: [],
+    customApis: [
+        { path: ModuleBE_AssetsAPI.apiDef.v1.deleteAll.path, accessLevel: DefaultAccessLevel_Admin.name },
+        { path: ModuleBE_AssetsAPI.apiDef.v1.patch.path, accessLevel: DefaultAccessLevel_Admin.name },
+        { path: ModuleBE_AssetsAPI.apiDef.v1.upsertAll.path, accessLevel: DefaultAccessLevel_Admin.name },
+        { path: ModuleBE_AssetsAPI.apiDef.v1.upsert.path, accessLevel: DefaultAccessLevel_Admin.name },
+        { path: ModuleBE_AssetsAPI.apiDef.v1.delete.path, accessLevel: DefaultAccessLevel_Delete.name },
+        { path: ModuleBE_AssetsAPI.apiDef.v1.queryUnique.path, accessLevel: DefaultAccessLevel_Read.name },
+        { path: ModuleBE_AssetsAPI.apiDef.v1.query.path, accessLevel: DefaultAccessLevel_Read.name },
+        { path: ApiDef_Assets.vv1.getReadSignedUrl.path, accessLevel: DefaultAccessLevel_Read.name },
+        { path: ApiDef_AssetUploader.vv1.getUploadUrl.path, accessLevel: DefaultAccessLevel_Write.name },
+        {
+            path: ApiDef_AssetUploader.vv1.processAssetManually.path,
+            domainId: Domain_Developer._id,
+            accessLevel: DefaultAccessLevel_Write.name
+        },
+    ]
+};
+export const PermissionsDomain_AssetsManager = Object.freeze(_PermissionsDomain_AssetsManager);
+const PermissionsGroupId_AssetsViewer = '0773dcf3b9fbe5e595ef6e2a596b8939';
+const PermissionsGroupId_AssetsManager = '3f5037358fba0ae1199047f2fa8add94';
+const _PermissionsGroup_AssetsViewer = {
+    _id: PermissionsGroupId_AssetsViewer,
+    name: 'Assets Viewer',
+    uiLabel: 'Assets Viewer',
+    accessLevels: {
+        [PermissionsDomain_AssetsManager.namespace]: DefaultAccessLevel_Read.name,
+    }
+};
+const _PermissionsGroup_AssetsManager = {
+    _id: PermissionsGroupId_AssetsManager,
+    name: 'Assets Manager',
+    uiLabel: 'Assets Manager',
+    accessLevels: {
+        [PermissionsDomain_AssetsManager.namespace]: DefaultAccessLevel_Delete.name,
+    }
+};
+export const PermissionsGroup_AssetsManager = Object.freeze(_PermissionsGroup_AssetsManager);
+export const PermissionsGroup_AssetsViewer = Object.freeze(_PermissionsGroup_AssetsViewer);
+export const PermissionsPackage_AssetsManager = {
+    name: PermissionsDomain_AssetsManager.namespace,
+    domains: [PermissionsDomain_AssetsManager],
+    groups: [PermissionsGroup_AssetsViewer, PermissionsGroup_AssetsManager]
+};

package/index.d.ts

@@ -0,0 +1,5 @@
+export * from './core/module-pack.js';
+export * from './modules/ModuleBE_AssetsDB.js';
+export * from './modules/ModuleBE_AssetsTemp.js';
+export * from './modules/ModuleBE_BucketListener.js';
+export * from './modules/ModuleBE_AssetUploader.js';

package/index.js

@@ -0,0 +1,23 @@
+/*
+ * Permissions management system, define access level for each of
+ * your server apis, and restrict users by giving them access levels
+ *
+ * Copyright (C) 2020 Adam van der Kruk aka TacB0sS
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+export * from './core/module-pack.js';
+export * from './modules/ModuleBE_AssetsDB.js';
+export * from './modules/ModuleBE_AssetsTemp.js';
+export * from './modules/ModuleBE_BucketListener.js';
+export * from './modules/ModuleBE_AssetUploader.js';

package/modules/ModuleBE_AssetUploader.d.ts

@@ -0,0 +1,19 @@
+import { ModuleBase_AssetUploader, UploaderConfig } from '@nu-art/file-upload-shared/modules/ModuleBase_AssetUploader';
+import { Axios_RequestConfig } from '@nu-art/thunderstorm-backend';
+import { TempSignedUrl, UI_Asset } from '@nu-art/file-upload-shared';
+import { ApiDef, BaseHttpRequest, TypedApi } from '@nu-art/thunderstorm-shared';
+export type ServerFilesToUpload = UI_Asset & {
+    file: Buffer;
+};
+type Config = UploaderConfig & {
+    requestConfig: Axios_RequestConfig;
+};
+export declare class ModuleBE_AssetUploader_Class extends ModuleBase_AssetUploader<Config> {
+    constructor();
+    init(): void;
+    createRequest<API extends TypedApi<any, any, any, any>>(uploadFile: ApiDef<API>): BaseHttpRequest<API>;
+    upload(files: ServerFilesToUpload[]): UI_Asset[];
+    protected subscribeToPush(toSubscribe: TempSignedUrl[]): Promise<void>;
+}
+export declare const ModuleBE_AssetUploader: ModuleBE_AssetUploader_Class;
+export {};

package/modules/ModuleBE_AssetUploader.js

@@ -0,0 +1,45 @@
+/*
+ * Permissions management system, define access level for each of
+ * your server apis, and restrict users by giving them access levels
+ *
+ * Copyright (C) 2020 Adam van der Kruk aka TacB0sS
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import { ModuleBase_AssetUploader, } from '@nu-art/file-upload-shared/modules/ModuleBase_AssetUploader';
+import { apiWithBodyAxios, apiWithQueryAxios, AxiosHttpModule } from '@nu-art/thunderstorm-backend';
+import { ApiDef_AssetUploader } from '@nu-art/file-upload-shared';
+export class ModuleBE_AssetUploader_Class extends ModuleBase_AssetUploader {
+    constructor() {
+        super();
+        this.vv1 = {
+            getUploadUrl: apiWithBodyAxios(ApiDef_AssetUploader.vv1.getUploadUrl),
+            processAssetManually: apiWithQueryAxios(ApiDef_AssetUploader.vv1.processAssetManually),
+        };
+    }
+    init() {
+        super.init();
+        AxiosHttpModule.setRequestOption(this.config.requestConfig);
+    }
+    createRequest(uploadFile) {
+        return AxiosHttpModule.createRequest(uploadFile);
+    }
+    upload(files) {
+        return this.uploadImpl(files);
+    }
+    async subscribeToPush(toSubscribe) {
+        // Not sure now
+        // We said timeout
+    }
+}
+export const ModuleBE_AssetUploader = new ModuleBE_AssetUploader_Class();

package/modules/ModuleBE_AssetsAPI.d.ts

@@ -0,0 +1,8 @@
+import { ModuleBE_BaseApi_Class } from '@nu-art/thunderstorm-backend';
+import { DBProto_Assets } from '@nu-art/file-upload-shared';
+export declare class ModuleBE_AssetsAPI_Class extends ModuleBE_BaseApi_Class<DBProto_Assets> {
+    constructor();
+    init(): void;
+    private getReadSignedUrl;
+}
+export declare const ModuleBE_AssetsAPI: ModuleBE_AssetsAPI_Class;

package/modules/ModuleBE_AssetsAPI.js

@@ -0,0 +1,22 @@
+import { addRoutes, createBodyServerApi, ModuleBE_BaseApi_Class } from '@nu-art/thunderstorm-backend';
+import { ModuleBE_AssetsDB } from './ModuleBE_AssetsDB.js';
+import { ApiDef_Assets } from '@nu-art/file-upload-shared';
+import { ModuleBE_AssetsStorage } from './ModuleBE_AssetsStorage.js';
+export class ModuleBE_AssetsAPI_Class extends ModuleBE_BaseApi_Class {
+    constructor() {
+        super(ModuleBE_AssetsDB);
+    }
+    init() {
+        super.init();
+        addRoutes([
+            createBodyServerApi(ApiDef_Assets.vv1.getReadSignedUrl, this.getReadSignedUrl)
+        ]);
+    }
+    getReadSignedUrl = async (body) => {
+        const dbAsset = await ModuleBE_AssetsDB.query.uniqueAssert(body._id);
+        return {
+            signedUrl: (await (await ModuleBE_AssetsStorage.getFile(dbAsset)).getReadSignedUrl()).signedUrl
+        };
+    };
+}
+export const ModuleBE_AssetsAPI = new ModuleBE_AssetsAPI_Class();

package/modules/ModuleBE_AssetsDB.d.ts

@@ -0,0 +1,52 @@
+import { TypedMap } from '@nu-art/ts-common';
+import { CleanupDetails, DBApiConfigV3, ModuleBE_BaseDB, OnCleanupSchedulerAct } from '@nu-art/thunderstorm-backend';
+import { Clause_Where } from '@nu-art/firebase-shared';
+import { OnAssetUploaded } from './ModuleBE_BucketListener.js';
+import { DB_Asset, DBProto_Assets, TempSignedUrl, UI_Asset } from '@nu-art/file-upload-shared';
+import { CollectionActionType, PostWriteProcessingData } from '@nu-art/firebase-backend/firestore-v3/FirestoreCollectionV3';
+import { firestore } from 'firebase-admin';
+import { FileMetadata } from '@google-cloud/storage';
+import { FileWrapper, FirestoreTransaction } from '@nu-art/firebase-backend';
+import Transaction = firestore.Transaction;
+type MyConfig = DBApiConfigV3<DBProto_Assets> & {
+    authKey: string;
+    bucketName?: string;
+    storagePath: string;
+    pathRegexp: string;
+};
+export type AssetContent = {
+    asset: DB_Asset;
+    content: Buffer;
+};
+export type FileTypeResult = {
+    ext: string;
+    mime: string;
+};
+export type FileTypeValidation = {
+    fileType?: string[];
+    minSize?: number;
+    maxSize?: number;
+    validator?: FileValidator;
+};
+export declare const DefaultMimetypeValidator: (file: FileWrapper, doc: DB_Asset) => Promise<import("file-type").FileTypeResult>;
+export type FileValidator = (file: FileWrapper, doc: DB_Asset) => Promise<FileTypeResult | undefined>;
+export declare const fileSizeValidator: (file: FileWrapper, metadata: FileMetadata, minSizeInBytes?: number, maxSizeInBytes?: number) => Promise<boolean>;
+export declare class ModuleBE_AssetsDB_Class extends ModuleBE_BaseDB<DBProto_Assets, MyConfig> implements OnCleanupSchedulerAct, OnAssetUploaded {
+    constructor();
+    mimeTypeValidator: TypedMap<FileValidator>;
+    fileValidator: TypedMap<FileTypeValidation>;
+    protected postWriteProcessing(data: PostWriteProcessingData<DBProto_Assets>, actionType: CollectionActionType, transaction?: Transaction): Promise<void>;
+    init(): void;
+    getAssetsContent(assetIds: string[]): Promise<AssetContent[]>;
+    registerTypeValidator(mimeType: string, validator: (file: FileWrapper, doc: DB_Asset) => Promise<void>): void;
+    queryUnique(where: Clause_Where<DB_Asset>, transaction?: FirestoreTransaction): Promise<DB_Asset>;
+    register: (key: string, validationConfig: FileTypeValidation) => void;
+    __onCleanupSchedulerAct(): CleanupDetails;
+    private cleanup;
+    getUrl: (files: UI_Asset[]) => Promise<TempSignedUrl[]>;
+    processAssetManually: (feId?: string) => Promise<void[]>;
+    __processAsset: (filePath?: string) => Promise<void>;
+    private notifyFrontend;
+}
+export declare const ModuleBE_AssetsDB: ModuleBE_AssetsDB_Class;
+export {};

package/modules/ModuleBE_AssetsDB.js

@@ -0,0 +1,243 @@
+/*
+ * Permissions management system, define access level for each of
+ * your server apis, and restrict users by giving them access levels
+ *
+ * Copyright (C) 2020 Adam van der Kruk aka TacB0sS
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import { ApiException, asArray, BadImplementationException, currentTimeMillis, Day, exists, filterInstances, generateHex, Hour, ImplementationMissingException, MB, Minute, MUSTNeverHappenException, ThisShouldNotHappenException } from '@nu-art/ts-common';
+import { ModuleBE_AssetsTemp } from './ModuleBE_AssetsTemp.js';
+import { addRoutes, createBodyServerApi, ModuleBE_BaseDB } from '@nu-art/thunderstorm-backend';
+import { ModuleBE_AssetsStorage } from './ModuleBE_AssetsStorage.js';
+import { ApiDef_AssetUploader, DBDef_Assets, FileStatus } from '@nu-art/file-upload-shared';
+import { PushMessageBE_FileUploadStatus } from '../core/messages.js';
+import { ModuleBE_AssetsDeleted } from './ModuleBE_AssetsDeleted.js';
+import { fileTypeFromBuffer } from 'file-type';
+export const DefaultMimetypeValidator = async (file, doc) => {
+    const buffer = await file.read();
+    const fileType = await fileTypeFromBuffer(buffer);
+    if (!fileType)
+        throw new ImplementationMissingException(`No validator defined for asset of mimetype: ${doc.mimeType}`);
+    if (fileType.mime !== doc.mimeType)
+        throw new BadImplementationException(`Original mimetype (${doc.mimeType}) does not match the resolved mimetype: (${fileType.mime})`);
+    return fileType;
+};
+export const fileSizeValidator = async (file, metadata, minSizeInBytes = 0, maxSizeInBytes = MB) => {
+    const fileSize = +(metadata.size ?? 0);
+    if (!fileSize)
+        throw new ThisShouldNotHappenException(`could not resolve metadata.size for file: ${file.path}`);
+    return fileSize >= minSizeInBytes && fileSize <= maxSizeInBytes;
+};
+export class ModuleBE_AssetsDB_Class extends ModuleBE_BaseDB {
+    constructor() {
+        super(DBDef_Assets);
+        this.setDefaultConfig({
+            ...this.config,
+            storagePath: 'assets',
+            pathRegexp: '^assets/.*',
+            authKey: 'file-uploader'
+        });
+    }
+    mimeTypeValidator = {};
+    fileValidator = {};
+    async postWriteProcessing(data, actionType, transaction) {
+        const deletedItems = data.deleted;
+        if (exists(deletedItems))
+            await ModuleBE_AssetsDeleted.create.all(asArray(deletedItems), transaction);
+    }
+    init() {
+        super.init();
+        addRoutes([
+            createBodyServerApi(ApiDef_AssetUploader.vv1.getUploadUrl, this.getUrl)
+        ]);
+        this.registerVersionUpgradeProcessor('1.0.1', async (assets) => {
+            assets.forEach(asset => {
+                // @ts-ignore
+                delete asset._audit;
+            });
+        });
+        const originalQuery = this.query;
+        this.query = {
+            ...originalQuery,
+            unique: async (_id, transaction) => {
+                const dbAsset = await originalQuery.uniqueAssert(_id, transaction);
+                const signedUrl = (dbAsset.signedUrl?.validUntil || 0) > currentTimeMillis() ? dbAsset.signedUrl : undefined;
+                if (!signedUrl) {
+                    const url = await ModuleBE_AssetsStorage.getReadSignedUrl(dbAsset);
+                    dbAsset.signedUrl = {
+                        url,
+                        validUntil: currentTimeMillis() + Day - Minute
+                    };
+                }
+                return dbAsset;
+            }
+        };
+    }
+    async getAssetsContent(assetIds) {
+        const assetsToSync = filterInstances(await ModuleBE_AssetsDB.query.all(assetIds));
+        const assetFiles = await Promise.all(assetsToSync.map(asset => ModuleBE_AssetsStorage.getFile(asset)));
+        const assetContent = await Promise.all(assetFiles.map(asset => asset.read()));
+        return assetIds.map((id, index) => ({ asset: assetsToSync[index], content: assetContent[index] }));
+    }
+    registerTypeValidator(mimeType, validator) {
+    }
+    async queryUnique(where, transaction) {
+        const dbAsset = await this.query.uniqueCustom({ where });
+        const signedUrl = (dbAsset.signedUrl?.validUntil || 0) > currentTimeMillis() ? dbAsset.signedUrl : undefined;
+        if (!signedUrl) {
+            const url = await ModuleBE_AssetsStorage.getReadSignedUrl(dbAsset);
+            dbAsset.signedUrl = {
+                url,
+                validUntil: currentTimeMillis() + Day - Minute
+            };
+        }
+        return dbAsset;
+    }
+    register = (key, validationConfig) => {
+        if (this.fileValidator[key] && this.fileValidator[key] !== validationConfig)
+            throw new BadImplementationException(`File Validator already exists for key: ${key}`);
+        this.fileValidator[key] = validationConfig;
+    };
+    __onCleanupSchedulerAct() {
+        return {
+            moduleKey: this.getName(),
+            interval: Day,
+            cleanup: () => this.cleanup(),
+        };
+    }
+    cleanup = async (interval = Hour, module = ModuleBE_AssetsTemp) => {
+        const entries = await module.query.custom({ where: { timestamp: { $lt: currentTimeMillis() - interval } } });
+        await Promise.all(entries.map(async (dbAsset) => {
+            const file = await ModuleBE_AssetsStorage.getFile(dbAsset);
+            if (!(await file.exists()))
+                return;
+            await file.delete();
+        }));
+        await module.delete.query({ where: { timestamp: { $lt: currentTimeMillis() - interval } } });
+    };
+    getUrl = async (files) => {
+        const bucketName = this.config?.bucketName;
+        const bucket = await ModuleBE_AssetsStorage.storage.getOrCreateBucket(bucketName);
+        return Promise.all(files.map(async (_file) => {
+            const key = _file.key || _file.mimeType;
+            // this will fail the entire request... should it?
+            if (!this.fileValidator[key])
+                throw new ImplementationMissingException(`Missing validator for type ${key}`);
+            const _id = generateHex(32);
+            const path = `${this.config.storagePath}/${_id}`;
+            const dbAsset = {
+                timestamp: currentTimeMillis(),
+                _id,
+                feId: _file.feId,
+                name: _file.name,
+                ext: _file.name.substring(_file.name.toLowerCase().lastIndexOf('.') + 1),
+                mimeType: _file.mimeType,
+                key,
+                path,
+                bucketName: bucket.bucketName
+            };
+            if (_file.public)
+                dbAsset.public = _file.public;
+            const dbTempMeta = await ModuleBE_AssetsTemp.set.item(dbAsset);
+            const fileWrapper = await bucket.getFile(dbTempMeta.path);
+            const url = await fileWrapper.getWriteSignedUrl(_file.mimeType, Hour);
+            return {
+                signedUrl: url.signedUrl,
+                asset: dbTempMeta
+            };
+        }));
+    };
+    processAssetManually = async (feId) => {
+        let query = { limit: 1 };
+        if (feId)
+            query = { where: { feId } };
+        const unprocessedFiles = await ModuleBE_AssetsTemp.query.custom(query);
+        return Promise.all(unprocessedFiles.map(asset => this.__processAsset(asset.path)));
+    };
+    __processAsset = async (filePath) => {
+        if (!filePath)
+            throw new MUSTNeverHappenException('Missing file path');
+        if (!filePath.match(this.config.pathRegexp))
+            return this.logVerbose(`File was added to storage in path: ${filePath}, NOT via file uploader`);
+        this.logDebug(`Looking for file with path: ${filePath}`);
+        let dbTempAsset;
+        try {
+            dbTempAsset = await ModuleBE_AssetsTemp.query.uniqueWhere({ path: filePath });
+        }
+        catch (e) {
+            return;
+        }
+        if (!dbTempAsset)
+            throw new ThisShouldNotHappenException(`Could not find meta for file with path: ${filePath}`);
+        await this.notifyFrontend(FileStatus.Processing, dbTempAsset);
+        this.logDebug(`Found temp meta with _id: ${dbTempAsset._id}`, dbTempAsset);
+        const validationConfig = this.fileValidator[dbTempAsset.key];
+        if (!validationConfig)
+            return this.notifyFrontend(FileStatus.ErrorNoConfig, dbTempAsset);
+        let mimetypeValidator = DefaultMimetypeValidator;
+        if (validationConfig.validator)
+            mimetypeValidator = validationConfig.validator;
+        if (!mimetypeValidator && validationConfig.fileType && validationConfig.fileType.includes(dbTempAsset.mimeType))
+            mimetypeValidator = this.mimeTypeValidator[dbTempAsset.mimeType];
+        if (!mimetypeValidator)
+            return this.notifyFrontend(FileStatus.ErrorNoValidator, dbTempAsset);
+        const file = await ModuleBE_AssetsStorage.getFile(dbTempAsset);
+        try {
+            const metadata = (await file.getMetadata());
+            if (!metadata)
+                return this.notifyFrontend(FileStatus.ErrorRetrievingMetadata, dbTempAsset);
+            await fileSizeValidator(file, metadata, validationConfig.minSize, validationConfig.maxSize);
+            const fileType = await mimetypeValidator(file, dbTempAsset);
+            dbTempAsset.md5Hash = metadata.md5Hash;
+            if (fileType && dbTempAsset.ext !== fileType.ext) {
+                this.logWarning(`renaming the file extension name: ${dbTempAsset.ext} => ${fileType.ext}`);
+                dbTempAsset.ext = fileType.ext;
+            }
+        }
+        catch (e) {
+            //TODO delete the file and the temp doc
+            this.logError(`Error while processing asset: ${dbTempAsset.name}`, e);
+            return this.notifyFrontend(FileStatus.ErrorWhileProcessing, dbTempAsset);
+        }
+        if (dbTempAsset.public) {
+            try {
+                // need to handle the response status!
+                await file.makePublic();
+            }
+            catch (e) {
+                return this.notifyFrontend(FileStatus.ErrorMakingPublic, dbTempAsset);
+            }
+        }
+        const finalDbAsset = await this.runTransaction(async (transaction) => {
+            const duplicatedAssets = await this.query.custom({ where: { md5Hash: dbTempAsset.md5Hash } }, transaction);
+            if (duplicatedAssets.length && duplicatedAssets[0]) {
+                this.logWarning(`${dbTempAsset.feId} is a duplicated entry for ${duplicatedAssets[0]._id}`);
+                return { ...duplicatedAssets[0], feId: dbTempAsset.feId };
+            }
+            const doc = this.collection.doc.item(dbTempAsset);
+            await ModuleBE_AssetsTemp.delete.unique(dbTempAsset._id, transaction);
+            return await doc.set(dbTempAsset, transaction);
+        });
+        return this.notifyFrontend(FileStatus.Completed, finalDbAsset);
+    };
+    notifyFrontend = async (status, asset, feId) => {
+        if (status !== FileStatus.Completed && status !== FileStatus.Processing) {
+            const message = `Error while processing asset: ${status}\n Failed on \n  Asset: ${asset.feId}\n    Key: ${asset.key}\n   Type: ${asset.mimeType}\n   File: ${asset.name}`;
+            throw new ApiException(500, message);
+        }
+        this.logDebug(`notify FE about asset ${feId}: ${status}`);
+        return PushMessageBE_FileUploadStatus.push({ status, asset }, { feId: feId || asset.feId });
+    };
+}
+export const ModuleBE_AssetsDB = new ModuleBE_AssetsDB_Class();

package/modules/ModuleBE_AssetsDeleted.d.ts

@@ -0,0 +1,6 @@
+import { DBProto_AssetsDeleted } from '@nu-art/file-upload-shared';
+import { ModuleBE_BaseDB } from '@nu-art/thunderstorm-backend';
+export declare class ModuleBE_AssetsDeleted_Class extends ModuleBE_BaseDB<DBProto_AssetsDeleted> {
+    constructor();
+}
+export declare const ModuleBE_AssetsDeleted: ModuleBE_AssetsDeleted_Class;

package/modules/ModuleBE_AssetsDeleted.js

@@ -0,0 +1,26 @@
+/*
+ * Permissions management system, define access level for each of
+ * your server apis, and restrict users by giving them access levels
+ *
+ * Copyright (C) 2020 Adam van der Kruk aka TacB0sS
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import { DBDef_TempDeleted } from '@nu-art/file-upload-shared';
+import { ModuleBE_BaseDB } from '@nu-art/thunderstorm-backend';
+export class ModuleBE_AssetsDeleted_Class extends ModuleBE_BaseDB {
+    constructor() {
+        super(DBDef_TempDeleted);
+    }
+}
+export const ModuleBE_AssetsDeleted = new ModuleBE_AssetsDeleted_Class();

package/modules/ModuleBE_AssetsStorage.d.ts

@@ -0,0 +1,17 @@
+import { OnSyncEnvCompleted } from '@nu-art/thunderstorm-backend';
+import { Module, TypedMap } from '@nu-art/ts-common';
+import { StorageWrapperBE } from '@nu-art/firebase-backend';
+import { DB_Asset } from '@nu-art/file-upload-shared';
+type Config = {
+    batchItemCount: number;
+};
+export declare class ModuleBE_AssetsStorage_Class extends Module<Config> implements OnSyncEnvCompleted {
+    readonly storage: StorageWrapperBE;
+    constructor();
+    init(): void;
+    __onSyncEnvCompleted(env: string, baseUrl: string, requiredHeaders: TypedMap<string>): Promise<void>;
+    getReadSignedUrl: (dbAsset: DB_Asset) => Promise<string>;
+    getFile: (dbAsset: DB_Asset) => Promise<import("@nu-art/firebase-backend").FileWrapper>;
+}
+export declare const ModuleBE_AssetsStorage: ModuleBE_AssetsStorage_Class;
+export {};

package/modules/ModuleBE_AssetsStorage.js

@@ -0,0 +1,77 @@
+/*
+ * Permissions management system, define access level for each of
+ * your server apis, and restrict users by giving them access levels
+ *
+ * Copyright (C) 2020 Adam van der Kruk aka TacB0sS
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import { AxiosHttpModule } from '@nu-art/thunderstorm-backend';
+import { filterInstances, Module, NotImplementedYetException } from '@nu-art/ts-common';
+import { ModuleBE_Firebase } from '@nu-art/firebase-backend';
+import { ApiDef_Assets } from '@nu-art/file-upload-shared';
+import { HttpMethod } from '@nu-art/thunderstorm-shared';
+import { ModuleBE_AssetsDB } from './ModuleBE_AssetsDB.js';
+export class ModuleBE_AssetsStorage_Class extends Module {
+    storage;
+    constructor() {
+        super();
+        this.setDefaultConfig({ batchItemCount: 10 });
+    }
+    init() {
+        super.init();
+        // @ts-ignore
+        this.storage = ModuleBE_Firebase.createAdminSession().getStorage();
+    }
+    async __onSyncEnvCompleted(env, baseUrl, requiredHeaders) {
+        this.logWarning('Not Implemented Yet', new NotImplementedYetException('Sync assets not implemented'));
+        let assetsToSync = [];
+        let page = 0;
+        const itemsCount = this.config.batchItemCount;
+        let dbAssets;
+        do {
+            dbAssets = await ModuleBE_AssetsDB.query.custom({ limit: { page: page++, itemsCount } });
+            if (dbAssets.length === 0)
+                break;
+            assetsToSync = filterInstances(await Promise.all(dbAssets.map(async (dbAsset) => {
+                const fileWrapper = await this.getFile(dbAsset);
+                if (await fileWrapper.exists())
+                    return;
+                return dbAsset;
+            })));
+            if (assetsToSync.length > 0)
+                continue;
+            assetsToSync.map(async (dbAsset) => {
+                let _signedUrl;
+                try {
+                    const { signedUrl } = await AxiosHttpModule
+                        .createRequest({ ...ApiDef_Assets.vv1.getReadSignedUrl, baseUrl, })
+                        .setHeaders(requiredHeaders)
+                        .setBodyAsJson({ assetId: dbAsset._id })
+                        .executeSync();
+                    _signedUrl = signedUrl;
+                }
+                catch (e) {
+                    console.log(e);
+                }
+                const fileContent = await AxiosHttpModule.createRequest({ method: HttpMethod.GET, fullUrl: _signedUrl, path: '' })
+                    .setResponseType('text')
+                    .executeSync();
+                await (await this.getFile(dbAsset)).write(fileContent);
+            });
+        } while (dbAssets.length > 0);
+    }
+    getReadSignedUrl = async (dbAsset) => (await (await this.getFile(dbAsset)).getReadSignedUrl()).signedUrl;
+    getFile = async (dbAsset) => this.storage.getFile(dbAsset.path, dbAsset.bucketName);
+}
+export const ModuleBE_AssetsStorage = new ModuleBE_AssetsStorage_Class();

package/modules/ModuleBE_AssetsTemp.d.ts

@@ -0,0 +1,6 @@
+import { DBProto_AssetsTemp } from '@nu-art/file-upload-shared';
+import { ModuleBE_BaseDB } from '@nu-art/thunderstorm-backend';
+export declare class ModuleBE_AssetsTemp_Class extends ModuleBE_BaseDB<DBProto_AssetsTemp> {
+    constructor();
+}
+export declare const ModuleBE_AssetsTemp: ModuleBE_AssetsTemp_Class;

package/modules/ModuleBE_AssetsTemp.js

@@ -0,0 +1,26 @@
+/*
+ * Permissions management system, define access level for each of
+ * your server apis, and restrict users by giving them access levels
+ *
+ * Copyright (C) 2020 Adam van der Kruk aka TacB0sS
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import { DBDef_TempAssets } from '@nu-art/file-upload-shared';
+import { ModuleBE_BaseDB } from '@nu-art/thunderstorm-backend';
+export class ModuleBE_AssetsTemp_Class extends ModuleBE_BaseDB {
+    constructor() {
+        super(DBDef_TempAssets);
+    }
+}
+export const ModuleBE_AssetsTemp = new ModuleBE_AssetsTemp_Class();

package/modules/ModuleBE_BucketListener.d.ts

@@ -0,0 +1,14 @@
+import { ModuleBE_StorageListener } from '@nu-art/firebase-backend';
+import { DefaultDef_ServiceAccount, RequiresServiceAccount, ServiceAccountCredentials } from '@nu-art/thunderstorm-backend/modules/_tdb/service-accounts';
+import { StorageEvent } from 'firebase-functions/storage';
+export interface OnAssetUploaded {
+    __processAsset(filePath?: string): void;
+}
+type Config = ServiceAccountCredentials & {};
+export declare class ModuleBE_BucketListener_Class extends ModuleBE_StorageListener<Config> implements RequiresServiceAccount {
+    constructor();
+    onFinalize(event: StorageEvent): Promise<any>;
+    __requiresServiceAccount(): DefaultDef_ServiceAccount;
+}
+export declare const ModuleBE_BucketListener: ModuleBE_BucketListener_Class;
+export {};

package/modules/ModuleBE_BucketListener.js

@@ -0,0 +1,54 @@
+/*
+ * A backend boilerplate with example apis
+ *
+ * Copyright (C) 2020 Adam van der Kruk aka TacB0sS
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import { BadImplementationException, Dispatcher } from '@nu-art/ts-common';
+import { MemStorage } from '@nu-art/ts-common/mem-storage/MemStorage';
+import { ModuleBE_StorageListener } from '@nu-art/firebase-backend';
+import { PermissionsGroup_PushMessanger } from '@nu-art/push-pub-sub-backend/core/permissions';
+import { MemKey_AccountId } from '@nu-art/user-account-backend';
+const dispatcher_onAssetUploaded = new Dispatcher('__processAsset');
+export class ModuleBE_BucketListener_Class extends ModuleBE_StorageListener {
+    constructor() {
+        super();
+    }
+    async onFinalize(event) {
+        // need to create a dispatch that collects a list of services that requires service account and
+        // service account details, the permissions  create project will create these account for the rest of the system ;
+        return new MemStorage().init(async () => {
+            const accountId = this.config.serviceAccount.accountId;
+            if (!accountId)
+                throw new BadImplementationException('Need to perform project setup to setup this feature');
+            MemKey_AccountId.set(accountId);
+            let filePath = event.data.name || '';
+            if (filePath.endsWith('}'))
+                filePath = filePath.substring(0, filePath.length - 1);
+            this.logInfo(`File was added to bucket: ${filePath}`);
+            await dispatcher_onAssetUploaded.dispatchModuleAsync(filePath);
+            this.logDebug('Object is ', event.data);
+            this.logDebug('event is ', event);
+        });
+    }
+    __requiresServiceAccount() {
+        const serviceAccount = {
+            moduleName: this.getName(),
+            email: 'bucket-manager@nu-art-software.com',
+            groupIds: [PermissionsGroup_PushMessanger._id]
+        };
+        return serviceAccount;
+    }
+}
+export const ModuleBE_BucketListener = new ModuleBE_BucketListener_Class();

package/package.json

@@ -0,0 +1,77 @@
+{
+  "name": "@nu-art/file-upload-backend",
+  "version": "0.400.7",
+  "description": "File Uploader - Express & Typescript based backend framework Backend",
+  "keywords": [
+    "TacB0sS",
+    "infra",
+    "nu-art",
+    "storm",
+    "thunderstorm",
+    "typescript"
+  ],
+  "homepage": "https://github.com/nu-art-js/file-upload",
+  "bugs": {
+    "url": "https://github.com/nu-art-js/file-upload/issues"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+ssh://git@github.com:nu-art-js/file-upload.git"
+  },
+  "publishConfig": {
+    "directory": "dist",
+    "linkDirectory": true
+  },
+  "license": "Apache-2.0",
+  "author": "TacB0sS",
+  "files": [
+    "**/*"
+  ],
+  "scripts": {
+    "build": "tsc"
+  },
+  "dependencies": {
+    "@nu-art/file-upload-shared": "0.400.7",
+    "@google-cloud/common": "^5.0.2",
+    "@google-cloud/firestore": "^7.11.0",
+    "@google-cloud/storage": "^7.15.0",
+    "@nu-art/firebase-backend": "0.400.7",
+    "@nu-art/firebase-shared": "0.400.7",
+    "@nu-art/permissions-backend": "0.400.7",
+    "@nu-art/permissions-shared": "0.400.7",
+    "@nu-art/push-pub-sub-backend": "0.400.7",
+    "@nu-art/push-pub-sub-shared": "0.400.7",
+    "@nu-art/thunderstorm-backend": "0.400.7",
+    "@nu-art/thunderstorm-shared": "0.400.7",
+    "@nu-art/ts-common": "0.400.7",
+    "@nu-art/user-account-backend": "0.400.7",
+    "@nu-art/user-account-shared": "0.400.7",
+    "express": "^4.18.2",
+    "file-type": "^21.0.0",
+    "firebase": "^11.9.0",
+    "firebase-admin": "13.4.0",
+    "firebase-functions": "6.3.2",
+    "google-auth-library": "^10.0.0",
+    "jszip": "^3.2.2",
+    "react": "^18.0.0"
+  },
+  "devDependencies": {
+    "@types/react": "^18.0.0",
+    "@types/compression": "^1.0.1",
+    "@types/express": "^4.17.17"
+  },
+  "unitConfig": {
+    "type": "typescript-lib"
+  },
+  "type": "module",
+  "exports": {
+    ".": {
+      "types": "./index.d.ts",
+      "import": "./index.js"
+    },
+    "./*": {
+      "types": "./*.d.ts",
+      "import": "./*.js"
+    }
+  }
+}