@lilaquadrat/studio 10.0.0-beta.9 → 10.1.0

package/lib/src/services/storage.service.js

@@ -8,31 +8,10 @@ import StorageModel from '../models/storage.model.js';
 import JetStreamService from './jetstream.service.js';
 import UploadService from './upload.service.js';
 import filenameHelper from '../helpers/filenameHelper.js';
+import jetstreamService from './jetstream.service.js';
 export class StorageService extends Immutable {
     constructor(options) {
         super();
-        this.buckets = {
-            cdn: {
-                public: true,
-                overwrite: false,
-                cache: 31536000,
-            },
-            hosting: {
-                public: false,
-                overwrite: true,
-                cache: 0,
-            },
-            secure: {
-                public: false,
-                overwrite: false,
-                cache: 31536000,
-            },
-            internal: {
-                public: false,
-                overwrite: false,
-                cache: 31536000,
-            },
-        };
         this.model = StorageModel;
         this.options = options;
     }
@@ -42,40 +21,48 @@ export class StorageService extends Immutable {
      * The token is cryptographically signed using HMAC-SHA256 and base64url-encoded to be
      * non-human readable and URL-safe.
      *
-     * @param fileId - The file identifier (ObjectId or string) to grant access to
      * @param expiresIn - Time in seconds until the token expires (default: 3600 = 1 hour)
      * @param limits - Optional restrictions to apply to the token
+     * @param limits.fileId - Restrict token to specific file (storage _id)
      * @param limits.company - Restrict token to specific company
      * @param limits.project - Restrict token to specific project
      * @param limits.app - Restrict token to specific app
      * @param limits.url - Restrict token to specific URL path prefix
      * @param limits.prefix - Restrict token to specific file prefix
+     * @param limits.customer - Restrict token to specific customer
+     * @param limits.list - Restrict token to specific list
      *
-     * @returns Base64url-encoded token containing: fileId:expires[:limits]:signature
+     * @returns Base64url-encoded token containing: expires[:limits]:signature
      *
      * @example
      * // Generate token valid for 1 hour with no restrictions
-     * const { token } = getSecureToken('507f1f77bcf86cd799439011', 3600);
+     * const { token } = getSecureToken(3600);
      *
      * @example
      * // Generate token limited to specific company and project
-     * const { token } = getSecureToken('507f1f77bcf86cd799439011', 3600, {
+     * const { token } = getSecureToken(3600, {
      *   company: 'acme',
      *   project: 'website'
      * });
      *
      * @example
+     * // Generate token limited to specific file
+     * const { token } = getSecureToken(3600, {
+     *   fileId: '507f1f77bcf86cd799439011'
+     * });
+     *
+     * @example
      * // Generate token limited to specific URL path
-     * const { token } = getSecureToken('507f1f77bcf86cd799439011', 3600, {
+     * const { token } = getSecureToken(3600, {
      *   url: '/company/project/files/'
      * });
      */
-    getSecureToken(fileId, expiresIn = 3600, limits) {
+    getSecureToken(expiresIn = 3600, limits) {
         const secret = process.env.SERVER_SECRET || 'secret';
         const now = Math.floor(Date.now() / 1000);
         const expires = now + expiresIn;
-        // Build payload: fileId:expires[:base64(limits)]
-        let payload = `${fileId}:${expires}`;
+        // Build payload: expires[:base64(limits)]
+        let payload = `${expires}`;
         if (limits && Object.keys(limits).length > 0) {
             // Base64 encode the JSON to avoid : separator conflicts
             const limitsJson = JSON.stringify(limits);
@@ -131,57 +118,101 @@ export class StorageService extends Immutable {
             decoded = Buffer.from(token, 'base64url').toString('utf-8');
         }
         catch {
+            console.debug('[validateSecureToken] failed to decode token from base64url');
             return false;
         }
         // Extract payload and signature (signature is after last colon)
         const lastColonIndex = decoded.lastIndexOf(':');
-        if (lastColonIndex === -1)
+        if (lastColonIndex === -1) {
+            console.debug('[validateSecureToken] decoded payload has no colon separator');
             return false;
+        }
         const payload = decoded.substring(0, lastColonIndex);
         const signature = decoded.substring(lastColonIndex + 1);
         // Verify signature matches expected HMAC
         const expectedSignature = createHmac('sha256', secret).update(payload).digest('hex');
-        if (signature !== expectedSignature)
+        if (signature !== expectedSignature) {
+            console.debug('[validateSecureToken] signature mismatch', {
+                expected: expectedSignature.substring(0, 8) + '...',
+                received: signature.substring(0, 8) + '...',
+            });
             return false;
-        // Parse payload structure: fileId:expires[:base64(limits)]
+        }
+        // Parse payload structure: expires[:base64(limits)]
         const parts = payload.split(':');
-        // parts[0] is fileId
-        // parts[1] is expires timestamp
-        // parts[2] is optional base64-encoded JSON limits object
-        if (parts.length < 2)
+        // parts[0] is expires timestamp
+        // parts[1] is optional base64-encoded JSON limits object
+        if (parts.length < 1) {
+            console.debug('[validateSecureToken] payload is empty', { parts });
             return false;
+        }
         // Check if token has expired
-        const expires = parseInt(parts[1], 10);
-        if (Date.now() / 1000 > expires)
+        const expires = parseInt(parts[0], 10);
+        if (Number.isNaN(expires)) {
+            console.debug('[validateSecureToken] expires is not a number', { parts });
             return false;
+        }
+        if (Date.now() / 1000 > expires) {
+            console.debug('[validateSecureToken] token expired', {
+                expires: new Date(expires * 1000).toISOString(),
+                now: new Date().toISOString(),
+            });
+            return false;
+        }
         // If token has limits, validate against checkLimits
-        if (parts.length > 2) {
+        if (parts.length > 1) {
             let tokenLimits;
             try {
                 // Decode from base64 first, then parse JSON
-                const limitsJson = Buffer.from(parts[2], 'base64').toString('utf-8');
+                const limitsJson = Buffer.from(parts[1], 'base64').toString('utf-8');
                 tokenLimits = JSON.parse(limitsJson);
             }
             catch {
+                console.debug('[validateSecureToken] failed to decode or parse token limits');
                 return false;
             }
             // Token has limits but no values provided to check
-            if (!checkLimits)
+            if (!checkLimits) {
+                console.debug('[validateSecureToken] token has limits but no checkLimits provided', { tokenLimits });
                 return false;
+            }
             // Validate each limit field if present in token
-            if (tokenLimits.company && tokenLimits.company !== checkLimits.company)
+            if (tokenLimits.fileId && tokenLimits.fileId !== checkLimits.fileId) {
+                console.debug('[validateSecureToken] fileId mismatch', { token: tokenLimits.fileId, provided: checkLimits.fileId });
+                return false;
+            }
+            if (tokenLimits.company && tokenLimits.company !== checkLimits.company) {
+                console.debug('[validateSecureToken] company mismatch', { token: tokenLimits.company, provided: checkLimits.company });
+                return false;
+            }
+            if (tokenLimits.project && tokenLimits.project !== checkLimits.project) {
+                console.debug('[validateSecureToken] project mismatch', { token: tokenLimits.project, provided: checkLimits.project });
+                return false;
+            }
+            if (tokenLimits.app && tokenLimits.app !== checkLimits.app) {
+                console.debug('[validateSecureToken] app mismatch', { token: tokenLimits.app, provided: checkLimits.app });
                 return false;
-            if (tokenLimits.project && tokenLimits.project !== checkLimits.project)
+            }
+            if (tokenLimits.prefix && tokenLimits.prefix !== checkLimits.prefix) {
+                console.debug('[validateSecureToken] prefix mismatch', { token: tokenLimits.prefix, provided: checkLimits.prefix });
                 return false;
-            if (tokenLimits.app && tokenLimits.app !== checkLimits.app)
+            }
+            if (tokenLimits.customer && tokenLimits.customer !== checkLimits.customer) {
+                console.debug('[validateSecureToken] customer mismatch', { token: tokenLimits.customer, provided: checkLimits.customer });
                 return false;
-            if (tokenLimits.prefix && tokenLimits.prefix !== checkLimits.prefix)
+            }
+            if (tokenLimits.list && tokenLimits.list !== checkLimits.list) {
+                console.debug('[validateSecureToken] list mismatch', { token: tokenLimits.list, provided: checkLimits.list });
                 return false;
+            }
             // For URL limit, check if requested URL starts with the token's allowed prefix
-            if (tokenLimits.url && checkLimits.url && !checkLimits.url.startsWith(tokenLimits.url))
+            if (tokenLimits.url && checkLimits.url && !checkLimits.url.startsWith(tokenLimits.url)) {
+                console.debug('[validateSecureToken] url mismatch', { tokenUrl: tokenLimits.url, providedUrl: checkLimits.url });
                 return false;
+            }
         }
         else if (checkLimits) {
+            console.debug('[validateSecureToken] checkLimits provided but token has no limits');
             return false;
         }
         return true;
@@ -211,6 +242,7 @@ export class StorageService extends Immutable {
             throw new Error('STORAGE_FILE_NOT_FOUND');
         // Merge provided limits with storage document context
         const tokenLimits = {
+            fileId: fileId.toString(),
             company: limits?.company || storage.company,
             project: limits?.project || storage.project,
             app: limits?.app || storage.app,
@@ -218,7 +250,7 @@ export class StorageService extends Immutable {
             url: limits?.url,
         };
         // Generate secure token
-        const tokenData = this.getSecureToken(fileId, expiresIn, tokenLimits);
+        const tokenData = this.getSecureToken(expiresIn, tokenLimits);
         // Construct download URL
         const baseUrl = process.env.SECURE_URL;
         const path = storage.prefix ? `${storage.prefix}/${storage.filename}` : storage.filename;
@@ -246,6 +278,21 @@ export class StorageService extends Immutable {
             await fs.promises.unlink(path.file);
             throw new Error('FILE_SIZE');
         }
+        // this is not correct.
+        // the idea is, the development does not work like that.
+        // needs to be reintroduced
+        // if (uploadFile.size) {
+        //
+        //   const chunkStat = await fs.promises.stat(path.file);
+        //   const existingSize = uploadFile.paths?.length
+        //     ? (await Promise.all(uploadFile.paths.map((p) => fs.promises.stat(p.path).then((s) => s.size).catch(() => 0))))
+        //       .reduce((sum, s) => sum + s, 0)
+        //     : 0;
+        //   if (existingSize + chunkStat.size > uploadFile.size) {
+        //     await fs.promises.unlink(path.file);
+        //     throw new Error('FILE_SIZE');
+        //   }
+        // }
         const updloadState = await UploadService.addPathToUpload(uploadInternalId, index, path.file);
         /**
         * if all chunks are there, complete the file and the upload
@@ -272,10 +319,13 @@ export class StorageService extends Immutable {
                 await this.concatenateFilesWithCat(data, pathAndFile.file);
             }
         }
-        else {
+        else if (type !== 'copy') {
             const writeStream = fs.createWriteStream(pathAndFile.file);
             await pipeline(data, writeStream);
         }
+        else {
+            await fs.promises.copyFile(data, pathAndFile.file);
+        }
         return pathAndFile;
     }
     async writeFileVersion(data, internalId, version) {
@@ -369,9 +419,7 @@ export class StorageService extends Immutable {
         catch (error) {
             if (error.code === 'ENOENT') {
                 console.warn(`File not found: ${fullPath}`);
-                return;
             }
-            throw error;
         }
     }
     /**
@@ -399,12 +447,6 @@ export class StorageService extends Immutable {
         const file = [baseFolder, filename].join('/');
         return { dir: baseFolder, file, relativePath: file };
     }
-    // async createUpload(filename: string, size: number, chunks: number, mimetype: string, options: UserAppOptionsRequired) {
-    //   const useFilename = this.sanitizeName(filename);
-    //   if (await this.exists(options.company, options.project, useFilename)) throw new Error('FILE_EXISTS');
-    //   if (await UploadService.uploadExists(options.company, options.project, useFilename)) throw new Error('UPLOAD_EXISTS');
-    //   return UploadService.createUpload(useFilename, size, chunks, mimetype, options);
-    // }
     static sanitizeName(name) {
         return name.replace(/[^a-z0-9-_.]/gi, '').toLowerCase();
     }
@@ -426,10 +468,17 @@ export class StorageService extends Immutable {
         }
         return useMetadata;
     }
-    get(company, project, app, query, limit) {
+    async get(select, limit) {
+        const { company, project, app, list, customer, assetId, query } = select;
         const realQuery = { company, project, app, parent: { $exists: false } };
+        if (list)
+            realQuery.list = list;
+        if (customer)
+            realQuery.customer = customer;
+        if (assetId?.length)
+            realQuery.assetId = { $in: assetId };
         let tags;
-        if (query.tags) {
+        if (query?.tags) {
             if (!Array.isArray(query.tags)) {
                 tags = [query.tags];
             }
@@ -438,15 +487,34 @@ export class StorageService extends Immutable {
             }
             realQuery['metadata.tags'] = { $in: tags };
         }
-        if (query.search) {
+        if (query?.search) {
             realQuery.filename = { $regex: query.search, $options: 'i' };
         }
+        const structureLookupStages = (assetId?.length || query?.populateAssetId)
+            ? [
+                { $lookup: { from: 'structures', localField: 'assetId', foreignField: 'assetId', as: '_structure' } },
+                { $unwind: { path: '$_structure', preserveNullAndEmptyArrays: true } },
+                { $addFields: { assetId: { id: '$assetId', title: { $ifNull: ['$_structure.title', '$_structure.question', ''] } } } },
+                { $project: { _structure: 0 } },
+            ]
+            : [];
+        let result;
         // If ignorePrefix is given, return all matching files without prefix filtering
-        if (query.ignorePrefix) {
-            return StorageModel.db.aggregate([
+        if (query?.ignorePrefix) {
+            // If prefix is provided with ignorePrefix, match all files where prefix starts with the given value
+            if (query?.prefix) {
+                const prefixWithSlash = query.prefix.endsWith('/') ? query.prefix : `${query.prefix}/`;
+                const escapedPrefix = prefixWithSlash.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+                realQuery.$or = [
+                    { prefix: query.prefix }, // Exact match
+                    { prefix: { $regex: `^${escapedPrefix}` } }, // Starts with prefix/
+                ];
+            }
+            result = await StorageModel.db.aggregate([
                 {
                     $match: realQuery,
                 },
+                ...structureLookupStages,
                 {
                     $facet: {
                         metadata: [{ $count: 'all' }],
@@ -454,7 +522,7 @@ export class StorageService extends Immutable {
                             { $sort: { _id: -1 } },
                             { $skip: limit?.skip || 0 },
                             { $limit: limit?.limit || 50 },
-                            { $project: { path: 0, bucket: 0 } },
+                            { $project: { path: 0 } },
                             { $addFields: { type: 'file' } },
                         ],
                     },
@@ -468,11 +536,11 @@ export class StorageService extends Immutable {
                 },
             ])
                 .toArray()
-                .then((result) => result[0] || { all: 0, data: [], count: 0 });
+                .then((r) => r[0] || { all: 0, data: [], count: 0 });
         }
         // If search or tags are given, show only files at the current level (no folders)
-        if (query.search || query.tags) {
-            const isRootLevel = query.prefix === undefined || query.prefix === '';
+        if (!result && (query?.search || query?.tags)) {
+            const isRootLevel = query?.prefix === undefined || query?.prefix === '';
             if (isRootLevel) {
                 // Root level: only files without prefix
                 realQuery.$or = [
@@ -485,10 +553,11 @@ export class StorageService extends Immutable {
                 // Non-root: only files with exact prefix match
                 realQuery.prefix = query.prefix;
             }
-            return StorageModel.db.aggregate([
+            result = await StorageModel.db.aggregate([
                 {
                     $match: realQuery,
                 },
+                ...structureLookupStages,
                 {
                     $facet: {
                         metadata: [{ $count: 'all' }],
@@ -496,7 +565,7 @@ export class StorageService extends Immutable {
                             { $sort: { _id: -1 } },
                             { $skip: limit?.skip || 0 },
                             { $limit: limit?.limit || 50 },
-                            { $project: { path: 0, bucket: 0 } },
+                            { $project: { path: 0 } },
                             { $addFields: { type: 'file' } },
                         ],
                     },
@@ -510,151 +579,213 @@ export class StorageService extends Immutable {
                 },
             ])
                 .toArray()
-                .then((result) => result[0] || { all: 0, data: [], count: 0 });
+                .then((r) => r[0] || { all: 0, data: [], count: 0 });
         }
         // Treat undefined prefix same as empty string (root level)
-        const isRootLevel = query.prefix === undefined || query.prefix === '';
-        if (isRootLevel) {
-            // Root level: show files without prefix and extract virtual folders
-            return StorageModel.db.aggregate([
-                {
-                    $match: realQuery,
-                },
-                {
-                    $addFields: {
-                        // Extract first segment of prefix for root-level folders
-                        immediateSubfolder: {
-                            $cond: {
-                                if: {
-                                    $or: [
-                                        { $eq: ['$prefix', null] },
-                                        { $eq: ['$prefix', ''] },
-                                        { $eq: [{ $type: '$prefix' }, 'missing'] },
-                                    ],
-                                },
-                                then: null, // File at root level (no prefix)
-                                else: {
-                                    // Extract first segment of prefix
-                                    $arrayElemAt: [{ $split: ['$prefix', '/'] }, 0],
+        if (!result) {
+            const isRootLevel = query?.prefix === undefined || query?.prefix === '';
+            if (isRootLevel) {
+                // Root level: show files without prefix and extract virtual folders
+                result = await StorageModel.db.aggregate([
+                    {
+                        $match: realQuery,
+                    },
+                    ...structureLookupStages,
+                    {
+                        $addFields: {
+                            // Extract first segment of prefix for root-level folders
+                            immediateSubfolder: {
+                                $cond: {
+                                    if: {
+                                        $or: [
+                                            { $eq: ['$prefix', null] },
+                                            { $eq: ['$prefix', ''] },
+                                            { $eq: [{ $type: '$prefix' }, 'missing'] },
+                                        ],
+                                    },
+                                    then: null, // File at root level (no prefix)
+                                    else: {
+                                        // Extract first segment of prefix
+                                        $arrayElemAt: [{ $split: ['$prefix', '/'] }, 0],
+                                    },
                                 },
                             },
                         },
                     },
-                },
-                {
-                    $facet: {
-                        files: [
-                            { $match: { immediateSubfolder: null } }, // Files at root level
-                            { $sort: { _id: -1 } },
-                            { $skip: limit?.skip || 0 },
-                            { $limit: limit?.limit || 50 },
-                            { $project: { path: 0, bucket: 0, immediateSubfolder: 0 } },
-                            { $addFields: { type: 'file' } },
-                        ],
-                        folders: [
-                            { $match: { immediateSubfolder: { $ne: null } } },
-                            { $group: { _id: '$immediateSubfolder' } }, // Unique folder names
-                            { $sort: { _id: 1 } },
-                            {
-                                $project: {
-                                    _id: 0,
-                                    prefix: '$_id',
-                                    name: '$_id',
-                                    type: { $literal: 'folder' },
+                    {
+                        $facet: {
+                            files: [
+                                { $match: { immediateSubfolder: null } }, // Files at root level
+                                { $sort: { _id: -1 } },
+                                { $skip: limit?.skip || 0 },
+                                { $limit: limit?.limit || 50 },
+                                { $project: { path: 0, immediateSubfolder: 0 } },
+                                { $addFields: { type: 'file' } },
+                            ],
+                            folders: [
+                                { $match: { immediateSubfolder: { $ne: null } } },
+                                { $group: { _id: '$immediateSubfolder' } }, // Unique folder names
+                                { $sort: { _id: 1 } },
+                                {
+                                    $project: {
+                                        _id: 0,
+                                        prefix: '$_id',
+                                        name: '$_id',
+                                        type: { $literal: 'folder' },
+                                    },
+                                },
+                            ],
+                            metadata: [
+                                { $match: { immediateSubfolder: null } },
+                                { $count: 'all' },
+                            ],
+                        },
+                    },
+                    {
+                        $project: {
+                            all: { $arrayElemAt: ['$metadata.all', 0] },
+                            data: { $concatArrays: ['$folders', '$files'] },
+                            count: { $add: [{ $size: '$folders' }, { $size: '$files' }] },
+                        },
+                    },
+                ])
+                    .toArray()
+                    .then((r) => r[0] || { all: 0, data: [], count: 0 });
+            }
+            else {
+                // Non-root: show exact prefix match and extract subfolders
+                const prefix = query.prefix;
+                const prefixWithSlash = prefix.endsWith('/') ? prefix : `${prefix}/`;
+                const escapedPrefix = prefixWithSlash.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+                // Build non-root query by adding prefix conditions to realQuery
+                const nonRootQuery = {
+                    ...realQuery,
+                    $or: [
+                        { prefix }, // Exact match files
+                        { prefix: { $regex: `^${escapedPrefix}` } }, // Deeper files for folder extraction
+                    ],
+                };
+                result = await StorageModel.db.aggregate([
+                    {
+                        $match: nonRootQuery,
+                    },
+                    ...structureLookupStages,
+                    {
+                        $addFields: {
+                            // Extract immediate subfolder name after current prefix
+                            immediateSubfolder: {
+                                $cond: {
+                                    if: { $eq: ['$prefix', prefix] },
+                                    then: null, // File at exact level
+                                    else: {
+                                        // Extract first segment after prefix
+                                        $arrayElemAt: [
+                                            { $split: [{ $substr: ['$prefix', prefixWithSlash.length, -1] }, '/'] },
+                                            0,
+                                        ],
+                                    },
                                 },
                             },
-                        ],
-                        metadata: [
-                            { $match: { immediateSubfolder: null } },
-                            { $count: 'all' },
-                        ],
+                        },
                     },
-                },
-                {
-                    $project: {
-                        all: { $arrayElemAt: ['$metadata.all', 0] },
-                        data: { $concatArrays: ['$folders', '$files'] },
-                        count: { $add: [{ $size: '$folders' }, { $size: '$files' }] },
+                    {
+                        $facet: {
+                            files: [
+                                { $match: { immediateSubfolder: null } }, // Files at exact level
+                                { $sort: { _id: -1 } },
+                                { $skip: limit?.skip || 0 },
+                                { $limit: limit?.limit || 50 },
+                                { $project: { path: 0, immediateSubfolder: 0 } },
+                                { $addFields: { type: 'file' } },
+                            ],
+                            folders: [
+                                { $match: { immediateSubfolder: { $ne: null } } },
+                                { $group: { _id: '$immediateSubfolder' } }, // Unique folder names
+                                { $sort: { _id: 1 } },
+                                {
+                                    $project: {
+                                        _id: 0,
+                                        prefix: { $concat: [prefixWithSlash, '$_id'] },
+                                        name: '$_id',
+                                        type: { $literal: 'folder' },
+                                    },
+                                },
+                            ],
+                            metadata: [
+                                { $match: { immediateSubfolder: null } },
+                                { $count: 'all' },
+                            ],
+                        },
                     },
-                },
-            ])
-                .toArray()
-                .then((result) => result[0] || { all: 0, data: [], count: 0 });
+                    {
+                        $project: {
+                            all: { $arrayElemAt: ['$metadata.all', 0] },
+                            data: { $concatArrays: ['$folders', '$files'] },
+                            count: { $add: [{ $size: '$folders' }, { $size: '$files' }] },
+                        },
+                    },
+                ])
+                    .toArray()
+                    .then((r) => r[0] || { all: 0, data: [], count: 0 });
+            }
         }
-        // Non-root: show exact prefix match and extract subfolders
-        const prefixWithSlash = query.prefix.endsWith('/') ? query.prefix : `${query.prefix}/`;
-        const escapedPrefix = prefixWithSlash.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
-        // Build non-root query by adding prefix conditions to realQuery
-        const nonRootQuery = {
-            ...realQuery,
-            $or: [
-                { prefix: query.prefix }, // Exact match files
-                { prefix: { $regex: `^${escapedPrefix}` } }, // Deeper files for folder extraction
-            ],
-        };
+        return result || { all: 0, data: [], count: 0 };
+    }
+    /**
+     * Get all files for a company/project with optional prefix filtering.
+     * Returns a flat list of all matching files without creating virtual folders.
+     *
+     * @param company - Company identifier
+     * @param project - Optional project identifier
+     * @param app - Optional app identifier
+     * @param prefix - Optional prefix to filter files (e.g., "folder/subfolder")
+     * @param limit - Optional pagination with skip and limit
+     * @returns Promise with all matching files and count
+     */
+    getAllFiles(company, project, app, prefix, limit) {
+        const query = { company, project, app, parent: { $exists: false } };
+        // If prefix is provided, filter files with that prefix
+        if (prefix) {
+            const prefixWithSlash = prefix.endsWith('/') ? prefix : `${prefix}/`;
+            const escapedPrefix = prefixWithSlash.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+            // Match files that start with the prefix (exact match or deeper)
+            query.$or = [
+                { prefix }, // Exact match
+                { prefix: { $regex: `^${escapedPrefix}` } }, // Starts with prefix/
+            ];
+        }
+        // Build data pipeline stages
+        const dataPipeline = [{ $sort: { _id: -1 } }];
+        // Only add pagination if limit is provided
+        if (limit) {
+            if (limit.skip)
+                dataPipeline.push({ $skip: limit.skip });
+            if (limit.limit)
+                dataPipeline.push({ $limit: limit.limit });
+        }
+        dataPipeline.push({ $project: { history: 0 } });
         return StorageModel.db.aggregate([
             {
-                $match: nonRootQuery,
-            },
-            {
-                $addFields: {
-                    // Extract immediate subfolder name after current prefix
-                    immediateSubfolder: {
-                        $cond: {
-                            if: { $eq: ['$prefix', query.prefix] },
-                            then: null, // File at exact level
-                            else: {
-                                // Extract first segment after prefix
-                                $arrayElemAt: [
-                                    { $split: [{ $substr: ['$prefix', prefixWithSlash.length, -1] }, '/'] },
-                                    0,
-                                ],
-                            },
-                        },
-                    },
-                },
+                $match: query,
             },
             {
                 $facet: {
-                    files: [
-                        { $match: { immediateSubfolder: null } }, // Files at exact level
-                        { $sort: { _id: -1 } },
-                        { $skip: limit?.skip || 0 },
-                        { $limit: limit?.limit || 50 },
-                        { $project: { path: 0, bucket: 0, immediateSubfolder: 0 } },
-                        { $addFields: { type: 'file' } },
-                    ],
-                    folders: [
-                        { $match: { immediateSubfolder: { $ne: null } } },
-                        { $group: { _id: '$immediateSubfolder' } }, // Unique folder names
-                        { $sort: { _id: 1 } },
-                        {
-                            $project: {
-                                _id: 0,
-                                prefix: { $concat: [prefixWithSlash, '$_id'] },
-                                name: '$_id',
-                                type: { $literal: 'folder' },
-                            },
-                        },
-                    ],
-                    metadata: [
-                        { $match: { immediateSubfolder: null } },
-                        { $count: 'all' },
-                    ],
+                    metadata: [{ $count: 'all' }],
+                    data: dataPipeline,
                 },
             },
             {
                 $project: {
                     all: { $arrayElemAt: ['$metadata.all', 0] },
-                    data: { $concatArrays: ['$folders', '$files'] },
-                    count: { $add: [{ $size: '$folders' }, { $size: '$files' }] },
+                    data: 1,
+                    count: { $size: '$data' },
                 },
             },
         ])
             .toArray()
             .then((result) => result[0] || { all: 0, data: [], count: 0 });
     }
-    getTags(company, project, search) {
+    async getTags(company, project, search) {
         let cleanSearch = search.toLowerCase().trim();
         let sortString = cleanSearch;
         let splitSearch = [];
@@ -666,9 +797,9 @@ export class StorageService extends Immutable {
         else {
             cleanSearch = new RegExp(`^(?!([a-z]+):).*(${cleanSearch}).*`);
         }
-        return StorageModel.db.find({ company, project, tags: { $regex: cleanSearch, $options: 'i' } }, { projection: { tags: 1 } })
-            .toArray()
-            .then((matchedDocuments) => this.matchSortTags(matchedDocuments, cleanSearch, sortString));
+        const matchedDocuments = await StorageModel.db.find({ company, project, 'metadata.tags': { $regex: cleanSearch, $options: 'i' } }, { projection: { 'metadata.tags': 1 } })
+            .toArray();
+        return this.matchSortTags(matchedDocuments, cleanSearch, sortString);
     }
     single(company, project, filename) {
         return StorageModel.db.aggregate([
@@ -770,21 +901,39 @@ export class StorageService extends Immutable {
     /**
      * checks if the file was already successfully uploaded
      */
-    exists(filename, prefix, app, options) {
-        return StorageModel.db.countDocuments({
+    exists(filename, prefix, app, customer, list, options) {
+        const query = {
             company: options.company,
             project: options.project,
             filename,
             prefix,
             app,
-        })
+        };
+        if (customer)
+            query.customer = customer;
+        if (list)
+            query.list = list;
+        return StorageModel.db.countDocuments(query)
             .then((exists) => exists >= 1);
     }
+    /**
+     * counts non-versioned storage documents bound to a specific assetId
+     */
+    async countByAssetId(assetId, app, customer, list, options) {
+        return StorageModel.db.countDocuments({
+            company: options.company,
+            project: options.project,
+            app,
+            assetId,
+            customer,
+            list,
+        });
+    }
     matchSortTags(documents, search, sort) {
         const mergedArray = [];
         const returnArray = [];
         documents.forEach((document) => {
-            document.tags.forEach((tag) => {
+            document.metadata?.tags?.forEach((tag) => {
                 if (mergedArray.find((tagArray) => tagArray.tag === tag))
                     return;
                 if (!tag.match(search))
@@ -801,14 +950,17 @@ export class StorageService extends Immutable {
     checkAllowedSize(filesize, allowedSize) {
         return filesize < allowedSize;
     }
-    stats(company, project) {
+    stats(company, project, customer) {
+        const match = {
+            company,
+            project,
+            parent: { $exists: false },
+        };
+        if (customer)
+            match.customer = customer;
         const aggregation = [
             {
-                $match: {
-                    company,
-                    project,
-                    parent: { $exists: false },
-                },
+                $match: match,
             },
             {
                 $group: {
@@ -833,28 +985,47 @@ export class StorageService extends Immutable {
             .then((value) => value[0] || { files: 0, size: 0 });
     }
     getByInternalId(internalId, options) {
-        return StorageModel.db.findOne({ _id: internalId, company: options.company, project: options.project });
+        return StorageModel.db.findOne({ _id: internalId, company: options.company, project: options.project }, { projection: { path: 0 } });
     }
-    getPathByFilename(filename, app, options) {
+    getPathByFilename(filename, app, customer, list, options) {
         const filenameObject = filenameHelper(filename);
-        return StorageModel.db.findOne({
+        const query = {
             filename: filenameObject.filename,
             prefix: filenameObject.directory,
             company: options.company,
             project: options.project,
             app,
-        }, {
+        };
+        if (customer)
+            query.customer = customer;
+        if (list)
+            query.list = list;
+        return StorageModel.db.findOne(query, {
             projection: { path: 1, metadata: 1 },
         });
     }
-    getByFilename(filename, prefix, bucket, options) {
-        return StorageModel.db.findOne({
+    getByFilename(filename, prefix, app, customer, list, options) {
+        const query = {
             filename,
             prefix,
             company: options.company,
             project: options.project,
-            bucket,
-        });
+            app,
+        };
+        if (customer)
+            query.customer = customer;
+        if (list)
+            query.list = list;
+        return StorageModel.db.findOne(query);
+    }
+    async removeBlobMembers(internalId, customer, options) {
+        const storage = await this.model.db.findOne({ _id: internalId, company: options.company, project: options.project, customer });
+        if (!storage)
+            throw new Error('BLOB_NOT_FOUND');
+        await fs.promises.unlink(`${this.options.dataFolder}/${storage.path}`);
+        await this.removeBlobVersion(internalId, options);
+        await this.delete(internalId, options.user, options.app);
+        await jetstreamService.publish(`edge.${storage.app}.removed`, storage);
     }
     async removeBlob(internalId, options) {
         const storage = await this.model.db.findOne({ _id: internalId, company: options.company, project: options.project });
@@ -862,14 +1033,16 @@ export class StorageService extends Immutable {
             throw new Error('BLOB_NOT_FOUND');
         await fs.promises.unlink(`${this.options.dataFolder}/${storage.path}`);
         await this.removeBlobVersion(internalId, options);
-        return this.delete(internalId, options.user, options.app);
+        await this.delete(internalId, options.user, options.app);
+        await jetstreamService.publish(`edge.${storage.app}.removed`, storage);
     }
     async removeBlobVersion(parentId, options) {
         const versions = await this.model.db.find({ parent: parentId, company: options.company, project: options.project }).toArray();
         if (versions.length) {
             await Promise.allSettled(versions.map(async (single) => {
                 await fs.promises.unlink(`${this.options.dataFolder}/${single.path}`);
-                return this.delete(single._id, options.user, options.app);
+                await this.delete(single._id, options.user, options.app);
+                await jetstreamService.publish(`edge.${single.app}.removed`, single);
             }));
         }
     }