@lenne.tech/nest-server 11.33.1 → 11.34.0

package/src/core/modules/hub/services/core-hub-db.service.ts

@@ -3,11 +3,26 @@ import { InjectConnection } from '@nestjs/mongoose';
 import * as mongo from 'mongodb';
 import { Connection, Types } from 'mongoose';
 
+import { ConfigService } from '../../../common/services/config.service';
+import { CoreS3Service } from '../../../common/services/core-s3.service';
 import { ModelRegistry } from '../../../common/services/model-registry.service';
+import {
+  DEFAULT_FILESYSTEM_DIR,
+  FILESYSTEM_FILES_COLLECTION,
+  FilesystemFileHelper,
+} from '../../file/filesystem-file.helper';
+import { S3_FILES_COLLECTION, S3FileHelper } from '../../file/s3-file.helper';
 import { HubActionMessage } from '../hub-action-messages';
 import { HUB_CONFIG } from '../hub.constants';
 import { buildErDiagram, HubModelDescriptor, HubModelField } from '../helpers/hub-mermaid.helper';
-import { HubDbData, HubFilesData, HubModelsData, HubUnavailable } from '../interfaces/hub-panels.interface';
+import {
+  HubDbData,
+  HubFilesData,
+  HubFileStore,
+  HubFileStoreSummary,
+  HubModelsData,
+  HubUnavailable,
+} from '../interfaces/hub-panels.interface';
 import { ResolvedHubConfig } from '../interfaces/hub-config.interface';
 
 /**
@@ -23,6 +38,10 @@ export class CoreHubDbService {
   constructor(
     @Inject(HUB_CONFIG) protected readonly config: ResolvedHubConfig,
     @Optional() @InjectConnection() protected readonly connection?: Connection,
+    // Provided and exported globally by `CoreModule`, but INERT without an `s3` config — so it is
+    // optional here for the same reason it is everywhere else: a project that uses no S3 must not
+    // be forced to install `@aws-sdk/client-s3` to open the Hub.
+    @Optional() protected readonly s3Service?: CoreS3Service,
   ) {}
 
   /** Database + per-collection statistics. */
@@ -85,10 +104,21 @@ export class CoreHubDbService {
   }
 
   /**
-   * Delete a GridFS file by id. When `expectedFilename` is given it must match the stored filename
-   * (the type-to-confirm keyword). Returns the filename; throws when not found or the name mismatches.
+   * Delete a file by id, from WHICHEVER store actually holds it.
+   *
+   * When `expectedFilename` is given it must match the stored filename (the type-to-confirm
+   * keyword). Returns the filename and the store it came from; throws when not found, when the name
+   * mismatches, or when the owning store cannot be reached.
+   *
+   * This used to delete from GridFS only, which meant an S3- or filesystem-backed file could not be
+   * removed from the Hub at all — the lookup answered `File not found.` for a file plainly sitting
+   * in the bucket.
    */
-  async deleteFile(id: string, expectedFilename?: string, bucket = 'fs'): Promise<{ filename: string; id: string }> {
+  async deleteFile(
+    id: string,
+    expectedFilename?: string,
+    bucket = 'fs',
+  ): Promise<{ filename: string; id: string; store: HubFileStore }> {
     const db = this.connection?.db;
     if (!db) {
       throw new Error(HubActionMessage.mongoUnavailable);
@@ -97,46 +127,140 @@ export class CoreHubDbService {
       throw new Error(HubActionMessage.invalidFileId);
     }
     const objectId = new Types.ObjectId(id);
-    const [doc] = await db.collection(`${bucket}.files`).find({ _id: objectId }, { limit: 1 }).toArray();
-    if (!doc) {
+
+    // Same probe order as `CoreFileService.deleteFile()`: S3, then filesystem, then GridFS as the
+    // fallthrough. Keeping the two in step matters — a Hub that resolved a file differently from the
+    // service would delete a different file than the one it displayed.
+    let store: HubFileStore | undefined;
+    let doc: any;
+    for (const candidate of this.fileStores(bucket)) {
+      const [found] = await db.collection(candidate.collection).find({ _id: objectId }, { limit: 1 }).toArray();
+      if (found) {
+        doc = found;
+        store = candidate.store;
+        break;
+      }
+    }
+    if (!doc || !store) {
       throw new Error(HubActionMessage.fileNotFound);
     }
     if (expectedFilename !== undefined && expectedFilename !== doc.filename) {
       throw new Error(HubActionMessage.confirmationFilenameMismatch);
     }
-    const gridFs = new mongo.GridFSBucket(db as unknown as mongo.Db, { bucketName: bucket });
-    await gridFs.delete(objectId as unknown as mongo.ObjectId);
-    return { filename: doc.filename, id };
+
+    switch (store) {
+      case 'filesystem': {
+        await FilesystemFileHelper.deleteFile(this.filesystemDir, db.collection(FILESYSTEM_FILES_COLLECTION), objectId);
+        break;
+      }
+      case 's3': {
+        // Refuse rather than half-delete — see HubActionMessage.s3Unavailable.
+        if (!this.s3Service?.enabled) {
+          throw new Error(HubActionMessage.s3Unavailable);
+        }
+        await S3FileHelper.deleteFile(this.s3Service, db.collection(S3_FILES_COLLECTION), objectId);
+        break;
+      }
+      default: {
+        const gridFs = new mongo.GridFSBucket(db as unknown as mongo.Db, { bucketName: bucket });
+        await gridFs.delete(objectId as unknown as mongo.ObjectId);
+      }
+    }
+
+    return { filename: doc.filename, id, store };
   }
 
-  /** GridFS file inventory for the given bucket (default `fs`). */
+  /**
+   * File inventory across ALL THREE metadata stores.
+   *
+   * Metadata always lives in MongoDB whichever store holds the bytes (`fs.files` / `s3-files` /
+   * `filesystem-files`), so one connection can see everything — which is exactly what
+   * `CoreFileService.findFileInfo()` does, and what this panel now mirrors.
+   *
+   * It used to read `fs.files` alone. Under `file.storage: 's3'` or `'filesystem'` that made the
+   * panel report **0 files** — not "this panel does not cover your driver", but a confident wrong
+   * answer, in the one tool an operator opens BECAUSE they are unsure. Someone checking whether an
+   * upload landed would have concluded it had not.
+   *
+   * Paging over a merge is the trap here: taking `skip`/`limit` from each store and concatenating
+   * returns the wrong ROWS, not merely the wrong order (the same defect `sortMergedFileInfo()` was
+   * fixed for). So each store yields its newest `skip + limit`, the union is re-sorted, and only
+   * then is the window cut — which is correct because the global newest `skip + limit` rows are
+   * necessarily contained in the per-store newest `skip + limit`.
+   *
+   * READS NEVER CREATE A COLLECTION: `find()` and `countDocuments()` on an absent collection answer
+   * empty. A GridFS-only deployment therefore does not grow an empty `s3-files` from the Hub merely
+   * looking at it — the same rule `ensureFilenameIndex()` follows on the write path.
+   */
   async getFiles(bucket = 'fs', skip = 0, limit = 100): Promise<HubFilesData | HubUnavailable> {
     const db = this.connection?.db;
     if (!db) {
       return { available: false, hint: 'No MongoDB connection is available.' };
     }
-    const filesCollection = `${bucket}.files`;
-    try {
-      const cursor = db
-        .collection(filesCollection)
-        .find({}, { limit: Math.min(limit, 500), skip, sort: { uploadDate: -1 } });
-      const docs = await cursor.toArray();
-      const total = await db.collection(filesCollection).countDocuments();
-      return {
-        bucket,
-        files: docs.map((doc) => ({
-          contentType: doc.contentType ?? doc.metadata?.contentType,
-          filename: doc.filename,
-          id: String(doc._id),
-          length: doc.length ?? 0,
-          uploadDate: doc.uploadDate ? new Date(doc.uploadDate).toISOString() : undefined,
-        })),
-        total,
-      };
-    } catch (error) {
-      this.logger.warn(`Failed to list GridFS files: ${error instanceof Error ? error.message : String(error)}`);
-      return { available: false, hint: `No GridFS bucket "${bucket}" or it is empty.` };
+
+    const safeSkip = Math.max(0, skip);
+    const safeLimit = Math.min(Math.max(1, limit), 500);
+    // Per-store window: enough to guarantee the merged page is complete, capped so a huge `skip`
+    // cannot turn one panel poll into an unbounded read.
+    const window = Math.min(safeSkip + safeLimit, 1000);
+
+    const stores: HubFileStoreSummary[] = [];
+    const rows: HubFilesData['files'] = [];
+
+    for (const source of this.fileStores(bucket)) {
+      try {
+        const docs = await db
+          .collection(source.collection)
+          .find({}, { limit: window, sort: { uploadDate: -1 } })
+          .toArray();
+        const count = await db.collection(source.collection).countDocuments();
+        stores.push({ collection: source.collection, count, store: source.store });
+        for (const doc of docs) {
+          rows.push({
+            contentType: doc.contentType ?? doc.metadata?.contentType,
+            filename: doc.filename,
+            id: String(doc._id),
+            length: doc.length ?? 0,
+            store: source.store,
+            uploadDate: doc.uploadDate ? new Date(doc.uploadDate).toISOString() : undefined,
+          });
+        }
+      } catch (error) {
+        // One unreadable store must not blank the panel: the other two still hold real answers, and
+        // an operator is better served by "two stores listed, this one errored" than by nothing.
+        const message = error instanceof Error ? error.message : String(error);
+        this.logger.warn(`Failed to list files in ${source.collection}: ${message}`);
+        stores.push({ collection: source.collection, count: 0, error: message, store: source.store });
+      }
     }
+
+    rows.sort((a, b) => (b.uploadDate ?? '').localeCompare(a.uploadDate ?? ''));
+
+    return {
+      bucket,
+      files: rows.slice(safeSkip, safeSkip + safeLimit),
+      stores,
+      total: stores.reduce((sum, entry) => sum + entry.count, 0),
+    };
+  }
+
+  /** Directory used by the `'filesystem'` driver — resolved exactly as `CoreFileService` does. */
+  protected get filesystemDir(): string {
+    return ConfigService.configFastButReadOnly?.file?.storageDir || DEFAULT_FILESYSTEM_DIR;
+  }
+
+  /**
+   * The three metadata stores, in the order both this service and `CoreFileService` probe them.
+   *
+   * GridFS is last because it is the fallthrough: its collection name depends on the bucket, and a
+   * document there carries no `storage` marker to identify it by.
+   */
+  protected fileStores(bucket: string): { collection: string; store: HubFileStore }[] {
+    return [
+      { collection: S3_FILES_COLLECTION, store: 's3' },
+      { collection: FILESYSTEM_FILES_COLLECTION, store: 'filesystem' },
+      { collection: `${bucket}.files`, store: 'gridfs' },
+    ];
   }
 
   /** Model inventory + a Mermaid ER diagram derived from the registered Mongoose schemas. */

package/src/server/modules/file/file.service.ts

@@ -5,6 +5,7 @@ import { Connection } from 'mongoose';
 import { RoleEnum } from '../../../core/common/enums/role.enum';
 import { ConfigService } from '../../../core/common/services/config.service';
 import { CoreS3Service } from '../../../core/common/services/core-s3.service';
+import { CoreFileInfo } from '../../../core/modules/file/core-file-info.model';
 import { CoreFileService, FileInputCheckType } from '../../../core/modules/file/core-file.service';
 import { FileServiceOptions } from '../../../core/modules/file/interfaces/file-service-options.interface';
 
@@ -22,10 +23,26 @@ export class FileService extends CoreFileService {
   }
 
   /**
-   * Duplicate file by name
+   * Duplicate file by name.
+   *
+   * Delegates instead of reaching into `this.files` directly. The direct GridFS
+   * pipe this used to be was wrong in four separate ways, and every one of them
+   * is the kind of thing a consuming project copies out of here:
+   *
+   * - it only ever worked on the GridFS driver — under `file.storage: 's3'` or
+   *   `'filesystem'` the source is simply not in the bucket;
+   * - it bypassed `checkRights()` entirely, so the owner rule below did not apply
+   *   to a duplicate at all;
+   * - it returned the write stream without awaiting it, so the caller was told the
+   *   copy existed while it was still being written;
+   * - neither stream carried an error handler, and an unhandled stream `'error'`
+   *   takes the whole process down.
+   *
+   * `duplicateByName()` answers all four. Forward the caller's context so the
+   * duplicate is COVERED by the ownership rule rather than exempt from it.
    */
-  async duplicate(fileName: string, newName: string): Promise<any> {
-    return this.files.openDownloadStreamByName(fileName).pipe(this.files.openUploadStream(newName));
+  async duplicate(fileName: string, newName: string, serviceOptions?: FileServiceOptions): Promise<CoreFileInfo> {
+    return this.duplicateByName(fileName, newName, serviceOptions);
   }
 
   /**