@lenne.tech/nest-server 11.33.0 → 11.34.0

package/src/core/modules/file/core-file.service.ts

@@ -1,8 +1,9 @@
 import { Logger, NotFoundException } from '@nestjs/common';
-import mongoose, { Connection, mongo, Types } from 'mongoose';
+import { Connection, mongo, Types } from 'mongoose';
 import { Readable } from 'stream';
 
 import { FilterArgs } from '../../common/args/filter.args';
+import { accessDeniedException } from '../../common/exceptions/access-denied.exception';
 import { getObjectIds, getStringIds } from '../../common/helpers/db.helper';
 import { convertFilterArgsToQuery } from '../../common/helpers/filter.helper';
 import { GridFSHelper } from '../../common/helpers/gridfs.helper';
@@ -195,73 +196,103 @@ export abstract class CoreFileService {
 
   /**
    * Duplicate file by name
+   *
+   * A duplicate is a READ of the source plus a WRITE of the copy, and it is
+   * authorized as exactly that: the source goes through `getFileInfoByName()` +
+   * `getFileStreamByName()` (`checkInputType: 'filename'`) and the copy through
+   * `createFile()` (`checkInputType: 'file'`) — the same public methods, with the
+   * same `checkRights()` hook, that every other caller uses.
+   *
+   * `serviceOptions` is OPTIONAL and additive. Omitting it keeps the pre-11.34.0
+   * behaviour for the framework default `checkRights()` (which returns `true`),
+   * and turns what used to be a crash into a clean refusal for a project that
+   * overrides it fail-closed: the GridFS branch used to bypass the hook
+   * altogether and copy the file unchecked, while the S3 / filesystem branches
+   * died inside a storage helper. System-internal callers say `{ force: true }`,
+   * the same idiom the rest of this service uses.
+   *
+   * The copy does NOT inherit the source's `metadata`. Copying it would silently
+   * hand the duplicate the source's owner, which is the one thing an ownership
+   * rule must not do behind the caller's back — pass `serviceOptions.metadata`
+   * to state the copy's own.
+   *
+   * @returns the file info of the COPY. Up to 11.33.1 the GridFS branch resolved
+   *   the raw `GridFSBucketWriteStream` instead, so a project reading anything
+   *   beyond `.id` / `.filename` off it has to adjust.
    */
-  async duplicateByName(name: string, newName: string): Promise<any> {
-    // Route through the storage dispatch like every other read/write. Going straight to GridFS
-    // meant that with `file.storage: 's3'` the source simply is not there — and the resulting
-    // FileNotFound arrives on a stream with NO error handler, so it becomes an uncaught
-    // exception that takes the process down instead of a 404.
-    const nonGridFsSource = (await this.findS3FileByName(name)) || (await this.findFilesystemFileByName(name));
-    if (nonGridFsSource) {
-      const source = await this.getFileStreamByName(name);
-      return this.createFile({
-        createReadStream: () => source,
-        filename: newName,
-        mimetype: nonGridFsSource.contentType || 'application/octet-stream',
-      });
+  async duplicateByName(name: string, newName: string, serviceOptions?: FileServiceOptions): Promise<CoreFileInfo> {
+    const source = await this.getFileInfoByName(name, serviceOptions);
+    if (!source) {
+      throw new NotFoundException(`File not found with filename ${name}`);
     }
-
-    return new Promise((resolve, reject) => {
-      const downloadStream = GridFSHelper.openDownloadStreamByName(this.files, name);
-      downloadStream.on('error', reject);
-      const uploadStream = GridFSHelper.openUploadStream(this.files, newName);
-      uploadStream.on('error', reject);
-      uploadStream.on('finish', () => resolve(uploadStream));
-      downloadStream.pipe(uploadStream);
-    });
+    return this.duplicateFile(source, newName, () => this.getFileStreamByName(name, serviceOptions), serviceOptions);
   }
 
   /**
    * Duplicate file by ID
+   *
+   * See {@link duplicateByName} for the authorization model and for why
+   * `serviceOptions` matters. The copy keeps the source's filename.
+   *
+   * @returns the id of the copy
    */
-  async duplicateById(id: string): Promise<string> {
+  async duplicateById(id: string | Types.ObjectId, serviceOptions?: FileServiceOptions): Promise<string> {
     const objectId = getObjectIds(id);
-    const file = await this.getFileInfo(objectId);
-
-    // Same dispatch as duplicateByName: a file stored outside GridFS has no GridFS counterpart.
-    const nonGridFsSource = (await this.findS3FileById(objectId)) || (await this.findFilesystemFileById(objectId));
-    if (nonGridFsSource) {
-      const source = await this.getFileStream(objectId);
-      const copy = await this.createFile({
-        createReadStream: () => source,
-        filename: file.filename,
-        mimetype: file.contentType || 'application/octet-stream',
-      });
-      return copy.id;
-    }
-
-    return new Promise((resolve, reject) => {
-      const downloadStream = GridFSHelper.openDownloadStream(this.files, objectId);
-
-      const newFileId = new mongoose.Types.ObjectId();
-      const uploadStream = GridFSHelper.openUploadStreamWithId(this.files, newFileId, file.filename, {
-        contentType: file.contentType,
-      });
-
-      downloadStream.pipe(uploadStream);
+    const source = await this.getFileInfo(objectId, serviceOptions);
+    if (!source) {
+      // Was a `TypeError` on `file.filename`: the source had been re-resolved with
+      // an empty context, so a fail-closed rule answered null and the null was then
+      // dereferenced. A refusal must read as a refusal, never as a crash.
+      throw new NotFoundException(`File not found with id ${getStringIds(objectId)}`);
+    }
+    const copy = await this.duplicateFile(
+      source,
+      source.filename,
+      () => this.getFileStream(objectId, serviceOptions),
+      serviceOptions,
+    );
+    return copy.id;
+  }
 
-      uploadStream.on('finish', () => {
-        resolve(getStringIds(newFileId));
-      });
+  /**
+   * Shared write half of the two duplicate methods.
+   *
+   * Deliberately driver-agnostic: it opens the source through the public read
+   * path (whichever store answers) and writes the copy through `createFile()`
+   * (whichever store is active). That is what makes a duplicate work ACROSS a
+   * driver change — a file still in GridFS is copied into S3 without a migration
+   * step — and what keeps the three drivers from drifting apart again.
+   */
+  protected async duplicateFile(
+    source: CoreFileInfo,
+    newName: string,
+    openSourceStream: () => Promise<Readable>,
+    serviceOptions?: FileServiceOptions,
+  ): Promise<CoreFileInfo> {
+    const stream = await openSourceStream();
+    if (!stream) {
+      // The read was refused between the two checks (or the bytes vanished).
+      // Same answer as an unknown file: never confirm that it exists.
+      throw new NotFoundException(`File not found with filename ${source.filename}`);
+    }
 
-      uploadStream.on('error', (err: { message: any }) => {
-        reject(new Error(`File duplication failed: ${err.message}`));
-      });
+    const copy = await this.createFile(
+      {
+        createReadStream: () => stream,
+        filename: newName,
+        mimetype: source.contentType || 'application/octet-stream',
+      },
+      serviceOptions,
+    );
 
-      downloadStream.on('error', (err: { message: any }) => {
-        reject(new Error(`File download failed: ${err.message}`));
-      });
-    });
+    if (!copy) {
+      // The WRITE half was refused. Unlike a refused read this is not an
+      // existence question — the caller has already been shown the source — so it
+      // answers with the framework's 401/403 policy rather than a 404.
+      stream.destroy?.();
+      throw accessDeniedException(serviceOptions?.currentUser);
+    }
+    return copy;
   }
 
   /**
@@ -317,8 +348,8 @@ export abstract class CoreFileService {
     const effective: [string, number][] = spec.length ? (spec as [string, number][]) : [['uploadDate', -1]];
     return [...docs].sort((a, b) => {
       for (const [field, direction] of effective) {
-        const left = a?.[field];
-        const right = b?.[field];
+        const left = this.readSortField(a, field);
+        const right = this.readSortField(b, field);
         if (left === right) {
           continue;
         }
@@ -336,6 +367,33 @@ export abstract class CoreFileService {
     });
   }
 
+  /**
+   * Read the value a `sort` key names, resolving DOTTED paths.
+   *
+   * `SortInput.field` is a free string, and MongoDB reads `metadata.ownerId` as a
+   * path into the document — so each store sorted correctly on its own while the
+   * merge above compared `doc['metadata.ownerId']`, which is `undefined` for every
+   * row. Every comparison then tied, the merged page came out grouped by store,
+   * and `skip`/`limit` over it returned the WRONG ROWS rather than the right rows
+   * in the wrong order.
+   *
+   * A path segment is only followed through plain objects. Anything else stops the
+   * walk and yields `undefined`, which the caller already sorts last.
+   */
+  protected readSortField(doc: Record<string, any>, field: string): any {
+    if (!field.includes('.')) {
+      return doc?.[field];
+    }
+    let current: any = doc;
+    for (const segment of field.split('.')) {
+      if (current === null || current === undefined || typeof current !== 'object') {
+        return undefined;
+      }
+      current = current[segment];
+    }
+    return current;
+  }
+
   /**
    * Get info about file via file ID
    */
@@ -531,6 +589,17 @@ export abstract class CoreFileService {
 
   /**
    * Delete file
+   *
+   * An unknown id answers `NotFoundException`, exactly as {@link deleteFileByName}
+   * does for an unknown filename. It used to fall through to GridFS and surface
+   * the driver's own `MongoRuntimeError: File not found for id …` — a **500** for
+   * the very condition its by-name sibling reported as a clean **404**, under all
+   * three storage drivers.
+   *
+   * A REFUSAL still answers `null` rather than throwing, which is the module-wide
+   * contract: a refusal must be indistinguishable from a file that is not there.
+   * Reaching the lookup below means `checkRights()` already said yes for this
+   * exact input, so a `null` here is genuinely a missing file.
    */
   async deleteFile(id: string | Types.ObjectId, serviceOptions?: FileServiceOptions): Promise<CoreFileInfo> {
     if (!(await this.checkRights(id, { ...serviceOptions, checkInputType: 'id' }))) {
@@ -538,6 +607,9 @@ export abstract class CoreFileService {
     }
     const objectId = getObjectIds(id);
     const fileInfo = await this.getFileInfo(objectId, serviceOptions);
+    if (!fileInfo) {
+      throw new NotFoundException(`File not found with id ${getStringIds(objectId)}`);
+    }
     if (await this.findS3FileById(objectId)) {
       await S3FileHelper.deleteFile(this.options.s3Service, this.s3Files, objectId);
       return fileInfo;
@@ -557,7 +629,13 @@ export abstract class CoreFileService {
     if (!(await this.checkRights(filename, { ...serviceOptions, checkInputType: 'filename' }))) {
       return null;
     }
-    const fileInfo = await this.getFileInfoByName(filename);
+    // Forward the caller's context, exactly as deleteFile() does for getFileInfo().
+    // Without it this method authorizes the caller and then re-resolves the file with
+    // an EMPTY context, so an overridden checkRights() is asked two different
+    // questions about one request. For an ownership rule that means the lookup denies
+    // — and the caller gets `File not found` for a file that exists and that they were
+    // just authorized for, which reads as a missing file rather than a refused one.
+    const fileInfo = await this.getFileInfoByName(filename, serviceOptions);
     if (!fileInfo) {
       throw new NotFoundException(`File not found with filename ${filename}`);
     }
@@ -665,20 +743,48 @@ export abstract class CoreFileService {
    * routes. Metadata to compare against must be written at upload time via
    * `serviceOptions.metadata` and read back with `getRawFileInfo()`.
    *
+   * **A missing `currentUser` must DENY.** This is the one part of the rule that
+   * is easy to get backwards: "no user in context" is NOT "system-internal call"
+   * — it is also exactly what an ANONYMOUS request looks like. The coarse gate
+   * turns those away while `downloadRoles` is narrower than `S_EVERYONE`, so an
+   * `if (!options.currentUser) return true` shortcut looks harmless — right up to
+   * the moment a project widens the gate, at which point it hands every file to
+   * everyone and the ownership rule evaporates precisely when it starts to
+   * matter. Genuinely internal callers say so with `force: true`, or pass the
+   * user they already have; see `src/server/modules/file` and
+   * `src/server/modules/user/avatar.controller.ts`.
+   *
+   * **Cover the `filename` branch too, not just `id`.** An id-only rule is
+   * enough while bytes are streamed, because the filename route resolves an id
+   * and checks it again — but NOT once `s3.presignedDownloads` is enabled, where
+   * the filename route authorizes on the by-name lookup alone and then redirects,
+   * and not for `deleteFileByName()`, which authorizes by name only.
+   *
+   * The example below is the rule `src/server/modules/file/file.service.ts`
+   * actually runs — it is compiled, type-checked and exercised end to end by
+   * every file-touching e2e spec, plus the dedicated contract test in
+   * `tests/file-ownership.e2e-spec.ts`. Prefer reading it there over copying
+   * from here.
+   *
    * @example
    * ```typescript
    * protected override async checkRights(
    *   input: any,
    *   options?: FileServiceOptions & { checkInputType: FileInputCheckType },
    * ): Promise<boolean> {
-   *   if (options?.checkInputType !== 'id' || options.force) {
+   *   if (options?.force || (options?.checkInputType !== 'filename' && options?.checkInputType !== 'id')) {
    *     return true;
    *   }
-   *   if (options.currentUser?.hasRole([RoleEnum.ADMIN])) {
+   *   if (options.currentUser?.hasRole?.([RoleEnum.ADMIN])) {
    *     return true;
    *   }
-   *   const raw = await this.getRawFileInfo(input);
-   *   return !!raw && String(raw.metadata?.ownerId) === String(options.currentUser?.id);
+   *   const raw = options.checkInputType === 'id'
+   *     ? await this.getRawFileInfo(input)
+   *     : await this.getRawFileInfoByName(input);
+   *   // Fails closed without a user: `String(undefined)` cannot equal a real owner id.
+   *   // Requiring the owner to be PRESENT is load-bearing too — comparing a missing
+   *   // ownerId against a missing user id would compare 'undefined' with 'undefined'.
+   *   return !!raw?.metadata?.ownerId && String(raw.metadata.ownerId) === String(options.currentUser?.id);
    * }
    * ```
    */

package/src/core/modules/hub/README.md

@@ -38,7 +38,7 @@ Do NOT set `hub` in the `production` block unless you intend the cockpit to be r
 | Database             | `/hub/db`             | dbStats / per-collection collStats                              |
 | Models / ERD         | `/hub/models`         | Mongoose schemas → Mermaid ER diagram                           |
 | Migrations           | `/hub/migrations`     | `MigrationRunner` status + run/rollback                         |
-| Files                | `/hub/files`          | GridFS listing + delete                                         |
+| Files                | `/hub/files`          | listing + delete across all three storage drivers               |
 | Config               | `/hub/config`         | full config, secrets masked                                     |
 | Auth Migration       | `/hub/auth-migration` | Legacy → IAM progress (BetterAuth, optional)                    |
 | Routes / Permissions | `/hub/routes`         | route + role + `@Restricted` map (Permissions module, optional) |
@@ -50,19 +50,45 @@ Do NOT set `hub` in the `production` block unless you intend the cockpit to be r
 Each panel has a `*.json` sidecar (the stable data contract) that the client polls. Optional sources
 degrade to an "unavailable" state instead of erroring.
 
+### Files panel
+
+The panel reads all three file metadata stores, not just GridFS — the same three
+`CoreFileService.findFileInfo()` consults, because metadata always lives in MongoDB whichever store
+holds the bytes:
+
+| Store        | Collection                            | Bytes live in                   |
+| ------------ | ------------------------------------- | ------------------------------- |
+| `s3`         | `s3-files`                            | the S3 bucket                   |
+| `filesystem` | `filesystem-files`                    | `file.storageDir` on local disk |
+| `gridfs`     | `<bucket>.files` (default `fs.files`) | MongoDB                         |
+
+Every row carries the `store` it was found in, and `files.json` reports a per-store `count` so an
+empty store reads as **empty** rather than as _"this panel does not cover your driver"_. Until
+11.34.0 it read `fs.files` alone, so under `file.storage: 's3'` or `'filesystem'` it confidently
+answered **0 files** — the worst answer available, since the Hub is what an operator opens precisely
+when they are unsure whether an upload landed.
+
+Two properties worth knowing:
+
+- **Delete dispatches to the owning store.** An S3-backed file needs `CoreS3Service` to be
+  configured in this process; without it the action REFUSES (`S3 storage is not available.`) rather
+  than deleting the metadata document and orphaning the object in the bucket.
+- **Reading never creates a collection.** A GridFS-only deployment does not grow an empty
+  `s3-files` / `filesystem-files` from the panel looking at them.
+
 ## Actions (mutating)
 
 Enabled by default (`actions: true`). Every mutating request requires the `X-Hub-Request: 1` header
 (CSRF defense) and destructive ones a server-validated `confirm` keyword:
 
-| Action                  | Endpoint                                   | Confirm      |
-| ----------------------- | ------------------------------------------ | ------------ |
-| Run pending migrations  | `POST /hub/actions/migrations/run`         | `RUN`        |
-| Rollback last migration | `POST /hub/actions/migrations/down`        | `DOWN`       |
-| Delete GridFS file      | `DELETE /hub/actions/files/:id`            | the filename |
-| Cron start/stop/trigger | `POST /hub/actions/cron/:name/:action`     | the job name |
-| Clear collector buffer  | `POST /hub/actions/collectors/:name/clear` | `CLEAR`      |
-| Send test mail          | `POST /hub/actions/email/test`             | —            |
+| Action                   | Endpoint                                   | Confirm      |
+| ------------------------ | ------------------------------------------ | ------------ |
+| Run pending migrations   | `POST /hub/actions/migrations/run`         | `RUN`        |
+| Rollback last migration  | `POST /hub/actions/migrations/down`        | `DOWN`       |
+| Delete file (any driver) | `DELETE /hub/actions/files/:id`            | the filename |
+| Cron start/stop/trigger  | `POST /hub/actions/cron/:name/:action`     | the job name |
+| Clear collector buffer   | `POST /hub/actions/collectors/:name/clear` | `CLEAR`      |
+| Send test mail           | `POST /hub/actions/email/test`             | —            |
 
 Every action writes an audit line: `[HUB-ACTION] <action> by user <id>`.
 

package/src/core/modules/hub/helpers/hub-client-js.helper.ts

@@ -506,9 +506,20 @@ export function getHubClientJs(): string {
 
   Hub.panels.files = function (content) {
     pollInto(content, 'files.json', function (c, d) {
-      c.appendChild(tiles([{ label: 'Files', value: d.total }, { label: 'Bucket', value: d.bucket }]));
+      // One tile per metadata store, so an empty store reads as "empty" rather than as "this panel
+      // does not cover your driver" — the ambiguity that made a GridFS-only listing look like the
+      // whole truth under file.storage: 's3'.
+      var storeTiles = [{ label: 'Files', value: d.total }];
+      (d.stores || []).forEach(function (s) {
+        storeTiles.push({ cls: s.error ? 'err' : '', label: s.store === 'gridfs' ? 'GridFS (' + d.bucket + ')' : s.store, value: s.error ? '!' : s.count });
+      });
+      c.appendChild(tiles(storeTiles));
+      (d.stores || []).filter(function (s) { return s.error; }).forEach(function (s) {
+        c.appendChild(el('p', { class: 'hub-hint', text: s.collection + ': ' + s.error }));
+      });
       c.appendChild(table([
         { key: 'filename', label: 'Filename' },
+        { key: 'store', label: 'Store', render: function (r) { return el('span', { class: 'hub-chip', text: r.store }); } },
         { key: 'length', label: 'Size', render: function (r) { return fmtBytes(r.length); } },
         { key: 'contentType', label: 'Type', render: function (r) { return r.contentType || '–'; } },
         { key: 'uploadDate', label: 'Uploaded', render: function (r) { return r.uploadDate ? relTime(r.uploadDate) : '–'; } },

package/src/core/modules/hub/hub-action-messages.ts

@@ -21,8 +21,16 @@ export const HubActionMessage = {
   confirmationFilenameRequired: 'Confirmation filename required.',
   /** Test mail: the EmailService is not wired into this app. */
   emailServiceUnavailable: 'EmailService is not available.',
-  /** Delete file: no GridFS file with the given id. */
+  /** Delete file: no file with the given id in any of the three metadata stores. */
   fileNotFound: 'File not found.',
+  /**
+   * Delete file: the file's bytes are in S3, but `CoreS3Service` is not configured in this process.
+   *
+   * Deliberately a REFUSAL rather than a metadata-only delete. Removing the document while the
+   * object stays in the bucket produces an orphan nothing can find again — unrecoverable in a way
+   * that "try again with S3 configured" is not.
+   */
+  s3Unavailable: 'S3 storage is not available.',
   /** Delete file: the id is not a valid ObjectId. */
   invalidFileId: 'Invalid file id.',
   /** Migrations action while the migrations panel is disabled. */

package/src/core/modules/hub/interfaces/hub-panels.interface.ts

@@ -79,9 +79,39 @@ export interface HubMigrationsData {
   source: 'collection' | 'runner';
 }
 
+/**
+ * Which metadata store a Hub file row came from.
+ *
+ * Mirrors `FileStorageDriver`, but is deliberately its OWN type: this one describes where a file
+ * was FOUND, which is a property of the row. `file.storage` describes where the next write GOES,
+ * which is a property of the config. Under a driver switch the two legitimately disagree for every
+ * pre-switch file — the whole reason this panel had to stop assuming one store.
+ */
+export type HubFileStore = 'filesystem' | 'gridfs' | 's3';
+
+/** Per-store summary for the Files panel, so an empty store reads as "empty", not "not covered". */
+export interface HubFileStoreSummary {
+  /** The MongoDB collection holding this store's metadata (`fs.files`, `s3-files`, …). */
+  collection: string;
+  /** Documents in that collection. `0` for a store this deployment has never written to. */
+  count: number;
+  /** Present only when this store could not be read; the other stores are still listed. */
+  error?: string;
+  store: HubFileStore;
+}
+
 export interface HubFilesData {
   bucket: string;
-  files: { contentType?: string; filename: string; id: string; length: number; uploadDate?: string }[];
+  files: {
+    contentType?: string;
+    filename: string;
+    id: string;
+    length: number;
+    store: HubFileStore;
+    uploadDate?: string;
+  }[];
+  /** Every store consulted, whether or not it holds anything. */
+  stores: HubFileStoreSummary[];
   total: number;
 }
 

package/src/core/modules/hub/services/core-hub-actions.service.ts

@@ -3,6 +3,7 @@ import { Injectable, Logger, Optional } from '@nestjs/common';
 import { EmailService } from '../../../common/services/email.service';
 import { RequestContext } from '../../../common/services/request-context.service';
 import { HubActionMessage } from '../hub-action-messages';
+import { HubFileStore } from '../interfaces/hub-panels.interface';
 import { CoreHubDbService } from './core-hub-db.service';
 import { CoreHubEmailService } from './core-hub-email.service';
 import { CoreHubMailboxService } from './core-hub-mailbox.service';
@@ -70,8 +71,15 @@ export class CoreHubActionsService {
     return { action, name };
   }
 
-  /** Delete a GridFS file (the confirm keyword must equal its filename). */
-  async deleteFile(id: string, expectedFilename: string): Promise<{ deleted: { filename: string; id: string } }> {
+  /**
+   * Delete a file from whichever storage driver holds it (the confirm keyword must equal its
+   * filename). The resolved store is echoed back so the `[HUB-ACTION]` audit trail records WHERE
+   * the bytes were removed from, not merely that a delete happened.
+   */
+  async deleteFile(
+    id: string,
+    expectedFilename: string,
+  ): Promise<{ deleted: { filename: string; id: string; store: HubFileStore } }> {
     this.audit(`delete file ${id}`);
     return { deleted: await this.dbService.deleteFile(id, expectedFilename) };
   }