@lenne.tech/nest-server 11.33.1 → 11.34.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude/rules/testing.md +181 -3
- package/FRAMEWORK-API.md +1 -1
- package/dist/core/common/helpers/gridfs.helper.d.ts +1 -0
- package/dist/core/common/helpers/gridfs.helper.js +43 -5
- package/dist/core/common/helpers/gridfs.helper.js.map +1 -1
- package/dist/core/common/helpers/validation-message.helper.d.ts +3 -0
- package/dist/core/common/helpers/validation-message.helper.js +41 -0
- package/dist/core/common/helpers/validation-message.helper.js.map +1 -0
- package/dist/core/common/pipes/map-and-validate.pipe.js +16 -6
- package/dist/core/common/pipes/map-and-validate.pipe.js.map +1 -1
- package/dist/core/common/services/core-s3.service.d.ts +4 -0
- package/dist/core/common/services/core-s3.service.js +28 -1
- package/dist/core/common/services/core-s3.service.js.map +1 -1
- package/dist/core/modules/file/core-file.service.d.ts +4 -2
- package/dist/core/modules/file/core-file.service.js +46 -47
- package/dist/core/modules/file/core-file.service.js.map +1 -1
- package/dist/core/modules/hub/helpers/hub-client-js.helper.js +12 -1
- package/dist/core/modules/hub/helpers/hub-client-js.helper.js.map +1 -1
- package/dist/core/modules/hub/hub-action-messages.d.ts +1 -0
- package/dist/core/modules/hub/hub-action-messages.js +1 -0
- package/dist/core/modules/hub/hub-action-messages.js.map +1 -1
- package/dist/core/modules/hub/interfaces/hub-panels.interface.d.ts +9 -0
- package/dist/core/modules/hub/services/core-hub-actions.service.d.ts +2 -0
- package/dist/core/modules/hub/services/core-hub-actions.service.js.map +1 -1
- package/dist/core/modules/hub/services/core-hub-db.service.d.ts +10 -2
- package/dist/core/modules/hub/services/core-hub-db.service.js +85 -29
- package/dist/core/modules/hub/services/core-hub-db.service.js.map +1 -1
- package/dist/server/modules/file/file.service.d.ts +2 -1
- package/dist/server/modules/file/file.service.js +2 -2
- package/dist/server/modules/file/file.service.js.map +1 -1
- package/dist/tsconfig.build.tsbuildinfo +1 -1
- package/migration-guides/11.23.x-to-11.24.0.md +1 -1
- package/migration-guides/11.32.x-to-11.33.x.md +2 -0
- package/migration-guides/11.33.x-to-11.34.x.md +394 -0
- package/migration-guides/11.34.0-to-11.34.1.md +132 -0
- package/package.json +3 -1
- package/src/core/common/helpers/gridfs.helper.ts +92 -7
- package/src/core/common/helpers/validation-message.helper.ts +83 -0
- package/src/core/common/pipes/map-and-validate.pipe.ts +25 -7
- package/src/core/common/services/core-s3.service.ts +71 -9
- package/src/core/modules/file/README.md +42 -6
- package/src/core/modules/file/core-file.service.ts +131 -59
- package/src/core/modules/hub/README.md +35 -9
- package/src/core/modules/hub/helpers/hub-client-js.helper.ts +12 -1
- package/src/core/modules/hub/hub-action-messages.ts +9 -1
- package/src/core/modules/hub/interfaces/hub-panels.interface.ts +31 -1
- package/src/core/modules/hub/services/core-hub-actions.service.ts +10 -2
- package/src/core/modules/hub/services/core-hub-db.service.ts +155 -31
- package/src/server/modules/file/file.service.ts +20 -3
|
@@ -1,8 +1,9 @@
|
|
|
1
1
|
import { Logger, NotFoundException } from '@nestjs/common';
|
|
2
|
-
import
|
|
2
|
+
import { Connection, mongo, Types } from 'mongoose';
|
|
3
3
|
import { Readable } from 'stream';
|
|
4
4
|
|
|
5
5
|
import { FilterArgs } from '../../common/args/filter.args';
|
|
6
|
+
import { accessDeniedException } from '../../common/exceptions/access-denied.exception';
|
|
6
7
|
import { getObjectIds, getStringIds } from '../../common/helpers/db.helper';
|
|
7
8
|
import { convertFilterArgsToQuery } from '../../common/helpers/filter.helper';
|
|
8
9
|
import { GridFSHelper } from '../../common/helpers/gridfs.helper';
|
|
@@ -195,73 +196,103 @@ export abstract class CoreFileService {
|
|
|
195
196
|
|
|
196
197
|
/**
|
|
197
198
|
* Duplicate file by name
|
|
199
|
+
*
|
|
200
|
+
* A duplicate is a READ of the source plus a WRITE of the copy, and it is
|
|
201
|
+
* authorized as exactly that: the source goes through `getFileInfoByName()` +
|
|
202
|
+
* `getFileStreamByName()` (`checkInputType: 'filename'`) and the copy through
|
|
203
|
+
* `createFile()` (`checkInputType: 'file'`) — the same public methods, with the
|
|
204
|
+
* same `checkRights()` hook, that every other caller uses.
|
|
205
|
+
*
|
|
206
|
+
* `serviceOptions` is OPTIONAL and additive. Omitting it keeps the pre-11.34.0
|
|
207
|
+
* behaviour for the framework default `checkRights()` (which returns `true`),
|
|
208
|
+
* and turns what used to be a crash into a clean refusal for a project that
|
|
209
|
+
* overrides it fail-closed: the GridFS branch used to bypass the hook
|
|
210
|
+
* altogether and copy the file unchecked, while the S3 / filesystem branches
|
|
211
|
+
* died inside a storage helper. System-internal callers say `{ force: true }`,
|
|
212
|
+
* the same idiom the rest of this service uses.
|
|
213
|
+
*
|
|
214
|
+
* The copy does NOT inherit the source's `metadata`. Copying it would silently
|
|
215
|
+
* hand the duplicate the source's owner, which is the one thing an ownership
|
|
216
|
+
* rule must not do behind the caller's back — pass `serviceOptions.metadata`
|
|
217
|
+
* to state the copy's own.
|
|
218
|
+
*
|
|
219
|
+
* @returns the file info of the COPY. Up to 11.33.1 the GridFS branch resolved
|
|
220
|
+
* the raw `GridFSBucketWriteStream` instead, so a project reading anything
|
|
221
|
+
* beyond `.id` / `.filename` off it has to adjust.
|
|
198
222
|
*/
|
|
199
|
-
async duplicateByName(name: string, newName: string): Promise<
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
// exception that takes the process down instead of a 404.
|
|
204
|
-
const nonGridFsSource = (await this.findS3FileByName(name)) || (await this.findFilesystemFileByName(name));
|
|
205
|
-
if (nonGridFsSource) {
|
|
206
|
-
const source = await this.getFileStreamByName(name);
|
|
207
|
-
return this.createFile({
|
|
208
|
-
createReadStream: () => source,
|
|
209
|
-
filename: newName,
|
|
210
|
-
mimetype: nonGridFsSource.contentType || 'application/octet-stream',
|
|
211
|
-
});
|
|
223
|
+
async duplicateByName(name: string, newName: string, serviceOptions?: FileServiceOptions): Promise<CoreFileInfo> {
|
|
224
|
+
const source = await this.getFileInfoByName(name, serviceOptions);
|
|
225
|
+
if (!source) {
|
|
226
|
+
throw new NotFoundException(`File not found with filename ${name}`);
|
|
212
227
|
}
|
|
213
|
-
|
|
214
|
-
return new Promise((resolve, reject) => {
|
|
215
|
-
const downloadStream = GridFSHelper.openDownloadStreamByName(this.files, name);
|
|
216
|
-
downloadStream.on('error', reject);
|
|
217
|
-
const uploadStream = GridFSHelper.openUploadStream(this.files, newName);
|
|
218
|
-
uploadStream.on('error', reject);
|
|
219
|
-
uploadStream.on('finish', () => resolve(uploadStream));
|
|
220
|
-
downloadStream.pipe(uploadStream);
|
|
221
|
-
});
|
|
228
|
+
return this.duplicateFile(source, newName, () => this.getFileStreamByName(name, serviceOptions), serviceOptions);
|
|
222
229
|
}
|
|
223
230
|
|
|
224
231
|
/**
|
|
225
232
|
* Duplicate file by ID
|
|
233
|
+
*
|
|
234
|
+
* See {@link duplicateByName} for the authorization model and for why
|
|
235
|
+
* `serviceOptions` matters. The copy keeps the source's filename.
|
|
236
|
+
*
|
|
237
|
+
* @returns the id of the copy
|
|
226
238
|
*/
|
|
227
|
-
async duplicateById(id: string): Promise<string> {
|
|
239
|
+
async duplicateById(id: string | Types.ObjectId, serviceOptions?: FileServiceOptions): Promise<string> {
|
|
228
240
|
const objectId = getObjectIds(id);
|
|
229
|
-
const
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
const downloadStream = GridFSHelper.openDownloadStream(this.files, objectId);
|
|
245
|
-
|
|
246
|
-
const newFileId = new mongoose.Types.ObjectId();
|
|
247
|
-
const uploadStream = GridFSHelper.openUploadStreamWithId(this.files, newFileId, file.filename, {
|
|
248
|
-
contentType: file.contentType,
|
|
249
|
-
});
|
|
250
|
-
|
|
251
|
-
downloadStream.pipe(uploadStream);
|
|
241
|
+
const source = await this.getFileInfo(objectId, serviceOptions);
|
|
242
|
+
if (!source) {
|
|
243
|
+
// Was a `TypeError` on `file.filename`: the source had been re-resolved with
|
|
244
|
+
// an empty context, so a fail-closed rule answered null and the null was then
|
|
245
|
+
// dereferenced. A refusal must read as a refusal, never as a crash.
|
|
246
|
+
throw new NotFoundException(`File not found with id ${getStringIds(objectId)}`);
|
|
247
|
+
}
|
|
248
|
+
const copy = await this.duplicateFile(
|
|
249
|
+
source,
|
|
250
|
+
source.filename,
|
|
251
|
+
() => this.getFileStream(objectId, serviceOptions),
|
|
252
|
+
serviceOptions,
|
|
253
|
+
);
|
|
254
|
+
return copy.id;
|
|
255
|
+
}
|
|
252
256
|
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
257
|
+
/**
|
|
258
|
+
* Shared write half of the two duplicate methods.
|
|
259
|
+
*
|
|
260
|
+
* Deliberately driver-agnostic: it opens the source through the public read
|
|
261
|
+
* path (whichever store answers) and writes the copy through `createFile()`
|
|
262
|
+
* (whichever store is active). That is what makes a duplicate work ACROSS a
|
|
263
|
+
* driver change — a file still in GridFS is copied into S3 without a migration
|
|
264
|
+
* step — and what keeps the three drivers from drifting apart again.
|
|
265
|
+
*/
|
|
266
|
+
protected async duplicateFile(
|
|
267
|
+
source: CoreFileInfo,
|
|
268
|
+
newName: string,
|
|
269
|
+
openSourceStream: () => Promise<Readable>,
|
|
270
|
+
serviceOptions?: FileServiceOptions,
|
|
271
|
+
): Promise<CoreFileInfo> {
|
|
272
|
+
const stream = await openSourceStream();
|
|
273
|
+
if (!stream) {
|
|
274
|
+
// The read was refused between the two checks (or the bytes vanished).
|
|
275
|
+
// Same answer as an unknown file: never confirm that it exists.
|
|
276
|
+
throw new NotFoundException(`File not found with filename ${source.filename}`);
|
|
277
|
+
}
|
|
256
278
|
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
279
|
+
const copy = await this.createFile(
|
|
280
|
+
{
|
|
281
|
+
createReadStream: () => stream,
|
|
282
|
+
filename: newName,
|
|
283
|
+
mimetype: source.contentType || 'application/octet-stream',
|
|
284
|
+
},
|
|
285
|
+
serviceOptions,
|
|
286
|
+
);
|
|
260
287
|
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
288
|
+
if (!copy) {
|
|
289
|
+
// The WRITE half was refused. Unlike a refused read this is not an
|
|
290
|
+
// existence question — the caller has already been shown the source — so it
|
|
291
|
+
// answers with the framework's 401/403 policy rather than a 404.
|
|
292
|
+
stream.destroy?.();
|
|
293
|
+
throw accessDeniedException(serviceOptions?.currentUser);
|
|
294
|
+
}
|
|
295
|
+
return copy;
|
|
265
296
|
}
|
|
266
297
|
|
|
267
298
|
/**
|
|
@@ -317,8 +348,8 @@ export abstract class CoreFileService {
|
|
|
317
348
|
const effective: [string, number][] = spec.length ? (spec as [string, number][]) : [['uploadDate', -1]];
|
|
318
349
|
return [...docs].sort((a, b) => {
|
|
319
350
|
for (const [field, direction] of effective) {
|
|
320
|
-
const left = a
|
|
321
|
-
const right = b
|
|
351
|
+
const left = this.readSortField(a, field);
|
|
352
|
+
const right = this.readSortField(b, field);
|
|
322
353
|
if (left === right) {
|
|
323
354
|
continue;
|
|
324
355
|
}
|
|
@@ -336,6 +367,33 @@ export abstract class CoreFileService {
|
|
|
336
367
|
});
|
|
337
368
|
}
|
|
338
369
|
|
|
370
|
+
/**
|
|
371
|
+
* Read the value a `sort` key names, resolving DOTTED paths.
|
|
372
|
+
*
|
|
373
|
+
* `SortInput.field` is a free string, and MongoDB reads `metadata.ownerId` as a
|
|
374
|
+
* path into the document — so each store sorted correctly on its own while the
|
|
375
|
+
* merge above compared `doc['metadata.ownerId']`, which is `undefined` for every
|
|
376
|
+
* row. Every comparison then tied, the merged page came out grouped by store,
|
|
377
|
+
* and `skip`/`limit` over it returned the WRONG ROWS rather than the right rows
|
|
378
|
+
* in the wrong order.
|
|
379
|
+
*
|
|
380
|
+
* A path segment is only followed through plain objects. Anything else stops the
|
|
381
|
+
* walk and yields `undefined`, which the caller already sorts last.
|
|
382
|
+
*/
|
|
383
|
+
protected readSortField(doc: Record<string, any>, field: string): any {
|
|
384
|
+
if (!field.includes('.')) {
|
|
385
|
+
return doc?.[field];
|
|
386
|
+
}
|
|
387
|
+
let current: any = doc;
|
|
388
|
+
for (const segment of field.split('.')) {
|
|
389
|
+
if (current === null || current === undefined || typeof current !== 'object') {
|
|
390
|
+
return undefined;
|
|
391
|
+
}
|
|
392
|
+
current = current[segment];
|
|
393
|
+
}
|
|
394
|
+
return current;
|
|
395
|
+
}
|
|
396
|
+
|
|
339
397
|
/**
|
|
340
398
|
* Get info about file via file ID
|
|
341
399
|
*/
|
|
@@ -531,6 +589,17 @@ export abstract class CoreFileService {
|
|
|
531
589
|
|
|
532
590
|
/**
|
|
533
591
|
* Delete file
|
|
592
|
+
*
|
|
593
|
+
* An unknown id answers `NotFoundException`, exactly as {@link deleteFileByName}
|
|
594
|
+
* does for an unknown filename. It used to fall through to GridFS and surface
|
|
595
|
+
* the driver's own `MongoRuntimeError: File not found for id …` — a **500** for
|
|
596
|
+
* the very condition its by-name sibling reported as a clean **404**, under all
|
|
597
|
+
* three storage drivers.
|
|
598
|
+
*
|
|
599
|
+
* A REFUSAL still answers `null` rather than throwing, which is the module-wide
|
|
600
|
+
* contract: a refusal must be indistinguishable from a file that is not there.
|
|
601
|
+
* Reaching the lookup below means `checkRights()` already said yes for this
|
|
602
|
+
* exact input, so a `null` here is genuinely a missing file.
|
|
534
603
|
*/
|
|
535
604
|
async deleteFile(id: string | Types.ObjectId, serviceOptions?: FileServiceOptions): Promise<CoreFileInfo> {
|
|
536
605
|
if (!(await this.checkRights(id, { ...serviceOptions, checkInputType: 'id' }))) {
|
|
@@ -538,6 +607,9 @@ export abstract class CoreFileService {
|
|
|
538
607
|
}
|
|
539
608
|
const objectId = getObjectIds(id);
|
|
540
609
|
const fileInfo = await this.getFileInfo(objectId, serviceOptions);
|
|
610
|
+
if (!fileInfo) {
|
|
611
|
+
throw new NotFoundException(`File not found with id ${getStringIds(objectId)}`);
|
|
612
|
+
}
|
|
541
613
|
if (await this.findS3FileById(objectId)) {
|
|
542
614
|
await S3FileHelper.deleteFile(this.options.s3Service, this.s3Files, objectId);
|
|
543
615
|
return fileInfo;
|
|
@@ -38,7 +38,7 @@ Do NOT set `hub` in the `production` block unless you intend the cockpit to be r
|
|
|
38
38
|
| Database | `/hub/db` | dbStats / per-collection collStats |
|
|
39
39
|
| Models / ERD | `/hub/models` | Mongoose schemas → Mermaid ER diagram |
|
|
40
40
|
| Migrations | `/hub/migrations` | `MigrationRunner` status + run/rollback |
|
|
41
|
-
| Files | `/hub/files` |
|
|
41
|
+
| Files | `/hub/files` | listing + delete across all three storage drivers |
|
|
42
42
|
| Config | `/hub/config` | full config, secrets masked |
|
|
43
43
|
| Auth Migration | `/hub/auth-migration` | Legacy → IAM progress (BetterAuth, optional) |
|
|
44
44
|
| Routes / Permissions | `/hub/routes` | route + role + `@Restricted` map (Permissions module, optional) |
|
|
@@ -50,19 +50,45 @@ Do NOT set `hub` in the `production` block unless you intend the cockpit to be r
|
|
|
50
50
|
Each panel has a `*.json` sidecar (the stable data contract) that the client polls. Optional sources
|
|
51
51
|
degrade to an "unavailable" state instead of erroring.
|
|
52
52
|
|
|
53
|
+
### Files panel
|
|
54
|
+
|
|
55
|
+
The panel reads all three file metadata stores, not just GridFS — the same three
|
|
56
|
+
`CoreFileService.findFileInfo()` consults, because metadata always lives in MongoDB whichever store
|
|
57
|
+
holds the bytes:
|
|
58
|
+
|
|
59
|
+
| Store | Collection | Bytes live in |
|
|
60
|
+
| ------------ | ------------------------------------- | ------------------------------- |
|
|
61
|
+
| `s3` | `s3-files` | the S3 bucket |
|
|
62
|
+
| `filesystem` | `filesystem-files` | `file.storageDir` on local disk |
|
|
63
|
+
| `gridfs` | `<bucket>.files` (default `fs.files`) | MongoDB |
|
|
64
|
+
|
|
65
|
+
Every row carries the `store` it was found in, and `files.json` reports a per-store `count` so an
|
|
66
|
+
empty store reads as **empty** rather than as _"this panel does not cover your driver"_. Until
|
|
67
|
+
11.34.0 it read `fs.files` alone, so under `file.storage: 's3'` or `'filesystem'` it confidently
|
|
68
|
+
answered **0 files** — the worst answer available, since the Hub is what an operator opens precisely
|
|
69
|
+
when they are unsure whether an upload landed.
|
|
70
|
+
|
|
71
|
+
Two properties worth knowing:
|
|
72
|
+
|
|
73
|
+
- **Delete dispatches to the owning store.** An S3-backed file needs `CoreS3Service` to be
|
|
74
|
+
configured in this process; without it the action REFUSES (`S3 storage is not available.`) rather
|
|
75
|
+
than deleting the metadata document and orphaning the object in the bucket.
|
|
76
|
+
- **Reading never creates a collection.** A GridFS-only deployment does not grow an empty
|
|
77
|
+
`s3-files` / `filesystem-files` from the panel looking at them.
|
|
78
|
+
|
|
53
79
|
## Actions (mutating)
|
|
54
80
|
|
|
55
81
|
Enabled by default (`actions: true`). Every mutating request requires the `X-Hub-Request: 1` header
|
|
56
82
|
(CSRF defense) and destructive ones a server-validated `confirm` keyword:
|
|
57
83
|
|
|
58
|
-
| Action
|
|
59
|
-
|
|
|
60
|
-
| Run pending migrations
|
|
61
|
-
| Rollback last migration
|
|
62
|
-
| Delete
|
|
63
|
-
| Cron start/stop/trigger
|
|
64
|
-
| Clear collector buffer
|
|
65
|
-
| Send test mail
|
|
84
|
+
| Action | Endpoint | Confirm |
|
|
85
|
+
| ------------------------ | ------------------------------------------ | ------------ |
|
|
86
|
+
| Run pending migrations | `POST /hub/actions/migrations/run` | `RUN` |
|
|
87
|
+
| Rollback last migration | `POST /hub/actions/migrations/down` | `DOWN` |
|
|
88
|
+
| Delete file (any driver) | `DELETE /hub/actions/files/:id` | the filename |
|
|
89
|
+
| Cron start/stop/trigger | `POST /hub/actions/cron/:name/:action` | the job name |
|
|
90
|
+
| Clear collector buffer | `POST /hub/actions/collectors/:name/clear` | `CLEAR` |
|
|
91
|
+
| Send test mail | `POST /hub/actions/email/test` | — |
|
|
66
92
|
|
|
67
93
|
Every action writes an audit line: `[HUB-ACTION] <action> by user <id>`.
|
|
68
94
|
|
|
@@ -506,9 +506,20 @@ export function getHubClientJs(): string {
|
|
|
506
506
|
|
|
507
507
|
Hub.panels.files = function (content) {
|
|
508
508
|
pollInto(content, 'files.json', function (c, d) {
|
|
509
|
-
|
|
509
|
+
// One tile per metadata store, so an empty store reads as "empty" rather than as "this panel
|
|
510
|
+
// does not cover your driver" — the ambiguity that made a GridFS-only listing look like the
|
|
511
|
+
// whole truth under file.storage: 's3'.
|
|
512
|
+
var storeTiles = [{ label: 'Files', value: d.total }];
|
|
513
|
+
(d.stores || []).forEach(function (s) {
|
|
514
|
+
storeTiles.push({ cls: s.error ? 'err' : '', label: s.store === 'gridfs' ? 'GridFS (' + d.bucket + ')' : s.store, value: s.error ? '!' : s.count });
|
|
515
|
+
});
|
|
516
|
+
c.appendChild(tiles(storeTiles));
|
|
517
|
+
(d.stores || []).filter(function (s) { return s.error; }).forEach(function (s) {
|
|
518
|
+
c.appendChild(el('p', { class: 'hub-hint', text: s.collection + ': ' + s.error }));
|
|
519
|
+
});
|
|
510
520
|
c.appendChild(table([
|
|
511
521
|
{ key: 'filename', label: 'Filename' },
|
|
522
|
+
{ key: 'store', label: 'Store', render: function (r) { return el('span', { class: 'hub-chip', text: r.store }); } },
|
|
512
523
|
{ key: 'length', label: 'Size', render: function (r) { return fmtBytes(r.length); } },
|
|
513
524
|
{ key: 'contentType', label: 'Type', render: function (r) { return r.contentType || '–'; } },
|
|
514
525
|
{ key: 'uploadDate', label: 'Uploaded', render: function (r) { return r.uploadDate ? relTime(r.uploadDate) : '–'; } },
|
|
@@ -21,8 +21,16 @@ export const HubActionMessage = {
|
|
|
21
21
|
confirmationFilenameRequired: 'Confirmation filename required.',
|
|
22
22
|
/** Test mail: the EmailService is not wired into this app. */
|
|
23
23
|
emailServiceUnavailable: 'EmailService is not available.',
|
|
24
|
-
/** Delete file: no
|
|
24
|
+
/** Delete file: no file with the given id in any of the three metadata stores. */
|
|
25
25
|
fileNotFound: 'File not found.',
|
|
26
|
+
/**
|
|
27
|
+
* Delete file: the file's bytes are in S3, but `CoreS3Service` is not configured in this process.
|
|
28
|
+
*
|
|
29
|
+
* Deliberately a REFUSAL rather than a metadata-only delete. Removing the document while the
|
|
30
|
+
* object stays in the bucket produces an orphan nothing can find again — unrecoverable in a way
|
|
31
|
+
* that "try again with S3 configured" is not.
|
|
32
|
+
*/
|
|
33
|
+
s3Unavailable: 'S3 storage is not available.',
|
|
26
34
|
/** Delete file: the id is not a valid ObjectId. */
|
|
27
35
|
invalidFileId: 'Invalid file id.',
|
|
28
36
|
/** Migrations action while the migrations panel is disabled. */
|
|
@@ -79,9 +79,39 @@ export interface HubMigrationsData {
|
|
|
79
79
|
source: 'collection' | 'runner';
|
|
80
80
|
}
|
|
81
81
|
|
|
82
|
+
/**
|
|
83
|
+
* Which metadata store a Hub file row came from.
|
|
84
|
+
*
|
|
85
|
+
* Mirrors `FileStorageDriver`, but is deliberately its OWN type: this one describes where a file
|
|
86
|
+
* was FOUND, which is a property of the row. `file.storage` describes where the next write GOES,
|
|
87
|
+
* which is a property of the config. Under a driver switch the two legitimately disagree for every
|
|
88
|
+
* pre-switch file — the whole reason this panel had to stop assuming one store.
|
|
89
|
+
*/
|
|
90
|
+
export type HubFileStore = 'filesystem' | 'gridfs' | 's3';
|
|
91
|
+
|
|
92
|
+
/** Per-store summary for the Files panel, so an empty store reads as "empty", not "not covered". */
|
|
93
|
+
export interface HubFileStoreSummary {
|
|
94
|
+
/** The MongoDB collection holding this store's metadata (`fs.files`, `s3-files`, …). */
|
|
95
|
+
collection: string;
|
|
96
|
+
/** Documents in that collection. `0` for a store this deployment has never written to. */
|
|
97
|
+
count: number;
|
|
98
|
+
/** Present only when this store could not be read; the other stores are still listed. */
|
|
99
|
+
error?: string;
|
|
100
|
+
store: HubFileStore;
|
|
101
|
+
}
|
|
102
|
+
|
|
82
103
|
export interface HubFilesData {
|
|
83
104
|
bucket: string;
|
|
84
|
-
files: {
|
|
105
|
+
files: {
|
|
106
|
+
contentType?: string;
|
|
107
|
+
filename: string;
|
|
108
|
+
id: string;
|
|
109
|
+
length: number;
|
|
110
|
+
store: HubFileStore;
|
|
111
|
+
uploadDate?: string;
|
|
112
|
+
}[];
|
|
113
|
+
/** Every store consulted, whether or not it holds anything. */
|
|
114
|
+
stores: HubFileStoreSummary[];
|
|
85
115
|
total: number;
|
|
86
116
|
}
|
|
87
117
|
|
|
@@ -3,6 +3,7 @@ import { Injectable, Logger, Optional } from '@nestjs/common';
|
|
|
3
3
|
import { EmailService } from '../../../common/services/email.service';
|
|
4
4
|
import { RequestContext } from '../../../common/services/request-context.service';
|
|
5
5
|
import { HubActionMessage } from '../hub-action-messages';
|
|
6
|
+
import { HubFileStore } from '../interfaces/hub-panels.interface';
|
|
6
7
|
import { CoreHubDbService } from './core-hub-db.service';
|
|
7
8
|
import { CoreHubEmailService } from './core-hub-email.service';
|
|
8
9
|
import { CoreHubMailboxService } from './core-hub-mailbox.service';
|
|
@@ -70,8 +71,15 @@ export class CoreHubActionsService {
|
|
|
70
71
|
return { action, name };
|
|
71
72
|
}
|
|
72
73
|
|
|
73
|
-
/**
|
|
74
|
-
|
|
74
|
+
/**
|
|
75
|
+
* Delete a file from whichever storage driver holds it (the confirm keyword must equal its
|
|
76
|
+
* filename). The resolved store is echoed back so the `[HUB-ACTION]` audit trail records WHERE
|
|
77
|
+
* the bytes were removed from, not merely that a delete happened.
|
|
78
|
+
*/
|
|
79
|
+
async deleteFile(
|
|
80
|
+
id: string,
|
|
81
|
+
expectedFilename: string,
|
|
82
|
+
): Promise<{ deleted: { filename: string; id: string; store: HubFileStore } }> {
|
|
75
83
|
this.audit(`delete file ${id}`);
|
|
76
84
|
return { deleted: await this.dbService.deleteFile(id, expectedFilename) };
|
|
77
85
|
}
|
|
@@ -3,11 +3,26 @@ import { InjectConnection } from '@nestjs/mongoose';
|
|
|
3
3
|
import * as mongo from 'mongodb';
|
|
4
4
|
import { Connection, Types } from 'mongoose';
|
|
5
5
|
|
|
6
|
+
import { ConfigService } from '../../../common/services/config.service';
|
|
7
|
+
import { CoreS3Service } from '../../../common/services/core-s3.service';
|
|
6
8
|
import { ModelRegistry } from '../../../common/services/model-registry.service';
|
|
9
|
+
import {
|
|
10
|
+
DEFAULT_FILESYSTEM_DIR,
|
|
11
|
+
FILESYSTEM_FILES_COLLECTION,
|
|
12
|
+
FilesystemFileHelper,
|
|
13
|
+
} from '../../file/filesystem-file.helper';
|
|
14
|
+
import { S3_FILES_COLLECTION, S3FileHelper } from '../../file/s3-file.helper';
|
|
7
15
|
import { HubActionMessage } from '../hub-action-messages';
|
|
8
16
|
import { HUB_CONFIG } from '../hub.constants';
|
|
9
17
|
import { buildErDiagram, HubModelDescriptor, HubModelField } from '../helpers/hub-mermaid.helper';
|
|
10
|
-
import {
|
|
18
|
+
import {
|
|
19
|
+
HubDbData,
|
|
20
|
+
HubFilesData,
|
|
21
|
+
HubFileStore,
|
|
22
|
+
HubFileStoreSummary,
|
|
23
|
+
HubModelsData,
|
|
24
|
+
HubUnavailable,
|
|
25
|
+
} from '../interfaces/hub-panels.interface';
|
|
11
26
|
import { ResolvedHubConfig } from '../interfaces/hub-config.interface';
|
|
12
27
|
|
|
13
28
|
/**
|
|
@@ -23,6 +38,10 @@ export class CoreHubDbService {
|
|
|
23
38
|
constructor(
|
|
24
39
|
@Inject(HUB_CONFIG) protected readonly config: ResolvedHubConfig,
|
|
25
40
|
@Optional() @InjectConnection() protected readonly connection?: Connection,
|
|
41
|
+
// Provided and exported globally by `CoreModule`, but INERT without an `s3` config — so it is
|
|
42
|
+
// optional here for the same reason it is everywhere else: a project that uses no S3 must not
|
|
43
|
+
// be forced to install `@aws-sdk/client-s3` to open the Hub.
|
|
44
|
+
@Optional() protected readonly s3Service?: CoreS3Service,
|
|
26
45
|
) {}
|
|
27
46
|
|
|
28
47
|
/** Database + per-collection statistics. */
|
|
@@ -85,10 +104,21 @@ export class CoreHubDbService {
|
|
|
85
104
|
}
|
|
86
105
|
|
|
87
106
|
/**
|
|
88
|
-
* Delete a
|
|
89
|
-
*
|
|
107
|
+
* Delete a file by id, from WHICHEVER store actually holds it.
|
|
108
|
+
*
|
|
109
|
+
* When `expectedFilename` is given it must match the stored filename (the type-to-confirm
|
|
110
|
+
* keyword). Returns the filename and the store it came from; throws when not found, when the name
|
|
111
|
+
* mismatches, or when the owning store cannot be reached.
|
|
112
|
+
*
|
|
113
|
+
* This used to delete from GridFS only, which meant an S3- or filesystem-backed file could not be
|
|
114
|
+
* removed from the Hub at all — the lookup answered `File not found.` for a file plainly sitting
|
|
115
|
+
* in the bucket.
|
|
90
116
|
*/
|
|
91
|
-
async deleteFile(
|
|
117
|
+
async deleteFile(
|
|
118
|
+
id: string,
|
|
119
|
+
expectedFilename?: string,
|
|
120
|
+
bucket = 'fs',
|
|
121
|
+
): Promise<{ filename: string; id: string; store: HubFileStore }> {
|
|
92
122
|
const db = this.connection?.db;
|
|
93
123
|
if (!db) {
|
|
94
124
|
throw new Error(HubActionMessage.mongoUnavailable);
|
|
@@ -97,46 +127,140 @@ export class CoreHubDbService {
|
|
|
97
127
|
throw new Error(HubActionMessage.invalidFileId);
|
|
98
128
|
}
|
|
99
129
|
const objectId = new Types.ObjectId(id);
|
|
100
|
-
|
|
101
|
-
|
|
130
|
+
|
|
131
|
+
// Same probe order as `CoreFileService.deleteFile()`: S3, then filesystem, then GridFS as the
|
|
132
|
+
// fallthrough. Keeping the two in step matters — a Hub that resolved a file differently from the
|
|
133
|
+
// service would delete a different file than the one it displayed.
|
|
134
|
+
let store: HubFileStore | undefined;
|
|
135
|
+
let doc: any;
|
|
136
|
+
for (const candidate of this.fileStores(bucket)) {
|
|
137
|
+
const [found] = await db.collection(candidate.collection).find({ _id: objectId }, { limit: 1 }).toArray();
|
|
138
|
+
if (found) {
|
|
139
|
+
doc = found;
|
|
140
|
+
store = candidate.store;
|
|
141
|
+
break;
|
|
142
|
+
}
|
|
143
|
+
}
|
|
144
|
+
if (!doc || !store) {
|
|
102
145
|
throw new Error(HubActionMessage.fileNotFound);
|
|
103
146
|
}
|
|
104
147
|
if (expectedFilename !== undefined && expectedFilename !== doc.filename) {
|
|
105
148
|
throw new Error(HubActionMessage.confirmationFilenameMismatch);
|
|
106
149
|
}
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
150
|
+
|
|
151
|
+
switch (store) {
|
|
152
|
+
case 'filesystem': {
|
|
153
|
+
await FilesystemFileHelper.deleteFile(this.filesystemDir, db.collection(FILESYSTEM_FILES_COLLECTION), objectId);
|
|
154
|
+
break;
|
|
155
|
+
}
|
|
156
|
+
case 's3': {
|
|
157
|
+
// Refuse rather than half-delete — see HubActionMessage.s3Unavailable.
|
|
158
|
+
if (!this.s3Service?.enabled) {
|
|
159
|
+
throw new Error(HubActionMessage.s3Unavailable);
|
|
160
|
+
}
|
|
161
|
+
await S3FileHelper.deleteFile(this.s3Service, db.collection(S3_FILES_COLLECTION), objectId);
|
|
162
|
+
break;
|
|
163
|
+
}
|
|
164
|
+
default: {
|
|
165
|
+
const gridFs = new mongo.GridFSBucket(db as unknown as mongo.Db, { bucketName: bucket });
|
|
166
|
+
await gridFs.delete(objectId as unknown as mongo.ObjectId);
|
|
167
|
+
}
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
return { filename: doc.filename, id, store };
|
|
110
171
|
}
|
|
111
172
|
|
|
112
|
-
/**
|
|
173
|
+
/**
|
|
174
|
+
* File inventory across ALL THREE metadata stores.
|
|
175
|
+
*
|
|
176
|
+
* Metadata always lives in MongoDB whichever store holds the bytes (`fs.files` / `s3-files` /
|
|
177
|
+
* `filesystem-files`), so one connection can see everything — which is exactly what
|
|
178
|
+
* `CoreFileService.findFileInfo()` does, and what this panel now mirrors.
|
|
179
|
+
*
|
|
180
|
+
* It used to read `fs.files` alone. Under `file.storage: 's3'` or `'filesystem'` that made the
|
|
181
|
+
* panel report **0 files** — not "this panel does not cover your driver", but a confident wrong
|
|
182
|
+
* answer, in the one tool an operator opens BECAUSE they are unsure. Someone checking whether an
|
|
183
|
+
* upload landed would have concluded it had not.
|
|
184
|
+
*
|
|
185
|
+
* Paging over a merge is the trap here: taking `skip`/`limit` from each store and concatenating
|
|
186
|
+
* returns the wrong ROWS, not merely the wrong order (the same defect `sortMergedFileInfo()` was
|
|
187
|
+
* fixed for). So each store yields its newest `skip + limit`, the union is re-sorted, and only
|
|
188
|
+
* then is the window cut — which is correct because the global newest `skip + limit` rows are
|
|
189
|
+
* necessarily contained in the per-store newest `skip + limit`.
|
|
190
|
+
*
|
|
191
|
+
* READS NEVER CREATE A COLLECTION: `find()` and `countDocuments()` on an absent collection answer
|
|
192
|
+
* empty. A GridFS-only deployment therefore does not grow an empty `s3-files` from the Hub merely
|
|
193
|
+
* looking at it — the same rule `ensureFilenameIndex()` follows on the write path.
|
|
194
|
+
*/
|
|
113
195
|
async getFiles(bucket = 'fs', skip = 0, limit = 100): Promise<HubFilesData | HubUnavailable> {
|
|
114
196
|
const db = this.connection?.db;
|
|
115
197
|
if (!db) {
|
|
116
198
|
return { available: false, hint: 'No MongoDB connection is available.' };
|
|
117
199
|
}
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
200
|
+
|
|
201
|
+
const safeSkip = Math.max(0, skip);
|
|
202
|
+
const safeLimit = Math.min(Math.max(1, limit), 500);
|
|
203
|
+
// Per-store window: enough to guarantee the merged page is complete, capped so a huge `skip`
|
|
204
|
+
// cannot turn one panel poll into an unbounded read.
|
|
205
|
+
const window = Math.min(safeSkip + safeLimit, 1000);
|
|
206
|
+
|
|
207
|
+
const stores: HubFileStoreSummary[] = [];
|
|
208
|
+
const rows: HubFilesData['files'] = [];
|
|
209
|
+
|
|
210
|
+
for (const source of this.fileStores(bucket)) {
|
|
211
|
+
try {
|
|
212
|
+
const docs = await db
|
|
213
|
+
.collection(source.collection)
|
|
214
|
+
.find({}, { limit: window, sort: { uploadDate: -1 } })
|
|
215
|
+
.toArray();
|
|
216
|
+
const count = await db.collection(source.collection).countDocuments();
|
|
217
|
+
stores.push({ collection: source.collection, count, store: source.store });
|
|
218
|
+
for (const doc of docs) {
|
|
219
|
+
rows.push({
|
|
220
|
+
contentType: doc.contentType ?? doc.metadata?.contentType,
|
|
221
|
+
filename: doc.filename,
|
|
222
|
+
id: String(doc._id),
|
|
223
|
+
length: doc.length ?? 0,
|
|
224
|
+
store: source.store,
|
|
225
|
+
uploadDate: doc.uploadDate ? new Date(doc.uploadDate).toISOString() : undefined,
|
|
226
|
+
});
|
|
227
|
+
}
|
|
228
|
+
} catch (error) {
|
|
229
|
+
// One unreadable store must not blank the panel: the other two still hold real answers, and
|
|
230
|
+
// an operator is better served by "two stores listed, this one errored" than by nothing.
|
|
231
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
232
|
+
this.logger.warn(`Failed to list files in ${source.collection}: ${message}`);
|
|
233
|
+
stores.push({ collection: source.collection, count: 0, error: message, store: source.store });
|
|
234
|
+
}
|
|
139
235
|
}
|
|
236
|
+
|
|
237
|
+
rows.sort((a, b) => (b.uploadDate ?? '').localeCompare(a.uploadDate ?? ''));
|
|
238
|
+
|
|
239
|
+
return {
|
|
240
|
+
bucket,
|
|
241
|
+
files: rows.slice(safeSkip, safeSkip + safeLimit),
|
|
242
|
+
stores,
|
|
243
|
+
total: stores.reduce((sum, entry) => sum + entry.count, 0),
|
|
244
|
+
};
|
|
245
|
+
}
|
|
246
|
+
|
|
247
|
+
/** Directory used by the `'filesystem'` driver — resolved exactly as `CoreFileService` does. */
|
|
248
|
+
protected get filesystemDir(): string {
|
|
249
|
+
return ConfigService.configFastButReadOnly?.file?.storageDir || DEFAULT_FILESYSTEM_DIR;
|
|
250
|
+
}
|
|
251
|
+
|
|
252
|
+
/**
|
|
253
|
+
* The three metadata stores, in the order both this service and `CoreFileService` probe them.
|
|
254
|
+
*
|
|
255
|
+
* GridFS is last because it is the fallthrough: its collection name depends on the bucket, and a
|
|
256
|
+
* document there carries no `storage` marker to identify it by.
|
|
257
|
+
*/
|
|
258
|
+
protected fileStores(bucket: string): { collection: string; store: HubFileStore }[] {
|
|
259
|
+
return [
|
|
260
|
+
{ collection: S3_FILES_COLLECTION, store: 's3' },
|
|
261
|
+
{ collection: FILESYSTEM_FILES_COLLECTION, store: 'filesystem' },
|
|
262
|
+
{ collection: `${bucket}.files`, store: 'gridfs' },
|
|
263
|
+
];
|
|
140
264
|
}
|
|
141
265
|
|
|
142
266
|
/** Model inventory + a Mermaid ER diagram derived from the registered Mongoose schemas. */
|