@lenne.tech/nest-server 11.32.3 → 11.33.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude/rules/architecture.md +16 -0
- package/.claude/rules/configurable-features.md +11 -3
- package/.claude/rules/testing.md +101 -1
- package/CLAUDE.md +5 -0
- package/FRAMEWORK-API.md +58 -3
- package/bin/migrate.js +13 -3
- package/dist/core/common/helpers/content-disposition.helper.d.ts +1 -0
- package/dist/core/common/helpers/content-disposition.helper.js +14 -0
- package/dist/core/common/helpers/content-disposition.helper.js.map +1 -0
- package/dist/core/common/helpers/file.helper.d.ts +21 -2
- package/dist/core/common/helpers/file.helper.js +69 -13
- package/dist/core/common/helpers/file.helper.js.map +1 -1
- package/dist/core/common/helpers/graceful-shutdown.helper.d.ts +2 -0
- package/dist/core/common/helpers/graceful-shutdown.helper.js +55 -0
- package/dist/core/common/helpers/graceful-shutdown.helper.js.map +1 -0
- package/dist/core/common/helpers/project-name.helper.d.ts +2 -0
- package/dist/core/common/helpers/project-name.helper.js +38 -0
- package/dist/core/common/helpers/project-name.helper.js.map +1 -0
- package/dist/core/common/interfaces/cron-job-config.interface.d.ts +1 -0
- package/dist/core/common/interfaces/server-options.interface.d.ts +39 -0
- package/dist/core/common/services/core-cron-jobs.initializer.d.ts +10 -0
- package/dist/core/common/services/core-cron-jobs.initializer.js +46 -0
- package/dist/core/common/services/core-cron-jobs.initializer.js.map +1 -0
- package/dist/core/common/services/core-cron-jobs.registry.d.ts +8 -0
- package/dist/core/common/services/core-cron-jobs.registry.js +12 -0
- package/dist/core/common/services/core-cron-jobs.registry.js.map +1 -0
- package/dist/core/common/services/core-cron-jobs.service.d.ts +37 -7
- package/dist/core/common/services/core-cron-jobs.service.js +278 -27
- package/dist/core/common/services/core-cron-jobs.service.js.map +1 -1
- package/dist/core/common/services/core-redis-pubsub.d.ts +20 -0
- package/dist/core/common/services/core-redis-pubsub.js +112 -0
- package/dist/core/common/services/core-redis-pubsub.js.map +1 -0
- package/dist/core/common/services/core-redis.service.d.ts +25 -0
- package/dist/core/common/services/core-redis.service.js +154 -0
- package/dist/core/common/services/core-redis.service.js.map +1 -0
- package/dist/core/common/services/core-s3.service.d.ts +51 -0
- package/dist/core/common/services/core-s3.service.js +261 -0
- package/dist/core/common/services/core-s3.service.js.map +1 -0
- package/dist/core/common/services/core-trust-proxy.initializer.d.ts +16 -0
- package/dist/core/common/services/core-trust-proxy.initializer.js +77 -0
- package/dist/core/common/services/core-trust-proxy.initializer.js.map +1 -0
- package/dist/core/common/services/rate-limit-store.d.ts +61 -0
- package/dist/core/common/services/rate-limit-store.js +235 -0
- package/dist/core/common/services/rate-limit-store.js.map +1 -0
- package/dist/core/modules/ai/core-ai-mcp.controller.d.ts +19 -3
- package/dist/core/modules/ai/core-ai-mcp.controller.js +148 -18
- package/dist/core/modules/ai/core-ai-mcp.controller.js.map +1 -1
- package/dist/core/modules/ai/services/core-ai.service.d.ts +7 -2
- package/dist/core/modules/ai/services/core-ai.service.js +24 -20
- package/dist/core/modules/ai/services/core-ai.service.js.map +1 -1
- package/dist/core/modules/auth/core-auth.module.js +4 -1
- package/dist/core/modules/auth/core-auth.module.js.map +1 -1
- package/dist/core/modules/auth/guards/legacy-auth-rate-limit.guard.d.ts +1 -1
- package/dist/core/modules/auth/guards/legacy-auth-rate-limit.guard.js +3 -14
- package/dist/core/modules/auth/guards/legacy-auth-rate-limit.guard.js.map +1 -1
- package/dist/core/modules/auth/services/legacy-auth-rate-limiter.service.d.ts +12 -10
- package/dist/core/modules/auth/services/legacy-auth-rate-limiter.service.js +30 -59
- package/dist/core/modules/auth/services/legacy-auth-rate-limiter.service.js.map +1 -1
- package/dist/core/modules/better-auth/core-better-auth-email-verification.service.d.ts +8 -2
- package/dist/core/modules/better-auth/core-better-auth-email-verification.service.js +118 -45
- package/dist/core/modules/better-auth/core-better-auth-email-verification.service.js.map +1 -1
- package/dist/core/modules/better-auth/core-better-auth-rate-limit.middleware.d.ts +1 -1
- package/dist/core/modules/better-auth/core-better-auth-rate-limit.middleware.js +2 -11
- package/dist/core/modules/better-auth/core-better-auth-rate-limit.middleware.js.map +1 -1
- package/dist/core/modules/better-auth/core-better-auth-rate-limiter.service.d.ts +12 -10
- package/dist/core/modules/better-auth/core-better-auth-rate-limiter.service.js +30 -85
- package/dist/core/modules/better-auth/core-better-auth-rate-limiter.service.js.map +1 -1
- package/dist/core/modules/better-auth/core-better-auth-user.mapper.js +0 -1
- package/dist/core/modules/better-auth/core-better-auth-user.mapper.js.map +1 -1
- package/dist/core/modules/file/core-file.controller.d.ts +15 -3
- package/dist/core/modules/file/core-file.controller.js +90 -20
- package/dist/core/modules/file/core-file.controller.js.map +1 -1
- package/dist/core/modules/file/core-file.resolver.d.ts +4 -4
- package/dist/core/modules/file/core-file.resolver.js +23 -16
- package/dist/core/modules/file/core-file.resolver.js.map +1 -1
- package/dist/core/modules/file/core-file.service.d.ts +41 -7
- package/dist/core/modules/file/core-file.service.js +227 -7
- package/dist/core/modules/file/core-file.service.js.map +1 -1
- package/dist/core/modules/file/file-metadata.helper.d.ts +16 -0
- package/dist/core/modules/file/file-metadata.helper.js +33 -0
- package/dist/core/modules/file/file-metadata.helper.js.map +1 -0
- package/dist/core/modules/file/file-roles.helper.d.ts +4 -0
- package/dist/core/modules/file/file-roles.helper.js +53 -0
- package/dist/core/modules/file/file-roles.helper.js.map +1 -0
- package/dist/core/modules/file/file-storage.helper.d.ts +13 -0
- package/dist/core/modules/file/file-storage.helper.js +65 -0
- package/dist/core/modules/file/file-storage.helper.js.map +1 -0
- package/dist/core/modules/file/filesystem-file.helper.d.ts +22 -0
- package/dist/core/modules/file/filesystem-file.helper.js +77 -0
- package/dist/core/modules/file/filesystem-file.helper.js.map +1 -0
- package/dist/core/modules/file/interfaces/file-service-options.interface.d.ts +1 -0
- package/dist/core/modules/file/interfaces/file-upload.interface.d.ts +7 -1
- package/dist/core/modules/file/s3-file.helper.d.ts +37 -0
- package/dist/core/modules/file/s3-file.helper.js +81 -0
- package/dist/core/modules/file/s3-file.helper.js.map +1 -0
- package/dist/core/modules/hub/core-hub.controller.d.ts +8 -8
- package/dist/core/modules/hub/core-hub.controller.js +24 -22
- package/dist/core/modules/hub/core-hub.controller.js.map +1 -1
- package/dist/core/modules/hub/helpers/hub-mask.helper.js +2 -2
- package/dist/core/modules/hub/helpers/hub-mask.helper.js.map +1 -1
- package/dist/core/modules/hub/hub-buffer.d.ts +23 -0
- package/dist/core/modules/hub/hub-buffer.js +101 -0
- package/dist/core/modules/hub/hub-buffer.js.map +1 -0
- package/dist/core/modules/hub/services/core-hub-mailbox.service.d.ts +7 -5
- package/dist/core/modules/hub/services/core-hub-mailbox.service.js +14 -10
- package/dist/core/modules/hub/services/core-hub-mailbox.service.js.map +1 -1
- package/dist/core/modules/hub/services/hub-log-buffer.service.d.ts +5 -2
- package/dist/core/modules/hub/services/hub-log-buffer.service.js +17 -8
- package/dist/core/modules/hub/services/hub-log-buffer.service.js.map +1 -1
- package/dist/core/modules/hub/services/hub-query-profiler.service.d.ts +4 -2
- package/dist/core/modules/hub/services/hub-query-profiler.service.js +13 -7
- package/dist/core/modules/hub/services/hub-query-profiler.service.js.map +1 -1
- package/dist/core/modules/hub/services/hub-trace-buffer.service.d.ts +4 -2
- package/dist/core/modules/hub/services/hub-trace-buffer.service.js +13 -8
- package/dist/core/modules/hub/services/hub-trace-buffer.service.js.map +1 -1
- package/dist/core/modules/migrate/cli/migrate-cli.d.ts +3 -1
- package/dist/core/modules/migrate/cli/migrate-cli.js +29 -4
- package/dist/core/modules/migrate/cli/migrate-cli.js.map +1 -1
- package/dist/core/modules/migrate/helpers/migration.helper.d.ts +3 -6
- package/dist/core/modules/migrate/helpers/migration.helper.js +55 -8
- package/dist/core/modules/migrate/helpers/migration.helper.js.map +1 -1
- package/dist/core/modules/migrate/migration-runner.d.ts +1 -0
- package/dist/core/modules/migrate/migration-runner.js +4 -0
- package/dist/core/modules/migrate/migration-runner.js.map +1 -1
- package/dist/core/modules/migrate/mongo-state-store.d.ts +1 -0
- package/dist/core/modules/migrate/mongo-state-store.js +98 -9
- package/dist/core/modules/migrate/mongo-state-store.js.map +1 -1
- package/dist/core/modules/system-setup/core-system-setup.service.d.ts +2 -0
- package/dist/core/modules/system-setup/core-system-setup.service.js +31 -1
- package/dist/core/modules/system-setup/core-system-setup.service.js.map +1 -1
- package/dist/core/modules/tenant/core-tenant.guard.d.ts +17 -3
- package/dist/core/modules/tenant/core-tenant.guard.js +92 -15
- package/dist/core/modules/tenant/core-tenant.guard.js.map +1 -1
- package/dist/core/modules/tenant/core-tenant.module.js +3 -4
- package/dist/core/modules/tenant/core-tenant.module.js.map +1 -1
- package/dist/core/modules/tus/core-tus.controller.d.ts +2 -0
- package/dist/core/modules/tus/core-tus.controller.js +29 -3
- package/dist/core/modules/tus/core-tus.controller.js.map +1 -1
- package/dist/core/modules/tus/core-tus.service.d.ts +24 -1
- package/dist/core/modules/tus/core-tus.service.js +182 -28
- package/dist/core/modules/tus/core-tus.service.js.map +1 -1
- package/dist/core/modules/tus/interfaces/tus-config.interface.js +3 -0
- package/dist/core/modules/tus/interfaces/tus-config.interface.js.map +1 -1
- package/dist/core/modules/tus/tus-redis-locker.d.ts +25 -0
- package/dist/core/modules/tus/tus-redis-locker.js +87 -0
- package/dist/core/modules/tus/tus-redis-locker.js.map +1 -0
- package/dist/core/modules/tus/tus.module.d.ts +1 -0
- package/dist/core/modules/tus/tus.module.js +30 -4
- package/dist/core/modules/tus/tus.module.js.map +1 -1
- package/dist/core.module.js +18 -1
- package/dist/core.module.js.map +1 -1
- package/dist/index.d.ts +10 -0
- package/dist/index.js +10 -0
- package/dist/index.js.map +1 -1
- package/dist/main.js +2 -1
- package/dist/main.js.map +1 -1
- package/dist/server/modules/file/file.controller.js.map +1 -1
- package/dist/server/modules/file/file.resolver.js +1 -12
- package/dist/server/modules/file/file.resolver.js.map +1 -1
- package/dist/server/modules/file/file.service.d.ts +5 -1
- package/dist/server/modules/file/file.service.js +12 -3
- package/dist/server/modules/file/file.service.js.map +1 -1
- package/dist/server/modules/user/avatar.controller.d.ts +5 -1
- package/dist/server/modules/user/avatar.controller.js +26 -10
- package/dist/server/modules/user/avatar.controller.js.map +1 -1
- package/dist/server/modules/user/user.model.js +3 -2
- package/dist/server/modules/user/user.model.js.map +1 -1
- package/dist/server/modules/user/user.module.js +6 -2
- package/dist/server/modules/user/user.module.js.map +1 -1
- package/dist/server/modules/user/user.service.d.ts +1 -1
- package/dist/server/modules/user/user.service.js +6 -13
- package/dist/server/modules/user/user.service.js.map +1 -1
- package/dist/tsconfig.build.tsbuildinfo +1 -1
- package/docs/REQUEST-LIFECYCLE.md +57 -8
- package/docs/security-overrides.md +30 -5
- package/migration-guides/11.32.3-to-11.32.4.md +335 -0
- package/migration-guides/11.32.x-to-11.33.x.md +1551 -0
- package/package.json +74 -42
- package/src/core/common/helpers/content-disposition.helper.ts +90 -0
- package/src/core/common/helpers/file.helper.ts +201 -20
- package/src/core/common/helpers/graceful-shutdown.helper.ts +116 -0
- package/src/core/common/helpers/project-name.helper.ts +71 -0
- package/src/core/common/interfaces/cron-job-config.interface.ts +16 -0
- package/src/core/common/interfaces/server-options.interface.ts +439 -2
- package/src/core/common/services/core-cron-jobs.initializer.ts +39 -0
- package/src/core/common/services/core-cron-jobs.registry.ts +45 -0
- package/src/core/common/services/core-cron-jobs.service.ts +589 -47
- package/src/core/common/services/core-redis-pubsub.ts +183 -0
- package/src/core/common/services/core-redis.service.ts +249 -0
- package/src/core/common/services/core-s3.service.ts +465 -0
- package/src/core/common/services/core-trust-proxy.initializer.ts +112 -0
- package/src/core/common/services/rate-limit-store.ts +492 -0
- package/src/core/modules/ai/README.md +25 -0
- package/src/core/modules/ai/core-ai-mcp.controller.ts +253 -22
- package/src/core/modules/ai/services/core-ai.service.ts +33 -24
- package/src/core/modules/auth/core-auth.module.ts +7 -1
- package/src/core/modules/auth/guards/legacy-auth-rate-limit.guard.ts +12 -21
- package/src/core/modules/auth/services/legacy-auth-rate-limiter.service.ts +49 -93
- package/src/core/modules/better-auth/core-better-auth-email-verification.service.ts +202 -66
- package/src/core/modules/better-auth/core-better-auth-rate-limit.middleware.ts +11 -18
- package/src/core/modules/better-auth/core-better-auth-rate-limiter.service.ts +51 -132
- package/src/core/modules/better-auth/core-better-auth-user.mapper.ts +7 -1
- package/src/core/modules/file/INTEGRATION-CHECKLIST.md +108 -0
- package/src/core/modules/file/README.md +324 -41
- package/src/core/modules/file/core-file.controller.ts +267 -19
- package/src/core/modules/file/core-file.resolver.ts +46 -12
- package/src/core/modules/file/core-file.service.ts +482 -16
- package/src/core/modules/file/file-metadata.helper.ts +142 -0
- package/src/core/modules/file/file-roles.helper.ts +110 -0
- package/src/core/modules/file/file-storage.helper.ts +163 -0
- package/src/core/modules/file/filesystem-file.helper.ts +184 -0
- package/src/core/modules/file/interfaces/file-service-options.interface.ts +14 -0
- package/src/core/modules/file/interfaces/file-upload.interface.ts +17 -1
- package/src/core/modules/file/s3-file.helper.ts +209 -0
- package/src/core/modules/hub/README.md +48 -0
- package/src/core/modules/hub/core-hub.controller.ts +18 -16
- package/src/core/modules/hub/helpers/hub-mask.helper.ts +21 -4
- package/src/core/modules/hub/hub-buffer.ts +198 -0
- package/src/core/modules/hub/services/core-hub-mailbox.service.ts +22 -12
- package/src/core/modules/hub/services/hub-log-buffer.service.ts +24 -8
- package/src/core/modules/hub/services/hub-query-profiler.service.ts +9 -6
- package/src/core/modules/hub/services/hub-trace-buffer.service.ts +14 -9
- package/src/core/modules/migrate/README.md +68 -6
- package/src/core/modules/migrate/cli/migrate-cli.ts +69 -6
- package/src/core/modules/migrate/helpers/migration.helper.ts +144 -10
- package/src/core/modules/migrate/migration-runner.ts +10 -1
- package/src/core/modules/migrate/mongo-state-store.ts +179 -11
- package/src/core/modules/system-setup/README.md +7 -2
- package/src/core/modules/system-setup/core-system-setup.service.ts +95 -1
- package/src/core/modules/tenant/README.md +21 -1
- package/src/core/modules/tenant/core-tenant.guard.ts +135 -14
- package/src/core/modules/tenant/core-tenant.module.ts +8 -4
- package/src/core/modules/tus/INTEGRATION-CHECKLIST.md +93 -21
- package/src/core/modules/tus/README.md +35 -2
- package/src/core/modules/tus/core-tus.controller.ts +64 -12
- package/src/core/modules/tus/core-tus.service.ts +358 -34
- package/src/core/modules/tus/interfaces/tus-config.interface.ts +9 -0
- package/src/core/modules/tus/tus-redis-locker.ts +139 -0
- package/src/core/modules/tus/tus.module.ts +78 -6
- package/src/core.module.ts +26 -1
- package/src/index.ts +10 -0
- package/src/main.ts +7 -1
- package/src/server/modules/file/file.controller.ts +7 -3
- package/src/server/modules/file/file.resolver.ts +5 -17
- package/src/server/modules/file/file.service.ts +42 -3
- package/src/server/modules/user/avatar.controller.ts +39 -9
- package/src/server/modules/user/user.model.ts +9 -3
- package/src/server/modules/user/user.module.ts +12 -3
- package/src/server/modules/user/user.service.ts +20 -17
- package/src/test/README.md +7 -2
- package/src/core/common/helpers/logging.helper.spec.ts +0 -61
- package/src/core/common/helpers/process-diagnostics.helper.spec.ts +0 -310
- package/src/core/common/services/brevo.service.spec.ts +0 -266
- package/src/core/modules/hub/core-hub.module.spec.ts +0 -108
- package/src/core/modules/hub/helpers/hub-command-shape.helper.spec.ts +0 -48
- package/src/core/modules/hub/helpers/hub-mask.helper.spec.ts +0 -67
- package/src/core/modules/hub/helpers/hub-mermaid.helper.spec.ts +0 -61
- package/src/core/modules/hub/helpers/hub-shell.helper.spec.ts +0 -106
- package/src/core/modules/hub/hub-config.helper.spec.ts +0 -108
- package/src/core/modules/hub/hub-ring-buffer.spec.ts +0 -95
- package/src/core/modules/hub/services/core-hub-mailbox.service.spec.ts +0 -116
- package/src/core/modules/hub/services/hub-trace-buffer.service.spec.ts +0 -112
|
@@ -4,9 +4,21 @@ import { Server, Upload } from '@tus/server';
|
|
|
4
4
|
import * as fs from 'fs';
|
|
5
5
|
import { Connection, mongo } from 'mongoose';
|
|
6
6
|
import * as path from 'path';
|
|
7
|
+
import { Readable } from 'stream';
|
|
7
8
|
|
|
8
9
|
import { GridFSHelper } from '../../common/helpers/gridfs.helper';
|
|
9
10
|
import { ITusConfig } from '../../common/interfaces/server-options.interface';
|
|
11
|
+
import { ConfigService } from '../../common/services/config.service';
|
|
12
|
+
import { CoreRedisService } from '../../common/services/core-redis.service';
|
|
13
|
+
import { CoreS3Service } from '../../common/services/core-s3.service';
|
|
14
|
+
import { resolveFileStorage } from '../file/file-storage.helper';
|
|
15
|
+
import {
|
|
16
|
+
DEFAULT_FILESYSTEM_DIR,
|
|
17
|
+
FILESYSTEM_FILES_COLLECTION,
|
|
18
|
+
FilesystemFileHelper,
|
|
19
|
+
} from '../file/filesystem-file.helper';
|
|
20
|
+
import { S3_FILES_COLLECTION, S3FileHelper } from '../file/s3-file.helper';
|
|
21
|
+
import { TusRedisLocker } from './tus-redis-locker';
|
|
10
22
|
import {
|
|
11
23
|
DEFAULT_TUS_ALLOWED_HEADERS,
|
|
12
24
|
DEFAULT_TUS_CONFIG,
|
|
@@ -14,11 +26,40 @@ import {
|
|
|
14
26
|
parseExpirationTime,
|
|
15
27
|
} from './interfaces/tus-config.interface';
|
|
16
28
|
|
|
29
|
+
/**
|
|
30
|
+
* Optional dependencies of CoreTusService.
|
|
31
|
+
*
|
|
32
|
+
* With an enabled CoreS3Service, in-progress uploads are staged in the S3
|
|
33
|
+
* staging bucket instead of on local disk (`tus.s3Staging`), which keeps
|
|
34
|
+
* resumable uploads working across replica restarts.
|
|
35
|
+
*/
|
|
36
|
+
export interface CoreTusServiceOptions {
|
|
37
|
+
configService?: ConfigService;
|
|
38
|
+
|
|
39
|
+
/** Central Redis; when present, upload locks are shared across replicas */
|
|
40
|
+
redisService?: CoreRedisService;
|
|
41
|
+
|
|
42
|
+
s3Service?: CoreS3Service;
|
|
43
|
+
}
|
|
44
|
+
|
|
17
45
|
/**
|
|
18
46
|
* Core TUS Service
|
|
19
47
|
*
|
|
20
48
|
* Provides integration with @tus/server for resumable file uploads.
|
|
21
|
-
* After upload completion, files are migrated to
|
|
49
|
+
* After upload completion, files are migrated to the configured file storage
|
|
50
|
+
* (S3 when `file.storage: 's3'`, otherwise GridFS).
|
|
51
|
+
*
|
|
52
|
+
* Uploads in progress are staged either on local disk (`@tus/file-store`) or,
|
|
53
|
+
* when S3 is configured and `tus.s3Staging` is not disabled, in the S3 staging
|
|
54
|
+
* bucket (`@tus/s3-store`). When BOTH ends are that same S3 store, the finished
|
|
55
|
+
* upload is moved by S3 itself (`CopyObject`) instead of being streamed through
|
|
56
|
+
* this process — see `canCopyWithinS3()`.
|
|
57
|
+
*
|
|
58
|
+
* NOTE: give the staging bucket a lifecycle rule that expires incomplete
|
|
59
|
+
* multipart uploads — aborted TUS uploads leave parts behind that nothing else
|
|
60
|
+
* cleans up (the local-disk store is swept by the expiration cleanup below).
|
|
61
|
+
* Objects of COMPLETED uploads are not covered by such a rule and are removed
|
|
62
|
+
* explicitly instead — see `deleteStagedObjects()`.
|
|
22
63
|
*
|
|
23
64
|
* This service follows the Module Inheritance Pattern and can be extended in projects.
|
|
24
65
|
*/
|
|
@@ -30,7 +71,23 @@ export class CoreTusService implements OnModuleDestroy, OnModuleInit {
|
|
|
30
71
|
private files: mongo.GridFSBucket;
|
|
31
72
|
private cleanupInterval: NodeJS.Timeout | null = null;
|
|
32
73
|
|
|
33
|
-
|
|
74
|
+
/** Datastore of the TUS server; set when uploads are staged in S3 */
|
|
75
|
+
protected s3Store: any = null;
|
|
76
|
+
|
|
77
|
+
/**
|
|
78
|
+
* Whether {@link CoreTusService.s3Store} was built from `options.s3Service`.
|
|
79
|
+
*
|
|
80
|
+
* Only then do staging and final storage share one endpoint, one region and one set of
|
|
81
|
+
* credentials — the precondition for letting S3 copy the finished upload internally. Tracked
|
|
82
|
+
* as a flag rather than inferred from `s3Store` being set, because a subclass may override
|
|
83
|
+
* {@link CoreTusService.createS3Store} to stage somewhere else entirely.
|
|
84
|
+
*/
|
|
85
|
+
protected s3StoreSharesEndpoint = false;
|
|
86
|
+
|
|
87
|
+
constructor(
|
|
88
|
+
private readonly connection: Connection,
|
|
89
|
+
protected readonly options?: CoreTusServiceOptions,
|
|
90
|
+
) {
|
|
34
91
|
// Initialize with defaults - will be configured in onModuleInit or via configure()
|
|
35
92
|
this.config = { ...DEFAULT_TUS_CONFIG };
|
|
36
93
|
}
|
|
@@ -52,6 +109,12 @@ export class CoreTusService implements OnModuleDestroy, OnModuleInit {
|
|
|
52
109
|
return;
|
|
53
110
|
}
|
|
54
111
|
|
|
112
|
+
// Idempotent on purpose: a second init would replace `tusServer` and `cleanupInterval`, and
|
|
113
|
+
// the replaced interval — no longer referenced — could never be cleared again.
|
|
114
|
+
if (this.tusServer) {
|
|
115
|
+
return;
|
|
116
|
+
}
|
|
117
|
+
|
|
55
118
|
// Initialize GridFS bucket
|
|
56
119
|
this.files = new mongo.GridFSBucket(this.connection.db, { bucketName: 'fs' });
|
|
57
120
|
|
|
@@ -60,7 +123,7 @@ export class CoreTusService implements OnModuleDestroy, OnModuleInit {
|
|
|
60
123
|
await this.ensureUploadDir(uploadDir);
|
|
61
124
|
|
|
62
125
|
// Create TUS server instance
|
|
63
|
-
this.tusServer = this.createTusServer(uploadDir);
|
|
126
|
+
this.tusServer = await this.createTusServer(uploadDir);
|
|
64
127
|
|
|
65
128
|
// Setup expiration cleanup if enabled
|
|
66
129
|
this.setupExpirationCleanup();
|
|
@@ -111,53 +174,290 @@ export class CoreTusService implements OnModuleDestroy, OnModuleInit {
|
|
|
111
174
|
* what happens after an upload completes.
|
|
112
175
|
*/
|
|
113
176
|
protected async onUploadComplete(upload: Upload): Promise<void> {
|
|
114
|
-
const uploadDir = this.config.uploadDir || DEFAULT_TUS_CONFIG.uploadDir;
|
|
115
|
-
const filePath = path.join(uploadDir, upload.id);
|
|
116
|
-
|
|
117
177
|
try {
|
|
118
178
|
// Extract metadata
|
|
119
179
|
const metadata = this.parseMetadata(upload.metadata);
|
|
120
180
|
const filename = metadata.filename || upload.id;
|
|
121
181
|
const contentType = metadata.filetype || 'application/octet-stream';
|
|
182
|
+
const fileMetadata = {
|
|
183
|
+
originalMetadata: metadata,
|
|
184
|
+
tusUploadId: upload.id,
|
|
185
|
+
uploadedAt: new Date(),
|
|
186
|
+
};
|
|
187
|
+
|
|
188
|
+
// S3 → S3: hand the object over INSIDE S3 rather than pulling it down and pushing it back
|
|
189
|
+
// up. The streaming path below is correct but pins the finishing PATCH request — and the
|
|
190
|
+
// upload lock it holds — for a full download plus a full upload of the whole file. At the
|
|
191
|
+
// 50 GB default cap that is unbounded; even a few GB overruns a typical reverse-proxy read
|
|
192
|
+
// timeout, and it pays for the traffic twice.
|
|
193
|
+
if (this.canCopyWithinS3(upload)) {
|
|
194
|
+
const fileInfo = await S3FileHelper.copyFile(
|
|
195
|
+
this.options.s3Service,
|
|
196
|
+
this.connection.db.collection(S3_FILES_COLLECTION),
|
|
197
|
+
{
|
|
198
|
+
contentLength: upload.size,
|
|
199
|
+
contentType,
|
|
200
|
+
filename,
|
|
201
|
+
metadata: fileMetadata,
|
|
202
|
+
sourceBucket: this.options.s3Service.getConfig().stagingBucket,
|
|
203
|
+
sourceKey: upload.id,
|
|
204
|
+
},
|
|
205
|
+
);
|
|
206
|
+
this.logger.debug(
|
|
207
|
+
`Upload ${upload.id} copied inside S3 to ${fileInfo._id} (filename: ${filename}) — no bytes through this process`,
|
|
208
|
+
);
|
|
209
|
+
await this.deleteStagedUpload(upload.id);
|
|
210
|
+
return;
|
|
211
|
+
}
|
|
122
212
|
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
.access(filePath)
|
|
126
|
-
.then(() => true)
|
|
127
|
-
.catch(() => false);
|
|
128
|
-
if (!fileExists) {
|
|
129
|
-
this.logger.warn(`Upload file not found at ${filePath}, skipping GridFS migration`);
|
|
213
|
+
const readStream = await this.readStagedUpload(upload.id);
|
|
214
|
+
if (!readStream) {
|
|
130
215
|
return;
|
|
131
216
|
}
|
|
132
217
|
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
218
|
+
if (this.fileStorageDriver === 'filesystem') {
|
|
219
|
+
const fileInfo = await FilesystemFileHelper.writeFile(
|
|
220
|
+
this.fileStorageDir,
|
|
221
|
+
this.connection.db.collection(FILESYSTEM_FILES_COLLECTION),
|
|
222
|
+
{ body: readStream, contentType, filename, metadata: fileMetadata },
|
|
223
|
+
);
|
|
224
|
+
this.logger.debug(`Upload ${upload.id} migrated to the filesystem as ${fileInfo._id} (filename: ${filename})`);
|
|
225
|
+
} else if (this.s3FileStorage) {
|
|
226
|
+
const fileInfo = await S3FileHelper.writeFile(
|
|
227
|
+
this.options.s3Service,
|
|
228
|
+
this.connection.db.collection(S3_FILES_COLLECTION),
|
|
229
|
+
// Streamed with the length TUS already knows, so a multi-GB resumable upload is never
|
|
230
|
+
// materialised in memory.
|
|
231
|
+
{ body: readStream, contentLength: upload.size, contentType, filename, metadata: fileMetadata },
|
|
232
|
+
);
|
|
233
|
+
this.logger.debug(`Upload ${upload.id} migrated to S3 as ${fileInfo._id} (filename: ${filename})`);
|
|
234
|
+
} else {
|
|
235
|
+
const fileInfo = await GridFSHelper.writeFileFromStream(this.files, readStream, {
|
|
236
|
+
contentType,
|
|
237
|
+
filename,
|
|
238
|
+
metadata: fileMetadata,
|
|
239
|
+
});
|
|
240
|
+
this.logger.debug(`Upload ${upload.id} migrated to GridFS as ${fileInfo._id} (filename: ${filename})`);
|
|
241
|
+
}
|
|
242
|
+
|
|
243
|
+
// Clean up the staged upload
|
|
244
|
+
await this.deleteStagedUpload(upload.id);
|
|
149
245
|
} catch (error) {
|
|
150
|
-
this.logger.error(`Failed to migrate upload ${upload.id}
|
|
246
|
+
this.logger.error(`Failed to migrate upload ${upload.id}: ${error.message}`);
|
|
151
247
|
throw error;
|
|
152
248
|
}
|
|
153
249
|
}
|
|
154
250
|
|
|
251
|
+
/**
|
|
252
|
+
* Which store a finished upload is migrated into.
|
|
253
|
+
*
|
|
254
|
+
* Deliberately the SAME resolution `CoreFileService` uses. A tus upload has to
|
|
255
|
+
* land where the download routes look for it — resolving this independently is
|
|
256
|
+
* how an upload ends up written to one store and read from another.
|
|
257
|
+
*/
|
|
258
|
+
protected get fileStorageDriver(): string {
|
|
259
|
+
return resolveFileStorage(ConfigService.configFastButReadOnly).driver;
|
|
260
|
+
}
|
|
261
|
+
|
|
262
|
+
/** Directory used when uploads are migrated to the local filesystem */
|
|
263
|
+
protected get fileStorageDir(): string {
|
|
264
|
+
return ConfigService.configFastButReadOnly?.file?.storageDir || DEFAULT_FILESYSTEM_DIR;
|
|
265
|
+
}
|
|
266
|
+
|
|
267
|
+
/**
|
|
268
|
+
* Whether finished uploads are stored in S3 (`file.storage: 's3'`) instead of GridFS
|
|
269
|
+
*/
|
|
270
|
+
protected get s3FileStorage(): boolean {
|
|
271
|
+
return !!this.options?.s3Service?.enabled && this.fileStorageDriver === 's3';
|
|
272
|
+
}
|
|
273
|
+
|
|
274
|
+
/**
|
|
275
|
+
* Whether the finished upload can be handed over inside S3 (server-side copy) instead of
|
|
276
|
+
* streaming through this process.
|
|
277
|
+
*
|
|
278
|
+
* Requires BOTH ends to be the same S3 store: staged by `createS3Store()` from
|
|
279
|
+
* `options.s3Service`, and a final driver of `'s3'` backed by that same service. Every other
|
|
280
|
+
* combination — S3 staging → GridFS, S3 staging → filesystem, local-disk staging → anything —
|
|
281
|
+
* genuinely has to transit the pod, because no store can reach into the other.
|
|
282
|
+
*/
|
|
283
|
+
protected canCopyWithinS3(upload: Upload): boolean {
|
|
284
|
+
if (!this.s3StoreSharesEndpoint || !this.s3FileStorage) {
|
|
285
|
+
return false;
|
|
286
|
+
}
|
|
287
|
+
|
|
288
|
+
// A single CopyObject cannot move more than 5 GB; above that S3 requires multipart
|
|
289
|
+
// upload-part-copy. We fall back to streaming rather than implement that — but never
|
|
290
|
+
// silently: this is exactly the size at which the streaming cost the copy avoids becomes
|
|
291
|
+
// severe, so an operator seeing slow, timing-out completions needs to find the reason in
|
|
292
|
+
// the log rather than guess at it.
|
|
293
|
+
if (typeof upload.size !== 'number') {
|
|
294
|
+
this.logger.warn(
|
|
295
|
+
`Upload ${upload.id} has no known size, so it cannot be size-checked against the ${CoreS3Service.MAX_COPY_OBJECT_BYTES}-byte ` +
|
|
296
|
+
'CopyObject limit — streaming it through this process instead.',
|
|
297
|
+
);
|
|
298
|
+
return false;
|
|
299
|
+
}
|
|
300
|
+
if (upload.size > CoreS3Service.MAX_COPY_OBJECT_BYTES) {
|
|
301
|
+
this.logger.warn(
|
|
302
|
+
`Upload ${upload.id} is ${upload.size} bytes, above the ${CoreS3Service.MAX_COPY_OBJECT_BYTES}-byte limit of a single ` +
|
|
303
|
+
'S3 CopyObject (multipart upload-part-copy would be required). Streaming it through this process instead — ' +
|
|
304
|
+
'expect the finishing request to run for the duration of a full download plus a full upload.',
|
|
305
|
+
);
|
|
306
|
+
return false;
|
|
307
|
+
}
|
|
308
|
+
|
|
309
|
+
return true;
|
|
310
|
+
}
|
|
311
|
+
|
|
312
|
+
/**
|
|
313
|
+
* Locker for upload exclusivity, or undefined to keep @tus/server's in-process default.
|
|
314
|
+
*/
|
|
315
|
+
protected createLocker(): any {
|
|
316
|
+
const redisService = this.options?.redisService;
|
|
317
|
+
if (!redisService?.enabled) {
|
|
318
|
+
return undefined;
|
|
319
|
+
}
|
|
320
|
+
this.logger.debug('TUS upload locks are shared via Redis');
|
|
321
|
+
return new TusRedisLocker(redisService);
|
|
322
|
+
}
|
|
323
|
+
|
|
324
|
+
/**
|
|
325
|
+
* Import the optional peer dependency `@tus/s3-store`.
|
|
326
|
+
* Separate method so tests can substitute the module.
|
|
327
|
+
*/
|
|
328
|
+
protected importS3Store(): Promise<any> {
|
|
329
|
+
return import('@tus/s3-store');
|
|
330
|
+
}
|
|
331
|
+
|
|
332
|
+
/**
|
|
333
|
+
* Create the S3 staging datastore.
|
|
334
|
+
* Returns null when S3 staging is not available, so the caller falls back to the local FileStore.
|
|
335
|
+
*/
|
|
336
|
+
protected async createS3Store(): Promise<any> {
|
|
337
|
+
const s3Service = this.options?.s3Service;
|
|
338
|
+
if (!s3Service?.enabled || this.config.s3Staging === false) {
|
|
339
|
+
return null;
|
|
340
|
+
}
|
|
341
|
+
|
|
342
|
+
let s3StoreModule: any;
|
|
343
|
+
try {
|
|
344
|
+
s3StoreModule = await this.importS3Store();
|
|
345
|
+
} catch {
|
|
346
|
+
this.logger.warn(
|
|
347
|
+
'S3 is configured but the optional peer dependency "@tus/s3-store" is not installed — TUS uploads are ' +
|
|
348
|
+
'staged on local disk. Run: pnpm add @tus/s3-store',
|
|
349
|
+
);
|
|
350
|
+
return null;
|
|
351
|
+
}
|
|
352
|
+
|
|
353
|
+
const { accessKeyId, endpoint, forcePathStyle, region, secretAccessKey, stagingBucket } = s3Service.getConfig();
|
|
354
|
+
this.s3Store = new s3StoreModule.S3Store({
|
|
355
|
+
s3ClientConfig: {
|
|
356
|
+
bucket: stagingBucket,
|
|
357
|
+
forcePathStyle,
|
|
358
|
+
region,
|
|
359
|
+
...(endpoint ? { endpoint } : {}),
|
|
360
|
+
...(accessKeyId && secretAccessKey ? { credentials: { accessKeyId, secretAccessKey } } : {}),
|
|
361
|
+
},
|
|
362
|
+
});
|
|
363
|
+
// Built from the shared CoreS3Service, so a finished upload can be copied inside S3.
|
|
364
|
+
this.s3StoreSharesEndpoint = true;
|
|
365
|
+
this.logger.log(`TUS uploads staged in S3 bucket ${stagingBucket}`);
|
|
366
|
+
return this.s3Store;
|
|
367
|
+
}
|
|
368
|
+
|
|
369
|
+
/**
|
|
370
|
+
* Read a completed upload from its staging location (S3 or local disk).
|
|
371
|
+
* Returns null when the staged data is gone.
|
|
372
|
+
*/
|
|
373
|
+
protected async readStagedUpload(uploadId: string): Promise<null | Readable> {
|
|
374
|
+
if (this.s3Store) {
|
|
375
|
+
return await this.s3Store.read(uploadId);
|
|
376
|
+
}
|
|
377
|
+
|
|
378
|
+
const uploadDir = this.config.uploadDir || DEFAULT_TUS_CONFIG.uploadDir;
|
|
379
|
+
const filePath = path.join(uploadDir, uploadId);
|
|
380
|
+
const fileExists = await fs.promises
|
|
381
|
+
.access(filePath)
|
|
382
|
+
.then(() => true)
|
|
383
|
+
.catch(() => false);
|
|
384
|
+
if (!fileExists) {
|
|
385
|
+
this.logger.warn(`Upload file not found at ${filePath}, skipping migration`);
|
|
386
|
+
return null;
|
|
387
|
+
}
|
|
388
|
+
return fs.createReadStream(filePath);
|
|
389
|
+
}
|
|
390
|
+
|
|
391
|
+
/**
|
|
392
|
+
* Delete a staged upload from its staging location (S3 or local disk)
|
|
393
|
+
*/
|
|
394
|
+
protected async deleteStagedUpload(uploadId: string): Promise<void> {
|
|
395
|
+
if (this.s3Store) {
|
|
396
|
+
try {
|
|
397
|
+
// Still worth calling: for an upload that never finished, this aborts the multipart and
|
|
398
|
+
// frees the uploaded parts, which a DeleteObject cannot do.
|
|
399
|
+
await this.s3Store.remove(uploadId);
|
|
400
|
+
} catch (error) {
|
|
401
|
+
// `remove()` issues AbortMultipartUpload FIRST, and for a COMPLETED upload S3 answers
|
|
402
|
+
// NoSuchUpload. @tus/s3-store v2 swallows that by testing `error.code` — an AWS SDK v2
|
|
403
|
+
// field that SDK v3 never populates (v3 uses `error.name` / `$metadata.httpStatusCode`) —
|
|
404
|
+
// so the error escapes and the deletion that follows it never runs. Every finished upload
|
|
405
|
+
// therefore left a full second copy of the file behind, in a bucket that by default IS the
|
|
406
|
+
// file bucket. No lifecycle rule reaches those: they are completed objects, not incomplete
|
|
407
|
+
// multiparts. Hence the explicit cleanup below, which does not depend on the peer's guard.
|
|
408
|
+
if (!this.isMissingUploadError(error)) {
|
|
409
|
+
this.logger.warn(`Failed to remove staged upload ${uploadId} from S3: ${error.message}`);
|
|
410
|
+
}
|
|
411
|
+
}
|
|
412
|
+
await this.deleteStagedObjects(uploadId);
|
|
413
|
+
return;
|
|
414
|
+
}
|
|
415
|
+
await this.deleteTemporaryFile(uploadId);
|
|
416
|
+
}
|
|
417
|
+
|
|
418
|
+
/**
|
|
419
|
+
* Delete the staged object and its `.info` sidecar from the staging bucket.
|
|
420
|
+
*
|
|
421
|
+
* Unconditional and idempotent: when `S3Store.remove()` already got there, these are no-ops
|
|
422
|
+
* (S3 answers a successful delete for a key that does not exist), and when it did not, this is
|
|
423
|
+
* what actually reclaims the space.
|
|
424
|
+
*/
|
|
425
|
+
protected async deleteStagedObjects(uploadId: string): Promise<void> {
|
|
426
|
+
const s3Service = this.options?.s3Service;
|
|
427
|
+
// Only when the staging store is OURS — a subclass may stage somewhere this service cannot see.
|
|
428
|
+
if (!this.s3StoreSharesEndpoint || !s3Service?.enabled) {
|
|
429
|
+
return;
|
|
430
|
+
}
|
|
431
|
+
const { stagingBucket } = s3Service.getConfig();
|
|
432
|
+
for (const key of [uploadId, `${uploadId}.info`]) {
|
|
433
|
+
try {
|
|
434
|
+
await s3Service.deleteObject(key, stagingBucket);
|
|
435
|
+
} catch (error) {
|
|
436
|
+
this.logger.warn(`Failed to delete staged object ${key} from S3 bucket ${stagingBucket}: ${error.message}`);
|
|
437
|
+
}
|
|
438
|
+
}
|
|
439
|
+
}
|
|
440
|
+
|
|
441
|
+
/**
|
|
442
|
+
* Whether an S3 error means "that upload/object is not there" — which, when cleaning up, is the
|
|
443
|
+
* outcome we wanted anyway.
|
|
444
|
+
*
|
|
445
|
+
* Matches on `name` and the HTTP status, never on `code`: that is the AWS SDK v2 shape, and
|
|
446
|
+
* relying on it is exactly the bug this guards against.
|
|
447
|
+
*/
|
|
448
|
+
protected isMissingUploadError(error: any): boolean {
|
|
449
|
+
const name = error?.name ?? error?.Code;
|
|
450
|
+
return (
|
|
451
|
+
name === 'NoSuchUpload' || name === 'NoSuchKey' || name === 'NotFound' || error?.$metadata?.httpStatusCode === 404
|
|
452
|
+
);
|
|
453
|
+
}
|
|
454
|
+
|
|
155
455
|
/**
|
|
156
456
|
* Handle upload termination (deletion)
|
|
157
457
|
*/
|
|
158
458
|
protected async onUploadTerminate(upload: Upload): Promise<void> {
|
|
159
459
|
this.logger.debug(`Upload ${upload.id} terminated`);
|
|
160
|
-
await this.
|
|
460
|
+
await this.deleteStagedUpload(upload.id);
|
|
161
461
|
}
|
|
162
462
|
|
|
163
463
|
/**
|
|
@@ -203,12 +503,21 @@ export class CoreTusService implements OnModuleDestroy, OnModuleInit {
|
|
|
203
503
|
/**
|
|
204
504
|
* Create the TUS server instance with configured extensions
|
|
205
505
|
*/
|
|
206
|
-
private createTusServer(uploadDir: string): Server {
|
|
207
|
-
const datastore = new FileStore({ directory: uploadDir });
|
|
506
|
+
private async createTusServer(uploadDir: string): Promise<Server> {
|
|
507
|
+
const datastore = (await this.createS3Store()) || new FileStore({ directory: uploadDir });
|
|
508
|
+
|
|
509
|
+
// @tus/server's default locker is in-memory, which is exclusive within ONE process only:
|
|
510
|
+
// behind a load balancer two replicas each hold their own lock for the same upload id and
|
|
511
|
+
// both accept a PATCH, interleaving two byte ranges into one file. Resumable uploads are
|
|
512
|
+
// long-lived and clients retry, so requests for one upload landing on different replicas is
|
|
513
|
+
// the normal case. With Redis the lock is shared; without it the default stands, which is
|
|
514
|
+
// correct for the single-replica deployment that configuration describes.
|
|
515
|
+
const locker = this.createLocker();
|
|
208
516
|
|
|
209
517
|
const server = new Server({
|
|
210
518
|
allowedHeaders: this.config.allowedHeaders || DEFAULT_TUS_ALLOWED_HEADERS,
|
|
211
519
|
datastore,
|
|
520
|
+
...(locker ? { locker } : {}),
|
|
212
521
|
maxSize: this.config.maxSize,
|
|
213
522
|
onUploadCreate: async (_req, upload) => {
|
|
214
523
|
// Validate file type if allowedTypes is configured
|
|
@@ -240,8 +549,16 @@ export class CoreTusService implements OnModuleDestroy, OnModuleInit {
|
|
|
240
549
|
await this.onUploadComplete(upload);
|
|
241
550
|
return {};
|
|
242
551
|
} catch (error) {
|
|
552
|
+
// Do NOT swallow this. `onUploadComplete` is what moves the finished upload into its
|
|
553
|
+
// permanent store; returning `{}` after it failed answers the client with a 204 that
|
|
554
|
+
// says the upload completed while nothing was persisted. The client then deletes its
|
|
555
|
+
// local copy and the file is gone — a silent data loss, and the one failure mode a
|
|
556
|
+
// resumable-upload protocol exists to prevent. Failing loudly lets the client retry.
|
|
243
557
|
this.logger.error(`Upload finish error: ${error.message}`);
|
|
244
|
-
|
|
558
|
+
throw Object.assign(new Error(`Upload could not be stored: ${error.message}`), {
|
|
559
|
+
body: 'Upload could not be stored',
|
|
560
|
+
status_code: 500,
|
|
561
|
+
});
|
|
245
562
|
}
|
|
246
563
|
},
|
|
247
564
|
path: this.config.path || DEFAULT_TUS_CONFIG.path,
|
|
@@ -349,6 +666,8 @@ export class CoreTusService implements OnModuleDestroy, OnModuleInit {
|
|
|
349
666
|
},
|
|
350
667
|
60 * 60 * 1000,
|
|
351
668
|
);
|
|
669
|
+
// An hourly sweep must never be the thing keeping the process alive.
|
|
670
|
+
this.cleanupInterval.unref?.();
|
|
352
671
|
|
|
353
672
|
this.logger.debug(`Expiration cleanup scheduled (expire after ${expiresIn})`);
|
|
354
673
|
}
|
|
@@ -357,6 +676,11 @@ export class CoreTusService implements OnModuleDestroy, OnModuleInit {
|
|
|
357
676
|
* Clean up expired incomplete uploads
|
|
358
677
|
*/
|
|
359
678
|
private async cleanupExpiredUploads(maxAgeMs: number): Promise<void> {
|
|
679
|
+
if (this.s3Store) {
|
|
680
|
+
// Aborted uploads live in the S3 staging bucket — expire them with a
|
|
681
|
+
// bucket lifecycle rule for incomplete multipart uploads instead.
|
|
682
|
+
return;
|
|
683
|
+
}
|
|
360
684
|
const uploadDir = this.config.uploadDir || DEFAULT_TUS_CONFIG.uploadDir;
|
|
361
685
|
const now = Date.now();
|
|
362
686
|
|
|
@@ -4,6 +4,7 @@
|
|
|
4
4
|
* The interfaces are defined in server-options.interface.ts to avoid circular imports.
|
|
5
5
|
* This file contains helper functions and defaults.
|
|
6
6
|
*/
|
|
7
|
+
import { RoleEnum } from '../../../common/enums/role.enum';
|
|
7
8
|
import { ITusConfig, ITusExpirationConfig } from '../../../common/interfaces/server-options.interface';
|
|
8
9
|
|
|
9
10
|
// Re-export for convenience
|
|
@@ -46,6 +47,14 @@ export const DEFAULT_TUS_CONFIG: Required<
|
|
|
46
47
|
expiration: { enabled: true, expiresIn: '24h' },
|
|
47
48
|
maxSize: 50 * 1024 * 1024 * 1024, // 50 GB
|
|
48
49
|
path: '/tus',
|
|
50
|
+
// A tus upload writes into the SAME file store that `file.downloadRoles`
|
|
51
|
+
// guards — GridFS or S3, whichever `file.storage` selects — and with the
|
|
52
|
+
// termination extension it can delete from it too. Requiring a session is the
|
|
53
|
+
// least that keeps the module's own posture coherent: anonymous 50 GB writes
|
|
54
|
+
// into a store only admins may read is not a defensible default. Widen
|
|
55
|
+
// deliberately via `tus: { roles: [...] }`.
|
|
56
|
+
roles: [RoleEnum.S_USER],
|
|
57
|
+
s3Staging: true,
|
|
49
58
|
termination: true,
|
|
50
59
|
uploadDir: 'uploads/tus',
|
|
51
60
|
};
|
|
@@ -0,0 +1,139 @@
|
|
|
1
|
+
import { Logger } from '@nestjs/common';
|
|
2
|
+
import { randomUUID } from 'node:crypto';
|
|
3
|
+
|
|
4
|
+
import { CoreRedisService } from '../../common/services/core-redis.service';
|
|
5
|
+
|
|
6
|
+
/** Release callback handed to the current lock holder when someone else wants the resource */
|
|
7
|
+
type RequestRelease = () => Promise<void> | void;
|
|
8
|
+
|
|
9
|
+
/**
|
|
10
|
+
* TUS lock that is exclusive across REPLICAS, not just within one process.
|
|
11
|
+
*
|
|
12
|
+
* `@tus/server` defaults to an in-memory locker, which is exactly right for one process and
|
|
13
|
+
* useless behind a load balancer: two replicas each hold their own lock for the same upload id
|
|
14
|
+
* and both accept a PATCH, so two byte ranges are written to one upload and the file ends up
|
|
15
|
+
* interleaved. Resumable uploads are long-lived and clients retry aggressively, so a request
|
|
16
|
+
* pair landing on different replicas is the normal case, not a rare one.
|
|
17
|
+
*
|
|
18
|
+
* The lock is a Redis key with a TTL held only while a request is being served, refreshed by a
|
|
19
|
+
* heartbeat so a slow but live PATCH keeps it, and released in `unlock()`. A holder that dies
|
|
20
|
+
* mid-request stops refreshing and the key expires — the upload becomes writable again instead
|
|
21
|
+
* of being stuck until someone intervenes.
|
|
22
|
+
*/
|
|
23
|
+
export class TusRedisLock {
|
|
24
|
+
protected heartbeat?: NodeJS.Timeout;
|
|
25
|
+
|
|
26
|
+
/** Random per-acquisition value, so only the actual holder can release or refresh the lock */
|
|
27
|
+
protected token?: string;
|
|
28
|
+
|
|
29
|
+
constructor(
|
|
30
|
+
protected readonly redisService: CoreRedisService,
|
|
31
|
+
protected readonly key: string,
|
|
32
|
+
protected readonly ttlMs: number,
|
|
33
|
+
protected readonly acquireTimeoutMs: number,
|
|
34
|
+
) {}
|
|
35
|
+
|
|
36
|
+
/**
|
|
37
|
+
* Wait for exclusive access to the upload, or throw once the timeout is reached.
|
|
38
|
+
*
|
|
39
|
+
* `cancelReq` is the TUS mechanism for cooperative handover: when a newer request wants an
|
|
40
|
+
* upload we already hold, the holder is asked to let go rather than made to wait it out.
|
|
41
|
+
*/
|
|
42
|
+
async lock(signal: AbortSignal, cancelReq: RequestRelease): Promise<void> {
|
|
43
|
+
const deadline = Date.now() + this.acquireTimeoutMs;
|
|
44
|
+
// randomUUID, not Math.random: this token is the ONLY thing separating "holds the lock" from
|
|
45
|
+
// "does not" in the compare-and-delete below and in the heartbeat's PEXPIRE. A collision lets
|
|
46
|
+
// one holder refresh or release another's lock, and two replicas then interleave byte ranges
|
|
47
|
+
// into one upload — the exact corruption this locker exists to prevent.
|
|
48
|
+
const token = `${process.pid}-${randomUUID()}`;
|
|
49
|
+
|
|
50
|
+
for (;;) {
|
|
51
|
+
const acquired = await this.redisService.getClient().set(this.key, token, 'PX', this.ttlMs, 'NX');
|
|
52
|
+
|
|
53
|
+
if (acquired === 'OK') {
|
|
54
|
+
this.token = token;
|
|
55
|
+
this.startHeartbeat();
|
|
56
|
+
return;
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
// Tell whoever holds it that someone else is waiting. Signalling on every attempt is
|
|
60
|
+
// deliberate: the holder may be on another replica and only learns of us through this.
|
|
61
|
+
await cancelReq();
|
|
62
|
+
|
|
63
|
+
if (signal.aborted || Date.now() > deadline) {
|
|
64
|
+
throw new Error(`Could not acquire the upload lock for "${this.key}" within ${this.acquireTimeoutMs}ms`);
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
await new Promise((resolve) => setTimeout(resolve, 100));
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
/** Release the lock — only if we still hold it */
|
|
72
|
+
async unlock(): Promise<void> {
|
|
73
|
+
this.stopHeartbeat();
|
|
74
|
+
const token = this.token;
|
|
75
|
+
this.token = undefined;
|
|
76
|
+
if (!token) {
|
|
77
|
+
return;
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
// Compare-and-delete: after our TTL expired the key may already belong to someone else,
|
|
81
|
+
// and deleting it blindly would unlock an upload another replica is actively writing.
|
|
82
|
+
const script = `if redis.call('GET', KEYS[1]) == ARGV[1] then return redis.call('DEL', KEYS[1]) end return 0`;
|
|
83
|
+
try {
|
|
84
|
+
await this.redisService.getClient().eval(script, 1, this.key, token);
|
|
85
|
+
} catch {
|
|
86
|
+
// The key expires on its own; a failed release costs at most one TTL of waiting.
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
/** Keep the lock alive while the request is still being served */
|
|
91
|
+
protected startHeartbeat(): void {
|
|
92
|
+
this.heartbeat = setInterval(
|
|
93
|
+
() => {
|
|
94
|
+
const token = this.token;
|
|
95
|
+
if (!token) {
|
|
96
|
+
return;
|
|
97
|
+
}
|
|
98
|
+
const script = `if redis.call('GET', KEYS[1]) == ARGV[1] then return redis.call('PEXPIRE', KEYS[1], ARGV[2]) end return 0`;
|
|
99
|
+
this.redisService
|
|
100
|
+
.getClient()
|
|
101
|
+
.eval(script, 1, this.key, token, this.ttlMs)
|
|
102
|
+
.catch(() => undefined);
|
|
103
|
+
},
|
|
104
|
+
Math.max(1000, Math.floor(this.ttlMs / 3)),
|
|
105
|
+
);
|
|
106
|
+
this.heartbeat.unref?.();
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
protected stopHeartbeat(): void {
|
|
110
|
+
if (this.heartbeat) {
|
|
111
|
+
clearInterval(this.heartbeat);
|
|
112
|
+
this.heartbeat = undefined;
|
|
113
|
+
}
|
|
114
|
+
}
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
/**
|
|
118
|
+
* Locker handed to `@tus/server` when Redis is configured (see {@link TusRedisLock}).
|
|
119
|
+
*/
|
|
120
|
+
export class TusRedisLocker {
|
|
121
|
+
protected readonly logger = new Logger(TusRedisLocker.name);
|
|
122
|
+
|
|
123
|
+
constructor(
|
|
124
|
+
protected readonly redisService: CoreRedisService,
|
|
125
|
+
/** How long a lock survives without a heartbeat — a dead holder's upload frees itself */
|
|
126
|
+
protected readonly ttlMs = 30_000,
|
|
127
|
+
/** How long a request waits for a busy upload before giving up with 500 */
|
|
128
|
+
protected readonly acquireTimeoutMs = 10_000,
|
|
129
|
+
) {}
|
|
130
|
+
|
|
131
|
+
newLock(id: string): TusRedisLock {
|
|
132
|
+
return new TusRedisLock(
|
|
133
|
+
this.redisService,
|
|
134
|
+
this.redisService.key('tus-lock', id),
|
|
135
|
+
this.ttlMs,
|
|
136
|
+
this.acquireTimeoutMs,
|
|
137
|
+
);
|
|
138
|
+
}
|
|
139
|
+
}
|