@lenne.tech/nest-server 11.32.3 → 11.33.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude/rules/architecture.md +16 -0
- package/.claude/rules/configurable-features.md +11 -3
- package/.claude/rules/testing.md +101 -1
- package/CLAUDE.md +5 -0
- package/FRAMEWORK-API.md +58 -3
- package/bin/migrate.js +13 -3
- package/dist/core/common/helpers/content-disposition.helper.d.ts +1 -0
- package/dist/core/common/helpers/content-disposition.helper.js +14 -0
- package/dist/core/common/helpers/content-disposition.helper.js.map +1 -0
- package/dist/core/common/helpers/file.helper.d.ts +21 -2
- package/dist/core/common/helpers/file.helper.js +69 -13
- package/dist/core/common/helpers/file.helper.js.map +1 -1
- package/dist/core/common/helpers/graceful-shutdown.helper.d.ts +2 -0
- package/dist/core/common/helpers/graceful-shutdown.helper.js +55 -0
- package/dist/core/common/helpers/graceful-shutdown.helper.js.map +1 -0
- package/dist/core/common/helpers/project-name.helper.d.ts +2 -0
- package/dist/core/common/helpers/project-name.helper.js +38 -0
- package/dist/core/common/helpers/project-name.helper.js.map +1 -0
- package/dist/core/common/interfaces/cron-job-config.interface.d.ts +1 -0
- package/dist/core/common/interfaces/server-options.interface.d.ts +39 -0
- package/dist/core/common/services/core-cron-jobs.initializer.d.ts +10 -0
- package/dist/core/common/services/core-cron-jobs.initializer.js +46 -0
- package/dist/core/common/services/core-cron-jobs.initializer.js.map +1 -0
- package/dist/core/common/services/core-cron-jobs.registry.d.ts +8 -0
- package/dist/core/common/services/core-cron-jobs.registry.js +12 -0
- package/dist/core/common/services/core-cron-jobs.registry.js.map +1 -0
- package/dist/core/common/services/core-cron-jobs.service.d.ts +37 -7
- package/dist/core/common/services/core-cron-jobs.service.js +278 -27
- package/dist/core/common/services/core-cron-jobs.service.js.map +1 -1
- package/dist/core/common/services/core-redis-pubsub.d.ts +20 -0
- package/dist/core/common/services/core-redis-pubsub.js +112 -0
- package/dist/core/common/services/core-redis-pubsub.js.map +1 -0
- package/dist/core/common/services/core-redis.service.d.ts +25 -0
- package/dist/core/common/services/core-redis.service.js +154 -0
- package/dist/core/common/services/core-redis.service.js.map +1 -0
- package/dist/core/common/services/core-s3.service.d.ts +51 -0
- package/dist/core/common/services/core-s3.service.js +261 -0
- package/dist/core/common/services/core-s3.service.js.map +1 -0
- package/dist/core/common/services/core-trust-proxy.initializer.d.ts +16 -0
- package/dist/core/common/services/core-trust-proxy.initializer.js +77 -0
- package/dist/core/common/services/core-trust-proxy.initializer.js.map +1 -0
- package/dist/core/common/services/rate-limit-store.d.ts +61 -0
- package/dist/core/common/services/rate-limit-store.js +235 -0
- package/dist/core/common/services/rate-limit-store.js.map +1 -0
- package/dist/core/modules/ai/core-ai-mcp.controller.d.ts +19 -3
- package/dist/core/modules/ai/core-ai-mcp.controller.js +148 -18
- package/dist/core/modules/ai/core-ai-mcp.controller.js.map +1 -1
- package/dist/core/modules/ai/services/core-ai.service.d.ts +7 -2
- package/dist/core/modules/ai/services/core-ai.service.js +24 -20
- package/dist/core/modules/ai/services/core-ai.service.js.map +1 -1
- package/dist/core/modules/auth/core-auth.module.js +4 -1
- package/dist/core/modules/auth/core-auth.module.js.map +1 -1
- package/dist/core/modules/auth/guards/legacy-auth-rate-limit.guard.d.ts +1 -1
- package/dist/core/modules/auth/guards/legacy-auth-rate-limit.guard.js +3 -14
- package/dist/core/modules/auth/guards/legacy-auth-rate-limit.guard.js.map +1 -1
- package/dist/core/modules/auth/services/legacy-auth-rate-limiter.service.d.ts +12 -10
- package/dist/core/modules/auth/services/legacy-auth-rate-limiter.service.js +30 -59
- package/dist/core/modules/auth/services/legacy-auth-rate-limiter.service.js.map +1 -1
- package/dist/core/modules/better-auth/core-better-auth-email-verification.service.d.ts +8 -2
- package/dist/core/modules/better-auth/core-better-auth-email-verification.service.js +118 -45
- package/dist/core/modules/better-auth/core-better-auth-email-verification.service.js.map +1 -1
- package/dist/core/modules/better-auth/core-better-auth-rate-limit.middleware.d.ts +1 -1
- package/dist/core/modules/better-auth/core-better-auth-rate-limit.middleware.js +2 -11
- package/dist/core/modules/better-auth/core-better-auth-rate-limit.middleware.js.map +1 -1
- package/dist/core/modules/better-auth/core-better-auth-rate-limiter.service.d.ts +12 -10
- package/dist/core/modules/better-auth/core-better-auth-rate-limiter.service.js +30 -85
- package/dist/core/modules/better-auth/core-better-auth-rate-limiter.service.js.map +1 -1
- package/dist/core/modules/better-auth/core-better-auth-user.mapper.js +0 -1
- package/dist/core/modules/better-auth/core-better-auth-user.mapper.js.map +1 -1
- package/dist/core/modules/file/core-file.controller.d.ts +15 -3
- package/dist/core/modules/file/core-file.controller.js +90 -20
- package/dist/core/modules/file/core-file.controller.js.map +1 -1
- package/dist/core/modules/file/core-file.resolver.d.ts +4 -4
- package/dist/core/modules/file/core-file.resolver.js +23 -16
- package/dist/core/modules/file/core-file.resolver.js.map +1 -1
- package/dist/core/modules/file/core-file.service.d.ts +41 -7
- package/dist/core/modules/file/core-file.service.js +227 -7
- package/dist/core/modules/file/core-file.service.js.map +1 -1
- package/dist/core/modules/file/file-metadata.helper.d.ts +16 -0
- package/dist/core/modules/file/file-metadata.helper.js +33 -0
- package/dist/core/modules/file/file-metadata.helper.js.map +1 -0
- package/dist/core/modules/file/file-roles.helper.d.ts +4 -0
- package/dist/core/modules/file/file-roles.helper.js +53 -0
- package/dist/core/modules/file/file-roles.helper.js.map +1 -0
- package/dist/core/modules/file/file-storage.helper.d.ts +13 -0
- package/dist/core/modules/file/file-storage.helper.js +65 -0
- package/dist/core/modules/file/file-storage.helper.js.map +1 -0
- package/dist/core/modules/file/filesystem-file.helper.d.ts +22 -0
- package/dist/core/modules/file/filesystem-file.helper.js +77 -0
- package/dist/core/modules/file/filesystem-file.helper.js.map +1 -0
- package/dist/core/modules/file/interfaces/file-service-options.interface.d.ts +1 -0
- package/dist/core/modules/file/interfaces/file-upload.interface.d.ts +7 -1
- package/dist/core/modules/file/s3-file.helper.d.ts +37 -0
- package/dist/core/modules/file/s3-file.helper.js +81 -0
- package/dist/core/modules/file/s3-file.helper.js.map +1 -0
- package/dist/core/modules/hub/core-hub.controller.d.ts +8 -8
- package/dist/core/modules/hub/core-hub.controller.js +24 -22
- package/dist/core/modules/hub/core-hub.controller.js.map +1 -1
- package/dist/core/modules/hub/helpers/hub-mask.helper.js +2 -2
- package/dist/core/modules/hub/helpers/hub-mask.helper.js.map +1 -1
- package/dist/core/modules/hub/hub-buffer.d.ts +23 -0
- package/dist/core/modules/hub/hub-buffer.js +101 -0
- package/dist/core/modules/hub/hub-buffer.js.map +1 -0
- package/dist/core/modules/hub/services/core-hub-mailbox.service.d.ts +7 -5
- package/dist/core/modules/hub/services/core-hub-mailbox.service.js +14 -10
- package/dist/core/modules/hub/services/core-hub-mailbox.service.js.map +1 -1
- package/dist/core/modules/hub/services/hub-log-buffer.service.d.ts +5 -2
- package/dist/core/modules/hub/services/hub-log-buffer.service.js +17 -8
- package/dist/core/modules/hub/services/hub-log-buffer.service.js.map +1 -1
- package/dist/core/modules/hub/services/hub-query-profiler.service.d.ts +4 -2
- package/dist/core/modules/hub/services/hub-query-profiler.service.js +13 -7
- package/dist/core/modules/hub/services/hub-query-profiler.service.js.map +1 -1
- package/dist/core/modules/hub/services/hub-trace-buffer.service.d.ts +4 -2
- package/dist/core/modules/hub/services/hub-trace-buffer.service.js +13 -8
- package/dist/core/modules/hub/services/hub-trace-buffer.service.js.map +1 -1
- package/dist/core/modules/migrate/cli/migrate-cli.d.ts +3 -1
- package/dist/core/modules/migrate/cli/migrate-cli.js +29 -4
- package/dist/core/modules/migrate/cli/migrate-cli.js.map +1 -1
- package/dist/core/modules/migrate/helpers/migration.helper.d.ts +3 -6
- package/dist/core/modules/migrate/helpers/migration.helper.js +55 -8
- package/dist/core/modules/migrate/helpers/migration.helper.js.map +1 -1
- package/dist/core/modules/migrate/migration-runner.d.ts +1 -0
- package/dist/core/modules/migrate/migration-runner.js +4 -0
- package/dist/core/modules/migrate/migration-runner.js.map +1 -1
- package/dist/core/modules/migrate/mongo-state-store.d.ts +1 -0
- package/dist/core/modules/migrate/mongo-state-store.js +98 -9
- package/dist/core/modules/migrate/mongo-state-store.js.map +1 -1
- package/dist/core/modules/system-setup/core-system-setup.service.d.ts +2 -0
- package/dist/core/modules/system-setup/core-system-setup.service.js +31 -1
- package/dist/core/modules/system-setup/core-system-setup.service.js.map +1 -1
- package/dist/core/modules/tenant/core-tenant.guard.d.ts +17 -3
- package/dist/core/modules/tenant/core-tenant.guard.js +92 -15
- package/dist/core/modules/tenant/core-tenant.guard.js.map +1 -1
- package/dist/core/modules/tenant/core-tenant.module.js +3 -4
- package/dist/core/modules/tenant/core-tenant.module.js.map +1 -1
- package/dist/core/modules/tus/core-tus.controller.d.ts +2 -0
- package/dist/core/modules/tus/core-tus.controller.js +29 -3
- package/dist/core/modules/tus/core-tus.controller.js.map +1 -1
- package/dist/core/modules/tus/core-tus.service.d.ts +24 -1
- package/dist/core/modules/tus/core-tus.service.js +182 -28
- package/dist/core/modules/tus/core-tus.service.js.map +1 -1
- package/dist/core/modules/tus/interfaces/tus-config.interface.js +3 -0
- package/dist/core/modules/tus/interfaces/tus-config.interface.js.map +1 -1
- package/dist/core/modules/tus/tus-redis-locker.d.ts +25 -0
- package/dist/core/modules/tus/tus-redis-locker.js +87 -0
- package/dist/core/modules/tus/tus-redis-locker.js.map +1 -0
- package/dist/core/modules/tus/tus.module.d.ts +1 -0
- package/dist/core/modules/tus/tus.module.js +30 -4
- package/dist/core/modules/tus/tus.module.js.map +1 -1
- package/dist/core.module.js +18 -1
- package/dist/core.module.js.map +1 -1
- package/dist/index.d.ts +10 -0
- package/dist/index.js +10 -0
- package/dist/index.js.map +1 -1
- package/dist/main.js +2 -1
- package/dist/main.js.map +1 -1
- package/dist/server/modules/file/file.controller.js.map +1 -1
- package/dist/server/modules/file/file.resolver.js +1 -12
- package/dist/server/modules/file/file.resolver.js.map +1 -1
- package/dist/server/modules/file/file.service.d.ts +5 -1
- package/dist/server/modules/file/file.service.js +12 -3
- package/dist/server/modules/file/file.service.js.map +1 -1
- package/dist/server/modules/user/avatar.controller.d.ts +5 -1
- package/dist/server/modules/user/avatar.controller.js +26 -10
- package/dist/server/modules/user/avatar.controller.js.map +1 -1
- package/dist/server/modules/user/user.model.js +3 -2
- package/dist/server/modules/user/user.model.js.map +1 -1
- package/dist/server/modules/user/user.module.js +6 -2
- package/dist/server/modules/user/user.module.js.map +1 -1
- package/dist/server/modules/user/user.service.d.ts +1 -1
- package/dist/server/modules/user/user.service.js +6 -13
- package/dist/server/modules/user/user.service.js.map +1 -1
- package/dist/tsconfig.build.tsbuildinfo +1 -1
- package/docs/REQUEST-LIFECYCLE.md +57 -8
- package/docs/security-overrides.md +30 -5
- package/migration-guides/11.32.3-to-11.32.4.md +335 -0
- package/migration-guides/11.32.x-to-11.33.x.md +1551 -0
- package/package.json +74 -42
- package/src/core/common/helpers/content-disposition.helper.ts +90 -0
- package/src/core/common/helpers/file.helper.ts +201 -20
- package/src/core/common/helpers/graceful-shutdown.helper.ts +116 -0
- package/src/core/common/helpers/project-name.helper.ts +71 -0
- package/src/core/common/interfaces/cron-job-config.interface.ts +16 -0
- package/src/core/common/interfaces/server-options.interface.ts +439 -2
- package/src/core/common/services/core-cron-jobs.initializer.ts +39 -0
- package/src/core/common/services/core-cron-jobs.registry.ts +45 -0
- package/src/core/common/services/core-cron-jobs.service.ts +589 -47
- package/src/core/common/services/core-redis-pubsub.ts +183 -0
- package/src/core/common/services/core-redis.service.ts +249 -0
- package/src/core/common/services/core-s3.service.ts +465 -0
- package/src/core/common/services/core-trust-proxy.initializer.ts +112 -0
- package/src/core/common/services/rate-limit-store.ts +492 -0
- package/src/core/modules/ai/README.md +25 -0
- package/src/core/modules/ai/core-ai-mcp.controller.ts +253 -22
- package/src/core/modules/ai/services/core-ai.service.ts +33 -24
- package/src/core/modules/auth/core-auth.module.ts +7 -1
- package/src/core/modules/auth/guards/legacy-auth-rate-limit.guard.ts +12 -21
- package/src/core/modules/auth/services/legacy-auth-rate-limiter.service.ts +49 -93
- package/src/core/modules/better-auth/core-better-auth-email-verification.service.ts +202 -66
- package/src/core/modules/better-auth/core-better-auth-rate-limit.middleware.ts +11 -18
- package/src/core/modules/better-auth/core-better-auth-rate-limiter.service.ts +51 -132
- package/src/core/modules/better-auth/core-better-auth-user.mapper.ts +7 -1
- package/src/core/modules/file/INTEGRATION-CHECKLIST.md +108 -0
- package/src/core/modules/file/README.md +324 -41
- package/src/core/modules/file/core-file.controller.ts +267 -19
- package/src/core/modules/file/core-file.resolver.ts +46 -12
- package/src/core/modules/file/core-file.service.ts +482 -16
- package/src/core/modules/file/file-metadata.helper.ts +142 -0
- package/src/core/modules/file/file-roles.helper.ts +110 -0
- package/src/core/modules/file/file-storage.helper.ts +163 -0
- package/src/core/modules/file/filesystem-file.helper.ts +184 -0
- package/src/core/modules/file/interfaces/file-service-options.interface.ts +14 -0
- package/src/core/modules/file/interfaces/file-upload.interface.ts +17 -1
- package/src/core/modules/file/s3-file.helper.ts +209 -0
- package/src/core/modules/hub/README.md +48 -0
- package/src/core/modules/hub/core-hub.controller.ts +18 -16
- package/src/core/modules/hub/helpers/hub-mask.helper.ts +21 -4
- package/src/core/modules/hub/hub-buffer.ts +198 -0
- package/src/core/modules/hub/services/core-hub-mailbox.service.ts +22 -12
- package/src/core/modules/hub/services/hub-log-buffer.service.ts +24 -8
- package/src/core/modules/hub/services/hub-query-profiler.service.ts +9 -6
- package/src/core/modules/hub/services/hub-trace-buffer.service.ts +14 -9
- package/src/core/modules/migrate/README.md +68 -6
- package/src/core/modules/migrate/cli/migrate-cli.ts +69 -6
- package/src/core/modules/migrate/helpers/migration.helper.ts +144 -10
- package/src/core/modules/migrate/migration-runner.ts +10 -1
- package/src/core/modules/migrate/mongo-state-store.ts +179 -11
- package/src/core/modules/system-setup/README.md +7 -2
- package/src/core/modules/system-setup/core-system-setup.service.ts +95 -1
- package/src/core/modules/tenant/README.md +21 -1
- package/src/core/modules/tenant/core-tenant.guard.ts +135 -14
- package/src/core/modules/tenant/core-tenant.module.ts +8 -4
- package/src/core/modules/tus/INTEGRATION-CHECKLIST.md +93 -21
- package/src/core/modules/tus/README.md +35 -2
- package/src/core/modules/tus/core-tus.controller.ts +64 -12
- package/src/core/modules/tus/core-tus.service.ts +358 -34
- package/src/core/modules/tus/interfaces/tus-config.interface.ts +9 -0
- package/src/core/modules/tus/tus-redis-locker.ts +139 -0
- package/src/core/modules/tus/tus.module.ts +78 -6
- package/src/core.module.ts +26 -1
- package/src/index.ts +10 -0
- package/src/main.ts +7 -1
- package/src/server/modules/file/file.controller.ts +7 -3
- package/src/server/modules/file/file.resolver.ts +5 -17
- package/src/server/modules/file/file.service.ts +42 -3
- package/src/server/modules/user/avatar.controller.ts +39 -9
- package/src/server/modules/user/user.model.ts +9 -3
- package/src/server/modules/user/user.module.ts +12 -3
- package/src/server/modules/user/user.service.ts +20 -17
- package/src/test/README.md +7 -2
- package/src/core/common/helpers/logging.helper.spec.ts +0 -61
- package/src/core/common/helpers/process-diagnostics.helper.spec.ts +0 -310
- package/src/core/common/services/brevo.service.spec.ts +0 -266
- package/src/core/modules/hub/core-hub.module.spec.ts +0 -108
- package/src/core/modules/hub/helpers/hub-command-shape.helper.spec.ts +0 -48
- package/src/core/modules/hub/helpers/hub-mask.helper.spec.ts +0 -67
- package/src/core/modules/hub/helpers/hub-mermaid.helper.spec.ts +0 -61
- package/src/core/modules/hub/helpers/hub-shell.helper.spec.ts +0 -106
- package/src/core/modules/hub/hub-config.helper.spec.ts +0 -108
- package/src/core/modules/hub/hub-ring-buffer.spec.ts +0 -95
- package/src/core/modules/hub/services/core-hub-mailbox.service.spec.ts +0 -116
- package/src/core/modules/hub/services/hub-trace-buffer.service.spec.ts +0 -112
|
@@ -7,12 +7,14 @@ import {
|
|
|
7
7
|
LogLevel,
|
|
8
8
|
OnModuleDestroy,
|
|
9
9
|
OnModuleInit,
|
|
10
|
+
Optional,
|
|
10
11
|
} from '@nestjs/common';
|
|
11
12
|
import { inspect } from 'util';
|
|
12
13
|
|
|
13
14
|
import { redactSensitiveText } from '../../../common/helpers/logging.helper';
|
|
15
|
+
import { CoreRedisService } from '../../../common/services/core-redis.service';
|
|
14
16
|
import { HUB_CONFIG } from '../hub.constants';
|
|
15
|
-
import {
|
|
17
|
+
import { HubBuffer } from '../hub-buffer';
|
|
16
18
|
import { HubLogRecord, HubLogsData } from '../interfaces/hub-panels.interface';
|
|
17
19
|
import { ResolvedHubConfig } from '../interfaces/hub-config.interface';
|
|
18
20
|
|
|
@@ -90,22 +92,32 @@ class HubDelegatingLogger implements LoggerService {
|
|
|
90
92
|
*/
|
|
91
93
|
@Injectable()
|
|
92
94
|
export class HubLogBufferService implements OnModuleDestroy, OnModuleInit {
|
|
93
|
-
private buffer?:
|
|
95
|
+
private buffer?: HubBuffer<BufferedLog>;
|
|
94
96
|
private cfg?: Exclude<ResolvedHubConfig['collectors']['logs'], false>;
|
|
95
97
|
private delegating?: HubDelegatingLogger;
|
|
96
98
|
private excludeContexts = new Set<string>();
|
|
99
|
+
|
|
100
|
+
/**
|
|
101
|
+
* Set by onModuleDestroy. The Nest Logger override is PROCESS-GLOBAL, and with a shutdown
|
|
102
|
+
* delay the HTTP server keeps serving while the app drains — so a Hub poll arriving in that
|
|
103
|
+
* window could re-install a logger belonging to an app that no longer exists.
|
|
104
|
+
*/
|
|
105
|
+
private destroyed = false;
|
|
97
106
|
private previousRef?: LoggerService;
|
|
98
107
|
|
|
99
|
-
constructor(
|
|
108
|
+
constructor(
|
|
109
|
+
@Inject(HUB_CONFIG) protected readonly config: ResolvedHubConfig,
|
|
110
|
+
@Optional() protected readonly redis?: CoreRedisService,
|
|
111
|
+
) {}
|
|
100
112
|
|
|
101
113
|
/** Idempotent install of the delegating logger. Public so reads can self-heal after a foreign override. */
|
|
102
114
|
attach(): void {
|
|
103
|
-
if (this.config.collectors.logs === false) {
|
|
115
|
+
if (this.destroyed || this.config.collectors.logs === false) {
|
|
104
116
|
return;
|
|
105
117
|
}
|
|
106
118
|
if (!this.cfg) {
|
|
107
119
|
this.cfg = this.config.collectors.logs;
|
|
108
|
-
this.buffer = new
|
|
120
|
+
this.buffer = new HubBuffer<BufferedLog>(this.cfg.capacity, 'logs', this.redis);
|
|
109
121
|
this.excludeContexts = new Set(this.cfg.excludeContexts);
|
|
110
122
|
}
|
|
111
123
|
const current = this.staticRef();
|
|
@@ -149,13 +161,17 @@ export class HubLogBufferService implements OnModuleDestroy, OnModuleInit {
|
|
|
149
161
|
return this.config.collectors.logs !== false;
|
|
150
162
|
}
|
|
151
163
|
|
|
152
|
-
getData(since?: number): HubLogsData {
|
|
164
|
+
async getData(since?: number): Promise<HubLogsData> {
|
|
153
165
|
this.selfHeal();
|
|
154
|
-
|
|
155
|
-
|
|
166
|
+
if (!this.buffer) {
|
|
167
|
+
return { cursor: -1, dropped: -1, records: [] };
|
|
168
|
+
}
|
|
169
|
+
const { cursor, dropped, entries } = await this.buffer.read(since);
|
|
170
|
+
return { cursor, dropped, records: entries };
|
|
156
171
|
}
|
|
157
172
|
|
|
158
173
|
onModuleDestroy(): void {
|
|
174
|
+
this.destroyed = true;
|
|
159
175
|
this.detach();
|
|
160
176
|
}
|
|
161
177
|
|
|
@@ -3,8 +3,9 @@ import { InjectConnection } from '@nestjs/mongoose';
|
|
|
3
3
|
import { Connection } from 'mongoose';
|
|
4
4
|
|
|
5
5
|
import { redactSensitiveText } from '../../../common/helpers/logging.helper';
|
|
6
|
+
import { CoreRedisService } from '../../../common/services/core-redis.service';
|
|
6
7
|
import { HUB_CONFIG, HUB_QUERY_PENDING_LIMIT } from '../hub.constants';
|
|
7
|
-
import {
|
|
8
|
+
import { HubBuffer } from '../hub-buffer';
|
|
8
9
|
import { normalizeCommandShape } from '../helpers/hub-command-shape.helper';
|
|
9
10
|
import { HubQueriesData, HubQueryRecord, HubQueryTemplate } from '../interfaces/hub-panels.interface';
|
|
10
11
|
import { ResolvedHubConfig } from '../interfaces/hub-config.interface';
|
|
@@ -58,7 +59,7 @@ const SHAPED_FIELDS = ['filter', 'query', 'pipeline', 'sort', 'projection', 'q',
|
|
|
58
59
|
export class HubQueryProfilerService implements OnModuleDestroy, OnModuleInit {
|
|
59
60
|
protected readonly logger = new Logger(HubQueryProfilerService.name);
|
|
60
61
|
|
|
61
|
-
private buffer?:
|
|
62
|
+
private buffer?: HubBuffer<ProfilerRecord>;
|
|
62
63
|
private client?: {
|
|
63
64
|
off?: (e: string, l: (...a: any[]) => void) => void;
|
|
64
65
|
on: (e: string, l: (...a: any[]) => void) => void;
|
|
@@ -78,6 +79,7 @@ export class HubQueryProfilerService implements OnModuleDestroy, OnModuleInit {
|
|
|
78
79
|
constructor(
|
|
79
80
|
@Inject(HUB_CONFIG) protected readonly config: ResolvedHubConfig,
|
|
80
81
|
@Optional() @InjectConnection() protected readonly connection?: Connection,
|
|
82
|
+
@Optional() protected readonly redis?: CoreRedisService,
|
|
81
83
|
) {}
|
|
82
84
|
|
|
83
85
|
/** Clear the buffer (Hub action). */
|
|
@@ -90,10 +92,11 @@ export class HubQueryProfilerService implements OnModuleDestroy, OnModuleInit {
|
|
|
90
92
|
return this.config.collectors.queries !== false;
|
|
91
93
|
}
|
|
92
94
|
|
|
93
|
-
getData(): HubQueriesData {
|
|
94
|
-
const
|
|
95
|
+
async getData(): Promise<HubQueriesData> {
|
|
96
|
+
const data = this.buffer ? await this.buffer.read() : { cursor: -1, entries: [] };
|
|
97
|
+
const records = data.entries;
|
|
95
98
|
return {
|
|
96
|
-
cursor:
|
|
99
|
+
cursor: data.cursor,
|
|
97
100
|
recent: records.slice(-100),
|
|
98
101
|
slowest: [...records].sort((a, b) => b.durationMs - a.durationMs).slice(0, 10),
|
|
99
102
|
summary: this.summary(records),
|
|
@@ -118,7 +121,7 @@ export class HubQueryProfilerService implements OnModuleDestroy, OnModuleInit {
|
|
|
118
121
|
return; // zero cost when disabled
|
|
119
122
|
}
|
|
120
123
|
this.cfg = this.config.collectors.queries;
|
|
121
|
-
this.buffer = new
|
|
124
|
+
this.buffer = new HubBuffer<ProfilerRecord>(this.cfg.capacity, 'queries', this.redis);
|
|
122
125
|
if (this.cfg.ignoreCommands?.length) {
|
|
123
126
|
this.ignore = new Set(this.cfg.ignoreCommands.map((c) => c.toLowerCase()));
|
|
124
127
|
}
|
|
@@ -1,8 +1,9 @@
|
|
|
1
|
-
import { Inject, Injectable } from '@nestjs/common';
|
|
1
|
+
import { Inject, Injectable, Optional } from '@nestjs/common';
|
|
2
2
|
import { Request, Response } from 'express';
|
|
3
3
|
|
|
4
|
+
import { CoreRedisService } from '../../../common/services/core-redis.service';
|
|
4
5
|
import { HUB_CONFIG } from '../hub.constants';
|
|
5
|
-
import {
|
|
6
|
+
import { HubBuffer } from '../hub-buffer';
|
|
6
7
|
import { HubTraceRecord, HubTracesData } from '../interfaces/hub-panels.interface';
|
|
7
8
|
import { ResolvedHubConfig } from '../interfaces/hub-config.interface';
|
|
8
9
|
|
|
@@ -19,14 +20,17 @@ interface TraceRecord extends HubTraceRecord {
|
|
|
19
20
|
*/
|
|
20
21
|
@Injectable()
|
|
21
22
|
export class HubTraceBufferService {
|
|
22
|
-
private readonly buffer?:
|
|
23
|
+
private readonly buffer?: HubBuffer<TraceRecord>;
|
|
23
24
|
private readonly cfg?: Exclude<ResolvedHubConfig['collectors']['traces'], false>;
|
|
24
25
|
private readonly excludePrefixes: string[];
|
|
25
26
|
|
|
26
|
-
constructor(
|
|
27
|
+
constructor(
|
|
28
|
+
@Inject(HUB_CONFIG) protected readonly config: ResolvedHubConfig,
|
|
29
|
+
@Optional() protected readonly redis?: CoreRedisService,
|
|
30
|
+
) {
|
|
27
31
|
if (config.collectors.traces !== false) {
|
|
28
32
|
this.cfg = config.collectors.traces;
|
|
29
|
-
this.buffer = new
|
|
33
|
+
this.buffer = new HubBuffer<TraceRecord>(this.cfg.capacity, 'traces', redis);
|
|
30
34
|
}
|
|
31
35
|
// Always exclude the Hub's own routes so polling does not flood the trace list with self-noise.
|
|
32
36
|
this.excludePrefixes = [...(this.cfg?.excludePaths ?? []), '/' + config.path];
|
|
@@ -40,8 +44,9 @@ export class HubTraceBufferService {
|
|
|
40
44
|
return this.config.collectors.traces !== false;
|
|
41
45
|
}
|
|
42
46
|
|
|
43
|
-
getData(since?: number): HubTracesData {
|
|
44
|
-
const
|
|
47
|
+
async getData(since?: number): Promise<HubTracesData> {
|
|
48
|
+
const data = this.buffer ? await this.buffer.read(since) : { cursor: -1, dropped: -1, entries: [] };
|
|
49
|
+
const records = data.entries;
|
|
45
50
|
let totalMs = 0;
|
|
46
51
|
let slowCount = 0;
|
|
47
52
|
let errorCount = 0;
|
|
@@ -55,8 +60,8 @@ export class HubTraceBufferService {
|
|
|
55
60
|
}
|
|
56
61
|
}
|
|
57
62
|
return {
|
|
58
|
-
cursor:
|
|
59
|
-
dropped:
|
|
63
|
+
cursor: data.cursor,
|
|
64
|
+
dropped: data.dropped,
|
|
60
65
|
summary: { avgMs: records.length ? totalMs / records.length : 0, errorCount, slowCount, total: records.length },
|
|
61
66
|
traces: records,
|
|
62
67
|
};
|
|
@@ -230,7 +230,7 @@ The migration state is stored in a MongoDB collection (default: `migrations`) as
|
|
|
230
230
|
|
|
231
231
|
### Locking Mechanism
|
|
232
232
|
|
|
233
|
-
|
|
233
|
+
How the lock works, regardless of which entry point uses it:
|
|
234
234
|
|
|
235
235
|
1. A unique index is created on the lock collection
|
|
236
236
|
2. The migration process attempts to insert a lock document
|
|
@@ -240,6 +240,27 @@ When using `synchronizedMigration` or `synchronizedUp`:
|
|
|
240
240
|
|
|
241
241
|
This ensures that in a cluster with multiple nodes, migrations run on only one machine at a time.
|
|
242
242
|
|
|
243
|
+
**Active by default for `migrate up`.** Stores built by `createMigrationStore()` use the
|
|
244
|
+
lock collection `migrations_lock` unless another name is given, and `MigrationRunner.up()`
|
|
245
|
+
(the CLI's `up` command) acquires that lock around the whole run. This matters because the
|
|
246
|
+
container entrypoint runs migrations on **every** boot: without the lock, N replicas
|
|
247
|
+
starting together each read the same empty state and apply the same pending migration N
|
|
248
|
+
times. A replica that waited re-reads the state inside the lock and finds nothing pending.
|
|
249
|
+
|
|
250
|
+
With a single replica nothing changes — the lock is acquired and released uncontended.
|
|
251
|
+
|
|
252
|
+
Opt out by passing an empty lock collection name:
|
|
253
|
+
|
|
254
|
+
```javascript
|
|
255
|
+
module.exports = createMigrationStore(uri, 'migrations', ''); // no locking
|
|
256
|
+
```
|
|
257
|
+
|
|
258
|
+
A `MongoStateStore` constructed **directly** still locks only when you pass
|
|
259
|
+
`lockCollectionName` yourself; the default lives in `createMigrationStore()`.
|
|
260
|
+
|
|
261
|
+
The legacy `synchronizedMigration` / `synchronizedUp` helpers use the same lock and still
|
|
262
|
+
require `lockCollectionName` to be set explicitly (they throw otherwise).
|
|
263
|
+
|
|
243
264
|
## Examples
|
|
244
265
|
|
|
245
266
|
### Example: Migration File
|
|
@@ -356,11 +377,17 @@ Factory function to create a migration store class for use with the migrate CLI:
|
|
|
356
377
|
const { createMigrationStore } = require('@lenne.tech/nest-server');
|
|
357
378
|
const config = require('../src/config.env');
|
|
358
379
|
|
|
359
|
-
module.exports = createMigrationStore(
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
380
|
+
module.exports = createMigrationStore(config.default.mongoose.uri);
|
|
381
|
+
```
|
|
382
|
+
|
|
383
|
+
**Locking is on by default.** The third parameter defaults to `'migrations_lock'`, so
|
|
384
|
+
`migrate up` is serialized across replicas without any project change (see
|
|
385
|
+
[Locking Mechanism](#locking-mechanism)):
|
|
386
|
+
|
|
387
|
+
```javascript
|
|
388
|
+
createMigrationStore(uri); // collection 'migrations', lock 'migrations_lock'
|
|
389
|
+
createMigrationStore(uri, 'migrations', 'custom_lock'); // custom lock collection
|
|
390
|
+
createMigrationStore(uri, 'migrations', ''); // opt out: no locking
|
|
364
391
|
```
|
|
365
392
|
|
|
366
393
|
### getDb()
|
|
@@ -387,6 +414,41 @@ const fileId = await uploadFileToGridFS('mongodb://localhost/mydb', '../assets/i
|
|
|
387
414
|
});
|
|
388
415
|
```
|
|
389
416
|
|
|
417
|
+
`relativePath` is resolved against the **helper module's** directory, not the migration file's — the
|
|
418
|
+
paths in this example and in the migration template are written accordingly.
|
|
419
|
+
|
|
420
|
+
The returned promise settles only after three guarantees hold:
|
|
421
|
+
|
|
422
|
+
| Guarantee | Behaviour |
|
|
423
|
+
| ------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
|
424
|
+
| The bytes are stored | Chunk completeness is verified before resolving (see `assertGridFsFileComplete()` below). An incomplete upload **rejects** and the incomplete file is removed, rather than returning an id that points at nothing |
|
|
425
|
+
| An unreadable source fails fast | A missing or unreadable file **rejects** with the underlying `ENOENT`. It used to hang forever, because `pipe()` does not forward read-stream errors |
|
|
426
|
+
| The connection is released | The MongoClient it opens is always closed, on the success and the failure path alike. A leaked connection keeps the Node event loop alive and `migrate up` never exits |
|
|
427
|
+
|
|
428
|
+
Since these are hard failures, a seed migration whose asset is missing or only partially stored now
|
|
429
|
+
fails the migration — and with the default `MIGRATE_FAILURE_POLICY=abort` in `docker-entrypoint.sh`,
|
|
430
|
+
the container refuses to start rather than booting with broken data. That is intentional: the point
|
|
431
|
+
of running migrations before the server is to find exactly this.
|
|
432
|
+
|
|
433
|
+
### assertGridFsFileComplete()
|
|
434
|
+
|
|
435
|
+
Verifies that every chunk of an already-stored GridFS file is present. `uploadFileToGridFS()` calls
|
|
436
|
+
it for you; call it directly to check files written by something else (a restored dump, another
|
|
437
|
+
service, a manual upload):
|
|
438
|
+
|
|
439
|
+
```typescript
|
|
440
|
+
import { assertGridFsFileComplete, getDb } from '@lenne.tech/nest-server';
|
|
441
|
+
|
|
442
|
+
const db = await getDb('mongodb://localhost/mydb');
|
|
443
|
+
await assertGridFsFileComplete(db, 'images', fileId, 'logo.png'); // throws if incomplete
|
|
444
|
+
```
|
|
445
|
+
|
|
446
|
+
Throws when the files document is missing entirely, or when fewer chunks are stored than its
|
|
447
|
+
`length` implies. It counts chunk documents rather than reading bytes back, so a chunk that was
|
|
448
|
+
written but truncated is **not** detected — the check targets the failure that actually occurs, a
|
|
449
|
+
connection lost mid-upload. Empty files are valid and pass: GridFS stores a zero-byte file with no
|
|
450
|
+
chunk documents at all.
|
|
451
|
+
|
|
390
452
|
### Migration Templates
|
|
391
453
|
|
|
392
454
|
Ready-to-use template for nest-server projects:
|
|
@@ -328,13 +328,76 @@ Environment Variables:
|
|
|
328
328
|
`);
|
|
329
329
|
}
|
|
330
330
|
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
331
|
+
/**
|
|
332
|
+
* Exit with `code`, but never before our own output has actually left the process.
|
|
333
|
+
*
|
|
334
|
+
* `process.exit()` does NOT drain stdout, and stdout is asynchronous whenever it
|
|
335
|
+
* is a pipe — which is exactly the Docker / CI / `| tee` case. Exiting straight
|
|
336
|
+
* after the completion log therefore discards the record of what this run did,
|
|
337
|
+
* and the line at risk is the last one ("All migrations completed successfully"),
|
|
338
|
+
* which is precisely what a CI step greps for. Measured: past the 64 KiB pipe
|
|
339
|
+
* buffer everything beyond it is lost.
|
|
340
|
+
*
|
|
341
|
+
* `process.exitCode` is assigned first so that a future `exitCode = 1` elsewhere
|
|
342
|
+
* is honoured rather than overwritten by a hardcoded 0. The timer is the safety
|
|
343
|
+
* net for the opposite failure — a consumer that never reads — and is `unref`'d
|
|
344
|
+
* so it cannot itself keep the process alive.
|
|
345
|
+
*/
|
|
346
|
+
const flushAndExit = async (code: number): Promise<void> => {
|
|
347
|
+
process.exitCode = code;
|
|
348
|
+
|
|
349
|
+
const guard = setTimeout(() => process.exit(code), 5000);
|
|
350
|
+
guard.unref();
|
|
351
|
+
|
|
352
|
+
await Promise.all(
|
|
353
|
+
[process.stdout, process.stderr].map(
|
|
354
|
+
(stream) =>
|
|
355
|
+
new Promise<void>((resolve) => {
|
|
356
|
+
if (!stream.writableLength) {
|
|
357
|
+
resolve();
|
|
358
|
+
return;
|
|
359
|
+
}
|
|
360
|
+
stream.write('', () => resolve());
|
|
361
|
+
}),
|
|
362
|
+
),
|
|
363
|
+
);
|
|
364
|
+
|
|
365
|
+
clearTimeout(guard);
|
|
366
|
+
process.exit(code);
|
|
367
|
+
};
|
|
368
|
+
|
|
369
|
+
/**
|
|
370
|
+
* Run the CLI and terminate the process when it is done.
|
|
371
|
+
*
|
|
372
|
+
* This is the entry point the `migrate` / `nest-migrate` bin uses. It exists
|
|
373
|
+
* separately from `main()` because the shell around it matters: migrations touch
|
|
374
|
+
* MongoDB, GridFS and — via the state store — a second connection, and any handle
|
|
375
|
+
* one of them leaves behind keeps Node alive forever. The CLI then prints
|
|
376
|
+
* "All migrations completed successfully" and simply never returns: a CI job
|
|
377
|
+
* blocks until its timeout, and a container entrypoint that runs migrations
|
|
378
|
+
* before `exec`ing the server never reaches the server at all.
|
|
379
|
+
*
|
|
380
|
+
* Note this must NOT be guarded by `require.main === module`: the shipped
|
|
381
|
+
* `bin/migrate.js` loads this module and calls in, so `require.main` is the shim,
|
|
382
|
+
* never this file. A guard here would make the exit unreachable on every path
|
|
383
|
+
* that actually ships.
|
|
384
|
+
*/
|
|
385
|
+
const runCli = async (): Promise<void> => {
|
|
386
|
+
try {
|
|
387
|
+
await main();
|
|
388
|
+
} catch (error) {
|
|
334
389
|
console.error('Fatal error:', error);
|
|
335
|
-
|
|
336
|
-
|
|
390
|
+
await flushAndExit(1);
|
|
391
|
+
return;
|
|
392
|
+
}
|
|
393
|
+
await flushAndExit(0);
|
|
394
|
+
};
|
|
395
|
+
|
|
396
|
+
// Run CLI if executed directly (`node migrate-cli.js`). The bin shim calls
|
|
397
|
+
// runCli() itself, so this only covers a direct invocation of the compiled file.
|
|
398
|
+
if (require.main === module) {
|
|
399
|
+
void runCli();
|
|
337
400
|
}
|
|
338
401
|
|
|
339
402
|
// parseArgs is exported for unit testing only (same pattern as resolveCliPath in bin/migrate.js)
|
|
340
|
-
export { main, parseArgs };
|
|
403
|
+
export { flushAndExit, main, parseArgs, runCli };
|
|
@@ -2,6 +2,8 @@ import * as fs from 'fs';
|
|
|
2
2
|
import { Db, GridFSBucket, MongoClient, ObjectId } from 'mongodb';
|
|
3
3
|
import * as path from 'path';
|
|
4
4
|
|
|
5
|
+
import { MongoStateStore } from '../mongo-state-store';
|
|
6
|
+
|
|
5
7
|
/**
|
|
6
8
|
* Migration helper functions for database operations
|
|
7
9
|
*/
|
|
@@ -67,13 +69,76 @@ export const getDb = async (mongoUrl: string): Promise<Db> => {
|
|
|
67
69
|
return client.db();
|
|
68
70
|
};
|
|
69
71
|
|
|
72
|
+
/**
|
|
73
|
+
* Throw unless every chunk of a stored GridFS file is present.
|
|
74
|
+
*
|
|
75
|
+
* A GridFS upload is not one write but many: N chunk documents plus the files
|
|
76
|
+
* document that describes them. The write stream's `'finish'` event says the
|
|
77
|
+
* stream ended — it does not prove every chunk is durably there, and a
|
|
78
|
+
* connection that goes away at the wrong moment can leave a files document
|
|
79
|
+
* behind that promises more bytes than exist. The upload then "succeeds", the
|
|
80
|
+
* caller stores the id, and the defect only surfaces much later as a broken
|
|
81
|
+
* download from a record that looks perfectly healthy.
|
|
82
|
+
*
|
|
83
|
+
* Counts documents rather than reading bytes, so a chunk that was written but
|
|
84
|
+
* truncated still passes. Catching that would mean streaming the whole file
|
|
85
|
+
* back on every upload; the cheap count catches the failure that actually
|
|
86
|
+
* occurs (a missing chunk) and is index-only via the GridFS default index.
|
|
87
|
+
*
|
|
88
|
+
* @param db - Database holding the bucket
|
|
89
|
+
* @param bucketName - GridFS bucket name, i.e. the prefix of `<bucket>.files` / `<bucket>.chunks`
|
|
90
|
+
* @param id - `_id` of the files document
|
|
91
|
+
* @param label - Optional human-readable name for the error message; defaults to the id
|
|
92
|
+
* @throws Error if the files document is missing or fewer chunks are stored than its length implies
|
|
93
|
+
*
|
|
94
|
+
* @example
|
|
95
|
+
* ```typescript
|
|
96
|
+
* await assertGridFsFileComplete(db, 'images', fileId, 'logo.png');
|
|
97
|
+
* ```
|
|
98
|
+
*/
|
|
99
|
+
export const assertGridFsFileComplete = async (
|
|
100
|
+
db: Db,
|
|
101
|
+
bucketName: string,
|
|
102
|
+
id: ObjectId,
|
|
103
|
+
label?: string,
|
|
104
|
+
): Promise<void> => {
|
|
105
|
+
const name = label || String(id);
|
|
106
|
+
const fileDoc = await db.collection(`${bucketName}.files`).findOne({ _id: id });
|
|
107
|
+
if (!fileDoc) {
|
|
108
|
+
throw new Error(`GridFS file '${name}' has no file document (id ${String(id)})`);
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
const chunkSize: number = fileDoc.chunkSize || 255 * 1024;
|
|
112
|
+
// NO `Math.max(1, …)` floor here: GridFS stores ZERO chunk documents for a
|
|
113
|
+
// zero-byte file — the driver's `writeRemnant()` returns early on `pos === 0`
|
|
114
|
+
// rather than inserting an empty chunk. A floor of 1 would reject every
|
|
115
|
+
// legitimately empty asset, and because the container entrypoint defaults to
|
|
116
|
+
// `MIGRATE_FAILURE_POLICY=abort`, that failure would keep the server from
|
|
117
|
+
// starting at all.
|
|
118
|
+
const expected = Math.ceil((fileDoc.length || 0) / chunkSize);
|
|
119
|
+
const actual = await db
|
|
120
|
+
.collection(`${bucketName}.chunks`)
|
|
121
|
+
// Read from the primary: this is a read-your-own-write, and a URI carrying
|
|
122
|
+
// `readPreference=secondaryPreferred` would otherwise fail a healthy upload
|
|
123
|
+
// against a secondary that has not caught up yet.
|
|
124
|
+
.countDocuments({ files_id: id }, { readPreference: 'primary' });
|
|
125
|
+
if (actual < expected) {
|
|
126
|
+
throw new Error(`GridFS file '${name}' is incomplete: ${actual} of ${expected} chunks stored (id ${String(id)})`);
|
|
127
|
+
}
|
|
128
|
+
};
|
|
129
|
+
|
|
70
130
|
/**
|
|
71
131
|
* Upload file to GridFS
|
|
72
132
|
*
|
|
133
|
+
* Resolves only once the upload is verified complete (see
|
|
134
|
+
* {@link assertGridFsFileComplete}); rejects — rather than hanging — when the
|
|
135
|
+
* source file cannot be read. Closes its own connection either way.
|
|
136
|
+
*
|
|
73
137
|
* @param mongoUrl - MongoDB connection URI
|
|
74
|
-
* @param relativePath -
|
|
138
|
+
* @param relativePath - Path to the file, resolved against this module's directory
|
|
75
139
|
* @param options - Optional bucket name and filename
|
|
76
140
|
* @returns Promise with ObjectId of uploaded file
|
|
141
|
+
* @throws Error if the source cannot be read, or if the stored file is incomplete
|
|
77
142
|
*
|
|
78
143
|
* @example
|
|
79
144
|
* ```typescript
|
|
@@ -100,19 +165,83 @@ export const uploadFileToGridFS = async (
|
|
|
100
165
|
};
|
|
101
166
|
|
|
102
167
|
const client = await MongoClient.connect(mongoUrl);
|
|
168
|
+
// Registered unconditionally — unlike `getDb()`, which only registers inside a
|
|
169
|
+
// migration context. This client is always ours to close, so `_endMigration()`
|
|
170
|
+
// stays a backstop for the case where the upload throws before `settle()` runs.
|
|
171
|
+
activeConnections.add(client);
|
|
172
|
+
|
|
103
173
|
const db = client.db();
|
|
104
174
|
const bucket = new GridFSBucket(db, { bucketName });
|
|
105
175
|
const writeStream = bucket.openUploadStream(filename);
|
|
106
176
|
|
|
107
|
-
const
|
|
177
|
+
const readStream = fs.createReadStream(path.resolve(__dirname, relativePath));
|
|
178
|
+
const rs = readStream.pipe(writeStream);
|
|
179
|
+
|
|
180
|
+
/**
|
|
181
|
+
* Read errors need their own handler — `pipe()` does not forward them.
|
|
182
|
+
*
|
|
183
|
+
* An unreadable source (missing file, wrong path inside a container image)
|
|
184
|
+
* emits on the READ stream, where nothing was listening: the write stream
|
|
185
|
+
* never finished, the promise below never settled, and the migration hung
|
|
186
|
+
* until something else timed out. Destroying the write stream routes it into
|
|
187
|
+
* the rejection path so the caller sees the actual cause.
|
|
188
|
+
*/
|
|
189
|
+
readStream.on('error', (err) => {
|
|
190
|
+
writeStream.destroy(err);
|
|
191
|
+
});
|
|
192
|
+
|
|
193
|
+
/**
|
|
194
|
+
* Close the connection before settling.
|
|
195
|
+
*
|
|
196
|
+
* This client used to be opened and never closed, and it was not registered
|
|
197
|
+
* either — so `_endMigration()` could not reach it. Every uploaded file left a
|
|
198
|
+
* live connection behind, and a live connection keeps an SDAM monitor timer
|
|
199
|
+
* alive, which keeps the Node event loop busy: `migrate up` finished its work,
|
|
200
|
+
* printed "All migrations completed successfully" and then never exited. That
|
|
201
|
+
* is invisible on a developer machine and blocks a CI job until its timeout.
|
|
202
|
+
*
|
|
203
|
+
* A failing `close()` must never become the error the caller sees: the reason
|
|
204
|
+
* the upload ended is what matters, and the settle path is also reached while
|
|
205
|
+
* rejecting. So the close error is logged and swallowed, and `settled` keeps a
|
|
206
|
+
* second call (success path falling into `.catch`) from closing twice.
|
|
207
|
+
*/
|
|
208
|
+
let settled = false;
|
|
209
|
+
const settle = async <T>(action: () => T): Promise<T> => {
|
|
210
|
+
if (!settled) {
|
|
211
|
+
settled = true;
|
|
212
|
+
try {
|
|
213
|
+
await client.close();
|
|
214
|
+
} catch (closeErr) {
|
|
215
|
+
console.warn('Failed to close migration connection:', closeErr);
|
|
216
|
+
} finally {
|
|
217
|
+
activeConnections.delete(client);
|
|
218
|
+
}
|
|
219
|
+
}
|
|
220
|
+
return action();
|
|
221
|
+
};
|
|
108
222
|
|
|
109
223
|
return new Promise<ObjectId>((resolve, reject) => {
|
|
110
224
|
rs.on('finish', () => {
|
|
111
|
-
|
|
225
|
+
const id = writeStream.id as ObjectId;
|
|
226
|
+
// Verify BEFORE closing: the check needs the same open client, and a
|
|
227
|
+
// failure has to reject rather than hand back an id that points at nothing.
|
|
228
|
+
assertGridFsFileComplete(db, bucketName, id, filename)
|
|
229
|
+
.then(() => settle(() => id))
|
|
230
|
+
.then(resolve)
|
|
231
|
+
.catch((err: unknown) => {
|
|
232
|
+
// Drop the incomplete file, otherwise every re-run of the migration
|
|
233
|
+
// leaves another orphaned files document (plus its partial chunks)
|
|
234
|
+
// behind, since a retry uploads under a fresh ObjectId.
|
|
235
|
+
bucket
|
|
236
|
+
.delete(id)
|
|
237
|
+
.catch(() => undefined)
|
|
238
|
+
.then(() => settle(() => undefined))
|
|
239
|
+
.finally(() => reject(err));
|
|
240
|
+
});
|
|
112
241
|
});
|
|
113
242
|
|
|
114
243
|
rs.on('error', (err) => {
|
|
115
|
-
reject(err);
|
|
244
|
+
settle(() => undefined).finally(() => reject(err));
|
|
116
245
|
});
|
|
117
246
|
});
|
|
118
247
|
};
|
|
@@ -120,9 +249,13 @@ export const uploadFileToGridFS = async (
|
|
|
120
249
|
/**
|
|
121
250
|
* Create a migration state store factory
|
|
122
251
|
*
|
|
252
|
+
* The lock collection is set by DEFAULT, so `migrate up` from N replicas booting at
|
|
253
|
+
* the same time serializes instead of applying the same migration twice (the container
|
|
254
|
+
* entrypoint runs migrations on every boot). Pass an empty string to opt out.
|
|
255
|
+
*
|
|
123
256
|
* @param mongoUrl - MongoDB connection URI
|
|
124
257
|
* @param collectionName - Optional collection name (default: 'migrations')
|
|
125
|
-
* @param lockCollectionName -
|
|
258
|
+
* @param lockCollectionName - Lock collection name (default: 'migrations_lock'); `''` disables locking
|
|
126
259
|
* @returns MongoStateStore class that can be used with migrate CLI
|
|
127
260
|
*
|
|
128
261
|
* @example
|
|
@@ -137,15 +270,16 @@ export const uploadFileToGridFS = async (
|
|
|
137
270
|
export const createMigrationStore = (
|
|
138
271
|
mongoUrl: string,
|
|
139
272
|
collectionName: string = 'migrations',
|
|
140
|
-
lockCollectionName
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
273
|
+
lockCollectionName: string = 'migrations_lock',
|
|
274
|
+
// Explicit return type: the returned class is anonymous to the declaration emitter, and
|
|
275
|
+
// MongoStateStore has private members, so an inferred type cannot be written to the .d.ts
|
|
276
|
+
// (TS4094).
|
|
277
|
+
): new () => MongoStateStore => {
|
|
144
278
|
return class MigrationStateStore extends MongoStateStore {
|
|
145
279
|
constructor() {
|
|
146
280
|
super({
|
|
147
281
|
collectionName,
|
|
148
|
-
lockCollectionName,
|
|
282
|
+
lockCollectionName: lockCollectionName || undefined,
|
|
149
283
|
uri: mongoUrl,
|
|
150
284
|
});
|
|
151
285
|
}
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import * as fs from 'fs';
|
|
2
2
|
import * as path from 'path';
|
|
3
3
|
|
|
4
|
-
import { MongoStateStore } from './mongo-state-store';
|
|
4
|
+
import { MongoStateStore, withMigrationLock } from './mongo-state-store';
|
|
5
5
|
|
|
6
6
|
/**
|
|
7
7
|
* Migration file interface
|
|
@@ -221,8 +221,17 @@ export class MigrationRunner {
|
|
|
221
221
|
|
|
222
222
|
/**
|
|
223
223
|
* Run all pending migrations (up)
|
|
224
|
+
*
|
|
225
|
+
* Serialized via the state store's lock collection when it has one (the default for
|
|
226
|
+
* stores built by `createMigrationStore()`). Reading the state INSIDE the lock is what
|
|
227
|
+
* makes it safe: a replica that waited sees the migrations the holder just applied and
|
|
228
|
+
* finds nothing pending, instead of applying them a second time.
|
|
224
229
|
*/
|
|
225
230
|
async up(): Promise<void> {
|
|
231
|
+
await withMigrationLock(this.options.stateStore, () => this.runUp());
|
|
232
|
+
}
|
|
233
|
+
|
|
234
|
+
protected async runUp(): Promise<void> {
|
|
226
235
|
const { _endMigration, _startMigration } = await import('./helpers/migration.helper');
|
|
227
236
|
|
|
228
237
|
const allMigrations = await this.loadMigrationFiles();
|