@fastybird/smart-panel-backend 1.1.0-alpha.63 → 1.1.0-alpha.65

package/dist/modules/extensions/services/managed-service-manager.service.d.ts

@@ -4,6 +4,9 @@ import { ConfigService } from '../../config/services/config.service';
 import { PluginConfigValidatorService } from '../../config/services/plugin-config-validator.service';
 import { NotificationsService } from '../../notifications/services/notifications.service';
 import { IManagedExtensionService, ManagedServiceOwnerKind, ServiceStatusExtended } from './managed-extension-service.interface';
+export interface ManagedRuntimeAdmission {
+    isCurrent(): boolean;
+}
 export declare class ManagedServiceManagerService implements OnApplicationBootstrap, OnModuleDestroy {
     private readonly configService;
     private readonly nestConfigService;
@@ -14,6 +17,11 @@ export declare class ManagedServiceManagerService implements OnApplicationBootst
     private readonly runtimeInfo;
     private readonly readinessRetries;
     private readonly serviceStartPromises;
+    private readonly lifecycleOwners;
+    private readonly serviceStopPromises;
+    private readonly issueWrites;
+    private readonly startAttempts;
+    private readonly startContext;
     private readonly openIssues;
     private readonly isCliMode;
     private startupComplete;
@@ -37,9 +45,15 @@ export declare class ManagedServiceManagerService implements OnApplicationBootst
     getRegisteredServices(): string[];
     private getServiceKey;
     private startServiceIfDesired;
+    captureCurrentStartAdmission(service: IManagedExtensionService): ManagedRuntimeAdmission | null;
+    private isCurrentRegistration;
+    private hasRetiredLifecycle;
+    private revokeAdmission;
     private startService;
     private startServiceOnce;
     private stopService;
+    private stopServiceOnce;
+    private stopServiceForConfig;
     private syncServiceState;
     private scheduleReadinessRetry;
     private clearReadinessRetryTimer;
@@ -54,4 +68,5 @@ export declare class ManagedServiceManagerService implements OnApplicationBootst
     private isConfigChangeResult;
     private raiseServiceError;
     private resolveServiceError;
+    private enqueueIssueWrite;
 }

package/dist/modules/extensions/services/managed-service-manager.service.js

@@ -10,6 +10,7 @@ var __metadata = (this && this.__metadata) || function (k, v) {
 };
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.ManagedServiceManagerService = void 0;
+const node_async_hooks_1 = require("node:async_hooks");
 const common_1 = require("@nestjs/common");
 const config_1 = require("@nestjs/config");
 const event_emitter_1 = require("@nestjs/event-emitter");
@@ -24,6 +25,7 @@ const notifications_service_1 = require("../../notifications/services/notificati
 const extensions_constants_1 = require("../extensions.constants");
 const READINESS_RETRY_INITIAL_DELAY_MS = 15_000;
 const READINESS_RETRY_MAX_ATTEMPTS = 3;
+const CONFIG_FORCE_STOP_TIMEOUT_MS = 10_000;
 let ManagedServiceManagerService = class ManagedServiceManagerService {
     constructor(configService, nestConfigService, pluginConfigValidator, notifications) {
         this.configService = configService;
@@ -34,8 +36,13 @@ let ManagedServiceManagerService = class ManagedServiceManagerService {
         this.services = new Map();
         this.runtimeInfo = new Map();
         this.readinessRetries = new Map();
-        this.serviceStartPromises = new Map();
-        this.openIssues = new Set();
+        this.serviceStartPromises = new WeakMap();
+        this.lifecycleOwners = new WeakMap();
+        this.serviceStopPromises = new WeakMap();
+        this.issueWrites = new Map();
+        this.startAttempts = new WeakMap();
+        this.startContext = new node_async_hooks_1.AsyncLocalStorage();
+        this.openIssues = new Map();
         this.startupComplete = false;
         this.shutdownInProgress = false;
         this.isCliMode = (0, config_utils_1.getEnvValue)(this.nestConfigService, 'FB_CLI', null) === 'on';
@@ -70,9 +77,11 @@ let ManagedServiceManagerService = class ManagedServiceManagerService {
         if (!this.services.has(key)) {
             return;
         }
+        const registration = this.services.get(key);
+        this.revokeAdmission(registration);
         this.services.delete(key);
         this.runtimeInfo.delete(key);
-        this.openIssues.delete(key);
+        void this.resolveServiceError(registration);
         this.clearReadinessRetry(key);
         this.logger.log(`Unregistered service: ${key}`);
     }
@@ -100,6 +109,8 @@ let ManagedServiceManagerService = class ManagedServiceManagerService {
         this.clearAllReadinessRetries();
         this.logger.log('Stopping all managed services for factory reset');
         const sorted = this.getSortedServices().reverse();
+        for (const registration of sorted)
+            this.revokeAdmission(registration);
         for (const registration of sorted) {
             const state = registration.service.getState();
             if (state === 'stopped')
@@ -121,6 +132,8 @@ let ManagedServiceManagerService = class ManagedServiceManagerService {
         this.clearAllReadinessRetries();
         this.logger.log('Stopping all managed services');
         const sorted = this.getSortedServices().reverse();
+        for (const registration of sorted)
+            this.revokeAdmission(registration);
         for (const registration of sorted) {
             const state = registration.service.getState();
             if (state === 'stopped') {
@@ -141,11 +154,13 @@ let ManagedServiceManagerService = class ManagedServiceManagerService {
         if (this.shutdownInProgress || !this.startupComplete) {
             return;
         }
-        for (const registration of this.services.values()) {
-            if ((payload?.type === 'plugin' || payload?.type === 'module') &&
-                (registration.owner.kind !== payload.type || registration.owner.type !== payload.source)) {
-                continue;
-            }
+        const registrations = Array.from(this.services.values()).filter((registration) => (payload?.type !== 'plugin' && payload?.type !== 'module') ||
+            (registration.owner.kind === payload.type && registration.owner.type === payload.source));
+        for (const registration of registrations) {
+            if (!this.shouldBeRunning(registration))
+                this.revokeAdmission(registration);
+        }
+        for (const registration of registrations) {
             await this.syncServiceState(registration);
         }
     }
@@ -244,11 +259,13 @@ let ManagedServiceManagerService = class ManagedServiceManagerService {
             return false;
         }
         await this.stopService(registration);
-        if (registration.service.getState() !== 'stopped') {
+        if (!this.isCurrentRegistration(registration) || registration.service.getState() !== 'stopped') {
             this.logger.warn(`Cannot restart service ${key}: stop did not reach the stopped state`);
             return false;
         }
         await this.startService(registration);
+        if (!this.isCurrentRegistration(registration))
+            return false;
         const resultingState = registration.service.getState();
         return resultingState === 'started' || resultingState === 'starting';
     }
@@ -264,11 +281,13 @@ let ManagedServiceManagerService = class ManagedServiceManagerService {
             return false;
         }
         const currentState = registration.service.getState();
-        if (currentState !== 'stopped' && currentState !== 'error') {
+        if (currentState !== 'stopped' && currentState !== 'error' && !this.hasRetiredLifecycle(registration)) {
             this.logger.warn(`Cannot start service ${key} from state ${currentState}`);
             return false;
         }
         await this.startService(registration);
+        if (!this.isCurrentRegistration(registration) || this.hasRetiredLifecycle(registration))
+            return false;
         const resultingState = registration.service.getState();
         return resultingState === 'started' || resultingState === 'starting';
     }
@@ -279,6 +298,7 @@ let ManagedServiceManagerService = class ManagedServiceManagerService {
             this.logger.warn(`Service not found: ${key}`);
             return false;
         }
+        this.revokeAdmission(registration);
         this.clearReadinessRetry(key);
         const currentState = registration.service.getState();
         if (currentState !== 'started' && currentState !== 'starting' && currentState !== 'error') {
@@ -286,7 +306,7 @@ let ManagedServiceManagerService = class ManagedServiceManagerService {
             return false;
         }
         await this.stopService(registration);
-        return registration.service.getState() === 'stopped';
+        return this.isCurrentRegistration(registration) && registration.service.getState() === 'stopped';
     }
     isRegistered(extensionKind, extensionType, serviceId) {
         return this.services.has(this.getServiceKey(extensionKind, extensionType, serviceId));
@@ -303,22 +323,92 @@ let ManagedServiceManagerService = class ManagedServiceManagerService {
         }
         await this.startService(registration);
     }
+    captureCurrentStartAdmission(service) {
+        const attempt = this.startContext.getStore();
+        return attempt?.active && attempt.registration.service === service && attempt.runtime.isCurrent()
+            ? attempt.runtime
+            : null;
+    }
+    isCurrentRegistration(registration) {
+        return this.services.get(this.getRegistrationKey(registration)) === registration;
+    }
+    hasRetiredLifecycle(registration) {
+        const owner = this.lifecycleOwners.get(registration.service);
+        return owner !== undefined && owner !== registration;
+    }
+    revokeAdmission(registration) {
+        this.startAttempts.delete(registration);
+        try {
+            registration.service.revokeCurrentAuthority?.();
+        }
+        catch (error) {
+            this.logger.error(`Failed to revoke service authority: ${this.getRegistrationKey(registration)}`, error instanceof Error ? error.stack : undefined);
+        }
+    }
     async startService(registration, requireDesiredState = true) {
-        const key = this.getRegistrationKey(registration);
-        const pendingStart = this.serviceStartPromises.get(key);
-        if (pendingStart !== undefined) {
-            await pendingStart;
+        if (!this.isCurrentRegistration(registration) ||
+            this.shutdownInProgress ||
+            (requireDesiredState && !this.shouldBeRunning(registration)))
+            return;
+        const pendingStop = this.serviceStopPromises.get(registration.service);
+        if (pendingStop !== undefined) {
+            await pendingStop;
+            await this.startService(registration, requireDesiredState);
             return;
         }
+        const pending = this.serviceStartPromises.get(registration.service);
+        if (pending) {
+            const current = this.startAttempts.get(registration) === pending.attempt;
+            await pending.promise;
+            if (current || !this.isCurrentRegistration(registration) || this.shutdownInProgress)
+                return;
+            await this.startService(registration, requireDesiredState);
+            return;
+        }
+        const state = registration.service.getState();
+        if (this.hasRetiredLifecycle(registration) && state !== 'stopped') {
+            await this.stopService(registration, true);
+            if (registration.service.getState() !== 'stopped')
+                return;
+            await this.startService(registration, requireDesiredState);
+            return;
+        }
+        if (state === 'started' || state === 'starting')
+            return;
+        const key = this.getRegistrationKey(registration);
         this.clearReadinessRetryTimer(key);
-        const startPromise = this.startServiceOnce(registration, key, requireDesiredState).finally(() => {
-            if (this.serviceStartPromises.get(key) === startPromise)
-                this.serviceStartPromises.delete(key);
-        });
-        this.serviceStartPromises.set(key, startPromise);
-        await startPromise;
+        const attempt = {
+            registration,
+            active: true,
+            invoked: false,
+            failed: false,
+            runtime: Object.freeze({
+                isCurrent: () => this.isCurrentRegistration(registration) &&
+                    this.startAttempts.get(registration) === attempt &&
+                    !this.shutdownInProgress &&
+                    !attempt.failed &&
+                    attempt.invoked &&
+                    (!requireDesiredState || this.shouldBeRunning(registration)),
+            }),
+        };
+        this.startAttempts.set(registration, attempt);
+        const entry = {
+            attempt,
+            promise: Promise.resolve()
+                .then(() => this.startServiceOnce(registration, key, requireDesiredState, attempt))
+                .finally(() => {
+                attempt.active = false;
+                if (this.serviceStartPromises.get(registration.service) === entry)
+                    this.serviceStartPromises.delete(registration.service);
+            }),
+        };
+        this.serviceStartPromises.set(registration.service, entry);
+        await entry.promise;
     }
-    async startServiceOnce(registration, key, requireDesiredState) {
+    async startServiceOnce(registration, key, requireDesiredState, attempt) {
+        const isCurrent = () => this.isCurrentRegistration(registration) && this.startAttempts.get(registration) === attempt;
+        if (!isCurrent())
+            return;
         const currentState = registration.service.getState();
         if (currentState === 'started' || currentState === 'starting') {
             return;
@@ -334,6 +424,8 @@ let ManagedServiceManagerService = class ManagedServiceManagerService {
                     validationResult = await this.pluginConfigValidator.validate(registration.owner.type, config);
                 }
                 catch {
+                    if (!isCurrent())
+                        return;
                     const warning = 'Configuration readiness check is temporarily unavailable';
                     (0, extension_logger_service_1.createExtensionLogger)(registration.owner.type, 'ManagedServiceManagerService').warn(`Service ${registration.serviceId} is enabled but not started because its ${warning.toLowerCase()}.`);
                     const runtime = this.runtimeInfo.get(key);
@@ -342,6 +434,8 @@ let ManagedServiceManagerService = class ManagedServiceManagerService {
                     await this.scheduleReadinessRetry(registration);
                     return;
                 }
+                if (!isCurrent())
+                    return;
                 if (!validationResult.valid && validationResult.transient) {
                     const warning = 'Configuration readiness check is temporarily unavailable';
                     (0, extension_logger_service_1.createExtensionLogger)(registration.owner.type, 'ManagedServiceManagerService').warn(`Service ${registration.serviceId} is enabled but not started because its ${warning.toLowerCase()}.`);
@@ -363,12 +457,18 @@ let ManagedServiceManagerService = class ManagedServiceManagerService {
                 }
             }
         }
+        if (!isCurrent())
+            return;
         if (requireDesiredState && (this.shutdownInProgress || !this.shouldBeRunning(registration))) {
             this.clearReadinessRetry(key);
             return;
         }
         try {
-            await registration.service.start();
+            attempt.invoked = true;
+            this.lifecycleOwners.set(registration.service, registration);
+            await this.startContext.run(attempt, () => registration.service.start());
+            if (!isCurrent())
+                return;
             const runtime = this.runtimeInfo.get(key);
             if (runtime) {
                 runtime.lastStartedAt = new Date();
@@ -380,6 +480,9 @@ let ManagedServiceManagerService = class ManagedServiceManagerService {
             await this.resolveServiceError(registration);
         }
         catch (error) {
+            attempt.failed = true;
+            if (!isCurrent())
+                return;
             const err = error;
             const runtime = this.runtimeInfo.get(key);
             const enteringError = runtime !== undefined && !this.openIssues.has(key);
@@ -388,20 +491,48 @@ let ManagedServiceManagerService = class ManagedServiceManagerService {
             }
             this.logger.error(`Failed to start service ${key}: ${err.message}`, err.stack);
             if (enteringError) {
-                this.openIssues.add(key);
+                this.openIssues.set(key, registration);
                 await this.raiseServiceError(registration, err.message);
             }
         }
     }
-    async stopService(registration) {
+    async stopService(registration, force = false) {
+        if (!this.isCurrentRegistration(registration))
+            return;
+        this.revokeAdmission(registration);
+        const pending = this.serviceStopPromises.get(registration.service);
+        if (pending !== undefined) {
+            await pending;
+            return;
+        }
+        const stopping = Promise.resolve()
+            .then(() => this.stopServiceOnce(registration, force))
+            .finally(() => {
+            if (this.serviceStopPromises.get(registration.service) === stopping)
+                this.serviceStopPromises.delete(registration.service);
+        });
+        this.serviceStopPromises.set(registration.service, stopping);
+        await stopping;
+    }
+    async stopServiceOnce(registration, force) {
         const key = this.getRegistrationKey(registration);
+        const owner = this.lifecycleOwners.get(registration.service);
+        const releaseStoppedOwner = () => {
+            if (registration.service.getState() === 'stopped' && this.lifecycleOwners.get(registration.service) === owner) {
+                this.lifecycleOwners.delete(registration.service);
+            }
+        };
         const currentState = registration.service.getState();
-        if (currentState === 'stopped' || currentState === 'stopping') {
+        if (currentState === 'stopped' || (!force && currentState === 'stopping')) {
+            releaseStoppedOwner();
             return;
         }
         this.logger.log(`Stopping service: ${key}`);
         try {
             await registration.service.stop();
+            releaseStoppedOwner();
+            if (!this.isCurrentRegistration(registration))
+                return;
             const runtime = this.runtimeInfo.get(key);
             if (runtime) {
                 runtime.lastStoppedAt = new Date();
@@ -409,6 +540,8 @@ let ManagedServiceManagerService = class ManagedServiceManagerService {
             this.logger.log(`Service stopped successfully: ${key}`);
         }
         catch (error) {
+            if (!this.isCurrentRegistration(registration))
+                return;
             const err = error;
             const runtime = this.runtimeInfo.get(key);
             if (runtime) {
@@ -417,35 +550,51 @@ let ManagedServiceManagerService = class ManagedServiceManagerService {
             this.logger.error(`Failed to stop service ${key}: ${err.message}`, err.stack);
         }
     }
+    async stopServiceForConfig(registration) {
+        let timer;
+        try {
+            await Promise.race([
+                this.stopService(registration, true),
+                new Promise((resolve) => {
+                    timer = setTimeout(() => {
+                        this.logger.warn(`Timeout waiting for forced config stop of ${this.getRegistrationKey(registration)}; ` +
+                            `service remains '${registration.service.getState()}', stop still pending`);
+                        resolve();
+                    }, CONFIG_FORCE_STOP_TIMEOUT_MS);
+                }),
+            ]);
+        }
+        finally {
+            if (timer !== undefined)
+                clearTimeout(timer);
+        }
+    }
     async syncServiceState(registration) {
+        if (!this.isCurrentRegistration(registration))
+            return;
         const key = this.getRegistrationKey(registration);
         let currentState = registration.service.getState();
         const shouldBeRunning = this.shouldBeRunning(registration);
-        if (!shouldBeRunning)
+        if (!shouldBeRunning) {
+            this.revokeAdmission(registration);
             this.clearReadinessRetry(key);
+        }
+        if (shouldBeRunning && this.hasRetiredLifecycle(registration)) {
+            await this.startService(registration);
+            return;
+        }
+        const attempt = this.startAttempts.get(registration);
+        const isCurrent = () => this.isCurrentRegistration(registration) && this.startAttempts.get(registration) === attempt;
         if (currentState === 'starting' || currentState === 'stopping') {
             const targetState = currentState === 'starting' ? 'started' : 'stopped';
             await this.waitForState(registration, targetState);
+            if (!isCurrent())
+                return;
             currentState = registration.service.getState();
         }
         if (!shouldBeRunning && (currentState === 'starting' || currentState === 'stopping')) {
             this.logger.log(`Service ${key} should be stopped while still in '${currentState}', forcing stop`);
-            try {
-                await registration.service.stop();
-                const runtime = this.runtimeInfo.get(key);
-                if (runtime) {
-                    runtime.lastStoppedAt = new Date();
-                }
-                this.logger.log(`Service force-stopped successfully: ${key}`);
-            }
-            catch (error) {
-                const err = error;
-                const runtime = this.runtimeInfo.get(key);
-                if (runtime) {
-                    runtime.lastError = err.message;
-                }
-                this.logger.error(`Failed to force-stop service ${key}: ${err.message}`, err.stack);
-            }
+            await this.stopServiceForConfig(registration);
             return;
         }
         if (shouldBeRunning && currentState === 'stopped') {
@@ -466,6 +615,8 @@ let ManagedServiceManagerService = class ManagedServiceManagerService {
         else if (shouldBeRunning && currentState === 'started' && registration.service.onConfigChanged) {
             try {
                 const result = await registration.service.onConfigChanged();
+                if (!isCurrent())
+                    return;
                 if (this.isConfigChangeResult(result) && result.restartRequired) {
                     this.logger.log(`Service ${key} requires restart after config change`);
                     await this.stopService(registration);
@@ -490,7 +641,7 @@ let ManagedServiceManagerService = class ManagedServiceManagerService {
             this.readinessRetries.delete(key);
             this.logger.warn(`Readiness retries exhausted for service: ${key}`);
             if (!this.openIssues.has(key)) {
-                this.openIssues.add(key);
+                this.openIssues.set(key, registration);
                 const runtime = this.runtimeInfo.get(key);
                 await this.raiseServiceError(registration, runtime?.lastError ?? 'Readiness retries exhausted');
             }
@@ -647,43 +798,62 @@ let ManagedServiceManagerService = class ManagedServiceManagerService {
     }
     async raiseServiceError(registration, lastError) {
         const key = this.getRegistrationKey(registration);
-        await this.notifications.notify({
-            source: registration.owner.type,
-            kind: notifications_constants_1.NotificationKind.ISSUE,
-            key: `service:${key}`,
-            severity: notifications_constants_1.NotificationSeverity.ERROR,
-            title: `Service ${registration.serviceId} of ${registration.owner.type} failed`,
-            message: (0, notifications_utils_1.sanitizeErrorMessage)(lastError),
-            actions: [
-                {
-                    type: notifications_constants_1.NotificationActionType.SERVICE,
-                    label: 'Restart service',
-                    extension_kind: registration.owner.kind,
-                    extension_type: registration.owner.type,
-                    service_id: registration.serviceId,
-                    operation: 'restart',
-                    primary: true,
-                },
-                {
-                    type: notifications_constants_1.NotificationActionType.LINK,
-                    label: 'Open services',
-                    url: `/extensions?tab=services&kind=${registration.owner.kind}`,
-                },
-            ],
+        await this.enqueueIssueWrite(key, async () => {
+            if (!this.isCurrentRegistration(registration) || this.openIssues.get(key) !== registration)
+                return;
+            await this.notifications.notify({
+                source: registration.owner.type,
+                kind: notifications_constants_1.NotificationKind.ISSUE,
+                key: `service:${key}`,
+                severity: notifications_constants_1.NotificationSeverity.ERROR,
+                title: `Service ${registration.serviceId} of ${registration.owner.type} failed`,
+                message: (0, notifications_utils_1.sanitizeErrorMessage)(lastError),
+                actions: [
+                    {
+                        type: notifications_constants_1.NotificationActionType.SERVICE,
+                        label: 'Restart service',
+                        extension_kind: registration.owner.kind,
+                        extension_type: registration.owner.type,
+                        service_id: registration.serviceId,
+                        operation: 'restart',
+                        primary: true,
+                    },
+                    {
+                        type: notifications_constants_1.NotificationActionType.LINK,
+                        label: 'Open services',
+                        url: `/extensions?tab=services&kind=${registration.owner.kind}`,
+                    },
+                ],
+            });
         });
     }
     async resolveServiceError(registration) {
         const key = this.getRegistrationKey(registration);
-        if (!this.openIssues.has(key)) {
+        if (this.openIssues.get(key) !== registration) {
+            await this.issueWrites.get(key);
             return;
         }
         this.openIssues.delete(key);
+        await this.enqueueIssueWrite(key, async () => {
+            try {
+                await this.notifications.resolve(registration.owner.type, `service:${key}`);
+            }
+            catch (error) {
+                const err = error;
+                this.logger.error(`Failed to resolve the service issue for ${key}: ${err.message}`, err.stack);
+            }
+        });
+    }
+    async enqueueIssueWrite(key, write) {
+        const previous = this.issueWrites.get(key) ?? Promise.resolve();
+        const pending = previous.catch(() => undefined).then(write);
+        this.issueWrites.set(key, pending);
         try {
-            await this.notifications.resolve(registration.owner.type, `service:${key}`);
+            await pending;
         }
-        catch (error) {
-            const err = error;
-            this.logger.error(`Failed to resolve the service issue for ${key}: ${err.message}`, err.stack);
+        finally {
+            if (this.issueWrites.get(key) === pending)
+                this.issueWrites.delete(key);
         }
     }
 };