@rvoh/psychic-workers 2.3.1 → 2.4.0

package/dist/cjs/src/background/index.js

@@ -7,6 +7,7 @@ import ActivatingBackgroundWorkersWithoutDefaultWorkerConnection from '../error/
 import ActivatingNamedQueueBackgroundWorkersWithoutWorkerConnection from '../error/background/ActivatingNamedQueueBackgroundWorkersWithoutWorkerConnection.js';
 import DefaultBullMQNativeOptionsMissingQueueConnectionAndDefaultQueueConnection from '../error/background/DefaultBullMQNativeOptionsMissingQueueConnectionAndDefaultQueueConnection.js';
 import NamedBullMQNativeOptionsMissingQueueConnectionAndDefaultQueueConnection from '../error/background/NamedBullMQNativeOptionsMissingQueueConnectionAndDefaultQueueConnection.js';
+import NoClassForSpecifiedGlobalName from '../error/background/NoClassForSpecifiedGlobalName.js';
 import NoQueueForSpecifiedQueueName from '../error/background/NoQueueForSpecifiedQueueName.js';
 import NoQueueForSpecifiedWorkstream from '../error/background/NoQueueForSpecifiedWorkstream.js';
 import EnvInternal from '../helpers/EnvInternal.js';
@@ -128,10 +129,66 @@ export class Background {
     get workers() {
         return [...this._workers];
     }
+    /**
+     * @internal
+     *
+     * the maximum number of milliseconds graceful shutdown is allowed
+     * to run before the process exits anyway, so that a hung hook,
+     * db close, or worker close can never keep a dying process alive
+     */
+    static SHUTDOWN_TIMEOUT_MS = 15000;
+    /**
+     * @internal
+     *
+     * set once a fatal error has begun graceful cleanup, so that a
+     * second fatal error during cleanup exits immediately instead of
+     * attempting cleanup again
+     */
+    fatalErrorShutdownBegun = false;
     async shutdownAndExit() {
-        await this.shutdown();
+        let exitCode = 0;
+        try {
+            await this.shutdownWithTimeout();
+        }
+        catch (error) {
+            PsychicApp.logWithLevel('error', '[psychic-workers] error during graceful shutdown:', error);
+            exitCode = 1;
+        }
         // https://docs.bullmq.io/guide/going-to-production#gracefully-shut-down-workers
-        process.exit(0);
+        process.exit(exitCode);
+    }
+    /**
+     * @internal
+     *
+     * after an uncaught exception or unhandled rejection, attempt
+     * best-effort graceful cleanup (bounded by SHUTDOWN_TIMEOUT_MS),
+     * then exit nonzero so orchestrators know to restart the process
+     */
+    async shutdownAndExitAfterFatalError() {
+        if (!this.fatalErrorShutdownBegun) {
+            this.fatalErrorShutdownBegun = true;
+            try {
+                await this.shutdownWithTimeout();
+            }
+            catch (error) {
+                PsychicApp.logWithLevel('error', '[psychic-workers] error during graceful shutdown after fatal error:', error);
+            }
+        }
+        process.exit(1);
+    }
+    /**
+     * @internal
+     *
+     * runs {@link Background.shutdown}, rejecting if it has not settled
+     * within SHUTDOWN_TIMEOUT_MS
+     */
+    async shutdownWithTimeout() {
+        await Promise.race([
+            this.shutdown(),
+            new Promise((_, reject) => {
+                setTimeout(() => reject(new Error(`[psychic-workers] graceful shutdown timed out after ${Background.SHUTDOWN_TIMEOUT_MS}ms`)), Background.SHUTDOWN_TIMEOUT_MS).unref();
+            }),
+        ]);
     }
     /**
      * Shuts down workers, closes all redis connections
@@ -153,7 +210,13 @@ export class Background {
         if (!EnvInternal.isTest)
             PsychicApp.log(`[psychic-workers] closeAllRedisConnections`);
         for (const worker of this.workers) {
-            await worker.close();
+            try {
+                await worker.close();
+            }
+            catch (error) {
+                if (!EnvInternal.isTest)
+                    PsychicApp.logWithLevel('error', `[psychic-workers] error closing worker:`, error);
+            }
         }
         for (const connection of this.redisConnections) {
             try {
@@ -379,24 +442,22 @@ export class Background {
      */
     work() {
         process.on('uncaughtException', (error) => {
-            PsychicApp.log('[psychic-workers] uncaughtException:', error);
+            PsychicApp.logWithLevel('error', '[psychic-workers] uncaughtException:', error);
+            void this.shutdownAndExitAfterFatalError();
         });
-        process.on('unhandledRejection', (error) => {
-            PsychicApp.log('[psychic-workers] unhandledRejection:', error);
+        process.on('unhandledRejection', (reason) => {
+            PsychicApp.logWithLevel('error', '[psychic-workers] unhandledRejection:', reason);
+            void this.shutdownAndExitAfterFatalError();
         });
         process.on('SIGTERM', () => {
             if (!EnvInternal.isTest)
                 PsychicApp.log('[psychic-workers] handle SIGTERM');
-            void this.shutdownAndExit()
-                .then(() => { })
-                .catch(() => { });
+            void this.shutdownAndExit();
         });
         process.on('SIGINT', () => {
             if (!EnvInternal.isTest)
                 PsychicApp.log('[psychic-workers] handle SIGINT');
-            void this.shutdownAndExit()
-                .then(() => { })
-                .catch(() => { });
+            void this.shutdownAndExit();
         });
         this.connect({ activateWorkers: true });
     }
@@ -617,22 +678,23 @@ export class Background {
                     objectClass = PsychicApp.lookupClassByGlobalName(globalName);
                 }
                 if (!objectClass)
-                    return;
+                    throw new NoClassForSpecifiedGlobalName(globalName);
                 // eslint-disable-next-line @typescript-eslint/no-unsafe-call, @typescript-eslint/no-unsafe-member-access
                 await objectClass[method](...args, job);
                 break;
-            case 'BackgroundJobQueueModelInstanceJob':
+            case 'BackgroundJobQueueModelInstanceJob': {
                 if (globalName) {
                     dreamClass = PsychicApp.lookupClassByGlobalName(globalName);
                 }
-                if (dreamClass) {
-                    const modelInstance = await dreamClass.connection('primary').find(id);
-                    if (!modelInstance)
-                        return;
-                    // eslint-disable-next-line @typescript-eslint/no-explicit-any, @typescript-eslint/no-unsafe-call, @typescript-eslint/no-unsafe-member-access
-                    await modelInstance[method](...args, job);
-                }
+                if (!dreamClass)
+                    throw new NoClassForSpecifiedGlobalName(globalName);
+                const modelInstance = await dreamClass.connection('primary').find(id);
+                if (!modelInstance)
+                    return;
+                // eslint-disable-next-line @typescript-eslint/no-explicit-any, @typescript-eslint/no-unsafe-call, @typescript-eslint/no-unsafe-member-access
+                await modelInstance[method](...args, job);
                 break;
+            }
         }
     }
 }

package/dist/cjs/src/error/background/NoClassForSpecifiedGlobalName.js

@@ -0,0 +1,15 @@
+export default class NoClassForSpecifiedGlobalName extends Error {
+    globalName;
+    constructor(globalName) {
+        super();
+        this.globalName = globalName;
+    }
+    get message() {
+        return `Error processing background job
+No class found for globalName "${this.globalName}"
+
+This usually means the class was renamed or removed after this job
+(or its repeating job scheduler) was enqueued.
+`;
+    }
+}

package/dist/cjs/src/package-exports/errors.js

@@ -1,2 +1,3 @@
+export { default as NoClassForSpecifiedGlobalName } from '../error/background/NoClassForSpecifiedGlobalName.js';
 export { default as NoQueueForSpecifiedQueueName } from '../error/background/NoQueueForSpecifiedQueueName.js';
 export { default as NoQueueForSpecifiedWorkstream } from '../error/background/NoQueueForSpecifiedWorkstream.js';

package/dist/esm/src/background/index.js

@@ -7,6 +7,7 @@ import ActivatingBackgroundWorkersWithoutDefaultWorkerConnection from '../error/
 import ActivatingNamedQueueBackgroundWorkersWithoutWorkerConnection from '../error/background/ActivatingNamedQueueBackgroundWorkersWithoutWorkerConnection.js';
 import DefaultBullMQNativeOptionsMissingQueueConnectionAndDefaultQueueConnection from '../error/background/DefaultBullMQNativeOptionsMissingQueueConnectionAndDefaultQueueConnection.js';
 import NamedBullMQNativeOptionsMissingQueueConnectionAndDefaultQueueConnection from '../error/background/NamedBullMQNativeOptionsMissingQueueConnectionAndDefaultQueueConnection.js';
+import NoClassForSpecifiedGlobalName from '../error/background/NoClassForSpecifiedGlobalName.js';
 import NoQueueForSpecifiedQueueName from '../error/background/NoQueueForSpecifiedQueueName.js';
 import NoQueueForSpecifiedWorkstream from '../error/background/NoQueueForSpecifiedWorkstream.js';
 import EnvInternal from '../helpers/EnvInternal.js';
@@ -128,10 +129,66 @@ export class Background {
     get workers() {
         return [...this._workers];
     }
+    /**
+     * @internal
+     *
+     * the maximum number of milliseconds graceful shutdown is allowed
+     * to run before the process exits anyway, so that a hung hook,
+     * db close, or worker close can never keep a dying process alive
+     */
+    static SHUTDOWN_TIMEOUT_MS = 15000;
+    /**
+     * @internal
+     *
+     * set once a fatal error has begun graceful cleanup, so that a
+     * second fatal error during cleanup exits immediately instead of
+     * attempting cleanup again
+     */
+    fatalErrorShutdownBegun = false;
     async shutdownAndExit() {
-        await this.shutdown();
+        let exitCode = 0;
+        try {
+            await this.shutdownWithTimeout();
+        }
+        catch (error) {
+            PsychicApp.logWithLevel('error', '[psychic-workers] error during graceful shutdown:', error);
+            exitCode = 1;
+        }
         // https://docs.bullmq.io/guide/going-to-production#gracefully-shut-down-workers
-        process.exit(0);
+        process.exit(exitCode);
+    }
+    /**
+     * @internal
+     *
+     * after an uncaught exception or unhandled rejection, attempt
+     * best-effort graceful cleanup (bounded by SHUTDOWN_TIMEOUT_MS),
+     * then exit nonzero so orchestrators know to restart the process
+     */
+    async shutdownAndExitAfterFatalError() {
+        if (!this.fatalErrorShutdownBegun) {
+            this.fatalErrorShutdownBegun = true;
+            try {
+                await this.shutdownWithTimeout();
+            }
+            catch (error) {
+                PsychicApp.logWithLevel('error', '[psychic-workers] error during graceful shutdown after fatal error:', error);
+            }
+        }
+        process.exit(1);
+    }
+    /**
+     * @internal
+     *
+     * runs {@link Background.shutdown}, rejecting if it has not settled
+     * within SHUTDOWN_TIMEOUT_MS
+     */
+    async shutdownWithTimeout() {
+        await Promise.race([
+            this.shutdown(),
+            new Promise((_, reject) => {
+                setTimeout(() => reject(new Error(`[psychic-workers] graceful shutdown timed out after ${Background.SHUTDOWN_TIMEOUT_MS}ms`)), Background.SHUTDOWN_TIMEOUT_MS).unref();
+            }),
+        ]);
     }
     /**
      * Shuts down workers, closes all redis connections
@@ -153,7 +210,13 @@ export class Background {
         if (!EnvInternal.isTest)
             PsychicApp.log(`[psychic-workers] closeAllRedisConnections`);
         for (const worker of this.workers) {
-            await worker.close();
+            try {
+                await worker.close();
+            }
+            catch (error) {
+                if (!EnvInternal.isTest)
+                    PsychicApp.logWithLevel('error', `[psychic-workers] error closing worker:`, error);
+            }
         }
         for (const connection of this.redisConnections) {
             try {
@@ -379,24 +442,22 @@ export class Background {
      */
     work() {
         process.on('uncaughtException', (error) => {
-            PsychicApp.log('[psychic-workers] uncaughtException:', error);
+            PsychicApp.logWithLevel('error', '[psychic-workers] uncaughtException:', error);
+            void this.shutdownAndExitAfterFatalError();
         });
-        process.on('unhandledRejection', (error) => {
-            PsychicApp.log('[psychic-workers] unhandledRejection:', error);
+        process.on('unhandledRejection', (reason) => {
+            PsychicApp.logWithLevel('error', '[psychic-workers] unhandledRejection:', reason);
+            void this.shutdownAndExitAfterFatalError();
         });
         process.on('SIGTERM', () => {
             if (!EnvInternal.isTest)
                 PsychicApp.log('[psychic-workers] handle SIGTERM');
-            void this.shutdownAndExit()
-                .then(() => { })
-                .catch(() => { });
+            void this.shutdownAndExit();
         });
         process.on('SIGINT', () => {
             if (!EnvInternal.isTest)
                 PsychicApp.log('[psychic-workers] handle SIGINT');
-            void this.shutdownAndExit()
-                .then(() => { })
-                .catch(() => { });
+            void this.shutdownAndExit();
         });
         this.connect({ activateWorkers: true });
     }
@@ -617,22 +678,23 @@ export class Background {
                     objectClass = PsychicApp.lookupClassByGlobalName(globalName);
                 }
                 if (!objectClass)
-                    return;
+                    throw new NoClassForSpecifiedGlobalName(globalName);
                 // eslint-disable-next-line @typescript-eslint/no-unsafe-call, @typescript-eslint/no-unsafe-member-access
                 await objectClass[method](...args, job);
                 break;
-            case 'BackgroundJobQueueModelInstanceJob':
+            case 'BackgroundJobQueueModelInstanceJob': {
                 if (globalName) {
                     dreamClass = PsychicApp.lookupClassByGlobalName(globalName);
                 }
-                if (dreamClass) {
-                    const modelInstance = await dreamClass.connection('primary').find(id);
-                    if (!modelInstance)
-                        return;
-                    // eslint-disable-next-line @typescript-eslint/no-explicit-any, @typescript-eslint/no-unsafe-call, @typescript-eslint/no-unsafe-member-access
-                    await modelInstance[method](...args, job);
-                }
+                if (!dreamClass)
+                    throw new NoClassForSpecifiedGlobalName(globalName);
+                const modelInstance = await dreamClass.connection('primary').find(id);
+                if (!modelInstance)
+                    return;
+                // eslint-disable-next-line @typescript-eslint/no-explicit-any, @typescript-eslint/no-unsafe-call, @typescript-eslint/no-unsafe-member-access
+                await modelInstance[method](...args, job);
                 break;
+            }
         }
     }
 }

package/dist/esm/src/error/background/NoClassForSpecifiedGlobalName.js

@@ -0,0 +1,15 @@
+export default class NoClassForSpecifiedGlobalName extends Error {
+    globalName;
+    constructor(globalName) {
+        super();
+        this.globalName = globalName;
+    }
+    get message() {
+        return `Error processing background job
+No class found for globalName "${this.globalName}"
+
+This usually means the class was renamed or removed after this job
+(or its repeating job scheduler) was enqueued.
+`;
+    }
+}

package/dist/esm/src/package-exports/errors.js

@@ -1,2 +1,3 @@
+export { default as NoClassForSpecifiedGlobalName } from '../error/background/NoClassForSpecifiedGlobalName.js';
 export { default as NoQueueForSpecifiedQueueName } from '../error/background/NoQueueForSpecifiedQueueName.js';
 export { default as NoQueueForSpecifiedWorkstream } from '../error/background/NoQueueForSpecifiedWorkstream.js';

package/dist/types/src/background/index.d.ts

@@ -88,7 +88,38 @@ export declare class Background {
      * Returns all the workers in your application
      */
     get workers(): Worker<any, any, string>[];
+    /**
+     * @internal
+     *
+     * the maximum number of milliseconds graceful shutdown is allowed
+     * to run before the process exits anyway, so that a hung hook,
+     * db close, or worker close can never keep a dying process alive
+     */
+    static readonly SHUTDOWN_TIMEOUT_MS = 15000;
+    /**
+     * @internal
+     *
+     * set once a fatal error has begun graceful cleanup, so that a
+     * second fatal error during cleanup exits immediately instead of
+     * attempting cleanup again
+     */
+    private fatalErrorShutdownBegun;
     private shutdownAndExit;
+    /**
+     * @internal
+     *
+     * after an uncaught exception or unhandled rejection, attempt
+     * best-effort graceful cleanup (bounded by SHUTDOWN_TIMEOUT_MS),
+     * then exit nonzero so orchestrators know to restart the process
+     */
+    private shutdownAndExitAfterFatalError;
+    /**
+     * @internal
+     *
+     * runs {@link Background.shutdown}, rejecting if it has not settled
+     * within SHUTDOWN_TIMEOUT_MS
+     */
+    private shutdownWithTimeout;
     /**
      * Shuts down workers, closes all redis connections
      */

package/dist/types/src/error/background/NoClassForSpecifiedGlobalName.d.ts

@@ -0,0 +1,5 @@
+export default class NoClassForSpecifiedGlobalName extends Error {
+    private globalName;
+    constructor(globalName: string | undefined);
+    get message(): string;
+}

package/dist/types/src/package-exports/errors.d.ts

@@ -1,2 +1,3 @@
+export { default as NoClassForSpecifiedGlobalName } from '../error/background/NoClassForSpecifiedGlobalName.js';
 export { default as NoQueueForSpecifiedQueueName } from '../error/background/NoQueueForSpecifiedQueueName.js';
 export { default as NoQueueForSpecifiedWorkstream } from '../error/background/NoQueueForSpecifiedWorkstream.js';

package/package.json

@@ -2,7 +2,7 @@
   "type": "module",
   "name": "@rvoh/psychic-workers",
   "description": "Background job system for Psychic applications",
-  "version": "2.3.1",
+  "version": "2.4.0",
   "author": "RVO Health",
   "publishConfig": {
     "access": "public"
@@ -44,8 +44,8 @@
     "psy": "NODE_ENV=${NODE_ENV:-test} pnpm psy:ts",
     "psy:js": "node ./dist/test-app/src/cli/index.js",
     "psy:ts": "tsx ./test-app/src/cli/index.ts",
-    "build": "echo \"building cjs...\" && rm -rf dist && npx tsc -p ./tsconfig.cjs.build.json && echo \"building esm...\" && npx tsc -p ./tsconfig.esm.build.json",
-    "build:test-app": "rm -rf dist && echo \"building test app to esm...\" && npx tsc -p ./tsconfig.esm.build.test-app.json && echo \"building test app to cjs...\" && npx tsc -p ./tsconfig.cjs.build.test-app.json",
+    "build": "echo \"building cjs...\" && rm -rf dist && tsc -p ./tsconfig.cjs.build.json && echo \"building esm...\" && tsc -p ./tsconfig.esm.build.json",
+    "build:test-app": "rm -rf dist && echo \"building test app to esm...\" && tsc -p ./tsconfig.esm.build.test-app.json && echo \"building test app to cjs...\" && tsc -p ./tsconfig.cjs.build.test-app.json",
     "uspec": "vitest --config ./spec/unit/vite.config.ts",
     "format": "pnpm prettier . --write",
     "lint": "pnpm eslint --no-warn-ignored \"src/**/*.ts\" && pnpm prettier . --check",
@@ -99,7 +99,7 @@
     "typescript-eslint": "^8.48.1",
     "vitest": "^4.1.0"
   },
-  "packageManager": "pnpm@11.1.3+sha512.c85357fe17ca12dd23dd7071822666dfd7e3cb76fe214e3370b5ea2fb34f2a231185509b63e717f3cd0acb38dd3f8d82bcd5e8172400ae678b70ea4fbed0896d",
+  "packageManager": "pnpm@11.9.0+sha512.bd682d5d03fe525ef7c9fd6780c6884d1e756ac4c9c9fe00c538782824310dcf90e3ddc4f53835f06dfaebd5085e41855e0bcbb3b60de2ac5bbab89e5036f03b",
   "dependencies": {
     "commander": "^14.0.2",
     "dotenv": "^17.2.3",