@orion-js/dogs 3.1.23 → 3.1.26

package/lib/defineJob/index.js

@@ -2,6 +2,11 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.defineJob = void 0;
 const defineJob = (options) => {
+    if (options.type === 'recurrent') {
+        if (!options.hasOwnProperty('priority')) {
+            options.priority = 100;
+        }
+    }
     return options;
 };
 exports.defineJob = defineJob;

package/lib/history.test.js

@@ -3,7 +3,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
 const helpers_1 = require("@orion-js/helpers");
 const _1 = require(".");
 const logger_1 = require("@orion-js/logger");
-(0, logger_1.setLogLevel)('error');
+(0, logger_1.setLogLevel)('none');
 describe('Test Jobs History', () => {
     it('Should save success history types', async () => {
         const jobId = (0, helpers_1.generateId)();
@@ -34,10 +34,11 @@ describe('Test Jobs History', () => {
         expect(execution.duration).toBeGreaterThan(49);
         expect(execution).toEqual({
             _id: expect.any(String),
+            jobId: expect.any(String),
             executionId: expect.any(String),
             jobName: jobId,
             type: 'event',
-            priority: 1,
+            priority: 100,
             tries: 1,
             startedAt: expect.any(Date),
             endedAt: expect.any(Date),
@@ -74,10 +75,11 @@ describe('Test Jobs History', () => {
         const execution = executions[0];
         expect(execution).toEqual({
             _id: expect.any(String),
+            jobId: expect.any(String),
             executionId: expect.any(String),
             jobName: jobId,
             type: 'event',
-            priority: 1,
+            priority: 100,
             tries: 1,
             startedAt: expect.any(Date),
             endedAt: expect.any(Date),
@@ -93,8 +95,10 @@ describe('Test Jobs History', () => {
         const jobId = (0, helpers_1.generateId)();
         const job = (0, _1.defineJob)({
             type: 'event',
-            async resolve() {
-                await (0, helpers_1.sleep)(100);
+            async resolve(_, context) {
+                if (context.tries === 1) {
+                    await (0, helpers_1.sleep)(100);
+                }
                 return { status: 'ok' };
             }
         });
@@ -112,15 +116,16 @@ describe('Test Jobs History', () => {
         await instance.stop();
         const executions = await _1.jobsHistoryRepo.getExecutions(jobId);
         expect(executions.length).toBe(2);
-        const execution1 = executions[1];
-        expect(execution1.duration).toBeGreaterThanOrEqual(10);
-        expect(execution1.duration).toBeLessThan(100);
-        expect(execution1).toEqual({
+        const execution = executions.find(e => e.status === 'stale');
+        expect(execution.duration).toBeGreaterThanOrEqual(100);
+        expect(execution.duration).toBeLessThan(200);
+        expect(execution).toEqual({
             _id: expect.any(String),
+            jobId: expect.any(String),
             executionId: expect.any(String),
             jobName: jobId,
             type: 'event',
-            priority: 1,
+            priority: 100,
             tries: 1,
             startedAt: expect.any(Date),
             endedAt: expect.any(Date),
@@ -129,24 +134,6 @@ describe('Test Jobs History', () => {
             status: 'stale',
             errorMessage: null,
             params: null,
-            result: null
-        });
-        const execution2 = executions[0];
-        expect(execution2.duration).toBeGreaterThanOrEqual(100);
-        expect(execution2).toEqual({
-            _id: expect.any(String),
-            executionId: expect.any(String),
-            jobName: jobId,
-            type: 'event',
-            priority: 1,
-            tries: 1,
-            startedAt: expect.any(Date),
-            endedAt: expect.any(Date),
-            duration: expect.any(Number),
-            expiresAt: expect.any(Date),
-            status: 'success',
-            errorMessage: null,
-            params: null,
             result: { status: 'ok' }
         });
     });

package/lib/repos/JobsHistoryRepo.js

@@ -9,6 +9,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.JobsHistoryRepo = void 0;
 const mongodb_1 = require("@orion-js/mongodb");
 const services_1 = require("@orion-js/services");
+const lodash_1 = require("lodash");
 const HistoryRecord_1 = require("../types/HistoryRecord");
 let JobsHistoryRepo = class JobsHistoryRepo {
     constructor() {
@@ -34,7 +35,14 @@ let JobsHistoryRepo = class JobsHistoryRepo {
         });
     }
     async saveExecution(record) {
-        await this.history.insertOne(record);
+        await this.history.upsert({ executionId: record.executionId }, {
+            $setOnInsert: {
+                status: record.status
+            },
+            $set: {
+                ...(0, lodash_1.omit)(record, 'status')
+            }
+        });
     }
     async getExecutions(jobName, limit, skip) {
         const cursor = this.history.find({ jobName }).sort({ startedAt: -1 });

package/lib/repos/JobsRepo.d.ts

@@ -5,10 +5,12 @@ import { JobToRun } from '../types/Worker';
 export declare class JobsRepo {
     jobs: import("@orion-js/mongodb").Collection<JobRecord>;
     getJobAndLock(jobNames: string[], lockTime: number): Promise<JobToRun>;
+    setJobRecordPriority(jobId: string, priority: number): Promise<void>;
     scheduleNextRun(options: {
         jobId: string;
         nextRunAt: Date;
         addTries: boolean;
+        priority: number;
     }): Promise<void>;
     deleteEventJob(jobId: string): Promise<void>;
     extendLockTime(jobId: string, extraTime: number): Promise<void>;

package/lib/repos/JobsRepo.js

@@ -7,6 +7,7 @@ var __decorate = (this && this.__decorate) || function (decorators, target, key,
 };
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.JobsRepo = void 0;
+const helpers_1 = require("@orion-js/helpers");
 const logger_1 = require("@orion-js/logger");
 const mongodb_1 = require("@orion-js/mongodb");
 const services_1 = require("@orion-js/services");
@@ -22,7 +23,7 @@ let JobsRepo = class JobsRepo {
                     keys: {
                         jobName: 1,
                         nextRunAt: 1,
-                        priority: 1,
+                        priority: -1,
                         lockedUntil: 1
                     }
                 },
@@ -58,7 +59,7 @@ let JobsRepo = class JobsRepo {
         }, {
             mongoOptions: {
                 sort: {
-                    priority: 1,
+                    priority: -1,
                     nextRunAt: 1
                 },
                 returnDocument: 'before'
@@ -74,6 +75,7 @@ let JobsRepo = class JobsRepo {
         }
         return {
             jobId: job._id,
+            executionId: (0, helpers_1.generateId)(),
             name: job.jobName,
             params: job.params,
             type: job.type,
@@ -83,9 +85,12 @@ let JobsRepo = class JobsRepo {
             uniqueIdentifier: job.uniqueIdentifier
         };
     }
+    async setJobRecordPriority(jobId, priority) {
+        await this.jobs.updateOne(jobId, { $set: { priority } });
+    }
     async scheduleNextRun(options) {
         const updator = {
-            $set: { nextRunAt: options.nextRunAt },
+            $set: { nextRunAt: options.nextRunAt, priority: options.priority },
             $unset: { lockedUntil: '' }
         };
         if (options.addTries) {
@@ -111,7 +116,7 @@ let JobsRepo = class JobsRepo {
         }, {
             $set: {
                 type: job.type,
-                priority: job.priority || 1
+                priority: job.priority
             },
             $setOnInsert: {
                 nextRunAt: new Date()

package/lib/services/EventsService.js

@@ -19,7 +19,7 @@ let EventsService = class EventsService {
         logger_1.logger.debug('Scheduling job...', options);
         await this.jobsRepo.scheduleJob({
             name: options.name,
-            priority: options.priority || 1,
+            priority: options.priority || 100,
             nextRunAt: (0, getNextRunDate_1.getNextRunDate)(options),
             params: options.params || null,
             uniqueIdentifier: options.uniqueIdentifier

package/lib/services/Executor.d.ts

@@ -16,5 +16,5 @@ export declare class Executor {
         jobToRun: JobToRun;
     }): Promise<void>;
     afterExecutionSuccess(job: JobDefinition, jobToRun: JobToRun, context: ExecutionContext): Promise<void>;
-    executeJob(jobs: JobsDefinition, jobToRun: JobToRun): Promise<void>;
+    executeJob(jobs: JobsDefinition, jobToRun: JobToRun, respawnWorker: Function): Promise<void>;
 }

package/lib/services/Executor.js

@@ -48,7 +48,8 @@ let Executor = class Executor {
                 await this.jobsRepo.scheduleNextRun({
                     jobId: jobToRun.jobId,
                     nextRunAt: (0, getNextRunDate_1.getNextRunDate)(job),
-                    addTries: false
+                    addTries: false,
+                    priority: job.priority
                 });
             }
         };
@@ -69,7 +70,8 @@ let Executor = class Executor {
             await this.jobsRepo.scheduleNextRun({
                 jobId: jobToRun.jobId,
                 nextRunAt: (0, getNextRunDate_1.getNextRunDate)(result),
-                addTries: true
+                addTries: true,
+                priority: job.type === 'recurrent' ? job.priority : jobToRun.priority
             });
         }
     }
@@ -80,7 +82,8 @@ let Executor = class Executor {
             const oneWeek = 1000 * 60 * 60 * 24 * 7;
             const saveExecutionsFor = job.saveExecutionsFor || oneWeek;
             await this.jobsHistoryRepo.saveExecution({
-                executionId: jobToRun.jobId,
+                jobId: jobToRun.jobId,
+                executionId: jobToRun.executionId,
                 jobName: jobToRun.name,
                 type: jobToRun.type,
                 priority: jobToRun.priority,
@@ -103,7 +106,8 @@ let Executor = class Executor {
             await this.jobsRepo.scheduleNextRun({
                 jobId: jobToRun.jobId,
                 nextRunAt: (0, getNextRunDate_1.getNextRunDate)(job),
-                addTries: false
+                addTries: false,
+                priority: job.priority
             });
         }
         if (job.type === 'event') {
@@ -111,12 +115,12 @@ let Executor = class Executor {
             await this.jobsRepo.deleteEventJob(jobToRun.jobId);
         }
     }
-    async executeJob(jobs, jobToRun) {
+    async executeJob(jobs, jobToRun, respawnWorker) {
         const job = this.getJobDefinition(jobToRun, jobs);
         if (!job)
             return;
         const startedAt = new Date();
-        const onStale = () => {
+        const onStale = async () => {
             if (job.onStale) {
                 context.logger.info(`Job "${jobToRun.name}" is stale`);
                 job.onStale(jobToRun.params, context);
@@ -124,6 +128,8 @@ let Executor = class Executor {
             else {
                 context.logger.error(`Job "${jobToRun.name}" is stale`);
             }
+            await this.jobsRepo.setJobRecordPriority(jobToRun.jobId, 0);
+            respawnWorker();
             this.saveExecution({
                 startedAt,
                 status: 'stale',

package/lib/services/WorkerService.d.ts

@@ -1,15 +1,16 @@
 import { JobDefinitionWithName, JobsDefinition } from '../types/JobsDefinition';
 import { StartWorkersConfig } from '../types/StartConfig';
-import { WorkersInstance } from '../types/Worker';
+import { WorkerInstance, WorkersInstance } from '../types/Worker';
 export declare class WorkerService {
     private jobsRepo;
     private executor;
     getJobNames(jobs: JobsDefinition): string[];
     getJobs(jobs: JobsDefinition): JobDefinitionWithName[];
-    runWorkerLoop(config: StartWorkersConfig): Promise<boolean>;
-    startWorker(config: StartWorkersConfig, workersInstance: WorkersInstance): Promise<void>;
+    runWorkerLoop(config: StartWorkersConfig, workerInstance: WorkerInstance): Promise<boolean>;
+    startWorker(config: StartWorkersConfig, workerInstance: WorkerInstance): Promise<void>;
     createWorkersInstanceDefinition(config: StartWorkersConfig): WorkersInstance;
     ensureRecords(config: StartWorkersConfig): Promise<void>;
+    startANewWorker(config: StartWorkersConfig, workersInstance: WorkersInstance): Promise<void>;
     runWorkers(config: StartWorkersConfig, workersInstance: WorkersInstance): Promise<void>;
     startWorkers(userConfig: Partial<StartWorkersConfig>): WorkersInstance;
 }

package/lib/services/WorkerService.js

@@ -28,33 +28,33 @@ let WorkerService = class WorkerService {
             };
         });
     }
-    async runWorkerLoop(config) {
+    async runWorkerLoop(config, workerInstance) {
         const names = this.getJobNames(config.jobs);
-        logger_1.logger.debug(`Running worker loop for jobs "${names.join(', ')}"...`);
+        logger_1.logger.debug(`Running worker loop [w${workerInstance.workerIndex}] for jobs "${names.join(', ')}"...`);
         const jobToRun = await this.jobsRepo.getJobAndLock(names, config.lockTime);
         if (!jobToRun) {
             logger_1.logger.debug('No job to run');
             return false;
         }
-        logger_1.logger.debug(`Got job to run:`, jobToRun);
-        await this.executor.executeJob(config.jobs, jobToRun);
+        logger_1.logger.debug(`Got job [w${workerInstance.workerIndex}] to run:`, jobToRun);
+        await this.executor.executeJob(config.jobs, jobToRun, workerInstance.respawn);
         return true;
     }
-    async startWorker(config, workersInstance) {
+    async startWorker(config, workerInstance) {
         while (true) {
-            if (!workersInstance.running) {
-                logger_1.logger.info('Got signal to stop. Stopping worker...');
+            if (!workerInstance.running) {
+                logger_1.logger.info(`Got signal to stop. Stopping worker [w${workerInstance.workerIndex}]...`);
                 return;
             }
             try {
-                const didRun = await this.runWorkerLoop(config);
+                const didRun = await this.runWorkerLoop(config, workerInstance);
                 if (!didRun)
                     await (0, helpers_1.sleep)(config.pollInterval);
                 if (didRun)
                     await (0, helpers_1.sleep)(config.cooldownPeriod);
             }
             catch (error) {
-                logger_1.logger.error(`Error in job runner. Waiting and running again`, error);
+                logger_1.logger.error(`Error in job runner.`, error);
                 await (0, helpers_1.sleep)(config.pollInterval);
             }
         }
@@ -67,7 +67,8 @@ let WorkerService = class WorkerService {
             stop: async () => {
                 logger_1.logger.debug('Stopping workers...', workersInstance.workers);
                 workersInstance.running = false;
-                await Promise.all(workersInstance.workers);
+                const stopingPromises = workersInstance.workers.map(worker => worker.stop());
+                await Promise.all(stopingPromises);
             }
         };
         return workersInstance;
@@ -81,13 +82,32 @@ let WorkerService = class WorkerService {
             await this.jobsRepo.ensureJobRecord(job);
         }));
     }
+    async startANewWorker(config, workersInstance) {
+        const workerIndex = workersInstance.workers.length;
+        logger_1.logger.info(`Starting worker [w${workerIndex}]`);
+        const workerInstance = {
+            running: true,
+            workerIndex,
+            stop: async () => {
+                logger_1.logger.info(`Stopping worker [w${workerIndex}]...`);
+                workerInstance.running = false;
+                await workerInstance.promise;
+            },
+            respawn: async () => {
+                logger_1.logger.info(`Respawning worker [w${workerIndex}]...`);
+                workerInstance.stop();
+                await this.startANewWorker(config, workersInstance);
+            }
+        };
+        const workerPromise = this.startWorker(config, workerInstance);
+        workerInstance.promise = workerPromise;
+        workersInstance.workers.push(workerInstance);
+    }
     async runWorkers(config, workersInstance) {
         logger_1.logger.debug('Will ensure records for recurrent jobs');
         await this.ensureRecords(config);
         for (const workerIndex of (0, lodash_1.range)(config.workersCount)) {
-            logger_1.logger.debug(`Starting worker ${workerIndex}`);
-            const workerPromise = this.startWorker(config, workersInstance);
-            workersInstance.workers.push(workerPromise);
+            this.startANewWorker(config, workersInstance);
         }
     }
     startWorkers(userConfig) {

package/lib/stale.test.d.ts

@@ -0,0 +1 @@
+export {};

package/lib/stale.test.js

@@ -0,0 +1,108 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const helpers_1 = require("@orion-js/helpers");
+const logger_1 = require("@orion-js/logger");
+const _1 = require(".");
+(0, logger_1.setLogLevel)('none');
+describe('Stale Jobs Management', () => {
+    it('Should spawn a new worker when a job is stale and kill the stale worker after it ends', async () => {
+        const jobName1 = 'job1' + (0, helpers_1.generateId)();
+        const job1 = (0, _1.defineJob)({
+            type: 'event',
+            async resolve(_, context) {
+                if (context.tries === 1) {
+                    await (0, helpers_1.sleep)(100);
+                }
+                return { status: 'finished' };
+            }
+        });
+        const jobName2 = 'job2' + (0, helpers_1.generateId)();
+        const job2 = (0, _1.defineJob)({
+            type: 'event',
+            async resolve() {
+                return { status: 'finished' };
+            }
+        });
+        const instance = (0, _1.startWorkers)({
+            jobs: { [jobName1]: job1, [jobName2]: job2 },
+            workersCount: 1,
+            pollInterval: 5,
+            cooldownPeriod: 5,
+            lockTime: 10
+        });
+        await (0, _1.scheduleJob)({ name: jobName1 });
+        await (0, _1.scheduleJob)({ name: jobName2 });
+        await (0, helpers_1.sleep)(150);
+        await instance.stop();
+        const executions1 = await _1.jobsHistoryRepo.getExecutions(jobName1);
+        expect(executions1.length).toBe(2);
+        const executions2 = await _1.jobsHistoryRepo.getExecutions(jobName2);
+        expect(executions2.length).toBe(1);
+        expect(instance.workers.length).toBe(2);
+    });
+    it('Should run stale jobs in the lowest priority', async () => {
+        const executions = [];
+        const priotities = [];
+        const jobName1 = 'job1' + (0, helpers_1.generateId)();
+        const job1 = (0, _1.defineJob)({
+            type: 'event',
+            async resolve(_, context) {
+                executions.push('stale');
+                priotities.push(context.record.priority);
+                if (context.tries === 1) {
+                    await (0, helpers_1.sleep)(100);
+                }
+            }
+        });
+        const jobName2 = 'job2' + (0, helpers_1.generateId)();
+        const job2 = (0, _1.defineJob)({
+            type: 'event',
+            async resolve() {
+                executions.push('success');
+            }
+        });
+        const instance = (0, _1.startWorkers)({
+            jobs: { [jobName1]: job1, [jobName2]: job2 },
+            workersCount: 1,
+            pollInterval: 5,
+            cooldownPeriod: 5,
+            lockTime: 10
+        });
+        await (0, _1.scheduleJob)({ name: jobName1 });
+        await (0, _1.scheduleJob)({ name: jobName2 });
+        await (0, helpers_1.sleep)(150);
+        await instance.stop();
+        expect(priotities).toEqual([100, 0]);
+        expect(executions).toEqual(['stale', 'success', 'stale']);
+    });
+    it('Should revert to original priority when execution was stale on recurrent jobs', async () => {
+        const priotities = [];
+        let didStale = false;
+        const jobName = 'job' + (0, helpers_1.generateId)();
+        const job = (0, _1.defineJob)({
+            type: 'recurrent',
+            runEvery: 5,
+            async resolve(_, context) {
+                priotities.push(context.record.priority);
+                if (!didStale) {
+                    didStale = true;
+                    await (0, helpers_1.sleep)(100);
+                }
+            }
+        });
+        const instance = (0, _1.startWorkers)({
+            jobs: { [jobName]: job },
+            workersCount: 1,
+            pollInterval: 5,
+            cooldownPeriod: 5,
+            lockTime: 10
+        });
+        await (0, helpers_1.sleep)(150);
+        await instance.stop();
+        const [first, second, ...others] = priotities;
+        expect(first).toBe(100);
+        expect(second).toBe(0);
+        expect(others.length).toBeGreaterThan(0);
+        expect(others).toEqual(Array(others.length).fill(100));
+    });
+});

package/lib/types/HistoryRecord.d.ts

@@ -3,6 +3,7 @@ export declare type PlainObject = {
 };
 export declare class HistoryRecord {
     _id: string;
+    jobId: string;
     executionId: string;
     jobName: string;
     type: 'recurrent' | 'event';

package/lib/types/HistoryRecord.js

@@ -17,6 +17,10 @@ __decorate([
     (0, typed_model_1.Prop)(),
     __metadata("design:type", String)
 ], HistoryRecord.prototype, "_id", void 0);
+__decorate([
+    (0, typed_model_1.Prop)(),
+    __metadata("design:type", String)
+], HistoryRecord.prototype, "jobId", void 0);
 __decorate([
     (0, typed_model_1.Prop)(),
     __metadata("design:type", String)

package/lib/types/JobsDefinition.d.ts

@@ -42,7 +42,7 @@ export interface RecurrentJobDefinition extends BaseJobDefinition {
      */
     runEvery?: number;
     /**
-     * The priority of the job. Higher is more priority. Default is 1.
+     * The priority of the job. Higher is more priority. Default is 100.
      */
     priority?: number;
 }

package/lib/types/Worker.d.ts

@@ -3,6 +3,7 @@ import { PlainObject } from './HistoryRecord';
 import { JobDefinition } from './JobsDefinition';
 export interface JobToRun {
     jobId: string;
+    executionId: string;
     name: string;
     type: 'event' | 'recurrent';
     params: PlainObject;
@@ -19,10 +20,17 @@ export interface ExecutionContext {
     extendLockTime: (extraTime: number) => Promise<void>;
     clearStaleTimeout: () => void;
 }
+export interface WorkerInstance {
+    running: boolean;
+    workerIndex: number;
+    stop: () => Promise<void>;
+    respawn: () => Promise<void>;
+    promise?: Promise<any>;
+}
 export interface WorkersInstance {
     running: boolean;
     workersCount: number;
-    workers: Promise<any>[];
+    workers: WorkerInstance[];
     /**
      * Stop all workers and wait for them to finish
      */

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@orion-js/dogs",
-  "version": "3.1.23",
+  "version": "3.1.26",
   "main": "lib/index.js",
   "types": "lib/index.d.ts",
   "files": [
@@ -18,7 +18,7 @@
   },
   "dependencies": {
     "@orion-js/helpers": "^3.1.12",
-    "@orion-js/mongodb": "^3.1.23",
+    "@orion-js/mongodb": "^3.1.26",
     "@orion-js/services": "^3.1.12",
     "@orion-js/typed-model": "^3.1.13"
   },
@@ -39,5 +39,5 @@
   "publishConfig": {
     "access": "public"
   },
-  "gitHead": "0194171f6f326dfa921cf6e9f341e5b733b7d11e"
+  "gitHead": "90c31b03722cbd10329e65376b9354fb08d7e852"
 }