@clairejs/server 3.15.1 → 3.16.0

package/dist/job/AwsJobScheduler.js

@@ -0,0 +1,405 @@
+var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
+    var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
+    if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
+    else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
+    return c > 3 && r && Object.defineProperty(target, key, r), r;
+};
+var __metadata = (this && this.__metadata) || function (k, v) {
+    if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
+};
+import { AbstractLogger, Errors, LogContext } from "@clairejs/core";
+import aws from "aws-sdk";
+import Redis from "ioredis";
+import { AbstractJobScheduler } from "./AbstractJobScheduler";
+import { JobInterval } from "./interfaces";
+const gcdBetween = (a, b) => {
+    const greater = a > b ? a : b;
+    const smaller = a > b ? b : a;
+    if (greater <= 0 || smaller <= 0) {
+        return 1;
+    }
+    else if (greater % smaller === 0) {
+        return smaller;
+    }
+    else {
+        const result = Math.trunc(greater / smaller);
+        return gcdBetween(smaller, greater - smaller * result);
+    }
+};
+const gcdOf = (val, arr) => {
+    if (!arr.length) {
+        return val;
+    }
+    else {
+        return gcdOf(gcdBetween(val, arr[0]), arr.slice(1));
+    }
+};
+const getWaitExpression = (interval) => `Wait ${interval} seconds`;
+const oneMinuteFunctionFactory = (lambdaFunctionARN, intervals) => {
+    //-- find most greatest common divisor of intervals
+    let gcd = JobInterval.EVERY_30S;
+    if (intervals.length) {
+        //-- re-calculate gcd
+        gcd = gcdOf(intervals[0], intervals.slice(1));
+        if (gcd < JobInterval.EVERY_5S) {
+            gcd = JobInterval.EVERY_5S;
+        }
+    }
+    //-- create loop array
+    const itemCount = Math.trunc(60 / gcd);
+    const itemArray = [];
+    for (let i = 0; i < itemCount; i++) {
+        itemArray.push(i * gcd);
+    }
+    const definition = `
+    {
+        "StartAt": "Create items",
+        "States": {
+          "Create items": {
+            "Type": "Pass",
+            "Next": "Loop",
+            "Result": ${JSON.stringify(itemArray)}
+          },
+          "Loop": {
+            "Type": "Map",
+            "End": true,
+            "Iterator": {
+              "StartAt": "${getWaitExpression(gcd)}",
+              "States": {
+                "${getWaitExpression(gcd)}": {
+                  "Type": "Wait",
+                  "Seconds": ${gcd},
+                  "Next": "Convert time to request object"
+                },
+                "Convert time to request object": {
+                  "Type": "Pass",
+                  "Next": "Lambda Invoke",
+                  "Result": {
+                    "requestContext": {
+                      "intervalScheduler": {
+                        "time": "$"
+                      }
+                    }
+                  }
+                },
+                "Lambda Invoke": {
+                  "Type": "Task",
+                  "Resource": "arn:aws:states:::lambda:invoke",
+                  "Parameters": {
+                    "Payload.$": "$",
+                    "FunctionName": "${lambdaFunctionARN}:$LATEST"
+                  },
+                  "End": true,
+                  "OutputPath": "$.Payload"
+                }
+              }
+            },
+            "MaxConcurrency": 1
+          }
+        },
+        "Comment": "One minute loop to trigger lambda function"
+      }
+    `;
+    return [gcd, definition];
+};
+let AwsJobScheduler = class AwsJobScheduler extends AbstractJobScheduler {
+    logger;
+    redisClient;
+    uniqueIdKey;
+    apiLambdaFunctionArn;
+    stepFunctionName;
+    iamRoleArn;
+    eventBusName;
+    jobNamePrefix;
+    oneMinuteRule;
+    eventbridge = new aws.EventBridge();
+    stepfunctions = new aws.StepFunctions({ apiVersion: "2016-11-23" });
+    constructor(logger, redisClient, uniqueIdKey, apiLambdaFunctionArn, stepFunctionName, 
+    /**
+     * This IAM role must have following permissions:
+     * - trigger any state machine (for one minute rule - as we don't now the state machine ARN until auto creation)
+     * - trigger lambda function specified in apiLambdaFunctionArn (so state machine can trigger API)
+     *
+     * In addition, the role which this API server assumes must have following permissions:
+     * - create event bridge rules, describe rules, remove rule, list and create rule targets
+     * - create state machines, list state machines
+     */
+    iamRoleArn, eventBusName = "default", jobNamePrefix = "claire-aws-job-", oneMinuteRule = "one-minute-step-function-trigger") {
+        super(logger);
+        this.logger = logger;
+        this.redisClient = redisClient;
+        this.uniqueIdKey = uniqueIdKey;
+        this.apiLambdaFunctionArn = apiLambdaFunctionArn;
+        this.stepFunctionName = stepFunctionName;
+        this.iamRoleArn = iamRoleArn;
+        this.eventBusName = eventBusName;
+        this.jobNamePrefix = jobNamePrefix;
+        this.oneMinuteRule = oneMinuteRule;
+    }
+    async handleInterval(interval) {
+        this.logger.debug(`Handle interval`, interval, typeof interval);
+        const allJobs = await this.getAvailableJobInfo();
+        const timedJobs = allJobs.filter((job) => job.interval && interval % job.interval === 0);
+        await Promise.all(timedJobs.map((job) => this.executeJob({ ...job, jobId: job.jobName })));
+    }
+    async handleCron(jobInfo) {
+        this.logger.debug(`Handle cron`, jobInfo);
+        await this.executeJob(jobInfo);
+    }
+    generateCronFromTimestamp(at) {
+        const date = new Date(at);
+        return [
+            date.getUTCMinutes(),
+            date.getUTCHours(),
+            date.getUTCDate(),
+            date.getUTCMonth(),
+            "?",
+            date.getUTCFullYear(),
+        ].join(" ");
+    }
+    isActiveScheduler() {
+        return false;
+    }
+    async getAllScheduledJobs() {
+        const availableJobs = await this.getAvailableJobInfo();
+        const allRules = await this.eventbridge
+            .listRules({
+            EventBusName: this.eventBusName,
+            NamePrefix: this.jobNamePrefix,
+        })
+            .promise();
+        const allJobs = await Promise.all((allRules.Rules || []).map(async (rule) => {
+            const ruleName = rule.Name;
+            const targets = await this.eventbridge
+                .listTargetsByRule({
+                EventBusName: this.eventBusName,
+                Rule: ruleName,
+            })
+                .promise();
+            const lambdaTarget = targets.Targets?.find((t) => t.Id === ruleName);
+            if (!lambdaTarget || !lambdaTarget.Input) {
+                return undefined;
+            }
+            const jobInfo = JSON.parse(lambdaTarget.Input).requestContext.cronScheduler.data;
+            return jobInfo;
+        }));
+        //-- concat with interval jobs because we don't register them
+        return allJobs
+            .filter((job) => !!job)
+            .concat(availableJobs.filter((j) => j.interval).map((j) => ({ ...j, jobId: j.jobName })));
+    }
+    async scheduleJob(jobInfo) {
+        this.logger.debug("Scheduling job: ", jobInfo);
+        if (jobInfo.cron || jobInfo.at) {
+            const uniqueId = await this.redisClient.incr(this.uniqueIdKey);
+            const jobId = `${this.jobNamePrefix}${uniqueId}`;
+            //-- generate pattern from cron (add * at the end for year) / at timestamp
+            const cronExpression = jobInfo.cron ? `${jobInfo.cron} *` : this.generateCronFromTimestamp(jobInfo.at);
+            this.logger.debug("Cron expression", cronExpression);
+            await this.eventbridge
+                .putRule({
+                Name: jobId,
+                Description: `${jobInfo.jobName} - ${jobInfo.cron || jobInfo.at}`,
+                EventBusName: this.eventBusName,
+                ScheduleExpression: `cron(${cronExpression})`,
+                State: "ENABLED",
+            })
+                .promise();
+            //-- this rule has only one target, which is the api lambda function, so id of target can be the same as id of rule
+            await this.eventbridge
+                .putTargets({
+                Rule: jobId,
+                Targets: [
+                    {
+                        Arn: this.apiLambdaFunctionArn,
+                        Id: jobId,
+                        //-- this json will be passed to lambda function as event input
+                        Input: JSON.stringify({
+                            requestContext: {
+                                cronScheduler: {
+                                    data: { ...jobInfo, jobId },
+                                },
+                            },
+                        }),
+                    },
+                ],
+            })
+                .promise();
+            return jobId;
+        }
+        else if (jobInfo.interval) {
+            //-- interval job does not need to persist, the step function will take care of it
+            return jobInfo.jobName;
+        }
+        else {
+            throw Errors.SYSTEM_ERROR(`Job does not have time config: ${jobInfo.jobName}`);
+        }
+    }
+    async syncJobs() {
+        //-- check and init step function: step function must trigger lambda every 5s
+        this.logger.debug("Checking interval scheduler");
+        await this.checkIntervalScheduler();
+        //-- check jobs
+        const scheduledJobs = await this.getAllScheduledJobs();
+        const allJobs = await this.getAvailableJobInfo();
+        //-- remove job that no more exist
+        const nomoreExistJobs = scheduledJobs.filter((job) => !allJobs.find((j) => j.jobName === job.jobName));
+        for (const job of nomoreExistJobs) {
+            this.logger.info(`Removing stale job: ${job.jobName} of id: ${job.jobId}`);
+            await this.removeJob(job.jobId);
+        }
+        if (nomoreExistJobs.length) {
+            this.logger.info(`Cleaned up: ${nomoreExistJobs.length} stale jobs`);
+        }
+        //-- remove scheduled cron jobs
+        this.logger.debug("Remove scheduled cron jobs");
+        const scheduledCronJobs = scheduledJobs.filter((j) => j.cron);
+        await Promise.all(scheduledCronJobs.map((j) => this.removeJob(j.jobId)));
+        //-- reschedule cron & interval jobs because we might have updated the cron expression / interval value
+        const cronJobs = allJobs.filter((job) => job.cron || job.interval);
+        this.logger.debug("Scheduling cron & interval jobs");
+        for (const job of cronJobs) {
+            await this.scheduleJob({
+                jobName: job.jobName,
+                cron: job.cron,
+                interval: job.interval,
+            });
+        }
+        //-- keep at timestamp jobs as is
+    }
+    async removeJob(jobId) {
+        await this.eventbridge
+            .removeTargets({
+            EventBusName: this.eventBusName,
+            Rule: jobId,
+            Ids: [jobId],
+        })
+            .promise();
+        await this.eventbridge
+            .deleteRule({
+            EventBusName: this.eventBusName,
+            Name: jobId,
+        })
+            .promise();
+    }
+    async checkIntervalScheduler() {
+        //-- check and create the step function
+        const allJobs = await this.getAvailableJobInfo();
+        const intervalJobs = allJobs.filter((j) => j.interval);
+        const allIntervals = intervalJobs.map((j) => j.interval);
+        const [interval, oneMinuteFunctionDefinition] = oneMinuteFunctionFactory(this.apiLambdaFunctionArn, allIntervals);
+        this.logger.debug("Listing all state machines");
+        const allMachines = await this.stepfunctions.listStateMachines({}).promise();
+        let oldStateMachineArn = allMachines.stateMachines.find((s) => s.name.includes(this.stepFunctionName))?.stateMachineArn;
+        let newStateMachineArn = oldStateMachineArn;
+        if (oldStateMachineArn) {
+            //-- check definition, if different then clear newStateMachineArn to create new one
+            const describeResult = await this.stepfunctions
+                .describeStateMachine({
+                stateMachineArn: oldStateMachineArn,
+            })
+                .promise();
+            //-- check the interval
+            if (!describeResult.definition.includes(getWaitExpression(interval))) {
+                this.logger.debug("Step function definition changed, create new state machine");
+                newStateMachineArn = "";
+            }
+        }
+        const awsStateMachineName = `${this.stepFunctionName}-${interval}`;
+        if (!newStateMachineArn) {
+            this.logger.debug(`Create new step function with interval ${interval} seconds`);
+            const result = await this.stepfunctions
+                .createStateMachine({
+                definition: oneMinuteFunctionDefinition,
+                name: awsStateMachineName,
+                roleArn: this.iamRoleArn,
+                type: "EXPRESS",
+            })
+                .promise();
+            newStateMachineArn = result.stateMachineArn;
+        }
+        this.logger.debug("Step function ARNs old / new", oldStateMachineArn, newStateMachineArn);
+        //-- check and create the one-minute-rule that trigger step function
+        this.logger.debug("Getting one minute rule");
+        const matchedRules = await this.eventbridge
+            .listRules({
+            EventBusName: this.eventBusName,
+            NamePrefix: this.oneMinuteRule,
+        })
+            .promise();
+        const oneMinuteRule = matchedRules.Rules?.find((r) => r.Name == this.oneMinuteRule);
+        if (!oneMinuteRule) {
+            this.logger.debug("Create new one minute rule");
+            //-- one minute rule does not exist, create new
+            await this.eventbridge
+                .putRule({
+                EventBusName: this.eventBusName,
+                Name: this.oneMinuteRule,
+                Description: "One minute trigger function for step function",
+                ScheduleExpression: "rate(1 minute)",
+                State: "ENABLED",
+            })
+                .promise();
+        }
+        this.logger.debug("Adding step function state machine as target");
+        if (newStateMachineArn !== oldStateMachineArn) {
+            //-- if there is old target then remove it
+            if (oldStateMachineArn) {
+                this.logger.debug("Removing old target", oldStateMachineArn);
+                //-- remove old target
+                await this.eventbridge
+                    .removeTargets({
+                    EventBusName: this.eventBusName,
+                    Rule: this.oneMinuteRule,
+                    Ids: [this.stepFunctionName],
+                })
+                    .promise();
+                this.logger.debug("Removing old state machine function");
+                await this.stepfunctions
+                    .deleteStateMachine({
+                    stateMachineArn: oldStateMachineArn,
+                })
+                    .promise();
+            }
+            //-- add the step function as call target
+            this.logger.debug("Adding new target", newStateMachineArn);
+            await this.eventbridge
+                .putTargets({
+                Rule: this.oneMinuteRule,
+                Targets: [
+                    {
+                        Arn: newStateMachineArn,
+                        Id: this.stepFunctionName,
+                        RoleArn: this.iamRoleArn,
+                    },
+                ],
+            })
+                .promise();
+        }
+        //-- if we don't have any interval job then disable the one minuter
+        if (!intervalJobs.length && oneMinuteRule?.State) {
+            this.logger.info("No interval job found, disable one minute rule");
+            await this.eventbridge
+                .disableRule({
+                EventBusName: this.eventBusName,
+                Name: this.oneMinuteRule,
+            })
+                .promise();
+        }
+        else {
+            this.logger.info("Interval job found, enable one minute rule");
+            await this.eventbridge
+                .enableRule({
+                EventBusName: this.eventBusName,
+                Name: this.oneMinuteRule,
+            })
+                .promise();
+        }
+    }
+};
+AwsJobScheduler = __decorate([
+    LogContext(),
+    __metadata("design:paramtypes", [AbstractLogger,
+        Redis, String, String, String, String, Object, Object, Object])
+], AwsJobScheduler);
+export { AwsJobScheduler };

package/dist/job/LocalJobScheduler.js

@@ -0,0 +1,273 @@
+var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
+    var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
+    if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
+    else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
+    return c > 3 && r && Object.defineProperty(target, key, r), r;
+};
+var __metadata = (this && this.__metadata) || function (k, v) {
+    if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
+};
+import { AbstractLogger, Errors, LogContext } from "@clairejs/core";
+import Redis from "ioredis";
+import Redlock from "redlock";
+import scheduler from "node-schedule";
+import { AbstractJobScheduler } from "./AbstractJobScheduler";
+import { AbstractJobRepository } from "./AbstractJobRepository";
+import { clearInterval, clearTimeout } from "timers";
+var CommunicationMessage;
+(function (CommunicationMessage) {
+    CommunicationMessage["SCHEDULE_JOB"] = "SCHEDULE_JOB";
+    CommunicationMessage["REMOVE_JOB"] = "REMOVE_JOB";
+    CommunicationMessage["SYNC_JOB"] = "SYNC_JOB";
+    CommunicationMessage["NOTIFY"] = "NOTIFY";
+})(CommunicationMessage || (CommunicationMessage = {}));
+let LocalJobScheduler = class LocalJobScheduler extends AbstractJobScheduler {
+    logger;
+    redisClient;
+    subscribeClient;
+    jobRepo;
+    lockMutexKey;
+    holdMutexKey;
+    uniqueIdKey;
+    multiClientChannel;
+    keyRetentionDurationSecond;
+    intervals = [];
+    mutexHoldInterval;
+    isActive = false;
+    notifyResolver = {};
+    jobHolder = {};
+    constructor(logger, redisClient, subscribeClient, jobRepo, 
+    /**
+     * Redis lock key to select active scheduler
+     */
+    lockMutexKey, 
+    /**
+     * Redis key to hold active scheduler role
+     */
+    holdMutexKey, 
+    /**
+     * Redis key to get unique incremental id
+     */
+    uniqueIdKey, 
+    /**
+     * The channel for communication between passive and active schedulers
+     */
+    multiClientChannel, 
+    /**
+     * The time to lock active scheduler
+     */
+    keyRetentionDurationSecond = 30) {
+        super(logger);
+        this.logger = logger;
+        this.redisClient = redisClient;
+        this.subscribeClient = subscribeClient;
+        this.jobRepo = jobRepo;
+        this.lockMutexKey = lockMutexKey;
+        this.holdMutexKey = holdMutexKey;
+        this.uniqueIdKey = uniqueIdKey;
+        this.multiClientChannel = multiClientChannel;
+        this.keyRetentionDurationSecond = keyRetentionDurationSecond;
+    }
+    sendJob(type, messageId, data) {
+        this.subscribeClient.publish(this.multiClientChannel, JSON.stringify({ type, messageId, data }));
+    }
+    async processMessage(type, messageId, data) {
+        switch (type) {
+            case CommunicationMessage.SYNC_JOB:
+                await this.syncJobs();
+                this.sendJob(CommunicationMessage.NOTIFY, messageId);
+                break;
+            case CommunicationMessage.NOTIFY:
+                const resolver = this.notifyResolver[messageId];
+                if (!resolver) {
+                    //-- resolver not found, ignore
+                    return;
+                }
+                resolver(data);
+                break;
+            case CommunicationMessage.SCHEDULE_JOB:
+                if (!this.isActive) {
+                    //-- not active scheduler, ignore
+                    return;
+                }
+                const jobInfo = data;
+                const scheduledId = await this.scheduleJobAt(jobInfo);
+                this.sendJob(CommunicationMessage.NOTIFY, messageId, scheduledId);
+                break;
+            case CommunicationMessage.REMOVE_JOB:
+                if (!this.isActive) {
+                    //-- not active scheduler, ignore
+                    return;
+                }
+                const jobId = data;
+                await this.removeJob(jobId);
+                this.sendJob(CommunicationMessage.NOTIFY, messageId);
+                break;
+            default:
+                this.logger.error(`Not recognize message type ${type}`);
+                break;
+        }
+    }
+    async extendMutexKey() {
+        //-- set expire the mutex key
+        if (!this.redisClient) {
+            return;
+        }
+        await this.redisClient.setex(this.holdMutexKey, this.keyRetentionDurationSecond, 1);
+        this.logger.debug("Scheduler extends mutex key");
+    }
+    async init() {
+        this.logger.debug("LocalJobScheduler init");
+        //-- subscribe to multi client channel
+        this.logger.debug("Listening on multi client channel");
+        this.redisClient.on("message", (channel, message) => {
+            if (channel === this.multiClientChannel) {
+                //-- process message
+                const payload = JSON.parse(message);
+                this.processMessage(payload.type, payload.messageId, payload.data).catch((err) => this.logger.error(`Fail to process message, ${payload}`, err));
+            }
+        });
+        await this.subscribeClient.subscribe(this.multiClientChannel);
+        this.logger.debug("Try to claim active scheduler");
+        //-- try to claim active scheduler
+        const locker = new Redlock([this.redisClient]);
+        try {
+            const lock = await locker.acquire([this.lockMutexKey], this.keyRetentionDurationSecond);
+            this.isActive = true;
+            await this.extendMutexKey();
+            await lock.release();
+            this.logger.debug("Being active scheduler");
+            //-- actively hold the mutex key
+            this.mutexHoldInterval = setInterval(() => {
+                this.extendMutexKey();
+            }, Math.trunc((this.keyRetentionDurationSecond * 1000) / 2) + 1);
+        }
+        catch (err) {
+            this.logger.info("Failed to lock mutex key, ignore", err);
+        }
+    }
+    exit() {
+        if (this.mutexHoldInterval) {
+            clearInterval(this.mutexHoldInterval);
+        }
+        this.redisClient.quit();
+        this.subscribeClient.quit();
+        for (const interval of this.intervals) {
+            clearInterval(interval);
+        }
+        this.logger.debug("LocalJobScheduler exit");
+    }
+    isActiveScheduler() {
+        return this.isActive;
+    }
+    async getAllScheduledJobs() {
+        return Object.values(this.jobHolder)
+            .filter((j) => !!j?.jobInfo)
+            .map((info) => info?.jobInfo);
+    }
+    async syncJobs() {
+        if (this.isActive) {
+            //-- schedule all cron & interval jobs
+            const allJobs = await this.getAvailableJobInfo();
+            for (const job of allJobs) {
+                if (job.cron || job.interval) {
+                    await this.scheduleJob(job);
+                }
+            }
+            //-- re-schedule jobs that are stored in repo
+            const allPersistedJobs = await this.jobRepo.getJobs();
+            //-- run job anyway, expired job will be removed then
+            for (const job of allPersistedJobs) {
+                await this.scheduleJob(job);
+            }
+        }
+        else {
+            const uniqueMessageId = await this.redisClient.incr(this.uniqueIdKey);
+            return new Promise((resolve) => {
+                this.notifyResolver[uniqueMessageId] = resolve;
+                this.sendJob(CommunicationMessage.SYNC_JOB, uniqueMessageId);
+            });
+        }
+        return;
+    }
+    async scheduleJob(jobInfo) {
+        if (this.isActive) {
+            //-- case each job type
+            if (jobInfo.at) {
+                //-- create new schedule job
+                const jobId = await this.jobRepo.saveJob({
+                    jobName: jobInfo.jobName,
+                    params: jobInfo.params,
+                    at: jobInfo.at,
+                });
+                //-- use the lib
+                const scheduledJob = { ...jobInfo, jobId };
+                const timeout = setTimeout(() => {
+                    this.executeJob(scheduledJob).catch((err) => this.logger.error(`Error execute job ${scheduledJob.jobName} with id: ${scheduledJob.jobId}`, err));
+                }, new Date(jobInfo.at).getTime() - Date.now());
+                this.jobHolder[jobId] = { jobCanceler: () => clearTimeout(timeout), jobInfo: { ...jobInfo, jobId } };
+                return jobId;
+            }
+            else if (jobInfo.interval) {
+                const jobId = jobInfo.jobName;
+                //-- set interval and does not need to persist
+                const scheduledJob = { ...jobInfo, jobId };
+                const interval = setInterval(() => {
+                    this.executeJob(scheduledJob).catch((err) => this.logger.error(`Error execute job ${scheduledJob.jobName} with id: ${scheduledJob.jobId}`, err));
+                }, jobInfo.interval);
+                this.jobHolder[jobId] = { jobCanceler: () => clearInterval(interval), jobInfo: { ...jobInfo, jobId } };
+                return jobId;
+            }
+            else if (jobInfo.cron) {
+                const jobId = jobInfo.jobName;
+                //-- set cron and does not need to persist
+                const scheduledJob = { ...jobInfo, jobId };
+                const job = scheduler.scheduleJob(jobInfo.cron, () => {
+                    this.executeJob(scheduledJob).catch((err) => this.logger.error(`Error execute job ${scheduledJob.jobName} with id: ${scheduledJob.jobId}`, err));
+                });
+                this.jobHolder[jobId] = { jobCanceler: () => job.cancel(), jobInfo: { ...jobInfo, jobId } };
+                return jobId;
+            }
+            else {
+                throw Errors.SYSTEM_ERROR(`Job does not have time config: ${jobInfo.jobName}`);
+            }
+        }
+        else {
+            //-- get unique message id
+            const uniqueMessageId = await this.redisClient.incr(this.uniqueIdKey);
+            return new Promise((resolve) => {
+                this.notifyResolver[uniqueMessageId] = resolve;
+                this.sendJob(CommunicationMessage.SCHEDULE_JOB, uniqueMessageId, jobInfo);
+            });
+        }
+    }
+    async removeJob(jobId) {
+        if (this.isActive) {
+            //-- remove from holder
+            const job = this.jobHolder[jobId];
+            if (job) {
+                job.jobCanceler();
+                this.jobHolder[jobId] = undefined;
+            }
+            //-- remove from persistence
+            await this.jobRepo.removeJobById(jobId);
+            return;
+        }
+        else {
+            //-- get unique message id
+            const uniqueMessageId = await this.redisClient.incr(this.uniqueIdKey);
+            return new Promise((resolve) => {
+                this.notifyResolver[uniqueMessageId] = resolve;
+                this.sendJob(CommunicationMessage.REMOVE_JOB, uniqueMessageId, jobId);
+            });
+        }
+    }
+};
+LocalJobScheduler = __decorate([
+    LogContext(),
+    __metadata("design:paramtypes", [AbstractLogger,
+        Redis,
+        Redis,
+        AbstractJobRepository, String, String, String, String, Number])
+], LocalJobScheduler);
+export { LocalJobScheduler };