@clairejs/server 3.26.1 → 3.27.0

package/README.md

@@ -1,5 +1,13 @@
 ## Change Log
 
+#### 3.27.0
+
+-   use aws scheduler for jobs
+
+#### 3.26.2
+
+-   clean up jobs after run
+
 #### 3.26.1
 
 -   remove the resolve url function in file upload handler

package/dist/job/AbstractJobScheduler.d.ts

@@ -29,6 +29,11 @@ export declare abstract class AbstractJobScheduler {
      * @param id The job id returned from scheduleJobAt function
      */
     protected abstract cancelJob(id: string): Promise<void>;
+    /**
+     * Clean up the job after it has been executed
+     * @param job The job to clean up
+     */
+    protected abstract cleanupJob(job: AbstractJob): Promise<void>;
     /**
      * Remove the scheduled job and prevent if from running in the future
      * @param id The job id returned from scheduleJobAt function

package/dist/job/AbstractJobScheduler.js

@@ -129,6 +129,8 @@ export class AbstractJobScheduler {
                 await jobHandler.handlerFn(job);
                 //-- job run success, update
                 update.lastSuccessAt = Date.now();
+                //-- clean up
+                await this.cleanupJob(job);
                 //-- reset retry count if this is cron job
                 if (job.cron) {
                     if (job.retryCount) {

package/dist/job/AwsJobScheduler.d.ts

@@ -1,24 +1,21 @@
 import { AbstractLogger } from "@clairejs/core";
 import { AbstractDbAdapter } from "@clairejs/orm";
-import aws from "aws-sdk";
+import { SchedulerClient } from "@aws-sdk/client-scheduler";
 import { AbstractJob } from "./interfaces";
 import { AbstractJobScheduler } from "./AbstractJobScheduler";
 import { AbstractJobRepository } from "./AbstractJobRepository";
-/**
- * Following EventBrige permissions is required: listRules, putRule, deleteRule, listTargetsByRule, putTargets, removeTargets.
- * Also update Lambda function to alow function invocation from events.amazonaws.com.
- */
 export declare class AwsJobScheduler extends AbstractJobScheduler {
     protected readonly logger: AbstractLogger;
     protected readonly db: AbstractDbAdapter;
     protected readonly jobRepo: AbstractJobRepository;
     protected readonly apiLambdaFunctionArn: string;
+    protected readonly apiLambdaFunctionRoleArn: string;
     protected readonly jobNamespace: string;
-    protected readonly eventBusName: string;
-    protected readonly eventbridge: aws.EventBridge;
-    constructor(logger: AbstractLogger, db: AbstractDbAdapter, jobRepo: AbstractJobRepository, apiLambdaFunctionArn: string, jobNamespace: string, eventBusName?: string);
+    protected readonly scheduler: SchedulerClient;
+    constructor(logger: AbstractLogger, db: AbstractDbAdapter, jobRepo: AbstractJobRepository, apiLambdaFunctionArn: string, apiLambdaFunctionRoleArn: string, jobNamespace: string);
     handleCron(jobInfo: AbstractJob): Promise<void>;
-    private generateCronFromTimestamp;
+    private convertCronToSchedulerExpression;
+    protected cleanupJob(job: AbstractJob): Promise<void>;
     protected getScheduledJobs(): Promise<AbstractJob[]>;
     protected _scheduleJob(jobInfo: AbstractJob): Promise<void>;
     cancelJob(jobId: string): Promise<void>;

package/dist/job/AwsJobScheduler.js

@@ -9,143 +9,173 @@ var __metadata = (this && this.__metadata) || function (k, v) {
 };
 import { AbstractLogger, Errors, LogContext } from "@clairejs/core";
 import { AbstractDbAdapter } from "@clairejs/orm";
-import assert from "assert";
-import aws from "aws-sdk";
+import { SchedulerClient, CreateScheduleCommand, DeleteScheduleCommand, GetScheduleCommand, ListSchedulesCommand, ScheduleState } from "@aws-sdk/client-scheduler";
 import { AbstractJobScheduler } from "./AbstractJobScheduler";
 import { AbstractJobRepository } from "./AbstractJobRepository";
 /**
- * Following EventBrige permissions is required: listRules, putRule, deleteRule, listTargetsByRule, putTargets, removeTargets.
- * Also update Lambda function to alow function invocation from events.amazonaws.com.
+ * AWS Scheduler API permissions required:
+ * scheduler:CreateSchedule, scheduler:DeleteSchedule, scheduler:GetSchedule,
+ * scheduler:ListSchedules, scheduler:UpdateSchedule
+ * Also update Lambda function to allow function invocation from scheduler.amazonaws.com.
  */
+const schedulerGroupName = "default";
 let AwsJobScheduler = class AwsJobScheduler extends AbstractJobScheduler {
     logger;
     db;
     jobRepo;
     apiLambdaFunctionArn;
+    apiLambdaFunctionRoleArn;
     jobNamespace;
-    eventBusName;
-    eventbridge = new aws.EventBridge();
-    constructor(logger, db, jobRepo, apiLambdaFunctionArn, jobNamespace, eventBusName = "default") {
+    scheduler = new SchedulerClient();
+    constructor(logger, db, jobRepo, apiLambdaFunctionArn, apiLambdaFunctionRoleArn, jobNamespace) {
         super(logger, db, jobRepo);
         this.logger = logger;
         this.db = db;
         this.jobRepo = jobRepo;
         this.apiLambdaFunctionArn = apiLambdaFunctionArn;
+        this.apiLambdaFunctionRoleArn = apiLambdaFunctionRoleArn;
         this.jobNamespace = jobNamespace;
-        this.eventBusName = eventBusName;
     }
     async handleCron(jobInfo) {
         this.logger.debug(`Handle cron`, jobInfo);
         await this.executeJob(jobInfo);
     }
-    generateCronFromTimestamp(at) {
-        const date = new Date(at);
-        return [
-            date.getUTCMinutes(),
-            date.getUTCHours(),
-            date.getUTCDate(),
-            date.getUTCMonth() + 1,
-            "?",
-            date.getUTCFullYear(),
-        ].join(" ");
+    convertCronToSchedulerExpression(cron) {
+        // AWS Scheduler uses a different cron format than standard cron
+        // Standard cron: minute hour day month day-of-week
+        // AWS Scheduler: minute hour day month day-of-week year
+        const parts = cron.split(" ");
+        if (parts.length === 5) {
+            // Add year field if not present
+            return `${cron} *`;
+        }
+        else if (parts.length === 6) {
+            return cron;
+        }
+        else {
+            throw Errors.SYSTEM_ERROR(`Invalid cron expression: ${cron}`);
+        }
+    }
+    async cleanupJob(job) {
+        if (job.at) {
+            await this.cancelJob(job.id);
+        }
     }
     async getScheduledJobs() {
-        const allRules = await this.eventbridge
-            .listRules({
-            EventBusName: this.eventBusName,
-            NamePrefix: this.jobNamespace,
-        })
-            .promise();
-        const allJobs = await Promise.all((allRules.Rules || []).map(async (rule) => {
-            const ruleName = rule.Name;
-            const targets = await this.eventbridge
-                .listTargetsByRule({
-                EventBusName: this.eventBusName,
-                Rule: ruleName,
-            })
-                .promise();
-            const lambdaTarget = targets.Targets?.find((t) => t.Id === ruleName);
-            if (!lambdaTarget || !lambdaTarget.Input) {
-                return undefined;
-            }
-            const jobInfo = JSON.parse(lambdaTarget.Input).requestContext.cronScheduler.data;
-            return jobInfo;
-        }));
-        //-- concat with interval jobs because we don't register them
-        return allJobs.flatMap((job) => (job ? [job] : []));
+        try {
+            const listCommand = new ListSchedulesCommand({
+                GroupName: schedulerGroupName,
+                NamePrefix: this.jobNamespace,
+            });
+            const allSchedules = await this.scheduler.send(listCommand);
+            const allJobs = await Promise.all((allSchedules.Schedules || []).map(async (schedule) => {
+                const scheduleName = schedule.Name;
+                try {
+                    const getCommand = new GetScheduleCommand({
+                        Name: scheduleName,
+                        GroupName: schedulerGroupName,
+                    });
+                    const scheduleDetails = await this.scheduler.send(getCommand);
+                    if (!scheduleDetails.Target?.Arn || !scheduleDetails.Target?.Input) {
+                        return undefined;
+                    }
+                    const jobInfo = JSON.parse(scheduleDetails.Target.Input).requestContext.cronScheduler.data;
+                    return jobInfo;
+                }
+                catch (error) {
+                    this.logger.warn(`Failed to get schedule details for ${scheduleName}`, error);
+                    return undefined;
+                }
+            }));
+            return allJobs.flatMap((job) => (job ? [job] : []));
+        }
+        catch (error) {
+            this.logger.error("Failed to get scheduled jobs", error);
+            return [];
+        }
     }
     async _scheduleJob(jobInfo) {
         this.logger.debug("Scheduling job: ", jobInfo);
         if (jobInfo.cron || jobInfo.at) {
             const jobId = `${this.jobNamespace}${jobInfo.id}`;
-            let cronExpression;
+            let scheduleExpression;
+            let flexibleTimeWindow;
             if (jobInfo.at) {
-                cronExpression = this.generateCronFromTimestamp(jobInfo.at);
+                // For one-time jobs, use at() expression
+                const date = new Date(jobInfo.at);
+                scheduleExpression = `at(${date.toISOString()})`;
+                // Add flexible time window for one-time jobs to handle slight delays
+                flexibleTimeWindow = {
+                    Mode: "OFF",
+                };
             }
             else if (jobInfo.cron) {
-                let cron = jobInfo.cron;
-                if (cron.endsWith(" *")) {
-                    cron = `${cron.slice(0, -2)} ?`;
-                }
-                cronExpression = `${cron} *`;
+                // For recurring jobs, use cron() expression
+                const schedulerCron = this.convertCronToSchedulerExpression(jobInfo.cron);
+                scheduleExpression = `cron(${schedulerCron})`;
+                // Add flexible time window for recurring jobs
+                flexibleTimeWindow = {
+                    Mode: "FLEXIBLE",
+                    MaximumWindowInMinutes: 5, // Allow 5-minute window for execution
+                };
+            }
+            else {
+                throw Errors.SYSTEM_ERROR(`Job does not have time config: ${jobInfo.jobName}`);
             }
-            this.logger.debug("Cron expression", cronExpression);
-            assert.ok(cronExpression);
-            await this.eventbridge
-                .putRule({
+            this.logger.debug("Schedule expression", scheduleExpression);
+            const scheduleParams = {
                 Name: jobId,
-                Description: `${jobInfo.jobName} - ${cronExpression || jobInfo.at}`,
-                EventBusName: this.eventBusName,
-                ScheduleExpression: `cron(${cronExpression})`,
-                State: "ENABLED",
-            })
-                .promise();
-            //-- this rule has only one target, which is the api lambda function, so id of target can be the same as id of rule
-            await this.eventbridge
-                .putTargets({
-                Rule: jobId,
-                Targets: [
-                    {
-                        Arn: this.apiLambdaFunctionArn,
-                        Id: jobId,
-                        //-- this json will be passed to lambda function as event input
-                        Input: JSON.stringify({
-                            requestContext: {
-                                cronScheduler: {
-                                    data: { ...jobInfo, jobId },
-                                },
+                GroupName: schedulerGroupName,
+                Description: `${jobInfo.jobName} - ${scheduleExpression}`,
+                ScheduleExpression: scheduleExpression,
+                FlexibleTimeWindow: flexibleTimeWindow,
+                Target: {
+                    Arn: this.apiLambdaFunctionArn,
+                    RoleArn: this.apiLambdaFunctionRoleArn,
+                    Input: JSON.stringify({
+                        requestContext: {
+                            cronScheduler: {
+                                data: { ...jobInfo, jobId },
                             },
-                        }),
-                    },
-                ],
-            })
-                .promise();
+                        },
+                    }),
+                },
+                State: ScheduleState.ENABLED,
+            };
+            try {
+                const createCommand = new CreateScheduleCommand(scheduleParams);
+                await this.scheduler.send(createCommand);
+                this.logger.debug(`Successfully created schedule: ${jobId}`);
+            }
+            catch (error) {
+                this.logger.error(`Failed to create schedule: ${jobId}`, error);
+                throw error;
+            }
         }
         else {
             throw Errors.SYSTEM_ERROR(`Job does not have time config: ${jobInfo.jobName}`);
         }
     }
     async cancelJob(jobId) {
-        const ruleId = `${this.jobNamespace}${jobId}`;
-        await this.eventbridge
-            .removeTargets({
-            EventBusName: this.eventBusName,
-            Rule: ruleId,
-            Ids: [ruleId],
-        })
-            .promise();
-        await this.eventbridge
-            .deleteRule({
-            EventBusName: this.eventBusName,
-            Name: ruleId,
-        })
-            .promise();
+        const scheduleName = `${this.jobNamespace}${jobId}`;
+        try {
+            const deleteCommand = new DeleteScheduleCommand({
+                Name: scheduleName,
+                GroupName: schedulerGroupName,
+            });
+            await this.scheduler.send(deleteCommand);
+            this.logger.debug(`Successfully deleted schedule: ${scheduleName}`);
+        }
+        catch (error) {
+            this.logger.error(`Failed to delete schedule: ${scheduleName}`, error);
+            throw error;
+        }
     }
 };
 AwsJobScheduler = __decorate([
     LogContext(),
     __metadata("design:paramtypes", [AbstractLogger,
         AbstractDbAdapter,
-        AbstractJobRepository, String, String, Object])
+        AbstractJobRepository, String, String, String])
 ], AwsJobScheduler);
 export { AwsJobScheduler };

package/dist/job/LocalJobScheduler.d.ts

@@ -50,6 +50,7 @@ export declare class LocalJobScheduler extends AbstractJobScheduler implements I
     private sendJob;
     private processMessage;
     private extendMutexKey;
+    protected cleanupJob(): Promise<void>;
     init(): Promise<void>;
     exit(): void;
     protected getScheduledJobs(): Promise<AbstractJob[]>;

package/dist/job/LocalJobScheduler.js

@@ -103,6 +103,9 @@ let LocalJobScheduler = class LocalJobScheduler extends AbstractJobScheduler {
         await this.redisClient.setex(this.holdMutexKey, this.keyRetentionDurationSeconds, 1);
         this.logger.debug("Scheduler extends mutex key");
     }
+    async cleanupJob() {
+        //-- do nothing
+    }
     async init() {
         this.logger.debug("LocalJobScheduler init");
         //-- subscribe to multi client channel

package/dist/system/LambdaWrapper.js

@@ -106,6 +106,7 @@ export class LambdaWrapper {
                 return toApiGatewayFormat({ code: 200, headers: corsHeaders, cookies: {} });
             }
             if (requestOptions.method === CRON_REQUEST_METHOD) {
+                console.log("handle cron", requestOptions.body);
                 await this.jobScheduler?.handleCron(requestOptions.body);
                 return toApiGatewayFormat({ code: 200, headers: corsHeaders, cookies: {} });
             }

package/dist/system/lamba-request-mapper.js

@@ -11,6 +11,7 @@ const parseCookie = (cookieArray) => {
     return result;
 };
 export const lambdaRequestMapper = (event) => {
+    console.log("lambda event", event);
     if (event.requestContext.eventType) {
         //-- socket
         const method = event.requestContext.eventType.toLowerCase();

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@clairejs/server",
-    "version": "3.26.1",
+    "version": "3.27.0",
     "description": "Claire server NodeJs framework written in Typescript.",
     "types": "dist/index.d.ts",
     "main": "dist/index.js",
@@ -15,6 +15,7 @@
     "author": "immort",
     "license": "ISC",
     "dependencies": {
+        "@aws-sdk/client-scheduler": "^3.848.0",
         "aws-sdk": "^2.841.0",
         "axios": "^1.6.2",
         "cookie-parser": "^1.4.6",