@adminforth/background-jobs 1.0.1

package/dist/index.js

@@ -0,0 +1,433 @@
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+import { AdminForthPlugin, Filters, Sorts } from "adminforth";
+import { afLogger } from "adminforth";
+import pLimit from 'p-limit';
+import { Level } from 'level';
+export default class extends AdminForthPlugin {
+    constructor(options) {
+        super(options, import.meta.url);
+        this.taskHandlers = {};
+        this.jobCustomComponents = {};
+        this.jobParallelLimits = {};
+        this.levelDbInstances = {};
+        this.options = options;
+        this.shouldHaveSingleInstancePerWholeApp = () => true;
+    }
+    getResourcePk() {
+        var _a;
+        const resourcePk = (_a = this.resourceConfig.columns.find(c => c.primaryKey)) === null || _a === void 0 ? void 0 : _a.name;
+        return resourcePk;
+    }
+    getResourceId() {
+        return this.resourceConfig.resourceId;
+    }
+    modifyResourceConfig(adminforth, resourceConfig) {
+        const _super = Object.create(null, {
+            modifyResourceConfig: { get: () => super.modifyResourceConfig }
+        });
+        return __awaiter(this, void 0, void 0, function* () {
+            var _a, _b;
+            _super.modifyResourceConfig.call(this, adminforth, resourceConfig);
+            if (!((_b = (_a = adminforth.config.customization) === null || _a === void 0 ? void 0 : _a.globalInjections) === null || _b === void 0 ? void 0 : _b.header)) {
+                adminforth.config.customization.globalInjections.header = [];
+            }
+            (adminforth.config.customization.globalInjections.header).push({
+                file: this.componentPath('NavbarJobs.vue'),
+                meta: {
+                    pluginInstanceId: this.pluginInstanceId,
+                }
+            });
+        });
+    }
+    checkIfFieldInResource(resourceConfig, fieldName, fieldString) {
+        if (!fieldName) {
+            throw new Error(`Field name for ${fieldString} is not provided. Please check your plugin options.`);
+        }
+        const fieldInConfig = resourceConfig.columns.find(f => f.name === fieldName);
+        if (!fieldInConfig) {
+            throw new Error(`Field ${fieldName} not found in resource config. Please check your plugin options.`);
+        }
+    }
+    createLevelDbTaskRecord(levelDb, taskId, initialState) {
+        return __awaiter(this, void 0, void 0, function* () {
+            //create record in level db with task id as key and initial state as value and status SCHEDULED
+            yield levelDb.put(taskId, JSON.stringify({ state: initialState, status: 'SCHEDULED' }));
+        });
+    }
+    setLevelDbTaskStateField(levelDb, taskId, state) {
+        return __awaiter(this, void 0, void 0, function* () {
+            //update record in level db with task id as key and new state as value
+            const status = yield this.getLevelDbTaskStatusField(levelDb, taskId);
+            yield levelDb.del(taskId);
+            yield levelDb.put(taskId, JSON.stringify({ state, status }));
+        });
+    }
+    setLevelDbTaskStatusField(levelDb, taskId, status) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const state = yield this.getLevelDbTaskStateField(levelDb, taskId);
+            yield levelDb.del(taskId);
+            yield levelDb.put(taskId, JSON.stringify({ state, status }));
+        });
+    }
+    getLevelDbTaskStateField(levelDb, taskId) {
+        return __awaiter(this, void 0, void 0, function* () {
+            //get record from level db with task id as key and return the value of the key in the state
+            const state = yield levelDb.get(taskId);
+            if (state) {
+                const parsedState = JSON.parse(state);
+                return parsedState.state;
+            }
+            return Promise.resolve(null);
+        });
+    }
+    getLevelDbTaskStatusField(levelDb, taskId) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const state = yield levelDb.get(taskId);
+            if (state) {
+                const parsedState = JSON.parse(state);
+                return parsedState.status;
+            }
+            return Promise.resolve(null);
+        });
+    }
+    registerTaskHandler(jobHandlerName, handler, customComponent, parrallelLimit = 3) {
+        //register the handler in a map with jobHandlerName as key and handler as value
+        this.taskHandlers[jobHandlerName] = handler;
+        this.jobParallelLimits[jobHandlerName] = parrallelLimit;
+        if (customComponent) {
+            this.jobCustomComponents[jobHandlerName] = customComponent;
+        }
+    }
+    startNewJob(jobName_1, adminUser_1, tasks_1) {
+        return __awaiter(this, arguments, void 0, function* (jobName, adminUser, tasks, initialFields = {}, jobHandlerName) {
+            const handleTask = this.taskHandlers[jobHandlerName];
+            if (!handleTask) {
+                throw new Error(`No handler registered for jobHandler ${jobHandlerName}. Please register a handler using the registerTaskHandler method before starting a job with this jobHandler.`);
+            }
+            const customComponent = this.jobCustomComponents[jobHandlerName];
+            const parrallelLimit = this.jobParallelLimits[jobHandlerName] || 3;
+            //create a record for the job in the database with status in progress
+            const objectToSave = {
+                [this.options.nameField]: jobName,
+                [this.options.startedByField]: adminUser.pk,
+                [this.options.stateField]: JSON.stringify(initialFields),
+                [this.options.progressField]: 0,
+                [this.options.statusField]: 'IN_PROGRESS',
+                [this.options.jobHandlerField]: jobHandlerName,
+            };
+            const creationResult = yield this.adminforth.resource(this.getResourceId()).create(objectToSave);
+            let createdRecord = null;
+            if (creationResult.ok === true) {
+                createdRecord = creationResult.createdRecord;
+            }
+            else {
+                throw new Error(`Failed to create a record for the job. Error: ${creationResult.error}`);
+            }
+            const jobId = createdRecord[this.getResourcePk()];
+            this.adminforth.websocket.publish('/background-jobs', {
+                jobId,
+                status: 'IN_PROGRESS',
+                name: jobName,
+                progress: 0,
+                createdAt: createdRecord[this.options.createdAtField],
+                customComponent,
+            });
+            //create a level db instance for the job with name as jobId
+            const jobLevelDb = new Level(`${this.options.levelDbPath || './background-jobs-dbs/'}job_${jobId}`, { valueEncoding: 'json' });
+            this.levelDbInstances[jobId] = jobLevelDb;
+            const limit2 = pLimit(parrallelLimit);
+            const createTaskRecordsPromises = tasks.map((task, index) => {
+                return limit2(() => this.createLevelDbTaskRecord(jobLevelDb, index.toString(), task.state));
+            });
+            yield Promise.all(createTaskRecordsPromises);
+            this.runProcessingTasks(tasks, jobLevelDb, jobId, handleTask, parrallelLimit);
+        });
+    }
+    runProcessingTasks(tasks, jobLevelDb, jobId, handleTask, parrallelLimit) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const totalTasks = tasks.length;
+            let completedTasks = 0;
+            let failedTasks = 0;
+            let lastJobStatus = 'IN_PROGRESS';
+            const taskHandler = (taskIndex, task) => __awaiter(this, void 0, void 0, function* () {
+                if (task.skip) {
+                    completedTasks = yield this.handleFinishTask(completedTasks, totalTasks, jobId, true);
+                    return;
+                }
+                if (lastJobStatus === 'CANCELLED') {
+                    afLogger.info(`Job ${jobId} was cancelled. Skipping task ${taskIndex}.`);
+                    return;
+                }
+                const currentJobStatus = yield this.getLastJobStatus(jobId);
+                if (currentJobStatus === 'CANCELLED') {
+                    lastJobStatus = currentJobStatus;
+                    afLogger.info(`Job ${jobId} was cancelled. Skipping task ${taskIndex}.`);
+                    return;
+                }
+                //define the setTaskStateField and getTaskStateField functions to pass to the task
+                const setTaskStateField = (state) => __awaiter(this, void 0, void 0, function* () {
+                    yield this.setLevelDbTaskStateField(jobLevelDb, taskIndex.toString(), state);
+                });
+                const getTaskStateField = () => __awaiter(this, void 0, void 0, function* () {
+                    return yield this.getLevelDbTaskStateField(jobLevelDb, taskIndex.toString());
+                });
+                yield this.setLevelDbTaskStatusField(jobLevelDb, taskIndex.toString(), 'IN_PROGRESS');
+                this.adminforth.websocket.publish(`/background-jobs-task-update/${jobId}`, { taskIndex, status: "IN_PROGRESS" });
+                //handling the task 
+                try {
+                    yield handleTask({ setTaskStateField, getTaskStateField });
+                    //Set task status to completed in level db
+                    yield this.setLevelDbTaskStatusField(jobLevelDb, taskIndex.toString(), 'DONE');
+                    this.adminforth.websocket.publish(`/background-jobs-task-update/${jobId}`, { taskIndex, status: "DONE" });
+                }
+                catch (error) {
+                    afLogger.error(`Error in handling task ${taskIndex} of job ${jobId}: ${error}`);
+                    yield this.setLevelDbTaskStatusField(jobLevelDb, taskIndex.toString(), 'FAILED');
+                    this.adminforth.websocket.publish(`/background-jobs-task-update/${jobId}`, { taskIndex, status: "FAILED" });
+                    failedTasks++;
+                    return;
+                }
+                finally {
+                    //Update progress
+                    const currentJobStatus = yield this.getLastJobStatus(jobId);
+                    if (currentJobStatus === 'CANCELLED') {
+                        lastJobStatus = currentJobStatus;
+                        afLogger.debug(`Job ${jobId} was cancelled during processing of task ${taskIndex}. Progress will not be updated.`);
+                        return;
+                    }
+                    completedTasks = yield this.handleFinishTask(completedTasks, totalTasks, jobId);
+                }
+            });
+            const limit = pLimit(parrallelLimit);
+            const tasksToExecute = tasks.map((task, taskIndex) => {
+                return limit(() => taskHandler(taskIndex, task));
+            });
+            yield Promise.all(tasksToExecute);
+            if (lastJobStatus !== 'CANCELLED' && failedTasks === 0) {
+                yield this.adminforth.resource(this.getResourceId()).update(jobId, {
+                    [this.options.statusField]: 'DONE',
+                });
+                this.adminforth.websocket.publish('/background-jobs', { jobId, status: 'DONE' });
+            }
+            else if (failedTasks > 0) {
+                yield this.adminforth.resource(this.getResourceId()).update(jobId, {
+                    [this.options.statusField]: 'DONE_WITH_ERRORS',
+                });
+                this.adminforth.websocket.publish('/background-jobs', { jobId, status: 'DONE_WITH_ERRORS' });
+            }
+        });
+    }
+    getLastJobStatus(jobId) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const currentJobRecord = yield this.adminforth.resource(this.getResourceId()).get(Filters.EQ(this.getResourcePk(), jobId));
+            const currentJobStatus = currentJobRecord[this.options.statusField];
+            return currentJobStatus;
+        });
+    }
+    handleFinishTask(completedTasks_1, totalTasks_1, jobId_1) {
+        return __awaiter(this, arguments, void 0, function* (completedTasks, totalTasks, jobId, wasTaskSkipped = false) {
+            completedTasks++;
+            if (wasTaskSkipped) {
+                return completedTasks;
+            }
+            const progress = Math.round((completedTasks / totalTasks) * 100);
+            yield this.adminforth.resource(this.getResourceId()).update(jobId, {
+                [this.options.progressField]: progress,
+            });
+            this.adminforth.websocket.publish('/background-jobs', { jobId, progress });
+            return completedTasks;
+        });
+    }
+    runProcessingUnfinishedTasks(job) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const levelDbPath = `${this.options.levelDbPath || './background-jobs-dbs/'}job_${job[this.getResourcePk()]}`;
+            const jobLevelDb = new Level(levelDbPath, { valueEncoding: 'json' });
+            this.levelDbInstances[job[this.getResourcePk()]] = jobLevelDb;
+            const jobHandlerName = job[this.options.jobHandlerField];
+            const handleTask = this.taskHandlers[jobHandlerName];
+            if (!handleTask) {
+                afLogger.error(`No handler registered for jobHandler ${jobHandlerName}. Cannot process unfinished tasks for job ${job[this.getResourcePk()]}.`);
+                return;
+            }
+            const parrallelLimit = this.jobParallelLimits[jobHandlerName] || 3;
+            const unfinishedTasks = [];
+            let taskIndex = 0;
+            while (true) {
+                const taskData = yield jobLevelDb.get(taskIndex.toString());
+                if (!taskData) {
+                    break;
+                }
+                let parsedTaskData;
+                try {
+                    parsedTaskData = JSON.parse(taskData);
+                }
+                catch (error) {
+                    afLogger.error(`Error parsing task data for task ${taskIndex} of job ${job[this.getResourcePk()]}: ${error}`);
+                    taskIndex++;
+                    continue;
+                }
+                if (parsedTaskData.status === 'IN_PROGRESS' || parsedTaskData.status === 'SCHEDULED') {
+                    unfinishedTasks.push({ state: parsedTaskData.state });
+                }
+                else {
+                    unfinishedTasks.push({ state: parsedTaskData.state, skip: true });
+                }
+                taskIndex++;
+            }
+            yield this.runProcessingTasks(unfinishedTasks, jobLevelDb, job[this.getResourcePk()], handleTask, parrallelLimit);
+        });
+    }
+    setJobField(jobId, key, value) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const jobRecord = yield this.adminforth.resource(this.getResourceId()).get(Filters.EQ(this.getResourcePk(), jobId));
+            const state = jobRecord[this.options.stateField];
+            const parsedState = JSON.parse(state);
+            parsedState[key] = value;
+            yield this.adminforth.resource(this.getResourceId()).update(jobId, {
+                [this.options.stateField]: JSON.stringify(parsedState),
+            });
+        });
+    }
+    getJobField(jobId, key) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const jobRecord = yield this.adminforth.resource(this.getResourceId()).get(Filters.EQ(this.getResourcePk(), jobId));
+            const state = jobRecord[this.options.stateField];
+            const parsedState = JSON.parse(state);
+            return parsedState[key];
+        });
+    }
+    getJobState(jobId) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const jobRecord = yield this.adminforth.resource(this.getResourceId()).get(Filters.EQ(this.getResourcePk(), jobId));
+            const state = jobRecord[this.options.stateField];
+            return JSON.parse(state);
+        });
+    }
+    processAllUnfinishedJobs() {
+        return __awaiter(this, void 0, void 0, function* () {
+            const resourceId = this.getResourceId();
+            const unprocessedJobs = yield this.adminforth.resource(resourceId).list(Filters.EQ(this.options.statusField, 'IN_PROGRESS'));
+            for (const job of unprocessedJobs) {
+                const jobName = job[this.options.nameField];
+                afLogger.info(`Processing unfinished job with name ${jobName} on startup.`);
+                this.runProcessingUnfinishedTasks(job);
+            }
+        });
+    }
+    validateConfigAfterDiscover(adminforth, resourceConfig) {
+        return __awaiter(this, void 0, void 0, function* () {
+            // optional method where you can safely check field types after database discovery was performed
+            this.checkIfFieldInResource(resourceConfig, this.options.createdAtField, 'createdAtField');
+            this.checkIfFieldInResource(resourceConfig, this.options.startedByField, 'startedByField');
+            this.checkIfFieldInResource(resourceConfig, this.options.stateField, 'stateField');
+            this.checkIfFieldInResource(resourceConfig, this.options.progressField, 'progressField');
+            this.checkIfFieldInResource(resourceConfig, this.options.statusField, 'statusField');
+            this.checkIfFieldInResource(resourceConfig, this.options.nameField, 'nameField');
+            //Add temp delay to make sure, that all resources active. Probably should be fixed
+            yield new Promise(resolve => setTimeout(resolve, 1000));
+            this.processAllUnfinishedJobs();
+        });
+    }
+    instanceUniqueRepresentation(pluginOptions) {
+        return `BackgroundJobsPlugin`;
+    }
+    setupEndpoints(server) {
+        server.endpoint({
+            method: 'POST',
+            path: `/plugin/${this.pluginInstanceId}/get-list-of-jobs`,
+            handler: (_a) => __awaiter(this, [_a], void 0, function* ({ adminUser }) {
+                const user = adminUser;
+                const startedByField = this.options.startedByField;
+                const resourcePk = this.getResourcePk();
+                const listOfJobs = yield this.adminforth.resource(this.resourceConfig.resourceId).list(Filters.EQ(startedByField, user.pk), 100, 0, Sorts.DESC(this.options.createdAtField));
+                const jobsToReturn = listOfJobs.map(job => {
+                    return {
+                        id: job[resourcePk],
+                        name: job[this.options.nameField],
+                        createdAt: job[this.options.createdAtField],
+                        status: job[this.options.statusField],
+                        progress: job[this.options.progressField],
+                        customComponent: this.jobCustomComponents[job[this.options.jobHandlerField]],
+                    };
+                });
+                return { jobs: jobsToReturn };
+            })
+        });
+        server.endpoint({
+            method: 'POST',
+            path: `/plugin/${this.pluginInstanceId}/cancel-job`,
+            handler: (_a) => __awaiter(this, [_a], void 0, function* ({ body }) {
+                const jobId = body.jobId;
+                const currentJob = yield this.adminforth.resource(this.getResourceId()).get(Filters.EQ(this.getResourcePk(), jobId));
+                const oldStatus = currentJob[this.options.statusField];
+                if (oldStatus === 'DONE' || oldStatus === 'DONE_WITH_ERRORS' || oldStatus === 'CANCELLED') {
+                    return { ok: false, message: `Cannot cancel a job with status ${oldStatus}.` };
+                }
+                try {
+                    yield this.adminforth.resource(this.getResourceId()).update(jobId, {
+                        [this.options.statusField]: 'CANCELLED',
+                    });
+                    this.adminforth.websocket.publish('/background-jobs', {
+                        jobId,
+                        status: 'CANCELLED',
+                    });
+                    return { ok: true };
+                }
+                catch (error) {
+                    return { ok: false, message: `Failed to cancel job with id ${jobId}.` };
+                }
+            })
+        });
+        server.endpoint({
+            method: 'POST',
+            path: `/plugin/${this.pluginInstanceId}/get-tasks`,
+            handler: (_a) => __awaiter(this, [_a], void 0, function* ({ body }) {
+                const { jobId, limit, offset } = body;
+                const levelDbPath = `${this.options.levelDbPath || './background-jobs-dbs/'}job_${jobId}`;
+                let jobLevelDb;
+                if (this.levelDbInstances[jobId]) {
+                    jobLevelDb = this.levelDbInstances[jobId];
+                }
+                else {
+                    try {
+                        jobLevelDb = new Level(levelDbPath, { valueEncoding: 'json' });
+                    }
+                    catch (error) {
+                        return { ok: false, message: `Failed to access tasks for job with id ${jobId}.` };
+                    }
+                }
+                const tasks = [];
+                let taskIndex = 0 + offset;
+                while (true) {
+                    if (limit && tasks.length >= limit) {
+                        break;
+                    }
+                    const taskData = yield jobLevelDb.get(taskIndex.toString());
+                    if (!taskData) {
+                        break;
+                    }
+                    let parsedTaskData;
+                    try {
+                        parsedTaskData = JSON.parse(taskData);
+                    }
+                    catch (error) {
+                        afLogger.error(`Error parsing task data for task ${taskIndex} of job ${jobId}: ${error}`);
+                        taskIndex++;
+                        continue;
+                    }
+                    tasks.push(parsedTaskData);
+                    taskIndex++;
+                }
+                return { ok: true, tasks };
+            })
+        });
+    }
+}

package/dist/types.js

@@ -0,0 +1 @@
+export {};