midway-fatcms 0.0.1-beta.6 → 0.0.1-beta.61

package/src/service/asyncTask/AsyncTaskRunnerService.ts

@@ -0,0 +1,319 @@
+import * as moment from 'moment';
+import { Inject, Provide } from '@midwayjs/core';
+import { Context, IMidwayKoaContext } from '@midwayjs/koa';
+import { BaseService } from '@/service/base/BaseService';
+import { IScheduleService } from '@/interface';
+import { CurdProService } from '@/service/curd/CurdProService';
+import { ISysAsyncTaskHandler, SysAsyncTaskEntity, SysAsyncTaskPartial, SysAsyncTaskStatus } from '@/models/AsyncTaskModel';
+import { GLOBAL_STATIC_CONFIG } from '@/libs/global-config/global-config';
+import { SystemTables } from '@/models/SystemTables';
+import { KeysOfSimpleSQL } from '@/libs/crud-pro/models/keys';
+import { errorToString } from '@/libs/utils/errorToString';
+import { ANONYMOUS_CONTEXT } from '@/schedule';
+import { RedisKeys } from '@/models/RedisKeys';
+import { getCurrentFullMoment } from '@/libs/utils/functions';
+import { ExportExcelAsyncTaskHandler } from './handler/ExportExcelAsyncTaskHandler';
+
+const END_STATUS_LIST = [SysAsyncTaskStatus.PART_SUCCEEDED, SysAsyncTaskStatus.SUCCEEDED, SysAsyncTaskStatus.FAILED, SysAsyncTaskStatus.CANCELLED, SysAsyncTaskStatus.PAUSED];
+
+class AsyncTaskRunner {
+  isBusy = false;
+  taskHandlerMap: Map<string, ISysAsyncTaskHandler> = new Map();
+
+  async executeTaskList(taskList: SysAsyncTaskEntity[]) {
+    if (!taskList || taskList.length === 0) {
+      return;
+    }
+
+    this.isBusy = true;
+
+    for (let i = 0; i < taskList.length; i++) {
+      const taskElement = taskList[i];
+      taskElement.task_status = SysAsyncTaskStatus.RUNNING;
+
+      try {
+        // 1. 更新当前这个任务的状态
+        await this.updateTaskStatus(taskElement, {
+          task_status: taskElement.task_status,
+          started_at: getCurrentFullMoment(),
+          updated_at: getCurrentFullMoment(),
+        });
+
+        // 任务状态的设置在自定义的执行器里面。
+        await this.executeTask(taskElement);
+
+        // 当前不是终止状态，并且没有异常就设置成成功。
+        if (!END_STATUS_LIST.includes(taskElement.task_status)) {
+          taskElement.task_status = SysAsyncTaskStatus.SUCCEEDED;
+        }
+      } catch (error) {
+        taskElement.task_status = SysAsyncTaskStatus.FAILED;
+        taskElement.error_message = errorToString(error);
+      }
+
+      try {
+        // 2. 更新当前这个任务的状态
+        await this.updateTaskStatus(taskElement, {
+          task_status: taskElement.task_status,
+          error_message: taskElement.error_message,
+          updated_at: getCurrentFullMoment(),
+          completed_at: getCurrentFullMoment(),
+        });
+      } catch (error) {
+        ANONYMOUS_CONTEXT.getApp().getCoreLogger().error('[AsyncTaskRunner] executeTaskList error', error);
+      }
+    }
+
+    this.isBusy = false;
+  }
+
+  private async executeTask(taskElement: SysAsyncTaskEntity) {
+    const taskType = taskElement.task_type;
+    const taskHandler = this.taskHandlerMap.get(taskType);
+    if (!taskHandler) {
+      throw new Error('TaskHandler not found , taskType = ' + taskType);
+    }
+    const updateTaskStatus = (updatePartials: SysAsyncTaskPartial) => {
+      return this.updateTaskStatus(taskElement, updatePartials);
+    };
+    await taskHandler.execute({ task: taskElement, updateTaskStatus });
+  }
+
+  /**
+   * 更新任务状态或任务进度
+   * @param taskElement
+   * @param updatePartials
+   * @private
+   */
+  private async updateTaskStatus(taskElement: SysAsyncTaskEntity, updatePartials: SysAsyncTaskPartial) {
+    if (!updatePartials) {
+      throw new Error('updatePartials not found');
+    }
+    return await ANONYMOUS_CONTEXT.runServiceAtAnonymousContext(async (ctx: IMidwayKoaContext) => {
+      const curdProService: CurdProService = await ctx.requestContext.getAsync('curdProService');
+      const { SystemDbName, SystemDbType } = GLOBAL_STATIC_CONFIG.getConfig();
+
+      // 一定要有一个更新时间字段
+      updatePartials.updated_at = getCurrentFullMoment();
+
+      const res = await curdProService.executeCrudByCfg(
+        {
+          condition: {
+            id: taskElement.id,
+          },
+          data: updatePartials,
+        },
+        {
+          sqlTable: SystemTables.sys_async_tasks,
+          sqlSimpleName: KeysOfSimpleSQL.SIMPLE_UPDATE,
+          sqlDatabase: SystemDbName,
+          sqlDbType: SystemDbType,
+        }
+      );
+      return res.getResModel().affected;
+    });
+  }
+
+  /**
+   * 获取当前进程可以处理的任务类型。
+   */
+  public getHandlerTaskTypeList(): string[] {
+    const keys = this.taskHandlerMap.keys();
+    return [...keys];
+  }
+}
+
+/**
+ * 业务可以扩展
+ */
+export const ASYNC_TASK_RUNNER = new AsyncTaskRunner();
+
+//内置的异步任务处理器：EXCEL导出
+ASYNC_TASK_RUNNER.taskHandlerMap.set('EXCEL_EXPORT', new ExportExcelAsyncTaskHandler());
+
+/**
+ * Redis锁
+ */
+const ASYNC_TASK_LOCK = RedisKeys.ASYNC_TASK_LOCK;
+const ASYNC_TASK_TIMEOUT_LOCK = RedisKeys.ASYNC_TASK_TIMEOUT_LOCK;
+
+/**
+ * 上次检查异步任务的时间点
+ */
+const ASYNC_TASK_RUNTIME_OBJ = {
+  LAST_CHECK_ASYNC_TASK_UPDATE_TIME: 0,
+};
+
+@Provide()
+export class AsyncTaskRunnerService extends BaseService implements IScheduleService {
+  @Inject()
+  protected ctx: Context;
+
+  @Inject()
+  private curdProService: CurdProService;
+
+  /**
+   * 定时任务执行
+   */
+  async runBySchedule() {
+    await this.runByScheduleForPendingTasks();
+    await this.runByScheduleForTimeoutTasks();
+  }
+
+  /**
+   * 更新超时的任务状态
+   * @returns
+   */
+  private async runByScheduleForTimeoutTasks() {
+    const { SystemDbName, SystemDbType } = GLOBAL_STATIC_CONFIG.getConfig();
+
+    // 每隔一小时执行一次，即可。
+    const lock = await this.redisService.set(ASYNC_TASK_TIMEOUT_LOCK, 1, 'EX', 60 * 60, 'NX');
+    if (lock !== 'OK') {
+      return Promise.resolve();
+    }
+
+    const timeoutTime = moment().subtract(24, 'hours').format('YYYY-MM-DD HH:mm:ss.SSS');
+    await this.curdProService.executeCrudByCfg(
+      {
+        condition: {
+          task_status: SysAsyncTaskStatus.RUNNING, // 虽然状态是RUNNING，但是已经超过24小时没有进展了。
+          updated_at: {
+            $lt: timeoutTime,
+          },
+        },
+        data: {
+          task_status: SysAsyncTaskStatus.FAILED,
+          error_message: '任务超时',
+        },
+      },
+      {
+        sqlTable: SystemTables.sys_async_tasks,
+        sqlSimpleName: KeysOfSimpleSQL.SIMPLE_UPDATE,
+        sqlDatabase: SystemDbName,
+        sqlDbType: SystemDbType,
+      }
+    );
+  }
+
+  /**
+   * 检查是否有新任务。如果有，就从数据库中获取一批任务，放到自己的任务队列里。
+   * @returns
+   */
+  private async runByScheduleForPendingTasks() {
+    // 1. 当前很忙
+    if (ASYNC_TASK_RUNNER.isBusy) {
+      return Promise.resolve();
+    }
+
+    // 2. 发现了新任务
+    const isExistNewTask = await this.isExistNewTask();
+    if (!isExistNewTask) {
+      return Promise.resolve();
+    }
+
+    // 这里的过期时间1分钟即可。fetchPendingTasks函数不可能超过一分钟。
+    // 因为这里只是从数据库中获取一批任务，放到自己的任务队列里。还没触发执行。
+    const lock = await this.redisService.set(ASYNC_TASK_LOCK, 1, 'EX', 60, 'NX');
+    if (lock !== 'OK') {
+      return Promise.resolve();
+    }
+
+    try {
+      await this.fetchPendingTasks();
+    } catch (e) {
+      console.error('[AsyncTaskRunnerService] fetchPendingTasks error', errorToString(e));
+    }
+
+    await this.redisService.del(ASYNC_TASK_LOCK);
+
+    return Promise.resolve();
+  }
+
+  async fetchPendingTasks(): Promise<void> {
+    const { SystemDbName, SystemDbType } = GLOBAL_STATIC_CONFIG.getConfig();
+
+    // 只获取本进程能够处理的任务类型。
+    const taskTypeList = ASYNC_TASK_RUNNER.getHandlerTaskTypeList();
+    if (taskTypeList.length === 0) {
+      return Promise.resolve();
+    }
+
+    // 查询等待处理的任务。
+    const queryRes = await this.curdProService.executeCrudByCfg(
+      {
+        condition: {
+          task_status: SysAsyncTaskStatus.PENDING,
+          task_type: {
+            $in: taskTypeList,
+          },
+        },
+        orderBy: 'id+',
+        limit: 10,
+      },
+      {
+        sqlTable: SystemTables.sys_async_tasks,
+        sqlSimpleName: KeysOfSimpleSQL.SIMPLE_QUERY_PAGE,
+        sqlDatabase: SystemDbName,
+        sqlDbType: SystemDbType,
+      }
+    );
+
+    const taskList = queryRes.getResRows();
+
+    if (taskList.length === 0) {
+      return Promise.resolve();
+    }
+    const taskIds = taskList.map(elem => elem.id).filter(Boolean);
+
+    // 将状态更新为处理中。防止其它进程重复处理。
+    await this.curdProService.executeCrudByCfg(
+      {
+        condition: {
+          id: {
+            $in: taskIds,
+          },
+        },
+        data: {
+          task_status: SysAsyncTaskStatus.RUNNING,
+        },
+      },
+      {
+        sqlTable: SystemTables.sys_async_tasks,
+        sqlSimpleName: KeysOfSimpleSQL.SIMPLE_UPDATE,
+        sqlDatabase: SystemDbName,
+        sqlDbType: SystemDbType,
+      }
+    );
+
+    // 开始执行。
+    await ASYNC_TASK_RUNNER.executeTaskList(taskList).then(() => {
+      console.log('[AsyncTaskRunnerService] ASYNC_TASK_RUNNER finished taskIds ==> ' + JSON.stringify(taskIds));
+    });
+  }
+
+  /**
+   * 是否存在新任务
+   * @private
+   */
+  private async isExistNewTask(): Promise<boolean> {
+    // 刚启动，没有检查过。
+    if (!ASYNC_TASK_RUNTIME_OBJ.LAST_CHECK_ASYNC_TASK_UPDATE_TIME) {
+      ASYNC_TASK_RUNTIME_OBJ.LAST_CHECK_ASYNC_TASK_UPDATE_TIME = Date.now();
+      return true;
+    }
+
+    //Redis没有任何内容
+    const updateTime = await this.redisService.get(RedisKeys.ASYNC_TASK_UPDATE_TIME);
+    if (!updateTime) {
+      ASYNC_TASK_RUNTIME_OBJ.LAST_CHECK_ASYNC_TASK_UPDATE_TIME = Date.now();
+      return false;
+    }
+
+    const updateTimeNumber = Number.parseInt(updateTime, 10);
+
+    const isExistNewTask: boolean = updateTimeNumber > ASYNC_TASK_RUNTIME_OBJ.LAST_CHECK_ASYNC_TASK_UPDATE_TIME;
+    ASYNC_TASK_RUNTIME_OBJ.LAST_CHECK_ASYNC_TASK_UPDATE_TIME = updateTimeNumber;
+    return isExistNewTask;
+  }
+}

package/src/service/asyncTask/AsyncTaskService.ts

@@ -0,0 +1,21 @@
+import { Inject, Provide } from '@midwayjs/core';
+import { Context } from '@midwayjs/koa';
+import { BaseService } from '@/service/base/BaseService';
+import { runScheduleTaskOnce } from '@/schedule';
+import { INNER_SCHEDULE_NAMES } from '@/schedule/scheduleNames';
+import { RedisKeys } from '@/models/RedisKeys';
+
+@Provide()
+export class AsyncTaskService extends BaseService {
+  @Inject()
+  protected ctx: Context;
+
+  async startTask() {
+    await this.redisService.set(RedisKeys.ASYNC_TASK_UPDATE_TIME, `${Date.now()}`);
+    runScheduleTaskOnce(INNER_SCHEDULE_NAMES.asyncTaskRunnerService).then(schedule => {
+      console.log(schedule);
+    });
+  }
+
+  async cancelTask(id: number) {}
+}

package/src/service/asyncTask/handler/ExcelInfoModel.ts

@@ -0,0 +1,11 @@
+export interface IExcelHeaderInfo {
+  title: string;
+  dataIndex: string;
+}
+
+export interface IExcelAsyncTaskHandler {
+  getExcelHeader(): Promise<IExcelHeaderInfo[]>;
+  getPageSize(): Promise<number>;
+  getTotalCount(): Promise<number>;
+  getExcelDataList(pageNo: number, pageSize: number): Promise<any[]>;
+}

package/src/service/asyncTask/handler/ExportExcelAsyncTaskHandler.ts

@@ -0,0 +1,245 @@
+import { IMidwayKoaContext } from '@midwayjs/koa';
+import { ISysAsyncTaskHandler, SysAsyncTaskContext, SysAsyncTaskStatus } from '@/models/AsyncTaskModel';
+import { ANONYMOUS_CONTEXT } from '@/schedule';
+import { IExcelHeaderInfo, IExcelAsyncTaskHandler } from './ExcelInfoModel';
+import { SysAsyncFileFormat } from '@/models/AsyncTaskModel';
+import { ExportExcelByStdCrudHandler } from './ExportExcelByStdCrudHandler';
+import { ExportExcelByInnerHttpHandler } from './ExportExcelByInnerHttpHandler';
+import { IStdCrudExportInputParams, StdCrudExportInputParamsAppType } from '@/models/bizmodels';
+
+import * as _ from 'lodash';
+import * as fastcsv from 'fast-csv';
+import * as fs from 'node:fs';
+import * as fs2 from 'node:fs/promises';
+import * as path from 'node:path';
+import { parseJsonObject } from '@/libs/utils/functions';
+import { UserSessionInfo } from '@/models/userSession';
+import { errorToString } from '@/libs/utils/errorToString';
+
+const indexRef = { current: 0 };
+
+async function createFileFolder(distFileFolder: string) {
+  try {
+    await fs2.access(distFileFolder);
+  } catch (e) {
+    await fs2.mkdir(distFileFolder, { recursive: true });
+  }
+}
+
+function getExcelAsyncTaskHandler(asyncTaskContext: SysAsyncTaskContext, ctx: IMidwayKoaContext): IExcelAsyncTaskHandler {
+  const inputParams: IStdCrudExportInputParams = parseJsonObject(asyncTaskContext?.task?.input_params);
+  if (inputParams && inputParams.appType === StdCrudExportInputParamsAppType.STD_CRUD) {
+    return new ExportExcelByStdCrudHandler(asyncTaskContext, inputParams, ctx);
+  }
+  if (inputParams && inputParams.appType === StdCrudExportInputParamsAppType.INNER_HTTP) {
+    return new ExportExcelByInnerHttpHandler(asyncTaskContext, inputParams, ctx);
+  }
+  throw new Error('[getExcelAsyncTaskHandler]appType不支持；appType = ' + inputParams.appType);
+}
+
+/**
+ * 多实例，每个任务一个实例。
+ * 导出EXCEL异步任务:
+ */
+class ExportExcelAsyncTask {
+  protected asyncTaskContext: SysAsyncTaskContext;
+  protected ctx: IMidwayKoaContext;
+  protected headerColumns: IExcelHeaderInfo[];
+  protected excelAsyncTaskHandler: IExcelAsyncTaskHandler;
+
+  constructor(asyncTaskContext: SysAsyncTaskContext, ctx: IMidwayKoaContext) {
+    this.asyncTaskContext = asyncTaskContext;
+    this.ctx = ctx;
+    this.excelAsyncTaskHandler = getExcelAsyncTaskHandler(asyncTaskContext, ctx);
+  }
+
+  private getDistFileFolder() {
+    const app = ANONYMOUS_CONTEXT.getApp();
+    const config = app.getConfig();
+    const fatcmsExportExcelTmpFilePath = config.fatcmsExportExcelTmpFilePath;
+    if (!fatcmsExportExcelTmpFilePath) {
+      return path.join(__dirname, '../../tmp');
+    }
+    return fatcmsExportExcelTmpFilePath;
+  }
+
+  async executeWithContext() {
+    const taskElement = this.asyncTaskContext.task;
+    const taskId = _.get(this.asyncTaskContext, 'task.id');
+
+    this.ctx.logger.info('[ExportExcelAsyncTask] executeWithContext start, taskId=' + taskId);
+
+    const pageSize = await this.excelAsyncTaskHandler.getPageSize();
+
+    const fileName = await this.getExcelFileName(); // 文件名
+    const distFileFolder = this.getDistFileFolder();
+    await createFileFolder(distFileFolder);
+    const diskFilePath = path.resolve(distFileFolder, fileName); // 磁盘文件路径
+
+    const totalCount = await this.getTotalCount();
+
+    await this.asyncTaskContext.updateTaskStatus({
+      output_total_records: totalCount,
+      output_file_format: SysAsyncFileFormat.CSV,
+      output_file_path: diskFilePath,
+    });
+
+    this.headerColumns = await this.getExcelHeader();
+
+    const ws = fs.createWriteStream(diskFilePath, {
+      encoding: 'utf-8',
+    });
+
+    const csvStream = fastcsv.format({ writeHeaders: false, headers: false, delimiter: ',' });
+    csvStream.pipe(ws);
+    csvStream.write('\ufeff'); // 写入BOM头，解决中文乱码问题
+
+    csvStream.write(this.formatToCsvHeader(this.headerColumns));
+
+    let processed_records = 0;
+
+    try {
+      if (typeof totalCount === 'number' && totalCount > 0) {
+        const batchCount = Math.ceil(totalCount / pageSize);
+        for (let i = 0; i < batchCount; i++) {
+          const num = await this.writeExcelBatchData(csvStream, i + 1, pageSize);
+          processed_records = processed_records + num;
+          const progress = Math.round((processed_records / totalCount) * 100);
+          await this.asyncTaskContext.updateTaskStatus({
+            processed_records,
+            progress,
+          });
+        }
+      }
+
+      taskElement.task_status = SysAsyncTaskStatus.SUCCEEDED;
+    } catch (error) {
+      taskElement.task_status = SysAsyncTaskStatus.FAILED;
+      taskElement.error_message = errorToString(error);
+      const progress = Math.round((processed_records / totalCount) * 100);
+      await this.asyncTaskContext.updateTaskStatus({
+        processed_records,
+        progress,
+        task_status: taskElement.task_status,
+        error_message: taskElement.error_message,
+      });
+    }
+
+    csvStream.end();
+    ws.on('finish', async () => {
+      this.ctx.logger.info('[ExportExcelAsyncTask] executeWithContext finish, taskId=' + taskId);
+      const fsStat = await fs2.stat(diskFilePath);
+      const output_file_size = fsStat.size;
+      this.asyncTaskContext.updateTaskStatus({
+        output_file_size,
+      });
+    });
+    this.ctx.logger.info('[ExportExcelAsyncTask] executeWithContext end, taskId=' + taskId);
+  }
+
+  private formatToCsvHeader(headerColumns: IExcelHeaderInfo[]): string[] {
+    return headerColumns.map(item => {
+      return item.title;
+    });
+  }
+
+  private async writeExcelBatchData(csvStream: fastcsv.CsvFormatterStream<any, any>, pageNo: number, pageSize: number): Promise<number> {
+    const dataList: any[] = await this.getExcelDataList(pageNo, pageSize);
+    for (let i = 0; i < dataList.length; i++) {
+      const csvRow = this.formatToCsvRow(dataList[i]);
+      csvStream.write(csvRow);
+    }
+    return dataList.length;
+  }
+
+  private formatToCsvRow(rowObj: any): string[] {
+    return this.headerColumns.map(obj => {
+      const dataIndex = obj.dataIndex;
+      const value = _.get(rowObj, dataIndex);
+      if (typeof value === 'undefined' || value === null) {
+        return '';
+      }
+      if (typeof value !== 'string') {
+        // boolean , number
+        return '' + value;
+      }
+      return value;
+    });
+  }
+
+  /**
+   * 获取文件名
+   * @returns 文件名
+   */
+  private async getExcelFileName() {
+    indexRef.current = indexRef.current + 1;
+    if (indexRef.current > 100000) {
+      indexRef.current = 1;
+    }
+    const num = `${indexRef.current}`.padStart(10, '0');
+    return `export_${Date.now()}_${num}.csv`;
+  }
+
+  /**
+   * 获取Excel的表头
+   * @returns
+   */
+  private async getExcelHeader(): Promise<IExcelHeaderInfo[]> {
+    return this.excelAsyncTaskHandler.getExcelHeader();
+    // const inputParams = parseJsonObject(this.asyncTaskContext?.task?.input_params);
+    // if (inputParams && inputParams.appType === 'STD_CRUD') {
+    //     // const appCode = inputParams.appCode;
+    //     console.log(this)
+    // }
+    // return [
+    //     { title: 'ID', dataIndex: 'id' },
+    //     { title: '姓名', dataIndex: 'name' },
+    //     { title: '邮箱', dataIndex: 'email' }
+    // ]
+  }
+
+  /**
+   * 获取文件总数
+   */
+  private async getTotalCount(): Promise<number> {
+    const totalCount = await this.excelAsyncTaskHandler.getTotalCount();
+    // 最多导出10万条数据
+    return Math.min(totalCount, 10 * 10000);
+  }
+
+  /**
+   * 获取数据内容数组
+   * @param pageNo 当前页号，从1开始
+   * @param pageSize 每页获取的数量。
+   */
+  private async getExcelDataList(pageNo: number, pageSize: number): Promise<any[]> {
+    return this.excelAsyncTaskHandler.getExcelDataList(pageNo, pageSize);
+    // const dataList = [];
+    // for (var i = 0; i < pageSize; i++) {
+    //     dataList.push({ id: i, name: 'nmae' + i, email: 'www.dss@dd.dom' })
+    // }
+    // return dataList;
+  }
+}
+
+/**
+ * 单例
+ */
+export class ExportExcelAsyncTaskHandler implements ISysAsyncTaskHandler {
+  public async execute(asyncTaskContext: SysAsyncTaskContext): Promise<any> {
+    const taskId = _.get(asyncTaskContext, 'task.id');
+
+    console.log('[ExportExcelAsyncTaskHandler] execute, taskId=>' + taskId);
+
+    const res = await ANONYMOUS_CONTEXT.runServiceAtAnonymousContext(async (ctx: IMidwayKoaContext) => {
+      const createdUserSession = parseJsonObject(asyncTaskContext?.task?.created_user_session);
+      ctx.userSession = new UserSessionInfo(createdUserSession, false);
+      const task = new ExportExcelAsyncTask(asyncTaskContext, ctx);
+      return task.executeWithContext();
+    });
+    if (res.error) {
+      throw res.error;
+    }
+    return res.result;
+  }
+}