midway-fatcms 0.0.1-beta.14 → 0.0.1-beta.17

package/src/models/AsyncTaskModel.ts

@@ -26,12 +26,14 @@ export interface SysAsyncTaskEntity {
   priority: number;
   retry_count: number;
   max_retries: number;
+
   parent_task_id: number | null;
   created_at: Date;
   started_at: Date | null;
-  completed_at: Date | null;
   updated_at: Date;
   expired_at: Date | null;
+  completed_at: Date | null;
+  created_user_session: string;
 }
 
 export enum SysAsyncTaskStatus {
@@ -58,11 +60,12 @@ export enum SysAsyncFileFormat {
 
 
 export interface ISysAsyncTaskHandler {
-  execute(ctx: SysAsyncTaskContext): Promise<void>;
+  execute(asyncTaskContext: SysAsyncTaskContext): Promise<void>;
 }
 
 export interface SysAsyncTaskContext {
   task: SysAsyncTaskEntity;
+  updateTaskStatus: ()=>any;
 }
 
 export interface SysAsyncTaskHandlerConfig {

package/src/schedule/anonymousContext.ts

@@ -0,0 +1,79 @@
+import * as koa from '@midwayjs/koa';
+import {ContextLogger} from "@/models/contextLogger";
+import {Transaction} from "@/libs/crud-pro/models/Transaction";
+import {ISessionInfo, SYS_ACCOUNT_TYPE, UserSessionInfo} from "@/models/userSession";
+import {IWorkbenchEntity} from "@/models/SystemEntities";
+
+
+const anonymousUserInfo:ISessionInfo = {
+  nickName: 'anonymous_user',
+  avatar: '',
+  roleCodes: [],
+  functionCodes: [],
+  loginName: 'anonymous_user',
+  sessionId: '',
+  accountId: '',
+  workbenchCode: '',
+  accountType: SYS_ACCOUNT_TYPE,
+};
+
+const anonymousWorkbenchInfo: IWorkbenchEntity = {
+  id: 0,
+  workbench_code: 'anonymous_workbench',
+  workbench_name: 'anonymous_workbench',
+  workbench_domain: 'anonymous_workbench',
+  workbench_desc: 'anonymous_workbench',
+  html_content: 'anonymous_workbench',
+  status: 1,
+  workbench_type: 0,
+};
+
+
+export type RunServiceAtAnonymousContextRunner = (ctx: koa.IMidwayKoaContext)=>any;
+
+
+class AnonymousContext {
+  private app: koa.Application;
+
+  setApp(app: koa.Application){
+    this.app = app;
+  }
+
+  getApp(){
+    return this.app;
+  }
+
+
+  async runServiceAtAnonymousContext( runner: RunServiceAtAnonymousContextRunner) {
+    const app = this.app;
+    const ctx: koa.IMidwayKoaContext = app.createAnonymousContext();
+
+    ctx.app = app;
+    ctx.contextLogger = new ContextLogger(ctx as any);
+    ctx.transaction = new Transaction();
+    ctx.userSession = new UserSessionInfo(anonymousUserInfo, true);
+    ctx.workbenchInfo = anonymousWorkbenchInfo;
+
+    let result: any = null;
+    let error: any = null;
+    try {
+      result = await runner(ctx);
+    } catch (e) {
+      error = e;
+      ctx.logger.error(`runServiceAtAnonymousContext error `, e);
+    } finally {
+      await ctx.transaction.releaseTx();
+    }
+    return {result: result, error: error};
+  }
+
+}
+
+
+const ANONYMOUS_CONTEXT = new AnonymousContext()
+
+
+
+export {
+  ANONYMOUS_CONTEXT
+}

package/src/schedule/index.ts

@@ -1,73 +1,14 @@
-import * as koa from '@midwayjs/koa';
-import { IScheduleService } from '../interface';
-import { ContextLogger } from '../models/contextLogger';
-import { Transaction } from '../libs/crud-pro/models/Transaction';
-import {ISessionInfo, SYS_ACCOUNT_TYPE, UserSessionInfo} from '../models/userSession';
-import { IWorkbenchEntity } from '../models/SystemEntities';
-
-const scheduleUserInfo:ISessionInfo = {
-  nickName: 'schedule_user',
-  avatar: '',
-  roleCodes: [],
-  functionCodes: [],
-  loginName: 'schedule_user',
-  sessionId: '',
-  accountId: '',
-  workbenchCode: '',
-  accountType: SYS_ACCOUNT_TYPE,
-};
-
-const scheduleWorkbenchInfo: IWorkbenchEntity = {
-  id: 0,
-  workbench_code: 'schedule_workbench',
-  workbench_name: 'schedule_workbench',
-  workbench_domain: 'schedule_workbench',
-  workbench_desc: 'schedule_workbench',
-  html_content: 'schedule_workbench',
-  status: 1,
-  workbench_type: 0,
-};
-
-async function runScheduleService(app: koa.Application, serviceName: string) {
-  const ctx = app.createAnonymousContext();
-
-  ctx.app = app;
-  ctx.contextLogger = new ContextLogger(ctx as any);
-  ctx.transaction = new Transaction();
-  ctx.userSession = new UserSessionInfo(scheduleUserInfo, true);
-  ctx.workbenchInfo = scheduleWorkbenchInfo;
-
-  try {
-    const serviceObject: IScheduleService = await ctx.requestContext.getAsync(serviceName);
-    if (serviceObject && serviceObject.runBySchedule) {
-      await serviceObject.runBySchedule();
-    } else {
-      ctx.logger.error(`runSchedule error , serviceName = ${serviceName}, 没有找到服务`);
-    }
-  } catch (e) {
-    ctx.logger.error(`runSchedule error , serviceName = ${serviceName}`, e);
-  } finally {
-    await ctx.transaction.releaseTx();
-  }
+import {startScheduleLoop, runScheduleTaskOnce} from './runSchedule'
+import {ANONYMOUS_CONTEXT} from './anonymousContext'
+import {
+  ALL_SCHEDULE_NAMES,
+  SCHEDULE_NAMES,
+} from './scheduleNames'
+
+export {
+  ALL_SCHEDULE_NAMES,
+  SCHEDULE_NAMES,
+  ANONYMOUS_CONTEXT,
+  startScheduleLoop,
+  runScheduleTaskOnce
 }
-
-async function runSchedule(app: koa.Application, serviceList: string[]) {
-  for (let i = 0; i < serviceList.length; i++) {
-    const serviceName = serviceList[i];
-    try {
-      await runScheduleService(app, serviceName);
-    } catch (e) {
-      const ctx = app.createAnonymousContext();
-      ctx.logger.error(`runSchedule error , serviceName = ${serviceName}`, e);
-    }
-  }
-}
-
-async function startSchedule(app: koa.Application, serviceList: string[]) {
-  await runSchedule(app, serviceList);
-  setInterval(() => {
-    runSchedule(app, serviceList);
-  }, 2 * 60 * 1000);
-}
-
-export { startSchedule, runSchedule };

package/src/schedule/runSchedule.ts

@@ -0,0 +1,35 @@
+import * as koa from '@midwayjs/koa';
+import { IScheduleService } from '@/interface';
+import { ANONYMOUS_CONTEXT } from "./anonymousContext";
+
+
+async function runScheduleTaskOnce(serviceName: string) {
+  return await ANONYMOUS_CONTEXT.runServiceAtAnonymousContext(async (ctx: koa.IMidwayKoaContext)=>{
+    const serviceObject: IScheduleService = await ctx.requestContext.getAsync(serviceName);
+    if (serviceObject && serviceObject.runBySchedule) {
+      await serviceObject.runBySchedule();
+    } else {
+      ctx.logger.error(`runSchedule error , serviceName = ${serviceName}, 没有找到服务`);
+    }
+  });
+}
+
+async function runScheduleServiceList(serviceList: string[]) {
+  for (let i = 0; i < serviceList.length; i++) {
+    const serviceName = serviceList[i];
+    try {
+      await runScheduleTaskOnce(serviceName);
+    } catch (e) {
+      ANONYMOUS_CONTEXT.getApp().getCoreLogger().error(`runSchedule error , serviceName = ${serviceName}`, e);
+    }
+  }
+}
+
+async function startScheduleLoop(serviceList: string[]) {
+  await runScheduleServiceList(serviceList);
+  setInterval(() => {
+    runScheduleServiceList(serviceList);
+  }, 2 * 60 * 1000);
+}
+
+export { startScheduleLoop, runScheduleTaskOnce };

package/src/schedule/scheduleNames.ts

@@ -0,0 +1,20 @@
+
+
+const SCHEDULE_NAMES = {
+  proxyApiLoadService: 'proxyApiLoadService',
+  workbenchService: 'workbenchService',
+  visitStatService: 'visitStatService',
+  asyncTaskRunnerService: 'asyncTaskRunnerService',
+}
+
+const ALL_SCHEDULE_NAMES = [
+  SCHEDULE_NAMES.proxyApiLoadService,
+  SCHEDULE_NAMES.workbenchService,
+  SCHEDULE_NAMES.visitStatService,
+  SCHEDULE_NAMES.asyncTaskRunnerService
+];
+
+export {
+  ALL_SCHEDULE_NAMES,
+  SCHEDULE_NAMES,
+}

package/src/service/asyncTask/AsyncTaskRunnerService.ts

@@ -1,5 +1,5 @@
 import {Inject, Provide} from '@midwayjs/core';
-import {Context} from '@midwayjs/koa';
+import {Context, IMidwayKoaContext} from '@midwayjs/koa';
 import {BaseService} from "@/service/base/BaseService";
 import {IScheduleService} from "@/interface";
 import {CurdProService} from "@/service/curd/CurdProService";
@@ -8,9 +8,10 @@ import {GLOBAL_STATIC_CONFIG} from "@/libs/global-config/global-config";
 import {SystemTables} from "@/models/SystemTables";
 import {KeysOfSimpleSQL} from "@/libs/crud-pro/models/keys";
 import {errorToString} from "@/libs/utils/errorToString";
+import {ANONYMOUS_CONTEXT} from "@/schedule";
 
 
-class AsyncTaskRunner  {
+class AsyncTaskRunner {
   isBusy: boolean = false;
   taskHandlerMap: Map<string, ISysAsyncTaskHandler> = new Map();
 
@@ -18,19 +19,29 @@ class AsyncTaskRunner  {
     if (!taskList || taskList.length === 0) {
       return;
     }
+
     this.isBusy = true;
 
     for (let i = 0; i < taskList.length; i++) {
       const taskElement = taskList[i];
+
       try {
+        // 任务状态的设置在自定义的执行器里面。
         await this.executeTask(taskElement);
-        taskElement.task_status = SysAsyncTaskStatus.SUCCEEDED;
       } catch (error) {
         taskElement.task_status = SysAsyncTaskStatus.FAILED;
         taskElement.error_message = errorToString(error)
       }
+
+      try {
+        await this.updateTaskStatus(taskElement);
+      } catch (error) {
+        ANONYMOUS_CONTEXT.getApp().getCoreLogger().error("executeTaskList error", error);
+      }
     }
 
+    this.isBusy = false;
+
   }
 
   private async executeTask(taskElement: SysAsyncTaskEntity) {
@@ -39,18 +50,67 @@ class AsyncTaskRunner  {
     if (!taskHandler) {
       throw new Error('TaskHandler not found , taskType = ' + taskType);
     }
-    await taskHandler.execute({task: taskElement});
+    const updateTaskStatus = () => {
+      return this.updateTaskStatus(taskElement);
+    }
+    await taskHandler.execute({task: taskElement, updateTaskStatus});
   }
+
+
+  /**
+   * 更新任务状态或任务进度
+   * @param taskElement
+   * @private
+   */
+  private async updateTaskStatus(taskElement: SysAsyncTaskEntity) {
+    return await ANONYMOUS_CONTEXT.runServiceAtAnonymousContext(async (ctx: IMidwayKoaContext) => {
+      const curdProService: CurdProService = await ctx.requestContext.getAsync("curdProService");
+      const {SystemDbName, SystemDbType} = GLOBAL_STATIC_CONFIG.getConfig();
+
+      const {id, ...otherTaskProps} = taskElement;
+
+      const res = await curdProService.executeCrudByCfg({
+        condition: {
+          id: id
+        },
+        data: otherTaskProps
+      }, {
+        sqlTable: SystemTables.sys_async_tasks,
+        sqlSimpleName: KeysOfSimpleSQL.SIMPLE_UPDATE,
+        sqlDatabase: SystemDbName,
+        sqlDbType: SystemDbType,
+      });
+      return res.getResModel().affected;
+    });
+
+  }
+
+
+  /**
+   * 获取当前进程可以处理的任务类型。
+   */
+  public getHandlerTaskTypeList(): string[] {
+    const keys = this.taskHandlerMap.keys();
+    return [...keys];
+  }
+
+
 }
 
 
-const ASYNC_TASK_RUNNER = new AsyncTaskRunner();
+/**
+ * 业务可以扩展
+ */
+export const ASYNC_TASK_RUNNER = new AsyncTaskRunner();
 
 
+/**
+ * Redis锁
+ */
 const LOCK_KEY_ASYNC_TASK_RUNNER = "LOCK_KEY_ASYNC_TASK_RUNNER"
 
 @Provide()
-export class AsyncTaskRunnerService extends BaseService implements IScheduleService{
+export class AsyncTaskRunnerService extends BaseService implements IScheduleService {
 
   @Inject()
   protected ctx: Context;
@@ -58,39 +118,44 @@ export class AsyncTaskRunnerService extends BaseService implements IScheduleServ
   @Inject()
   private curdProService: CurdProService;
 
-  async runBySchedule () {
-    if (ASYNC_TASK_RUNNER.isBusy) {
-      return Promise.resolve();
-    }
 
-    const lock = await this.redisService.set(LOCK_KEY_ASYNC_TASK_RUNNER, 1,"EX", 3600, "NX");
-    if (lock !== 'OK') {
+  async fetchPendingTasks(): Promise<void> {
+
+    const {SystemDbName, SystemDbType} = GLOBAL_STATIC_CONFIG.getConfig();
+
+    // 只获取本进程能够处理的任务类型。
+    const taskTypeList = ASYNC_TASK_RUNNER.getHandlerTaskTypeList();
+    if (taskTypeList.length === 0) {
       return Promise.resolve();
     }
 
 
-    const { SystemDbName, SystemDbType } = GLOBAL_STATIC_CONFIG.getConfig();
-
+    // 查询等待处理的任务。
     const queryRes = await this.curdProService.executeCrudByCfg({
       condition: {
-        task_status: SysAsyncTaskStatus.PENDING
+        task_status: SysAsyncTaskStatus.PENDING,
+        task_type: {
+          "$in": taskTypeList
+        }
       },
+      orderBy: 'id+',
       limit: 10
-    },{
+    }, {
       sqlTable: SystemTables.sys_async_tasks,
       sqlSimpleName: KeysOfSimpleSQL.SIMPLE_QUERY_PAGE,
       sqlDatabase: SystemDbName,
       sqlDbType: SystemDbType,
     });
 
+
     const taskList = queryRes.getResRows();
 
     if (taskList.length === 0) {
       return Promise.resolve();
     }
-
     const taskIds = taskList.map(elem => elem.id).filter(Boolean)
 
+    // 将状态更新为处理中。防止其它进程重复处理。
     await this.curdProService.executeCrudByCfg({
       condition: {
         id: {
@@ -100,7 +165,7 @@ export class AsyncTaskRunnerService extends BaseService implements IScheduleServ
       data: {
         task_status: SysAsyncTaskStatus.RUNNING
       }
-    },{
+    }, {
       sqlTable: SystemTables.sys_async_tasks,
       sqlSimpleName: KeysOfSimpleSQL.SIMPLE_UPDATE,
       sqlDatabase: SystemDbName,
@@ -108,7 +173,36 @@ export class AsyncTaskRunnerService extends BaseService implements IScheduleServ
     });
 
 
-    ASYNC_TASK_RUNNER.executeTaskList(taskList);
+    // 开始执行。
+    ASYNC_TASK_RUNNER.executeTaskList(taskList).then(() => {
+      console.log("ASYNC_TASK_RUNNER finished taskIds ==> " + JSON.stringify(taskIds))
+    })
+
+  }
+
+
+  async runBySchedule() {
+    if (ASYNC_TASK_RUNNER.isBusy) {
+      return Promise.resolve();
+    }
+
+    // 这里的过期时间1分钟即可。fetchPendingTasks函数不可能超过一分钟。
+    // 因为这里只是从数据库中获取一批任务，放到自己的任务队列里。还没触发执行。
+    const lock = await this.redisService.set(LOCK_KEY_ASYNC_TASK_RUNNER, 1, "EX", 60, "NX");
+    if (lock !== 'OK') {
+      return Promise.resolve();
+    }
+
+    try {
+      await this.fetchPendingTasks();
+    } catch (e) {
+      console.error("fetchPendingTasks error", errorToString(e));
+    }
+
+    await this.redisService.del(LOCK_KEY_ASYNC_TASK_RUNNER);
+
     return Promise.resolve();
   }
+
+
 }

package/src/service/asyncTask/AsyncTaskService.ts

@@ -1,7 +1,8 @@
 import {Inject, Provide} from '@midwayjs/core';
 import {Context} from '@midwayjs/koa';
 import {BaseService} from "@/service/base/BaseService";
-import {runSchedule} from "@/schedule";
+import {runScheduleTaskOnce} from "@/schedule";
+import {SCHEDULE_NAMES} from "@/schedule/scheduleNames";
 
 @Provide()
 export class AsyncTaskService extends BaseService {
@@ -9,7 +10,7 @@ export class AsyncTaskService extends BaseService {
   protected ctx: Context;
 
   async startTask() {
-    runSchedule(this.app, ['asyncTaskRunnerService']).then(schedule => {
+    runScheduleTaskOnce(SCHEDULE_NAMES.asyncTaskRunnerService).then(schedule => {
       console.log(schedule);
     })
   }

package/src/service/base/BaseService.ts

@@ -3,8 +3,8 @@ import { Context } from '@midwayjs/koa';
 import { RedisService } from '@midwayjs/redis';
 import { OSSService, OSSServiceFactory } from '@midwayjs/oss';
 import * as koa from '@midwayjs/koa';
-import { consoleLogger, ContextLogger, ILoggerContext } from '../../models/contextLogger';
-import { isEnableDebug, isLocalEnv } from '../../libs/utils/fatcms-request';
+import { consoleLogger, ContextLogger, ILoggerContext } from '@/models/contextLogger';
+import { isEnableDebug, isLocalEnv } from '@/libs/utils/fatcms-request';
 
 
 @Provide()

package/src/service/crudstd/CrudStdService.ts

@@ -2,18 +2,19 @@ import { Inject, Provide } from '@midwayjs/core';
 import { Context } from '@midwayjs/koa';
 import { CurdMixService } from '../curd/CurdMixService';
 import { IRequestCfgModel, IRequestModel } from '../../libs/crud-pro/interfaces';
-import { KeysOfAuthType, KeysOfSimpleSQL } from '../../libs/crud-pro/models/keys';
+import { KeysOfAuthType, KeysOfSimpleSQL, SqlDbType } from '../../libs/crud-pro/models/keys';
 import { parseJsonObject } from '../../libs/utils/functions';
 import { ICrudStdAppInfo, ICrudStdAppInfoForSettingKey } from '../../models/bizmodels';
 import { BizException } from '../../models/devops';
 import { ExecuteContext } from '../../libs/crud-pro/models/ExecuteContext';
-import { SystemTables} from '../../models/SystemTables';
+import { SystemTables } from '../../models/SystemTables';
 import { CrudStdActionService } from './CrudStdActionService';
 import { CrudStdRelationService } from './CrudStdRelationService';
 import * as _ from 'lodash';
 import { ApiBaseService } from '../base/ApiBaseService';
-import {parseDatabaseName} from "../../libs/crud-pro/utils/DatabaseName";
+import { parseDatabaseName } from '../../libs/crud-pro/utils/DatabaseName';
 import { GLOBAL_STATIC_CONFIG } from '../../libs/global-config/global-config';
+import { MixinUtils } from '@/libs/crud-pro/utils/MixinUtils';
 
 export const SPECIAL_SETTING_KEY = {
   QUERY_LIST: 'QUERY_LIST',
@@ -69,6 +70,9 @@ export class CrudStdService extends ApiBaseService {
     // 根据用户配置，设置关联查询的数据信息。
     await this.crudStdRelationService.addCfgModelColumnsRelation(cfgModel, appInfo);
 
+    // 根据表结构的数据类型，修正数据类型
+    await this.fixDataFieldTypeBySqlTableField(params, cfgModel, appInfo, { dbType, dbName });
+
     // 业务系统自定义的修改cfgModel
     await GLOBAL_STATIC_CONFIG.getConfig().bizUpdateCfgModelForCrudStd(params, cfgModel, appInfo, this.ctx);
 
@@ -138,11 +142,8 @@ export class CrudStdService extends ApiBaseService {
    * @private
    */
   private async getCrudStdAppInfo(appCode?: string) {
-
     const { SystemDbName, SystemDbType } = GLOBAL_STATIC_CONFIG.getConfig();
 
-
-
     this.logInfo('getCrudStdAppInfo', appCode);
 
     const ss = {
@@ -195,4 +196,31 @@ export class CrudStdService extends ApiBaseService {
     }
     return true; // 无需鉴权
   }
+
+  private async fixDataFieldTypeBySqlTableField(params: IRequestModel, cfgModel: IRequestCfgModel, appInfo: ICrudStdAppInfoForSettingKey, arg3: { dbType: SqlDbType; dbName: string }) {
+    const tableFields = _.get(appInfo, 'stdCrudCfgObj.tableFields');
+
+    const tableFieldMap = MixinUtils.toMap(tableFields, (e: any) => {
+      return e.fieldIndex;
+    });
+
+    if (arg3.dbType === SqlDbType.postgres) {
+      if (cfgModel.sqlSimpleName === KeysOfSimpleSQL.SIMPLE_INSERT || cfgModel.sqlSimpleName === KeysOfSimpleSQL.SIMPLE_UPDATE) {
+        const data = params.data || {};
+        const dataKeys = Object.keys(data);
+        for (let index = 0; index < dataKeys.length; index++) {
+          const dataKey = dataKeys[index];
+          const dataValue = data[dataKey];
+          const fieldInfo = tableFieldMap[dataKey];
+          const fieldType = ('' + _.get(fieldInfo, 'type')).toUpperCase();
+          if (fieldType === 'ARRAY') {
+            const dataValueArray = parseJsonObject(dataValue);
+            if (Array.isArray(dataValueArray)) {
+              data[dataKey] = `{${dataValueArray.map(v => `"${v}"`).join(',')}}`;
+            }
+          }
+        }
+      }
+    }
+  }
 }