midway-fatcms 0.0.1-beta.18 → 0.0.1-beta.19

package/src/service/VisitStatService.ts

@@ -1,14 +1,14 @@
-import {Inject, Provide} from '@midwayjs/core';
-import {Context} from '@midwayjs/koa';
-import {IScheduleService} from '../interface';
-import {BaseService} from "./base/BaseService";
-import {CurdProService} from "./curd/CurdProService";
-import { SystemTables} from "../models/SystemTables";
-import {KeysOfSimpleSQL} from "../libs/crud-pro/models/keys";
-import {FILE_GET_TYPES} from "../models/bizmodels";
-import {CommonResult} from "../libs/utils/common-dto";
+import { Inject, Provide } from '@midwayjs/core';
+import { Context } from '@midwayjs/koa';
+import { IScheduleService } from '@/interface';
+import { BaseService } from './base/BaseService';
+import { CurdProService } from './curd/CurdProService';
+import { SystemTables } from '@/models/SystemTables';
+import { KeysOfSimpleSQL } from '@/libs/crud-pro/models/keys';
+import { FILE_GET_TYPES } from '@/models/bizmodels';
+import { CommonResult } from '@/libs/utils/common-dto';
 import { GLOBAL_STATIC_CONFIG } from '@/libs/global-config/global-config';
-
+import { RedisKeys } from '@/models/RedisKeys';
 
 // 3天 的秒数
 const EXPIRE_TIME = 3 * 24 * 60 * 60;
@@ -19,7 +19,6 @@ const STAT_TYPES = {
   uv_uid: 'uv_uid',
 };
 
-
 @Provide()
 export class VisitStatService extends BaseService implements IScheduleService {
   @Inject()
@@ -30,34 +29,32 @@ export class VisitStatService extends BaseService implements IScheduleService {
 
   async runBySchedule(): Promise<any> {
     const hours = new Date().getHours();
-    this.logInfo("[VisitStatService] runBySchedule called, hours = " + hours);
+    this.logInfo('[VisitStatService] runBySchedule called, hours = ' + hours);
     return this.flushVisitStatToDB();
   }
 
-  async flushVisitStatToDB():  Promise<any>   {
-
+  async flushVisitStatToDB(): Promise<any> {
     const { SystemDbName, SystemDbType } = GLOBAL_STATIC_CONFIG.getConfig();
 
     //  统计昨天的数据。
     const stat_date = new Date(Date.now() - 24 * 3600 * 1000).toISOString().split('T')[0];
 
-    const lockKey = `vs_lock_${stat_date}`;
+    const lockKey = `${RedisKeys.VISIT_STAT_LOCK_PREFIX}${stat_date}`;
     // 每天只能执行一次
     const client = this.redisService;
-    const nxRes = await client.set(lockKey, 1, 'EX', EXPIRE_TIME, 'NX')
+    const nxRes = await client.set(lockKey, 1, 'EX', EXPIRE_TIME, 'NX');
     if (nxRes !== 'OK') {
-      this.logInfo("[VisitStatService] flushVisitStatToDB 每天只能执行一次 ");
-      return CommonResult.errorRes("每天只能执行一次")
+      this.logInfo('[VisitStatService] flushVisitStatToDB 每天只能执行一次 ');
+      return CommonResult.errorRes('每天只能执行一次');
     }
 
-    this.logInfo("[VisitStatService] flushVisitStatToDB start ");
+    this.logInfo('[VisitStatService] flushVisitStatToDB start ');
 
-    let totalCount = 0 ;
+    let totalCount = 0;
     let successCount = 0;
 
     try {
-
-      const keys = await client.keys(`vs_${stat_date}::*`);
+      const keys = await client.keys(`${RedisKeys.VISIT_STAT_DATE_PREFIX}${stat_date}::*`);
       // `vs_${date}::${req_host}::${resource_type}::${stat_type}::${resource}`;
       totalCount = keys.length;
 
@@ -71,112 +68,99 @@ export class VisitStatService extends BaseService implements IScheduleService {
             const stat_resource = parts[4];
 
             let stat_count: number;
-            if (stat_type === STAT_TYPES.uv_ip || stat_type == STAT_TYPES.uv_uid) {
+            if (stat_type === STAT_TYPES.uv_ip || stat_type === STAT_TYPES.uv_uid) {
               stat_count = await client.scard(key);
             } else {
               stat_count = parseInt(await client.get(key)) || 0;
             }
 
-            await this.curdProService.executeCrudByCfg({
-              data: {
-                stat_date,
-                stat_resource,
-                stat_type,
-                resource_type,
-                stat_count,
-                req_host
+            await this.curdProService.executeCrudByCfg(
+              {
+                data: {
+                  stat_date,
+                  stat_resource,
+                  stat_type,
+                  resource_type,
+                  stat_count,
+                  req_host,
+                },
+              },
+              {
+                sqlSimpleName: KeysOfSimpleSQL.SIMPLE_INSERT,
+                sqlDatabase: SystemDbName,
+                sqlDbType: SystemDbType,
+                sqlTable: SystemTables.sys_visit_stats,
               }
-            }, {
-              sqlSimpleName: KeysOfSimpleSQL.SIMPLE_INSERT,
-              sqlDatabase: SystemDbName,
-              sqlDbType: SystemDbType,
-              sqlTable: SystemTables.sys_visit_stats,
-            })
+            );
 
             await client.del(key);
             successCount++;
           }
-        } catch (e){
+        } catch (e) {
           this.logError('[VisitStatService] flushVisitStatToDB: key = ' + key, e);
           console.log('[VisitStatService] flushVisitStatToDB: key = ' + key, e);
         }
       }
     } catch (error) {
-      this.logError('[VisitStatService] flushVisitStatToDB' , error);
-      console.log('[VisitStatService] flushVisitStatToDB' , error);
+      this.logError('[VisitStatService] flushVisitStatToDB', error);
+      console.log('[VisitStatService] flushVisitStatToDB', error);
     }
 
-    this.logInfo("[VisitStatService] flushVisitStatToDB end ");
+    this.logInfo('[VisitStatService] flushVisitStatToDB end ');
 
-    return CommonResult.successRes({successCount, totalCount})
+    return CommonResult.successRes({ successCount, totalCount });
   }
 
-
   /**
    * 统计请求数据
    */
   async trackRequest(resource: string, resourceType: string) {
-
     // 文件下载定制
     if (resource.startsWith('/ns/gw/file/get/')) {
       for (let i = 0; i < FILE_GET_TYPES.length; i++) {
         const fileGetType = FILE_GET_TYPES[i];
-        const pathPrefix = `/ns/gw/file/get/${fileGetType}/`
+        const pathPrefix = `/ns/gw/file/get/${fileGetType}/`;
         if (resource.startsWith(pathPrefix)) {
           const res2 = resource.substring(pathPrefix.length);
-          return this.trackRequestImpl(res2, fileGetType)
+          return this.trackRequestImpl(res2, fileGetType);
         }
       }
     }
 
     // 文档访问定制
     if (resource.startsWith('/ns/app/doc-editor/libview')) {
-      const {docLibId,docId} = this.ctx.query || {};
-      return this.trackRequestImpl(`${docLibId},${docId}`, 'docLibView')
+      const { docLibId, docId } = this.ctx.query || {};
+      return this.trackRequestImpl(`${docLibId},${docId}`, 'docLibView');
     }
 
-
-    return this.trackRequestImpl(resource, resourceType)
+    return this.trackRequestImpl(resource, resourceType);
   }
 
-
   private async trackRequestImpl(resource: string, resource_type: string) {
     const client = this.redisService;
     const ip = this.ctx.headers['x-real-ip'];
-    const userSession = this.ctx.userSession
-    const req_host = this.ctx?.request?.host || "null" // 域名
+    const userSession = this.ctx.userSession;
+    const req_host = this.ctx?.request?.host || 'null'; // 域名
     const date = new Date().toISOString().split('T')[0];
 
     const toDateResourceKey = (stat_type: string): string => {
-      return `vs_${date}::${req_host}::${resource_type}::${stat_type}::${resource}`;
-    }
+      return `${RedisKeys.VISIT_STAT_DATE_PREFIX}${date}::${req_host}::${resource_type}::${stat_type}::${resource}`;
+    };
 
     const pv_key = toDateResourceKey(STAT_TYPES.pv);
     const uv_ip_key = toDateResourceKey(STAT_TYPES.uv_ip);
     const uv_uid_key = toDateResourceKey(STAT_TYPES.uv_uid);
 
-
     // 统计 PV
-    await Promise.all([
-      client.incr(pv_key),
-      client.expire(pv_key, EXPIRE_TIME)
-    ]);
+    await Promise.all([client.incr(pv_key), client.expire(pv_key, EXPIRE_TIME)]);
 
     // IP 访问 UV
-    await Promise.all([
-      client.sadd(uv_ip_key, `${ip}`),
-      client.expire(uv_ip_key, EXPIRE_TIME)
-    ]);
+    await Promise.all([client.sadd(uv_ip_key, `${ip}`), client.expire(uv_ip_key, EXPIRE_TIME)]);
 
     if (userSession && userSession.isLogin()) {
       const accountId = userSession.getSessionInfo().accountId;
       // 统计 UV
-      await Promise.all([
-        client.sadd(uv_uid_key, accountId),
-        client.expire(uv_uid_key, EXPIRE_TIME)
-      ]);
+      await Promise.all([client.sadd(uv_uid_key, accountId), client.expire(uv_uid_key, EXPIRE_TIME)]);
     }
-
-  };
-
+  }
 }

package/src/service/asyncTask/AsyncTaskRunnerService.ts

@@ -1,18 +1,18 @@
-import {Inject, Provide} from '@midwayjs/core';
-import {Context, IMidwayKoaContext} from '@midwayjs/koa';
-import {BaseService} from "@/service/base/BaseService";
-import {IScheduleService} from "@/interface";
-import {CurdProService} from "@/service/curd/CurdProService";
-import {ISysAsyncTaskHandler, SysAsyncTaskEntity, SysAsyncTaskStatus} from "@/models/AsyncTaskModel";
-import {GLOBAL_STATIC_CONFIG} from "@/libs/global-config/global-config";
-import {SystemTables} from "@/models/SystemTables";
-import {KeysOfSimpleSQL} from "@/libs/crud-pro/models/keys";
-import {errorToString} from "@/libs/utils/errorToString";
-import {ANONYMOUS_CONTEXT} from "@/schedule";
-
+import { Inject, Provide } from '@midwayjs/core';
+import { Context, IMidwayKoaContext } from '@midwayjs/koa';
+import { BaseService } from '@/service/base/BaseService';
+import { IScheduleService } from '@/interface';
+import { CurdProService } from '@/service/curd/CurdProService';
+import { ISysAsyncTaskHandler, SysAsyncTaskEntity, SysAsyncTaskPartial, SysAsyncTaskStatus } from '@/models/AsyncTaskModel';
+import { GLOBAL_STATIC_CONFIG } from '@/libs/global-config/global-config';
+import { SystemTables } from '@/models/SystemTables';
+import { KeysOfSimpleSQL } from '@/libs/crud-pro/models/keys';
+import { errorToString } from '@/libs/utils/errorToString';
+import { ANONYMOUS_CONTEXT } from '@/schedule';
+import { RedisKeys } from '@/models/RedisKeys';
 
 class AsyncTaskRunner {
-  isBusy: boolean = false;
+  isBusy = false;
   taskHandlerMap: Map<string, ISysAsyncTaskHandler> = new Map();
 
   async executeTaskList(taskList: SysAsyncTaskEntity[]) {
@@ -30,18 +30,20 @@ class AsyncTaskRunner {
         await this.executeTask(taskElement);
       } catch (error) {
         taskElement.task_status = SysAsyncTaskStatus.FAILED;
-        taskElement.error_message = errorToString(error)
+        taskElement.error_message = errorToString(error);
       }
 
       try {
-        await this.updateTaskStatus(taskElement);
+        await this.updateTaskStatus(taskElement, {
+          task_status: taskElement.task_status,
+          error_message: taskElement.error_message,
+        });
       } catch (error) {
-        ANONYMOUS_CONTEXT.getApp().getCoreLogger().error("executeTaskList error", error);
+        ANONYMOUS_CONTEXT.getApp().getCoreLogger().error('executeTaskList error', error);
       }
     }
 
     this.isBusy = false;
-
   }
 
   private async executeTask(taskElement: SysAsyncTaskEntity) {
@@ -50,42 +52,44 @@ class AsyncTaskRunner {
     if (!taskHandler) {
       throw new Error('TaskHandler not found , taskType = ' + taskType);
     }
-    const updateTaskStatus = () => {
-      return this.updateTaskStatus(taskElement);
-    }
-    await taskHandler.execute({task: taskElement, updateTaskStatus});
+    const updateTaskStatus = (updatePartials: SysAsyncTaskPartial) => {
+      return this.updateTaskStatus(taskElement, updatePartials);
+    };
+    await taskHandler.execute({ task: taskElement, updateTaskStatus });
   }
 
-
   /**
    * 更新任务状态或任务进度
    * @param taskElement
+   * @param updatePartials
    * @private
    */
-  private async updateTaskStatus(taskElement: SysAsyncTaskEntity) {
+  private async updateTaskStatus(taskElement: SysAsyncTaskEntity, updatePartials: SysAsyncTaskPartial) {
+    if (!updatePartials) {
+      throw new Error('updatePartials not found');
+    }
     return await ANONYMOUS_CONTEXT.runServiceAtAnonymousContext(async (ctx: IMidwayKoaContext) => {
-      const curdProService: CurdProService = await ctx.requestContext.getAsync("curdProService");
-      const {SystemDbName, SystemDbType} = GLOBAL_STATIC_CONFIG.getConfig();
-
-      const {id, ...otherTaskProps} = taskElement;
-
-      const res = await curdProService.executeCrudByCfg({
-        condition: {
-          id: id
+      const curdProService: CurdProService = await ctx.requestContext.getAsync('curdProService');
+      const { SystemDbName, SystemDbType } = GLOBAL_STATIC_CONFIG.getConfig();
+
+      const res = await curdProService.executeCrudByCfg(
+        {
+          condition: {
+            id: taskElement.id,
+          },
+          data: updatePartials,
         },
-        data: otherTaskProps
-      }, {
-        sqlTable: SystemTables.sys_async_tasks,
-        sqlSimpleName: KeysOfSimpleSQL.SIMPLE_UPDATE,
-        sqlDatabase: SystemDbName,
-        sqlDbType: SystemDbType,
-      });
+        {
+          sqlTable: SystemTables.sys_async_tasks,
+          sqlSimpleName: KeysOfSimpleSQL.SIMPLE_UPDATE,
+          sqlDatabase: SystemDbName,
+          sqlDbType: SystemDbType,
+        }
+      );
       return res.getResModel().affected;
     });
-
   }
 
-
   /**
    * 获取当前进程可以处理的任务类型。
    */
@@ -93,36 +97,32 @@ class AsyncTaskRunner {
     const keys = this.taskHandlerMap.keys();
     return [...keys];
   }
-
-
 }
 
-
 /**
  * 业务可以扩展
  */
 export const ASYNC_TASK_RUNNER = new AsyncTaskRunner();
 
-
 /**
  * Redis锁
  */
-const LOCK_KEY_ASYNC_TASK_RUNNER = "LOCK_KEY_ASYNC_TASK_RUNNER"
+const ASYNC_TASK_LOCK = RedisKeys.ASYNC_TASK_LOCK;
+
+const ASYNC_TASK_RUNTIME_OBJ = {
+  LAST_CHECK_ASYNC_TASK_UPDATE_TIME: 0,
+};
 
 @Provide()
 export class AsyncTaskRunnerService extends BaseService implements IScheduleService {
-
-
   @Inject()
   protected ctx: Context;
 
   @Inject()
   private curdProService: CurdProService;
 
-
   async fetchPendingTasks(): Promise<void> {
-
-    const {SystemDbName, SystemDbType} = GLOBAL_STATIC_CONFIG.getConfig();
+    const { SystemDbName, SystemDbType } = GLOBAL_STATIC_CONFIG.getConfig();
 
     // 只获取本进程能够处理的任务类型。
     const taskTypeList = ASYNC_TASK_RUNNER.getHandlerTaskTypeList();
@@ -130,66 +130,74 @@ export class AsyncTaskRunnerService extends BaseService implements IScheduleServ
       return Promise.resolve();
     }
 
-
     // 查询等待处理的任务。
-    const queryRes = await this.curdProService.executeCrudByCfg({
-      condition: {
-        task_status: SysAsyncTaskStatus.PENDING,
-        task_type: {
-          "$in": taskTypeList
-        }
+    const queryRes = await this.curdProService.executeCrudByCfg(
+      {
+        condition: {
+          task_status: SysAsyncTaskStatus.PENDING,
+          task_type: {
+            $in: taskTypeList,
+          },
+        },
+        orderBy: 'id+',
+        limit: 10,
       },
-      orderBy: 'id+',
-      limit: 10
-    }, {
-      sqlTable: SystemTables.sys_async_tasks,
-      sqlSimpleName: KeysOfSimpleSQL.SIMPLE_QUERY_PAGE,
-      sqlDatabase: SystemDbName,
-      sqlDbType: SystemDbType,
-    });
-
+      {
+        sqlTable: SystemTables.sys_async_tasks,
+        sqlSimpleName: KeysOfSimpleSQL.SIMPLE_QUERY_PAGE,
+        sqlDatabase: SystemDbName,
+        sqlDbType: SystemDbType,
+      }
+    );
 
     const taskList = queryRes.getResRows();
 
     if (taskList.length === 0) {
       return Promise.resolve();
     }
-    const taskIds = taskList.map(elem => elem.id).filter(Boolean)
+    const taskIds = taskList.map(elem => elem.id).filter(Boolean);
 
     // 将状态更新为处理中。防止其它进程重复处理。
-    await this.curdProService.executeCrudByCfg({
-      condition: {
-        id: {
-          "$in": taskIds
-        }
+    await this.curdProService.executeCrudByCfg(
+      {
+        condition: {
+          id: {
+            $in: taskIds,
+          },
+        },
+        data: {
+          task_status: SysAsyncTaskStatus.RUNNING,
+        },
       },
-      data: {
-        task_status: SysAsyncTaskStatus.RUNNING
+      {
+        sqlTable: SystemTables.sys_async_tasks,
+        sqlSimpleName: KeysOfSimpleSQL.SIMPLE_UPDATE,
+        sqlDatabase: SystemDbName,
+        sqlDbType: SystemDbType,
       }
-    }, {
-      sqlTable: SystemTables.sys_async_tasks,
-      sqlSimpleName: KeysOfSimpleSQL.SIMPLE_UPDATE,
-      sqlDatabase: SystemDbName,
-      sqlDbType: SystemDbType,
-    });
-
+    );
 
     // 开始执行。
     ASYNC_TASK_RUNNER.executeTaskList(taskList).then(() => {
-      console.log("ASYNC_TASK_RUNNER finished taskIds ==> " + JSON.stringify(taskIds))
-    })
-
+      console.log('ASYNC_TASK_RUNNER finished taskIds ==> ' + JSON.stringify(taskIds));
+    });
   }
 
-
   async runBySchedule() {
+    // 1. 当前很忙
     if (ASYNC_TASK_RUNNER.isBusy) {
       return Promise.resolve();
     }
 
+    // 2. 发现了新任务
+    const isExistNewTask = await this.isExistNewTask();
+    if (!isExistNewTask) {
+      return Promise.resolve();
+    }
+
     // 这里的过期时间1分钟即可。fetchPendingTasks函数不可能超过一分钟。
     // 因为这里只是从数据库中获取一批任务，放到自己的任务队列里。还没触发执行。
-    const lock = await this.redisService.set(LOCK_KEY_ASYNC_TASK_RUNNER, 1, "EX", 60, "NX");
+    const lock = await this.redisService.set(ASYNC_TASK_LOCK, 1, 'EX', 60, 'NX');
     if (lock !== 'OK') {
       return Promise.resolve();
     }
@@ -197,12 +205,35 @@ export class AsyncTaskRunnerService extends BaseService implements IScheduleServ
     try {
       await this.fetchPendingTasks();
     } catch (e) {
-      console.error("fetchPendingTasks error", errorToString(e));
+      console.error('fetchPendingTasks error', errorToString(e));
     }
 
-    await this.redisService.del(LOCK_KEY_ASYNC_TASK_RUNNER);
+    await this.redisService.del(ASYNC_TASK_LOCK);
 
     return Promise.resolve();
   }
 
+  /**
+   * 是否存在新任务
+   * @private
+   */
+  private async isExistNewTask(): Promise<boolean> {
+    // 刚启动，没有检查过。
+    if (!ASYNC_TASK_RUNTIME_OBJ.LAST_CHECK_ASYNC_TASK_UPDATE_TIME) {
+      return true;
+    }
+
+    //Redis没有任何内容
+    const updateTime = await this.redisService.get(RedisKeys.ASYNC_TASK_UPDATE_TIME);
+    if (!updateTime) {
+      ASYNC_TASK_RUNTIME_OBJ.LAST_CHECK_ASYNC_TASK_UPDATE_TIME = Date.now();
+      return false;
+    }
+
+    const updateTimeNumber = Number.parseInt(updateTime, 10);
+
+    const isExistNewTask: boolean = updateTimeNumber > ASYNC_TASK_RUNTIME_OBJ.LAST_CHECK_ASYNC_TASK_UPDATE_TIME;
+    ASYNC_TASK_RUNTIME_OBJ.LAST_CHECK_ASYNC_TASK_UPDATE_TIME = updateTimeNumber;
+    return isExistNewTask;
+  }
 }

package/src/service/asyncTask/AsyncTaskService.ts

@@ -1,8 +1,9 @@
-import {Inject, Provide} from '@midwayjs/core';
-import {Context} from '@midwayjs/koa';
-import {BaseService} from "@/service/base/BaseService";
-import {runScheduleTaskOnce} from "@/schedule";
-import {INNER_SCHEDULE_NAMES} from "@/schedule/scheduleNames";
+import { Inject, Provide } from '@midwayjs/core';
+import { Context } from '@midwayjs/koa';
+import { BaseService } from '@/service/base/BaseService';
+import { runScheduleTaskOnce } from '@/schedule';
+import { INNER_SCHEDULE_NAMES } from '@/schedule/scheduleNames';
+import { RedisKeys } from '@/models/RedisKeys';
 
 @Provide()
 export class AsyncTaskService extends BaseService {
@@ -10,12 +11,11 @@ export class AsyncTaskService extends BaseService {
   protected ctx: Context;
 
   async startTask() {
+    await this.redisService.set(RedisKeys.ASYNC_TASK_UPDATE_TIME, `${Date.now()}`);
     runScheduleTaskOnce(INNER_SCHEDULE_NAMES.asyncTaskRunnerService).then(schedule => {
       console.log(schedule);
-    })
+    });
   }
 
-  async cancelTask(id: number) {
-
-  }
+  async cancelTask(id: number) {}
 }