midway-fatcms 0.0.1-beta.17 → 0.0.1-beta.19
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.eslintrc.json +3 -1
- package/dist/configuration.js +18 -7
- package/dist/controller/gateway/AsyncTaskController.js +3 -3
- package/dist/controller/gateway/DocGatewayController.js +1 -1
- package/dist/controller/gateway/PublicApiController.js +4 -6
- package/dist/controller/gateway/StaticController.js +25 -22
- package/dist/controller/helpers.controller.d.ts +1 -1
- package/dist/controller/manage/CrudStandardDesignApi.d.ts +1 -1
- package/dist/controller/manage/CrudStandardDesignApi.js +4 -16
- package/dist/controller/manage/DataDictManageApi.d.ts +1 -1
- package/dist/controller/manage/DeployManageApi.d.ts +1 -1
- package/dist/controller/manage/DeployManageApi.js +32 -30
- package/dist/controller/manage/MenuManageApi.js +1 -1
- package/dist/controller/manage/SuperAdminManageApi.d.ts +2 -2
- package/dist/controller/manage/SysConfigMangeApi.d.ts +1 -1
- package/dist/controller/manage/SysConfigMangeApi.js +3 -3
- package/dist/controller/manage/SystemInfoManageApi.d.ts +1 -1
- package/dist/controller/manage/SystemInfoManageApi.js +7 -1
- package/dist/controller/manage/UserAccountManageApi.d.ts +1 -1
- package/dist/controller/manage/WorkbenchMangeApi.js +1 -1
- package/dist/controller/myinfo/AuthController.js +1 -1
- package/dist/controller/render/AppRenderController.js +6 -2
- package/dist/controller/test.controller.d.ts +1 -1
- package/dist/controller/test.controller.js +4 -4
- package/dist/libs/crud-pro/models/ExecuteContext.d.ts +1 -1
- package/dist/libs/crud-pro/models/TransactionSqlServer.js +1 -1
- package/dist/libs/crud-pro/services/CrudProCachedCfgService.js +1 -1
- package/dist/libs/crud-pro/services/CrudProExecuteSqlService.js +7 -6
- package/dist/libs/crud-pro/services/CrudProGenSqlCondition.d.ts +1 -1
- package/dist/libs/crud-pro/services/CrudProGenSqlCondition.js +2 -4
- package/dist/libs/crud-pro/services/CrudProGenSqlService.js +6 -15
- package/dist/libs/crud-pro/services/CrudProOriginToExecuteSql.js +5 -4
- package/dist/libs/crud-pro/services/CrudProTableMetaService.js +2 -2
- package/dist/libs/crud-pro/utils/MixinUtils.js +1 -1
- package/dist/libs/crud-pro/utils/ValidateUtils.js +1 -1
- package/dist/libs/utils/fatcms-request.js +2 -2
- package/dist/middleware/forbidden.middleware.js +4 -20
- package/dist/middleware/global.middleware.js +1 -4
- package/dist/models/AsyncTaskModel.d.ts +2 -1
- package/dist/models/RedisKeys.d.ts +8 -0
- package/dist/models/RedisKeys.js +11 -0
- package/dist/schedule/index.d.ts +3 -3
- package/dist/schedule/index.js +3 -4
- package/dist/schedule/runSchedule.d.ts +11 -2
- package/dist/schedule/runSchedule.js +49 -15
- package/dist/schedule/scheduleNames.d.ts +8 -3
- package/dist/schedule/scheduleNames.js +12 -12
- package/dist/service/UserSessionService.js +2 -1
- package/dist/service/VisitStatService.d.ts +1 -1
- package/dist/service/VisitStatService.js +16 -25
- package/dist/service/asyncTask/AsyncTaskRunnerService.d.ts +9 -3
- package/dist/service/asyncTask/AsyncTaskRunnerService.js +57 -21
- package/dist/service/asyncTask/AsyncTaskService.d.ts +1 -1
- package/dist/service/asyncTask/AsyncTaskService.js +4 -3
- package/package.json +1 -1
- package/src/configuration.ts +23 -11
- package/src/controller/base/BaseApiController.ts +6 -6
- package/src/controller/gateway/AsyncTaskController.ts +17 -22
- package/src/controller/gateway/CrudMtdGatewayController.ts +7 -7
- package/src/controller/gateway/CrudStdGatewayController.ts +4 -4
- package/src/controller/gateway/DocGatewayController.ts +17 -17
- package/src/controller/gateway/FileController.ts +8 -9
- package/src/controller/gateway/ProxyApiGatewayController.ts +4 -4
- package/src/controller/gateway/PublicApiController.ts +19 -22
- package/src/controller/gateway/StaticController.ts +234 -264
- package/src/controller/helpers.controller.ts +1 -1
- package/src/controller/home.controller.ts +0 -5
- package/src/controller/manage/AnyApiMangeApi.ts +4 -4
- package/src/controller/manage/AppLogMangeApi.ts +3 -3
- package/src/controller/manage/AppMangeApi.ts +5 -5
- package/src/controller/manage/AppPageMangeApi.ts +3 -3
- package/src/controller/manage/AppSchemaHistoryApi.ts +1 -1
- package/src/controller/manage/CrudMethodsMangeApi.ts +3 -3
- package/src/controller/manage/CrudStandardDesignApi.ts +25 -77
- package/src/controller/manage/DataDictManageApi.ts +4 -4
- package/src/controller/manage/DeployManageApi.ts +84 -91
- package/src/controller/manage/DocLibManageApi.ts +3 -3
- package/src/controller/manage/DocManageApi.ts +3 -3
- package/src/controller/manage/MenuManageApi.ts +9 -14
- package/src/controller/manage/SuperAdminManageApi.ts +2 -10
- package/src/controller/manage/SysConfigMangeApi.ts +8 -11
- package/src/controller/manage/SystemInfoManageApi.ts +11 -6
- package/src/controller/manage/UserAccountManageApi.ts +2 -2
- package/src/controller/manage/WorkbenchMangeApi.ts +6 -6
- package/src/controller/myinfo/AuthController.ts +4 -8
- package/src/controller/render/AppRenderController.ts +8 -8
- package/src/controller/test.controller.ts +17 -17
- package/src/index.ts +0 -1
- package/src/libs/crud-pro/CrudPro.ts +0 -1
- package/src/libs/crud-pro/interfaces.ts +1 -3
- package/src/libs/crud-pro/models/ExecuteContext.ts +1 -4
- package/src/libs/crud-pro/models/ExecuteContextFunc.ts +1 -2
- package/src/libs/crud-pro/models/RequestModel.ts +1 -1
- package/src/libs/crud-pro/models/ResModel.ts +7 -12
- package/src/libs/crud-pro/models/SqlCfgModel.ts +1 -1
- package/src/libs/crud-pro/models/Transaction.ts +8 -9
- package/src/libs/crud-pro/models/TransactionPostgres.ts +1 -1
- package/src/libs/crud-pro/models/TransactionSqlServer.ts +8 -13
- package/src/libs/crud-pro/services/CrudProCachedCfgService.ts +1 -3
- package/src/libs/crud-pro/services/CrudProExecuteSqlService.ts +36 -48
- package/src/libs/crud-pro/services/CrudProGenSqlCondition.ts +19 -38
- package/src/libs/crud-pro/services/CrudProGenSqlService.ts +15 -32
- package/src/libs/crud-pro/services/CrudProOriginToExecuteSql.ts +23 -27
- package/src/libs/crud-pro/services/CrudProServiceBase.ts +1 -2
- package/src/libs/crud-pro/services/CrudProTableMetaService.ts +6 -14
- package/src/libs/crud-pro/utils/DateTimeUtils.ts +2 -2
- package/src/libs/crud-pro/utils/MixinUtils.ts +1 -1
- package/src/libs/crud-pro/utils/ValidateUtils.ts +1 -3
- package/src/libs/crud-pro/utils/pool/MySQLUtils.ts +1 -1
- package/src/libs/crud-pro/utils/pool/PostgresUtils.ts +3 -3
- package/src/libs/crud-pro/utils/pool/SqlServerUtils.ts +3 -3
- package/src/libs/global-config/global-config.ts +0 -5
- package/src/libs/utils/crypto-utils.ts +2 -4
- package/src/libs/utils/errorToString.ts +3 -8
- package/src/libs/utils/fatcms-request.ts +9 -21
- package/src/libs/utils/ordernum-utils.ts +2 -6
- package/src/libs/utils/parseConfig.ts +7 -15
- package/src/middleware/forbidden.middleware.ts +6 -25
- package/src/middleware/global.middleware.ts +10 -21
- package/src/models/AsyncTaskModel.ts +3 -2
- package/src/models/RedisKeys.ts +13 -0
- package/src/models/bizmodels.ts +1 -2
- package/src/schedule/index.ts +4 -6
- package/src/schedule/runSchedule.ts +65 -17
- package/src/schedule/scheduleNames.ts +12 -11
- package/src/service/UserSessionService.ts +5 -4
- package/src/service/VisitStatService.ts +55 -72
- package/src/service/anyapi/AnyApiService.ts +7 -7
- package/src/service/asyncTask/AsyncTaskRunnerService.ts +118 -87
- package/src/service/asyncTask/AsyncTaskService.ts +10 -10
- package/src/service/proxyapi/ProxyApiLoadService.ts +4 -3
|
@@ -1,14 +1,14 @@
|
|
|
1
|
-
import {Inject, Provide} from '@midwayjs/core';
|
|
2
|
-
import {Context} from '@midwayjs/koa';
|
|
3
|
-
import {IScheduleService} from '
|
|
4
|
-
import {BaseService} from
|
|
5
|
-
import {CurdProService} from
|
|
6
|
-
import { SystemTables} from
|
|
7
|
-
import {KeysOfSimpleSQL} from
|
|
8
|
-
import {FILE_GET_TYPES} from
|
|
9
|
-
import {CommonResult} from
|
|
1
|
+
import { Inject, Provide } from '@midwayjs/core';
|
|
2
|
+
import { Context } from '@midwayjs/koa';
|
|
3
|
+
import { IScheduleService } from '@/interface';
|
|
4
|
+
import { BaseService } from './base/BaseService';
|
|
5
|
+
import { CurdProService } from './curd/CurdProService';
|
|
6
|
+
import { SystemTables } from '@/models/SystemTables';
|
|
7
|
+
import { KeysOfSimpleSQL } from '@/libs/crud-pro/models/keys';
|
|
8
|
+
import { FILE_GET_TYPES } from '@/models/bizmodels';
|
|
9
|
+
import { CommonResult } from '@/libs/utils/common-dto';
|
|
10
10
|
import { GLOBAL_STATIC_CONFIG } from '@/libs/global-config/global-config';
|
|
11
|
-
|
|
11
|
+
import { RedisKeys } from '@/models/RedisKeys';
|
|
12
12
|
|
|
13
13
|
// 3天 的秒数
|
|
14
14
|
const EXPIRE_TIME = 3 * 24 * 60 * 60;
|
|
@@ -19,7 +19,6 @@ const STAT_TYPES = {
|
|
|
19
19
|
uv_uid: 'uv_uid',
|
|
20
20
|
};
|
|
21
21
|
|
|
22
|
-
|
|
23
22
|
@Provide()
|
|
24
23
|
export class VisitStatService extends BaseService implements IScheduleService {
|
|
25
24
|
@Inject()
|
|
@@ -30,35 +29,32 @@ export class VisitStatService extends BaseService implements IScheduleService {
|
|
|
30
29
|
|
|
31
30
|
async runBySchedule(): Promise<any> {
|
|
32
31
|
const hours = new Date().getHours();
|
|
33
|
-
this.logInfo(
|
|
32
|
+
this.logInfo('[VisitStatService] runBySchedule called, hours = ' + hours);
|
|
34
33
|
return this.flushVisitStatToDB();
|
|
35
34
|
}
|
|
36
35
|
|
|
37
|
-
|
|
38
|
-
async flushVisitStatToDB(): Promise<any> {
|
|
39
|
-
|
|
36
|
+
async flushVisitStatToDB(): Promise<any> {
|
|
40
37
|
const { SystemDbName, SystemDbType } = GLOBAL_STATIC_CONFIG.getConfig();
|
|
41
38
|
|
|
42
39
|
// 统计昨天的数据。
|
|
43
40
|
const stat_date = new Date(Date.now() - 24 * 3600 * 1000).toISOString().split('T')[0];
|
|
44
41
|
|
|
45
|
-
const lockKey =
|
|
42
|
+
const lockKey = `${RedisKeys.VISIT_STAT_LOCK_PREFIX}${stat_date}`;
|
|
46
43
|
// 每天只能执行一次
|
|
47
44
|
const client = this.redisService;
|
|
48
|
-
const nxRes = await client.set(lockKey, 1, 'EX', EXPIRE_TIME, 'NX')
|
|
45
|
+
const nxRes = await client.set(lockKey, 1, 'EX', EXPIRE_TIME, 'NX');
|
|
49
46
|
if (nxRes !== 'OK') {
|
|
50
|
-
this.logInfo(
|
|
51
|
-
return CommonResult.errorRes(
|
|
47
|
+
this.logInfo('[VisitStatService] flushVisitStatToDB 每天只能执行一次 ');
|
|
48
|
+
return CommonResult.errorRes('每天只能执行一次');
|
|
52
49
|
}
|
|
53
50
|
|
|
54
|
-
this.logInfo(
|
|
51
|
+
this.logInfo('[VisitStatService] flushVisitStatToDB start ');
|
|
55
52
|
|
|
56
|
-
let totalCount = 0
|
|
53
|
+
let totalCount = 0;
|
|
57
54
|
let successCount = 0;
|
|
58
55
|
|
|
59
56
|
try {
|
|
60
|
-
|
|
61
|
-
const keys = await client.keys(`vs_${stat_date}::*`);
|
|
57
|
+
const keys = await client.keys(`${RedisKeys.VISIT_STAT_DATE_PREFIX}${stat_date}::*`);
|
|
62
58
|
// `vs_${date}::${req_host}::${resource_type}::${stat_type}::${resource}`;
|
|
63
59
|
totalCount = keys.length;
|
|
64
60
|
|
|
@@ -72,112 +68,99 @@ export class VisitStatService extends BaseService implements IScheduleService {
|
|
|
72
68
|
const stat_resource = parts[4];
|
|
73
69
|
|
|
74
70
|
let stat_count: number;
|
|
75
|
-
if (stat_type === STAT_TYPES.uv_ip || stat_type
|
|
71
|
+
if (stat_type === STAT_TYPES.uv_ip || stat_type === STAT_TYPES.uv_uid) {
|
|
76
72
|
stat_count = await client.scard(key);
|
|
77
73
|
} else {
|
|
78
74
|
stat_count = parseInt(await client.get(key)) || 0;
|
|
79
75
|
}
|
|
80
76
|
|
|
81
|
-
await this.curdProService.executeCrudByCfg(
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
77
|
+
await this.curdProService.executeCrudByCfg(
|
|
78
|
+
{
|
|
79
|
+
data: {
|
|
80
|
+
stat_date,
|
|
81
|
+
stat_resource,
|
|
82
|
+
stat_type,
|
|
83
|
+
resource_type,
|
|
84
|
+
stat_count,
|
|
85
|
+
req_host,
|
|
86
|
+
},
|
|
87
|
+
},
|
|
88
|
+
{
|
|
89
|
+
sqlSimpleName: KeysOfSimpleSQL.SIMPLE_INSERT,
|
|
90
|
+
sqlDatabase: SystemDbName,
|
|
91
|
+
sqlDbType: SystemDbType,
|
|
92
|
+
sqlTable: SystemTables.sys_visit_stats,
|
|
89
93
|
}
|
|
90
|
-
|
|
91
|
-
sqlSimpleName: KeysOfSimpleSQL.SIMPLE_INSERT,
|
|
92
|
-
sqlDatabase: SystemDbName,
|
|
93
|
-
sqlDbType: SystemDbType,
|
|
94
|
-
sqlTable: SystemTables.sys_visit_stats,
|
|
95
|
-
})
|
|
94
|
+
);
|
|
96
95
|
|
|
97
96
|
await client.del(key);
|
|
98
97
|
successCount++;
|
|
99
98
|
}
|
|
100
|
-
} catch (e){
|
|
99
|
+
} catch (e) {
|
|
101
100
|
this.logError('[VisitStatService] flushVisitStatToDB: key = ' + key, e);
|
|
102
101
|
console.log('[VisitStatService] flushVisitStatToDB: key = ' + key, e);
|
|
103
102
|
}
|
|
104
103
|
}
|
|
105
104
|
} catch (error) {
|
|
106
|
-
this.logError('[VisitStatService] flushVisitStatToDB'
|
|
107
|
-
console.log('[VisitStatService] flushVisitStatToDB'
|
|
105
|
+
this.logError('[VisitStatService] flushVisitStatToDB', error);
|
|
106
|
+
console.log('[VisitStatService] flushVisitStatToDB', error);
|
|
108
107
|
}
|
|
109
108
|
|
|
110
|
-
this.logInfo(
|
|
109
|
+
this.logInfo('[VisitStatService] flushVisitStatToDB end ');
|
|
111
110
|
|
|
112
|
-
return CommonResult.successRes({successCount, totalCount})
|
|
111
|
+
return CommonResult.successRes({ successCount, totalCount });
|
|
113
112
|
}
|
|
114
113
|
|
|
115
|
-
|
|
116
114
|
/**
|
|
117
115
|
* 统计请求数据
|
|
118
116
|
*/
|
|
119
117
|
async trackRequest(resource: string, resourceType: string) {
|
|
120
|
-
|
|
121
118
|
// 文件下载定制
|
|
122
119
|
if (resource.startsWith('/ns/gw/file/get/')) {
|
|
123
120
|
for (let i = 0; i < FILE_GET_TYPES.length; i++) {
|
|
124
121
|
const fileGetType = FILE_GET_TYPES[i];
|
|
125
|
-
const pathPrefix = `/ns/gw/file/get/${fileGetType}
|
|
122
|
+
const pathPrefix = `/ns/gw/file/get/${fileGetType}/`;
|
|
126
123
|
if (resource.startsWith(pathPrefix)) {
|
|
127
124
|
const res2 = resource.substring(pathPrefix.length);
|
|
128
|
-
return this.trackRequestImpl(res2, fileGetType)
|
|
125
|
+
return this.trackRequestImpl(res2, fileGetType);
|
|
129
126
|
}
|
|
130
127
|
}
|
|
131
128
|
}
|
|
132
129
|
|
|
133
130
|
// 文档访问定制
|
|
134
131
|
if (resource.startsWith('/ns/app/doc-editor/libview')) {
|
|
135
|
-
const {docLibId,docId} = this.ctx.query || {};
|
|
136
|
-
return this.trackRequestImpl(`${docLibId},${docId}`, 'docLibView')
|
|
132
|
+
const { docLibId, docId } = this.ctx.query || {};
|
|
133
|
+
return this.trackRequestImpl(`${docLibId},${docId}`, 'docLibView');
|
|
137
134
|
}
|
|
138
135
|
|
|
139
|
-
|
|
140
|
-
return this.trackRequestImpl(resource, resourceType)
|
|
136
|
+
return this.trackRequestImpl(resource, resourceType);
|
|
141
137
|
}
|
|
142
138
|
|
|
143
|
-
|
|
144
139
|
private async trackRequestImpl(resource: string, resource_type: string) {
|
|
145
140
|
const client = this.redisService;
|
|
146
141
|
const ip = this.ctx.headers['x-real-ip'];
|
|
147
|
-
const userSession = this.ctx.userSession
|
|
148
|
-
const req_host = this.ctx?.request?.host ||
|
|
142
|
+
const userSession = this.ctx.userSession;
|
|
143
|
+
const req_host = this.ctx?.request?.host || 'null'; // 域名
|
|
149
144
|
const date = new Date().toISOString().split('T')[0];
|
|
150
145
|
|
|
151
146
|
const toDateResourceKey = (stat_type: string): string => {
|
|
152
|
-
return
|
|
153
|
-
}
|
|
147
|
+
return `${RedisKeys.VISIT_STAT_DATE_PREFIX}${date}::${req_host}::${resource_type}::${stat_type}::${resource}`;
|
|
148
|
+
};
|
|
154
149
|
|
|
155
150
|
const pv_key = toDateResourceKey(STAT_TYPES.pv);
|
|
156
151
|
const uv_ip_key = toDateResourceKey(STAT_TYPES.uv_ip);
|
|
157
152
|
const uv_uid_key = toDateResourceKey(STAT_TYPES.uv_uid);
|
|
158
153
|
|
|
159
|
-
|
|
160
154
|
// 统计 PV
|
|
161
|
-
await Promise.all([
|
|
162
|
-
client.incr(pv_key),
|
|
163
|
-
client.expire(pv_key, EXPIRE_TIME)
|
|
164
|
-
]);
|
|
155
|
+
await Promise.all([client.incr(pv_key), client.expire(pv_key, EXPIRE_TIME)]);
|
|
165
156
|
|
|
166
157
|
// IP 访问 UV
|
|
167
|
-
await Promise.all([
|
|
168
|
-
client.sadd(uv_ip_key, `${ip}`),
|
|
169
|
-
client.expire(uv_ip_key, EXPIRE_TIME)
|
|
170
|
-
]);
|
|
158
|
+
await Promise.all([client.sadd(uv_ip_key, `${ip}`), client.expire(uv_ip_key, EXPIRE_TIME)]);
|
|
171
159
|
|
|
172
160
|
if (userSession && userSession.isLogin()) {
|
|
173
161
|
const accountId = userSession.getSessionInfo().accountId;
|
|
174
162
|
// 统计 UV
|
|
175
|
-
await Promise.all([
|
|
176
|
-
client.sadd(uv_uid_key, accountId),
|
|
177
|
-
client.expire(uv_uid_key, EXPIRE_TIME)
|
|
178
|
-
]);
|
|
163
|
+
await Promise.all([client.sadd(uv_uid_key, accountId), client.expire(uv_uid_key, EXPIRE_TIME)]);
|
|
179
164
|
}
|
|
180
|
-
|
|
181
|
-
};
|
|
182
|
-
|
|
165
|
+
}
|
|
183
166
|
}
|
|
@@ -1,17 +1,17 @@
|
|
|
1
1
|
import { Inject, Provide } from '@midwayjs/core';
|
|
2
2
|
import { Context } from '@midwayjs/koa';
|
|
3
|
-
import { KeysOfSimpleSQL } from '
|
|
4
|
-
import { SystemTables} from '
|
|
3
|
+
import { KeysOfSimpleSQL } from '@/libs/crud-pro/models/keys';
|
|
4
|
+
import { SystemTables} from '@/models/SystemTables';
|
|
5
5
|
import { LRUCache } from 'lru-cache';
|
|
6
|
-
import { BizException } from '
|
|
6
|
+
import { BizException } from '@/models/devops';
|
|
7
7
|
import { CurdMixService } from '../curd/CurdMixService';
|
|
8
|
-
import { ISysAnyApiEntity } from '
|
|
8
|
+
import { ISysAnyApiEntity } from '@/models/SystemEntities';
|
|
9
9
|
import { AnyApiSandboxService, IRunInSandboxParams } from './AnyApiSandboxService';
|
|
10
|
-
import { parseJsonObject } from '
|
|
10
|
+
import { parseJsonObject } from '@/libs/utils/functions';
|
|
11
11
|
import * as _ from 'lodash';
|
|
12
|
-
import { MixinUtils } from '
|
|
12
|
+
import { MixinUtils } from '@/libs/crud-pro/utils/MixinUtils';
|
|
13
13
|
import { WorkbenchService } from '../WorkbenchService';
|
|
14
|
-
import { validateByCfgString } from '
|
|
14
|
+
import { validateByCfgString } from '@/libs/crud-pro/utils/ValidateUtils';
|
|
15
15
|
import { API_BASE_TYPE, ApiBaseService } from '../base/ApiBaseService';
|
|
16
16
|
import { GLOBAL_STATIC_CONFIG } from '@/libs/global-config/global-config';
|
|
17
17
|
|
|
@@ -1,18 +1,18 @@
|
|
|
1
|
-
import {Inject, Provide} from '@midwayjs/core';
|
|
2
|
-
import {Context, IMidwayKoaContext} from '@midwayjs/koa';
|
|
3
|
-
import {BaseService} from
|
|
4
|
-
import {IScheduleService} from
|
|
5
|
-
import {CurdProService} from
|
|
6
|
-
import {ISysAsyncTaskHandler, SysAsyncTaskEntity, SysAsyncTaskStatus} from
|
|
7
|
-
import {GLOBAL_STATIC_CONFIG} from
|
|
8
|
-
import {SystemTables} from
|
|
9
|
-
import {KeysOfSimpleSQL} from
|
|
10
|
-
import {errorToString} from
|
|
11
|
-
import {ANONYMOUS_CONTEXT} from
|
|
12
|
-
|
|
1
|
+
import { Inject, Provide } from '@midwayjs/core';
|
|
2
|
+
import { Context, IMidwayKoaContext } from '@midwayjs/koa';
|
|
3
|
+
import { BaseService } from '@/service/base/BaseService';
|
|
4
|
+
import { IScheduleService } from '@/interface';
|
|
5
|
+
import { CurdProService } from '@/service/curd/CurdProService';
|
|
6
|
+
import { ISysAsyncTaskHandler, SysAsyncTaskEntity, SysAsyncTaskPartial, SysAsyncTaskStatus } from '@/models/AsyncTaskModel';
|
|
7
|
+
import { GLOBAL_STATIC_CONFIG } from '@/libs/global-config/global-config';
|
|
8
|
+
import { SystemTables } from '@/models/SystemTables';
|
|
9
|
+
import { KeysOfSimpleSQL } from '@/libs/crud-pro/models/keys';
|
|
10
|
+
import { errorToString } from '@/libs/utils/errorToString';
|
|
11
|
+
import { ANONYMOUS_CONTEXT } from '@/schedule';
|
|
12
|
+
import { RedisKeys } from '@/models/RedisKeys';
|
|
13
13
|
|
|
14
14
|
class AsyncTaskRunner {
|
|
15
|
-
isBusy
|
|
15
|
+
isBusy = false;
|
|
16
16
|
taskHandlerMap: Map<string, ISysAsyncTaskHandler> = new Map();
|
|
17
17
|
|
|
18
18
|
async executeTaskList(taskList: SysAsyncTaskEntity[]) {
|
|
@@ -30,18 +30,20 @@ class AsyncTaskRunner {
|
|
|
30
30
|
await this.executeTask(taskElement);
|
|
31
31
|
} catch (error) {
|
|
32
32
|
taskElement.task_status = SysAsyncTaskStatus.FAILED;
|
|
33
|
-
taskElement.error_message = errorToString(error)
|
|
33
|
+
taskElement.error_message = errorToString(error);
|
|
34
34
|
}
|
|
35
35
|
|
|
36
36
|
try {
|
|
37
|
-
await this.updateTaskStatus(taskElement
|
|
37
|
+
await this.updateTaskStatus(taskElement, {
|
|
38
|
+
task_status: taskElement.task_status,
|
|
39
|
+
error_message: taskElement.error_message,
|
|
40
|
+
});
|
|
38
41
|
} catch (error) {
|
|
39
|
-
ANONYMOUS_CONTEXT.getApp().getCoreLogger().error(
|
|
42
|
+
ANONYMOUS_CONTEXT.getApp().getCoreLogger().error('executeTaskList error', error);
|
|
40
43
|
}
|
|
41
44
|
}
|
|
42
45
|
|
|
43
46
|
this.isBusy = false;
|
|
44
|
-
|
|
45
47
|
}
|
|
46
48
|
|
|
47
49
|
private async executeTask(taskElement: SysAsyncTaskEntity) {
|
|
@@ -50,42 +52,44 @@ class AsyncTaskRunner {
|
|
|
50
52
|
if (!taskHandler) {
|
|
51
53
|
throw new Error('TaskHandler not found , taskType = ' + taskType);
|
|
52
54
|
}
|
|
53
|
-
const updateTaskStatus = () => {
|
|
54
|
-
return this.updateTaskStatus(taskElement);
|
|
55
|
-
}
|
|
56
|
-
await taskHandler.execute({task: taskElement, updateTaskStatus});
|
|
55
|
+
const updateTaskStatus = (updatePartials: SysAsyncTaskPartial) => {
|
|
56
|
+
return this.updateTaskStatus(taskElement, updatePartials);
|
|
57
|
+
};
|
|
58
|
+
await taskHandler.execute({ task: taskElement, updateTaskStatus });
|
|
57
59
|
}
|
|
58
60
|
|
|
59
|
-
|
|
60
61
|
/**
|
|
61
62
|
* 更新任务状态或任务进度
|
|
62
63
|
* @param taskElement
|
|
64
|
+
* @param updatePartials
|
|
63
65
|
* @private
|
|
64
66
|
*/
|
|
65
|
-
private async updateTaskStatus(taskElement: SysAsyncTaskEntity) {
|
|
67
|
+
private async updateTaskStatus(taskElement: SysAsyncTaskEntity, updatePartials: SysAsyncTaskPartial) {
|
|
68
|
+
if (!updatePartials) {
|
|
69
|
+
throw new Error('updatePartials not found');
|
|
70
|
+
}
|
|
66
71
|
return await ANONYMOUS_CONTEXT.runServiceAtAnonymousContext(async (ctx: IMidwayKoaContext) => {
|
|
67
|
-
const curdProService: CurdProService = await ctx.requestContext.getAsync(
|
|
68
|
-
const {SystemDbName, SystemDbType} = GLOBAL_STATIC_CONFIG.getConfig();
|
|
69
|
-
|
|
70
|
-
const
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
72
|
+
const curdProService: CurdProService = await ctx.requestContext.getAsync('curdProService');
|
|
73
|
+
const { SystemDbName, SystemDbType } = GLOBAL_STATIC_CONFIG.getConfig();
|
|
74
|
+
|
|
75
|
+
const res = await curdProService.executeCrudByCfg(
|
|
76
|
+
{
|
|
77
|
+
condition: {
|
|
78
|
+
id: taskElement.id,
|
|
79
|
+
},
|
|
80
|
+
data: updatePartials,
|
|
75
81
|
},
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
82
|
+
{
|
|
83
|
+
sqlTable: SystemTables.sys_async_tasks,
|
|
84
|
+
sqlSimpleName: KeysOfSimpleSQL.SIMPLE_UPDATE,
|
|
85
|
+
sqlDatabase: SystemDbName,
|
|
86
|
+
sqlDbType: SystemDbType,
|
|
87
|
+
}
|
|
88
|
+
);
|
|
83
89
|
return res.getResModel().affected;
|
|
84
90
|
});
|
|
85
|
-
|
|
86
91
|
}
|
|
87
92
|
|
|
88
|
-
|
|
89
93
|
/**
|
|
90
94
|
* 获取当前进程可以处理的任务类型。
|
|
91
95
|
*/
|
|
@@ -93,35 +97,32 @@ class AsyncTaskRunner {
|
|
|
93
97
|
const keys = this.taskHandlerMap.keys();
|
|
94
98
|
return [...keys];
|
|
95
99
|
}
|
|
96
|
-
|
|
97
|
-
|
|
98
100
|
}
|
|
99
101
|
|
|
100
|
-
|
|
101
102
|
/**
|
|
102
103
|
* 业务可以扩展
|
|
103
104
|
*/
|
|
104
105
|
export const ASYNC_TASK_RUNNER = new AsyncTaskRunner();
|
|
105
106
|
|
|
106
|
-
|
|
107
107
|
/**
|
|
108
108
|
* Redis锁
|
|
109
109
|
*/
|
|
110
|
-
const
|
|
110
|
+
const ASYNC_TASK_LOCK = RedisKeys.ASYNC_TASK_LOCK;
|
|
111
|
+
|
|
112
|
+
const ASYNC_TASK_RUNTIME_OBJ = {
|
|
113
|
+
LAST_CHECK_ASYNC_TASK_UPDATE_TIME: 0,
|
|
114
|
+
};
|
|
111
115
|
|
|
112
116
|
@Provide()
|
|
113
117
|
export class AsyncTaskRunnerService extends BaseService implements IScheduleService {
|
|
114
|
-
|
|
115
118
|
@Inject()
|
|
116
119
|
protected ctx: Context;
|
|
117
120
|
|
|
118
121
|
@Inject()
|
|
119
122
|
private curdProService: CurdProService;
|
|
120
123
|
|
|
121
|
-
|
|
122
124
|
async fetchPendingTasks(): Promise<void> {
|
|
123
|
-
|
|
124
|
-
const {SystemDbName, SystemDbType} = GLOBAL_STATIC_CONFIG.getConfig();
|
|
125
|
+
const { SystemDbName, SystemDbType } = GLOBAL_STATIC_CONFIG.getConfig();
|
|
125
126
|
|
|
126
127
|
// 只获取本进程能够处理的任务类型。
|
|
127
128
|
const taskTypeList = ASYNC_TASK_RUNNER.getHandlerTaskTypeList();
|
|
@@ -129,66 +130,74 @@ export class AsyncTaskRunnerService extends BaseService implements IScheduleServ
|
|
|
129
130
|
return Promise.resolve();
|
|
130
131
|
}
|
|
131
132
|
|
|
132
|
-
|
|
133
133
|
// 查询等待处理的任务。
|
|
134
|
-
const queryRes = await this.curdProService.executeCrudByCfg(
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
134
|
+
const queryRes = await this.curdProService.executeCrudByCfg(
|
|
135
|
+
{
|
|
136
|
+
condition: {
|
|
137
|
+
task_status: SysAsyncTaskStatus.PENDING,
|
|
138
|
+
task_type: {
|
|
139
|
+
$in: taskTypeList,
|
|
140
|
+
},
|
|
141
|
+
},
|
|
142
|
+
orderBy: 'id+',
|
|
143
|
+
limit: 10,
|
|
140
144
|
},
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
});
|
|
149
|
-
|
|
145
|
+
{
|
|
146
|
+
sqlTable: SystemTables.sys_async_tasks,
|
|
147
|
+
sqlSimpleName: KeysOfSimpleSQL.SIMPLE_QUERY_PAGE,
|
|
148
|
+
sqlDatabase: SystemDbName,
|
|
149
|
+
sqlDbType: SystemDbType,
|
|
150
|
+
}
|
|
151
|
+
);
|
|
150
152
|
|
|
151
153
|
const taskList = queryRes.getResRows();
|
|
152
154
|
|
|
153
155
|
if (taskList.length === 0) {
|
|
154
156
|
return Promise.resolve();
|
|
155
157
|
}
|
|
156
|
-
const taskIds = taskList.map(elem => elem.id).filter(Boolean)
|
|
158
|
+
const taskIds = taskList.map(elem => elem.id).filter(Boolean);
|
|
157
159
|
|
|
158
160
|
// 将状态更新为处理中。防止其它进程重复处理。
|
|
159
|
-
await this.curdProService.executeCrudByCfg(
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
161
|
+
await this.curdProService.executeCrudByCfg(
|
|
162
|
+
{
|
|
163
|
+
condition: {
|
|
164
|
+
id: {
|
|
165
|
+
$in: taskIds,
|
|
166
|
+
},
|
|
167
|
+
},
|
|
168
|
+
data: {
|
|
169
|
+
task_status: SysAsyncTaskStatus.RUNNING,
|
|
170
|
+
},
|
|
164
171
|
},
|
|
165
|
-
|
|
166
|
-
|
|
172
|
+
{
|
|
173
|
+
sqlTable: SystemTables.sys_async_tasks,
|
|
174
|
+
sqlSimpleName: KeysOfSimpleSQL.SIMPLE_UPDATE,
|
|
175
|
+
sqlDatabase: SystemDbName,
|
|
176
|
+
sqlDbType: SystemDbType,
|
|
167
177
|
}
|
|
168
|
-
|
|
169
|
-
sqlTable: SystemTables.sys_async_tasks,
|
|
170
|
-
sqlSimpleName: KeysOfSimpleSQL.SIMPLE_UPDATE,
|
|
171
|
-
sqlDatabase: SystemDbName,
|
|
172
|
-
sqlDbType: SystemDbType,
|
|
173
|
-
});
|
|
174
|
-
|
|
178
|
+
);
|
|
175
179
|
|
|
176
180
|
// 开始执行。
|
|
177
181
|
ASYNC_TASK_RUNNER.executeTaskList(taskList).then(() => {
|
|
178
|
-
console.log(
|
|
179
|
-
})
|
|
180
|
-
|
|
182
|
+
console.log('ASYNC_TASK_RUNNER finished taskIds ==> ' + JSON.stringify(taskIds));
|
|
183
|
+
});
|
|
181
184
|
}
|
|
182
185
|
|
|
183
|
-
|
|
184
186
|
async runBySchedule() {
|
|
187
|
+
// 1. 当前很忙
|
|
185
188
|
if (ASYNC_TASK_RUNNER.isBusy) {
|
|
186
189
|
return Promise.resolve();
|
|
187
190
|
}
|
|
188
191
|
|
|
192
|
+
// 2. 发现了新任务
|
|
193
|
+
const isExistNewTask = await this.isExistNewTask();
|
|
194
|
+
if (!isExistNewTask) {
|
|
195
|
+
return Promise.resolve();
|
|
196
|
+
}
|
|
197
|
+
|
|
189
198
|
// 这里的过期时间1分钟即可。fetchPendingTasks函数不可能超过一分钟。
|
|
190
199
|
// 因为这里只是从数据库中获取一批任务,放到自己的任务队列里。还没触发执行。
|
|
191
|
-
const lock = await this.redisService.set(
|
|
200
|
+
const lock = await this.redisService.set(ASYNC_TASK_LOCK, 1, 'EX', 60, 'NX');
|
|
192
201
|
if (lock !== 'OK') {
|
|
193
202
|
return Promise.resolve();
|
|
194
203
|
}
|
|
@@ -196,13 +205,35 @@ export class AsyncTaskRunnerService extends BaseService implements IScheduleServ
|
|
|
196
205
|
try {
|
|
197
206
|
await this.fetchPendingTasks();
|
|
198
207
|
} catch (e) {
|
|
199
|
-
console.error(
|
|
208
|
+
console.error('fetchPendingTasks error', errorToString(e));
|
|
200
209
|
}
|
|
201
210
|
|
|
202
|
-
await this.redisService.del(
|
|
211
|
+
await this.redisService.del(ASYNC_TASK_LOCK);
|
|
203
212
|
|
|
204
213
|
return Promise.resolve();
|
|
205
214
|
}
|
|
206
215
|
|
|
216
|
+
/**
|
|
217
|
+
* 是否存在新任务
|
|
218
|
+
* @private
|
|
219
|
+
*/
|
|
220
|
+
private async isExistNewTask(): Promise<boolean> {
|
|
221
|
+
// 刚启动,没有检查过。
|
|
222
|
+
if (!ASYNC_TASK_RUNTIME_OBJ.LAST_CHECK_ASYNC_TASK_UPDATE_TIME) {
|
|
223
|
+
return true;
|
|
224
|
+
}
|
|
225
|
+
|
|
226
|
+
//Redis没有任何内容
|
|
227
|
+
const updateTime = await this.redisService.get(RedisKeys.ASYNC_TASK_UPDATE_TIME);
|
|
228
|
+
if (!updateTime) {
|
|
229
|
+
ASYNC_TASK_RUNTIME_OBJ.LAST_CHECK_ASYNC_TASK_UPDATE_TIME = Date.now();
|
|
230
|
+
return false;
|
|
231
|
+
}
|
|
232
|
+
|
|
233
|
+
const updateTimeNumber = Number.parseInt(updateTime, 10);
|
|
207
234
|
|
|
235
|
+
const isExistNewTask: boolean = updateTimeNumber > ASYNC_TASK_RUNTIME_OBJ.LAST_CHECK_ASYNC_TASK_UPDATE_TIME;
|
|
236
|
+
ASYNC_TASK_RUNTIME_OBJ.LAST_CHECK_ASYNC_TASK_UPDATE_TIME = updateTimeNumber;
|
|
237
|
+
return isExistNewTask;
|
|
238
|
+
}
|
|
208
239
|
}
|
|
@@ -1,8 +1,9 @@
|
|
|
1
|
-
import {Inject, Provide} from '@midwayjs/core';
|
|
2
|
-
import {Context} from '@midwayjs/koa';
|
|
3
|
-
import {BaseService} from
|
|
4
|
-
import {runScheduleTaskOnce} from
|
|
5
|
-
import {
|
|
1
|
+
import { Inject, Provide } from '@midwayjs/core';
|
|
2
|
+
import { Context } from '@midwayjs/koa';
|
|
3
|
+
import { BaseService } from '@/service/base/BaseService';
|
|
4
|
+
import { runScheduleTaskOnce } from '@/schedule';
|
|
5
|
+
import { INNER_SCHEDULE_NAMES } from '@/schedule/scheduleNames';
|
|
6
|
+
import { RedisKeys } from '@/models/RedisKeys';
|
|
6
7
|
|
|
7
8
|
@Provide()
|
|
8
9
|
export class AsyncTaskService extends BaseService {
|
|
@@ -10,12 +11,11 @@ export class AsyncTaskService extends BaseService {
|
|
|
10
11
|
protected ctx: Context;
|
|
11
12
|
|
|
12
13
|
async startTask() {
|
|
13
|
-
|
|
14
|
+
await this.redisService.set(RedisKeys.ASYNC_TASK_UPDATE_TIME, `${Date.now()}`);
|
|
15
|
+
runScheduleTaskOnce(INNER_SCHEDULE_NAMES.asyncTaskRunnerService).then(schedule => {
|
|
14
16
|
console.log(schedule);
|
|
15
|
-
})
|
|
17
|
+
});
|
|
16
18
|
}
|
|
17
19
|
|
|
18
|
-
async cancelTask(id: number) {
|
|
19
|
-
|
|
20
|
-
}
|
|
20
|
+
async cancelTask(id: number) {}
|
|
21
21
|
}
|
|
@@ -86,7 +86,7 @@ export class ProxyApiLoadService extends BaseService implements IScheduleService
|
|
|
86
86
|
}
|
|
87
87
|
|
|
88
88
|
private async buildUpstreamMap() {
|
|
89
|
-
|
|
89
|
+
|
|
90
90
|
const { SystemDbName, SystemDbType } = GLOBAL_STATIC_CONFIG.getConfig();
|
|
91
91
|
|
|
92
92
|
// biz_tag={'upstream'}
|
|
@@ -114,10 +114,10 @@ export class ProxyApiLoadService extends BaseService implements IScheduleService
|
|
|
114
114
|
}
|
|
115
115
|
|
|
116
116
|
private async loadProxyApiEntity(workbench_code: string, upstreamMap: Record<string, IUpstreamInfo>): Promise<IProxyApiEntity[]> {
|
|
117
|
-
|
|
117
|
+
|
|
118
118
|
const { SystemDbName, SystemDbType } = GLOBAL_STATIC_CONFIG.getConfig();
|
|
119
119
|
|
|
120
|
-
|
|
120
|
+
|
|
121
121
|
const res = await this.curdProService.executeCrudByCfg(
|
|
122
122
|
{
|
|
123
123
|
condition: {},
|
|
@@ -170,4 +170,5 @@ export class ProxyApiLoadService extends BaseService implements IScheduleService
|
|
|
170
170
|
}
|
|
171
171
|
}
|
|
172
172
|
}
|
|
173
|
+
|
|
173
174
|
}
|