oak-backend-base 4.1.28 → 4.1.29
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/Synchronizer.d.ts +10 -1
- package/lib/Synchronizer.js +51 -8
- package/lib/routines/update.d.ts +65 -0
- package/lib/routines/update.js +784 -0
- package/lib/utils/dbPriority.js +1 -1
- package/package.json +12 -8
package/lib/Synchronizer.d.ts
CHANGED
|
@@ -2,12 +2,21 @@ import { EntityDict, StorageSchema, EndpointItem, SyncConfig } from 'oak-domain/
|
|
|
2
2
|
import { VolatileTrigger } from 'oak-domain/lib/types/Trigger';
|
|
3
3
|
import { EntityDict as BaseEntityDict } from 'oak-domain/lib/base-app-domain';
|
|
4
4
|
import { BackendRuntimeContext } from 'oak-frontend-base/lib/context/BackendRuntimeContext';
|
|
5
|
+
export type FetchFn = (url: string, options: {
|
|
6
|
+
method: string;
|
|
7
|
+
headers: Record<string, string>;
|
|
8
|
+
body: string;
|
|
9
|
+
}, timeout?: number) => Promise<{
|
|
10
|
+
status: number;
|
|
11
|
+
json: () => Promise<any>;
|
|
12
|
+
}>;
|
|
5
13
|
export default class Synchronizer<ED extends EntityDict & BaseEntityDict, Cxt extends BackendRuntimeContext<ED>> {
|
|
6
14
|
private config;
|
|
7
15
|
private schema;
|
|
8
16
|
private remotePullInfoMap;
|
|
9
17
|
private channelDict;
|
|
10
18
|
private contextBuilder;
|
|
19
|
+
private fetchFn;
|
|
11
20
|
private pushAccessMap;
|
|
12
21
|
private startChannel2;
|
|
13
22
|
/**开始同步这些channel上的oper。注意,这时候即使某个channel上失败了,也不应影响本事务提交(其它的channel成功了) */
|
|
@@ -22,7 +31,7 @@ export default class Synchronizer<ED extends EntityDict & BaseEntityDict, Cxt ex
|
|
|
22
31
|
*/
|
|
23
32
|
private trySynchronizeOpers;
|
|
24
33
|
private makeCreateOperTrigger;
|
|
25
|
-
constructor(config: SyncConfig<ED, Cxt>, schema: StorageSchema<ED>, contextBuilder: () => Cxt);
|
|
34
|
+
constructor(config: SyncConfig<ED, Cxt>, schema: StorageSchema<ED>, contextBuilder: () => Cxt, fetchFn?: FetchFn);
|
|
26
35
|
/**
|
|
27
36
|
* 根据sync的定义,生成对应的 commit triggers
|
|
28
37
|
* @returns
|
package/lib/Synchronizer.js
CHANGED
|
@@ -5,11 +5,11 @@ const crypto_1 = require("crypto");
|
|
|
5
5
|
const types_1 = require("oak-domain/lib/types");
|
|
6
6
|
const relationPath_1 = require("oak-domain/lib/utils/relationPath");
|
|
7
7
|
const assert_1 = tslib_1.__importDefault(require("assert"));
|
|
8
|
-
const path_1 = require("path");
|
|
9
8
|
const lodash_1 = require("oak-domain/lib/utils/lodash");
|
|
10
9
|
const filter_1 = require("oak-domain/lib/store/filter");
|
|
11
10
|
const uuid_1 = require("oak-domain/lib/utils/uuid");
|
|
12
11
|
const lodash_2 = require("lodash");
|
|
12
|
+
const domain_1 = require("oak-domain/lib/utils/domain");
|
|
13
13
|
const OAK_SYNC_HEADER_ENTITY = 'oak-sync-entity';
|
|
14
14
|
const OAK_SYNC_HEADER_ENTITY_ID = 'oak-sync-entity-id';
|
|
15
15
|
const OAK_SYNC_HEADER_TIMESTAMP = 'oak-sync-timestamp';
|
|
@@ -67,6 +67,7 @@ class Synchronizer {
|
|
|
67
67
|
remotePullInfoMap = {};
|
|
68
68
|
channelDict = {};
|
|
69
69
|
contextBuilder;
|
|
70
|
+
fetchFn;
|
|
70
71
|
pushAccessMap = {};
|
|
71
72
|
async startChannel2(context, channel) {
|
|
72
73
|
const { queue, api, selfEncryptInfo, entity, entityId, onFailed, timeout = 5000 } = channel;
|
|
@@ -78,12 +79,12 @@ class Synchronizer {
|
|
|
78
79
|
seq: ele.$$seq$$,
|
|
79
80
|
}))), 'txnId:', context.getCurrentTxnId());
|
|
80
81
|
}
|
|
81
|
-
const finalApi = (0,
|
|
82
|
+
const finalApi = (0, domain_1.urlJoin)(api, selfEncryptInfo.id);
|
|
82
83
|
channel.queue = [];
|
|
83
84
|
try {
|
|
84
85
|
const body = JSON.stringify(opers);
|
|
85
86
|
const { ts, nonce, signature } = await sign(selfEncryptInfo.privateKey, body);
|
|
86
|
-
const res = await
|
|
87
|
+
const res = await this.fetchFn(finalApi, {
|
|
87
88
|
method: 'post',
|
|
88
89
|
headers: {
|
|
89
90
|
'Content-Type': 'application/json',
|
|
@@ -190,7 +191,7 @@ class Synchronizer {
|
|
|
190
191
|
if (!this.channelDict[userId]) {
|
|
191
192
|
// channel上缓存这些信息,暂不支持动态更新
|
|
192
193
|
this.channelDict[userId] = {
|
|
193
|
-
api: (0,
|
|
194
|
+
api: (0, domain_1.urlJoin)(url, 'endpoint', endpoint),
|
|
194
195
|
queue: [],
|
|
195
196
|
entity: remoteEntity,
|
|
196
197
|
entityId: remoteEntityId,
|
|
@@ -205,7 +206,7 @@ class Synchronizer {
|
|
|
205
206
|
(0, assert_1.default)(this.channelDict[userId].onFailed === onFailed);
|
|
206
207
|
}
|
|
207
208
|
const channel = this.channelDict[userId];
|
|
208
|
-
(0, assert_1.default)(channel.api === (0,
|
|
209
|
+
(0, assert_1.default)(channel.api === (0, domain_1.urlJoin)(url, 'endpoint', endpoint));
|
|
209
210
|
(0, assert_1.default)(channel.entity === remoteEntity);
|
|
210
211
|
(0, assert_1.default)(channel.entityId === remoteEntityId);
|
|
211
212
|
if (channel.queue.find(ele => ele.oper.id === oper.id)) {
|
|
@@ -343,8 +344,19 @@ class Synchronizer {
|
|
|
343
344
|
}, { dontCollect: true, forUpdate: true });
|
|
344
345
|
dirtyOpers = dirtyOpers.filter(ele => !!ele[types_1.TriggerUuidAttribute]);
|
|
345
346
|
if (dirtyOpers.length > 0) {
|
|
347
|
+
// 检查所有 channel 的 queue 是否已清空
|
|
346
348
|
for (const c in this.channelDict) {
|
|
347
|
-
|
|
349
|
+
// 在生产环境不使用assert,而是清空队列,以防止脏数据堆积影响后续同步
|
|
350
|
+
if (this.channelDict[c].queue.length > 0) {
|
|
351
|
+
if (process.env.NODE_ENV !== 'production') {
|
|
352
|
+
// 构建详细的错误信息
|
|
353
|
+
const queuedOperIds = this.channelDict[c].queue.map(q => q.oper.id).join(', ');
|
|
354
|
+
(0, assert_1.default)(false, `发现 channel ${c} 的 queue 未清空,包含 oper IDs: [${queuedOperIds}]`);
|
|
355
|
+
}
|
|
356
|
+
else {
|
|
357
|
+
this.channelDict[c].queue = [];
|
|
358
|
+
}
|
|
359
|
+
}
|
|
348
360
|
}
|
|
349
361
|
const pushedIds = [];
|
|
350
362
|
const unPushedIds = [];
|
|
@@ -394,15 +406,45 @@ class Synchronizer {
|
|
|
394
406
|
makeCreateOperTrigger() {
|
|
395
407
|
const { config } = this;
|
|
396
408
|
const { remotes, self } = config;
|
|
409
|
+
const entityActionMap = new Map();
|
|
397
410
|
// 根据remotes定义,建立从entity到需要同步的远端结点信息的Map
|
|
398
411
|
remotes.forEach((remote) => {
|
|
399
412
|
const { getPushInfo, pushEntities: pushEntityDefs, endpoint, pathToUser, relationName: rnRemote, onFailed, timeout } = remote;
|
|
400
413
|
if (pushEntityDefs) {
|
|
401
414
|
const pushEntities = [];
|
|
402
|
-
const endpoint2 = (0,
|
|
415
|
+
const endpoint2 = (0, domain_1.urlJoin)(endpoint || 'sync', self.entity);
|
|
403
416
|
for (const def of pushEntityDefs) {
|
|
404
417
|
const { pathToRemoteEntity, pathToSelfEntity, relationName, recursive, entity, actions, onSynchronized } = def;
|
|
405
418
|
pushEntities.push(entity);
|
|
419
|
+
if (!entityActionMap.has(entity)) {
|
|
420
|
+
entityActionMap.set(entity, {
|
|
421
|
+
actions: new Set(),
|
|
422
|
+
matchAllActions: false,
|
|
423
|
+
});
|
|
424
|
+
}
|
|
425
|
+
const tracker = entityActionMap.get(entity);
|
|
426
|
+
//如果当前定义是 wildcard(所有 action)
|
|
427
|
+
if (!actions || actions.length === 0) {
|
|
428
|
+
if (tracker.matchAllActions) {
|
|
429
|
+
throw new Error(`PushEntityDef 配置错误:entity「${entity}」的所有 action 被定义了多次`);
|
|
430
|
+
}
|
|
431
|
+
if (tracker.actions.size > 0) {
|
|
432
|
+
throw new Error(`PushEntityDef 配置错误:entity「${entity}」已定义特定 action「${Array.from(tracker.actions).join(', ')}」,不能再定义所有 action`);
|
|
433
|
+
}
|
|
434
|
+
tracker.matchAllActions = true;
|
|
435
|
+
}
|
|
436
|
+
else {
|
|
437
|
+
// 如果当前定义是特定 action
|
|
438
|
+
if (tracker.matchAllActions) {
|
|
439
|
+
throw new Error(`PushEntityDef 配置错误:entity「${entity}」已定义所有 action,不能再定义特定 action「${actions.join(', ')}」`);
|
|
440
|
+
}
|
|
441
|
+
for (const action of actions) {
|
|
442
|
+
if (tracker.actions.has(action)) {
|
|
443
|
+
throw new Error(`PushEntityDef 配置错误:entity「${entity}」的action「${action}」被定义了多次`);
|
|
444
|
+
}
|
|
445
|
+
tracker.actions.add(action);
|
|
446
|
+
}
|
|
447
|
+
}
|
|
406
448
|
const relationName2 = relationName || rnRemote;
|
|
407
449
|
const path2 = pathToUser ? `${pathToRemoteEntity}.${pathToUser}` : pathToRemoteEntity;
|
|
408
450
|
(0, assert_1.default)(!recursive);
|
|
@@ -507,10 +549,11 @@ class Synchronizer {
|
|
|
507
549
|
};
|
|
508
550
|
return createOperTrigger;
|
|
509
551
|
}
|
|
510
|
-
constructor(config, schema, contextBuilder) {
|
|
552
|
+
constructor(config, schema, contextBuilder, fetchFn) {
|
|
511
553
|
this.config = config;
|
|
512
554
|
this.schema = schema;
|
|
513
555
|
this.contextBuilder = contextBuilder;
|
|
556
|
+
this.fetchFn = fetchFn || fetchWithTimeout;
|
|
514
557
|
}
|
|
515
558
|
/**
|
|
516
559
|
* 根据sync的定义,生成对应的 commit triggers
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
import { EntityDict as TypeEntityDict, StorageSchema } from 'oak-domain/lib/types';
|
|
2
|
+
import { AsyncContext } from 'oak-domain/lib/store/AsyncRowStore';
|
|
3
|
+
import BackendRuntimeContext from 'oak-frontend-base/lib/context/BackendRuntimeContext';
|
|
4
|
+
import { EntityDict as BaseEntityDict } from 'oak-domain/lib/base-app-domain';
|
|
5
|
+
type EntityDict = TypeEntityDict & BaseEntityDict;
|
|
6
|
+
/**
|
|
7
|
+
* 反向引用描述
|
|
8
|
+
* 用于记录哪些实体引用了当前实体,便于在删除或更新时维护引用完整性
|
|
9
|
+
*/
|
|
10
|
+
export type ReverseRefDesc<ED extends EntityDict> = {
|
|
11
|
+
type: 'ref' | 'refs';
|
|
12
|
+
attrName: string;
|
|
13
|
+
sourceEntity: keyof ED;
|
|
14
|
+
};
|
|
15
|
+
/**
|
|
16
|
+
* 更新计划配置
|
|
17
|
+
* 定义了数据同步的完整策略和生命周期钩子
|
|
18
|
+
*/
|
|
19
|
+
export type UpdatePlan<ED extends EntityDict> = {
|
|
20
|
+
dontCreateOper?: boolean;
|
|
21
|
+
plan: {
|
|
22
|
+
[entityName in keyof ED]: {
|
|
23
|
+
/**
|
|
24
|
+
* 是否允许数据唯一性冲突时使用新数据覆盖旧数据(仅data中冲突)
|
|
25
|
+
* @deprecated ID必须相同但会导致ID检查不通过,弃用
|
|
26
|
+
*/
|
|
27
|
+
allowDataUniqueReWrite?: boolean;
|
|
28
|
+
/**
|
|
29
|
+
* 在插入数据库遇到唯一性冲突时的处理方式(数据库和data冲突)
|
|
30
|
+
* skip表示跳过插入新数据,会将新数据的反指关系更新到已有数据上
|
|
31
|
+
* update表示使用新数据更新已有数据,会自动更新现有数据的所有反指关系
|
|
32
|
+
* error表示抛出错误
|
|
33
|
+
*/
|
|
34
|
+
onUniqueViolation?: 'skip' | 'update' | 'error';
|
|
35
|
+
/**
|
|
36
|
+
* 处理数据库中存在但是数据文件中不存在的数据的方式
|
|
37
|
+
* skip表示跳过不处理
|
|
38
|
+
* delete表示做逻辑删除,可能导致插入新数据时id冲突
|
|
39
|
+
* 设置为physicalDelete可以保证最强的一致性
|
|
40
|
+
*/
|
|
41
|
+
onOnlyExistingInDb?: 'skip' | 'delete' | 'physicalDelete';
|
|
42
|
+
};
|
|
43
|
+
};
|
|
44
|
+
beforeCheck?: (options: {
|
|
45
|
+
context: BackendRuntimeContext<ED>;
|
|
46
|
+
data: {
|
|
47
|
+
[entityName in keyof ED]?: Array<Partial<ED[entityName]['CreateOperationData']>>;
|
|
48
|
+
};
|
|
49
|
+
reverseRefs: {
|
|
50
|
+
[entityName in keyof ED]?: ReverseRefDesc<ED>[];
|
|
51
|
+
};
|
|
52
|
+
}) => Promise<void>;
|
|
53
|
+
afterUpdate?: (options: {
|
|
54
|
+
context: BackendRuntimeContext<ED>;
|
|
55
|
+
data: {
|
|
56
|
+
[entityName in keyof EntityDict]?: Array<Partial<EntityDict[entityName]['CreateOperationData']>>;
|
|
57
|
+
};
|
|
58
|
+
reverseRefs: {
|
|
59
|
+
[entityName in keyof EntityDict]?: ReverseRefDesc<ED>[];
|
|
60
|
+
};
|
|
61
|
+
}) => Promise<void>;
|
|
62
|
+
};
|
|
63
|
+
export declare const createUpdatePlan: <ED extends EntityDict>(options: UpdatePlan<ED>) => <Cxt extends AsyncContext<ED>>(context: Cxt) => Promise<void>;
|
|
64
|
+
export declare function checkPathValue(itemId: string, destEntity: keyof EntityDict, path: string, storageSchema: StorageSchema<EntityDict>): boolean;
|
|
65
|
+
export {};
|
|
@@ -0,0 +1,784 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.createUpdatePlan = void 0;
|
|
4
|
+
exports.checkPathValue = checkPathValue;
|
|
5
|
+
const tslib_1 = require("tslib");
|
|
6
|
+
/**
|
|
7
|
+
* 数据同步工具
|
|
8
|
+
*
|
|
9
|
+
* 核心职责:将项目 lib/data 目录下的数据文件同步到数据库,同时处理数据冲突、引用关系和删除策略
|
|
10
|
+
*
|
|
11
|
+
* ==================== 配置项说明 ====================
|
|
12
|
+
*
|
|
13
|
+
* 1. allowDataUniqueReWrite(已弃用)
|
|
14
|
+
* - 类型:boolean
|
|
15
|
+
* - 默认:false
|
|
16
|
+
* - 作用:当数据文件内部存在唯一索引冲突时,是否用后加载的数据覆盖先加载的数据
|
|
17
|
+
* - 限制:要求冲突数据的 ID 必须完全相同,否则会抛出错误
|
|
18
|
+
* - 弃用原因:ID 相同但会导致后续 ID 检查不通过
|
|
19
|
+
*
|
|
20
|
+
* 2. onUniqueViolation
|
|
21
|
+
* - 类型:'skip' | 'update' | 'error'
|
|
22
|
+
* - 默认:'error'
|
|
23
|
+
* - 作用:当新数据的唯一索引与数据库已有数据冲突时的处理策略
|
|
24
|
+
* - 行为说明:
|
|
25
|
+
* - 'error': 直接抛出错误,终止执行
|
|
26
|
+
* - 'skip': 跳过新数据的插入,但会将所有反向引用从旧数据 ID 更新到新数据 ID
|
|
27
|
+
* - 'update': 插入新数据,将反向引用更新到新数据 ID,然后删除旧数据
|
|
28
|
+
*
|
|
29
|
+
* 3. onOnlyExistingInDb
|
|
30
|
+
* - 类型:'skip' | 'delete' | 'physicalDelete'
|
|
31
|
+
* - 默认:'skip'
|
|
32
|
+
* - 作用:处理数据库中存在但数据文件中不存在的数据
|
|
33
|
+
* - 行为说明:
|
|
34
|
+
* - 'skip': 不处理,保留数据库中的数据
|
|
35
|
+
* - 'delete': 执行逻辑删除(设置 $$deleteAt$$),可能在插入新数据时导致 ID 冲突
|
|
36
|
+
* - 'physicalDelete': 执行物理删除,保证最强一致性,删除后可复用 ID
|
|
37
|
+
*
|
|
38
|
+
* 4. dontCreateOper
|
|
39
|
+
* - 类型:boolean
|
|
40
|
+
* - 作用:是否跳过创建操作记录(operation log)
|
|
41
|
+
* - 适用场景:批量数据迁移、初始化等不需要记录操作历史的场景
|
|
42
|
+
*
|
|
43
|
+
* 5. beforeCheck
|
|
44
|
+
* - 类型:异步函数
|
|
45
|
+
* - 执行时机:在所有数据校验之后、数据库操作之前
|
|
46
|
+
* - 参数:{ context, data, reverseRefs }
|
|
47
|
+
* - 作用:执行自定义的业务校验逻辑
|
|
48
|
+
*
|
|
49
|
+
* 6. afterUpdate
|
|
50
|
+
* - 类型:异步函数
|
|
51
|
+
* - 执行时机:所有数据库操作完成之后
|
|
52
|
+
* - 参数:{ context, data, reverseRefs }
|
|
53
|
+
* - 作用:执行后置处理,如缓存刷新、通知发送等
|
|
54
|
+
*
|
|
55
|
+
* ==================== 执行流程 ====================
|
|
56
|
+
*
|
|
57
|
+
* 阶段 1:数据加载与校验
|
|
58
|
+
* ├─ 1.1 加载 lib/data 目录下的所有实体数据
|
|
59
|
+
* ├─ 1.2 执行实体级别的前置检查器(entityBeforeCheckers)
|
|
60
|
+
* │ └─ 示例:path 实体会校验路径格式和引用合法性
|
|
61
|
+
* └─ 1.3 分析所有实体的反向引用关系(reverseRefs)
|
|
62
|
+
* ├─ 处理静态引用:xxxId 字段
|
|
63
|
+
* └─ 处理动态引用:entity + entityId 字段组合
|
|
64
|
+
*
|
|
65
|
+
* 阶段 2:数据差异对比(按实体逐个处理)
|
|
66
|
+
* ├─ 2.1 校验配置项合法性
|
|
67
|
+
* ├─ 2.2 构建 ID → 数据对象映射,检查 ID 重复
|
|
68
|
+
* ├─ 2.3 校验引用完整性
|
|
69
|
+
* │ ├─ 检查 ref 类型字段指向的记录是否存在
|
|
70
|
+
* │ └─ 检查动态引用(entity/entityId)的完整性
|
|
71
|
+
* ├─ 2.4 从数据库查询所有已存在的记录(使用 forUpdate 锁定)
|
|
72
|
+
* ├─ 2.5 对比数据差异
|
|
73
|
+
* │ ├─ 已存在:使用 diffAttrs 对比属性,生成待更新列表
|
|
74
|
+
* │ ├─ 不存在:加入待新增列表
|
|
75
|
+
* │ └─ 特殊处理:physicalDelete 模式下,已删除数据视为新增
|
|
76
|
+
* └─ 2.6 校验唯一索引
|
|
77
|
+
* ├─ 检查数据文件内部冲突(allowDataUniqueReWrite)
|
|
78
|
+
* └─ 检查与数据库冲突(onUniqueViolation)
|
|
79
|
+
*
|
|
80
|
+
* 阶段 3:数据库写入(按实体逐个处理)
|
|
81
|
+
* ├─ 3.1 物理删除已标记删除的数据(如果需要)
|
|
82
|
+
* ├─ 3.2 执行新增操作
|
|
83
|
+
* │ ├─ 调用 context.operate 创建记录
|
|
84
|
+
* │ └─ 如果存在反向引用更新需求(toUpdateRefs):
|
|
85
|
+
* │ ├─ 更新所有引用旧 ID 的记录为新 ID
|
|
86
|
+
* │ └─ 删除旧数据
|
|
87
|
+
* ├─ 3.3 执行更新操作
|
|
88
|
+
* │ └─ 仅更新有差异的字段
|
|
89
|
+
* └─ 3.4 收集待删除的数据 ID
|
|
90
|
+
*
|
|
91
|
+
* 阶段 4:依赖分析与删除
|
|
92
|
+
* ├─ 4.1 根据 reverseRefs 构建实体依赖图
|
|
93
|
+
* │ └─ 规则:引用方必须在被引用方之前删除
|
|
94
|
+
* ├─ 4.2 拓扑排序生成删除顺序
|
|
95
|
+
* └─ 4.3 按顺序执行删除操作
|
|
96
|
+
* └─ 根据 onOnlyExistingInDb 决定逻辑删除或物理删除
|
|
97
|
+
*
|
|
98
|
+
* 阶段 5:后置处理
|
|
99
|
+
* └─ 5.1 执行 afterUpdate 钩子
|
|
100
|
+
*
|
|
101
|
+
* ==================== 关键决策点 ====================
|
|
102
|
+
*
|
|
103
|
+
* 1. 唯一索引冲突处理的选择
|
|
104
|
+
* - skip: 适用于幂等性场景,多次执行不改变数据库状态
|
|
105
|
+
* - update: 适用于数据迁移,需要用新数据完全替换旧数据
|
|
106
|
+
* - error: 适用于严格模式,任何冲突都需要人工介入
|
|
107
|
+
*
|
|
108
|
+
* 2. 反向引用更新时机
|
|
109
|
+
* - 在 skip 模式:新数据不插入,但引用关系需要指向已存在的旧数据
|
|
110
|
+
* - 在 update 模式:新数据插入后,所有引用都需要更新到新数据
|
|
111
|
+
*
|
|
112
|
+
* 3. 删除顺序的必要性
|
|
113
|
+
* - 必须先删除引用方,再删除被引用方,避免外键约束错误
|
|
114
|
+
* - 使用拓扑排序保证删除的正确性
|
|
115
|
+
*
|
|
116
|
+
* ==================== 注意事项 ====================
|
|
117
|
+
*
|
|
118
|
+
* 1. 所有数据库操作都使用 blockTrigger: true 避免触发器干扰
|
|
119
|
+
* 2. 使用 forUpdate 锁定记录,避免并发修改导致的数据不一致
|
|
120
|
+
* 3. diffAttrs 会忽略系统字段($$createAt$$、$$updateAt$$、$$deleteAt$$)
|
|
121
|
+
* 4. compareInner 会深度比较对象和数组,支持 null/undefined 互换
|
|
122
|
+
* 5. 反向引用分为静态引用(ref)和动态引用(refs),处理逻辑不同
|
|
123
|
+
*/
|
|
124
|
+
const assert_1 = tslib_1.__importDefault(require("assert"));
|
|
125
|
+
const path_1 = require("path");
|
|
126
|
+
const dependencyBuilder_1 = require("oak-domain/lib/compiler/dependencyBuilder");
|
|
127
|
+
const types_1 = require("oak-domain/lib/types");
|
|
128
|
+
const uuid_1 = require("oak-domain/lib/utils/uuid");
|
|
129
|
+
const requirePrj_1 = tslib_1.__importDefault(require("../utils/requirePrj"));
|
|
130
|
+
const i18n_1 = tslib_1.__importDefault(require("oak-domain/lib/data/i18n"));
|
|
131
|
+
/**
|
|
132
|
+
* 对比两个对象的属性差异,返回需要更新的属性集合
|
|
133
|
+
*
|
|
134
|
+
* 行为说明:
|
|
135
|
+
* - 忽略系统内置属性($$createAt$$、$$updateAt$$、$$deleteAt$$)
|
|
136
|
+
* - 使用深度对比(compareInner)判断属性值是否真正发生变化
|
|
137
|
+
* - 只返回发生变化的属性,减少不必要的数据库更新
|
|
138
|
+
*
|
|
139
|
+
* @param oldData 旧数据(数据库中的数据)
|
|
140
|
+
* @param newData 新数据(数据文件中的数据)
|
|
141
|
+
* @returns 发生变化的属性集合
|
|
142
|
+
*/
|
|
143
|
+
const diffAttrs = (oldData, newData) => {
|
|
144
|
+
const compared = new Set();
|
|
145
|
+
const diffs = {};
|
|
146
|
+
for (const key in newData) {
|
|
147
|
+
// 忽略$$createAt$$和$$updateAt$$,$$deleteAt$$属性
|
|
148
|
+
if (types_1.initinctiveAttributes.includes(key)) {
|
|
149
|
+
compared.add(key);
|
|
150
|
+
continue;
|
|
151
|
+
}
|
|
152
|
+
if (newData[key] !== oldData[key]) {
|
|
153
|
+
if (!compareInner(newData[key], oldData[key])) {
|
|
154
|
+
diffs[key] = newData[key];
|
|
155
|
+
}
|
|
156
|
+
compared.add(key);
|
|
157
|
+
}
|
|
158
|
+
}
|
|
159
|
+
for (const key in oldData) {
|
|
160
|
+
if (types_1.initinctiveAttributes.includes(key)) {
|
|
161
|
+
continue;
|
|
162
|
+
}
|
|
163
|
+
if (compared.has(key)) {
|
|
164
|
+
continue;
|
|
165
|
+
}
|
|
166
|
+
// 这里表示之前有值在old里,但是新的里没有这个属性了,视为需要更新为null
|
|
167
|
+
if ((oldData[key] !== undefined && oldData[key] !== null) && newData[key] === undefined) {
|
|
168
|
+
diffs[key] = null; // 这里将undefined的属性更新为null,表示删除这个属性
|
|
169
|
+
}
|
|
170
|
+
}
|
|
171
|
+
return diffs;
|
|
172
|
+
};
|
|
173
|
+
/**
|
|
174
|
+
* 深度比较两个值是否相等
|
|
175
|
+
*
|
|
176
|
+
* 特殊处理:
|
|
177
|
+
* - null 和 undefined 视为相等
|
|
178
|
+
* - 递归比较对象和数组
|
|
179
|
+
* - 支持嵌套结构的完整对比
|
|
180
|
+
*
|
|
181
|
+
* @param a 值 A
|
|
182
|
+
* @param b 值 B
|
|
183
|
+
* @returns 是否相等
|
|
184
|
+
*/
|
|
185
|
+
const compareInner = (a, b) => {
|
|
186
|
+
if (typeof a !== typeof b) {
|
|
187
|
+
return false;
|
|
188
|
+
}
|
|
189
|
+
if (typeof a === 'object' && a !== null && b !== null) {
|
|
190
|
+
const aKeys = Object.keys(a);
|
|
191
|
+
const bKeys = Object.keys(b);
|
|
192
|
+
if (aKeys.length !== bKeys.length) {
|
|
193
|
+
return false;
|
|
194
|
+
}
|
|
195
|
+
for (const key of aKeys) {
|
|
196
|
+
if (!compareInner(a[key], b[key])) {
|
|
197
|
+
return false;
|
|
198
|
+
}
|
|
199
|
+
}
|
|
200
|
+
return true;
|
|
201
|
+
}
|
|
202
|
+
else if (Array.isArray(a) && Array.isArray(b)) {
|
|
203
|
+
if (a.length !== b.length) {
|
|
204
|
+
return false;
|
|
205
|
+
}
|
|
206
|
+
for (let i = 0; i < a.length; i++) {
|
|
207
|
+
if (!compareInner(a[i], b[i])) {
|
|
208
|
+
return false;
|
|
209
|
+
}
|
|
210
|
+
}
|
|
211
|
+
return true;
|
|
212
|
+
}
|
|
213
|
+
else {
|
|
214
|
+
return (a === b) || (a === undefined && b === null) || (a === null && b === undefined);
|
|
215
|
+
}
|
|
216
|
+
};
|
|
217
|
+
const createUpdatePlan = (options) => {
|
|
218
|
+
const entityPlans = options.plan;
|
|
219
|
+
const beforeCheckFn = options.beforeCheck || (async () => { });
|
|
220
|
+
const afterUpdateFn = options.afterUpdate || (async () => { });
|
|
221
|
+
const globalOptions = Object.assign({}, options.dontCreateOper ? { dontCreateOper: true } : {});
|
|
222
|
+
const update = async (context) => {
|
|
223
|
+
const pwd = process.cwd();
|
|
224
|
+
const schema = context.getSchema();
|
|
225
|
+
const needUpdateEntities = Object.keys(options.plan);
|
|
226
|
+
const depGraph = (0, dependencyBuilder_1.analyzeDepedency)(pwd).ascOrder;
|
|
227
|
+
const data = (0, requirePrj_1.default)(pwd, (0, path_1.join)('lib', 'data', 'index'), depGraph);
|
|
228
|
+
data.i18n.push(...i18n_1.default); // 合并domain的i18n数据
|
|
229
|
+
for (const entityName in data) {
|
|
230
|
+
if (!needUpdateEntities.includes(entityName)) {
|
|
231
|
+
continue;
|
|
232
|
+
}
|
|
233
|
+
(0, assert_1.default)(schema[entityName], `Entity ${entityName} not found in schema`);
|
|
234
|
+
if (entityBeforeCheckers[entityName]) {
|
|
235
|
+
const checkers = entityBeforeCheckers[entityName];
|
|
236
|
+
const entityDataArray = data[entityName];
|
|
237
|
+
for (const item of entityDataArray) {
|
|
238
|
+
for (const checker of checkers) {
|
|
239
|
+
await checker.check({
|
|
240
|
+
context: context,
|
|
241
|
+
item: item,
|
|
242
|
+
schema
|
|
243
|
+
});
|
|
244
|
+
}
|
|
245
|
+
}
|
|
246
|
+
}
|
|
247
|
+
}
|
|
248
|
+
// 所有的反指关系
|
|
249
|
+
const reverseRefs = {};
|
|
250
|
+
for (const entityName in schema) {
|
|
251
|
+
const storageDesc = schema[entityName];
|
|
252
|
+
for (const attrName in storageDesc.attributes) {
|
|
253
|
+
const columnDesc = storageDesc.attributes[attrName];
|
|
254
|
+
(0, assert_1.default)(columnDesc, `Attribute ${attrName} not found in entity ${entityName}`);
|
|
255
|
+
if (columnDesc.type === 'ref' && attrName.endsWith('Id')) {
|
|
256
|
+
(0, assert_1.default)(columnDesc.ref, `Attribute ${attrName} in entity ${entityName} is ref type but missing ref`);
|
|
257
|
+
if (!reverseRefs[columnDesc.ref]) {
|
|
258
|
+
reverseRefs[columnDesc.ref] = [];
|
|
259
|
+
}
|
|
260
|
+
reverseRefs[columnDesc.ref].push({
|
|
261
|
+
type: 'ref',
|
|
262
|
+
attrName,
|
|
263
|
+
sourceEntity: entityName,
|
|
264
|
+
});
|
|
265
|
+
}
|
|
266
|
+
else if (attrName === 'entity' && columnDesc.params?.length === 32) {
|
|
267
|
+
if (!columnDesc.ref) {
|
|
268
|
+
console.log(`Attribute entity in entity ${entityName} has no ref, skipping dynamic reverse ref analysis`);
|
|
269
|
+
continue;
|
|
270
|
+
}
|
|
271
|
+
const columnDesc2 = (storageDesc.attributes)['entityId'];
|
|
272
|
+
if (columnDesc2.params?.length === 64) {
|
|
273
|
+
// 这个情况说明是动态反指
|
|
274
|
+
(0, assert_1.default)(Array.isArray(columnDesc.ref), `Attribute entityId in entity ${entityName} should have array ref`);
|
|
275
|
+
for (const refEntity of columnDesc.ref) {
|
|
276
|
+
if (!reverseRefs[refEntity]) {
|
|
277
|
+
reverseRefs[refEntity] = [];
|
|
278
|
+
}
|
|
279
|
+
reverseRefs[refEntity].push({
|
|
280
|
+
type: 'refs',
|
|
281
|
+
attrName: 'entityId',
|
|
282
|
+
sourceEntity: entityName,
|
|
283
|
+
});
|
|
284
|
+
}
|
|
285
|
+
}
|
|
286
|
+
}
|
|
287
|
+
}
|
|
288
|
+
}
|
|
289
|
+
// 先执行前置检查
|
|
290
|
+
await beforeCheckFn({ context, data, reverseRefs });
|
|
291
|
+
const toDelete = {};
|
|
292
|
+
const needPhysicalDelete = {};
|
|
293
|
+
for (const entityName of needUpdateEntities) {
|
|
294
|
+
const entityPlan = entityPlans[entityName];
|
|
295
|
+
// 先确保配置是正确的
|
|
296
|
+
if (entityPlan.onUniqueViolation) {
|
|
297
|
+
(0, assert_1.default)(['skip', 'update', 'error'].includes(entityPlan.onUniqueViolation), `Invalid onUniqueViolation option for entity ${entityName}`);
|
|
298
|
+
}
|
|
299
|
+
if (entityPlan.onOnlyExistingInDb) {
|
|
300
|
+
(0, assert_1.default)(['skip', 'delete', 'physicalDelete'].includes(entityPlan.onOnlyExistingInDb), `Invalid onOnlyExistingInDb option for entity ${entityName}`);
|
|
301
|
+
}
|
|
302
|
+
(0, assert_1.default)(!entityPlan.allowDataUniqueReWrite || typeof entityPlan.allowDataUniqueReWrite === 'boolean', `Invalid allowDataUniqueReWrite option for entity ${entityName}`);
|
|
303
|
+
const storageDesc = schema[entityName];
|
|
304
|
+
(0, assert_1.default)(storageDesc, `Entity ${entityName} not found in schema`);
|
|
305
|
+
const entityDataArray = data[entityName];
|
|
306
|
+
(0, assert_1.default)(Array.isArray(entityDataArray), `Data for entity ${entityName} should be an array`);
|
|
307
|
+
const idToDataMap = new Map();
|
|
308
|
+
for (const item of entityDataArray) {
|
|
309
|
+
(0, assert_1.default)(item.id, `Item in entity ${entityName} is missing id`);
|
|
310
|
+
// ID不允许重复
|
|
311
|
+
if (idToDataMap.has(item.id)) {
|
|
312
|
+
throw new Error(`Duplicate id ${item.id} in entity ${entityName}`);
|
|
313
|
+
}
|
|
314
|
+
idToDataMap.set(item.id, item);
|
|
315
|
+
}
|
|
316
|
+
const fullProjection = {};
|
|
317
|
+
for (const attrName in storageDesc.attributes) {
|
|
318
|
+
Object.assign(fullProjection, { [attrName]: 1 });
|
|
319
|
+
// 检查指针关系
|
|
320
|
+
const columnDesc = storageDesc.attributes[attrName];
|
|
321
|
+
(0, assert_1.default)(columnDesc, `Attribute ${attrName} not found in entity ${entityName}`);
|
|
322
|
+
if (columnDesc.notNull) {
|
|
323
|
+
for (const item of entityDataArray) {
|
|
324
|
+
const value = item[attrName];
|
|
325
|
+
(0, assert_1.default)(value !== null && value !== undefined, `In entity ${entityName}, item id ${item.id} attribute ${attrName} is notNull but missing in data`);
|
|
326
|
+
}
|
|
327
|
+
}
|
|
328
|
+
if (columnDesc.type === 'ref' && attrName.endsWith('Id')) {
|
|
329
|
+
(0, assert_1.default)(columnDesc.ref, `Attribute ${attrName} in entity ${entityName} is ref type but missing ref`);
|
|
330
|
+
// 在现有数据中查找entity为ref,id为属性值的记录
|
|
331
|
+
const exEntityData = data[columnDesc.ref] || [];
|
|
332
|
+
for (const item of entityDataArray) {
|
|
333
|
+
const value = item[attrName];
|
|
334
|
+
if (!value && !columnDesc.notNull) {
|
|
335
|
+
continue;
|
|
336
|
+
}
|
|
337
|
+
const findRecord = exEntityData.find(rec => rec.id === item[attrName]);
|
|
338
|
+
if (findRecord) {
|
|
339
|
+
continue;
|
|
340
|
+
}
|
|
341
|
+
// 查询数据库
|
|
342
|
+
const count = await context.count(columnDesc.ref, {
|
|
343
|
+
filter: {
|
|
344
|
+
id: value
|
|
345
|
+
}
|
|
346
|
+
}, { blockTrigger: true, dontCollect: true });
|
|
347
|
+
(0, assert_1.default)(count === 1, `In entity ${entityName}, item id ${item.id} attribute ${attrName} references non-existing record id ${value} in entity ${columnDesc.ref}`);
|
|
348
|
+
}
|
|
349
|
+
}
|
|
350
|
+
else if (attrName === 'entity' && columnDesc.params?.length === 32) {
|
|
351
|
+
const allowNullEntity = columnDesc.notNull === false;
|
|
352
|
+
if (storageDesc.attributes['entityId']) {
|
|
353
|
+
const columnDesc2 = storageDesc.attributes['entityId'];
|
|
354
|
+
if (columnDesc2.params?.length === 64) {
|
|
355
|
+
// 这个情况说明是动态反指
|
|
356
|
+
for (const item of entityDataArray) {
|
|
357
|
+
if (allowNullEntity && !item['entity']) {
|
|
358
|
+
continue;
|
|
359
|
+
}
|
|
360
|
+
const entityName = item['entity'];
|
|
361
|
+
const entityId = item['entityId'];
|
|
362
|
+
if (!columnDesc2.notNull && (!entityName || !entityId)) {
|
|
363
|
+
continue;
|
|
364
|
+
}
|
|
365
|
+
(0, assert_1.default)(entityName && entityId, `In entity ${entityName}, item id ${item.id} dynamic attribute entity/entityId is missing`);
|
|
366
|
+
const exEntityData = data[entityName] || [];
|
|
367
|
+
const findRecord = exEntityData.find(rec => rec.id === entityId);
|
|
368
|
+
if (findRecord) {
|
|
369
|
+
continue;
|
|
370
|
+
}
|
|
371
|
+
// 查询数据库
|
|
372
|
+
const count = await context.count(entityName, {
|
|
373
|
+
filter: {
|
|
374
|
+
id: entityId
|
|
375
|
+
}
|
|
376
|
+
}, { blockTrigger: true, dontCollect: true });
|
|
377
|
+
(0, assert_1.default)(count === 1, `In entity ${entityName}, item id ${item.id} dynamic attribute entity/entityId references non-existing record id ${entityId} in entity ${entityName}`);
|
|
378
|
+
}
|
|
379
|
+
}
|
|
380
|
+
}
|
|
381
|
+
}
|
|
382
|
+
// 通用的检查
|
|
383
|
+
switch (columnDesc.type) {
|
|
384
|
+
case 'varchar': {
|
|
385
|
+
const length = columnDesc.params?.length || 0;
|
|
386
|
+
for (const item of entityDataArray) {
|
|
387
|
+
const value = item[attrName];
|
|
388
|
+
if (value && value.length > length) {
|
|
389
|
+
throw new Error(`In entity ${entityName}, item id ${item.id} attribute ${attrName} exceeds varchar length ${length}`);
|
|
390
|
+
}
|
|
391
|
+
}
|
|
392
|
+
break;
|
|
393
|
+
}
|
|
394
|
+
case 'object': {
|
|
395
|
+
// 这种情况说明是一个合法的json对象即可
|
|
396
|
+
for (const item of entityDataArray) {
|
|
397
|
+
const value = item[attrName];
|
|
398
|
+
if (value !== null && value !== undefined) {
|
|
399
|
+
(0, assert_1.default)(typeof value === 'object', `In entity ${entityName}, item id ${item.id} attribute ${attrName} should be an object`);
|
|
400
|
+
}
|
|
401
|
+
}
|
|
402
|
+
}
|
|
403
|
+
}
|
|
404
|
+
}
|
|
405
|
+
const existingRecords = await context.select(entityName, {
|
|
406
|
+
filter: {
|
|
407
|
+
id: { $in: Array.from(idToDataMap.keys()) }
|
|
408
|
+
},
|
|
409
|
+
data: {
|
|
410
|
+
id: 1,
|
|
411
|
+
...fullProjection,
|
|
412
|
+
...(entityPlan.onOnlyExistingInDb === 'physicalDelete' ? { $$deleteAt$$: 1 } : {})
|
|
413
|
+
}
|
|
414
|
+
}, {
|
|
415
|
+
blockTrigger: true,
|
|
416
|
+
dontCollect: true,
|
|
417
|
+
forUpdate: true,
|
|
418
|
+
...(entityPlan.onOnlyExistingInDb === 'physicalDelete' ? { includedDeleted: true } : {})
|
|
419
|
+
});
|
|
420
|
+
const toAdd = [];
|
|
421
|
+
const toUpdate = [];
|
|
422
|
+
for (const item of entityDataArray) {
|
|
423
|
+
const existing = existingRecords.find(rec => rec.id === item.id);
|
|
424
|
+
if (existing) {
|
|
425
|
+
if (entityPlan.onOnlyExistingInDb === 'physicalDelete' && existing['$$deleteAt$$']) {
|
|
426
|
+
// 视为新增,但是在新增之前就需要物理删除
|
|
427
|
+
if (!needPhysicalDelete[entityName]) {
|
|
428
|
+
needPhysicalDelete[entityName] = [];
|
|
429
|
+
}
|
|
430
|
+
needPhysicalDelete[entityName].push(item.id);
|
|
431
|
+
toAdd.push(item);
|
|
432
|
+
continue;
|
|
433
|
+
}
|
|
434
|
+
// 对比属性差异
|
|
435
|
+
const diffs = diffAttrs(existing, item);
|
|
436
|
+
if (Object.keys(diffs).length > 0) {
|
|
437
|
+
const updateData = Object.assign({ id: item.id }, diffs);
|
|
438
|
+
toUpdate.push(updateData);
|
|
439
|
+
}
|
|
440
|
+
}
|
|
441
|
+
else {
|
|
442
|
+
(0, assert_1.default)(item.id, `New item in entity ${entityName} missing id`);
|
|
443
|
+
toAdd.push(item);
|
|
444
|
+
}
|
|
445
|
+
}
|
|
446
|
+
const toUpdateRefs = {};
|
|
447
|
+
// 检查所有unique的属性在新增和更新中的唯一性
|
|
448
|
+
for (const i in storageDesc.indexes) {
|
|
449
|
+
const indexDesc = storageDesc.indexes[i];
|
|
450
|
+
const indexName = indexDesc.name;
|
|
451
|
+
(0, assert_1.default)(indexDesc, `Index ${indexName} not found in entity ${entityName}`);
|
|
452
|
+
(0, assert_1.default)(typeof indexDesc !== 'number', `Index ${indexName} in entity ${entityName} should not be a number`);
|
|
453
|
+
(0, assert_1.default)(typeof indexDesc !== 'function', `Index ${indexName} in entity ${entityName} should not be a function`);
|
|
454
|
+
if (!indexDesc?.config?.unique) {
|
|
455
|
+
continue;
|
|
456
|
+
}
|
|
457
|
+
const projection = {};
|
|
458
|
+
for (const attr of indexDesc.attributes) {
|
|
459
|
+
(0, assert_1.default)(attr.name, `Index attribute name missing in index ${indexName} of entity ${entityName}`);
|
|
460
|
+
Object.assign(projection, { [attr.name]: 1 });
|
|
461
|
+
}
|
|
462
|
+
// 这里对每一条数据进行检查unique索引
|
|
463
|
+
const toCheck = [...toAdd];
|
|
464
|
+
for (const item of toCheck) {
|
|
465
|
+
// 检查toCheck列表内部是否有重复
|
|
466
|
+
const duplicates = entityDataArray.filter(dataItem => {
|
|
467
|
+
let allMatch = true;
|
|
468
|
+
for (const attr of indexDesc.attributes) {
|
|
469
|
+
if (!compareInner(dataItem[attr.name], item[attr.name])) {
|
|
470
|
+
allMatch = false;
|
|
471
|
+
break;
|
|
472
|
+
}
|
|
473
|
+
}
|
|
474
|
+
return allMatch;
|
|
475
|
+
});
|
|
476
|
+
if (duplicates.length > 1) {
|
|
477
|
+
if (!entityPlan.allowDataUniqueReWrite) {
|
|
478
|
+
throw new Error(`Unique index violation on entity ${entityName} for index ${indexName} when adding id ${item.id}`);
|
|
479
|
+
}
|
|
480
|
+
else {
|
|
481
|
+
// 必须id一样,不然不允许覆盖
|
|
482
|
+
const allSameId = duplicates.every(dup => dup.id === item.id);
|
|
483
|
+
(0, assert_1.default)(allSameId, `当允许唯一性冲突覆盖时,冲突的数据ID必须相同,实体 ${entityName} 索引 ${indexName} 冲突数据ID列表: ${duplicates.map(d => d.id).join(', ')}`);
|
|
484
|
+
console.log(`发现实体 ${entityName} 唯一性索引${indexName}冲突,将保留数据 ${duplicates[duplicates.length - 1]}`);
|
|
485
|
+
for (let i = 0; i < duplicates.length - 2; i++) { // 保留最后一条,因为在requireSth时,是先加载依赖,再加载项目,所以后面的数据应该是项目自己的
|
|
486
|
+
const indexInAdd = toAdd.findIndex(dataItem => dataItem.id === duplicates[i].id);
|
|
487
|
+
if (indexInAdd >= 0) {
|
|
488
|
+
toAdd.splice(indexInAdd, 1);
|
|
489
|
+
}
|
|
490
|
+
}
|
|
491
|
+
}
|
|
492
|
+
}
|
|
493
|
+
// 检查数据库中是否有重复
|
|
494
|
+
const filter = {};
|
|
495
|
+
for (const attr of indexDesc.attributes) {
|
|
496
|
+
Object.assign(filter, { [attr.name]: item[attr.name] });
|
|
497
|
+
}
|
|
498
|
+
const count = await context.count(entityName, {
|
|
499
|
+
filter,
|
|
500
|
+
}, { blockTrigger: true, dontCollect: true });
|
|
501
|
+
if (count > 0) {
|
|
502
|
+
if (!entityPlan.onUniqueViolation || entityPlan.onUniqueViolation === 'error') {
|
|
503
|
+
throw new Error(`Unique index violation on entity ${entityName} for index ${indexName} when adding id ${item.id}, attr is ${JSON.stringify(filter)}`);
|
|
504
|
+
}
|
|
505
|
+
else if (entityPlan.onUniqueViolation === 'skip') {
|
|
506
|
+
// 从待添加列表中移除
|
|
507
|
+
const indexInAdd = toAdd.findIndex(dataItem => dataItem.id === item.id);
|
|
508
|
+
if (indexInAdd >= 0) {
|
|
509
|
+
toAdd.splice(indexInAdd, 1);
|
|
510
|
+
}
|
|
511
|
+
// 查出老数据ID,放到toUpdateRefs更新列表中
|
|
512
|
+
const sexistingRecords = await context.select(entityName, {
|
|
513
|
+
filter,
|
|
514
|
+
data: { id: 1 }
|
|
515
|
+
}, { blockTrigger: true, dontCollect: true });
|
|
516
|
+
(0, assert_1.default)(sexistingRecords.length === 1, `Existing record not found for unique violation skip on entity ${entityName}`);
|
|
517
|
+
// 是要在全部完成插入之后,把新id都更新成旧id,其实就说明新数据不会被新增,直接复用老数据
|
|
518
|
+
toUpdateRefs[sexistingRecords[0].id] = {
|
|
519
|
+
entityId: item.id,
|
|
520
|
+
};
|
|
521
|
+
}
|
|
522
|
+
else if (entityPlan.onUniqueViolation === 'update') {
|
|
523
|
+
// 查询出原始数据ID,放到toUpdateRefs更新列表中
|
|
524
|
+
const sexistingRecords = await context.select(entityName, {
|
|
525
|
+
filter,
|
|
526
|
+
data: { id: 1 }
|
|
527
|
+
}, { blockTrigger: true, dontCollect: true });
|
|
528
|
+
for (const existingRecord of sexistingRecords) {
|
|
529
|
+
(0, assert_1.default)(existingRecord, `Existing record not found for unique violation update on entity ${entityName}`);
|
|
530
|
+
(0, assert_1.default)(existingRecord.id !== item.id, `Existing record id should be different from new item id for unique violation update on entity ${entityName}`);
|
|
531
|
+
toUpdateRefs[item.id] = {
|
|
532
|
+
entityId: existingRecord.id,
|
|
533
|
+
};
|
|
534
|
+
}
|
|
535
|
+
}
|
|
536
|
+
}
|
|
537
|
+
}
|
|
538
|
+
}
|
|
539
|
+
if (needPhysicalDelete[entityName]) {
|
|
540
|
+
console.log(`将物理删除实体 ${entityName} ID列表:`, needPhysicalDelete[entityName]);
|
|
541
|
+
await context.operate(entityName, {
|
|
542
|
+
id: await (0, uuid_1.generateNewIdAsync)(),
|
|
543
|
+
action: "remove",
|
|
544
|
+
data: {},
|
|
545
|
+
filter: {
|
|
546
|
+
id: {
|
|
547
|
+
$in: needPhysicalDelete[entityName]
|
|
548
|
+
},
|
|
549
|
+
}
|
|
550
|
+
}, { dontCollect: true, blockTrigger: true, ...globalOptions, deletePhysically: true, includedDeleted: true });
|
|
551
|
+
}
|
|
552
|
+
// 检查完毕数据正确性,开始检查更新
|
|
553
|
+
if (toAdd.length > 0) {
|
|
554
|
+
console.log(`将新增实体 ${entityName} ID列表:`, toAdd.map(item => item.id));
|
|
555
|
+
for (const item of toAdd) {
|
|
556
|
+
await context.operate(entityName, {
|
|
557
|
+
id: await (0, uuid_1.generateNewIdAsync)(),
|
|
558
|
+
action: "create",
|
|
559
|
+
data: item,
|
|
560
|
+
}, { dontCollect: true, blockTrigger: true, ...globalOptions });
|
|
561
|
+
// 是否需要更新反指
|
|
562
|
+
if (toUpdateRefs[item.id]) {
|
|
563
|
+
console.log(`在实体 ${entityName} 中需要将旧的引用ID ${toUpdateRefs[item.id].entityId} 的所有反指针更新为新的ID ${item.id} `);
|
|
564
|
+
const reverseRefDescs = reverseRefs[entityName];
|
|
565
|
+
if (reverseRefDescs) {
|
|
566
|
+
for (const reverseRefDesc of reverseRefDescs) {
|
|
567
|
+
if (reverseRefDesc.type === 'ref') {
|
|
568
|
+
// 单指
|
|
569
|
+
const updateData = {
|
|
570
|
+
[reverseRefDesc.attrName]: item.id,
|
|
571
|
+
};
|
|
572
|
+
console.log(`Updating reverse ref in entity ${String(reverseRefDesc.sourceEntity)} attr ${reverseRefDesc.attrName} to new id ${item.id} for original id ${toUpdateRefs[item.id].entityId}`);
|
|
573
|
+
await context.operate(reverseRefDesc.sourceEntity, {
|
|
574
|
+
id: await (0, uuid_1.generateNewIdAsync)(),
|
|
575
|
+
action: "update",
|
|
576
|
+
data: updateData,
|
|
577
|
+
filter: {
|
|
578
|
+
[reverseRefDesc.attrName]: toUpdateRefs[item.id].entityId,
|
|
579
|
+
}
|
|
580
|
+
}, { dontCollect: true, blockTrigger: true, ...globalOptions });
|
|
581
|
+
}
|
|
582
|
+
else {
|
|
583
|
+
(0, assert_1.default)(reverseRefDesc.type === 'refs', `Unknown reverse ref type ${reverseRefDesc.type} in entity ${entityName}`);
|
|
584
|
+
// 动态反指
|
|
585
|
+
const existingRecords = await context.select(reverseRefDesc.sourceEntity, {
|
|
586
|
+
filter: {
|
|
587
|
+
entity: entityName,
|
|
588
|
+
entityId: toUpdateRefs[item.id].entityId,
|
|
589
|
+
},
|
|
590
|
+
data: {
|
|
591
|
+
id: 1,
|
|
592
|
+
entityId: 1,
|
|
593
|
+
}
|
|
594
|
+
}, { blockTrigger: true, dontCollect: true });
|
|
595
|
+
const updateData = {
|
|
596
|
+
entityId: item.id,
|
|
597
|
+
};
|
|
598
|
+
console.log(`Updating reverse refs in entity ${String(reverseRefDesc.sourceEntity)} attr ${reverseRefDesc.attrName} to new id ${item.id} for original id ${toUpdateRefs[item.id].entityId}`);
|
|
599
|
+
await context.operate(reverseRefDesc.sourceEntity, {
|
|
600
|
+
id: await (0, uuid_1.generateNewIdAsync)(),
|
|
601
|
+
action: "update",
|
|
602
|
+
data: updateData,
|
|
603
|
+
filter: {
|
|
604
|
+
id: {
|
|
605
|
+
$in: existingRecords.map(rec => rec.id)
|
|
606
|
+
},
|
|
607
|
+
}
|
|
608
|
+
}, { dontCollect: true, blockTrigger: true, ...globalOptions });
|
|
609
|
+
}
|
|
610
|
+
}
|
|
611
|
+
// 删除已存在的数据
|
|
612
|
+
console.log(`Deleting original record id ${toUpdateRefs[item.id].entityId} in entity ${entityName} after reverse ref update`);
|
|
613
|
+
await context.operate(entityName, {
|
|
614
|
+
id: await (0, uuid_1.generateNewIdAsync)(),
|
|
615
|
+
action: "remove",
|
|
616
|
+
data: {},
|
|
617
|
+
filter: {
|
|
618
|
+
id: toUpdateRefs[item.id].entityId,
|
|
619
|
+
}
|
|
620
|
+
}, { dontCollect: true, blockTrigger: true, ...globalOptions });
|
|
621
|
+
}
|
|
622
|
+
}
|
|
623
|
+
}
|
|
624
|
+
}
|
|
625
|
+
else {
|
|
626
|
+
console.log(`实体 ${entityName} 无需新增数据`);
|
|
627
|
+
}
|
|
628
|
+
if (toUpdate.length > 0) {
|
|
629
|
+
console.log(`将更新实体 ${entityName} ID列表:`, toUpdate.map(item => item.id));
|
|
630
|
+
for (const item of toUpdate) {
|
|
631
|
+
(0, assert_1.default)(item.id, `Update item in entity ${entityName} missing id`);
|
|
632
|
+
await context.operate(entityName, {
|
|
633
|
+
id: await (0, uuid_1.generateNewIdAsync)(),
|
|
634
|
+
action: "update",
|
|
635
|
+
data: item,
|
|
636
|
+
filter: {
|
|
637
|
+
id: item.id,
|
|
638
|
+
}
|
|
639
|
+
}, { dontCollect: true, blockTrigger: true, ...globalOptions });
|
|
640
|
+
}
|
|
641
|
+
}
|
|
642
|
+
else {
|
|
643
|
+
console.log(`实体 ${entityName} 无需更新数据`);
|
|
644
|
+
}
|
|
645
|
+
// 需要删除的数据
|
|
646
|
+
if (entityPlan.onOnlyExistingInDb === 'delete' || entityPlan.onOnlyExistingInDb === 'physicalDelete') {
|
|
647
|
+
const nonExistingRecords = await context.select(entityName, {
|
|
648
|
+
filter: {
|
|
649
|
+
id: { $nin: Array.from(idToDataMap.keys()) }
|
|
650
|
+
},
|
|
651
|
+
data: { id: 1 }
|
|
652
|
+
}, { blockTrigger: true, dontCollect: true, ...entityPlan.onOnlyExistingInDb === 'physicalDelete' ? { includedDeleted: true } : {} });
|
|
653
|
+
if (nonExistingRecords.length > 0) {
|
|
654
|
+
toDelete[entityName] = nonExistingRecords.map(rec => rec.id);
|
|
655
|
+
}
|
|
656
|
+
}
|
|
657
|
+
else {
|
|
658
|
+
console.log(`实体 ${entityName} 无需删除数据`);
|
|
659
|
+
}
|
|
660
|
+
}
|
|
661
|
+
// 执行删除操作,根据 reverseRefs 构建删除顺序
|
|
662
|
+
// 构建每个实体被哪些实体依赖的映射(被引用方 -> 引用方列表)
|
|
663
|
+
const entityDeps = new Map();
|
|
664
|
+
const entitiesToDelete = new Set(Object.keys(toDelete));
|
|
665
|
+
// 初始化所有待删除实体
|
|
666
|
+
entitiesToDelete.forEach(entityName => {
|
|
667
|
+
if (!entityDeps.has(entityName)) {
|
|
668
|
+
entityDeps.set(entityName, new Set());
|
|
669
|
+
}
|
|
670
|
+
});
|
|
671
|
+
// 根据 reverseRefs 构建依赖关系
|
|
672
|
+
for (const [targetEntity, refList] of Object.entries(reverseRefs)) {
|
|
673
|
+
if (!entitiesToDelete.has(targetEntity)) {
|
|
674
|
+
continue;
|
|
675
|
+
}
|
|
676
|
+
for (const refDesc of refList || []) {
|
|
677
|
+
if (entitiesToDelete.has(refDesc.sourceEntity)) {
|
|
678
|
+
// sourceEntity 引用了 targetEntity
|
|
679
|
+
// 删除时应该先删除 sourceEntity
|
|
680
|
+
if (!entityDeps.has(targetEntity)) {
|
|
681
|
+
entityDeps.set(targetEntity, new Set());
|
|
682
|
+
}
|
|
683
|
+
entityDeps.get(targetEntity).add(refDesc.sourceEntity);
|
|
684
|
+
}
|
|
685
|
+
}
|
|
686
|
+
}
|
|
687
|
+
// 拓扑排序:优先删除没有依赖的实体(引用方)
|
|
688
|
+
const deleteOrder = [];
|
|
689
|
+
const visited = new Set();
|
|
690
|
+
const visit = (entityName) => {
|
|
691
|
+
if (visited.has(entityName)) {
|
|
692
|
+
return;
|
|
693
|
+
}
|
|
694
|
+
visited.add(entityName);
|
|
695
|
+
const deps = entityDeps.get(entityName);
|
|
696
|
+
if (deps) {
|
|
697
|
+
deps.forEach(dep => {
|
|
698
|
+
visit(dep);
|
|
699
|
+
});
|
|
700
|
+
}
|
|
701
|
+
deleteOrder.push(entityName);
|
|
702
|
+
};
|
|
703
|
+
entitiesToDelete.forEach(entityName => {
|
|
704
|
+
visit(entityName);
|
|
705
|
+
});
|
|
706
|
+
// 按照拓扑排序的顺序执行删除
|
|
707
|
+
for (const entityName of deleteOrder) {
|
|
708
|
+
const idsToDelete = toDelete[entityName];
|
|
709
|
+
if (!idsToDelete || idsToDelete.length === 0) {
|
|
710
|
+
continue;
|
|
711
|
+
}
|
|
712
|
+
console.log(`删除实体 ${String(entityName)},ID 列表:`, idsToDelete);
|
|
713
|
+
await context.operate(entityName, {
|
|
714
|
+
id: await (0, uuid_1.generateNewIdAsync)(),
|
|
715
|
+
action: "remove",
|
|
716
|
+
data: {},
|
|
717
|
+
filter: {
|
|
718
|
+
id: { $in: idsToDelete }
|
|
719
|
+
}
|
|
720
|
+
}, { dontCollect: true, blockTrigger: true, ...globalOptions, ...entityPlans[entityName].onOnlyExistingInDb === 'physicalDelete' ? { deletePhysically: true, includedDeleted: true } : {} });
|
|
721
|
+
}
|
|
722
|
+
// 执行后置处理
|
|
723
|
+
await afterUpdateFn({ context, data, reverseRefs });
|
|
724
|
+
console.log('数据更新完成', JSON.stringify(context.opResult, null, 2));
|
|
725
|
+
};
|
|
726
|
+
return update;
|
|
727
|
+
};
|
|
728
|
+
exports.createUpdatePlan = createUpdatePlan;
|
|
729
|
+
/**
|
|
730
|
+
* 这里做一些特殊的检查,比如path的合法性检查等,通用的检查放在update函数里
|
|
731
|
+
*/
|
|
732
|
+
const entityBeforeCheckers = {
|
|
733
|
+
'path': [
|
|
734
|
+
{
|
|
735
|
+
check: async ({ context, item, schema }) => {
|
|
736
|
+
(0, assert_1.default)(item.destEntity, `Path item id ${item.id} missing destEntity`);
|
|
737
|
+
(0, assert_1.default)(item.value !== undefined && item.value !== null, `Path item id ${item.id} missing value`);
|
|
738
|
+
(0, assert_1.default)(schema[item.sourceEntity], `Path item id ${item.id} has invalid sourceEntity ${item.sourceEntity}`);
|
|
739
|
+
checkPathValue(item.id, item.destEntity, item.value, schema);
|
|
740
|
+
}
|
|
741
|
+
}
|
|
742
|
+
]
|
|
743
|
+
};
|
|
744
|
+
function checkPathValue(itemId, destEntity, path, storageSchema) {
|
|
745
|
+
const checkInner = (currentEntity, subPath, schema) => {
|
|
746
|
+
if (!subPath) {
|
|
747
|
+
return true;
|
|
748
|
+
}
|
|
749
|
+
// 第一层path
|
|
750
|
+
const pathParts = subPath.split('.');
|
|
751
|
+
const firstPart = pathParts[0];
|
|
752
|
+
(0, assert_1.default)(firstPart, `[Path校验失败] 数据 ${itemId} 路径格式错误,路径 "${subPath}" 不能为空或无效`);
|
|
753
|
+
if (firstPart.includes("$")) {
|
|
754
|
+
const [entity, attr] = firstPart.split('$');
|
|
755
|
+
const entitySchema = schema[entity];
|
|
756
|
+
(0, assert_1.default)(entitySchema, `[Path校验失败] 数据 ${itemId} 引用路径错误:实体 "${entity}" 在 schema 中不存在,请检查路径 "${path}" 中的实体名称是否正确`);
|
|
757
|
+
const attrDesc = entitySchema.attributes[`${attr}Id`];
|
|
758
|
+
(0, assert_1.default)(attrDesc, `[Path校验失败] 数据 ${itemId} 引用路径错误:实体 "${entity}" 中不存在属性 "${attr}",请检查 schema 定义`);
|
|
759
|
+
(0, assert_1.default)(attrDesc.type === 'ref', `[Path校验失败] 数据 ${itemId} 引用路径错误:实体 "${entity}" 的属性 "${attr}" 不是引用类型 (ref),实际类型为 ${attrDesc.type},请检查 schema 定义`);
|
|
760
|
+
return checkInner(entity, pathParts.slice(1).join('.'), schema);
|
|
761
|
+
}
|
|
762
|
+
else {
|
|
763
|
+
const entitySchema = schema[currentEntity];
|
|
764
|
+
(0, assert_1.default)(entitySchema, `[Path校验失败] 数据 ${itemId} 引用路径错误:实体 "${currentEntity}" 在 schema 中不存在`);
|
|
765
|
+
const attrDesc = entitySchema.attributes[`${firstPart}Id`];
|
|
766
|
+
if (!attrDesc) {
|
|
767
|
+
// 尝试查找entity和entityId的动态反指
|
|
768
|
+
const entityDesc = entitySchema.attributes['entity'];
|
|
769
|
+
const entityIdDesc = entitySchema.attributes['entityId'];
|
|
770
|
+
if (entityDesc && entityIdDesc && entityDesc.params?.length === 32 && entityIdDesc.params?.length === 64) {
|
|
771
|
+
(0, assert_1.default)(entityDesc.ref?.includes(firstPart), `[Path校验失败] 数据 ${itemId} 引用路径错误:实体中的entity多态反指不包含 "${firstPart}",无法建立到 "${firstPart}" 的引用关系`);
|
|
772
|
+
// 这是动态反指
|
|
773
|
+
return checkInner(firstPart, pathParts.slice(1).join('.'), schema);
|
|
774
|
+
}
|
|
775
|
+
}
|
|
776
|
+
(0, assert_1.default)(attrDesc, `[Path校验失败] 数据 ${itemId} 引用路径错误:实体中不存在属性 "${firstPart}Id",无法建立到 "${firstPart}" 的引用关系`);
|
|
777
|
+
(0, assert_1.default)(attrDesc.type === 'ref', `[Path校验失败] 数据 ${itemId} 引用路径错误:属性 "${firstPart}Id" 不是引用类型(ref),实际类型为 ${attrDesc.type}`);
|
|
778
|
+
const distEntity = attrDesc.ref;
|
|
779
|
+
(0, assert_1.default)(distEntity, `[Path校验失败] 数据 ${itemId} 引用路径错误:属性 "${firstPart}Id" 的 ref 配置为空,无法确定目标实体`);
|
|
780
|
+
return checkInner(distEntity, pathParts.slice(1).join('.'), schema);
|
|
781
|
+
}
|
|
782
|
+
};
|
|
783
|
+
return checkInner(destEntity, path, storageSchema);
|
|
784
|
+
}
|
package/lib/utils/dbPriority.js
CHANGED
|
@@ -38,7 +38,7 @@ const getDbConfig = (path) => {
|
|
|
38
38
|
// do nothing
|
|
39
39
|
}
|
|
40
40
|
}
|
|
41
|
-
throw new Error(`没有找到数据库配置文件,请在configuration目录下添加任一配置文件:${Object.keys(exports.dbList).map(ele => `${ele}.json`).join('、')}`);
|
|
41
|
+
throw new Error(`没有找到数据库配置文件,请在configuration目录下添加任一配置文件:${Object.keys(exports.dbList).map(ele => `${ele}.json`).join('、')}, 当前环境: ${process.env.NODE_ENV}`);
|
|
42
42
|
};
|
|
43
43
|
exports.getDbConfig = getDbConfig;
|
|
44
44
|
const getDbStoreClass = (config) => {
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "oak-backend-base",
|
|
3
|
-
"version": "4.1.
|
|
3
|
+
"version": "4.1.29",
|
|
4
4
|
"description": "oak-backend-base",
|
|
5
5
|
"main": "lib/index",
|
|
6
6
|
"author": {
|
|
@@ -13,7 +13,10 @@
|
|
|
13
13
|
"copy-files": "copyfiles -u 1 src/**/*.json lib/",
|
|
14
14
|
"test": "ts-node test/test.ts",
|
|
15
15
|
"test2": "ts-node test/testDbStore.ts",
|
|
16
|
-
"
|
|
16
|
+
"test:sync": "vitest run",
|
|
17
|
+
"test:sync:ui": "vitest --ui",
|
|
18
|
+
"build": "node ./scripts/build.js && npm run copy-files",
|
|
19
|
+
"make:test:domain": "ts-node scripts/makeTestDomain.ts"
|
|
17
20
|
},
|
|
18
21
|
"dependencies": {
|
|
19
22
|
"@types/koa": "^2.15.0",
|
|
@@ -21,10 +24,10 @@
|
|
|
21
24
|
"mysql": "^2.18.1",
|
|
22
25
|
"mysql2": "^2.3.3",
|
|
23
26
|
"node-schedule": "^2.1.0",
|
|
24
|
-
"oak-common-aspect": "^3.0.
|
|
25
|
-
"oak-db": "^3.3.
|
|
26
|
-
"oak-domain": "^5.1.
|
|
27
|
-
"oak-frontend-base": "^5.3.
|
|
27
|
+
"oak-common-aspect": "^3.0.6",
|
|
28
|
+
"oak-db": "^3.3.14",
|
|
29
|
+
"oak-domain": "^5.1.36",
|
|
30
|
+
"oak-frontend-base": "^5.3.46",
|
|
28
31
|
"socket.io": "^4.8.1",
|
|
29
32
|
"socket.io-client": "^4.7.2",
|
|
30
33
|
"uuid": "^8.3.2"
|
|
@@ -35,9 +38,10 @@
|
|
|
35
38
|
"@types/node": "^20.6.0",
|
|
36
39
|
"@types/node-schedule": "^2.1.0",
|
|
37
40
|
"@types/uuid": "^8.3.4",
|
|
41
|
+
"@vitest/ui": "^4.0.18",
|
|
38
42
|
"copyfiles": "^2.4.1",
|
|
39
|
-
"ts-node": "^10.9.1",
|
|
40
43
|
"tslib": "^2.4.0",
|
|
41
|
-
"typescript": "^5.2.2"
|
|
44
|
+
"typescript": "^5.2.2",
|
|
45
|
+
"vitest": "^4.0.18"
|
|
42
46
|
}
|
|
43
47
|
}
|