oak-backend-base 4.1.27 → 4.1.29
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/AppLoader.d.ts +33 -2
- package/lib/AppLoader.js +160 -43
- package/lib/DbStore.d.ts +4 -1
- package/lib/DbStore.js +3 -1
- package/lib/Synchronizer.d.ts +10 -1
- package/lib/Synchronizer.js +51 -8
- package/lib/routines/update.d.ts +65 -0
- package/lib/routines/update.js +784 -0
- package/lib/utils/dbPriority.d.ts +8 -3
- package/lib/utils/dbPriority.js +16 -9
- package/package.json +12 -8
package/lib/AppLoader.d.ts
CHANGED
|
@@ -19,6 +19,7 @@ export declare class AppLoader<ED extends EntityDict & BaseEntityDict, Cxt exten
|
|
|
19
19
|
private watcherTimerId?;
|
|
20
20
|
private scheduledJobs;
|
|
21
21
|
private internalErrorHandlers;
|
|
22
|
+
private watcherExecutingData;
|
|
22
23
|
regAllExceptionHandler(): void;
|
|
23
24
|
/**
|
|
24
25
|
* 注册一个内部错误处理器
|
|
@@ -32,9 +33,15 @@ export declare class AppLoader<ED extends EntityDict & BaseEntityDict, Cxt exten
|
|
|
32
33
|
private requireSth;
|
|
33
34
|
protected makeContext(cxtStr?: string, headers?: IncomingHttpHeaders): Promise<Cxt>;
|
|
34
35
|
/**
|
|
35
|
-
*
|
|
36
|
+
* 获取数据库配置
|
|
37
|
+
* @returns 读取数据库配置
|
|
36
38
|
*/
|
|
37
|
-
private
|
|
39
|
+
private getDbConfig;
|
|
40
|
+
/**
|
|
41
|
+
* 获取同步配置
|
|
42
|
+
* @returns 读取同步配置
|
|
43
|
+
*/
|
|
44
|
+
private getSyncConfig;
|
|
38
45
|
constructor(path: string, nsSubscribe?: Namespace, nsSocket?: Namespace, nsServer?: Namespace);
|
|
39
46
|
protected registerTrigger(trigger: Trigger<ED, keyof ED, Cxt>): void;
|
|
40
47
|
protected initTriggers(): void;
|
|
@@ -55,6 +62,30 @@ export declare class AppLoader<ED extends EntityDict & BaseEntityDict, Cxt exten
|
|
|
55
62
|
}>][];
|
|
56
63
|
protected operateInWatcher<T extends keyof ED>(entity: T, operation: ED[T]['Update'], context: Cxt, singleton?: true): Promise<OperationResult<ED>>;
|
|
57
64
|
protected selectInWatcher<T extends keyof ED>(entity: T, selection: ED[T]['Selection'], context: Cxt, forUpdate?: true, singleton?: true): Promise<Partial<ED[T]["Schema"]>[]>;
|
|
65
|
+
/**
|
|
66
|
+
* 检查某个数据是否正在被watcher执行
|
|
67
|
+
* @param name watcher名称
|
|
68
|
+
* @param dataId 数据ID
|
|
69
|
+
* @returns 如果没有正在执行则返回true,否则返回false
|
|
70
|
+
*/
|
|
71
|
+
private checkDataExecuting;
|
|
72
|
+
/**
|
|
73
|
+
* 过滤出未在执行中的数据行,并标记为执行中
|
|
74
|
+
* @returns [过滤后的行, 是否有行被跳过]
|
|
75
|
+
*/
|
|
76
|
+
private filterAndMarkExecutingRows;
|
|
77
|
+
/**
|
|
78
|
+
* 清理执行标记
|
|
79
|
+
*/
|
|
80
|
+
private cleanupExecutingMarks;
|
|
81
|
+
/**
|
|
82
|
+
* 解析 filter 和 projection(支持函数或静态值)
|
|
83
|
+
*/
|
|
84
|
+
private resolveFilterAndProjection;
|
|
85
|
+
/**
|
|
86
|
+
* 执行 WB 类型 watcher 的查询操作
|
|
87
|
+
*/
|
|
88
|
+
private selectForWBWatcher;
|
|
58
89
|
protected execWatcher(watcher: Watcher<ED, keyof ED, Cxt>): Promise<OperationResult<ED> | undefined>;
|
|
59
90
|
protected getCheckpointTs(): number;
|
|
60
91
|
protected checkpoint(): Promise<number>;
|
package/lib/AppLoader.js
CHANGED
|
@@ -30,6 +30,7 @@ class AppLoader extends types_1.AppLoader {
|
|
|
30
30
|
watcherTimerId;
|
|
31
31
|
scheduledJobs = {};
|
|
32
32
|
internalErrorHandlers = new Array();
|
|
33
|
+
watcherExecutingData = new Map();
|
|
33
34
|
regAllExceptionHandler() {
|
|
34
35
|
const handlers = this.requireSth('lib/configuration/exception');
|
|
35
36
|
if (Array.isArray(handlers)) {
|
|
@@ -92,20 +93,24 @@ class AppLoader extends types_1.AppLoader {
|
|
|
92
93
|
return context;
|
|
93
94
|
}
|
|
94
95
|
/**
|
|
95
|
-
*
|
|
96
|
+
* 获取数据库配置
|
|
97
|
+
* @returns 读取数据库配置
|
|
96
98
|
*/
|
|
97
|
-
|
|
98
|
-
|
|
99
|
+
getDbConfig() {
|
|
100
|
+
return (0, dbPriority_1.getDbConfig)(this.path);
|
|
101
|
+
}
|
|
102
|
+
/**
|
|
103
|
+
* 获取同步配置
|
|
104
|
+
* @returns 读取同步配置
|
|
105
|
+
*/
|
|
106
|
+
getSyncConfig() {
|
|
99
107
|
const syncConfigFile = (0, path_1.join)(this.path, 'lib', 'configuration', 'sync.js');
|
|
100
108
|
const syncConfigs = (0, fs_1.existsSync)(syncConfigFile) && require(syncConfigFile).default;
|
|
101
|
-
return
|
|
102
|
-
dbConfig: dbConfig,
|
|
103
|
-
syncConfig: syncConfigs,
|
|
104
|
-
};
|
|
109
|
+
return syncConfigs;
|
|
105
110
|
}
|
|
106
111
|
constructor(path, nsSubscribe, nsSocket, nsServer) {
|
|
107
112
|
super(path);
|
|
108
|
-
const
|
|
113
|
+
const dbConfig = this.getDbConfig();
|
|
109
114
|
const { storageSchema } = require(`${path}/lib/oak-app-domain/Storage`);
|
|
110
115
|
const depGraph = (0, dependencyBuilder_1.analyzeDepedency)(process.cwd());
|
|
111
116
|
this.externalDependencies = depGraph.ascOrder;
|
|
@@ -173,7 +178,7 @@ class AppLoader extends types_1.AppLoader {
|
|
|
173
178
|
async mount(initialize) {
|
|
174
179
|
const { path } = this;
|
|
175
180
|
if (!initialize) {
|
|
176
|
-
const
|
|
181
|
+
const syncConfig = this.getSyncConfig();
|
|
177
182
|
if (syncConfig) {
|
|
178
183
|
this.synchronizer = new Synchronizer_1.default(syncConfig, this.dbStore.getSchema(), () => this.contextBuilder(this.dbStore));
|
|
179
184
|
}
|
|
@@ -182,7 +187,7 @@ class AppLoader extends types_1.AppLoader {
|
|
|
182
187
|
}
|
|
183
188
|
const { importations, exportations } = require(`${path}/lib/ports/index`);
|
|
184
189
|
(0, index_1.registerPorts)(importations || [], exportations || []);
|
|
185
|
-
this.dbStore.connect();
|
|
190
|
+
return this.dbStore.connect();
|
|
186
191
|
}
|
|
187
192
|
async unmount() {
|
|
188
193
|
if (this.watcherTimerId) {
|
|
@@ -368,60 +373,172 @@ class AppLoader extends types_1.AppLoader {
|
|
|
368
373
|
forUpdate,
|
|
369
374
|
});
|
|
370
375
|
}
|
|
376
|
+
/**
|
|
377
|
+
* 检查某个数据是否正在被watcher执行
|
|
378
|
+
* @param name watcher名称
|
|
379
|
+
* @param dataId 数据ID
|
|
380
|
+
* @returns 如果没有正在执行则返回true,否则返回false
|
|
381
|
+
*/
|
|
382
|
+
checkDataExecuting(name, dataId) {
|
|
383
|
+
let dataSet = this.watcherExecutingData.get(name);
|
|
384
|
+
if (!dataSet) {
|
|
385
|
+
dataSet = new Map();
|
|
386
|
+
this.watcherExecutingData.set(name, dataSet);
|
|
387
|
+
}
|
|
388
|
+
const existingTs = dataSet.get(dataId);
|
|
389
|
+
if (existingTs) {
|
|
390
|
+
const now = Date.now();
|
|
391
|
+
if (now - existingTs > 10 * 60 * 1000) {
|
|
392
|
+
console.error(`检测到执行器【${name}】的数据【${dataId}】标记超时(${Math.floor((now - existingTs) / 1000)}秒),请立即检查是否存在死循环或长时间阻塞的操作`);
|
|
393
|
+
}
|
|
394
|
+
return false;
|
|
395
|
+
}
|
|
396
|
+
dataSet.set(dataId, Date.now());
|
|
397
|
+
return true;
|
|
398
|
+
}
|
|
399
|
+
/**
|
|
400
|
+
* 过滤出未在执行中的数据行,并标记为执行中
|
|
401
|
+
* @returns [过滤后的行, 是否有行被跳过]
|
|
402
|
+
*/
|
|
403
|
+
filterAndMarkExecutingRows(watcher, rows) {
|
|
404
|
+
if (watcher.exclusive !== true) {
|
|
405
|
+
// 不需要排他执行,直接返回所有行
|
|
406
|
+
return [rows, false];
|
|
407
|
+
}
|
|
408
|
+
const rowsWithoutExecuting = [];
|
|
409
|
+
let hasSkiped = false;
|
|
410
|
+
const watcherName = watcher.name;
|
|
411
|
+
for (const row of rows) {
|
|
412
|
+
if (!row.id) {
|
|
413
|
+
console.error(`实例【${process.env.OAK_INSTANCE_ID || '单机'}】执行器【${watcherName}】获取的数据没有ID,跳过此数据的并发检查处理:`, row);
|
|
414
|
+
rowsWithoutExecuting.push(row);
|
|
415
|
+
continue;
|
|
416
|
+
}
|
|
417
|
+
if (this.checkDataExecuting(watcherName, row.id)) {
|
|
418
|
+
rowsWithoutExecuting.push(row);
|
|
419
|
+
}
|
|
420
|
+
else {
|
|
421
|
+
hasSkiped = true;
|
|
422
|
+
console.warn(`实例【${process.env.OAK_INSTANCE_ID || '单机'}】执行器【${watcherName}】将跳过正在被执行的数据ID:【${row.id}】,请检查是否执行超时`);
|
|
423
|
+
}
|
|
424
|
+
}
|
|
425
|
+
return [rowsWithoutExecuting, hasSkiped];
|
|
426
|
+
}
|
|
427
|
+
/**
|
|
428
|
+
* 清理执行标记
|
|
429
|
+
*/
|
|
430
|
+
cleanupExecutingMarks(watcherName, rows) {
|
|
431
|
+
for (const row of rows) {
|
|
432
|
+
if (row.id) {
|
|
433
|
+
const dataSet = this.watcherExecutingData.get(watcherName);
|
|
434
|
+
if (dataSet) {
|
|
435
|
+
dataSet.delete(row.id);
|
|
436
|
+
}
|
|
437
|
+
}
|
|
438
|
+
}
|
|
439
|
+
}
|
|
440
|
+
/**
|
|
441
|
+
* 解析 filter 和 projection(支持函数或静态值)
|
|
442
|
+
*/
|
|
443
|
+
async resolveFilterAndProjection(filter, projection) {
|
|
444
|
+
const filter2 = typeof filter === 'function' ? await filter() : (0, lodash_1.cloneDeep)(filter);
|
|
445
|
+
const projection2 = typeof projection === 'function' ? await projection() : (0, lodash_1.cloneDeep)(projection);
|
|
446
|
+
return [filter2, projection2];
|
|
447
|
+
}
|
|
448
|
+
/**
|
|
449
|
+
* 执行 WB 类型 watcher 的查询操作
|
|
450
|
+
*/
|
|
451
|
+
async selectForWBWatcher(watcher, context) {
|
|
452
|
+
const { entity, projection, filter, singleton, forUpdate } = watcher;
|
|
453
|
+
const [filter2, projection2] = await this.resolveFilterAndProjection(filter, projection);
|
|
454
|
+
return await this.selectInWatcher(entity, {
|
|
455
|
+
data: projection2,
|
|
456
|
+
filter: filter2,
|
|
457
|
+
}, context, forUpdate, singleton);
|
|
458
|
+
}
|
|
371
459
|
async execWatcher(watcher) {
|
|
372
460
|
let result;
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
const rows = await this.selectInWatcher(entity, {
|
|
379
|
-
data: projection2,
|
|
380
|
-
filter: filter2,
|
|
381
|
-
}, selectContext, forUpdate, singleton);
|
|
382
|
-
if (rows.length > 0) {
|
|
383
|
-
result = await fn(() => this.makeContext(), rows);
|
|
461
|
+
// BBWatcher:直接操作,无需查询
|
|
462
|
+
if (watcher.hasOwnProperty('actionData')) {
|
|
463
|
+
// 如果配置了 exclusive,BBWatcher 不支持
|
|
464
|
+
if (watcher.exclusive === true) {
|
|
465
|
+
console.warn(`BBWatcher【${watcher.name}】配置了 exclusive=true,但 BBWatcher 不支持排他执行,将忽略此配置,请使用 WBWatcher 或 WBFreeWatcher 来实现排他执行`);
|
|
384
466
|
}
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
const context = await this.makeContext();
|
|
388
|
-
try {
|
|
389
|
-
if (watcher.hasOwnProperty('actionData')) {
|
|
467
|
+
const context = await this.makeContext();
|
|
468
|
+
try {
|
|
390
469
|
const { entity, action, filter, actionData, singleton } = watcher;
|
|
391
470
|
const filter2 = typeof filter === 'function' ? await filter() : (0, lodash_1.cloneDeep)(filter);
|
|
392
|
-
const data = typeof actionData === 'function' ? await
|
|
471
|
+
const data = typeof actionData === 'function' ? await actionData() : (0, lodash_1.cloneDeep)(actionData);
|
|
393
472
|
result = await this.operateInWatcher(entity, {
|
|
394
473
|
id: await (0, uuid_1.generateNewIdAsync)(),
|
|
395
474
|
action: action,
|
|
396
475
|
data,
|
|
397
476
|
filter: filter2,
|
|
398
477
|
}, context, singleton);
|
|
478
|
+
await context.commit();
|
|
479
|
+
return result;
|
|
399
480
|
}
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
filter: filter2,
|
|
407
|
-
}, context, forUpdate, singleton);
|
|
408
|
-
if (rows.length > 0) {
|
|
409
|
-
result = await fn(context, rows);
|
|
481
|
+
catch (err) {
|
|
482
|
+
if (err instanceof types_1.OakPartialSuccess) {
|
|
483
|
+
await context.commit();
|
|
484
|
+
}
|
|
485
|
+
else {
|
|
486
|
+
await context.rollback();
|
|
410
487
|
}
|
|
488
|
+
throw err;
|
|
489
|
+
}
|
|
490
|
+
}
|
|
491
|
+
// WBFreeWatcher 和 WBWatcher:查询后执行
|
|
492
|
+
const isFreeType = watcher.hasOwnProperty('type') &&
|
|
493
|
+
watcher.type === 'free';
|
|
494
|
+
// 1. 执行查询(WBFreeWatcher 使用独立 context)
|
|
495
|
+
const selectContext = isFreeType ? await this.makeContext() : await this.makeContext();
|
|
496
|
+
const rows = await this.selectForWBWatcher(watcher, selectContext);
|
|
497
|
+
if (isFreeType) {
|
|
498
|
+
await selectContext.commit();
|
|
499
|
+
}
|
|
500
|
+
// 2. 并发检查:过滤出未在执行中的数据
|
|
501
|
+
const [rowsWithoutExecuting, hasSkipped] = this.filterAndMarkExecutingRows(watcher, rows);
|
|
502
|
+
if (rowsWithoutExecuting.length === 0) {
|
|
503
|
+
if (!isFreeType) {
|
|
504
|
+
await selectContext.commit();
|
|
505
|
+
}
|
|
506
|
+
// 全部行都被跳过,直接返回,实际没有执行任何操作,hasSkipped也是其他执行流在处理
|
|
507
|
+
// this.cleanupExecutingMarks(watcher.name, hasSkipped);
|
|
508
|
+
if (hasSkipped) {
|
|
509
|
+
console.log(`执行器【${watcher.name}】本次没有可执行的数据行,全部数据行均被跳过`);
|
|
411
510
|
}
|
|
412
|
-
await context.commit();
|
|
413
511
|
return result;
|
|
414
512
|
}
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
513
|
+
// 3. 执行业务逻辑
|
|
514
|
+
try {
|
|
515
|
+
if (isFreeType) {
|
|
516
|
+
const { fn } = watcher;
|
|
517
|
+
result = await fn(() => this.makeContext(), rowsWithoutExecuting);
|
|
418
518
|
}
|
|
419
519
|
else {
|
|
420
|
-
|
|
520
|
+
const { fn } = watcher;
|
|
521
|
+
result = await fn(selectContext, rowsWithoutExecuting);
|
|
522
|
+
await selectContext.commit();
|
|
523
|
+
}
|
|
524
|
+
return result;
|
|
525
|
+
}
|
|
526
|
+
catch (err) {
|
|
527
|
+
if (!isFreeType) {
|
|
528
|
+
if (err instanceof types_1.OakPartialSuccess) {
|
|
529
|
+
await selectContext.commit();
|
|
530
|
+
}
|
|
531
|
+
else {
|
|
532
|
+
await selectContext.rollback();
|
|
533
|
+
}
|
|
421
534
|
}
|
|
422
|
-
// 不能在这里publish,因为这个方法可能是在timer中调用,也可能是在routine中调用
|
|
423
535
|
throw err;
|
|
424
536
|
}
|
|
537
|
+
finally {
|
|
538
|
+
// 清理执行标记
|
|
539
|
+
this.cleanupExecutingMarks(watcher.name, rowsWithoutExecuting);
|
|
540
|
+
// 这里只需要清理被执行的行,因为被跳过的行本来就不是这一次执行被占用的。
|
|
541
|
+
}
|
|
425
542
|
}
|
|
426
543
|
getCheckpointTs() {
|
|
427
544
|
const now = Date.now();
|
package/lib/DbStore.d.ts
CHANGED
|
@@ -2,6 +2,7 @@ import { DbConfiguration } from 'oak-db/src/types/configuration';
|
|
|
2
2
|
import { EntityDict, StorageSchema, Trigger, Checker, SelectFreeEntities, UpdateFreeDict, AuthDeduceRelationMap, VolatileTrigger, OperateOption } from 'oak-domain/lib/types';
|
|
3
3
|
import { EntityDict as BaseEntityDict } from 'oak-domain/lib/base-app-domain';
|
|
4
4
|
import { BackendRuntimeContext } from 'oak-frontend-base/lib/context/BackendRuntimeContext';
|
|
5
|
+
import { DbTypeSymbol } from './utils/dbPriority';
|
|
5
6
|
import { CascadeStore } from 'oak-domain/lib/store/CascadeStore';
|
|
6
7
|
import { DbStore } from 'oak-db/lib/types/dbStore';
|
|
7
8
|
export type TriggerStore<ED extends EntityDict & BaseEntityDict, Cxt extends BackendRuntimeContext<ED>> = {
|
|
@@ -13,4 +14,6 @@ export type TriggerStore<ED extends EntityDict & BaseEntityDict, Cxt extends Bac
|
|
|
13
14
|
independentCheckPoint(name: string, ts: number, instanceCount?: number, instanceId?: number): Promise<number>;
|
|
14
15
|
};
|
|
15
16
|
export type AppDbStore<ED extends EntityDict & BaseEntityDict, Cxt extends BackendRuntimeContext<ED>> = DbStore<ED, Cxt> & CascadeStore<ED> & TriggerStore<ED, Cxt>;
|
|
16
|
-
export declare const createDbStore: <ED extends EntityDict & BaseEntityDict, Cxt extends BackendRuntimeContext<ED>>(storageSchema: StorageSchema<ED>, contextBuilder: () => Cxt, dbConfiguration: DbConfiguration
|
|
17
|
+
export declare const createDbStore: <ED extends EntityDict & BaseEntityDict, Cxt extends BackendRuntimeContext<ED>>(storageSchema: StorageSchema<ED>, contextBuilder: () => Cxt, dbConfiguration: DbConfiguration & {
|
|
18
|
+
[DbTypeSymbol]?: string;
|
|
19
|
+
}, authDeduceRelationMap: AuthDeduceRelationMap<ED>, selectFreeEntities?: SelectFreeEntities<ED>, updateFreeDict?: UpdateFreeDict<ED>, onVolatileTrigger?: <T extends keyof ED>(entity: T, trigger: VolatileTrigger<ED, T, Cxt>, ids: string[], cxtStr: string, option: OperateOption) => Promise<void>) => AppDbStore<ED, Cxt>;
|
package/lib/DbStore.js
CHANGED
|
@@ -5,8 +5,10 @@ const TriggerExecutor_1 = require("oak-domain/lib/store/TriggerExecutor");
|
|
|
5
5
|
const RelationAuth_1 = require("oak-domain/lib/store/RelationAuth");
|
|
6
6
|
const dbPriority_1 = require("./utils/dbPriority");
|
|
7
7
|
const createDbStore = (storageSchema, contextBuilder, dbConfiguration, authDeduceRelationMap, selectFreeEntities = [], updateFreeDict = {}, onVolatileTrigger) => {
|
|
8
|
-
|
|
8
|
+
// TODO: 这里的类型检查会过不去,因为ts不知道上层已经实现这个抽象类。
|
|
9
|
+
const BaseStoreClass = (0, dbPriority_1.getDbStoreClass)(dbConfiguration);
|
|
9
10
|
// 动态创建继承类
|
|
11
|
+
// @ts-ignore
|
|
10
12
|
class DynamicDbStore extends BaseStoreClass {
|
|
11
13
|
executor;
|
|
12
14
|
relationAuth;
|
package/lib/Synchronizer.d.ts
CHANGED
|
@@ -2,12 +2,21 @@ import { EntityDict, StorageSchema, EndpointItem, SyncConfig } from 'oak-domain/
|
|
|
2
2
|
import { VolatileTrigger } from 'oak-domain/lib/types/Trigger';
|
|
3
3
|
import { EntityDict as BaseEntityDict } from 'oak-domain/lib/base-app-domain';
|
|
4
4
|
import { BackendRuntimeContext } from 'oak-frontend-base/lib/context/BackendRuntimeContext';
|
|
5
|
+
export type FetchFn = (url: string, options: {
|
|
6
|
+
method: string;
|
|
7
|
+
headers: Record<string, string>;
|
|
8
|
+
body: string;
|
|
9
|
+
}, timeout?: number) => Promise<{
|
|
10
|
+
status: number;
|
|
11
|
+
json: () => Promise<any>;
|
|
12
|
+
}>;
|
|
5
13
|
export default class Synchronizer<ED extends EntityDict & BaseEntityDict, Cxt extends BackendRuntimeContext<ED>> {
|
|
6
14
|
private config;
|
|
7
15
|
private schema;
|
|
8
16
|
private remotePullInfoMap;
|
|
9
17
|
private channelDict;
|
|
10
18
|
private contextBuilder;
|
|
19
|
+
private fetchFn;
|
|
11
20
|
private pushAccessMap;
|
|
12
21
|
private startChannel2;
|
|
13
22
|
/**开始同步这些channel上的oper。注意,这时候即使某个channel上失败了,也不应影响本事务提交(其它的channel成功了) */
|
|
@@ -22,7 +31,7 @@ export default class Synchronizer<ED extends EntityDict & BaseEntityDict, Cxt ex
|
|
|
22
31
|
*/
|
|
23
32
|
private trySynchronizeOpers;
|
|
24
33
|
private makeCreateOperTrigger;
|
|
25
|
-
constructor(config: SyncConfig<ED, Cxt>, schema: StorageSchema<ED>, contextBuilder: () => Cxt);
|
|
34
|
+
constructor(config: SyncConfig<ED, Cxt>, schema: StorageSchema<ED>, contextBuilder: () => Cxt, fetchFn?: FetchFn);
|
|
26
35
|
/**
|
|
27
36
|
* 根据sync的定义,生成对应的 commit triggers
|
|
28
37
|
* @returns
|
package/lib/Synchronizer.js
CHANGED
|
@@ -5,11 +5,11 @@ const crypto_1 = require("crypto");
|
|
|
5
5
|
const types_1 = require("oak-domain/lib/types");
|
|
6
6
|
const relationPath_1 = require("oak-domain/lib/utils/relationPath");
|
|
7
7
|
const assert_1 = tslib_1.__importDefault(require("assert"));
|
|
8
|
-
const path_1 = require("path");
|
|
9
8
|
const lodash_1 = require("oak-domain/lib/utils/lodash");
|
|
10
9
|
const filter_1 = require("oak-domain/lib/store/filter");
|
|
11
10
|
const uuid_1 = require("oak-domain/lib/utils/uuid");
|
|
12
11
|
const lodash_2 = require("lodash");
|
|
12
|
+
const domain_1 = require("oak-domain/lib/utils/domain");
|
|
13
13
|
const OAK_SYNC_HEADER_ENTITY = 'oak-sync-entity';
|
|
14
14
|
const OAK_SYNC_HEADER_ENTITY_ID = 'oak-sync-entity-id';
|
|
15
15
|
const OAK_SYNC_HEADER_TIMESTAMP = 'oak-sync-timestamp';
|
|
@@ -67,6 +67,7 @@ class Synchronizer {
|
|
|
67
67
|
remotePullInfoMap = {};
|
|
68
68
|
channelDict = {};
|
|
69
69
|
contextBuilder;
|
|
70
|
+
fetchFn;
|
|
70
71
|
pushAccessMap = {};
|
|
71
72
|
async startChannel2(context, channel) {
|
|
72
73
|
const { queue, api, selfEncryptInfo, entity, entityId, onFailed, timeout = 5000 } = channel;
|
|
@@ -78,12 +79,12 @@ class Synchronizer {
|
|
|
78
79
|
seq: ele.$$seq$$,
|
|
79
80
|
}))), 'txnId:', context.getCurrentTxnId());
|
|
80
81
|
}
|
|
81
|
-
const finalApi = (0,
|
|
82
|
+
const finalApi = (0, domain_1.urlJoin)(api, selfEncryptInfo.id);
|
|
82
83
|
channel.queue = [];
|
|
83
84
|
try {
|
|
84
85
|
const body = JSON.stringify(opers);
|
|
85
86
|
const { ts, nonce, signature } = await sign(selfEncryptInfo.privateKey, body);
|
|
86
|
-
const res = await
|
|
87
|
+
const res = await this.fetchFn(finalApi, {
|
|
87
88
|
method: 'post',
|
|
88
89
|
headers: {
|
|
89
90
|
'Content-Type': 'application/json',
|
|
@@ -190,7 +191,7 @@ class Synchronizer {
|
|
|
190
191
|
if (!this.channelDict[userId]) {
|
|
191
192
|
// channel上缓存这些信息,暂不支持动态更新
|
|
192
193
|
this.channelDict[userId] = {
|
|
193
|
-
api: (0,
|
|
194
|
+
api: (0, domain_1.urlJoin)(url, 'endpoint', endpoint),
|
|
194
195
|
queue: [],
|
|
195
196
|
entity: remoteEntity,
|
|
196
197
|
entityId: remoteEntityId,
|
|
@@ -205,7 +206,7 @@ class Synchronizer {
|
|
|
205
206
|
(0, assert_1.default)(this.channelDict[userId].onFailed === onFailed);
|
|
206
207
|
}
|
|
207
208
|
const channel = this.channelDict[userId];
|
|
208
|
-
(0, assert_1.default)(channel.api === (0,
|
|
209
|
+
(0, assert_1.default)(channel.api === (0, domain_1.urlJoin)(url, 'endpoint', endpoint));
|
|
209
210
|
(0, assert_1.default)(channel.entity === remoteEntity);
|
|
210
211
|
(0, assert_1.default)(channel.entityId === remoteEntityId);
|
|
211
212
|
if (channel.queue.find(ele => ele.oper.id === oper.id)) {
|
|
@@ -343,8 +344,19 @@ class Synchronizer {
|
|
|
343
344
|
}, { dontCollect: true, forUpdate: true });
|
|
344
345
|
dirtyOpers = dirtyOpers.filter(ele => !!ele[types_1.TriggerUuidAttribute]);
|
|
345
346
|
if (dirtyOpers.length > 0) {
|
|
347
|
+
// 检查所有 channel 的 queue 是否已清空
|
|
346
348
|
for (const c in this.channelDict) {
|
|
347
|
-
|
|
349
|
+
// 在生产环境不使用assert,而是清空队列,以防止脏数据堆积影响后续同步
|
|
350
|
+
if (this.channelDict[c].queue.length > 0) {
|
|
351
|
+
if (process.env.NODE_ENV !== 'production') {
|
|
352
|
+
// 构建详细的错误信息
|
|
353
|
+
const queuedOperIds = this.channelDict[c].queue.map(q => q.oper.id).join(', ');
|
|
354
|
+
(0, assert_1.default)(false, `发现 channel ${c} 的 queue 未清空,包含 oper IDs: [${queuedOperIds}]`);
|
|
355
|
+
}
|
|
356
|
+
else {
|
|
357
|
+
this.channelDict[c].queue = [];
|
|
358
|
+
}
|
|
359
|
+
}
|
|
348
360
|
}
|
|
349
361
|
const pushedIds = [];
|
|
350
362
|
const unPushedIds = [];
|
|
@@ -394,15 +406,45 @@ class Synchronizer {
|
|
|
394
406
|
makeCreateOperTrigger() {
|
|
395
407
|
const { config } = this;
|
|
396
408
|
const { remotes, self } = config;
|
|
409
|
+
const entityActionMap = new Map();
|
|
397
410
|
// 根据remotes定义,建立从entity到需要同步的远端结点信息的Map
|
|
398
411
|
remotes.forEach((remote) => {
|
|
399
412
|
const { getPushInfo, pushEntities: pushEntityDefs, endpoint, pathToUser, relationName: rnRemote, onFailed, timeout } = remote;
|
|
400
413
|
if (pushEntityDefs) {
|
|
401
414
|
const pushEntities = [];
|
|
402
|
-
const endpoint2 = (0,
|
|
415
|
+
const endpoint2 = (0, domain_1.urlJoin)(endpoint || 'sync', self.entity);
|
|
403
416
|
for (const def of pushEntityDefs) {
|
|
404
417
|
const { pathToRemoteEntity, pathToSelfEntity, relationName, recursive, entity, actions, onSynchronized } = def;
|
|
405
418
|
pushEntities.push(entity);
|
|
419
|
+
if (!entityActionMap.has(entity)) {
|
|
420
|
+
entityActionMap.set(entity, {
|
|
421
|
+
actions: new Set(),
|
|
422
|
+
matchAllActions: false,
|
|
423
|
+
});
|
|
424
|
+
}
|
|
425
|
+
const tracker = entityActionMap.get(entity);
|
|
426
|
+
//如果当前定义是 wildcard(所有 action)
|
|
427
|
+
if (!actions || actions.length === 0) {
|
|
428
|
+
if (tracker.matchAllActions) {
|
|
429
|
+
throw new Error(`PushEntityDef 配置错误:entity「${entity}」的所有 action 被定义了多次`);
|
|
430
|
+
}
|
|
431
|
+
if (tracker.actions.size > 0) {
|
|
432
|
+
throw new Error(`PushEntityDef 配置错误:entity「${entity}」已定义特定 action「${Array.from(tracker.actions).join(', ')}」,不能再定义所有 action`);
|
|
433
|
+
}
|
|
434
|
+
tracker.matchAllActions = true;
|
|
435
|
+
}
|
|
436
|
+
else {
|
|
437
|
+
// 如果当前定义是特定 action
|
|
438
|
+
if (tracker.matchAllActions) {
|
|
439
|
+
throw new Error(`PushEntityDef 配置错误:entity「${entity}」已定义所有 action,不能再定义特定 action「${actions.join(', ')}」`);
|
|
440
|
+
}
|
|
441
|
+
for (const action of actions) {
|
|
442
|
+
if (tracker.actions.has(action)) {
|
|
443
|
+
throw new Error(`PushEntityDef 配置错误:entity「${entity}」的action「${action}」被定义了多次`);
|
|
444
|
+
}
|
|
445
|
+
tracker.actions.add(action);
|
|
446
|
+
}
|
|
447
|
+
}
|
|
406
448
|
const relationName2 = relationName || rnRemote;
|
|
407
449
|
const path2 = pathToUser ? `${pathToRemoteEntity}.${pathToUser}` : pathToRemoteEntity;
|
|
408
450
|
(0, assert_1.default)(!recursive);
|
|
@@ -507,10 +549,11 @@ class Synchronizer {
|
|
|
507
549
|
};
|
|
508
550
|
return createOperTrigger;
|
|
509
551
|
}
|
|
510
|
-
constructor(config, schema, contextBuilder) {
|
|
552
|
+
constructor(config, schema, contextBuilder, fetchFn) {
|
|
511
553
|
this.config = config;
|
|
512
554
|
this.schema = schema;
|
|
513
555
|
this.contextBuilder = contextBuilder;
|
|
556
|
+
this.fetchFn = fetchFn || fetchWithTimeout;
|
|
514
557
|
}
|
|
515
558
|
/**
|
|
516
559
|
* 根据sync的定义,生成对应的 commit triggers
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
import { EntityDict as TypeEntityDict, StorageSchema } from 'oak-domain/lib/types';
|
|
2
|
+
import { AsyncContext } from 'oak-domain/lib/store/AsyncRowStore';
|
|
3
|
+
import BackendRuntimeContext from 'oak-frontend-base/lib/context/BackendRuntimeContext';
|
|
4
|
+
import { EntityDict as BaseEntityDict } from 'oak-domain/lib/base-app-domain';
|
|
5
|
+
type EntityDict = TypeEntityDict & BaseEntityDict;
|
|
6
|
+
/**
|
|
7
|
+
* 反向引用描述
|
|
8
|
+
* 用于记录哪些实体引用了当前实体,便于在删除或更新时维护引用完整性
|
|
9
|
+
*/
|
|
10
|
+
export type ReverseRefDesc<ED extends EntityDict> = {
|
|
11
|
+
type: 'ref' | 'refs';
|
|
12
|
+
attrName: string;
|
|
13
|
+
sourceEntity: keyof ED;
|
|
14
|
+
};
|
|
15
|
+
/**
|
|
16
|
+
* 更新计划配置
|
|
17
|
+
* 定义了数据同步的完整策略和生命周期钩子
|
|
18
|
+
*/
|
|
19
|
+
export type UpdatePlan<ED extends EntityDict> = {
|
|
20
|
+
dontCreateOper?: boolean;
|
|
21
|
+
plan: {
|
|
22
|
+
[entityName in keyof ED]: {
|
|
23
|
+
/**
|
|
24
|
+
* 是否允许数据唯一性冲突时使用新数据覆盖旧数据(仅data中冲突)
|
|
25
|
+
* @deprecated ID必须相同但会导致ID检查不通过,弃用
|
|
26
|
+
*/
|
|
27
|
+
allowDataUniqueReWrite?: boolean;
|
|
28
|
+
/**
|
|
29
|
+
* 在插入数据库遇到唯一性冲突时的处理方式(数据库和data冲突)
|
|
30
|
+
* skip表示跳过插入新数据,会将新数据的反指关系更新到已有数据上
|
|
31
|
+
* update表示使用新数据更新已有数据,会自动更新现有数据的所有反指关系
|
|
32
|
+
* error表示抛出错误
|
|
33
|
+
*/
|
|
34
|
+
onUniqueViolation?: 'skip' | 'update' | 'error';
|
|
35
|
+
/**
|
|
36
|
+
* 处理数据库中存在但是数据文件中不存在的数据的方式
|
|
37
|
+
* skip表示跳过不处理
|
|
38
|
+
* delete表示做逻辑删除,可能导致插入新数据时id冲突
|
|
39
|
+
* 设置为physicalDelete可以保证最强的一致性
|
|
40
|
+
*/
|
|
41
|
+
onOnlyExistingInDb?: 'skip' | 'delete' | 'physicalDelete';
|
|
42
|
+
};
|
|
43
|
+
};
|
|
44
|
+
beforeCheck?: (options: {
|
|
45
|
+
context: BackendRuntimeContext<ED>;
|
|
46
|
+
data: {
|
|
47
|
+
[entityName in keyof ED]?: Array<Partial<ED[entityName]['CreateOperationData']>>;
|
|
48
|
+
};
|
|
49
|
+
reverseRefs: {
|
|
50
|
+
[entityName in keyof ED]?: ReverseRefDesc<ED>[];
|
|
51
|
+
};
|
|
52
|
+
}) => Promise<void>;
|
|
53
|
+
afterUpdate?: (options: {
|
|
54
|
+
context: BackendRuntimeContext<ED>;
|
|
55
|
+
data: {
|
|
56
|
+
[entityName in keyof EntityDict]?: Array<Partial<EntityDict[entityName]['CreateOperationData']>>;
|
|
57
|
+
};
|
|
58
|
+
reverseRefs: {
|
|
59
|
+
[entityName in keyof EntityDict]?: ReverseRefDesc<ED>[];
|
|
60
|
+
};
|
|
61
|
+
}) => Promise<void>;
|
|
62
|
+
};
|
|
63
|
+
export declare const createUpdatePlan: <ED extends EntityDict>(options: UpdatePlan<ED>) => <Cxt extends AsyncContext<ED>>(context: Cxt) => Promise<void>;
|
|
64
|
+
export declare function checkPathValue(itemId: string, destEntity: keyof EntityDict, path: string, storageSchema: StorageSchema<EntityDict>): boolean;
|
|
65
|
+
export {};
|