oak-backend-base 3.4.1 → 3.5.0

package/lib/Synchronizer.js

@@ -15,8 +15,8 @@ class Synchronizer {
     config;
     schema;
     remotePullInfoMap = {};
-    pullMaxBornAtMap = {};
     channelDict = {};
+    contextBuilder;
     pushAccessMap = {};
     /**
      * 向某一个远端对象push opers。根据幂等性，这里如果失败了必须反复推送
@@ -30,7 +30,12 @@ class Synchronizer {
             // todo 加密
             const queue = channel.queue;
             const opers = queue.map(ele => ele.oper);
-            console.log('向远端结点sync数据', api, JSON.stringify(opers));
+            if (process.env.NODE_ENV === 'development') {
+                console.log('向远端结点sync oper', JSON.stringify(opers.map(ele => ({
+                    id: ele.id,
+                    seq: ele.$$seq$$,
+                }))), 'txnId:', context.getCurrentTxnId());
+            }
             const finalApi = (0, path_1.join)(api, selfEncryptInfo.id);
             const res = await fetch(finalApi, {
                 method: 'post',
@@ -61,6 +66,9 @@ class Synchronizer {
         /**
          * 返回结构见this.getSelfEndpoint
          */
+        if (process.env.NODE_ENV === 'development') {
+            console.log('同步oper返回结果', JSON.stringify(json), 'txnId:', context.getCurrentTxnId());
+        }
         const { successIds, failed, redundantIds } = json;
         if (failed) {
             const { id, error } = failed;
@@ -76,7 +84,7 @@ class Synchronizer {
                 aliveOperIds.push(...this.channelDict[k].queue.map(ele => ele.oper.id));
             }
         }
-        const overIds = (0, lodash_1.difference)(successIds, aliveOperIds);
+        const overIds = (0, lodash_1.difference)(successIds.concat(redundantIds), aliveOperIds);
         if (overIds.length > 0) {
             await context.operate('oper', {
                 id: await (0, uuid_1.generateNewIdAsync)(),
@@ -112,7 +120,7 @@ class Synchronizer {
         if (channel.queue.length > 0) {
             // 最大延迟redo时间512秒
             const retryDelay = Math.pow(2, Math.min(9, retry)) * 1000;
-            console.error(`有${channel.queue.length}个oper同步失败，将于${retryDelay}毫秒后重试`);
+            console.error(`有${channel.queue.length}个oper同步失败，id是「${channel.queue.map(ele => ele.oper.id).join(',')}」，将于${retryDelay}毫秒后重试`);
             return new Promise((resolve) => {
                 setTimeout(async () => {
                     await this.startChannel(context, channel, retry + 1);
@@ -149,11 +157,32 @@ class Synchronizer {
         (0, assert_1.default)(channel.api === (0, path_1.join)(url, 'endpoint', endpoint));
         (0, assert_1.default)(channel.entity === remoteEntity);
         (0, assert_1.default)(channel.entityId === remoteEntityId);
+        if (channel.queue.find(ele => ele.oper.id === oper.id)) {
+            console.error('aaaaa');
+        }
         channel.queue.push({
             oper,
             onSynchronized,
         });
     }
+    refineOperData(oper, rowIds) {
+        const { action, id, targetEntity, data, $$seq$$, filter } = oper;
+        const data2 = (action === 'create' && data instanceof Array) ? data.filter(ele => rowIds.includes(ele.id)) : data;
+        // 过滤掉数据中的跨事务trigger信息
+        if (data2 instanceof Array) {
+            data2.forEach((d) => {
+                (0, lodash_2.unset)(d, types_1.TriggerDataAttribute);
+                (0, lodash_2.unset)(d, types_1.TriggerUuidAttribute);
+            });
+        }
+        else {
+            (0, lodash_2.unset)(data2, types_1.TriggerDataAttribute);
+            (0, lodash_2.unset)(data2, types_1.TriggerUuidAttribute);
+        }
+        return {
+            id, action, targetEntity, data: data2, $$seq$$, filter,
+        };
+    }
     async dispatchOperToChannels(oper, context) {
         const { operatorId, targetEntity, filter, action, data } = oper;
         const entityIds = (0, filter_1.getRelevantIds)(filter);
@@ -192,18 +221,7 @@ class Synchronizer {
                         }
                         const selfEncryptInfo = encryptInfoDict[selfEntityId];
                         // 推送到远端结点的oper
-                        const oper2 = {
-                            id: oper.id,
-                            action: action,
-                            data: (action === 'create' && data instanceof Array) ? data.filter(ele => rowIds.includes(ele.id)) : data,
-                            filter: {
-                                id: rowIds.length === 1 ? rowIds[0] : {
-                                    $in: rowIds,
-                                }
-                            },
-                            $$seq$$: oper.$$seq$$,
-                            targetEntity,
-                        };
+                        const oper2 = this.refineOperData(oper, rowIds);
                         const { url } = await getRemoteAccessInfo(context, {
                             userId,
                             remoteEntityId: entityId,
@@ -227,69 +245,85 @@ class Synchronizer {
      * 每个进程都保证把当前所有的oper顺序处理掉，就不会有乱序的问题，大家通过database上的锁来完成同步
      * @param context
      */
-    async trySynchronizeOpers(context) {
-        let dirtyOpers = await context.select('oper', {
-            data: {
-                id: 1,
-            },
-            filter: {
-                [types_1.TriggerDataAttribute]: {
-                    $exists: true,
-                },
-            }
-        }, { dontCollect: true });
-        if (dirtyOpers.length > 0) {
-            // 这一步是加锁，保证只有一个进程完成推送，推送者提交前会将$$triggerData$$清零
-            const ids = dirtyOpers.map(ele => ele.id);
-            dirtyOpers = await context.select('oper', {
+    async trySynchronizeOpers() {
+        const context = this.contextBuilder();
+        await context.begin();
+        // 暂时全用root身份去执行（未来不一定对)
+        await context.initialize();
+        context.openRootMode();
+        try {
+            let dirtyOpers = await context.select('oper', {
                 data: {
                     id: 1,
-                    action: 1,
-                    data: 1,
-                    targetEntity: 1,
-                    operatorId: 1,
-                    [types_1.TriggerDataAttribute]: 1,
-                    bornAt: 1,
-                    $$createAt$$: 1,
-                    $$seq$$: 1,
-                    filter: 1,
                 },
                 filter: {
-                    id: { $in: ids },
-                },
-            }, { dontCollect: true, forUpdate: true });
-            dirtyOpers = dirtyOpers.filter(ele => !!ele[types_1.TriggerDataAttribute]);
+                    [types_1.TriggerDataAttribute]: {
+                        $exists: true,
+                    },
+                }
+            }, { dontCollect: true });
             if (dirtyOpers.length > 0) {
-                const pushedIds = [];
-                const unpushedIds = [];
-                await Promise.all(dirtyOpers.map(async (oper) => {
-                    const result = await this.dispatchOperToChannels(oper, context);
-                    if (result) {
-                        pushedIds.push(oper.id);
-                    }
-                    else {
-                        unpushedIds.push(oper.id);
+                // 这一步是加锁，保证只有一个进程完成推送，推送者提交前会将$$triggerData$$清零
+                const ids = dirtyOpers.map(ele => ele.id);
+                dirtyOpers = await context.select('oper', {
+                    data: {
+                        id: 1,
+                        action: 1,
+                        data: 1,
+                        targetEntity: 1,
+                        operatorId: 1,
+                        [types_1.TriggerDataAttribute]: 1,
+                        bornAt: 1,
+                        $$createAt$$: 1,
+                        $$seq$$: 1,
+                        filter: 1,
+                    },
+                    filter: {
+                        id: { $in: ids },
+                    },
+                }, { dontCollect: true, forUpdate: true });
+                dirtyOpers = dirtyOpers.filter(ele => !!ele[types_1.TriggerDataAttribute]);
+                if (dirtyOpers.length > 0) {
+                    for (const c in this.channelDict) {
+                        (0, assert_1.default)(this.channelDict[c].queue.length === 0);
                     }
-                }));
-                if (unpushedIds.length > 0) {
-                    await context.operate('oper', {
-                        id: await (0, uuid_1.generateNewIdAsync)(),
-                        action: 'update',
-                        data: {
-                            [types_1.TriggerDataAttribute]: null,
-                            [types_1.TriggerUuidAttribute]: null,
-                        },
-                        filter: {
-                            id: {
-                                $in: unpushedIds,
-                            }
+                    const pushedIds = [];
+                    const unpushedIds = [];
+                    await Promise.all(dirtyOpers.map(async (oper) => {
+                        const result = await this.dispatchOperToChannels(oper, context);
+                        if (result) {
+                            pushedIds.push(oper.id);
                         }
-                    }, {});
-                }
-                if (pushedIds.length > 0) {
-                    await this.startAllChannel(context);
+                        else {
+                            unpushedIds.push(oper.id);
+                        }
+                    }));
+                    if (unpushedIds.length > 0) {
+                        await context.operate('oper', {
+                            id: await (0, uuid_1.generateNewIdAsync)(),
+                            action: 'update',
+                            data: {
+                                [types_1.TriggerDataAttribute]: null,
+                                [types_1.TriggerUuidAttribute]: null,
+                            },
+                            filter: {
+                                id: {
+                                    $in: unpushedIds,
+                                }
+                            }
+                        }, {});
+                    }
+                    if (pushedIds.length > 0) {
+                        await this.startAllChannel(context);
+                    }
                 }
             }
+            await context.commit();
+        }
+        catch (err) {
+            await context.rollback();
+            console.error(err);
+            throw err;
         }
     }
     makeCreateOperTrigger() {
@@ -395,18 +429,19 @@ class Synchronizer {
                 return pushEntities.includes(data.targetEntity)
                     && !!this.pushAccessMap[targetEntity].find(({ actions }) => !actions || actions.includes(action));
             },
-            fn: async ({ ids }, context) => {
+            fn: async ({ ids }) => {
                 (0, assert_1.default)(ids.length === 1);
-                this.trySynchronizeOpers(context);
+                this.trySynchronizeOpers();
                 // 内部自主处理triggerData，因此不需要让triggerExecutor处理
                 throw new types_1.OakMakeSureByMySelfException();
             }
         };
         return createOperTrigger;
     }
-    constructor(config, schema) {
+    constructor(config, schema, contextBuilder) {
         this.config = config;
         this.schema = schema;
+        this.contextBuilder = contextBuilder;
     }
     /**
      * 根据sync的定义，生成对应的 commit triggers
@@ -418,8 +453,8 @@ class Synchronizer {
     getSyncRoutine() {
         return {
             name: 'checkpoint routine for sync',
-            routine: async (context) => {
-                this.trySynchronizeOpers(context);
+            routine: async () => {
+                this.trySynchronizeOpers();
                 return {};
             },
         };
@@ -433,7 +468,9 @@ class Synchronizer {
                 // body中是传过来的oper数组信息
                 const { entity, entityId } = params;
                 const { [OAK_SYNC_HEADER_ENTITY]: meEntity, [OAK_SYNC_HEADER_ENTITYID]: meEntityId } = headers;
-                console.log('接收到来自远端的sync数据', entity, JSON.stringify(body));
+                if (process.env.NODE_ENV === 'development') {
+                    console.log('接收到来自远端的sync数据', entity, JSON.stringify(body));
+                }
                 const successIds = [];
                 const redundantIds = [];
                 let failed;
@@ -447,6 +484,7 @@ class Synchronizer {
                     if (pullEntities) {
                         pullEntities.forEach((def) => pullEntityDict[def.entity] = def);
                     }
+                    const closeFn = context.openRootMode();
                     this.remotePullInfoMap[entity][entityId] = {
                         pullInfo: await getPullInfo(context, {
                             selfId: meEntityId,
@@ -454,6 +492,7 @@ class Synchronizer {
                         }),
                         pullEntityDict,
                     };
+                    closeFn();
                 }
                 const { pullInfo, pullEntityDict } = this.remotePullInfoMap[entity][entityId];
                 const { userId, algorithm, publicKey, cxtInfo } = pullInfo;
@@ -463,36 +502,30 @@ class Synchronizer {
                     await context.initialize(cxtInfo);
                 }
                 // todo 解密
-                if (!this.pullMaxBornAtMap.hasOwnProperty(entityId)) {
-                    const [maxHisOper] = await context.select('oper', {
-                        data: {
-                            id: 1,
-                            bornAt: 1,
-                        },
-                        filter: {
-                            operatorId: userId,
+                const opers = body;
+                const ids = opers.map(ele => ele.id);
+                const existeIds = (await context.select('oper', {
+                    data: {
+                        id: 1,
+                    },
+                    filter: {
+                        id: {
+                            $in: ids,
                         },
-                        sorter: [
-                            {
-                                $attr: {
-                                    bornAt: 1,
-                                },
-                                $direction: 'desc',
-                            },
-                        ],
-                        indexFrom: 0,
-                        count: 1,
-                    }, { dontCollect: true });
-                    this.pullMaxBornAtMap[entityId] = maxHisOper?.bornAt || 0;
+                    }
+                }, {})).map(ele => ele.id);
+                const staleOpers = opers.filter(ele => existeIds.includes(ele.id));
+                const freshOpers = opers.filter(ele => !existeIds.includes(ele.id));
+                if (process.env.NODE_ENV !== 'production') {
+                    const maxStaleSeq = Math.max(...staleOpers.map(ele => ele.$$seq$$));
+                    for (const oper of freshOpers) {
+                        (0, assert_1.default)(oper.$$seq$$ > maxStaleSeq, '发现了seq没有按序进行同步');
+                    }
                 }
-                let maxBornAt = this.pullMaxBornAtMap[entityId];
-                const opers = body;
-                const outdatedOpers = opers.filter(ele => ele.$$seq$$ <= maxBornAt);
-                const freshOpers = opers.filter(ele => ele.$$seq$$ > maxBornAt);
                 await Promise.all([
                     // 无法严格保证推送按bornAt，所以一旦还有outdatedOpers，检查其已经被apply
                     (async () => {
-                        const ids = outdatedOpers.map(ele => ele.id);
+                        const ids = staleOpers.map(ele => ele.id);
                         if (ids.length > 0) {
                             const opersExisted = await context.select('oper', {
                                 data: {
@@ -538,7 +571,6 @@ class Synchronizer {
                                 };
                                 await context.operate(targetEntity, operation, {});
                                 successIds.push(id);
-                                maxBornAt = $$seq$$;
                             }
                             catch (err) {
                                 console.error(err);
@@ -552,7 +584,6 @@ class Synchronizer {
                         }
                     })()
                 ]);
-                this.pullMaxBornAtMap[entityId] = maxBornAt;
                 return {
                     successIds,
                     failed,

package/lib/cluster/DataSubscriber.d.ts

@@ -12,8 +12,7 @@ import { Namespace } from 'socket.io';
 export default class DataSubscriber<ED extends EntityDict & BaseEntityDict, Context extends BackendRuntimeContext<ED>> {
     private ns;
     private nsServer?;
-    private contextBuilder;
-    constructor(ns: Namespace, contextBuilder: (scene?: string) => Promise<Context>, nsServer?: Namespace);
+    constructor(ns: Namespace, nsServer?: Namespace);
     /**
      * 来自外部的socket连接，监听数据变化
      */

package/lib/cluster/DataSubscriber.js

@@ -12,11 +12,9 @@ const console_1 = require("console");
 class DataSubscriber {
     ns;
     nsServer;
-    contextBuilder;
-    constructor(ns, contextBuilder, nsServer) {
+    constructor(ns, nsServer) {
         this.ns = ns;
         this.nsServer = nsServer;
-        this.contextBuilder = contextBuilder;
         this.startup();
     }
     /**

package/lib/types/Sync.d.ts

@@ -1,16 +1,16 @@
-import { EntityDict } from 'oak-domain/lib/types';
-import { EntityDict as BaseEntityDict } from 'oak-domain/lib/base-app-domain';
-import { BackendRuntimeContext } from 'oak-frontend-base/lib/context/BackendRuntimeContext';
-import { RemotePushInfo, RemotePullInfo, SelfEncryptInfo, SyncRemoteConfigBase, SyncSelfConfigBase, SyncConfig } from 'oak-domain/lib/types/Sync';
-interface SyncRemoteConfigWrapper<ED extends EntityDict & BaseEntityDict, Cxt extends BackendRuntimeContext<ED>> extends SyncRemoteConfigBase<ED, Cxt> {
-    getRemotePushInfo: (userId: string) => Promise<RemotePushInfo>;
-    getRemotePullInfo: (id: string) => Promise<RemotePullInfo>;
-}
-interface SyncSelfConfigWrapper<ED extends EntityDict & BaseEntityDict> extends SyncSelfConfigBase<ED> {
-    getSelfEncryptInfo: () => Promise<SelfEncryptInfo>;
-}
-export interface SyncConfigWrapper<ED extends EntityDict & BaseEntityDict, Cxt extends BackendRuntimeContext<ED>> {
-    self: SyncSelfConfigWrapper<ED>;
-    remotes: Array<SyncRemoteConfigWrapper<ED, Cxt>>;
-}
-export { RemotePushInfo, RemotePullInfo, SelfEncryptInfo, SyncConfig, };
+import { EntityDict } from 'oak-domain/lib/types';
+import { EntityDict as BaseEntityDict } from 'oak-domain/lib/base-app-domain';
+import { BackendRuntimeContext } from 'oak-frontend-base/lib/context/BackendRuntimeContext';
+import { RemotePushInfo, RemotePullInfo, SelfEncryptInfo, SyncRemoteConfigBase, SyncSelfConfigBase, SyncConfig } from 'oak-domain/lib/types/Sync';
+interface SyncRemoteConfigWrapper<ED extends EntityDict & BaseEntityDict, Cxt extends BackendRuntimeContext<ED>> extends SyncRemoteConfigBase<ED, Cxt> {
+    getRemotePushInfo: (userId: string) => Promise<RemotePushInfo>;
+    getRemotePullInfo: (id: string) => Promise<RemotePullInfo>;
+}
+interface SyncSelfConfigWrapper<ED extends EntityDict & BaseEntityDict> extends SyncSelfConfigBase<ED> {
+    getSelfEncryptInfo: () => Promise<SelfEncryptInfo>;
+}
+export interface SyncConfigWrapper<ED extends EntityDict & BaseEntityDict, Cxt extends BackendRuntimeContext<ED>> {
+    self: SyncSelfConfigWrapper<ED>;
+    remotes: Array<SyncRemoteConfigWrapper<ED, Cxt>>;
+}
+export { RemotePushInfo, RemotePullInfo, SelfEncryptInfo, SyncConfig, };

package/lib/types/Sync.js

@@ -1,5 +1,5 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-;
-;
-;
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+;
+;
+;

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "oak-backend-base",
-    "version": "3.4.1",
+    "version": "3.5.0",
     "description": "oak-backend-base",
     "main": "lib/index",
     "author": {
@@ -20,10 +20,10 @@
         "mysql": "^2.18.1",
         "mysql2": "^2.3.3",
         "node-schedule": "^2.1.0",
-        "oak-common-aspect": "^2.3.0",
-        "oak-db": "^3.2.0",
-        "oak-domain": "^4.3.1",
-        "oak-frontend-base": "^4.3.0",
+        "oak-common-aspect": "^2.3.1",
+        "oak-db": "^3.2.1",
+        "oak-domain": "^4.5.0",
+        "oak-frontend-base": "^4.5.0",
         "socket.io": "^4.7.2",
         "socket.io-client": "^4.7.2",
         "uuid": "^8.3.2"