oak-backend-base 4.1.28 → 5.0.0

package/lib/AppLoader.d.ts

@@ -1,6 +1,6 @@
 import { EntityDict as BaseEntityDict } from 'oak-domain/lib/base-app-domain';
 import { AppLoader as GeneralAppLoader, Trigger, EntityDict, Watcher, OpRecord, FreeTimer, OperationResult, BaseTimer } from "oak-domain/lib/types";
-import { BackendRuntimeContext } from 'oak-frontend-base/lib/context/BackendRuntimeContext';
+import { BackendRuntimeContext } from 'oak-domain/lib/context/BackendRuntimeContext';
 import { IncomingHttpHeaders, IncomingMessage } from 'http';
 import { Namespace } from 'socket.io';
 import DataSubscriber from './cluster/DataSubscriber';
@@ -8,6 +8,7 @@ import Synchronizer from './Synchronizer';
 import { AsyncContext } from 'oak-domain/lib/store/AsyncRowStore';
 import { InternalErrorHandler } from './types';
 import { AppDbStore } from './DbStore';
+import { AppLoaderUpgradeOptions, AppLoaderUpgradeResult } from './upgrade';
 export declare class AppLoader<ED extends EntityDict & BaseEntityDict, Cxt extends BackendRuntimeContext<ED>> extends GeneralAppLoader<ED, Cxt> {
     protected dbStore: AppDbStore<ED, Cxt>;
     private aspectDict;
@@ -54,6 +55,7 @@ export declare class AppLoader<ED extends EntityDict & BaseEntityDict, Cxt exten
         message?: string;
     }>;
     initialize(ifExists?: 'drop' | 'omit' | 'dropIfNotStatic'): Promise<void>;
+    upgrade(options?: AppLoaderUpgradeOptions): Promise<AppLoaderUpgradeResult>;
     getStore(): AppDbStore<ED, Cxt>;
     getEndpoints(prefix: string): [string, "post" | "get" | "put" | "delete", string, (params: Record<string, string>, headers: IncomingHttpHeaders, req: IncomingMessage, body?: any) => Promise<{
         headers?: Record<string, string | string[]>;
@@ -87,7 +89,9 @@ export declare class AppLoader<ED extends EntityDict & BaseEntityDict, Cxt exten
      */
     private selectForWBWatcher;
     protected execWatcher(watcher: Watcher<ED, keyof ED, Cxt>): Promise<OperationResult<ED> | undefined>;
+    private lastCheckpointTs;
     protected getCheckpointTs(): number;
+    protected setCheckpointTs(ts: number): void;
     protected checkpoint(): Promise<number>;
     startWatchers(): void;
     protected execBaseTimer(timer: BaseTimer<ED, Cxt>, context: Cxt): Promise<OperationResult<ED>> | undefined;

package/lib/AppLoader.js

@@ -15,10 +15,11 @@ const dependencyBuilder_1 = require("oak-domain/lib/compiler/dependencyBuilder")
 const DataSubscriber_1 = tslib_1.__importDefault(require("./cluster/DataSubscriber"));
 const env_1 = require("./cluster/env");
 const Synchronizer_1 = tslib_1.__importDefault(require("./Synchronizer"));
-const i18n_1 = tslib_1.__importDefault(require("oak-domain/lib/data/i18n"));
 const requirePrj_1 = tslib_1.__importDefault(require("./utils/requirePrj"));
 const dbPriority_1 = require("./utils/dbPriority");
 const DbStore_1 = require("./DbStore");
+const upgrade_1 = require("./upgrade");
+const initializeData_1 = require("./utils/initializeData");
 class AppLoader extends types_1.AppLoader {
     dbStore;
     aspectDict;
@@ -241,13 +242,11 @@ class AppLoader extends types_1.AppLoader {
     async initialize(ifExists) {
         await this.dbStore.initialize({ ifExists });
         const data = this.requireSth('lib/data/index');
-        // oak-domain中只有i18n
-        (0, assert_1.default)(data.i18n);
-        data.i18n.push(...i18n_1.default);
         const context = this.contextBuilder(this.dbStore);
         context.openRootMode();
-        for (const entity in data) {
-            let rows = data[entity];
+        const entities = (0, initializeData_1.getInitializationOrder)(data, this.dbStore.getSchema());
+        for (const entity of entities) {
+            const rows = data[entity];
             if (rows.length > 0) {
                 await context.begin();
                 // 如果是static的对象，只要表中有数据就pass
@@ -267,23 +266,16 @@ class AppLoader extends types_1.AppLoader {
                 }
                 // 再插入所有的行
                 try {
-                    const insertRows = async (idx) => {
-                        const rows2 = rows.slice(idx, 1000);
-                        if (rows2.length > 0) {
-                            await this.dbStore.operate(entity, {
-                                data: rows,
-                                action: 'create',
-                            }, context, {
-                                dontCollect: true,
-                                dontCreateOper: true,
-                                blockTrigger: true,
-                            });
-                            if (rows2.length === 1000) {
-                                await insertRows(idx + 1000);
-                            }
-                        }
-                    };
-                    await insertRows(0);
+                    for (const rows2 of (0, initializeData_1.chunkRows)(rows)) {
+                        await this.dbStore.operate(entity, {
+                            data: rows2,
+                            action: 'create',
+                        }, context, {
+                            dontCollect: true,
+                            dontCreateOper: true,
+                            blockTrigger: true,
+                        });
+                    }
                     await context.commit();
                     console.log(`data in ${entity} initialized, ${rows.length} rows inserted`);
                 }
@@ -296,6 +288,30 @@ class AppLoader extends types_1.AppLoader {
         }
         // await this.dbStore.disconnect(); // 不需要马上断开连接，在initialize后可能还会有操作，unmount时会断开
     }
+    async upgrade(options = {}) {
+        const plan = await this.dbStore.makeUpgradePlan({
+            largeTableRowThreshold: options.largeTableRowThreshold,
+        });
+        let outputDir;
+        let files;
+        if (options.outputDir) {
+            outputDir = (0, path_1.resolve)(this.path, options.outputDir);
+            files = await (0, upgrade_1.writeUpgradeArtifacts)(plan, outputDir, this.dbStore.supportsTransactionalDdl());
+        }
+        const executed = await (0, upgrade_1.applyUpgradePlan)(this.dbStore, plan, options);
+        const remainingPlan = executed.total > 0
+            ? await this.dbStore.makeUpgradePlan({
+                largeTableRowThreshold: options.largeTableRowThreshold,
+            })
+            : undefined;
+        return {
+            plan,
+            remainingPlan,
+            outputDir,
+            files,
+            executed,
+        };
+    }
     getStore() {
         return this.dbStore;
     }
@@ -540,12 +556,18 @@ class AppLoader extends types_1.AppLoader {
             // 这里只需要清理被执行的行，因为被跳过的行本来就不是这一次执行被占用的。
         }
     }
+    lastCheckpointTs = 0;
     getCheckpointTs() {
-        const now = Date.now();
-        return process.env.NODE_ENV === 'development' ? now - 30 * 1000 : now - 120 * 1000;
+        return this.lastCheckpointTs;
     }
-    checkpoint() {
-        return this.dbStore.checkpoint(this.getCheckpointTs());
+    setCheckpointTs(ts) {
+        this.lastCheckpointTs = ts;
+    }
+    async checkpoint() {
+        const now = Date.now();
+        const count = await this.dbStore.checkpoint(this.getCheckpointTs());
+        this.setCheckpointTs(now);
+        return count;
     }
     startWatchers() {
         const watchers = this.requireSth('lib/watchers/index');

package/lib/ClusterAppLoader.d.ts

@@ -1,6 +1,6 @@
 import { EntityDict as BaseEntityDict } from 'oak-domain/lib/base-app-domain';
 import { EntityDict, OperationResult, Trigger, Watcher, FreeTimer, BaseTimer } from 'oak-domain/lib/types';
-import { BackendRuntimeContext } from 'oak-frontend-base/lib/context/BackendRuntimeContext';
+import { BackendRuntimeContext } from 'oak-domain/lib/context/BackendRuntimeContext';
 import { AppLoader } from './AppLoader';
 import { Namespace } from 'socket.io';
 import { Socket } from 'socket.io-client';

package/lib/ClusterAppLoader.js

@@ -183,6 +183,7 @@ class ClusterAppLoader extends AppLoader_1.AppLoader {
     async checkpoint() {
         const { instanceCount, instanceId } = (0, env_1.getClusterInfo)();
         let count = 0;
+        const now = Date.now();
         for (const name in this.commitTriggers) {
             if (this.commitTriggers[name]) {
                 count += await this.dbStore.independentCheckPoint(name, this.getCheckpointTs(), instanceCount, instanceId);
@@ -191,6 +192,7 @@ class ClusterAppLoader extends AppLoader_1.AppLoader {
                 count += await this.dbStore.independentCheckPoint(name, this.getCheckpointTs());
             }
         }
+        this.setCheckpointTs(now);
         return count;
     }
 }

package/lib/DbStore.d.ts

@@ -1,7 +1,7 @@
 import { DbConfiguration } from 'oak-db/src/types/configuration';
 import { EntityDict, StorageSchema, Trigger, Checker, SelectFreeEntities, UpdateFreeDict, AuthDeduceRelationMap, VolatileTrigger, OperateOption } from 'oak-domain/lib/types';
 import { EntityDict as BaseEntityDict } from 'oak-domain/lib/base-app-domain';
-import { BackendRuntimeContext } from 'oak-frontend-base/lib/context/BackendRuntimeContext';
+import { BackendRuntimeContext } from 'oak-domain/lib/context/BackendRuntimeContext';
 import { DbTypeSymbol } from './utils/dbPriority';
 import { CascadeStore } from 'oak-domain/lib/store/CascadeStore';
 import { DbStore } from 'oak-db/lib/types/dbStore';

package/lib/Synchronizer.d.ts

@@ -1,13 +1,22 @@
 import { EntityDict, StorageSchema, EndpointItem, SyncConfig } from 'oak-domain/lib/types';
 import { VolatileTrigger } from 'oak-domain/lib/types/Trigger';
 import { EntityDict as BaseEntityDict } from 'oak-domain/lib/base-app-domain';
-import { BackendRuntimeContext } from 'oak-frontend-base/lib/context/BackendRuntimeContext';
+import { BackendRuntimeContext } from 'oak-domain/lib/context/BackendRuntimeContext';
+export type FetchFn = (url: string, options: {
+    method: string;
+    headers: Record<string, string>;
+    body: string;
+}, timeout?: number) => Promise<{
+    status: number;
+    json: () => Promise<any>;
+}>;
 export default class Synchronizer<ED extends EntityDict & BaseEntityDict, Cxt extends BackendRuntimeContext<ED>> {
     private config;
     private schema;
     private remotePullInfoMap;
     private channelDict;
     private contextBuilder;
+    private fetchFn;
     private pushAccessMap;
     private startChannel2;
     /**开始同步这些channel上的oper。注意，这时候即使某个channel上失败了，也不应影响本事务提交（其它的channel成功了） */
@@ -22,7 +31,7 @@ export default class Synchronizer<ED extends EntityDict & BaseEntityDict, Cxt ex
      */
     private trySynchronizeOpers;
     private makeCreateOperTrigger;
-    constructor(config: SyncConfig<ED, Cxt>, schema: StorageSchema<ED>, contextBuilder: () => Cxt);
+    constructor(config: SyncConfig<ED, Cxt>, schema: StorageSchema<ED>, contextBuilder: () => Cxt, fetchFn?: FetchFn);
     /**
      * 根据sync的定义，生成对应的 commit triggers
      * @returns

package/lib/Synchronizer.js

@@ -5,11 +5,11 @@ const crypto_1 = require("crypto");
 const types_1 = require("oak-domain/lib/types");
 const relationPath_1 = require("oak-domain/lib/utils/relationPath");
 const assert_1 = tslib_1.__importDefault(require("assert"));
-const path_1 = require("path");
 const lodash_1 = require("oak-domain/lib/utils/lodash");
 const filter_1 = require("oak-domain/lib/store/filter");
 const uuid_1 = require("oak-domain/lib/utils/uuid");
 const lodash_2 = require("lodash");
+const domain_1 = require("oak-domain/lib/utils/domain");
 const OAK_SYNC_HEADER_ENTITY = 'oak-sync-entity';
 const OAK_SYNC_HEADER_ENTITY_ID = 'oak-sync-entity-id';
 const OAK_SYNC_HEADER_TIMESTAMP = 'oak-sync-timestamp';
@@ -67,6 +67,7 @@ class Synchronizer {
     remotePullInfoMap = {};
     channelDict = {};
     contextBuilder;
+    fetchFn;
     pushAccessMap = {};
     async startChannel2(context, channel) {
         const { queue, api, selfEncryptInfo, entity, entityId, onFailed, timeout = 5000 } = channel;
@@ -78,12 +79,12 @@ class Synchronizer {
                 seq: ele.$$seq$$,
             }))), 'txnId:', context.getCurrentTxnId());
         }
-        const finalApi = (0, path_1.join)(api, selfEncryptInfo.id);
+        const finalApi = (0, domain_1.urlJoin)(api, selfEncryptInfo.id);
         channel.queue = [];
         try {
             const body = JSON.stringify(opers);
             const { ts, nonce, signature } = await sign(selfEncryptInfo.privateKey, body);
-            const res = await fetchWithTimeout(finalApi, {
+            const res = await this.fetchFn(finalApi, {
                 method: 'post',
                 headers: {
                     'Content-Type': 'application/json',
@@ -190,7 +191,7 @@ class Synchronizer {
         if (!this.channelDict[userId]) {
             // channel上缓存这些信息，暂不支持动态更新
             this.channelDict[userId] = {
-                api: (0, path_1.join)(url, 'endpoint', endpoint),
+                api: (0, domain_1.urlJoin)(url, 'endpoint', endpoint),
                 queue: [],
                 entity: remoteEntity,
                 entityId: remoteEntityId,
@@ -205,7 +206,7 @@ class Synchronizer {
             (0, assert_1.default)(this.channelDict[userId].onFailed === onFailed);
         }
         const channel = this.channelDict[userId];
-        (0, assert_1.default)(channel.api === (0, path_1.join)(url, 'endpoint', endpoint));
+        (0, assert_1.default)(channel.api === (0, domain_1.urlJoin)(url, 'endpoint', endpoint));
         (0, assert_1.default)(channel.entity === remoteEntity);
         (0, assert_1.default)(channel.entityId === remoteEntityId);
         if (channel.queue.find(ele => ele.oper.id === oper.id)) {
@@ -343,8 +344,19 @@ class Synchronizer {
             }, { dontCollect: true, forUpdate: true });
             dirtyOpers = dirtyOpers.filter(ele => !!ele[types_1.TriggerUuidAttribute]);
             if (dirtyOpers.length > 0) {
+                // 检查所有 channel 的 queue 是否已清空
                 for (const c in this.channelDict) {
-                    (0, assert_1.default)(this.channelDict[c].queue.length === 0);
+                    // 在生产环境不使用assert，而是清空队列，以防止脏数据堆积影响后续同步
+                    if (this.channelDict[c].queue.length > 0) {
+                        if (process.env.NODE_ENV !== 'production') {
+                            // 构建详细的错误信息
+                            const queuedOperIds = this.channelDict[c].queue.map(q => q.oper.id).join(', ');
+                            (0, assert_1.default)(false, `发现 channel ${c} 的 queue 未清空，包含 oper IDs: [${queuedOperIds}]`);
+                        }
+                        else {
+                            this.channelDict[c].queue = [];
+                        }
+                    }
                 }
                 const pushedIds = [];
                 const unPushedIds = [];
@@ -394,15 +406,30 @@ class Synchronizer {
     makeCreateOperTrigger() {
         const { config } = this;
         const { remotes, self } = config;
+        const checkActionsMap = {};
         // 根据remotes定义，建立从entity到需要同步的远端结点信息的Map
         remotes.forEach((remote) => {
             const { getPushInfo, pushEntities: pushEntityDefs, endpoint, pathToUser, relationName: rnRemote, onFailed, timeout } = remote;
             if (pushEntityDefs) {
                 const pushEntities = [];
-                const endpoint2 = (0, path_1.join)(endpoint || 'sync', self.entity);
+                const endpoint2 = (0, domain_1.urlJoin)(endpoint || 'sync', self.entity);
                 for (const def of pushEntityDefs) {
                     const { pathToRemoteEntity, pathToSelfEntity, relationName, recursive, entity, actions, onSynchronized } = def;
                     pushEntities.push(entity);
+                    (0, assert_1.default)(pathToRemoteEntity, 'pushEntityDef必须定义pathToRemoteEntity');
+                    (0, assert_1.default)(pathToSelfEntity, 'pushEntityDef必须定义pathToSelfEntity');
+                    (0, assert_1.default)(actions && actions.length > 0, 'pushEntityDef必须定义actions，并且长度必须大于0');
+                    const checkKey = `${String(entity)}-${pathToRemoteEntity}-${pathToSelfEntity}`;
+                    if (!checkActionsMap[checkKey]) {
+                        checkActionsMap[checkKey] = new Set();
+                    }
+                    const checkActions = checkActionsMap[checkKey];
+                    for (const action of actions) {
+                        if (checkActions.has(action)) {
+                            throw new Error(`pushEntityDef中发现重复的entity/pathToRemoteEntity/pathToSelfEntity/action组合，重复的action是「${action}」，entity是「${String(entity)}」，pathToRemoteEntity是「${pathToRemoteEntity}」，pathToSelfEntity是「${pathToSelfEntity}」`);
+                        }
+                        checkActions.add(action);
+                    }
                     const relationName2 = relationName || rnRemote;
                     const path2 = pathToUser ? `${pathToRemoteEntity}.${pathToUser}` : pathToRemoteEntity;
                     (0, assert_1.default)(!recursive);
@@ -507,10 +534,11 @@ class Synchronizer {
         };
         return createOperTrigger;
     }
-    constructor(config, schema, contextBuilder) {
+    constructor(config, schema, contextBuilder, fetchFn) {
         this.config = config;
         this.schema = schema;
         this.contextBuilder = contextBuilder;
+        this.fetchFn = fetchFn || fetchWithTimeout;
     }
     /**
      * 根据sync的定义，生成对应的 commit triggers

package/lib/cluster/DataSubscriber.d.ts

@@ -1,6 +1,6 @@
 import { EntityDict, OpRecord } from 'oak-domain/lib/types';
 import { EntityDict as BaseEntityDict } from 'oak-domain/lib/base-app-domain';
-import { BackendRuntimeContext } from 'oak-frontend-base/lib/context/BackendRuntimeContext';
+import { BackendRuntimeContext } from 'oak-domain/lib/context/BackendRuntimeContext';
 import { Namespace } from 'socket.io';
 /**
  * 集群行为备忘：

package/lib/index.d.ts

@@ -1,3 +1,4 @@
 export { AppLoader } from './AppLoader';
 export { ClusterAppLoader } from './ClusterAppLoader';
 export * from './cluster/env';
+export * from './upgrade';

package/lib/index.js

@@ -7,3 +7,4 @@ Object.defineProperty(exports, "AppLoader", { enumerable: true, get: function ()
 var ClusterAppLoader_1 = require("./ClusterAppLoader");
 Object.defineProperty(exports, "ClusterAppLoader", { enumerable: true, get: function () { return ClusterAppLoader_1.ClusterAppLoader; } });
 tslib_1.__exportStar(require("./cluster/env"), exports);
+tslib_1.__exportStar(require("./upgrade"), exports);

package/lib/routines/update.d.ts

@@ -0,0 +1,65 @@
+import { EntityDict as TypeEntityDict, StorageSchema } from 'oak-domain/lib/types';
+import { AsyncContext } from 'oak-domain/lib/store/AsyncRowStore';
+import BackendRuntimeContext from 'oak-domain/lib/context/BackendRuntimeContext';
+import { EntityDict as BaseEntityDict } from 'oak-domain/lib/base-app-domain';
+type EntityDict = TypeEntityDict & BaseEntityDict;
+/**
+ * 反向引用描述
+ * 用于记录哪些实体引用了当前实体，便于在删除或更新时维护引用完整性
+ */
+export type ReverseRefDesc<ED extends EntityDict> = {
+    type: 'ref' | 'refs';
+    attrName: string;
+    sourceEntity: keyof ED;
+};
+/**
+ * 更新计划配置
+ * 定义了数据同步的完整策略和生命周期钩子
+ */
+export type UpdatePlan<ED extends EntityDict> = {
+    dontCreateOper?: boolean;
+    plan: {
+        [entityName in keyof ED]: {
+            /**
+             * 是否允许数据唯一性冲突时使用新数据覆盖旧数据（仅data中冲突）
+             * @deprecated ID必须相同但会导致ID检查不通过，弃用
+             */
+            allowDataUniqueReWrite?: boolean;
+            /**
+             * 在插入数据库遇到唯一性冲突时的处理方式（数据库和data冲突）
+             * skip表示跳过插入新数据，会将新数据的反指关系更新到已有数据上
+             * update表示使用新数据更新已有数据，会自动更新现有数据的所有反指关系
+             * error表示抛出错误
+             */
+            onUniqueViolation?: 'skip' | 'update' | 'error';
+            /**
+             * 处理数据库中存在但是数据文件中不存在的数据的方式
+             * skip表示跳过不处理
+             * delete表示做逻辑删除，可能导致插入新数据时id冲突
+             * 设置为physicalDelete可以保证最强的一致性
+             */
+            onOnlyExistingInDb?: 'skip' | 'delete' | 'physicalDelete';
+        };
+    };
+    beforeCheck?: (options: {
+        context: BackendRuntimeContext<ED>;
+        data: {
+            [entityName in keyof ED]?: Array<Partial<ED[entityName]['CreateOperationData']>>;
+        };
+        reverseRefs: {
+            [entityName in keyof ED]?: ReverseRefDesc<ED>[];
+        };
+    }) => Promise<void>;
+    afterUpdate?: (options: {
+        context: BackendRuntimeContext<ED>;
+        data: {
+            [entityName in keyof EntityDict]?: Array<Partial<EntityDict[entityName]['CreateOperationData']>>;
+        };
+        reverseRefs: {
+            [entityName in keyof EntityDict]?: ReverseRefDesc<ED>[];
+        };
+    }) => Promise<void>;
+};
+export declare const createUpdatePlan: <ED extends EntityDict>(options: UpdatePlan<ED>) => <Cxt extends AsyncContext<ED>>(context: Cxt) => Promise<void>;
+export declare function checkPathValue(itemId: string, destEntity: keyof EntityDict, path: string, storageSchema: StorageSchema<EntityDict>): boolean;
+export {};