oak-backend-base 4.1.28 → 5.0.0

package/lib/types/Sync.d.ts

@@ -1,16 +1,16 @@
-import { EntityDict } from 'oak-domain/lib/types';
-import { EntityDict as BaseEntityDict } from 'oak-domain/lib/base-app-domain';
-import { BackendRuntimeContext } from 'oak-frontend-base/lib/context/BackendRuntimeContext';
-import { RemotePushInfo, RemotePullInfo, SelfEncryptInfo, SyncRemoteConfigBase, SyncSelfConfigBase, SyncConfig } from 'oak-domain/lib/types/Sync';
-interface SyncRemoteConfigWrapper<ED extends EntityDict & BaseEntityDict, Cxt extends BackendRuntimeContext<ED>> extends SyncRemoteConfigBase<ED, Cxt> {
-    getRemotePushInfo: (userId: string) => Promise<RemotePushInfo>;
-    getRemotePullInfo: (id: string) => Promise<RemotePullInfo>;
-}
-interface SyncSelfConfigWrapper<ED extends EntityDict & BaseEntityDict> extends SyncSelfConfigBase<ED> {
-    getSelfEncryptInfo: () => Promise<SelfEncryptInfo>;
-}
-export interface SyncConfigWrapper<ED extends EntityDict & BaseEntityDict, Cxt extends BackendRuntimeContext<ED>> {
-    self: SyncSelfConfigWrapper<ED>;
-    remotes: Array<SyncRemoteConfigWrapper<ED, Cxt>>;
-}
-export { RemotePushInfo, RemotePullInfo, SelfEncryptInfo, SyncConfig, };
+import { EntityDict } from 'oak-domain/lib/types';
+import { EntityDict as BaseEntityDict } from 'oak-domain/lib/base-app-domain';
+import { BackendRuntimeContext } from 'oak-domain/lib/context/BackendRuntimeContext';
+import { RemotePushInfo, RemotePullInfo, SelfEncryptInfo, SyncRemoteConfigBase, SyncSelfConfigBase, SyncConfig } from 'oak-domain/lib/types/Sync';
+interface SyncRemoteConfigWrapper<ED extends EntityDict & BaseEntityDict, Cxt extends BackendRuntimeContext<ED>> extends SyncRemoteConfigBase<ED, Cxt> {
+    getRemotePushInfo: (userId: string) => Promise<RemotePushInfo>;
+    getRemotePullInfo: (id: string) => Promise<RemotePullInfo>;
+}
+interface SyncSelfConfigWrapper<ED extends EntityDict & BaseEntityDict> extends SyncSelfConfigBase<ED> {
+    getSelfEncryptInfo: () => Promise<SelfEncryptInfo>;
+}
+export interface SyncConfigWrapper<ED extends EntityDict & BaseEntityDict, Cxt extends BackendRuntimeContext<ED>> {
+    self: SyncSelfConfigWrapper<ED>;
+    remotes: Array<SyncRemoteConfigWrapper<ED, Cxt>>;
+}
+export { RemotePushInfo, RemotePullInfo, SelfEncryptInfo, SyncConfig, };

package/lib/upgrade.d.ts

@@ -0,0 +1,50 @@
+import { MigrationPlanningOptions, Plan } from 'oak-db';
+export interface UpgradeExecutionOptions {
+    execute?: boolean;
+}
+export interface AppLoaderUpgradeOptions extends UpgradeExecutionOptions, MigrationPlanningOptions {
+    outputDir?: string;
+}
+export interface UpgradeOutputFiles {
+    migrationSql: string;
+    rollbackSql: string;
+    summary: string;
+    tableChanges: string;
+    warnings: string;
+    renameCandidates: string;
+}
+export interface UpgradeExecutionSummary {
+    prepareSql: number;
+    forwardSql: number;
+    onlineSql: number;
+    manualSql: number;
+    backwardSql: number;
+    total: number;
+}
+export interface AppLoaderUpgradeResult {
+    plan: Plan;
+    remainingPlan?: Plan;
+    outputDir?: string;
+    files?: UpgradeOutputFiles;
+    executed: UpgradeExecutionSummary;
+}
+type SqlExecutor = {
+    exec(sql: string, txnId?: string): Promise<void>;
+    begin(): Promise<string>;
+    commit(txnId: string): Promise<void>;
+    rollback(txnId: string): Promise<void>;
+    supportsTransactionalDdl(): boolean;
+};
+type SqlCategory = keyof Omit<UpgradeExecutionSummary, 'total'>;
+type OrderedSqlStep = {
+    sql: string;
+    category: SqlCategory;
+    group: number;
+    table?: string;
+    order: number;
+};
+export declare function buildOrderedUpgradeSteps(plan: Plan): OrderedSqlStep[];
+export declare function buildOrderedRollbackSteps(plan: Plan): OrderedSqlStep[];
+export declare function writeUpgradeArtifacts(plan: Plan, outputDir: string, supportsTransactionalDdl?: boolean): Promise<UpgradeOutputFiles>;
+export declare function applyUpgradePlan(executor: SqlExecutor, plan: Plan, options?: UpgradeExecutionOptions): Promise<UpgradeExecutionSummary>;
+export {};

package/lib/upgrade.js

@@ -0,0 +1,262 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.buildOrderedUpgradeSteps = buildOrderedUpgradeSteps;
+exports.buildOrderedRollbackSteps = buildOrderedRollbackSteps;
+exports.writeUpgradeArtifacts = writeUpgradeArtifacts;
+exports.applyUpgradePlan = applyUpgradePlan;
+const promises_1 = require("fs/promises");
+const path_1 = require("path");
+const categoryOrder = [
+    'prepareSql',
+    'forwardSql',
+    'onlineSql',
+    'manualSql',
+    'backwardSql',
+];
+const executableCategoryOrder = [
+    'prepareSql',
+    'forwardSql',
+    'onlineSql',
+    'manualSql',
+];
+function makeExecutionSummary() {
+    return {
+        prepareSql: 0,
+        forwardSql: 0,
+        onlineSql: 0,
+        manualSql: 0,
+        backwardSql: 0,
+        total: 0,
+    };
+}
+function extractTableName(sql) {
+    const patterns = [
+        /alter\s+table\s+[`"]([^`"]+)[`"]/i,
+        /create\s+table(?:\s+if\s+not\s+exists)?\s+[`"]([^`"]+)[`"]/i,
+        /drop\s+table(?:\s+if\s+exists)?\s+[`"]([^`"]+)[`"]/i,
+        /create\s+(?:unique\s+)?index(?:\s+concurrently)?(?:\s+if\s+not\s+exists)?\s+[`"][^`"]+[`"]\s+on\s+[`"]([^`"]+)[`"]/i,
+    ];
+    for (const pattern of patterns) {
+        const match = sql.match(pattern);
+        if (match) {
+            return match[1];
+        }
+    }
+    return undefined;
+}
+function isCreateTableSql(sql) {
+    return /create\s+table/i.test(sql);
+}
+function isDropTableSql(sql) {
+    return /drop\s+table/i.test(sql);
+}
+function isForeignKeyDropSql(sql) {
+    return /drop\s+(?:foreign\s+key|constraint)/i.test(sql);
+}
+function isForeignKeyAddSql(sql) {
+    return /add\s+constraint\s+.*foreign\s+key|add\s+foreign\s+key/i.test(sql);
+}
+function isColumnSql(sql) {
+    return /add\s+column|modify\s+column|alter\s+column|rename\s+column/i.test(sql);
+}
+function isIndexSql(sql) {
+    return /rename\s+index|create\s+(?:unique\s+)?index|add\s+(?:unique\s+|fulltext\s+|spatial\s+)?index|drop\s+index|alter\s+index/i.test(sql);
+}
+function getCategoryPriority(category) {
+    switch (category) {
+        case 'prepareSql':
+            return 0;
+        case 'manualSql':
+            return 1;
+        case 'forwardSql':
+            return 2;
+        case 'onlineSql':
+            return 3;
+        case 'backwardSql':
+            return 4;
+    }
+}
+function classifySqlGroup(sql, category) {
+    if (category === 'prepareSql') {
+        return 0;
+    }
+    if (category === 'manualSql') {
+        // manualSql 往往已经带着 planner 计算好的执行顺序，
+        // 例如 PostgreSQL enum 重建需要 rename/create/alter/drop 保持原样串行。
+        return 0;
+    }
+    if (category === 'backwardSql') {
+        if (isForeignKeyDropSql(sql)) {
+            return 60;
+        }
+        if (isIndexSql(sql)) {
+            return 70;
+        }
+        if (/drop\s+column/i.test(sql)) {
+            return 80;
+        }
+        if (isDropTableSql(sql)) {
+            return 90;
+        }
+        return 85;
+    }
+    if (isCreateTableSql(sql)) {
+        return 10;
+    }
+    if (isForeignKeyDropSql(sql)) {
+        return 15;
+    }
+    if (isColumnSql(sql)) {
+        return 20;
+    }
+    if (isIndexSql(sql) || category === 'onlineSql') {
+        return 30;
+    }
+    if (isForeignKeyAddSql(sql)) {
+        return 40;
+    }
+    return 50;
+}
+function buildOrderedSteps(plan, categories) {
+    let order = 0;
+    const steps = [];
+    for (const category of categories) {
+        for (const sql of plan[category]) {
+            steps.push({
+                sql,
+                category,
+                group: classifySqlGroup(sql, category),
+                table: extractTableName(sql),
+                order: order++,
+            });
+        }
+    }
+    return steps.sort((left, right) => getCategoryPriority(left.category) - getCategoryPriority(right.category)
+        || left.group - right.group
+        || left.order - right.order);
+}
+function buildOrderedUpgradeSteps(plan) {
+    return buildOrderedSteps(plan, executableCategoryOrder);
+}
+function buildOrderedRollbackSteps(plan) {
+    return buildOrderedSteps(plan, ['backwardSql']);
+}
+function toSqlFileContent(sqls) {
+    return sqls.length > 0 ? `${sqls.join('\n')}\n` : '';
+}
+function buildArtifactSqlStatements(steps, supportsTransactionalDdl = false) {
+    const sqls = [];
+    let openedTransaction = false;
+    const flushTransaction = () => {
+        if (!openedTransaction) {
+            return;
+        }
+        sqls.push('COMMIT;');
+        openedTransaction = false;
+    };
+    for (const step of steps) {
+        const transactional = supportsTransactionalDdl
+            && step.category !== 'prepareSql'
+            && step.category !== 'onlineSql';
+        if (transactional) {
+            if (!openedTransaction) {
+                sqls.push('BEGIN;');
+                openedTransaction = true;
+            }
+        }
+        else {
+            flushTransaction();
+        }
+        sqls.push(step.sql);
+    }
+    flushTransaction();
+    return sqls;
+}
+function getTableChanges(plan) {
+    return plan.tableChanges || [];
+}
+function buildTableChangeSummary(plan) {
+    const tableChanges = getTableChanges(plan);
+    return {
+        total: tableChanges.length,
+        create: tableChanges.filter((item) => item.lifecycle === 'create').length,
+        alter: tableChanges.filter((item) => item.lifecycle === 'alter').length,
+        drop: tableChanges.filter((item) => item.lifecycle === 'drop').length,
+    };
+}
+async function writeUpgradeArtifacts(plan, outputDir, supportsTransactionalDdl = false) {
+    await (0, promises_1.mkdir)(outputDir, {
+        recursive: true,
+    });
+    const files = {
+        migrationSql: (0, path_1.join)(outputDir, 'migration.sql'),
+        rollbackSql: (0, path_1.join)(outputDir, 'rollback.sql'),
+        summary: (0, path_1.join)(outputDir, 'summary.json'),
+        tableChanges: (0, path_1.join)(outputDir, 'table-changes.json'),
+        warnings: (0, path_1.join)(outputDir, 'warnings.json'),
+        renameCandidates: (0, path_1.join)(outputDir, 'rename-candidates.json'),
+    };
+    const orderedSql = buildArtifactSqlStatements(buildOrderedUpgradeSteps(plan), supportsTransactionalDdl);
+    const rollbackSql = buildArtifactSqlStatements(buildOrderedRollbackSteps(plan), supportsTransactionalDdl);
+    await Promise.all([
+        (0, promises_1.writeFile)(files.migrationSql, toSqlFileContent(orderedSql), 'utf8'),
+        (0, promises_1.writeFile)(files.rollbackSql, toSqlFileContent(rollbackSql), 'utf8'),
+        (0, promises_1.writeFile)(files.summary, `${JSON.stringify({
+            summary: plan.summary,
+            tables: buildTableChangeSummary(plan),
+            generatedAt: new Date().toISOString(),
+        }, null, 2)}\n`, 'utf8'),
+        (0, promises_1.writeFile)(files.tableChanges, `${JSON.stringify(getTableChanges(plan), null, 2)}\n`, 'utf8'),
+        (0, promises_1.writeFile)(files.warnings, `${JSON.stringify(plan.warnings, null, 2)}\n`, 'utf8'),
+        (0, promises_1.writeFile)(files.renameCandidates, `${JSON.stringify(plan.renameCandidates, null, 2)}\n`, 'utf8'),
+    ]);
+    return files;
+}
+async function executeOrderedSqlSteps(executor, steps, summary) {
+    if (steps.length === 0) {
+        return;
+    }
+    let txnId;
+    const supportsTransactionalDdl = executor.supportsTransactionalDdl();
+    const flushTransaction = async () => {
+        if (!txnId) {
+            return;
+        }
+        await executor.commit(txnId);
+        txnId = undefined;
+    };
+    try {
+        for (const step of steps) {
+            const transactional = supportsTransactionalDdl
+                && step.category !== 'prepareSql'
+                && step.category !== 'onlineSql';
+            if (!transactional) {
+                await flushTransaction();
+                await executor.exec(step.sql);
+            }
+            else {
+                if (!txnId) {
+                    txnId = await executor.begin();
+                }
+                await executor.exec(step.sql, txnId);
+            }
+            summary[step.category] += 1;
+            summary.total += 1;
+        }
+        await flushTransaction();
+    }
+    catch (err) {
+        if (txnId) {
+            await executor.rollback(txnId);
+        }
+        throw err;
+    }
+}
+async function applyUpgradePlan(executor, plan, options = {}) {
+    const summary = makeExecutionSummary();
+    if (!options.execute) {
+        return summary;
+    }
+    await executeOrderedSqlSteps(executor, buildOrderedUpgradeSteps(plan), summary);
+    return summary;
+}

package/lib/utils/dbPriority.d.ts

@@ -2,7 +2,7 @@ import { MysqlStore, PostgreSQLStore } from "oak-db";
 import { DbConfiguration } from "oak-db/src/types/configuration";
 import { EntityDict, StorageSchema } from 'oak-domain/lib/types';
 import { EntityDict as BaseEntityDict } from 'oak-domain/lib/base-app-domain';
-import { BackendRuntimeContext } from 'oak-frontend-base/lib/context/BackendRuntimeContext';
+import { BackendRuntimeContext } from 'oak-domain/lib/context/BackendRuntimeContext';
 import { CascadeStore } from "oak-domain/lib/store/CascadeStore";
 import { DbStore } from "oak-db/lib/types/dbStore";
 /**

package/lib/utils/dbPriority.js

@@ -38,7 +38,7 @@ const getDbConfig = (path) => {
             // do nothing
         }
     }
-    throw new Error(`没有找到数据库配置文件，请在configuration目录下添加任一配置文件：${Object.keys(exports.dbList).map(ele => `${ele}.json`).join('、')}`);
+    throw new Error(`没有找到数据库配置文件，请在configuration目录下添加任一配置文件：${Object.keys(exports.dbList).map(ele => `${ele}.json`).join('、')}, 当前环境: ${process.env.NODE_ENV}`);
 };
 exports.getDbConfig = getDbConfig;
 const getDbStoreClass = (config) => {

package/lib/utils/initializeData.d.ts

@@ -0,0 +1,5 @@
+import { EntityDict, StorageSchema } from 'oak-domain/lib/types';
+type SerializedData = Record<string, unknown[]>;
+export declare function getInitializationOrder<ED extends EntityDict>(data: SerializedData, schema: StorageSchema<ED>): string[];
+export declare function chunkRows<T>(rows: T[], size?: number): Generator<T[], void, unknown>;
+export {};

package/lib/utils/initializeData.js

@@ -0,0 +1,61 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.getInitializationOrder = getInitializationOrder;
+exports.chunkRows = chunkRows;
+function collectEntityDependencies(entity, schema, entitySet) {
+    const desc = schema[entity];
+    if (!desc) {
+        return new Set();
+    }
+    const dependencies = new Set();
+    for (const attr of Object.values(desc.attributes || {})) {
+        if (attr.type !== 'ref' || !attr.ref) {
+            continue;
+        }
+        const refs = Array.isArray(attr.ref) ? attr.ref : [attr.ref];
+        for (const ref of refs) {
+            if (ref !== entity && entitySet.has(ref)) {
+                dependencies.add(ref);
+            }
+        }
+    }
+    return dependencies;
+}
+function getInitializationOrder(data, schema) {
+    const entities = Object.keys(data).filter((entity) => Array.isArray(data[entity]));
+    const entitySet = new Set(entities);
+    const dependencies = Object.fromEntries(entities.map((entity) => [
+        entity,
+        collectEntityDependencies(entity, schema, entitySet),
+    ]));
+    const ordered = [];
+    const initialized = new Set();
+    const pending = new Set(entities);
+    while (pending.size > 0) {
+        let progressed = false;
+        for (const entity of entities) {
+            if (!pending.has(entity)) {
+                continue;
+            }
+            const deps = dependencies[entity];
+            if ([...deps].every((dep) => initialized.has(dep))) {
+                ordered.push(entity);
+                initialized.add(entity);
+                pending.delete(entity);
+                progressed = true;
+            }
+        }
+        if (!progressed) {
+            const rest = entities.filter((entity) => pending.has(entity));
+            console.warn(`data initialization order has circular or unresolved dependencies, fallback to original order: ${rest.join(', ')}`);
+            ordered.push(...rest);
+            break;
+        }
+    }
+    return ordered;
+}
+function* chunkRows(rows, size = 1000) {
+    for (let idx = 0; idx < rows.length; idx += size) {
+        yield rows.slice(idx, idx + size);
+    }
+}

package/package.json

@@ -1,7 +1,10 @@
 {
     "name": "oak-backend-base",
-    "version": "4.1.28",
+    "version": "5.0.0",
     "description": "oak-backend-base",
+    "oak": {
+        "package": true
+    },
     "main": "lib/index",
     "author": {
         "name": "XuChang"
@@ -13,7 +16,10 @@
         "copy-files": "copyfiles -u 1 src/**/*.json lib/",
         "test": "ts-node test/test.ts",
         "test2": "ts-node test/testDbStore.ts",
-        "build": "tsc && npm run copy-files"
+        "test:sync": "vitest run",
+        "test:sync:ui": "vitest --ui",
+        "build": "node ./scripts/build.js && npm run copy-files",
+        "make:test:domain": "ts-node scripts/makeTestDomain.ts"
     },
     "dependencies": {
         "@types/koa": "^2.15.0",
@@ -21,10 +27,9 @@
         "mysql": "^2.18.1",
         "mysql2": "^2.3.3",
         "node-schedule": "^2.1.0",
-        "oak-common-aspect": "^3.0.5",
-        "oak-db": "^3.3.13",
-        "oak-domain": "^5.1.35",
-        "oak-frontend-base": "^5.3.45",
+        "oak-common-aspect": "^4.0.0",
+        "oak-db": "^4.0.1",
+        "oak-domain": "^6.0.0",
         "socket.io": "^4.8.1",
         "socket.io-client": "^4.7.2",
         "uuid": "^8.3.2"
@@ -35,9 +40,10 @@
         "@types/node": "^20.6.0",
         "@types/node-schedule": "^2.1.0",
         "@types/uuid": "^8.3.4",
+        "@vitest/ui": "^4.0.18",
         "copyfiles": "^2.4.1",
-        "ts-node": "^10.9.1",
         "tslib": "^2.4.0",
-        "typescript": "^5.2.2"
+        "typescript": "^5.2.2",
+        "vitest": "^4.0.18"
     }
 }