@nocobase/plugin-user-data-sync 1.4.0-alpha

package/dist/server/models/sync-source.js

@@ -0,0 +1,38 @@
+/**
+ * This file is part of the NocoBase (R) project.
+ * Copyright (c) 2020-2024 NocoBase Co., Ltd.
+ * Authors: NocoBase Team.
+ *
+ * This project is dual-licensed under AGPL-3.0 and NocoBase Commercial License.
+ * For more information, please refer to: https://www.nocobase.com/agreement.
+ */
+
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+var sync_source_exports = {};
+__export(sync_source_exports, {
+  SyncSourceModel: () => SyncSourceModel
+});
+module.exports = __toCommonJS(sync_source_exports);
+var import_database = require("@nocobase/database");
+class SyncSourceModel extends import_database.Model {
+}
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  SyncSourceModel
+});

package/dist/server/plugin.d.ts

@@ -0,0 +1,27 @@
+/**
+ * This file is part of the NocoBase (R) project.
+ * Copyright (c) 2020-2024 NocoBase Co., Ltd.
+ * Authors: NocoBase Team.
+ *
+ * This project is dual-licensed under AGPL-3.0 and NocoBase Commercial License.
+ * For more information, please refer to: https://www.nocobase.com/agreement.
+ */
+import { Plugin } from '@nocobase/server';
+import { UserDataResourceManager } from './user-data-resource-manager';
+import { UserDataSyncService } from './user-data-sync-service';
+import { SyncSourceManager } from './sync-source-manager';
+import { Logger } from '@nocobase/logger';
+export declare class PluginUserDataSyncServer extends Plugin {
+    sourceManager: SyncSourceManager;
+    resourceManager: UserDataResourceManager;
+    syncService: UserDataSyncService;
+    afterAdd(): Promise<void>;
+    beforeLoad(): Promise<void>;
+    getLogger(): Logger;
+    load(): Promise<void>;
+    install(): Promise<void>;
+    afterEnable(): Promise<void>;
+    afterDisable(): Promise<void>;
+    remove(): Promise<void>;
+}
+export default PluginUserDataSyncServer;

package/dist/server/plugin.js

@@ -0,0 +1,100 @@
+/**
+ * This file is part of the NocoBase (R) project.
+ * Copyright (c) 2020-2024 NocoBase Co., Ltd.
+ * Authors: NocoBase Team.
+ *
+ * This project is dual-licensed under AGPL-3.0 and NocoBase Commercial License.
+ * For more information, please refer to: https://www.nocobase.com/agreement.
+ */
+
+var __create = Object.create;
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __getProtoOf = Object.getPrototypeOf;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
+  // If the importer is in node compatibility mode or this is not an ESM
+  // file that has been converted to a CommonJS file using a Babel-
+  // compatible transform (i.e. "__esModule" has not been set), then set
+  // "default" to the CommonJS "module.exports" for node compatibility.
+  isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
+  mod
+));
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+var plugin_exports = {};
+__export(plugin_exports, {
+  PluginUserDataSyncServer: () => PluginUserDataSyncServer,
+  default: () => plugin_default
+});
+module.exports = __toCommonJS(plugin_exports);
+var import_server = require("@nocobase/server");
+var import_user_data_resource_manager = require("./user-data-resource-manager");
+var import_user_data_sync_service = require("./user-data-sync-service");
+var import_user_data = __toESM(require("./actions/user-data"));
+var import_sync_source_manager = require("./sync-source-manager");
+var import_sync_source = require("./models/sync-source");
+class PluginUserDataSyncServer extends import_server.Plugin {
+  sourceManager;
+  resourceManager;
+  syncService;
+  async afterAdd() {
+  }
+  async beforeLoad() {
+    this.app.db.registerModels({ SyncSourceModel: import_sync_source.SyncSourceModel });
+    this.sourceManager = new import_sync_source_manager.SyncSourceManager();
+    this.resourceManager = new import_user_data_resource_manager.UserDataResourceManager();
+  }
+  getLogger() {
+    const logger = this.createLogger({
+      dirname: "user-data-sync",
+      filename: "%DATE%.log",
+      format: "json"
+    });
+    return logger;
+  }
+  async load() {
+    const logger = this.getLogger();
+    this.resourceManager.db = this.app.db;
+    this.resourceManager.logger = this.app.logger;
+    this.syncService = new import_user_data_sync_service.UserDataSyncService(this.resourceManager, this.sourceManager, logger);
+    this.app.resourceManager.define({
+      name: "userData",
+      actions: {
+        listSyncTypes: import_user_data.default.listSyncTypes,
+        pull: import_user_data.default.pull,
+        push: import_user_data.default.push,
+        retry: import_user_data.default.retry
+      }
+    });
+    this.app.acl.registerSnippet({
+      name: `pm.${this.name}`,
+      actions: ["userData:*", "userDataSyncSources:*"]
+    });
+  }
+  async install() {
+  }
+  async afterEnable() {
+  }
+  async afterDisable() {
+  }
+  async remove() {
+  }
+}
+var plugin_default = PluginUserDataSyncServer;
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  PluginUserDataSyncServer
+});

package/dist/server/sync-source-manager.d.ts

@@ -0,0 +1,28 @@
+/**
+ * This file is part of the NocoBase (R) project.
+ * Copyright (c) 2020-2024 NocoBase Co., Ltd.
+ * Authors: NocoBase Team.
+ *
+ * This project is dual-licensed under AGPL-3.0 and NocoBase Commercial License.
+ * For more information, please refer to: https://www.nocobase.com/agreement.
+ */
+import { Registry } from '@nocobase/utils';
+import { SyncSource, SyncSourceExtend } from './sync-source';
+import { Context } from '@nocobase/actions';
+import { SyncSourceModel } from './models/sync-source';
+type SyncSourceConfig = {
+    syncSource: SyncSourceExtend<SyncSource>;
+    title?: string;
+};
+export declare class SyncSourceManager {
+    protected syncSourceTypes: Registry<SyncSourceConfig>;
+    registerType(syncSourceType: string, syncSourceConfig: SyncSourceConfig): void;
+    listTypes(): {
+        name: string;
+        title: string;
+    }[];
+    getByName(name: string, ctx: Context): Promise<SyncSource>;
+    getById(id: number, ctx: Context): Promise<SyncSource>;
+    create(sourceInstance: SyncSourceModel, ctx: Context): SyncSource;
+}
+export {};

package/dist/server/sync-source-manager.js

@@ -0,0 +1,71 @@
+/**
+ * This file is part of the NocoBase (R) project.
+ * Copyright (c) 2020-2024 NocoBase Co., Ltd.
+ * Authors: NocoBase Team.
+ *
+ * This project is dual-licensed under AGPL-3.0 and NocoBase Commercial License.
+ * For more information, please refer to: https://www.nocobase.com/agreement.
+ */
+
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+var sync_source_manager_exports = {};
+__export(sync_source_manager_exports, {
+  SyncSourceManager: () => SyncSourceManager
+});
+module.exports = __toCommonJS(sync_source_manager_exports);
+var import_utils = require("@nocobase/utils");
+class SyncSourceManager {
+  syncSourceTypes = new import_utils.Registry();
+  registerType(syncSourceType, syncSourceConfig) {
+    this.syncSourceTypes.register(syncSourceType, syncSourceConfig);
+  }
+  listTypes() {
+    return Array.from(this.syncSourceTypes.getEntities()).map(([syncSourceType, source]) => ({
+      name: syncSourceType,
+      title: source.title
+    }));
+  }
+  async getByName(name, ctx) {
+    const repo = ctx.db.getRepository("userDataSyncSources");
+    const sourceInstance = await repo.findOne({ filter: { enabled: true, name } });
+    if (!sourceInstance) {
+      throw new Error(`SyncSource [${name}] is not found.`);
+    }
+    return this.create(sourceInstance, ctx);
+  }
+  async getById(id, ctx) {
+    const repo = ctx.db.getRepository("userDataSyncSources");
+    const sourceInstance = await repo.findOne({ filter: { enabled: true }, filterByTk: id });
+    if (!sourceInstance) {
+      throw new Error(`SyncSource [${id}] is not found.`);
+    }
+    return this.create(sourceInstance, ctx);
+  }
+  create(sourceInstance, ctx) {
+    const { syncSource } = this.syncSourceTypes.get(sourceInstance.sourceType) || {};
+    if (!syncSource) {
+      throw new Error(`SyncSourceType [${sourceInstance.sourceType}] is not found.`);
+    }
+    return new syncSource({ sourceInstance, options: sourceInstance.options, ctx });
+  }
+}
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  SyncSourceManager
+});

package/dist/server/sync-source.d.ts

@@ -0,0 +1,42 @@
+/**
+ * This file is part of the NocoBase (R) project.
+ * Copyright (c) 2020-2024 NocoBase Co., Ltd.
+ * Authors: NocoBase Team.
+ *
+ * This project is dual-licensed under AGPL-3.0 and NocoBase Commercial License.
+ * For more information, please refer to: https://www.nocobase.com/agreement.
+ */
+import { Context } from '@nocobase/actions';
+import { SyncSourceModel } from './models/sync-source';
+import { UserData } from './user-data-resource-manager';
+export type SyncSourceConfig = {
+    sourceInstance: SyncSourceModel;
+    options: {
+        [key: string]: any;
+    };
+    ctx: Context;
+};
+interface ISyncSource {
+    pull(): Promise<UserData[]>;
+}
+export declare abstract class SyncSource implements ISyncSource {
+    instance: SyncSourceModel;
+    protected options: {
+        [key: string]: any;
+    };
+    protected ctx: Context;
+    constructor(config: SyncSourceConfig);
+    abstract pull(): Promise<UserData[]>;
+    newTask(): Promise<any>;
+    beginTask(taskId: number): Promise<void>;
+    endTask(params: EndTaskParams): Promise<void>;
+    retryTask(taskId: number): Promise<any>;
+}
+export type SyncSourceExtend<T extends SyncSource> = new (config: SyncSourceConfig) => T;
+type EndTaskParams = {
+    taskId: number;
+    success: boolean;
+    cost?: number;
+    message?: string;
+};
+export {};

package/dist/server/sync-source.js

@@ -0,0 +1,107 @@
+/**
+ * This file is part of the NocoBase (R) project.
+ * Copyright (c) 2020-2024 NocoBase Co., Ltd.
+ * Authors: NocoBase Team.
+ *
+ * This project is dual-licensed under AGPL-3.0 and NocoBase Commercial License.
+ * For more information, please refer to: https://www.nocobase.com/agreement.
+ */
+
+var __create = Object.create;
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __getProtoOf = Object.getPrototypeOf;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
+  // If the importer is in node compatibility mode or this is not an ESM
+  // file that has been converted to a CommonJS file using a Babel-
+  // compatible transform (i.e. "__esModule" has not been set), then set
+  // "default" to the CommonJS "module.exports" for node compatibility.
+  isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
+  mod
+));
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+var sync_source_exports = {};
+__export(sync_source_exports, {
+  SyncSource: () => SyncSource
+});
+module.exports = __toCommonJS(sync_source_exports);
+var import_dayjs = __toESM(require("dayjs"));
+class SyncSource {
+  instance;
+  options;
+  ctx;
+  constructor(config) {
+    const { options, ctx, sourceInstance } = config;
+    this.instance = sourceInstance;
+    this.options = options;
+    this.ctx = ctx;
+  }
+  async newTask() {
+    const batch = generateUniqueNumber();
+    return await this.instance.createTask({ batch, status: "init" });
+  }
+  async beginTask(taskId) {
+    const tasks = await this.instance.getTasks({ where: { id: taskId } });
+    if (!tasks && !tasks.length) {
+      throw new Error(`Task [${taskId}] is not found.`);
+    }
+    const task = tasks[0];
+    if (task.status !== "init") {
+      throw new Error(`Task [${taskId}] is not init.`);
+    }
+    task.status = "processing";
+    await task.save();
+  }
+  async endTask(params) {
+    const { taskId, success, cost, message } = params;
+    const tasks = await this.instance.getTasks({ where: { id: taskId } });
+    if (!tasks && !tasks.length) {
+      throw new Error(`Task [${taskId}] is not found.`);
+    }
+    const task = tasks[0];
+    if (task.status !== "processing") {
+      throw new Error(`Task [${taskId}] is not processing.`);
+    }
+    task.status = success ? "success" : "failed";
+    task.cost = cost;
+    task.message = message;
+    await task.save();
+  }
+  async retryTask(taskId) {
+    const tasks = await this.instance.getTasks({ where: { id: taskId } });
+    if (!tasks && !tasks.length) {
+      throw new Error(`Task [${taskId}] is not found.`);
+    }
+    const task = tasks[0];
+    if (task.status !== "failed") {
+      throw new Error(`Task [${taskId}] is not failed.`);
+    }
+    task.status = "processing";
+    task.message = "";
+    await task.save();
+    return task;
+  }
+}
+function generateUniqueNumber() {
+  const formattedDate = (0, import_dayjs.default)().format("YYYYMMDDHHmmss");
+  const randomDigits = Math.floor(1e5 + Math.random() * 9e5);
+  return formattedDate + randomDigits;
+}
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  SyncSource
+});

package/dist/server/user-data-resource-manager.d.ts

@@ -0,0 +1,108 @@
+/**
+ * This file is part of the NocoBase (R) project.
+ * Copyright (c) 2020-2024 NocoBase Co., Ltd.
+ * Authors: NocoBase Team.
+ *
+ * This project is dual-licensed under AGPL-3.0 and NocoBase Commercial License.
+ * For more information, please refer to: https://www.nocobase.com/agreement.
+ */
+import { Toposort, ToposortOptions } from '@nocobase/utils';
+import Database, { Repository } from '@nocobase/database';
+import { SystemLogger } from '@nocobase/logger';
+export type FormatUser = {
+    uid: string;
+    username?: string;
+    email?: string;
+    nickname?: string;
+    phone?: string;
+    departments?: (string | FormatUserDepartment)[];
+    isDeleted?: boolean;
+    [key: string]: any;
+};
+export type FormatDepartment = {
+    uid: string;
+    title?: string;
+    parentUid?: string;
+    isDeleted?: boolean;
+    [key: string]: any;
+};
+export type FormatUserDepartment = {
+    uid: string;
+    isOwner?: boolean;
+    isMain?: boolean;
+};
+export type UserDataRecord = FormatUser | FormatDepartment;
+export type SyncDataType = 'user' | 'department';
+export type SyncAccept = SyncDataType;
+export type OriginRecord = {
+    id: number;
+    sourceName: string;
+    sourceUk: string;
+    dataType: SyncDataType;
+    metaData: UserDataRecord;
+    resources: {
+        resource: string;
+        resourcePk: string;
+    }[];
+};
+export type UserData = {
+    dataType: SyncDataType;
+    matchKey?: string;
+    records: UserDataRecord[];
+    sourceName: string;
+};
+export type PrimaryKey = number | string;
+export type RecordResourceChanged = {
+    resourcesPk: PrimaryKey;
+    isDeleted: boolean;
+};
+export declare abstract class UserDataResource {
+    name: string;
+    accepts: SyncAccept[];
+    db: Database;
+    logger: SystemLogger;
+    constructor(db: Database, logger: SystemLogger);
+    abstract update(record: OriginRecord, resourcePks: PrimaryKey[], matchKey?: string): Promise<RecordResourceChanged[]>;
+    abstract create(record: OriginRecord, matchKey: string): Promise<RecordResourceChanged[]>;
+    get syncRecordRepo(): Repository<any, any>;
+    get syncRecordResourceRepo(): Repository<any, any>;
+}
+export type SyncResult = {
+    resource: string;
+    detail: {
+        count: {
+            all: number;
+            success: number;
+            failed: number;
+        };
+        failedRecords: {
+            record: UserDataRecord;
+            message: string;
+        }[];
+    };
+};
+export declare class UserDataResourceManager {
+    resources: Toposort<UserDataResource>;
+    syncRecordRepo: Repository;
+    syncRecordResourceRepo: Repository;
+    logger: SystemLogger;
+    registerResource(resource: UserDataResource, options?: ToposortOptions): void;
+    set db(value: Database);
+    saveOriginRecords(data: UserData): Promise<void>;
+    findOriginRecords({ sourceName, dataType, sourceUks }: {
+        sourceName: any;
+        dataType: any;
+        sourceUks: any;
+    }): Promise<OriginRecord[]>;
+    addResourceToOriginRecord({ recordId, resource, resourcePk }: {
+        recordId: any;
+        resource: any;
+        resourcePk: any;
+    }): Promise<void>;
+    removeResourceFromOriginRecord({ recordId, resource, resourcePk }: {
+        recordId: any;
+        resource: any;
+        resourcePk: any;
+    }): Promise<void>;
+    updateOrCreate(data: UserData): Promise<SyncResult[]>;
+}