@nocobase/plugin-user-data-sync 1.4.0-alpha

package/dist/server/user-data-resource-manager.js

@@ -0,0 +1,214 @@
+/**
+ * This file is part of the NocoBase (R) project.
+ * Copyright (c) 2020-2024 NocoBase Co., Ltd.
+ * Authors: NocoBase Team.
+ *
+ * This project is dual-licensed under AGPL-3.0 and NocoBase Commercial License.
+ * For more information, please refer to: https://www.nocobase.com/agreement.
+ */
+
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+var user_data_resource_manager_exports = {};
+__export(user_data_resource_manager_exports, {
+  UserDataResource: () => UserDataResource,
+  UserDataResourceManager: () => UserDataResourceManager
+});
+module.exports = __toCommonJS(user_data_resource_manager_exports);
+var import_utils = require("@nocobase/utils");
+class UserDataResource {
+  name;
+  accepts;
+  db;
+  logger;
+  constructor(db, logger) {
+    this.db = db;
+    this.logger = logger;
+  }
+  get syncRecordRepo() {
+    return this.db.getRepository("userDataSyncRecords");
+  }
+  get syncRecordResourceRepo() {
+    return this.db.getRepository("userDataSyncRecordsResources");
+  }
+}
+class UserDataResourceManager {
+  resources = new import_utils.Toposort();
+  syncRecordRepo;
+  syncRecordResourceRepo;
+  logger;
+  registerResource(resource, options) {
+    if (!resource.name) {
+      throw new Error('"name" for user data synchronize resource is required');
+    }
+    if (!resource.accepts) {
+      throw new Error('"accepts" for user data synchronize resource is required');
+    }
+    this.resources.add(resource, { tag: resource.name, ...options });
+  }
+  set db(value) {
+    this.syncRecordRepo = value.getRepository("userDataSyncRecords");
+    this.syncRecordResourceRepo = value.getRepository("userDataSyncRecordsResources");
+  }
+  async saveOriginRecords(data) {
+    for (const record of data.records) {
+      if (record.uid === void 0) {
+        throw new Error(`record must has uid, error record: ${JSON.stringify(record)}`);
+      }
+      const syncRecord = await this.syncRecordRepo.findOne({
+        where: {
+          sourceName: data.sourceName,
+          sourceUk: record.uid,
+          dataType: data.dataType
+        }
+      });
+      if (syncRecord) {
+        syncRecord.lastMetaData = syncRecord.metaData;
+        syncRecord.metaData = record;
+        await syncRecord.save();
+      } else {
+        await this.syncRecordRepo.create({
+          values: {
+            sourceName: data.sourceName,
+            sourceUk: record.uid,
+            dataType: data.dataType,
+            metaData: record
+          }
+        });
+      }
+    }
+  }
+  async findOriginRecords({ sourceName, dataType, sourceUks }) {
+    return await this.syncRecordRepo.find({
+      appends: ["resources"],
+      filter: { sourceName, dataType, sourceUk: { $in: sourceUks } }
+    });
+  }
+  async addResourceToOriginRecord({ recordId, resource, resourcePk }) {
+    const syncRecord = await this.syncRecordRepo.findOne({
+      filter: {
+        id: recordId
+      }
+    });
+    if (syncRecord) {
+      await syncRecord.createResource({
+        resource,
+        resourcePk
+      });
+    }
+  }
+  async removeResourceFromOriginRecord({ recordId, resource, resourcePk }) {
+    const recordResource = await this.syncRecordResourceRepo.findOne({
+      where: {
+        recordId,
+        resource,
+        resourcePk
+      }
+    });
+    if (recordResource) {
+      await recordResource.destroy();
+    }
+  }
+  async updateOrCreate(data) {
+    var _a, _b, _c, _d, _e;
+    await this.saveOriginRecords(data);
+    const { dataType, sourceName, records, matchKey } = data;
+    const sourceUks = records.map((record) => record.uid);
+    let processed = false;
+    const syncResults = [];
+    for (const resource of this.resources.nodes) {
+      if (!resource.accepts.includes(dataType)) {
+        continue;
+      }
+      const associateResource = resource.name;
+      processed = true;
+      const originRecords = await this.findOriginRecords({ sourceName, sourceUks, dataType });
+      if (!(originRecords && originRecords.length)) {
+        continue;
+      }
+      const successRecords = [];
+      const failedRecords = [];
+      for (const originRecord of originRecords) {
+        const resourceRecords = (_a = originRecord.resources) == null ? void 0 : _a.filter(
+          (r) => r.resource === associateResource
+        );
+        let recordResourceChangeds;
+        if (resourceRecords && resourceRecords.length > 0) {
+          const resourcePks = resourceRecords.map((r) => r.resourcePk);
+          try {
+            recordResourceChangeds = await resource.update(originRecord, resourcePks, matchKey);
+            (_b = this.logger) == null ? void 0 : _b.debug(`update record success. Data changed: ${JSON.stringify(recordResourceChangeds)}`);
+            successRecords.push(originRecord.metaData);
+          } catch (error) {
+            (_c = this.logger) == null ? void 0 : _c.warn(`update record error: ${error.message}`, { originRecord });
+            failedRecords.push({ record: originRecord.metaData, message: error.message });
+            continue;
+          }
+        } else {
+          try {
+            recordResourceChangeds = await resource.create(originRecord, matchKey);
+            (_d = this.logger) == null ? void 0 : _d.debug(`create record success. Data changed: ${JSON.stringify(recordResourceChangeds)}`);
+            successRecords.push(originRecord.metaData);
+          } catch (error) {
+            (_e = this.logger) == null ? void 0 : _e.warn(`create record error: ${error.message}`, { originRecord });
+            failedRecords.push({ record: originRecord.metaData, message: error.message });
+            continue;
+          }
+        }
+        if (!recordResourceChangeds || recordResourceChangeds.length === 0) {
+          continue;
+        }
+        for (const { resourcesPk, isDeleted } of recordResourceChangeds) {
+          if (isDeleted) {
+            await this.removeResourceFromOriginRecord({
+              recordId: originRecord.id,
+              resource: associateResource,
+              resourcePk: resourcesPk
+            });
+          } else {
+            await this.addResourceToOriginRecord({
+              recordId: originRecord.id,
+              resource: associateResource,
+              resourcePk: resourcesPk
+            });
+          }
+        }
+      }
+      syncResults.push({
+        resource: associateResource,
+        detail: {
+          count: {
+            all: originRecords.length,
+            success: successRecords.length,
+            failed: failedRecords.length
+          },
+          failedRecords
+        }
+      });
+    }
+    if (!processed) {
+      throw new Error(`dataType "${dataType}" is not support`);
+    }
+    return syncResults;
+  }
+}
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  UserDataResource,
+  UserDataResourceManager
+});

package/dist/server/user-data-sync-service.d.ts

@@ -0,0 +1,23 @@
+/**
+ * This file is part of the NocoBase (R) project.
+ * Copyright (c) 2020-2024 NocoBase Co., Ltd.
+ * Authors: NocoBase Team.
+ *
+ * This project is dual-licensed under AGPL-3.0 and NocoBase Commercial License.
+ * For more information, please refer to: https://www.nocobase.com/agreement.
+ */
+import { SyncResult, UserDataResourceManager } from './user-data-resource-manager';
+import { SyncSourceManager } from './sync-source-manager';
+import { Context } from '@nocobase/actions';
+import { SyncSource } from './sync-source';
+import { Logger } from '@nocobase/logger';
+export declare class UserDataSyncService {
+    resourceManager: UserDataResourceManager;
+    sourceManager: SyncSourceManager;
+    logger: Logger;
+    constructor(resourceManager: UserDataResourceManager, sourceManager: SyncSourceManager, logger: Logger);
+    pull(sourceName: string, ctx: Context): Promise<void>;
+    push(data: any): Promise<SyncResult[]>;
+    retry(sourceId: number, taskId: number, ctx: Context): Promise<void>;
+    runSync(source: SyncSource, task: any, ctx: Context): Promise<void>;
+}

package/dist/server/user-data-sync-service.js

@@ -0,0 +1,130 @@
+/**
+ * This file is part of the NocoBase (R) project.
+ * Copyright (c) 2020-2024 NocoBase Co., Ltd.
+ * Authors: NocoBase Team.
+ *
+ * This project is dual-licensed under AGPL-3.0 and NocoBase Commercial License.
+ * For more information, please refer to: https://www.nocobase.com/agreement.
+ */
+
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+var user_data_sync_service_exports = {};
+__export(user_data_sync_service_exports, {
+  UserDataSyncService: () => UserDataSyncService
+});
+module.exports = __toCommonJS(user_data_sync_service_exports);
+var import_error = require("./error");
+class UserDataSyncService {
+  resourceManager;
+  sourceManager;
+  logger;
+  constructor(resourceManager, sourceManager, logger) {
+    this.resourceManager = resourceManager;
+    this.sourceManager = sourceManager;
+    this.logger = logger;
+  }
+  async pull(sourceName, ctx) {
+    const source = await this.sourceManager.getByName(sourceName, ctx);
+    const task = await source.newTask();
+    await source.beginTask(task.id);
+    ctx.log.info("begin sync task of source", { source: sourceName, sourceType: source.instance.sourceType });
+    this.runSync(source, task, ctx);
+  }
+  async push(data) {
+    const { dataType, records } = data;
+    if (dataType === void 0) {
+      throw new Error("dataType for user data synchronize is required");
+    }
+    if (dataType !== "user" && dataType !== "department") {
+      throw new Error("dataType must be user or department");
+    }
+    if (records === void 0) {
+      throw new Error("records for user data synchronize is required");
+    }
+    if (records.length === 0) {
+      throw new Error("records must have at least one piece of data");
+    }
+    const userData = {
+      dataType: data.dataType,
+      matchKey: data.matchKey,
+      records: data.records,
+      sourceName: data.sourceName ? data.sourceName : "api"
+    };
+    this.logger.info({
+      source: data.sourceName ? data.sourceName : "api",
+      sourceType: "api",
+      data
+    });
+    return await this.resourceManager.updateOrCreate(userData);
+  }
+  async retry(sourceId, taskId, ctx) {
+    const source = await this.sourceManager.getById(sourceId, ctx);
+    const task = await source.retryTask(taskId);
+    ctx.log.info("retry sync task of source", {
+      source: source.instance.name,
+      sourceType: source.instance.name,
+      task: task.id
+    });
+    this.runSync(source, task, ctx);
+  }
+  async runSync(source, task, ctx) {
+    const currentTimeMillis = (/* @__PURE__ */ new Date()).getTime();
+    try {
+      ctx.log.info("begin pull data of source", {
+        source: source.instance.name,
+        sourceType: source.instance.sourceType
+      });
+      const data = await source.pull();
+      this.logger.info({
+        source: source.instance.name,
+        sourceType: source.instance.sourceType,
+        batch: task.batch,
+        data
+      });
+      ctx.log.info("end pull data of source", { source: source.instance.name, sourceType: source.instance.sourceType });
+      ctx.log.info("begin update data of source", {
+        source: source.instance.name,
+        sourceType: source.instance.sourceType
+      });
+      for (const item of data) {
+        await this.resourceManager.updateOrCreate(item);
+      }
+      ctx.log.info("end update data of source", {
+        source: source.instance.name,
+        sourceType: source.instance.sourceType
+      });
+      const costTime = (/* @__PURE__ */ new Date()).getTime() - currentTimeMillis;
+      await source.endTask({ taskId: task.id, success: true, cost: costTime });
+    } catch (err) {
+      ctx.log.error(
+        `sync task of source: ${source.instance.name} sourceType: ${source.instance.sourceType} error: ${err.message}`,
+        { method: "runSync", err: err.stack, cause: err.cause }
+      );
+      let message = err.message;
+      if (err instanceof import_error.ExternalAPIError) {
+        message = "The sync source API call failed. Please check the logs to troubleshoot the issue.";
+      }
+      await source.endTask({ taskId: task.id, success: false, message });
+    }
+  }
+}
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  UserDataSyncService
+});

package/dist/swagger/index.d.ts

@@ -0,0 +1,116 @@
+/**
+ * This file is part of the NocoBase (R) project.
+ * Copyright (c) 2020-2024 NocoBase Co., Ltd.
+ * Authors: NocoBase Team.
+ *
+ * This project is dual-licensed under AGPL-3.0 and NocoBase Commercial License.
+ * For more information, please refer to: https://www.nocobase.com/agreement.
+ */
+declare const _default: {
+    info: {
+        title: string;
+    };
+    paths: {
+        '/userData:push': {
+            post: {
+                description: string;
+                tags: string[];
+                security: any[];
+                requestBody: {
+                    content: {
+                        'application/json': {
+                            schema: {
+                                type: string;
+                                items: {
+                                    $ref: string;
+                                };
+                            };
+                        };
+                    };
+                };
+                responses: {
+                    200: {
+                        description: string;
+                    };
+                };
+            };
+        };
+    };
+    components: {
+        schemas: {
+            userData: {
+                type: string;
+                description: string;
+                properties: {
+                    dataType: {
+                        type: string;
+                        description: string;
+                    };
+                    uniqueKey: {
+                        type: string;
+                        description: string;
+                    };
+                    records: {
+                        type: string;
+                        description: string;
+                        items: {
+                            type: string;
+                        };
+                    };
+                    sourceName: {
+                        type: string;
+                        description: string;
+                    };
+                };
+            };
+            user: {
+                type: string;
+                description: string;
+                properties: {
+                    id: {
+                        type: string;
+                        description: string;
+                    };
+                    nickname: {
+                        type: string;
+                        description: string;
+                    };
+                    email: {
+                        type: string;
+                        description: string;
+                    };
+                    phone: {
+                        type: string;
+                        description: string;
+                    };
+                    departments: {
+                        type: string;
+                        description: string;
+                        items: {
+                            type: string;
+                        };
+                    };
+                };
+            };
+            department: {
+                type: string;
+                description: string;
+                properties: {
+                    id: {
+                        type: string;
+                        description: string;
+                    };
+                    name: {
+                        type: string;
+                        description: string;
+                    };
+                    parentId: {
+                        type: string;
+                        description: string;
+                    };
+                };
+            };
+        };
+    };
+};
+export default _default;

package/dist/swagger/index.js

@@ -0,0 +1,138 @@
+/**
+ * This file is part of the NocoBase (R) project.
+ * Copyright (c) 2020-2024 NocoBase Co., Ltd.
+ * Authors: NocoBase Team.
+ *
+ * This project is dual-licensed under AGPL-3.0 and NocoBase Commercial License.
+ * For more information, please refer to: https://www.nocobase.com/agreement.
+ */
+
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+var swagger_exports = {};
+__export(swagger_exports, {
+  default: () => swagger_default
+});
+module.exports = __toCommonJS(swagger_exports);
+var swagger_default = {
+  info: {
+    title: "NocoBase API - User data synchronization plugin"
+  },
+  paths: {
+    "/userData:push": {
+      post: {
+        description: "Push user data",
+        tags: ["Push"],
+        security: [],
+        requestBody: {
+          content: {
+            "application/json": {
+              schema: {
+                type: "array",
+                items: {
+                  $ref: "#/components/schemas/userData"
+                }
+              }
+            }
+          }
+        },
+        responses: {
+          200: {
+            description: "ok"
+          }
+        }
+      }
+    }
+  },
+  components: {
+    schemas: {
+      userData: {
+        type: "object",
+        description: "\u7528\u6237\u6570\u636E",
+        properties: {
+          dataType: {
+            type: "string",
+            description: "\u6570\u636E\u7C7B\u578B, \u76EE\u524D\u53EF\u9009\u503C\u4E3A: user, department"
+          },
+          uniqueKey: {
+            type: "string",
+            description: "\u552F\u4E00\u952E"
+          },
+          records: {
+            type: "array",
+            description: "\u6570\u636E, \u82E5 dataType \u4E3A user, \u5219\u4E3A\u7528\u6237\u6570\u636E\u5B57\u6BB5\u89C1schemas/user, \u82E5 dataType \u4E3A department, \u5219\u4E3A\u90E8\u95E8\u6570\u636E\u5B57\u6BB5\u89C1schemas/department",
+            items: {
+              type: "object"
+            }
+          },
+          sourceName: {
+            type: "string",
+            description: "\u6570\u636E\u6E90\u540D\u79F0"
+          }
+        }
+      },
+      user: {
+        type: "object",
+        description: "\u7528\u6237",
+        properties: {
+          id: {
+            type: "integer",
+            description: "ID"
+          },
+          nickname: {
+            type: "string",
+            description: "\u6635\u79F0"
+          },
+          email: {
+            type: "string",
+            description: "\u90AE\u7BB1"
+          },
+          phone: {
+            type: "string",
+            description: "\u624B\u673A\u53F7"
+          },
+          departments: {
+            type: "array",
+            description: "\u6240\u5C5E\u90E8\u95E8, \u90E8\u95E8ID \u6570\u7EC4",
+            items: {
+              type: "string"
+            }
+          }
+        }
+      },
+      department: {
+        type: "object",
+        description: "\u90E8\u95E8",
+        properties: {
+          id: {
+            type: "string",
+            description: "ID"
+          },
+          name: {
+            type: "string",
+            description: "\u540D\u79F0"
+          },
+          parentId: {
+            type: "string",
+            description: "\u7236\u7EA7\u90E8\u95E8ID"
+          }
+        }
+      }
+    }
+  }
+};

package/package.json

@@ -0,0 +1,18 @@
+{
+  "name": "@nocobase/plugin-user-data-sync",
+  "displayName": "User data synchronization",
+  "displayName.zh-CN": "用户数据同步",
+  "description": "Provide user data source management and user data synchronization interface. The data source can be DingTalk, WeCom, etc., and can be expanded.",
+  "description.zh-CN": "提供用户数据源管理，用户数据同步接口，数据源可为钉钉、企业微信等，可扩展。",
+  "version": "1.4.0-alpha",
+  "main": "dist/server/index.js",
+  "peerDependencies": {
+    "@nocobase/client": "1.x",
+    "@nocobase/server": "1.x",
+    "@nocobase/test": "1.x"
+  },
+  "keywords": [
+    "Users & permissions"
+  ],
+  "gitHead": "f097a2bddec152522b5645bd5d451f4c866d2060"
+}

package/server.d.ts

@@ -0,0 +1,2 @@
+export * from './dist/server';
+export { default } from './dist/server';

package/server.js

@@ -0,0 +1 @@
+module.exports = require('./dist/server/index.js');