@nocobase/plugin-user-data-sync 1.4.0-alpha.20240826214041

package/dist/server/sync-source-manager.d.ts

@@ -0,0 +1,28 @@
+/**
+ * This file is part of the NocoBase (R) project.
+ * Copyright (c) 2020-2024 NocoBase Co., Ltd.
+ * Authors: NocoBase Team.
+ *
+ * This project is dual-licensed under AGPL-3.0 and NocoBase Commercial License.
+ * For more information, please refer to: https://www.nocobase.com/agreement.
+ */
+import { Registry } from '@nocobase/utils';
+import { SyncSource, SyncSourceExtend } from './sync-source';
+import { Context } from '@nocobase/actions';
+import { SyncSourceModel } from './models/sync-source';
+type SyncSourceConfig = {
+    syncSource: SyncSourceExtend<SyncSource>;
+    title?: string;
+};
+export declare class SyncSourceManager {
+    protected syncSourceTypes: Registry<SyncSourceConfig>;
+    registerType(syncSourceType: string, syncSourceConfig: SyncSourceConfig): void;
+    listTypes(): {
+        name: string;
+        title: string;
+    }[];
+    getByName(name: string, ctx: Context): Promise<SyncSource>;
+    getById(id: number, ctx: Context): Promise<SyncSource>;
+    create(sourceInstance: SyncSourceModel, ctx: Context): SyncSource;
+}
+export {};

package/dist/server/sync-source-manager.js

@@ -0,0 +1,71 @@
+/**
+ * This file is part of the NocoBase (R) project.
+ * Copyright (c) 2020-2024 NocoBase Co., Ltd.
+ * Authors: NocoBase Team.
+ *
+ * This project is dual-licensed under AGPL-3.0 and NocoBase Commercial License.
+ * For more information, please refer to: https://www.nocobase.com/agreement.
+ */
+
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+var sync_source_manager_exports = {};
+__export(sync_source_manager_exports, {
+  SyncSourceManager: () => SyncSourceManager
+});
+module.exports = __toCommonJS(sync_source_manager_exports);
+var import_utils = require("@nocobase/utils");
+class SyncSourceManager {
+  syncSourceTypes = new import_utils.Registry();
+  registerType(syncSourceType, syncSourceConfig) {
+    this.syncSourceTypes.register(syncSourceType, syncSourceConfig);
+  }
+  listTypes() {
+    return Array.from(this.syncSourceTypes.getEntities()).map(([syncSourceType, source]) => ({
+      name: syncSourceType,
+      title: source.title
+    }));
+  }
+  async getByName(name, ctx) {
+    const repo = ctx.db.getRepository("userDataSyncSources");
+    const sourceInstance = await repo.findOne({ filter: { enabled: true, name } });
+    if (!sourceInstance) {
+      throw new Error(`SyncSource [${name}] is not found.`);
+    }
+    return this.create(sourceInstance, ctx);
+  }
+  async getById(id, ctx) {
+    const repo = ctx.db.getRepository("userDataSyncSources");
+    const sourceInstance = await repo.findOne({ filter: { enabled: true }, filterByTk: id });
+    if (!sourceInstance) {
+      throw new Error(`SyncSource [${id}] is not found.`);
+    }
+    return this.create(sourceInstance, ctx);
+  }
+  create(sourceInstance, ctx) {
+    const { syncSource } = this.syncSourceTypes.get(sourceInstance.sourceType) || {};
+    if (!syncSource) {
+      throw new Error(`SyncSourceType [${sourceInstance.sourceType}] is not found.`);
+    }
+    return new syncSource({ sourceInstance, options: sourceInstance.options, ctx });
+  }
+}
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  SyncSourceManager
+});

package/dist/server/sync-source.d.ts

@@ -0,0 +1,42 @@
+/**
+ * This file is part of the NocoBase (R) project.
+ * Copyright (c) 2020-2024 NocoBase Co., Ltd.
+ * Authors: NocoBase Team.
+ *
+ * This project is dual-licensed under AGPL-3.0 and NocoBase Commercial License.
+ * For more information, please refer to: https://www.nocobase.com/agreement.
+ */
+import { Context } from '@nocobase/actions';
+import { SyncSourceModel } from './models/sync-source';
+import { UserData } from './user-data-resource-manager';
+export type SyncSourceConfig = {
+    sourceInstance: SyncSourceModel;
+    options: {
+        [key: string]: any;
+    };
+    ctx: Context;
+};
+interface ISyncSource {
+    pull(): Promise<UserData[]>;
+}
+export declare abstract class SyncSource implements ISyncSource {
+    instance: SyncSourceModel;
+    protected options: {
+        [key: string]: any;
+    };
+    protected ctx: Context;
+    constructor(config: SyncSourceConfig);
+    abstract pull(): Promise<UserData[]>;
+    newTask(): Promise<any>;
+    beginTask(taskId: number): Promise<void>;
+    endTask(params: EndTaskParams): Promise<void>;
+    retryTask(taskId: number): Promise<any>;
+}
+export type SyncSourceExtend<T extends SyncSource> = new (config: SyncSourceConfig) => T;
+type EndTaskParams = {
+    taskId: number;
+    success: boolean;
+    cost?: number;
+    message?: string;
+};
+export {};

package/dist/server/sync-source.js

@@ -0,0 +1,107 @@
+/**
+ * This file is part of the NocoBase (R) project.
+ * Copyright (c) 2020-2024 NocoBase Co., Ltd.
+ * Authors: NocoBase Team.
+ *
+ * This project is dual-licensed under AGPL-3.0 and NocoBase Commercial License.
+ * For more information, please refer to: https://www.nocobase.com/agreement.
+ */
+
+var __create = Object.create;
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __getProtoOf = Object.getPrototypeOf;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
+  // If the importer is in node compatibility mode or this is not an ESM
+  // file that has been converted to a CommonJS file using a Babel-
+  // compatible transform (i.e. "__esModule" has not been set), then set
+  // "default" to the CommonJS "module.exports" for node compatibility.
+  isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
+  mod
+));
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+var sync_source_exports = {};
+__export(sync_source_exports, {
+  SyncSource: () => SyncSource
+});
+module.exports = __toCommonJS(sync_source_exports);
+var import_dayjs = __toESM(require("dayjs"));
+class SyncSource {
+  instance;
+  options;
+  ctx;
+  constructor(config) {
+    const { options, ctx, sourceInstance } = config;
+    this.instance = sourceInstance;
+    this.options = options;
+    this.ctx = ctx;
+  }
+  async newTask() {
+    const batch = generateUniqueNumber();
+    return await this.instance.createTask({ batch, status: "init" });
+  }
+  async beginTask(taskId) {
+    const tasks = await this.instance.getTasks({ where: { id: taskId } });
+    if (!tasks && !tasks.length) {
+      throw new Error(`Task [${taskId}] is not found.`);
+    }
+    const task = tasks[0];
+    if (task.status !== "init") {
+      throw new Error(`Task [${taskId}] is not init.`);
+    }
+    task.status = "processing";
+    await task.save();
+  }
+  async endTask(params) {
+    const { taskId, success, cost, message } = params;
+    const tasks = await this.instance.getTasks({ where: { id: taskId } });
+    if (!tasks && !tasks.length) {
+      throw new Error(`Task [${taskId}] is not found.`);
+    }
+    const task = tasks[0];
+    if (task.status !== "processing") {
+      throw new Error(`Task [${taskId}] is not processing.`);
+    }
+    task.status = success ? "success" : "failed";
+    task.cost = cost;
+    task.message = message;
+    await task.save();
+  }
+  async retryTask(taskId) {
+    const tasks = await this.instance.getTasks({ where: { id: taskId } });
+    if (!tasks && !tasks.length) {
+      throw new Error(`Task [${taskId}] is not found.`);
+    }
+    const task = tasks[0];
+    if (task.status !== "failed") {
+      throw new Error(`Task [${taskId}] is not failed.`);
+    }
+    task.status = "processing";
+    task.message = "";
+    await task.save();
+    return task;
+  }
+}
+function generateUniqueNumber() {
+  const formattedDate = (0, import_dayjs.default)().format("YYYYMMDDHHmmss");
+  const randomDigits = Math.floor(1e5 + Math.random() * 9e5);
+  return formattedDate + randomDigits;
+}
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  SyncSource
+});

package/dist/server/user-data-resource-manager.d.ts

@@ -0,0 +1,103 @@
+/**
+ * This file is part of the NocoBase (R) project.
+ * Copyright (c) 2020-2024 NocoBase Co., Ltd.
+ * Authors: NocoBase Team.
+ *
+ * This project is dual-licensed under AGPL-3.0 and NocoBase Commercial License.
+ * For more information, please refer to: https://www.nocobase.com/agreement.
+ */
+import { Toposort, ToposortOptions } from '@nocobase/utils';
+import Database, { Repository } from '@nocobase/database';
+import { SystemLogger } from '@nocobase/logger';
+export type FormatUser = {
+    uid: string;
+    username?: string;
+    email?: string;
+    nickname?: string;
+    phone?: string;
+    departments?: string[];
+    isDeleted?: boolean;
+    [key: string]: any;
+};
+export type FormatDepartment = {
+    uid: string;
+    title?: string;
+    parentUid?: string;
+    isDeleted?: boolean;
+    [key: string]: any;
+};
+export type UserDataRecord = FormatUser | FormatDepartment;
+export type SyncDataType = 'user' | 'department';
+export type SyncAccept = SyncDataType;
+export type OriginRecord = {
+    id: number;
+    sourceName: string;
+    sourceUk: string;
+    dataType: SyncDataType;
+    metaData: UserDataRecord;
+    resources: {
+        resource: string;
+        resourcePk: string;
+    }[];
+};
+export type UserData = {
+    dataType: SyncDataType;
+    matchKey?: string;
+    records: UserDataRecord[];
+    sourceName: string;
+};
+export type PrimaryKey = number | string;
+export type RecordResourceChanged = {
+    resourcesPk: PrimaryKey;
+    isDeleted: boolean;
+};
+export declare abstract class UserDataResource {
+    name: string;
+    accepts: SyncAccept[];
+    db: Database;
+    logger: SystemLogger;
+    constructor(db: Database, logger: SystemLogger);
+    abstract update(record: OriginRecord, resourcePks: PrimaryKey[], matchKey?: string): Promise<RecordResourceChanged[]>;
+    abstract create(record: OriginRecord, matchKey: string): Promise<RecordResourceChanged[]>;
+    get syncRecordRepo(): Repository<any, any>;
+    get syncRecordResourceRepo(): Repository<any, any>;
+}
+export type SyncResult = {
+    resource: string;
+    detail: {
+        count: {
+            all: number;
+            success: number;
+            failed: number;
+        };
+        failedRecords: {
+            record: UserDataRecord;
+            message: string;
+        }[];
+    };
+};
+export declare class UserDataResourceManager {
+    resources: Toposort<UserDataResource>;
+    syncRecordRepo: Repository;
+    syncRecordResourceRepo: Repository;
+    logger: SystemLogger;
+    registerResource(resource: UserDataResource, options?: ToposortOptions): void;
+    set db(value: Database);
+    saveOriginRecords(data: UserData): Promise<void>;
+    findOriginRecords({ sourceName, dataType, sourceUks }: {
+        sourceName: any;
+        dataType: any;
+        sourceUks: any;
+    }): Promise<OriginRecord[]>;
+    addResourceToOriginRecord({ recordId, resource, resourcePk }: {
+        recordId: any;
+        resource: any;
+        resourcePk: any;
+    }): Promise<void>;
+    removeResourceFromOriginRecord({ recordId, resource, resourcePk }: {
+        recordId: any;
+        resource: any;
+        resourcePk: any;
+    }): Promise<void>;
+    updateOrCreate(data: UserData): Promise<SyncResult[]>;
+}

package/dist/server/user-data-resource-manager.js

@@ -0,0 +1,214 @@
+/**
+ * This file is part of the NocoBase (R) project.
+ * Copyright (c) 2020-2024 NocoBase Co., Ltd.
+ * Authors: NocoBase Team.
+ *
+ * This project is dual-licensed under AGPL-3.0 and NocoBase Commercial License.
+ * For more information, please refer to: https://www.nocobase.com/agreement.
+ */
+
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+var user_data_resource_manager_exports = {};
+__export(user_data_resource_manager_exports, {
+  UserDataResource: () => UserDataResource,
+  UserDataResourceManager: () => UserDataResourceManager
+});
+module.exports = __toCommonJS(user_data_resource_manager_exports);
+var import_utils = require("@nocobase/utils");
+class UserDataResource {
+  name;
+  accepts;
+  db;
+  logger;
+  constructor(db, logger) {
+    this.db = db;
+    this.logger = logger;
+  }
+  get syncRecordRepo() {
+    return this.db.getRepository("userDataSyncRecords");
+  }
+  get syncRecordResourceRepo() {
+    return this.db.getRepository("userDataSyncRecordsResources");
+  }
+}
+class UserDataResourceManager {
+  resources = new import_utils.Toposort();
+  syncRecordRepo;
+  syncRecordResourceRepo;
+  logger;
+  registerResource(resource, options) {
+    if (!resource.name) {
+      throw new Error('"name" for user data synchronize resource is required');
+    }
+    if (!resource.accepts) {
+      throw new Error('"accepts" for user data synchronize resource is required');
+    }
+    this.resources.add(resource, { tag: resource.name, ...options });
+  }
+  set db(value) {
+    this.syncRecordRepo = value.getRepository("userDataSyncRecords");
+    this.syncRecordResourceRepo = value.getRepository("userDataSyncRecordsResources");
+  }
+  async saveOriginRecords(data) {
+    for (const record of data.records) {
+      if (record.uid === void 0) {
+        throw new Error(`record must has uid, error record: ${JSON.stringify(record)}`);
+      }
+      const syncRecord = await this.syncRecordRepo.findOne({
+        where: {
+          sourceName: data.sourceName,
+          sourceUk: record.uid,
+          dataType: data.dataType
+        }
+      });
+      if (syncRecord) {
+        syncRecord.lastMetaData = syncRecord.metaData;
+        syncRecord.metaData = record;
+        await syncRecord.save();
+      } else {
+        await this.syncRecordRepo.create({
+          values: {
+            sourceName: data.sourceName,
+            sourceUk: record.uid,
+            dataType: data.dataType,
+            metaData: record
+          }
+        });
+      }
+    }
+  }
+  async findOriginRecords({ sourceName, dataType, sourceUks }) {
+    return await this.syncRecordRepo.find({
+      appends: ["resources"],
+      filter: { sourceName, dataType, sourceUk: { $in: sourceUks } }
+    });
+  }
+  async addResourceToOriginRecord({ recordId, resource, resourcePk }) {
+    const syncRecord = await this.syncRecordRepo.findOne({
+      filter: {
+        id: recordId
+      }
+    });
+    if (syncRecord) {
+      await syncRecord.createResource({
+        resource,
+        resourcePk
+      });
+    }
+  }
+  async removeResourceFromOriginRecord({ recordId, resource, resourcePk }) {
+    const recordResource = await this.syncRecordResourceRepo.findOne({
+      where: {
+        recordId,
+        resource,
+        resourcePk
+      }
+    });
+    if (recordResource) {
+      await recordResource.destroy();
+    }
+  }
+  async updateOrCreate(data) {
+    var _a, _b, _c, _d, _e;
+    await this.saveOriginRecords(data);
+    const { dataType, sourceName, records, matchKey } = data;
+    const sourceUks = records.map((record) => record.uid);
+    let processed = false;
+    const syncResults = [];
+    for (const resource of this.resources.nodes) {
+      if (!resource.accepts.includes(dataType)) {
+        continue;
+      }
+      const associateResource = resource.name;
+      processed = true;
+      const originRecords = await this.findOriginRecords({ sourceName, sourceUks, dataType });
+      if (!(originRecords && originRecords.length)) {
+        continue;
+      }
+      const successRecords = [];
+      const failedRecords = [];
+      for (const originRecord of originRecords) {
+        const resourceRecords = (_a = originRecord.resources) == null ? void 0 : _a.filter(
+          (r) => r.resource === associateResource
+        );
+        let recordResourceChangeds;
+        if (resourceRecords && resourceRecords.length > 0) {
+          const resourcePks = resourceRecords.map((r) => r.resourcePk);
+          try {
+            recordResourceChangeds = await resource.update(originRecord, resourcePks, matchKey);
+            (_b = this.logger) == null ? void 0 : _b.debug(`update record success. Data changed: ${JSON.stringify(recordResourceChangeds)}`);
+            successRecords.push(originRecord.metaData);
+          } catch (error) {
+            (_c = this.logger) == null ? void 0 : _c.warn(`update record error: ${error.message}`, { originRecord });
+            failedRecords.push({ record: originRecord.metaData, message: error.message });
+            continue;
+          }
+        } else {
+          try {
+            recordResourceChangeds = await resource.create(originRecord, matchKey);
+            (_d = this.logger) == null ? void 0 : _d.debug(`create record success. Data changed: ${JSON.stringify(recordResourceChangeds)}`);
+            successRecords.push(originRecord.metaData);
+          } catch (error) {
+            (_e = this.logger) == null ? void 0 : _e.warn(`create record error: ${error.message}`, { originRecord });
+            failedRecords.push({ record: originRecord.metaData, message: error.message });
+            continue;
+          }
+        }
+        if (!recordResourceChangeds || recordResourceChangeds.length === 0) {
+          continue;
+        }
+        for (const { resourcesPk, isDeleted } of recordResourceChangeds) {
+          if (isDeleted) {
+            await this.removeResourceFromOriginRecord({
+              recordId: originRecord.id,
+              resource: associateResource,
+              resourcePk: resourcesPk
+            });
+          } else {
+            await this.addResourceToOriginRecord({
+              recordId: originRecord.id,
+              resource: associateResource,
+              resourcePk: resourcesPk
+            });
+          }
+        }
+      }
+      syncResults.push({
+        resource: associateResource,
+        detail: {
+          count: {
+            all: originRecords.length,
+            success: successRecords.length,
+            failed: failedRecords.length
+          },
+          failedRecords
+        }
+      });
+    }
+    if (!processed) {
+      throw new Error(`dataType "${dataType}" is not support`);
+    }
+    return syncResults;
+  }
+}
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  UserDataResource,
+  UserDataResourceManager
+});

package/dist/server/user-data-sync-service.d.ts

@@ -0,0 +1,23 @@
+/**
+ * This file is part of the NocoBase (R) project.
+ * Copyright (c) 2020-2024 NocoBase Co., Ltd.
+ * Authors: NocoBase Team.
+ *
+ * This project is dual-licensed under AGPL-3.0 and NocoBase Commercial License.
+ * For more information, please refer to: https://www.nocobase.com/agreement.
+ */
+import { SyncResult, UserDataResourceManager } from './user-data-resource-manager';
+import { SyncSourceManager } from './sync-source-manager';
+import { Context } from '@nocobase/actions';
+import { SyncSource } from './sync-source';
+import { Logger } from '@nocobase/logger';
+export declare class UserDataSyncService {
+    resourceManager: UserDataResourceManager;
+    sourceManager: SyncSourceManager;
+    logger: Logger;
+    constructor(resourceManager: UserDataResourceManager, sourceManager: SyncSourceManager, logger: Logger);
+    pull(sourceName: string, ctx: Context): Promise<void>;
+    push(data: any): Promise<SyncResult[]>;
+    retry(sourceId: number, taskId: number, ctx: Context): Promise<void>;
+    runSync(source: SyncSource, task: any, ctx: Context): Promise<void>;
+}