@nocobase/plugin-localization 2.1.0-beta.29 → 2.1.0-beta.32

package/dist/server/actions/aiTranslate.js

@@ -0,0 +1,127 @@
+/**
+ * This file is part of the NocoBase (R) project.
+ * Copyright (c) 2020-2024 NocoBase Co., Ltd.
+ * Authors: NocoBase Team.
+ *
+ * This project is dual-licensed under AGPL-3.0 and NocoBase Commercial License.
+ * For more information, please refer to: https://www.nocobase.com/agreement.
+ */
+
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+var aiTranslate_exports = {};
+__export(aiTranslate_exports, {
+  default: () => aiTranslate_default
+});
+module.exports = __toCommonJS(aiTranslate_exports);
+var import_localization_ai_translate = require("../tasks/localization-ai-translate");
+const validateParams = (ctx) => {
+  const { mode, locale, employeeUsername = "lina", model, textIds } = ctx.action.params.values || {};
+  if (!["full", "incremental", "selected"].includes(mode)) {
+    ctx.throw(400, "Invalid translation mode");
+  }
+  if (mode === "selected" && (!Array.isArray(textIds) || !textIds.length)) {
+    ctx.throw(400, "Please select the records you want to translate");
+  }
+  return {
+    mode,
+    locale: locale || ctx.get("X-Locale") || "en-US",
+    employeeUsername,
+    model,
+    textIds
+  };
+};
+const buildFindTextsOptions = (mode, locale, textIds) => {
+  const options = {};
+  if (mode === "selected") {
+    options.filter = {
+      id: {
+        $in: textIds || []
+      }
+    };
+  }
+  if (mode === "incremental") {
+    options.include = [{ association: "translations", where: { locale }, required: false }];
+    options.where = {
+      "$translations.id$": null
+    };
+  }
+  return options;
+};
+const getAITranslatePreview = async (ctx) => {
+  const { mode, locale, employeeUsername, model, textIds } = validateParams(ctx);
+  const aiPlugin = ctx.app.pm.get("ai");
+  if (!(aiPlugin == null ? void 0 : aiPlugin.aiEmployeesManager)) {
+    ctx.throw(500, "AI plugin is not available");
+  }
+  const employee = await aiPlugin.aiEmployeesManager.getEmployee(employeeUsername);
+  if (!employee) {
+    ctx.throw(400, `AI employee "${employeeUsername}" not found`);
+  }
+  const resolvedModel = await aiPlugin.aiEmployeesManager.resolveModel(employee, model);
+  const { model: modelName, service } = await aiPlugin.aiManager.getLLMService(resolvedModel);
+  const providerMeta = aiPlugin.aiManager.llmProviders.get(service.provider);
+  const count = await ctx.db.getRepository("localizationTexts").count(buildFindTextsOptions(mode, locale, textIds));
+  return {
+    mode,
+    locale,
+    count,
+    provider: service.provider,
+    providerTitle: providerMeta == null ? void 0 : providerMeta.title,
+    llmService: service.name,
+    llmServiceTitle: service.title,
+    model: modelName
+  };
+};
+const aiTranslatePreview = async (ctx, next) => {
+  ctx.body = await getAITranslatePreview(ctx);
+  await next();
+};
+const aiTranslate = async (ctx, next) => {
+  var _a, _b, _c, _d;
+  const { mode, locale, employeeUsername, model, textIds } = validateParams(ctx);
+  const currentUserId = ((_b = (_a = ctx.auth) == null ? void 0 : _a.user) == null ? void 0 : _b.id) || ((_d = (_c = ctx.state) == null ? void 0 : _c.currentUser) == null ? void 0 : _d.id);
+  const taskManager = ctx.app.container.get("AsyncTaskManager");
+  if (!taskManager) {
+    ctx.throw(500, "AsyncTaskManager is not available");
+  }
+  const task = await taskManager.createTask(
+    {
+      origin: "localization",
+      type: import_localization_ai_translate.LOCALIZATION_AI_TRANSLATE_TASK_TYPE,
+      title: mode === "full" ? "AI full localization translation" : "AI incremental localization translation",
+      params: {
+        mode,
+        locale,
+        employeeUsername,
+        model,
+        userId: currentUserId,
+        textIds
+      },
+      createdById: currentUserId,
+      cancelable: true
+    },
+    {
+      useQueue: true,
+      context: ctx
+    }
+  );
+  ctx.body = task.toJSON();
+  await next();
+};
+var aiTranslate_default = { aiTranslate, aiTranslatePreview };

package/dist/server/actions/localization.js

@@ -35,15 +35,23 @@ const sync = async (ctx, next) => {
   const plugin = ctx.app.pm.get("localization");
   const resourcesInstance = plugin.resources;
   const locale = ctx.get("X-Locale") || "en-US";
-  const { types = [] } = ctx.action.params.values || {};
+  const { types = [], resetTranslations = false } = ctx.action.params.values || {};
   if (!types.length) {
     ctx.throw(400, ctx.t("Please provide synchronization source."));
   }
-  const resources = await plugin.sourceManager.sync(ctx, types);
-  let textValues = [];
+  const resources = await ctx.app.localeManager.syncSources(ctx, types);
+  const normalizedResources = {};
   Object.entries(resources).forEach(([module2, resource]) => {
+    const normalizedModule = plugin.normalizeResourceModule(module2).replace("resources.", "");
+    normalizedResources[normalizedModule] = {
+      ...normalizedResources[normalizedModule] || {},
+      ...resource
+    };
+  });
+  let textValues = [];
+  Object.entries(normalizedResources).forEach(([module2, resource]) => {
     Object.keys(resource).forEach((text) => {
-      textValues.push({ module: `resources.${module2}`, text });
+      textValues.push({ module: plugin.normalizeResourceModule(module2), text });
     });
   });
   textValues = await resourcesInstance.filterExists(textValues);
@@ -53,26 +61,34 @@ const sync = async (ctx, next) => {
     });
     const texts = await ctx.db.getModel("localizationTexts").findAll({
       include: [{ association: "translations", where: { locale }, required: false }],
-      where: { "$translations.id$": null },
+      where: resetTranslations ? void 0 : { "$translations.id$": null },
       transaction: t
     });
     const translationValues = texts.filter((text) => {
       var _a;
-      const module2 = text.module.replace("resources.", "");
-      return (_a = resources[module2]) == null ? void 0 : _a[text.text];
+      const module2 = plugin.normalizeResourceModule(text.module).replace("resources.", "");
+      return (_a = normalizedResources[module2]) == null ? void 0 : _a[text.text];
     }).map((text) => {
       var _a;
-      const module2 = text.module.replace("resources.", "");
+      const module2 = plugin.normalizeResourceModule(text.module).replace("resources.", "");
       return {
         locale,
         textId: text.id,
-        translation: (_a = resources[module2]) == null ? void 0 : _a[text.text]
+        translation: (_a = normalizedResources[module2]) == null ? void 0 : _a[text.text]
       };
     });
-    await ctx.db.getModel("localizationTranslations").bulkCreate(translationValues, {
-      transaction: t
-    });
+    if (resetTranslations) {
+      await ctx.db.getModel("localizationTranslations").bulkCreate(translationValues, {
+        updateOnDuplicate: ["translation"],
+        transaction: t
+      });
+    } else {
+      await ctx.db.getModel("localizationTranslations").bulkCreate(translationValues, {
+        transaction: t
+      });
+    }
     await resourcesInstance.updateCacheTexts(newTexts);
+    await resourcesInstance.reset();
   });
   ctx.logger.info(`Sync localization resources done, ${Date.now() - startTime}ms`);
   await next();
@@ -82,9 +98,8 @@ const publish = async (ctx, next) => {
   await next();
 };
 const getSources = async (ctx, next) => {
-  const plugin = ctx.app.pm.get("localization");
-  const sources = Array.from(plugin.sourceManager.sources.getEntities());
-  ctx.body = sources.map(([name, source]) => ({
+  const sources = Array.from(ctx.app.localeManager.sources.getEntities());
+  ctx.body = sources.filter(([, source]) => source.sync).map(([name, source]) => ({
     name,
     title: source.title
   }));

package/dist/server/actions/localizationTexts.js

@@ -104,18 +104,17 @@ const list = async (ctx, next) => {
   const [rows, count] = await listText(ctx.db, { module: module2, keyword, hasTranslation, locale, options });
   const cache = ctx.app.cache;
   const pm = ctx.app.pm;
-  const plugin = pm.get("localization");
   const plugins = await cache.wrap(`lm-plugins:${locale}`, () => pm.list({ locale }));
-  const sources = Array.from(plugin.sourceManager.sources.getValues());
+  const sources = Array.from(ctx.app.localeManager.sources.getValues());
   const extendModules = sources.filter((source) => source.namespace).map((source) => ({
     value: source.namespace,
     label: source.title
   }));
   const modules = [
     ...extendModules,
-    ...plugins.map((plugin2) => ({
-      value: plugin2.alias || plugin2.name,
-      label: plugin2.displayName
+    ...plugins.map((plugin) => ({
+      value: plugin.alias || plugin.name,
+      label: plugin.displayName
     }))
   ];
   for (const row of rows) {
@@ -133,9 +132,9 @@ const list = async (ctx, next) => {
     totalPage: Math.ceil(count / pageSize),
     modules: [
       ...extendModules,
-      ...plugins.map((plugin2) => ({
-        value: plugin2.alias || plugin2.name,
-        label: plugin2.displayName
+      ...plugins.map((plugin) => ({
+        value: plugin.alias || plugin.name,
+        label: plugin.displayName
       }))
     ]
   };
@@ -163,7 +162,7 @@ const missing = async (ctx, next) => {
     const plugin = (_a = ctx.app.pm) == null ? void 0 : _a.get("localization");
     const currentLocale = locale || ctx.get("X-Locale") || "en-US";
     await (plugin == null ? void 0 : plugin.addNewTexts(
-      keys.map((key) => ({ text: key.text, module: `resources.${key.ns}` })),
+      keys.map((key) => ({ text: key.text, module: plugin.normalizeResourceModule(key.ns) })),
       {
         locale: currentLocale
       }

package/dist/server/migrations/20260511230000-delete-official-plugin-package-resource-modules.d.ts

@@ -0,0 +1,14 @@
+/**
+ * This file is part of the NocoBase (R) project.
+ * Copyright (c) 2020-2024 NocoBase Co., Ltd.
+ * Authors: NocoBase Team.
+ *
+ * This project is dual-licensed under AGPL-3.0 and NocoBase Commercial License.
+ * For more information, please refer to: https://www.nocobase.com/agreement.
+ */
+import { Migration } from '@nocobase/server';
+export default class extends Migration {
+    on: string;
+    appVersion: string;
+    up(): Promise<void>;
+}

package/dist/server/migrations/20260511230000-delete-official-plugin-package-resource-modules.js

@@ -0,0 +1,64 @@
+/**
+ * This file is part of the NocoBase (R) project.
+ * Copyright (c) 2020-2024 NocoBase Co., Ltd.
+ * Authors: NocoBase Team.
+ *
+ * This project is dual-licensed under AGPL-3.0 and NocoBase Commercial License.
+ * For more information, please refer to: https://www.nocobase.com/agreement.
+ */
+
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+var delete_official_plugin_package_resource_modules_exports = {};
+__export(delete_official_plugin_package_resource_modules_exports, {
+  default: () => delete_official_plugin_package_resource_modules_default
+});
+module.exports = __toCommonJS(delete_official_plugin_package_resource_modules_exports);
+var import_database = require("@nocobase/database");
+var import_server = require("@nocobase/server");
+class delete_official_plugin_package_resource_modules_default extends import_server.Migration {
+  on = "afterLoad";
+  // 'beforeLoad' or 'afterLoad'
+  appVersion = "<2.2.0";
+  async up() {
+    const textRepo = this.db.getRepository("localizationTexts");
+    const translationRepo = this.db.getRepository("localizationTranslations");
+    const texts = await textRepo.find({
+      filter: {
+        module: {
+          [import_database.Op.like]: `resources.${import_server.OFFICIAL_PLUGIN_PREFIX}%`
+        }
+      },
+      fields: ["id"]
+    });
+    const textIds = texts.map((text) => text.get("id"));
+    if (!textIds.length) {
+      return;
+    }
+    await translationRepo.destroy({
+      filter: {
+        textId: {
+          $in: textIds
+        }
+      }
+    });
+    await textRepo.destroy({
+      filterByTk: textIds
+    });
+  }
+}

package/dist/server/plugin.d.ts

@@ -9,10 +9,11 @@
 /// <reference types="node" />
 import { InstallOptions, Plugin } from '@nocobase/server';
 import Resources from './resources';
-import { SourceManager } from './source-manager';
 export declare class PluginLocalizationServer extends Plugin {
     resources: Resources;
-    sourceManager: SourceManager;
+    private aiTranslateTaskRegistered;
+    private localeSourceTextHookKeys;
+    normalizeResourceModule(module: string): string;
     addNewTexts: (texts: {
         text: string;
         module: string;
@@ -22,7 +23,9 @@ export declare class PluginLocalizationServer extends Plugin {
     }) => Promise<void>;
     afterAdd(): void;
     beforeLoad(): void;
+    private registerAITranslateTaskType;
     load(): Promise<void>;
+    private handleLocaleSourceTextsSaved;
     handleSyncMessage(message: any): Promise<void>;
     install(options?: InstallOptions): Promise<void>;
     afterEnable(): Promise<void>;

package/dist/server/plugin.js

@@ -42,17 +42,29 @@ __export(plugin_exports, {
 module.exports = __toCommonJS(plugin_exports);
 var import_server = require("@nocobase/server");
 var import_localization = __toESM(require("./actions/localization"));
+var import_aiTranslate = __toESM(require("./actions/aiTranslate"));
 var import_localizationTexts = __toESM(require("./actions/localizationTexts"));
 var import_resources = __toESM(require("./resources"));
 var import_utils = require("./utils");
 var import_constants = require("./constants");
-var import_source_manager = require("./source-manager");
 var import_utils2 = require("@nocobase/utils");
+var import_localization_ai_translate = require("./tasks/localization-ai-translate");
 var import_package = __toESM(require("../../package.json"));
 class PluginLocalizationServer extends import_server.Plugin {
   resources;
-  sourceManager = new import_source_manager.SourceManager();
+  aiTranslateTaskRegistered = false;
+  localeSourceTextHookKeys = /* @__PURE__ */ new Set();
+  normalizeResourceModule(module2) {
+    const prefix = "resources.";
+    const namespace = module2.startsWith(prefix) ? module2.slice(prefix.length) : module2;
+    const normalizedNamespace = namespace.startsWith(import_server.OFFICIAL_PLUGIN_PREFIX) ? namespace.replace(import_server.OFFICIAL_PLUGIN_PREFIX, "") : namespace;
+    return `${prefix}${normalizedNamespace}`;
+  }
   addNewTexts = async (texts, options) => {
+    texts = texts.map(({ text, module: module2 }) => ({
+      text,
+      module: this.normalizeResourceModule(module2)
+    }));
     texts = await this.resources.filterExists(texts, options == null ? void 0 : options.transaction);
     await this.db.getModel("localizationTexts").bulkCreate(
       texts.map(({ text, module: module2 }) => ({
@@ -88,18 +100,35 @@ class PluginLocalizationServer extends import_server.Plugin {
     });
   };
   afterAdd() {
-    this.app.on("afterLoad", () => this.sourceManager.handleTextsSaved(this.db, this.addNewTexts));
+    this.app.on("afterLoad", () => this.handleLocaleSourceTextsSaved());
+    this.app.on("afterLoad", () => this.registerAITranslateTaskType());
   }
   beforeLoad() {
   }
+  registerAITranslateTaskType() {
+    if (this.aiTranslateTaskRegistered) {
+      return;
+    }
+    try {
+      const taskManager = this.app.container.get("AsyncTaskManager");
+      taskManager.registerTaskType(import_localization_ai_translate.LocalizationAITranslateTask);
+      this.aiTranslateTaskRegistered = true;
+    } catch (error) {
+      this.log.warn("AsyncTaskManager is not available, skip localization AI translate task registration.");
+    }
+  }
   async load() {
+    this.registerAITranslateTaskType();
     this.app.resourceManager.define({
       name: "localizationTexts",
       actions: import_localizationTexts.default
     });
     this.app.resourceManager.define({
       name: "localization",
-      actions: import_localization.default
+      actions: {
+        ...import_localization.default,
+        ...import_aiTranslate.default
+      }
     });
     this.app.acl.registerSnippet({
       name: `pm.${this.name}.localization`,
@@ -119,24 +148,22 @@ class PluginLocalizationServer extends import_server.Plugin {
       store: "memory"
     });
     this.resources = new import_resources.default(this.db, cache);
-    this.sourceManager.registerSource("local", {
+    this.app.localeManager.registerSource("local", {
       title: (0, import_utils2.tval)("System & Plugins", { ns: import_package.default.name }),
       sync: async (ctx) => {
-        const resources = await ctx.app.localeManager.getCacheResources(ctx.get("X-Locale") || "en-US");
+        const resources = await ctx.app.localeManager.getBuiltInResources(ctx.get("X-Locale") || "en-US");
         const result = {};
         Object.entries(resources).forEach(([module2, resource]) => {
-          if (module2.startsWith(import_server.OFFICIAL_PLUGIN_PREFIX)) {
-            const name = module2.replace(import_server.OFFICIAL_PLUGIN_PREFIX, "");
-            if (resources[name]) {
-              return;
-            }
-          }
-          result[module2] = resource;
+          const name = module2.startsWith(import_server.OFFICIAL_PLUGIN_PREFIX) ? module2.replace(import_server.OFFICIAL_PLUGIN_PREFIX, "") : module2;
+          result[name] = {
+            ...result[name] || {},
+            ...resource
+          };
         });
         return result;
       }
     });
-    this.sourceManager.registerSource("db", {
+    this.app.localeManager.registerSource("db", {
       title: (0, import_utils2.tval)("Collections & Fields", { ns: import_package.default.name }),
       namespace: import_constants.NAMESPACE_COLLECTIONS,
       sync: async (ctx) => {
@@ -185,6 +212,37 @@ class PluginLocalizationServer extends import_server.Plugin {
       await this.addNewTexts(texts, options);
     });
   }
+  handleLocaleSourceTextsSaved() {
+    for (const [sourceName, source] of this.app.localeManager.sources.getEntities()) {
+      if (!source.collections) {
+        continue;
+      }
+      for (const [index, collectionOptions] of source.collections.entries()) {
+        const hookKey = `${sourceName}:${index}:${collectionOptions.collection}`;
+        if (this.localeSourceTextHookKeys.has(hookKey)) {
+          continue;
+        }
+        this.localeSourceTextHookKeys.add(hookKey);
+        this.db.on(`${collectionOptions.collection}.afterSave`, async (instance, options) => {
+          let texts = [];
+          if (collectionOptions.getTexts) {
+            texts = await collectionOptions.getTexts(instance, options);
+          } else {
+            const fields = collectionOptions.fields || [];
+            const changedFields = fields.filter((field) => instance["_changed"].has(field));
+            if (!changedFields.length) {
+              return;
+            }
+            texts = changedFields.map((field) => instance.get(field)).filter(Boolean).map((text) => ({ text, module: `resources.${source.namespace}` }));
+          }
+          if (!(texts == null ? void 0 : texts.length)) {
+            return;
+          }
+          await this.addNewTexts(texts, options);
+        });
+      }
+    }
+  }
   async handleSyncMessage(message) {
     switch (message.type) {
       case "updateCacheTexts":

package/dist/server/tasks/localization-ai-translate.d.ts

@@ -0,0 +1,31 @@
+/**
+ * This file is part of the NocoBase (R) project.
+ * Copyright (c) 2020-2024 NocoBase Co., Ltd.
+ * Authors: NocoBase Team.
+ *
+ * This project is dual-licensed under AGPL-3.0 and NocoBase Commercial License.
+ * For more information, please refer to: https://www.nocobase.com/agreement.
+ */
+import { TaskType } from '@nocobase/plugin-async-task-manager';
+export declare const LOCALIZATION_AI_TRANSLATE_TASK_TYPE = "localization:ai-translate";
+export declare class LocalizationAITranslateTask extends TaskType {
+    static type: string;
+    execute(): Promise<{
+        translated: number;
+        total: number;
+    }>;
+    private countTexts;
+    private buildFindTextsOptions;
+    private normalizeTextRecord;
+    private getLocaleReferences;
+    private getSystemDefaultLocale;
+    private getModuleName;
+    private isBuiltInText;
+    private getBuiltInReference;
+    private runTranslationWorker;
+    private translateText;
+    private buildProviderContext;
+    private buildTranslationTerms;
+    private getLanguageName;
+    private extractTextContent;
+}