@procore/ai-translations 0.4.0 → 0.6.0

package/dist/modern/index.mjs

@@ -46,6 +46,9 @@ var Config = class _Config {
   setToolName(name) {
     this.toolName = name;
   }
+  isBackendTranslationStrategy() {
+    return this.ToolConfig.strategy === "backend_translations";
+  }
   getToolName() {
     return this.toolName;
   }
@@ -239,10 +242,10 @@ var Hash = class {
 var QueueManager = class {
   constructor() {
     if (!globalThis.__AI_TRANSLATION_FRONTEND_QUEUE__) {
-      globalThis.__AI_TRANSLATION_FRONTEND_QUEUE__ = /* @__PURE__ */ new Set();
+      globalThis.__AI_TRANSLATION_FRONTEND_QUEUE__ = /* @__PURE__ */ new Map();
     }
     if (!globalThis.__AI_TRANSLATION_BACKEND_QUEUE__) {
-      globalThis.__AI_TRANSLATION_BACKEND_QUEUE__ = /* @__PURE__ */ new Set();
+      globalThis.__AI_TRANSLATION_BACKEND_QUEUE__ = /* @__PURE__ */ new Map();
     }
   }
   getQueue(strategy) {
@@ -255,41 +258,20 @@ var QueueManager = class {
     return void 0;
   }
   add(entry) {
-    this.getQueue(entry.translationStrategy)?.add(entry);
-  }
-  *generateBatches(strategy, config2) {
-    const queue = this.getQueue(strategy);
-    if (!queue) {
-      throw new Error("Invalid translation strategy");
-    }
-    if (queue.size === 0) {
-      console.warn("No translations to process");
-      return;
-    }
-    const batch = config2.getToolConfig().translationBatchSize;
-    let batches = [];
-    for (const entry of queue.values()) {
-      batches.push(entry);
-      if (batches.length >= batch) {
-        yield batches;
-        batches = [];
-      }
-    }
-    if (batches.length > 0) {
-      queue.clear();
-      yield batches;
-    }
+    const queue = this.getQueue(entry.translationStrategy);
+    if (!queue || queue.has(entry.id)) return;
+    queue.set(entry.id, entry);
   }
-  *generateBatchesV2(config2) {
+  *generateBatches(config2) {
     const queue = this.getQueue(config2.getToolConfig().strategy);
     if (!queue) throw new Error("Invalid translation strategy");
     const batchSize = config2.getToolConfig().translationBatchSize;
     while (queue.size > 0) {
       const batch = [];
       while (batch.length < batchSize && queue.size > 0) {
-        const next = queue.values().next().value;
-        queue.delete(next);
-        batch.push(next);
+        const [key, entry] = queue.entries().next().value;
+        queue.delete(key);
+        batch.push(entry);
       }
       yield batch;
     }
@@ -410,6 +392,19 @@ var Storage = class _Storage {
     }
   }
   // ------------------------------------
+  // 🗑️ Clear Translation by ID
+  // ------------------------------------
+  static async deleteById(id) {
+    try {
+      const db = await _Storage.getDB();
+      await db.delete(_Storage.STORE_NAME, id);
+      return true;
+    } catch (error) {
+      console.error("[Storage] Failed to delete by id:", error);
+      return false;
+    }
+  }
+  // ------------------------------------
   // 🗑️ Clear All Translations
   // ------------------------------------
   static async clearAll() {
@@ -438,6 +433,7 @@ var TranslationRegistry = class _TranslationRegistry {
   static memoryCache = /* @__PURE__ */ new Map();
   static originalTextIndex = /* @__PURE__ */ new Map();
   static nonRetryableTexts = /* @__PURE__ */ new Set();
+  static enqueuedItems = /* @__PURE__ */ new Set();
   tool;
   constructor(tool) {
     this.tool = tool;
@@ -491,10 +487,42 @@ var TranslationRegistry = class _TranslationRegistry {
   isNonRetryable(key) {
     return _TranslationRegistry.nonRetryableTexts.has(key);
   }
-  async get(text, targetLanguage, tool) {
+  enqueue(enqueuedKey) {
+    _TranslationRegistry.enqueuedItems.add(enqueuedKey);
+  }
+  isEnqueued(enqueuedKey) {
+    return _TranslationRegistry.enqueuedItems.has(enqueuedKey);
+  }
+  clearEnqueued() {
+    _TranslationRegistry.enqueuedItems.clear();
+  }
+  async removeTextsFromEnqueued(texts, targetLanguage, strategy) {
+    for (const text of texts) {
+      const id = await Storage.generateId(text, targetLanguage, this.tool);
+      _TranslationRegistry.enqueuedItems.delete(
+        this.generateEnqueuedKey(id, strategy)
+      );
+    }
+  }
+  static removeFromEnqueued(key) {
+    _TranslationRegistry.enqueuedItems.delete(key);
+  }
+  generateEnqueuedKey(id, strategy) {
+    return `${id}:${strategy}`;
+  }
+  async get(text, targetLanguage, tool, config2) {
     const key = await Hash.generateFromMultiple([text, targetLanguage, tool]);
-    if (_TranslationRegistry.memoryCache.has(key)) {
-      return _TranslationRegistry.memoryCache.get(key);
+    const cached = _TranslationRegistry.memoryCache.get(key);
+    if (cached) {
+      if (cached.translationStrategy === "frontend_translations" && config2.isBackendTranslationStrategy()) {
+        _TranslationRegistry.memoryCache.delete(key);
+        if (cached.translatedText) {
+          _TranslationRegistry.originalTextIndex.delete(cached.translatedText);
+        }
+        await Storage.deleteById(key);
+        return void 0;
+      }
+      return cached;
     }
     const translation = await Storage.getTranslation(
       text,
@@ -502,6 +530,10 @@ var TranslationRegistry = class _TranslationRegistry {
       tool
     );
     if (translation !== void 0) {
+      if (translation.translation_strategy === "frontend_translations" && config2.isBackendTranslationStrategy()) {
+        await Storage.deleteById(translation.id);
+        return void 0;
+      }
       return this.toTranslationRegistryEntry(translation);
     }
     return void 0;
@@ -516,6 +548,7 @@ var TranslationRegistry = class _TranslationRegistry {
   async clear() {
     _TranslationRegistry.memoryCache.clear();
     _TranslationRegistry.originalTextIndex.clear();
+    _TranslationRegistry.enqueuedItems.clear();
     await Storage.clearAll();
     return;
   }
@@ -523,6 +556,7 @@ var TranslationRegistry = class _TranslationRegistry {
     await Storage.deleteByToolAndStrategy(tool, strategy);
     _TranslationRegistry.memoryCache.clear();
     _TranslationRegistry.originalTextIndex.clear();
+    _TranslationRegistry.enqueuedItems.clear();
     return;
   }
   toTranslationRegistryEntry(translation) {
@@ -544,10 +578,16 @@ var TranslationRegistry = class _TranslationRegistry {
         translation.id
       );
       const newTranslation = this.toTranslationRegistryEntry(translation);
+      const enqueuedKey = this.generateEnqueuedKey(
+        translation.id,
+        translation.translation_strategy
+      );
       if (currentTranslation && !this.shouldUpdateTranslation(currentTranslation, newTranslation)) {
+        _TranslationRegistry.removeFromEnqueued(enqueuedKey);
         continue;
       }
       this.setTranslationToRegistry(newTranslation);
+      _TranslationRegistry.removeFromEnqueued(enqueuedKey);
     }
     return;
   }
@@ -597,12 +637,13 @@ var EventHandler = class {
     this.aiTranslationEvents = new SystemEvents(this.sourceEventIdentifier);
     this.toolName = toolName;
   }
-  publishTranslationCompleteEvent(sourceTranslatedTexts, nonRetryableTexts) {
+  publishTranslationCompleteEvent(sourceTranslatedTexts, nonRetryableTexts, retryableFailedTexts = []) {
     this.aiTranslationEvents.publish(
       this.withEventName(TRANSLATION_COMPLETE_EVENT_NAME),
       {
         sourceTranslatedTexts,
-        nonRetryableTexts
+        nonRetryableTexts,
+        retryableFailedTexts
       }
     );
   }
@@ -614,7 +655,14 @@ var EventHandler = class {
           detail.data?.sourceTranslatedTexts
         ) ? detail.data.sourceTranslatedTexts : [];
         const nonRetryableTexts = Array.isArray(detail.data?.nonRetryableTexts) ? detail.data.nonRetryableTexts : [];
-        callback(sourceTranslatedTexts, nonRetryableTexts);
+        const retryableFailedTexts = Array.isArray(
+          detail.data?.retryableFailedTexts
+        ) ? detail.data.retryableFailedTexts : [];
+        callback(
+          sourceTranslatedTexts,
+          nonRetryableTexts,
+          retryableFailedTexts
+        );
       }
     );
   }
@@ -703,6 +751,7 @@ var TranslationManager = class {
   currentTranslatorStrategy;
   sourceTexts = /* @__PURE__ */ new Map();
   nonRetryableTexts = /* @__PURE__ */ new Set();
+  retryableFailedTexts = /* @__PURE__ */ new Map();
   progressEventHandler;
   translationProgress = {
     progress: 0,
@@ -731,19 +780,21 @@ var TranslationManager = class {
     this.translationProgress.total = queueManager.queueSize(
       this.currentTranslatorStrategy
     );
-    for (const batch of queueManager.generateBatchesV2(
-      this.translator.getConfig()
-    )) {
-      batch.forEach((entry) => this.mfeToBeNotified.add(entry.tool));
-      const result = await this.translator.processTranslations(
-        batch.map(
-          (entry) => this.convertTranslationQueueEntryToTranslationRequest(entry)
-        )
-      );
-      this.setTranslationProgress(batch.length);
-      await this.updateDatabaseWithTranslations(result);
-      await this.notifyTranslationCompleted();
-    }
+    do {
+      for (const batch of queueManager.generateBatches(
+        this.translator.getConfig()
+      )) {
+        batch.forEach((entry) => this.mfeToBeNotified.add(entry.tool));
+        const result = await this.translator.processTranslations(
+          batch.map(
+            (entry) => this.convertTranslationQueueEntryToTranslationRequest(entry)
+          )
+        );
+        this.setTranslationProgress(batch.length);
+        await this.updateDatabaseWithTranslations(result);
+        await this.notifyTranslationCompleted();
+      }
+    } while (queueManager.queueSize(this.currentTranslatorStrategy) > 0);
     this.resetTranslationProgress();
     this.publishTranslationProgress();
     if (this.currentTranslatorStrategy === "frontend_translations") {
@@ -762,6 +813,7 @@ var TranslationManager = class {
           continue;
         }
         if (!translation.isTranslated) {
+          this.addRetryableFailedText(response.tool, translation.sourceText);
           continue;
         }
         this.addSourceTextToMap(response.tool, translation.sourceText);
@@ -781,12 +833,14 @@ var TranslationManager = class {
       const eventhandler = new EventHandler(mfe);
       eventhandler.publishTranslationCompleteEvent(
         Array.from(this.sourceTexts.get(mfe) ?? /* @__PURE__ */ new Set()),
-        Array.from(this.nonRetryableTexts)
+        Array.from(this.nonRetryableTexts),
+        Array.from(this.retryableFailedTexts.get(mfe) ?? /* @__PURE__ */ new Set())
       );
     }
     this.mfeToBeNotified.clear();
     this.sourceTexts.clear();
     this.nonRetryableTexts.clear();
+    this.retryableFailedTexts.clear();
   }
   setTranslationProgress(batchSize) {
     this.translationProgress.current += batchSize;
@@ -823,6 +877,11 @@ var TranslationManager = class {
     existing.add(sourceText);
     this.sourceTexts.set(tool, existing);
   }
+  addRetryableFailedText(tool, sourceText) {
+    const existing = this.retryableFailedTexts.get(tool) ?? /* @__PURE__ */ new Set();
+    existing.add(sourceText);
+    this.retryableFailedTexts.set(tool, existing);
+  }
   publishTranslationProgress() {
     this.progressEventHandler.publishTranslationProgressEvent(
       this.translationProgress.progress,
@@ -889,8 +948,16 @@ var Client = class {
         method: "POST",
         body: JSON.stringify(requests)
       });
+      const responseBody = Array.isArray(data) ? data.map((group) => ({
+        ...group,
+        translations: Array.isArray(group.translations) ? group.translations.map((t) => ({
+          ...t,
+          isTranslated: t.isTranslated ?? (!!t.translation && t.translation !== ""),
+          retryable: t.retryable ?? true
+        })) : []
+      })) : data;
       return {
-        responseBody: data,
+        responseBody,
         success: true
       };
     } catch (error) {
@@ -967,12 +1034,55 @@ var getModelDownloadEventHandler = () => {
   }
   return _modelDownloadEventHandler;
 };
+var SUPPORTED_LANGUAGES = /* @__PURE__ */ new Set([
+  "ar",
+  "bn",
+  "bg",
+  "zh",
+  "hr",
+  "cs",
+  "da",
+  "nl",
+  "en",
+  "fi",
+  "fr",
+  "de",
+  "el",
+  "he",
+  "hi",
+  "hu",
+  "id",
+  "it",
+  "ja",
+  "kn",
+  "ko",
+  "lt",
+  "mr",
+  "no",
+  "pl",
+  "pt",
+  "ro",
+  "ru",
+  "sk",
+  "sl",
+  "es",
+  "sv",
+  "ta",
+  "te",
+  "th",
+  "tr",
+  "uk",
+  "vi"
+]);
 var ChromeTranslator = class _ChromeTranslator {
   translator;
   constructor(translator) {
     this.translator = translator;
   }
-  static translatorReadyPromise = Promise.resolve();
+  static translatorReadyPromises = /* @__PURE__ */ new Map();
+  static isSupportedLanguage(lang) {
+    return SUPPORTED_LANGUAGES.has(lang) || SUPPORTED_LANGUAGES.has(lang.split("-")[0] ?? "");
+  }
   static ensureRegistry() {
     if (!globalThis.chromeTranslatorRegistry) {
       globalThis.chromeTranslatorRegistry = /* @__PURE__ */ new Map();
@@ -994,30 +1104,31 @@ var ChromeTranslator = class _ChromeTranslator {
     );
   }
   static async createTranslator(source, target) {
+    const key = this.generateKey(source, target);
     let resolveReady;
-    this.translatorReadyPromise = new Promise((resolve) => {
+    const readyPromise = new Promise((resolve) => {
       resolveReady = resolve;
     });
+    this.translatorReadyPromises.set(key, readyPromise);
     let isDownloading = false;
     const translator = await Translator.create({
       sourceLanguage: source,
       targetLanguage: target,
       monitor(m) {
-        m.addEventListener(
-          "downloadprogress",
-          (e) => {
-            isDownloading = true;
-            const progress = e.total > 0 ? Math.round(e.loaded / e.total * 100) : 0;
-            getModelDownloadEventHandler().publishModelDownloadProgressEvent(
-              e.loaded,
-              e.total,
-              progress
-            );
-            if (e.loaded === e.total) {
-              resolveReady();
-            }
+        m.addEventListener("downloadprogress", (e) => {
+          isDownloading = true;
+          const loaded = e.loaded ?? 0;
+          const total = e.total ?? 0;
+          const progress = total > 0 ? Math.round(loaded / total * 100) : 0;
+          getModelDownloadEventHandler().publishModelDownloadProgressEvent(
+            loaded,
+            total,
+            progress
+          );
+          if (total > 0 && loaded >= total) {
+            resolveReady();
           }
-        );
+        });
       }
     });
     if (!isDownloading) {
@@ -1025,8 +1136,8 @@ var ChromeTranslator = class _ChromeTranslator {
     }
     return translator;
   }
-  static waitForReady() {
-    return this.translatorReadyPromise;
+  static waitForReady(key) {
+    return this.translatorReadyPromises.get(key) ?? Promise.resolve();
   }
   static async translate(text, targetLanguage) {
     this.ensureRegistry();
@@ -1034,8 +1145,17 @@ var ChromeTranslator = class _ChromeTranslator {
       const detector = await ChromeLanguageDetector.getInstance();
       const sourceLanguage = await detector.detectLanguage(text);
       const key = this.generateKey(sourceLanguage, targetLanguage);
-      let instance = globalThis.chromeTranslatorRegistry.get(key);
-      if (!instance) {
+      if (!this.isSupportedLanguage(sourceLanguage) || !this.isSupportedLanguage(targetLanguage)) {
+        return {
+          translation: text,
+          sourceLanguage,
+          success: false,
+          retryable: false,
+          errorMessage: `Unsupported language pair: ${sourceLanguage} \u2192 ${targetLanguage}`
+        };
+      }
+      const existingInstance = globalThis.chromeTranslatorRegistry.get(key);
+      if (!existingInstance) {
         const translatorCapabilities = await Translator.availability({
           sourceLanguage,
           targetLanguage
@@ -1053,10 +1173,16 @@ var ChromeTranslator = class _ChromeTranslator {
           sourceLanguage,
           targetLanguage
         );
-        instance = new _ChromeTranslator(chromeTranslator);
-        globalThis.chromeTranslatorRegistry.set(key, instance);
+        if (!globalThis.chromeTranslatorRegistry.has(key)) {
+          globalThis.chromeTranslatorRegistry.set(
+            key,
+            new _ChromeTranslator(chromeTranslator)
+          );
+        }
       }
-      await this.waitForReady();
+      const instance = globalThis.chromeTranslatorRegistry.get(key);
+      await this.waitForReady(key);
+      this.translatorReadyPromises.delete(key);
       const translation = await instance.translator?.translate(text);
       if (!translation) {
         return {
@@ -1064,7 +1190,7 @@ var ChromeTranslator = class _ChromeTranslator {
           sourceLanguage,
           success: false,
           errorMessage: "Translation failed",
-          retryable: true
+          retryable: false
         };
       }
       return {
@@ -1074,11 +1200,12 @@ var ChromeTranslator = class _ChromeTranslator {
         retryable: true
       };
     } catch (error) {
+      const message = error instanceof Error ? error.message : "Unknown error";
       return {
         translation: text,
         sourceLanguage: targetLanguage,
         success: false,
-        errorMessage: error instanceof Error ? error.message : "Unknown error",
+        errorMessage: message,
         retryable: true
       };
     }
@@ -1235,13 +1362,22 @@ var aitFunction = async (text, translationRegistry, config2, tool) => {
   const existingByOriginal = await translationRegistry.get(
     text,
     targetLanguage,
-    tool
+    tool,
+    config2
   );
   if (existingByOriginal && existingByOriginal.isTranslated && existingByOriginal.translatedText && existingByOriginal.translatedText.trim() !== "") {
     return existingByOriginal.translatedText;
   }
   if (!existingByOriginal) {
+    const id = await Storage.generateId(text, targetLanguage, tool);
+    const strategy = config2.getToolConfig().strategy;
+    const enqueuedKey = translationRegistry.generateEnqueuedKey(id, strategy);
+    if (translationRegistry.isEnqueued(enqueuedKey)) {
+      return text;
+    }
+    translationRegistry.enqueue(enqueuedKey);
     queueManager.add({
+      id,
       originalText: text,
       targetLanguage,
       tool,
@@ -1305,12 +1441,17 @@ function AITranslationInnerProvider(props) {
   });
   useEffect(() => {
     const unsubscribe = eventHandler.current.subscribeToTranslationCompleteEvent(
-      async (sourceTranslatedTexts, nonRetryableTexts) => {
+      async (sourceTranslatedTexts, nonRetryableTexts, retryableFailedTexts) => {
         try {
           await translationRegistry.current.populateRegistryFromDatabase();
           translationRegistry.current.populateNonRetryableTexts(
             nonRetryableTexts
           );
+          translationRegistry.current.removeTextsFromEnqueued(
+            retryableFailedTexts,
+            locale,
+            config2.getToolConfig().strategy
+          );
           eventHandler.current.publishRerenderEvent(sourceTranslatedTexts);
         } catch (error) {
           console.error(
@@ -1321,7 +1462,7 @@ function AITranslationInnerProvider(props) {
       }
     );
     return () => unsubscribe();
-  }, []);
+  }, [locale, config2]);
   useEffect(() => {
     const unsubscribe = eventHandler.current.subscribeToTranslationProgressEvent(
       (progress, current, total) => {
@@ -1339,15 +1480,15 @@ function AITranslationInnerProvider(props) {
     return () => unsubscribe();
   }, []);
   useEffect(() => {
-    if (isFetched && remoteConfig && remoteConfig[tool]) {
-      setConfig((prevConfig) => {
-        const clonedConfig = prevConfig.clone();
-        clonedConfig.setTargetLanguage(locale);
-        clonedConfig.setToolConfig(remoteConfig[tool]);
-        translator.current.updateConfig(clonedConfig);
-        return clonedConfig;
-      });
-    }
+    if (!isFetched || !remoteConfig) return;
+    const toolConfig = remoteConfig[tool] ?? remoteConfig;
+    setConfig((prevConfig) => {
+      const clonedConfig = prevConfig.clone();
+      clonedConfig.setTargetLanguage(locale);
+      clonedConfig.setToolConfig(toolConfig);
+      translator.current.updateConfig(clonedConfig);
+      return clonedConfig;
+    });
   }, [tool, remoteConfig, isFetched, locale]);
   config2.setTargetLanguage(locale);
   config2.setToolName(tool);
@@ -1410,6 +1551,7 @@ var TranslatedIcon = ({
       className,
       xmlns: "http://www.w3.org/2000/svg",
       "aria-hidden": "true",
+      style: { marginRight: "10px", flexShrink: 0 },
       children: /* @__PURE__ */ jsx2(
         "path",
         {
@@ -1444,12 +1586,14 @@ var AITranslateText = ({
           return;
         }
         if (sourceTexts.includes(text)) {
-          setDisplayText(await context.ait(text));
+          const translatedText = await context.ait(text);
+          setDisplayText(translatedText);
+          setShowHighlightState(showHighlight && translatedText !== text);
         }
       }
     );
     return () => unsubscribe();
-  }, [context, text]);
+  }, [context, text, showHighlight]);
   const reset = useCallback2(
     (displayValue = text) => {
       setDisplayText(displayValue);
@@ -1472,7 +1616,7 @@ var AITranslateText = ({
         const translatedText = await context.ait(text);
         if (cancelled) return;
         setDisplayText(translatedText);
-        setShowHighlightState(showHighlight);
+        setShowHighlightState(showHighlight && translatedText !== text);
       } catch {
         if (cancelled) return;
         reset();
@@ -1484,8 +1628,8 @@ var AITranslateText = ({
     };
   }, [text, shouldTranslate, context, showHighlight, reset]);
   return /* @__PURE__ */ jsxs(Fragment, { children: [
-    displayText,
-    showHighlightState && /* @__PURE__ */ jsx3(TranslatedIcon, { ...translatedIconProps })
+    showHighlightState && /* @__PURE__ */ jsx3(TranslatedIcon, { ...translatedIconProps }),
+    displayText
   ] });
 };
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@procore/ai-translations",
-  "version": "0.4.0",
+  "version": "0.6.0",
   "description": "Library that provides a solution to use AI to translate text into a language",
   "main": "dist/legacy/index.js",
   "types": "dist/legacy/index.d.ts",