@adminforth/i18n 1.0.18-next.1 → 1.0.18-next.11

package/dist/index.js

@@ -13,6 +13,9 @@ import path from 'path';
 import fs from 'fs-extra';
 import chokidar from 'chokidar';
 import { AsyncQueue } from '@sapphire/async-queue';
+console.log = (...args) => {
+    process.stdout.write(args.join(" ") + "\n");
+};
 const processFrontendMessagesQueue = new AsyncQueue();
 const SLAVIC_PLURAL_EXAMPLES = {
     uk: 'яблук | Яблуко | Яблука | Яблук', // zero | singular | 2-4 | 5+
@@ -141,6 +144,7 @@ export default class I18N extends AdminForthPlugin {
             if (!enColumn) {
                 throw new Error(`Field ${this.enFieldName} not found column to store english original string in resource ${resourceConfig.resourceId}`);
             }
+            enColumn.editReadonly = true;
             // if sourceFieldName defined, check it exists
             if (this.options.sourceFieldName) {
                 if (!resourceConfig.columns.find(c => c.name === this.options.sourceFieldName)) {
@@ -282,15 +286,16 @@ export default class I18N extends AdminForthPlugin {
                             translatedCount = yield this.bulkTranslate({ selectedIds });
                         }
                         catch (e) {
+                            process.env.HEAVY_DEBUG && console.error('🪲⛔ bulkTranslate error', e);
                             if (e instanceof AiTranslateError) {
-                                process.env.HEAVY_DEBUG && console.error('🪲⛔ bulkTranslate error', e);
                                 return { ok: false, error: e.message };
                             }
+                            throw e;
                         }
-                        process.env.HEAVY_DEBUG && console.log('🪲bulkTranslate done', selectedIds);
                         this.updateUntranslatedMenuBadge();
                         return {
-                            ok: true, error: undefined,
+                            ok: true,
+                            error: undefined,
                             successMessage: yield tr(`Translated {count} items`, 'backend', {
                                 count: translatedCount,
                             }),
@@ -323,6 +328,82 @@ export default class I18N extends AdminForthPlugin {
             });
         });
     }
+    translateToLang(langIsoCode_1, strings_1) {
+        return __awaiter(this, arguments, void 0, function* (langIsoCode, strings, plurals = false, translations, updateStrings = {}) {
+            const maxKeysInOneReq = 10;
+            if (strings.length === 0) {
+                return [];
+            }
+            if (strings.length > maxKeysInOneReq) {
+                let totalTranslated = [];
+                for (let i = 0; i < strings.length; i += maxKeysInOneReq) {
+                    const slicedStrings = strings.slice(i, i + maxKeysInOneReq);
+                    process.env.HEAVY_DEBUG && console.log('🪲🔪slicedStrings len', slicedStrings.length);
+                    const madeKeys = yield this.translateToLang(langIsoCode, slicedStrings, plurals, translations, updateStrings);
+                    totalTranslated = totalTranslated.concat(madeKeys);
+                }
+                return totalTranslated;
+            }
+            const lang = langIsoCode;
+            const langName = iso6391.getName(lang);
+            const requestSlavicPlurals = Object.keys(SLAVIC_PLURAL_EXAMPLES).includes(lang) && plurals;
+            const prompt = `
+I need to translate strings in JSON to ${lang} (${langName}) language from English for my web app.
+${requestSlavicPlurals ? `You should provide 4 translations (in format zero | singular | 2-4 | 5+) e.g. ${SLAVIC_PLURAL_EXAMPLES[lang]}` : ''}
+Keep keys, as is, write translation into values! Here are the strings:
+
+\`\`\`json
+${JSON.stringify(strings.reduce((acc, s) => {
+                acc[s.en_string] = '';
+                return acc;
+            }, {}), null, 2)}
+\`\`\`
+`;
+            // call OpenAI
+            const resp = yield this.options.completeAdapter.complete(prompt, [], 300);
+            if (resp.error) {
+                throw new AiTranslateError(resp.error);
+            }
+            // parse response like
+            // Here are the translations for the strings you provided:
+            // ```json
+            // [{"live": "canlı"}, {"Table Games": "Masa Oyunları"}]
+            // ```
+            let res;
+            try {
+                res = resp.content.split("```json")[1].split("```")[0];
+            }
+            catch (e) {
+                console.error(`Error in parsing LLM resp: ${resp}\n Prompt was: ${prompt}\n Resp was: ${JSON.stringify(resp)}`);
+                return [];
+            }
+            try {
+                res = JSON.parse(res);
+            }
+            catch (e) {
+                console.error(`Error in parsing LLM resp json: ${resp}\n Prompt was: ${prompt}\n Resp was: ${JSON.stringify(resp)}`);
+                return [];
+            }
+            for (const [enStr, translatedStr] of Object.entries(res)) {
+                const translationsTargeted = translations.filter(t => t[this.enFieldName] === enStr);
+                // might be several with same en_string
+                for (const translation of translationsTargeted) {
+                    //translation[this.trFieldNames[lang]] = translatedStr;
+                    // process.env.HEAVY_DEBUG && console.log(`🪲translated to ${lang} ${translation.en_string}, ${translatedStr}`)
+                    if (!updateStrings[translation[this.primaryKeyFieldName]]) {
+                        updateStrings[translation[this.primaryKeyFieldName]] = {
+                            updates: {},
+                            translatedStr,
+                            category: translation[this.options.categoryFieldName],
+                            strId: translation[this.primaryKeyFieldName],
+                        };
+                    }
+                    updateStrings[translation[this.primaryKeyFieldName]].updates[this.trFieldNames[lang]] = translatedStr;
+                }
+            }
+            return Object.keys(updateStrings);
+        });
+    }
     // returns translated count
     bulkTranslate(_a) {
         return __awaiter(this, arguments, void 0, function* ({ selectedIds }) {
@@ -345,83 +426,16 @@ export default class I18N extends AdminForthPlugin {
                     }
                 }
             }
-            const maxKeysInOneReq = 10;
             const updateStrings = {};
-            const translateToLang = (langIsoCode_1, strings_1, ...args_1) => __awaiter(this, [langIsoCode_1, strings_1, ...args_1], void 0, function* (langIsoCode, strings, plurals = false) {
-                if (strings.length === 0) {
-                    return 0;
-                }
-                if (strings.length > maxKeysInOneReq) {
-                    let totalTranslated = 0;
-                    for (let i = 0; i < strings.length; i += maxKeysInOneReq) {
-                        const slicedStrings = strings.slice(i, i + maxKeysInOneReq);
-                        process.env.HEAVY_DEBUG && console.log('🪲🔪slicedStrings len', slicedStrings.length);
-                        totalTranslated += yield translateToLang(langIsoCode, slicedStrings, plurals);
-                    }
-                    return totalTranslated;
-                }
-                const lang = langIsoCode;
-                const requestSlavicPlurals = Object.keys(SLAVIC_PLURAL_EXAMPLES).includes(lang) && plurals;
-                const prompt = `
-I need to translate strings in JSON to ${lang} language from English for my web app.
-${requestSlavicPlurals ? `You should provide 4 translations (in format zero | singular | 2-4 | 5+) e.g. ${SLAVIC_PLURAL_EXAMPLES[lang]}` : ''}
-Keep keys, as is, write translation into values! Here are the strings:
-
-\`\`\`json
-${JSON.stringify(strings.reduce((acc, s) => {
-                    acc[s.en_string] = '';
-                    return acc;
-                }, {}), null, 2)}
-\`\`\`
-`;
-                process.env.HEAVY_DEBUG && console.log('🧠 llm prompt', prompt);
-                // call OpenAI
-                const resp = yield this.options.completeAdapter.complete(prompt, [], 300);
-                process.env.HEAVY_DEBUG && console.log('🧠 llm resp', resp);
-                if (resp.error) {
-                    throw new AiTranslateError(resp.error);
-                }
-                // parse response like
-                // Here are the translations for the strings you provided:
-                // ```json
-                // [{"live": "canlı"}, {"Table Games": "Masa Oyunları"}]
-                // ```
-                let res;
-                try {
-                    res = resp.content.split("```json")[1].split("```")[0];
-                }
-                catch (e) {
-                    console.error('error in parsing OpenAI', resp);
-                    throw new AiTranslateError('Error in parsing OpenAI response');
-                }
-                res = JSON.parse(res);
-                for (const [enStr, translatedStr] of Object.entries(res)) {
-                    const translationsTargeted = translations.filter(t => t[this.enFieldName] === enStr);
-                    // might be several with same en_string
-                    for (const translation of translationsTargeted) {
-                        //translation[this.trFieldNames[lang]] = translatedStr;
-                        // process.env.HEAVY_DEBUG && console.log(`🪲translated to ${lang} ${translation.en_string}, ${translatedStr}`)
-                        if (!updateStrings[translation[this.primaryKeyFieldName]]) {
-                            updateStrings[translation[this.primaryKeyFieldName]] = {
-                                updates: {},
-                                translatedStr,
-                                category: translation[this.options.categoryFieldName],
-                                strId: translation[this.primaryKeyFieldName],
-                            };
-                        }
-                        updateStrings[translation[this.primaryKeyFieldName]].updates[this.trFieldNames[lang]] = translatedStr;
-                    }
-                }
-                return Object.keys(updateStrings).length;
-            });
             const langsInvolved = new Set(Object.keys(needToTranslateByLang));
-            let totalTranslated = 0;
+            let totalTranslated = [];
             yield Promise.all(Object.entries(needToTranslateByLang).map((_b) => __awaiter(this, [_b], void 0, function* ([lang, strings]) {
                 // first translate without plurals
                 const stringsWithoutPlurals = strings.filter(s => !s.en_string.includes('|'));
-                totalTranslated += yield translateToLang(lang, stringsWithoutPlurals, false);
+                const noPluralKeys = yield this.translateToLang(lang, stringsWithoutPlurals, false, translations, updateStrings);
                 const stringsWithPlurals = strings.filter(s => s.en_string.includes('|'));
-                totalTranslated += yield translateToLang(lang, stringsWithPlurals, true);
+                const pluralKeys = yield this.translateToLang(lang, stringsWithPlurals, true, translations, updateStrings);
+                totalTranslated = totalTranslated.concat(noPluralKeys, pluralKeys);
             })));
             yield Promise.all(Object.entries(updateStrings).map((_c) => __awaiter(this, [_c], void 0, function* ([_, { updates, strId }]) {
                 // because this will translate all languages, we can set completedLangs to all languages
@@ -429,12 +443,12 @@ ${JSON.stringify(strings.reduce((acc, s) => {
                 yield this.adminforth.resource(this.resourceConfig.resourceId).update(strId, Object.assign(Object.assign({}, updates), { [this.options.completedFieldName]: futureCompletedFieldValue }));
             })));
             for (const lang of langsInvolved) {
-                this.cache.clear(`${this.resourceConfig.resourceId}:frontend:${lang}`);
+                yield this.cache.clear(`${this.resourceConfig.resourceId}:frontend:${lang}`);
                 for (const [enStr, { category }] of Object.entries(updateStrings)) {
-                    this.cache.clear(`${this.resourceConfig.resourceId}:${category}:${lang}:${enStr}`);
+                    yield this.cache.clear(`${this.resourceConfig.resourceId}:${category}:${lang}:${enStr}`);
                 }
             }
-            return totalTranslated;
+            return new Set(totalTranslated).size;
         });
     }
     processExtractedMessages(adminforth, filePath) {
@@ -506,6 +520,9 @@ ${JSON.stringify(strings.reduce((acc, s) => {
     validateConfigAfterDiscover(adminforth, resourceConfig) {
         // optional method where you can safely check field types after database discovery was performed
         // ensure each trFieldName (apart from enFieldName) is nullable column of type string
+        if (this.options.completeAdapter) {
+            this.options.completeAdapter.validate();
+        }
         for (const lang of this.options.supportedLanguages) {
             if (lang === 'en') {
                 continue;

package/index.ts

@@ -7,6 +7,11 @@ import fs from 'fs-extra';
 import chokidar from 'chokidar';
 import  { AsyncQueue } from '@sapphire/async-queue';
 
+
+console.log = (...args) => {
+  process.stdout.write(args.join(" ") + "\n");
+};
+
 const processFrontendMessagesQueue = new AsyncQueue();
 
 const SLAVIC_PLURAL_EXAMPLES = {
@@ -156,6 +161,8 @@ export default class I18N extends AdminForthPlugin {
       throw new Error(`Field ${this.enFieldName} not found column to store english original string in resource ${resourceConfig.resourceId}`);
     }
 
+    enColumn.editReadonly = true;
+
     // if sourceFieldName defined, check it exists
     if (this.options.sourceFieldName) {
       if (!resourceConfig.columns.find(c => c.name === this.options.sourceFieldName)) {
@@ -322,15 +329,16 @@ export default class I18N extends AdminForthPlugin {
             try {
               translatedCount = await this.bulkTranslate({ selectedIds });
             } catch (e) {
+              process.env.HEAVY_DEBUG && console.error('🪲⛔ bulkTranslate error', e);
               if (e instanceof AiTranslateError) {
-                process.env.HEAVY_DEBUG && console.error('🪲⛔ bulkTranslate error', e);
                 return { ok: false, error: e.message };
-              }
+              } 
+              throw e;
             }
-            process.env.HEAVY_DEBUG && console.log('🪲bulkTranslate done', selectedIds);
             this.updateUntranslatedMenuBadge();
             return { 
-              ok: true, error: undefined, 
+              ok: true, 
+              error: undefined, 
               successMessage: await tr(`Translated {count} items`, 'backend', {
                 count: translatedCount,
               }),
@@ -364,6 +372,103 @@ export default class I18N extends AdminForthPlugin {
     });
   }
 
+  async translateToLang (
+      langIsoCode: LanguageCode, 
+      strings: { en_string: string, category: string }[], 
+      plurals=false,
+      translations: any[],
+      updateStrings: Record<string, { updates: any, category: string, strId: string, translatedStr: string }> = {}
+  ): Promise<string[]> {
+    const maxKeysInOneReq = 10;
+    if (strings.length === 0) {
+      return [];
+    }
+
+    if (strings.length > maxKeysInOneReq) {
+      let totalTranslated = [];
+      for (let i = 0; i < strings.length; i += maxKeysInOneReq) {
+        const slicedStrings = strings.slice(i, i + maxKeysInOneReq);
+        process.env.HEAVY_DEBUG && console.log('🪲🔪slicedStrings len', slicedStrings.length);
+        const madeKeys = await this.translateToLang(langIsoCode, slicedStrings, plurals, translations, updateStrings);
+        totalTranslated = totalTranslated.concat(madeKeys);
+      }
+      return totalTranslated;
+    }
+    const lang = langIsoCode;
+    const langName = iso6391.getName(lang);
+    const requestSlavicPlurals = Object.keys(SLAVIC_PLURAL_EXAMPLES).includes(lang) && plurals;
+
+    const prompt = `
+I need to translate strings in JSON to ${lang} (${langName}) language from English for my web app.
+${requestSlavicPlurals ? `You should provide 4 translations (in format zero | singular | 2-4 | 5+) e.g. ${SLAVIC_PLURAL_EXAMPLES[lang]}` : ''}
+Keep keys, as is, write translation into values! Here are the strings:
+
+\`\`\`json
+${
+JSON.stringify(strings.reduce((acc: object, s: { en_string: string }): object => {
+  acc[s.en_string] = '';
+  return acc;
+}, {}), null, 2)
+}
+\`\`\`
+`;  
+
+    // call OpenAI
+    const resp = await this.options.completeAdapter.complete(
+      prompt,
+      [],
+      300,
+    );
+
+    if (resp.error) {
+      throw new AiTranslateError(resp.error);
+    }
+
+    // parse response like
+    // Here are the translations for the strings you provided:
+    // ```json
+    // [{"live": "canlı"}, {"Table Games": "Masa Oyunları"}]
+    // ```
+    let res;
+    try {
+      res = resp.content.split("```json")[1].split("```")[0];
+    } catch (e) {
+      console.error(`Error in parsing LLM resp: ${resp}\n Prompt was: ${prompt}\n Resp was: ${JSON.stringify(resp)}`, );
+      return [];
+    }
+
+    try {
+      res = JSON.parse(res);
+    } catch (e) {
+      console.error(`Error in parsing LLM resp json: ${resp}\n Prompt was: ${prompt}\n Resp was: ${JSON.stringify(resp)}`, );
+      return [];
+    }
+
+
+    for (const [enStr, translatedStr] of Object.entries(res) as [string, string][]) {
+      const translationsTargeted = translations.filter(t => t[this.enFieldName] === enStr);
+      // might be several with same en_string
+      for (const translation of translationsTargeted) {
+        //translation[this.trFieldNames[lang]] = translatedStr;
+        // process.env.HEAVY_DEBUG && console.log(`🪲translated to ${lang} ${translation.en_string}, ${translatedStr}`)
+        if (!updateStrings[translation[this.primaryKeyFieldName]]) {
+
+          updateStrings[translation[this.primaryKeyFieldName]] = {
+            updates: {},
+            translatedStr,
+            category: translation[this.options.categoryFieldName],
+            strId: translation[this.primaryKeyFieldName],
+          };
+        }
+        updateStrings[
+          translation[this.primaryKeyFieldName]
+        ].updates[this.trFieldNames[lang]] = translatedStr;
+      }
+    }
+
+    return Object.keys(updateStrings);
+  }
+
   // returns translated count
   async bulkTranslate({ selectedIds }: { selectedIds: string[] }): Promise<number> {
 
@@ -397,8 +502,6 @@ export default class I18N extends AdminForthPlugin {
       }
     }
 
-    const maxKeysInOneReq = 10;
-
     const updateStrings: Record<string, { 
       updates: any, 
       category: string,
@@ -406,104 +509,27 @@ export default class I18N extends AdminForthPlugin {
       translatedStr: string
      }> = {};
 
-    const translateToLang = async (langIsoCode: LanguageCode, strings: { en_string: string, category: string }[], plurals=false): Promise<number> => {
-      if (strings.length === 0) {
-        return 0;
-      }
-
-      if (strings.length > maxKeysInOneReq) {
-        let totalTranslated = 0;
-        for (let i = 0; i < strings.length; i += maxKeysInOneReq) {
-          const slicedStrings = strings.slice(i, i + maxKeysInOneReq);
-          process.env.HEAVY_DEBUG && console.log('🪲🔪slicedStrings len', slicedStrings.length);
-          totalTranslated += await translateToLang(langIsoCode, slicedStrings, plurals);
-        }
-        return totalTranslated;
-      }
-      const lang = langIsoCode;
-
-      const requestSlavicPlurals = Object.keys(SLAVIC_PLURAL_EXAMPLES).includes(lang) && plurals;
 
-      const prompt = `
-I need to translate strings in JSON to ${lang} language from English for my web app.
-${requestSlavicPlurals ? `You should provide 4 translations (in format zero | singular | 2-4 | 5+) e.g. ${SLAVIC_PLURAL_EXAMPLES[lang]}` : ''}
-Keep keys, as is, write translation into values! Here are the strings:
+    const langsInvolved = new Set(Object.keys(needToTranslateByLang));
 
-\`\`\`json
-${
-  JSON.stringify(strings.reduce((acc: object, s: { en_string: string }): object => {
-    acc[s.en_string] = '';
-    return acc;
-  }, {}), null, 2)
-}
-\`\`\`
-`;
+    let totalTranslated = [];
 
-      process.env.HEAVY_DEBUG && console.log('🧠 llm prompt', prompt);
+    await Promise.all(
+      Object.entries(needToTranslateByLang).map(
+        async ([lang, strings]: [LanguageCode, { en_string: string, category: string }[]]) => {
+          // first translate without plurals
+          const stringsWithoutPlurals = strings.filter(s => !s.en_string.includes('|'));
+          const noPluralKeys = await this.translateToLang(lang, stringsWithoutPlurals, false, translations, updateStrings);
 
-      // call OpenAI
-      const resp = await this.options.completeAdapter.complete(
-        prompt,
-        [],
-        300,
-      );
 
-      process.env.HEAVY_DEBUG && console.log('🧠 llm resp', resp);
+          const stringsWithPlurals = strings.filter(s => s.en_string.includes('|'));
 
-      if (resp.error) {
-        throw new AiTranslateError(resp.error);
-      }
+          const pluralKeys = await this.translateToLang(lang, stringsWithPlurals, true, translations, updateStrings);
 
-      // parse response like
-      // Here are the translations for the strings you provided:
-      // ```json
-      // [{"live": "canlı"}, {"Table Games": "Masa Oyunları"}]
-      // ```
-      let res;
-      try {
-        res = resp.content.split("```json")[1].split("```")[0];
-      } catch (e) {
-        console.error('error in parsing OpenAI', resp);
-        throw new AiTranslateError('Error in parsing OpenAI response');
-      }
-      res = JSON.parse(res);
-
-
-      for (const [enStr, translatedStr] of Object.entries(res) as [string, string][]) {
-        const translationsTargeted = translations.filter(t => t[this.enFieldName] === enStr);
-        // might be several with same en_string
-        for (const translation of translationsTargeted) {
-          //translation[this.trFieldNames[lang]] = translatedStr;
-          // process.env.HEAVY_DEBUG && console.log(`🪲translated to ${lang} ${translation.en_string}, ${translatedStr}`)
-          if (!updateStrings[translation[this.primaryKeyFieldName]]) {
-
-            updateStrings[translation[this.primaryKeyFieldName]] = {
-              updates: {},
-              translatedStr,
-              category: translation[this.options.categoryFieldName],
-              strId: translation[this.primaryKeyFieldName],
-            };
-          }
-          updateStrings[
-            translation[this.primaryKeyFieldName]
-          ].updates[this.trFieldNames[lang]] = translatedStr;
+          totalTranslated = totalTranslated.concat(noPluralKeys, pluralKeys);
         }
-      }
-
-      return Object.keys(updateStrings).length;
-    }
-
-    const langsInvolved = new Set(Object.keys(needToTranslateByLang));
-
-    let totalTranslated = 0;
-    await Promise.all(Object.entries(needToTranslateByLang).map(async ([lang, strings]: [LanguageCode, { en_string: string, category: string }[]]) => {
-      // first translate without plurals
-      const stringsWithoutPlurals = strings.filter(s => !s.en_string.includes('|'));
-      totalTranslated += await translateToLang(lang, stringsWithoutPlurals, false);
-
-      const stringsWithPlurals = strings.filter(s => s.en_string.includes('|'));
-      totalTranslated += await translateToLang(lang, stringsWithPlurals, true);
-    }));    
+      )
+    );
 
     await Promise.all(
       Object.entries(updateStrings).map(
@@ -520,13 +546,13 @@ ${
     );
 
     for (const lang of langsInvolved) {
-      this.cache.clear(`${this.resourceConfig.resourceId}:frontend:${lang}`);
+      await this.cache.clear(`${this.resourceConfig.resourceId}:frontend:${lang}`);
       for (const [enStr, { category }] of Object.entries(updateStrings)) {
-        this.cache.clear(`${this.resourceConfig.resourceId}:${category}:${lang}:${enStr}`);
+        await this.cache.clear(`${this.resourceConfig.resourceId}:${category}:${lang}:${enStr}`);
       }
     }
 
-    return totalTranslated;
+    return new Set(totalTranslated).size;
 
   }
 
@@ -593,12 +619,10 @@ ${
     });
     w.on('change', () => {
       process.env.HEAVY_DEBUG && console.log('🪲🔔messagesFile change', messagesFile);
-
       this.processExtractedMessages(adminforth, messagesFile);
     });
     w.on('add', () => {
       process.env.HEAVY_DEBUG && console.log('🪲🔔messagesFile add', messagesFile);
-
       this.processExtractedMessages(adminforth, messagesFile);
     });
 
@@ -607,6 +631,10 @@ ${
   validateConfigAfterDiscover(adminforth: IAdminForth, resourceConfig: AdminForthResource) {
     // optional method where you can safely check field types after database discovery was performed
     // ensure each trFieldName (apart from enFieldName) is nullable column of type string
+    if (this.options.completeAdapter) {
+      this.options.completeAdapter.validate();
+    }
+
     for (const lang of this.options.supportedLanguages) {
       if (lang === 'en') {
         continue;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@adminforth/i18n",
-  "version": "1.0.18-next.1",
+  "version": "1.0.18-next.11",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
   "type": "module",