@intlayer/cli 7.3.11 → 7.3.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -96,7 +96,8 @@ const translateDictionary = async (task, configuration, options) => {
|
|
|
96
96
|
].join(""), { colSize: 5 });
|
|
97
97
|
};
|
|
98
98
|
appLogger(`${task.dictionaryPreset}${localePreset} Preparing ${(0, __intlayer_config.colorizePath)((0, node_path.basename)(targetLocaleDictionary.filePath))}`, { level: "info" });
|
|
99
|
-
const
|
|
99
|
+
const isContentStructured = typeof dictionaryToProcess.content === "object" && dictionaryToProcess.content !== null || Array.isArray(dictionaryToProcess.content);
|
|
100
|
+
const chunkedJsonContent = (0, __intlayer_chokidar.chunkJSON)(isContentStructured ? dictionaryToProcess.content : { __INTLAYER_ROOT_PRIMITIVE_CONTENT__: dictionaryToProcess.content }, CHUNK_SIZE);
|
|
100
101
|
const nbOfChunks = chunkedJsonContent.length;
|
|
101
102
|
if (nbOfChunks > 1) appLogger(`${task.dictionaryPreset}${localePreset} Split into ${(0, __intlayer_config.colorizeNumber)(nbOfChunks)} chunks for translation`, { level: "info" });
|
|
102
103
|
const chunkResult = [];
|
|
@@ -104,7 +105,7 @@ const translateDictionary = async (task, configuration, options) => {
|
|
|
104
105
|
const chunkPreset = createChunkPreset(chunk.index, chunk.total);
|
|
105
106
|
if (nbOfChunks > 1) appLogger(`${task.dictionaryPreset}${localePreset}${chunkPreset} Translating chunk`, { level: "info" });
|
|
106
107
|
const chunkContent = (0, __intlayer_chokidar.reconstructFromSingleChunk)(chunk);
|
|
107
|
-
const presetOutputContent = (0, __intlayer_chokidar.reduceObjectFormat)(targetLocaleDictionary.content, chunkContent);
|
|
108
|
+
const presetOutputContent = (0, __intlayer_chokidar.reduceObjectFormat)(isContentStructured ? targetLocaleDictionary.content : { __INTLAYER_ROOT_PRIMITIVE_CONTENT__: targetLocaleDictionary.content }, chunkContent);
|
|
108
109
|
const executeTranslation = async () => {
|
|
109
110
|
return await (0, __intlayer_config.retryManager)(async () => {
|
|
110
111
|
let translationResult;
|
|
@@ -153,12 +154,12 @@ const translateDictionary = async (task, configuration, options) => {
|
|
|
153
154
|
(await Promise.all(chunkPromises)).sort((chunkA, chunkB) => chunkA.chunk.index - chunkB.chunk.index).forEach(({ result }) => {
|
|
154
155
|
chunkResult.push(result);
|
|
155
156
|
});
|
|
156
|
-
|
|
157
|
+
let finalContent = (0, __intlayer_core.mergeDictionaries)(chunkResult.map((chunk) => ({
|
|
157
158
|
...dictionaryToProcess,
|
|
158
159
|
content: chunk
|
|
159
|
-
})));
|
|
160
|
-
|
|
161
|
-
if (typeof baseUnmergedDictionary.locale === "string") finalContent = require_fill_deepMergeContent.deepMergeContent(targetLocaleDictionary.content ?? {},
|
|
160
|
+
}))).content;
|
|
161
|
+
if (!isContentStructured) finalContent = finalContent?.__INTLAYER_ROOT_PRIMITIVE_CONTENT__;
|
|
162
|
+
if (typeof baseUnmergedDictionary.locale === "string") finalContent = require_fill_deepMergeContent.deepMergeContent(targetLocaleDictionary.content ?? {}, finalContent);
|
|
162
163
|
return [targetLocale, finalContent];
|
|
163
164
|
}));
|
|
164
165
|
const translatedContent = Object.fromEntries(translatedContentResults);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"translateDictionary.cjs","names":["baseUnmergedDictionary: Dictionary | undefined","metadata:\n | Pick<Dictionary, 'description' | 'title' | 'tags'>\n | undefined","targetLocaleDictionary: Dictionary","getFilterMissingContentPerLocale","ANSIColors","chunkedJsonContent: JsonChunk[]","chunkResult: JsonChunk[]","translationResult: any","chunkPreset","deepMergeContent","translatedContent: Partial<Record<Locale, Dictionary['content']>>","dictionaryOutput: Dictionary"],"sources":["../../../src/fill/translateDictionary.ts"],"sourcesContent":["import { basename } from 'node:path';\nimport type { AIConfig } from '@intlayer/ai';\nimport { type AIOptions, getIntlayerAPIProxy } from '@intlayer/api';\nimport {\n chunkJSON,\n formatLocale,\n type JsonChunk,\n reconstructFromSingleChunk,\n reduceObjectFormat,\n verifyIdenticObjectFormat,\n} from '@intlayer/chokidar';\nimport {\n ANSIColors,\n colon,\n colorize,\n colorizeNumber,\n colorizePath,\n getAppLogger,\n retryManager,\n} from '@intlayer/config';\nimport {\n getFilterMissingTranslationsDictionary,\n getMultilingualDictionary,\n getPerLocaleDictionary,\n insertContentInDictionary,\n mergeDictionaries,\n} from '@intlayer/core';\nimport type { Dictionary, IntlayerConfig, Locale } from '@intlayer/types';\nimport { getUnmergedDictionaries } from '@intlayer/unmerged-dictionaries-entry';\nimport type { AIClient } from '../utils/setupAI';\nimport { deepMergeContent } from './deepMergeContent';\nimport { getFilterMissingContentPerLocale } from './getFilterMissingContentPerLocale';\nimport type { TranslationTask } from './listTranslationsTasks';\n\ntype TranslateDictionaryResult = TranslationTask & {\n dictionaryOutput: Dictionary | null;\n};\n\ntype TranslateDictionaryOptions = {\n mode: 'complete' | 'review';\n aiOptions?: AIOptions;\n fillMetadata?: boolean;\n onHandle?: ReturnType<typeof import('@intlayer/chokidar').getGlobalLimiter>;\n onSuccess?: () => void;\n onError?: (error: unknown) => void;\n getAbortError?: () => Error | null;\n aiClient?: AIClient;\n aiConfig?: AIConfig;\n};\n\nconst hasMissingMetadata = (dictionary: Dictionary) =>\n !dictionary.description || !dictionary.title || !dictionary.tags;\n\nconst CHUNK_SIZE = 7000; // GPT-5 Mini safe input size\nconst GROUP_MAX_RETRY = 2;\nconst MAX_RETRY = 3;\nconst RETRY_DELAY = 1000 * 10; // 10 seconds\n\nconst MAX_FOLLOWING_ERRORS = 10; // 10 errors in a row, hard exit the process\nlet followingErrors = 0;\n\nexport const translateDictionary = async (\n task: TranslationTask,\n configuration: IntlayerConfig,\n options?: TranslateDictionaryOptions\n): Promise<TranslateDictionaryResult> => {\n const appLogger = getAppLogger(configuration);\n const intlayerAPI = getIntlayerAPIProxy(undefined, configuration);\n\n const { mode, aiOptions, fillMetadata, aiClient, aiConfig } = {\n mode: 'complete',\n fillMetadata: true,\n ...options,\n } as const;\n\n const notifySuccess = () => {\n followingErrors = 0;\n options?.onSuccess?.();\n };\n\n const result = await retryManager(\n async () => {\n const unmergedDictionariesRecord = getUnmergedDictionaries(configuration);\n\n const baseUnmergedDictionary: Dictionary | undefined =\n unmergedDictionariesRecord[task.dictionaryKey].find(\n (dict) => dict.localId === task.dictionaryLocalId\n );\n\n if (!baseUnmergedDictionary) {\n appLogger(\n `${task.dictionaryPreset}Dictionary not found in unmergedDictionariesRecord. Skipping.`,\n {\n level: 'warn',\n }\n );\n return { ...task, dictionaryOutput: null };\n }\n\n let metadata:\n | Pick<Dictionary, 'description' | 'title' | 'tags'>\n | undefined;\n\n if (\n fillMetadata &&\n (hasMissingMetadata(baseUnmergedDictionary) || mode === 'review')\n ) {\n const defaultLocaleDictionary = getPerLocaleDictionary(\n baseUnmergedDictionary,\n configuration.internationalization.defaultLocale\n );\n\n appLogger(\n `${task.dictionaryPreset} Filling missing metadata for ${colorizePath(basename(baseUnmergedDictionary.filePath!))}`,\n {\n level: 'info',\n }\n );\n\n const runAudit = async () => {\n if (aiClient && aiConfig) {\n const result = await aiClient.auditDictionaryMetadata({\n fileContent: JSON.stringify(defaultLocaleDictionary),\n aiConfig,\n });\n\n return {\n data: result,\n };\n }\n\n return await intlayerAPI.ai.auditContentDeclarationMetadata({\n fileContent: JSON.stringify(defaultLocaleDictionary),\n aiOptions,\n });\n };\n\n const metadataResult = options?.onHandle\n ? await options.onHandle(runAudit)\n : await runAudit();\n\n metadata = metadataResult.data?.fileContent;\n }\n\n const translatedContentResults = await Promise.all(\n task.targetLocales.map(async (targetLocale) => {\n /**\n * In complete mode, for large dictionaries, we want to filter all content that is already translated\n *\n * targetLocale: fr\n *\n * { test1: t({ ar: 'Hello', en: 'Hello', fr: 'Bonjour' } }) -> {}\n * { test2: t({ ar: 'Hello', en: 'Hello' }) } -> { test2: t({ ar: 'Hello', en: 'Hello' }) }\n *\n */\n // Reset to base dictionary for each locale to ensure we filter from the original\n let dictionaryToProcess = structuredClone(baseUnmergedDictionary);\n\n let targetLocaleDictionary: Dictionary;\n\n if (typeof baseUnmergedDictionary.locale === 'string') {\n // For per-locale files, the content is already in simple JSON format (not translation nodes)\n // The base dictionary is already the source locale content\n\n // Load the existing target locale dictionary\n const targetLocaleFilePath =\n baseUnmergedDictionary.filePath?.replace(\n new RegExp(`/${task.sourceLocale}/`, 'g'),\n `/${targetLocale}/`\n );\n\n // Find the target locale dictionary in unmerged dictionaries\n const targetUnmergedDictionary = targetLocaleFilePath\n ? unmergedDictionariesRecord[task.dictionaryKey]?.find(\n (dict) =>\n dict.filePath === targetLocaleFilePath &&\n dict.locale === targetLocale\n )\n : undefined;\n\n targetLocaleDictionary = targetUnmergedDictionary ?? {\n key: baseUnmergedDictionary.key,\n content: {},\n filePath: targetLocaleFilePath,\n locale: targetLocale,\n };\n\n // In complete mode, filter out already translated content\n if (mode === 'complete') {\n dictionaryToProcess = getFilterMissingContentPerLocale(\n dictionaryToProcess,\n targetUnmergedDictionary\n );\n }\n } else {\n // For multilingual dictionaries\n if (mode === 'complete') {\n // Remove all nodes that don't have any content to translate\n dictionaryToProcess = getFilterMissingTranslationsDictionary(\n dictionaryToProcess,\n targetLocale\n );\n }\n\n dictionaryToProcess = getPerLocaleDictionary(\n dictionaryToProcess,\n task.sourceLocale\n );\n\n targetLocaleDictionary = getPerLocaleDictionary(\n baseUnmergedDictionary,\n targetLocale\n );\n }\n\n const localePreset = colon(\n [\n colorize('[', ANSIColors.GREY_DARK),\n formatLocale(targetLocale),\n colorize(']', ANSIColors.GREY_DARK),\n ].join(''),\n { colSize: 18 }\n );\n\n const createChunkPreset = (\n chunkIndex: number,\n totalChunks: number\n ) => {\n if (totalChunks <= 1) return '';\n return colon(\n [\n colorize('[', ANSIColors.GREY_DARK),\n colorizeNumber(chunkIndex + 1),\n colorize(`/${totalChunks}`, ANSIColors.GREY_DARK),\n colorize(']', ANSIColors.GREY_DARK),\n ].join(''),\n { colSize: 5 }\n );\n };\n\n appLogger(\n `${task.dictionaryPreset}${localePreset} Preparing ${colorizePath(basename(targetLocaleDictionary.filePath!))}`,\n {\n level: 'info',\n }\n );\n\n const chunkedJsonContent: JsonChunk[] = chunkJSON(\n dictionaryToProcess.content,\n CHUNK_SIZE\n );\n\n const nbOfChunks = chunkedJsonContent.length;\n\n if (nbOfChunks > 1) {\n appLogger(\n `${task.dictionaryPreset}${localePreset} Split into ${colorizeNumber(nbOfChunks)} chunks for translation`,\n {\n level: 'info',\n }\n );\n }\n\n const chunkResult: JsonChunk[] = [];\n\n // Process chunks in parallel (globally throttled) to allow concurrent translation\n const chunkPromises = chunkedJsonContent.map((chunk) => {\n const chunkPreset = createChunkPreset(chunk.index, chunk.total);\n\n if (nbOfChunks > 1) {\n appLogger(\n `${task.dictionaryPreset}${localePreset}${chunkPreset} Translating chunk`,\n {\n level: 'info',\n }\n );\n }\n\n // Reconstruct partial JSON content from this chunk's patches\n const chunkContent = reconstructFromSingleChunk(chunk);\n const presetOutputContent = reduceObjectFormat(\n targetLocaleDictionary.content,\n chunkContent\n ) as unknown as JSON;\n\n const executeTranslation = async () => {\n return await retryManager(\n async () => {\n let translationResult: any;\n\n if (aiClient && aiConfig) {\n translationResult = await aiClient.translateJSON({\n entryFileContent: chunkContent as unknown as JSON,\n presetOutputContent,\n dictionaryDescription:\n dictionaryToProcess.description ??\n metadata?.description ??\n '',\n entryLocale: task.sourceLocale,\n outputLocale: targetLocale,\n mode,\n aiConfig,\n });\n } else {\n translationResult = await intlayerAPI.ai\n .translateJSON({\n entryFileContent: chunkContent as unknown as JSON,\n presetOutputContent,\n dictionaryDescription:\n dictionaryToProcess.description ??\n metadata?.description ??\n '',\n entryLocale: task.sourceLocale,\n outputLocale: targetLocale,\n mode,\n aiOptions,\n })\n .then((result) => result.data);\n }\n\n if (!translationResult?.fileContent) {\n throw new Error('No content result');\n }\n\n const { isIdentic } = verifyIdenticObjectFormat(\n translationResult.fileContent,\n chunkContent\n );\n if (!isIdentic) {\n throw new Error(\n 'Translation result does not match expected format'\n );\n }\n\n notifySuccess();\n return translationResult.fileContent;\n },\n {\n maxRetry: MAX_RETRY,\n delay: RETRY_DELAY,\n onError: ({ error, attempt, maxRetry }) => {\n const chunkPreset = createChunkPreset(\n chunk.index,\n chunk.total\n );\n appLogger(\n `${task.dictionaryPreset}${localePreset}${chunkPreset} ${colorize('Error filling:', ANSIColors.RED)} ${colorize(typeof error === 'string' ? error : JSON.stringify(error), ANSIColors.GREY_DARK)} - Attempt ${colorizeNumber(attempt + 1)} of ${colorizeNumber(maxRetry)}`,\n {\n level: 'error',\n }\n );\n\n followingErrors += 1;\n\n if (followingErrors >= MAX_FOLLOWING_ERRORS) {\n appLogger(`There is something wrong.`, {\n level: 'error',\n });\n process.exit(1); // 1 for error\n }\n },\n }\n )();\n };\n\n const wrapped = options?.onHandle\n ? options.onHandle(executeTranslation) // queued in global limiter\n : executeTranslation(); // no global limiter\n\n return wrapped.then((result) => ({ chunk, result }));\n });\n\n // Wait for all chunks for this locale in parallel (still capped by global limiter)\n const chunkResults = await Promise.all(chunkPromises);\n\n // Maintain order\n chunkResults\n .sort((chunkA, chunkB) => chunkA.chunk.index - chunkB.chunk.index)\n .forEach(({ result }) => {\n chunkResult.push(result);\n });\n\n // Merge partial JSON objects produced from each chunk into a single object\n const merged = mergeDictionaries(\n chunkResult.map((chunk) => ({\n ...dictionaryToProcess,\n content: chunk,\n }))\n );\n\n // For per-locale files, merge the newly translated content with existing target content\n let finalContent = merged.content;\n if (typeof baseUnmergedDictionary.locale === 'string') {\n // Deep merge: existing content + newly translated content\n finalContent = deepMergeContent(\n targetLocaleDictionary.content ?? {},\n merged.content\n );\n }\n\n return [targetLocale, finalContent] as const;\n })\n );\n\n const translatedContent: Partial<Record<Locale, Dictionary['content']>> =\n Object.fromEntries(translatedContentResults);\n\n const baseDictionary = baseUnmergedDictionary.locale\n ? {\n ...baseUnmergedDictionary,\n key: baseUnmergedDictionary.key!,\n content: {},\n }\n : baseUnmergedDictionary;\n\n let dictionaryOutput: Dictionary = {\n ...getMultilingualDictionary(baseDictionary),\n locale: undefined, // Ensure the dictionary is multilingual\n ...metadata,\n };\n\n for (const targetLocale of task.targetLocales) {\n if (translatedContent[targetLocale]) {\n dictionaryOutput = insertContentInDictionary(\n dictionaryOutput,\n translatedContent[targetLocale],\n targetLocale\n );\n }\n }\n\n appLogger(\n `${task.dictionaryPreset} ${colorize('Translation completed successfully', ANSIColors.GREEN)} for ${colorizePath(basename(dictionaryOutput.filePath!))}`,\n {\n level: 'info',\n }\n );\n\n if (\n baseUnmergedDictionary.locale &&\n (baseUnmergedDictionary.fill === true ||\n baseUnmergedDictionary.fill === undefined) &&\n baseUnmergedDictionary.location === 'local'\n ) {\n const dictionaryFilePath = baseUnmergedDictionary\n .filePath!.split('.')\n .slice(0, -1);\n\n const contentIndex = dictionaryFilePath[dictionaryFilePath.length - 1];\n\n return JSON.parse(\n JSON.stringify({\n ...task,\n dictionaryOutput: {\n ...dictionaryOutput,\n fill: undefined,\n filled: true,\n },\n }).replaceAll(\n new RegExp(`\\\\.${contentIndex}\\\\.[a-zA-Z0-9]+`, 'g'),\n `.filled.${contentIndex}.json`\n )\n );\n }\n\n return {\n ...task,\n dictionaryOutput,\n };\n },\n {\n maxRetry: GROUP_MAX_RETRY,\n delay: RETRY_DELAY,\n onError: ({ error, attempt, maxRetry }) =>\n appLogger(\n `${task.dictionaryPreset} ${colorize('Error fill command:', ANSIColors.RED)} ${colorize(typeof error === 'string' ? error : JSON.stringify(error), ANSIColors.GREY_DARK)} - Attempt ${colorizeNumber(attempt + 1)} of ${colorizeNumber(maxRetry)}`,\n {\n level: 'error',\n }\n ),\n onMaxTryReached: ({ error }) =>\n appLogger(\n `${task.dictionaryPreset} ${colorize('Maximum number of retries reached:', ANSIColors.RED)} ${colorize(typeof error === 'string' ? error : JSON.stringify(error), ANSIColors.GREY_DARK)}`,\n {\n level: 'error',\n }\n ),\n }\n )();\n\n return result as TranslateDictionaryResult;\n};\n"],"mappings":";;;;;;;;;;;AAkDA,MAAM,sBAAsB,eAC1B,CAAC,WAAW,eAAe,CAAC,WAAW,SAAS,CAAC,WAAW;AAE9D,MAAM,aAAa;AACnB,MAAM,kBAAkB;AACxB,MAAM,YAAY;AAClB,MAAM,cAAc,MAAO;AAE3B,MAAM,uBAAuB;AAC7B,IAAI,kBAAkB;AAEtB,MAAa,sBAAsB,OACjC,MACA,eACA,YACuC;CACvC,MAAM,gDAAyB,cAAc;CAC7C,MAAM,sDAAkC,QAAW,cAAc;CAEjE,MAAM,EAAE,MAAM,WAAW,cAAc,UAAU,aAAa;EAC5D,MAAM;EACN,cAAc;EACd,GAAG;EACJ;CAED,MAAM,sBAAsB;AAC1B,oBAAkB;AAClB,WAAS,aAAa;;AA6ZxB,QA1Ze,0CACb,YAAY;EACV,MAAM,iGAAqD,cAAc;EAEzE,MAAMA,yBACJ,2BAA2B,KAAK,eAAe,MAC5C,SAAS,KAAK,YAAY,KAAK,kBACjC;AAEH,MAAI,CAAC,wBAAwB;AAC3B,aACE,GAAG,KAAK,iBAAiB,gEACzB,EACE,OAAO,QACR,CACF;AACD,UAAO;IAAE,GAAG;IAAM,kBAAkB;IAAM;;EAG5C,IAAIC;AAIJ,MACE,iBACC,mBAAmB,uBAAuB,IAAI,SAAS,WACxD;GACA,MAAM,sEACJ,wBACA,cAAc,qBAAqB,cACpC;AAED,aACE,GAAG,KAAK,iBAAiB,4FAAsD,uBAAuB,SAAU,CAAC,IACjH,EACE,OAAO,QACR,CACF;GAED,MAAM,WAAW,YAAY;AAC3B,QAAI,YAAY,SAMd,QAAO,EACL,MANa,MAAM,SAAS,wBAAwB;KACpD,aAAa,KAAK,UAAU,wBAAwB;KACpD;KACD,CAAC,EAID;AAGH,WAAO,MAAM,YAAY,GAAG,gCAAgC;KAC1D,aAAa,KAAK,UAAU,wBAAwB;KACpD;KACD,CAAC;;AAOJ,eAJuB,SAAS,WAC5B,MAAM,QAAQ,SAAS,SAAS,GAChC,MAAM,UAAU,EAEM,MAAM;;EAGlC,MAAM,2BAA2B,MAAM,QAAQ,IAC7C,KAAK,cAAc,IAAI,OAAO,iBAAiB;;;;;;;;;;GAW7C,IAAI,sBAAsB,gBAAgB,uBAAuB;GAEjE,IAAIC;AAEJ,OAAI,OAAO,uBAAuB,WAAW,UAAU;IAKrD,MAAM,uBACJ,uBAAuB,UAAU,QAC/B,IAAI,OAAO,IAAI,KAAK,aAAa,IAAI,IAAI,EACzC,IAAI,aAAa,GAClB;IAGH,MAAM,2BAA2B,uBAC7B,2BAA2B,KAAK,gBAAgB,MAC7C,SACC,KAAK,aAAa,wBAClB,KAAK,WAAW,aACnB,GACD;AAEJ,6BAAyB,4BAA4B;KACnD,KAAK,uBAAuB;KAC5B,SAAS,EAAE;KACX,UAAU;KACV,QAAQ;KACT;AAGD,QAAI,SAAS,WACX,uBAAsBC,+EACpB,qBACA,yBACD;UAEE;AAEL,QAAI,SAAS,WAEX,mFACE,qBACA,aACD;AAGH,sEACE,qBACA,KAAK,aACN;AAED,yEACE,wBACA,aACD;;GAGH,MAAM,4CACJ;oCACW,KAAKC,6BAAW,UAAU;0CACtB,aAAa;oCACjB,KAAKA,6BAAW,UAAU;IACpC,CAAC,KAAK,GAAG,EACV,EAAE,SAAS,IAAI,CAChB;GAED,MAAM,qBACJ,YACA,gBACG;AACH,QAAI,eAAe,EAAG,QAAO;AAC7B,wCACE;qCACW,KAAKA,6BAAW,UAAU;2CACpB,aAAa,EAAE;qCACrB,IAAI,eAAeA,6BAAW,UAAU;qCACxC,KAAKA,6BAAW,UAAU;KACpC,CAAC,KAAK,GAAG,EACV,EAAE,SAAS,GAAG,CACf;;AAGH,aACE,GAAG,KAAK,mBAAmB,aAAa,yEAAmC,uBAAuB,SAAU,CAAC,IAC7G,EACE,OAAO,QACR,CACF;GAED,MAAMC,wDACJ,oBAAoB,SACpB,WACD;GAED,MAAM,aAAa,mBAAmB;AAEtC,OAAI,aAAa,EACf,WACE,GAAG,KAAK,mBAAmB,aAAa,oDAA6B,WAAW,CAAC,0BACjF,EACE,OAAO,QACR,CACF;GAGH,MAAMC,cAA2B,EAAE;GAGnC,MAAM,gBAAgB,mBAAmB,KAAK,UAAU;IACtD,MAAM,cAAc,kBAAkB,MAAM,OAAO,MAAM,MAAM;AAE/D,QAAI,aAAa,EACf,WACE,GAAG,KAAK,mBAAmB,eAAe,YAAY,qBACtD,EACE,OAAO,QACR,CACF;IAIH,MAAM,mEAA0C,MAAM;IACtD,MAAM,kEACJ,uBAAuB,SACvB,aACD;IAED,MAAM,qBAAqB,YAAY;AACrC,YAAO,0CACL,YAAY;MACV,IAAIC;AAEJ,UAAI,YAAY,SACd,qBAAoB,MAAM,SAAS,cAAc;OAC/C,kBAAkB;OAClB;OACA,uBACE,oBAAoB,eACpB,UAAU,eACV;OACF,aAAa,KAAK;OAClB,cAAc;OACd;OACA;OACD,CAAC;UAEF,qBAAoB,MAAM,YAAY,GACnC,cAAc;OACb,kBAAkB;OAClB;OACA,uBACE,oBAAoB,eACpB,UAAU,eACV;OACF,aAAa,KAAK;OAClB,cAAc;OACd;OACA;OACD,CAAC,CACD,MAAM,WAAW,OAAO,KAAK;AAGlC,UAAI,CAAC,mBAAmB,YACtB,OAAM,IAAI,MAAM,oBAAoB;MAGtC,MAAM,EAAE,iEACN,kBAAkB,aAClB,aACD;AACD,UAAI,CAAC,UACH,OAAM,IAAI,MACR,oDACD;AAGH,qBAAe;AACf,aAAO,kBAAkB;QAE3B;MACE,UAAU;MACV,OAAO;MACP,UAAU,EAAE,OAAO,SAAS,eAAe;OACzC,MAAMC,gBAAc,kBAClB,MAAM,OACN,MAAM,MACP;AACD,iBACE,GAAG,KAAK,mBAAmB,eAAeA,cAAY,mCAAY,kBAAkBJ,6BAAW,IAAI,CAAC,mCAAY,OAAO,UAAU,WAAW,QAAQ,KAAK,UAAU,MAAM,EAAEA,6BAAW,UAAU,CAAC,mDAA4B,UAAU,EAAE,CAAC,4CAAqB,SAAS,IACxQ,EACE,OAAO,SACR,CACF;AAED,0BAAmB;AAEnB,WAAI,mBAAmB,sBAAsB;AAC3C,kBAAU,6BAA6B,EACrC,OAAO,SACR,CAAC;AACF,gBAAQ,KAAK,EAAE;;;MAGpB,CACF,EAAE;;AAOL,YAJgB,SAAS,WACrB,QAAQ,SAAS,mBAAmB,GACpC,oBAAoB,EAET,MAAM,YAAY;KAAE;KAAO;KAAQ,EAAE;KACpD;AAMF,IAHqB,MAAM,QAAQ,IAAI,cAAc,EAIlD,MAAM,QAAQ,WAAW,OAAO,MAAM,QAAQ,OAAO,MAAM,MAAM,CACjE,SAAS,EAAE,aAAa;AACvB,gBAAY,KAAK,OAAO;KACxB;GAGJ,MAAM,gDACJ,YAAY,KAAK,WAAW;IAC1B,GAAG;IACH,SAAS;IACV,EAAE,CACJ;GAGD,IAAI,eAAe,OAAO;AAC1B,OAAI,OAAO,uBAAuB,WAAW,SAE3C,gBAAeK,+CACb,uBAAuB,WAAW,EAAE,EACpC,OAAO,QACR;AAGH,UAAO,CAAC,cAAc,aAAa;IACnC,CACH;EAED,MAAMC,oBACJ,OAAO,YAAY,yBAAyB;EAU9C,IAAIC,mBAA+B;GACjC,kDATqB,uBAAuB,SAC1C;IACE,GAAG;IACH,KAAK,uBAAuB;IAC5B,SAAS,EAAE;IACZ,GACD,uBAG0C;GAC5C,QAAQ;GACR,GAAG;GACJ;AAED,OAAK,MAAM,gBAAgB,KAAK,cAC9B,KAAI,kBAAkB,cACpB,mEACE,kBACA,kBAAkB,eAClB,aACD;AAIL,YACE,GAAG,KAAK,iBAAiB,mCAAY,sCAAsCP,6BAAW,MAAM,CAAC,mEAA6B,iBAAiB,SAAU,CAAC,IACtJ,EACE,OAAO,QACR,CACF;AAED,MACE,uBAAuB,WACtB,uBAAuB,SAAS,QAC/B,uBAAuB,SAAS,WAClC,uBAAuB,aAAa,SACpC;GACA,MAAM,qBAAqB,uBACxB,SAAU,MAAM,IAAI,CACpB,MAAM,GAAG,GAAG;GAEf,MAAM,eAAe,mBAAmB,mBAAmB,SAAS;AAEpE,UAAO,KAAK,MACV,KAAK,UAAU;IACb,GAAG;IACH,kBAAkB;KAChB,GAAG;KACH,MAAM;KACN,QAAQ;KACT;IACF,CAAC,CAAC,WACD,IAAI,OAAO,MAAM,aAAa,kBAAkB,IAAI,EACpD,WAAW,aAAa,OACzB,CACF;;AAGH,SAAO;GACL,GAAG;GACH;GACD;IAEH;EACE,UAAU;EACV,OAAO;EACP,UAAU,EAAE,OAAO,SAAS,eAC1B,UACE,GAAG,KAAK,iBAAiB,mCAAY,uBAAuBA,6BAAW,IAAI,CAAC,mCAAY,OAAO,UAAU,WAAW,QAAQ,KAAK,UAAU,MAAM,EAAEA,6BAAW,UAAU,CAAC,mDAA4B,UAAU,EAAE,CAAC,4CAAqB,SAAS,IAChP,EACE,OAAO,SACR,CACF;EACH,kBAAkB,EAAE,YAClB,UACE,GAAG,KAAK,iBAAiB,mCAAY,sCAAsCA,6BAAW,IAAI,CAAC,mCAAY,OAAO,UAAU,WAAW,QAAQ,KAAK,UAAU,MAAM,EAAEA,6BAAW,UAAU,IACvL,EACE,OAAO,SACR,CACF;EACJ,CACF,EAAE"}
|
|
1
|
+
{"version":3,"file":"translateDictionary.cjs","names":["baseUnmergedDictionary: Dictionary | undefined","metadata:\n | Pick<Dictionary, 'description' | 'title' | 'tags'>\n | undefined","targetLocaleDictionary: Dictionary","getFilterMissingContentPerLocale","ANSIColors","chunkedJsonContent: JsonChunk[]","chunkResult: JsonChunk[]","translationResult: any","chunkPreset","deepMergeContent","translatedContent: Partial<Record<Locale, Dictionary['content']>>","dictionaryOutput: Dictionary"],"sources":["../../../src/fill/translateDictionary.ts"],"sourcesContent":["import { basename } from 'node:path';\nimport type { AIConfig } from '@intlayer/ai';\nimport { type AIOptions, getIntlayerAPIProxy } from '@intlayer/api';\nimport {\n chunkJSON,\n formatLocale,\n type JsonChunk,\n reconstructFromSingleChunk,\n reduceObjectFormat,\n verifyIdenticObjectFormat,\n} from '@intlayer/chokidar';\nimport {\n ANSIColors,\n colon,\n colorize,\n colorizeNumber,\n colorizePath,\n getAppLogger,\n retryManager,\n} from '@intlayer/config';\nimport {\n getFilterMissingTranslationsDictionary,\n getMultilingualDictionary,\n getPerLocaleDictionary,\n insertContentInDictionary,\n mergeDictionaries,\n} from '@intlayer/core';\nimport type { Dictionary, IntlayerConfig, Locale } from '@intlayer/types';\nimport { getUnmergedDictionaries } from '@intlayer/unmerged-dictionaries-entry';\nimport type { AIClient } from '../utils/setupAI';\nimport { deepMergeContent } from './deepMergeContent';\nimport { getFilterMissingContentPerLocale } from './getFilterMissingContentPerLocale';\nimport type { TranslationTask } from './listTranslationsTasks';\n\ntype TranslateDictionaryResult = TranslationTask & {\n dictionaryOutput: Dictionary | null;\n};\n\ntype TranslateDictionaryOptions = {\n mode: 'complete' | 'review';\n aiOptions?: AIOptions;\n fillMetadata?: boolean;\n onHandle?: ReturnType<typeof import('@intlayer/chokidar').getGlobalLimiter>;\n onSuccess?: () => void;\n onError?: (error: unknown) => void;\n getAbortError?: () => Error | null;\n aiClient?: AIClient;\n aiConfig?: AIConfig;\n};\n\nconst hasMissingMetadata = (dictionary: Dictionary) =>\n !dictionary.description || !dictionary.title || !dictionary.tags;\n\nconst CHUNK_SIZE = 7000; // GPT-5 Mini safe input size\nconst GROUP_MAX_RETRY = 2;\nconst MAX_RETRY = 3;\nconst RETRY_DELAY = 1000 * 10; // 10 seconds\n\nconst MAX_FOLLOWING_ERRORS = 10; // 10 errors in a row, hard exit the process\nlet followingErrors = 0;\n\nexport const translateDictionary = async (\n task: TranslationTask,\n configuration: IntlayerConfig,\n options?: TranslateDictionaryOptions\n): Promise<TranslateDictionaryResult> => {\n const appLogger = getAppLogger(configuration);\n const intlayerAPI = getIntlayerAPIProxy(undefined, configuration);\n\n const { mode, aiOptions, fillMetadata, aiClient, aiConfig } = {\n mode: 'complete',\n fillMetadata: true,\n ...options,\n } as const;\n\n const notifySuccess = () => {\n followingErrors = 0;\n options?.onSuccess?.();\n };\n\n const result = await retryManager(\n async () => {\n const unmergedDictionariesRecord = getUnmergedDictionaries(configuration);\n\n const baseUnmergedDictionary: Dictionary | undefined =\n unmergedDictionariesRecord[task.dictionaryKey].find(\n (dict) => dict.localId === task.dictionaryLocalId\n );\n\n if (!baseUnmergedDictionary) {\n appLogger(\n `${task.dictionaryPreset}Dictionary not found in unmergedDictionariesRecord. Skipping.`,\n {\n level: 'warn',\n }\n );\n return { ...task, dictionaryOutput: null };\n }\n\n let metadata:\n | Pick<Dictionary, 'description' | 'title' | 'tags'>\n | undefined;\n\n if (\n fillMetadata &&\n (hasMissingMetadata(baseUnmergedDictionary) || mode === 'review')\n ) {\n const defaultLocaleDictionary = getPerLocaleDictionary(\n baseUnmergedDictionary,\n configuration.internationalization.defaultLocale\n );\n\n appLogger(\n `${task.dictionaryPreset} Filling missing metadata for ${colorizePath(basename(baseUnmergedDictionary.filePath!))}`,\n {\n level: 'info',\n }\n );\n\n const runAudit = async () => {\n if (aiClient && aiConfig) {\n const result = await aiClient.auditDictionaryMetadata({\n fileContent: JSON.stringify(defaultLocaleDictionary),\n aiConfig,\n });\n\n return {\n data: result,\n };\n }\n\n return await intlayerAPI.ai.auditContentDeclarationMetadata({\n fileContent: JSON.stringify(defaultLocaleDictionary),\n aiOptions,\n });\n };\n\n const metadataResult = options?.onHandle\n ? await options.onHandle(runAudit)\n : await runAudit();\n\n metadata = metadataResult.data?.fileContent;\n }\n\n const translatedContentResults = await Promise.all(\n task.targetLocales.map(async (targetLocale) => {\n /**\n * In complete mode, for large dictionaries, we want to filter all content that is already translated\n *\n * targetLocale: fr\n *\n * { test1: t({ ar: 'Hello', en: 'Hello', fr: 'Bonjour' } }) -> {}\n * { test2: t({ ar: 'Hello', en: 'Hello' }) } -> { test2: t({ ar: 'Hello', en: 'Hello' }) }\n *\n */\n // Reset to base dictionary for each locale to ensure we filter from the original\n let dictionaryToProcess = structuredClone(baseUnmergedDictionary);\n\n let targetLocaleDictionary: Dictionary;\n\n if (typeof baseUnmergedDictionary.locale === 'string') {\n // For per-locale files, the content is already in simple JSON format (not translation nodes)\n // The base dictionary is already the source locale content\n\n // Load the existing target locale dictionary\n const targetLocaleFilePath =\n baseUnmergedDictionary.filePath?.replace(\n new RegExp(`/${task.sourceLocale}/`, 'g'),\n `/${targetLocale}/`\n );\n\n // Find the target locale dictionary in unmerged dictionaries\n const targetUnmergedDictionary = targetLocaleFilePath\n ? unmergedDictionariesRecord[task.dictionaryKey]?.find(\n (dict) =>\n dict.filePath === targetLocaleFilePath &&\n dict.locale === targetLocale\n )\n : undefined;\n\n targetLocaleDictionary = targetUnmergedDictionary ?? {\n key: baseUnmergedDictionary.key,\n content: {},\n filePath: targetLocaleFilePath,\n locale: targetLocale,\n };\n\n // In complete mode, filter out already translated content\n if (mode === 'complete') {\n dictionaryToProcess = getFilterMissingContentPerLocale(\n dictionaryToProcess,\n targetUnmergedDictionary\n );\n }\n } else {\n // For multilingual dictionaries\n if (mode === 'complete') {\n // Remove all nodes that don't have any content to translate\n dictionaryToProcess = getFilterMissingTranslationsDictionary(\n dictionaryToProcess,\n targetLocale\n );\n }\n\n dictionaryToProcess = getPerLocaleDictionary(\n dictionaryToProcess,\n task.sourceLocale\n );\n\n targetLocaleDictionary = getPerLocaleDictionary(\n baseUnmergedDictionary,\n targetLocale\n );\n }\n\n const localePreset = colon(\n [\n colorize('[', ANSIColors.GREY_DARK),\n formatLocale(targetLocale),\n colorize(']', ANSIColors.GREY_DARK),\n ].join(''),\n { colSize: 18 }\n );\n\n const createChunkPreset = (\n chunkIndex: number,\n totalChunks: number\n ) => {\n if (totalChunks <= 1) return '';\n return colon(\n [\n colorize('[', ANSIColors.GREY_DARK),\n colorizeNumber(chunkIndex + 1),\n colorize(`/${totalChunks}`, ANSIColors.GREY_DARK),\n colorize(']', ANSIColors.GREY_DARK),\n ].join(''),\n { colSize: 5 }\n );\n };\n\n appLogger(\n `${task.dictionaryPreset}${localePreset} Preparing ${colorizePath(basename(targetLocaleDictionary.filePath!))}`,\n {\n level: 'info',\n }\n );\n\n const isContentStructured =\n (typeof dictionaryToProcess.content === 'object' &&\n dictionaryToProcess.content !== null) ||\n Array.isArray(dictionaryToProcess.content);\n\n const contentToProcess = isContentStructured\n ? dictionaryToProcess.content\n : {\n __INTLAYER_ROOT_PRIMITIVE_CONTENT__:\n dictionaryToProcess.content,\n };\n\n const chunkedJsonContent: JsonChunk[] = chunkJSON(\n contentToProcess as unknown as Record<string, any>,\n CHUNK_SIZE\n );\n\n const nbOfChunks = chunkedJsonContent.length;\n\n if (nbOfChunks > 1) {\n appLogger(\n `${task.dictionaryPreset}${localePreset} Split into ${colorizeNumber(nbOfChunks)} chunks for translation`,\n {\n level: 'info',\n }\n );\n }\n\n const chunkResult: JsonChunk[] = [];\n\n // Process chunks in parallel (globally throttled) to allow concurrent translation\n const chunkPromises = chunkedJsonContent.map((chunk) => {\n const chunkPreset = createChunkPreset(chunk.index, chunk.total);\n\n if (nbOfChunks > 1) {\n appLogger(\n `${task.dictionaryPreset}${localePreset}${chunkPreset} Translating chunk`,\n {\n level: 'info',\n }\n );\n }\n\n // Reconstruct partial JSON content from this chunk's patches\n const chunkContent = reconstructFromSingleChunk(chunk);\n const presetOutputContent = reduceObjectFormat(\n isContentStructured\n ? targetLocaleDictionary.content\n : {\n __INTLAYER_ROOT_PRIMITIVE_CONTENT__:\n targetLocaleDictionary.content,\n },\n chunkContent\n ) as unknown as JSON;\n\n const executeTranslation = async () => {\n return await retryManager(\n async () => {\n let translationResult: any;\n\n if (aiClient && aiConfig) {\n translationResult = await aiClient.translateJSON({\n entryFileContent: chunkContent as unknown as JSON,\n presetOutputContent,\n dictionaryDescription:\n dictionaryToProcess.description ??\n metadata?.description ??\n '',\n entryLocale: task.sourceLocale,\n outputLocale: targetLocale,\n mode,\n aiConfig,\n });\n } else {\n translationResult = await intlayerAPI.ai\n .translateJSON({\n entryFileContent: chunkContent as unknown as JSON,\n presetOutputContent,\n dictionaryDescription:\n dictionaryToProcess.description ??\n metadata?.description ??\n '',\n entryLocale: task.sourceLocale,\n outputLocale: targetLocale,\n mode,\n aiOptions,\n })\n .then((result) => result.data);\n }\n\n if (!translationResult?.fileContent) {\n throw new Error('No content result');\n }\n\n const { isIdentic } = verifyIdenticObjectFormat(\n translationResult.fileContent,\n chunkContent\n );\n if (!isIdentic) {\n throw new Error(\n 'Translation result does not match expected format'\n );\n }\n\n notifySuccess();\n return translationResult.fileContent;\n },\n {\n maxRetry: MAX_RETRY,\n delay: RETRY_DELAY,\n onError: ({ error, attempt, maxRetry }) => {\n const chunkPreset = createChunkPreset(\n chunk.index,\n chunk.total\n );\n appLogger(\n `${task.dictionaryPreset}${localePreset}${chunkPreset} ${colorize('Error filling:', ANSIColors.RED)} ${colorize(typeof error === 'string' ? error : JSON.stringify(error), ANSIColors.GREY_DARK)} - Attempt ${colorizeNumber(attempt + 1)} of ${colorizeNumber(maxRetry)}`,\n {\n level: 'error',\n }\n );\n\n followingErrors += 1;\n\n if (followingErrors >= MAX_FOLLOWING_ERRORS) {\n appLogger(`There is something wrong.`, {\n level: 'error',\n });\n process.exit(1); // 1 for error\n }\n },\n }\n )();\n };\n\n const wrapped = options?.onHandle\n ? options.onHandle(executeTranslation) // queued in global limiter\n : executeTranslation(); // no global limiter\n\n return wrapped.then((result) => ({ chunk, result }));\n });\n\n // Wait for all chunks for this locale in parallel (still capped by global limiter)\n const chunkResults = await Promise.all(chunkPromises);\n\n // Maintain order\n chunkResults\n .sort((chunkA, chunkB) => chunkA.chunk.index - chunkB.chunk.index)\n .forEach(({ result }) => {\n chunkResult.push(result);\n });\n\n // Merge partial JSON objects produced from each chunk into a single object\n const merged = mergeDictionaries(\n chunkResult.map((chunk) => ({\n ...dictionaryToProcess,\n content: chunk,\n }))\n );\n\n // For per-locale files, merge the newly translated content with existing target content\n let finalContent = merged.content;\n\n if (!isContentStructured) {\n finalContent = (finalContent as any)\n ?.__INTLAYER_ROOT_PRIMITIVE_CONTENT__;\n }\n\n if (typeof baseUnmergedDictionary.locale === 'string') {\n // Deep merge: existing content + newly translated content\n finalContent = deepMergeContent(\n targetLocaleDictionary.content ?? {},\n finalContent\n );\n }\n\n return [targetLocale, finalContent] as const;\n })\n );\n\n const translatedContent: Partial<Record<Locale, Dictionary['content']>> =\n Object.fromEntries(translatedContentResults);\n\n const baseDictionary = baseUnmergedDictionary.locale\n ? {\n ...baseUnmergedDictionary,\n key: baseUnmergedDictionary.key!,\n content: {},\n }\n : baseUnmergedDictionary;\n\n let dictionaryOutput: Dictionary = {\n ...getMultilingualDictionary(baseDictionary),\n locale: undefined, // Ensure the dictionary is multilingual\n ...metadata,\n };\n\n for (const targetLocale of task.targetLocales) {\n if (translatedContent[targetLocale]) {\n dictionaryOutput = insertContentInDictionary(\n dictionaryOutput,\n translatedContent[targetLocale],\n targetLocale\n );\n }\n }\n\n appLogger(\n `${task.dictionaryPreset} ${colorize('Translation completed successfully', ANSIColors.GREEN)} for ${colorizePath(basename(dictionaryOutput.filePath!))}`,\n {\n level: 'info',\n }\n );\n\n if (\n baseUnmergedDictionary.locale &&\n (baseUnmergedDictionary.fill === true ||\n baseUnmergedDictionary.fill === undefined) &&\n baseUnmergedDictionary.location === 'local'\n ) {\n const dictionaryFilePath = baseUnmergedDictionary\n .filePath!.split('.')\n .slice(0, -1);\n\n const contentIndex = dictionaryFilePath[dictionaryFilePath.length - 1];\n\n return JSON.parse(\n JSON.stringify({\n ...task,\n dictionaryOutput: {\n ...dictionaryOutput,\n fill: undefined,\n filled: true,\n },\n }).replaceAll(\n new RegExp(`\\\\.${contentIndex}\\\\.[a-zA-Z0-9]+`, 'g'),\n `.filled.${contentIndex}.json`\n )\n );\n }\n\n return {\n ...task,\n dictionaryOutput,\n };\n },\n {\n maxRetry: GROUP_MAX_RETRY,\n delay: RETRY_DELAY,\n onError: ({ error, attempt, maxRetry }) =>\n appLogger(\n `${task.dictionaryPreset} ${colorize('Error fill command:', ANSIColors.RED)} ${colorize(typeof error === 'string' ? error : JSON.stringify(error), ANSIColors.GREY_DARK)} - Attempt ${colorizeNumber(attempt + 1)} of ${colorizeNumber(maxRetry)}`,\n {\n level: 'error',\n }\n ),\n onMaxTryReached: ({ error }) =>\n appLogger(\n `${task.dictionaryPreset} ${colorize('Maximum number of retries reached:', ANSIColors.RED)} ${colorize(typeof error === 'string' ? error : JSON.stringify(error), ANSIColors.GREY_DARK)}`,\n {\n level: 'error',\n }\n ),\n }\n )();\n\n return result as TranslateDictionaryResult;\n};\n"],"mappings":";;;;;;;;;;;AAkDA,MAAM,sBAAsB,eAC1B,CAAC,WAAW,eAAe,CAAC,WAAW,SAAS,CAAC,WAAW;AAE9D,MAAM,aAAa;AACnB,MAAM,kBAAkB;AACxB,MAAM,YAAY;AAClB,MAAM,cAAc,MAAO;AAE3B,MAAM,uBAAuB;AAC7B,IAAI,kBAAkB;AAEtB,MAAa,sBAAsB,OACjC,MACA,eACA,YACuC;CACvC,MAAM,gDAAyB,cAAc;CAC7C,MAAM,sDAAkC,QAAW,cAAc;CAEjE,MAAM,EAAE,MAAM,WAAW,cAAc,UAAU,aAAa;EAC5D,MAAM;EACN,cAAc;EACd,GAAG;EACJ;CAED,MAAM,sBAAsB;AAC1B,oBAAkB;AAClB,WAAS,aAAa;;AAobxB,QAjbe,0CACb,YAAY;EACV,MAAM,iGAAqD,cAAc;EAEzE,MAAMA,yBACJ,2BAA2B,KAAK,eAAe,MAC5C,SAAS,KAAK,YAAY,KAAK,kBACjC;AAEH,MAAI,CAAC,wBAAwB;AAC3B,aACE,GAAG,KAAK,iBAAiB,gEACzB,EACE,OAAO,QACR,CACF;AACD,UAAO;IAAE,GAAG;IAAM,kBAAkB;IAAM;;EAG5C,IAAIC;AAIJ,MACE,iBACC,mBAAmB,uBAAuB,IAAI,SAAS,WACxD;GACA,MAAM,sEACJ,wBACA,cAAc,qBAAqB,cACpC;AAED,aACE,GAAG,KAAK,iBAAiB,4FAAsD,uBAAuB,SAAU,CAAC,IACjH,EACE,OAAO,QACR,CACF;GAED,MAAM,WAAW,YAAY;AAC3B,QAAI,YAAY,SAMd,QAAO,EACL,MANa,MAAM,SAAS,wBAAwB;KACpD,aAAa,KAAK,UAAU,wBAAwB;KACpD;KACD,CAAC,EAID;AAGH,WAAO,MAAM,YAAY,GAAG,gCAAgC;KAC1D,aAAa,KAAK,UAAU,wBAAwB;KACpD;KACD,CAAC;;AAOJ,eAJuB,SAAS,WAC5B,MAAM,QAAQ,SAAS,SAAS,GAChC,MAAM,UAAU,EAEM,MAAM;;EAGlC,MAAM,2BAA2B,MAAM,QAAQ,IAC7C,KAAK,cAAc,IAAI,OAAO,iBAAiB;;;;;;;;;;GAW7C,IAAI,sBAAsB,gBAAgB,uBAAuB;GAEjE,IAAIC;AAEJ,OAAI,OAAO,uBAAuB,WAAW,UAAU;IAKrD,MAAM,uBACJ,uBAAuB,UAAU,QAC/B,IAAI,OAAO,IAAI,KAAK,aAAa,IAAI,IAAI,EACzC,IAAI,aAAa,GAClB;IAGH,MAAM,2BAA2B,uBAC7B,2BAA2B,KAAK,gBAAgB,MAC7C,SACC,KAAK,aAAa,wBAClB,KAAK,WAAW,aACnB,GACD;AAEJ,6BAAyB,4BAA4B;KACnD,KAAK,uBAAuB;KAC5B,SAAS,EAAE;KACX,UAAU;KACV,QAAQ;KACT;AAGD,QAAI,SAAS,WACX,uBAAsBC,+EACpB,qBACA,yBACD;UAEE;AAEL,QAAI,SAAS,WAEX,mFACE,qBACA,aACD;AAGH,sEACE,qBACA,KAAK,aACN;AAED,yEACE,wBACA,aACD;;GAGH,MAAM,4CACJ;oCACW,KAAKC,6BAAW,UAAU;0CACtB,aAAa;oCACjB,KAAKA,6BAAW,UAAU;IACpC,CAAC,KAAK,GAAG,EACV,EAAE,SAAS,IAAI,CAChB;GAED,MAAM,qBACJ,YACA,gBACG;AACH,QAAI,eAAe,EAAG,QAAO;AAC7B,wCACE;qCACW,KAAKA,6BAAW,UAAU;2CACpB,aAAa,EAAE;qCACrB,IAAI,eAAeA,6BAAW,UAAU;qCACxC,KAAKA,6BAAW,UAAU;KACpC,CAAC,KAAK,GAAG,EACV,EAAE,SAAS,GAAG,CACf;;AAGH,aACE,GAAG,KAAK,mBAAmB,aAAa,yEAAmC,uBAAuB,SAAU,CAAC,IAC7G,EACE,OAAO,QACR,CACF;GAED,MAAM,sBACH,OAAO,oBAAoB,YAAY,YACtC,oBAAoB,YAAY,QAClC,MAAM,QAAQ,oBAAoB,QAAQ;GAS5C,MAAMC,wDAPmB,sBACrB,oBAAoB,UACpB,EACE,qCACE,oBAAoB,SACvB,EAIH,WACD;GAED,MAAM,aAAa,mBAAmB;AAEtC,OAAI,aAAa,EACf,WACE,GAAG,KAAK,mBAAmB,aAAa,oDAA6B,WAAW,CAAC,0BACjF,EACE,OAAO,QACR,CACF;GAGH,MAAMC,cAA2B,EAAE;GAGnC,MAAM,gBAAgB,mBAAmB,KAAK,UAAU;IACtD,MAAM,cAAc,kBAAkB,MAAM,OAAO,MAAM,MAAM;AAE/D,QAAI,aAAa,EACf,WACE,GAAG,KAAK,mBAAmB,eAAe,YAAY,qBACtD,EACE,OAAO,QACR,CACF;IAIH,MAAM,mEAA0C,MAAM;IACtD,MAAM,kEACJ,sBACI,uBAAuB,UACvB,EACE,qCACE,uBAAuB,SAC1B,EACL,aACD;IAED,MAAM,qBAAqB,YAAY;AACrC,YAAO,0CACL,YAAY;MACV,IAAIC;AAEJ,UAAI,YAAY,SACd,qBAAoB,MAAM,SAAS,cAAc;OAC/C,kBAAkB;OAClB;OACA,uBACE,oBAAoB,eACpB,UAAU,eACV;OACF,aAAa,KAAK;OAClB,cAAc;OACd;OACA;OACD,CAAC;UAEF,qBAAoB,MAAM,YAAY,GACnC,cAAc;OACb,kBAAkB;OAClB;OACA,uBACE,oBAAoB,eACpB,UAAU,eACV;OACF,aAAa,KAAK;OAClB,cAAc;OACd;OACA;OACD,CAAC,CACD,MAAM,WAAW,OAAO,KAAK;AAGlC,UAAI,CAAC,mBAAmB,YACtB,OAAM,IAAI,MAAM,oBAAoB;MAGtC,MAAM,EAAE,iEACN,kBAAkB,aAClB,aACD;AACD,UAAI,CAAC,UACH,OAAM,IAAI,MACR,oDACD;AAGH,qBAAe;AACf,aAAO,kBAAkB;QAE3B;MACE,UAAU;MACV,OAAO;MACP,UAAU,EAAE,OAAO,SAAS,eAAe;OACzC,MAAMC,gBAAc,kBAClB,MAAM,OACN,MAAM,MACP;AACD,iBACE,GAAG,KAAK,mBAAmB,eAAeA,cAAY,mCAAY,kBAAkBJ,6BAAW,IAAI,CAAC,mCAAY,OAAO,UAAU,WAAW,QAAQ,KAAK,UAAU,MAAM,EAAEA,6BAAW,UAAU,CAAC,mDAA4B,UAAU,EAAE,CAAC,4CAAqB,SAAS,IACxQ,EACE,OAAO,SACR,CACF;AAED,0BAAmB;AAEnB,WAAI,mBAAmB,sBAAsB;AAC3C,kBAAU,6BAA6B,EACrC,OAAO,SACR,CAAC;AACF,gBAAQ,KAAK,EAAE;;;MAGpB,CACF,EAAE;;AAOL,YAJgB,SAAS,WACrB,QAAQ,SAAS,mBAAmB,GACpC,oBAAoB,EAET,MAAM,YAAY;KAAE;KAAO;KAAQ,EAAE;KACpD;AAMF,IAHqB,MAAM,QAAQ,IAAI,cAAc,EAIlD,MAAM,QAAQ,WAAW,OAAO,MAAM,QAAQ,OAAO,MAAM,MAAM,CACjE,SAAS,EAAE,aAAa;AACvB,gBAAY,KAAK,OAAO;KACxB;GAWJ,IAAI,sDAPF,YAAY,KAAK,WAAW;IAC1B,GAAG;IACH,SAAS;IACV,EAAE,CACJ,CAGyB;AAE1B,OAAI,CAAC,oBACH,gBAAgB,cACZ;AAGN,OAAI,OAAO,uBAAuB,WAAW,SAE3C,gBAAeK,+CACb,uBAAuB,WAAW,EAAE,EACpC,aACD;AAGH,UAAO,CAAC,cAAc,aAAa;IACnC,CACH;EAED,MAAMC,oBACJ,OAAO,YAAY,yBAAyB;EAU9C,IAAIC,mBAA+B;GACjC,kDATqB,uBAAuB,SAC1C;IACE,GAAG;IACH,KAAK,uBAAuB;IAC5B,SAAS,EAAE;IACZ,GACD,uBAG0C;GAC5C,QAAQ;GACR,GAAG;GACJ;AAED,OAAK,MAAM,gBAAgB,KAAK,cAC9B,KAAI,kBAAkB,cACpB,mEACE,kBACA,kBAAkB,eAClB,aACD;AAIL,YACE,GAAG,KAAK,iBAAiB,mCAAY,sCAAsCP,6BAAW,MAAM,CAAC,mEAA6B,iBAAiB,SAAU,CAAC,IACtJ,EACE,OAAO,QACR,CACF;AAED,MACE,uBAAuB,WACtB,uBAAuB,SAAS,QAC/B,uBAAuB,SAAS,WAClC,uBAAuB,aAAa,SACpC;GACA,MAAM,qBAAqB,uBACxB,SAAU,MAAM,IAAI,CACpB,MAAM,GAAG,GAAG;GAEf,MAAM,eAAe,mBAAmB,mBAAmB,SAAS;AAEpE,UAAO,KAAK,MACV,KAAK,UAAU;IACb,GAAG;IACH,kBAAkB;KAChB,GAAG;KACH,MAAM;KACN,QAAQ;KACT;IACF,CAAC,CAAC,WACD,IAAI,OAAO,MAAM,aAAa,kBAAkB,IAAI,EACpD,WAAW,aAAa,OACzB,CACF;;AAGH,SAAO;GACL,GAAG;GACH;GACD;IAEH;EACE,UAAU;EACV,OAAO;EACP,UAAU,EAAE,OAAO,SAAS,eAC1B,UACE,GAAG,KAAK,iBAAiB,mCAAY,uBAAuBA,6BAAW,IAAI,CAAC,mCAAY,OAAO,UAAU,WAAW,QAAQ,KAAK,UAAU,MAAM,EAAEA,6BAAW,UAAU,CAAC,mDAA4B,UAAU,EAAE,CAAC,4CAAqB,SAAS,IAChP,EACE,OAAO,SACR,CACF;EACH,kBAAkB,EAAE,YAClB,UACE,GAAG,KAAK,iBAAiB,mCAAY,sCAAsCA,6BAAW,IAAI,CAAC,mCAAY,OAAO,UAAU,WAAW,QAAQ,KAAK,UAAU,MAAM,EAAEA,6BAAW,UAAU,IACvL,EACE,OAAO,SACR,CACF;EACJ,CACF,EAAE"}
|
|
@@ -95,7 +95,8 @@ const translateDictionary = async (task, configuration, options) => {
|
|
|
95
95
|
].join(""), { colSize: 5 });
|
|
96
96
|
};
|
|
97
97
|
appLogger(`${task.dictionaryPreset}${localePreset} Preparing ${colorizePath(basename(targetLocaleDictionary.filePath))}`, { level: "info" });
|
|
98
|
-
const
|
|
98
|
+
const isContentStructured = typeof dictionaryToProcess.content === "object" && dictionaryToProcess.content !== null || Array.isArray(dictionaryToProcess.content);
|
|
99
|
+
const chunkedJsonContent = chunkJSON(isContentStructured ? dictionaryToProcess.content : { __INTLAYER_ROOT_PRIMITIVE_CONTENT__: dictionaryToProcess.content }, CHUNK_SIZE);
|
|
99
100
|
const nbOfChunks = chunkedJsonContent.length;
|
|
100
101
|
if (nbOfChunks > 1) appLogger(`${task.dictionaryPreset}${localePreset} Split into ${colorizeNumber(nbOfChunks)} chunks for translation`, { level: "info" });
|
|
101
102
|
const chunkResult = [];
|
|
@@ -103,7 +104,7 @@ const translateDictionary = async (task, configuration, options) => {
|
|
|
103
104
|
const chunkPreset = createChunkPreset(chunk.index, chunk.total);
|
|
104
105
|
if (nbOfChunks > 1) appLogger(`${task.dictionaryPreset}${localePreset}${chunkPreset} Translating chunk`, { level: "info" });
|
|
105
106
|
const chunkContent = reconstructFromSingleChunk(chunk);
|
|
106
|
-
const presetOutputContent = reduceObjectFormat(targetLocaleDictionary.content, chunkContent);
|
|
107
|
+
const presetOutputContent = reduceObjectFormat(isContentStructured ? targetLocaleDictionary.content : { __INTLAYER_ROOT_PRIMITIVE_CONTENT__: targetLocaleDictionary.content }, chunkContent);
|
|
107
108
|
const executeTranslation = async () => {
|
|
108
109
|
return await retryManager(async () => {
|
|
109
110
|
let translationResult;
|
|
@@ -152,12 +153,12 @@ const translateDictionary = async (task, configuration, options) => {
|
|
|
152
153
|
(await Promise.all(chunkPromises)).sort((chunkA, chunkB) => chunkA.chunk.index - chunkB.chunk.index).forEach(({ result }) => {
|
|
153
154
|
chunkResult.push(result);
|
|
154
155
|
});
|
|
155
|
-
|
|
156
|
+
let finalContent = mergeDictionaries(chunkResult.map((chunk) => ({
|
|
156
157
|
...dictionaryToProcess,
|
|
157
158
|
content: chunk
|
|
158
|
-
})));
|
|
159
|
-
|
|
160
|
-
if (typeof baseUnmergedDictionary.locale === "string") finalContent = deepMergeContent(targetLocaleDictionary.content ?? {},
|
|
159
|
+
}))).content;
|
|
160
|
+
if (!isContentStructured) finalContent = finalContent?.__INTLAYER_ROOT_PRIMITIVE_CONTENT__;
|
|
161
|
+
if (typeof baseUnmergedDictionary.locale === "string") finalContent = deepMergeContent(targetLocaleDictionary.content ?? {}, finalContent);
|
|
161
162
|
return [targetLocale, finalContent];
|
|
162
163
|
}));
|
|
163
164
|
const translatedContent = Object.fromEntries(translatedContentResults);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"translateDictionary.mjs","names":["baseUnmergedDictionary: Dictionary | undefined","metadata:\n | Pick<Dictionary, 'description' | 'title' | 'tags'>\n | undefined","targetLocaleDictionary: Dictionary","chunkedJsonContent: JsonChunk[]","chunkResult: JsonChunk[]","translationResult: any","chunkPreset","translatedContent: Partial<Record<Locale, Dictionary['content']>>","dictionaryOutput: Dictionary"],"sources":["../../../src/fill/translateDictionary.ts"],"sourcesContent":["import { basename } from 'node:path';\nimport type { AIConfig } from '@intlayer/ai';\nimport { type AIOptions, getIntlayerAPIProxy } from '@intlayer/api';\nimport {\n chunkJSON,\n formatLocale,\n type JsonChunk,\n reconstructFromSingleChunk,\n reduceObjectFormat,\n verifyIdenticObjectFormat,\n} from '@intlayer/chokidar';\nimport {\n ANSIColors,\n colon,\n colorize,\n colorizeNumber,\n colorizePath,\n getAppLogger,\n retryManager,\n} from '@intlayer/config';\nimport {\n getFilterMissingTranslationsDictionary,\n getMultilingualDictionary,\n getPerLocaleDictionary,\n insertContentInDictionary,\n mergeDictionaries,\n} from '@intlayer/core';\nimport type { Dictionary, IntlayerConfig, Locale } from '@intlayer/types';\nimport { getUnmergedDictionaries } from '@intlayer/unmerged-dictionaries-entry';\nimport type { AIClient } from '../utils/setupAI';\nimport { deepMergeContent } from './deepMergeContent';\nimport { getFilterMissingContentPerLocale } from './getFilterMissingContentPerLocale';\nimport type { TranslationTask } from './listTranslationsTasks';\n\ntype TranslateDictionaryResult = TranslationTask & {\n dictionaryOutput: Dictionary | null;\n};\n\ntype TranslateDictionaryOptions = {\n mode: 'complete' | 'review';\n aiOptions?: AIOptions;\n fillMetadata?: boolean;\n onHandle?: ReturnType<typeof import('@intlayer/chokidar').getGlobalLimiter>;\n onSuccess?: () => void;\n onError?: (error: unknown) => void;\n getAbortError?: () => Error | null;\n aiClient?: AIClient;\n aiConfig?: AIConfig;\n};\n\nconst hasMissingMetadata = (dictionary: Dictionary) =>\n !dictionary.description || !dictionary.title || !dictionary.tags;\n\nconst CHUNK_SIZE = 7000; // GPT-5 Mini safe input size\nconst GROUP_MAX_RETRY = 2;\nconst MAX_RETRY = 3;\nconst RETRY_DELAY = 1000 * 10; // 10 seconds\n\nconst MAX_FOLLOWING_ERRORS = 10; // 10 errors in a row, hard exit the process\nlet followingErrors = 0;\n\nexport const translateDictionary = async (\n task: TranslationTask,\n configuration: IntlayerConfig,\n options?: TranslateDictionaryOptions\n): Promise<TranslateDictionaryResult> => {\n const appLogger = getAppLogger(configuration);\n const intlayerAPI = getIntlayerAPIProxy(undefined, configuration);\n\n const { mode, aiOptions, fillMetadata, aiClient, aiConfig } = {\n mode: 'complete',\n fillMetadata: true,\n ...options,\n } as const;\n\n const notifySuccess = () => {\n followingErrors = 0;\n options?.onSuccess?.();\n };\n\n const result = await retryManager(\n async () => {\n const unmergedDictionariesRecord = getUnmergedDictionaries(configuration);\n\n const baseUnmergedDictionary: Dictionary | undefined =\n unmergedDictionariesRecord[task.dictionaryKey].find(\n (dict) => dict.localId === task.dictionaryLocalId\n );\n\n if (!baseUnmergedDictionary) {\n appLogger(\n `${task.dictionaryPreset}Dictionary not found in unmergedDictionariesRecord. Skipping.`,\n {\n level: 'warn',\n }\n );\n return { ...task, dictionaryOutput: null };\n }\n\n let metadata:\n | Pick<Dictionary, 'description' | 'title' | 'tags'>\n | undefined;\n\n if (\n fillMetadata &&\n (hasMissingMetadata(baseUnmergedDictionary) || mode === 'review')\n ) {\n const defaultLocaleDictionary = getPerLocaleDictionary(\n baseUnmergedDictionary,\n configuration.internationalization.defaultLocale\n );\n\n appLogger(\n `${task.dictionaryPreset} Filling missing metadata for ${colorizePath(basename(baseUnmergedDictionary.filePath!))}`,\n {\n level: 'info',\n }\n );\n\n const runAudit = async () => {\n if (aiClient && aiConfig) {\n const result = await aiClient.auditDictionaryMetadata({\n fileContent: JSON.stringify(defaultLocaleDictionary),\n aiConfig,\n });\n\n return {\n data: result,\n };\n }\n\n return await intlayerAPI.ai.auditContentDeclarationMetadata({\n fileContent: JSON.stringify(defaultLocaleDictionary),\n aiOptions,\n });\n };\n\n const metadataResult = options?.onHandle\n ? await options.onHandle(runAudit)\n : await runAudit();\n\n metadata = metadataResult.data?.fileContent;\n }\n\n const translatedContentResults = await Promise.all(\n task.targetLocales.map(async (targetLocale) => {\n /**\n * In complete mode, for large dictionaries, we want to filter all content that is already translated\n *\n * targetLocale: fr\n *\n * { test1: t({ ar: 'Hello', en: 'Hello', fr: 'Bonjour' } }) -> {}\n * { test2: t({ ar: 'Hello', en: 'Hello' }) } -> { test2: t({ ar: 'Hello', en: 'Hello' }) }\n *\n */\n // Reset to base dictionary for each locale to ensure we filter from the original\n let dictionaryToProcess = structuredClone(baseUnmergedDictionary);\n\n let targetLocaleDictionary: Dictionary;\n\n if (typeof baseUnmergedDictionary.locale === 'string') {\n // For per-locale files, the content is already in simple JSON format (not translation nodes)\n // The base dictionary is already the source locale content\n\n // Load the existing target locale dictionary\n const targetLocaleFilePath =\n baseUnmergedDictionary.filePath?.replace(\n new RegExp(`/${task.sourceLocale}/`, 'g'),\n `/${targetLocale}/`\n );\n\n // Find the target locale dictionary in unmerged dictionaries\n const targetUnmergedDictionary = targetLocaleFilePath\n ? unmergedDictionariesRecord[task.dictionaryKey]?.find(\n (dict) =>\n dict.filePath === targetLocaleFilePath &&\n dict.locale === targetLocale\n )\n : undefined;\n\n targetLocaleDictionary = targetUnmergedDictionary ?? {\n key: baseUnmergedDictionary.key,\n content: {},\n filePath: targetLocaleFilePath,\n locale: targetLocale,\n };\n\n // In complete mode, filter out already translated content\n if (mode === 'complete') {\n dictionaryToProcess = getFilterMissingContentPerLocale(\n dictionaryToProcess,\n targetUnmergedDictionary\n );\n }\n } else {\n // For multilingual dictionaries\n if (mode === 'complete') {\n // Remove all nodes that don't have any content to translate\n dictionaryToProcess = getFilterMissingTranslationsDictionary(\n dictionaryToProcess,\n targetLocale\n );\n }\n\n dictionaryToProcess = getPerLocaleDictionary(\n dictionaryToProcess,\n task.sourceLocale\n );\n\n targetLocaleDictionary = getPerLocaleDictionary(\n baseUnmergedDictionary,\n targetLocale\n );\n }\n\n const localePreset = colon(\n [\n colorize('[', ANSIColors.GREY_DARK),\n formatLocale(targetLocale),\n colorize(']', ANSIColors.GREY_DARK),\n ].join(''),\n { colSize: 18 }\n );\n\n const createChunkPreset = (\n chunkIndex: number,\n totalChunks: number\n ) => {\n if (totalChunks <= 1) return '';\n return colon(\n [\n colorize('[', ANSIColors.GREY_DARK),\n colorizeNumber(chunkIndex + 1),\n colorize(`/${totalChunks}`, ANSIColors.GREY_DARK),\n colorize(']', ANSIColors.GREY_DARK),\n ].join(''),\n { colSize: 5 }\n );\n };\n\n appLogger(\n `${task.dictionaryPreset}${localePreset} Preparing ${colorizePath(basename(targetLocaleDictionary.filePath!))}`,\n {\n level: 'info',\n }\n );\n\n const chunkedJsonContent: JsonChunk[] = chunkJSON(\n dictionaryToProcess.content,\n CHUNK_SIZE\n );\n\n const nbOfChunks = chunkedJsonContent.length;\n\n if (nbOfChunks > 1) {\n appLogger(\n `${task.dictionaryPreset}${localePreset} Split into ${colorizeNumber(nbOfChunks)} chunks for translation`,\n {\n level: 'info',\n }\n );\n }\n\n const chunkResult: JsonChunk[] = [];\n\n // Process chunks in parallel (globally throttled) to allow concurrent translation\n const chunkPromises = chunkedJsonContent.map((chunk) => {\n const chunkPreset = createChunkPreset(chunk.index, chunk.total);\n\n if (nbOfChunks > 1) {\n appLogger(\n `${task.dictionaryPreset}${localePreset}${chunkPreset} Translating chunk`,\n {\n level: 'info',\n }\n );\n }\n\n // Reconstruct partial JSON content from this chunk's patches\n const chunkContent = reconstructFromSingleChunk(chunk);\n const presetOutputContent = reduceObjectFormat(\n targetLocaleDictionary.content,\n chunkContent\n ) as unknown as JSON;\n\n const executeTranslation = async () => {\n return await retryManager(\n async () => {\n let translationResult: any;\n\n if (aiClient && aiConfig) {\n translationResult = await aiClient.translateJSON({\n entryFileContent: chunkContent as unknown as JSON,\n presetOutputContent,\n dictionaryDescription:\n dictionaryToProcess.description ??\n metadata?.description ??\n '',\n entryLocale: task.sourceLocale,\n outputLocale: targetLocale,\n mode,\n aiConfig,\n });\n } else {\n translationResult = await intlayerAPI.ai\n .translateJSON({\n entryFileContent: chunkContent as unknown as JSON,\n presetOutputContent,\n dictionaryDescription:\n dictionaryToProcess.description ??\n metadata?.description ??\n '',\n entryLocale: task.sourceLocale,\n outputLocale: targetLocale,\n mode,\n aiOptions,\n })\n .then((result) => result.data);\n }\n\n if (!translationResult?.fileContent) {\n throw new Error('No content result');\n }\n\n const { isIdentic } = verifyIdenticObjectFormat(\n translationResult.fileContent,\n chunkContent\n );\n if (!isIdentic) {\n throw new Error(\n 'Translation result does not match expected format'\n );\n }\n\n notifySuccess();\n return translationResult.fileContent;\n },\n {\n maxRetry: MAX_RETRY,\n delay: RETRY_DELAY,\n onError: ({ error, attempt, maxRetry }) => {\n const chunkPreset = createChunkPreset(\n chunk.index,\n chunk.total\n );\n appLogger(\n `${task.dictionaryPreset}${localePreset}${chunkPreset} ${colorize('Error filling:', ANSIColors.RED)} ${colorize(typeof error === 'string' ? error : JSON.stringify(error), ANSIColors.GREY_DARK)} - Attempt ${colorizeNumber(attempt + 1)} of ${colorizeNumber(maxRetry)}`,\n {\n level: 'error',\n }\n );\n\n followingErrors += 1;\n\n if (followingErrors >= MAX_FOLLOWING_ERRORS) {\n appLogger(`There is something wrong.`, {\n level: 'error',\n });\n process.exit(1); // 1 for error\n }\n },\n }\n )();\n };\n\n const wrapped = options?.onHandle\n ? options.onHandle(executeTranslation) // queued in global limiter\n : executeTranslation(); // no global limiter\n\n return wrapped.then((result) => ({ chunk, result }));\n });\n\n // Wait for all chunks for this locale in parallel (still capped by global limiter)\n const chunkResults = await Promise.all(chunkPromises);\n\n // Maintain order\n chunkResults\n .sort((chunkA, chunkB) => chunkA.chunk.index - chunkB.chunk.index)\n .forEach(({ result }) => {\n chunkResult.push(result);\n });\n\n // Merge partial JSON objects produced from each chunk into a single object\n const merged = mergeDictionaries(\n chunkResult.map((chunk) => ({\n ...dictionaryToProcess,\n content: chunk,\n }))\n );\n\n // For per-locale files, merge the newly translated content with existing target content\n let finalContent = merged.content;\n if (typeof baseUnmergedDictionary.locale === 'string') {\n // Deep merge: existing content + newly translated content\n finalContent = deepMergeContent(\n targetLocaleDictionary.content ?? {},\n merged.content\n );\n }\n\n return [targetLocale, finalContent] as const;\n })\n );\n\n const translatedContent: Partial<Record<Locale, Dictionary['content']>> =\n Object.fromEntries(translatedContentResults);\n\n const baseDictionary = baseUnmergedDictionary.locale\n ? {\n ...baseUnmergedDictionary,\n key: baseUnmergedDictionary.key!,\n content: {},\n }\n : baseUnmergedDictionary;\n\n let dictionaryOutput: Dictionary = {\n ...getMultilingualDictionary(baseDictionary),\n locale: undefined, // Ensure the dictionary is multilingual\n ...metadata,\n };\n\n for (const targetLocale of task.targetLocales) {\n if (translatedContent[targetLocale]) {\n dictionaryOutput = insertContentInDictionary(\n dictionaryOutput,\n translatedContent[targetLocale],\n targetLocale\n );\n }\n }\n\n appLogger(\n `${task.dictionaryPreset} ${colorize('Translation completed successfully', ANSIColors.GREEN)} for ${colorizePath(basename(dictionaryOutput.filePath!))}`,\n {\n level: 'info',\n }\n );\n\n if (\n baseUnmergedDictionary.locale &&\n (baseUnmergedDictionary.fill === true ||\n baseUnmergedDictionary.fill === undefined) &&\n baseUnmergedDictionary.location === 'local'\n ) {\n const dictionaryFilePath = baseUnmergedDictionary\n .filePath!.split('.')\n .slice(0, -1);\n\n const contentIndex = dictionaryFilePath[dictionaryFilePath.length - 1];\n\n return JSON.parse(\n JSON.stringify({\n ...task,\n dictionaryOutput: {\n ...dictionaryOutput,\n fill: undefined,\n filled: true,\n },\n }).replaceAll(\n new RegExp(`\\\\.${contentIndex}\\\\.[a-zA-Z0-9]+`, 'g'),\n `.filled.${contentIndex}.json`\n )\n );\n }\n\n return {\n ...task,\n dictionaryOutput,\n };\n },\n {\n maxRetry: GROUP_MAX_RETRY,\n delay: RETRY_DELAY,\n onError: ({ error, attempt, maxRetry }) =>\n appLogger(\n `${task.dictionaryPreset} ${colorize('Error fill command:', ANSIColors.RED)} ${colorize(typeof error === 'string' ? error : JSON.stringify(error), ANSIColors.GREY_DARK)} - Attempt ${colorizeNumber(attempt + 1)} of ${colorizeNumber(maxRetry)}`,\n {\n level: 'error',\n }\n ),\n onMaxTryReached: ({ error }) =>\n appLogger(\n `${task.dictionaryPreset} ${colorize('Maximum number of retries reached:', ANSIColors.RED)} ${colorize(typeof error === 'string' ? error : JSON.stringify(error), ANSIColors.GREY_DARK)}`,\n {\n level: 'error',\n }\n ),\n }\n )();\n\n return result as TranslateDictionaryResult;\n};\n"],"mappings":";;;;;;;;;;AAkDA,MAAM,sBAAsB,eAC1B,CAAC,WAAW,eAAe,CAAC,WAAW,SAAS,CAAC,WAAW;AAE9D,MAAM,aAAa;AACnB,MAAM,kBAAkB;AACxB,MAAM,YAAY;AAClB,MAAM,cAAc,MAAO;AAE3B,MAAM,uBAAuB;AAC7B,IAAI,kBAAkB;AAEtB,MAAa,sBAAsB,OACjC,MACA,eACA,YACuC;CACvC,MAAM,YAAY,aAAa,cAAc;CAC7C,MAAM,cAAc,oBAAoB,QAAW,cAAc;CAEjE,MAAM,EAAE,MAAM,WAAW,cAAc,UAAU,aAAa;EAC5D,MAAM;EACN,cAAc;EACd,GAAG;EACJ;CAED,MAAM,sBAAsB;AAC1B,oBAAkB;AAClB,WAAS,aAAa;;AA6ZxB,QA1Ze,MAAM,aACnB,YAAY;EACV,MAAM,6BAA6B,wBAAwB,cAAc;EAEzE,MAAMA,yBACJ,2BAA2B,KAAK,eAAe,MAC5C,SAAS,KAAK,YAAY,KAAK,kBACjC;AAEH,MAAI,CAAC,wBAAwB;AAC3B,aACE,GAAG,KAAK,iBAAiB,gEACzB,EACE,OAAO,QACR,CACF;AACD,UAAO;IAAE,GAAG;IAAM,kBAAkB;IAAM;;EAG5C,IAAIC;AAIJ,MACE,iBACC,mBAAmB,uBAAuB,IAAI,SAAS,WACxD;GACA,MAAM,0BAA0B,uBAC9B,wBACA,cAAc,qBAAqB,cACpC;AAED,aACE,GAAG,KAAK,iBAAiB,gCAAgC,aAAa,SAAS,uBAAuB,SAAU,CAAC,IACjH,EACE,OAAO,QACR,CACF;GAED,MAAM,WAAW,YAAY;AAC3B,QAAI,YAAY,SAMd,QAAO,EACL,MANa,MAAM,SAAS,wBAAwB;KACpD,aAAa,KAAK,UAAU,wBAAwB;KACpD;KACD,CAAC,EAID;AAGH,WAAO,MAAM,YAAY,GAAG,gCAAgC;KAC1D,aAAa,KAAK,UAAU,wBAAwB;KACpD;KACD,CAAC;;AAOJ,eAJuB,SAAS,WAC5B,MAAM,QAAQ,SAAS,SAAS,GAChC,MAAM,UAAU,EAEM,MAAM;;EAGlC,MAAM,2BAA2B,MAAM,QAAQ,IAC7C,KAAK,cAAc,IAAI,OAAO,iBAAiB;;;;;;;;;;GAW7C,IAAI,sBAAsB,gBAAgB,uBAAuB;GAEjE,IAAIC;AAEJ,OAAI,OAAO,uBAAuB,WAAW,UAAU;IAKrD,MAAM,uBACJ,uBAAuB,UAAU,QAC/B,IAAI,OAAO,IAAI,KAAK,aAAa,IAAI,IAAI,EACzC,IAAI,aAAa,GAClB;IAGH,MAAM,2BAA2B,uBAC7B,2BAA2B,KAAK,gBAAgB,MAC7C,SACC,KAAK,aAAa,wBAClB,KAAK,WAAW,aACnB,GACD;AAEJ,6BAAyB,4BAA4B;KACnD,KAAK,uBAAuB;KAC5B,SAAS,EAAE;KACX,UAAU;KACV,QAAQ;KACT;AAGD,QAAI,SAAS,WACX,uBAAsB,iCACpB,qBACA,yBACD;UAEE;AAEL,QAAI,SAAS,WAEX,uBAAsB,uCACpB,qBACA,aACD;AAGH,0BAAsB,uBACpB,qBACA,KAAK,aACN;AAED,6BAAyB,uBACvB,wBACA,aACD;;GAGH,MAAM,eAAe,MACnB;IACE,SAAS,KAAK,WAAW,UAAU;IACnC,aAAa,aAAa;IAC1B,SAAS,KAAK,WAAW,UAAU;IACpC,CAAC,KAAK,GAAG,EACV,EAAE,SAAS,IAAI,CAChB;GAED,MAAM,qBACJ,YACA,gBACG;AACH,QAAI,eAAe,EAAG,QAAO;AAC7B,WAAO,MACL;KACE,SAAS,KAAK,WAAW,UAAU;KACnC,eAAe,aAAa,EAAE;KAC9B,SAAS,IAAI,eAAe,WAAW,UAAU;KACjD,SAAS,KAAK,WAAW,UAAU;KACpC,CAAC,KAAK,GAAG,EACV,EAAE,SAAS,GAAG,CACf;;AAGH,aACE,GAAG,KAAK,mBAAmB,aAAa,aAAa,aAAa,SAAS,uBAAuB,SAAU,CAAC,IAC7G,EACE,OAAO,QACR,CACF;GAED,MAAMC,qBAAkC,UACtC,oBAAoB,SACpB,WACD;GAED,MAAM,aAAa,mBAAmB;AAEtC,OAAI,aAAa,EACf,WACE,GAAG,KAAK,mBAAmB,aAAa,cAAc,eAAe,WAAW,CAAC,0BACjF,EACE,OAAO,QACR,CACF;GAGH,MAAMC,cAA2B,EAAE;GAGnC,MAAM,gBAAgB,mBAAmB,KAAK,UAAU;IACtD,MAAM,cAAc,kBAAkB,MAAM,OAAO,MAAM,MAAM;AAE/D,QAAI,aAAa,EACf,WACE,GAAG,KAAK,mBAAmB,eAAe,YAAY,qBACtD,EACE,OAAO,QACR,CACF;IAIH,MAAM,eAAe,2BAA2B,MAAM;IACtD,MAAM,sBAAsB,mBAC1B,uBAAuB,SACvB,aACD;IAED,MAAM,qBAAqB,YAAY;AACrC,YAAO,MAAM,aACX,YAAY;MACV,IAAIC;AAEJ,UAAI,YAAY,SACd,qBAAoB,MAAM,SAAS,cAAc;OAC/C,kBAAkB;OAClB;OACA,uBACE,oBAAoB,eACpB,UAAU,eACV;OACF,aAAa,KAAK;OAClB,cAAc;OACd;OACA;OACD,CAAC;UAEF,qBAAoB,MAAM,YAAY,GACnC,cAAc;OACb,kBAAkB;OAClB;OACA,uBACE,oBAAoB,eACpB,UAAU,eACV;OACF,aAAa,KAAK;OAClB,cAAc;OACd;OACA;OACD,CAAC,CACD,MAAM,WAAW,OAAO,KAAK;AAGlC,UAAI,CAAC,mBAAmB,YACtB,OAAM,IAAI,MAAM,oBAAoB;MAGtC,MAAM,EAAE,cAAc,0BACpB,kBAAkB,aAClB,aACD;AACD,UAAI,CAAC,UACH,OAAM,IAAI,MACR,oDACD;AAGH,qBAAe;AACf,aAAO,kBAAkB;QAE3B;MACE,UAAU;MACV,OAAO;MACP,UAAU,EAAE,OAAO,SAAS,eAAe;OACzC,MAAMC,gBAAc,kBAClB,MAAM,OACN,MAAM,MACP;AACD,iBACE,GAAG,KAAK,mBAAmB,eAAeA,cAAY,GAAG,SAAS,kBAAkB,WAAW,IAAI,CAAC,GAAG,SAAS,OAAO,UAAU,WAAW,QAAQ,KAAK,UAAU,MAAM,EAAE,WAAW,UAAU,CAAC,aAAa,eAAe,UAAU,EAAE,CAAC,MAAM,eAAe,SAAS,IACxQ,EACE,OAAO,SACR,CACF;AAED,0BAAmB;AAEnB,WAAI,mBAAmB,sBAAsB;AAC3C,kBAAU,6BAA6B,EACrC,OAAO,SACR,CAAC;AACF,gBAAQ,KAAK,EAAE;;;MAGpB,CACF,EAAE;;AAOL,YAJgB,SAAS,WACrB,QAAQ,SAAS,mBAAmB,GACpC,oBAAoB,EAET,MAAM,YAAY;KAAE;KAAO;KAAQ,EAAE;KACpD;AAMF,IAHqB,MAAM,QAAQ,IAAI,cAAc,EAIlD,MAAM,QAAQ,WAAW,OAAO,MAAM,QAAQ,OAAO,MAAM,MAAM,CACjE,SAAS,EAAE,aAAa;AACvB,gBAAY,KAAK,OAAO;KACxB;GAGJ,MAAM,SAAS,kBACb,YAAY,KAAK,WAAW;IAC1B,GAAG;IACH,SAAS;IACV,EAAE,CACJ;GAGD,IAAI,eAAe,OAAO;AAC1B,OAAI,OAAO,uBAAuB,WAAW,SAE3C,gBAAe,iBACb,uBAAuB,WAAW,EAAE,EACpC,OAAO,QACR;AAGH,UAAO,CAAC,cAAc,aAAa;IACnC,CACH;EAED,MAAMC,oBACJ,OAAO,YAAY,yBAAyB;EAU9C,IAAIC,mBAA+B;GACjC,GAAG,0BATkB,uBAAuB,SAC1C;IACE,GAAG;IACH,KAAK,uBAAuB;IAC5B,SAAS,EAAE;IACZ,GACD,uBAG0C;GAC5C,QAAQ;GACR,GAAG;GACJ;AAED,OAAK,MAAM,gBAAgB,KAAK,cAC9B,KAAI,kBAAkB,cACpB,oBAAmB,0BACjB,kBACA,kBAAkB,eAClB,aACD;AAIL,YACE,GAAG,KAAK,iBAAiB,GAAG,SAAS,sCAAsC,WAAW,MAAM,CAAC,OAAO,aAAa,SAAS,iBAAiB,SAAU,CAAC,IACtJ,EACE,OAAO,QACR,CACF;AAED,MACE,uBAAuB,WACtB,uBAAuB,SAAS,QAC/B,uBAAuB,SAAS,WAClC,uBAAuB,aAAa,SACpC;GACA,MAAM,qBAAqB,uBACxB,SAAU,MAAM,IAAI,CACpB,MAAM,GAAG,GAAG;GAEf,MAAM,eAAe,mBAAmB,mBAAmB,SAAS;AAEpE,UAAO,KAAK,MACV,KAAK,UAAU;IACb,GAAG;IACH,kBAAkB;KAChB,GAAG;KACH,MAAM;KACN,QAAQ;KACT;IACF,CAAC,CAAC,WACD,IAAI,OAAO,MAAM,aAAa,kBAAkB,IAAI,EACpD,WAAW,aAAa,OACzB,CACF;;AAGH,SAAO;GACL,GAAG;GACH;GACD;IAEH;EACE,UAAU;EACV,OAAO;EACP,UAAU,EAAE,OAAO,SAAS,eAC1B,UACE,GAAG,KAAK,iBAAiB,GAAG,SAAS,uBAAuB,WAAW,IAAI,CAAC,GAAG,SAAS,OAAO,UAAU,WAAW,QAAQ,KAAK,UAAU,MAAM,EAAE,WAAW,UAAU,CAAC,aAAa,eAAe,UAAU,EAAE,CAAC,MAAM,eAAe,SAAS,IAChP,EACE,OAAO,SACR,CACF;EACH,kBAAkB,EAAE,YAClB,UACE,GAAG,KAAK,iBAAiB,GAAG,SAAS,sCAAsC,WAAW,IAAI,CAAC,GAAG,SAAS,OAAO,UAAU,WAAW,QAAQ,KAAK,UAAU,MAAM,EAAE,WAAW,UAAU,IACvL,EACE,OAAO,SACR,CACF;EACJ,CACF,EAAE"}
|
|
1
|
+
{"version":3,"file":"translateDictionary.mjs","names":["baseUnmergedDictionary: Dictionary | undefined","metadata:\n | Pick<Dictionary, 'description' | 'title' | 'tags'>\n | undefined","targetLocaleDictionary: Dictionary","chunkedJsonContent: JsonChunk[]","chunkResult: JsonChunk[]","translationResult: any","chunkPreset","translatedContent: Partial<Record<Locale, Dictionary['content']>>","dictionaryOutput: Dictionary"],"sources":["../../../src/fill/translateDictionary.ts"],"sourcesContent":["import { basename } from 'node:path';\nimport type { AIConfig } from '@intlayer/ai';\nimport { type AIOptions, getIntlayerAPIProxy } from '@intlayer/api';\nimport {\n chunkJSON,\n formatLocale,\n type JsonChunk,\n reconstructFromSingleChunk,\n reduceObjectFormat,\n verifyIdenticObjectFormat,\n} from '@intlayer/chokidar';\nimport {\n ANSIColors,\n colon,\n colorize,\n colorizeNumber,\n colorizePath,\n getAppLogger,\n retryManager,\n} from '@intlayer/config';\nimport {\n getFilterMissingTranslationsDictionary,\n getMultilingualDictionary,\n getPerLocaleDictionary,\n insertContentInDictionary,\n mergeDictionaries,\n} from '@intlayer/core';\nimport type { Dictionary, IntlayerConfig, Locale } from '@intlayer/types';\nimport { getUnmergedDictionaries } from '@intlayer/unmerged-dictionaries-entry';\nimport type { AIClient } from '../utils/setupAI';\nimport { deepMergeContent } from './deepMergeContent';\nimport { getFilterMissingContentPerLocale } from './getFilterMissingContentPerLocale';\nimport type { TranslationTask } from './listTranslationsTasks';\n\ntype TranslateDictionaryResult = TranslationTask & {\n dictionaryOutput: Dictionary | null;\n};\n\ntype TranslateDictionaryOptions = {\n mode: 'complete' | 'review';\n aiOptions?: AIOptions;\n fillMetadata?: boolean;\n onHandle?: ReturnType<typeof import('@intlayer/chokidar').getGlobalLimiter>;\n onSuccess?: () => void;\n onError?: (error: unknown) => void;\n getAbortError?: () => Error | null;\n aiClient?: AIClient;\n aiConfig?: AIConfig;\n};\n\nconst hasMissingMetadata = (dictionary: Dictionary) =>\n !dictionary.description || !dictionary.title || !dictionary.tags;\n\nconst CHUNK_SIZE = 7000; // GPT-5 Mini safe input size\nconst GROUP_MAX_RETRY = 2;\nconst MAX_RETRY = 3;\nconst RETRY_DELAY = 1000 * 10; // 10 seconds\n\nconst MAX_FOLLOWING_ERRORS = 10; // 10 errors in a row, hard exit the process\nlet followingErrors = 0;\n\nexport const translateDictionary = async (\n task: TranslationTask,\n configuration: IntlayerConfig,\n options?: TranslateDictionaryOptions\n): Promise<TranslateDictionaryResult> => {\n const appLogger = getAppLogger(configuration);\n const intlayerAPI = getIntlayerAPIProxy(undefined, configuration);\n\n const { mode, aiOptions, fillMetadata, aiClient, aiConfig } = {\n mode: 'complete',\n fillMetadata: true,\n ...options,\n } as const;\n\n const notifySuccess = () => {\n followingErrors = 0;\n options?.onSuccess?.();\n };\n\n const result = await retryManager(\n async () => {\n const unmergedDictionariesRecord = getUnmergedDictionaries(configuration);\n\n const baseUnmergedDictionary: Dictionary | undefined =\n unmergedDictionariesRecord[task.dictionaryKey].find(\n (dict) => dict.localId === task.dictionaryLocalId\n );\n\n if (!baseUnmergedDictionary) {\n appLogger(\n `${task.dictionaryPreset}Dictionary not found in unmergedDictionariesRecord. Skipping.`,\n {\n level: 'warn',\n }\n );\n return { ...task, dictionaryOutput: null };\n }\n\n let metadata:\n | Pick<Dictionary, 'description' | 'title' | 'tags'>\n | undefined;\n\n if (\n fillMetadata &&\n (hasMissingMetadata(baseUnmergedDictionary) || mode === 'review')\n ) {\n const defaultLocaleDictionary = getPerLocaleDictionary(\n baseUnmergedDictionary,\n configuration.internationalization.defaultLocale\n );\n\n appLogger(\n `${task.dictionaryPreset} Filling missing metadata for ${colorizePath(basename(baseUnmergedDictionary.filePath!))}`,\n {\n level: 'info',\n }\n );\n\n const runAudit = async () => {\n if (aiClient && aiConfig) {\n const result = await aiClient.auditDictionaryMetadata({\n fileContent: JSON.stringify(defaultLocaleDictionary),\n aiConfig,\n });\n\n return {\n data: result,\n };\n }\n\n return await intlayerAPI.ai.auditContentDeclarationMetadata({\n fileContent: JSON.stringify(defaultLocaleDictionary),\n aiOptions,\n });\n };\n\n const metadataResult = options?.onHandle\n ? await options.onHandle(runAudit)\n : await runAudit();\n\n metadata = metadataResult.data?.fileContent;\n }\n\n const translatedContentResults = await Promise.all(\n task.targetLocales.map(async (targetLocale) => {\n /**\n * In complete mode, for large dictionaries, we want to filter all content that is already translated\n *\n * targetLocale: fr\n *\n * { test1: t({ ar: 'Hello', en: 'Hello', fr: 'Bonjour' } }) -> {}\n * { test2: t({ ar: 'Hello', en: 'Hello' }) } -> { test2: t({ ar: 'Hello', en: 'Hello' }) }\n *\n */\n // Reset to base dictionary for each locale to ensure we filter from the original\n let dictionaryToProcess = structuredClone(baseUnmergedDictionary);\n\n let targetLocaleDictionary: Dictionary;\n\n if (typeof baseUnmergedDictionary.locale === 'string') {\n // For per-locale files, the content is already in simple JSON format (not translation nodes)\n // The base dictionary is already the source locale content\n\n // Load the existing target locale dictionary\n const targetLocaleFilePath =\n baseUnmergedDictionary.filePath?.replace(\n new RegExp(`/${task.sourceLocale}/`, 'g'),\n `/${targetLocale}/`\n );\n\n // Find the target locale dictionary in unmerged dictionaries\n const targetUnmergedDictionary = targetLocaleFilePath\n ? unmergedDictionariesRecord[task.dictionaryKey]?.find(\n (dict) =>\n dict.filePath === targetLocaleFilePath &&\n dict.locale === targetLocale\n )\n : undefined;\n\n targetLocaleDictionary = targetUnmergedDictionary ?? {\n key: baseUnmergedDictionary.key,\n content: {},\n filePath: targetLocaleFilePath,\n locale: targetLocale,\n };\n\n // In complete mode, filter out already translated content\n if (mode === 'complete') {\n dictionaryToProcess = getFilterMissingContentPerLocale(\n dictionaryToProcess,\n targetUnmergedDictionary\n );\n }\n } else {\n // For multilingual dictionaries\n if (mode === 'complete') {\n // Remove all nodes that don't have any content to translate\n dictionaryToProcess = getFilterMissingTranslationsDictionary(\n dictionaryToProcess,\n targetLocale\n );\n }\n\n dictionaryToProcess = getPerLocaleDictionary(\n dictionaryToProcess,\n task.sourceLocale\n );\n\n targetLocaleDictionary = getPerLocaleDictionary(\n baseUnmergedDictionary,\n targetLocale\n );\n }\n\n const localePreset = colon(\n [\n colorize('[', ANSIColors.GREY_DARK),\n formatLocale(targetLocale),\n colorize(']', ANSIColors.GREY_DARK),\n ].join(''),\n { colSize: 18 }\n );\n\n const createChunkPreset = (\n chunkIndex: number,\n totalChunks: number\n ) => {\n if (totalChunks <= 1) return '';\n return colon(\n [\n colorize('[', ANSIColors.GREY_DARK),\n colorizeNumber(chunkIndex + 1),\n colorize(`/${totalChunks}`, ANSIColors.GREY_DARK),\n colorize(']', ANSIColors.GREY_DARK),\n ].join(''),\n { colSize: 5 }\n );\n };\n\n appLogger(\n `${task.dictionaryPreset}${localePreset} Preparing ${colorizePath(basename(targetLocaleDictionary.filePath!))}`,\n {\n level: 'info',\n }\n );\n\n const isContentStructured =\n (typeof dictionaryToProcess.content === 'object' &&\n dictionaryToProcess.content !== null) ||\n Array.isArray(dictionaryToProcess.content);\n\n const contentToProcess = isContentStructured\n ? dictionaryToProcess.content\n : {\n __INTLAYER_ROOT_PRIMITIVE_CONTENT__:\n dictionaryToProcess.content,\n };\n\n const chunkedJsonContent: JsonChunk[] = chunkJSON(\n contentToProcess as unknown as Record<string, any>,\n CHUNK_SIZE\n );\n\n const nbOfChunks = chunkedJsonContent.length;\n\n if (nbOfChunks > 1) {\n appLogger(\n `${task.dictionaryPreset}${localePreset} Split into ${colorizeNumber(nbOfChunks)} chunks for translation`,\n {\n level: 'info',\n }\n );\n }\n\n const chunkResult: JsonChunk[] = [];\n\n // Process chunks in parallel (globally throttled) to allow concurrent translation\n const chunkPromises = chunkedJsonContent.map((chunk) => {\n const chunkPreset = createChunkPreset(chunk.index, chunk.total);\n\n if (nbOfChunks > 1) {\n appLogger(\n `${task.dictionaryPreset}${localePreset}${chunkPreset} Translating chunk`,\n {\n level: 'info',\n }\n );\n }\n\n // Reconstruct partial JSON content from this chunk's patches\n const chunkContent = reconstructFromSingleChunk(chunk);\n const presetOutputContent = reduceObjectFormat(\n isContentStructured\n ? targetLocaleDictionary.content\n : {\n __INTLAYER_ROOT_PRIMITIVE_CONTENT__:\n targetLocaleDictionary.content,\n },\n chunkContent\n ) as unknown as JSON;\n\n const executeTranslation = async () => {\n return await retryManager(\n async () => {\n let translationResult: any;\n\n if (aiClient && aiConfig) {\n translationResult = await aiClient.translateJSON({\n entryFileContent: chunkContent as unknown as JSON,\n presetOutputContent,\n dictionaryDescription:\n dictionaryToProcess.description ??\n metadata?.description ??\n '',\n entryLocale: task.sourceLocale,\n outputLocale: targetLocale,\n mode,\n aiConfig,\n });\n } else {\n translationResult = await intlayerAPI.ai\n .translateJSON({\n entryFileContent: chunkContent as unknown as JSON,\n presetOutputContent,\n dictionaryDescription:\n dictionaryToProcess.description ??\n metadata?.description ??\n '',\n entryLocale: task.sourceLocale,\n outputLocale: targetLocale,\n mode,\n aiOptions,\n })\n .then((result) => result.data);\n }\n\n if (!translationResult?.fileContent) {\n throw new Error('No content result');\n }\n\n const { isIdentic } = verifyIdenticObjectFormat(\n translationResult.fileContent,\n chunkContent\n );\n if (!isIdentic) {\n throw new Error(\n 'Translation result does not match expected format'\n );\n }\n\n notifySuccess();\n return translationResult.fileContent;\n },\n {\n maxRetry: MAX_RETRY,\n delay: RETRY_DELAY,\n onError: ({ error, attempt, maxRetry }) => {\n const chunkPreset = createChunkPreset(\n chunk.index,\n chunk.total\n );\n appLogger(\n `${task.dictionaryPreset}${localePreset}${chunkPreset} ${colorize('Error filling:', ANSIColors.RED)} ${colorize(typeof error === 'string' ? error : JSON.stringify(error), ANSIColors.GREY_DARK)} - Attempt ${colorizeNumber(attempt + 1)} of ${colorizeNumber(maxRetry)}`,\n {\n level: 'error',\n }\n );\n\n followingErrors += 1;\n\n if (followingErrors >= MAX_FOLLOWING_ERRORS) {\n appLogger(`There is something wrong.`, {\n level: 'error',\n });\n process.exit(1); // 1 for error\n }\n },\n }\n )();\n };\n\n const wrapped = options?.onHandle\n ? options.onHandle(executeTranslation) // queued in global limiter\n : executeTranslation(); // no global limiter\n\n return wrapped.then((result) => ({ chunk, result }));\n });\n\n // Wait for all chunks for this locale in parallel (still capped by global limiter)\n const chunkResults = await Promise.all(chunkPromises);\n\n // Maintain order\n chunkResults\n .sort((chunkA, chunkB) => chunkA.chunk.index - chunkB.chunk.index)\n .forEach(({ result }) => {\n chunkResult.push(result);\n });\n\n // Merge partial JSON objects produced from each chunk into a single object\n const merged = mergeDictionaries(\n chunkResult.map((chunk) => ({\n ...dictionaryToProcess,\n content: chunk,\n }))\n );\n\n // For per-locale files, merge the newly translated content with existing target content\n let finalContent = merged.content;\n\n if (!isContentStructured) {\n finalContent = (finalContent as any)\n ?.__INTLAYER_ROOT_PRIMITIVE_CONTENT__;\n }\n\n if (typeof baseUnmergedDictionary.locale === 'string') {\n // Deep merge: existing content + newly translated content\n finalContent = deepMergeContent(\n targetLocaleDictionary.content ?? {},\n finalContent\n );\n }\n\n return [targetLocale, finalContent] as const;\n })\n );\n\n const translatedContent: Partial<Record<Locale, Dictionary['content']>> =\n Object.fromEntries(translatedContentResults);\n\n const baseDictionary = baseUnmergedDictionary.locale\n ? {\n ...baseUnmergedDictionary,\n key: baseUnmergedDictionary.key!,\n content: {},\n }\n : baseUnmergedDictionary;\n\n let dictionaryOutput: Dictionary = {\n ...getMultilingualDictionary(baseDictionary),\n locale: undefined, // Ensure the dictionary is multilingual\n ...metadata,\n };\n\n for (const targetLocale of task.targetLocales) {\n if (translatedContent[targetLocale]) {\n dictionaryOutput = insertContentInDictionary(\n dictionaryOutput,\n translatedContent[targetLocale],\n targetLocale\n );\n }\n }\n\n appLogger(\n `${task.dictionaryPreset} ${colorize('Translation completed successfully', ANSIColors.GREEN)} for ${colorizePath(basename(dictionaryOutput.filePath!))}`,\n {\n level: 'info',\n }\n );\n\n if (\n baseUnmergedDictionary.locale &&\n (baseUnmergedDictionary.fill === true ||\n baseUnmergedDictionary.fill === undefined) &&\n baseUnmergedDictionary.location === 'local'\n ) {\n const dictionaryFilePath = baseUnmergedDictionary\n .filePath!.split('.')\n .slice(0, -1);\n\n const contentIndex = dictionaryFilePath[dictionaryFilePath.length - 1];\n\n return JSON.parse(\n JSON.stringify({\n ...task,\n dictionaryOutput: {\n ...dictionaryOutput,\n fill: undefined,\n filled: true,\n },\n }).replaceAll(\n new RegExp(`\\\\.${contentIndex}\\\\.[a-zA-Z0-9]+`, 'g'),\n `.filled.${contentIndex}.json`\n )\n );\n }\n\n return {\n ...task,\n dictionaryOutput,\n };\n },\n {\n maxRetry: GROUP_MAX_RETRY,\n delay: RETRY_DELAY,\n onError: ({ error, attempt, maxRetry }) =>\n appLogger(\n `${task.dictionaryPreset} ${colorize('Error fill command:', ANSIColors.RED)} ${colorize(typeof error === 'string' ? error : JSON.stringify(error), ANSIColors.GREY_DARK)} - Attempt ${colorizeNumber(attempt + 1)} of ${colorizeNumber(maxRetry)}`,\n {\n level: 'error',\n }\n ),\n onMaxTryReached: ({ error }) =>\n appLogger(\n `${task.dictionaryPreset} ${colorize('Maximum number of retries reached:', ANSIColors.RED)} ${colorize(typeof error === 'string' ? error : JSON.stringify(error), ANSIColors.GREY_DARK)}`,\n {\n level: 'error',\n }\n ),\n }\n )();\n\n return result as TranslateDictionaryResult;\n};\n"],"mappings":";;;;;;;;;;AAkDA,MAAM,sBAAsB,eAC1B,CAAC,WAAW,eAAe,CAAC,WAAW,SAAS,CAAC,WAAW;AAE9D,MAAM,aAAa;AACnB,MAAM,kBAAkB;AACxB,MAAM,YAAY;AAClB,MAAM,cAAc,MAAO;AAE3B,MAAM,uBAAuB;AAC7B,IAAI,kBAAkB;AAEtB,MAAa,sBAAsB,OACjC,MACA,eACA,YACuC;CACvC,MAAM,YAAY,aAAa,cAAc;CAC7C,MAAM,cAAc,oBAAoB,QAAW,cAAc;CAEjE,MAAM,EAAE,MAAM,WAAW,cAAc,UAAU,aAAa;EAC5D,MAAM;EACN,cAAc;EACd,GAAG;EACJ;CAED,MAAM,sBAAsB;AAC1B,oBAAkB;AAClB,WAAS,aAAa;;AAobxB,QAjbe,MAAM,aACnB,YAAY;EACV,MAAM,6BAA6B,wBAAwB,cAAc;EAEzE,MAAMA,yBACJ,2BAA2B,KAAK,eAAe,MAC5C,SAAS,KAAK,YAAY,KAAK,kBACjC;AAEH,MAAI,CAAC,wBAAwB;AAC3B,aACE,GAAG,KAAK,iBAAiB,gEACzB,EACE,OAAO,QACR,CACF;AACD,UAAO;IAAE,GAAG;IAAM,kBAAkB;IAAM;;EAG5C,IAAIC;AAIJ,MACE,iBACC,mBAAmB,uBAAuB,IAAI,SAAS,WACxD;GACA,MAAM,0BAA0B,uBAC9B,wBACA,cAAc,qBAAqB,cACpC;AAED,aACE,GAAG,KAAK,iBAAiB,gCAAgC,aAAa,SAAS,uBAAuB,SAAU,CAAC,IACjH,EACE,OAAO,QACR,CACF;GAED,MAAM,WAAW,YAAY;AAC3B,QAAI,YAAY,SAMd,QAAO,EACL,MANa,MAAM,SAAS,wBAAwB;KACpD,aAAa,KAAK,UAAU,wBAAwB;KACpD;KACD,CAAC,EAID;AAGH,WAAO,MAAM,YAAY,GAAG,gCAAgC;KAC1D,aAAa,KAAK,UAAU,wBAAwB;KACpD;KACD,CAAC;;AAOJ,eAJuB,SAAS,WAC5B,MAAM,QAAQ,SAAS,SAAS,GAChC,MAAM,UAAU,EAEM,MAAM;;EAGlC,MAAM,2BAA2B,MAAM,QAAQ,IAC7C,KAAK,cAAc,IAAI,OAAO,iBAAiB;;;;;;;;;;GAW7C,IAAI,sBAAsB,gBAAgB,uBAAuB;GAEjE,IAAIC;AAEJ,OAAI,OAAO,uBAAuB,WAAW,UAAU;IAKrD,MAAM,uBACJ,uBAAuB,UAAU,QAC/B,IAAI,OAAO,IAAI,KAAK,aAAa,IAAI,IAAI,EACzC,IAAI,aAAa,GAClB;IAGH,MAAM,2BAA2B,uBAC7B,2BAA2B,KAAK,gBAAgB,MAC7C,SACC,KAAK,aAAa,wBAClB,KAAK,WAAW,aACnB,GACD;AAEJ,6BAAyB,4BAA4B;KACnD,KAAK,uBAAuB;KAC5B,SAAS,EAAE;KACX,UAAU;KACV,QAAQ;KACT;AAGD,QAAI,SAAS,WACX,uBAAsB,iCACpB,qBACA,yBACD;UAEE;AAEL,QAAI,SAAS,WAEX,uBAAsB,uCACpB,qBACA,aACD;AAGH,0BAAsB,uBACpB,qBACA,KAAK,aACN;AAED,6BAAyB,uBACvB,wBACA,aACD;;GAGH,MAAM,eAAe,MACnB;IACE,SAAS,KAAK,WAAW,UAAU;IACnC,aAAa,aAAa;IAC1B,SAAS,KAAK,WAAW,UAAU;IACpC,CAAC,KAAK,GAAG,EACV,EAAE,SAAS,IAAI,CAChB;GAED,MAAM,qBACJ,YACA,gBACG;AACH,QAAI,eAAe,EAAG,QAAO;AAC7B,WAAO,MACL;KACE,SAAS,KAAK,WAAW,UAAU;KACnC,eAAe,aAAa,EAAE;KAC9B,SAAS,IAAI,eAAe,WAAW,UAAU;KACjD,SAAS,KAAK,WAAW,UAAU;KACpC,CAAC,KAAK,GAAG,EACV,EAAE,SAAS,GAAG,CACf;;AAGH,aACE,GAAG,KAAK,mBAAmB,aAAa,aAAa,aAAa,SAAS,uBAAuB,SAAU,CAAC,IAC7G,EACE,OAAO,QACR,CACF;GAED,MAAM,sBACH,OAAO,oBAAoB,YAAY,YACtC,oBAAoB,YAAY,QAClC,MAAM,QAAQ,oBAAoB,QAAQ;GAS5C,MAAMC,qBAAkC,UAPf,sBACrB,oBAAoB,UACpB,EACE,qCACE,oBAAoB,SACvB,EAIH,WACD;GAED,MAAM,aAAa,mBAAmB;AAEtC,OAAI,aAAa,EACf,WACE,GAAG,KAAK,mBAAmB,aAAa,cAAc,eAAe,WAAW,CAAC,0BACjF,EACE,OAAO,QACR,CACF;GAGH,MAAMC,cAA2B,EAAE;GAGnC,MAAM,gBAAgB,mBAAmB,KAAK,UAAU;IACtD,MAAM,cAAc,kBAAkB,MAAM,OAAO,MAAM,MAAM;AAE/D,QAAI,aAAa,EACf,WACE,GAAG,KAAK,mBAAmB,eAAe,YAAY,qBACtD,EACE,OAAO,QACR,CACF;IAIH,MAAM,eAAe,2BAA2B,MAAM;IACtD,MAAM,sBAAsB,mBAC1B,sBACI,uBAAuB,UACvB,EACE,qCACE,uBAAuB,SAC1B,EACL,aACD;IAED,MAAM,qBAAqB,YAAY;AACrC,YAAO,MAAM,aACX,YAAY;MACV,IAAIC;AAEJ,UAAI,YAAY,SACd,qBAAoB,MAAM,SAAS,cAAc;OAC/C,kBAAkB;OAClB;OACA,uBACE,oBAAoB,eACpB,UAAU,eACV;OACF,aAAa,KAAK;OAClB,cAAc;OACd;OACA;OACD,CAAC;UAEF,qBAAoB,MAAM,YAAY,GACnC,cAAc;OACb,kBAAkB;OAClB;OACA,uBACE,oBAAoB,eACpB,UAAU,eACV;OACF,aAAa,KAAK;OAClB,cAAc;OACd;OACA;OACD,CAAC,CACD,MAAM,WAAW,OAAO,KAAK;AAGlC,UAAI,CAAC,mBAAmB,YACtB,OAAM,IAAI,MAAM,oBAAoB;MAGtC,MAAM,EAAE,cAAc,0BACpB,kBAAkB,aAClB,aACD;AACD,UAAI,CAAC,UACH,OAAM,IAAI,MACR,oDACD;AAGH,qBAAe;AACf,aAAO,kBAAkB;QAE3B;MACE,UAAU;MACV,OAAO;MACP,UAAU,EAAE,OAAO,SAAS,eAAe;OACzC,MAAMC,gBAAc,kBAClB,MAAM,OACN,MAAM,MACP;AACD,iBACE,GAAG,KAAK,mBAAmB,eAAeA,cAAY,GAAG,SAAS,kBAAkB,WAAW,IAAI,CAAC,GAAG,SAAS,OAAO,UAAU,WAAW,QAAQ,KAAK,UAAU,MAAM,EAAE,WAAW,UAAU,CAAC,aAAa,eAAe,UAAU,EAAE,CAAC,MAAM,eAAe,SAAS,IACxQ,EACE,OAAO,SACR,CACF;AAED,0BAAmB;AAEnB,WAAI,mBAAmB,sBAAsB;AAC3C,kBAAU,6BAA6B,EACrC,OAAO,SACR,CAAC;AACF,gBAAQ,KAAK,EAAE;;;MAGpB,CACF,EAAE;;AAOL,YAJgB,SAAS,WACrB,QAAQ,SAAS,mBAAmB,GACpC,oBAAoB,EAET,MAAM,YAAY;KAAE;KAAO;KAAQ,EAAE;KACpD;AAMF,IAHqB,MAAM,QAAQ,IAAI,cAAc,EAIlD,MAAM,QAAQ,WAAW,OAAO,MAAM,QAAQ,OAAO,MAAM,MAAM,CACjE,SAAS,EAAE,aAAa;AACvB,gBAAY,KAAK,OAAO;KACxB;GAWJ,IAAI,eARW,kBACb,YAAY,KAAK,WAAW;IAC1B,GAAG;IACH,SAAS;IACV,EAAE,CACJ,CAGyB;AAE1B,OAAI,CAAC,oBACH,gBAAgB,cACZ;AAGN,OAAI,OAAO,uBAAuB,WAAW,SAE3C,gBAAe,iBACb,uBAAuB,WAAW,EAAE,EACpC,aACD;AAGH,UAAO,CAAC,cAAc,aAAa;IACnC,CACH;EAED,MAAMC,oBACJ,OAAO,YAAY,yBAAyB;EAU9C,IAAIC,mBAA+B;GACjC,GAAG,0BATkB,uBAAuB,SAC1C;IACE,GAAG;IACH,KAAK,uBAAuB;IAC5B,SAAS,EAAE;IACZ,GACD,uBAG0C;GAC5C,QAAQ;GACR,GAAG;GACJ;AAED,OAAK,MAAM,gBAAgB,KAAK,cAC9B,KAAI,kBAAkB,cACpB,oBAAmB,0BACjB,kBACA,kBAAkB,eAClB,aACD;AAIL,YACE,GAAG,KAAK,iBAAiB,GAAG,SAAS,sCAAsC,WAAW,MAAM,CAAC,OAAO,aAAa,SAAS,iBAAiB,SAAU,CAAC,IACtJ,EACE,OAAO,QACR,CACF;AAED,MACE,uBAAuB,WACtB,uBAAuB,SAAS,QAC/B,uBAAuB,SAAS,WAClC,uBAAuB,aAAa,SACpC;GACA,MAAM,qBAAqB,uBACxB,SAAU,MAAM,IAAI,CACpB,MAAM,GAAG,GAAG;GAEf,MAAM,eAAe,mBAAmB,mBAAmB,SAAS;AAEpE,UAAO,KAAK,MACV,KAAK,UAAU;IACb,GAAG;IACH,kBAAkB;KAChB,GAAG;KACH,MAAM;KACN,QAAQ;KACT;IACF,CAAC,CAAC,WACD,IAAI,OAAO,MAAM,aAAa,kBAAkB,IAAI,EACpD,WAAW,aAAa,OACzB,CACF;;AAGH,SAAO;GACL,GAAG;GACH;GACD;IAEH;EACE,UAAU;EACV,OAAO;EACP,UAAU,EAAE,OAAO,SAAS,eAC1B,UACE,GAAG,KAAK,iBAAiB,GAAG,SAAS,uBAAuB,WAAW,IAAI,CAAC,GAAG,SAAS,OAAO,UAAU,WAAW,QAAQ,KAAK,UAAU,MAAM,EAAE,WAAW,UAAU,CAAC,aAAa,eAAe,UAAU,EAAE,CAAC,MAAM,eAAe,SAAS,IAChP,EACE,OAAO,SACR,CACF;EACH,kBAAkB,EAAE,YAClB,UACE,GAAG,KAAK,iBAAiB,GAAG,SAAS,sCAAsC,WAAW,IAAI,CAAC,GAAG,SAAS,OAAO,UAAU,WAAW,QAAQ,KAAK,UAAU,MAAM,EAAE,WAAW,UAAU,IACvL,EACE,OAAO,SACR,CACF;EACJ,CACF,EAAE"}
|
package/dist/types/pull.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"pull.d.ts","names":[],"sources":["../../src/pull.ts"],"sourcesContent":[],"mappings":";;;KAmBK,WAAA;;EAAA,mBAAW,CAAA,EAAA,MAGE;EAcL,aA8OZ,CAAA,EA5PiB,
|
|
1
|
+
{"version":3,"file":"pull.d.ts","names":[],"sources":["../../src/pull.ts"],"sourcesContent":[],"mappings":";;;KAmBK,WAAA;;EAAA,mBAAW,CAAA,EAAA,MAGE;EAcL,aA8OZ,CAAA,EA5PiB,uBAcwC;;;;;;cAA7C,iBAAwB,gBAAc"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@intlayer/cli",
|
|
3
|
-
"version": "7.3.
|
|
3
|
+
"version": "7.3.13",
|
|
4
4
|
"private": false,
|
|
5
5
|
"description": "Provides uniform command-line interface scripts for Intlayer, used in packages like intlayer-cli and intlayer.",
|
|
6
6
|
"keywords": [
|
|
@@ -68,14 +68,14 @@
|
|
|
68
68
|
},
|
|
69
69
|
"dependencies": {
|
|
70
70
|
"@clack/prompts": "^0.11.0",
|
|
71
|
-
"@intlayer/api": "7.3.
|
|
72
|
-
"@intlayer/chokidar": "7.3.
|
|
73
|
-
"@intlayer/config": "7.3.
|
|
74
|
-
"@intlayer/core": "7.3.
|
|
75
|
-
"@intlayer/dictionaries-entry": "7.3.
|
|
76
|
-
"@intlayer/remote-dictionaries-entry": "7.3.
|
|
77
|
-
"@intlayer/types": "7.3.
|
|
78
|
-
"@intlayer/unmerged-dictionaries-entry": "7.3.
|
|
71
|
+
"@intlayer/api": "7.3.13",
|
|
72
|
+
"@intlayer/chokidar": "7.3.13",
|
|
73
|
+
"@intlayer/config": "7.3.13",
|
|
74
|
+
"@intlayer/core": "7.3.13",
|
|
75
|
+
"@intlayer/dictionaries-entry": "7.3.13",
|
|
76
|
+
"@intlayer/remote-dictionaries-entry": "7.3.13",
|
|
77
|
+
"@intlayer/types": "7.3.13",
|
|
78
|
+
"@intlayer/unmerged-dictionaries-entry": "7.3.13",
|
|
79
79
|
"commander": "14.0.1",
|
|
80
80
|
"eventsource": "3.0.7",
|
|
81
81
|
"fast-glob": "3.3.3"
|
|
@@ -88,10 +88,10 @@
|
|
|
88
88
|
"rimraf": "6.1.2",
|
|
89
89
|
"tsdown": "0.16.8",
|
|
90
90
|
"typescript": "5.9.3",
|
|
91
|
-
"vitest": "4.0.
|
|
91
|
+
"vitest": "4.0.15"
|
|
92
92
|
},
|
|
93
93
|
"peerDependencies": {
|
|
94
|
-
"@intlayer/ai": "7.3.
|
|
94
|
+
"@intlayer/ai": "7.3.13"
|
|
95
95
|
},
|
|
96
96
|
"peerDependenciesMeta": {
|
|
97
97
|
"@intlayer/ai": {
|