@intlayer/cli 8.6.1 → 8.6.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cjs/fill/extractTranslatableContent.cjs +58 -0
- package/dist/cjs/fill/extractTranslatableContent.cjs.map +1 -0
- package/dist/cjs/fill/fill.cjs +8 -2
- package/dist/cjs/fill/fill.cjs.map +1 -1
- package/dist/cjs/fill/translateDictionary.cjs +42 -67
- package/dist/cjs/fill/translateDictionary.cjs.map +1 -1
- package/dist/cjs/listContentDeclaration.cjs +5 -3
- package/dist/cjs/listContentDeclaration.cjs.map +1 -1
- package/dist/cjs/reviewDoc/reviewDoc.cjs +6 -0
- package/dist/cjs/reviewDoc/reviewDoc.cjs.map +1 -1
- package/dist/cjs/translateDoc/translateDoc.cjs +6 -0
- package/dist/cjs/translateDoc/translateDoc.cjs.map +1 -1
- package/dist/esm/fill/extractTranslatableContent.mjs +55 -0
- package/dist/esm/fill/extractTranslatableContent.mjs.map +1 -0
- package/dist/esm/fill/fill.mjs +9 -3
- package/dist/esm/fill/fill.mjs.map +1 -1
- package/dist/esm/fill/translateDictionary.mjs +43 -68
- package/dist/esm/fill/translateDictionary.mjs.map +1 -1
- package/dist/esm/listContentDeclaration.mjs +5 -3
- package/dist/esm/listContentDeclaration.mjs.map +1 -1
- package/dist/esm/reviewDoc/reviewDoc.mjs +7 -1
- package/dist/esm/reviewDoc/reviewDoc.mjs.map +1 -1
- package/dist/esm/translateDoc/translateDoc.mjs +7 -1
- package/dist/esm/translateDoc/translateDoc.mjs.map +1 -1
- package/dist/types/fill/extractTranslatableContent.d.ts +20 -0
- package/dist/types/fill/extractTranslatableContent.d.ts.map +1 -0
- package/dist/types/fill/fill.d.ts.map +1 -1
- package/dist/types/fill/translateDictionary.d.ts.map +1 -1
- package/dist/types/listContentDeclaration.d.ts +3 -3
- package/dist/types/listContentDeclaration.d.ts.map +1 -1
- package/dist/types/reviewDoc/reviewDoc.d.ts.map +1 -1
- package/dist/types/translateDoc/translateDoc.d.ts.map +1 -1
- package/package.json +12 -12
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"translateDoc.cjs","names":["setupAI","performance","getOutputFilePath","checkFileModifiedRange","translateFile","ANSIColors"],"sources":["../../../src/translateDoc/translateDoc.ts"],"sourcesContent":["import { existsSync, mkdirSync, writeFileSync } from 'node:fs';\nimport { dirname, join } from 'node:path';\nimport { performance } from 'node:perf_hooks';\nimport { listGitFiles, logConfigDetails } from '@intlayer/chokidar/cli';\nimport { parallelize, pLimit } from '@intlayer/chokidar/utils';\nimport * as ANSIColors from '@intlayer/config/colors';\nimport {\n colorize,\n colorizeNumber,\n getAppLogger,\n} from '@intlayer/config/logger';\nimport { getConfiguration } from '@intlayer/config/node';\nimport type { Locale } from '@intlayer/types/allLocales';\nimport fg from 'fast-glob';\nimport { checkFileModifiedRange } from '../utils/checkFileModifiedRange';\nimport { getOutputFilePath } from '../utils/getOutputFilePath';\nimport { setupAI } from '../utils/setupAI';\nimport { translateFile } from './translateFile';\nimport type { ErrorState, TranslateDocOptions } from './types';\n\nexport const translateDoc = async ({\n docPattern,\n locales,\n excludedGlobPattern,\n baseLocale,\n aiOptions,\n nbSimultaneousFileProcessed = 20, // Default to a higher concurrency for chunks\n configOptions,\n customInstructions,\n skipIfModifiedBefore,\n skipIfModifiedAfter,\n skipIfExists,\n gitOptions,\n flushStrategy = 'incremental',\n}: TranslateDocOptions) => {\n const configuration = getConfiguration(configOptions);\n logConfigDetails(configOptions);\n\n const appLogger = getAppLogger(configuration);\n\n // 1. GLOBAL QUEUE SETUP\n // We use pLimit to create a single bottleneck for AI requests.\n // This queue is shared across all files and locales.\n const maxConcurrentChunks = nbSimultaneousFileProcessed;\n const globalChunkLimiter = pLimit(maxConcurrentChunks);\n\n let docList: string[] = await fg(docPattern, {\n ignore: excludedGlobPattern,\n });\n\n const aiResult = await setupAI(configuration, aiOptions);\n if (!aiResult?.hasAIAccess) return;\n const { aiClient, aiConfig } = aiResult;\n\n if (gitOptions) {\n const gitChangedFiles = await listGitFiles(gitOptions);\n if (gitChangedFiles) {\n docList = docList.filter((path) =>\n gitChangedFiles.some((gitFile) => join(process.cwd(), path) === gitFile)\n );\n }\n }\n\n const batchStartTime = performance.now();\n\n appLogger(\n `Translating ${colorizeNumber(docList.length)} files to ${colorizeNumber(locales.length)} locales. \\n` +\n `Global Concurrency: ${colorizeNumber(maxConcurrentChunks)} chunks in parallel.`\n );\n\n const errorState: ErrorState = {\n count: 0,\n maxErrors: 5,\n shouldStop: false,\n };\n\n //
|
|
1
|
+
{"version":3,"file":"translateDoc.cjs","names":["setupAI","x","performance","getOutputFilePath","checkFileModifiedRange","translateFile","ANSIColors"],"sources":["../../../src/translateDoc/translateDoc.ts"],"sourcesContent":["import { existsSync, mkdirSync, writeFileSync } from 'node:fs';\nimport { dirname, join } from 'node:path';\nimport { performance } from 'node:perf_hooks';\nimport { checkAISDKAccess } from '@intlayer/ai';\nimport { listGitFiles, logConfigDetails } from '@intlayer/chokidar/cli';\nimport { parallelize, pLimit } from '@intlayer/chokidar/utils';\nimport * as ANSIColors from '@intlayer/config/colors';\nimport {\n colorize,\n colorizeNumber,\n getAppLogger,\n x,\n} from '@intlayer/config/logger';\nimport { getConfiguration } from '@intlayer/config/node';\nimport type { Locale } from '@intlayer/types/allLocales';\nimport fg from 'fast-glob';\nimport { checkFileModifiedRange } from '../utils/checkFileModifiedRange';\nimport { getOutputFilePath } from '../utils/getOutputFilePath';\nimport { setupAI } from '../utils/setupAI';\nimport { translateFile } from './translateFile';\nimport type { ErrorState, TranslateDocOptions } from './types';\n\nexport const translateDoc = async ({\n docPattern,\n locales,\n excludedGlobPattern,\n baseLocale,\n aiOptions,\n nbSimultaneousFileProcessed = 20, // Default to a higher concurrency for chunks\n configOptions,\n customInstructions,\n skipIfModifiedBefore,\n skipIfModifiedAfter,\n skipIfExists,\n gitOptions,\n flushStrategy = 'incremental',\n}: TranslateDocOptions) => {\n const configuration = getConfiguration(configOptions);\n logConfigDetails(configOptions);\n\n const appLogger = getAppLogger(configuration);\n\n // 1. GLOBAL QUEUE SETUP\n // We use pLimit to create a single bottleneck for AI requests.\n // This queue is shared across all files and locales.\n const maxConcurrentChunks = nbSimultaneousFileProcessed;\n const globalChunkLimiter = pLimit(maxConcurrentChunks);\n\n let docList: string[] = await fg(docPattern, {\n ignore: excludedGlobPattern,\n });\n\n const aiResult = await setupAI(configuration, aiOptions);\n if (!aiResult?.hasAIAccess) return;\n\n const { aiClient, aiConfig } = aiResult;\n\n const { hasAIAccess, error } = await checkAISDKAccess(aiConfig!);\n if (!hasAIAccess) {\n appLogger(`${x} ${error}`);\n return;\n }\n\n if (gitOptions) {\n const gitChangedFiles = await listGitFiles(gitOptions);\n if (gitChangedFiles) {\n docList = docList.filter((path) =>\n gitChangedFiles.some((gitFile) => join(process.cwd(), path) === gitFile)\n );\n }\n }\n\n const batchStartTime = performance.now();\n\n appLogger(\n `Translating ${colorizeNumber(docList.length)} files to ${colorizeNumber(locales.length)} locales. \\n` +\n `Global Concurrency: ${colorizeNumber(maxConcurrentChunks)} chunks in parallel.`\n );\n\n const errorState: ErrorState = {\n count: 0,\n maxErrors: 5,\n shouldStop: false,\n };\n\n // FLATTENED TASK LIST\n // We create a task for every File x Locale combination.\n const allTasks = docList.flatMap((docPath) =>\n locales.map((locale) => async () => {\n if (errorState.shouldStop) return;\n\n const absoluteBaseFilePath = join(configuration.system.baseDir, docPath);\n const outputFilePath = getOutputFilePath(\n absoluteBaseFilePath,\n locale,\n baseLocale\n );\n\n // Skip logic\n if (skipIfExists && existsSync(outputFilePath)) return;\n\n if (flushStrategy === 'incremental' && !existsSync(outputFilePath)) {\n mkdirSync(dirname(outputFilePath), { recursive: true });\n writeFileSync(outputFilePath, '');\n }\n\n const fileModificationData = checkFileModifiedRange(outputFilePath, {\n skipIfModifiedBefore,\n skipIfModifiedAfter,\n });\n\n if (fileModificationData.isSkipped) {\n appLogger(fileModificationData.message);\n return;\n }\n\n // Execute translation using the SHARED limiter\n await translateFile({\n baseFilePath: absoluteBaseFilePath,\n outputFilePath,\n locale: locale as Locale,\n baseLocale,\n configuration,\n errorState,\n aiOptions,\n customInstructions,\n aiClient,\n aiConfig,\n flushStrategy,\n limit: globalChunkLimiter, // Pass the global queue\n });\n })\n );\n\n // HIGH-THROUGHPUT FILE OPENER\n // We open many files simultaneously (e.g., 50) to ensure the global chunk queue\n // is always saturated with work.\n // If we open too few files, the chunk queue might drain faster than we can read new files.\n const FILE_OPEN_LIMIT = 50;\n\n await parallelize(allTasks, (task) => task(), FILE_OPEN_LIMIT);\n\n const batchEndTime = performance.now();\n const batchDuration = ((batchEndTime - batchStartTime) / 1000).toFixed(2);\n\n if (errorState.count > 0) {\n appLogger(`Finished with ${errorState.count} errors in ${batchDuration}s.`);\n } else {\n appLogger(\n `${colorize('✔', ANSIColors.GREEN)} Batch completed successfully in ${colorizeNumber(batchDuration)}s.`\n );\n }\n};\n"],"mappings":";;;;;;;;;;;;;;;;;;;;AAsBA,MAAa,eAAe,OAAO,EACjC,YACA,SACA,qBACA,YACA,WACA,8BAA8B,IAC9B,eACA,oBACA,sBACA,qBACA,cACA,YACA,gBAAgB,oBACS;CACzB,MAAM,4DAAiC,cAAc;AACrD,8CAAiB,cAAc;CAE/B,MAAM,sDAAyB,cAAc;CAK7C,MAAM,sBAAsB;CAC5B,MAAM,0DAA4B,oBAAoB;CAEtD,IAAI,UAAoB,6BAAS,YAAY,EAC3C,QAAQ,qBACT,CAAC;CAEF,MAAM,WAAW,MAAMA,8BAAQ,eAAe,UAAU;AACxD,KAAI,CAAC,UAAU,YAAa;CAE5B,MAAM,EAAE,UAAU,aAAa;CAE/B,MAAM,EAAE,aAAa,UAAU,yCAAuB,SAAU;AAChE,KAAI,CAAC,aAAa;AAChB,YAAU,GAAGC,0BAAE,GAAG,QAAQ;AAC1B;;AAGF,KAAI,YAAY;EACd,MAAM,kBAAkB,+CAAmB,WAAW;AACtD,MAAI,gBACF,WAAU,QAAQ,QAAQ,SACxB,gBAAgB,MAAM,gCAAiB,QAAQ,KAAK,EAAE,KAAK,KAAK,QAAQ,CACzE;;CAIL,MAAM,iBAAiBC,4BAAY,KAAK;AAExC,WACE,2DAA8B,QAAQ,OAAO,CAAC,wDAA2B,QAAQ,OAAO,CAAC,8EACjD,oBAAoB,CAAC,sBAC9D;CAED,MAAM,aAAyB;EAC7B,OAAO;EACP,WAAW;EACX,YAAY;EACb;AAyDD,iDArDiB,QAAQ,SAAS,YAChC,QAAQ,KAAK,WAAW,YAAY;AAClC,MAAI,WAAW,WAAY;EAE3B,MAAM,2CAA4B,cAAc,OAAO,SAAS,QAAQ;EACxE,MAAM,iBAAiBC,kDACrB,sBACA,QACA,WACD;AAGD,MAAI,wCAA2B,eAAe,CAAE;AAEhD,MAAI,kBAAkB,iBAAiB,yBAAY,eAAe,EAAE;AAClE,iDAAkB,eAAe,EAAE,EAAE,WAAW,MAAM,CAAC;AACvD,8BAAc,gBAAgB,GAAG;;EAGnC,MAAM,uBAAuBC,4DAAuB,gBAAgB;GAClE;GACA;GACD,CAAC;AAEF,MAAI,qBAAqB,WAAW;AAClC,aAAU,qBAAqB,QAAQ;AACvC;;AAIF,QAAMC,iDAAc;GAClB,cAAc;GACd;GACQ;GACR;GACA;GACA;GACA;GACA;GACA;GACA;GACA;GACA,OAAO;GACR,CAAC;GACF,CACH,GAQ4B,SAAS,MAAM,EAFpB,GAEsC;CAG9D,MAAM,kBADeH,4BAAY,KAAK,GACC,kBAAkB,KAAM,QAAQ,EAAE;AAEzE,KAAI,WAAW,QAAQ,EACrB,WAAU,iBAAiB,WAAW,MAAM,aAAa,cAAc,IAAI;KAE3E,WACE,yCAAY,KAAKI,wBAAW,MAAM,CAAC,+EAAkD,cAAc,CAAC,IACrG"}
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
//#region src/fill/extractTranslatableContent.ts
|
|
2
|
+
const extractTranslatableContent = (content, currentPath = [], state = {
|
|
3
|
+
currentIndex: 1,
|
|
4
|
+
extractedContent: [],
|
|
5
|
+
translatableDictionary: {}
|
|
6
|
+
}) => {
|
|
7
|
+
if (typeof content === "string") {
|
|
8
|
+
const regex = /[{]+.*?[}]+/g;
|
|
9
|
+
const replacement = {};
|
|
10
|
+
let varIndex = 1;
|
|
11
|
+
const modifiedContent = content.replace(regex, (matchStr) => {
|
|
12
|
+
const placeholder = `<${varIndex}>`;
|
|
13
|
+
replacement[placeholder] = matchStr;
|
|
14
|
+
varIndex++;
|
|
15
|
+
return placeholder;
|
|
16
|
+
});
|
|
17
|
+
const contentWithoutPlaceholders = modifiedContent.replace(/<\d+>/g, "");
|
|
18
|
+
if (/[\p{L}\p{N}]/u.test(contentWithoutPlaceholders)) {
|
|
19
|
+
state.extractedContent.push({
|
|
20
|
+
index: state.currentIndex,
|
|
21
|
+
path: currentPath,
|
|
22
|
+
value: modifiedContent,
|
|
23
|
+
replacement: Object.keys(replacement).length > 0 ? replacement : void 0
|
|
24
|
+
});
|
|
25
|
+
state.translatableDictionary[state.currentIndex] = modifiedContent;
|
|
26
|
+
state.currentIndex++;
|
|
27
|
+
}
|
|
28
|
+
} else if (Array.isArray(content)) content.forEach((item, index) => {
|
|
29
|
+
extractTranslatableContent(item, [...currentPath, index], state);
|
|
30
|
+
});
|
|
31
|
+
else if (typeof content === "object" && content !== null) {
|
|
32
|
+
for (const key in content) if (Object.hasOwn(content, key)) extractTranslatableContent(content[key], [...currentPath, key], state);
|
|
33
|
+
}
|
|
34
|
+
return {
|
|
35
|
+
extractedContent: state.extractedContent,
|
|
36
|
+
translatableDictionary: state.translatableDictionary
|
|
37
|
+
};
|
|
38
|
+
};
|
|
39
|
+
const reinsertTranslatedContent = (baseContent, extractedContentProps, translatedDictionary) => {
|
|
40
|
+
const result = structuredClone(baseContent);
|
|
41
|
+
for (const { index, path, replacement } of extractedContentProps) {
|
|
42
|
+
let translatedValue = translatedDictionary[index];
|
|
43
|
+
if (translatedValue !== void 0) {
|
|
44
|
+
if (replacement) for (const [placeholder, originalVar] of Object.entries(replacement)) translatedValue = translatedValue.replace(placeholder, originalVar);
|
|
45
|
+
let current = result;
|
|
46
|
+
for (let i = 0; i < path.length - 1; i++) current = current[path[i]];
|
|
47
|
+
current[path[path.length - 1]] = translatedValue;
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
return result;
|
|
51
|
+
};
|
|
52
|
+
|
|
53
|
+
//#endregion
|
|
54
|
+
export { extractTranslatableContent, reinsertTranslatedContent };
|
|
55
|
+
//# sourceMappingURL=extractTranslatableContent.mjs.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"extractTranslatableContent.mjs","names":[],"sources":["../../../src/fill/extractTranslatableContent.ts"],"sourcesContent":["export type ExtractedContentProps = {\n index: number;\n path: (string | number)[];\n value: string;\n replacement?: Record<string, string>;\n};\n\nexport type ExtractedContentResult = {\n extractedContent: ExtractedContentProps[];\n translatableDictionary: Record<number, string>;\n};\n\nexport const extractTranslatableContent = (\n content: any,\n currentPath: (string | number)[] = [],\n state = {\n currentIndex: 1,\n extractedContent: [] as ExtractedContentProps[],\n translatableDictionary: {} as Record<number, string>,\n }\n): ExtractedContentResult => {\n if (typeof content === 'string') {\n const regex = /[{]+.*?[}]+/g;\n const replacement: Record<string, string> = {};\n let varIndex = 1;\n\n const modifiedContent = content.replace(regex, (matchStr) => {\n const placeholder = `<${varIndex}>`;\n replacement[placeholder] = matchStr;\n varIndex++;\n return placeholder;\n });\n\n // Only extract strings that contain at least one letter or number outside of placeholders.\n // This avoids extracting strings that are only spaces, special characters, or just variables.\n const contentWithoutPlaceholders = modifiedContent.replace(/<\\d+>/g, '');\n if (/[\\p{L}\\p{N}]/u.test(contentWithoutPlaceholders)) {\n state.extractedContent.push({\n index: state.currentIndex,\n path: currentPath,\n value: modifiedContent,\n replacement:\n Object.keys(replacement).length > 0 ? replacement : undefined,\n });\n state.translatableDictionary[state.currentIndex] = modifiedContent;\n state.currentIndex++;\n }\n } else if (Array.isArray(content)) {\n content.forEach((item, index) => {\n extractTranslatableContent(item, [...currentPath, index], state);\n });\n } else if (typeof content === 'object' && content !== null) {\n for (const key in content) {\n if (Object.hasOwn(content, key)) {\n extractTranslatableContent(content[key], [...currentPath, key], state);\n }\n }\n }\n\n return {\n extractedContent: state.extractedContent,\n translatableDictionary: state.translatableDictionary,\n };\n};\n\nexport const reinsertTranslatedContent = (\n baseContent: any,\n extractedContentProps: ExtractedContentProps[],\n translatedDictionary: Record<number, string>\n): any => {\n const result = structuredClone(baseContent);\n\n for (const { index, path, replacement } of extractedContentProps) {\n let translatedValue = translatedDictionary[index];\n\n if (translatedValue !== undefined) {\n if (replacement) {\n for (const [placeholder, originalVar] of Object.entries(replacement)) {\n translatedValue = translatedValue.replace(placeholder, originalVar);\n }\n }\n\n let current = result;\n for (let i = 0; i < path.length - 1; i++) {\n current = current[path[i]];\n }\n current[path[path.length - 1]] = translatedValue;\n }\n }\n\n return result;\n};\n"],"mappings":";AAYA,MAAa,8BACX,SACA,cAAmC,EAAE,EACrC,QAAQ;CACN,cAAc;CACd,kBAAkB,EAAE;CACpB,wBAAwB,EAAE;CAC3B,KAC0B;AAC3B,KAAI,OAAO,YAAY,UAAU;EAC/B,MAAM,QAAQ;EACd,MAAM,cAAsC,EAAE;EAC9C,IAAI,WAAW;EAEf,MAAM,kBAAkB,QAAQ,QAAQ,QAAQ,aAAa;GAC3D,MAAM,cAAc,IAAI,SAAS;AACjC,eAAY,eAAe;AAC3B;AACA,UAAO;IACP;EAIF,MAAM,6BAA6B,gBAAgB,QAAQ,UAAU,GAAG;AACxE,MAAI,gBAAgB,KAAK,2BAA2B,EAAE;AACpD,SAAM,iBAAiB,KAAK;IAC1B,OAAO,MAAM;IACb,MAAM;IACN,OAAO;IACP,aACE,OAAO,KAAK,YAAY,CAAC,SAAS,IAAI,cAAc;IACvD,CAAC;AACF,SAAM,uBAAuB,MAAM,gBAAgB;AACnD,SAAM;;YAEC,MAAM,QAAQ,QAAQ,CAC/B,SAAQ,SAAS,MAAM,UAAU;AAC/B,6BAA2B,MAAM,CAAC,GAAG,aAAa,MAAM,EAAE,MAAM;GAChE;UACO,OAAO,YAAY,YAAY,YAAY,MACpD;OAAK,MAAM,OAAO,QAChB,KAAI,OAAO,OAAO,SAAS,IAAI,CAC7B,4BAA2B,QAAQ,MAAM,CAAC,GAAG,aAAa,IAAI,EAAE,MAAM;;AAK5E,QAAO;EACL,kBAAkB,MAAM;EACxB,wBAAwB,MAAM;EAC/B;;AAGH,MAAa,6BACX,aACA,uBACA,yBACQ;CACR,MAAM,SAAS,gBAAgB,YAAY;AAE3C,MAAK,MAAM,EAAE,OAAO,MAAM,iBAAiB,uBAAuB;EAChE,IAAI,kBAAkB,qBAAqB;AAE3C,MAAI,oBAAoB,QAAW;AACjC,OAAI,YACF,MAAK,MAAM,CAAC,aAAa,gBAAgB,OAAO,QAAQ,YAAY,CAClE,mBAAkB,gBAAgB,QAAQ,aAAa,YAAY;GAIvE,IAAI,UAAU;AACd,QAAK,IAAI,IAAI,GAAG,IAAI,KAAK,SAAS,GAAG,IACnC,WAAU,QAAQ,KAAK;AAEzB,WAAQ,KAAK,KAAK,SAAS,MAAM;;;AAIrC,QAAO"}
|
package/dist/esm/fill/fill.mjs
CHANGED
|
@@ -8,8 +8,9 @@ import { loadContentDeclarations, prepareIntlayer, writeContentDeclaration } fro
|
|
|
8
8
|
import { logConfigDetails } from "@intlayer/chokidar/cli";
|
|
9
9
|
import { formatPath, getGlobalLimiter, getTaskLimiter } from "@intlayer/chokidar/utils";
|
|
10
10
|
import * as ANSIColors from "@intlayer/config/colors";
|
|
11
|
-
import { colorize, colorizeKey, colorizePath, getAppLogger } from "@intlayer/config/logger";
|
|
11
|
+
import { colorize, colorizeKey, colorizePath, getAppLogger, x } from "@intlayer/config/logger";
|
|
12
12
|
import { getConfiguration } from "@intlayer/config/node";
|
|
13
|
+
import { checkAISDKAccess } from "@intlayer/ai";
|
|
13
14
|
|
|
14
15
|
//#region src/fill/fill.ts
|
|
15
16
|
const NB_CONCURRENT_TRANSLATIONS = 7;
|
|
@@ -29,10 +30,15 @@ const fill = async (options) => {
|
|
|
29
30
|
const aiResult = await setupAI(configuration, options?.aiOptions);
|
|
30
31
|
if (!aiResult?.hasAIAccess) return;
|
|
31
32
|
const { aiClient, aiConfig } = aiResult;
|
|
33
|
+
const { hasAIAccess, error } = await checkAISDKAccess(aiConfig);
|
|
34
|
+
if (!hasAIAccess) {
|
|
35
|
+
appLogger(`${x} ${error}`);
|
|
36
|
+
return;
|
|
37
|
+
}
|
|
32
38
|
const targetUnmergedDictionaries = await getTargetUnmergedDictionaries(options);
|
|
33
|
-
const sourceDictionaries = await loadContentDeclarations([...new Set(targetUnmergedDictionaries.map((unmergedDictionary) => unmergedDictionary.filePath).filter(Boolean))].map((sourcePath) => join(configuration.system.baseDir, sourcePath)), configuration);
|
|
39
|
+
const sourceDictionaries = await loadContentDeclarations([...new Set(targetUnmergedDictionaries.filter((unmergedDictionary) => unmergedDictionary.location !== "remote").map((unmergedDictionary) => unmergedDictionary.filePath).filter(Boolean))].map((sourcePath) => join(configuration.system.baseDir, sourcePath)), configuration, void 0, { logError: false });
|
|
34
40
|
const originalFillByPath = /* @__PURE__ */ new Map();
|
|
35
|
-
for (const
|
|
41
|
+
for (const dictionary of sourceDictionaries) if (dictionary.filePath) originalFillByPath.set(dictionary.filePath, dictionary.fill);
|
|
36
42
|
const affectedDictionaryKeys = /* @__PURE__ */ new Set();
|
|
37
43
|
targetUnmergedDictionaries.forEach((dict) => {
|
|
38
44
|
affectedDictionaryKeys.add(dict.key);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"fill.mjs","names":[],"sources":["../../../src/fill/fill.ts"],"sourcesContent":["import { basename, join, relative } from 'node:path';\nimport type { AIOptions } from '@intlayer/api';\nimport {\n loadContentDeclarations,\n prepareIntlayer,\n writeContentDeclaration,\n} from '@intlayer/chokidar/build';\nimport {\n type ListGitFilesOptions,\n logConfigDetails,\n} from '@intlayer/chokidar/cli';\nimport {\n formatPath,\n getGlobalLimiter,\n getTaskLimiter,\n} from '@intlayer/chokidar/utils';\nimport * as ANSIColors from '@intlayer/config/colors';\nimport {\n colorize,\n colorizeKey,\n colorizePath,\n getAppLogger,\n} from '@intlayer/config/logger';\nimport { getConfiguration } from '@intlayer/config/node';\nimport type { Locale } from '@intlayer/types/allLocales';\nimport type { Fill } from '@intlayer/types/dictionary';\nimport {\n ensureArray,\n type GetTargetDictionaryOptions,\n getTargetUnmergedDictionaries,\n} from '../getTargetDictionary';\nimport { setupAI } from '../utils/setupAI';\nimport {\n listTranslationsTasks,\n type TranslationTask,\n} from './listTranslationsTasks';\nimport { translateDictionary } from './translateDictionary';\nimport { writeFill } from './writeFill';\n\nconst NB_CONCURRENT_TRANSLATIONS = 7;\n\n// Arguments for the fill function\nexport type FillOptions = {\n sourceLocale?: Locale;\n outputLocales?: Locale | Locale[];\n mode?: 'complete' | 'review';\n gitOptions?: ListGitFilesOptions;\n aiOptions?: AIOptions; // Added aiOptions to be passed to translateJSON\n verbose?: boolean;\n nbConcurrentTranslations?: number;\n nbConcurrentTasks?: number; // NEW: number of tasks that may run at once\n build?: boolean;\n skipMetadata?: boolean;\n} & GetTargetDictionaryOptions;\n\n/**\n * Fill translations based on the provided options.\n */\nexport const fill = async (options?: FillOptions): Promise<void> => {\n const configuration = getConfiguration(options?.configOptions);\n logConfigDetails(options?.configOptions);\n\n const appLogger = getAppLogger(configuration);\n\n if (options?.build === true) {\n await prepareIntlayer(configuration, { forceRun: true });\n } else if (typeof options?.build === 'undefined') {\n await prepareIntlayer(configuration);\n }\n\n const { defaultLocale, locales } = configuration.internationalization;\n const mode = options?.mode ?? 'complete';\n const baseLocale = options?.sourceLocale ?? defaultLocale;\n\n const outputLocales = options?.outputLocales\n ? ensureArray(options.outputLocales)\n : locales;\n\n const aiResult = await setupAI(configuration, options?.aiOptions);\n\n if (!aiResult?.hasAIAccess) return;\n\n const { aiClient, aiConfig } = aiResult;\n\n const targetUnmergedDictionaries =\n await getTargetUnmergedDictionaries(options);\n\n // Load the original source content declaration files to recover function-type\n // `fill` values that are lost when dictionaries are JSON-serialised into\n // unmerged_dictionaries.cjs. Dictionary-level fill takes priority over the\n // config-level fill, but we can only know that by reading the source files.\n const uniqueSourcePaths = [\n ...new Set(\n targetUnmergedDictionaries\n .map((unmergedDictionary) => unmergedDictionary.filePath)\n .filter(Boolean) as string[]\n ),\n ];\n const sourceDictionaries = await loadContentDeclarations(\n uniqueSourcePaths.map((sourcePath) =>\n join(configuration.system.baseDir, sourcePath)\n ),\n configuration\n );\n // Map relative filePath → original fill value from the source file\n const originalFillByPath = new Map<string, Fill | undefined>();\n\n for (const dict of sourceDictionaries) {\n if (dict.filePath) {\n originalFillByPath.set(dict.filePath, dict.fill as Fill | undefined);\n }\n }\n\n const affectedDictionaryKeys = new Set<string>();\n targetUnmergedDictionaries.forEach((dict) => {\n affectedDictionaryKeys.add(dict.key);\n });\n\n const keysToProcess = Array.from(affectedDictionaryKeys);\n\n appLogger([\n 'Affected dictionary keys for processing:',\n keysToProcess.length > 0\n ? keysToProcess.map((key) => colorizeKey(key)).join(', ')\n : colorize('No keys found', ANSIColors.YELLOW),\n ]);\n\n if (keysToProcess.length === 0) return;\n\n /**\n * List the translations tasks\n *\n * Create a list of per-locale dictionaries to translate\n *\n * In 'complete' mode, filter only the missing locales to translate\n */\n const translationTasks: TranslationTask[] = listTranslationsTasks(\n targetUnmergedDictionaries.map((dictionary) => dictionary.localId!),\n outputLocales,\n mode,\n baseLocale,\n configuration\n );\n\n // AI calls in flight at once (translateJSON + metadata audit)\n const nbConcurrentTranslations =\n options?.nbConcurrentTranslations ?? NB_CONCURRENT_TRANSLATIONS;\n const globalLimiter = getGlobalLimiter(nbConcurrentTranslations);\n\n // NEW: number of *tasks* that may run at once (start/prepare/log/write)\n const nbConcurrentTasks = Math.max(\n 1,\n Math.min(\n options?.nbConcurrentTasks ?? nbConcurrentTranslations,\n translationTasks.length\n )\n );\n\n const taskLimiter = getTaskLimiter(nbConcurrentTasks);\n\n const runners = translationTasks.map((task) =>\n taskLimiter(async () => {\n const relativePath = relative(\n configuration?.system?.baseDir ?? process.cwd(),\n task?.dictionaryFilePath ?? ''\n );\n\n // log AFTER acquiring a task slot\n appLogger(\n `${task.dictionaryPreset} Processing ${colorizePath(basename(relativePath))}`,\n { level: 'info' }\n );\n\n const translationTaskResult = await translateDictionary(\n task,\n configuration,\n {\n mode,\n aiOptions: options?.aiOptions,\n fillMetadata: !options?.skipMetadata,\n onHandle: globalLimiter,\n aiClient,\n aiConfig,\n }\n );\n\n if (!translationTaskResult?.dictionaryOutput) return;\n\n const { dictionaryOutput, sourceLocale } = translationTaskResult;\n\n // Determine if we should write to separate files\n // - If dictionary has explicit fill setting (string, function, or object), use it\n // - If dictionary is per-locale AND has no explicit fill=false, use global fill config\n // - If dictionary is multilingual (no locale property), always write to same file\n //\n // NOTE: function-type fill values are lost during JSON serialisation of\n // unmerged_dictionaries.cjs. We recover them by checking the original\n // source file that was loaded above (originalFillByPath). Dictionary-level\n // fill always takes priority over config-level fill.\n const originalFill = originalFillByPath.get(\n dictionaryOutput.filePath ?? ''\n );\n\n // originalFill is undefined when the source file had no fill property; use\n // the (possibly JSON-preserved) dictionaryOutput.fill as a fallback so that\n // string/boolean fill values set directly on the dict still work.\n const dictFill: Fill | undefined =\n originalFill !== undefined ? originalFill : dictionaryOutput.fill;\n\n const hasDictionaryLevelFill =\n typeof dictFill === 'string' ||\n typeof dictFill === 'function' ||\n (typeof dictFill === 'object' && dictFill !== null);\n\n const isPerLocale = typeof dictionaryOutput.locale === 'string';\n\n const effectiveFill = hasDictionaryLevelFill\n ? dictFill\n : isPerLocale\n ? (configuration.dictionary?.fill ?? true)\n : (configuration.dictionary?.fill ?? false); // Multilingual dictionaries use config-level fill if set\n\n const isFillOtherFile =\n typeof effectiveFill === 'string' ||\n typeof effectiveFill === 'function' ||\n (typeof effectiveFill === 'object' && effectiveFill !== null);\n\n if (isFillOtherFile) {\n await writeFill(\n {\n ...dictionaryOutput,\n // Ensure fill is set on the dictionary for writeFill to use\n fill: effectiveFill,\n },\n outputLocales,\n [sourceLocale],\n configuration\n );\n } else {\n await writeContentDeclaration(dictionaryOutput, configuration);\n\n if (dictionaryOutput.filePath) {\n appLogger(\n `${task.dictionaryPreset} Content declaration written to ${formatPath(basename(dictionaryOutput.filePath))}`,\n { level: 'info' }\n );\n }\n }\n })\n );\n\n await Promise.all(runners);\n await (globalLimiter as any).onIdle();\n};\n"],"mappings":";;;;;;;;;;;;;;AAuCA,MAAM,6BAA6B;;;;AAmBnC,MAAa,OAAO,OAAO,YAAyC;CAClE,MAAM,gBAAgB,iBAAiB,SAAS,cAAc;AAC9D,kBAAiB,SAAS,cAAc;CAExC,MAAM,YAAY,aAAa,cAAc;AAE7C,KAAI,SAAS,UAAU,KACrB,OAAM,gBAAgB,eAAe,EAAE,UAAU,MAAM,CAAC;UAC/C,OAAO,SAAS,UAAU,YACnC,OAAM,gBAAgB,cAAc;CAGtC,MAAM,EAAE,eAAe,YAAY,cAAc;CACjD,MAAM,OAAO,SAAS,QAAQ;CAC9B,MAAM,aAAa,SAAS,gBAAgB;CAE5C,MAAM,gBAAgB,SAAS,gBAC3B,YAAY,QAAQ,cAAc,GAClC;CAEJ,MAAM,WAAW,MAAM,QAAQ,eAAe,SAAS,UAAU;AAEjE,KAAI,CAAC,UAAU,YAAa;CAE5B,MAAM,EAAE,UAAU,aAAa;CAE/B,MAAM,6BACJ,MAAM,8BAA8B,QAAQ;CAa9C,MAAM,qBAAqB,MAAM,wBAPP,CACxB,GAAG,IAAI,IACL,2BACG,KAAK,uBAAuB,mBAAmB,SAAS,CACxD,OAAO,QAAQ,CACnB,CACF,CAEmB,KAAK,eACrB,KAAK,cAAc,OAAO,SAAS,WAAW,CAC/C,EACD,cACD;CAED,MAAM,qCAAqB,IAAI,KAA+B;AAE9D,MAAK,MAAM,QAAQ,mBACjB,KAAI,KAAK,SACP,oBAAmB,IAAI,KAAK,UAAU,KAAK,KAAyB;CAIxE,MAAM,yCAAyB,IAAI,KAAa;AAChD,4BAA2B,SAAS,SAAS;AAC3C,yBAAuB,IAAI,KAAK,IAAI;GACpC;CAEF,MAAM,gBAAgB,MAAM,KAAK,uBAAuB;AAExD,WAAU,CACR,4CACA,cAAc,SAAS,IACnB,cAAc,KAAK,QAAQ,YAAY,IAAI,CAAC,CAAC,KAAK,KAAK,GACvD,SAAS,iBAAiB,WAAW,OAAO,CACjD,CAAC;AAEF,KAAI,cAAc,WAAW,EAAG;;;;;;;;CAShC,MAAM,mBAAsC,sBAC1C,2BAA2B,KAAK,eAAe,WAAW,QAAS,EACnE,eACA,MACA,YACA,cACD;CAGD,MAAM,2BACJ,SAAS,4BAA4B;CACvC,MAAM,gBAAgB,iBAAiB,yBAAyB;CAWhE,MAAM,cAAc,eARM,KAAK,IAC7B,GACA,KAAK,IACH,SAAS,qBAAqB,0BAC9B,iBAAiB,OAClB,CACF,CAEoD;CAErD,MAAM,UAAU,iBAAiB,KAAK,SACpC,YAAY,YAAY;EACtB,MAAM,eAAe,SACnB,eAAe,QAAQ,WAAW,QAAQ,KAAK,EAC/C,MAAM,sBAAsB,GAC7B;AAGD,YACE,GAAG,KAAK,iBAAiB,cAAc,aAAa,SAAS,aAAa,CAAC,IAC3E,EAAE,OAAO,QAAQ,CAClB;EAED,MAAM,wBAAwB,MAAM,oBAClC,MACA,eACA;GACE;GACA,WAAW,SAAS;GACpB,cAAc,CAAC,SAAS;GACxB,UAAU;GACV;GACA;GACD,CACF;AAED,MAAI,CAAC,uBAAuB,iBAAkB;EAE9C,MAAM,EAAE,kBAAkB,iBAAiB;EAW3C,MAAM,eAAe,mBAAmB,IACtC,iBAAiB,YAAY,GAC9B;EAKD,MAAM,WACJ,iBAAiB,SAAY,eAAe,iBAAiB;EAE/D,MAAM,yBACJ,OAAO,aAAa,YACpB,OAAO,aAAa,cACnB,OAAO,aAAa,YAAY,aAAa;EAEhD,MAAM,cAAc,OAAO,iBAAiB,WAAW;EAEvD,MAAM,gBAAgB,yBAClB,WACA,cACG,cAAc,YAAY,QAAQ,OAClC,cAAc,YAAY,QAAQ;AAOzC,MAJE,OAAO,kBAAkB,YACzB,OAAO,kBAAkB,cACxB,OAAO,kBAAkB,YAAY,kBAAkB,KAGxD,OAAM,UACJ;GACE,GAAG;GAEH,MAAM;GACP,EACD,eACA,CAAC,aAAa,EACd,cACD;OACI;AACL,SAAM,wBAAwB,kBAAkB,cAAc;AAE9D,OAAI,iBAAiB,SACnB,WACE,GAAG,KAAK,iBAAiB,kCAAkC,WAAW,SAAS,iBAAiB,SAAS,CAAC,IAC1G,EAAE,OAAO,QAAQ,CAClB;;GAGL,CACH;AAED,OAAM,QAAQ,IAAI,QAAQ;AAC1B,OAAO,cAAsB,QAAQ"}
|
|
1
|
+
{"version":3,"file":"fill.mjs","names":[],"sources":["../../../src/fill/fill.ts"],"sourcesContent":["import { basename, join, relative } from 'node:path';\nimport { checkAISDKAccess } from '@intlayer/ai';\nimport type { AIOptions } from '@intlayer/api';\nimport {\n loadContentDeclarations,\n prepareIntlayer,\n writeContentDeclaration,\n} from '@intlayer/chokidar/build';\nimport {\n type ListGitFilesOptions,\n logConfigDetails,\n} from '@intlayer/chokidar/cli';\nimport {\n formatPath,\n getGlobalLimiter,\n getTaskLimiter,\n} from '@intlayer/chokidar/utils';\nimport * as ANSIColors from '@intlayer/config/colors';\nimport {\n colorize,\n colorizeKey,\n colorizePath,\n getAppLogger,\n x,\n} from '@intlayer/config/logger';\nimport { getConfiguration } from '@intlayer/config/node';\nimport type { Locale } from '@intlayer/types/allLocales';\nimport type { Fill } from '@intlayer/types/dictionary';\nimport {\n ensureArray,\n type GetTargetDictionaryOptions,\n getTargetUnmergedDictionaries,\n} from '../getTargetDictionary';\nimport { setupAI } from '../utils/setupAI';\nimport {\n listTranslationsTasks,\n type TranslationTask,\n} from './listTranslationsTasks';\nimport { translateDictionary } from './translateDictionary';\nimport { writeFill } from './writeFill';\n\nconst NB_CONCURRENT_TRANSLATIONS = 7;\n\n// Arguments for the fill function\nexport type FillOptions = {\n sourceLocale?: Locale;\n outputLocales?: Locale | Locale[];\n mode?: 'complete' | 'review';\n gitOptions?: ListGitFilesOptions;\n aiOptions?: AIOptions; // Added aiOptions to be passed to translateJSON\n verbose?: boolean;\n nbConcurrentTranslations?: number;\n nbConcurrentTasks?: number; // NEW: number of tasks that may run at once\n build?: boolean;\n skipMetadata?: boolean;\n} & GetTargetDictionaryOptions;\n\n/**\n * Fill translations based on the provided options.\n */\nexport const fill = async (options?: FillOptions): Promise<void> => {\n const configuration = getConfiguration(options?.configOptions);\n logConfigDetails(options?.configOptions);\n\n const appLogger = getAppLogger(configuration);\n\n if (options?.build === true) {\n await prepareIntlayer(configuration, { forceRun: true });\n } else if (typeof options?.build === 'undefined') {\n await prepareIntlayer(configuration);\n }\n\n const { defaultLocale, locales } = configuration.internationalization;\n const mode = options?.mode ?? 'complete';\n const baseLocale = options?.sourceLocale ?? defaultLocale;\n\n const outputLocales = options?.outputLocales\n ? ensureArray(options.outputLocales)\n : locales;\n\n const aiResult = await setupAI(configuration, options?.aiOptions);\n\n if (!aiResult?.hasAIAccess) return;\n\n const { aiClient, aiConfig } = aiResult;\n\n const { hasAIAccess, error } = await checkAISDKAccess(aiConfig!);\n if (!hasAIAccess) {\n appLogger(`${x} ${error}`);\n return;\n }\n\n const targetUnmergedDictionaries =\n await getTargetUnmergedDictionaries(options);\n\n // Load the original source content declaration files to recover function-type\n // `fill` values that are lost when dictionaries are JSON-serialised into\n // unmerged_dictionaries.cjs. Dictionary-level fill takes priority over the\n // config-level fill, but we can only know that by reading the source files.\n const uniqueSourcePaths = [\n ...new Set(\n targetUnmergedDictionaries\n .filter(\n (unmergedDictionary) => unmergedDictionary.location !== 'remote'\n )\n .map((unmergedDictionary) => unmergedDictionary.filePath)\n .filter(Boolean) as string[]\n ),\n ];\n const sourceDictionaries = await loadContentDeclarations(\n uniqueSourcePaths.map((sourcePath) =>\n join(configuration.system.baseDir, sourcePath)\n ),\n configuration,\n undefined,\n {\n logError: false,\n }\n );\n // Map relative filePath → original fill value from the source file\n const originalFillByPath = new Map<string, Fill | undefined>();\n\n for (const dictionary of sourceDictionaries) {\n if (dictionary.filePath) {\n originalFillByPath.set(\n dictionary.filePath,\n dictionary.fill as Fill | undefined\n );\n }\n }\n\n const affectedDictionaryKeys = new Set<string>();\n\n targetUnmergedDictionaries.forEach((dict) => {\n affectedDictionaryKeys.add(dict.key);\n });\n\n const keysToProcess = Array.from(affectedDictionaryKeys);\n\n appLogger([\n 'Affected dictionary keys for processing:',\n keysToProcess.length > 0\n ? keysToProcess.map((key) => colorizeKey(key)).join(', ')\n : colorize('No keys found', ANSIColors.YELLOW),\n ]);\n\n if (keysToProcess.length === 0) return;\n\n /**\n * List the translations tasks\n *\n * Create a list of per-locale dictionaries to translate\n *\n * In 'complete' mode, filter only the missing locales to translate\n */\n const translationTasks: TranslationTask[] = listTranslationsTasks(\n targetUnmergedDictionaries.map((dictionary) => dictionary.localId!),\n outputLocales,\n mode,\n baseLocale,\n configuration\n );\n\n // AI calls in flight at once (translateJSON + metadata audit)\n const nbConcurrentTranslations =\n options?.nbConcurrentTranslations ?? NB_CONCURRENT_TRANSLATIONS;\n const globalLimiter = getGlobalLimiter(nbConcurrentTranslations);\n\n // NEW: number of *tasks* that may run at once (start/prepare/log/write)\n const nbConcurrentTasks = Math.max(\n 1,\n Math.min(\n options?.nbConcurrentTasks ?? nbConcurrentTranslations,\n translationTasks.length\n )\n );\n\n const taskLimiter = getTaskLimiter(nbConcurrentTasks);\n\n const runners = translationTasks.map((task) =>\n taskLimiter(async () => {\n const relativePath = relative(\n configuration?.system?.baseDir ?? process.cwd(),\n task?.dictionaryFilePath ?? ''\n );\n\n // log AFTER acquiring a task slot\n appLogger(\n `${task.dictionaryPreset} Processing ${colorizePath(basename(relativePath))}`,\n { level: 'info' }\n );\n\n const translationTaskResult = await translateDictionary(\n task,\n configuration,\n {\n mode,\n aiOptions: options?.aiOptions,\n fillMetadata: !options?.skipMetadata,\n onHandle: globalLimiter,\n aiClient,\n aiConfig,\n }\n );\n\n if (!translationTaskResult?.dictionaryOutput) return;\n\n const { dictionaryOutput, sourceLocale } = translationTaskResult;\n\n // Determine if we should write to separate files\n // - If dictionary has explicit fill setting (string, function, or object), use it\n // - If dictionary is per-locale AND has no explicit fill=false, use global fill config\n // - If dictionary is multilingual (no locale property), always write to same file\n //\n // NOTE: function-type fill values are lost during JSON serialisation of\n // unmerged_dictionaries.cjs. We recover them by checking the original\n // source file that was loaded above (originalFillByPath). Dictionary-level\n // fill always takes priority over config-level fill.\n const originalFill = originalFillByPath.get(\n dictionaryOutput.filePath ?? ''\n );\n\n // originalFill is undefined when the source file had no fill property; use\n // the (possibly JSON-preserved) dictionaryOutput.fill as a fallback so that\n // string/boolean fill values set directly on the dict still work.\n const dictFill: Fill | undefined =\n originalFill !== undefined ? originalFill : dictionaryOutput.fill;\n\n const hasDictionaryLevelFill =\n typeof dictFill === 'string' ||\n typeof dictFill === 'function' ||\n (typeof dictFill === 'object' && dictFill !== null);\n\n const isPerLocale = typeof dictionaryOutput.locale === 'string';\n\n const effectiveFill = hasDictionaryLevelFill\n ? dictFill\n : isPerLocale\n ? (configuration.dictionary?.fill ?? true)\n : (configuration.dictionary?.fill ?? false); // Multilingual dictionaries use config-level fill if set\n\n const isFillOtherFile =\n typeof effectiveFill === 'string' ||\n typeof effectiveFill === 'function' ||\n (typeof effectiveFill === 'object' && effectiveFill !== null);\n\n if (isFillOtherFile) {\n await writeFill(\n {\n ...dictionaryOutput,\n // Ensure fill is set on the dictionary for writeFill to use\n fill: effectiveFill,\n },\n outputLocales,\n [sourceLocale],\n configuration\n );\n } else {\n await writeContentDeclaration(dictionaryOutput, configuration);\n\n if (dictionaryOutput.filePath) {\n appLogger(\n `${task.dictionaryPreset} Content declaration written to ${formatPath(basename(dictionaryOutput.filePath))}`,\n { level: 'info' }\n );\n }\n }\n })\n );\n\n await Promise.all(runners);\n await (globalLimiter as any).onIdle();\n};\n"],"mappings":";;;;;;;;;;;;;;;AAyCA,MAAM,6BAA6B;;;;AAmBnC,MAAa,OAAO,OAAO,YAAyC;CAClE,MAAM,gBAAgB,iBAAiB,SAAS,cAAc;AAC9D,kBAAiB,SAAS,cAAc;CAExC,MAAM,YAAY,aAAa,cAAc;AAE7C,KAAI,SAAS,UAAU,KACrB,OAAM,gBAAgB,eAAe,EAAE,UAAU,MAAM,CAAC;UAC/C,OAAO,SAAS,UAAU,YACnC,OAAM,gBAAgB,cAAc;CAGtC,MAAM,EAAE,eAAe,YAAY,cAAc;CACjD,MAAM,OAAO,SAAS,QAAQ;CAC9B,MAAM,aAAa,SAAS,gBAAgB;CAE5C,MAAM,gBAAgB,SAAS,gBAC3B,YAAY,QAAQ,cAAc,GAClC;CAEJ,MAAM,WAAW,MAAM,QAAQ,eAAe,SAAS,UAAU;AAEjE,KAAI,CAAC,UAAU,YAAa;CAE5B,MAAM,EAAE,UAAU,aAAa;CAE/B,MAAM,EAAE,aAAa,UAAU,MAAM,iBAAiB,SAAU;AAChE,KAAI,CAAC,aAAa;AAChB,YAAU,GAAG,EAAE,GAAG,QAAQ;AAC1B;;CAGF,MAAM,6BACJ,MAAM,8BAA8B,QAAQ;CAgB9C,MAAM,qBAAqB,MAAM,wBAVP,CACxB,GAAG,IAAI,IACL,2BACG,QACE,uBAAuB,mBAAmB,aAAa,SACzD,CACA,KAAK,uBAAuB,mBAAmB,SAAS,CACxD,OAAO,QAAQ,CACnB,CACF,CAEmB,KAAK,eACrB,KAAK,cAAc,OAAO,SAAS,WAAW,CAC/C,EACD,eACA,QACA,EACE,UAAU,OACX,CACF;CAED,MAAM,qCAAqB,IAAI,KAA+B;AAE9D,MAAK,MAAM,cAAc,mBACvB,KAAI,WAAW,SACb,oBAAmB,IACjB,WAAW,UACX,WAAW,KACZ;CAIL,MAAM,yCAAyB,IAAI,KAAa;AAEhD,4BAA2B,SAAS,SAAS;AAC3C,yBAAuB,IAAI,KAAK,IAAI;GACpC;CAEF,MAAM,gBAAgB,MAAM,KAAK,uBAAuB;AAExD,WAAU,CACR,4CACA,cAAc,SAAS,IACnB,cAAc,KAAK,QAAQ,YAAY,IAAI,CAAC,CAAC,KAAK,KAAK,GACvD,SAAS,iBAAiB,WAAW,OAAO,CACjD,CAAC;AAEF,KAAI,cAAc,WAAW,EAAG;;;;;;;;CAShC,MAAM,mBAAsC,sBAC1C,2BAA2B,KAAK,eAAe,WAAW,QAAS,EACnE,eACA,MACA,YACA,cACD;CAGD,MAAM,2BACJ,SAAS,4BAA4B;CACvC,MAAM,gBAAgB,iBAAiB,yBAAyB;CAWhE,MAAM,cAAc,eARM,KAAK,IAC7B,GACA,KAAK,IACH,SAAS,qBAAqB,0BAC9B,iBAAiB,OAClB,CACF,CAEoD;CAErD,MAAM,UAAU,iBAAiB,KAAK,SACpC,YAAY,YAAY;EACtB,MAAM,eAAe,SACnB,eAAe,QAAQ,WAAW,QAAQ,KAAK,EAC/C,MAAM,sBAAsB,GAC7B;AAGD,YACE,GAAG,KAAK,iBAAiB,cAAc,aAAa,SAAS,aAAa,CAAC,IAC3E,EAAE,OAAO,QAAQ,CAClB;EAED,MAAM,wBAAwB,MAAM,oBAClC,MACA,eACA;GACE;GACA,WAAW,SAAS;GACpB,cAAc,CAAC,SAAS;GACxB,UAAU;GACV;GACA;GACD,CACF;AAED,MAAI,CAAC,uBAAuB,iBAAkB;EAE9C,MAAM,EAAE,kBAAkB,iBAAiB;EAW3C,MAAM,eAAe,mBAAmB,IACtC,iBAAiB,YAAY,GAC9B;EAKD,MAAM,WACJ,iBAAiB,SAAY,eAAe,iBAAiB;EAE/D,MAAM,yBACJ,OAAO,aAAa,YACpB,OAAO,aAAa,cACnB,OAAO,aAAa,YAAY,aAAa;EAEhD,MAAM,cAAc,OAAO,iBAAiB,WAAW;EAEvD,MAAM,gBAAgB,yBAClB,WACA,cACG,cAAc,YAAY,QAAQ,OAClC,cAAc,YAAY,QAAQ;AAOzC,MAJE,OAAO,kBAAkB,YACzB,OAAO,kBAAkB,cACxB,OAAO,kBAAkB,YAAY,kBAAkB,KAGxD,OAAM,UACJ;GACE,GAAG;GAEH,MAAM;GACP,EACD,eACA,CAAC,aAAa,EACd,cACD;OACI;AACL,SAAM,wBAAwB,kBAAkB,cAAc;AAE9D,OAAI,iBAAiB,SACnB,WACE,GAAG,KAAK,iBAAiB,kCAAkC,WAAW,SAAS,iBAAiB,SAAS,CAAC,IAC1G,EAAE,OAAO,QAAQ,CAClB;;GAGL,CACH;AAED,OAAM,QAAQ,IAAI,QAAQ;AAC1B,OAAO,cAAsB,QAAQ"}
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { deepMergeContent } from "./deepMergeContent.mjs";
|
|
2
|
-
import {
|
|
2
|
+
import { extractTranslatableContent, reinsertTranslatedContent } from "./extractTranslatableContent.mjs";
|
|
3
3
|
import { basename } from "node:path";
|
|
4
|
-
import { chunkJSON, formatLocale, mergeChunks, reconstructFromSingleChunk,
|
|
4
|
+
import { chunkJSON, excludeObjectFormat, formatLocale, mergeChunks, reconstructFromSingleChunk, verifyIdenticObjectFormat } from "@intlayer/chokidar/utils";
|
|
5
5
|
import * as ANSIColors from "@intlayer/config/colors";
|
|
6
6
|
import { colon, colorize, colorizeNumber, colorizePath, getAppLogger } from "@intlayer/config/logger";
|
|
7
7
|
import { getUnmergedDictionaries } from "@intlayer/unmerged-dictionaries-entry";
|
|
@@ -10,39 +10,26 @@ import { getIntlayerAPIProxy } from "@intlayer/api";
|
|
|
10
10
|
import { retryManager } from "@intlayer/config/utils";
|
|
11
11
|
|
|
12
12
|
//#region src/fill/translateDictionary.ts
|
|
13
|
+
const createChunkPreset = (chunkIndex, totalChunks) => {
|
|
14
|
+
if (totalChunks <= 1) return "";
|
|
15
|
+
return colon([
|
|
16
|
+
colorize("[", ANSIColors.GREY_DARK),
|
|
17
|
+
colorizeNumber(chunkIndex + 1),
|
|
18
|
+
colorize(`/${totalChunks}`, ANSIColors.GREY_DARK),
|
|
19
|
+
colorize("]", ANSIColors.GREY_DARK)
|
|
20
|
+
].join(""), { colSize: 5 });
|
|
21
|
+
};
|
|
13
22
|
const hasMissingMetadata = (dictionary) => !dictionary.description || !dictionary.title || !dictionary.tags;
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
content: obj,
|
|
22
|
-
nulls: void 0,
|
|
23
|
-
hasNulls: false
|
|
24
|
-
};
|
|
25
|
-
const content = {};
|
|
26
|
-
const nulls = {};
|
|
27
|
-
let hasNulls = false;
|
|
28
|
-
for (const [key, value] of Object.entries(obj)) if (value === null) {
|
|
29
|
-
nulls[key] = null;
|
|
30
|
-
hasNulls = true;
|
|
31
|
-
} else {
|
|
32
|
-
const child = stripNullValues(value);
|
|
33
|
-
content[key] = child.content;
|
|
34
|
-
if (child.hasNulls) {
|
|
35
|
-
nulls[key] = child.nulls;
|
|
36
|
-
hasNulls = true;
|
|
37
|
-
}
|
|
23
|
+
const serializeError = (error) => {
|
|
24
|
+
if (error instanceof Error) return error.cause ? `${error.message} (cause: ${String(error.cause)})` : error.message;
|
|
25
|
+
if (typeof error === "string") return error;
|
|
26
|
+
try {
|
|
27
|
+
return JSON.stringify(error);
|
|
28
|
+
} catch {
|
|
29
|
+
return String(error);
|
|
38
30
|
}
|
|
39
|
-
return {
|
|
40
|
-
content,
|
|
41
|
-
nulls: hasNulls ? nulls : void 0,
|
|
42
|
-
hasNulls
|
|
43
|
-
};
|
|
44
31
|
};
|
|
45
|
-
const CHUNK_SIZE =
|
|
32
|
+
const CHUNK_SIZE = 1500;
|
|
46
33
|
const GROUP_MAX_RETRY = 2;
|
|
47
34
|
const MAX_RETRY = 3;
|
|
48
35
|
const RETRY_DELAY = 1e3 * 10;
|
|
@@ -100,51 +87,42 @@ const translateDictionary = async (task, configuration, options) => {
|
|
|
100
87
|
let targetLocaleDictionary;
|
|
101
88
|
if (typeof baseUnmergedDictionary.locale === "string") {
|
|
102
89
|
const targetLocaleFilePath = baseUnmergedDictionary.filePath?.replace(new RegExp(`/${task.sourceLocale}/`, "g"), `/${targetLocale}/`);
|
|
103
|
-
|
|
104
|
-
targetLocaleDictionary = targetUnmergedDictionary ?? {
|
|
90
|
+
targetLocaleDictionary = (targetLocaleFilePath ? unmergedDictionariesRecord[task.dictionaryKey]?.find((dict) => dict.filePath === targetLocaleFilePath && dict.locale === targetLocale) : void 0) ?? {
|
|
105
91
|
key: baseUnmergedDictionary.key,
|
|
106
92
|
content: {},
|
|
107
93
|
filePath: targetLocaleFilePath,
|
|
108
94
|
locale: targetLocale
|
|
109
95
|
};
|
|
110
|
-
if (mode === "complete") dictionaryToProcess = getFilterMissingContentPerLocale(dictionaryToProcess, targetUnmergedDictionary);
|
|
111
96
|
} else {
|
|
112
97
|
if (mode === "complete") dictionaryToProcess = getFilterMissingTranslationsDictionary(dictionaryToProcess, targetLocale);
|
|
113
98
|
dictionaryToProcess = getPerLocaleDictionary(dictionaryToProcess, task.sourceLocale);
|
|
114
99
|
targetLocaleDictionary = getPerLocaleDictionary(baseUnmergedDictionary, targetLocale);
|
|
115
100
|
}
|
|
101
|
+
if (mode === "complete") dictionaryToProcess = {
|
|
102
|
+
...dictionaryToProcess,
|
|
103
|
+
content: excludeObjectFormat(dictionaryToProcess.content, targetLocaleDictionary.content) ?? {}
|
|
104
|
+
};
|
|
116
105
|
const localePreset = colon([
|
|
117
106
|
colorize("[", ANSIColors.GREY_DARK),
|
|
118
107
|
formatLocale(targetLocale),
|
|
119
108
|
colorize("]", ANSIColors.GREY_DARK)
|
|
120
109
|
].join(""), { colSize: 18 });
|
|
121
|
-
const createChunkPreset = (chunkIndex, totalChunks) => {
|
|
122
|
-
if (totalChunks <= 1) return "";
|
|
123
|
-
return colon([
|
|
124
|
-
colorize("[", ANSIColors.GREY_DARK),
|
|
125
|
-
colorizeNumber(chunkIndex + 1),
|
|
126
|
-
colorize(`/${totalChunks}`, ANSIColors.GREY_DARK),
|
|
127
|
-
colorize("]", ANSIColors.GREY_DARK)
|
|
128
|
-
].join(""), { colSize: 5 });
|
|
129
|
-
};
|
|
130
110
|
appLogger(`${task.dictionaryPreset}${localePreset} Preparing ${colorizePath(basename(targetLocaleDictionary.filePath))}`, { level: "info" });
|
|
131
|
-
const
|
|
132
|
-
const { content: contentToProcess, nulls: strippedNullValues } = stripNullValues(isContentStructured ? dictionaryToProcess.content : { __INTLAYER_ROOT_PRIMITIVE_CONTENT__: dictionaryToProcess.content });
|
|
133
|
-
const chunkedJsonContent = chunkJSON(contentToProcess, CHUNK_SIZE);
|
|
111
|
+
const chunkedJsonContent = chunkJSON(dictionaryToProcess.content, CHUNK_SIZE);
|
|
134
112
|
const nbOfChunks = chunkedJsonContent.length;
|
|
135
113
|
if (nbOfChunks > 1) appLogger(`${task.dictionaryPreset}${localePreset} Split into ${colorizeNumber(nbOfChunks)} chunks for translation`, { level: "info" });
|
|
136
114
|
const chunkResult = [];
|
|
137
|
-
const chunkPromises = chunkedJsonContent.map((chunk) => {
|
|
115
|
+
const chunkPromises = chunkedJsonContent.map(async (chunk) => {
|
|
138
116
|
const chunkPreset = createChunkPreset(chunk.index, chunk.total);
|
|
139
117
|
if (nbOfChunks > 1) appLogger(`${task.dictionaryPreset}${localePreset}${chunkPreset} Translating chunk`, { level: "info" });
|
|
140
|
-
const
|
|
141
|
-
const
|
|
118
|
+
const reconstructedChunk = reconstructFromSingleChunk(chunk);
|
|
119
|
+
const { extractedContent: chunkExtractedContent, translatableDictionary: chunkTranslatableDictionary } = extractTranslatableContent(reconstructedChunk);
|
|
142
120
|
const executeTranslation = async () => {
|
|
143
121
|
return await retryManager(async () => {
|
|
144
122
|
let translationResult;
|
|
145
123
|
if (aiClient && aiConfig) translationResult = await aiClient.translateJSON({
|
|
146
|
-
entryFileContent:
|
|
147
|
-
presetOutputContent,
|
|
124
|
+
entryFileContent: chunkTranslatableDictionary,
|
|
125
|
+
presetOutputContent: chunkTranslatableDictionary,
|
|
148
126
|
dictionaryDescription: dictionaryToProcess.description ?? metadata?.description ?? "",
|
|
149
127
|
entryLocale: task.sourceLocale,
|
|
150
128
|
outputLocale: targetLocale,
|
|
@@ -152,8 +130,8 @@ const translateDictionary = async (task, configuration, options) => {
|
|
|
152
130
|
aiConfig
|
|
153
131
|
});
|
|
154
132
|
else translationResult = await intlayerAPI.ai.translateJSON({
|
|
155
|
-
entryFileContent:
|
|
156
|
-
presetOutputContent,
|
|
133
|
+
entryFileContent: chunkTranslatableDictionary,
|
|
134
|
+
presetOutputContent: chunkTranslatableDictionary,
|
|
157
135
|
dictionaryDescription: dictionaryToProcess.description ?? metadata?.description ?? "",
|
|
158
136
|
entryLocale: task.sourceLocale,
|
|
159
137
|
outputLocale: targetLocale,
|
|
@@ -161,16 +139,16 @@ const translateDictionary = async (task, configuration, options) => {
|
|
|
161
139
|
aiOptions
|
|
162
140
|
}).then((result) => result.data);
|
|
163
141
|
if (!translationResult?.fileContent) throw new Error("No content result");
|
|
164
|
-
const { isIdentic } = verifyIdenticObjectFormat(translationResult.fileContent,
|
|
165
|
-
if (!isIdentic) throw new Error(
|
|
142
|
+
const { isIdentic, error } = verifyIdenticObjectFormat(translationResult.fileContent, chunkTranslatableDictionary);
|
|
143
|
+
if (!isIdentic) throw new Error(`Translation result does not match expected format: ${error}`);
|
|
166
144
|
notifySuccess();
|
|
167
|
-
return translationResult.fileContent;
|
|
145
|
+
return reinsertTranslatedContent(reconstructedChunk, chunkExtractedContent, translationResult.fileContent);
|
|
168
146
|
}, {
|
|
169
147
|
maxRetry: MAX_RETRY,
|
|
170
148
|
delay: RETRY_DELAY,
|
|
171
149
|
onError: ({ error, attempt, maxRetry }) => {
|
|
172
150
|
const chunkPreset = createChunkPreset(chunk.index, chunk.total);
|
|
173
|
-
appLogger(`${task.dictionaryPreset}${localePreset}${chunkPreset} ${colorize("Error filling:", ANSIColors.RED)} ${colorize(
|
|
151
|
+
appLogger(`${task.dictionaryPreset}${localePreset}${chunkPreset} ${colorize("Error filling:", ANSIColors.RED)} ${colorize(serializeError(error), ANSIColors.GREY_DARK)} - Attempt ${colorizeNumber(attempt + 1)} of ${colorizeNumber(maxRetry)}`, { level: "error" });
|
|
174
152
|
followingErrors += 1;
|
|
175
153
|
if (followingErrors >= MAX_FOLLOWING_ERRORS) {
|
|
176
154
|
appLogger(`There is something wrong.`, { level: "error" });
|
|
@@ -187,15 +165,12 @@ const translateDictionary = async (task, configuration, options) => {
|
|
|
187
165
|
(await Promise.all(chunkPromises)).sort((chunkA, chunkB) => chunkA.chunk.index - chunkB.chunk.index).forEach(({ result }) => {
|
|
188
166
|
chunkResult.push(result);
|
|
189
167
|
});
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
let finalContent = {
|
|
168
|
+
const reinsertedContent = mergeChunks(chunkResult);
|
|
169
|
+
const merged = {
|
|
193
170
|
...dictionaryToProcess,
|
|
194
|
-
content:
|
|
195
|
-
}
|
|
196
|
-
|
|
197
|
-
if (typeof baseUnmergedDictionary.locale === "string") finalContent = deepMergeContent(targetLocaleDictionary.content ?? {}, finalContent);
|
|
198
|
-
return [targetLocale, finalContent];
|
|
171
|
+
content: reinsertedContent
|
|
172
|
+
};
|
|
173
|
+
return [targetLocale, deepMergeContent(targetLocaleDictionary.content ?? {}, merged.content)];
|
|
199
174
|
}));
|
|
200
175
|
const translatedContent = Object.fromEntries(translatedContentResults);
|
|
201
176
|
let dictionaryOutput = {
|
|
@@ -229,8 +204,8 @@ const translateDictionary = async (task, configuration, options) => {
|
|
|
229
204
|
}, {
|
|
230
205
|
maxRetry: GROUP_MAX_RETRY,
|
|
231
206
|
delay: RETRY_DELAY,
|
|
232
|
-
onError: ({ error, attempt, maxRetry }) => appLogger(`${task.dictionaryPreset} ${colorize("Error:", ANSIColors.RED)} ${colorize(
|
|
233
|
-
onMaxTryReached: ({ error }) => appLogger(`${task.dictionaryPreset} ${colorize("Maximum number of retries reached:", ANSIColors.RED)} ${colorize(
|
|
207
|
+
onError: ({ error, attempt, maxRetry }) => appLogger(`${task.dictionaryPreset} ${colorize("Error:", ANSIColors.RED)} ${colorize(serializeError(error), ANSIColors.GREY_DARK)} - Attempt ${colorizeNumber(attempt + 1)} of ${colorizeNumber(maxRetry)}`, { level: "error" }),
|
|
208
|
+
onMaxTryReached: ({ error }) => appLogger(`${task.dictionaryPreset} ${colorize("Maximum number of retries reached:", ANSIColors.RED)} ${colorize(serializeError(error), ANSIColors.GREY_DARK)}`, { level: "error" })
|
|
234
209
|
})();
|
|
235
210
|
};
|
|
236
211
|
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"translateDictionary.mjs","names":[],"sources":["../../../src/fill/translateDictionary.ts"],"sourcesContent":["import { basename } from 'node:path';\nimport type { AIConfig } from '@intlayer/ai';\nimport { type AIOptions, getIntlayerAPIProxy } from '@intlayer/api';\nimport {\n chunkJSON,\n formatLocale,\n type JsonChunk,\n mergeChunks,\n reconstructFromSingleChunk,\n reduceObjectFormat,\n verifyIdenticObjectFormat,\n} from '@intlayer/chokidar/utils';\nimport * as ANSIColors from '@intlayer/config/colors';\nimport {\n colon,\n colorize,\n colorizeNumber,\n colorizePath,\n getAppLogger,\n} from '@intlayer/config/logger';\nimport { retryManager } from '@intlayer/config/utils';\nimport {\n getFilterMissingTranslationsDictionary,\n getMultilingualDictionary,\n getPerLocaleDictionary,\n insertContentInDictionary,\n} from '@intlayer/core/plugins';\nimport type { Locale } from '@intlayer/types/allLocales';\nimport type { IntlayerConfig } from '@intlayer/types/config';\nimport type { Dictionary } from '@intlayer/types/dictionary';\nimport { getUnmergedDictionaries } from '@intlayer/unmerged-dictionaries-entry';\nimport type { AIClient } from '../utils/setupAI';\nimport { deepMergeContent } from './deepMergeContent';\nimport { getFilterMissingContentPerLocale } from './getFilterMissingContentPerLocale';\nimport type { TranslationTask } from './listTranslationsTasks';\n\ntype TranslateDictionaryResult = TranslationTask & {\n dictionaryOutput: Dictionary | null;\n};\n\ntype TranslateDictionaryOptions = {\n mode: 'complete' | 'review';\n aiOptions?: AIOptions;\n fillMetadata?: boolean;\n onHandle?: ReturnType<\n typeof import('@intlayer/chokidar/utils').getGlobalLimiter\n >;\n onSuccess?: () => void;\n onError?: (error: unknown) => void;\n getAbortError?: () => Error | null;\n aiClient?: AIClient;\n aiConfig?: AIConfig;\n};\n\nconst hasMissingMetadata = (dictionary: Dictionary) =>\n !dictionary.description || !dictionary.title || !dictionary.tags;\n\n/**\n * Recursively strips null values from an object, returning the cleaned content\n * and a separate object containing only the null-valued paths so they can be\n * re-injected after AI translation (nulls don't need translation).\n */\nconst stripNullValues = (\n obj: any\n): { content: any; nulls: any; hasNulls: boolean } => {\n if (typeof obj !== 'object' || obj === null || Array.isArray(obj)) {\n return { content: obj, nulls: undefined, hasNulls: false };\n }\n\n const content: any = {};\n const nulls: any = {};\n let hasNulls = false;\n\n for (const [key, value] of Object.entries(obj)) {\n if (value === null) {\n nulls[key] = null;\n hasNulls = true;\n } else {\n const child = stripNullValues(value);\n content[key] = child.content;\n if (child.hasNulls) {\n nulls[key] = child.nulls;\n hasNulls = true;\n }\n }\n }\n\n return { content, nulls: hasNulls ? nulls : undefined, hasNulls };\n};\n\nconst CHUNK_SIZE = 7000; // GPT-5 Mini safe input size\nconst GROUP_MAX_RETRY = 2;\nconst MAX_RETRY = 3;\nconst RETRY_DELAY = 1000 * 10; // 10 seconds\n\nconst MAX_FOLLOWING_ERRORS = 10; // 10 errors in a row, hard exit the process\nlet followingErrors = 0;\n\nexport const translateDictionary = async (\n task: TranslationTask,\n configuration: IntlayerConfig,\n options?: TranslateDictionaryOptions\n): Promise<TranslateDictionaryResult> => {\n const appLogger = getAppLogger(configuration);\n const intlayerAPI = getIntlayerAPIProxy(undefined, configuration);\n\n const { mode, aiOptions, fillMetadata, aiClient, aiConfig } = {\n mode: 'complete',\n fillMetadata: true,\n ...options,\n } as const;\n\n const notifySuccess = () => {\n followingErrors = 0;\n options?.onSuccess?.();\n };\n\n const result = await retryManager(\n async () => {\n const unmergedDictionariesRecord = getUnmergedDictionaries(configuration);\n\n const baseUnmergedDictionary: Dictionary | undefined =\n unmergedDictionariesRecord[task.dictionaryKey].find(\n (dict) => dict.localId === task.dictionaryLocalId\n );\n\n if (!baseUnmergedDictionary) {\n appLogger(\n `${task.dictionaryPreset}Dictionary not found in unmergedDictionariesRecord. Skipping.`,\n {\n level: 'warn',\n }\n );\n return { ...task, dictionaryOutput: null };\n }\n\n let metadata:\n | Pick<Dictionary, 'description' | 'title' | 'tags'>\n | undefined;\n\n if (\n fillMetadata &&\n (hasMissingMetadata(baseUnmergedDictionary) || mode === 'review')\n ) {\n const defaultLocaleDictionary = getPerLocaleDictionary(\n baseUnmergedDictionary,\n configuration.internationalization.defaultLocale\n );\n\n appLogger(\n `${task.dictionaryPreset} Filling missing metadata for ${colorizePath(basename(baseUnmergedDictionary.filePath!))}`,\n {\n level: 'info',\n }\n );\n\n const runAudit = async () => {\n if (aiClient && aiConfig) {\n const result = await aiClient.auditDictionaryMetadata({\n fileContent: JSON.stringify(defaultLocaleDictionary),\n aiConfig,\n });\n\n return {\n data: result,\n };\n }\n\n return await intlayerAPI.ai.auditContentDeclarationMetadata({\n fileContent: JSON.stringify(defaultLocaleDictionary),\n aiOptions,\n });\n };\n\n const metadataResult = options?.onHandle\n ? await options.onHandle(runAudit)\n : await runAudit();\n\n metadata = metadataResult.data?.fileContent;\n }\n\n const translatedContentResults = await Promise.all(\n task.targetLocales.map(async (targetLocale) => {\n /**\n * In complete mode, for large dictionaries, we want to filter all content that is already translated\n *\n * targetLocale: fr\n *\n * { test1: t({ ar: 'Hello', en: 'Hello', fr: 'Bonjour' } }) -> {}\n * { test2: t({ ar: 'Hello', en: 'Hello' }) } -> { test2: t({ ar: 'Hello', en: 'Hello' }) }\n *\n */\n // Reset to base dictionary for each locale to ensure we filter from the original\n let dictionaryToProcess = structuredClone(baseUnmergedDictionary);\n\n let targetLocaleDictionary: Dictionary;\n\n if (typeof baseUnmergedDictionary.locale === 'string') {\n // For per-locale files, the content is already in simple JSON format (not translation nodes)\n // The base dictionary is already the source locale content\n\n // Load the existing target locale dictionary\n const targetLocaleFilePath =\n baseUnmergedDictionary.filePath?.replace(\n new RegExp(`/${task.sourceLocale}/`, 'g'),\n `/${targetLocale}/`\n );\n\n // Find the target locale dictionary in unmerged dictionaries\n const targetUnmergedDictionary = targetLocaleFilePath\n ? unmergedDictionariesRecord[task.dictionaryKey]?.find(\n (dict) =>\n dict.filePath === targetLocaleFilePath &&\n dict.locale === targetLocale\n )\n : undefined;\n\n targetLocaleDictionary = targetUnmergedDictionary ?? {\n key: baseUnmergedDictionary.key,\n content: {},\n filePath: targetLocaleFilePath,\n locale: targetLocale,\n };\n\n // In complete mode, filter out already translated content\n if (mode === 'complete') {\n dictionaryToProcess = getFilterMissingContentPerLocale(\n dictionaryToProcess,\n targetUnmergedDictionary\n );\n }\n } else {\n // For multilingual dictionaries\n if (mode === 'complete') {\n // Remove all nodes that don't have any content to translate\n dictionaryToProcess = getFilterMissingTranslationsDictionary(\n dictionaryToProcess,\n targetLocale\n );\n }\n\n dictionaryToProcess = getPerLocaleDictionary(\n dictionaryToProcess,\n task.sourceLocale\n );\n\n targetLocaleDictionary = getPerLocaleDictionary(\n baseUnmergedDictionary,\n targetLocale\n );\n }\n\n const localePreset = colon(\n [\n colorize('[', ANSIColors.GREY_DARK),\n formatLocale(targetLocale),\n colorize(']', ANSIColors.GREY_DARK),\n ].join(''),\n { colSize: 18 }\n );\n\n const createChunkPreset = (\n chunkIndex: number,\n totalChunks: number\n ) => {\n if (totalChunks <= 1) return '';\n return colon(\n [\n colorize('[', ANSIColors.GREY_DARK),\n colorizeNumber(chunkIndex + 1),\n colorize(`/${totalChunks}`, ANSIColors.GREY_DARK),\n colorize(']', ANSIColors.GREY_DARK),\n ].join(''),\n { colSize: 5 }\n );\n };\n\n appLogger(\n `${task.dictionaryPreset}${localePreset} Preparing ${colorizePath(basename(targetLocaleDictionary.filePath!))}`,\n {\n level: 'info',\n }\n );\n\n const isContentStructured =\n (typeof dictionaryToProcess.content === 'object' &&\n dictionaryToProcess.content !== null) ||\n Array.isArray(dictionaryToProcess.content);\n\n const rawContentToProcess = isContentStructured\n ? dictionaryToProcess.content\n : {\n __INTLAYER_ROOT_PRIMITIVE_CONTENT__:\n dictionaryToProcess.content,\n };\n\n // Strip null values before sending to AI — nulls need no translation\n // and confuse the model. They will be re-injected after merging.\n const { content: contentToProcess, nulls: strippedNullValues } =\n stripNullValues(rawContentToProcess);\n\n const chunkedJsonContent: JsonChunk[] = chunkJSON(\n contentToProcess as unknown as Record<string, any>,\n CHUNK_SIZE\n );\n\n const nbOfChunks = chunkedJsonContent.length;\n\n if (nbOfChunks > 1) {\n appLogger(\n `${task.dictionaryPreset}${localePreset} Split into ${colorizeNumber(nbOfChunks)} chunks for translation`,\n {\n level: 'info',\n }\n );\n }\n\n const chunkResult: JsonChunk[] = [];\n\n // Process chunks in parallel (globally throttled) to allow concurrent translation\n const chunkPromises = chunkedJsonContent.map((chunk) => {\n const chunkPreset = createChunkPreset(chunk.index, chunk.total);\n\n if (nbOfChunks > 1) {\n appLogger(\n `${task.dictionaryPreset}${localePreset}${chunkPreset} Translating chunk`,\n {\n level: 'info',\n }\n );\n }\n\n // Reconstruct partial JSON content from this chunk's patches\n const chunkContent = reconstructFromSingleChunk(chunk);\n const presetOutputContent = reduceObjectFormat(\n isContentStructured\n ? targetLocaleDictionary.content\n : {\n __INTLAYER_ROOT_PRIMITIVE_CONTENT__:\n targetLocaleDictionary.content,\n },\n chunkContent\n ) as unknown as JSON;\n\n const executeTranslation = async () => {\n return await retryManager(\n async () => {\n let translationResult: any;\n\n if (aiClient && aiConfig) {\n translationResult = await aiClient.translateJSON({\n entryFileContent: chunkContent as unknown as JSON,\n presetOutputContent,\n dictionaryDescription:\n dictionaryToProcess.description ??\n metadata?.description ??\n '',\n entryLocale: task.sourceLocale,\n outputLocale: targetLocale,\n mode,\n aiConfig,\n });\n } else {\n translationResult = await intlayerAPI.ai\n .translateJSON({\n entryFileContent: chunkContent as unknown as JSON,\n presetOutputContent,\n dictionaryDescription:\n dictionaryToProcess.description ??\n metadata?.description ??\n '',\n entryLocale: task.sourceLocale,\n outputLocale: targetLocale,\n mode,\n aiOptions,\n })\n .then((result) => result.data);\n }\n\n if (!translationResult?.fileContent) {\n throw new Error('No content result');\n }\n\n const { isIdentic } = verifyIdenticObjectFormat(\n translationResult.fileContent,\n chunkContent\n );\n\n if (!isIdentic) {\n throw new Error(\n 'Translation result does not match expected format'\n );\n }\n\n notifySuccess();\n return translationResult.fileContent;\n },\n {\n maxRetry: MAX_RETRY,\n delay: RETRY_DELAY,\n onError: ({ error, attempt, maxRetry }) => {\n const chunkPreset = createChunkPreset(\n chunk.index,\n chunk.total\n );\n appLogger(\n `${task.dictionaryPreset}${localePreset}${chunkPreset} ${colorize('Error filling:', ANSIColors.RED)} ${colorize(typeof error === 'string' ? error : JSON.stringify(error), ANSIColors.GREY_DARK)} - Attempt ${colorizeNumber(attempt + 1)} of ${colorizeNumber(maxRetry)}`,\n {\n level: 'error',\n }\n );\n\n followingErrors += 1;\n\n if (followingErrors >= MAX_FOLLOWING_ERRORS) {\n appLogger(`There is something wrong.`, {\n level: 'error',\n });\n process.exit(1); // 1 for error\n }\n },\n }\n )();\n };\n\n const wrapped = options?.onHandle\n ? options.onHandle(executeTranslation) // queued in global limiter\n : executeTranslation(); // no global limiter\n\n return wrapped.then((result) => ({ chunk, result }));\n });\n\n // Wait for all chunks for this locale in parallel (still capped by global limiter)\n const chunkResults = await Promise.all(chunkPromises);\n\n // Maintain order\n chunkResults\n .sort((chunkA, chunkB) => chunkA.chunk.index - chunkB.chunk.index)\n .forEach(({ result }) => {\n chunkResult.push(result);\n });\n\n // Merge partial JSON objects produced from each chunk into a single object\n let mergedContent = mergeChunks(chunkResult);\n\n // Re-inject null values that were stripped before AI translation\n if (strippedNullValues) {\n mergedContent = deepMergeContent(mergedContent, strippedNullValues);\n }\n\n const merged = {\n ...dictionaryToProcess,\n content: mergedContent,\n };\n\n // For per-locale files, merge the newly translated content with existing target content\n let finalContent = merged.content;\n\n if (!isContentStructured) {\n finalContent = (finalContent as any)\n ?.__INTLAYER_ROOT_PRIMITIVE_CONTENT__;\n }\n\n if (typeof baseUnmergedDictionary.locale === 'string') {\n // Deep merge: existing content + newly translated content\n finalContent = deepMergeContent(\n targetLocaleDictionary.content ?? {},\n finalContent\n );\n }\n\n return [targetLocale, finalContent] as const;\n })\n );\n\n const translatedContent: Partial<Record<Locale, Dictionary['content']>> =\n Object.fromEntries(translatedContentResults);\n\n const baseDictionary = baseUnmergedDictionary.locale\n ? {\n ...baseUnmergedDictionary,\n key: baseUnmergedDictionary.key!,\n content: {},\n }\n : baseUnmergedDictionary;\n\n let dictionaryOutput: Dictionary = {\n ...getMultilingualDictionary(baseDictionary),\n locale: undefined, // Ensure the dictionary is multilingual\n ...metadata,\n };\n\n for (const targetLocale of task.targetLocales) {\n if (translatedContent[targetLocale]) {\n dictionaryOutput = insertContentInDictionary(\n dictionaryOutput,\n translatedContent[targetLocale],\n targetLocale\n );\n }\n }\n\n appLogger(\n `${task.dictionaryPreset} ${colorize('Translation completed successfully', ANSIColors.GREEN)} for ${colorizePath(basename(dictionaryOutput.filePath!))}`,\n {\n level: 'info',\n }\n );\n\n // The dict-level `fill` value may have been lost during JSON serialisation\n // (functions can't be serialised to JSON). Fall back to the config-level\n // fill so that an explicit fill in intlayer.config.ts is honoured.\n const effectiveFillForCheck =\n baseUnmergedDictionary.fill ?? configuration.dictionary?.fill;\n\n if (\n baseUnmergedDictionary.locale &&\n (effectiveFillForCheck === true ||\n effectiveFillForCheck === undefined) &&\n baseUnmergedDictionary.location === 'local'\n ) {\n const dictionaryFilePath = baseUnmergedDictionary\n .filePath!.split('.')\n .slice(0, -1);\n\n const contentIndex = dictionaryFilePath[dictionaryFilePath.length - 1];\n\n return JSON.parse(\n JSON.stringify({\n ...task,\n dictionaryOutput: {\n ...dictionaryOutput,\n fill: undefined,\n filled: true,\n },\n }).replaceAll(\n new RegExp(`\\\\.${contentIndex}\\\\.[a-zA-Z0-9]+`, 'g'),\n `.filled.${contentIndex}.json`\n )\n );\n }\n\n return {\n ...task,\n dictionaryOutput,\n };\n },\n {\n maxRetry: GROUP_MAX_RETRY,\n delay: RETRY_DELAY,\n onError: ({ error, attempt, maxRetry }) =>\n appLogger(\n `${task.dictionaryPreset} ${colorize('Error:', ANSIColors.RED)} ${colorize(typeof error === 'string' ? error : JSON.stringify(error), ANSIColors.GREY_DARK)} - Attempt ${colorizeNumber(attempt + 1)} of ${colorizeNumber(maxRetry)}`,\n {\n level: 'error',\n }\n ),\n onMaxTryReached: ({ error }) =>\n appLogger(\n `${task.dictionaryPreset} ${colorize('Maximum number of retries reached:', ANSIColors.RED)} ${colorize(typeof error === 'string' ? error : JSON.stringify(error), ANSIColors.GREY_DARK)}`,\n {\n level: 'error',\n }\n ),\n }\n )();\n\n return result as TranslateDictionaryResult;\n};\n"],"mappings":";;;;;;;;;;;;AAsDA,MAAM,sBAAsB,eAC1B,CAAC,WAAW,eAAe,CAAC,WAAW,SAAS,CAAC,WAAW;;;;;;AAO9D,MAAM,mBACJ,QACoD;AACpD,KAAI,OAAO,QAAQ,YAAY,QAAQ,QAAQ,MAAM,QAAQ,IAAI,CAC/D,QAAO;EAAE,SAAS;EAAK,OAAO;EAAW,UAAU;EAAO;CAG5D,MAAM,UAAe,EAAE;CACvB,MAAM,QAAa,EAAE;CACrB,IAAI,WAAW;AAEf,MAAK,MAAM,CAAC,KAAK,UAAU,OAAO,QAAQ,IAAI,CAC5C,KAAI,UAAU,MAAM;AAClB,QAAM,OAAO;AACb,aAAW;QACN;EACL,MAAM,QAAQ,gBAAgB,MAAM;AACpC,UAAQ,OAAO,MAAM;AACrB,MAAI,MAAM,UAAU;AAClB,SAAM,OAAO,MAAM;AACnB,cAAW;;;AAKjB,QAAO;EAAE;EAAS,OAAO,WAAW,QAAQ;EAAW;EAAU;;AAGnE,MAAM,aAAa;AACnB,MAAM,kBAAkB;AACxB,MAAM,YAAY;AAClB,MAAM,cAAc,MAAO;AAE3B,MAAM,uBAAuB;AAC7B,IAAI,kBAAkB;AAEtB,MAAa,sBAAsB,OACjC,MACA,eACA,YACuC;CACvC,MAAM,YAAY,aAAa,cAAc;CAC7C,MAAM,cAAc,oBAAoB,QAAW,cAAc;CAEjE,MAAM,EAAE,MAAM,WAAW,cAAc,UAAU,aAAa;EAC5D,MAAM;EACN,cAAc;EACd,GAAG;EACJ;CAED,MAAM,sBAAsB;AAC1B,oBAAkB;AAClB,WAAS,aAAa;;AAqcxB,QAlce,MAAM,aACnB,YAAY;EACV,MAAM,6BAA6B,wBAAwB,cAAc;EAEzE,MAAM,yBACJ,2BAA2B,KAAK,eAAe,MAC5C,SAAS,KAAK,YAAY,KAAK,kBACjC;AAEH,MAAI,CAAC,wBAAwB;AAC3B,aACE,GAAG,KAAK,iBAAiB,gEACzB,EACE,OAAO,QACR,CACF;AACD,UAAO;IAAE,GAAG;IAAM,kBAAkB;IAAM;;EAG5C,IAAI;AAIJ,MACE,iBACC,mBAAmB,uBAAuB,IAAI,SAAS,WACxD;GACA,MAAM,0BAA0B,uBAC9B,wBACA,cAAc,qBAAqB,cACpC;AAED,aACE,GAAG,KAAK,iBAAiB,gCAAgC,aAAa,SAAS,uBAAuB,SAAU,CAAC,IACjH,EACE,OAAO,QACR,CACF;GAED,MAAM,WAAW,YAAY;AAC3B,QAAI,YAAY,SAMd,QAAO,EACL,MANa,MAAM,SAAS,wBAAwB;KACpD,aAAa,KAAK,UAAU,wBAAwB;KACpD;KACD,CAAC,EAID;AAGH,WAAO,MAAM,YAAY,GAAG,gCAAgC;KAC1D,aAAa,KAAK,UAAU,wBAAwB;KACpD;KACD,CAAC;;AAOJ,eAJuB,SAAS,WAC5B,MAAM,QAAQ,SAAS,SAAS,GAChC,MAAM,UAAU,EAEM,MAAM;;EAGlC,MAAM,2BAA2B,MAAM,QAAQ,IAC7C,KAAK,cAAc,IAAI,OAAO,iBAAiB;;;;;;;;;;GAW7C,IAAI,sBAAsB,gBAAgB,uBAAuB;GAEjE,IAAI;AAEJ,OAAI,OAAO,uBAAuB,WAAW,UAAU;IAKrD,MAAM,uBACJ,uBAAuB,UAAU,QAC/B,IAAI,OAAO,IAAI,KAAK,aAAa,IAAI,IAAI,EACzC,IAAI,aAAa,GAClB;IAGH,MAAM,2BAA2B,uBAC7B,2BAA2B,KAAK,gBAAgB,MAC7C,SACC,KAAK,aAAa,wBAClB,KAAK,WAAW,aACnB,GACD;AAEJ,6BAAyB,4BAA4B;KACnD,KAAK,uBAAuB;KAC5B,SAAS,EAAE;KACX,UAAU;KACV,QAAQ;KACT;AAGD,QAAI,SAAS,WACX,uBAAsB,iCACpB,qBACA,yBACD;UAEE;AAEL,QAAI,SAAS,WAEX,uBAAsB,uCACpB,qBACA,aACD;AAGH,0BAAsB,uBACpB,qBACA,KAAK,aACN;AAED,6BAAyB,uBACvB,wBACA,aACD;;GAGH,MAAM,eAAe,MACnB;IACE,SAAS,KAAK,WAAW,UAAU;IACnC,aAAa,aAAa;IAC1B,SAAS,KAAK,WAAW,UAAU;IACpC,CAAC,KAAK,GAAG,EACV,EAAE,SAAS,IAAI,CAChB;GAED,MAAM,qBACJ,YACA,gBACG;AACH,QAAI,eAAe,EAAG,QAAO;AAC7B,WAAO,MACL;KACE,SAAS,KAAK,WAAW,UAAU;KACnC,eAAe,aAAa,EAAE;KAC9B,SAAS,IAAI,eAAe,WAAW,UAAU;KACjD,SAAS,KAAK,WAAW,UAAU;KACpC,CAAC,KAAK,GAAG,EACV,EAAE,SAAS,GAAG,CACf;;AAGH,aACE,GAAG,KAAK,mBAAmB,aAAa,aAAa,aAAa,SAAS,uBAAuB,SAAU,CAAC,IAC7G,EACE,OAAO,QACR,CACF;GAED,MAAM,sBACH,OAAO,oBAAoB,YAAY,YACtC,oBAAoB,YAAY,QAClC,MAAM,QAAQ,oBAAoB,QAAQ;GAW5C,MAAM,EAAE,SAAS,kBAAkB,OAAO,uBACxC,gBAV0B,sBACxB,oBAAoB,UACpB,EACE,qCACE,oBAAoB,SACvB,CAKiC;GAEtC,MAAM,qBAAkC,UACtC,kBACA,WACD;GAED,MAAM,aAAa,mBAAmB;AAEtC,OAAI,aAAa,EACf,WACE,GAAG,KAAK,mBAAmB,aAAa,cAAc,eAAe,WAAW,CAAC,0BACjF,EACE,OAAO,QACR,CACF;GAGH,MAAM,cAA2B,EAAE;GAGnC,MAAM,gBAAgB,mBAAmB,KAAK,UAAU;IACtD,MAAM,cAAc,kBAAkB,MAAM,OAAO,MAAM,MAAM;AAE/D,QAAI,aAAa,EACf,WACE,GAAG,KAAK,mBAAmB,eAAe,YAAY,qBACtD,EACE,OAAO,QACR,CACF;IAIH,MAAM,eAAe,2BAA2B,MAAM;IACtD,MAAM,sBAAsB,mBAC1B,sBACI,uBAAuB,UACvB,EACE,qCACE,uBAAuB,SAC1B,EACL,aACD;IAED,MAAM,qBAAqB,YAAY;AACrC,YAAO,MAAM,aACX,YAAY;MACV,IAAI;AAEJ,UAAI,YAAY,SACd,qBAAoB,MAAM,SAAS,cAAc;OAC/C,kBAAkB;OAClB;OACA,uBACE,oBAAoB,eACpB,UAAU,eACV;OACF,aAAa,KAAK;OAClB,cAAc;OACd;OACA;OACD,CAAC;UAEF,qBAAoB,MAAM,YAAY,GACnC,cAAc;OACb,kBAAkB;OAClB;OACA,uBACE,oBAAoB,eACpB,UAAU,eACV;OACF,aAAa,KAAK;OAClB,cAAc;OACd;OACA;OACD,CAAC,CACD,MAAM,WAAW,OAAO,KAAK;AAGlC,UAAI,CAAC,mBAAmB,YACtB,OAAM,IAAI,MAAM,oBAAoB;MAGtC,MAAM,EAAE,cAAc,0BACpB,kBAAkB,aAClB,aACD;AAED,UAAI,CAAC,UACH,OAAM,IAAI,MACR,oDACD;AAGH,qBAAe;AACf,aAAO,kBAAkB;QAE3B;MACE,UAAU;MACV,OAAO;MACP,UAAU,EAAE,OAAO,SAAS,eAAe;OACzC,MAAM,cAAc,kBAClB,MAAM,OACN,MAAM,MACP;AACD,iBACE,GAAG,KAAK,mBAAmB,eAAe,YAAY,GAAG,SAAS,kBAAkB,WAAW,IAAI,CAAC,GAAG,SAAS,OAAO,UAAU,WAAW,QAAQ,KAAK,UAAU,MAAM,EAAE,WAAW,UAAU,CAAC,aAAa,eAAe,UAAU,EAAE,CAAC,MAAM,eAAe,SAAS,IACxQ,EACE,OAAO,SACR,CACF;AAED,0BAAmB;AAEnB,WAAI,mBAAmB,sBAAsB;AAC3C,kBAAU,6BAA6B,EACrC,OAAO,SACR,CAAC;AACF,gBAAQ,KAAK,EAAE;;;MAGpB,CACF,EAAE;;AAOL,YAJgB,SAAS,WACrB,QAAQ,SAAS,mBAAmB,GACpC,oBAAoB,EAET,MAAM,YAAY;KAAE;KAAO;KAAQ,EAAE;KACpD;AAMF,IAHqB,MAAM,QAAQ,IAAI,cAAc,EAIlD,MAAM,QAAQ,WAAW,OAAO,MAAM,QAAQ,OAAO,MAAM,MAAM,CACjE,SAAS,EAAE,aAAa;AACvB,gBAAY,KAAK,OAAO;KACxB;GAGJ,IAAI,gBAAgB,YAAY,YAAY;AAG5C,OAAI,mBACF,iBAAgB,iBAAiB,eAAe,mBAAmB;GASrE,IAAI,eANW;IACb,GAAG;IACH,SAAS;IACV,CAGyB;AAE1B,OAAI,CAAC,oBACH,gBAAgB,cACZ;AAGN,OAAI,OAAO,uBAAuB,WAAW,SAE3C,gBAAe,iBACb,uBAAuB,WAAW,EAAE,EACpC,aACD;AAGH,UAAO,CAAC,cAAc,aAAa;IACnC,CACH;EAED,MAAM,oBACJ,OAAO,YAAY,yBAAyB;EAU9C,IAAI,mBAA+B;GACjC,GAAG,0BATkB,uBAAuB,SAC1C;IACE,GAAG;IACH,KAAK,uBAAuB;IAC5B,SAAS,EAAE;IACZ,GACD,uBAG0C;GAC5C,QAAQ;GACR,GAAG;GACJ;AAED,OAAK,MAAM,gBAAgB,KAAK,cAC9B,KAAI,kBAAkB,cACpB,oBAAmB,0BACjB,kBACA,kBAAkB,eAClB,aACD;AAIL,YACE,GAAG,KAAK,iBAAiB,GAAG,SAAS,sCAAsC,WAAW,MAAM,CAAC,OAAO,aAAa,SAAS,iBAAiB,SAAU,CAAC,IACtJ,EACE,OAAO,QACR,CACF;EAKD,MAAM,wBACJ,uBAAuB,QAAQ,cAAc,YAAY;AAE3D,MACE,uBAAuB,WACtB,0BAA0B,QACzB,0BAA0B,WAC5B,uBAAuB,aAAa,SACpC;GACA,MAAM,qBAAqB,uBACxB,SAAU,MAAM,IAAI,CACpB,MAAM,GAAG,GAAG;GAEf,MAAM,eAAe,mBAAmB,mBAAmB,SAAS;AAEpE,UAAO,KAAK,MACV,KAAK,UAAU;IACb,GAAG;IACH,kBAAkB;KAChB,GAAG;KACH,MAAM;KACN,QAAQ;KACT;IACF,CAAC,CAAC,WACD,IAAI,OAAO,MAAM,aAAa,kBAAkB,IAAI,EACpD,WAAW,aAAa,OACzB,CACF;;AAGH,SAAO;GACL,GAAG;GACH;GACD;IAEH;EACE,UAAU;EACV,OAAO;EACP,UAAU,EAAE,OAAO,SAAS,eAC1B,UACE,GAAG,KAAK,iBAAiB,GAAG,SAAS,UAAU,WAAW,IAAI,CAAC,GAAG,SAAS,OAAO,UAAU,WAAW,QAAQ,KAAK,UAAU,MAAM,EAAE,WAAW,UAAU,CAAC,aAAa,eAAe,UAAU,EAAE,CAAC,MAAM,eAAe,SAAS,IACnO,EACE,OAAO,SACR,CACF;EACH,kBAAkB,EAAE,YAClB,UACE,GAAG,KAAK,iBAAiB,GAAG,SAAS,sCAAsC,WAAW,IAAI,CAAC,GAAG,SAAS,OAAO,UAAU,WAAW,QAAQ,KAAK,UAAU,MAAM,EAAE,WAAW,UAAU,IACvL,EACE,OAAO,SACR,CACF;EACJ,CACF,EAAE"}
|
|
1
|
+
{"version":3,"file":"translateDictionary.mjs","names":[],"sources":["../../../src/fill/translateDictionary.ts"],"sourcesContent":["import { basename } from 'node:path';\nimport type { AIConfig } from '@intlayer/ai';\nimport { type AIOptions, getIntlayerAPIProxy } from '@intlayer/api';\nimport {\n chunkJSON,\n excludeObjectFormat,\n formatLocale,\n type JsonChunk,\n mergeChunks,\n reconstructFromSingleChunk,\n verifyIdenticObjectFormat,\n} from '@intlayer/chokidar/utils';\nimport * as ANSIColors from '@intlayer/config/colors';\nimport {\n colon,\n colorize,\n colorizeNumber,\n colorizePath,\n getAppLogger,\n} from '@intlayer/config/logger';\nimport { retryManager } from '@intlayer/config/utils';\nimport {\n getFilterMissingTranslationsDictionary,\n getMultilingualDictionary,\n getPerLocaleDictionary,\n insertContentInDictionary,\n} from '@intlayer/core/plugins';\nimport type { Locale } from '@intlayer/types/allLocales';\nimport type { IntlayerConfig } from '@intlayer/types/config';\nimport type { Dictionary } from '@intlayer/types/dictionary';\nimport { getUnmergedDictionaries } from '@intlayer/unmerged-dictionaries-entry';\nimport type { AIClient } from '../utils/setupAI';\nimport { deepMergeContent } from './deepMergeContent';\nimport {\n extractTranslatableContent,\n reinsertTranslatedContent,\n} from './extractTranslatableContent';\nimport type { TranslationTask } from './listTranslationsTasks';\n\ntype TranslateDictionaryResult = TranslationTask & {\n dictionaryOutput: Dictionary | null;\n};\n\ntype TranslateDictionaryOptions = {\n mode: 'complete' | 'review';\n aiOptions?: AIOptions;\n fillMetadata?: boolean;\n onHandle?: ReturnType<\n typeof import('@intlayer/chokidar/utils').getGlobalLimiter\n >;\n onSuccess?: () => void;\n onError?: (error: unknown) => void;\n getAbortError?: () => Error | null;\n aiClient?: AIClient;\n aiConfig?: AIConfig;\n};\n\nconst createChunkPreset = (chunkIndex: number, totalChunks: number) => {\n if (totalChunks <= 1) return '';\n return colon(\n [\n colorize('[', ANSIColors.GREY_DARK),\n colorizeNumber(chunkIndex + 1),\n colorize(`/${totalChunks}`, ANSIColors.GREY_DARK),\n colorize(']', ANSIColors.GREY_DARK),\n ].join(''),\n { colSize: 5 }\n );\n};\n\nconst hasMissingMetadata = (dictionary: Dictionary) =>\n !dictionary.description || !dictionary.title || !dictionary.tags;\n\nconst serializeError = (error: unknown): string => {\n if (error instanceof Error) {\n return error.cause\n ? `${error.message} (cause: ${String(error.cause)})`\n : error.message;\n }\n if (typeof error === 'string') return error;\n try {\n return JSON.stringify(error);\n } catch {\n return String(error);\n }\n};\n\nconst CHUNK_SIZE = 1500; // Smaller chunks for better accuracy and structural integrity\nconst GROUP_MAX_RETRY = 2;\nconst MAX_RETRY = 3;\nconst RETRY_DELAY = 1000 * 10; // 10 seconds\n\nconst MAX_FOLLOWING_ERRORS = 10; // 10 errors in a row, hard exit the process\nlet followingErrors = 0;\n\nexport const translateDictionary = async (\n task: TranslationTask,\n configuration: IntlayerConfig,\n options?: TranslateDictionaryOptions\n): Promise<TranslateDictionaryResult> => {\n const appLogger = getAppLogger(configuration);\n const intlayerAPI = getIntlayerAPIProxy(undefined, configuration);\n\n const { mode, aiOptions, fillMetadata, aiClient, aiConfig } = {\n mode: 'complete',\n fillMetadata: true,\n ...options,\n } as const;\n\n const notifySuccess = () => {\n followingErrors = 0;\n options?.onSuccess?.();\n };\n\n const result = await retryManager(\n async () => {\n const unmergedDictionariesRecord = getUnmergedDictionaries(configuration);\n\n const baseUnmergedDictionary: Dictionary | undefined =\n unmergedDictionariesRecord[task.dictionaryKey].find(\n (dict) => dict.localId === task.dictionaryLocalId\n );\n\n if (!baseUnmergedDictionary) {\n appLogger(\n `${task.dictionaryPreset}Dictionary not found in unmergedDictionariesRecord. Skipping.`,\n {\n level: 'warn',\n }\n );\n return { ...task, dictionaryOutput: null };\n }\n\n let metadata:\n | Pick<Dictionary, 'description' | 'title' | 'tags'>\n | undefined;\n\n if (\n fillMetadata &&\n (hasMissingMetadata(baseUnmergedDictionary) || mode === 'review')\n ) {\n const defaultLocaleDictionary = getPerLocaleDictionary(\n baseUnmergedDictionary,\n configuration.internationalization.defaultLocale\n );\n\n appLogger(\n `${task.dictionaryPreset} Filling missing metadata for ${colorizePath(basename(baseUnmergedDictionary.filePath!))}`,\n {\n level: 'info',\n }\n );\n\n const runAudit = async () => {\n if (aiClient && aiConfig) {\n const result = await aiClient.auditDictionaryMetadata({\n fileContent: JSON.stringify(defaultLocaleDictionary),\n aiConfig,\n });\n\n return {\n data: result,\n };\n }\n\n return await intlayerAPI.ai.auditContentDeclarationMetadata({\n fileContent: JSON.stringify(defaultLocaleDictionary),\n aiOptions,\n });\n };\n\n const metadataResult = options?.onHandle\n ? await options.onHandle(runAudit)\n : await runAudit();\n\n metadata = metadataResult.data?.fileContent;\n }\n\n const translatedContentResults = await Promise.all(\n task.targetLocales.map(async (targetLocale) => {\n /**\n * In complete mode, for large dictionaries, we want to filter all content that is already translated\n *\n * targetLocale: fr\n *\n * { test1: t({ ar: 'Hello', en: 'Hello', fr: 'Bonjour' } }) -> {}\n * { test2: t({ ar: 'Hello', en: 'Hello' }) } -> { test2: t({ ar: 'Hello', en: 'Hello' }) }\n *\n */\n // Reset to base dictionary for each locale to ensure we filter from the original\n let dictionaryToProcess = structuredClone(baseUnmergedDictionary);\n\n let targetLocaleDictionary: Dictionary;\n\n if (typeof baseUnmergedDictionary.locale === 'string') {\n // For per-locale files, the content is already in simple JSON format (not translation nodes)\n // The base dictionary is already the source locale content\n\n // Load the existing target locale dictionary\n const targetLocaleFilePath =\n baseUnmergedDictionary.filePath?.replace(\n new RegExp(`/${task.sourceLocale}/`, 'g'),\n `/${targetLocale}/`\n );\n\n // Find the target locale dictionary in unmerged dictionaries\n const targetUnmergedDictionary = targetLocaleFilePath\n ? unmergedDictionariesRecord[task.dictionaryKey]?.find(\n (dict) =>\n dict.filePath === targetLocaleFilePath &&\n dict.locale === targetLocale\n )\n : undefined;\n\n targetLocaleDictionary = targetUnmergedDictionary ?? {\n key: baseUnmergedDictionary.key,\n content: {},\n filePath: targetLocaleFilePath,\n locale: targetLocale,\n };\n } else {\n // For multilingual dictionaries\n if (mode === 'complete') {\n // Remove all nodes that don't have any content to translate\n dictionaryToProcess = getFilterMissingTranslationsDictionary(\n dictionaryToProcess,\n targetLocale\n );\n }\n\n dictionaryToProcess = getPerLocaleDictionary(\n dictionaryToProcess,\n task.sourceLocale\n );\n\n targetLocaleDictionary = getPerLocaleDictionary(\n baseUnmergedDictionary,\n targetLocale\n );\n }\n\n // Filter to only untranslated fields, preserving explicit null values as\n // default-locale fallback markers. Applied after both paths converge so\n // the same logic covers per-locale and multilingual dictionaries.\n if (mode === 'complete') {\n dictionaryToProcess = {\n ...dictionaryToProcess,\n content:\n excludeObjectFormat(\n dictionaryToProcess.content,\n targetLocaleDictionary.content\n ) ?? {},\n };\n }\n\n const localePreset = colon(\n [\n colorize('[', ANSIColors.GREY_DARK),\n formatLocale(targetLocale),\n colorize(']', ANSIColors.GREY_DARK),\n ].join(''),\n { colSize: 18 }\n );\n\n appLogger(\n `${task.dictionaryPreset}${localePreset} Preparing ${colorizePath(basename(targetLocaleDictionary.filePath!))}`,\n {\n level: 'info',\n }\n );\n\n const chunkedJsonContent: JsonChunk[] = chunkJSON(\n dictionaryToProcess.content as unknown as Record<string, any>,\n CHUNK_SIZE\n );\n\n const nbOfChunks = chunkedJsonContent.length;\n\n if (nbOfChunks > 1) {\n appLogger(\n `${task.dictionaryPreset}${localePreset} Split into ${colorizeNumber(nbOfChunks)} chunks for translation`,\n {\n level: 'info',\n }\n );\n }\n\n const chunkResult: JsonChunk[] = [];\n\n // Process chunks in parallel (globally throttled) to allow concurrent translation\n const chunkPromises = chunkedJsonContent.map(async (chunk) => {\n const chunkPreset = createChunkPreset(chunk.index, chunk.total);\n\n if (nbOfChunks > 1) {\n appLogger(\n `${task.dictionaryPreset}${localePreset}${chunkPreset} Translating chunk`,\n {\n level: 'info',\n }\n );\n }\n\n const reconstructedChunk = reconstructFromSingleChunk(chunk);\n const {\n extractedContent: chunkExtractedContent,\n translatableDictionary: chunkTranslatableDictionary,\n } = extractTranslatableContent(reconstructedChunk);\n\n const executeTranslation = async () => {\n return await retryManager(\n async () => {\n let translationResult: any;\n\n if (aiClient && aiConfig) {\n translationResult = await aiClient.translateJSON({\n entryFileContent:\n chunkTranslatableDictionary as unknown as JSON,\n presetOutputContent: chunkTranslatableDictionary,\n dictionaryDescription:\n dictionaryToProcess.description ??\n metadata?.description ??\n '',\n entryLocale: task.sourceLocale,\n outputLocale: targetLocale,\n mode,\n aiConfig,\n });\n } else {\n translationResult = await intlayerAPI.ai\n .translateJSON({\n entryFileContent:\n chunkTranslatableDictionary as unknown as JSON,\n presetOutputContent: chunkTranslatableDictionary,\n dictionaryDescription:\n dictionaryToProcess.description ??\n metadata?.description ??\n '',\n entryLocale: task.sourceLocale,\n outputLocale: targetLocale,\n mode,\n aiOptions,\n })\n .then((result) => result.data);\n }\n\n if (!translationResult?.fileContent) {\n throw new Error('No content result');\n }\n\n const { isIdentic, error } = verifyIdenticObjectFormat(\n translationResult.fileContent,\n chunkTranslatableDictionary\n );\n\n if (!isIdentic) {\n throw new Error(\n `Translation result does not match expected format: ${error}`\n );\n }\n\n notifySuccess();\n return reinsertTranslatedContent(\n reconstructedChunk,\n chunkExtractedContent,\n translationResult.fileContent as Record<number, string>\n );\n },\n {\n maxRetry: MAX_RETRY,\n delay: RETRY_DELAY,\n onError: ({ error, attempt, maxRetry }) => {\n const chunkPreset = createChunkPreset(\n chunk.index,\n chunk.total\n );\n appLogger(\n `${task.dictionaryPreset}${localePreset}${chunkPreset} ${colorize('Error filling:', ANSIColors.RED)} ${colorize(serializeError(error), ANSIColors.GREY_DARK)} - Attempt ${colorizeNumber(attempt + 1)} of ${colorizeNumber(maxRetry)}`,\n {\n level: 'error',\n }\n );\n\n followingErrors += 1;\n\n if (followingErrors >= MAX_FOLLOWING_ERRORS) {\n appLogger(`There is something wrong.`, {\n level: 'error',\n });\n process.exit(1); // 1 for error\n }\n },\n }\n )();\n };\n\n const wrapped = options?.onHandle\n ? options.onHandle(executeTranslation) // queued in global limiter\n : executeTranslation(); // no global limiter\n\n return wrapped.then((result) => ({ chunk, result }));\n });\n\n // Wait for all chunks for this locale in parallel (still capped by global limiter)\n const chunkResults = await Promise.all(chunkPromises);\n\n // Maintain order\n chunkResults\n .sort((chunkA, chunkB) => chunkA.chunk.index - chunkB.chunk.index)\n .forEach(({ result }) => {\n chunkResult.push(result);\n });\n\n // Merge translated chunk contents back into a single content object\n const reinsertedContent = mergeChunks(chunkResult);\n\n const merged = {\n ...dictionaryToProcess,\n content: reinsertedContent,\n };\n\n // Merge newly translated content (including explicit null fallbacks) back\n // into the existing target locale content. Applies to both per-locale and\n // multilingual paths so the target always retains previously translated\n // fields and receives null markers where the source has no translation.\n const finalContent = deepMergeContent(\n targetLocaleDictionary.content ?? {},\n merged.content\n );\n\n return [targetLocale, finalContent] as const;\n })\n );\n\n const translatedContent: Partial<Record<Locale, Dictionary['content']>> =\n Object.fromEntries(translatedContentResults);\n\n const baseDictionary = baseUnmergedDictionary.locale\n ? {\n ...baseUnmergedDictionary,\n key: baseUnmergedDictionary.key!,\n content: {},\n }\n : baseUnmergedDictionary;\n\n let dictionaryOutput: Dictionary = {\n ...getMultilingualDictionary(baseDictionary),\n locale: undefined, // Ensure the dictionary is multilingual\n ...metadata,\n };\n\n for (const targetLocale of task.targetLocales) {\n if (translatedContent[targetLocale]) {\n dictionaryOutput = insertContentInDictionary(\n dictionaryOutput,\n translatedContent[targetLocale],\n targetLocale\n );\n }\n }\n\n appLogger(\n `${task.dictionaryPreset} ${colorize('Translation completed successfully', ANSIColors.GREEN)} for ${colorizePath(basename(dictionaryOutput.filePath!))}`,\n {\n level: 'info',\n }\n );\n\n // The dict-level `fill` value may have been lost during JSON serialisation\n // (functions can't be serialised to JSON). Fall back to the config-level\n // fill so that an explicit fill in intlayer.config.ts is honoured.\n const effectiveFillForCheck =\n baseUnmergedDictionary.fill ?? configuration.dictionary?.fill;\n\n if (\n baseUnmergedDictionary.locale &&\n (effectiveFillForCheck === true ||\n effectiveFillForCheck === undefined) &&\n baseUnmergedDictionary.location === 'local'\n ) {\n const dictionaryFilePath = baseUnmergedDictionary\n .filePath!.split('.')\n .slice(0, -1);\n\n const contentIndex = dictionaryFilePath[dictionaryFilePath.length - 1];\n\n return JSON.parse(\n JSON.stringify({\n ...task,\n dictionaryOutput: {\n ...dictionaryOutput,\n fill: undefined,\n filled: true,\n },\n }).replaceAll(\n new RegExp(`\\\\.${contentIndex}\\\\.[a-zA-Z0-9]+`, 'g'),\n `.filled.${contentIndex}.json`\n )\n );\n }\n\n return {\n ...task,\n dictionaryOutput,\n };\n },\n {\n maxRetry: GROUP_MAX_RETRY,\n delay: RETRY_DELAY,\n onError: ({ error, attempt, maxRetry }) =>\n appLogger(\n `${task.dictionaryPreset} ${colorize('Error:', ANSIColors.RED)} ${colorize(serializeError(error), ANSIColors.GREY_DARK)} - Attempt ${colorizeNumber(attempt + 1)} of ${colorizeNumber(maxRetry)}`,\n {\n level: 'error',\n }\n ),\n onMaxTryReached: ({ error }) =>\n appLogger(\n `${task.dictionaryPreset} ${colorize('Maximum number of retries reached:', ANSIColors.RED)} ${colorize(serializeError(error), ANSIColors.GREY_DARK)}`,\n {\n level: 'error',\n }\n ),\n }\n )();\n\n return result as TranslateDictionaryResult;\n};\n"],"mappings":";;;;;;;;;;;;AAyDA,MAAM,qBAAqB,YAAoB,gBAAwB;AACrE,KAAI,eAAe,EAAG,QAAO;AAC7B,QAAO,MACL;EACE,SAAS,KAAK,WAAW,UAAU;EACnC,eAAe,aAAa,EAAE;EAC9B,SAAS,IAAI,eAAe,WAAW,UAAU;EACjD,SAAS,KAAK,WAAW,UAAU;EACpC,CAAC,KAAK,GAAG,EACV,EAAE,SAAS,GAAG,CACf;;AAGH,MAAM,sBAAsB,eAC1B,CAAC,WAAW,eAAe,CAAC,WAAW,SAAS,CAAC,WAAW;AAE9D,MAAM,kBAAkB,UAA2B;AACjD,KAAI,iBAAiB,MACnB,QAAO,MAAM,QACT,GAAG,MAAM,QAAQ,WAAW,OAAO,MAAM,MAAM,CAAC,KAChD,MAAM;AAEZ,KAAI,OAAO,UAAU,SAAU,QAAO;AACtC,KAAI;AACF,SAAO,KAAK,UAAU,MAAM;SACtB;AACN,SAAO,OAAO,MAAM;;;AAIxB,MAAM,aAAa;AACnB,MAAM,kBAAkB;AACxB,MAAM,YAAY;AAClB,MAAM,cAAc,MAAO;AAE3B,MAAM,uBAAuB;AAC7B,IAAI,kBAAkB;AAEtB,MAAa,sBAAsB,OACjC,MACA,eACA,YACuC;CACvC,MAAM,YAAY,aAAa,cAAc;CAC7C,MAAM,cAAc,oBAAoB,QAAW,cAAc;CAEjE,MAAM,EAAE,MAAM,WAAW,cAAc,UAAU,aAAa;EAC5D,MAAM;EACN,cAAc;EACd,GAAG;EACJ;CAED,MAAM,sBAAsB;AAC1B,oBAAkB;AAClB,WAAS,aAAa;;AA8ZxB,QA3Ze,MAAM,aACnB,YAAY;EACV,MAAM,6BAA6B,wBAAwB,cAAc;EAEzE,MAAM,yBACJ,2BAA2B,KAAK,eAAe,MAC5C,SAAS,KAAK,YAAY,KAAK,kBACjC;AAEH,MAAI,CAAC,wBAAwB;AAC3B,aACE,GAAG,KAAK,iBAAiB,gEACzB,EACE,OAAO,QACR,CACF;AACD,UAAO;IAAE,GAAG;IAAM,kBAAkB;IAAM;;EAG5C,IAAI;AAIJ,MACE,iBACC,mBAAmB,uBAAuB,IAAI,SAAS,WACxD;GACA,MAAM,0BAA0B,uBAC9B,wBACA,cAAc,qBAAqB,cACpC;AAED,aACE,GAAG,KAAK,iBAAiB,gCAAgC,aAAa,SAAS,uBAAuB,SAAU,CAAC,IACjH,EACE,OAAO,QACR,CACF;GAED,MAAM,WAAW,YAAY;AAC3B,QAAI,YAAY,SAMd,QAAO,EACL,MANa,MAAM,SAAS,wBAAwB;KACpD,aAAa,KAAK,UAAU,wBAAwB;KACpD;KACD,CAAC,EAID;AAGH,WAAO,MAAM,YAAY,GAAG,gCAAgC;KAC1D,aAAa,KAAK,UAAU,wBAAwB;KACpD;KACD,CAAC;;AAOJ,eAJuB,SAAS,WAC5B,MAAM,QAAQ,SAAS,SAAS,GAChC,MAAM,UAAU,EAEM,MAAM;;EAGlC,MAAM,2BAA2B,MAAM,QAAQ,IAC7C,KAAK,cAAc,IAAI,OAAO,iBAAiB;;;;;;;;;;GAW7C,IAAI,sBAAsB,gBAAgB,uBAAuB;GAEjE,IAAI;AAEJ,OAAI,OAAO,uBAAuB,WAAW,UAAU;IAKrD,MAAM,uBACJ,uBAAuB,UAAU,QAC/B,IAAI,OAAO,IAAI,KAAK,aAAa,IAAI,IAAI,EACzC,IAAI,aAAa,GAClB;AAWH,8BARiC,uBAC7B,2BAA2B,KAAK,gBAAgB,MAC7C,SACC,KAAK,aAAa,wBAClB,KAAK,WAAW,aACnB,GACD,WAEiD;KACnD,KAAK,uBAAuB;KAC5B,SAAS,EAAE;KACX,UAAU;KACV,QAAQ;KACT;UACI;AAEL,QAAI,SAAS,WAEX,uBAAsB,uCACpB,qBACA,aACD;AAGH,0BAAsB,uBACpB,qBACA,KAAK,aACN;AAED,6BAAyB,uBACvB,wBACA,aACD;;AAMH,OAAI,SAAS,WACX,uBAAsB;IACpB,GAAG;IACH,SACE,oBACE,oBAAoB,SACpB,uBAAuB,QACxB,IAAI,EAAE;IACV;GAGH,MAAM,eAAe,MACnB;IACE,SAAS,KAAK,WAAW,UAAU;IACnC,aAAa,aAAa;IAC1B,SAAS,KAAK,WAAW,UAAU;IACpC,CAAC,KAAK,GAAG,EACV,EAAE,SAAS,IAAI,CAChB;AAED,aACE,GAAG,KAAK,mBAAmB,aAAa,aAAa,aAAa,SAAS,uBAAuB,SAAU,CAAC,IAC7G,EACE,OAAO,QACR,CACF;GAED,MAAM,qBAAkC,UACtC,oBAAoB,SACpB,WACD;GAED,MAAM,aAAa,mBAAmB;AAEtC,OAAI,aAAa,EACf,WACE,GAAG,KAAK,mBAAmB,aAAa,cAAc,eAAe,WAAW,CAAC,0BACjF,EACE,OAAO,QACR,CACF;GAGH,MAAM,cAA2B,EAAE;GAGnC,MAAM,gBAAgB,mBAAmB,IAAI,OAAO,UAAU;IAC5D,MAAM,cAAc,kBAAkB,MAAM,OAAO,MAAM,MAAM;AAE/D,QAAI,aAAa,EACf,WACE,GAAG,KAAK,mBAAmB,eAAe,YAAY,qBACtD,EACE,OAAO,QACR,CACF;IAGH,MAAM,qBAAqB,2BAA2B,MAAM;IAC5D,MAAM,EACJ,kBAAkB,uBAClB,wBAAwB,gCACtB,2BAA2B,mBAAmB;IAElD,MAAM,qBAAqB,YAAY;AACrC,YAAO,MAAM,aACX,YAAY;MACV,IAAI;AAEJ,UAAI,YAAY,SACd,qBAAoB,MAAM,SAAS,cAAc;OAC/C,kBACE;OACF,qBAAqB;OACrB,uBACE,oBAAoB,eACpB,UAAU,eACV;OACF,aAAa,KAAK;OAClB,cAAc;OACd;OACA;OACD,CAAC;UAEF,qBAAoB,MAAM,YAAY,GACnC,cAAc;OACb,kBACE;OACF,qBAAqB;OACrB,uBACE,oBAAoB,eACpB,UAAU,eACV;OACF,aAAa,KAAK;OAClB,cAAc;OACd;OACA;OACD,CAAC,CACD,MAAM,WAAW,OAAO,KAAK;AAGlC,UAAI,CAAC,mBAAmB,YACtB,OAAM,IAAI,MAAM,oBAAoB;MAGtC,MAAM,EAAE,WAAW,UAAU,0BAC3B,kBAAkB,aAClB,4BACD;AAED,UAAI,CAAC,UACH,OAAM,IAAI,MACR,sDAAsD,QACvD;AAGH,qBAAe;AACf,aAAO,0BACL,oBACA,uBACA,kBAAkB,YACnB;QAEH;MACE,UAAU;MACV,OAAO;MACP,UAAU,EAAE,OAAO,SAAS,eAAe;OACzC,MAAM,cAAc,kBAClB,MAAM,OACN,MAAM,MACP;AACD,iBACE,GAAG,KAAK,mBAAmB,eAAe,YAAY,GAAG,SAAS,kBAAkB,WAAW,IAAI,CAAC,GAAG,SAAS,eAAe,MAAM,EAAE,WAAW,UAAU,CAAC,aAAa,eAAe,UAAU,EAAE,CAAC,MAAM,eAAe,SAAS,IACpO,EACE,OAAO,SACR,CACF;AAED,0BAAmB;AAEnB,WAAI,mBAAmB,sBAAsB;AAC3C,kBAAU,6BAA6B,EACrC,OAAO,SACR,CAAC;AACF,gBAAQ,KAAK,EAAE;;;MAGpB,CACF,EAAE;;AAOL,YAJgB,SAAS,WACrB,QAAQ,SAAS,mBAAmB,GACpC,oBAAoB,EAET,MAAM,YAAY;KAAE;KAAO;KAAQ,EAAE;KACpD;AAMF,IAHqB,MAAM,QAAQ,IAAI,cAAc,EAIlD,MAAM,QAAQ,WAAW,OAAO,MAAM,QAAQ,OAAO,MAAM,MAAM,CACjE,SAAS,EAAE,aAAa;AACvB,gBAAY,KAAK,OAAO;KACxB;GAGJ,MAAM,oBAAoB,YAAY,YAAY;GAElD,MAAM,SAAS;IACb,GAAG;IACH,SAAS;IACV;AAWD,UAAO,CAAC,cALa,iBACnB,uBAAuB,WAAW,EAAE,EACpC,OAAO,QACR,CAEkC;IACnC,CACH;EAED,MAAM,oBACJ,OAAO,YAAY,yBAAyB;EAU9C,IAAI,mBAA+B;GACjC,GAAG,0BATkB,uBAAuB,SAC1C;IACE,GAAG;IACH,KAAK,uBAAuB;IAC5B,SAAS,EAAE;IACZ,GACD,uBAG0C;GAC5C,QAAQ;GACR,GAAG;GACJ;AAED,OAAK,MAAM,gBAAgB,KAAK,cAC9B,KAAI,kBAAkB,cACpB,oBAAmB,0BACjB,kBACA,kBAAkB,eAClB,aACD;AAIL,YACE,GAAG,KAAK,iBAAiB,GAAG,SAAS,sCAAsC,WAAW,MAAM,CAAC,OAAO,aAAa,SAAS,iBAAiB,SAAU,CAAC,IACtJ,EACE,OAAO,QACR,CACF;EAKD,MAAM,wBACJ,uBAAuB,QAAQ,cAAc,YAAY;AAE3D,MACE,uBAAuB,WACtB,0BAA0B,QACzB,0BAA0B,WAC5B,uBAAuB,aAAa,SACpC;GACA,MAAM,qBAAqB,uBACxB,SAAU,MAAM,IAAI,CACpB,MAAM,GAAG,GAAG;GAEf,MAAM,eAAe,mBAAmB,mBAAmB,SAAS;AAEpE,UAAO,KAAK,MACV,KAAK,UAAU;IACb,GAAG;IACH,kBAAkB;KAChB,GAAG;KACH,MAAM;KACN,QAAQ;KACT;IACF,CAAC,CAAC,WACD,IAAI,OAAO,MAAM,aAAa,kBAAkB,IAAI,EACpD,WAAW,aAAa,OACzB,CACF;;AAGH,SAAO;GACL,GAAG;GACH;GACD;IAEH;EACE,UAAU;EACV,OAAO;EACP,UAAU,EAAE,OAAO,SAAS,eAC1B,UACE,GAAG,KAAK,iBAAiB,GAAG,SAAS,UAAU,WAAW,IAAI,CAAC,GAAG,SAAS,eAAe,MAAM,EAAE,WAAW,UAAU,CAAC,aAAa,eAAe,UAAU,EAAE,CAAC,MAAM,eAAe,SAAS,IAC/L,EACE,OAAO,SACR,CACF;EACH,kBAAkB,EAAE,YAClB,UACE,GAAG,KAAK,iBAAiB,GAAG,SAAS,sCAAsC,WAAW,IAAI,CAAC,GAAG,SAAS,eAAe,MAAM,EAAE,WAAW,UAAU,IACnJ,EACE,OAAO,SACR,CACF;EACJ,CACF,EAAE"}
|
|
@@ -1,20 +1,22 @@
|
|
|
1
1
|
import { relative } from "node:path";
|
|
2
|
+
import { prepareIntlayer } from "@intlayer/chokidar/cli";
|
|
2
3
|
import { formatPath } from "@intlayer/chokidar/utils";
|
|
3
4
|
import { colon, colorizeKey, colorizeNumber, getAppLogger } from "@intlayer/config/logger";
|
|
4
5
|
import { getConfiguration } from "@intlayer/config/node";
|
|
5
6
|
import { getUnmergedDictionaries } from "@intlayer/unmerged-dictionaries-entry";
|
|
6
7
|
|
|
7
8
|
//#region src/listContentDeclaration.ts
|
|
8
|
-
const listContentDeclarationRows = (options) => {
|
|
9
|
+
const listContentDeclarationRows = async (options) => {
|
|
9
10
|
const config = getConfiguration(options?.configOptions);
|
|
11
|
+
await prepareIntlayer(config);
|
|
10
12
|
const unmergedDictionariesRecord = getUnmergedDictionaries(config);
|
|
11
13
|
return Object.values(unmergedDictionariesRecord).flat().map((dictionary) => ({
|
|
12
14
|
key: dictionary.key ?? "",
|
|
13
15
|
path: options?.absolute ? dictionary.filePath ?? "Remote" : relative(config.system.baseDir, dictionary.filePath ?? "Remote")
|
|
14
16
|
}));
|
|
15
17
|
};
|
|
16
|
-
const listContentDeclaration = (options) => {
|
|
17
|
-
const rows = listContentDeclarationRows(options);
|
|
18
|
+
const listContentDeclaration = async (options) => {
|
|
19
|
+
const rows = await listContentDeclarationRows(options);
|
|
18
20
|
if (options?.json) {
|
|
19
21
|
console.log(JSON.stringify(rows));
|
|
20
22
|
return;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"listContentDeclaration.mjs","names":[],"sources":["../../src/listContentDeclaration.ts"],"sourcesContent":["import { relative } from 'node:path';\nimport { formatPath } from '@intlayer/chokidar/utils';\nimport {\n colon,\n colorizeKey,\n colorizeNumber,\n getAppLogger,\n} from '@intlayer/config/logger';\nimport {\n type GetConfigurationOptions,\n getConfiguration,\n} from '@intlayer/config/node';\nimport { getUnmergedDictionaries } from '@intlayer/unmerged-dictionaries-entry';\n\ntype ListContentDeclarationOptions = {\n configOptions?: GetConfigurationOptions;\n json?: boolean;\n absolute?: boolean;\n};\n\nexport const listContentDeclarationRows = (\n options?: ListContentDeclarationOptions\n) => {\n const config = getConfiguration(options?.configOptions);\n\n const unmergedDictionariesRecord = getUnmergedDictionaries(config);\n\n const rows = Object.values(unmergedDictionariesRecord)\n .flat()\n .map((dictionary) => ({\n key: dictionary.key ?? '',\n path: options?.absolute\n ? (dictionary.filePath ?? 'Remote')\n : relative(config.system.baseDir, dictionary.filePath ?? 'Remote'),\n }));\n return rows;\n};\n\nexport const listContentDeclaration = (\n options?: ListContentDeclarationOptions\n) => {\n const rows = listContentDeclarationRows(options);\n\n if (options?.json) {\n console.log(JSON.stringify(rows));\n return;\n }\n\n const config = getConfiguration(options?.configOptions);\n const appLogger = getAppLogger(config);\n\n const lines = rows.map((row) =>\n [\n colon(` - ${colorizeKey(row.key)}`, {\n colSize: rows.map((row) => row.key.length),\n maxSize: 60,\n }),\n ' - ',\n formatPath(row.path),\n ].join('')\n );\n\n appLogger(`Content declaration files:`);\n\n lines.forEach((line) => {\n appLogger(line, {\n level: 'info',\n });\n });\n\n appLogger(`Total content declaration files: ${colorizeNumber(rows.length)}`);\n};\n"],"mappings":"
|
|
1
|
+
{"version":3,"file":"listContentDeclaration.mjs","names":[],"sources":["../../src/listContentDeclaration.ts"],"sourcesContent":["import { relative } from 'node:path';\nimport { prepareIntlayer } from '@intlayer/chokidar/cli';\nimport { formatPath } from '@intlayer/chokidar/utils';\nimport {\n colon,\n colorizeKey,\n colorizeNumber,\n getAppLogger,\n} from '@intlayer/config/logger';\nimport {\n type GetConfigurationOptions,\n getConfiguration,\n} from '@intlayer/config/node';\nimport { getUnmergedDictionaries } from '@intlayer/unmerged-dictionaries-entry';\n\ntype ListContentDeclarationOptions = {\n configOptions?: GetConfigurationOptions;\n json?: boolean;\n absolute?: boolean;\n};\n\nexport const listContentDeclarationRows = async (\n options?: ListContentDeclarationOptions\n) => {\n const config = getConfiguration(options?.configOptions);\n\n await prepareIntlayer(config);\n\n const unmergedDictionariesRecord = getUnmergedDictionaries(config);\n\n const rows = Object.values(unmergedDictionariesRecord)\n .flat()\n .map((dictionary) => ({\n key: dictionary.key ?? '',\n path: options?.absolute\n ? (dictionary.filePath ?? 'Remote')\n : relative(config.system.baseDir, dictionary.filePath ?? 'Remote'),\n }));\n return rows;\n};\n\nexport const listContentDeclaration = async (\n options?: ListContentDeclarationOptions\n) => {\n const rows = await listContentDeclarationRows(options);\n\n if (options?.json) {\n console.log(JSON.stringify(rows));\n return;\n }\n\n const config = getConfiguration(options?.configOptions);\n const appLogger = getAppLogger(config);\n\n const lines = rows.map((row) =>\n [\n colon(` - ${colorizeKey(row.key)}`, {\n colSize: rows.map((row) => row.key.length),\n maxSize: 60,\n }),\n ' - ',\n formatPath(row.path),\n ].join('')\n );\n\n appLogger(`Content declaration files:`);\n\n lines.forEach((line) => {\n appLogger(line, {\n level: 'info',\n });\n });\n\n appLogger(`Total content declaration files: ${colorizeNumber(rows.length)}`);\n};\n"],"mappings":";;;;;;;;AAqBA,MAAa,6BAA6B,OACxC,YACG;CACH,MAAM,SAAS,iBAAiB,SAAS,cAAc;AAEvD,OAAM,gBAAgB,OAAO;CAE7B,MAAM,6BAA6B,wBAAwB,OAAO;AAUlE,QARa,OAAO,OAAO,2BAA2B,CACnD,MAAM,CACN,KAAK,gBAAgB;EACpB,KAAK,WAAW,OAAO;EACvB,MAAM,SAAS,WACV,WAAW,YAAY,WACxB,SAAS,OAAO,OAAO,SAAS,WAAW,YAAY,SAAS;EACrE,EAAE;;AAIP,MAAa,yBAAyB,OACpC,YACG;CACH,MAAM,OAAO,MAAM,2BAA2B,QAAQ;AAEtD,KAAI,SAAS,MAAM;AACjB,UAAQ,IAAI,KAAK,UAAU,KAAK,CAAC;AACjC;;CAIF,MAAM,YAAY,aADH,iBAAiB,SAAS,cAAc,CACjB;CAEtC,MAAM,QAAQ,KAAK,KAAK,QACtB;EACE,MAAM,MAAM,YAAY,IAAI,IAAI,IAAI;GAClC,SAAS,KAAK,KAAK,QAAQ,IAAI,IAAI,OAAO;GAC1C,SAAS;GACV,CAAC;EACF;EACA,WAAW,IAAI,KAAK;EACrB,CAAC,KAAK,GAAG,CACX;AAED,WAAU,6BAA6B;AAEvC,OAAM,SAAS,SAAS;AACtB,YAAU,MAAM,EACd,OAAO,QACR,CAAC;GACF;AAEF,WAAU,oCAAoC,eAAe,KAAK,OAAO,GAAG"}
|
|
@@ -7,8 +7,9 @@ import { join, relative } from "node:path";
|
|
|
7
7
|
import { listGitFiles, listGitLines, logConfigDetails } from "@intlayer/chokidar/cli";
|
|
8
8
|
import { formatLocale, formatPath, parallelize } from "@intlayer/chokidar/utils";
|
|
9
9
|
import * as ANSIColors from "@intlayer/config/colors";
|
|
10
|
-
import { colorize, colorizeNumber, getAppLogger } from "@intlayer/config/logger";
|
|
10
|
+
import { colorize, colorizeNumber, getAppLogger, x } from "@intlayer/config/logger";
|
|
11
11
|
import { getConfiguration } from "@intlayer/config/node";
|
|
12
|
+
import { checkAISDKAccess } from "@intlayer/ai";
|
|
12
13
|
import fg from "fast-glob";
|
|
13
14
|
|
|
14
15
|
//#region src/reviewDoc/reviewDoc.ts
|
|
@@ -23,6 +24,11 @@ const reviewDoc = async ({ docPattern, locales, excludedGlobPattern, baseLocale,
|
|
|
23
24
|
const aiResult = await setupAI(configuration, aiOptions);
|
|
24
25
|
if (!aiResult?.hasAIAccess) return;
|
|
25
26
|
const { aiClient, aiConfig } = aiResult;
|
|
27
|
+
const { hasAIAccess, error } = await checkAISDKAccess(aiConfig);
|
|
28
|
+
if (!hasAIAccess) {
|
|
29
|
+
appLogger(`${x} ${error}`);
|
|
30
|
+
return;
|
|
31
|
+
}
|
|
26
32
|
if (nbSimultaneousFileProcessed && nbSimultaneousFileProcessed > 10) {
|
|
27
33
|
appLogger(`Warning: nbSimultaneousFileProcessed is set to ${nbSimultaneousFileProcessed}, which is greater than 10. Setting it to 10.`);
|
|
28
34
|
nbSimultaneousFileProcessed = 10;
|