kythia-core 0.10.1-beta → 0.11.0-beta

package/src/cli/commands/LangCheckCommand.js

@@ -0,0 +1,367 @@
+/**
+ * 🕵️‍♂️ Translation Integrity Linter
+ *
+ * @file src/cli/commands/LangCheckCommand.js
+ * @copyright © 2025 kenndeclouv
+ * @assistant chaa & graa
+ * @version 0.11.0-beta
+ *
+ * @description
+ * Performs a deep AST analysis of the codebase to find `t()` translation function calls.
+ * Verifies that every used key exists in the language files (JSON) and reports usage errors.
+ *
+ * ✨ Core Features:
+ * -  AST Parsing: Uses Babel parser for accurate key detection (handles dynamic patterns).
+ * -  Key Verification: Recursively checks nested JSON structures.
+ * -  Unused Key Detection: Reports keys defined in JSON but never used in code.
+ */
+
+const Command = require('../Command');
+const fs = require('node:fs');
+const path = require('node:path');
+const glob = require('glob');
+const parser = require('@babel/parser');
+const traverse = require('@babel/traverse').default;
+
+function getAllKeys(obj, prefix = '') {
+	Object.keys(obj).forEach((key) => {
+		if (key === '_value' || key === 'text') {
+			if (Object.keys(obj).length === 1) return;
+			if (prefix) allDefinedKeys.add(prefix);
+			return;
+		}
+		const fullKey = prefix ? `${prefix}.${key}` : key;
+		if (typeof obj[key] === 'object' && obj[key] !== null) {
+			if (key !== 'jobs' && key !== 'shop') {
+				getAllKeys(obj[key], fullKey);
+			} else {
+				allDefinedKeys.add(fullKey);
+			}
+		} else {
+			allDefinedKeys.add(fullKey);
+		}
+	});
+}
+
+class LangCheckCommand extends Command {
+	signature = 'lang:check';
+	description =
+		'Lint translation key usage in code and language files (AST-based)';
+
+	async handle() {
+		const PROJECT_ROOT = path.join(__dirname, '..', '..', '..');
+		const SCAN_DIRECTORIES = ['addons', 'src'];
+		const LANG_DIR = path.join(PROJECT_ROOT, 'src', 'lang');
+		const DEFAULT_LANG = 'en';
+		const IGNORE_PATTERNS = [
+			'**/node_modules/**',
+			'**/dist/**',
+			'**/tests/**',
+			'**/assets/**',
+			'**/dashboard/web/public/**',
+			'**/temp/**',
+			'**/leetMap.js',
+			'**/generate_*.js',
+			'**/refactor_*.js',
+			'**/undo_*.js',
+			'**/*.d.ts',
+		];
+
+		const locales = {};
+		const usedStaticKeys = new Set();
+		const usedDynamicKeys = new Set();
+		const unanalyzableKeys = new Set();
+		let filesScanned = 0;
+		let filesWithErrors = 0;
+
+		console.log('--- Kythia AST Translation Linter ---');
+
+		function hasNestedKey(obj, pathExpr) {
+			if (!obj || !pathExpr) return false;
+			const parts = pathExpr.split('.');
+			let current = obj;
+			for (const part of parts) {
+				if (
+					typeof current !== 'object' ||
+					current === null ||
+					!Object.hasOwn(current, part)
+				) {
+					return false;
+				}
+				current = current[part];
+			}
+			return true;
+		}
+
+		function _loadLocales() {
+			console.log(`\n🔍 Reading language files from: ${LANG_DIR}`);
+			try {
+				const langFiles = fs
+					.readdirSync(LANG_DIR)
+					.filter(
+						(file) =>
+							file.endsWith('.json') &&
+							!file.includes('_flat') &&
+							!file.includes('_FLAT'),
+					);
+				if (langFiles.length === 0) {
+					console.error(
+						'\x1b[31m%s\x1b[0m',
+						'❌ No .json files found in the language folder.',
+					);
+					return false;
+				}
+				for (const file of langFiles) {
+					const lang = file.replace('.json', '');
+					const content = fs.readFileSync(path.join(LANG_DIR, file), 'utf8');
+					try {
+						locales[lang] = JSON.parse(content);
+						console.log(`  > Successfully loaded: ${file}`);
+					} catch (jsonError) {
+						console.error(
+							`\x1b[31m%s\x1b[0m`,
+							`❌ Failed to parse JSON: ${file} - ${jsonError.message}`,
+						);
+						filesWithErrors++;
+					}
+				}
+				if (!locales[DEFAULT_LANG]) {
+					console.error(
+						`\x1b[31m%s\x1b[0m`,
+						`❌ Default language (${DEFAULT_LANG}) not found!`,
+					);
+					return false;
+				}
+				return true;
+			} catch (error) {
+				console.error(
+					'\x1b[31m%s\x1b[0m',
+					`❌ Failed to load language files: ${error.message}`,
+				);
+				return false;
+			}
+		}
+
+		if (!_loadLocales()) {
+			console.error('\x1b[31mCannot proceed (language files invalid).\x1b[0m');
+			process.exit(1);
+		}
+
+		console.log(
+			`\nScanning .js/.ts files in: ${SCAN_DIRECTORIES.join(', ')}...`,
+		);
+		SCAN_DIRECTORIES.forEach((dirName) => {
+			const dirPath = path.join(PROJECT_ROOT, dirName);
+			const files = glob.sync(`${dirPath}/**/*.{js,ts}`, {
+				ignore: IGNORE_PATTERNS,
+				dot: true,
+			});
+			files.forEach((filePath) => {
+				filesScanned++;
+				process.stdout.write(`\rScanning: ${filesScanned} files...`);
+
+				try {
+					const code = fs.readFileSync(filePath, 'utf8');
+					const ast = parser.parse(code, {
+						sourceType: 'module',
+						plugins: [
+							'typescript',
+							'jsx',
+							'classProperties',
+							'objectRestSpread',
+						],
+						errorRecovery: true,
+					});
+
+					traverse(ast, {
+						CallExpression(nodePath) {
+							const node = nodePath.node;
+							if (
+								node.callee.type === 'Identifier' &&
+								node.callee.name === 't'
+							) {
+								if (node.arguments.length >= 2) {
+									const keyArg = node.arguments[1];
+									if (keyArg.type === 'StringLiteral') {
+										usedStaticKeys.add(keyArg.value);
+									} else if (keyArg.type === 'TemplateLiteral') {
+										let pattern = '';
+										keyArg.quasis.forEach((quasi, _i) => {
+											pattern += quasi.value.raw;
+											if (!quasi.tail) {
+												pattern += '*';
+											}
+										});
+										pattern = pattern.replace(/_/g, '.');
+										usedDynamicKeys.add(pattern);
+									} else if (
+										keyArg.type === 'BinaryExpression' &&
+										keyArg.operator === '+'
+									) {
+										if (keyArg.left.type === 'StringLiteral') {
+											const pattern = `${keyArg.left.value.replace(/_/g, '.')}*`;
+											usedDynamicKeys.add(pattern);
+										} else {
+											unanalyzableKeys.add(
+												`Complex (+) at ${path.relative(PROJECT_ROOT, filePath)}:${node.loc?.start.line}`,
+											);
+										}
+									} else {
+										unanalyzableKeys.add(
+											`Variable/Other at ${path.relative(PROJECT_ROOT, filePath)}:${node.loc?.start.line}`,
+										);
+									}
+								}
+							}
+						},
+					});
+				} catch (parseError) {
+					if (parseError.message.includes('Unexpected token')) {
+						console.warn(
+							`\n\x1b[33m[WARN] Syntax Error parsing ${path.relative(
+								PROJECT_ROOT,
+								filePath,
+							)}:${parseError.loc?.line} - ${parseError.message}\x1b[0m`,
+						);
+					} else {
+						console.error(
+							`\n\x1b[31m[ERROR] Failed to parse ${path.relative(
+								PROJECT_ROOT,
+								filePath,
+							)}: ${parseError.message}\x1b[0m`,
+						);
+					}
+					filesWithErrors++;
+				}
+			});
+		});
+		process.stdout.write(`${'\r'.padEnd(process.stdout.columns || 60)}\r`);
+
+		console.log(`\nScan completed. Total ${filesScanned} files processed.`);
+		console.log(`  > Found \x1b[33m${usedStaticKeys.size}\x1b[0m static keys.`);
+		console.log(
+			`  > Found \x1b[33m${usedDynamicKeys.size}\x1b[0m dynamic key patterns (check manually!).`,
+		);
+
+		if (unanalyzableKeys.size > 0) {
+			console.log(
+				`  > \x1b[31m${unanalyzableKeys.size}\x1b[0m t() calls could not be analyzed (variable/complex).`,
+			);
+		}
+
+		console.log('\nVerifying static keys against language files...');
+
+		let totalMissingStatic = 0;
+		for (const lang in locales) {
+			const missingInLang = [];
+			for (const staticKey of usedStaticKeys) {
+				if (!hasNestedKey(locales[lang], staticKey)) {
+					missingInLang.push(staticKey);
+				}
+			}
+			if (missingInLang.length > 0) {
+				console.log(
+					`\n❌ \x1b[31m[${lang.toUpperCase()}] Found ${missingInLang.length} missing static keys:\x1b[0m`,
+				);
+				missingInLang.sort().forEach((key) => {
+					console.log(`  - ${key}`);
+				});
+				totalMissingStatic += missingInLang.length;
+				filesWithErrors++;
+			} else {
+				console.log(
+					`\n✅ \x1b[32m[${lang.toUpperCase()}] All static keys found!\x1b[0m`,
+				);
+			}
+		}
+
+		if (usedDynamicKeys.size > 0) {
+			console.log(
+				`\n\n⚠️ \x1b[33mDynamic Key Patterns Detected (Check Manually):\x1b[0m`,
+			);
+			[...usedDynamicKeys].sort().forEach((pattern) => {
+				console.log(`  - ${pattern}`);
+			});
+			console.log(
+				`   (Ensure all possible keys from these patterns exist in the language files)`,
+			);
+		}
+
+		if (unanalyzableKeys.size > 0) {
+			console.log(
+				`\n\n⚠️ \x1b[31mComplex/Unanalyzable t() Calls (Check Manually):\x1b[0m`,
+			);
+			[...unanalyzableKeys].sort().forEach((loc) => {
+				console.log(`  - ${loc}`);
+			});
+		}
+
+		console.log(`\nChecking UNUSED keys (based on ${DEFAULT_LANG}.json)...`);
+
+		const defaultLocale = locales[DEFAULT_LANG];
+		const allDefinedKeys = new Set();
+
+		if (defaultLocale) {
+			try {
+				getAllKeys(defaultLocale);
+			} catch (e) {
+				console.error('Error collecting defined keys:', e);
+			}
+
+			const unusedKeys = [];
+			for (const definedKey of allDefinedKeys) {
+				if (!usedStaticKeys.has(definedKey)) {
+					let matchedByDynamic = false;
+					for (const dynamicPattern of usedDynamicKeys) {
+						const regexPattern = `^${dynamicPattern
+							.replace(/\./g, '\\.')
+							.replace(/\*/g, '[^.]+?')}$`;
+						if (new RegExp(regexPattern).test(definedKey)) {
+							matchedByDynamic = true;
+							break;
+						}
+					}
+					if (!matchedByDynamic) {
+						unusedKeys.push(definedKey);
+					}
+				}
+			}
+			if (unusedKeys.length > 0) {
+				console.log(
+					`\n⚠️ \x1b[33mFound ${unusedKeys.length} UNUSED keys in ${DEFAULT_LANG}.json (don't match static/dynamic patterns):\x1b[0m`,
+				);
+				unusedKeys.sort().forEach((key) => {
+					console.log(`  - ${key}`);
+				});
+			} else {
+				console.log(
+					`\n✅ \x1b[32m[${DEFAULT_LANG.toUpperCase()}] No unused keys found.\x1b[0m`,
+				);
+			}
+		} else {
+			console.warn(
+				`\n\x1b[33m[WARN] Cannot check unused keys because ${DEFAULT_LANG}.json failed to load.\x1b[0m`,
+			);
+		}
+
+		console.log('\n--- Done ---');
+
+		if (filesWithErrors > 0 || totalMissingStatic > 0) {
+			console.log(
+				`\x1b[31mTotal ${totalMissingStatic} missing static key errors + ${filesWithErrors - totalMissingStatic} file errors found. Please fix them.\x1b[0m`,
+			);
+			process.exit(1);
+		} else {
+			console.log(
+				'\x1b[32mCongratulations! Language files (for static keys) are already synced with the code.\x1b[0m',
+			);
+			if (usedDynamicKeys.size > 0 || unanalyzableKeys.size > 0) {
+				console.log(
+					"\x1b[33mHowever, don't forget to manually check the reported dynamic/complex keys above!\x1b[0m",
+				);
+			}
+		}
+	}
+}
+
+module.exports = LangCheckCommand;

package/src/cli/commands/LangTranslateCommand.js

@@ -0,0 +1,336 @@
+/**
+ * 🌐 AI Localization Assistant
+ *
+ * @file src/cli/commands/LangTranslateCommand.js
+ * @copyright © 2025 kenndeclouv
+ * @assistant chaa & graa
+ * @version 0.11.0-beta
+ *
+ * @description
+ * Automates the translation of the core language file (`en.json`) to a target language
+ * using Google's Gemini AI. Handles JSON flattening, batch processing, and placeholder preservation.
+ *
+ * ✨ Core Features:
+ * -  AI-Powered: Uses Gemini 2.5 Flash for natural translations.
+ * -  Smart Batching: Splits large files to avoid token limits.
+ * -  Context Aware: Preserves keys and complex placeholders.
+ */
+
+const Command = require('../Command');
+const pc = require('picocolors');
+
+class LangTranslateCommand extends Command {
+	signature = 'lang:translate <target>';
+	description = 'Translate en.json to target language using Gemini AI';
+
+	async handle(_options, target) {
+		require('@dotenvx/dotenvx/config');
+		const fs = require('node:fs');
+		const path = require('node:path');
+		const { GoogleGenAI } = require('@google/genai');
+
+		const API_KEYS = (process.env.GEMINI_API_KEYS || '')
+			.split(',')
+			.filter(Boolean);
+
+		const targetLang = target || 'ja';
+
+		const TARGET_LANGUAGE =
+			targetLang === 'ja' ? 'Japan (ja)' : `${targetLang}`;
+
+		const rootDir = process.cwd();
+		const INPUT_FILE_SAFE = path.join(
+			rootDir,
+			'addons',
+			'core',
+			'lang',
+			'en.json',
+		);
+		const OUTPUT_FILE_SAFE = path.join(
+			rootDir,
+			'addons',
+			'core',
+			'lang',
+			`${targetLang}.json`,
+		);
+
+		const BATCH_SIZE = 80;
+		const GEMINI_MODEL = 'gemini-2.5-flash';
+		const DELAY_BETWEEN_BATCHES_MS = 5000;
+		const DELAY_ON_ERROR_MS = 5000;
+
+		if (API_KEYS.length === 0) {
+			console.error(pc.red('❌ FATAL: GEMINI_API_KEYS not found in .env!'));
+			process.exit(1);
+		}
+
+		let keyIndex = 0;
+		function getNextGenAI(nextIndex = null) {
+			if (typeof nextIndex === 'number') {
+				keyIndex = nextIndex % API_KEYS.length;
+			}
+			const apiKey = API_KEYS[keyIndex];
+			keyIndex = (keyIndex + 1) % API_KEYS.length;
+			console.log(
+				`[Key Rotator] Using API Key #${keyIndex === 0 ? API_KEYS.length : keyIndex}`,
+			);
+			return new GoogleGenAI({ apiKey });
+		}
+
+		function flattenObject(obj, parentKey = '', result = {}) {
+			for (const key in obj) {
+				const newKey = parentKey ? `${parentKey}.${key}` : key;
+				if (
+					typeof obj[key] === 'object' &&
+					obj[key] !== null &&
+					!Array.isArray(obj[key])
+				) {
+					flattenObject(obj[key], newKey, result);
+				} else {
+					result[newKey] = obj[key];
+				}
+			}
+			return result;
+		}
+
+		function unflattenObject(obj) {
+			const result = {};
+			for (const key in obj) {
+				const keys = key.split('.');
+				keys.reduce((acc, cur, i) => {
+					if (i === keys.length - 1) {
+						acc[cur] = obj[key];
+					} else {
+						acc[cur] = acc[cur] || {};
+					}
+					return acc[cur];
+				}, result);
+			}
+			return result;
+		}
+
+		async function translateBatch(batch) {
+			const placeholderMap = new Map();
+			let placeholderCounter = 0;
+
+			const processedBatch = JSON.parse(
+				JSON.stringify(batch),
+				(_key, value) => {
+					if (typeof value !== 'string') return value;
+					return value.replace(/{([^{}]*)}/g, (match) => {
+						if (!placeholderMap.has(match)) {
+							placeholderMap.set(`__P_${placeholderCounter}__`, match);
+							placeholderCounter++;
+						}
+						for (const [token, ph] of placeholderMap.entries()) {
+							if (ph === match) return token;
+						}
+						return match;
+					});
+				},
+			);
+
+			const prompt = `
+You are a professional localization expert. Translate the JSON values from english to ${TARGET_LANGUAGE}.
+
+- **Target Locale:** ${targetLang} (choose naturally)
+- **DO NOT** translate the JSON keys.
+- **DO NOT** translate any placeholder tokens that look like \`__P_N__\`. Keep them exactly as they are.
+- **KEEP** all original markdown (\`##\`, \`*\`, \`\\\`\`, \`\n\`).
+- Respond ONLY with the translated JSON object, in a VALID JSON format.
+
+Input:
+${JSON.stringify(processedBatch, null, 2)}
+
+Output:
+`;
+
+			let attempt = 1;
+			let usedKeyIndex = keyIndex;
+
+			while (true) {
+				let genAI = getNextGenAI();
+				const GEMINI_API_CLIENT = genAI;
+				try {
+					console.log(`[Batch] Attempt #${attempt}...`);
+
+					const response = await GEMINI_API_CLIENT.models.generateContent({
+						model: GEMINI_MODEL,
+						contents: [{ role: 'user', parts: [{ text: prompt }] }],
+					});
+
+					let text;
+					if (response && typeof response.text === 'function') {
+						text = response.text();
+					} else if (response && typeof response.text === 'string') {
+						text = response.text;
+					}
+					text = typeof text === 'string' ? text.trim() : '';
+
+					if (text.startsWith('```json')) {
+						text = text.substring(7, text.length - 3).trim();
+					} else if (text.startsWith('```')) {
+						text = text
+							.replace(/^```[a-z]*\n?/, '')
+							.replace(/```$/, '')
+							.trim();
+					}
+
+					let translatedBatch = JSON.parse(text);
+					translatedBatch = JSON.parse(
+						JSON.stringify(translatedBatch),
+						(_key, value) => {
+							if (typeof value !== 'string') return value;
+							return value.replace(
+								/__P_(\d+)__/g,
+								(match) => placeholderMap.get(match) || match,
+							);
+						},
+					);
+
+					return translatedBatch;
+				} catch (e) {
+					const errorMessage = e.message || '';
+					console.error(
+						pc.red(`❌ Error in batch (Attempt ${attempt})...`),
+						errorMessage,
+					);
+
+					if (
+						errorMessage.includes('429') ||
+						errorMessage.includes('RESOURCE_EXHAUSTED')
+					) {
+						usedKeyIndex = (usedKeyIndex + 1) % API_KEYS.length;
+						console.warn(
+							pc.yellow(
+								`[RATE LIMIT] Got 429! Rotating to next API key [#${usedKeyIndex + 1}] and retrying.`,
+							),
+						);
+						genAI = getNextGenAI(usedKeyIndex);
+					} else {
+						console.warn(
+							pc.yellow(
+								`[OTHER ERROR] Waiting ${DELAY_ON_ERROR_MS / 1000} seconds...`,
+							),
+						);
+						await new Promise((resolve) =>
+							setTimeout(resolve, DELAY_ON_ERROR_MS),
+						);
+					}
+					attempt++;
+				}
+			}
+		}
+
+		console.log(
+			pc.cyan(`🚀 Starting translation process (to ${targetLang})...`),
+		);
+		console.log(pc.dim(`   Input: ${INPUT_FILE_SAFE}`));
+		console.log(pc.dim(`   Output: ${OUTPUT_FILE_SAFE}`));
+
+		if (!fs.existsSync(INPUT_FILE_SAFE)) {
+			console.error(pc.red(`❌ Input file not found: ${INPUT_FILE_SAFE}`));
+			process.exit(1);
+		}
+
+		const idJsonString = fs.readFileSync(INPUT_FILE_SAFE, 'utf8');
+		const idJson = JSON.parse(idJsonString);
+
+		console.log(pc.dim('Flattening JSON...'));
+		const flatIdJson = flattenObject(idJson);
+		const flatLangJson = {};
+		const allKeys = Object.keys(flatIdJson);
+
+		let existingLangJson = {};
+		if (fs.existsSync(OUTPUT_FILE_SAFE)) {
+			console.log(
+				pc.yellow(`[INFO] File ${OUTPUT_FILE_SAFE} exists, continuing work...`),
+			);
+			try {
+				existingLangJson = flattenObject(
+					JSON.parse(fs.readFileSync(OUTPUT_FILE_SAFE, 'utf8')),
+				);
+			} catch (_e) {
+				console.warn(
+					pc.yellow(
+						`[WARN] File ${OUTPUT_FILE_SAFE} is corrupted, will be overwritten.`,
+					),
+				);
+			}
+		}
+
+		const keysToTranslate = allKeys.filter(
+			(key) =>
+				typeof flatIdJson[key] === 'string' &&
+				(!existingLangJson[key] || existingLangJson[key] === flatIdJson[key]),
+		);
+
+		allKeys.forEach((key) => {
+			if (!keysToTranslate.includes(key)) {
+				flatLangJson[key] = existingLangJson[key] || flatIdJson[key];
+			}
+		});
+
+		const totalBatches = Math.ceil(keysToTranslate.length / BATCH_SIZE);
+
+		console.log(pc.green(`✅ Total of ${allKeys.length} keys.`));
+		console.log(
+			pc.green(
+				`✅ Found ${keysToTranslate.length} keys that need translation.`,
+			),
+		);
+		console.log(
+			pc.green(
+				`✅ Divided into ${totalBatches} batches (up to ${BATCH_SIZE} keys per batch).`,
+			),
+		);
+
+		for (let i = 0; i < totalBatches; i++) {
+			console.log(
+				pc.cyan(`--- 🏃 Working on Batch ${i + 1} / ${totalBatches} ---`),
+			);
+			const batchKeys = keysToTranslate.slice(
+				i * BATCH_SIZE,
+				(i + 1) * BATCH_SIZE,
+			);
+			const batchToTranslate = {};
+			batchKeys.forEach((key) => {
+				batchToTranslate[key] = flatIdJson[key];
+			});
+
+			if (Object.keys(batchToTranslate).length > 0) {
+				const translatedBatch = await translateBatch(batchToTranslate);
+				if (translatedBatch) {
+					Object.assign(flatLangJson, translatedBatch);
+				} else {
+					Object.assign(flatLangJson, batchToTranslate);
+				}
+			}
+
+			if (i < totalBatches - 1) {
+				console.log(
+					pc.yellow(
+						`--- 😴 Waiting ${DELAY_BETWEEN_BATCHES_MS / 1000} seconds between batches ---`,
+					),
+				);
+				await new Promise((resolve) =>
+					setTimeout(resolve, DELAY_BETWEEN_BATCHES_MS),
+				);
+			}
+		}
+
+		console.log(pc.dim('Unflattening JSON...'));
+		const langJson = unflattenObject(flatLangJson);
+
+		console.log(pc.green(`Saving to file: ${OUTPUT_FILE_SAFE}`));
+		fs.writeFileSync(
+			OUTPUT_FILE_SAFE,
+			JSON.stringify(langJson, null, 2),
+			'utf8',
+		);
+
+		console.log(pc.green('🎉 Done! Translation file generated successfully.'));
+	}
+}
+
+module.exports = LangTranslateCommand;