locai 2.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +928 -0
- package/dist/cli.js +718 -0
- package/dist/commands/review.js +357 -0
- package/dist/commands/translator.js +29 -0
- package/dist/config/index.js +52 -0
- package/dist/core/adapters/base-adapter.js +1 -0
- package/dist/core/adapters/factory.js +44 -0
- package/dist/core/adapters/json-adapter.js +20 -0
- package/dist/core/adapters/po-adapter.js +50 -0
- package/dist/core/adapters/properties-adapter.js +31 -0
- package/dist/core/adapters/yaml-adapter.js +24 -0
- package/dist/core/context-processor.js +247 -0
- package/dist/core/detection/framework-detector.js +56 -0
- package/dist/core/detection/structure-detector.js +30 -0
- package/dist/core/fallback-provider.js +442 -0
- package/dist/core/orchestrator.js +438 -0
- package/dist/core/pipeline/Pipeline.js +48 -0
- package/dist/core/pipeline/PipelineStep.js +13 -0
- package/dist/core/pipeline/context.js +21 -0
- package/dist/core/pipeline/steps/CacheReadStep.js +80 -0
- package/dist/core/pipeline/steps/CacheWriteStep.js +32 -0
- package/dist/core/pipeline/steps/ConfidenceCheckStep.js +67 -0
- package/dist/core/pipeline/steps/GlossaryPostStep.js +17 -0
- package/dist/core/pipeline/steps/GlossaryPreStep.js +20 -0
- package/dist/core/pipeline/steps/InputValidationStep.js +26 -0
- package/dist/core/pipeline/steps/QualityCheckStep.js +48 -0
- package/dist/core/pipeline/steps/ToneCheckStep.js +58 -0
- package/dist/core/pipeline/steps/TranslationStep.js +65 -0
- package/dist/core/pipeline/steps/VectorCacheReadStep.js +84 -0
- package/dist/core/pipeline/steps/VectorCacheWriteStep.js +42 -0
- package/dist/core/pipeline/steps/__tests__/ToneCheckStep.test.js +100 -0
- package/dist/core/provider-factory.js +139 -0
- package/dist/providers/base-provider.js +230 -0
- package/dist/providers/dashscope.js +128 -0
- package/dist/providers/deepseek.js +111 -0
- package/dist/providers/gemini.js +135 -0
- package/dist/providers/openai.js +203 -0
- package/dist/providers/xai.js +111 -0
- package/dist/services/embedding-provider.js +50 -0
- package/dist/services/translation-service.js +609 -0
- package/dist/services/vector-store.js +79 -0
- package/dist/utils/ai-context-analyzer.js +460 -0
- package/dist/utils/confidence-scorer.js +277 -0
- package/dist/utils/error-helper.js +386 -0
- package/dist/utils/file-manager.js +357 -0
- package/dist/utils/glossary-manager.js +194 -0
- package/dist/utils/graceful-shutdown.js +118 -0
- package/dist/utils/input-validator.js +663 -0
- package/dist/utils/logger.js +374 -0
- package/dist/utils/object-transformer.js +177 -0
- package/dist/utils/progress-tracker.js +277 -0
- package/dist/utils/prompt-templates.js +415 -0
- package/dist/utils/quality/__tests__/tone-verifier.test.js +91 -0
- package/dist/utils/quality/base-checker.js +73 -0
- package/dist/utils/quality/code-block-checker.js +214 -0
- package/dist/utils/quality/html-tag-checker.js +64 -0
- package/dist/utils/quality/index.js +220 -0
- package/dist/utils/quality/length-checker.js +140 -0
- package/dist/utils/quality/markdown-checker.js +220 -0
- package/dist/utils/quality/placeholder-checker.js +142 -0
- package/dist/utils/quality/punctuation-checker.js +50 -0
- package/dist/utils/quality/quote-balance-checker.js +143 -0
- package/dist/utils/quality/special-characters-checker.js +229 -0
- package/dist/utils/quality/style-guide-checker.js +192 -0
- package/dist/utils/quality/text-sanitizer.js +120 -0
- package/dist/utils/quality/tone-verifier.js +91 -0
- package/dist/utils/rate-limiter.js +398 -0
- package/dist/utils/retry-helper.js +166 -0
- package/dist/utils/state-manager.js +162 -0
- package/dist/utils/statistics-manager.js +51 -0
- package/dist/utils/ui-manager.js +98 -0
- package/package.json +80 -0
|
@@ -0,0 +1,609 @@
|
|
|
1
|
+
import path from "path";
|
|
2
|
+
import { FileManager } from "../utils/file-manager.js";
|
|
3
|
+
import ObjectTransformer from "../utils/object-transformer.js";
|
|
4
|
+
import Orchestrator from "../core/orchestrator.js";
|
|
5
|
+
import QualityChecker from "../utils/quality/index.js";
|
|
6
|
+
import StateManager from "../utils/state-manager.js";
|
|
7
|
+
import InputValidator from "../utils/input-validator.js";
|
|
8
|
+
import gracefulShutdown from "../utils/graceful-shutdown.js";
|
|
9
|
+
import uiManager from "../utils/ui-manager.js";
|
|
10
|
+
import statisticsManager from "../utils/statistics-manager.js";
|
|
11
|
+
/**
|
|
12
|
+
* Service for handling file translation operations.
|
|
13
|
+
* Orchestrates the translation process including state management,
|
|
14
|
+
* verification, and statistics collection.
|
|
15
|
+
*/
|
|
16
|
+
class TranslationService {
|
|
17
|
+
/**
|
|
18
|
+
* Check if text only contains placeholders and whitespace.
|
|
19
|
+
* @param {string} text - Text to check.
|
|
20
|
+
* @returns {boolean} - True if text only contains placeholders.
|
|
21
|
+
*/
|
|
22
|
+
isPlaceholderOnlyText(text) {
|
|
23
|
+
if (!text || typeof text !== "string") {
|
|
24
|
+
return false;
|
|
25
|
+
}
|
|
26
|
+
// Remove all placeholders from the text
|
|
27
|
+
const placeholderRegex = /\{[^}]+\}|\$\{[^}]+\}|%[sd]/g;
|
|
28
|
+
const textWithoutPlaceholders = text.replace(placeholderRegex, "").trim();
|
|
29
|
+
return textWithoutPlaceholders.length === 0;
|
|
30
|
+
}
|
|
31
|
+
/**
|
|
32
|
+
* Main translator function to process source file and create translations.
|
|
33
|
+
* @param {string} file - Path to source file.
|
|
34
|
+
* @param {TranslationOptions} options - Translation options.
|
|
35
|
+
* @returns {Promise<GlobalStats>} - Global statistics object.
|
|
36
|
+
* @throws {Error} If validation fails or translation process errors.
|
|
37
|
+
*/
|
|
38
|
+
async translateFile(file, options) {
|
|
39
|
+
await uiManager.log(`\nProcessing File: "${path.basename(file)}"`);
|
|
40
|
+
try {
|
|
41
|
+
const resolvedFile = await this.validateTranslationInputs(file, options);
|
|
42
|
+
const startTime = Date.now();
|
|
43
|
+
const sourceContent = await FileManager.readJSON(resolvedFile);
|
|
44
|
+
const flattenedSource = ObjectTransformer.flatten(sourceContent);
|
|
45
|
+
const totalKeys = Object.keys(flattenedSource).length;
|
|
46
|
+
await uiManager.log(`Source file contains ${totalKeys} translation keys`);
|
|
47
|
+
const { stateManager, projectRoot, currentState, comparison } = await this.initializeTranslationState(resolvedFile, flattenedSource, options);
|
|
48
|
+
gracefulShutdown.registerCallback(async () => {
|
|
49
|
+
try {
|
|
50
|
+
await stateManager.saveState(projectRoot, currentState);
|
|
51
|
+
console.log("State saved during shutdown");
|
|
52
|
+
}
|
|
53
|
+
catch (error) {
|
|
54
|
+
console.error("Failed to save state during shutdown:", error.message);
|
|
55
|
+
}
|
|
56
|
+
});
|
|
57
|
+
statisticsManager.reset();
|
|
58
|
+
const globalStats = statisticsManager.getStats();
|
|
59
|
+
try {
|
|
60
|
+
await this.processAllLanguages(resolvedFile, flattenedSource, options, globalStats, comparison);
|
|
61
|
+
await this.finalizeTranslation(stateManager, projectRoot, currentState, globalStats, startTime, options);
|
|
62
|
+
return globalStats;
|
|
63
|
+
}
|
|
64
|
+
catch (error) {
|
|
65
|
+
await uiManager.log(`\nTranslation error: ${error.message}`);
|
|
66
|
+
globalStats.error = {
|
|
67
|
+
message: error.message,
|
|
68
|
+
time: new Date().toISOString(),
|
|
69
|
+
stack: process.env.DEBUG ? error.stack : undefined,
|
|
70
|
+
};
|
|
71
|
+
throw error;
|
|
72
|
+
}
|
|
73
|
+
finally {
|
|
74
|
+
if (process.env.DEBUG) {
|
|
75
|
+
await uiManager.log("\nCache statistics:");
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
catch (validationError) {
|
|
80
|
+
await uiManager.log(`\nInput validation error: ${validationError.message}`);
|
|
81
|
+
throw validationError;
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
/**
|
|
85
|
+
* Validate input parameters for translation process.
|
|
86
|
+
* @param {string} file - Path to source file.
|
|
87
|
+
* @param {TranslationOptions} options - Translation options.
|
|
88
|
+
* @returns {Promise<string>} - Resolved absolute path to source file.
|
|
89
|
+
* @throws {Error} If inputs are invalid.
|
|
90
|
+
*/
|
|
91
|
+
async validateTranslationInputs(file, options) {
|
|
92
|
+
if (!file || typeof file !== "string") {
|
|
93
|
+
throw new Error("File path must be a non-empty string");
|
|
94
|
+
}
|
|
95
|
+
if (!options || typeof options !== "object") {
|
|
96
|
+
throw new Error("Options must be an object");
|
|
97
|
+
}
|
|
98
|
+
if (options.source) {
|
|
99
|
+
options.source = InputValidator.validateLanguageCode(options.source, "source language");
|
|
100
|
+
}
|
|
101
|
+
if (options.targets && Array.isArray(options.targets)) {
|
|
102
|
+
options.targets = InputValidator.validateLanguageCodes(options.targets, "target languages");
|
|
103
|
+
}
|
|
104
|
+
if (options.localesDir) {
|
|
105
|
+
options.localesDir = InputValidator.validateDirectoryPath(options.localesDir, "locales directory");
|
|
106
|
+
}
|
|
107
|
+
const resolvedFile = path.resolve(file);
|
|
108
|
+
const cwd = process.cwd();
|
|
109
|
+
if (!resolvedFile.startsWith(cwd)) {
|
|
110
|
+
throw new Error(`Source file '${file}' is outside working directory (resolved: ${resolvedFile})`);
|
|
111
|
+
}
|
|
112
|
+
return resolvedFile;
|
|
113
|
+
}
|
|
114
|
+
/**
|
|
115
|
+
* Initialize translation state and sync analysis.
|
|
116
|
+
* @param {string} resolvedFile - Path to resolved source file.
|
|
117
|
+
* @param {Record<string, any>} flattenedSource - Flattened source content.
|
|
118
|
+
* @param {TranslationOptions} options - Translation options.
|
|
119
|
+
* @returns {Promise<Object>} - Initialized state components.
|
|
120
|
+
*/
|
|
121
|
+
async initializeTranslationState(resolvedFile, flattenedSource, options) {
|
|
122
|
+
const stateManager = new StateManager(options.syncOptions);
|
|
123
|
+
const projectRoot = process.cwd();
|
|
124
|
+
const previousState = await stateManager.loadState(projectRoot);
|
|
125
|
+
const currentState = stateManager.generateStateFromSource(flattenedSource);
|
|
126
|
+
const comparison = stateManager.compareStates(previousState, currentState);
|
|
127
|
+
const stats = stateManager.getComparisonStats(comparison);
|
|
128
|
+
if (stats.hasChanges) {
|
|
129
|
+
await uiManager.log(`\nSync Analysis:`);
|
|
130
|
+
await uiManager.log(` New keys: ${stats.newCount}`);
|
|
131
|
+
await uiManager.log(` Modified keys: ${stats.modifiedCount}`);
|
|
132
|
+
await uiManager.log(` Deleted keys: ${stats.deletedCount}`);
|
|
133
|
+
// Handle deleted keys - remove them from all target files
|
|
134
|
+
const syncEnabled = options.syncOptions?.enabled !== false;
|
|
135
|
+
const removeDeletedEnabled = options.syncOptions?.removeDeletedKeys !== false;
|
|
136
|
+
if (comparison?.deletedKeys &&
|
|
137
|
+
Array.isArray(comparison.deletedKeys) &&
|
|
138
|
+
comparison.deletedKeys.length > 0 &&
|
|
139
|
+
syncEnabled &&
|
|
140
|
+
removeDeletedEnabled) {
|
|
141
|
+
await uiManager.log(`\nRemoving ${comparison.deletedKeys.length} deleted keys from target files...`);
|
|
142
|
+
await this.removeDeletedKeysFromTargets(resolvedFile, comparison.deletedKeys, options);
|
|
143
|
+
}
|
|
144
|
+
}
|
|
145
|
+
else if (previousState &&
|
|
146
|
+
typeof previousState === "object" &&
|
|
147
|
+
Object.keys(previousState).length > 0) {
|
|
148
|
+
await uiManager.log(`No changes detected in source file`);
|
|
149
|
+
}
|
|
150
|
+
else {
|
|
151
|
+
await uiManager.log(`First run - will process all keys`);
|
|
152
|
+
}
|
|
153
|
+
return { stateManager, projectRoot, currentState, comparison };
|
|
154
|
+
}
|
|
155
|
+
/**
|
|
156
|
+
* Process all target languages with concurrency.
|
|
157
|
+
* @param {string} resolvedFile - Path to source file.
|
|
158
|
+
* @param {Record<string, any>} flattenedSource - Flattened source content.
|
|
159
|
+
* @param {TranslationOptions} options - Translation options.
|
|
160
|
+
* @param {GlobalStats} globalStats - Global statistics object to update.
|
|
161
|
+
* @param {any} comparison - State comparison result.
|
|
162
|
+
*/
|
|
163
|
+
async processAllLanguages(resolvedFile, flattenedSource, options, globalStats, comparison) {
|
|
164
|
+
const languageConcurrency = options.concurrencyLimit || 3;
|
|
165
|
+
const targetLanguages = [...(options.targets || [])];
|
|
166
|
+
await uiManager.log(`Processing ${targetLanguages.length} languages with concurrency of ${languageConcurrency}`);
|
|
167
|
+
// Create shared orchestrators array to collect review queues
|
|
168
|
+
const orchestrators = [];
|
|
169
|
+
for (let i = 0; i < targetLanguages.length; i += languageConcurrency) {
|
|
170
|
+
const currentBatch = targetLanguages.slice(i, i + languageConcurrency);
|
|
171
|
+
// Respect user's progress options, but default logToConsole to false for batch processing
|
|
172
|
+
// unless explicitly forced by the user in the config (not just inherited default)
|
|
173
|
+
const progressOptions = {
|
|
174
|
+
...options.progressOptions,
|
|
175
|
+
// If not explicitly set in options, default to false for batch mode to prevent garbled output
|
|
176
|
+
logToConsole: options.progressOptions?.logToConsole ?? false,
|
|
177
|
+
};
|
|
178
|
+
const batchResults = await Promise.all(currentBatch.map((targetLang) => {
|
|
179
|
+
const orchestrator = new Orchestrator({
|
|
180
|
+
...options,
|
|
181
|
+
concurrencyLimit: 1,
|
|
182
|
+
progressOptions,
|
|
183
|
+
});
|
|
184
|
+
orchestrators.push(orchestrator);
|
|
185
|
+
return this.processLanguage(targetLang, resolvedFile, flattenedSource, orchestrator, { ...options, progressOptions }, globalStats, comparison);
|
|
186
|
+
}));
|
|
187
|
+
await uiManager.logBatchResults(batchResults);
|
|
188
|
+
}
|
|
189
|
+
// Store orchestrators in globalStats for later access
|
|
190
|
+
globalStats.orchestrators = orchestrators;
|
|
191
|
+
}
|
|
192
|
+
/**
|
|
193
|
+
* Process a single language translation.
|
|
194
|
+
* @param {string} targetLang - Target language code.
|
|
195
|
+
* @param {string} sourceFile - Path to source file.
|
|
196
|
+
* @param {Record<string, any>} flattenedSource - Flattened source content.
|
|
197
|
+
* @param {Orchestrator} orchestrator - Orchestrator instance.
|
|
198
|
+
* @param {TranslationOptions} options - Translation options.
|
|
199
|
+
* @param {GlobalStats} globalStats - Global statistics object.
|
|
200
|
+
* @param {any} comparison - State comparison result.
|
|
201
|
+
* @returns {Promise<LanguageProcessResult>} - Status and messages for this language.
|
|
202
|
+
*/
|
|
203
|
+
async processLanguage(targetLang, sourceFile, flattenedSource, orchestrator, options, globalStats, comparison) {
|
|
204
|
+
const langStartTime = Date.now();
|
|
205
|
+
try {
|
|
206
|
+
if (!targetLang || typeof targetLang !== "string") {
|
|
207
|
+
throw new Error("Invalid target language provided");
|
|
208
|
+
}
|
|
209
|
+
if (!sourceFile || typeof sourceFile !== "string") {
|
|
210
|
+
throw new Error("Invalid source file path provided");
|
|
211
|
+
}
|
|
212
|
+
if (!flattenedSource || typeof flattenedSource !== "object") {
|
|
213
|
+
throw new Error("Invalid flattened source data provided");
|
|
214
|
+
}
|
|
215
|
+
if (!orchestrator || typeof orchestrator.processTranslations !== "function") {
|
|
216
|
+
throw new Error("Invalid orchestrator instance provided");
|
|
217
|
+
}
|
|
218
|
+
if (!globalStats || typeof globalStats !== "object") {
|
|
219
|
+
throw new Error("Invalid global stats object provided");
|
|
220
|
+
}
|
|
221
|
+
const safeTargetLang = InputValidator.validateLanguageCode(targetLang, "target language");
|
|
222
|
+
await uiManager.log(`Starting translations for ${safeTargetLang}`);
|
|
223
|
+
let finalStatus = null;
|
|
224
|
+
let savedMessage = undefined;
|
|
225
|
+
statisticsManager.initLanguageStats(safeTargetLang);
|
|
226
|
+
const sourceDir = path.dirname(sourceFile);
|
|
227
|
+
const safeTargetFilename = `${safeTargetLang}.json`;
|
|
228
|
+
const targetPath = InputValidator.createSafeFilePath(sourceDir, safeTargetFilename);
|
|
229
|
+
let targetContent = {};
|
|
230
|
+
try {
|
|
231
|
+
targetContent = await FileManager.readJSON(targetPath);
|
|
232
|
+
if (!targetContent || typeof targetContent !== "object") {
|
|
233
|
+
console.warn(`Invalid content in ${targetPath}, using empty object`);
|
|
234
|
+
targetContent = {};
|
|
235
|
+
}
|
|
236
|
+
}
|
|
237
|
+
catch (err) {
|
|
238
|
+
if (err.code === "ENOENT") {
|
|
239
|
+
await uiManager.log(`\ud83c\udd95 Creating new translation file for ${safeTargetLang}`);
|
|
240
|
+
}
|
|
241
|
+
else {
|
|
242
|
+
if (options.debug) {
|
|
243
|
+
console.warn(`Error reading ${targetPath}: ${err.message}, using empty object`);
|
|
244
|
+
}
|
|
245
|
+
}
|
|
246
|
+
targetContent = {};
|
|
247
|
+
}
|
|
248
|
+
let flattenedTarget = {};
|
|
249
|
+
try {
|
|
250
|
+
flattenedTarget = ObjectTransformer.flatten(targetContent);
|
|
251
|
+
if (!flattenedTarget || typeof flattenedTarget !== "object") {
|
|
252
|
+
flattenedTarget = {};
|
|
253
|
+
}
|
|
254
|
+
}
|
|
255
|
+
catch (err) {
|
|
256
|
+
console.warn(`Error flattening target content: ${err.message}, using empty object`);
|
|
257
|
+
flattenedTarget = {};
|
|
258
|
+
}
|
|
259
|
+
const missingKeys = [];
|
|
260
|
+
let hasPlaceholderOnlyChanges = false;
|
|
261
|
+
for (const [key, sourceText] of Object.entries(flattenedSource)) {
|
|
262
|
+
try {
|
|
263
|
+
InputValidator.validateKey(key, "translation key");
|
|
264
|
+
InputValidator.validateText(sourceText, "source text");
|
|
265
|
+
}
|
|
266
|
+
catch (keyError) {
|
|
267
|
+
await uiManager.log(`\u26a0\ufe0f Skipping invalid key/text: ${keyError.message}`);
|
|
268
|
+
globalStats.languages[safeTargetLang].failed++;
|
|
269
|
+
globalStats.failed++;
|
|
270
|
+
continue;
|
|
271
|
+
}
|
|
272
|
+
globalStats.languages[safeTargetLang].processed++;
|
|
273
|
+
if (this.isPlaceholderOnlyText(sourceText)) {
|
|
274
|
+
const existingValue = flattenedTarget[key];
|
|
275
|
+
if (existingValue !== sourceText) {
|
|
276
|
+
flattenedTarget[key] = sourceText;
|
|
277
|
+
hasPlaceholderOnlyChanges = true;
|
|
278
|
+
if (options.debug) {
|
|
279
|
+
await uiManager.log(` Copying placeholder-only text: "${sourceText}"`);
|
|
280
|
+
}
|
|
281
|
+
}
|
|
282
|
+
globalStats.languages[safeTargetLang].skipped++;
|
|
283
|
+
globalStats.skipped++;
|
|
284
|
+
continue;
|
|
285
|
+
}
|
|
286
|
+
const isNewKey = comparison.newKeys.includes(key);
|
|
287
|
+
const isModifiedKey = comparison.modifiedKeys.includes(key);
|
|
288
|
+
const keyExistsInTarget = key in flattenedTarget;
|
|
289
|
+
if (keyExistsInTarget && !options.forceUpdate && !isModifiedKey) {
|
|
290
|
+
globalStats.languages[safeTargetLang].skipped++;
|
|
291
|
+
globalStats.skipped++;
|
|
292
|
+
continue;
|
|
293
|
+
}
|
|
294
|
+
missingKeys.push({
|
|
295
|
+
key,
|
|
296
|
+
text: sourceText,
|
|
297
|
+
targetLang: safeTargetLang,
|
|
298
|
+
existingTranslation: flattenedTarget[key],
|
|
299
|
+
isModified: isModifiedKey,
|
|
300
|
+
isNew: isNewKey,
|
|
301
|
+
});
|
|
302
|
+
}
|
|
303
|
+
if (missingKeys.length === 0 && !hasPlaceholderOnlyChanges) {
|
|
304
|
+
await uiManager.log(`All translations exist for ${safeTargetLang}`);
|
|
305
|
+
globalStats.languages[safeTargetLang].timeMs = Date.now() - langStartTime;
|
|
306
|
+
return { status: { completed: 0, total: 0, language: safeTargetLang } };
|
|
307
|
+
}
|
|
308
|
+
if (missingKeys.length > 0) {
|
|
309
|
+
await uiManager.log(`Found ${missingKeys.length} missing translations for ${safeTargetLang}`);
|
|
310
|
+
}
|
|
311
|
+
let results = [];
|
|
312
|
+
if (missingKeys.length > 0) {
|
|
313
|
+
try {
|
|
314
|
+
results = await orchestrator.processTranslations(missingKeys);
|
|
315
|
+
if (!Array.isArray(results)) {
|
|
316
|
+
console.warn("Invalid results from orchestrator, using empty array");
|
|
317
|
+
results = [];
|
|
318
|
+
}
|
|
319
|
+
}
|
|
320
|
+
catch (err) {
|
|
321
|
+
console.error(`Error processing translations for ${safeTargetLang}: ${err.message}`);
|
|
322
|
+
results = [];
|
|
323
|
+
if (globalStats.languages[safeTargetLang]) {
|
|
324
|
+
globalStats.languages[safeTargetLang].failed += missingKeys.length;
|
|
325
|
+
globalStats.failed += missingKeys.length;
|
|
326
|
+
}
|
|
327
|
+
}
|
|
328
|
+
}
|
|
329
|
+
else {
|
|
330
|
+
if (hasPlaceholderOnlyChanges) {
|
|
331
|
+
await uiManager.log(`Found placeholder-only changes for ${safeTargetLang}`);
|
|
332
|
+
}
|
|
333
|
+
}
|
|
334
|
+
try {
|
|
335
|
+
if (orchestrator?.progress &&
|
|
336
|
+
typeof orchestrator.progress.getStatus === "function") {
|
|
337
|
+
finalStatus = orchestrator.progress.getStatus();
|
|
338
|
+
}
|
|
339
|
+
}
|
|
340
|
+
catch (err) {
|
|
341
|
+
console.warn(`Error getting orchestrator status: ${err.message}`);
|
|
342
|
+
}
|
|
343
|
+
let validResults = [];
|
|
344
|
+
try {
|
|
345
|
+
validResults = results.filter((result) => result && result.success === true);
|
|
346
|
+
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
347
|
+
validResults = validResults.filter((result) => result.key &&
|
|
348
|
+
typeof result.key === "string" &&
|
|
349
|
+
result.translated &&
|
|
350
|
+
typeof result.translated === "string");
|
|
351
|
+
}
|
|
352
|
+
catch (err) {
|
|
353
|
+
console.warn(`Error filtering results: ${err.message}`);
|
|
354
|
+
validResults = [];
|
|
355
|
+
}
|
|
356
|
+
if (validResults.length > 0) {
|
|
357
|
+
validResults.forEach(({ key, translated }) => {
|
|
358
|
+
if (translated) {
|
|
359
|
+
flattenedTarget[key] = translated;
|
|
360
|
+
}
|
|
361
|
+
});
|
|
362
|
+
}
|
|
363
|
+
const hasChangesToSave = validResults.length > 0 || hasPlaceholderOnlyChanges;
|
|
364
|
+
if (hasChangesToSave) {
|
|
365
|
+
const unflattened = ObjectTransformer.unflatten(flattenedTarget);
|
|
366
|
+
await FileManager.writeJSON(targetPath, unflattened);
|
|
367
|
+
globalStats.total += validResults.length;
|
|
368
|
+
globalStats.success += validResults.length;
|
|
369
|
+
globalStats.failed += results.length - validResults.length;
|
|
370
|
+
globalStats.languages[safeTargetLang].added += validResults.length;
|
|
371
|
+
globalStats.languages[safeTargetLang].failed +=
|
|
372
|
+
results.length - validResults.length;
|
|
373
|
+
validResults.forEach((result) => {
|
|
374
|
+
if (result.context) {
|
|
375
|
+
const category = result.context.category || "general";
|
|
376
|
+
globalStats.byCategory[category] =
|
|
377
|
+
(globalStats.byCategory[category] || 0) + 1;
|
|
378
|
+
if (!globalStats.details[category]) {
|
|
379
|
+
globalStats.details[category] = {
|
|
380
|
+
totalConfidence: 0,
|
|
381
|
+
samples: 0,
|
|
382
|
+
};
|
|
383
|
+
}
|
|
384
|
+
globalStats.details[category].totalConfidence +=
|
|
385
|
+
result.context.confidence || 0;
|
|
386
|
+
globalStats.details[category].samples++;
|
|
387
|
+
}
|
|
388
|
+
});
|
|
389
|
+
savedMessage = `\n\ud83d\udcbe Translations saved: ${safeTargetLang}.json`;
|
|
390
|
+
}
|
|
391
|
+
globalStats.languages[safeTargetLang].timeMs = Date.now() - langStartTime;
|
|
392
|
+
return { status: finalStatus, savedMessage };
|
|
393
|
+
}
|
|
394
|
+
catch (error) {
|
|
395
|
+
const safeError = error.message.includes("outside working directory")
|
|
396
|
+
? "Invalid file path detected"
|
|
397
|
+
: error.message;
|
|
398
|
+
await uiManager.log(`\n\u274c Error processing ${targetLang}: ${safeError}`);
|
|
399
|
+
if (globalStats.languages[targetLang]) {
|
|
400
|
+
globalStats.languages[targetLang].error = safeError;
|
|
401
|
+
globalStats.languages[targetLang].timeMs = Date.now() - langStartTime;
|
|
402
|
+
}
|
|
403
|
+
return { status: null, error: safeError };
|
|
404
|
+
}
|
|
405
|
+
}
|
|
406
|
+
/**
|
|
407
|
+
* Finalize translation process and save state.
|
|
408
|
+
* @param {StateManager} stateManager - State manager instance.
|
|
409
|
+
* @param {string} projectRoot - Project root directory.
|
|
410
|
+
* @param {any} currentState - Current state object.
|
|
411
|
+
* @param {GlobalStats} globalStats - Global statistics object.
|
|
412
|
+
* @param {number} startTime - Process start timestamp.
|
|
413
|
+
* @param {TranslationOptions} options - Translation options.
|
|
414
|
+
*/
|
|
415
|
+
async finalizeTranslation(stateManager, projectRoot, currentState, globalStats, startTime, options) {
|
|
416
|
+
globalStats.endTime = new Date().toISOString();
|
|
417
|
+
globalStats.totalDuration = (Date.now() - startTime) / 1000;
|
|
418
|
+
await uiManager.displayGlobalSummary(globalStats, (options.targets || []).length);
|
|
419
|
+
// Save review queue if confidence scoring was enabled
|
|
420
|
+
if (options.saveReviewQueue || options.minConfidence !== undefined) {
|
|
421
|
+
if (globalStats.orchestrators && globalStats.orchestrators.length > 0) {
|
|
422
|
+
// Collect all review queues from all orchestrators
|
|
423
|
+
const allReviewItems = [];
|
|
424
|
+
globalStats.orchestrators.forEach((orchestrator) => {
|
|
425
|
+
if (orchestrator.confidenceSettings &&
|
|
426
|
+
orchestrator.confidenceSettings.reviewQueue) {
|
|
427
|
+
allReviewItems.push(...orchestrator.confidenceSettings.reviewQueue);
|
|
428
|
+
}
|
|
429
|
+
});
|
|
430
|
+
// If we have items to review, save them
|
|
431
|
+
if (allReviewItems.length > 0) {
|
|
432
|
+
// Use first orchestrator to save, but update its review queue
|
|
433
|
+
const orchestrator = globalStats.orchestrators[0];
|
|
434
|
+
orchestrator.confidenceSettings.reviewQueue = allReviewItems;
|
|
435
|
+
orchestrator.saveReviewQueue();
|
|
436
|
+
}
|
|
437
|
+
}
|
|
438
|
+
}
|
|
439
|
+
try {
|
|
440
|
+
await stateManager.saveState(projectRoot, currentState);
|
|
441
|
+
if (options.debug) {
|
|
442
|
+
await uiManager.log(`State saved for future sync operations`);
|
|
443
|
+
}
|
|
444
|
+
}
|
|
445
|
+
catch (error) {
|
|
446
|
+
await uiManager.log(`Warning: Could not save state: ${error.message}`);
|
|
447
|
+
}
|
|
448
|
+
}
|
|
449
|
+
/**
|
|
450
|
+
* Remove deleted keys from all target language files.
|
|
451
|
+
* @param {string} sourceFile - Path to source file.
|
|
452
|
+
* @param {string[]} deletedKeys - Array of keys to delete.
|
|
453
|
+
* @param {TranslationOptions} options - Translation options.
|
|
454
|
+
*/
|
|
455
|
+
async removeDeletedKeysFromTargets(sourceFile, deletedKeys, options) {
|
|
456
|
+
const sourceDir = path.dirname(sourceFile);
|
|
457
|
+
let totalRemoved = 0;
|
|
458
|
+
let filesProcessed = 0;
|
|
459
|
+
for (const targetLang of options.targets || []) {
|
|
460
|
+
try {
|
|
461
|
+
const safeTargetFilename = `${targetLang}.json`;
|
|
462
|
+
const targetPath = InputValidator.createSafeFilePath(sourceDir, safeTargetFilename);
|
|
463
|
+
// Check if target file exists
|
|
464
|
+
const fileExists = await FileManager.exists(targetPath);
|
|
465
|
+
if (!fileExists) {
|
|
466
|
+
await uiManager.log(` Skipping ${targetLang}.json (file doesn't exist)`);
|
|
467
|
+
continue;
|
|
468
|
+
}
|
|
469
|
+
// Read target file
|
|
470
|
+
const targetContent = await FileManager.readJSON(targetPath);
|
|
471
|
+
const flattenedTarget = ObjectTransformer.flatten(targetContent);
|
|
472
|
+
// Remove deleted keys
|
|
473
|
+
let removedFromThisFile = 0;
|
|
474
|
+
for (const key of deletedKeys) {
|
|
475
|
+
if (key in flattenedTarget) {
|
|
476
|
+
delete flattenedTarget[key];
|
|
477
|
+
removedFromThisFile++;
|
|
478
|
+
totalRemoved++;
|
|
479
|
+
}
|
|
480
|
+
}
|
|
481
|
+
// Save updated file only if we removed something
|
|
482
|
+
if (removedFromThisFile > 0) {
|
|
483
|
+
const unflattened = ObjectTransformer.unflatten(flattenedTarget);
|
|
484
|
+
await FileManager.writeJSON(targetPath, unflattened);
|
|
485
|
+
await uiManager.log(` ${targetLang}.json: Removed ${removedFromThisFile} keys`);
|
|
486
|
+
}
|
|
487
|
+
else {
|
|
488
|
+
await uiManager.log(` No keys to remove from ${targetLang}.json`);
|
|
489
|
+
}
|
|
490
|
+
filesProcessed++;
|
|
491
|
+
}
|
|
492
|
+
catch (error) {
|
|
493
|
+
await uiManager.log(` \u274c Error processing ${targetLang}.json: ${error.message}`);
|
|
494
|
+
}
|
|
495
|
+
}
|
|
496
|
+
await uiManager.log(`Cleanup Summary: Removed ${totalRemoved} keys from ${filesProcessed} files\n`);
|
|
497
|
+
}
|
|
498
|
+
/**
|
|
499
|
+
* Validate and fix existing translations that have length issues.
|
|
500
|
+
* @param {string} file - Path to source file.
|
|
501
|
+
* @param {TranslationOptions} options - Translation options.
|
|
502
|
+
* @throws {Error} If validation process fails.
|
|
503
|
+
*/
|
|
504
|
+
async validateAndFixExistingTranslations(file, options) {
|
|
505
|
+
await uiManager.log(`\nChecking existing translations in: "${path.basename(file)}"`);
|
|
506
|
+
const sourceContent = await FileManager.readJSON(file);
|
|
507
|
+
const flattenedSource = ObjectTransformer.flatten(sourceContent);
|
|
508
|
+
const orchestrator = new Orchestrator(options);
|
|
509
|
+
const languageResults = await Promise.all((options.targets || []).map(async (targetLang) => {
|
|
510
|
+
try {
|
|
511
|
+
const targetPath = path.join(path.dirname(file), `${targetLang}.json`);
|
|
512
|
+
const targetContent = await FileManager.readJSON(targetPath);
|
|
513
|
+
return { targetLang, targetPath, content: targetContent, success: true };
|
|
514
|
+
}
|
|
515
|
+
catch (err) {
|
|
516
|
+
await uiManager.log(`Could not read ${targetLang}.json: ${err.message}`);
|
|
517
|
+
return {
|
|
518
|
+
targetLang,
|
|
519
|
+
targetPath: "",
|
|
520
|
+
content: {},
|
|
521
|
+
success: false,
|
|
522
|
+
error: err.message,
|
|
523
|
+
};
|
|
524
|
+
}
|
|
525
|
+
}));
|
|
526
|
+
const validLanguages = languageResults.filter((result) => result.success);
|
|
527
|
+
try {
|
|
528
|
+
let totalFixed = 0;
|
|
529
|
+
let totalIssues = 0;
|
|
530
|
+
const qualityChecker = new QualityChecker({
|
|
531
|
+
styleGuide: options.styleGuide,
|
|
532
|
+
context: options.context,
|
|
533
|
+
lengthControl: options.lengthControl,
|
|
534
|
+
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
535
|
+
...(options.qualityChecks?.rules || {}),
|
|
536
|
+
enabled: true,
|
|
537
|
+
});
|
|
538
|
+
// Process languages sequentially to avoid overwhelming the API
|
|
539
|
+
for (const langData of validLanguages) {
|
|
540
|
+
const { targetLang, targetPath, content } = langData;
|
|
541
|
+
const flattenedTarget = ObjectTransformer.flatten(content);
|
|
542
|
+
const invalidItems = [];
|
|
543
|
+
for (const [key, translatedText] of Object.entries(flattenedTarget)) {
|
|
544
|
+
const sourceText = flattenedSource[key];
|
|
545
|
+
if (!sourceText)
|
|
546
|
+
continue;
|
|
547
|
+
const checkResult = qualityChecker.validate(sourceText, translatedText, {
|
|
548
|
+
...options,
|
|
549
|
+
targetLang,
|
|
550
|
+
});
|
|
551
|
+
const lengthIssue = checkResult.issues.find((i) => i.type === "length");
|
|
552
|
+
if (lengthIssue) {
|
|
553
|
+
invalidItems.push({
|
|
554
|
+
key,
|
|
555
|
+
text: sourceText,
|
|
556
|
+
targetLang,
|
|
557
|
+
existingTranslation: translatedText,
|
|
558
|
+
issueDetails: lengthIssue,
|
|
559
|
+
isModified: false,
|
|
560
|
+
isNew: false,
|
|
561
|
+
});
|
|
562
|
+
}
|
|
563
|
+
}
|
|
564
|
+
if (invalidItems.length > 0) {
|
|
565
|
+
totalIssues += invalidItems.length;
|
|
566
|
+
await uiManager.log(`Found ${invalidItems.length} length issues in ${targetLang}`);
|
|
567
|
+
// Batch process fixes to improve performance
|
|
568
|
+
const results = await orchestrator.processTranslations(invalidItems);
|
|
569
|
+
const fixedCount = results.filter((r) => r.success).length;
|
|
570
|
+
totalFixed += fixedCount;
|
|
571
|
+
results.forEach(({ key, translated, success }) => {
|
|
572
|
+
if (success && translated) {
|
|
573
|
+
flattenedTarget[key] = translated;
|
|
574
|
+
}
|
|
575
|
+
});
|
|
576
|
+
const unflattened = ObjectTransformer.unflatten(flattenedTarget);
|
|
577
|
+
await FileManager.writeJSON(targetPath, unflattened);
|
|
578
|
+
await uiManager.log(`Fixed ${fixedCount}/${invalidItems.length} translations in ${targetLang}`);
|
|
579
|
+
}
|
|
580
|
+
else {
|
|
581
|
+
await uiManager.log(`No length issues found in ${targetLang}`);
|
|
582
|
+
}
|
|
583
|
+
}
|
|
584
|
+
if (totalIssues > 0) {
|
|
585
|
+
await uiManager.log(`\nFix Length Summary: Fixed ${totalFixed} of ${totalIssues} issues (${Math.round((totalFixed / totalIssues) * 100)}%)`);
|
|
586
|
+
}
|
|
587
|
+
else {
|
|
588
|
+
await uiManager.log(`\nNo length issues found in any language`);
|
|
589
|
+
}
|
|
590
|
+
}
|
|
591
|
+
catch (error) {
|
|
592
|
+
await uiManager.log(`\nValidation error: ${error.message}`);
|
|
593
|
+
throw error;
|
|
594
|
+
}
|
|
595
|
+
}
|
|
596
|
+
/**
|
|
597
|
+
* Find locale files
|
|
598
|
+
*/
|
|
599
|
+
async findLocaleFiles(localesDir, sourceLang) {
|
|
600
|
+
try {
|
|
601
|
+
return await FileManager.findLocaleFiles(localesDir, sourceLang);
|
|
602
|
+
}
|
|
603
|
+
catch (error) {
|
|
604
|
+
await uiManager.log(`Error finding locale files: ${error.message}`);
|
|
605
|
+
return [];
|
|
606
|
+
}
|
|
607
|
+
}
|
|
608
|
+
}
|
|
609
|
+
export default new TranslationService();
|
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
import { LocalIndex } from "vectra";
|
|
2
|
+
import path from "path";
|
|
3
|
+
import fs from "fs";
|
|
4
|
+
export class VectorStore {
|
|
5
|
+
index = null;
|
|
6
|
+
config;
|
|
7
|
+
initialized = false;
|
|
8
|
+
DIMENSION = 1536; // OpenAI text-embedding-3-small dimension
|
|
9
|
+
constructor(config) {
|
|
10
|
+
this.config = config;
|
|
11
|
+
}
|
|
12
|
+
async initialize() {
|
|
13
|
+
if (!this.config.enabled)
|
|
14
|
+
return;
|
|
15
|
+
if (this.initialized)
|
|
16
|
+
return;
|
|
17
|
+
try {
|
|
18
|
+
const dbPath = path.resolve(process.cwd(), this.config.vectorDbPath);
|
|
19
|
+
if (!fs.existsSync(dbPath)) {
|
|
20
|
+
fs.mkdirSync(dbPath, { recursive: true });
|
|
21
|
+
}
|
|
22
|
+
this.index = new LocalIndex(dbPath);
|
|
23
|
+
if (!(await this.index.isIndexCreated())) {
|
|
24
|
+
await this.index.createIndex();
|
|
25
|
+
}
|
|
26
|
+
this.initialized = true;
|
|
27
|
+
// log(`Vector store initialized at ${dbPath}`, true);
|
|
28
|
+
}
|
|
29
|
+
catch (error) {
|
|
30
|
+
console.error(`Failed to initialize vector store: ${error.message}`);
|
|
31
|
+
this.config.enabled = false; // Disable if init fails
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
async addItem(text, translation, sourceLang, targetLang, embedding, metadata = {}) {
|
|
35
|
+
if (!this.initialized || !this.index)
|
|
36
|
+
return;
|
|
37
|
+
try {
|
|
38
|
+
await this.index.insertItem({
|
|
39
|
+
vector: embedding,
|
|
40
|
+
metadata: {
|
|
41
|
+
text,
|
|
42
|
+
translation,
|
|
43
|
+
sourceLang,
|
|
44
|
+
targetLang,
|
|
45
|
+
...metadata,
|
|
46
|
+
timestamp: Date.now(),
|
|
47
|
+
},
|
|
48
|
+
});
|
|
49
|
+
}
|
|
50
|
+
catch (error) {
|
|
51
|
+
console.warn(`Failed to add item to vector store: ${error.message}`);
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
async search(embedding, targetLang, limit = 3) {
|
|
55
|
+
if (!this.initialized || !this.index)
|
|
56
|
+
return [];
|
|
57
|
+
try {
|
|
58
|
+
// @ts-ignore - Vectra types might be mismatched in this version
|
|
59
|
+
const results = await this.index.queryItems(embedding, limit);
|
|
60
|
+
// Filter by target language and map to result format
|
|
61
|
+
return results
|
|
62
|
+
.filter((result) => result.item.metadata.targetLang === targetLang)
|
|
63
|
+
.map((result) => ({
|
|
64
|
+
item: {
|
|
65
|
+
text: result.item.metadata.text,
|
|
66
|
+
translation: result.item.metadata.translation,
|
|
67
|
+
sourceLang: result.item.metadata.sourceLang,
|
|
68
|
+
targetLang: result.item.metadata.targetLang,
|
|
69
|
+
metadata: result.item.metadata,
|
|
70
|
+
},
|
|
71
|
+
score: result.score,
|
|
72
|
+
}));
|
|
73
|
+
}
|
|
74
|
+
catch (error) {
|
|
75
|
+
console.warn(`Vector search failed: ${error.message}`);
|
|
76
|
+
return [];
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
}
|