locai 2.6.2 → 2.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +23 -25
- package/dist/cli/helpers.d.ts +34 -2
- package/dist/cli/helpers.js +96 -37
- package/dist/cli.js +40 -39
- package/dist/commands/review.d.ts +33 -8
- package/dist/commands/review.js +89 -22
- package/dist/commands/translator.d.ts +4 -2
- package/dist/commands/translator.js +1 -1
- package/dist/config/index.d.ts +53 -66
- package/dist/config/index.js +2 -1
- package/dist/config/setup.d.ts +3 -2
- package/dist/config/setup.js +5 -3
- package/dist/core/adapters/android-adapter.d.ts +3 -2
- package/dist/core/adapters/android-adapter.js +12 -13
- package/dist/core/adapters/base-adapter.d.ts +13 -2
- package/dist/core/adapters/ios-adapter.d.ts +3 -2
- package/dist/core/adapters/json-adapter.d.ts +4 -3
- package/dist/core/adapters/json-adapter.js +1 -3
- package/dist/core/adapters/po-adapter.d.ts +3 -2
- package/dist/core/adapters/po-adapter.js +4 -5
- package/dist/core/adapters/properties-adapter.d.ts +3 -2
- package/dist/core/adapters/properties-adapter.js +2 -4
- package/dist/core/adapters/yaml-adapter.d.ts +4 -3
- package/dist/core/adapters/yaml-adapter.js +1 -3
- package/dist/core/context-processor.d.ts +21 -6
- package/dist/core/context-processor.js +2 -7
- package/dist/core/detection/framework-detector.js +1 -1
- package/dist/core/detection/structure-detector.js +1 -1
- package/dist/core/fallback-provider.d.ts +3 -4
- package/dist/core/fallback-provider.js +15 -21
- package/dist/core/orchestrator.d.ts +98 -26
- package/dist/core/orchestrator.js +56 -116
- package/dist/core/pipeline/Pipeline.d.ts +2 -2
- package/dist/core/pipeline/Pipeline.js +1 -3
- package/dist/core/pipeline/context.d.ts +69 -13
- package/dist/core/pipeline/context.js +1 -7
- package/dist/core/pipeline/steps/CacheReadStep.d.ts +10 -7
- package/dist/core/pipeline/steps/CacheReadStep.js +1 -12
- package/dist/core/pipeline/steps/CacheWriteStep.d.ts +2 -2
- package/dist/core/pipeline/steps/CacheWriteStep.js +1 -8
- package/dist/core/pipeline/steps/ConfidenceCheckStep.d.ts +5 -0
- package/dist/core/pipeline/steps/ConfidenceCheckStep.js +24 -44
- package/dist/core/pipeline/steps/ContextEnrichmentStep.js +3 -14
- package/dist/core/pipeline/steps/QualityCheckStep.d.ts +1 -1
- package/dist/core/pipeline/steps/QualityCheckStep.js +0 -2
- package/dist/core/pipeline/steps/ToneCheckStep.js +1 -1
- package/dist/core/pipeline/steps/TranslationStep.d.ts +13 -3
- package/dist/core/pipeline/steps/TranslationStep.js +21 -23
- package/dist/core/pipeline/steps/VectorCacheReadStep.d.ts +9 -1
- package/dist/core/pipeline/steps/VectorCacheReadStep.js +3 -5
- package/dist/core/pipeline/steps/VectorCacheWriteStep.d.ts +2 -1
- package/dist/core/pipeline/steps/VectorCacheWriteStep.js +2 -1
- package/dist/core/provider-factory.d.ts +15 -9
- package/dist/core/provider-factory.js +42 -32
- package/dist/providers/anthropic.d.ts +5 -4
- package/dist/providers/anthropic.js +3 -2
- package/dist/providers/api-types.d.ts +74 -0
- package/dist/providers/api-types.js +5 -0
- package/dist/providers/base-provider.d.ts +27 -31
- package/dist/providers/base-provider.js +22 -46
- package/dist/providers/dashscope.js +0 -2
- package/dist/providers/deepseek.js +0 -1
- package/dist/providers/gemini.js +0 -1
- package/dist/providers/openai.d.ts +3 -3
- package/dist/providers/openai.js +0 -1
- package/dist/providers/xai.js +0 -1
- package/dist/services/embedding-provider.js +4 -2
- package/dist/services/language-processor.d.ts +5 -3
- package/dist/services/language-processor.js +38 -43
- package/dist/services/source-analyzer.d.ts +24 -0
- package/dist/services/source-analyzer.js +161 -68
- package/dist/services/translation-service.d.ts +31 -13
- package/dist/services/translation-service.js +116 -71
- package/dist/services/vector-store.d.ts +15 -2
- package/dist/services/vector-store.js +14 -10
- package/dist/types/index.d.ts +31 -33
- package/dist/utils/ai-context-analyzer.d.ts +12 -3
- package/dist/utils/ai-context-analyzer.js +18 -10
- package/dist/utils/confidence-scorer.d.ts +16 -19
- package/dist/utils/confidence-scorer.js +36 -125
- package/dist/utils/error-helper.d.ts +27 -6
- package/dist/utils/error-helper.js +10 -9
- package/dist/utils/file-manager.d.ts +8 -7
- package/dist/utils/file-manager.js +22 -23
- package/dist/utils/glossary-manager.d.ts +22 -3
- package/dist/utils/glossary-manager.js +47 -28
- package/dist/utils/graceful-shutdown.d.ts +7 -1
- package/dist/utils/graceful-shutdown.js +1 -2
- package/dist/utils/input-validator.d.ts +53 -66
- package/dist/utils/input-validator.js +100 -151
- package/dist/utils/locale-paths.d.ts +7 -0
- package/dist/utils/locale-paths.js +16 -0
- package/dist/utils/logger.d.ts +8 -8
- package/dist/utils/logger.js +7 -15
- package/dist/utils/object-transformer.d.ts +20 -5
- package/dist/utils/object-transformer.js +20 -28
- package/dist/utils/progress-tracker.js +2 -3
- package/dist/utils/prompt-templates.d.ts +45 -5
- package/dist/utils/prompt-templates.js +61 -5
- package/dist/utils/quality/base-checker.d.ts +40 -18
- package/dist/utils/quality/base-checker.js +1 -2
- package/dist/utils/quality/code-block-checker.d.ts +13 -5
- package/dist/utils/quality/index.d.ts +2 -1
- package/dist/utils/quality/index.js +8 -27
- package/dist/utils/quality/length-checker.d.ts +35 -5
- package/dist/utils/quality/length-checker.js +2 -3
- package/dist/utils/quality/markdown-checker.d.ts +14 -7
- package/dist/utils/quality/special-characters-checker.d.ts +18 -1
- package/dist/utils/quality/special-characters-checker.js +0 -1
- package/dist/utils/quality/style-guide-checker.d.ts +1 -1
- package/dist/utils/quality/style-guide-checker.js +0 -2
- package/dist/utils/quality/tone-verifier.js +1 -1
- package/dist/utils/rate-limiter.d.ts +17 -1
- package/dist/utils/rate-limiter.js +12 -11
- package/dist/utils/retry-helper.d.ts +21 -3
- package/dist/utils/retry-helper.js +11 -3
- package/dist/utils/review-queue.d.ts +45 -0
- package/dist/utils/review-queue.js +53 -0
- package/dist/utils/state-manager.d.ts +12 -3
- package/dist/utils/state-manager.js +32 -7
- package/dist/utils/statistics-manager.d.ts +11 -1
- package/dist/utils/statistics-manager.js +8 -0
- package/dist/utils/ui-manager.d.ts +2 -1
- package/dist/utils/ui-manager.js +15 -1
- package/package.json +14 -9
- package/dist/commands/interactive-review.d.ts +0 -9
- package/dist/commands/interactive-review.js +0 -109
- package/dist/core/pipeline/steps/__tests__/ToneCheckStep.test.d.ts +0 -1
- package/dist/core/pipeline/steps/__tests__/ToneCheckStep.test.js +0 -100
- package/dist/utils/quality/__tests__/tone-verifier.test.d.ts +0 -1
- package/dist/utils/quality/__tests__/tone-verifier.test.js +0 -91
|
@@ -7,8 +7,6 @@ import { LRUCache } from "lru-cache";
|
|
|
7
7
|
import os from "os";
|
|
8
8
|
import gracefulShutdown from "../utils/graceful-shutdown.js";
|
|
9
9
|
import GlossaryManager from "../utils/glossary-manager.js";
|
|
10
|
-
import fs from "fs";
|
|
11
|
-
import path from "path";
|
|
12
10
|
import { log } from "../utils/logger.js";
|
|
13
11
|
// Pipeline imports
|
|
14
12
|
import Pipeline from "./pipeline/Pipeline.js";
|
|
@@ -24,6 +22,7 @@ import ConfidenceCheckStep from "./pipeline/steps/ConfidenceCheckStep.js";
|
|
|
24
22
|
import CacheWriteStep from "./pipeline/steps/CacheWriteStep.js";
|
|
25
23
|
import ContextEnrichmentStep from "./pipeline/steps/ContextEnrichmentStep.js";
|
|
26
24
|
import { SourceCodeAnalyzer } from "../services/source-analyzer.js";
|
|
25
|
+
import { updateReviewQueue } from "../utils/review-queue.js";
|
|
27
26
|
// Vector Memory Imports
|
|
28
27
|
import { VectorStore } from "../services/vector-store.js";
|
|
29
28
|
import { EmbeddingProvider } from "../services/embedding-provider.js";
|
|
@@ -36,7 +35,6 @@ class Orchestrator {
|
|
|
36
35
|
qualityChecker;
|
|
37
36
|
advanced;
|
|
38
37
|
glossaryManager;
|
|
39
|
-
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
40
38
|
translationCache;
|
|
41
39
|
cacheStats;
|
|
42
40
|
// Vector Memory
|
|
@@ -87,47 +85,29 @@ class Orchestrator {
|
|
|
87
85
|
max: options.cacheSize || 1000,
|
|
88
86
|
ttl: options.cacheTTL || 1000 * 60 * 60 * 24,
|
|
89
87
|
updateAgeOnGet: options.updateAgeOnGet !== false,
|
|
90
|
-
allowStale: options.allowStaleCache !== false,
|
|
91
|
-
// Simplified fetchMethod for stale-while-revalidate pattern
|
|
92
|
-
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
93
|
-
fetchMethod: async (key, staleValue, { context }) => {
|
|
94
|
-
if (staleValue && options.staleWhileRevalidate !== false) {
|
|
95
|
-
// Background refresh using a mini-pipeline or direct logic
|
|
96
|
-
this._refreshCacheEntry(key, context).catch((err) => {
|
|
97
|
-
console.warn(`Cache refresh failed for key ${key}: ${err.message}`);
|
|
98
|
-
});
|
|
99
|
-
return staleValue;
|
|
100
|
-
}
|
|
101
|
-
return null;
|
|
102
|
-
},
|
|
103
88
|
});
|
|
104
89
|
this.cacheStats = {
|
|
105
90
|
hits: 0,
|
|
106
91
|
misses: 0,
|
|
107
|
-
staleHits: 0,
|
|
108
92
|
stored: 0,
|
|
109
|
-
refreshes: 0,
|
|
110
93
|
};
|
|
111
|
-
// Confidence scoring
|
|
94
|
+
// Confidence scoring: turned on by `confidenceScoring.enabled` or by the top-level
|
|
95
|
+
// minConfidence / saveReviewQueue (the --min-confidence / --save-review-queue flags)
|
|
96
|
+
const scoring = options.confidenceScoring;
|
|
97
|
+
const sectionReviewThresholds = [scoring?.minConfidence, scoring?.reviewThreshold].filter((threshold) => threshold !== undefined);
|
|
112
98
|
this.confidenceSettings = {
|
|
113
99
|
enabled: options.minConfidence !== undefined ||
|
|
114
|
-
options.saveReviewQueue ||
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
saveReviewQueue: options.saveReviewQueue || options.confidenceScoring?.saveReviewQueue || false,
|
|
100
|
+
Boolean(options.saveReviewQueue || scoring?.enabled),
|
|
101
|
+
autoApproveThreshold: scoring?.autoApproveThreshold ?? 0.9,
|
|
102
|
+
reviewThreshold: options.minConfidence ??
|
|
103
|
+
(sectionReviewThresholds.length > 0 ? Math.max(...sectionReviewThresholds) : 0.7),
|
|
104
|
+
rejectThreshold: scoring?.rejectThreshold ?? 0.5,
|
|
105
|
+
saveReviewQueue: Boolean(options.saveReviewQueue || scoring?.saveReviewQueue),
|
|
121
106
|
reviewQueue: [],
|
|
107
|
+
scored: [],
|
|
122
108
|
autoApprovedCount: 0,
|
|
123
109
|
rejectedCount: 0,
|
|
124
110
|
};
|
|
125
|
-
// Use minConfidence from options if present as specific threshold logic override or mapping
|
|
126
|
-
if (options.minConfidence) {
|
|
127
|
-
// This logic was slightly fuzzy in JS, preserved here:
|
|
128
|
-
// If minConfidence is set, it might override others or act as a floor.
|
|
129
|
-
// The original code used it.
|
|
130
|
-
}
|
|
131
111
|
this.concurrencyLimit = options.concurrencyLimit || 5;
|
|
132
112
|
if (this.advanced.autoOptimize) {
|
|
133
113
|
this._applyAutoOptimizations();
|
|
@@ -223,7 +203,7 @@ class Orchestrator {
|
|
|
223
203
|
enabled: this.options.styleGuide?.enforceTone,
|
|
224
204
|
debug: this.advanced.debug,
|
|
225
205
|
}));
|
|
226
|
-
// 7. Confidence
|
|
206
|
+
// 7. Confidence thresholds (scores are computed by TranslationStep)
|
|
227
207
|
this.pipeline.use(new ConfidenceCheckStep(this.confidenceSettings, this.advanced.debug));
|
|
228
208
|
// 8. Cache Write
|
|
229
209
|
if (this.translationCache) {
|
|
@@ -232,15 +212,37 @@ class Orchestrator {
|
|
|
232
212
|
}));
|
|
233
213
|
}
|
|
234
214
|
}
|
|
235
|
-
async processTranslation(key, text, targetLang,
|
|
236
|
-
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
237
|
-
contextData,
|
|
238
|
-
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
239
|
-
existingTranslation) {
|
|
215
|
+
async processTranslation(key, text, targetLang, contextData, existingTranslation) {
|
|
240
216
|
const context = createTranslationContext(key, text, this.options.source, targetLang, this.options, contextData, existingTranslation);
|
|
241
217
|
await this.pipeline.execute(context);
|
|
218
|
+
this._recordConfidence(context);
|
|
242
219
|
return context.result;
|
|
243
220
|
}
|
|
221
|
+
/** Count a scored translation and queue it when it needs review or was rejected. */
|
|
222
|
+
_recordConfidence({ key, sourceText, sourceLang, targetLang, meta, result, }) {
|
|
223
|
+
const settings = this.confidenceSettings;
|
|
224
|
+
if (!settings.enabled || !result.confidence) {
|
|
225
|
+
return;
|
|
226
|
+
}
|
|
227
|
+
settings.scored.push({ language: targetLang, key });
|
|
228
|
+
if (result.autoApproved)
|
|
229
|
+
settings.autoApprovedCount++;
|
|
230
|
+
if (result.rejected)
|
|
231
|
+
settings.rejectedCount++;
|
|
232
|
+
if ((result.needsReview || result.rejected) && result.translated) {
|
|
233
|
+
settings.reviewQueue.push({
|
|
234
|
+
key,
|
|
235
|
+
language: targetLang,
|
|
236
|
+
source: sourceText,
|
|
237
|
+
translation: result.translated,
|
|
238
|
+
confidence: result.confidence,
|
|
239
|
+
sourceLang,
|
|
240
|
+
category: meta.category || "general",
|
|
241
|
+
rejected: result.rejected === true,
|
|
242
|
+
timestamp: new Date().toISOString(),
|
|
243
|
+
});
|
|
244
|
+
}
|
|
245
|
+
}
|
|
244
246
|
async processTranslations(items) {
|
|
245
247
|
this.progress.start(items.length, items[0].targetLang);
|
|
246
248
|
const limit = pLimit(this.concurrencyLimit);
|
|
@@ -249,17 +251,16 @@ class Orchestrator {
|
|
|
249
251
|
if (this.advanced.debug) {
|
|
250
252
|
log(`Processing ${items.length} items with concurrency limit ${this.concurrencyLimit}`, true);
|
|
251
253
|
}
|
|
252
|
-
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
253
254
|
const allPromises = [];
|
|
254
255
|
for (const chunk of chunks) {
|
|
255
256
|
const texts = chunk.map((item) => item.text);
|
|
256
|
-
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
257
257
|
let contextResults = [];
|
|
258
258
|
try {
|
|
259
259
|
contextResults = await this.contextProcessor.analyzeBatch(texts);
|
|
260
260
|
}
|
|
261
261
|
catch (error) {
|
|
262
|
-
|
|
262
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
263
|
+
console.warn(`Context analysis failed for batch: ${message}`);
|
|
263
264
|
}
|
|
264
265
|
for (let i = 0; i < chunk.length; i++) {
|
|
265
266
|
const item = chunk[i];
|
|
@@ -275,7 +276,7 @@ class Orchestrator {
|
|
|
275
276
|
return {
|
|
276
277
|
key: item.key,
|
|
277
278
|
translated: item.text,
|
|
278
|
-
error: error.message,
|
|
279
|
+
error: error instanceof Error ? error.message : String(error),
|
|
279
280
|
success: false,
|
|
280
281
|
context: {},
|
|
281
282
|
};
|
|
@@ -288,9 +289,7 @@ class Orchestrator {
|
|
|
288
289
|
return processedResults;
|
|
289
290
|
}
|
|
290
291
|
// Helper methods
|
|
291
|
-
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
292
292
|
_chunkArray(array, chunkSize) {
|
|
293
|
-
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
294
293
|
const chunks = [];
|
|
295
294
|
for (let i = 0; i < array.length; i += chunkSize) {
|
|
296
295
|
chunks.push(array.slice(i, i + chunkSize));
|
|
@@ -316,7 +315,8 @@ class Orchestrator {
|
|
|
316
315
|
}
|
|
317
316
|
}
|
|
318
317
|
catch (error) {
|
|
319
|
-
|
|
318
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
319
|
+
console.warn("Failed to auto-optimize settings:", message);
|
|
320
320
|
}
|
|
321
321
|
}
|
|
322
322
|
clearCache() {
|
|
@@ -329,12 +329,9 @@ class Orchestrator {
|
|
|
329
329
|
this.cacheStats = {
|
|
330
330
|
hits: 0,
|
|
331
331
|
misses: 0,
|
|
332
|
-
staleHits: 0,
|
|
333
332
|
stored: 0,
|
|
334
|
-
refreshes: 0,
|
|
335
333
|
};
|
|
336
334
|
}
|
|
337
|
-
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
338
335
|
getCacheStats() {
|
|
339
336
|
if (!this.translationCache)
|
|
340
337
|
return {};
|
|
@@ -345,30 +342,6 @@ class Orchestrator {
|
|
|
345
342
|
hitRate: this.cacheStats.hits / (this.cacheStats.hits + this.cacheStats.misses) || 0,
|
|
346
343
|
};
|
|
347
344
|
}
|
|
348
|
-
// Used by background revalidation.
|
|
349
|
-
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
350
|
-
async _refreshCacheEntry(key, context) {
|
|
351
|
-
if (!context || !context.text || !context.targetLang) {
|
|
352
|
-
return;
|
|
353
|
-
}
|
|
354
|
-
try {
|
|
355
|
-
// Use the full pipeline for refresh, but skip cache read to force re-translation
|
|
356
|
-
// pass specific options merged with global options
|
|
357
|
-
const refreshOptions = {
|
|
358
|
-
...this.options,
|
|
359
|
-
skipCache: true,
|
|
360
|
-
staleWhileRevalidate: false, // Prevent infinite recursion matching this condition
|
|
361
|
-
};
|
|
362
|
-
const ctx = createTranslationContext(key, context.text, this.options.source, context.targetLang, refreshOptions, context.contextData);
|
|
363
|
-
await this.pipeline.execute(ctx);
|
|
364
|
-
// Stats are handled within steps (TranslationStep adds confidence, CacheWriteStep updates cache & stats)
|
|
365
|
-
this.cacheStats.refreshes++;
|
|
366
|
-
}
|
|
367
|
-
catch (error) {
|
|
368
|
-
console.warn(`Failed to refresh cache entry: ${error.message}`);
|
|
369
|
-
}
|
|
370
|
-
}
|
|
371
|
-
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
372
345
|
getStatus() {
|
|
373
346
|
return {
|
|
374
347
|
cache: this.getCacheStats(),
|
|
@@ -388,58 +361,25 @@ class Orchestrator {
|
|
|
388
361
|
: null,
|
|
389
362
|
};
|
|
390
363
|
}
|
|
364
|
+
/**
|
|
365
|
+
* Add the translations this orchestrator flagged to `.localize-cache/review-queue.json`,
|
|
366
|
+
* when saving the review queue is enabled.
|
|
367
|
+
*/
|
|
391
368
|
saveReviewQueue() {
|
|
392
|
-
if (!this.confidenceSettings.saveReviewQueue
|
|
393
|
-
this.confidenceSettings.reviewQueue.length === 0) {
|
|
369
|
+
if (!this.confidenceSettings.saveReviewQueue) {
|
|
394
370
|
return;
|
|
395
371
|
}
|
|
396
372
|
try {
|
|
397
|
-
const
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
const reviewFile = path.join(cacheDir, "review-queue.json");
|
|
402
|
-
const data = {
|
|
403
|
-
timestamp: new Date().toISOString(),
|
|
404
|
-
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
405
|
-
minConfidence: this.options.minConfidence,
|
|
406
|
-
items: this.confidenceSettings.reviewQueue,
|
|
407
|
-
stats: {
|
|
408
|
-
total: this.confidenceSettings.reviewQueue.length,
|
|
409
|
-
byLanguage: this._groupByLanguage(this.confidenceSettings.reviewQueue),
|
|
410
|
-
byConfidenceLevel: this._groupByConfidenceLevel(this.confidenceSettings.reviewQueue),
|
|
411
|
-
},
|
|
412
|
-
};
|
|
413
|
-
fs.writeFileSync(reviewFile, JSON.stringify(data, null, 2));
|
|
414
|
-
console.log(`\nReview queue saved: ${this.confidenceSettings.reviewQueue.length} items need review`);
|
|
415
|
-
console.log(` File: ${reviewFile}`);
|
|
416
|
-
console.log(` Run 'localize review' to start interactive review\n`);
|
|
373
|
+
const { file, size } = updateReviewQueue(this.confidenceSettings.scored, this.confidenceSettings.reviewQueue);
|
|
374
|
+
console.log(`\nReview queue saved: ${size} items need review`);
|
|
375
|
+
console.log(` File: ${file}`);
|
|
376
|
+
console.log(` Run 'locai review' to start interactive review\n`);
|
|
417
377
|
}
|
|
418
378
|
catch (error) {
|
|
419
|
-
|
|
379
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
380
|
+
console.warn(`Warning: Failed to save review queue: ${message}`);
|
|
420
381
|
}
|
|
421
382
|
}
|
|
422
|
-
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
423
|
-
_groupByLanguage(items) {
|
|
424
|
-
const grouped = {};
|
|
425
|
-
items.forEach((item) => {
|
|
426
|
-
if (!grouped[item.language]) {
|
|
427
|
-
grouped[item.language] = 0;
|
|
428
|
-
}
|
|
429
|
-
grouped[item.language]++;
|
|
430
|
-
});
|
|
431
|
-
return grouped;
|
|
432
|
-
}
|
|
433
|
-
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
434
|
-
_groupByConfidenceLevel(items) {
|
|
435
|
-
const grouped = { high: 0, medium: 0, low: 0, very_low: 0 };
|
|
436
|
-
items.forEach((item) => {
|
|
437
|
-
if (grouped[item.confidence.level] !== undefined) {
|
|
438
|
-
grouped[item.confidence.level]++;
|
|
439
|
-
}
|
|
440
|
-
});
|
|
441
|
-
return grouped;
|
|
442
|
-
}
|
|
443
383
|
destroy() {
|
|
444
384
|
if (this.translationCache) {
|
|
445
385
|
this.translationCache.clear();
|
|
@@ -5,14 +5,14 @@ import { TranslationContext } from "./context.js";
|
|
|
5
5
|
* Allows composing a sequence of steps that can process a context object.
|
|
6
6
|
*/
|
|
7
7
|
declare class Pipeline {
|
|
8
|
-
steps:
|
|
8
|
+
steps: PipelineStep[];
|
|
9
9
|
constructor();
|
|
10
10
|
/**
|
|
11
11
|
* Add a step to the pipeline.
|
|
12
12
|
* @param step - A step instance with an execute(context, next) method.
|
|
13
13
|
* @returns - Returns self for chaining.
|
|
14
14
|
*/
|
|
15
|
-
use(step: PipelineStep
|
|
15
|
+
use(step: PipelineStep): Pipeline;
|
|
16
16
|
/**
|
|
17
17
|
* Execute the pipeline with the given context.
|
|
18
18
|
* @param context - The context object to be processed.
|
|
@@ -3,8 +3,7 @@
|
|
|
3
3
|
* Allows composing a sequence of steps that can process a context object.
|
|
4
4
|
*/
|
|
5
5
|
class Pipeline {
|
|
6
|
-
|
|
7
|
-
steps = []; // allow loose typing initially for step inputs, but ideally PipelineStep
|
|
6
|
+
steps = [];
|
|
8
7
|
constructor() {
|
|
9
8
|
this.steps = [];
|
|
10
9
|
}
|
|
@@ -13,7 +12,6 @@ class Pipeline {
|
|
|
13
12
|
* @param step - A step instance with an execute(context, next) method.
|
|
14
13
|
* @returns - Returns self for chaining.
|
|
15
14
|
*/
|
|
16
|
-
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
17
15
|
use(step) {
|
|
18
16
|
if (!step || typeof step.execute !== "function") {
|
|
19
17
|
throw new Error("Pipeline step must implement an execute(context, next) method.");
|
|
@@ -1,29 +1,85 @@
|
|
|
1
|
+
import type { CodeContext } from "../../types/context.js";
|
|
2
|
+
import type { AnalysisResult } from "../context-processor.js";
|
|
3
|
+
import type { OrchestratorOptions } from "../orchestrator.js";
|
|
4
|
+
import type { ConfidenceResult } from "../../utils/confidence-scorer.js";
|
|
5
|
+
import type { ProtectedTerm } from "../../utils/glossary-manager.js";
|
|
6
|
+
import type { FlatTranslationValue } from "../../utils/object-transformer.js";
|
|
7
|
+
import type { FixResult, QualityIssue } from "../../utils/quality/base-checker.js";
|
|
8
|
+
/** A stored translation of similar source text, found in vector memory. */
|
|
9
|
+
export interface SimilarTranslation {
|
|
10
|
+
source: string;
|
|
11
|
+
target: string;
|
|
12
|
+
similarity: number;
|
|
13
|
+
}
|
|
14
|
+
/**
|
|
15
|
+
* What is known about the text being translated: its detected category (see
|
|
16
|
+
* ContextProcessor) plus what the pipeline steps add. Forwarded to the provider as
|
|
17
|
+
* `detectedContext` and returned as the result's `context`.
|
|
18
|
+
*/
|
|
19
|
+
export interface TranslationMeta extends Partial<AnalysisResult> {
|
|
20
|
+
/** Where the key is used in the project's source code (ContextEnrichmentStep). */
|
|
21
|
+
code?: CodeContext;
|
|
22
|
+
/** Embedding of the source text, computed by VectorCacheReadStep and reused by VectorCacheWriteStep. */
|
|
23
|
+
embedding?: number[];
|
|
24
|
+
/** Close vector-memory match, offered to the provider as a reference translation. */
|
|
25
|
+
similarTranslation?: SimilarTranslation;
|
|
26
|
+
/** Set on results served directly from vector memory. */
|
|
27
|
+
matchType?: "vector-exact";
|
|
28
|
+
similarity?: number;
|
|
29
|
+
}
|
|
30
|
+
/** Tone mismatch reported by ToneCheckStep. */
|
|
31
|
+
export interface ToneIssue {
|
|
32
|
+
type: "tone_mismatch";
|
|
33
|
+
message: string;
|
|
34
|
+
severity: "warning";
|
|
35
|
+
suggestion?: string;
|
|
36
|
+
score: number;
|
|
37
|
+
}
|
|
38
|
+
/** Quality findings of a translation: the quality checker's fix result and any tone issues. */
|
|
39
|
+
export interface QualityReport extends Partial<Omit<FixResult, "issues">> {
|
|
40
|
+
issues: (QualityIssue | ToneIssue)[];
|
|
41
|
+
}
|
|
1
42
|
export interface TranslationResult {
|
|
2
43
|
key: string;
|
|
3
44
|
translated: string | null;
|
|
4
45
|
success: boolean;
|
|
5
|
-
context:
|
|
46
|
+
context: TranslationMeta;
|
|
6
47
|
error?: string;
|
|
7
48
|
fromCache?: boolean;
|
|
8
|
-
qualityChecks?:
|
|
9
|
-
|
|
49
|
+
qualityChecks?: QualityReport;
|
|
50
|
+
/** Set on results served directly from vector memory. */
|
|
51
|
+
meta?: {
|
|
52
|
+
vectorMatch: boolean;
|
|
53
|
+
similarity: number;
|
|
54
|
+
};
|
|
55
|
+
confidence?: ConfidenceResult;
|
|
56
|
+
autoApproved?: boolean;
|
|
57
|
+
rejected?: boolean;
|
|
58
|
+
rejectionReason?: string;
|
|
59
|
+
needsReview?: boolean;
|
|
10
60
|
}
|
|
61
|
+
/** Options visible to the pipeline steps: the orchestrator options. */
|
|
62
|
+
export type PipelineOptions = OrchestratorOptions;
|
|
63
|
+
/** State shared by the pipeline steps while one key is translated. */
|
|
11
64
|
export interface TranslationContext {
|
|
12
65
|
key: string;
|
|
13
66
|
sourceText: string;
|
|
14
67
|
sourceLang: string;
|
|
15
68
|
targetLang: string;
|
|
16
|
-
options:
|
|
17
|
-
meta:
|
|
18
|
-
|
|
69
|
+
options: PipelineOptions;
|
|
70
|
+
meta: TranslationMeta;
|
|
71
|
+
/** Current value of the key in the target file, or null when there is none. */
|
|
72
|
+
existingTranslation: FlatTranslationValue;
|
|
19
73
|
result: TranslationResult;
|
|
74
|
+
/** Cache key computed by CacheReadStep, used by CacheWriteStep. */
|
|
75
|
+
cacheKey?: string;
|
|
76
|
+
/** Source text with glossary terms replaced by placeholder tokens (GlossaryPreStep). */
|
|
20
77
|
protectedText?: string;
|
|
21
|
-
|
|
78
|
+
/** Placeholder token → protected glossary term, used by GlossaryPostStep. */
|
|
79
|
+
termMap?: Map<string, ProtectedTerm>;
|
|
22
80
|
translatedText?: string;
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
qualityChecks?: any;
|
|
27
|
-
[key: string]: any;
|
|
81
|
+
/** Confidence score of the translation, when the provider supports scoring. */
|
|
82
|
+
confidence?: ConfidenceResult;
|
|
83
|
+
qualityResult?: QualityReport;
|
|
28
84
|
}
|
|
29
|
-
export declare const createTranslationContext: (key: string, sourceText: string, sourceLang: string, targetLang: string, options:
|
|
85
|
+
export declare const createTranslationContext: (key: string, sourceText: string, sourceLang: string, targetLang: string, options: PipelineOptions, meta?: TranslationMeta, existingTranslation?: FlatTranslationValue) => TranslationContext;
|
|
@@ -1,10 +1,4 @@
|
|
|
1
|
-
export const createTranslationContext = (key, sourceText, sourceLang, targetLang,
|
|
2
|
-
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
3
|
-
options,
|
|
4
|
-
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
5
|
-
meta = {},
|
|
6
|
-
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
7
|
-
existingTranslation = null) => ({
|
|
1
|
+
export const createTranslationContext = (key, sourceText, sourceLang, targetLang, options, meta = {}, existingTranslation = null) => ({
|
|
8
2
|
key,
|
|
9
3
|
sourceText,
|
|
10
4
|
sourceLang,
|
|
@@ -1,24 +1,27 @@
|
|
|
1
1
|
import PipelineStep from "../PipelineStep.js";
|
|
2
|
-
import { TranslationContext } from "../context.js";
|
|
2
|
+
import { TranslationContext, TranslationResult } from "../context.js";
|
|
3
|
+
/** Translation cache counters, shared by the orchestrator and the cache steps. */
|
|
3
4
|
export interface CacheStats {
|
|
4
5
|
hits: number;
|
|
5
|
-
staleHits: number;
|
|
6
6
|
misses: number;
|
|
7
7
|
stored: number;
|
|
8
|
-
[key: string]: any;
|
|
9
8
|
}
|
|
9
|
+
/** The part of the translation cache (an LRUCache) the cache steps use. */
|
|
10
10
|
export interface Cache {
|
|
11
11
|
has(key: string): boolean;
|
|
12
|
-
get(key: string):
|
|
13
|
-
set(key: string, value:
|
|
12
|
+
get(key: string): TranslationResult | undefined;
|
|
13
|
+
set(key: string, value: TranslationResult): void;
|
|
14
14
|
delete(key: string): void;
|
|
15
|
-
|
|
15
|
+
}
|
|
16
|
+
export interface CacheStepOptions {
|
|
17
|
+
/** Set to false to bypass the cache (default enabled). */
|
|
18
|
+
enabled?: boolean;
|
|
16
19
|
}
|
|
17
20
|
declare class CacheReadStep extends PipelineStep {
|
|
18
21
|
private cache;
|
|
19
22
|
private stats;
|
|
20
23
|
private options;
|
|
21
|
-
constructor(cache: Cache, cacheStats: CacheStats, options?:
|
|
24
|
+
constructor(cache: Cache, cacheStats: CacheStats, options?: CacheStepOptions);
|
|
22
25
|
execute(context: TranslationContext, next: () => Promise<void>): Promise<void>;
|
|
23
26
|
_generateCacheKey(text: string, targetLang: string, category?: string): string;
|
|
24
27
|
}
|
|
@@ -3,7 +3,6 @@ import crypto from "crypto";
|
|
|
3
3
|
class CacheReadStep extends PipelineStep {
|
|
4
4
|
cache;
|
|
5
5
|
stats;
|
|
6
|
-
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
7
6
|
options;
|
|
8
7
|
constructor(cache, cacheStats, options = {}) {
|
|
9
8
|
super();
|
|
@@ -12,7 +11,7 @@ class CacheReadStep extends PipelineStep {
|
|
|
12
11
|
this.options = options;
|
|
13
12
|
}
|
|
14
13
|
async execute(context, next) {
|
|
15
|
-
if (this.options.enabled === false
|
|
14
|
+
if (this.options.enabled === false) {
|
|
16
15
|
await next();
|
|
17
16
|
return;
|
|
18
17
|
}
|
|
@@ -21,16 +20,6 @@ class CacheReadStep extends PipelineStep {
|
|
|
21
20
|
const cachedResult = this.cache.get(cacheKey);
|
|
22
21
|
if (cachedResult && typeof cachedResult === "object") {
|
|
23
22
|
this.stats.hits++;
|
|
24
|
-
// Check for stale hit
|
|
25
|
-
try {
|
|
26
|
-
const ttl = this.cache.getRemainingTTL?.(cacheKey);
|
|
27
|
-
if (typeof ttl === "number" && ttl <= 0) {
|
|
28
|
-
this.stats.staleHits++;
|
|
29
|
-
}
|
|
30
|
-
}
|
|
31
|
-
catch (_error) {
|
|
32
|
-
// Ignore cache errors
|
|
33
|
-
}
|
|
34
23
|
// Hydrate context result from cache
|
|
35
24
|
context.result = {
|
|
36
25
|
...cachedResult,
|
|
@@ -1,11 +1,11 @@
|
|
|
1
1
|
import PipelineStep from "../PipelineStep.js";
|
|
2
2
|
import { TranslationContext } from "../context.js";
|
|
3
|
-
import { Cache, CacheStats } from "./CacheReadStep.js";
|
|
3
|
+
import { Cache, CacheStats, CacheStepOptions } from "./CacheReadStep.js";
|
|
4
4
|
declare class CacheWriteStep extends PipelineStep {
|
|
5
5
|
private cache;
|
|
6
6
|
private stats;
|
|
7
7
|
private options;
|
|
8
|
-
constructor(cache: Cache, cacheStats: CacheStats, options?:
|
|
8
|
+
constructor(cache: Cache, cacheStats: CacheStats, options?: CacheStepOptions);
|
|
9
9
|
execute(context: TranslationContext, next: () => Promise<void>): Promise<void>;
|
|
10
10
|
}
|
|
11
11
|
export default CacheWriteStep;
|
|
@@ -2,7 +2,6 @@ import PipelineStep from "../PipelineStep.js";
|
|
|
2
2
|
class CacheWriteStep extends PipelineStep {
|
|
3
3
|
cache;
|
|
4
4
|
stats;
|
|
5
|
-
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
6
5
|
options;
|
|
7
6
|
constructor(cache, cacheStats, options = {}) {
|
|
8
7
|
super();
|
|
@@ -17,13 +16,7 @@ class CacheWriteStep extends PipelineStep {
|
|
|
17
16
|
context.result.success &&
|
|
18
17
|
!context.result.fromCache &&
|
|
19
18
|
context.cacheKey) {
|
|
20
|
-
this.cache.set(context.cacheKey, context.result
|
|
21
|
-
context: {
|
|
22
|
-
text: context.sourceText,
|
|
23
|
-
targetLang: context.targetLang,
|
|
24
|
-
contextData: context.meta,
|
|
25
|
-
},
|
|
26
|
-
});
|
|
19
|
+
this.cache.set(context.cacheKey, context.result);
|
|
27
20
|
this.stats.stored++;
|
|
28
21
|
}
|
|
29
22
|
await next();
|
|
@@ -1,6 +1,11 @@
|
|
|
1
1
|
import PipelineStep from "../PipelineStep.js";
|
|
2
2
|
import { TranslationContext } from "../context.js";
|
|
3
3
|
import { ConfidenceSettings } from "./TranslationStep.js";
|
|
4
|
+
/**
|
|
5
|
+
* Classifies a scored translation by the confidence thresholds: below rejectThreshold it is
|
|
6
|
+
* rejected (the result fails, so the translation is not written), below reviewThreshold it
|
|
7
|
+
* needs review, at or above autoApproveThreshold it is auto-approved.
|
|
8
|
+
*/
|
|
4
9
|
declare class ConfidenceCheckStep extends PipelineStep {
|
|
5
10
|
private confidenceSettings;
|
|
6
11
|
private debug;
|
|
@@ -1,5 +1,10 @@
|
|
|
1
1
|
import PipelineStep from "../PipelineStep.js";
|
|
2
2
|
import { log } from "../../../utils/logger.js";
|
|
3
|
+
/**
|
|
4
|
+
* Classifies a scored translation by the confidence thresholds: below rejectThreshold it is
|
|
5
|
+
* rejected (the result fails, so the translation is not written), below reviewThreshold it
|
|
6
|
+
* needs review, at or above autoApproveThreshold it is auto-approved.
|
|
7
|
+
*/
|
|
3
8
|
class ConfidenceCheckStep extends PipelineStep {
|
|
4
9
|
confidenceSettings;
|
|
5
10
|
debug;
|
|
@@ -9,57 +14,32 @@ class ConfidenceCheckStep extends PipelineStep {
|
|
|
9
14
|
this.debug = debug;
|
|
10
15
|
}
|
|
11
16
|
async execute(context, next) {
|
|
12
|
-
|
|
13
|
-
if (!
|
|
17
|
+
const { confidence, result } = context;
|
|
18
|
+
if (!this.confidenceSettings.enabled || !confidence || !result.success) {
|
|
14
19
|
await next();
|
|
15
20
|
return;
|
|
16
21
|
}
|
|
17
|
-
|
|
18
|
-
const
|
|
19
|
-
const result = context.result;
|
|
22
|
+
const { autoApproveThreshold = 0.9, reviewThreshold = 0.7, rejectThreshold = 0.5, } = this.confidenceSettings;
|
|
23
|
+
const score = confidence.score;
|
|
20
24
|
result.confidence = confidence;
|
|
21
|
-
|
|
22
|
-
if (
|
|
23
|
-
|
|
24
|
-
result.autoApproved = true;
|
|
25
|
-
if (this.confidenceSettings.autoApprovedCount !== undefined) {
|
|
26
|
-
this.confidenceSettings.autoApprovedCount++;
|
|
27
|
-
}
|
|
28
|
-
if (this.debug) {
|
|
29
|
-
log(`Auto-approved: ${context.key} (score: ${confidence.score.toFixed(3)})`, true);
|
|
30
|
-
}
|
|
31
|
-
}
|
|
32
|
-
// Auto-reject if below threshold
|
|
33
|
-
else if (this.confidenceSettings.rejectThreshold !== undefined &&
|
|
34
|
-
confidence.score < this.confidenceSettings.rejectThreshold) {
|
|
25
|
+
let outcome = "Accepted";
|
|
26
|
+
if (score < rejectThreshold) {
|
|
27
|
+
outcome = "Rejected";
|
|
35
28
|
result.rejected = true;
|
|
36
|
-
result.rejectionReason = `
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
}
|
|
40
|
-
if (this.debug) {
|
|
41
|
-
log(`Auto-rejected: ${context.key} (score: ${confidence.score.toFixed(3)})`, true);
|
|
42
|
-
}
|
|
43
|
-
// Keep original text for rejected translations
|
|
44
|
-
result.translated = context.sourceText;
|
|
29
|
+
result.rejectionReason = `Confidence ${score.toFixed(3)} is below the reject threshold ${rejectThreshold}`;
|
|
30
|
+
result.error = result.rejectionReason;
|
|
31
|
+
result.success = false;
|
|
45
32
|
}
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
this.confidenceSettings.reviewThreshold !== undefined &&
|
|
49
|
-
confidence.score < this.confidenceSettings.reviewThreshold) {
|
|
33
|
+
else if (score < reviewThreshold) {
|
|
34
|
+
outcome = "Needs review";
|
|
50
35
|
result.needsReview = true;
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
sourceLang: context.options.source,
|
|
59
|
-
category: context.meta?.category || "general",
|
|
60
|
-
timestamp: new Date().toISOString(),
|
|
61
|
-
});
|
|
62
|
-
}
|
|
36
|
+
}
|
|
37
|
+
else if (score >= autoApproveThreshold) {
|
|
38
|
+
outcome = "Auto-approved";
|
|
39
|
+
result.autoApproved = true;
|
|
40
|
+
}
|
|
41
|
+
if (this.debug) {
|
|
42
|
+
log(`${outcome}: ${context.key} (score: ${score.toFixed(3)})`, true);
|
|
63
43
|
}
|
|
64
44
|
await next();
|
|
65
45
|
}
|