@lingo.dev/compiler 0.1.2 → 0.1.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/build/plugin/build-translator.cjs +3 -3
- package/build/plugin/build-translator.mjs +3 -3
- package/build/plugin/build-translator.mjs.map +1 -1
- package/build/plugin/next.cjs +3 -3
- package/build/plugin/next.d.cts.map +1 -1
- package/build/plugin/next.d.mts.map +1 -1
- package/build/plugin/next.mjs +3 -3
- package/build/plugin/next.mjs.map +1 -1
- package/build/plugin/unplugin.cjs +81 -2
- package/build/plugin/unplugin.d.cts.map +1 -1
- package/build/plugin/unplugin.d.mts.map +1 -1
- package/build/plugin/unplugin.mjs +81 -2
- package/build/plugin/unplugin.mjs.map +1 -1
- package/build/react/server/ServerLingoProvider.d.cts +2 -2
- package/build/react/shared/LingoProvider.d.cts +2 -2
- package/build/react/shared/LocaleSwitcher.d.cts +2 -2
- package/build/translation-server/translation-server.cjs +7 -17
- package/build/translation-server/translation-server.mjs +7 -17
- package/build/translation-server/translation-server.mjs.map +1 -1
- package/build/translators/cache-factory.mjs.map +1 -1
- package/build/translators/lingo/model-factory.cjs +5 -10
- package/build/translators/lingo/model-factory.mjs +5 -10
- package/build/translators/lingo/model-factory.mjs.map +1 -1
- package/build/translators/lingo/provider-details.cjs +69 -0
- package/build/translators/lingo/provider-details.mjs +69 -0
- package/build/translators/lingo/provider-details.mjs.map +1 -0
- package/build/translators/lingo/{service.cjs → translator.cjs} +11 -13
- package/build/translators/lingo/{service.mjs → translator.mjs} +12 -14
- package/build/translators/lingo/translator.mjs.map +1 -0
- package/build/translators/memory-cache.cjs +47 -0
- package/build/translators/memory-cache.mjs +47 -0
- package/build/translators/memory-cache.mjs.map +1 -0
- package/build/translators/pluralization/service.cjs +19 -44
- package/build/translators/pluralization/service.mjs +19 -44
- package/build/translators/pluralization/service.mjs.map +1 -1
- package/build/translators/pseudotranslator/index.cjs +2 -10
- package/build/translators/pseudotranslator/index.mjs +2 -10
- package/build/translators/pseudotranslator/index.mjs.map +1 -1
- package/build/translators/translation-service.cjs +55 -57
- package/build/translators/translation-service.mjs +55 -57
- package/build/translators/translation-service.mjs.map +1 -1
- package/build/utils/observability.cjs +84 -0
- package/build/utils/observability.mjs +83 -0
- package/build/utils/observability.mjs.map +1 -0
- package/build/utils/rc.cjs +21 -0
- package/build/utils/rc.mjs +17 -0
- package/build/utils/rc.mjs.map +1 -0
- package/build/utils/repository-id.cjs +64 -0
- package/build/utils/repository-id.mjs +64 -0
- package/build/utils/repository-id.mjs.map +1 -0
- package/build/utils/tracking-events.cjs +28 -0
- package/build/utils/tracking-events.mjs +25 -0
- package/build/utils/tracking-events.mjs.map +1 -0
- package/package.json +12 -8
- package/build/translators/lingo/service.mjs.map +0 -1
- package/build/translators/translator-factory.cjs +0 -49
- package/build/translators/translator-factory.mjs +0 -50
- package/build/translators/translator-factory.mjs.map +0 -1
|
@@ -13,23 +13,17 @@ import { generateText } from "ai";
|
|
|
13
13
|
*/
|
|
14
14
|
var PluralizationService = class {
|
|
15
15
|
languageModel;
|
|
16
|
-
modelName;
|
|
17
16
|
cache = /* @__PURE__ */ new Map();
|
|
18
17
|
prompt;
|
|
19
18
|
sourceLocale;
|
|
20
19
|
constructor(config, logger) {
|
|
21
20
|
this.logger = logger;
|
|
22
21
|
const localeModel = parseModelString(config.model);
|
|
23
|
-
if (!localeModel) throw new Error(`Invalid model format: "${config.model}"`);
|
|
24
|
-
|
|
25
|
-
this.logger.info("Validating API keys for pluralization...");
|
|
26
|
-
const validatedKeys = validateAndGetApiKeys(modelsConfig);
|
|
27
|
-
this.logger.info("✅ API keys validated for pluralization");
|
|
28
|
-
this.languageModel = createAiModel(localeModel, validatedKeys);
|
|
29
|
-
this.modelName = `${localeModel.provider}:${localeModel.name}`;
|
|
22
|
+
if (!localeModel) throw new Error(`Invalid model format in pluralization service: "${config.model}"`);
|
|
23
|
+
this.languageModel = createAiModel(localeModel, validateAndGetApiKeys({ "*:*": config.model }));
|
|
30
24
|
this.sourceLocale = config.sourceLocale;
|
|
31
25
|
this.prompt = getSystemPrompt({ sourceLocale: config.sourceLocale });
|
|
32
|
-
this.logger.
|
|
26
|
+
this.logger.debug(`Initialized pluralization service with ${localeModel.provider}:${localeModel.name}`);
|
|
33
27
|
}
|
|
34
28
|
/**
|
|
35
29
|
* Generate ICU formats for multiple candidates in a single batch
|
|
@@ -39,23 +33,20 @@ var PluralizationService = class {
|
|
|
39
33
|
* @returns Map of hash -> ICU generation result
|
|
40
34
|
*/
|
|
41
35
|
async generateBatch(candidates, batchSize = 10) {
|
|
42
|
-
const results =
|
|
43
|
-
const uncachedCandidates = candidates.filter((c) => {
|
|
36
|
+
const { uncachedCandidates, results } = candidates.reduce((acc, c) => {
|
|
44
37
|
const cached = this.cache.get(c.hash);
|
|
45
|
-
if (cached)
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
38
|
+
if (cached) acc.results.set(c.hash, cached);
|
|
39
|
+
else acc.uncachedCandidates.push(c);
|
|
40
|
+
return acc;
|
|
41
|
+
}, {
|
|
42
|
+
uncachedCandidates: [],
|
|
43
|
+
results: /* @__PURE__ */ new Map()
|
|
50
44
|
});
|
|
51
|
-
if (uncachedCandidates.length === 0)
|
|
52
|
-
|
|
53
|
-
return results;
|
|
54
|
-
}
|
|
55
|
-
this.logger.info(`Processing ${uncachedCandidates.length} candidates (${candidates.length - uncachedCandidates.length} cached)`);
|
|
45
|
+
if (uncachedCandidates.length === 0) return results;
|
|
46
|
+
this.logger.debug(`Processing ${uncachedCandidates.length} candidates (${candidates.length - uncachedCandidates.length} cached)`);
|
|
56
47
|
for (let i = 0; i < uncachedCandidates.length; i += batchSize) {
|
|
57
48
|
const batch = uncachedCandidates.slice(i, i + batchSize);
|
|
58
|
-
this.logger.
|
|
49
|
+
this.logger.debug(`Processing batch ${Math.floor(i / batchSize) + 1}/${Math.ceil(uncachedCandidates.length / batchSize)} (${batch.length} candidates)`);
|
|
59
50
|
const batchResults = await this.processBatch(batch);
|
|
60
51
|
for (const [hash, result] of batchResults) {
|
|
61
52
|
results.set(hash, result);
|
|
@@ -97,7 +88,7 @@ var PluralizationService = class {
|
|
|
97
88
|
content: obj2xml(batchRequest)
|
|
98
89
|
}
|
|
99
90
|
]
|
|
100
|
-
}), DEFAULT_TIMEOUTS.AI_API * 2, `Pluralization with ${this.
|
|
91
|
+
}), DEFAULT_TIMEOUTS.AI_API * 2, `Pluralization with ${this.languageModel}`)).text.trim();
|
|
101
92
|
this.logger.debug(`LLM XML response: ${responseText.substring(0, 200)}...`);
|
|
102
93
|
const parsed = parseXmlFromResponseText(responseText);
|
|
103
94
|
const resultArray = Array.isArray(parsed.results.result) ? parsed.results.result : [parsed.results.result];
|
|
@@ -123,7 +114,7 @@ var PluralizationService = class {
|
|
|
123
114
|
}
|
|
124
115
|
}
|
|
125
116
|
for (const candidate of candidates) if (!results.has(candidate.hash)) {
|
|
126
|
-
this.logger.warn(`No result returned for candidate: ${candidate.sourceText}`);
|
|
117
|
+
this.logger.warn(`No result returned for a candidate: ${candidate.sourceText}`);
|
|
127
118
|
results.set(candidate.hash, {
|
|
128
119
|
success: false,
|
|
129
120
|
error: "No result returned by LLM"
|
|
@@ -163,9 +154,9 @@ var PluralizationService = class {
|
|
|
163
154
|
failed: 0,
|
|
164
155
|
durationMs: 0
|
|
165
156
|
};
|
|
166
|
-
this.logger.
|
|
157
|
+
this.logger.debug(`Starting pluralization processing for ${totalEntries} entries`);
|
|
167
158
|
const candidates = detectPluralCandidates(Object.fromEntries(Object.entries(metadata.entries).map(([hash, entry]) => [hash, entry.sourceText])), this.logger);
|
|
168
|
-
this.logger.
|
|
159
|
+
this.logger.debug(`Found ${candidates.length} plural candidates (${(candidates.length / totalEntries * 100).toFixed(1)}%)`);
|
|
169
160
|
if (candidates.length === 0) return {
|
|
170
161
|
total: totalEntries,
|
|
171
162
|
candidates: 0,
|
|
@@ -209,12 +200,12 @@ var PluralizationService = class {
|
|
|
209
200
|
failed++;
|
|
210
201
|
continue;
|
|
211
202
|
}
|
|
212
|
-
this.logger.
|
|
203
|
+
this.logger.debug(`Pluralizing: "${entry.sourceText}" -> "${result.icuText}"`);
|
|
213
204
|
entry.sourceText = result.icuText;
|
|
214
205
|
pluralized++;
|
|
215
206
|
}
|
|
216
207
|
const duration = performance.now() - startTime;
|
|
217
|
-
this.logger.
|
|
208
|
+
this.logger.debug(`Pluralization completed: ${pluralized} pluralized, ${rejected} rejected, ${failed} failed in ${duration.toFixed(0)}ms`);
|
|
218
209
|
return {
|
|
219
210
|
total: totalEntries,
|
|
220
211
|
candidates: candidates.length,
|
|
@@ -224,22 +215,6 @@ var PluralizationService = class {
|
|
|
224
215
|
durationMs: duration
|
|
225
216
|
};
|
|
226
217
|
}
|
|
227
|
-
/**
|
|
228
|
-
* Clear the cache
|
|
229
|
-
*/
|
|
230
|
-
clearCache() {
|
|
231
|
-
this.cache.clear();
|
|
232
|
-
this.logger.debug("Pluralization cache cleared");
|
|
233
|
-
}
|
|
234
|
-
/**
|
|
235
|
-
* Get cache statistics
|
|
236
|
-
*/
|
|
237
|
-
getCacheStats() {
|
|
238
|
-
return {
|
|
239
|
-
size: this.cache.size,
|
|
240
|
-
hits: 0
|
|
241
|
-
};
|
|
242
|
-
}
|
|
243
218
|
};
|
|
244
219
|
|
|
245
220
|
//#endregion
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"service.mjs","names":["logger: Logger","modelsConfig: Record<string, string>","batchRequest: PluralizationBatch"],"sources":["../../../src/translators/pluralization/service.ts"],"sourcesContent":["/**\n * Pluralization service with batching and caching\n */\n\nimport type { LanguageModel } from \"ai\";\nimport { generateText } from \"ai\";\nimport type {\n ICUGenerationResult,\n PluralCandidate,\n PluralizationBatch,\n PluralizationConfig,\n PluralizationResponse,\n PluralizationStats,\n} from \"./types\";\nimport {\n createAiModel,\n parseModelString,\n validateAndGetApiKeys,\n} from \"../lingo/model-factory\";\nimport { Logger } from \"../../utils/logger\";\nimport { DEFAULT_TIMEOUTS, withTimeout } from \"../../utils/timeout\";\nimport { getSystemPrompt } from \"./prompt\";\nimport { obj2xml, parseXmlFromResponseText } from \"../parse-xml\";\nimport { shots } from \"./shots\";\nimport type { MetadataSchema } from \"../../types\";\nimport { detectPluralCandidates } from \"./pattern-detector\";\nimport { validateICU } from \"./icu-validator\";\n\n/**\n * Pluralization service with batching and model reuse\n */\nexport class PluralizationService {\n private readonly languageModel: LanguageModel;\n private readonly modelName: string;\n private cache = new Map<string, ICUGenerationResult>();\n private readonly prompt: string;\n private readonly sourceLocale: string;\n\n constructor(\n config: PluralizationConfig,\n private logger: Logger,\n ) {\n const localeModel = parseModelString(config.model);\n if (!localeModel) {\n throw new Error(`Invalid model format: \"${config.model}\"`);\n }\n\n // Validate and fetch API keys for the pluralization provider\n // We need to create a models config that validateAndFetchApiKeys can use\n const modelsConfig: Record<string, string> = {\n \"*:*\": config.model, // Single model for pluralization\n };\n\n this.logger.info(\"Validating API keys for pluralization...\");\n const validatedKeys = validateAndGetApiKeys(modelsConfig);\n this.logger.info(\"✅ API keys validated for pluralization\");\n\n this.languageModel = createAiModel(localeModel, validatedKeys);\n this.modelName = `${localeModel.provider}:${localeModel.name}`;\n this.sourceLocale = config.sourceLocale;\n this.prompt = getSystemPrompt({ sourceLocale: config.sourceLocale });\n\n this.logger.info(\n `Initialized pluralization service with ${this.modelName}`,\n );\n }\n\n /**\n * Generate ICU formats for multiple candidates in a single batch\n *\n * @param candidates Array of plural candidates\n * @param batchSize Maximum candidates per batch (default: 10)\n * @returns Map of hash -> ICU generation result\n */\n async generateBatch(\n candidates: PluralCandidate[],\n batchSize: number = 10,\n ): Promise<Map<string, ICUGenerationResult>> {\n const results = new Map<string, ICUGenerationResult>();\n\n // Check cache first\n const uncachedCandidates = candidates.filter((c) => {\n const cached = this.cache.get(c.hash);\n if (cached) {\n results.set(c.hash, cached);\n return false;\n }\n return true;\n });\n\n if (uncachedCandidates.length === 0) {\n this.logger.debug(\n `All ${candidates.length} candidates found in cache, skipping LLM call`,\n );\n return results;\n }\n\n this.logger.info(\n `Processing ${uncachedCandidates.length} candidates (${candidates.length - uncachedCandidates.length} cached)`,\n );\n\n // Process in batches\n for (let i = 0; i < uncachedCandidates.length; i += batchSize) {\n const batch = uncachedCandidates.slice(i, i + batchSize);\n\n this.logger.info(\n `Processing batch ${Math.floor(i / batchSize) + 1}/${Math.ceil(uncachedCandidates.length / batchSize)} (${batch.length} candidates)`,\n );\n\n const batchResults = await this.processBatch(batch);\n\n // Store results and cache them\n for (const [hash, result] of batchResults) {\n results.set(hash, result);\n this.cache.set(hash, result);\n }\n }\n\n return results;\n }\n\n /**\n * Process a single batch of candidates\n */\n private async processBatch(\n candidates: PluralCandidate[],\n ): Promise<Map<string, ICUGenerationResult>> {\n const results = new Map<string, ICUGenerationResult>();\n\n try {\n // Prepare batch request in XML format\n const batchRequest: PluralizationBatch = {\n version: 0.1,\n sourceLocale: this.sourceLocale,\n candidates: {\n candidate: candidates.map((c) => ({\n hash: c.hash,\n text: c.sourceText,\n })),\n },\n };\n\n // Call LLM with XML format and few-shot examples\n const response = await withTimeout(\n generateText({\n model: this.languageModel,\n messages: [\n {\n role: \"system\",\n content: this.prompt,\n },\n // Add few-shot examples\n ...shots.flatMap((shotsTuple) => [\n {\n role: \"user\" as const,\n content: obj2xml(shotsTuple[0]),\n },\n {\n role: \"assistant\" as const,\n content: obj2xml(shotsTuple[1]),\n },\n ]),\n {\n role: \"user\",\n content: obj2xml(batchRequest),\n },\n ],\n }),\n DEFAULT_TIMEOUTS.AI_API * 2, // Double timeout for batch\n `Pluralization with ${this.modelName}`,\n );\n\n const responseText = response.text.trim();\n this.logger.debug(\n `LLM XML response: ${responseText.substring(0, 200)}...`,\n );\n // Parse XML response\n const parsed =\n parseXmlFromResponseText<PluralizationResponse>(responseText);\n\n // Process results\n const resultArray = Array.isArray(parsed.results.result)\n ? parsed.results.result\n : [parsed.results.result];\n\n for (const result of resultArray) {\n const candidate = candidates.find((c) => c.hash === result.hash);\n if (!candidate) {\n this.logger.warn(`No candidate found for hash: ${result.hash}`);\n continue;\n }\n\n if (result.shouldPluralize && result.icuText) {\n this.logger.debug(\n `✓ ICU format generated for \"${candidate.sourceText}\": \"${result.icuText}\"`,\n );\n results.set(result.hash, {\n success: true,\n icuText: result.icuText,\n reasoning: result.reasoning,\n });\n } else {\n this.logger.debug(\n `✗ Pluralization not appropriate for \"${candidate.sourceText}\": ${result.reasoning}`,\n );\n results.set(result.hash, {\n success: false,\n reasoning: result.reasoning,\n });\n }\n }\n\n // Handle missing results (LLM didn't return result for some candidates)\n for (const candidate of candidates) {\n if (!results.has(candidate.hash)) {\n this.logger.warn(\n `No result returned for candidate: ${candidate.sourceText}`,\n );\n results.set(candidate.hash, {\n success: false,\n error: \"No result returned by LLM\",\n });\n }\n }\n } catch (error) {\n const errorMsg = error instanceof Error ? error.message : \"Unknown error\";\n this.logger.error(`Failed to process batch: ${errorMsg}`);\n\n // Mark all candidates as failed\n for (const candidate of candidates) {\n results.set(candidate.hash, {\n success: false,\n error: errorMsg,\n });\n }\n }\n\n return results;\n }\n\n /**\n * Process metadata entries for pluralization\n *\n * This is the main entry point that:\n * 1. Detects plural candidates using pattern matching\n * 2. Generates ICU format using LLM (batched)\n * 3. Validates the ICU format\n * 4. Updates metadata entries in-place (modifies sourceText)\n * 5. Returns statistics\n * @param metadata Metadata schema with translation entries\n\n * @returns Statistics about the pluralization process\n */\n async process(metadata: MetadataSchema): Promise<PluralizationStats> {\n const startTime = performance.now();\n const totalEntries = Object.keys(metadata.entries).length;\n\n if (totalEntries === 0) {\n return {\n total: 0,\n candidates: 0,\n pluralized: 0,\n rejected: 0,\n failed: 0,\n durationMs: 0,\n };\n }\n\n this.logger.info(\n `Starting pluralization processing for ${totalEntries} entries`,\n );\n\n // Step 1: Detect plural candidates using pattern matching\n const entriesMap: Record<string, string> = Object.fromEntries(\n Object.entries(metadata.entries).map(([hash, entry]) => [\n hash,\n entry.sourceText,\n ]),\n );\n\n const candidates = detectPluralCandidates(entriesMap, this.logger);\n\n this.logger.info(\n `Found ${candidates.length} plural candidates (${((candidates.length / totalEntries) * 100).toFixed(1)}%)`,\n );\n\n if (candidates.length === 0) {\n const endTime = performance.now();\n return {\n total: totalEntries,\n candidates: 0,\n pluralized: 0,\n rejected: 0,\n failed: 0,\n durationMs: endTime - startTime,\n };\n }\n\n // Step 2: Generate ICU formats with batching\n this.logger.debug(\"Generating ICU formats with batching...\");\n const icuResults = await this.generateBatch(candidates, 10);\n\n // Step 3: Validate and update metadata entries\n this.logger.debug(\"Validating and updating entries...\");\n let pluralized = 0;\n let rejected = 0;\n let failed = 0;\n\n for (const candidate of candidates) {\n const result = icuResults.get(candidate.hash);\n const entry = metadata.entries[candidate.hash];\n this.logger.debug(`Processing candidate: ${candidate.sourceText}`);\n if (!entry) {\n this.logger.warn(`Entry not found for hash: ${candidate.hash}`);\n failed++;\n continue;\n }\n\n if (!result) {\n this.logger.warn(`No result for hash: ${candidate.hash}`);\n failed++;\n continue;\n }\n\n if (result.error) {\n this.logger.warn(\n `Error generating ICU for \"${candidate.sourceText}\": ${result.error}`,\n );\n failed++;\n continue;\n }\n\n if (!result.success || !result.icuText) {\n this.logger.debug(\n `Rejected pluralization for \"${candidate.sourceText}\": ${result.reasoning}`,\n );\n rejected++;\n continue;\n }\n\n const isValid = validateICU(\n result.icuText,\n candidate.sourceText,\n this.logger,\n );\n\n if (!isValid) {\n this.logger.warn(\n `Invalid ICU format generated for \"${candidate.sourceText}\", falling back to original`,\n );\n failed++;\n continue;\n }\n\n // Update metadata entry in-place\n this.logger.info(\n `Pluralizing: \"${entry.sourceText}\" -> \"${result.icuText}\"`,\n );\n entry.sourceText = result.icuText;\n pluralized++;\n }\n\n const endTime = performance.now();\n const duration = endTime - startTime;\n\n this.logger.info(\n `Pluralization completed: ${pluralized} pluralized, ${rejected} rejected, ${failed} failed in ${duration.toFixed(0)}ms`,\n );\n\n return {\n total: totalEntries,\n candidates: candidates.length,\n pluralized,\n rejected,\n failed,\n durationMs: duration,\n };\n }\n\n /**\n * Clear the cache\n */\n clearCache(): void {\n this.cache.clear();\n this.logger.debug(\"Pluralization cache cleared\");\n }\n\n /**\n * Get cache statistics\n */\n getCacheStats(): { size: number; hits: number } {\n return {\n size: this.cache.size,\n hits: 0, // We don't track hits currently\n };\n }\n}\n"],"mappings":";;;;;;;;;;;;;AA+BA,IAAa,uBAAb,MAAkC;CAChC,AAAiB;CACjB,AAAiB;CACjB,AAAQ,wBAAQ,IAAI,KAAkC;CACtD,AAAiB;CACjB,AAAiB;CAEjB,YACE,QACA,AAAQA,QACR;EADQ;EAER,MAAM,cAAc,iBAAiB,OAAO,MAAM;AAClD,MAAI,CAAC,YACH,OAAM,IAAI,MAAM,0BAA0B,OAAO,MAAM,GAAG;EAK5D,MAAMC,eAAuC,EAC3C,OAAO,OAAO,OACf;AAED,OAAK,OAAO,KAAK,2CAA2C;EAC5D,MAAM,gBAAgB,sBAAsB,aAAa;AACzD,OAAK,OAAO,KAAK,yCAAyC;AAE1D,OAAK,gBAAgB,cAAc,aAAa,cAAc;AAC9D,OAAK,YAAY,GAAG,YAAY,SAAS,GAAG,YAAY;AACxD,OAAK,eAAe,OAAO;AAC3B,OAAK,SAAS,gBAAgB,EAAE,cAAc,OAAO,cAAc,CAAC;AAEpE,OAAK,OAAO,KACV,0CAA0C,KAAK,YAChD;;;;;;;;;CAUH,MAAM,cACJ,YACA,YAAoB,IACuB;EAC3C,MAAM,0BAAU,IAAI,KAAkC;EAGtD,MAAM,qBAAqB,WAAW,QAAQ,MAAM;GAClD,MAAM,SAAS,KAAK,MAAM,IAAI,EAAE,KAAK;AACrC,OAAI,QAAQ;AACV,YAAQ,IAAI,EAAE,MAAM,OAAO;AAC3B,WAAO;;AAET,UAAO;IACP;AAEF,MAAI,mBAAmB,WAAW,GAAG;AACnC,QAAK,OAAO,MACV,OAAO,WAAW,OAAO,+CAC1B;AACD,UAAO;;AAGT,OAAK,OAAO,KACV,cAAc,mBAAmB,OAAO,eAAe,WAAW,SAAS,mBAAmB,OAAO,UACtG;AAGD,OAAK,IAAI,IAAI,GAAG,IAAI,mBAAmB,QAAQ,KAAK,WAAW;GAC7D,MAAM,QAAQ,mBAAmB,MAAM,GAAG,IAAI,UAAU;AAExD,QAAK,OAAO,KACV,oBAAoB,KAAK,MAAM,IAAI,UAAU,GAAG,EAAE,GAAG,KAAK,KAAK,mBAAmB,SAAS,UAAU,CAAC,IAAI,MAAM,OAAO,cACxH;GAED,MAAM,eAAe,MAAM,KAAK,aAAa,MAAM;AAGnD,QAAK,MAAM,CAAC,MAAM,WAAW,cAAc;AACzC,YAAQ,IAAI,MAAM,OAAO;AACzB,SAAK,MAAM,IAAI,MAAM,OAAO;;;AAIhC,SAAO;;;;;CAMT,MAAc,aACZ,YAC2C;EAC3C,MAAM,0BAAU,IAAI,KAAkC;AAEtD,MAAI;GAEF,MAAMC,eAAmC;IACvC,SAAS;IACT,cAAc,KAAK;IACnB,YAAY,EACV,WAAW,WAAW,KAAK,OAAO;KAChC,MAAM,EAAE;KACR,MAAM,EAAE;KACT,EAAE,EACJ;IACF;GAgCD,MAAM,gBA7BW,MAAM,YACrB,aAAa;IACX,OAAO,KAAK;IACZ,UAAU;KACR;MACE,MAAM;MACN,SAAS,KAAK;MACf;KAED,GAAG,MAAM,SAAS,eAAe,CAC/B;MACE,MAAM;MACN,SAAS,QAAQ,WAAW,GAAG;MAChC,EACD;MACE,MAAM;MACN,SAAS,QAAQ,WAAW,GAAG;MAChC,CACF,CAAC;KACF;MACE,MAAM;MACN,SAAS,QAAQ,aAAa;MAC/B;KACF;IACF,CAAC,EACF,iBAAiB,SAAS,GAC1B,sBAAsB,KAAK,YAC5B,EAE6B,KAAK,MAAM;AACzC,QAAK,OAAO,MACV,qBAAqB,aAAa,UAAU,GAAG,IAAI,CAAC,KACrD;GAED,MAAM,SACJ,yBAAgD,aAAa;GAG/D,MAAM,cAAc,MAAM,QAAQ,OAAO,QAAQ,OAAO,GACpD,OAAO,QAAQ,SACf,CAAC,OAAO,QAAQ,OAAO;AAE3B,QAAK,MAAM,UAAU,aAAa;IAChC,MAAM,YAAY,WAAW,MAAM,MAAM,EAAE,SAAS,OAAO,KAAK;AAChE,QAAI,CAAC,WAAW;AACd,UAAK,OAAO,KAAK,gCAAgC,OAAO,OAAO;AAC/D;;AAGF,QAAI,OAAO,mBAAmB,OAAO,SAAS;AAC5C,UAAK,OAAO,MACV,+BAA+B,UAAU,WAAW,MAAM,OAAO,QAAQ,GAC1E;AACD,aAAQ,IAAI,OAAO,MAAM;MACvB,SAAS;MACT,SAAS,OAAO;MAChB,WAAW,OAAO;MACnB,CAAC;WACG;AACL,UAAK,OAAO,MACV,wCAAwC,UAAU,WAAW,KAAK,OAAO,YAC1E;AACD,aAAQ,IAAI,OAAO,MAAM;MACvB,SAAS;MACT,WAAW,OAAO;MACnB,CAAC;;;AAKN,QAAK,MAAM,aAAa,WACtB,KAAI,CAAC,QAAQ,IAAI,UAAU,KAAK,EAAE;AAChC,SAAK,OAAO,KACV,qCAAqC,UAAU,aAChD;AACD,YAAQ,IAAI,UAAU,MAAM;KAC1B,SAAS;KACT,OAAO;KACR,CAAC;;WAGC,OAAO;GACd,MAAM,WAAW,iBAAiB,QAAQ,MAAM,UAAU;AAC1D,QAAK,OAAO,MAAM,4BAA4B,WAAW;AAGzD,QAAK,MAAM,aAAa,WACtB,SAAQ,IAAI,UAAU,MAAM;IAC1B,SAAS;IACT,OAAO;IACR,CAAC;;AAIN,SAAO;;;;;;;;;;;;;;;CAgBT,MAAM,QAAQ,UAAuD;EACnE,MAAM,YAAY,YAAY,KAAK;EACnC,MAAM,eAAe,OAAO,KAAK,SAAS,QAAQ,CAAC;AAEnD,MAAI,iBAAiB,EACnB,QAAO;GACL,OAAO;GACP,YAAY;GACZ,YAAY;GACZ,UAAU;GACV,QAAQ;GACR,YAAY;GACb;AAGH,OAAK,OAAO,KACV,yCAAyC,aAAa,UACvD;EAUD,MAAM,aAAa,uBAPwB,OAAO,YAChD,OAAO,QAAQ,SAAS,QAAQ,CAAC,KAAK,CAAC,MAAM,WAAW,CACtD,MACA,MAAM,WACP,CAAC,CACH,EAEqD,KAAK,OAAO;AAElE,OAAK,OAAO,KACV,SAAS,WAAW,OAAO,uBAAwB,WAAW,SAAS,eAAgB,KAAK,QAAQ,EAAE,CAAC,IACxG;AAED,MAAI,WAAW,WAAW,EAExB,QAAO;GACL,OAAO;GACP,YAAY;GACZ,YAAY;GACZ,UAAU;GACV,QAAQ;GACR,YAPc,YAAY,KAAK,GAOT;GACvB;AAIH,OAAK,OAAO,MAAM,0CAA0C;EAC5D,MAAM,aAAa,MAAM,KAAK,cAAc,YAAY,GAAG;AAG3D,OAAK,OAAO,MAAM,qCAAqC;EACvD,IAAI,aAAa;EACjB,IAAI,WAAW;EACf,IAAI,SAAS;AAEb,OAAK,MAAM,aAAa,YAAY;GAClC,MAAM,SAAS,WAAW,IAAI,UAAU,KAAK;GAC7C,MAAM,QAAQ,SAAS,QAAQ,UAAU;AACzC,QAAK,OAAO,MAAM,yBAAyB,UAAU,aAAa;AAClE,OAAI,CAAC,OAAO;AACV,SAAK,OAAO,KAAK,6BAA6B,UAAU,OAAO;AAC/D;AACA;;AAGF,OAAI,CAAC,QAAQ;AACX,SAAK,OAAO,KAAK,uBAAuB,UAAU,OAAO;AACzD;AACA;;AAGF,OAAI,OAAO,OAAO;AAChB,SAAK,OAAO,KACV,6BAA6B,UAAU,WAAW,KAAK,OAAO,QAC/D;AACD;AACA;;AAGF,OAAI,CAAC,OAAO,WAAW,CAAC,OAAO,SAAS;AACtC,SAAK,OAAO,MACV,+BAA+B,UAAU,WAAW,KAAK,OAAO,YACjE;AACD;AACA;;AASF,OAAI,CANY,YACd,OAAO,SACP,UAAU,YACV,KAAK,OACN,EAEa;AACZ,SAAK,OAAO,KACV,qCAAqC,UAAU,WAAW,6BAC3D;AACD;AACA;;AAIF,QAAK,OAAO,KACV,iBAAiB,MAAM,WAAW,QAAQ,OAAO,QAAQ,GAC1D;AACD,SAAM,aAAa,OAAO;AAC1B;;EAIF,MAAM,WADU,YAAY,KAAK,GACN;AAE3B,OAAK,OAAO,KACV,4BAA4B,WAAW,eAAe,SAAS,aAAa,OAAO,aAAa,SAAS,QAAQ,EAAE,CAAC,IACrH;AAED,SAAO;GACL,OAAO;GACP,YAAY,WAAW;GACvB;GACA;GACA;GACA,YAAY;GACb;;;;;CAMH,aAAmB;AACjB,OAAK,MAAM,OAAO;AAClB,OAAK,OAAO,MAAM,8BAA8B;;;;;CAMlD,gBAAgD;AAC9C,SAAO;GACL,MAAM,KAAK,MAAM;GACjB,MAAM;GACP"}
|
|
1
|
+
{"version":3,"file":"service.mjs","names":["logger: Logger","batchRequest: PluralizationBatch"],"sources":["../../../src/translators/pluralization/service.ts"],"sourcesContent":["/**\n * Pluralization service with batching and caching\n */\n\nimport type { LanguageModel } from \"ai\";\nimport { generateText } from \"ai\";\nimport type {\n ICUGenerationResult,\n PluralCandidate,\n PluralizationBatch,\n PluralizationConfig,\n PluralizationResponse,\n PluralizationStats,\n} from \"./types\";\nimport {\n createAiModel,\n parseModelString,\n validateAndGetApiKeys,\n} from \"../lingo/model-factory\";\nimport { Logger } from \"../../utils/logger\";\nimport { DEFAULT_TIMEOUTS, withTimeout } from \"../../utils/timeout\";\nimport { getSystemPrompt } from \"./prompt\";\nimport { obj2xml, parseXmlFromResponseText } from \"../parse-xml\";\nimport { shots } from \"./shots\";\nimport type { MetadataSchema } from \"../../types\";\nimport { detectPluralCandidates } from \"./pattern-detector\";\nimport { validateICU } from \"./icu-validator\";\n\n/**\n * Pluralization service with batching and model reuse\n */\nexport class PluralizationService {\n private readonly languageModel: LanguageModel;\n private cache = new Map<string, ICUGenerationResult>();\n private readonly prompt: string;\n private readonly sourceLocale: string;\n\n constructor(\n config: PluralizationConfig,\n private logger: Logger,\n ) {\n const localeModel = parseModelString(config.model);\n if (!localeModel) {\n throw new Error(`Invalid model format in pluralization service: \"${config.model}\"`);\n }\n\n // Validate and fetch API keys for the pluralization provider\n // We need to create a models config that validateAndFetchApiKeys can use\n const modelsConfig: Record<string, string> = {\n \"*:*\": config.model,\n };\n\n const validatedKeys = validateAndGetApiKeys(modelsConfig);\n\n this.languageModel = createAiModel(localeModel, validatedKeys);\n this.sourceLocale = config.sourceLocale;\n this.prompt = getSystemPrompt({ sourceLocale: config.sourceLocale });\n\n this.logger.debug(\n `Initialized pluralization service with ${localeModel.provider}:${localeModel.name}`,\n );\n }\n\n /**\n * Generate ICU formats for multiple candidates in a single batch\n *\n * @param candidates Array of plural candidates\n * @param batchSize Maximum candidates per batch (default: 10)\n * @returns Map of hash -> ICU generation result\n */\n async generateBatch(\n candidates: PluralCandidate[],\n batchSize: number = 10,\n ): Promise<Map<string, ICUGenerationResult>> {\n const { uncachedCandidates, results } = candidates.reduce(\n (acc, c) => {\n const cached = this.cache.get(c.hash);\n if (cached) {\n acc.results.set(c.hash, cached);\n } else {\n acc.uncachedCandidates.push(c);\n }\n return acc;\n },\n {\n uncachedCandidates: [] as PluralCandidate[],\n results: new Map<string, ICUGenerationResult>(),\n },\n );\n\n if (uncachedCandidates.length === 0) {\n return results;\n }\n\n this.logger.debug(\n `Processing ${uncachedCandidates.length} candidates (${candidates.length - uncachedCandidates.length} cached)`,\n );\n\n // Process in batches\n for (let i = 0; i < uncachedCandidates.length; i += batchSize) {\n const batch = uncachedCandidates.slice(i, i + batchSize);\n\n this.logger.debug(\n `Processing batch ${Math.floor(i / batchSize) + 1}/${Math.ceil(uncachedCandidates.length / batchSize)} (${batch.length} candidates)`,\n );\n\n const batchResults = await this.processBatch(batch);\n\n // Store results and cache them\n for (const [hash, result] of batchResults) {\n results.set(hash, result);\n this.cache.set(hash, result);\n }\n }\n\n return results;\n }\n\n /**\n * Process a single batch of candidates\n */\n private async processBatch(\n candidates: PluralCandidate[],\n ): Promise<Map<string, ICUGenerationResult>> {\n const results = new Map<string, ICUGenerationResult>();\n\n try {\n // Prepare batch request in XML format\n const batchRequest: PluralizationBatch = {\n version: 0.1,\n sourceLocale: this.sourceLocale,\n candidates: {\n candidate: candidates.map((c) => ({\n hash: c.hash,\n text: c.sourceText,\n })),\n },\n };\n\n // Call LLM with XML format and few-shot examples\n const response = await withTimeout(\n generateText({\n model: this.languageModel,\n messages: [\n {\n role: \"system\",\n content: this.prompt,\n },\n // Add few-shot examples\n ...shots.flatMap((shotsTuple) => [\n {\n role: \"user\" as const,\n content: obj2xml(shotsTuple[0]),\n },\n {\n role: \"assistant\" as const,\n content: obj2xml(shotsTuple[1]),\n },\n ]),\n {\n role: \"user\",\n content: obj2xml(batchRequest),\n },\n ],\n }),\n DEFAULT_TIMEOUTS.AI_API * 2, // Double timeout for batch\n `Pluralization with ${this.languageModel}`,\n );\n\n const responseText = response.text.trim();\n this.logger.debug(\n `LLM XML response: ${responseText.substring(0, 200)}...`,\n );\n // Parse XML response\n const parsed =\n parseXmlFromResponseText<PluralizationResponse>(responseText);\n\n // Process results\n const resultArray = Array.isArray(parsed.results.result)\n ? parsed.results.result\n : [parsed.results.result];\n\n for (const result of resultArray) {\n const candidate = candidates.find((c) => c.hash === result.hash);\n if (!candidate) {\n this.logger.warn(`No candidate found for hash: ${result.hash}`);\n continue;\n }\n\n if (result.shouldPluralize && result.icuText) {\n this.logger.debug(\n `✓ ICU format generated for \"${candidate.sourceText}\": \"${result.icuText}\"`,\n );\n results.set(result.hash, {\n success: true,\n icuText: result.icuText,\n reasoning: result.reasoning,\n });\n } else {\n this.logger.debug(\n `✗ Pluralization not appropriate for \"${candidate.sourceText}\": ${result.reasoning}`,\n );\n results.set(result.hash, {\n success: false,\n reasoning: result.reasoning,\n });\n }\n }\n\n // Handle missing results (LLM didn't return result for some candidates)\n for (const candidate of candidates) {\n if (!results.has(candidate.hash)) {\n this.logger.warn(\n `No result returned for a candidate: ${candidate.sourceText}`,\n );\n results.set(candidate.hash, {\n success: false,\n error: \"No result returned by LLM\",\n });\n }\n }\n } catch (error) {\n const errorMsg = error instanceof Error ? error.message : \"Unknown error\";\n this.logger.error(`Failed to process batch: ${errorMsg}`);\n\n // Mark all candidates as failed\n for (const candidate of candidates) {\n results.set(candidate.hash, {\n success: false,\n error: errorMsg,\n });\n }\n }\n\n return results;\n }\n\n /**\n * Process metadata entries for pluralization\n *\n * This is the main entry point that:\n * 1. Detects plural candidates using pattern matching\n * 2. Generates ICU format using LLM (batched)\n * 3. Validates the ICU format\n * 4. Updates metadata entries in-place (modifies sourceText)\n * 5. Returns statistics\n * @param metadata Metadata schema with translation entries\n\n * @returns Statistics about the pluralization process\n */\n async process(metadata: MetadataSchema): Promise<PluralizationStats> {\n const startTime = performance.now();\n const totalEntries = Object.keys(metadata.entries).length;\n\n if (totalEntries === 0) {\n return {\n total: 0,\n candidates: 0,\n pluralized: 0,\n rejected: 0,\n failed: 0,\n durationMs: 0,\n };\n }\n\n this.logger.debug(\n `Starting pluralization processing for ${totalEntries} entries`,\n );\n\n // Step 1: Detect plural candidates using pattern matching\n const entriesMap: Record<string, string> = Object.fromEntries(\n Object.entries(metadata.entries).map(([hash, entry]) => [\n hash,\n entry.sourceText,\n ]),\n );\n\n const candidates = detectPluralCandidates(entriesMap, this.logger);\n\n this.logger.debug(\n `Found ${candidates.length} plural candidates (${((candidates.length / totalEntries) * 100).toFixed(1)}%)`,\n );\n\n if (candidates.length === 0) {\n const endTime = performance.now();\n return {\n total: totalEntries,\n candidates: 0,\n pluralized: 0,\n rejected: 0,\n failed: 0,\n durationMs: endTime - startTime,\n };\n }\n\n // Step 2: Generate ICU formats with batching\n this.logger.debug(\"Generating ICU formats with batching...\");\n const icuResults = await this.generateBatch(candidates, 10);\n\n // Step 3: Validate and update metadata entries\n this.logger.debug(\"Validating and updating entries...\");\n let pluralized = 0;\n let rejected = 0;\n let failed = 0;\n\n for (const candidate of candidates) {\n const result = icuResults.get(candidate.hash);\n const entry = metadata.entries[candidate.hash];\n this.logger.debug(`Processing candidate: ${candidate.sourceText}`);\n if (!entry) {\n this.logger.warn(`Entry not found for hash: ${candidate.hash}`);\n failed++;\n continue;\n }\n\n if (!result) {\n this.logger.warn(`No result for hash: ${candidate.hash}`);\n failed++;\n continue;\n }\n\n if (result.error) {\n this.logger.warn(\n `Error generating ICU for \"${candidate.sourceText}\": ${result.error}`,\n );\n failed++;\n continue;\n }\n\n if (!result.success || !result.icuText) {\n this.logger.debug(\n `Rejected pluralization for \"${candidate.sourceText}\": ${result.reasoning}`,\n );\n rejected++;\n continue;\n }\n\n const isValid = validateICU(\n result.icuText,\n candidate.sourceText,\n this.logger,\n );\n\n if (!isValid) {\n this.logger.warn(\n `Invalid ICU format generated for \"${candidate.sourceText}\", falling back to original`,\n );\n failed++;\n continue;\n }\n\n this.logger.debug(\n `Pluralizing: \"${entry.sourceText}\" -> \"${result.icuText}\"`,\n );\n entry.sourceText = result.icuText;\n pluralized++;\n }\n\n const endTime = performance.now();\n const duration = endTime - startTime;\n\n this.logger.debug(\n `Pluralization completed: ${pluralized} pluralized, ${rejected} rejected, ${failed} failed in ${duration.toFixed(0)}ms`,\n );\n\n return {\n total: totalEntries,\n candidates: candidates.length,\n pluralized,\n rejected,\n failed,\n durationMs: duration,\n };\n }\n}\n"],"mappings":";;;;;;;;;;;;;AA+BA,IAAa,uBAAb,MAAkC;CAChC,AAAiB;CACjB,AAAQ,wBAAQ,IAAI,KAAkC;CACtD,AAAiB;CACjB,AAAiB;CAEjB,YACE,QACA,AAAQA,QACR;EADQ;EAER,MAAM,cAAc,iBAAiB,OAAO,MAAM;AAClD,MAAI,CAAC,YACH,OAAM,IAAI,MAAM,mDAAmD,OAAO,MAAM,GAAG;AAWrF,OAAK,gBAAgB,cAAc,aAFb,sBAJuB,EAC3C,OAAO,OAAO,OACf,CAEwD,CAEK;AAC9D,OAAK,eAAe,OAAO;AAC3B,OAAK,SAAS,gBAAgB,EAAE,cAAc,OAAO,cAAc,CAAC;AAEpE,OAAK,OAAO,MACV,0CAA0C,YAAY,SAAS,GAAG,YAAY,OAC/E;;;;;;;;;CAUH,MAAM,cACJ,YACA,YAAoB,IACuB;EAC3C,MAAM,EAAE,oBAAoB,YAAY,WAAW,QAChD,KAAK,MAAM;GACV,MAAM,SAAS,KAAK,MAAM,IAAI,EAAE,KAAK;AACrC,OAAI,OACF,KAAI,QAAQ,IAAI,EAAE,MAAM,OAAO;OAE/B,KAAI,mBAAmB,KAAK,EAAE;AAEhC,UAAO;KAET;GACE,oBAAoB,EAAE;GACtB,yBAAS,IAAI,KAAkC;GAChD,CACF;AAED,MAAI,mBAAmB,WAAW,EAChC,QAAO;AAGT,OAAK,OAAO,MACV,cAAc,mBAAmB,OAAO,eAAe,WAAW,SAAS,mBAAmB,OAAO,UACtG;AAGD,OAAK,IAAI,IAAI,GAAG,IAAI,mBAAmB,QAAQ,KAAK,WAAW;GAC7D,MAAM,QAAQ,mBAAmB,MAAM,GAAG,IAAI,UAAU;AAExD,QAAK,OAAO,MACV,oBAAoB,KAAK,MAAM,IAAI,UAAU,GAAG,EAAE,GAAG,KAAK,KAAK,mBAAmB,SAAS,UAAU,CAAC,IAAI,MAAM,OAAO,cACxH;GAED,MAAM,eAAe,MAAM,KAAK,aAAa,MAAM;AAGnD,QAAK,MAAM,CAAC,MAAM,WAAW,cAAc;AACzC,YAAQ,IAAI,MAAM,OAAO;AACzB,SAAK,MAAM,IAAI,MAAM,OAAO;;;AAIhC,SAAO;;;;;CAMT,MAAc,aACZ,YAC2C;EAC3C,MAAM,0BAAU,IAAI,KAAkC;AAEtD,MAAI;GAEF,MAAMC,eAAmC;IACvC,SAAS;IACT,cAAc,KAAK;IACnB,YAAY,EACV,WAAW,WAAW,KAAK,OAAO;KAChC,MAAM,EAAE;KACR,MAAM,EAAE;KACT,EAAE,EACJ;IACF;GAgCD,MAAM,gBA7BW,MAAM,YACrB,aAAa;IACX,OAAO,KAAK;IACZ,UAAU;KACR;MACE,MAAM;MACN,SAAS,KAAK;MACf;KAED,GAAG,MAAM,SAAS,eAAe,CAC/B;MACE,MAAM;MACN,SAAS,QAAQ,WAAW,GAAG;MAChC,EACD;MACE,MAAM;MACN,SAAS,QAAQ,WAAW,GAAG;MAChC,CACF,CAAC;KACF;MACE,MAAM;MACN,SAAS,QAAQ,aAAa;MAC/B;KACF;IACF,CAAC,EACF,iBAAiB,SAAS,GAC1B,sBAAsB,KAAK,gBAC5B,EAE6B,KAAK,MAAM;AACzC,QAAK,OAAO,MACV,qBAAqB,aAAa,UAAU,GAAG,IAAI,CAAC,KACrD;GAED,MAAM,SACJ,yBAAgD,aAAa;GAG/D,MAAM,cAAc,MAAM,QAAQ,OAAO,QAAQ,OAAO,GACpD,OAAO,QAAQ,SACf,CAAC,OAAO,QAAQ,OAAO;AAE3B,QAAK,MAAM,UAAU,aAAa;IAChC,MAAM,YAAY,WAAW,MAAM,MAAM,EAAE,SAAS,OAAO,KAAK;AAChE,QAAI,CAAC,WAAW;AACd,UAAK,OAAO,KAAK,gCAAgC,OAAO,OAAO;AAC/D;;AAGF,QAAI,OAAO,mBAAmB,OAAO,SAAS;AAC5C,UAAK,OAAO,MACV,+BAA+B,UAAU,WAAW,MAAM,OAAO,QAAQ,GAC1E;AACD,aAAQ,IAAI,OAAO,MAAM;MACvB,SAAS;MACT,SAAS,OAAO;MAChB,WAAW,OAAO;MACnB,CAAC;WACG;AACL,UAAK,OAAO,MACV,wCAAwC,UAAU,WAAW,KAAK,OAAO,YAC1E;AACD,aAAQ,IAAI,OAAO,MAAM;MACvB,SAAS;MACT,WAAW,OAAO;MACnB,CAAC;;;AAKN,QAAK,MAAM,aAAa,WACtB,KAAI,CAAC,QAAQ,IAAI,UAAU,KAAK,EAAE;AAChC,SAAK,OAAO,KACV,uCAAuC,UAAU,aAClD;AACD,YAAQ,IAAI,UAAU,MAAM;KAC1B,SAAS;KACT,OAAO;KACR,CAAC;;WAGC,OAAO;GACd,MAAM,WAAW,iBAAiB,QAAQ,MAAM,UAAU;AAC1D,QAAK,OAAO,MAAM,4BAA4B,WAAW;AAGzD,QAAK,MAAM,aAAa,WACtB,SAAQ,IAAI,UAAU,MAAM;IAC1B,SAAS;IACT,OAAO;IACR,CAAC;;AAIN,SAAO;;;;;;;;;;;;;;;CAgBT,MAAM,QAAQ,UAAuD;EACnE,MAAM,YAAY,YAAY,KAAK;EACnC,MAAM,eAAe,OAAO,KAAK,SAAS,QAAQ,CAAC;AAEnD,MAAI,iBAAiB,EACnB,QAAO;GACL,OAAO;GACP,YAAY;GACZ,YAAY;GACZ,UAAU;GACV,QAAQ;GACR,YAAY;GACb;AAGH,OAAK,OAAO,MACV,yCAAyC,aAAa,UACvD;EAUD,MAAM,aAAa,uBAPwB,OAAO,YAChD,OAAO,QAAQ,SAAS,QAAQ,CAAC,KAAK,CAAC,MAAM,WAAW,CACtD,MACA,MAAM,WACP,CAAC,CACH,EAEqD,KAAK,OAAO;AAElE,OAAK,OAAO,MACV,SAAS,WAAW,OAAO,uBAAwB,WAAW,SAAS,eAAgB,KAAK,QAAQ,EAAE,CAAC,IACxG;AAED,MAAI,WAAW,WAAW,EAExB,QAAO;GACL,OAAO;GACP,YAAY;GACZ,YAAY;GACZ,UAAU;GACV,QAAQ;GACR,YAPc,YAAY,KAAK,GAOT;GACvB;AAIH,OAAK,OAAO,MAAM,0CAA0C;EAC5D,MAAM,aAAa,MAAM,KAAK,cAAc,YAAY,GAAG;AAG3D,OAAK,OAAO,MAAM,qCAAqC;EACvD,IAAI,aAAa;EACjB,IAAI,WAAW;EACf,IAAI,SAAS;AAEb,OAAK,MAAM,aAAa,YAAY;GAClC,MAAM,SAAS,WAAW,IAAI,UAAU,KAAK;GAC7C,MAAM,QAAQ,SAAS,QAAQ,UAAU;AACzC,QAAK,OAAO,MAAM,yBAAyB,UAAU,aAAa;AAClE,OAAI,CAAC,OAAO;AACV,SAAK,OAAO,KAAK,6BAA6B,UAAU,OAAO;AAC/D;AACA;;AAGF,OAAI,CAAC,QAAQ;AACX,SAAK,OAAO,KAAK,uBAAuB,UAAU,OAAO;AACzD;AACA;;AAGF,OAAI,OAAO,OAAO;AAChB,SAAK,OAAO,KACV,6BAA6B,UAAU,WAAW,KAAK,OAAO,QAC/D;AACD;AACA;;AAGF,OAAI,CAAC,OAAO,WAAW,CAAC,OAAO,SAAS;AACtC,SAAK,OAAO,MACV,+BAA+B,UAAU,WAAW,KAAK,OAAO,YACjE;AACD;AACA;;AASF,OAAI,CANY,YACd,OAAO,SACP,UAAU,YACV,KAAK,OACN,EAEa;AACZ,SAAK,OAAO,KACV,qCAAqC,UAAU,WAAW,6BAC3D;AACD;AACA;;AAGF,QAAK,OAAO,MACV,iBAAiB,MAAM,WAAW,QAAQ,OAAO,QAAQ,GAC1D;AACD,SAAM,aAAa,OAAO;AAC1B;;EAIF,MAAM,WADU,YAAY,KAAK,GACN;AAE3B,OAAK,OAAO,MACV,4BAA4B,WAAW,eAAe,SAAS,aAAa,OAAO,aAAa,SAAS,QAAQ,EAAE,CAAC,IACrH;AAED,SAAO;GACL,OAAO;GACP,YAAY,WAAW;GACvB;GACA;GACA;GACA,YAAY;GACb"}
|
|
@@ -10,22 +10,14 @@ var PseudoTranslator = class {
|
|
|
10
10
|
this.logger = logger;
|
|
11
11
|
}
|
|
12
12
|
translate(locale, entries) {
|
|
13
|
-
this.logger.debug(`[TRACE-PSEUDO] translate() ENTERED for ${locale} with ${Object.keys(entries).length} entries`);
|
|
14
13
|
const delay = this.config?.delayMedian ?? 0;
|
|
15
14
|
const actualDelay = this.getRandomDelay(delay);
|
|
16
|
-
this.logger.debug(`[TRACE-PSEUDO] Config delay: ${delay}ms, actual delay: ${actualDelay}ms`);
|
|
17
15
|
return new Promise((resolve) => {
|
|
18
|
-
this.logger.debug(`[TRACE-PSEUDO] Promise created, scheduling setTimeout for ${actualDelay}ms`);
|
|
19
16
|
setTimeout(() => {
|
|
20
|
-
|
|
21
|
-
const result = Object.fromEntries(Object.entries(entries).map(([hash, entry]) => {
|
|
17
|
+
resolve(Object.fromEntries(Object.entries(entries).map(([hash, entry]) => {
|
|
22
18
|
return [hash, `${locale}/${pseudolocalize(entry.text)}`];
|
|
23
|
-
}));
|
|
24
|
-
this.logger.debug(`[TRACE-PSEUDO] Pseudolocalization complete, resolving with ${Object.keys(result).length} translations`);
|
|
25
|
-
resolve(result);
|
|
26
|
-
this.logger.debug(`[TRACE-PSEUDO] Promise resolved for ${locale}`);
|
|
19
|
+
})));
|
|
27
20
|
}, actualDelay);
|
|
28
|
-
this.logger.debug(`[TRACE-PSEUDO] setTimeout scheduled, returning promise`);
|
|
29
21
|
});
|
|
30
22
|
}
|
|
31
23
|
getRandomDelay(median) {
|
|
@@ -9,22 +9,14 @@ var PseudoTranslator = class {
|
|
|
9
9
|
this.logger = logger;
|
|
10
10
|
}
|
|
11
11
|
translate(locale, entries) {
|
|
12
|
-
this.logger.debug(`[TRACE-PSEUDO] translate() ENTERED for ${locale} with ${Object.keys(entries).length} entries`);
|
|
13
12
|
const delay = this.config?.delayMedian ?? 0;
|
|
14
13
|
const actualDelay = this.getRandomDelay(delay);
|
|
15
|
-
this.logger.debug(`[TRACE-PSEUDO] Config delay: ${delay}ms, actual delay: ${actualDelay}ms`);
|
|
16
14
|
return new Promise((resolve) => {
|
|
17
|
-
this.logger.debug(`[TRACE-PSEUDO] Promise created, scheduling setTimeout for ${actualDelay}ms`);
|
|
18
15
|
setTimeout(() => {
|
|
19
|
-
|
|
20
|
-
const result = Object.fromEntries(Object.entries(entries).map(([hash, entry]) => {
|
|
16
|
+
resolve(Object.fromEntries(Object.entries(entries).map(([hash, entry]) => {
|
|
21
17
|
return [hash, `${locale}/${pseudolocalize(entry.text)}`];
|
|
22
|
-
}));
|
|
23
|
-
this.logger.debug(`[TRACE-PSEUDO] Pseudolocalization complete, resolving with ${Object.keys(result).length} translations`);
|
|
24
|
-
resolve(result);
|
|
25
|
-
this.logger.debug(`[TRACE-PSEUDO] Promise resolved for ${locale}`);
|
|
18
|
+
})));
|
|
26
19
|
}, actualDelay);
|
|
27
|
-
this.logger.debug(`[TRACE-PSEUDO] setTimeout scheduled, returning promise`);
|
|
28
20
|
});
|
|
29
21
|
}
|
|
30
22
|
getRandomDelay(median) {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.mjs","names":["config: PseudoTranslatorConfig","logger: Logger","PSEUDO_MAP: Record<string, string>","parts: Array<{ text: string; preserve: boolean }>","match: RegExpExecArray | null"],"sources":["../../../src/translators/pseudotranslator/index.ts"],"sourcesContent":["/**\n * Pseudotranslator for testing without actual translation APIs\n */\n\nimport type { TranslatableEntry, Translator } from \"../api\";\nimport { Logger } from \"../../utils/logger\";\nimport type { LocaleCode } from \"lingo.dev/spec\";\n\nexport interface PseudoTranslatorConfig {\n delayMedian?: number;\n}\n\n/**\n * Pseudo-translator that uses pseudolocalization\n * Useful for testing i18n without actual translation APIs\n */\nexport class PseudoTranslator implements Translator<PseudoTranslatorConfig> {\n constructor(\n readonly config: PseudoTranslatorConfig,\n private readonly logger: Logger,\n ) {}\n\n translate(locale: LocaleCode, entries: Record<string, TranslatableEntry>) {\n
|
|
1
|
+
{"version":3,"file":"index.mjs","names":["config: PseudoTranslatorConfig","logger: Logger","PSEUDO_MAP: Record<string, string>","parts: Array<{ text: string; preserve: boolean }>","match: RegExpExecArray | null"],"sources":["../../../src/translators/pseudotranslator/index.ts"],"sourcesContent":["/**\n * Pseudotranslator for testing without actual translation APIs\n */\n\nimport type { TranslatableEntry, Translator } from \"../api\";\nimport { Logger } from \"../../utils/logger\";\nimport type { LocaleCode } from \"lingo.dev/spec\";\n\nexport interface PseudoTranslatorConfig {\n delayMedian?: number;\n}\n\n/**\n * Pseudo-translator that uses pseudolocalization\n * Useful for testing i18n without actual translation APIs\n */\nexport class PseudoTranslator implements Translator<PseudoTranslatorConfig> {\n constructor(\n readonly config: PseudoTranslatorConfig,\n private readonly logger: Logger,\n ) {}\n\n translate(locale: LocaleCode, entries: Record<string, TranslatableEntry>) {\n const delay = this.config?.delayMedian ?? 0;\n const actualDelay = this.getRandomDelay(delay);\n\n return new Promise<Record<string, string>>((resolve) => {\n setTimeout(() => {\n\n const result = Object.fromEntries(\n Object.entries(entries).map(([hash, entry]) => {\n return [hash, `${locale}/${pseudolocalize(entry.text)}`];\n }),\n );\n\n resolve(result);\n }, actualDelay);\n });\n }\n\n private getRandomDelay(median: number): number {\n if (median === 0) return 0;\n // Generate random delay with distribution around median\n // Use a simple approach: median ± 50%\n const min = median * 0.5;\n const max = median * 1.5;\n return Math.floor(Math.random() * (max - min + 1)) + min;\n }\n}\n\n/**\n * Character map for pseudolocalization\n */\nconst PSEUDO_MAP: Record<string, string> = {\n a: \"á\",\n b: \"ḅ\",\n c: \"ç\",\n d: \"ḍ\",\n e: \"é\",\n f: \"ƒ\",\n g: \"ĝ\",\n h: \"ĥ\",\n i: \"í\",\n j: \"ĵ\",\n k: \"ḳ\",\n l: \"ĺ\",\n m: \"ṁ\",\n n: \"ñ\",\n o: \"ó\",\n p: \"ṗ\",\n q: \"ɋ\",\n r: \"ŕ\",\n s: \"ś\",\n t: \"ţ\",\n u: \"ú\",\n v: \"ṿ\",\n w: \"ŵ\",\n x: \"ẋ\",\n y: \"ý\",\n z: \"ẑ\",\n A: \"Á\",\n B: \"Ḅ\",\n C: \"Ç\",\n D: \"Ḍ\",\n E: \"É\",\n F: \"Ƒ\",\n G: \"Ĝ\",\n H: \"Ĥ\",\n I: \"Í\",\n J: \"Ĵ\",\n K: \"Ḳ\",\n L: \"Ĺ\",\n M: \"Ṁ\",\n N: \"Ñ\",\n O: \"Ó\",\n P: \"Ṗ\",\n Q: \"Ɋ\",\n R: \"Ŕ\",\n S: \"Ś\",\n T: \"Ţ\",\n U: \"Ú\",\n V: \"Ṿ\",\n W: \"Ŵ\",\n X: \"Ẋ\",\n Y: \"Ý\",\n Z: \"Ẑ\",\n};\n\n/**\n * Pseudolocalize a string\n * Adds brackets, expands length by ~30%, and uses accented characters\n * Preserves variable placeholders {name} and component tags <a0>, </a0>\n */\nexport function pseudolocalize(text: string): string {\n // Don't pseudolocalize if it's just whitespace or a variable placeholder\n if (!text.trim() || text.match(/^{.*}$/)) {\n return text;\n }\n\n // Regular expression to match patterns we should NOT translate:\n // - Variable placeholders: {varName}\n // - Component tags: <tagName> or </tagName>\n const preserveRegex = /(\\{\\w+}|<\\/?\\w+\\/?>)/g;\n\n // Split text into parts that should be preserved and parts that should be translated\n const parts: Array<{ text: string; preserve: boolean }> = [];\n let lastIndex = 0;\n let match: RegExpExecArray | null;\n\n while ((match = preserveRegex.exec(text)) !== null) {\n // Add text before the match (to be translated)\n if (match.index > lastIndex) {\n parts.push({\n text: text.substring(lastIndex, match.index),\n preserve: false,\n });\n }\n\n // Add the matched pattern (to be preserved)\n parts.push({\n text: match[0],\n preserve: true,\n });\n\n lastIndex = preserveRegex.lastIndex;\n }\n\n // Add remaining text (to be translated)\n if (lastIndex < text.length) {\n parts.push({\n text: text.substring(lastIndex),\n preserve: false,\n });\n }\n\n // Convert characters in translatable parts only\n let result = \"\";\n for (const part of parts) {\n if (part.preserve) {\n // Keep placeholders and tags as-is\n result += part.text;\n } else {\n // Pseudolocalize the text\n for (const char of part.text) {\n result += PSEUDO_MAP[char] || char;\n }\n }\n }\n\n // Add padding to simulate longer translations (~30% longer)\n const padding = \" \".repeat(Math.ceil(text.length * 0.3));\n\n // Wrap in brackets to identify translated strings\n return `${result}${padding}`;\n}\n"],"mappings":";;;;;AAgBA,IAAa,mBAAb,MAA4E;CAC1E,YACE,AAASA,QACT,AAAiBC,QACjB;EAFS;EACQ;;CAGnB,UAAU,QAAoB,SAA4C;EACxE,MAAM,QAAQ,KAAK,QAAQ,eAAe;EAC1C,MAAM,cAAc,KAAK,eAAe,MAAM;AAE9C,SAAO,IAAI,SAAiC,YAAY;AACtD,oBAAiB;AAQf,YANe,OAAO,YACpB,OAAO,QAAQ,QAAQ,CAAC,KAAK,CAAC,MAAM,WAAW;AAC7C,YAAO,CAAC,MAAM,GAAG,OAAO,GAAG,eAAe,MAAM,KAAK,GAAG;MACxD,CACH,CAEc;MACd,YAAY;IACf;;CAGJ,AAAQ,eAAe,QAAwB;AAC7C,MAAI,WAAW,EAAG,QAAO;EAGzB,MAAM,MAAM,SAAS;EACrB,MAAM,MAAM,SAAS;AACrB,SAAO,KAAK,MAAM,KAAK,QAAQ,IAAI,MAAM,MAAM,GAAG,GAAG;;;;;;AAOzD,MAAMC,aAAqC;CACzC,GAAG;CACH,GAAG;CACH,GAAG;CACH,GAAG;CACH,GAAG;CACH,GAAG;CACH,GAAG;CACH,GAAG;CACH,GAAG;CACH,GAAG;CACH,GAAG;CACH,GAAG;CACH,GAAG;CACH,GAAG;CACH,GAAG;CACH,GAAG;CACH,GAAG;CACH,GAAG;CACH,GAAG;CACH,GAAG;CACH,GAAG;CACH,GAAG;CACH,GAAG;CACH,GAAG;CACH,GAAG;CACH,GAAG;CACH,GAAG;CACH,GAAG;CACH,GAAG;CACH,GAAG;CACH,GAAG;CACH,GAAG;CACH,GAAG;CACH,GAAG;CACH,GAAG;CACH,GAAG;CACH,GAAG;CACH,GAAG;CACH,GAAG;CACH,GAAG;CACH,GAAG;CACH,GAAG;CACH,GAAG;CACH,GAAG;CACH,GAAG;CACH,GAAG;CACH,GAAG;CACH,GAAG;CACH,GAAG;CACH,GAAG;CACH,GAAG;CACH,GAAG;CACJ;;;;;;AAOD,SAAgB,eAAe,MAAsB;AAEnD,KAAI,CAAC,KAAK,MAAM,IAAI,KAAK,MAAM,SAAS,CACtC,QAAO;CAMT,MAAM,gBAAgB;CAGtB,MAAMC,QAAoD,EAAE;CAC5D,IAAI,YAAY;CAChB,IAAIC;AAEJ,SAAQ,QAAQ,cAAc,KAAK,KAAK,MAAM,MAAM;AAElD,MAAI,MAAM,QAAQ,UAChB,OAAM,KAAK;GACT,MAAM,KAAK,UAAU,WAAW,MAAM,MAAM;GAC5C,UAAU;GACX,CAAC;AAIJ,QAAM,KAAK;GACT,MAAM,MAAM;GACZ,UAAU;GACX,CAAC;AAEF,cAAY,cAAc;;AAI5B,KAAI,YAAY,KAAK,OACnB,OAAM,KAAK;EACT,MAAM,KAAK,UAAU,UAAU;EAC/B,UAAU;EACX,CAAC;CAIJ,IAAI,SAAS;AACb,MAAK,MAAM,QAAQ,MACjB,KAAI,KAAK,SAEP,WAAU,KAAK;KAGf,MAAK,MAAM,QAAQ,KAAK,KACtB,WAAU,WAAW,SAAS;CAMpC,MAAM,UAAU,IAAI,OAAO,KAAK,KAAK,KAAK,SAAS,GAAI,CAAC;AAGxD,QAAO,GAAG,SAAS"}
|
|
@@ -1,23 +1,44 @@
|
|
|
1
1
|
const require_index = require('./pseudotranslator/index.cjs');
|
|
2
|
+
const require_translator = require('./lingo/translator.cjs');
|
|
2
3
|
const require_service = require('./pluralization/service.cjs');
|
|
4
|
+
const require_cache_factory = require('./cache-factory.cjs');
|
|
5
|
+
const require_memory_cache = require('./memory-cache.cjs');
|
|
3
6
|
|
|
4
7
|
//#region src/translators/translation-service.ts
|
|
5
8
|
var TranslationService = class {
|
|
6
|
-
useCache = true;
|
|
7
9
|
pluralizationService;
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
10
|
+
translator;
|
|
11
|
+
cache;
|
|
12
|
+
constructor(config, logger) {
|
|
11
13
|
this.config = config;
|
|
12
14
|
this.logger = logger;
|
|
13
|
-
const
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
this.
|
|
17
|
-
this.
|
|
15
|
+
const isDev = config.environment === "development";
|
|
16
|
+
if (isDev && config.dev?.usePseudotranslator) {
|
|
17
|
+
this.logger.info("📝 Using pseudotranslator (dev.usePseudotranslator enabled)");
|
|
18
|
+
this.translator = new require_index.PseudoTranslator({ delayMedian: 100 }, logger);
|
|
19
|
+
this.cache = new require_memory_cache.MemoryTranslationCache();
|
|
20
|
+
} else try {
|
|
21
|
+
const models = config.models;
|
|
22
|
+
this.logger.debug(`Creating Lingo translator with models: ${JSON.stringify(models)}`);
|
|
23
|
+
this.cache = require_cache_factory.createCache(config);
|
|
24
|
+
this.translator = new require_translator.LingoTranslator({
|
|
25
|
+
models,
|
|
26
|
+
sourceLocale: config.sourceLocale,
|
|
27
|
+
prompt: config.prompt
|
|
28
|
+
}, this.logger);
|
|
29
|
+
if (this.config.pluralization?.enabled) this.pluralizationService = new require_service.PluralizationService({
|
|
18
30
|
...this.config.pluralization,
|
|
19
31
|
sourceLocale: this.config.sourceLocale
|
|
20
32
|
}, this.logger);
|
|
33
|
+
} catch (error) {
|
|
34
|
+
if (isDev) {
|
|
35
|
+
const errorMsg = error instanceof Error ? error.message : String(error);
|
|
36
|
+
this.logger.warn(`⚠️ Translation setup error: \n${errorMsg}\n
|
|
37
|
+
⚠️ Auto-fallback to pseudotranslator in development mode.
|
|
38
|
+
Set the required API keys for real translations.`);
|
|
39
|
+
this.translator = new require_index.PseudoTranslator({ delayMedian: 100 }, this.logger);
|
|
40
|
+
this.cache = new require_memory_cache.MemoryTranslationCache();
|
|
41
|
+
} else throw error;
|
|
21
42
|
}
|
|
22
43
|
}
|
|
23
44
|
/**
|
|
@@ -31,77 +52,59 @@ var TranslationService = class {
|
|
|
31
52
|
async translate(locale, metadata, requestedHashes) {
|
|
32
53
|
const startTime = performance.now();
|
|
33
54
|
const workingHashes = requestedHashes || Object.keys(metadata.entries);
|
|
34
|
-
this.logger.
|
|
35
|
-
this.
|
|
36
|
-
const cacheStartTime = performance.now();
|
|
37
|
-
const cachedTranslations = this.useCache ? await this.cache.get(locale) : {};
|
|
38
|
-
const cacheEndTime = performance.now();
|
|
39
|
-
this.logger.debug(`[TRACE] Cache check completed in ${(cacheEndTime - cacheStartTime).toFixed(2)}ms, found ${Object.keys(cachedTranslations).length} entries`);
|
|
55
|
+
this.logger.debug(`Translation requested for ${workingHashes.length} hashes in locale: ${locale}`);
|
|
56
|
+
const cachedTranslations = await this.cache.get(locale);
|
|
40
57
|
const uncachedHashes = workingHashes.filter((hash) => !cachedTranslations[hash]);
|
|
41
|
-
this.logger.debug(
|
|
58
|
+
this.logger.debug(`${uncachedHashes.length} hashes need processing, ${workingHashes.length - uncachedHashes.length} are cached`);
|
|
42
59
|
const cachedCount = workingHashes.length - uncachedHashes.length;
|
|
43
|
-
if (uncachedHashes.length === 0) {
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
}
|
|
55
|
-
};
|
|
56
|
-
}
|
|
57
|
-
this.logger.info(`Generating translations for ${uncachedHashes.length} uncached hashes in ${locale}...`);
|
|
60
|
+
if (uncachedHashes.length === 0) return {
|
|
61
|
+
translations: this.pickTranslations(cachedTranslations, workingHashes),
|
|
62
|
+
errors: [],
|
|
63
|
+
stats: {
|
|
64
|
+
total: workingHashes.length,
|
|
65
|
+
cached: cachedCount,
|
|
66
|
+
translated: 0,
|
|
67
|
+
failed: 0
|
|
68
|
+
}
|
|
69
|
+
};
|
|
70
|
+
this.logger.debug(`Generating translations for ${uncachedHashes.length} uncached hashes in ${locale}...`);
|
|
58
71
|
const filteredMetadata = {
|
|
59
72
|
...metadata,
|
|
60
73
|
entries: Object.fromEntries(uncachedHashes.map((hash) => [hash, metadata.entries[hash]]).filter(([_, entry]) => entry !== void 0))
|
|
61
74
|
};
|
|
62
75
|
if (this.pluralizationService) {
|
|
63
|
-
this.logger.
|
|
76
|
+
this.logger.debug(`Processing pluralization for ${Object.keys(filteredMetadata.entries).length} entries...`);
|
|
64
77
|
const pluralStats = await this.pluralizationService.process(filteredMetadata);
|
|
65
|
-
this.logger.
|
|
78
|
+
this.logger.debug(`Pluralization stats: ${pluralStats.pluralized} pluralized, ${pluralStats.rejected} rejected, ${pluralStats.failed} failed`);
|
|
66
79
|
}
|
|
67
80
|
const overriddenTranslations = {};
|
|
68
81
|
const hashesNeedingTranslation = [];
|
|
69
|
-
this.logger.debug(`
|
|
82
|
+
this.logger.debug(`Checking for overrides in ${uncachedHashes.length} entries`);
|
|
70
83
|
for (const hash of uncachedHashes) {
|
|
71
84
|
const entry = filteredMetadata.entries[hash];
|
|
72
85
|
if (!entry) continue;
|
|
73
86
|
if (entry.overrides && entry.overrides[locale]) {
|
|
74
87
|
overriddenTranslations[hash] = entry.overrides[locale];
|
|
75
|
-
this.logger.debug(`
|
|
88
|
+
this.logger.debug(`Using override for ${hash} in locale ${locale}: "${entry.overrides[locale]}"`);
|
|
76
89
|
} else hashesNeedingTranslation.push(hash);
|
|
77
90
|
}
|
|
78
|
-
|
|
79
|
-
if (overrideCount > 0) this.logger.info(`Found ${overrideCount} override(s) for locale ${locale}, skipping AI translation for these entries`);
|
|
80
|
-
this.logger.debug(`[TRACE] Preparing ${hashesNeedingTranslation.length} entries for translation (after overrides)`);
|
|
91
|
+
Object.keys(overriddenTranslations).length;
|
|
81
92
|
const entriesToTranslate = this.prepareEntries(filteredMetadata, hashesNeedingTranslation);
|
|
82
|
-
this.logger.debug(`[TRACE] Prepared ${Object.keys(entriesToTranslate).length} entries`);
|
|
83
93
|
let newTranslations = { ...overriddenTranslations };
|
|
84
94
|
const errors = [];
|
|
85
95
|
if (locale === this.config.sourceLocale) {
|
|
86
|
-
this.logger.debug(`
|
|
96
|
+
this.logger.debug(`Source locale detected, returning sourceText for ${hashesNeedingTranslation.length} entries`);
|
|
87
97
|
for (const [hash, entry] of Object.entries(entriesToTranslate)) newTranslations[hash] = entry.text;
|
|
88
98
|
} else if (Object.keys(entriesToTranslate).length > 0) {
|
|
89
99
|
try {
|
|
90
|
-
this.logger.debug(`
|
|
91
|
-
this.logger.debug(`[TRACE] About to await translator.translate()...`);
|
|
92
|
-
const translateStartTime = performance.now();
|
|
93
|
-
this.logger.debug(`[TRACE] Executing translator.translate() NOW`);
|
|
100
|
+
this.logger.debug(`Translating ${locale} with ${Object.keys(entriesToTranslate).length} entries`);
|
|
94
101
|
const translatedTexts = await this.translator.translate(locale, entriesToTranslate);
|
|
95
|
-
this.logger.debug(`[TRACE] translator.translate() returned`);
|
|
96
102
|
newTranslations = {
|
|
97
103
|
...overriddenTranslations,
|
|
98
104
|
...translatedTexts
|
|
99
105
|
};
|
|
100
|
-
const translateEndTime = performance.now();
|
|
101
|
-
this.logger.debug(`[TRACE] translator.translate() completed in ${(translateEndTime - translateStartTime).toFixed(2)}ms`);
|
|
102
|
-
this.logger.debug(`[TRACE] Received ${Object.keys(translatedTexts).length} translations (+ ${overrideCount} overrides)`);
|
|
103
106
|
} catch (error) {
|
|
104
|
-
this.logger.error(`Translation failed
|
|
107
|
+
this.logger.error(`Translation failed:`, error);
|
|
105
108
|
return {
|
|
106
109
|
translations: this.pickTranslations(cachedTranslations, workingHashes),
|
|
107
110
|
errors: [{
|
|
@@ -122,17 +125,12 @@ var TranslationService = class {
|
|
|
122
125
|
errors.push({
|
|
123
126
|
hash,
|
|
124
127
|
sourceText: entry?.sourceText || "",
|
|
125
|
-
error: "
|
|
128
|
+
error: "Translator doesn't return translation"
|
|
126
129
|
});
|
|
127
130
|
}
|
|
128
131
|
}
|
|
129
|
-
if (
|
|
130
|
-
this.logger.debug(`[TRACE] Updating cache with ${Object.keys(newTranslations).length} translations for ${locale}`);
|
|
131
|
-
const updateStartTime = performance.now();
|
|
132
|
+
if (Object.keys(newTranslations).length > 0) try {
|
|
132
133
|
await this.cache.update(locale, newTranslations);
|
|
133
|
-
const updateEndTime = performance.now();
|
|
134
|
-
this.logger.debug(`[TRACE] Cache update completed in ${(updateEndTime - updateStartTime).toFixed(2)}ms`);
|
|
135
|
-
this.logger.info(`Updated cache with ${Object.keys(newTranslations).length} translations for ${locale}`);
|
|
136
134
|
} catch (error) {
|
|
137
135
|
this.logger.error(`Failed to update cache:`, error);
|
|
138
136
|
}
|
|
@@ -142,7 +140,7 @@ var TranslationService = class {
|
|
|
142
140
|
};
|
|
143
141
|
const result = this.pickTranslations(allTranslations, workingHashes);
|
|
144
142
|
const endTime = performance.now();
|
|
145
|
-
this.logger.
|
|
143
|
+
this.logger.debug(`Translation completed for ${locale}: ${Object.keys(newTranslations).length} new, ${cachedCount} cached, ${errors.length} errors in ${(endTime - startTime).toFixed(2)}ms`);
|
|
146
144
|
return {
|
|
147
145
|
translations: result,
|
|
148
146
|
errors,
|