@babelize/engine 0.1.0 → 0.1.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/analysis/analyze.js +1 -1
- package/dist/detection/stack.d.ts +42 -0
- package/dist/detection/stack.js +277 -0
- package/dist/django/catalog.d.ts +92 -0
- package/dist/django/catalog.js +677 -0
- package/dist/django/extract.d.ts +20 -0
- package/dist/django/extract.js +79 -0
- package/dist/django/inject-file.d.ts +6 -0
- package/dist/django/inject-file.js +52 -0
- package/dist/django/inject.d.ts +11 -0
- package/dist/django/inject.js +205 -0
- package/dist/django/keys.d.ts +21 -0
- package/dist/django/keys.js +63 -0
- package/dist/django/locale-names.d.ts +13 -0
- package/dist/django/locale-names.js +60 -0
- package/dist/django/mo-file.d.ts +6 -0
- package/dist/django/mo-file.js +60 -0
- package/dist/django/plural-forms.d.ts +34 -0
- package/dist/django/plural-forms.js +216 -0
- package/dist/django/po-file.d.ts +38 -0
- package/dist/django/po-file.js +273 -0
- package/dist/django/python-extract.d.ts +34 -0
- package/dist/django/python-extract.js +619 -0
- package/dist/django/python-lexer.d.ts +65 -0
- package/dist/django/python-lexer.js +398 -0
- package/dist/django/template-extract.d.ts +32 -0
- package/dist/django/template-extract.js +680 -0
- package/dist/django/types.d.ts +49 -0
- package/dist/django/types.js +15 -0
- package/dist/index.d.ts +5 -0
- package/dist/index.js +16 -2
- package/dist/local/pseudo-provider.js +5 -4
- package/dist/local/translate-core.d.ts +5 -2
- package/dist/local/translate-core.js +43 -5
- package/dist/local/translate-messages.js +7 -9
- package/dist/modules/extractor/extract.js +22 -14
- package/dist/modules/finalization/django-finalize.d.ts +15 -0
- package/dist/modules/finalization/django-finalize.js +156 -19
- package/dist/modules/finalization/finalize.d.ts +2 -0
- package/dist/modules/finalization/finalize.js +58 -30
- package/dist/modules/finalization/nextjs-finalize.d.ts +4 -1
- package/dist/modules/finalization/nextjs-finalize.js +8 -9
- package/dist/modules/finalization/writes.d.ts +29 -0
- package/dist/modules/finalization/writes.js +78 -0
- package/dist/modules/format-injection/index.js +3 -2
- package/dist/modules/glossary/index.d.ts +1 -2
- package/dist/modules/glossary/index.js +14 -6
- package/dist/modules/glossary/rules.d.ts +81 -0
- package/dist/modules/glossary/rules.js +244 -0
- package/dist/modules/injection/inject.js +14 -30
- package/dist/modules/keygen/keygen.js +7 -0
- package/dist/modules/locale-builder/locales.js +23 -0
- package/dist/modules/lockfile/index.d.ts +1 -1
- package/dist/modules/lockfile/index.js +2 -1
- package/dist/modules/lockfile/lockfile-manager.d.ts +9 -0
- package/dist/modules/lockfile/lockfile-manager.js +16 -1
- package/dist/modules/lockfile/schema.d.ts +42 -0
- package/dist/modules/lockfile/schema.js +11 -0
- package/dist/modules/translator/icu-branches.d.ts +138 -0
- package/dist/modules/translator/icu-branches.js +393 -0
- package/dist/modules/translator/translate.d.ts +40 -4
- package/dist/modules/translator/translate.js +297 -173
- package/dist/modules/workspace/validate.js +67 -32
- package/dist/orchestrator.d.ts +5 -0
- package/dist/orchestrator.js +125 -14
- package/dist/providers/types.d.ts +10 -2
- package/dist/qa/check-project.d.ts +2 -1
- package/dist/qa/check-project.js +50 -7
- package/dist/qa/gettext-files.d.ts +15 -0
- package/dist/qa/gettext-files.js +115 -0
- package/dist/qa/locale-qa.js +20 -3
- package/dist/runtime/config.js +1 -0
- package/dist/runtime/types.d.ts +64 -0
- package/dist/utils/extraction-options.d.ts +9 -3
- package/dist/utils/extraction-options.js +13 -3
- package/dist/utils/file-utils.js +7 -2
- package/dist/utils/icu-message.d.ts +2 -2
- package/dist/utils/icu-message.js +43 -7
- package/dist/utils/jsx-message.js +7 -0
- package/dist/utils/metadata.d.ts +4 -2
- package/dist/utils/metadata.js +16 -10
- package/dist/utils/not-injected.js +2 -0
- package/dist/utils/placeholder-protection.d.ts +18 -1
- package/dist/utils/placeholder-protection.js +58 -13
- package/dist/utils/translation-sanitizer.d.ts +1 -0
- package/dist/utils/translation-sanitizer.js +23 -9
- package/dist/utils/translator-load-balancer.js +5 -1
- package/dist/utils/untranslated.d.ts +15 -0
- package/dist/utils/untranslated.js +34 -0
- package/package.json +2 -6
- package/dist/injection/django/django-template-injector.d.ts +0 -29
- package/dist/injection/django/django-template-injector.js +0 -216
- package/dist/modules/glossary/glossary.d.ts +0 -9
- package/dist/modules/glossary/glossary.js +0 -27
- package/dist/modules/glossary/masker.d.ts +0 -23
- package/dist/modules/glossary/masker.js +0 -56
- package/dist/modules/locale-builder/po.d.ts +0 -33
- package/dist/modules/locale-builder/po.js +0 -131
- package/dist/modules/translator/translator.d.ts +0 -5
- package/dist/modules/translator/translator.js +0 -46
- package/dist/utils/dtl-mask.d.ts +0 -25
- package/dist/utils/dtl-mask.js +0 -58
- package/dist/utils/py-extractor.d.ts +0 -14
- package/dist/utils/py-extractor.js +0 -285
|
@@ -43,27 +43,46 @@ const fast_glob_1 = __importDefault(require("fast-glob"));
|
|
|
43
43
|
const types_1 = require("@babelize/types");
|
|
44
44
|
const logger_1 = require("../../utils/logger");
|
|
45
45
|
const file_utils_1 = require("../../utils/file-utils");
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
46
|
+
const stack_1 = require("../../detection/stack");
|
|
47
|
+
const django_finalize_1 = require("../finalization/django-finalize");
|
|
48
|
+
const locale_names_1 = require("../../django/locale-names");
|
|
49
|
+
const PY_IGNORE = ['**/node_modules/**', '**/.venv/**', '**/venv/**', '**/site-packages/**', '**/.git/**'];
|
|
50
|
+
/**
|
|
51
|
+
* The Django project root (directory of the shallowest manage.py), relative
|
|
52
|
+
* to the workspace, or null when this isn't a Django project. A root-level
|
|
53
|
+
* package.json with a supported JS framework wins over a nested manage.py.
|
|
54
|
+
*/
|
|
55
|
+
async function detectDjangoRoot(workspacePath, rootJsFramework) {
|
|
56
|
+
if (await (0, file_utils_1.pathExists)(path.join(workspacePath, 'manage.py')))
|
|
57
|
+
return '.';
|
|
58
|
+
if (rootJsFramework !== 'unknown')
|
|
59
|
+
return null;
|
|
60
|
+
const manage = await (0, fast_glob_1.default)('**/manage.py', { cwd: workspacePath, onlyFiles: true, deep: 4, ignore: PY_IGNORE });
|
|
61
|
+
if (manage.length > 0) {
|
|
62
|
+
const shallowest = manage.sort((a, b) => a.split('/').length - b.split('/').length || a.localeCompare(b))[0];
|
|
63
|
+
return path.posix.dirname(shallowest) || '.';
|
|
51
64
|
}
|
|
52
|
-
|
|
53
|
-
const
|
|
54
|
-
for (const file of configFiles) {
|
|
65
|
+
const depFiles = ['pyproject.toml', 'requirements.txt', 'Pipfile', 'setup.py', 'setup.cfg'];
|
|
66
|
+
for (const file of depFiles) {
|
|
55
67
|
const content = await (0, file_utils_1.readFile)(path.join(workspacePath, file)).catch(() => '');
|
|
56
|
-
if (content &&
|
|
57
|
-
return '
|
|
58
|
-
}
|
|
59
|
-
}
|
|
60
|
-
// Check for settings.py
|
|
61
|
-
const settingsFiles = await (0, fast_glob_1.default)('**/settings.py', { cwd: workspacePath, onlyFiles: true, deep: 4 });
|
|
62
|
-
if (settingsFiles.length > 0) {
|
|
63
|
-
return 'django';
|
|
68
|
+
if (content && (0, stack_1.mentionsPythonPackage)(content, 'django'))
|
|
69
|
+
return '.';
|
|
64
70
|
}
|
|
65
71
|
return null;
|
|
66
72
|
}
|
|
73
|
+
const SUPPORTED_LIST = 'React, Next.js, Vue, Django and plain HTML/CSS/JS';
|
|
74
|
+
/** The clearest "not supported" message for a workspace we couldn't classify. */
|
|
75
|
+
async function unsupportedMessage(workspacePath, fallback) {
|
|
76
|
+
try {
|
|
77
|
+
const stack = await (0, stack_1.detectStack)(await (0, stack_1.localRepoFiles)(workspacePath));
|
|
78
|
+
if (stack.unsupported)
|
|
79
|
+
return stack.message;
|
|
80
|
+
}
|
|
81
|
+
catch {
|
|
82
|
+
// fall through
|
|
83
|
+
}
|
|
84
|
+
return fallback;
|
|
85
|
+
}
|
|
67
86
|
async function validateWorkspace(config) {
|
|
68
87
|
const { workspacePath } = config;
|
|
69
88
|
if (!(await (0, file_utils_1.pathExists)(workspacePath))) {
|
|
@@ -80,14 +99,18 @@ async function validateWorkspace(config) {
|
|
|
80
99
|
framework = config.framework;
|
|
81
100
|
logger_1.engineLogger.info('Using explicit framework override', { framework });
|
|
82
101
|
}
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
102
|
+
let djangoRoot = null;
|
|
103
|
+
if (!framework || framework === 'unknown' || framework === 'django') {
|
|
104
|
+
djangoRoot = await detectDjangoRoot(workspacePath, packageJson ? detectFramework(packageJson) : 'unknown');
|
|
105
|
+
if (djangoRoot !== null) {
|
|
106
|
+
framework = 'django';
|
|
107
|
+
logger_1.engineLogger.info('Detected Django project', { djangoRoot });
|
|
108
|
+
}
|
|
109
|
+
else if (framework === 'django') {
|
|
110
|
+
throw new Error('This project was set up as Django, but no manage.py or Django dependency was found.');
|
|
88
111
|
}
|
|
89
112
|
}
|
|
90
|
-
if (!packageJson) {
|
|
113
|
+
if (!packageJson && framework !== 'django') {
|
|
91
114
|
const htmlFiles = await (0, fast_glob_1.default)('**/*.html', { cwd: workspacePath, onlyFiles: true, deep: 6 });
|
|
92
115
|
if (htmlFiles.length > 0) {
|
|
93
116
|
// Check for server-template markers
|
|
@@ -108,11 +131,10 @@ async function validateWorkspace(config) {
|
|
|
108
131
|
}
|
|
109
132
|
}
|
|
110
133
|
else if (framework === 'unknown') {
|
|
111
|
-
throw new Error(
|
|
112
|
-
'No package.json or HTML files found.');
|
|
134
|
+
throw new Error(await unsupportedMessage(workspacePath, `Babelize supports ${SUPPORTED_LIST} projects. No package.json, manage.py or HTML files were found.`));
|
|
113
135
|
}
|
|
114
136
|
}
|
|
115
|
-
else if (framework === 'unknown') {
|
|
137
|
+
else if (framework === 'unknown' && packageJson) {
|
|
116
138
|
framework = detectFramework(packageJson);
|
|
117
139
|
}
|
|
118
140
|
if (framework === 'unknown' && !isStatic) {
|
|
@@ -156,10 +178,16 @@ async function validateWorkspace(config) {
|
|
|
156
178
|
}
|
|
157
179
|
}
|
|
158
180
|
if (!types_1.SUPPORTED_FRAMEWORKS.includes(framework)) {
|
|
159
|
-
throw new Error(
|
|
160
|
-
|
|
181
|
+
throw new Error(await unsupportedMessage(workspacePath, `Babelize supports ${SUPPORTED_LIST} projects. This project's framework isn't supported yet.`));
|
|
182
|
+
}
|
|
183
|
+
if (framework === 'static') {
|
|
184
|
+
// A stray HTML file in a Rails/Laravel/Flutter repo doesn't make it a static site.
|
|
185
|
+
const stack = await (0, stack_1.detectStack)(await (0, stack_1.localRepoFiles)(workspacePath)).catch(() => null);
|
|
186
|
+
if (stack?.unsupported)
|
|
187
|
+
throw new Error(stack.message);
|
|
161
188
|
}
|
|
162
|
-
|
|
189
|
+
// Django: the whole project is scanned (templates/ and apps live anywhere).
|
|
190
|
+
const sourceDirs = framework === 'django' ? ['.'] : await findSourceDirectories(workspacePath);
|
|
163
191
|
if (sourceDirs.length === 0) {
|
|
164
192
|
logger_1.engineLogger.info('No standard source directories found, defaulting to root');
|
|
165
193
|
sourceDirs.push('.');
|
|
@@ -168,16 +196,18 @@ async function validateWorkspace(config) {
|
|
|
168
196
|
const localeFormat = frameworkConfig.localeFormat;
|
|
169
197
|
const existingLocales = [];
|
|
170
198
|
let hasLocalesDir = false;
|
|
199
|
+
let localeDirRel = '';
|
|
171
200
|
if (localeFormat === 'po') {
|
|
172
|
-
// Django:
|
|
173
|
-
|
|
201
|
+
// Django: <LOCALE_PATHS[0]>/<lang>/LC_MESSAGES/django.po (default <BASE_DIR>/locale)
|
|
202
|
+
localeDirRel = await (0, django_finalize_1.resolveDjangoLocaleDir)(workspacePath, djangoRoot ?? '.');
|
|
203
|
+
const djangoLocaleDir = path.join(workspacePath, localeDirRel);
|
|
174
204
|
hasLocalesDir = await (0, file_utils_1.isDirectory)(djangoLocaleDir);
|
|
175
205
|
if (hasLocalesDir) {
|
|
176
206
|
const langDirs = await (0, file_utils_1.readDir)(djangoLocaleDir);
|
|
177
207
|
for (const langDir of langDirs) {
|
|
178
208
|
const poFile = path.join(djangoLocaleDir, langDir, 'LC_MESSAGES', 'django.po');
|
|
179
209
|
if (await (0, file_utils_1.pathExists)(poFile)) {
|
|
180
|
-
existingLocales.push(langDir);
|
|
210
|
+
existingLocales.push((0, locale_names_1.localeFromDjangoName)(langDir));
|
|
181
211
|
}
|
|
182
212
|
}
|
|
183
213
|
}
|
|
@@ -208,6 +238,10 @@ async function validateWorkspace(config) {
|
|
|
208
238
|
sourceDirs.push('.');
|
|
209
239
|
publicDir = '.';
|
|
210
240
|
}
|
|
241
|
+
if (localeFormat === 'po') {
|
|
242
|
+
// For gettext projects publicDir is the locale directory.
|
|
243
|
+
publicDir = localeDirRel;
|
|
244
|
+
}
|
|
211
245
|
const babelizeDir = path.join(workspacePath, '.babelize');
|
|
212
246
|
await (0, file_utils_1.ensureDir)(babelizeDir);
|
|
213
247
|
const workspace = {
|
|
@@ -218,8 +252,9 @@ async function validateWorkspace(config) {
|
|
|
218
252
|
hasLocalesDir,
|
|
219
253
|
existingLocales,
|
|
220
254
|
publicDir,
|
|
221
|
-
packageJson: packageJson,
|
|
255
|
+
packageJson: (packageJson ?? {}),
|
|
222
256
|
formatConfig: config.formatConfig,
|
|
257
|
+
...(framework === 'django' ? { djangoRoot: djangoRoot ?? '.' } : {}),
|
|
223
258
|
};
|
|
224
259
|
return { workspace };
|
|
225
260
|
}
|
package/dist/orchestrator.d.ts
CHANGED
|
@@ -5,4 +5,9 @@ import type { EngineConfigV2, EngineResultV2 } from './runtime/types';
|
|
|
5
5
|
* This is the single, clean wiring layer for the pipeline.
|
|
6
6
|
* Workers/controllers call `runEngineV2()`.
|
|
7
7
|
*/
|
|
8
|
+
/** Thrown at a progress checkpoint once the run's AbortSignal has fired. */
|
|
9
|
+
export declare class RunAbortedError extends Error {
|
|
10
|
+
readonly reason: string;
|
|
11
|
+
constructor(reason: string);
|
|
12
|
+
}
|
|
8
13
|
export declare function runEngineV2(config: EngineConfigV2): Promise<EngineResultV2>;
|
package/dist/orchestrator.js
CHANGED
|
@@ -33,6 +33,7 @@ var __importStar = (this && this.__importStar) || (function () {
|
|
|
33
33
|
};
|
|
34
34
|
})();
|
|
35
35
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
36
|
+
exports.RunAbortedError = void 0;
|
|
36
37
|
exports.runEngineV2 = runEngineV2;
|
|
37
38
|
const fs = __importStar(require("fs-extra"));
|
|
38
39
|
const logger_1 = require("./utils/logger");
|
|
@@ -43,6 +44,7 @@ const extract_1 = require("./modules/extractor/extract");
|
|
|
43
44
|
const keygen_1 = require("./modules/keygen");
|
|
44
45
|
const locale_builder_1 = require("./modules/locale-builder");
|
|
45
46
|
const translate_1 = require("./modules/translator/translate");
|
|
47
|
+
const glossary_1 = require("./modules/glossary");
|
|
46
48
|
const translator_caller_1 = require("./utils/translator-caller");
|
|
47
49
|
const injection_1 = require("./modules/injection");
|
|
48
50
|
const summary_1 = require("./modules/summary");
|
|
@@ -54,13 +56,15 @@ const not_injected_1 = require("./utils/not-injected");
|
|
|
54
56
|
const language_codes_1 = require("./modules/translator/language-codes");
|
|
55
57
|
const format_injection_1 = require("./modules/format-injection");
|
|
56
58
|
const extraction_options_1 = require("./utils/extraction-options");
|
|
59
|
+
const catalog_1 = require("./django/catalog");
|
|
57
60
|
/**
|
|
58
|
-
* Whether the run's translator is an LLM (BYOK or an injected provider
|
|
59
|
-
*
|
|
60
|
-
*
|
|
61
|
+
* Whether the run's translator is an LLM (BYOK or an injected provider that
|
|
62
|
+
* isn't plain-text MT). Only then do JSX sentences get ICU plurals in the
|
|
63
|
+
* source; NLLB translates existing ICU messages branch by branch.
|
|
64
|
+
* `capabilities.icu` overrides the guess.
|
|
61
65
|
*/
|
|
62
66
|
function llmTranslator(config) {
|
|
63
|
-
return config.capabilities?.icu ?? (!!config.byok || !!config.providers?.translator);
|
|
67
|
+
return config.capabilities?.icu ?? (!!config.byok || (!!config.providers?.translator && !config.providers.translator.plainText));
|
|
64
68
|
}
|
|
65
69
|
/**
|
|
66
70
|
* Engine v2 Orchestrator (backend runtime)
|
|
@@ -68,18 +72,54 @@ function llmTranslator(config) {
|
|
|
68
72
|
* This is the single, clean wiring layer for the pipeline.
|
|
69
73
|
* Workers/controllers call `runEngineV2()`.
|
|
70
74
|
*/
|
|
75
|
+
/** Thrown at a progress checkpoint once the run's AbortSignal has fired. */
|
|
76
|
+
class RunAbortedError extends Error {
|
|
77
|
+
constructor(reason) {
|
|
78
|
+
super(reason === 'timeout' ? 'Run timed out' : reason === 'cancelled' ? 'Run cancelled' : `Run aborted: ${reason}`);
|
|
79
|
+
this.reason = reason;
|
|
80
|
+
this.name = 'RunAbortedError';
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
exports.RunAbortedError = RunAbortedError;
|
|
84
|
+
function abortReason(signal) {
|
|
85
|
+
const reason = signal.reason;
|
|
86
|
+
if (typeof reason === 'string')
|
|
87
|
+
return reason;
|
|
88
|
+
if (reason instanceof Error)
|
|
89
|
+
return reason.message;
|
|
90
|
+
return 'aborted';
|
|
91
|
+
}
|
|
71
92
|
async function runEngineV2(config) {
|
|
72
93
|
if (config.logger) {
|
|
73
94
|
const { logger, ...rest } = config;
|
|
74
95
|
return (0, logger_1.runWithEngineLogger)(logger, () => runEngineV2(rest));
|
|
75
96
|
}
|
|
76
97
|
const startTime = Date.now();
|
|
98
|
+
// Fresh translations so far, for billing a run that is stopped mid-way.
|
|
99
|
+
const translationProgress = { freshTranslations: 0 };
|
|
77
100
|
try {
|
|
78
101
|
logger_1.engineLogger.info('Starting Babelize engine v2 pipeline', {
|
|
79
102
|
workspacePath: config.workspacePath,
|
|
80
103
|
targetLocales: config.targetLocales,
|
|
81
104
|
dryRun: config.dryRun,
|
|
82
105
|
});
|
|
106
|
+
// Cooperative cancellation: every phase (and every translation batch)
|
|
107
|
+
// reports progress, so the run stops at the next report after the signal
|
|
108
|
+
// fires. The engine has no other way to be stopped mid-run.
|
|
109
|
+
if (config.signal) {
|
|
110
|
+
const signal = config.signal;
|
|
111
|
+
const report = config.onProgress;
|
|
112
|
+
if (signal.aborted)
|
|
113
|
+
throw new RunAbortedError(abortReason(signal));
|
|
114
|
+
config = {
|
|
115
|
+
...config,
|
|
116
|
+
onProgress: (message, step, status) => {
|
|
117
|
+
if (signal.aborted)
|
|
118
|
+
throw new RunAbortedError(abortReason(signal));
|
|
119
|
+
report?.(message, step, status);
|
|
120
|
+
},
|
|
121
|
+
};
|
|
122
|
+
}
|
|
83
123
|
// The file-list cache is per run: a long-lived worker (or a CLI rerun in
|
|
84
124
|
// the same process) must see files added since the previous run.
|
|
85
125
|
(0, file_utils_1.clearFindFilesCache)();
|
|
@@ -94,7 +134,7 @@ async function runEngineV2(config) {
|
|
|
94
134
|
config.onProgress?.('Extracting strings from codebase...', 'extraction', 'running');
|
|
95
135
|
// One resolved ICU decision for extraction, translation and injection (#248).
|
|
96
136
|
const icuOptions = await (0, extraction_options_1.resolveIcuOptions)(workspace, llmTranslator(config));
|
|
97
|
-
if (!icuOptions.sentences) {
|
|
137
|
+
if (!icuOptions.sentences && workspace.framework !== 'django') {
|
|
98
138
|
logger_1.engineLogger.warn('i18next-icu not loaded by the project i18n config; keeping JSX sentences as fragments');
|
|
99
139
|
}
|
|
100
140
|
const { extraction } = await (0, extract_1.extractStrings)(workspace, config.filesToProcess, icuOptions);
|
|
@@ -129,6 +169,11 @@ async function runEngineV2(config) {
|
|
|
129
169
|
// Use only translatable strings for the rest of the pipeline
|
|
130
170
|
config.onProgress?.('Generating translation keys...', 'keygen', 'running');
|
|
131
171
|
const { keyGeneration } = (0, keygen_1.generateKeys)(translatableStrings, existingData);
|
|
172
|
+
// gettext projects (Django): the catalogue of messages and the project's
|
|
173
|
+
// existing .po translations, which are kept and never re-translated.
|
|
174
|
+
const gettext = workspace.localeFormat === 'po'
|
|
175
|
+
? await catalog_1.GettextCatalog.create(workspace, translatableStrings, keyGeneration.keys, config.targetLocales)
|
|
176
|
+
: null;
|
|
132
177
|
// ── Lockfile: partition keys ─────────────────────────────
|
|
133
178
|
let lockfileStats = { reused: 0, translated: 0, changed: 0 };
|
|
134
179
|
let tmHits = 0;
|
|
@@ -168,9 +213,20 @@ async function runEngineV2(config) {
|
|
|
168
213
|
// is a heuristic (renamed translators, other file types, templates), so
|
|
169
214
|
// it never changes what is translated, kept or written.
|
|
170
215
|
keyUsage = await (0, key_usage_1.reportKeyUsage)(workspace, flatKeys, new Set(keyGeneration.keys.map((k) => k.key)));
|
|
216
|
+
// Translations already in the project's .po files win over the lockfile.
|
|
217
|
+
if (gettext)
|
|
218
|
+
lockfile = gettext.seedLockfile(lockfile, flatKeys);
|
|
171
219
|
// Pseudo-localization (--provider pseudo) is not a translation: a real
|
|
172
220
|
// run treats pseudo values in the lockfile as missing and translates them.
|
|
173
|
-
|
|
221
|
+
// The glossary is loaded once, here: keys whose glossary rules changed
|
|
222
|
+
// since their stored translation are re-translated (and only those).
|
|
223
|
+
// If the rules failed to load, stored translations are kept as they are
|
|
224
|
+
// rather than redone without the glossary.
|
|
225
|
+
const rules = await (0, translate_1.loadProjectTranslationRules)(config.projectId, config.providers?.rules);
|
|
226
|
+
const lockfileRead = {
|
|
227
|
+
pseudo: config.providers?.translator?.pseudo === true,
|
|
228
|
+
glossaryHash: rules.complete ? (source, locale) => (0, glossary_1.glossaryFingerprint)(source, locale, rules.glossary) : undefined,
|
|
229
|
+
};
|
|
174
230
|
const partition = (0, lockfile_1.partitionKeys)(lockfile, flatKeys, config.targetLocales, lockfileRead);
|
|
175
231
|
lockfileStats = (0, lockfile_1.computeLockfileStats)(partition);
|
|
176
232
|
logger_1.engineLogger.info('Lockfile partition result', {
|
|
@@ -203,12 +259,18 @@ async function runEngineV2(config) {
|
|
|
203
259
|
}
|
|
204
260
|
else if (translatorHealthy) {
|
|
205
261
|
const keysToTranslateByLocale = (0, lockfile_1.buildKeysToTranslateByTargetLocale)(partition, config.targetLocales);
|
|
206
|
-
const result = await (0, translate_1.translateLocales)(workspace, localeHandling.sourceLocale, config.targetLocales, config.forceTranslation, config.dryRun, config.projectId, (msg) => config.onProgress?.(msg, 'translation', 'running'), keysToTranslateByLocale, config.providers?.cache, config.byok, contextMap, extractionContextMap, config.providers, { targetPlurals: icuOptions.targetPlurals });
|
|
262
|
+
const result = await (0, translate_1.translateLocales)(workspace, localeHandling.sourceLocale, config.targetLocales, config.forceTranslation, config.dryRun, config.projectId, (msg) => config.onProgress?.(msg, 'translation', 'running'), keysToTranslateByLocale, config.providers?.cache, config.byok, contextMap, extractionContextMap, config.providers, { targetPlurals: icuOptions.targetPlurals, pluralValuePlaceholders: gettext?.pluralValuePlaceholders(), rules, progress: translationProgress });
|
|
207
263
|
translation = result.translation;
|
|
264
|
+
// A translation that wouldn't compile into the .po file is a failure, not output.
|
|
265
|
+
gettext?.validate(translation, flatKeys);
|
|
208
266
|
tmHits = result.stats.tmHits;
|
|
209
267
|
translationMetadata = result.metadata;
|
|
210
268
|
translatedThisRun = freshTranslations(translation, translationMetadata);
|
|
211
|
-
|
|
269
|
+
// Strings reused from the lockfile still make a good run: only fail
|
|
270
|
+
// when nothing at all is usable. Otherwise one string that keeps
|
|
271
|
+
// failing (and is retried every run) failed the whole project.
|
|
272
|
+
if (!hasCachedTranslations(cachedTranslations))
|
|
273
|
+
assertSomethingTranslated(translation);
|
|
212
274
|
}
|
|
213
275
|
else {
|
|
214
276
|
const errorMsg = 'Translator service unavailable — target locale files (hi.json, fr.json, etc.) were NOT generated. Only source locale (en.json) was created. Check that TRANSLATOR_URL or BYOK provider is configured and the service is running.';
|
|
@@ -264,6 +326,15 @@ async function runEngineV2(config) {
|
|
|
264
326
|
locale.data._meta = { dir: (0, language_codes_1.isRTL)(locale.locale) ? 'rtl' : 'ltr', language: locale.locale };
|
|
265
327
|
}
|
|
266
328
|
}
|
|
329
|
+
// The translator wrote only this run's strings to disk; write the merged
|
|
330
|
+
// locale (lockfile + new) so the workspace — which the download ZIP is
|
|
331
|
+
// built from — has every translation, including fully cached locales.
|
|
332
|
+
if (!config.dryRun && workspace.localeFormat !== 'po') {
|
|
333
|
+
for (const locale of translation.locales) {
|
|
334
|
+
const { file } = (0, file_utils_1.localeFilePath)(workspace.path, workspace.publicDir, workspace.localeFormat, locale.locale);
|
|
335
|
+
await (0, file_utils_1.writeJsonFile)(file, locale.data, true);
|
|
336
|
+
}
|
|
337
|
+
}
|
|
267
338
|
// ── Lockfile: update after translation ─────────────────
|
|
268
339
|
config.onProgress?.('Updating lockfile...', 'lockfile', 'running');
|
|
269
340
|
const allTranslatedPerLocale = {};
|
|
@@ -333,7 +404,7 @@ async function runEngineV2(config) {
|
|
|
333
404
|
config, startTime, workspace, allTranslatedPerLocale,
|
|
334
405
|
lockfileStats, tmHits, translationMetadata,
|
|
335
406
|
translatedThisRun, keyUsage,
|
|
336
|
-
formatStrings, frameworkConfig,
|
|
407
|
+
formatStrings, frameworkConfig, gettext,
|
|
337
408
|
});
|
|
338
409
|
return result;
|
|
339
410
|
}
|
|
@@ -358,7 +429,7 @@ async function runEngineV2(config) {
|
|
|
358
429
|
config.onProgress?.(msg, 'translation', 'warning');
|
|
359
430
|
}
|
|
360
431
|
const translationResult = translatorHealthy
|
|
361
|
-
? await (0, translate_1.translateLocales)(workspace, localeHandling.sourceLocale, config.targetLocales, config.forceTranslation, config.dryRun, config.projectId, (msg) => config.onProgress?.(msg, 'translation', 'running'), undefined, config.providers?.cache, config.byok, contextMap, extractionContextMap, config.providers, { targetPlurals: icuOptions.targetPlurals })
|
|
432
|
+
? await (0, translate_1.translateLocales)(workspace, localeHandling.sourceLocale, config.targetLocales, config.forceTranslation, config.dryRun, config.projectId, (msg) => config.onProgress?.(msg, 'translation', 'running'), undefined, config.providers?.cache, config.byok, contextMap, extractionContextMap, config.providers, { targetPlurals: icuOptions.targetPlurals, pluralValuePlaceholders: gettext?.pluralValuePlaceholders(), progress: translationProgress })
|
|
362
433
|
: {
|
|
363
434
|
translation: {
|
|
364
435
|
locales: [],
|
|
@@ -368,6 +439,7 @@ async function runEngineV2(config) {
|
|
|
368
439
|
metadata: {}
|
|
369
440
|
};
|
|
370
441
|
const translation = translationResult.translation;
|
|
442
|
+
gettext?.validate(translation, (0, keygen_1.flattenLocaleData)(localeHandling.sourceLocale.data));
|
|
371
443
|
tmHits = translationResult.stats.tmHits;
|
|
372
444
|
translationMetadata = translationResult.metadata;
|
|
373
445
|
translatedThisRun = freshTranslations(translation, translationMetadata);
|
|
@@ -387,7 +459,7 @@ async function runEngineV2(config) {
|
|
|
387
459
|
config, startTime, workspace, allTranslatedPerLocale,
|
|
388
460
|
lockfileStats: undefined, tmHits, translationMetadata,
|
|
389
461
|
translatedThisRun, keyUsage,
|
|
390
|
-
formatStrings, frameworkConfig,
|
|
462
|
+
formatStrings, frameworkConfig, gettext,
|
|
391
463
|
});
|
|
392
464
|
return result;
|
|
393
465
|
}
|
|
@@ -396,6 +468,9 @@ async function runEngineV2(config) {
|
|
|
396
468
|
return {
|
|
397
469
|
success: false,
|
|
398
470
|
error: error?.message ?? String(error),
|
|
471
|
+
...(error instanceof RunAbortedError
|
|
472
|
+
? { aborted: error.reason, partialTranslations: translationProgress.freshTranslations }
|
|
473
|
+
: {}),
|
|
399
474
|
stats: {
|
|
400
475
|
filesScanned: 0,
|
|
401
476
|
filesWithStrings: 0,
|
|
@@ -510,6 +585,9 @@ function setNestedKey(obj, dottedKey, value) {
|
|
|
510
585
|
}
|
|
511
586
|
current[parts[parts.length - 1]] = value;
|
|
512
587
|
}
|
|
588
|
+
function hasCachedTranslations(cached) {
|
|
589
|
+
return Object.values(cached).some((locale) => Object.keys(locale).length > 0);
|
|
590
|
+
}
|
|
513
591
|
/**
|
|
514
592
|
* A run where translation was needed but no string in any locale could be
|
|
515
593
|
* translated is a failed run, not an empty success: fail it with the reasons
|
|
@@ -526,8 +604,24 @@ function assertSomethingTranslated(translation) {
|
|
|
526
604
|
const sample = failures.find((f) => f.message)?.message;
|
|
527
605
|
throw new Error(`No strings could be translated (${summary})${sample ? `: ${sample}` : ''}`);
|
|
528
606
|
}
|
|
607
|
+
/**
|
|
608
|
+
* Add a changed file to the run's list, folding a later write to a file that is
|
|
609
|
+
* already listed (an entry point that was injected, then wired up) into one
|
|
610
|
+
* entry: the original from the first write, the content from the last.
|
|
611
|
+
*/
|
|
612
|
+
function mergeChangedFile(list, changed) {
|
|
613
|
+
const existing = list.find((f) => f.file === changed.file);
|
|
614
|
+
if (!existing) {
|
|
615
|
+
list.push(changed);
|
|
616
|
+
return;
|
|
617
|
+
}
|
|
618
|
+
existing.modifiedContent = changed.modifiedContent;
|
|
619
|
+
existing.injections.push(...changed.injections);
|
|
620
|
+
existing.importAdded = existing.importAdded || changed.importAdded;
|
|
621
|
+
existing.hookAdded = existing.hookAdded || changed.hookAdded;
|
|
622
|
+
}
|
|
529
623
|
async function finalizePipeline(params) {
|
|
530
|
-
const { extraction, keyGeneration, localeHandling, translation, config, startTime, workspace, allTranslatedPerLocale, lockfileStats, tmHits, translationMetadata, formatStrings, frameworkConfig, translatedThisRun, keyUsage, } = params;
|
|
624
|
+
const { extraction, keyGeneration, localeHandling, translation, config, startTime, workspace, allTranslatedPerLocale, lockfileStats, tmHits, translationMetadata, formatStrings, frameworkConfig, translatedThisRun, keyUsage, gettext, } = params;
|
|
531
625
|
const keyMapping = (0, keygen_1.createKeyMapping)(keyGeneration.keys);
|
|
532
626
|
// Injection: AST-based i18n hook injection (skipped for frameworks like Django)
|
|
533
627
|
let injection;
|
|
@@ -554,7 +648,8 @@ async function finalizePipeline(params) {
|
|
|
554
648
|
config.onProgress?.('Injecting Intl.* format replacements...', 'injection', 'running');
|
|
555
649
|
const formatInjection = await (0, format_injection_1.injectFormats)(workspace, formatStrings, config.formatConfig, config.dryRun);
|
|
556
650
|
if (formatInjection.totalInjections > 0) {
|
|
557
|
-
|
|
651
|
+
for (const changed of formatInjection.modifiedFiles)
|
|
652
|
+
mergeChangedFile(injection.modifiedFiles, changed);
|
|
558
653
|
injection.totalInjections += formatInjection.totalInjections;
|
|
559
654
|
injection.skippedFiles.push(...formatInjection.skippedFiles);
|
|
560
655
|
}
|
|
@@ -562,8 +657,18 @@ async function finalizePipeline(params) {
|
|
|
562
657
|
totalInjections: formatInjection.totalInjections,
|
|
563
658
|
});
|
|
564
659
|
}
|
|
565
|
-
//
|
|
660
|
+
// gettext: merge this run's translations into the project's .po files.
|
|
566
661
|
const finalizationNotes = [];
|
|
662
|
+
if (gettext) {
|
|
663
|
+
gettext.refreshReferences(injection);
|
|
664
|
+
const written = await gettext.write(config.targetLocales, allTranslatedPerLocale, {
|
|
665
|
+
dryRun: config.dryRun,
|
|
666
|
+
pseudo: config.providers?.translator?.pseudo === true,
|
|
667
|
+
});
|
|
668
|
+
translation.locales = written.locales;
|
|
669
|
+
finalizationNotes.push(...written.notes);
|
|
670
|
+
}
|
|
671
|
+
// Finalization: install i18n deps, create config files, modify entry points
|
|
567
672
|
if (frameworkConfig.phases.includes('finalize')) {
|
|
568
673
|
config.onProgress?.('Finalizing project...', 'finalization', 'running');
|
|
569
674
|
const { finalization } = await (0, finalization_1.finalizeProject)(workspace, config.dryRun, {
|
|
@@ -572,6 +677,12 @@ async function finalizePipeline(params) {
|
|
|
572
677
|
installDependencies: config.installDependencies,
|
|
573
678
|
});
|
|
574
679
|
finalizationNotes.push(...(finalization.warnings ?? []));
|
|
680
|
+
// Every file finalization wrote (package.json, i18n config, entry point,
|
|
681
|
+
// next.config, layout, i18n.js, settings.py, a lockfile it updated) ships
|
|
682
|
+
// with the run like injected files — hosts stage modifiedFiles for the PR.
|
|
683
|
+
for (const changed of finalization.changedFiles ?? []) {
|
|
684
|
+
mergeChangedFile(injection.modifiedFiles, { ...changed, injections: [], importAdded: false, hookAdded: false });
|
|
685
|
+
}
|
|
575
686
|
if (finalization.dependenciesInvalid) {
|
|
576
687
|
if (finalization.packagesInstalled) {
|
|
577
688
|
logger_1.engineLogger.info('New dependencies installed successfully');
|
|
@@ -89,6 +89,13 @@ export interface TranslationProvider {
|
|
|
89
89
|
* in the lockfile and replaced by the next real run.
|
|
90
90
|
*/
|
|
91
91
|
readonly pseudo?: boolean;
|
|
92
|
+
/**
|
|
93
|
+
* Machine translation of plain text only (e.g. the NLLB service): it can't
|
|
94
|
+
* be given ICU syntax, prompts or hints. ICU plural/select messages are
|
|
95
|
+
* translated branch by branch for it, as for the default NLLB translator.
|
|
96
|
+
* Unset means an LLM-class provider.
|
|
97
|
+
*/
|
|
98
|
+
readonly plainText?: boolean;
|
|
92
99
|
/**
|
|
93
100
|
* Translate a batch. Resolves with one KeyResult per requested entry.
|
|
94
101
|
* Rejects only when the request as a whole failed (network, auth, 5xx) or
|
|
@@ -134,8 +141,9 @@ export interface EngineProviders {
|
|
|
134
141
|
/**
|
|
135
142
|
* Second chance for keys the (LLM) translator failed with `provider_error`
|
|
136
143
|
* (rate limit, outage), tried once after the repair pass. Treated as a
|
|
137
|
-
* non-LLM service (e.g. NLLB): ICU
|
|
138
|
-
*
|
|
144
|
+
* non-LLM service (e.g. NLLB): ICU keys go to it branch by branch,
|
|
145
|
+
* mixed-script keys never, and its output isn't cached or written to
|
|
146
|
+
* translation memory. Keys it
|
|
139
147
|
* translated get `fallback: true` in their metadata.
|
|
140
148
|
*/
|
|
141
149
|
fallbackTranslator?: TranslationProvider;
|
|
@@ -12,7 +12,8 @@ export interface ProjectLocaleConfig {
|
|
|
12
12
|
export declare function readProjectLocaleConfig(projectDir: string): Promise<ProjectLocaleConfig>;
|
|
13
13
|
export interface LocaleSetReport {
|
|
14
14
|
dir: string;
|
|
15
|
-
|
|
15
|
+
/** `gettext`: Django .po catalogues (<dir>/<lang>/LC_MESSAGES/django.po). */
|
|
16
|
+
layout: LocaleFileSet['layout'] | 'gettext';
|
|
16
17
|
sourceLocale: string;
|
|
17
18
|
/** Source keys checked against. */
|
|
18
19
|
sourceKeys: number;
|
package/dist/qa/check-project.js
CHANGED
|
@@ -44,6 +44,8 @@ const fs_1 = require("fs");
|
|
|
44
44
|
const path = __importStar(require("path"));
|
|
45
45
|
const locale_qa_1 = require("./locale-qa");
|
|
46
46
|
const locale_files_1 = require("./locale-files");
|
|
47
|
+
const gettext_files_1 = require("./gettext-files");
|
|
48
|
+
const locale_names_1 = require("../django/locale-names");
|
|
47
49
|
/**
|
|
48
50
|
* The parts of `.babelize/config.json` the local tools use. Missing or
|
|
49
51
|
* malformed config is not an error: everything in it is optional.
|
|
@@ -65,19 +67,39 @@ async function readProjectLocaleConfig(projectDir) {
|
|
|
65
67
|
targetLocales: list(raw.targetLocales) ?? list(raw.targetLanguageIds) ?? list(raw.targetLanguages),
|
|
66
68
|
};
|
|
67
69
|
const g = raw.glossary;
|
|
70
|
+
const neverTranslate = [];
|
|
71
|
+
const alwaysTranslateAs = [];
|
|
68
72
|
if (g && typeof g === 'object') {
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
73
|
+
if (Array.isArray(g.neverTranslate))
|
|
74
|
+
neverTranslate.push(...g.neverTranslate.filter((t) => typeof t === 'string'));
|
|
75
|
+
if (Array.isArray(g.alwaysTranslateAs)) {
|
|
76
|
+
alwaysTranslateAs.push(...g.alwaysTranslateAs.filter((e) => !!e && typeof e === 'object' && typeof e.term === 'string' &&
|
|
77
|
+
!!e.translations && typeof e.translations === 'object'));
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
// Older configs: `customTranslations: { term: { neverTranslate?, translations? } }`.
|
|
81
|
+
// Read into the same glossary so `translate` and `check` agree on it.
|
|
82
|
+
const legacy = raw.customTranslations;
|
|
83
|
+
if (legacy && typeof legacy === 'object' && !Array.isArray(legacy)) {
|
|
84
|
+
for (const [term, value] of Object.entries(legacy)) {
|
|
85
|
+
if (!value || typeof value !== 'object')
|
|
86
|
+
continue;
|
|
87
|
+
const entry = value;
|
|
88
|
+
if (entry.neverTranslate === true)
|
|
89
|
+
neverTranslate.push(term);
|
|
90
|
+
else if (entry.translations && typeof entry.translations === 'object') {
|
|
91
|
+
alwaysTranslateAs.push({ term, translations: entry.translations });
|
|
92
|
+
}
|
|
93
|
+
}
|
|
76
94
|
}
|
|
95
|
+
if (neverTranslate.length || alwaysTranslateAs.length)
|
|
96
|
+
config.glossary = { neverTranslate, alwaysTranslateAs };
|
|
77
97
|
return config;
|
|
78
98
|
}
|
|
79
99
|
/** The file a locale issue lives in, relative to the project root. */
|
|
80
100
|
function localeIssueFile(set, issue) {
|
|
101
|
+
if (set.layout === 'gettext')
|
|
102
|
+
return `${set.dir === '.' ? '' : `${set.dir}/`}${(0, locale_names_1.djangoLocaleName)(issue.locale)}/LC_MESSAGES/django.po`;
|
|
81
103
|
if (set.layout === 'namespaced') {
|
|
82
104
|
const ns = issue.key.includes(':') ? issue.key.split(':')[0] : 'translation';
|
|
83
105
|
return `${set.dir}/${issue.locale}/${ns}.json`;
|
|
@@ -146,5 +168,26 @@ async function checkProjectLocales(projectDir, options = {}) {
|
|
|
146
168
|
fileErrors,
|
|
147
169
|
});
|
|
148
170
|
}
|
|
171
|
+
// gettext catalogues: the msgids are the source.
|
|
172
|
+
for (const set of await (0, gettext_files_1.findGettextSets)(projectDir, options.ignoreDirs)) {
|
|
173
|
+
const loaded = await (0, gettext_files_1.loadGettextSet)(projectDir, set);
|
|
174
|
+
const available = Object.keys(set.files).filter((l) => l !== sourceLocale).sort();
|
|
175
|
+
const wanted = options.locales?.length
|
|
176
|
+
? options.locales.filter((l) => l !== sourceLocale)
|
|
177
|
+
: [...new Set([...available, ...expected])].filter((l) => l !== sourceLocale).sort();
|
|
178
|
+
const locales = [];
|
|
179
|
+
for (const locale of wanted.filter((l) => set.files[l])) {
|
|
180
|
+
locales.push((0, locale_qa_1.checkLocalePair)(loaded.source, loaded.byLocale[locale] ?? {}, locale, { sourceLocale, glossary, keys: options.keys }));
|
|
181
|
+
}
|
|
182
|
+
reports.push({
|
|
183
|
+
dir: set.dir,
|
|
184
|
+
layout: 'gettext',
|
|
185
|
+
sourceLocale,
|
|
186
|
+
sourceKeys: Object.keys(loaded.source).length,
|
|
187
|
+
locales,
|
|
188
|
+
missingLocales: wanted.filter((l) => !set.files[l]),
|
|
189
|
+
fileErrors: loaded.errors,
|
|
190
|
+
});
|
|
191
|
+
}
|
|
149
192
|
return { sourceLocale, sets: reports, skippedSets };
|
|
150
193
|
}
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
export interface GettextSet {
|
|
2
|
+
dir: string;
|
|
3
|
+
/** Babelize locale code → relative path of its django.po. */
|
|
4
|
+
files: Record<string, string>;
|
|
5
|
+
}
|
|
6
|
+
export interface GettextMessages {
|
|
7
|
+
source: Record<string, string>;
|
|
8
|
+
byLocale: Record<string, Record<string, string>>;
|
|
9
|
+
errors: Array<{
|
|
10
|
+
file: string;
|
|
11
|
+
message: string;
|
|
12
|
+
}>;
|
|
13
|
+
}
|
|
14
|
+
export declare function findGettextSets(projectDir: string, ignoreDirs?: readonly string[]): Promise<GettextSet[]>;
|
|
15
|
+
export declare function loadGettextSet(projectDir: string, set: GettextSet): Promise<GettextMessages>;
|