@babelize/engine 0.2.1 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/detection/i18n.d.ts +19 -0
- package/dist/detection/i18n.js +144 -0
- package/dist/detection/stack.d.ts +2 -0
- package/dist/index.d.ts +4 -1
- package/dist/index.js +9 -2
- package/dist/local/hosted-provider.d.ts +30 -2
- package/dist/local/hosted-provider.js +50 -3
- package/dist/local/translate-core.js +4 -0
- package/dist/modules/lockfile/index.d.ts +1 -1
- package/dist/modules/lockfile/index.js +2 -1
- package/dist/modules/lockfile/lockfile-manager.d.ts +9 -0
- package/dist/modules/lockfile/lockfile-manager.js +43 -1
- package/dist/modules/lockfile/schema.d.ts +14 -0
- package/dist/modules/lockfile/schema.js +2 -0
- package/dist/modules/overrides/index.d.ts +41 -0
- package/dist/modules/overrides/index.js +50 -0
- package/dist/modules/translator/budget.d.ts +25 -0
- package/dist/modules/translator/budget.js +46 -0
- package/dist/modules/translator/translate.d.ts +11 -0
- package/dist/modules/translator/translate.js +50 -10
- package/dist/orchestrator.js +54 -6
- package/dist/providers/types.d.ts +6 -0
- package/dist/runtime/config.js +1 -0
- package/dist/runtime/types.d.ts +21 -0
- package/dist/utils/translator-caller.js +6 -3
- package/package.json +1 -1
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
import type { RepoFiles } from './stack';
|
|
2
|
+
export type I18nLibraryId = 'next-intl' | 'next-i18next' | 'react-i18next' | 'i18next' | 'vue-i18n' | 'react-intl' | 'lingui' | 'django';
|
|
3
|
+
export declare const I18N_LIBRARY_NAMES: Record<I18nLibraryId, string>;
|
|
4
|
+
export type SourceLanguageBasis = 'config' | 'files' | 'default';
|
|
5
|
+
export interface DetectedI18n {
|
|
6
|
+
library: {
|
|
7
|
+
id: I18nLibraryId;
|
|
8
|
+
name: string;
|
|
9
|
+
} | null;
|
|
10
|
+
/** Locale codes the repo already has files for, as written (en, pt-BR, zh_Hant). */
|
|
11
|
+
locales: string[];
|
|
12
|
+
/** Where the locale files live, e.g. "messages" or "public/locales". */
|
|
13
|
+
localeDir: string | null;
|
|
14
|
+
sourceLanguage: {
|
|
15
|
+
code: string;
|
|
16
|
+
basis: SourceLanguageBasis;
|
|
17
|
+
};
|
|
18
|
+
}
|
|
19
|
+
export declare function detectI18n(files: RepoFiles, framework?: string): Promise<DetectedI18n>;
|
|
@@ -0,0 +1,144 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.I18N_LIBRARY_NAMES = void 0;
|
|
4
|
+
exports.detectI18n = detectI18n;
|
|
5
|
+
/**
|
|
6
|
+
* What a repository already has for i18n, read from a file listing and a few
|
|
7
|
+
* small files (no clone): the i18n library, existing locale files, and the
|
|
8
|
+
* source language. Runs on the same `RepoFiles` as `detectStack`, so the
|
|
9
|
+
* new-project flow can say "Next.js · English · next-intl detected" and
|
|
10
|
+
* pre-select the languages the repo already ships.
|
|
11
|
+
*/
|
|
12
|
+
const locale_files_1 = require("../qa/locale-files");
|
|
13
|
+
exports.I18N_LIBRARY_NAMES = {
|
|
14
|
+
'next-intl': 'next-intl',
|
|
15
|
+
'next-i18next': 'next-i18next',
|
|
16
|
+
'react-i18next': 'react-i18next',
|
|
17
|
+
i18next: 'i18next',
|
|
18
|
+
'vue-i18n': 'vue-i18n',
|
|
19
|
+
'react-intl': 'react-intl',
|
|
20
|
+
lingui: 'Lingui',
|
|
21
|
+
django: 'Django i18n',
|
|
22
|
+
};
|
|
23
|
+
/** Most specific first: next-i18next depends on react-i18next, which depends on i18next. */
|
|
24
|
+
const LIBRARY_DEPS = [
|
|
25
|
+
['next-intl', ['next-intl']],
|
|
26
|
+
['next-i18next', ['next-i18next']],
|
|
27
|
+
['vue-i18n', ['vue-i18n', '@nuxtjs/i18n']],
|
|
28
|
+
['react-i18next', ['react-i18next']],
|
|
29
|
+
['react-intl', ['react-intl']],
|
|
30
|
+
['lingui', ['@lingui/core', '@lingui/react']],
|
|
31
|
+
['i18next', ['i18next']],
|
|
32
|
+
];
|
|
33
|
+
const IGNORED = /(?:^|\/)(?:node_modules|\.git|vendor|dist|build|\.next|out|coverage|__tests__|__fixtures__|fixtures|tests?|e2e)\//;
|
|
34
|
+
/** Files that commonly declare the default locale, checked in this order. */
|
|
35
|
+
const CONFIG_FILE = /(?:^|\/)(?:i18n(?:\/(?:routing|request|config|index))?|i18n\.config|next\.config|next-i18next\.config|nuxt\.config|src\/i18n(?:\/index)?|src\/plugins\/i18n|lingui\.config)\.(?:[cm]?[jt]s|json)$/;
|
|
36
|
+
/** defaultLocale: 'de', fallbackLng: "fr", sourceLocale: 'en', locale: 'en' (vue-i18n createI18n). */
|
|
37
|
+
const DEFAULT_LOCALE_PATTERNS = [
|
|
38
|
+
/\bdefaultLocale\s*[:=]\s*['"`]([A-Za-z_-]{2,12})['"`]/,
|
|
39
|
+
/\bsourceLocale\s*[:=]\s*['"`]([A-Za-z_-]{2,12})['"`]/,
|
|
40
|
+
/\bfallbackLng\s*[:=]\s*['"`]([A-Za-z_-]{2,12})['"`]/,
|
|
41
|
+
/\bfallbackLocale\s*[:=]\s*['"`]([A-Za-z_-]{2,12})['"`]/,
|
|
42
|
+
];
|
|
43
|
+
const DJANGO_LANGUAGE_CODE = /^\s*LANGUAGE_CODE\s*=\s*['"]([A-Za-z_-]{2,12})['"]/m;
|
|
44
|
+
/** Django: locale/<code>/LC_MESSAGES/django.po */
|
|
45
|
+
const DJANGO_PO = /(?:^|\/)locale\/([A-Za-z_-]{2,12})\/LC_MESSAGES\/django\.po$/;
|
|
46
|
+
function depth(p) {
|
|
47
|
+
return p.split('/').length - 1;
|
|
48
|
+
}
|
|
49
|
+
async function readDeps(files, paths) {
|
|
50
|
+
const pkgs = paths
|
|
51
|
+
.filter((p) => (p === 'package.json' || p.endsWith('/package.json')) && depth(p) <= 2)
|
|
52
|
+
.sort((a, b) => depth(a) - depth(b))
|
|
53
|
+
.slice(0, 6);
|
|
54
|
+
const deps = new Set();
|
|
55
|
+
for (const p of pkgs) {
|
|
56
|
+
const raw = await files.read(p);
|
|
57
|
+
if (!raw)
|
|
58
|
+
continue;
|
|
59
|
+
try {
|
|
60
|
+
const pkg = JSON.parse(raw);
|
|
61
|
+
for (const name of Object.keys({ ...pkg.dependencies, ...pkg.devDependencies }))
|
|
62
|
+
deps.add(name);
|
|
63
|
+
}
|
|
64
|
+
catch {
|
|
65
|
+
// not JSON: skip
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
return deps;
|
|
69
|
+
}
|
|
70
|
+
/** "en-us" (Django) → "en-US"; "pt_BR" stays as written. */
|
|
71
|
+
function tidyCode(code) {
|
|
72
|
+
const m = /^([a-z]{2,3})-([a-z]{2})$/.exec(code);
|
|
73
|
+
return m ? `${m[1]}-${m[2].toUpperCase()}` : code;
|
|
74
|
+
}
|
|
75
|
+
async function detectI18n(files, framework) {
|
|
76
|
+
const paths = files.paths.filter((p) => !IGNORED.test(p));
|
|
77
|
+
// Library
|
|
78
|
+
let library = null;
|
|
79
|
+
if (framework === 'django') {
|
|
80
|
+
library = { id: 'django', name: exports.I18N_LIBRARY_NAMES.django };
|
|
81
|
+
}
|
|
82
|
+
else {
|
|
83
|
+
const deps = await readDeps(files, paths);
|
|
84
|
+
const found = LIBRARY_DEPS.find(([, names]) => names.some((n) => deps.has(n)));
|
|
85
|
+
if (found)
|
|
86
|
+
library = { id: found[0], name: exports.I18N_LIBRARY_NAMES[found[0]] };
|
|
87
|
+
}
|
|
88
|
+
// Existing locale files: the set with the most locales wins.
|
|
89
|
+
let locales = [];
|
|
90
|
+
let localeDir = null;
|
|
91
|
+
let largestByLocale = new Map();
|
|
92
|
+
const sets = (0, locale_files_1.groupLocaleFiles)(paths);
|
|
93
|
+
if (sets.length > 0) {
|
|
94
|
+
const best = sets.reduce((a, b) => (Object.keys(b.files).length > Object.keys(a.files).length ? b : a));
|
|
95
|
+
locales = Object.keys(best.files).sort();
|
|
96
|
+
localeDir = best.dir;
|
|
97
|
+
if (files.size) {
|
|
98
|
+
largestByLocale = new Map(locales.map((l) => [l, best.files[l].reduce((sum, f) => sum + (files.size(f) ?? 0), 0)]));
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
else {
|
|
102
|
+
const po = paths.map((p) => DJANGO_PO.exec(p)).filter((m) => !!m);
|
|
103
|
+
if (po.length > 0) {
|
|
104
|
+
locales = [...new Set(po.map((m) => m[1]))].sort();
|
|
105
|
+
localeDir = po[0][0].split('/').slice(0, -3).join('/') || '.';
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
// Source language: declared in config, else inferred from the locale files.
|
|
109
|
+
let sourceLanguage = { code: 'en', basis: 'default' };
|
|
110
|
+
const configCandidates = paths
|
|
111
|
+
.filter((p) => CONFIG_FILE.test(p) && depth(p) <= 3)
|
|
112
|
+
.sort((a, b) => depth(a) - depth(b))
|
|
113
|
+
.slice(0, 6);
|
|
114
|
+
if (framework === 'django') {
|
|
115
|
+
configCandidates.unshift(...paths.filter((p) => /(?:^|\/)settings(?:\/base)?\.py$/.test(p) && depth(p) <= 3).slice(0, 2));
|
|
116
|
+
}
|
|
117
|
+
for (const p of configCandidates) {
|
|
118
|
+
const content = await files.read(p);
|
|
119
|
+
if (!content)
|
|
120
|
+
continue;
|
|
121
|
+
const m = p.endsWith('.py')
|
|
122
|
+
? DJANGO_LANGUAGE_CODE.exec(content)
|
|
123
|
+
: DEFAULT_LOCALE_PATTERNS.map((re) => re.exec(content)).find(Boolean);
|
|
124
|
+
const code = m?.[1] ? tidyCode(m[1]) : null;
|
|
125
|
+
if (code && (0, locale_files_1.isLocaleCode)(code)) {
|
|
126
|
+
sourceLanguage = { code, basis: 'config' };
|
|
127
|
+
break;
|
|
128
|
+
}
|
|
129
|
+
}
|
|
130
|
+
if (sourceLanguage.basis === 'default' && locales.length > 0) {
|
|
131
|
+
if (locales.length === 1) {
|
|
132
|
+
sourceLanguage = { code: locales[0], basis: 'files' };
|
|
133
|
+
}
|
|
134
|
+
else if (locales.some((l) => l === 'en' || l.startsWith('en-') || l.startsWith('en_'))) {
|
|
135
|
+
sourceLanguage = { code: locales.find((l) => l === 'en') ?? locales.find((l) => l.startsWith('en')), basis: 'files' };
|
|
136
|
+
}
|
|
137
|
+
else if (largestByLocale.size > 0) {
|
|
138
|
+
// The source usually has every key: the biggest file set.
|
|
139
|
+
const [code] = [...largestByLocale.entries()].reduce((a, b) => (b[1] > a[1] ? b : a));
|
|
140
|
+
sourceLanguage = { code, basis: 'files' };
|
|
141
|
+
}
|
|
142
|
+
}
|
|
143
|
+
return { library, locales, localeDir, sourceLanguage };
|
|
144
|
+
}
|
|
@@ -32,6 +32,8 @@ export interface RepoFiles {
|
|
|
32
32
|
read(relPath: string): Promise<string | null>;
|
|
33
33
|
/** The listing is incomplete (very large repository). */
|
|
34
34
|
truncated?: boolean;
|
|
35
|
+
/** File size in bytes, when the listing knows it. */
|
|
36
|
+
size?(relPath: string): number | undefined;
|
|
35
37
|
}
|
|
36
38
|
export declare const FRAMEWORK_DISPLAY_NAMES: Record<string, string>;
|
|
37
39
|
/** Does a Python dependency file list Django itself (not just django-* add-ons)? */
|
package/dist/index.d.ts
CHANGED
|
@@ -18,6 +18,7 @@ export * from './providers';
|
|
|
18
18
|
export { runWithEngineLogger, setEngineLogger } from './utils/logger';
|
|
19
19
|
export type { EngineLogger, LogContext } from './utils/logger';
|
|
20
20
|
export * from './modules/lockfile';
|
|
21
|
+
export { applyTranslationOverrides, setLocaleValue, type OverrideReport, type TranslationOverride } from './modules/overrides';
|
|
21
22
|
export * from './modules/translator/language-codes';
|
|
22
23
|
export type { MemoryEntry, FuzzyMatch, MemoryLookupResult } from './modules/translator/memory/types';
|
|
23
24
|
export { matchFromMemory } from './modules/translator/memory/memory-matcher';
|
|
@@ -26,8 +27,10 @@ export type { SyncDetectionResult, SyncDetectedString, SyncMissingLocaleKey } fr
|
|
|
26
27
|
export type { TranslationEntry, WorkspaceInfo } from './runtime/types';
|
|
27
28
|
export { addBabelToHistory, detectProjectInfo, getBabelizeSummary, getProjectInfo, saveCachedStrings, saveProjectConfig, } from './utils/metadata';
|
|
28
29
|
export type { BabelHistoryEntry, ExtractedStringEntry, ProjectInfo } from './utils/metadata';
|
|
29
|
-
export { detectStack, localRepoFiles, FRAMEWORK_DISPLAY_NAMES } from './detection/stack';
|
|
30
|
+
export { detectStack, localRepoFiles, FRAMEWORK_DISPLAY_NAMES, SUPPORTED_SUMMARY } from './detection/stack';
|
|
30
31
|
export type { DetectedStack, RepoFiles, StackFramework, PackageManagerId } from './detection/stack';
|
|
32
|
+
export { detectI18n, I18N_LIBRARY_NAMES } from './detection/i18n';
|
|
33
|
+
export type { DetectedI18n, I18nLibraryId, SourceLanguageBasis } from './detection/i18n';
|
|
31
34
|
export { parseCode } from './utils/ast-helpers';
|
|
32
35
|
export { extractPlaceholders, hasICUSyntax, hashString, protectPlaceholders, restorePlaceholders, } from './utils/string-utils';
|
|
33
36
|
export { sanitizeTranslation, validateRawTokens } from './utils/translation-sanitizer';
|
package/dist/index.js
CHANGED
|
@@ -21,8 +21,8 @@ var __exportStar = (this && this.__exportStar) || function(m, exports) {
|
|
|
21
21
|
for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
|
|
22
22
|
};
|
|
23
23
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
24
|
-
exports.
|
|
25
|
-
exports.TRANSLATION_CACHE_TTL_SECONDS = exports.translationCacheKey = exports.hashText = exports.readProjectLocaleConfig = exports.localeIssueFile = exports.checkProjectLocales = exports.ReviewError = exports.reviewLocaleChanges = exports.loadLocale = exports.isLocaleCode = exports.isIgnoredLocalePath = exports.groupLocaleFiles = exports.findLocaleFileSets = exports.DEFAULT_LOCALE_IGNORE_DIRS = exports.glossaryFingerprint = exports.normalizeGlossary = exports.localeValue = exports.containsTerm = exports.canonicalizeLocaleKeys = exports.canonicalLocale = exports.tallyLocaleIssues = exports.flattenLocaleMessages = exports.checkLocalePair = exports.frameworkSupport = void 0;
|
|
24
|
+
exports.CircuitBreaker = exports.validateBatchQuality = exports.validateTranslationQuality = exports.translatorLoadBalancer = exports.warmupTranslator = exports.checkTranslatorHealth = exports.callTranslator = exports.requiredPluralCategories = exports.parseIcu = exports.isIcuMessage = exports.compareIcuMessages = exports.sameProtectedSpans = exports.restoreText = exports.protectText = exports.hasNothingToTranslate = exports.checkTokens = exports.validateRawTokens = exports.sanitizeTranslation = exports.restorePlaceholders = exports.protectPlaceholders = exports.hashString = exports.hasICUSyntax = exports.extractPlaceholders = exports.parseCode = exports.I18N_LIBRARY_NAMES = exports.detectI18n = exports.SUPPORTED_SUMMARY = exports.FRAMEWORK_DISPLAY_NAMES = exports.localRepoFiles = exports.detectStack = exports.saveProjectConfig = exports.saveCachedStrings = exports.getProjectInfo = exports.getBabelizeSummary = exports.detectProjectInfo = exports.addBabelToHistory = exports.formatSyncNotificationMessage = exports.detectSyncIssues = exports.matchFromMemory = exports.setLocaleValue = exports.applyTranslationOverrides = exports.setEngineLogger = exports.runWithEngineLogger = exports.validateWorkspace = exports.scanKeyReferences = exports.RunAbortedError = exports.validateEngineConfigV2 = exports.runEngineV2 = exports.loadBabelizeConfig = exports.BabelizeEngine = void 0;
|
|
25
|
+
exports.TRANSLATION_CACHE_TTL_SECONDS = exports.translationCacheKey = exports.hashText = exports.readProjectLocaleConfig = exports.localeIssueFile = exports.checkProjectLocales = exports.ReviewError = exports.reviewLocaleChanges = exports.loadLocale = exports.isLocaleCode = exports.isIgnoredLocalePath = exports.groupLocaleFiles = exports.findLocaleFileSets = exports.DEFAULT_LOCALE_IGNORE_DIRS = exports.glossaryFingerprint = exports.normalizeGlossary = exports.localeValue = exports.containsTerm = exports.canonicalizeLocaleKeys = exports.canonicalLocale = exports.tallyLocaleIssues = exports.flattenLocaleMessages = exports.checkLocalePair = exports.frameworkSupport = exports.analyzeWorkspace = exports.analyzeProject = exports.localeExtension = exports.getCircuitBreaker = exports.CircuitState = void 0;
|
|
26
26
|
var BabelizeEngine_1 = require("./BabelizeEngine");
|
|
27
27
|
Object.defineProperty(exports, "BabelizeEngine", { enumerable: true, get: function () { return BabelizeEngine_1.BabelizeEngine; } });
|
|
28
28
|
var config_1 = require("./config");
|
|
@@ -46,6 +46,9 @@ Object.defineProperty(exports, "runWithEngineLogger", { enumerable: true, get: f
|
|
|
46
46
|
Object.defineProperty(exports, "setEngineLogger", { enumerable: true, get: function () { return logger_1.setEngineLogger; } });
|
|
47
47
|
// ── Pipeline pieces the backend uses directly ────────────────
|
|
48
48
|
__exportStar(require("./modules/lockfile"), exports);
|
|
49
|
+
var overrides_1 = require("./modules/overrides");
|
|
50
|
+
Object.defineProperty(exports, "applyTranslationOverrides", { enumerable: true, get: function () { return overrides_1.applyTranslationOverrides; } });
|
|
51
|
+
Object.defineProperty(exports, "setLocaleValue", { enumerable: true, get: function () { return overrides_1.setLocaleValue; } });
|
|
49
52
|
__exportStar(require("./modules/translator/language-codes"), exports);
|
|
50
53
|
var memory_matcher_1 = require("./modules/translator/memory/memory-matcher");
|
|
51
54
|
Object.defineProperty(exports, "matchFromMemory", { enumerable: true, get: function () { return memory_matcher_1.matchFromMemory; } });
|
|
@@ -64,6 +67,10 @@ var stack_1 = require("./detection/stack");
|
|
|
64
67
|
Object.defineProperty(exports, "detectStack", { enumerable: true, get: function () { return stack_1.detectStack; } });
|
|
65
68
|
Object.defineProperty(exports, "localRepoFiles", { enumerable: true, get: function () { return stack_1.localRepoFiles; } });
|
|
66
69
|
Object.defineProperty(exports, "FRAMEWORK_DISPLAY_NAMES", { enumerable: true, get: function () { return stack_1.FRAMEWORK_DISPLAY_NAMES; } });
|
|
70
|
+
Object.defineProperty(exports, "SUPPORTED_SUMMARY", { enumerable: true, get: function () { return stack_1.SUPPORTED_SUMMARY; } });
|
|
71
|
+
var i18n_1 = require("./detection/i18n");
|
|
72
|
+
Object.defineProperty(exports, "detectI18n", { enumerable: true, get: function () { return i18n_1.detectI18n; } });
|
|
73
|
+
Object.defineProperty(exports, "I18N_LIBRARY_NAMES", { enumerable: true, get: function () { return i18n_1.I18N_LIBRARY_NAMES; } });
|
|
67
74
|
var ast_helpers_1 = require("./utils/ast-helpers");
|
|
68
75
|
Object.defineProperty(exports, "parseCode", { enumerable: true, get: function () { return ast_helpers_1.parseCode; } });
|
|
69
76
|
var string_utils_1 = require("./utils/string-utils");
|
|
@@ -2,7 +2,8 @@ import type { KeyResult, TranslationBatchRequest, TranslationProvider } from '..
|
|
|
2
2
|
/**
|
|
3
3
|
* Babelize-hosted translation for `babelize translate`: a TranslationProvider
|
|
4
4
|
* that sends the engine's already placeholder-protected strings (never code)
|
|
5
|
-
* to POST {apiUrl}/v1/translate/batch with a Babelize API key
|
|
5
|
+
* to POST {apiUrl}/v1/translate/batch with a Babelize API key or a signed-in
|
|
6
|
+
* CLI session (the endpoint accepts both and bills the same account). The engine
|
|
6
7
|
* still validates every result and runs its repair retry locally; model,
|
|
7
8
|
* quota and credits are decided by the server.
|
|
8
9
|
*/
|
|
@@ -36,8 +37,19 @@ export declare class HostedTranslationError extends Error {
|
|
|
36
37
|
get fatal(): boolean;
|
|
37
38
|
}
|
|
38
39
|
export interface HostedProviderOptions {
|
|
39
|
-
/** Babelize API key (bblz_sk_…). Never printed. */
|
|
40
|
+
/** Babelize API key (bblz_sk_…) or a session access token. Never printed. */
|
|
40
41
|
apiKey: string;
|
|
42
|
+
/**
|
|
43
|
+
* What `apiKey` is, for the message when it's rejected: an API key from
|
|
44
|
+
* BABELIZE_API_KEY (default) or the CLI's signed-in session.
|
|
45
|
+
*/
|
|
46
|
+
credential?: 'api_key' | 'session';
|
|
47
|
+
/**
|
|
48
|
+
* Called once when a request is rejected with 401: return a fresh token
|
|
49
|
+
* (a session refresh) to retry with it, or null to give up. Sessions expire
|
|
50
|
+
* during long runs; API keys don't, so API-key callers leave this out.
|
|
51
|
+
*/
|
|
52
|
+
refreshToken?: () => Promise<string | null>;
|
|
41
53
|
/** e.g. https://api.babelize.co/api */
|
|
42
54
|
apiUrl: string;
|
|
43
55
|
fetch?: typeof fetch;
|
|
@@ -57,10 +69,26 @@ export declare function parseRetryAfterMs(value: string | null): number | undefi
|
|
|
57
69
|
export declare class HostedTranslationProvider implements TranslationProvider {
|
|
58
70
|
/** Model the server reported (X-Babelize-Model), once a request succeeded. */
|
|
59
71
|
model?: string;
|
|
72
|
+
/**
|
|
73
|
+
* True when the server translates with NLLB rather than an LLM: the engine
|
|
74
|
+
* then splits ICU messages and multi-line text and protects glossary terms
|
|
75
|
+
* before sending, as it does for cloud runs. Set by detectCapabilities().
|
|
76
|
+
*/
|
|
77
|
+
plainText: boolean;
|
|
60
78
|
/** Set once a request fails in a way no retry can fix (bad key, no credits). */
|
|
61
79
|
fatalError?: HostedTranslationError;
|
|
62
80
|
private readonly opts;
|
|
81
|
+
/** The token in use; replaced when a session is refreshed. */
|
|
82
|
+
private token;
|
|
83
|
+
/** In-flight refresh, shared by concurrent batches that all got a 401. */
|
|
84
|
+
private refreshing?;
|
|
63
85
|
constructor(options: HostedProviderOptions);
|
|
86
|
+
/**
|
|
87
|
+
* Ask the server what serves translation (GET /v1/translate/capabilities)
|
|
88
|
+
* before a run, so requests are shaped for it. A server without the endpoint
|
|
89
|
+
* (older) or any failure keeps the LLM shape; the run itself reports errors.
|
|
90
|
+
*/
|
|
91
|
+
detectCapabilities(): Promise<void>;
|
|
64
92
|
translateBatch(request: TranslationBatchRequest): Promise<KeyResult[]>;
|
|
65
93
|
private post;
|
|
66
94
|
private backoff;
|
|
@@ -8,7 +8,8 @@ const types_1 = require("@babelize/types");
|
|
|
8
8
|
/**
|
|
9
9
|
* Babelize-hosted translation for `babelize translate`: a TranslationProvider
|
|
10
10
|
* that sends the engine's already placeholder-protected strings (never code)
|
|
11
|
-
* to POST {apiUrl}/v1/translate/batch with a Babelize API key
|
|
11
|
+
* to POST {apiUrl}/v1/translate/batch with a Babelize API key or a signed-in
|
|
12
|
+
* CLI session (the endpoint accepts both and bills the same account). The engine
|
|
12
13
|
* still validates every result and runs its repair retry locally; model,
|
|
13
14
|
* quota and credits are decided by the server.
|
|
14
15
|
*/
|
|
@@ -48,6 +49,13 @@ function parseRetryAfterMs(value) {
|
|
|
48
49
|
}
|
|
49
50
|
class HostedTranslationProvider {
|
|
50
51
|
constructor(options) {
|
|
52
|
+
/**
|
|
53
|
+
* True when the server translates with NLLB rather than an LLM: the engine
|
|
54
|
+
* then splits ICU messages and multi-line text and protects glossary terms
|
|
55
|
+
* before sending, as it does for cloud runs. Set by detectCapabilities().
|
|
56
|
+
*/
|
|
57
|
+
this.plainText = false;
|
|
58
|
+
this.token = options.apiKey;
|
|
51
59
|
this.opts = {
|
|
52
60
|
fetch: globalThis.fetch.bind(globalThis),
|
|
53
61
|
maxRetries: 3,
|
|
@@ -59,6 +67,31 @@ class HostedTranslationProvider {
|
|
|
59
67
|
apiUrl: options.apiUrl.replace(/\/+$/, ''),
|
|
60
68
|
};
|
|
61
69
|
}
|
|
70
|
+
/**
|
|
71
|
+
* Ask the server what serves translation (GET /v1/translate/capabilities)
|
|
72
|
+
* before a run, so requests are shaped for it. A server without the endpoint
|
|
73
|
+
* (older) or any failure keeps the LLM shape; the run itself reports errors.
|
|
74
|
+
*/
|
|
75
|
+
async detectCapabilities() {
|
|
76
|
+
try {
|
|
77
|
+
const res = await this.opts.fetch(`${this.opts.apiUrl}/v1/translate/capabilities`, {
|
|
78
|
+
headers: {
|
|
79
|
+
Authorization: `Bearer ${this.token}`,
|
|
80
|
+
...(this.opts.userAgent ? { 'User-Agent': this.opts.userAgent } : {}),
|
|
81
|
+
},
|
|
82
|
+
signal: AbortSignal.timeout(15000),
|
|
83
|
+
});
|
|
84
|
+
if (!res.ok)
|
|
85
|
+
return;
|
|
86
|
+
const body = (await res.json());
|
|
87
|
+
this.plainText = body.plainText === true;
|
|
88
|
+
if (body.model)
|
|
89
|
+
this.model ?? (this.model = body.model);
|
|
90
|
+
}
|
|
91
|
+
catch {
|
|
92
|
+
// keep the defaults
|
|
93
|
+
}
|
|
94
|
+
}
|
|
62
95
|
async translateBatch(request) {
|
|
63
96
|
// Strings over the server's limits never leave the machine: they fail
|
|
64
97
|
// locally, and the rest of the batch is still sent.
|
|
@@ -122,6 +155,7 @@ class HostedTranslationProvider {
|
|
|
122
155
|
// replays it instead of translating and charging again.
|
|
123
156
|
const idempotencyKey = (0, node_crypto_1.randomUUID)();
|
|
124
157
|
let attempt = 0;
|
|
158
|
+
let refreshed = false;
|
|
125
159
|
for (;;) {
|
|
126
160
|
let res;
|
|
127
161
|
try {
|
|
@@ -129,7 +163,7 @@ class HostedTranslationProvider {
|
|
|
129
163
|
method: 'POST',
|
|
130
164
|
headers: {
|
|
131
165
|
'Content-Type': 'application/json',
|
|
132
|
-
Authorization: `Bearer ${this.
|
|
166
|
+
Authorization: `Bearer ${this.token}`,
|
|
133
167
|
'Idempotency-Key': idempotencyKey,
|
|
134
168
|
...(this.opts.userAgent ? { 'User-Agent': this.opts.userAgent } : {}),
|
|
135
169
|
},
|
|
@@ -162,6 +196,17 @@ class HostedTranslationProvider {
|
|
|
162
196
|
this.model = res.headers.get(exports.HOSTED_MODEL_HEADER) ?? this.model;
|
|
163
197
|
return Array.isArray(data.results) ? data.results : [];
|
|
164
198
|
}
|
|
199
|
+
// An expired session: refresh it once and try again with the same key.
|
|
200
|
+
if (res.status === 401 && this.opts.refreshToken && !refreshed) {
|
|
201
|
+
refreshed = true;
|
|
202
|
+
const sent = this.token;
|
|
203
|
+
this.refreshing ?? (this.refreshing = this.opts.refreshToken().finally(() => { this.refreshing = undefined; }));
|
|
204
|
+
const fresh = this.token !== sent ? this.token : await this.refreshing;
|
|
205
|
+
if (fresh) {
|
|
206
|
+
this.token = fresh;
|
|
207
|
+
continue;
|
|
208
|
+
}
|
|
209
|
+
}
|
|
165
210
|
if (RETRYABLE_STATUS.has(res.status) && attempt < this.opts.maxRetries && !(await isHostedOff(res))) {
|
|
166
211
|
const retryAfter = parseRetryAfterMs(res.headers.get('retry-after'));
|
|
167
212
|
await this.opts.sleep(Math.min(retryAfter ?? this.backoff(attempt), this.opts.maxRetryAfterMs));
|
|
@@ -183,7 +228,9 @@ class HostedTranslationProvider {
|
|
|
183
228
|
switch (res.status) {
|
|
184
229
|
case 401:
|
|
185
230
|
case 403:
|
|
186
|
-
return new HostedTranslationError(
|
|
231
|
+
return new HostedTranslationError(this.opts.credential === 'session'
|
|
232
|
+
? `Your Babelize sign-in was rejected (${res.status}). Sign in again with \`babelize auth login\`, then re-run.`
|
|
233
|
+
: `BABELIZE_API_KEY was rejected (${res.status}). Create a new key at ${exports.API_KEYS_URL} and set BABELIZE_API_KEY.`, 'hosted_auth', res.status);
|
|
187
234
|
case 402:
|
|
188
235
|
return new HostedTranslationError(payload?.message ??
|
|
189
236
|
`Your Babelize account is out of AI credits. Top up or upgrade at ${exports.BILLING_URL}, or use your own LLM key (e.g. set ANTHROPIC_API_KEY).`, 'hosted_credits', 402);
|
|
@@ -283,6 +283,10 @@ async function runScoped(opts, deps, engineLog, state) {
|
|
|
283
283
|
if (!opts.byok && !translator) {
|
|
284
284
|
throw new LocalTranslateError('No translation provider configured.', 'no_translator');
|
|
285
285
|
}
|
|
286
|
+
// Shape requests for what the server translates with (NLLB needs ICU and
|
|
287
|
+
// multi-line text split and glossary terms protected; an LLM doesn't).
|
|
288
|
+
if (deps.hosted)
|
|
289
|
+
await deps.hosted.detectCapabilities();
|
|
286
290
|
const files = await listProjectFiles(projectDir);
|
|
287
291
|
const temp = await (0, workspace_copy_1.createTempCopy)(projectDir, { files, tmpRoot: deps.tmpRoot });
|
|
288
292
|
try {
|
|
@@ -1,3 +1,3 @@
|
|
|
1
1
|
export { hashSourceString } from './hasher';
|
|
2
2
|
export { type LockfileData, type LockfileEntry, type LockfilePartition, type LockfileStats, type PartitionedKey, createEmptyLockfile, } from './schema';
|
|
3
|
-
export { buildKeysToTranslateByTargetLocale, clearLockfile, removeLocaleFromLockfile, collectCachedTranslations, computeLockfileStats, getLockfilePath, loadLockfile, partitionKeys, saveLockfile, updateLockfile, lockfileChanged, dropFailedTranslations, isStalePseudoEntry, glossaryChanged, type LockfileReadOptions, } from './lockfile-manager';
|
|
3
|
+
export { buildKeysToTranslateByTargetLocale, clearLockfile, removeLocaleFromLockfile, removeLockfileEntries, collectCachedTranslations, computeLockfileStats, getLockfilePath, loadLockfile, partitionKeys, saveLockfile, updateLockfile, lockfileChanged, dropFailedTranslations, isStalePseudoEntry, glossaryChanged, type LockfileReadOptions, } from './lockfile-manager';
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.glossaryChanged = exports.isStalePseudoEntry = exports.dropFailedTranslations = exports.lockfileChanged = exports.updateLockfile = exports.saveLockfile = exports.partitionKeys = exports.loadLockfile = exports.getLockfilePath = exports.computeLockfileStats = exports.collectCachedTranslations = exports.removeLocaleFromLockfile = exports.clearLockfile = exports.buildKeysToTranslateByTargetLocale = exports.createEmptyLockfile = exports.hashSourceString = void 0;
|
|
3
|
+
exports.glossaryChanged = exports.isStalePseudoEntry = exports.dropFailedTranslations = exports.lockfileChanged = exports.updateLockfile = exports.saveLockfile = exports.partitionKeys = exports.loadLockfile = exports.getLockfilePath = exports.computeLockfileStats = exports.collectCachedTranslations = exports.removeLockfileEntries = exports.removeLocaleFromLockfile = exports.clearLockfile = exports.buildKeysToTranslateByTargetLocale = exports.createEmptyLockfile = exports.hashSourceString = void 0;
|
|
4
4
|
var hasher_1 = require("./hasher");
|
|
5
5
|
Object.defineProperty(exports, "hashSourceString", { enumerable: true, get: function () { return hasher_1.hashSourceString; } });
|
|
6
6
|
var schema_1 = require("./schema");
|
|
@@ -9,6 +9,7 @@ var lockfile_manager_1 = require("./lockfile-manager");
|
|
|
9
9
|
Object.defineProperty(exports, "buildKeysToTranslateByTargetLocale", { enumerable: true, get: function () { return lockfile_manager_1.buildKeysToTranslateByTargetLocale; } });
|
|
10
10
|
Object.defineProperty(exports, "clearLockfile", { enumerable: true, get: function () { return lockfile_manager_1.clearLockfile; } });
|
|
11
11
|
Object.defineProperty(exports, "removeLocaleFromLockfile", { enumerable: true, get: function () { return lockfile_manager_1.removeLocaleFromLockfile; } });
|
|
12
|
+
Object.defineProperty(exports, "removeLockfileEntries", { enumerable: true, get: function () { return lockfile_manager_1.removeLockfileEntries; } });
|
|
12
13
|
Object.defineProperty(exports, "collectCachedTranslations", { enumerable: true, get: function () { return lockfile_manager_1.collectCachedTranslations; } });
|
|
13
14
|
Object.defineProperty(exports, "computeLockfileStats", { enumerable: true, get: function () { return lockfile_manager_1.computeLockfileStats; } });
|
|
14
15
|
Object.defineProperty(exports, "getLockfilePath", { enumerable: true, get: function () { return lockfile_manager_1.getLockfilePath; } });
|
|
@@ -32,6 +32,15 @@ export declare function clearLockfile(projectId: string, dataDir?: string): Prom
|
|
|
32
32
|
* Used when a language is removed from a project so re-adding it
|
|
33
33
|
* triggers fresh translation instead of resurrecting stale cache.
|
|
34
34
|
*/
|
|
35
|
+
/**
|
|
36
|
+
* Forget the stored translation of some (key, locale) pairs, so the next run
|
|
37
|
+
* translates them again (a dashboard edit that was reverted). Other locales
|
|
38
|
+
* of the key stay.
|
|
39
|
+
*/
|
|
40
|
+
export declare function removeLockfileEntries(projectId: string, entries: Array<{
|
|
41
|
+
key: string;
|
|
42
|
+
locale: string;
|
|
43
|
+
}>, dataDir?: string): Promise<number>;
|
|
35
44
|
export declare function removeLocaleFromLockfile(projectId: string, localeCode: string, dataDir?: string): Promise<void>;
|
|
36
45
|
/**
|
|
37
46
|
* Partition generated keys into unchanged / new / changed / missingLocales
|
|
@@ -38,6 +38,7 @@ exports.getLockfilePath = getLockfilePath;
|
|
|
38
38
|
exports.loadLockfile = loadLockfile;
|
|
39
39
|
exports.saveLockfile = saveLockfile;
|
|
40
40
|
exports.clearLockfile = clearLockfile;
|
|
41
|
+
exports.removeLockfileEntries = removeLockfileEntries;
|
|
41
42
|
exports.removeLocaleFromLockfile = removeLocaleFromLockfile;
|
|
42
43
|
exports.glossaryChanged = glossaryChanged;
|
|
43
44
|
exports.isStalePseudoEntry = isStalePseudoEntry;
|
|
@@ -242,6 +243,44 @@ async function clearLockfile(projectId, dataDir) {
|
|
|
242
243
|
* Used when a language is removed from a project so re-adding it
|
|
243
244
|
* triggers fresh translation instead of resurrecting stale cache.
|
|
244
245
|
*/
|
|
246
|
+
/**
|
|
247
|
+
* Forget the stored translation of some (key, locale) pairs, so the next run
|
|
248
|
+
* translates them again (a dashboard edit that was reverted). Other locales
|
|
249
|
+
* of the key stay.
|
|
250
|
+
*/
|
|
251
|
+
async function removeLockfileEntries(projectId, entries, dataDir) {
|
|
252
|
+
const lockfilePath = getLockfilePath(projectId, dataDir);
|
|
253
|
+
if (entries.length === 0 || !(await fs.pathExists(lockfilePath)))
|
|
254
|
+
return 0;
|
|
255
|
+
const lockDir = lockfilePath + '.lock';
|
|
256
|
+
await acquireLock(lockDir);
|
|
257
|
+
try {
|
|
258
|
+
// Read under the lock: a run saving meanwhile must not be lost.
|
|
259
|
+
const lockfile = await loadLockfile(projectId, dataDir);
|
|
260
|
+
let removed = 0;
|
|
261
|
+
for (const { key, locale } of entries) {
|
|
262
|
+
const entry = lockfile.keys[key];
|
|
263
|
+
if (!entry || !(locale in entry.languages))
|
|
264
|
+
continue;
|
|
265
|
+
delete entry.languages[locale];
|
|
266
|
+
if (entry.metadata)
|
|
267
|
+
delete entry.metadata[locale];
|
|
268
|
+
if (Object.keys(entry.languages).length === 0)
|
|
269
|
+
delete lockfile.keys[key];
|
|
270
|
+
removed++;
|
|
271
|
+
}
|
|
272
|
+
if (removed > 0) {
|
|
273
|
+
lockfile.updatedAt = new Date().toISOString();
|
|
274
|
+
const tmpPath = lockfilePath + '.tmp';
|
|
275
|
+
await fs.writeJson(tmpPath, lockfile, { spaces: 2 });
|
|
276
|
+
await fs.rename(tmpPath, lockfilePath);
|
|
277
|
+
}
|
|
278
|
+
return removed;
|
|
279
|
+
}
|
|
280
|
+
finally {
|
|
281
|
+
await releaseLock(lockDir);
|
|
282
|
+
}
|
|
283
|
+
}
|
|
245
284
|
async function removeLocaleFromLockfile(projectId, localeCode, dataDir) {
|
|
246
285
|
const lockfilePath = getLockfilePath(projectId, dataDir);
|
|
247
286
|
try {
|
|
@@ -297,7 +336,8 @@ function glossaryChanged(entry, source, locale, options = {}) {
|
|
|
297
336
|
if (!options.glossaryHash)
|
|
298
337
|
return false;
|
|
299
338
|
const meta = entry.metadata?.[locale];
|
|
300
|
-
|
|
339
|
+
// The project's own files, or a dashboard edit: the glossary doesn't rewrite them.
|
|
340
|
+
if (meta?.external || meta?.manual)
|
|
301
341
|
return false;
|
|
302
342
|
return (meta?.glossaryHash ?? '') !== options.glossaryHash(source, locale);
|
|
303
343
|
}
|
|
@@ -394,6 +434,8 @@ function pickLockfileMetadata(meta) {
|
|
|
394
434
|
out.glossaryHash = meta.glossaryHash;
|
|
395
435
|
if (meta.external === true)
|
|
396
436
|
out.external = true;
|
|
437
|
+
if (meta.manual === true)
|
|
438
|
+
out.manual = true;
|
|
397
439
|
return out;
|
|
398
440
|
}
|
|
399
441
|
function sameJson(a, b) {
|
|
@@ -25,6 +25,8 @@ export declare const lockfileEntrySchema: z.ZodObject<{
|
|
|
25
25
|
* catalogue), not from Babelize: glossary edits don't overwrite it.
|
|
26
26
|
*/
|
|
27
27
|
external: z.ZodOptional<z.ZodBoolean>;
|
|
28
|
+
/** Edited in the dashboard (a translation override), not translated by Babelize. */
|
|
29
|
+
manual: z.ZodOptional<z.ZodBoolean>;
|
|
28
30
|
}, "strip", z.ZodTypeAny, {
|
|
29
31
|
sourceHash?: string | undefined;
|
|
30
32
|
fromMemory?: boolean | undefined;
|
|
@@ -32,6 +34,7 @@ export declare const lockfileEntrySchema: z.ZodObject<{
|
|
|
32
34
|
pseudo?: boolean | undefined;
|
|
33
35
|
glossaryHash?: string | undefined;
|
|
34
36
|
external?: boolean | undefined;
|
|
37
|
+
manual?: boolean | undefined;
|
|
35
38
|
}, {
|
|
36
39
|
sourceHash?: string | undefined;
|
|
37
40
|
fromMemory?: boolean | undefined;
|
|
@@ -39,6 +42,7 @@ export declare const lockfileEntrySchema: z.ZodObject<{
|
|
|
39
42
|
pseudo?: boolean | undefined;
|
|
40
43
|
glossaryHash?: string | undefined;
|
|
41
44
|
external?: boolean | undefined;
|
|
45
|
+
manual?: boolean | undefined;
|
|
42
46
|
}>>>;
|
|
43
47
|
}, "strip", z.ZodTypeAny, {
|
|
44
48
|
sourceHash: string;
|
|
@@ -51,6 +55,7 @@ export declare const lockfileEntrySchema: z.ZodObject<{
|
|
|
51
55
|
pseudo?: boolean | undefined;
|
|
52
56
|
glossaryHash?: string | undefined;
|
|
53
57
|
external?: boolean | undefined;
|
|
58
|
+
manual?: boolean | undefined;
|
|
54
59
|
}> | undefined;
|
|
55
60
|
}, {
|
|
56
61
|
sourceHash: string;
|
|
@@ -63,6 +68,7 @@ export declare const lockfileEntrySchema: z.ZodObject<{
|
|
|
63
68
|
pseudo?: boolean | undefined;
|
|
64
69
|
glossaryHash?: string | undefined;
|
|
65
70
|
external?: boolean | undefined;
|
|
71
|
+
manual?: boolean | undefined;
|
|
66
72
|
}> | undefined;
|
|
67
73
|
}>;
|
|
68
74
|
export type LockfileEntry = z.infer<typeof lockfileEntrySchema>;
|
|
@@ -96,6 +102,8 @@ export declare const lockfileDataSchema: z.ZodObject<{
|
|
|
96
102
|
* catalogue), not from Babelize: glossary edits don't overwrite it.
|
|
97
103
|
*/
|
|
98
104
|
external: z.ZodOptional<z.ZodBoolean>;
|
|
105
|
+
/** Edited in the dashboard (a translation override), not translated by Babelize. */
|
|
106
|
+
manual: z.ZodOptional<z.ZodBoolean>;
|
|
99
107
|
}, "strip", z.ZodTypeAny, {
|
|
100
108
|
sourceHash?: string | undefined;
|
|
101
109
|
fromMemory?: boolean | undefined;
|
|
@@ -103,6 +111,7 @@ export declare const lockfileDataSchema: z.ZodObject<{
|
|
|
103
111
|
pseudo?: boolean | undefined;
|
|
104
112
|
glossaryHash?: string | undefined;
|
|
105
113
|
external?: boolean | undefined;
|
|
114
|
+
manual?: boolean | undefined;
|
|
106
115
|
}, {
|
|
107
116
|
sourceHash?: string | undefined;
|
|
108
117
|
fromMemory?: boolean | undefined;
|
|
@@ -110,6 +119,7 @@ export declare const lockfileDataSchema: z.ZodObject<{
|
|
|
110
119
|
pseudo?: boolean | undefined;
|
|
111
120
|
glossaryHash?: string | undefined;
|
|
112
121
|
external?: boolean | undefined;
|
|
122
|
+
manual?: boolean | undefined;
|
|
113
123
|
}>>>;
|
|
114
124
|
}, "strip", z.ZodTypeAny, {
|
|
115
125
|
sourceHash: string;
|
|
@@ -122,6 +132,7 @@ export declare const lockfileDataSchema: z.ZodObject<{
|
|
|
122
132
|
pseudo?: boolean | undefined;
|
|
123
133
|
glossaryHash?: string | undefined;
|
|
124
134
|
external?: boolean | undefined;
|
|
135
|
+
manual?: boolean | undefined;
|
|
125
136
|
}> | undefined;
|
|
126
137
|
}, {
|
|
127
138
|
sourceHash: string;
|
|
@@ -134,6 +145,7 @@ export declare const lockfileDataSchema: z.ZodObject<{
|
|
|
134
145
|
pseudo?: boolean | undefined;
|
|
135
146
|
glossaryHash?: string | undefined;
|
|
136
147
|
external?: boolean | undefined;
|
|
148
|
+
manual?: boolean | undefined;
|
|
137
149
|
}> | undefined;
|
|
138
150
|
}>>;
|
|
139
151
|
}, "strip", z.ZodTypeAny, {
|
|
@@ -148,6 +160,7 @@ export declare const lockfileDataSchema: z.ZodObject<{
|
|
|
148
160
|
pseudo?: boolean | undefined;
|
|
149
161
|
glossaryHash?: string | undefined;
|
|
150
162
|
external?: boolean | undefined;
|
|
163
|
+
manual?: boolean | undefined;
|
|
151
164
|
}> | undefined;
|
|
152
165
|
}>;
|
|
153
166
|
projectId: string;
|
|
@@ -165,6 +178,7 @@ export declare const lockfileDataSchema: z.ZodObject<{
|
|
|
165
178
|
pseudo?: boolean | undefined;
|
|
166
179
|
glossaryHash?: string | undefined;
|
|
167
180
|
external?: boolean | undefined;
|
|
181
|
+
manual?: boolean | undefined;
|
|
168
182
|
}> | undefined;
|
|
169
183
|
}>;
|
|
170
184
|
projectId: string;
|
|
@@ -32,6 +32,8 @@ exports.lockfileEntrySchema = zod_1.z.object({
|
|
|
32
32
|
* catalogue), not from Babelize: glossary edits don't overwrite it.
|
|
33
33
|
*/
|
|
34
34
|
external: zod_1.z.boolean().optional(),
|
|
35
|
+
/** Edited in the dashboard (a translation override), not translated by Babelize. */
|
|
36
|
+
manual: zod_1.z.boolean().optional(),
|
|
35
37
|
})).optional(),
|
|
36
38
|
});
|
|
37
39
|
// ─────────────────────────────────────────────────────────────
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Translation overrides: translations a person edited in the dashboard.
|
|
3
|
+
*
|
|
4
|
+
* They win over the lockfile, translation memory and the translator, on every
|
|
5
|
+
* run, for as long as the source text is the one that was edited. When the
|
|
6
|
+
* source text changes the edit no longer fits: it is not applied, the key is
|
|
7
|
+
* translated as usual, and the run reports it as replaced.
|
|
8
|
+
*/
|
|
9
|
+
import type { LocaleData } from '../../runtime/types';
|
|
10
|
+
export interface TranslationOverride {
|
|
11
|
+
key: string;
|
|
12
|
+
locale: string;
|
|
13
|
+
text: string;
|
|
14
|
+
/** The source text the edit was made against. */
|
|
15
|
+
sourceText: string;
|
|
16
|
+
}
|
|
17
|
+
export interface OverrideReport {
|
|
18
|
+
/** Edits written into this run's locale files. */
|
|
19
|
+
applied: Array<{
|
|
20
|
+
key: string;
|
|
21
|
+
locale: string;
|
|
22
|
+
}>;
|
|
23
|
+
/** Edits whose source text has changed since: not applied, translated as usual. */
|
|
24
|
+
replaced: Array<{
|
|
25
|
+
key: string;
|
|
26
|
+
locale: string;
|
|
27
|
+
text: string;
|
|
28
|
+
sourceText: string;
|
|
29
|
+
}>;
|
|
30
|
+
}
|
|
31
|
+
/** Set `key` in locale data: a literal flat key when the file has one, else a nested path. */
|
|
32
|
+
export declare function setLocaleValue(data: LocaleData, key: string, value: string): void;
|
|
33
|
+
/**
|
|
34
|
+
* Apply edits to the run's locale data in place. `source` is the flat source
|
|
35
|
+
* locale (key → text). Edits for a key the run no longer has, or a locale it
|
|
36
|
+
* doesn't write, are left alone (kept for when they come back).
|
|
37
|
+
*/
|
|
38
|
+
export declare function applyTranslationOverrides(locales: Array<{
|
|
39
|
+
locale: string;
|
|
40
|
+
data: LocaleData;
|
|
41
|
+
}>, source: Record<string, string>, overrides: TranslationOverride[]): OverrideReport;
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* Translation overrides: translations a person edited in the dashboard.
|
|
4
|
+
*
|
|
5
|
+
* They win over the lockfile, translation memory and the translator, on every
|
|
6
|
+
* run, for as long as the source text is the one that was edited. When the
|
|
7
|
+
* source text changes the edit no longer fits: it is not applied, the key is
|
|
8
|
+
* translated as usual, and the run reports it as replaced.
|
|
9
|
+
*/
|
|
10
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
11
|
+
exports.setLocaleValue = setLocaleValue;
|
|
12
|
+
exports.applyTranslationOverrides = applyTranslationOverrides;
|
|
13
|
+
/** Set `key` in locale data: a literal flat key when the file has one, else a nested path. */
|
|
14
|
+
function setLocaleValue(data, key, value) {
|
|
15
|
+
if (typeof data[key] === 'string' || !key.includes('.')) {
|
|
16
|
+
data[key] = value;
|
|
17
|
+
return;
|
|
18
|
+
}
|
|
19
|
+
const parts = key.split('.');
|
|
20
|
+
let current = data;
|
|
21
|
+
for (let i = 0; i < parts.length - 1; i++) {
|
|
22
|
+
const part = parts[i];
|
|
23
|
+
if (typeof current[part] !== 'object' || current[part] === null)
|
|
24
|
+
current[part] = {};
|
|
25
|
+
current = current[part];
|
|
26
|
+
}
|
|
27
|
+
current[parts[parts.length - 1]] = value;
|
|
28
|
+
}
|
|
29
|
+
/**
|
|
30
|
+
* Apply edits to the run's locale data in place. `source` is the flat source
|
|
31
|
+
* locale (key → text). Edits for a key the run no longer has, or a locale it
|
|
32
|
+
* doesn't write, are left alone (kept for when they come back).
|
|
33
|
+
*/
|
|
34
|
+
function applyTranslationOverrides(locales, source, overrides) {
|
|
35
|
+
const report = { applied: [], replaced: [] };
|
|
36
|
+
const byLocale = new Map(locales.map((l) => [l.locale, l]));
|
|
37
|
+
for (const o of overrides) {
|
|
38
|
+
const target = byLocale.get(o.locale);
|
|
39
|
+
const current = source[o.key];
|
|
40
|
+
if (!target || current === undefined)
|
|
41
|
+
continue;
|
|
42
|
+
if (current !== o.sourceText) {
|
|
43
|
+
report.replaced.push({ key: o.key, locale: o.locale, text: o.text, sourceText: o.sourceText });
|
|
44
|
+
continue;
|
|
45
|
+
}
|
|
46
|
+
setLocaleValue(target.data, o.key, o.text);
|
|
47
|
+
report.applied.push({ key: o.key, locale: o.locale });
|
|
48
|
+
}
|
|
49
|
+
return report;
|
|
50
|
+
}
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* A run's translation budget: the most (key, locale) pairs it may send to the
|
|
3
|
+
* translator (the caller's credit budget, see EngineConfigV2.maxTranslations).
|
|
4
|
+
*
|
|
5
|
+
* Locales are translated concurrently, so the budget is granted in slot order
|
|
6
|
+
* (the locale's position in the run) rather than first come, first served:
|
|
7
|
+
* each locale claims what it needs once its cache and translation-memory
|
|
8
|
+
* lookups are done, and a claim is only settled after every earlier slot has
|
|
9
|
+
* claimed. The same project and budget therefore always translate the same
|
|
10
|
+
* pairs: earlier languages are completed first, keys in source order.
|
|
11
|
+
*
|
|
12
|
+
* Every slot must claim exactly once (0 when it has nothing to send), or the
|
|
13
|
+
* slots after it wait forever.
|
|
14
|
+
*/
|
|
15
|
+
export declare class TranslationBudget {
|
|
16
|
+
private remaining;
|
|
17
|
+
private nextSlot;
|
|
18
|
+
private readonly pending;
|
|
19
|
+
constructor(limit: number);
|
|
20
|
+
/** Resolves with how many of `wanted` pairs slot `slot` may translate. */
|
|
21
|
+
claim(slot: number, wanted: number): Promise<number>;
|
|
22
|
+
private settle;
|
|
23
|
+
}
|
|
24
|
+
/** A `maxTranslations` value that caps the run: a finite number >= 0. */
|
|
25
|
+
export declare function isTranslationCap(max: number | undefined): max is number;
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.TranslationBudget = void 0;
|
|
4
|
+
exports.isTranslationCap = isTranslationCap;
|
|
5
|
+
/**
|
|
6
|
+
* A run's translation budget: the most (key, locale) pairs it may send to the
|
|
7
|
+
* translator (the caller's credit budget, see EngineConfigV2.maxTranslations).
|
|
8
|
+
*
|
|
9
|
+
* Locales are translated concurrently, so the budget is granted in slot order
|
|
10
|
+
* (the locale's position in the run) rather than first come, first served:
|
|
11
|
+
* each locale claims what it needs once its cache and translation-memory
|
|
12
|
+
* lookups are done, and a claim is only settled after every earlier slot has
|
|
13
|
+
* claimed. The same project and budget therefore always translate the same
|
|
14
|
+
* pairs: earlier languages are completed first, keys in source order.
|
|
15
|
+
*
|
|
16
|
+
* Every slot must claim exactly once (0 when it has nothing to send), or the
|
|
17
|
+
* slots after it wait forever.
|
|
18
|
+
*/
|
|
19
|
+
class TranslationBudget {
|
|
20
|
+
constructor(limit) {
|
|
21
|
+
this.nextSlot = 0;
|
|
22
|
+
this.pending = new Map();
|
|
23
|
+
this.remaining = Math.max(0, Math.floor(limit));
|
|
24
|
+
}
|
|
25
|
+
/** Resolves with how many of `wanted` pairs slot `slot` may translate. */
|
|
26
|
+
claim(slot, wanted) {
|
|
27
|
+
return new Promise((resolve) => {
|
|
28
|
+
this.pending.set(slot, { wanted: Math.max(0, Math.floor(wanted)), resolve });
|
|
29
|
+
this.settle();
|
|
30
|
+
});
|
|
31
|
+
}
|
|
32
|
+
settle() {
|
|
33
|
+
for (let next = this.pending.get(this.nextSlot); next; next = this.pending.get(this.nextSlot)) {
|
|
34
|
+
this.pending.delete(this.nextSlot);
|
|
35
|
+
this.nextSlot++;
|
|
36
|
+
const granted = Math.min(next.wanted, this.remaining);
|
|
37
|
+
this.remaining -= granted;
|
|
38
|
+
next.resolve(granted);
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
exports.TranslationBudget = TranslationBudget;
|
|
43
|
+
/** A `maxTranslations` value that caps the run: a finite number >= 0. */
|
|
44
|
+
function isTranslationCap(max) {
|
|
45
|
+
return typeof max === 'number' && Number.isFinite(max) && max >= 0;
|
|
46
|
+
}
|
|
@@ -32,6 +32,13 @@ export interface TranslateOptions {
|
|
|
32
32
|
progress?: {
|
|
33
33
|
freshTranslations: number;
|
|
34
34
|
};
|
|
35
|
+
/**
|
|
36
|
+
* Most (key, locale) pairs to send to the translator this call (the run's
|
|
37
|
+
* credit budget). Cache, TM and passthrough reuse don't count. Locales are
|
|
38
|
+
* filled in `targetLocales` order, keys in source order; the rest are
|
|
39
|
+
* returned as `skippedForBudget`. Undefined or negative: no cap.
|
|
40
|
+
*/
|
|
41
|
+
maxTranslations?: number;
|
|
35
42
|
}
|
|
36
43
|
export interface LoadedTranslationRules {
|
|
37
44
|
glossary: GlossaryRules;
|
|
@@ -59,6 +66,10 @@ export declare function translateLocales(workspace: WorkspaceInfo, sourceLocale:
|
|
|
59
66
|
error: string;
|
|
60
67
|
}[];
|
|
61
68
|
failures: TranslationFailure[];
|
|
69
|
+
skippedForBudget: {
|
|
70
|
+
key: string;
|
|
71
|
+
locale: string;
|
|
72
|
+
}[];
|
|
62
73
|
};
|
|
63
74
|
stats: {
|
|
64
75
|
tmHits: number;
|
|
@@ -24,6 +24,7 @@ const translation_sanitizer_1 = require("../../utils/translation-sanitizer");
|
|
|
24
24
|
const glossary_match_1 = require("../../utils/glossary-match");
|
|
25
25
|
const icu_spans_1 = require("../../utils/icu-spans");
|
|
26
26
|
const icu_branches_1 = require("./icu-branches");
|
|
27
|
+
const budget_1 = require("./budget");
|
|
27
28
|
const BATCH_SIZE = 15;
|
|
28
29
|
/** QA note on keys the backup (NLLB) translator produced. */
|
|
29
30
|
exports.FALLBACK_QA_WARNING = 'translated with the backup translator; will be retried with the AI translator next run';
|
|
@@ -214,6 +215,7 @@ async function translateLocales(workspace, sourceLocale, targetLocales, forceTra
|
|
|
214
215
|
const locales = [];
|
|
215
216
|
const errors = [];
|
|
216
217
|
const failures = [];
|
|
218
|
+
const skippedForBudget = [];
|
|
217
219
|
let tmHitsTotal = 0;
|
|
218
220
|
const perLocaleMetadata = {};
|
|
219
221
|
// Ensure locale output directories exist for all target locales
|
|
@@ -248,7 +250,23 @@ async function translateLocales(workspace, sourceLocale, targetLocales, forceTra
|
|
|
248
250
|
}
|
|
249
251
|
const localeTasks = targetLocales
|
|
250
252
|
.filter((l) => l !== sourceLocale.locale && (0, language_codes_1.isSupported)(l));
|
|
251
|
-
|
|
253
|
+
// The run's translation budget, granted in locale order (see TranslationBudget).
|
|
254
|
+
const budget = (0, budget_1.isTranslationCap)(options.maxTranslations) ? new budget_1.TranslationBudget(options.maxTranslations) : null;
|
|
255
|
+
// Every locale claims its share of the budget exactly once, even when it has
|
|
256
|
+
// nothing to send or fails, or the locales after it never get theirs.
|
|
257
|
+
const withBudget = (fn) => async (targetLocale, slot) => {
|
|
258
|
+
if (!budget)
|
|
259
|
+
return fn(targetLocale);
|
|
260
|
+
let claimed = false;
|
|
261
|
+
try {
|
|
262
|
+
return await fn(targetLocale, (wanted) => { claimed = true; return budget.claim(slot, wanted); });
|
|
263
|
+
}
|
|
264
|
+
finally {
|
|
265
|
+
if (!claimed)
|
|
266
|
+
void budget.claim(slot, 0);
|
|
267
|
+
}
|
|
268
|
+
};
|
|
269
|
+
const { results: localeResults, errors: localeErrors } = await (0, concurrent_map_1.concurrentMap)(localeTasks, withBudget(async (targetLocale, claimBudget) => {
|
|
252
270
|
const targetLangCode = (0, language_codes_1.toFlores)(targetLocale);
|
|
253
271
|
const localePaths = (0, file_utils_1.localeFilePath)(workspace.path, workspace.publicDir, workspace.localeFormat, targetLocale);
|
|
254
272
|
const targetPath = localePaths.file;
|
|
@@ -269,6 +287,7 @@ async function translateLocales(workspace, sourceLocale, targetLocales, forceTra
|
|
|
269
287
|
tmHits: 0,
|
|
270
288
|
metadata: {},
|
|
271
289
|
failures: {},
|
|
290
|
+
skippedForBudget: [],
|
|
272
291
|
error: null,
|
|
273
292
|
};
|
|
274
293
|
}
|
|
@@ -276,7 +295,7 @@ async function translateLocales(workspace, sourceLocale, targetLocales, forceTra
|
|
|
276
295
|
// LLM translators (BYOK, or an injected one such as Babelize-hosted)
|
|
277
296
|
// take the brand voice in the prompt; NLLB output is polished after.
|
|
278
297
|
const hasBrandVoiceInPrompt = !!((byokConfig || (providers.translator && !providers.translator.plainText)) && brandVoice);
|
|
279
|
-
const translationResult = await translateEntries(entriesToTranslate, sourceLangCode, targetLangCode, projectId, onProgress, glossary, targetLocale, cacheProvider, byokConfig, contextMap, extractionContextMap, hasBrandVoiceInPrompt ? (brandVoice || undefined) : undefined, dryRun, forceTranslation, providers, rulesComplete, options);
|
|
298
|
+
const translationResult = await translateEntries(entriesToTranslate, sourceLangCode, targetLangCode, projectId, onProgress, glossary, targetLocale, cacheProvider, byokConfig, contextMap, extractionContextMap, hasBrandVoiceInPrompt ? (brandVoice || undefined) : undefined, dryRun, forceTranslation, providers, rulesComplete, options, claimBudget);
|
|
280
299
|
const entriesResult = translationResult.result;
|
|
281
300
|
const batchStats = translationResult.stats;
|
|
282
301
|
const batchMetadata = translationResult.metadata;
|
|
@@ -284,10 +303,13 @@ async function translateLocales(workspace, sourceLocale, targetLocales, forceTra
|
|
|
284
303
|
if (Object.keys(entriesResult).length === 0 && Object.keys(entriesToTranslate).length > 0) {
|
|
285
304
|
// Nothing translated: write no file. Every key is in keyFailures with
|
|
286
305
|
// its reason, so the caller can report exactly what went wrong.
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
306
|
+
// (Not an error when the run's budget simply didn't reach this locale.)
|
|
307
|
+
if (Object.keys(keyFailures).length > 0 || translationResult.skippedForBudget.length === 0) {
|
|
308
|
+
logger_1.engineLogger.error(`No strings could be translated for locale "${targetLocale}"`, {
|
|
309
|
+
locale: targetLocale, sourceKeys: Object.keys(entriesToTranslate).length,
|
|
310
|
+
});
|
|
311
|
+
}
|
|
312
|
+
return { locale: targetLocale, file: null, tmHits: batchStats.tmHits, metadata: batchMetadata, failures: keyFailures, skippedForBudget: translationResult.skippedForBudget, error: null };
|
|
291
313
|
}
|
|
292
314
|
let translatedEntries = entriesResult;
|
|
293
315
|
// Post-hoc brand voice polish only for the NLLB service; LLM
|
|
@@ -355,15 +377,18 @@ async function translateLocales(workspace, sourceLocale, targetLocales, forceTra
|
|
|
355
377
|
tmHits: batchStats.tmHits,
|
|
356
378
|
metadata: batchMetadata,
|
|
357
379
|
failures: keyFailures,
|
|
380
|
+
skippedForBudget: translationResult.skippedForBudget,
|
|
358
381
|
error: null,
|
|
359
382
|
};
|
|
360
|
-
}, TRANSLATE_CONCURRENCY);
|
|
383
|
+
}), TRANSLATE_CONCURRENCY);
|
|
361
384
|
for (const r of localeResults) {
|
|
362
385
|
if (!r)
|
|
363
386
|
continue;
|
|
364
387
|
for (const [key, f] of Object.entries(r.failures ?? {})) {
|
|
365
388
|
failures.push({ locale: r.locale, key, source: flatSource[key] ?? '', ...f });
|
|
366
389
|
}
|
|
390
|
+
for (const key of r.skippedForBudget ?? [])
|
|
391
|
+
skippedForBudget.push({ key, locale: r.locale });
|
|
367
392
|
if (r.error) {
|
|
368
393
|
errors.push({ locale: r.locale, error: r.error });
|
|
369
394
|
}
|
|
@@ -385,13 +410,19 @@ async function translateLocales(workspace, sourceLocale, targetLocales, forceTra
|
|
|
385
410
|
if (byokConfig && (errors.length > 0 || localeErrors.length > 0)) {
|
|
386
411
|
throw localeErrors.length > 0 ? localeErrors[0] : new Error(errors[0].error);
|
|
387
412
|
}
|
|
413
|
+
if (skippedForBudget.length > 0) {
|
|
414
|
+
logger_1.engineLogger.warn('Translation budget reached; remaining strings are left for the next run', {
|
|
415
|
+
maxTranslations: options.maxTranslations,
|
|
416
|
+
skipped: skippedForBudget.length,
|
|
417
|
+
});
|
|
418
|
+
}
|
|
388
419
|
return {
|
|
389
|
-
translation: { locales, errors, failures },
|
|
420
|
+
translation: { locales, errors, failures, skippedForBudget },
|
|
390
421
|
stats: { tmHits: tmHitsTotal },
|
|
391
422
|
metadata: perLocaleMetadata
|
|
392
423
|
};
|
|
393
424
|
}
|
|
394
|
-
async function translateEntries(entries, sourceLang, targetLang, projectId, onProgress, glossary = glossary_1.EMPTY_GLOSSARY, targetLocale = '', cacheProvider, byokConfig, contextMap, extractionContextMap, brandVoice, dryRun = false, forceTranslation = false, providers = {}, rulesComplete = true, options = {}) {
|
|
425
|
+
async function translateEntries(entries, sourceLang, targetLang, projectId, onProgress, glossary = glossary_1.EMPTY_GLOSSARY, targetLocale = '', cacheProvider, byokConfig, contextMap, extractionContextMap, brandVoice, dryRun = false, forceTranslation = false, providers = {}, rulesComplete = true, options = {}, claimBudget) {
|
|
395
426
|
const result = {};
|
|
396
427
|
const metadata = {};
|
|
397
428
|
const keys = Object.keys(entries);
|
|
@@ -546,6 +577,7 @@ async function translateEntries(entries, sourceLang, targetLang, projectId, onPr
|
|
|
546
577
|
});
|
|
547
578
|
}
|
|
548
579
|
const failures = {};
|
|
580
|
+
const skippedForBudget = [];
|
|
549
581
|
// LLM providers get structure-aware prompts and repair attempts. The NLLB
|
|
550
582
|
// service translates plain text only: ICU messages go branch by branch
|
|
551
583
|
// (./icu-branches), and mixed-script text fails up front.
|
|
@@ -618,6 +650,14 @@ async function translateEntries(entries, sourceLang, targetLang, projectId, onPr
|
|
|
618
650
|
: undefined,
|
|
619
651
|
});
|
|
620
652
|
}
|
|
653
|
+
// 1b. The run's budget: only as many keys as it covers go to the
|
|
654
|
+
// translator (in source order). The rest get no value, like a failed
|
|
655
|
+
// key: not in the locale file or lockfile, so the next run retries them.
|
|
656
|
+
if (claimBudget) {
|
|
657
|
+
const granted = await claimBudget(prepared.length);
|
|
658
|
+
for (const entry of prepared.splice(granted))
|
|
659
|
+
skippedForBudget.push(entry.key);
|
|
660
|
+
}
|
|
621
661
|
// 2. Translate in batches; judge every key independently.
|
|
622
662
|
const outcomes = new Map();
|
|
623
663
|
const judge = (entry, keyResult, nllb = !isLlm) => {
|
|
@@ -970,7 +1010,7 @@ async function translateEntries(entries, sourceLang, targetLang, projectId, onPr
|
|
|
970
1010
|
meta.qa = [...(meta.qa ?? []), ...notes];
|
|
971
1011
|
metadata[key] = meta;
|
|
972
1012
|
}
|
|
973
|
-
return { result, stats: { tmHits }, metadata, failures };
|
|
1013
|
+
return { result, stats: { tmHits }, metadata, failures, skippedForBudget };
|
|
974
1014
|
}
|
|
975
1015
|
/**
|
|
976
1016
|
* Persist translations to the project owner's TM. Never throws: a TM failure
|
package/dist/orchestrator.js
CHANGED
|
@@ -52,6 +52,7 @@ const finalization_1 = require("./modules/finalization");
|
|
|
52
52
|
const lockfile_1 = require("./modules/lockfile");
|
|
53
53
|
const file_utils_1 = require("./utils/file-utils");
|
|
54
54
|
const key_usage_1 = require("./utils/key-usage");
|
|
55
|
+
const overrides_1 = require("./modules/overrides");
|
|
55
56
|
const not_injected_1 = require("./utils/not-injected");
|
|
56
57
|
const language_codes_1 = require("./modules/translator/language-codes");
|
|
57
58
|
const format_injection_1 = require("./modules/format-injection");
|
|
@@ -180,6 +181,8 @@ async function runEngineV2(config) {
|
|
|
180
181
|
// (key, locale) translations the translator produced this run — the
|
|
181
182
|
// billable work. Lockfile/TM/cache reuse and failures are excluded.
|
|
182
183
|
let translatedThisRun = 0;
|
|
184
|
+
// (key, locale) pairs left for the next run: the run's maxTranslations ran out.
|
|
185
|
+
let skippedForBudget = 0;
|
|
183
186
|
let keyUsage;
|
|
184
187
|
let translationMetadata = {};
|
|
185
188
|
if (config.projectId) {
|
|
@@ -239,6 +242,20 @@ async function runEngineV2(config) {
|
|
|
239
242
|
// lack some locales (a newly added language): their existing locales are
|
|
240
243
|
// still valid and must not drop out of the other locale files.
|
|
241
244
|
const cachedTranslations = (0, lockfile_1.collectCachedTranslations)(lockfile, [...partition.unchanged, ...partition.missingLocales], config.targetLocales, lockfileRead);
|
|
245
|
+
// Dashboard edits (overrides), loaded before translating: a pair with a
|
|
246
|
+
// current edit is never sent to the translator (it would be thrown away).
|
|
247
|
+
let overrides = [];
|
|
248
|
+
if (config.providers?.overrides) {
|
|
249
|
+
try {
|
|
250
|
+
overrides = await config.providers.overrides.getOverrides(config.projectId);
|
|
251
|
+
}
|
|
252
|
+
catch (err) {
|
|
253
|
+
logger_1.engineLogger.warn('Could not load translation overrides; translating without them', {
|
|
254
|
+
error: err instanceof Error ? err.message : String(err),
|
|
255
|
+
});
|
|
256
|
+
}
|
|
257
|
+
}
|
|
258
|
+
const overridden = new Set(overrides.filter((o) => flatKeys[o.key] === o.sourceText).map((o) => `${o.locale}\u0000${o.key}`));
|
|
242
259
|
config.onProgress?.('Translating to target locales...', 'translation', 'running');
|
|
243
260
|
const translatorHealthy = await isTranslatorHealthy(config);
|
|
244
261
|
if (!translatorHealthy) {
|
|
@@ -259,13 +276,19 @@ async function runEngineV2(config) {
|
|
|
259
276
|
}
|
|
260
277
|
else if (translatorHealthy) {
|
|
261
278
|
const keysToTranslateByLocale = (0, lockfile_1.buildKeysToTranslateByTargetLocale)(partition, config.targetLocales);
|
|
262
|
-
|
|
279
|
+
if (overridden.size > 0) {
|
|
280
|
+
for (const [locale, keys] of Object.entries(keysToTranslateByLocale)) {
|
|
281
|
+
keysToTranslateByLocale[locale] = keys.filter((k) => !overridden.has(`${locale}\u0000${k}`));
|
|
282
|
+
}
|
|
283
|
+
}
|
|
284
|
+
const result = await (0, translate_1.translateLocales)(workspace, localeHandling.sourceLocale, config.targetLocales, config.forceTranslation, config.dryRun, config.projectId, (msg) => config.onProgress?.(msg, 'translation', 'running'), keysToTranslateByLocale, config.providers?.cache, config.byok, contextMap, extractionContextMap, config.providers, { targetPlurals: icuOptions.targetPlurals, pluralValuePlaceholders: gettext?.pluralValuePlaceholders(), rules, progress: translationProgress, maxTranslations: config.maxTranslations });
|
|
263
285
|
translation = result.translation;
|
|
264
286
|
// A translation that wouldn't compile into the .po file is a failure, not output.
|
|
265
287
|
gettext?.validate(translation, flatKeys);
|
|
266
288
|
tmHits = result.stats.tmHits;
|
|
267
289
|
translationMetadata = result.metadata;
|
|
268
290
|
translatedThisRun = freshTranslations(translation, translationMetadata);
|
|
291
|
+
skippedForBudget = translation.skippedForBudget?.length ?? 0;
|
|
269
292
|
// Strings reused from the lockfile still make a good run: only fail
|
|
270
293
|
// when nothing at all is usable. Otherwise one string that keeps
|
|
271
294
|
// failing (and is retried every run) failed the whole project.
|
|
@@ -326,6 +349,24 @@ async function runEngineV2(config) {
|
|
|
326
349
|
locale.data._meta = { dir: (0, language_codes_1.isRTL)(locale.locale) ? 'rtl' : 'ltr', language: locale.locale };
|
|
327
350
|
}
|
|
328
351
|
}
|
|
352
|
+
// Dashboard edits win over the lockfile and the translator (while their
|
|
353
|
+
// source text is unchanged); the lockfile marks them as manual.
|
|
354
|
+
let overrideReport;
|
|
355
|
+
if (overrides.length > 0) {
|
|
356
|
+
overrideReport = (0, overrides_1.applyTranslationOverrides)(translation.locales, flatKeys, overrides);
|
|
357
|
+
for (const { key, locale } of overrideReport.applied) {
|
|
358
|
+
translationMetadata[locale] = { ...(translationMetadata[locale] ?? {}) };
|
|
359
|
+
// Keep what the lockfile knew (glossary fingerprint, inTm): losing it would re-translate the key every run.
|
|
360
|
+
// Not a fallback translation (that flag would drop it from the lockfile).
|
|
361
|
+
const kept = { ...(lockfile.keys[key]?.metadata?.[locale] ?? {}), ...(translationMetadata[locale][key] ?? {}) };
|
|
362
|
+
delete kept.fallback;
|
|
363
|
+
translationMetadata[locale][key] = { ...kept, manual: true };
|
|
364
|
+
}
|
|
365
|
+
logger_1.engineLogger.info('Applied translation overrides', {
|
|
366
|
+
applied: overrideReport.applied.length,
|
|
367
|
+
replaced: overrideReport.replaced.length,
|
|
368
|
+
});
|
|
369
|
+
}
|
|
329
370
|
// The translator wrote only this run's strings to disk; write the merged
|
|
330
371
|
// locale (lockfile + new) so the workspace — which the download ZIP is
|
|
331
372
|
// built from — has every translation, including fully cached locales.
|
|
@@ -384,6 +425,9 @@ async function runEngineV2(config) {
|
|
|
384
425
|
const previousLockfile = lockfile;
|
|
385
426
|
lockfile = (0, lockfile_1.updateLockfile)(lockfile, flatKeys, allTranslatedPerLocale, config.projectId, lockfileMetadata);
|
|
386
427
|
lockfile = (0, lockfile_1.dropFailedTranslations)(lockfile, translation.failures ?? []);
|
|
428
|
+
// Pairs the budget didn't cover are retried next run the same way
|
|
429
|
+
// (a changed key must not keep its old translation as current).
|
|
430
|
+
lockfile = (0, lockfile_1.dropFailedTranslations)(lockfile, translation.skippedForBudget ?? []);
|
|
387
431
|
// Backup-translator (NLLB) output is shipped in the locale file but not
|
|
388
432
|
// recorded as done, so the next run retries it with the AI translator.
|
|
389
433
|
lockfile = (0, lockfile_1.dropFailedTranslations)(lockfile, fallbackTranslations(translationMetadata));
|
|
@@ -403,8 +447,8 @@ async function runEngineV2(config) {
|
|
|
403
447
|
extraction, keyGeneration, localeHandling, translation,
|
|
404
448
|
config, startTime, workspace, allTranslatedPerLocale,
|
|
405
449
|
lockfileStats, tmHits, translationMetadata,
|
|
406
|
-
translatedThisRun, keyUsage,
|
|
407
|
-
formatStrings, frameworkConfig, gettext,
|
|
450
|
+
translatedThisRun, skippedForBudget, keyUsage,
|
|
451
|
+
formatStrings, frameworkConfig, gettext, overrideReport,
|
|
408
452
|
});
|
|
409
453
|
return result;
|
|
410
454
|
}
|
|
@@ -429,7 +473,7 @@ async function runEngineV2(config) {
|
|
|
429
473
|
config.onProgress?.(msg, 'translation', 'warning');
|
|
430
474
|
}
|
|
431
475
|
const translationResult = translatorHealthy
|
|
432
|
-
? await (0, translate_1.translateLocales)(workspace, localeHandling.sourceLocale, config.targetLocales, config.forceTranslation, config.dryRun, config.projectId, (msg) => config.onProgress?.(msg, 'translation', 'running'), undefined, config.providers?.cache, config.byok, contextMap, extractionContextMap, config.providers, { targetPlurals: icuOptions.targetPlurals, pluralValuePlaceholders: gettext?.pluralValuePlaceholders(), progress: translationProgress })
|
|
476
|
+
? await (0, translate_1.translateLocales)(workspace, localeHandling.sourceLocale, config.targetLocales, config.forceTranslation, config.dryRun, config.projectId, (msg) => config.onProgress?.(msg, 'translation', 'running'), undefined, config.providers?.cache, config.byok, contextMap, extractionContextMap, config.providers, { targetPlurals: icuOptions.targetPlurals, pluralValuePlaceholders: gettext?.pluralValuePlaceholders(), progress: translationProgress, maxTranslations: config.maxTranslations })
|
|
433
477
|
: {
|
|
434
478
|
translation: {
|
|
435
479
|
locales: [],
|
|
@@ -443,6 +487,7 @@ async function runEngineV2(config) {
|
|
|
443
487
|
tmHits = translationResult.stats.tmHits;
|
|
444
488
|
translationMetadata = translationResult.metadata;
|
|
445
489
|
translatedThisRun = freshTranslations(translation, translationMetadata);
|
|
490
|
+
skippedForBudget = translation.skippedForBudget?.length ?? 0;
|
|
446
491
|
assertSomethingTranslated(translation);
|
|
447
492
|
// Fail immediately when translation errors occurred — before injection/finalization
|
|
448
493
|
if (translation.errors && translation.errors.length > 0) {
|
|
@@ -458,7 +503,7 @@ async function runEngineV2(config) {
|
|
|
458
503
|
extraction, keyGeneration, localeHandling, translation,
|
|
459
504
|
config, startTime, workspace, allTranslatedPerLocale,
|
|
460
505
|
lockfileStats: undefined, tmHits, translationMetadata,
|
|
461
|
-
translatedThisRun, keyUsage,
|
|
506
|
+
translatedThisRun, skippedForBudget, keyUsage,
|
|
462
507
|
formatStrings, frameworkConfig, gettext,
|
|
463
508
|
});
|
|
464
509
|
return result;
|
|
@@ -621,7 +666,7 @@ function mergeChangedFile(list, changed) {
|
|
|
621
666
|
existing.hookAdded = existing.hookAdded || changed.hookAdded;
|
|
622
667
|
}
|
|
623
668
|
async function finalizePipeline(params) {
|
|
624
|
-
const { extraction, keyGeneration, localeHandling, translation, config, startTime, workspace, allTranslatedPerLocale, lockfileStats, tmHits, translationMetadata, formatStrings, frameworkConfig, translatedThisRun, keyUsage, gettext, } = params;
|
|
669
|
+
const { extraction, keyGeneration, localeHandling, translation, config, startTime, workspace, allTranslatedPerLocale, lockfileStats, tmHits, translationMetadata, formatStrings, frameworkConfig, translatedThisRun, skippedForBudget, keyUsage, gettext, overrideReport, } = params;
|
|
625
670
|
const keyMapping = (0, keygen_1.createKeyMapping)(keyGeneration.keys);
|
|
626
671
|
// Injection: AST-based i18n hook injection (skipped for frameworks like Django)
|
|
627
672
|
let injection;
|
|
@@ -701,6 +746,8 @@ async function finalizePipeline(params) {
|
|
|
701
746
|
config.onProgress?.('Generating summary...', 'summary', 'running');
|
|
702
747
|
const { stats, warnings, diffs } = (0, summary_1.generateSummary)(extraction, keyGeneration, localeHandling, translation, injection, startTime, lockfileStats, tmHits);
|
|
703
748
|
stats.translatedThisRun = Math.max(0, translatedThisRun);
|
|
749
|
+
if (skippedForBudget && skippedForBudget > 0)
|
|
750
|
+
stats.skippedForBudget = skippedForBudget;
|
|
704
751
|
logger_1.engineLogger.info('Processing complete', stats);
|
|
705
752
|
logger_1.engineLogger.debug('Readable summary', { summary: (0, summary_1.generateReadableSummary)(stats) });
|
|
706
753
|
const allLocales = [localeHandling.sourceLocale, ...translation.locales];
|
|
@@ -723,5 +770,6 @@ async function finalizePipeline(params) {
|
|
|
723
770
|
targets: allTranslatedPerLocale,
|
|
724
771
|
metadata: translationMetadata,
|
|
725
772
|
},
|
|
773
|
+
...(overrideReport ? { overrides: overrideReport } : {}),
|
|
726
774
|
};
|
|
727
775
|
}
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import type { TranslationEntry } from '../runtime/types';
|
|
2
2
|
import type { MemoryEntry } from '../modules/translator/memory/types';
|
|
3
3
|
import type { GlossaryEntryType } from '../modules/glossary/types';
|
|
4
|
+
import type { TranslationOverride } from '../modules/overrides';
|
|
4
5
|
/**
|
|
5
6
|
* Everything the engine needs from the outside world, as interfaces. The
|
|
6
7
|
* engine never imports a database or cache client: the cloud backend injects
|
|
@@ -129,6 +130,9 @@ export interface ProjectRulesStore {
|
|
|
129
130
|
getGlossary(projectId: string): Promise<GlossaryRule[]>;
|
|
130
131
|
getBrandVoice(projectId: string): Promise<string | null>;
|
|
131
132
|
}
|
|
133
|
+
export interface TranslationOverrideStore {
|
|
134
|
+
getOverrides(projectId: string): Promise<TranslationOverride[]>;
|
|
135
|
+
}
|
|
132
136
|
export interface TranslationCache {
|
|
133
137
|
get(key: string): Promise<{
|
|
134
138
|
hit: boolean;
|
|
@@ -150,4 +154,6 @@ export interface EngineProviders {
|
|
|
150
154
|
memory?: TranslationMemoryStore;
|
|
151
155
|
rules?: ProjectRulesStore;
|
|
152
156
|
cache?: TranslationCache;
|
|
157
|
+
/** Translations edited in the dashboard; they win over every other source. */
|
|
158
|
+
overrides?: TranslationOverrideStore;
|
|
153
159
|
}
|
package/dist/runtime/config.js
CHANGED
|
@@ -24,6 +24,7 @@ function validateEngineConfigV2(config) {
|
|
|
24
24
|
confidenceThreshold,
|
|
25
25
|
dryRun: config.dryRun ?? false,
|
|
26
26
|
forceTranslation: config.forceTranslation ?? false,
|
|
27
|
+
maxTranslations: config.maxTranslations,
|
|
27
28
|
onProgress: config.onProgress,
|
|
28
29
|
signal: config.signal,
|
|
29
30
|
projectId: config.projectId,
|
package/dist/runtime/types.d.ts
CHANGED
|
@@ -25,6 +25,15 @@ export interface EngineConfigV2 {
|
|
|
25
25
|
dryRun: boolean;
|
|
26
26
|
/** Force translation even if locale files exist */
|
|
27
27
|
forceTranslation?: boolean;
|
|
28
|
+
/**
|
|
29
|
+
* Most (key, locale) pairs this run may send to the translator: the
|
|
30
|
+
* caller's credit budget. Pairs past it are left untranslated (not in the
|
|
31
|
+
* locale files or lockfile, so the next run picks them up) and counted in
|
|
32
|
+
* `stats.skippedForBudget`. Lockfile, translation-memory, cache and
|
|
33
|
+
* passthrough reuse never count against it. Languages are filled in
|
|
34
|
+
* `targetLocales` order, keys in source order. Undefined or negative: no cap.
|
|
35
|
+
*/
|
|
36
|
+
maxTranslations?: number;
|
|
28
37
|
/** Project ID for scoping lockfile/memory */
|
|
29
38
|
projectId?: string;
|
|
30
39
|
/**
|
|
@@ -266,6 +275,11 @@ export interface TranslationResult {
|
|
|
266
275
|
}>;
|
|
267
276
|
/** Keys with no translation for a locale this run (not written, retried next run). */
|
|
268
277
|
failures?: TranslationFailure[];
|
|
278
|
+
/** (key, locale) pairs not sent to the translator because the run's `maxTranslations` ran out (retried next run). */
|
|
279
|
+
skippedForBudget?: Array<{
|
|
280
|
+
key: string;
|
|
281
|
+
locale: string;
|
|
282
|
+
}>;
|
|
269
283
|
}
|
|
270
284
|
export interface InjectionLocation {
|
|
271
285
|
file: string;
|
|
@@ -339,6 +353,11 @@ export interface ProcessingStats {
|
|
|
339
353
|
* and failed keys.
|
|
340
354
|
*/
|
|
341
355
|
translatedThisRun?: number;
|
|
356
|
+
/**
|
|
357
|
+
* (key, locale) translations left for the next run because the run's
|
|
358
|
+
* `maxTranslations` budget ran out. Set only when some were skipped.
|
|
359
|
+
*/
|
|
360
|
+
skippedForBudget?: number;
|
|
342
361
|
}
|
|
343
362
|
export interface NotInjectedString {
|
|
344
363
|
file: string;
|
|
@@ -398,6 +417,8 @@ export interface EngineResultV2 {
|
|
|
398
417
|
targets: Record<string, Record<string, string>>;
|
|
399
418
|
metadata?: Record<string, Record<string, any>>;
|
|
400
419
|
};
|
|
420
|
+
/** Dashboard edits this run applied, and those whose source text changed (lockfile runs only). */
|
|
421
|
+
overrides?: import('../modules/overrides').OverrideReport;
|
|
401
422
|
}
|
|
402
423
|
export interface Phase0Result {
|
|
403
424
|
workspace: WorkspaceInfo;
|
|
@@ -92,7 +92,7 @@ async function callTranslator(entries, sourceLang, targetLang, byokConfig, brand
|
|
|
92
92
|
if (needsFallback) {
|
|
93
93
|
logger_1.engineLogger.warn('Rust translator returned untargeted translations or unavailable, falling back to HF Inference API');
|
|
94
94
|
try {
|
|
95
|
-
const hfResults = await callHuggingFaceNLLB(entries, sourceLang, targetLang);
|
|
95
|
+
const hfResults = await callHuggingFaceNLLB(entries, sourceLang, targetLang, outerSignal);
|
|
96
96
|
if (hfResults.translations.length > 0) {
|
|
97
97
|
return { translations: hfResults.translations, metadata: { ...rustResult?.metadata, fallback: 'hf-inference-api' } };
|
|
98
98
|
}
|
|
@@ -137,12 +137,15 @@ async function callByokTranslator(entries, sourceLang, targetLang, byokConfig, b
|
|
|
137
137
|
}
|
|
138
138
|
return { translations, metadata: result.metadata };
|
|
139
139
|
}
|
|
140
|
-
async function callHuggingFaceNLLB(entries, sourceLang, targetLang
|
|
140
|
+
async function callHuggingFaceNLLB(entries, sourceLang, targetLang,
|
|
141
|
+
/** The caller gave up (time budget, client gone): stop instead of finishing every entry. */
|
|
142
|
+
outerSignal) {
|
|
141
143
|
const hfToken = process.env.HF_TOKEN || process.env.HF_API_TOKEN;
|
|
142
144
|
const model = 'facebook/nllb-200-distilled-600M';
|
|
143
145
|
const baseUrl = `https://api-inference.huggingface.co/models/${model}`;
|
|
144
146
|
const translations = [];
|
|
145
147
|
for (const entry of entries) {
|
|
148
|
+
outerSignal?.throwIfAborted();
|
|
146
149
|
const response = await fetch(baseUrl, {
|
|
147
150
|
method: 'POST',
|
|
148
151
|
headers: {
|
|
@@ -153,7 +156,7 @@ async function callHuggingFaceNLLB(entries, sourceLang, targetLang) {
|
|
|
153
156
|
inputs: entry.value,
|
|
154
157
|
parameters: { src_lang: sourceLang, tgt_lang: targetLang },
|
|
155
158
|
}),
|
|
156
|
-
signal: AbortSignal.timeout(60000),
|
|
159
|
+
signal: outerSignal ? AbortSignal.any([AbortSignal.timeout(60000), outerSignal]) : AbortSignal.timeout(60000),
|
|
157
160
|
});
|
|
158
161
|
if (!response.ok) {
|
|
159
162
|
const body = await response.text().catch(() => '');
|
package/package.json
CHANGED