@babelize/engine 0.2.1 → 0.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/detection/i18n.d.ts +19 -0
- package/dist/detection/i18n.js +144 -0
- package/dist/detection/stack.d.ts +2 -0
- package/dist/index.d.ts +4 -1
- package/dist/index.js +9 -2
- package/dist/local/hosted-provider.d.ts +34 -2
- package/dist/local/hosted-provider.js +58 -4
- package/dist/local/report.d.ts +2 -0
- package/dist/local/report.js +1 -0
- package/dist/local/translate-core.d.ts +2 -0
- package/dist/local/translate-core.js +5 -0
- package/dist/modules/lockfile/index.d.ts +1 -1
- package/dist/modules/lockfile/index.js +2 -1
- package/dist/modules/lockfile/lockfile-manager.d.ts +9 -0
- package/dist/modules/lockfile/lockfile-manager.js +43 -1
- package/dist/modules/lockfile/schema.d.ts +14 -0
- package/dist/modules/lockfile/schema.js +2 -0
- package/dist/modules/overrides/index.d.ts +41 -0
- package/dist/modules/overrides/index.js +50 -0
- package/dist/modules/translator/budget.d.ts +25 -0
- package/dist/modules/translator/budget.js +46 -0
- package/dist/modules/translator/icu-branches.d.ts +7 -0
- package/dist/modules/translator/icu-branches.js +29 -3
- package/dist/modules/translator/translate.d.ts +11 -0
- package/dist/modules/translator/translate.js +50 -10
- package/dist/orchestrator.js +54 -6
- package/dist/providers/types.d.ts +6 -0
- package/dist/runtime/config.js +1 -0
- package/dist/runtime/types.d.ts +21 -0
- package/dist/utils/icu-message.js +25 -5
- package/dist/utils/translator-caller.js +6 -3
- package/package.json +1 -1
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
import type { RepoFiles } from './stack';
|
|
2
|
+
export type I18nLibraryId = 'next-intl' | 'next-i18next' | 'react-i18next' | 'i18next' | 'vue-i18n' | 'react-intl' | 'lingui' | 'django';
|
|
3
|
+
export declare const I18N_LIBRARY_NAMES: Record<I18nLibraryId, string>;
|
|
4
|
+
export type SourceLanguageBasis = 'config' | 'files' | 'default';
|
|
5
|
+
export interface DetectedI18n {
|
|
6
|
+
library: {
|
|
7
|
+
id: I18nLibraryId;
|
|
8
|
+
name: string;
|
|
9
|
+
} | null;
|
|
10
|
+
/** Locale codes the repo already has files for, as written (en, pt-BR, zh_Hant). */
|
|
11
|
+
locales: string[];
|
|
12
|
+
/** Where the locale files live, e.g. "messages" or "public/locales". */
|
|
13
|
+
localeDir: string | null;
|
|
14
|
+
sourceLanguage: {
|
|
15
|
+
code: string;
|
|
16
|
+
basis: SourceLanguageBasis;
|
|
17
|
+
};
|
|
18
|
+
}
|
|
19
|
+
export declare function detectI18n(files: RepoFiles, framework?: string): Promise<DetectedI18n>;
|
|
@@ -0,0 +1,144 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.I18N_LIBRARY_NAMES = void 0;
|
|
4
|
+
exports.detectI18n = detectI18n;
|
|
5
|
+
/**
|
|
6
|
+
* What a repository already has for i18n, read from a file listing and a few
|
|
7
|
+
* small files (no clone): the i18n library, existing locale files, and the
|
|
8
|
+
* source language. Runs on the same `RepoFiles` as `detectStack`, so the
|
|
9
|
+
* new-project flow can say "Next.js · English · next-intl detected" and
|
|
10
|
+
* pre-select the languages the repo already ships.
|
|
11
|
+
*/
|
|
12
|
+
const locale_files_1 = require("../qa/locale-files");
|
|
13
|
+
exports.I18N_LIBRARY_NAMES = {
|
|
14
|
+
'next-intl': 'next-intl',
|
|
15
|
+
'next-i18next': 'next-i18next',
|
|
16
|
+
'react-i18next': 'react-i18next',
|
|
17
|
+
i18next: 'i18next',
|
|
18
|
+
'vue-i18n': 'vue-i18n',
|
|
19
|
+
'react-intl': 'react-intl',
|
|
20
|
+
lingui: 'Lingui',
|
|
21
|
+
django: 'Django i18n',
|
|
22
|
+
};
|
|
23
|
+
/** Most specific first: next-i18next depends on react-i18next, which depends on i18next. */
|
|
24
|
+
const LIBRARY_DEPS = [
|
|
25
|
+
['next-intl', ['next-intl']],
|
|
26
|
+
['next-i18next', ['next-i18next']],
|
|
27
|
+
['vue-i18n', ['vue-i18n', '@nuxtjs/i18n']],
|
|
28
|
+
['react-i18next', ['react-i18next']],
|
|
29
|
+
['react-intl', ['react-intl']],
|
|
30
|
+
['lingui', ['@lingui/core', '@lingui/react']],
|
|
31
|
+
['i18next', ['i18next']],
|
|
32
|
+
];
|
|
33
|
+
const IGNORED = /(?:^|\/)(?:node_modules|\.git|vendor|dist|build|\.next|out|coverage|__tests__|__fixtures__|fixtures|tests?|e2e)\//;
|
|
34
|
+
/** Files that commonly declare the default locale, checked in this order. */
|
|
35
|
+
const CONFIG_FILE = /(?:^|\/)(?:i18n(?:\/(?:routing|request|config|index))?|i18n\.config|next\.config|next-i18next\.config|nuxt\.config|src\/i18n(?:\/index)?|src\/plugins\/i18n|lingui\.config)\.(?:[cm]?[jt]s|json)$/;
|
|
36
|
+
/** defaultLocale: 'de', fallbackLng: "fr", sourceLocale: 'en', locale: 'en' (vue-i18n createI18n). */
|
|
37
|
+
const DEFAULT_LOCALE_PATTERNS = [
|
|
38
|
+
/\bdefaultLocale\s*[:=]\s*['"`]([A-Za-z_-]{2,12})['"`]/,
|
|
39
|
+
/\bsourceLocale\s*[:=]\s*['"`]([A-Za-z_-]{2,12})['"`]/,
|
|
40
|
+
/\bfallbackLng\s*[:=]\s*['"`]([A-Za-z_-]{2,12})['"`]/,
|
|
41
|
+
/\bfallbackLocale\s*[:=]\s*['"`]([A-Za-z_-]{2,12})['"`]/,
|
|
42
|
+
];
|
|
43
|
+
const DJANGO_LANGUAGE_CODE = /^\s*LANGUAGE_CODE\s*=\s*['"]([A-Za-z_-]{2,12})['"]/m;
|
|
44
|
+
/** Django: locale/<code>/LC_MESSAGES/django.po */
|
|
45
|
+
const DJANGO_PO = /(?:^|\/)locale\/([A-Za-z_-]{2,12})\/LC_MESSAGES\/django\.po$/;
|
|
46
|
+
function depth(p) {
|
|
47
|
+
return p.split('/').length - 1;
|
|
48
|
+
}
|
|
49
|
+
async function readDeps(files, paths) {
|
|
50
|
+
const pkgs = paths
|
|
51
|
+
.filter((p) => (p === 'package.json' || p.endsWith('/package.json')) && depth(p) <= 2)
|
|
52
|
+
.sort((a, b) => depth(a) - depth(b))
|
|
53
|
+
.slice(0, 6);
|
|
54
|
+
const deps = new Set();
|
|
55
|
+
for (const p of pkgs) {
|
|
56
|
+
const raw = await files.read(p);
|
|
57
|
+
if (!raw)
|
|
58
|
+
continue;
|
|
59
|
+
try {
|
|
60
|
+
const pkg = JSON.parse(raw);
|
|
61
|
+
for (const name of Object.keys({ ...pkg.dependencies, ...pkg.devDependencies }))
|
|
62
|
+
deps.add(name);
|
|
63
|
+
}
|
|
64
|
+
catch {
|
|
65
|
+
// not JSON: skip
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
return deps;
|
|
69
|
+
}
|
|
70
|
+
/** "en-us" (Django) → "en-US"; "pt_BR" stays as written. */
|
|
71
|
+
function tidyCode(code) {
|
|
72
|
+
const m = /^([a-z]{2,3})-([a-z]{2})$/.exec(code);
|
|
73
|
+
return m ? `${m[1]}-${m[2].toUpperCase()}` : code;
|
|
74
|
+
}
|
|
75
|
+
async function detectI18n(files, framework) {
|
|
76
|
+
const paths = files.paths.filter((p) => !IGNORED.test(p));
|
|
77
|
+
// Library
|
|
78
|
+
let library = null;
|
|
79
|
+
if (framework === 'django') {
|
|
80
|
+
library = { id: 'django', name: exports.I18N_LIBRARY_NAMES.django };
|
|
81
|
+
}
|
|
82
|
+
else {
|
|
83
|
+
const deps = await readDeps(files, paths);
|
|
84
|
+
const found = LIBRARY_DEPS.find(([, names]) => names.some((n) => deps.has(n)));
|
|
85
|
+
if (found)
|
|
86
|
+
library = { id: found[0], name: exports.I18N_LIBRARY_NAMES[found[0]] };
|
|
87
|
+
}
|
|
88
|
+
// Existing locale files: the set with the most locales wins.
|
|
89
|
+
let locales = [];
|
|
90
|
+
let localeDir = null;
|
|
91
|
+
let largestByLocale = new Map();
|
|
92
|
+
const sets = (0, locale_files_1.groupLocaleFiles)(paths);
|
|
93
|
+
if (sets.length > 0) {
|
|
94
|
+
const best = sets.reduce((a, b) => (Object.keys(b.files).length > Object.keys(a.files).length ? b : a));
|
|
95
|
+
locales = Object.keys(best.files).sort();
|
|
96
|
+
localeDir = best.dir;
|
|
97
|
+
if (files.size) {
|
|
98
|
+
largestByLocale = new Map(locales.map((l) => [l, best.files[l].reduce((sum, f) => sum + (files.size(f) ?? 0), 0)]));
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
else {
|
|
102
|
+
const po = paths.map((p) => DJANGO_PO.exec(p)).filter((m) => !!m);
|
|
103
|
+
if (po.length > 0) {
|
|
104
|
+
locales = [...new Set(po.map((m) => m[1]))].sort();
|
|
105
|
+
localeDir = po[0][0].split('/').slice(0, -3).join('/') || '.';
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
// Source language: declared in config, else inferred from the locale files.
|
|
109
|
+
let sourceLanguage = { code: 'en', basis: 'default' };
|
|
110
|
+
const configCandidates = paths
|
|
111
|
+
.filter((p) => CONFIG_FILE.test(p) && depth(p) <= 3)
|
|
112
|
+
.sort((a, b) => depth(a) - depth(b))
|
|
113
|
+
.slice(0, 6);
|
|
114
|
+
if (framework === 'django') {
|
|
115
|
+
configCandidates.unshift(...paths.filter((p) => /(?:^|\/)settings(?:\/base)?\.py$/.test(p) && depth(p) <= 3).slice(0, 2));
|
|
116
|
+
}
|
|
117
|
+
for (const p of configCandidates) {
|
|
118
|
+
const content = await files.read(p);
|
|
119
|
+
if (!content)
|
|
120
|
+
continue;
|
|
121
|
+
const m = p.endsWith('.py')
|
|
122
|
+
? DJANGO_LANGUAGE_CODE.exec(content)
|
|
123
|
+
: DEFAULT_LOCALE_PATTERNS.map((re) => re.exec(content)).find(Boolean);
|
|
124
|
+
const code = m?.[1] ? tidyCode(m[1]) : null;
|
|
125
|
+
if (code && (0, locale_files_1.isLocaleCode)(code)) {
|
|
126
|
+
sourceLanguage = { code, basis: 'config' };
|
|
127
|
+
break;
|
|
128
|
+
}
|
|
129
|
+
}
|
|
130
|
+
if (sourceLanguage.basis === 'default' && locales.length > 0) {
|
|
131
|
+
if (locales.length === 1) {
|
|
132
|
+
sourceLanguage = { code: locales[0], basis: 'files' };
|
|
133
|
+
}
|
|
134
|
+
else if (locales.some((l) => l === 'en' || l.startsWith('en-') || l.startsWith('en_'))) {
|
|
135
|
+
sourceLanguage = { code: locales.find((l) => l === 'en') ?? locales.find((l) => l.startsWith('en')), basis: 'files' };
|
|
136
|
+
}
|
|
137
|
+
else if (largestByLocale.size > 0) {
|
|
138
|
+
// The source usually has every key: the biggest file set.
|
|
139
|
+
const [code] = [...largestByLocale.entries()].reduce((a, b) => (b[1] > a[1] ? b : a));
|
|
140
|
+
sourceLanguage = { code, basis: 'files' };
|
|
141
|
+
}
|
|
142
|
+
}
|
|
143
|
+
return { library, locales, localeDir, sourceLanguage };
|
|
144
|
+
}
|
|
@@ -32,6 +32,8 @@ export interface RepoFiles {
|
|
|
32
32
|
read(relPath: string): Promise<string | null>;
|
|
33
33
|
/** The listing is incomplete (very large repository). */
|
|
34
34
|
truncated?: boolean;
|
|
35
|
+
/** File size in bytes, when the listing knows it. */
|
|
36
|
+
size?(relPath: string): number | undefined;
|
|
35
37
|
}
|
|
36
38
|
export declare const FRAMEWORK_DISPLAY_NAMES: Record<string, string>;
|
|
37
39
|
/** Does a Python dependency file list Django itself (not just django-* add-ons)? */
|
package/dist/index.d.ts
CHANGED
|
@@ -18,6 +18,7 @@ export * from './providers';
|
|
|
18
18
|
export { runWithEngineLogger, setEngineLogger } from './utils/logger';
|
|
19
19
|
export type { EngineLogger, LogContext } from './utils/logger';
|
|
20
20
|
export * from './modules/lockfile';
|
|
21
|
+
export { applyTranslationOverrides, setLocaleValue, type OverrideReport, type TranslationOverride } from './modules/overrides';
|
|
21
22
|
export * from './modules/translator/language-codes';
|
|
22
23
|
export type { MemoryEntry, FuzzyMatch, MemoryLookupResult } from './modules/translator/memory/types';
|
|
23
24
|
export { matchFromMemory } from './modules/translator/memory/memory-matcher';
|
|
@@ -26,8 +27,10 @@ export type { SyncDetectionResult, SyncDetectedString, SyncMissingLocaleKey } fr
|
|
|
26
27
|
export type { TranslationEntry, WorkspaceInfo } from './runtime/types';
|
|
27
28
|
export { addBabelToHistory, detectProjectInfo, getBabelizeSummary, getProjectInfo, saveCachedStrings, saveProjectConfig, } from './utils/metadata';
|
|
28
29
|
export type { BabelHistoryEntry, ExtractedStringEntry, ProjectInfo } from './utils/metadata';
|
|
29
|
-
export { detectStack, localRepoFiles, FRAMEWORK_DISPLAY_NAMES } from './detection/stack';
|
|
30
|
+
export { detectStack, localRepoFiles, FRAMEWORK_DISPLAY_NAMES, SUPPORTED_SUMMARY } from './detection/stack';
|
|
30
31
|
export type { DetectedStack, RepoFiles, StackFramework, PackageManagerId } from './detection/stack';
|
|
32
|
+
export { detectI18n, I18N_LIBRARY_NAMES } from './detection/i18n';
|
|
33
|
+
export type { DetectedI18n, I18nLibraryId, SourceLanguageBasis } from './detection/i18n';
|
|
31
34
|
export { parseCode } from './utils/ast-helpers';
|
|
32
35
|
export { extractPlaceholders, hasICUSyntax, hashString, protectPlaceholders, restorePlaceholders, } from './utils/string-utils';
|
|
33
36
|
export { sanitizeTranslation, validateRawTokens } from './utils/translation-sanitizer';
|
package/dist/index.js
CHANGED
|
@@ -21,8 +21,8 @@ var __exportStar = (this && this.__exportStar) || function(m, exports) {
|
|
|
21
21
|
for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
|
|
22
22
|
};
|
|
23
23
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
24
|
-
exports.
|
|
25
|
-
exports.TRANSLATION_CACHE_TTL_SECONDS = exports.translationCacheKey = exports.hashText = exports.readProjectLocaleConfig = exports.localeIssueFile = exports.checkProjectLocales = exports.ReviewError = exports.reviewLocaleChanges = exports.loadLocale = exports.isLocaleCode = exports.isIgnoredLocalePath = exports.groupLocaleFiles = exports.findLocaleFileSets = exports.DEFAULT_LOCALE_IGNORE_DIRS = exports.glossaryFingerprint = exports.normalizeGlossary = exports.localeValue = exports.containsTerm = exports.canonicalizeLocaleKeys = exports.canonicalLocale = exports.tallyLocaleIssues = exports.flattenLocaleMessages = exports.checkLocalePair = exports.frameworkSupport = void 0;
|
|
24
|
+
exports.CircuitBreaker = exports.validateBatchQuality = exports.validateTranslationQuality = exports.translatorLoadBalancer = exports.warmupTranslator = exports.checkTranslatorHealth = exports.callTranslator = exports.requiredPluralCategories = exports.parseIcu = exports.isIcuMessage = exports.compareIcuMessages = exports.sameProtectedSpans = exports.restoreText = exports.protectText = exports.hasNothingToTranslate = exports.checkTokens = exports.validateRawTokens = exports.sanitizeTranslation = exports.restorePlaceholders = exports.protectPlaceholders = exports.hashString = exports.hasICUSyntax = exports.extractPlaceholders = exports.parseCode = exports.I18N_LIBRARY_NAMES = exports.detectI18n = exports.SUPPORTED_SUMMARY = exports.FRAMEWORK_DISPLAY_NAMES = exports.localRepoFiles = exports.detectStack = exports.saveProjectConfig = exports.saveCachedStrings = exports.getProjectInfo = exports.getBabelizeSummary = exports.detectProjectInfo = exports.addBabelToHistory = exports.formatSyncNotificationMessage = exports.detectSyncIssues = exports.matchFromMemory = exports.setLocaleValue = exports.applyTranslationOverrides = exports.setEngineLogger = exports.runWithEngineLogger = exports.validateWorkspace = exports.scanKeyReferences = exports.RunAbortedError = exports.validateEngineConfigV2 = exports.runEngineV2 = exports.loadBabelizeConfig = exports.BabelizeEngine = void 0;
|
|
25
|
+
exports.TRANSLATION_CACHE_TTL_SECONDS = exports.translationCacheKey = exports.hashText = exports.readProjectLocaleConfig = exports.localeIssueFile = exports.checkProjectLocales = exports.ReviewError = exports.reviewLocaleChanges = exports.loadLocale = exports.isLocaleCode = exports.isIgnoredLocalePath = exports.groupLocaleFiles = exports.findLocaleFileSets = exports.DEFAULT_LOCALE_IGNORE_DIRS = exports.glossaryFingerprint = exports.normalizeGlossary = exports.localeValue = exports.containsTerm = exports.canonicalizeLocaleKeys = exports.canonicalLocale = exports.tallyLocaleIssues = exports.flattenLocaleMessages = exports.checkLocalePair = exports.frameworkSupport = exports.analyzeWorkspace = exports.analyzeProject = exports.localeExtension = exports.getCircuitBreaker = exports.CircuitState = void 0;
|
|
26
26
|
var BabelizeEngine_1 = require("./BabelizeEngine");
|
|
27
27
|
Object.defineProperty(exports, "BabelizeEngine", { enumerable: true, get: function () { return BabelizeEngine_1.BabelizeEngine; } });
|
|
28
28
|
var config_1 = require("./config");
|
|
@@ -46,6 +46,9 @@ Object.defineProperty(exports, "runWithEngineLogger", { enumerable: true, get: f
|
|
|
46
46
|
Object.defineProperty(exports, "setEngineLogger", { enumerable: true, get: function () { return logger_1.setEngineLogger; } });
|
|
47
47
|
// ── Pipeline pieces the backend uses directly ────────────────
|
|
48
48
|
__exportStar(require("./modules/lockfile"), exports);
|
|
49
|
+
var overrides_1 = require("./modules/overrides");
|
|
50
|
+
Object.defineProperty(exports, "applyTranslationOverrides", { enumerable: true, get: function () { return overrides_1.applyTranslationOverrides; } });
|
|
51
|
+
Object.defineProperty(exports, "setLocaleValue", { enumerable: true, get: function () { return overrides_1.setLocaleValue; } });
|
|
49
52
|
__exportStar(require("./modules/translator/language-codes"), exports);
|
|
50
53
|
var memory_matcher_1 = require("./modules/translator/memory/memory-matcher");
|
|
51
54
|
Object.defineProperty(exports, "matchFromMemory", { enumerable: true, get: function () { return memory_matcher_1.matchFromMemory; } });
|
|
@@ -64,6 +67,10 @@ var stack_1 = require("./detection/stack");
|
|
|
64
67
|
Object.defineProperty(exports, "detectStack", { enumerable: true, get: function () { return stack_1.detectStack; } });
|
|
65
68
|
Object.defineProperty(exports, "localRepoFiles", { enumerable: true, get: function () { return stack_1.localRepoFiles; } });
|
|
66
69
|
Object.defineProperty(exports, "FRAMEWORK_DISPLAY_NAMES", { enumerable: true, get: function () { return stack_1.FRAMEWORK_DISPLAY_NAMES; } });
|
|
70
|
+
Object.defineProperty(exports, "SUPPORTED_SUMMARY", { enumerable: true, get: function () { return stack_1.SUPPORTED_SUMMARY; } });
|
|
71
|
+
var i18n_1 = require("./detection/i18n");
|
|
72
|
+
Object.defineProperty(exports, "detectI18n", { enumerable: true, get: function () { return i18n_1.detectI18n; } });
|
|
73
|
+
Object.defineProperty(exports, "I18N_LIBRARY_NAMES", { enumerable: true, get: function () { return i18n_1.I18N_LIBRARY_NAMES; } });
|
|
67
74
|
var ast_helpers_1 = require("./utils/ast-helpers");
|
|
68
75
|
Object.defineProperty(exports, "parseCode", { enumerable: true, get: function () { return ast_helpers_1.parseCode; } });
|
|
69
76
|
var string_utils_1 = require("./utils/string-utils");
|
|
@@ -2,7 +2,8 @@ import type { KeyResult, TranslationBatchRequest, TranslationProvider } from '..
|
|
|
2
2
|
/**
|
|
3
3
|
* Babelize-hosted translation for `babelize translate`: a TranslationProvider
|
|
4
4
|
* that sends the engine's already placeholder-protected strings (never code)
|
|
5
|
-
* to POST {apiUrl}/v1/translate/batch with a Babelize API key
|
|
5
|
+
* to POST {apiUrl}/v1/translate/batch with a Babelize API key or a signed-in
|
|
6
|
+
* CLI session (the endpoint accepts both and bills the same account). The engine
|
|
6
7
|
* still validates every result and runs its repair retry locally; model,
|
|
7
8
|
* quota and credits are decided by the server.
|
|
8
9
|
*/
|
|
@@ -15,6 +16,8 @@ export declare const HOSTED_MAX_BATCH: 100;
|
|
|
15
16
|
export declare const HOSTED_REQUEST_TIMEOUT_MS = 150000;
|
|
16
17
|
/** Response header naming the model that served a batch. */
|
|
17
18
|
export declare const HOSTED_MODEL_HEADER = "x-babelize-model";
|
|
19
|
+
/** Whole credits the server charged for one batch. */
|
|
20
|
+
export declare const HOSTED_CREDITS_HEADER = "x-credits-charged";
|
|
18
21
|
export declare const API_KEYS_URL = "https://app.babelize.co/api-keys";
|
|
19
22
|
export declare const BILLING_URL = "https://app.babelize.co/billing";
|
|
20
23
|
export type HostedErrorCode =
|
|
@@ -36,8 +39,19 @@ export declare class HostedTranslationError extends Error {
|
|
|
36
39
|
get fatal(): boolean;
|
|
37
40
|
}
|
|
38
41
|
export interface HostedProviderOptions {
|
|
39
|
-
/** Babelize API key (bblz_sk_…). Never printed. */
|
|
42
|
+
/** Babelize API key (bblz_sk_…) or a session access token. Never printed. */
|
|
40
43
|
apiKey: string;
|
|
44
|
+
/**
|
|
45
|
+
* What `apiKey` is, for the message when it's rejected: an API key from
|
|
46
|
+
* BABELIZE_API_KEY (default) or the CLI's signed-in session.
|
|
47
|
+
*/
|
|
48
|
+
credential?: 'api_key' | 'session';
|
|
49
|
+
/**
|
|
50
|
+
* Called once when a request is rejected with 401: return a fresh token
|
|
51
|
+
* (a session refresh) to retry with it, or null to give up. Sessions expire
|
|
52
|
+
* during long runs; API keys don't, so API-key callers leave this out.
|
|
53
|
+
*/
|
|
54
|
+
refreshToken?: () => Promise<string | null>;
|
|
41
55
|
/** e.g. https://api.babelize.co/api */
|
|
42
56
|
apiUrl: string;
|
|
43
57
|
fetch?: typeof fetch;
|
|
@@ -57,10 +71,28 @@ export declare function parseRetryAfterMs(value: string | null): number | undefi
|
|
|
57
71
|
export declare class HostedTranslationProvider implements TranslationProvider {
|
|
58
72
|
/** Model the server reported (X-Babelize-Model), once a request succeeded. */
|
|
59
73
|
model?: string;
|
|
74
|
+
/**
|
|
75
|
+
* True when the server translates with NLLB rather than an LLM: the engine
|
|
76
|
+
* then splits ICU messages and multi-line text and protects glossary terms
|
|
77
|
+
* before sending, as it does for cloud runs. Set by detectCapabilities().
|
|
78
|
+
*/
|
|
79
|
+
plainText: boolean;
|
|
80
|
+
/** Credits charged so far by this provider's batches (X-Credits-Charged). */
|
|
81
|
+
creditsCharged: number;
|
|
60
82
|
/** Set once a request fails in a way no retry can fix (bad key, no credits). */
|
|
61
83
|
fatalError?: HostedTranslationError;
|
|
62
84
|
private readonly opts;
|
|
85
|
+
/** The token in use; replaced when a session is refreshed. */
|
|
86
|
+
private token;
|
|
87
|
+
/** In-flight refresh, shared by concurrent batches that all got a 401. */
|
|
88
|
+
private refreshing?;
|
|
63
89
|
constructor(options: HostedProviderOptions);
|
|
90
|
+
/**
|
|
91
|
+
* Ask the server what serves translation (GET /v1/translate/capabilities)
|
|
92
|
+
* before a run, so requests are shaped for it. A server without the endpoint
|
|
93
|
+
* (older) or any failure keeps the LLM shape; the run itself reports errors.
|
|
94
|
+
*/
|
|
95
|
+
detectCapabilities(): Promise<void>;
|
|
64
96
|
translateBatch(request: TranslationBatchRequest): Promise<KeyResult[]>;
|
|
65
97
|
private post;
|
|
66
98
|
private backoff;
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.HostedTranslationProvider = exports.HostedTranslationError = exports.BILLING_URL = exports.API_KEYS_URL = exports.HOSTED_MODEL_HEADER = exports.HOSTED_REQUEST_TIMEOUT_MS = exports.HOSTED_MAX_BATCH = void 0;
|
|
3
|
+
exports.HostedTranslationProvider = exports.HostedTranslationError = exports.BILLING_URL = exports.API_KEYS_URL = exports.HOSTED_CREDITS_HEADER = exports.HOSTED_MODEL_HEADER = exports.HOSTED_REQUEST_TIMEOUT_MS = exports.HOSTED_MAX_BATCH = void 0;
|
|
4
4
|
exports.parseRetryAfterMs = parseRetryAfterMs;
|
|
5
5
|
const node_crypto_1 = require("node:crypto");
|
|
6
6
|
const language_codes_1 = require("../modules/translator/language-codes");
|
|
@@ -8,7 +8,8 @@ const types_1 = require("@babelize/types");
|
|
|
8
8
|
/**
|
|
9
9
|
* Babelize-hosted translation for `babelize translate`: a TranslationProvider
|
|
10
10
|
* that sends the engine's already placeholder-protected strings (never code)
|
|
11
|
-
* to POST {apiUrl}/v1/translate/batch with a Babelize API key
|
|
11
|
+
* to POST {apiUrl}/v1/translate/batch with a Babelize API key or a signed-in
|
|
12
|
+
* CLI session (the endpoint accepts both and bills the same account). The engine
|
|
12
13
|
* still validates every result and runs its repair retry locally; model,
|
|
13
14
|
* quota and credits are decided by the server.
|
|
14
15
|
*/
|
|
@@ -21,6 +22,8 @@ exports.HOSTED_MAX_BATCH = types_1.TRANSLATE_BATCH_LIMITS.maxEntries;
|
|
|
21
22
|
exports.HOSTED_REQUEST_TIMEOUT_MS = 150000;
|
|
22
23
|
/** Response header naming the model that served a batch. */
|
|
23
24
|
exports.HOSTED_MODEL_HEADER = 'x-babelize-model';
|
|
25
|
+
/** Whole credits the server charged for one batch. */
|
|
26
|
+
exports.HOSTED_CREDITS_HEADER = 'x-credits-charged';
|
|
24
27
|
exports.API_KEYS_URL = 'https://app.babelize.co/api-keys';
|
|
25
28
|
exports.BILLING_URL = 'https://app.babelize.co/billing';
|
|
26
29
|
class HostedTranslationError extends Error {
|
|
@@ -48,6 +51,15 @@ function parseRetryAfterMs(value) {
|
|
|
48
51
|
}
|
|
49
52
|
class HostedTranslationProvider {
|
|
50
53
|
constructor(options) {
|
|
54
|
+
/**
|
|
55
|
+
* True when the server translates with NLLB rather than an LLM: the engine
|
|
56
|
+
* then splits ICU messages and multi-line text and protects glossary terms
|
|
57
|
+
* before sending, as it does for cloud runs. Set by detectCapabilities().
|
|
58
|
+
*/
|
|
59
|
+
this.plainText = false;
|
|
60
|
+
/** Credits charged so far by this provider's batches (X-Credits-Charged). */
|
|
61
|
+
this.creditsCharged = 0;
|
|
62
|
+
this.token = options.apiKey;
|
|
51
63
|
this.opts = {
|
|
52
64
|
fetch: globalThis.fetch.bind(globalThis),
|
|
53
65
|
maxRetries: 3,
|
|
@@ -59,6 +71,31 @@ class HostedTranslationProvider {
|
|
|
59
71
|
apiUrl: options.apiUrl.replace(/\/+$/, ''),
|
|
60
72
|
};
|
|
61
73
|
}
|
|
74
|
+
/**
|
|
75
|
+
* Ask the server what serves translation (GET /v1/translate/capabilities)
|
|
76
|
+
* before a run, so requests are shaped for it. A server without the endpoint
|
|
77
|
+
* (older) or any failure keeps the LLM shape; the run itself reports errors.
|
|
78
|
+
*/
|
|
79
|
+
async detectCapabilities() {
|
|
80
|
+
try {
|
|
81
|
+
const res = await this.opts.fetch(`${this.opts.apiUrl}/v1/translate/capabilities`, {
|
|
82
|
+
headers: {
|
|
83
|
+
Authorization: `Bearer ${this.token}`,
|
|
84
|
+
...(this.opts.userAgent ? { 'User-Agent': this.opts.userAgent } : {}),
|
|
85
|
+
},
|
|
86
|
+
signal: AbortSignal.timeout(15000),
|
|
87
|
+
});
|
|
88
|
+
if (!res.ok)
|
|
89
|
+
return;
|
|
90
|
+
const body = (await res.json());
|
|
91
|
+
this.plainText = body.plainText === true;
|
|
92
|
+
if (body.model)
|
|
93
|
+
this.model ?? (this.model = body.model);
|
|
94
|
+
}
|
|
95
|
+
catch {
|
|
96
|
+
// keep the defaults
|
|
97
|
+
}
|
|
98
|
+
}
|
|
62
99
|
async translateBatch(request) {
|
|
63
100
|
// Strings over the server's limits never leave the machine: they fail
|
|
64
101
|
// locally, and the rest of the batch is still sent.
|
|
@@ -122,6 +159,7 @@ class HostedTranslationProvider {
|
|
|
122
159
|
// replays it instead of translating and charging again.
|
|
123
160
|
const idempotencyKey = (0, node_crypto_1.randomUUID)();
|
|
124
161
|
let attempt = 0;
|
|
162
|
+
let refreshed = false;
|
|
125
163
|
for (;;) {
|
|
126
164
|
let res;
|
|
127
165
|
try {
|
|
@@ -129,7 +167,7 @@ class HostedTranslationProvider {
|
|
|
129
167
|
method: 'POST',
|
|
130
168
|
headers: {
|
|
131
169
|
'Content-Type': 'application/json',
|
|
132
|
-
Authorization: `Bearer ${this.
|
|
170
|
+
Authorization: `Bearer ${this.token}`,
|
|
133
171
|
'Idempotency-Key': idempotencyKey,
|
|
134
172
|
...(this.opts.userAgent ? { 'User-Agent': this.opts.userAgent } : {}),
|
|
135
173
|
},
|
|
@@ -160,8 +198,22 @@ class HostedTranslationProvider {
|
|
|
160
198
|
throw new HostedTranslationError(`Lost the response from Babelize at ${this.opts.apiUrl} (${err.message}).`, 'hosted_unavailable');
|
|
161
199
|
}
|
|
162
200
|
this.model = res.headers.get(exports.HOSTED_MODEL_HEADER) ?? this.model;
|
|
201
|
+
const charged = Number(res.headers.get(exports.HOSTED_CREDITS_HEADER));
|
|
202
|
+
if (Number.isFinite(charged) && charged > 0)
|
|
203
|
+
this.creditsCharged += charged;
|
|
163
204
|
return Array.isArray(data.results) ? data.results : [];
|
|
164
205
|
}
|
|
206
|
+
// An expired session: refresh it once and try again with the same key.
|
|
207
|
+
if (res.status === 401 && this.opts.refreshToken && !refreshed) {
|
|
208
|
+
refreshed = true;
|
|
209
|
+
const sent = this.token;
|
|
210
|
+
this.refreshing ?? (this.refreshing = this.opts.refreshToken().finally(() => { this.refreshing = undefined; }));
|
|
211
|
+
const fresh = this.token !== sent ? this.token : await this.refreshing;
|
|
212
|
+
if (fresh) {
|
|
213
|
+
this.token = fresh;
|
|
214
|
+
continue;
|
|
215
|
+
}
|
|
216
|
+
}
|
|
165
217
|
if (RETRYABLE_STATUS.has(res.status) && attempt < this.opts.maxRetries && !(await isHostedOff(res))) {
|
|
166
218
|
const retryAfter = parseRetryAfterMs(res.headers.get('retry-after'));
|
|
167
219
|
await this.opts.sleep(Math.min(retryAfter ?? this.backoff(attempt), this.opts.maxRetryAfterMs));
|
|
@@ -183,7 +235,9 @@ class HostedTranslationProvider {
|
|
|
183
235
|
switch (res.status) {
|
|
184
236
|
case 401:
|
|
185
237
|
case 403:
|
|
186
|
-
return new HostedTranslationError(
|
|
238
|
+
return new HostedTranslationError(this.opts.credential === 'session'
|
|
239
|
+
? `Your Babelize sign-in was rejected (${res.status}). Sign in again with \`babelize auth login\`, then re-run.`
|
|
240
|
+
: `BABELIZE_API_KEY was rejected (${res.status}). Create a new key at ${exports.API_KEYS_URL} and set BABELIZE_API_KEY.`, 'hosted_auth', res.status);
|
|
187
241
|
case 402:
|
|
188
242
|
return new HostedTranslationError(payload?.message ??
|
|
189
243
|
`Your Babelize account is out of AI credits. Top up or upgrade at ${exports.BILLING_URL}, or use your own LLM key (e.g. set ANTHROPIC_API_KEY).`, 'hosted_credits', 402);
|
package/dist/local/report.d.ts
CHANGED
package/dist/local/report.js
CHANGED
|
@@ -107,6 +107,7 @@ function buildTranslateReport(r) {
|
|
|
107
107
|
framework: r.framework,
|
|
108
108
|
frameworkStatus: r.frameworkStatus,
|
|
109
109
|
provider: r.provider ? { provider: r.provider.provider, name: r.provider.name, ...(r.provider.model ? { model: r.provider.model } : {}) } : null,
|
|
110
|
+
...(r.creditsCharged !== undefined ? { creditsCharged: r.creditsCharged } : {}),
|
|
110
111
|
sourceLocale: r.sourceLocale,
|
|
111
112
|
targetLocales: r.targetLocales,
|
|
112
113
|
keptLocales: r.keptLocales,
|
|
@@ -123,6 +123,8 @@ export interface LocalTranslateReport {
|
|
|
123
123
|
name: string;
|
|
124
124
|
model?: string;
|
|
125
125
|
};
|
|
126
|
+
/** Babelize credits this run was charged; set for hosted runs only. */
|
|
127
|
+
creditsCharged?: number;
|
|
126
128
|
sourceLocale: string;
|
|
127
129
|
/** Locales translated in this run: `--to` plus earlier locales (unless replace). */
|
|
128
130
|
targetLocales: string[];
|
|
@@ -283,6 +283,10 @@ async function runScoped(opts, deps, engineLog, state) {
|
|
|
283
283
|
if (!opts.byok && !translator) {
|
|
284
284
|
throw new LocalTranslateError('No translation provider configured.', 'no_translator');
|
|
285
285
|
}
|
|
286
|
+
// Shape requests for what the server translates with (NLLB needs ICU and
|
|
287
|
+
// multi-line text split and glossary terms protected; an LLM doesn't).
|
|
288
|
+
if (deps.hosted)
|
|
289
|
+
await deps.hosted.detectCapabilities();
|
|
286
290
|
const files = await listProjectFiles(projectDir);
|
|
287
291
|
const temp = await (0, workspace_copy_1.createTempCopy)(projectDir, { files, tmpRoot: deps.tmpRoot });
|
|
288
292
|
try {
|
|
@@ -458,6 +462,7 @@ async function runScoped(opts, deps, engineLog, state) {
|
|
|
458
462
|
framework,
|
|
459
463
|
frameworkStatus,
|
|
460
464
|
provider,
|
|
465
|
+
...(deps.hosted ? { creditsCharged: deps.hosted.creditsCharged } : {}),
|
|
461
466
|
sourceLocale: opts.sourceLocale,
|
|
462
467
|
targetLocales,
|
|
463
468
|
keptLocales,
|
|
@@ -1,3 +1,3 @@
|
|
|
1
1
|
export { hashSourceString } from './hasher';
|
|
2
2
|
export { type LockfileData, type LockfileEntry, type LockfilePartition, type LockfileStats, type PartitionedKey, createEmptyLockfile, } from './schema';
|
|
3
|
-
export { buildKeysToTranslateByTargetLocale, clearLockfile, removeLocaleFromLockfile, collectCachedTranslations, computeLockfileStats, getLockfilePath, loadLockfile, partitionKeys, saveLockfile, updateLockfile, lockfileChanged, dropFailedTranslations, isStalePseudoEntry, glossaryChanged, type LockfileReadOptions, } from './lockfile-manager';
|
|
3
|
+
export { buildKeysToTranslateByTargetLocale, clearLockfile, removeLocaleFromLockfile, removeLockfileEntries, collectCachedTranslations, computeLockfileStats, getLockfilePath, loadLockfile, partitionKeys, saveLockfile, updateLockfile, lockfileChanged, dropFailedTranslations, isStalePseudoEntry, glossaryChanged, type LockfileReadOptions, } from './lockfile-manager';
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.glossaryChanged = exports.isStalePseudoEntry = exports.dropFailedTranslations = exports.lockfileChanged = exports.updateLockfile = exports.saveLockfile = exports.partitionKeys = exports.loadLockfile = exports.getLockfilePath = exports.computeLockfileStats = exports.collectCachedTranslations = exports.removeLocaleFromLockfile = exports.clearLockfile = exports.buildKeysToTranslateByTargetLocale = exports.createEmptyLockfile = exports.hashSourceString = void 0;
|
|
3
|
+
exports.glossaryChanged = exports.isStalePseudoEntry = exports.dropFailedTranslations = exports.lockfileChanged = exports.updateLockfile = exports.saveLockfile = exports.partitionKeys = exports.loadLockfile = exports.getLockfilePath = exports.computeLockfileStats = exports.collectCachedTranslations = exports.removeLockfileEntries = exports.removeLocaleFromLockfile = exports.clearLockfile = exports.buildKeysToTranslateByTargetLocale = exports.createEmptyLockfile = exports.hashSourceString = void 0;
|
|
4
4
|
var hasher_1 = require("./hasher");
|
|
5
5
|
Object.defineProperty(exports, "hashSourceString", { enumerable: true, get: function () { return hasher_1.hashSourceString; } });
|
|
6
6
|
var schema_1 = require("./schema");
|
|
@@ -9,6 +9,7 @@ var lockfile_manager_1 = require("./lockfile-manager");
|
|
|
9
9
|
Object.defineProperty(exports, "buildKeysToTranslateByTargetLocale", { enumerable: true, get: function () { return lockfile_manager_1.buildKeysToTranslateByTargetLocale; } });
|
|
10
10
|
Object.defineProperty(exports, "clearLockfile", { enumerable: true, get: function () { return lockfile_manager_1.clearLockfile; } });
|
|
11
11
|
Object.defineProperty(exports, "removeLocaleFromLockfile", { enumerable: true, get: function () { return lockfile_manager_1.removeLocaleFromLockfile; } });
|
|
12
|
+
Object.defineProperty(exports, "removeLockfileEntries", { enumerable: true, get: function () { return lockfile_manager_1.removeLockfileEntries; } });
|
|
12
13
|
Object.defineProperty(exports, "collectCachedTranslations", { enumerable: true, get: function () { return lockfile_manager_1.collectCachedTranslations; } });
|
|
13
14
|
Object.defineProperty(exports, "computeLockfileStats", { enumerable: true, get: function () { return lockfile_manager_1.computeLockfileStats; } });
|
|
14
15
|
Object.defineProperty(exports, "getLockfilePath", { enumerable: true, get: function () { return lockfile_manager_1.getLockfilePath; } });
|
|
@@ -32,6 +32,15 @@ export declare function clearLockfile(projectId: string, dataDir?: string): Prom
|
|
|
32
32
|
* Used when a language is removed from a project so re-adding it
|
|
33
33
|
* triggers fresh translation instead of resurrecting stale cache.
|
|
34
34
|
*/
|
|
35
|
+
/**
|
|
36
|
+
* Forget the stored translation of some (key, locale) pairs, so the next run
|
|
37
|
+
* translates them again (a dashboard edit that was reverted). Other locales
|
|
38
|
+
* of the key stay.
|
|
39
|
+
*/
|
|
40
|
+
export declare function removeLockfileEntries(projectId: string, entries: Array<{
|
|
41
|
+
key: string;
|
|
42
|
+
locale: string;
|
|
43
|
+
}>, dataDir?: string): Promise<number>;
|
|
35
44
|
export declare function removeLocaleFromLockfile(projectId: string, localeCode: string, dataDir?: string): Promise<void>;
|
|
36
45
|
/**
|
|
37
46
|
* Partition generated keys into unchanged / new / changed / missingLocales
|
|
@@ -38,6 +38,7 @@ exports.getLockfilePath = getLockfilePath;
|
|
|
38
38
|
exports.loadLockfile = loadLockfile;
|
|
39
39
|
exports.saveLockfile = saveLockfile;
|
|
40
40
|
exports.clearLockfile = clearLockfile;
|
|
41
|
+
exports.removeLockfileEntries = removeLockfileEntries;
|
|
41
42
|
exports.removeLocaleFromLockfile = removeLocaleFromLockfile;
|
|
42
43
|
exports.glossaryChanged = glossaryChanged;
|
|
43
44
|
exports.isStalePseudoEntry = isStalePseudoEntry;
|
|
@@ -242,6 +243,44 @@ async function clearLockfile(projectId, dataDir) {
|
|
|
242
243
|
* Used when a language is removed from a project so re-adding it
|
|
243
244
|
* triggers fresh translation instead of resurrecting stale cache.
|
|
244
245
|
*/
|
|
246
|
+
/**
|
|
247
|
+
* Forget the stored translation of some (key, locale) pairs, so the next run
|
|
248
|
+
* translates them again (a dashboard edit that was reverted). Other locales
|
|
249
|
+
* of the key stay.
|
|
250
|
+
*/
|
|
251
|
+
async function removeLockfileEntries(projectId, entries, dataDir) {
|
|
252
|
+
const lockfilePath = getLockfilePath(projectId, dataDir);
|
|
253
|
+
if (entries.length === 0 || !(await fs.pathExists(lockfilePath)))
|
|
254
|
+
return 0;
|
|
255
|
+
const lockDir = lockfilePath + '.lock';
|
|
256
|
+
await acquireLock(lockDir);
|
|
257
|
+
try {
|
|
258
|
+
// Read under the lock: a run saving meanwhile must not be lost.
|
|
259
|
+
const lockfile = await loadLockfile(projectId, dataDir);
|
|
260
|
+
let removed = 0;
|
|
261
|
+
for (const { key, locale } of entries) {
|
|
262
|
+
const entry = lockfile.keys[key];
|
|
263
|
+
if (!entry || !(locale in entry.languages))
|
|
264
|
+
continue;
|
|
265
|
+
delete entry.languages[locale];
|
|
266
|
+
if (entry.metadata)
|
|
267
|
+
delete entry.metadata[locale];
|
|
268
|
+
if (Object.keys(entry.languages).length === 0)
|
|
269
|
+
delete lockfile.keys[key];
|
|
270
|
+
removed++;
|
|
271
|
+
}
|
|
272
|
+
if (removed > 0) {
|
|
273
|
+
lockfile.updatedAt = new Date().toISOString();
|
|
274
|
+
const tmpPath = lockfilePath + '.tmp';
|
|
275
|
+
await fs.writeJson(tmpPath, lockfile, { spaces: 2 });
|
|
276
|
+
await fs.rename(tmpPath, lockfilePath);
|
|
277
|
+
}
|
|
278
|
+
return removed;
|
|
279
|
+
}
|
|
280
|
+
finally {
|
|
281
|
+
await releaseLock(lockDir);
|
|
282
|
+
}
|
|
283
|
+
}
|
|
245
284
|
async function removeLocaleFromLockfile(projectId, localeCode, dataDir) {
|
|
246
285
|
const lockfilePath = getLockfilePath(projectId, dataDir);
|
|
247
286
|
try {
|
|
@@ -297,7 +336,8 @@ function glossaryChanged(entry, source, locale, options = {}) {
|
|
|
297
336
|
if (!options.glossaryHash)
|
|
298
337
|
return false;
|
|
299
338
|
const meta = entry.metadata?.[locale];
|
|
300
|
-
|
|
339
|
+
// The project's own files, or a dashboard edit: the glossary doesn't rewrite them.
|
|
340
|
+
if (meta?.external || meta?.manual)
|
|
301
341
|
return false;
|
|
302
342
|
return (meta?.glossaryHash ?? '') !== options.glossaryHash(source, locale);
|
|
303
343
|
}
|
|
@@ -394,6 +434,8 @@ function pickLockfileMetadata(meta) {
|
|
|
394
434
|
out.glossaryHash = meta.glossaryHash;
|
|
395
435
|
if (meta.external === true)
|
|
396
436
|
out.external = true;
|
|
437
|
+
if (meta.manual === true)
|
|
438
|
+
out.manual = true;
|
|
397
439
|
return out;
|
|
398
440
|
}
|
|
399
441
|
function sameJson(a, b) {
|