ai-i18n-tools 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +157 -0
- package/dist/api/openrouter.d.ts +115 -0
- package/dist/api/openrouter.d.ts.map +1 -0
- package/dist/api/openrouter.js +399 -0
- package/dist/api/openrouter.js.map +1 -0
- package/dist/cli/doc-translate.d.ts +90 -0
- package/dist/cli/doc-translate.d.ts.map +1 -0
- package/dist/cli/doc-translate.js +1153 -0
- package/dist/cli/doc-translate.js.map +1 -0
- package/dist/cli/export-ui-xliff.d.ts +32 -0
- package/dist/cli/export-ui-xliff.d.ts.map +1 -0
- package/dist/cli/export-ui-xliff.js +153 -0
- package/dist/cli/export-ui-xliff.js.map +1 -0
- package/dist/cli/extract-strings.d.ts +12 -0
- package/dist/cli/extract-strings.d.ts.map +1 -0
- package/dist/cli/extract-strings.js +80 -0
- package/dist/cli/extract-strings.js.map +1 -0
- package/dist/cli/file-utils.d.ts +10 -0
- package/dist/cli/file-utils.d.ts.map +1 -0
- package/dist/cli/file-utils.js +77 -0
- package/dist/cli/file-utils.js.map +1 -0
- package/dist/cli/format.d.ts +21 -0
- package/dist/cli/format.d.ts.map +1 -0
- package/dist/cli/format.js +75 -0
- package/dist/cli/format.js.map +1 -0
- package/dist/cli/helpers.d.ts +27 -0
- package/dist/cli/helpers.d.ts.map +1 -0
- package/dist/cli/helpers.js +84 -0
- package/dist/cli/helpers.js.map +1 -0
- package/dist/cli/index.d.ts +3 -0
- package/dist/cli/index.d.ts.map +1 -0
- package/dist/cli/index.js +772 -0
- package/dist/cli/index.js.map +1 -0
- package/dist/cli/log-output.d.ts +13 -0
- package/dist/cli/log-output.d.ts.map +1 -0
- package/dist/cli/log-output.js +75 -0
- package/dist/cli/log-output.js.map +1 -0
- package/dist/cli/translate-svg.d.ts +7 -0
- package/dist/cli/translate-svg.d.ts.map +1 -0
- package/dist/cli/translate-svg.js +167 -0
- package/dist/cli/translate-svg.js.map +1 -0
- package/dist/cli/translate-ui-strings.d.ts +27 -0
- package/dist/cli/translate-ui-strings.d.ts.map +1 -0
- package/dist/cli/translate-ui-strings.js +357 -0
- package/dist/cli/translate-ui-strings.js.map +1 -0
- package/dist/core/cache-tracking-keys.d.ts +12 -0
- package/dist/core/cache-tracking-keys.d.ts.map +1 -0
- package/dist/core/cache-tracking-keys.js +20 -0
- package/dist/core/cache-tracking-keys.js.map +1 -0
- package/dist/core/cache.d.ts +153 -0
- package/dist/core/cache.d.ts.map +1 -0
- package/dist/core/cache.js +546 -0
- package/dist/core/cache.js.map +1 -0
- package/dist/core/config.d.ts +58 -0
- package/dist/core/config.d.ts.map +1 -0
- package/dist/core/config.js +392 -0
- package/dist/core/config.js.map +1 -0
- package/dist/core/doc-file-tracking.d.ts +8 -0
- package/dist/core/doc-file-tracking.d.ts.map +1 -0
- package/dist/core/doc-file-tracking.js +27 -0
- package/dist/core/doc-file-tracking.js.map +1 -0
- package/dist/core/errors.d.ts +19 -0
- package/dist/core/errors.d.ts.map +1 -0
- package/dist/core/errors.js +23 -0
- package/dist/core/errors.js.map +1 -0
- package/dist/core/locale-utils.d.ts +20 -0
- package/dist/core/locale-utils.d.ts.map +1 -0
- package/dist/core/locale-utils.js +75 -0
- package/dist/core/locale-utils.js.map +1 -0
- package/dist/core/output-paths.d.ts +22 -0
- package/dist/core/output-paths.d.ts.map +1 -0
- package/dist/core/output-paths.js +130 -0
- package/dist/core/output-paths.js.map +1 -0
- package/dist/core/prompt-builder.d.ts +62 -0
- package/dist/core/prompt-builder.d.ts.map +1 -0
- package/dist/core/prompt-builder.js +232 -0
- package/dist/core/prompt-builder.js.map +1 -0
- package/dist/core/prompts.d.ts +27 -0
- package/dist/core/prompts.d.ts.map +1 -0
- package/dist/core/prompts.js +57 -0
- package/dist/core/prompts.js.map +1 -0
- package/dist/core/svg-asset-paths.d.ts +40 -0
- package/dist/core/svg-asset-paths.d.ts.map +1 -0
- package/dist/core/svg-asset-paths.js +107 -0
- package/dist/core/svg-asset-paths.js.map +1 -0
- package/dist/core/types.d.ts +388 -0
- package/dist/core/types.d.ts.map +1 -0
- package/dist/core/types.js +265 -0
- package/dist/core/types.js.map +1 -0
- package/dist/core/ui-languages.d.ts +66 -0
- package/dist/core/ui-languages.d.ts.map +1 -0
- package/dist/core/ui-languages.js +277 -0
- package/dist/core/ui-languages.js.map +1 -0
- package/dist/core/user-edited-model.d.ts +3 -0
- package/dist/core/user-edited-model.d.ts.map +1 -0
- package/dist/core/user-edited-model.js +3 -0
- package/dist/core/user-edited-model.js.map +1 -0
- package/dist/edit-cache-app/app.js +1326 -0
- package/dist/edit-cache-app/index.html +287 -0
- package/dist/edit-cache-app/styles.css +664 -0
- package/dist/extractors/base-extractor.d.ts +15 -0
- package/dist/extractors/base-extractor.d.ts.map +1 -0
- package/dist/extractors/base-extractor.js +23 -0
- package/dist/extractors/base-extractor.js.map +1 -0
- package/dist/extractors/classify-segment.d.ts +6 -0
- package/dist/extractors/classify-segment.d.ts.map +1 -0
- package/dist/extractors/classify-segment.js +20 -0
- package/dist/extractors/classify-segment.js.map +1 -0
- package/dist/extractors/json-extractor.d.ts +16 -0
- package/dist/extractors/json-extractor.d.ts.map +1 -0
- package/dist/extractors/json-extractor.js +128 -0
- package/dist/extractors/json-extractor.js.map +1 -0
- package/dist/extractors/markdown-extractor.d.ts +15 -0
- package/dist/extractors/markdown-extractor.d.ts.map +1 -0
- package/dist/extractors/markdown-extractor.js +205 -0
- package/dist/extractors/markdown-extractor.js.map +1 -0
- package/dist/extractors/svg-extractor.d.ts +19 -0
- package/dist/extractors/svg-extractor.d.ts.map +1 -0
- package/dist/extractors/svg-extractor.js +132 -0
- package/dist/extractors/svg-extractor.js.map +1 -0
- package/dist/extractors/ui-string-extractor.d.ts +40 -0
- package/dist/extractors/ui-string-extractor.d.ts.map +1 -0
- package/dist/extractors/ui-string-extractor.js +146 -0
- package/dist/extractors/ui-string-extractor.js.map +1 -0
- package/dist/extractors/ui-string-locations.d.ts +23 -0
- package/dist/extractors/ui-string-locations.d.ts.map +1 -0
- package/dist/extractors/ui-string-locations.js +138 -0
- package/dist/extractors/ui-string-locations.js.map +1 -0
- package/dist/glossary/glossary.d.ts +34 -0
- package/dist/glossary/glossary.d.ts.map +1 -0
- package/dist/glossary/glossary.js +260 -0
- package/dist/glossary/glossary.js.map +1 -0
- package/dist/glossary/matcher.d.ts +10 -0
- package/dist/glossary/matcher.d.ts.map +1 -0
- package/dist/glossary/matcher.js +12 -0
- package/dist/glossary/matcher.js.map +1 -0
- package/dist/index.d.ts +44 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +44 -0
- package/dist/index.js.map +1 -0
- package/dist/processors/admonition-placeholders.d.ts +8 -0
- package/dist/processors/admonition-placeholders.d.ts.map +1 -0
- package/dist/processors/admonition-placeholders.js +59 -0
- package/dist/processors/admonition-placeholders.js.map +1 -0
- package/dist/processors/anchor-placeholders.d.ts +8 -0
- package/dist/processors/anchor-placeholders.d.ts.map +1 -0
- package/dist/processors/anchor-placeholders.js +37 -0
- package/dist/processors/anchor-placeholders.js.map +1 -0
- package/dist/processors/batch-processor.d.ts +10 -0
- package/dist/processors/batch-processor.d.ts.map +1 -0
- package/dist/processors/batch-processor.js +33 -0
- package/dist/processors/batch-processor.js.map +1 -0
- package/dist/processors/bold-code-placeholders.d.ts +14 -0
- package/dist/processors/bold-code-placeholders.d.ts.map +1 -0
- package/dist/processors/bold-code-placeholders.js +116 -0
- package/dist/processors/bold-code-placeholders.js.map +1 -0
- package/dist/processors/doc-postprocess.d.ts +51 -0
- package/dist/processors/doc-postprocess.d.ts.map +1 -0
- package/dist/processors/doc-postprocess.js +215 -0
- package/dist/processors/doc-postprocess.js.map +1 -0
- package/dist/processors/emphasis-placeholders.d.ts +6 -0
- package/dist/processors/emphasis-placeholders.d.ts.map +1 -0
- package/dist/processors/emphasis-placeholders.js +262 -0
- package/dist/processors/emphasis-placeholders.js.map +1 -0
- package/dist/processors/flat-link-rewrite.d.ts +32 -0
- package/dist/processors/flat-link-rewrite.d.ts.map +1 -0
- package/dist/processors/flat-link-rewrite.js +90 -0
- package/dist/processors/flat-link-rewrite.js.map +1 -0
- package/dist/processors/glossary-force-placeholders.d.ts +12 -0
- package/dist/processors/glossary-force-placeholders.d.ts.map +1 -0
- package/dist/processors/glossary-force-placeholders.js +58 -0
- package/dist/processors/glossary-force-placeholders.js.map +1 -0
- package/dist/processors/inline-code-placeholders.d.ts +11 -0
- package/dist/processors/inline-code-placeholders.d.ts.map +1 -0
- package/dist/processors/inline-code-placeholders.js +87 -0
- package/dist/processors/inline-code-placeholders.js.map +1 -0
- package/dist/processors/placeholder-handler.d.ts +38 -0
- package/dist/processors/placeholder-handler.d.ts.map +1 -0
- package/dist/processors/placeholder-handler.js +55 -0
- package/dist/processors/placeholder-handler.js.map +1 -0
- package/dist/processors/translation-placeholder-leaks.d.ts +2 -0
- package/dist/processors/translation-placeholder-leaks.d.ts.map +1 -0
- package/dist/processors/translation-placeholder-leaks.js +9 -0
- package/dist/processors/translation-placeholder-leaks.js.map +1 -0
- package/dist/processors/url-placeholders.d.ts +10 -0
- package/dist/processors/url-placeholders.d.ts.map +1 -0
- package/dist/processors/url-placeholders.js +29 -0
- package/dist/processors/url-placeholders.js.map +1 -0
- package/dist/processors/validator.d.ts +23 -0
- package/dist/processors/validator.d.ts.map +1 -0
- package/dist/processors/validator.js +186 -0
- package/dist/processors/validator.js.map +1 -0
- package/dist/runtime/i18next-helpers.d.ts +146 -0
- package/dist/runtime/i18next-helpers.d.ts.map +1 -0
- package/dist/runtime/i18next-helpers.js +192 -0
- package/dist/runtime/i18next-helpers.js.map +1 -0
- package/dist/runtime/index.d.ts +4 -0
- package/dist/runtime/index.d.ts.map +1 -0
- package/dist/runtime/index.js +4 -0
- package/dist/runtime/index.js.map +1 -0
- package/dist/runtime/template.d.ts +21 -0
- package/dist/runtime/template.d.ts.map +1 -0
- package/dist/runtime/template.js +28 -0
- package/dist/runtime/template.js.map +1 -0
- package/dist/runtime/ui-language-display.d.ts +16 -0
- package/dist/runtime/ui-language-display.d.ts.map +1 -0
- package/dist/runtime/ui-language-display.js +26 -0
- package/dist/runtime/ui-language-display.js.map +1 -0
- package/dist/server/translation-editor.d.ts +25 -0
- package/dist/server/translation-editor.d.ts.map +1 -0
- package/dist/server/translation-editor.js +583 -0
- package/dist/server/translation-editor.js.map +1 -0
- package/dist/utils/concurrency.d.ts +31 -0
- package/dist/utils/concurrency.d.ts.map +1 -0
- package/dist/utils/concurrency.js +103 -0
- package/dist/utils/concurrency.js.map +1 -0
- package/dist/utils/hash.d.ts +5 -0
- package/dist/utils/hash.d.ts.map +1 -0
- package/dist/utils/hash.js +9 -0
- package/dist/utils/hash.js.map +1 -0
- package/dist/utils/ignore-parser.d.ts +7 -0
- package/dist/utils/ignore-parser.d.ts.map +1 -0
- package/dist/utils/ignore-parser.js +26 -0
- package/dist/utils/ignore-parser.js.map +1 -0
- package/dist/utils/logger.d.ts +45 -0
- package/dist/utils/logger.d.ts.map +1 -0
- package/dist/utils/logger.js +158 -0
- package/dist/utils/logger.js.map +1 -0
- package/docs/GETTING_STARTED.md +697 -0
- package/docs/PACKAGE_OVERVIEW.md +427 -0
- package/docs/ai-i18n-tools-context.md +481 -0
- package/package.json +117 -0
- package/translated-docs/README.de.md +157 -0
- package/translated-docs/README.es.md +157 -0
- package/translated-docs/README.fr.md +157 -0
- package/translated-docs/README.hi.md +157 -0
- package/translated-docs/README.ja.md +157 -0
- package/translated-docs/README.ko.md +157 -0
- package/translated-docs/README.pt-BR.md +157 -0
- package/translated-docs/README.zh-CN.md +157 -0
- package/translated-docs/README.zh-TW.md +157 -0
- package/translated-docs/docs/GETTING_STARTED.de.md +682 -0
- package/translated-docs/docs/GETTING_STARTED.es.md +682 -0
- package/translated-docs/docs/GETTING_STARTED.fr.md +682 -0
- package/translated-docs/docs/GETTING_STARTED.hi.md +682 -0
- package/translated-docs/docs/GETTING_STARTED.ja.md +682 -0
- package/translated-docs/docs/GETTING_STARTED.ko.md +682 -0
- package/translated-docs/docs/GETTING_STARTED.pt-BR.md +682 -0
- package/translated-docs/docs/GETTING_STARTED.zh-CN.md +682 -0
- package/translated-docs/docs/GETTING_STARTED.zh-TW.md +682 -0
- package/translated-docs/docs/PACKAGE_OVERVIEW.de.md +428 -0
- package/translated-docs/docs/PACKAGE_OVERVIEW.es.md +428 -0
- package/translated-docs/docs/PACKAGE_OVERVIEW.fr.md +428 -0
- package/translated-docs/docs/PACKAGE_OVERVIEW.hi.md +428 -0
- package/translated-docs/docs/PACKAGE_OVERVIEW.ja.md +428 -0
- package/translated-docs/docs/PACKAGE_OVERVIEW.ko.md +428 -0
- package/translated-docs/docs/PACKAGE_OVERVIEW.pt-BR.md +428 -0
- package/translated-docs/docs/PACKAGE_OVERVIEW.zh-CN.md +428 -0
- package/translated-docs/docs/PACKAGE_OVERVIEW.zh-TW.md +428 -0
|
@@ -0,0 +1,1153 @@
|
|
|
1
|
+
import fs from "fs";
|
|
2
|
+
import path from "path";
|
|
3
|
+
import { matter, stringify as matterStringify } from "gray-matter-es";
|
|
4
|
+
import chalk from "chalk";
|
|
5
|
+
import { segmentTranslationText } from "../core/types.js";
|
|
6
|
+
import { documentationFileTrackingKey } from "../core/doc-file-tracking.js";
|
|
7
|
+
import { BatchTranslationError } from "../core/types.js";
|
|
8
|
+
import { timestamp, formatElapsedMmSs, printModelsTryInOrder } from "./format.js";
|
|
9
|
+
import { TranslationCache } from "../core/cache.js";
|
|
10
|
+
import { MarkdownExtractor } from "../extractors/markdown-extractor.js";
|
|
11
|
+
import { JsonExtractor } from "../extractors/json-extractor.js";
|
|
12
|
+
import { SvgExtractor } from "../extractors/svg-extractor.js";
|
|
13
|
+
import { PlaceholderHandler } from "../processors/placeholder-handler.js";
|
|
14
|
+
import { protectGlossaryForcedTerms, restoreGlossaryForcedTerms, } from "../processors/glossary-force-placeholders.js";
|
|
15
|
+
import { splitTranslatableIntoBatches } from "../processors/batch-processor.js";
|
|
16
|
+
import { Glossary } from "../glossary/glossary.js";
|
|
17
|
+
import { OpenRouterClient } from "../api/openrouter.js";
|
|
18
|
+
import { validateDocTranslatePair, validateTranslation } from "../processors/validator.js";
|
|
19
|
+
import { computeFlatLinkRewritePrefixes, normalizeMarkdownRelPath, rewriteDocLinksForFlatOutput, } from "../processors/flat-link-rewrite.js";
|
|
20
|
+
import { shouldRewriteFlatMarkdownLinks } from "../core/output-paths.js";
|
|
21
|
+
import { applyMarkdownLanguageListPostProcessing, applyMarkdownPostProcessing, } from "../processors/doc-postprocess.js";
|
|
22
|
+
import { resolveSvgAssetOutputPath, svgAssetCacheFilepath, svgTranslationFilepathMetadata, } from "../core/svg-asset-paths.js";
|
|
23
|
+
import { ensureDirForFile, hashFileContent, resolveTranslatedOutputPath, writeAtomicUtf8, } from "./helpers.js";
|
|
24
|
+
import { normalizeLocale } from "../core/config.js";
|
|
25
|
+
import { runMapWithConcurrency, AsyncSemaphore, AsyncMutex } from "../utils/concurrency.js";
|
|
26
|
+
export function translatePromptFormatToResponseFormat(fmt) {
|
|
27
|
+
if (fmt === "json-array") {
|
|
28
|
+
return "json-array";
|
|
29
|
+
}
|
|
30
|
+
if (fmt === "json-object") {
|
|
31
|
+
return "json-object";
|
|
32
|
+
}
|
|
33
|
+
return "xml-tags";
|
|
34
|
+
}
|
|
35
|
+
function emptyMarkdownProtectShell() {
|
|
36
|
+
return {
|
|
37
|
+
openMap: [],
|
|
38
|
+
endMap: [],
|
|
39
|
+
htmlAnchors: [],
|
|
40
|
+
docusaurusHeadingIds: [],
|
|
41
|
+
urlMap: [],
|
|
42
|
+
boldCodeMap: [],
|
|
43
|
+
ilcMap: [],
|
|
44
|
+
};
|
|
45
|
+
}
|
|
46
|
+
/** Glossary forced tokens first, then markdown URL/admonition/emphasis protection when `useMarkdownPlaceholders`. */
|
|
47
|
+
export function protectSegmentForTranslation(raw, glossary, locale, useMarkdownPlaceholders) {
|
|
48
|
+
const g = protectGlossaryForcedTerms(raw, glossary, locale);
|
|
49
|
+
const glossaryForceReplacements = g.replacements;
|
|
50
|
+
if (useMarkdownPlaceholders) {
|
|
51
|
+
const ph = new PlaceholderHandler();
|
|
52
|
+
const st = ph.protectForTranslation(g.text);
|
|
53
|
+
return {
|
|
54
|
+
text: st.text,
|
|
55
|
+
state: {
|
|
56
|
+
...st,
|
|
57
|
+
glossaryForceReplacements: glossaryForceReplacements.length > 0 ? glossaryForceReplacements : undefined,
|
|
58
|
+
},
|
|
59
|
+
};
|
|
60
|
+
}
|
|
61
|
+
return {
|
|
62
|
+
text: g.text,
|
|
63
|
+
state: {
|
|
64
|
+
text: g.text,
|
|
65
|
+
...emptyMarkdownProtectShell(),
|
|
66
|
+
glossaryForceReplacements: glossaryForceReplacements.length > 0 ? glossaryForceReplacements : undefined,
|
|
67
|
+
},
|
|
68
|
+
};
|
|
69
|
+
}
|
|
70
|
+
function restoreSegmentTranslation(ph, raw, st) {
|
|
71
|
+
if (!st) {
|
|
72
|
+
return raw;
|
|
73
|
+
}
|
|
74
|
+
let out = ph.restoreAfterTranslation(raw, st);
|
|
75
|
+
out = restoreGlossaryForcedTerms(out, st.glossaryForceReplacements ?? []);
|
|
76
|
+
return out;
|
|
77
|
+
}
|
|
78
|
+
function segmentOriginalContent(s, originalContentByHash) {
|
|
79
|
+
return { ...s, content: originalContentByHash.get(s.hash) ?? s.content };
|
|
80
|
+
}
|
|
81
|
+
/** Same style as {@link OpenRouterClient} model switch warnings; used when post-restore quality checks fail. */
|
|
82
|
+
function warnDocQualityModelSwitch(locale, relativePath, failedModel, nextModel, detail,
|
|
83
|
+
/** Same shape as successful batch lines, e.g. `segments 28ā52/155`. */
|
|
84
|
+
segmentsLabel) {
|
|
85
|
+
const loc = relativePath != null ? `${locale} ${relativePath}` : locale;
|
|
86
|
+
const seg = segmentsLabel ? `: ${segmentsLabel}` : "";
|
|
87
|
+
console.warn(chalk.yellow(` ā ļø ${loc}${seg}: ${failedModel} output failed quality check (${detail}). Trying ${nextModel}ā¦`));
|
|
88
|
+
}
|
|
89
|
+
function safeDocNameForLogFilename(relativePath) {
|
|
90
|
+
return relativePath.replace(/[/\\:*?"<>|]/g, "_");
|
|
91
|
+
}
|
|
92
|
+
/**
|
|
93
|
+
* Writes prompt, raw response, and validation details when doc translation quality checks fail.
|
|
94
|
+
* Filename: `{iso}-FAILED-TRANSLATION_{document}_{ms}.log` under `cacheDirAbs`.
|
|
95
|
+
* Returns absolute path; callers print `š Failure log: ā¦` after warnings (retry) or before throw (fatal).
|
|
96
|
+
*/
|
|
97
|
+
function writeDocTranslationFailureLog(opts) {
|
|
98
|
+
const tsIso = new Date().toISOString().replace(/:/g, "-");
|
|
99
|
+
const ms = Date.now();
|
|
100
|
+
const doc = safeDocNameForLogFilename(opts.relativePath);
|
|
101
|
+
const fileName = `${tsIso}-FAILED-TRANSLATION_${doc}_${ms}.log`;
|
|
102
|
+
const abs = path.join(opts.cacheDirAbs, fileName);
|
|
103
|
+
try {
|
|
104
|
+
ensureDirForFile(abs);
|
|
105
|
+
const lines = [
|
|
106
|
+
"=== ai-i18n-tools doc translation failure ===",
|
|
107
|
+
`logFilePath: ${abs}`,
|
|
108
|
+
`isoTime: ${new Date().toISOString()}`,
|
|
109
|
+
`locale: ${opts.locale}`,
|
|
110
|
+
`document: ${opts.relativePath}`,
|
|
111
|
+
`segments: ${opts.segmentsLabel}`,
|
|
112
|
+
`outcome: ${opts.outcome}`,
|
|
113
|
+
`failedModel: ${opts.failedModel}`,
|
|
114
|
+
opts.nextModel ? `nextModel: ${opts.nextModel}` : "",
|
|
115
|
+
"",
|
|
116
|
+
"--- quality / validation errors ---",
|
|
117
|
+
...opts.qualityErrors.map((e) => ` ${e}`),
|
|
118
|
+
"",
|
|
119
|
+
"--- per-segment validation ---",
|
|
120
|
+
...opts.perSegmentLines.map((e) => ` ${e}`),
|
|
121
|
+
"",
|
|
122
|
+
"--- system prompt ---",
|
|
123
|
+
opts.systemPrompt,
|
|
124
|
+
"",
|
|
125
|
+
"--- user content ---",
|
|
126
|
+
opts.userContent,
|
|
127
|
+
"",
|
|
128
|
+
"--- raw assistant response ---",
|
|
129
|
+
opts.rawAssistantContent,
|
|
130
|
+
"",
|
|
131
|
+
].filter((l) => l !== "");
|
|
132
|
+
fs.writeFileSync(abs, lines.join("\n"), "utf8");
|
|
133
|
+
return abs;
|
|
134
|
+
}
|
|
135
|
+
catch (e) {
|
|
136
|
+
console.warn(chalk.yellow(` ā ļø Could not write translation failure log: ${e}`));
|
|
137
|
+
return undefined;
|
|
138
|
+
}
|
|
139
|
+
}
|
|
140
|
+
/** Elapsed time as `M:SS` (minutes not zero-padded), matching reference translate-docs. */
|
|
141
|
+
function formatElapsedFileTime(ms) {
|
|
142
|
+
const totalSeconds = Math.floor(ms / 1000);
|
|
143
|
+
const minutes = Math.floor(totalSeconds / 60);
|
|
144
|
+
const seconds = totalSeconds % 60;
|
|
145
|
+
return `${minutes}:${seconds.toString().padStart(2, "0")}`;
|
|
146
|
+
}
|
|
147
|
+
function logTranslateFileComplete(relPath, outPath, segmentsCached, segmentsNew, elapsedMs, costUsd) {
|
|
148
|
+
const fileTimeFormatted = formatElapsedFileTime(elapsedMs);
|
|
149
|
+
const fileCostStr = costUsd > 0 ? `$${costUsd.toFixed(4)}` : "$0.0000";
|
|
150
|
+
console.log(chalk.blue(`ā
${relPath} ā ${path.basename(outPath)} (${segmentsCached} cached, ${segmentsNew} new) - ${fileTimeFormatted} - ${fileCostStr}`));
|
|
151
|
+
}
|
|
152
|
+
export function shouldRunMarkdown(opts, config) {
|
|
153
|
+
if (!config.features.translateMarkdown) {
|
|
154
|
+
return false;
|
|
155
|
+
}
|
|
156
|
+
if (opts.typeFilter === "json") {
|
|
157
|
+
return false;
|
|
158
|
+
}
|
|
159
|
+
if (opts.jsonOnly) {
|
|
160
|
+
return false;
|
|
161
|
+
}
|
|
162
|
+
return true;
|
|
163
|
+
}
|
|
164
|
+
export function shouldRunJson(opts, config) {
|
|
165
|
+
if (!config.features.translateJSON || !config.documentation.jsonSource?.trim()) {
|
|
166
|
+
return false;
|
|
167
|
+
}
|
|
168
|
+
if (opts.noJson) {
|
|
169
|
+
return false;
|
|
170
|
+
}
|
|
171
|
+
if (opts.typeFilter === "markdown") {
|
|
172
|
+
return false;
|
|
173
|
+
}
|
|
174
|
+
return true;
|
|
175
|
+
}
|
|
176
|
+
export function matchesPathFilter(relPath, filter) {
|
|
177
|
+
if (!filter?.trim()) {
|
|
178
|
+
return true;
|
|
179
|
+
}
|
|
180
|
+
const f = filter.replace(/\\/g, "/");
|
|
181
|
+
const r = relPath.replace(/\\/g, "/");
|
|
182
|
+
return r === f || r.startsWith(f.endsWith("/") ? f : `${f}/`);
|
|
183
|
+
}
|
|
184
|
+
async function withCacheMutex(mutex, fn) {
|
|
185
|
+
if (!mutex) {
|
|
186
|
+
return fn();
|
|
187
|
+
}
|
|
188
|
+
return mutex.runExclusive(async () => fn());
|
|
189
|
+
}
|
|
190
|
+
/** Match reference transrewrt `addTranslationMetadata` for translated markdown outputs. */
|
|
191
|
+
function addTranslationMetadata(content, sourceFileMtime, sourceFileHash, locale, relativePath, translationModels) {
|
|
192
|
+
const { data: frontMatter, content: body } = matter(content);
|
|
193
|
+
const base = typeof frontMatter === "object" && frontMatter !== null && !Array.isArray(frontMatter)
|
|
194
|
+
? frontMatter
|
|
195
|
+
: {};
|
|
196
|
+
const fm = { ...base };
|
|
197
|
+
fm.translation_last_updated = new Date().toISOString();
|
|
198
|
+
fm.source_file_mtime = sourceFileMtime;
|
|
199
|
+
fm.source_file_hash = sourceFileHash;
|
|
200
|
+
fm.translation_language = locale;
|
|
201
|
+
fm.source_file_path = relativePath;
|
|
202
|
+
if (translationModels.length > 0) {
|
|
203
|
+
fm.translation_models = translationModels;
|
|
204
|
+
}
|
|
205
|
+
return matterStringify(body, fm);
|
|
206
|
+
}
|
|
207
|
+
/** Unique non-empty model ids from segment translations, sorted for stable front matter. */
|
|
208
|
+
function collectTranslationModelsFromSegments(segments, translations) {
|
|
209
|
+
const set = new Set();
|
|
210
|
+
for (const s of segments) {
|
|
211
|
+
if (!s.translatable) {
|
|
212
|
+
continue;
|
|
213
|
+
}
|
|
214
|
+
const m = translations.get(s.hash)?.modelUsed?.trim();
|
|
215
|
+
if (m) {
|
|
216
|
+
set.add(m);
|
|
217
|
+
}
|
|
218
|
+
}
|
|
219
|
+
return [...set].sort((a, b) => a.localeCompare(b));
|
|
220
|
+
}
|
|
221
|
+
/** 1-based segment range label for translate-docs logs (indices are 0-based in the full segment list). */
|
|
222
|
+
function segRangeLabel(indices0, totalSegments) {
|
|
223
|
+
if (indices0.length === 0) {
|
|
224
|
+
return "";
|
|
225
|
+
}
|
|
226
|
+
const sorted = [...indices0].sort((a, b) => a - b);
|
|
227
|
+
const a = sorted[0] + 1;
|
|
228
|
+
const b = sorted[sorted.length - 1] + 1;
|
|
229
|
+
if (a === b) {
|
|
230
|
+
return `segment ${a}/${totalSegments}`;
|
|
231
|
+
}
|
|
232
|
+
return `segments ${a}ā${b}/${totalSegments}`;
|
|
233
|
+
}
|
|
234
|
+
async function translateSegmentsBatched(batchable, placeholderById,
|
|
235
|
+
/** Pre-placeholder markdown source per segment hash; empty for json/svg (original is on the segment). */
|
|
236
|
+
originalContentByHash, locale, glossary, client, dryRun, verbose, batchSize, maxBatchChars, contentType, batchConcurrency,
|
|
237
|
+
/** OpenRouter batch: XML <seg>/<t> vs JSON array/object (see `--prompt-format`). */
|
|
238
|
+
batchResponseFormat, docLog,
|
|
239
|
+
/** Absolute `cacheDir` path; writes `*-FAILED-TRANSLATION-*.log` on markdown quality failures. */
|
|
240
|
+
failureLogDirAbs) {
|
|
241
|
+
const out = new Map();
|
|
242
|
+
let inTok = 0;
|
|
243
|
+
let outTok = 0;
|
|
244
|
+
let cost = 0;
|
|
245
|
+
const batches = splitTranslatableIntoBatches(batchable, { batchSize, maxBatchChars });
|
|
246
|
+
if (batches.length === 0) {
|
|
247
|
+
return { map: out, inTok, outTok, cost };
|
|
248
|
+
}
|
|
249
|
+
const batchDocIndices = [];
|
|
250
|
+
if (docLog && docLog.segmentIndicesInDoc.length === batchable.length) {
|
|
251
|
+
let off = 0;
|
|
252
|
+
for (const b of batches) {
|
|
253
|
+
batchDocIndices.push(docLog.segmentIndicesInDoc.slice(off, off + b.length));
|
|
254
|
+
off += b.length;
|
|
255
|
+
}
|
|
256
|
+
}
|
|
257
|
+
const apiConc = Math.max(1, Math.floor(batchConcurrency));
|
|
258
|
+
const sem = !dryRun && client && batches.length > 0 && apiConc > 1 ? new AsyncSemaphore(apiConc) : null;
|
|
259
|
+
const processBatch = async (bi) => {
|
|
260
|
+
const partial = new Map();
|
|
261
|
+
let localIn = 0;
|
|
262
|
+
let localOut = 0;
|
|
263
|
+
let localCost = 0;
|
|
264
|
+
const batch = batches[bi];
|
|
265
|
+
const hintText = batch.map((s) => s.content).join("\n");
|
|
266
|
+
const hints = glossary.findTermsInText(hintText, locale);
|
|
267
|
+
const markdownQuality = contentType === "markdown";
|
|
268
|
+
if (dryRun || !client) {
|
|
269
|
+
if (verbose) {
|
|
270
|
+
console.log(chalk.yellow(` ${timestamp()} - [dry-run] batch ${bi + 1}/${batches.length} (${batch.length} segments)`));
|
|
271
|
+
}
|
|
272
|
+
for (const s of batch) {
|
|
273
|
+
partial.set(s.hash, { text: s.content });
|
|
274
|
+
}
|
|
275
|
+
return { partial, inTok: localIn, outTok: localOut, cost: localCost };
|
|
276
|
+
}
|
|
277
|
+
const logBatchSuccess = (res) => {
|
|
278
|
+
if (docLog && batchDocIndices[bi]) {
|
|
279
|
+
const idxs = batchDocIndices[bi];
|
|
280
|
+
const n = batch.length;
|
|
281
|
+
const totalTok = res.usage.totalTokens;
|
|
282
|
+
console.log(chalk.green(`āļø ${locale} ${docLog.relativePath}: ${segRangeLabel(idxs, docLog.totalSegments)} (${n} segment${n === 1 ? "" : "s"} in batch, ${totalTok} tokens)`));
|
|
283
|
+
}
|
|
284
|
+
};
|
|
285
|
+
if (!markdownQuality) {
|
|
286
|
+
try {
|
|
287
|
+
const res = await client.translateDocumentBatch(batch, locale, hints, {
|
|
288
|
+
contentType,
|
|
289
|
+
responseFormat: batchResponseFormat,
|
|
290
|
+
docLogContext: docLog ? { relativePath: docLog.relativePath } : undefined,
|
|
291
|
+
});
|
|
292
|
+
localIn += res.usage.inputTokens;
|
|
293
|
+
localOut += res.usage.outputTokens;
|
|
294
|
+
localCost += res.cost ?? 0;
|
|
295
|
+
const ph = new PlaceholderHandler();
|
|
296
|
+
for (let i = 0; i < batch.length; i++) {
|
|
297
|
+
const s = batch[i];
|
|
298
|
+
const raw = res.translations.get(i);
|
|
299
|
+
if (raw === undefined) {
|
|
300
|
+
throw new Error(`Missing translation for batch index ${i}`);
|
|
301
|
+
}
|
|
302
|
+
const st = placeholderById.get(s.id);
|
|
303
|
+
const restored = restoreSegmentTranslation(ph, raw, st);
|
|
304
|
+
partial.set(s.hash, { text: restored, modelUsed: res.model });
|
|
305
|
+
}
|
|
306
|
+
logBatchSuccess(res);
|
|
307
|
+
}
|
|
308
|
+
catch (e) {
|
|
309
|
+
if (e instanceof BatchTranslationError && client) {
|
|
310
|
+
if (verbose) {
|
|
311
|
+
console.warn(chalk.yellow(`ā ļø ${locale} ${docLog?.relativePath ?? "?"}: batch failed (expected ${e.expected} translated segments, got ${e.received}); falling back to single-segment API calls`));
|
|
312
|
+
}
|
|
313
|
+
const ph = new PlaceholderHandler();
|
|
314
|
+
for (const s of batch) {
|
|
315
|
+
const st = placeholderById.get(s.id);
|
|
316
|
+
const single = await client.translateDocumentSegment(s.content, locale, hints, {
|
|
317
|
+
contentType,
|
|
318
|
+
docLogContext: docLog && docLog.relativePath
|
|
319
|
+
? { locale, relativePath: docLog.relativePath }
|
|
320
|
+
: undefined,
|
|
321
|
+
});
|
|
322
|
+
localIn += single.usage.inputTokens;
|
|
323
|
+
localOut += single.usage.outputTokens;
|
|
324
|
+
localCost += single.cost ?? 0;
|
|
325
|
+
const restored = restoreSegmentTranslation(ph, single.content, st);
|
|
326
|
+
partial.set(s.hash, { text: restored, modelUsed: single.model });
|
|
327
|
+
}
|
|
328
|
+
}
|
|
329
|
+
else {
|
|
330
|
+
throw e;
|
|
331
|
+
}
|
|
332
|
+
}
|
|
333
|
+
return { partial, inTok: localIn, outTok: localOut, cost: localCost };
|
|
334
|
+
}
|
|
335
|
+
const ph = new PlaceholderHandler();
|
|
336
|
+
const models = client.getConfiguredModels();
|
|
337
|
+
try {
|
|
338
|
+
let startModelIndex = 0;
|
|
339
|
+
let finished = false;
|
|
340
|
+
while (!finished && startModelIndex < models.length) {
|
|
341
|
+
const res = await client.translateDocumentBatch(batch, locale, hints, {
|
|
342
|
+
contentType,
|
|
343
|
+
responseFormat: batchResponseFormat,
|
|
344
|
+
docLogContext: docLog ? { relativePath: docLog.relativePath } : undefined,
|
|
345
|
+
startModelIndex,
|
|
346
|
+
});
|
|
347
|
+
localIn += res.usage.inputTokens;
|
|
348
|
+
localOut += res.usage.outputTokens;
|
|
349
|
+
localCost += res.cost ?? 0;
|
|
350
|
+
const qualityErrors = [];
|
|
351
|
+
const batchPartial = new Map();
|
|
352
|
+
const perSegmentLines = [];
|
|
353
|
+
for (let i = 0; i < batch.length; i++) {
|
|
354
|
+
const s = batch[i];
|
|
355
|
+
const raw = res.translations.get(i);
|
|
356
|
+
if (raw === undefined) {
|
|
357
|
+
throw new Error(`Missing translation for batch index ${i}`);
|
|
358
|
+
}
|
|
359
|
+
const st = placeholderById.get(s.id);
|
|
360
|
+
const restored = restoreSegmentTranslation(ph, raw, st);
|
|
361
|
+
const origSeg = segmentOriginalContent(s, originalContentByHash);
|
|
362
|
+
const v = await validateDocTranslatePair(origSeg, restored);
|
|
363
|
+
if (!v.ok) {
|
|
364
|
+
qualityErrors.push(...v.errors);
|
|
365
|
+
}
|
|
366
|
+
const docIdx0 = batchDocIndices[bi]?.[i];
|
|
367
|
+
const label = docLog && docIdx0 !== undefined
|
|
368
|
+
? `doc #${docIdx0 + 1}/${docLog.totalSegments} (hash ${s.hash})`
|
|
369
|
+
: `batch index ${i} (hash ${s.hash})`;
|
|
370
|
+
perSegmentLines.push(`${label}: ${v.ok ? "OK" : v.errors.join("; ")}`);
|
|
371
|
+
batchPartial.set(s.hash, restored);
|
|
372
|
+
}
|
|
373
|
+
if (qualityErrors.length === 0) {
|
|
374
|
+
for (const [h, t] of batchPartial) {
|
|
375
|
+
partial.set(h, { text: t, modelUsed: res.model });
|
|
376
|
+
}
|
|
377
|
+
logBatchSuccess(res);
|
|
378
|
+
finished = true;
|
|
379
|
+
}
|
|
380
|
+
else {
|
|
381
|
+
const nextStart = models.indexOf(res.model) + 1;
|
|
382
|
+
const segLabel = docLog && batchDocIndices[bi] && batchDocIndices[bi].length > 0
|
|
383
|
+
? segRangeLabel(batchDocIndices[bi], docLog.totalSegments)
|
|
384
|
+
: "";
|
|
385
|
+
let failureLogPath;
|
|
386
|
+
if (failureLogDirAbs && docLog) {
|
|
387
|
+
failureLogPath = writeDocTranslationFailureLog({
|
|
388
|
+
cacheDirAbs: failureLogDirAbs,
|
|
389
|
+
relativePath: docLog.relativePath,
|
|
390
|
+
locale,
|
|
391
|
+
segmentsLabel: segLabel || "(segment range unknown)",
|
|
392
|
+
outcome: nextStart >= models.length ? "fatal" : "retrying_next_model",
|
|
393
|
+
failedModel: res.model,
|
|
394
|
+
nextModel: nextStart < models.length ? models[nextStart] : undefined,
|
|
395
|
+
qualityErrors,
|
|
396
|
+
perSegmentLines,
|
|
397
|
+
systemPrompt: res.debugPrompt?.systemPrompt ?? "",
|
|
398
|
+
userContent: res.debugPrompt?.userContent ?? "",
|
|
399
|
+
rawAssistantContent: res.rawAssistantContent ?? "(missing raw response; rebuild ai-i18n-tools)",
|
|
400
|
+
});
|
|
401
|
+
}
|
|
402
|
+
if (nextStart >= models.length) {
|
|
403
|
+
if (failureLogPath) {
|
|
404
|
+
console.warn(chalk.gray(` š Failure log: ${failureLogPath}`));
|
|
405
|
+
}
|
|
406
|
+
throw new Error(`Doc translation quality failed (${docLog?.relativePath ?? "?"}): ${segLabel ? `${segLabel}: ` : ""}${qualityErrors.join("; ")}`);
|
|
407
|
+
}
|
|
408
|
+
warnDocQualityModelSwitch(locale, docLog?.relativePath, res.model, models[nextStart], qualityErrors.join("; "), segLabel || undefined);
|
|
409
|
+
if (failureLogPath) {
|
|
410
|
+
console.warn(chalk.gray(` š Failure log: ${failureLogPath}`));
|
|
411
|
+
}
|
|
412
|
+
startModelIndex = nextStart;
|
|
413
|
+
}
|
|
414
|
+
}
|
|
415
|
+
}
|
|
416
|
+
catch (e) {
|
|
417
|
+
if (e instanceof BatchTranslationError && client) {
|
|
418
|
+
if (verbose) {
|
|
419
|
+
console.warn(chalk.yellow(`ā ļø ${locale} ${docLog?.relativePath ?? "?"}: batch failed (expected ${e.expected} translated segments, got ${e.received}); falling back to single-segment API calls`));
|
|
420
|
+
}
|
|
421
|
+
for (let si = 0; si < batch.length; si++) {
|
|
422
|
+
const s = batch[si];
|
|
423
|
+
const st = placeholderById.get(s.id);
|
|
424
|
+
const origSeg = segmentOriginalContent(s, originalContentByHash);
|
|
425
|
+
const docIdx0 = batchDocIndices[bi]?.[si];
|
|
426
|
+
const segLabelSingle = docLog && docIdx0 !== undefined ? segRangeLabel([docIdx0], docLog.totalSegments) : "";
|
|
427
|
+
let startIdx = 0;
|
|
428
|
+
while (startIdx < models.length) {
|
|
429
|
+
const single = await client.translateDocumentSegment(s.content, locale, hints, {
|
|
430
|
+
contentType,
|
|
431
|
+
startModelIndex: startIdx,
|
|
432
|
+
docLogContext: docLog && docLog.relativePath
|
|
433
|
+
? { locale, relativePath: docLog.relativePath }
|
|
434
|
+
: undefined,
|
|
435
|
+
});
|
|
436
|
+
localIn += single.usage.inputTokens;
|
|
437
|
+
localOut += single.usage.outputTokens;
|
|
438
|
+
localCost += single.cost ?? 0;
|
|
439
|
+
const restored = restoreSegmentTranslation(ph, single.content, st);
|
|
440
|
+
const v = await validateDocTranslatePair(origSeg, restored);
|
|
441
|
+
if (v.ok) {
|
|
442
|
+
partial.set(s.hash, { text: restored, modelUsed: single.model });
|
|
443
|
+
break;
|
|
444
|
+
}
|
|
445
|
+
const nextIdx = models.indexOf(single.model) + 1;
|
|
446
|
+
const perSegDetail = [
|
|
447
|
+
docLog && docIdx0 !== undefined
|
|
448
|
+
? `doc #${docIdx0 + 1}/${docLog.totalSegments} (hash ${s.hash}): ${v.errors.join("; ")}`
|
|
449
|
+
: `batch index ${si} (hash ${s.hash}): ${v.errors.join("; ")}`,
|
|
450
|
+
];
|
|
451
|
+
let failureLogPathSingle;
|
|
452
|
+
if (failureLogDirAbs && docLog) {
|
|
453
|
+
failureLogPathSingle = writeDocTranslationFailureLog({
|
|
454
|
+
cacheDirAbs: failureLogDirAbs,
|
|
455
|
+
relativePath: docLog.relativePath,
|
|
456
|
+
locale,
|
|
457
|
+
segmentsLabel: segLabelSingle || "(segment range unknown)",
|
|
458
|
+
outcome: nextIdx >= models.length ? "fatal" : "retrying_next_model",
|
|
459
|
+
failedModel: single.model,
|
|
460
|
+
nextModel: nextIdx < models.length ? models[nextIdx] : undefined,
|
|
461
|
+
qualityErrors: v.errors,
|
|
462
|
+
perSegmentLines: perSegDetail,
|
|
463
|
+
systemPrompt: single.debugPrompt?.systemPrompt ?? "",
|
|
464
|
+
userContent: single.debugPrompt?.userContent ?? "",
|
|
465
|
+
rawAssistantContent: single.rawAssistantContent ?? "(missing raw response; rebuild ai-i18n-tools)",
|
|
466
|
+
});
|
|
467
|
+
}
|
|
468
|
+
if (nextIdx >= models.length) {
|
|
469
|
+
if (failureLogPathSingle) {
|
|
470
|
+
console.warn(chalk.gray(` š Failure log: ${failureLogPathSingle}`));
|
|
471
|
+
}
|
|
472
|
+
throw new Error(`Doc translation quality failed (${docLog?.relativePath ?? "?"}): ${segLabelSingle ? `${segLabelSingle}: ` : ""}${v.errors.join("; ")}`);
|
|
473
|
+
}
|
|
474
|
+
warnDocQualityModelSwitch(locale, docLog?.relativePath, single.model, models[nextIdx], v.errors.join("; "), segLabelSingle || undefined);
|
|
475
|
+
if (failureLogPathSingle) {
|
|
476
|
+
console.warn(chalk.gray(` š Failure log: ${failureLogPathSingle}`));
|
|
477
|
+
}
|
|
478
|
+
startIdx = nextIdx;
|
|
479
|
+
}
|
|
480
|
+
}
|
|
481
|
+
}
|
|
482
|
+
else {
|
|
483
|
+
throw e;
|
|
484
|
+
}
|
|
485
|
+
}
|
|
486
|
+
return { partial, inTok: localIn, outTok: localOut, cost: localCost };
|
|
487
|
+
};
|
|
488
|
+
if (sem) {
|
|
489
|
+
const results = await Promise.all(batches.map((_, bi) => sem.use(() => processBatch(bi))));
|
|
490
|
+
for (const r of results) {
|
|
491
|
+
inTok += r.inTok;
|
|
492
|
+
outTok += r.outTok;
|
|
493
|
+
cost += r.cost;
|
|
494
|
+
for (const [h, t] of r.partial) {
|
|
495
|
+
out.set(h, t);
|
|
496
|
+
}
|
|
497
|
+
}
|
|
498
|
+
}
|
|
499
|
+
else {
|
|
500
|
+
for (let bi = 0; bi < batches.length; bi++) {
|
|
501
|
+
const r = await processBatch(bi);
|
|
502
|
+
inTok += r.inTok;
|
|
503
|
+
outTok += r.outTok;
|
|
504
|
+
cost += r.cost;
|
|
505
|
+
for (const [h, t] of r.partial) {
|
|
506
|
+
out.set(h, t);
|
|
507
|
+
}
|
|
508
|
+
}
|
|
509
|
+
}
|
|
510
|
+
return { map: out, inTok, outTok, cost };
|
|
511
|
+
}
|
|
512
|
+
export async function translateMarkdownFile(absSource, relPath, locale, config, cache, client, glossary, opts, hitKeys, translatedMarkdownRelPaths) {
|
|
513
|
+
const totals = {
|
|
514
|
+
filesWritten: 0,
|
|
515
|
+
filesSkipped: 0,
|
|
516
|
+
inputTokens: 0,
|
|
517
|
+
outputTokens: 0,
|
|
518
|
+
costUsd: 0,
|
|
519
|
+
};
|
|
520
|
+
const content = fs.readFileSync(absSource, "utf8");
|
|
521
|
+
const fileHash = hashFileContent(content);
|
|
522
|
+
const sourceFileMtime = fs.statSync(absSource).mtime.toISOString();
|
|
523
|
+
/** Cwd-relative posix path for `translations.filepath` metadata (aligned with JSON/SVG). */
|
|
524
|
+
const translationFilepathMeta = relPath.split(path.sep).join("/");
|
|
525
|
+
const outPath = resolveTranslatedOutputPath(config, opts.cwd, locale, relPath, "markdown");
|
|
526
|
+
const blockIdx = opts.documentationBlockIndex ?? 0;
|
|
527
|
+
const fileTrackingKey = documentationFileTrackingKey(blockIdx, relPath);
|
|
528
|
+
if (opts.force && cache && !opts.noCache) {
|
|
529
|
+
await withCacheMutex(opts.cacheMutex, () => cache.clearFile(fileTrackingKey, locale));
|
|
530
|
+
if (opts.verbose) {
|
|
531
|
+
console.log(chalk.yellow(` š Force mode: cleared file tracking for ${relPath} (${locale})`));
|
|
532
|
+
}
|
|
533
|
+
}
|
|
534
|
+
const cachedFileHash = cache && !opts.noCache
|
|
535
|
+
? await withCacheMutex(opts.cacheMutex, () => cache.getFileHash(fileTrackingKey, locale))
|
|
536
|
+
: null;
|
|
537
|
+
if (!opts.force &&
|
|
538
|
+
!opts.forceUpdate &&
|
|
539
|
+
cache &&
|
|
540
|
+
!opts.noCache &&
|
|
541
|
+
cachedFileHash === fileHash &&
|
|
542
|
+
fs.existsSync(outPath)) {
|
|
543
|
+
if (opts.verbose) {
|
|
544
|
+
console.log(chalk.gray(`āļø ${timestamp()} - ${locale} ${relPath} (unchanged)`));
|
|
545
|
+
}
|
|
546
|
+
totals.filesSkipped = 1;
|
|
547
|
+
totals.filesProcessed = 0;
|
|
548
|
+
return { skipped: true, totals };
|
|
549
|
+
}
|
|
550
|
+
const fileStartTime = Date.now();
|
|
551
|
+
const md = new MarkdownExtractor();
|
|
552
|
+
const langListCfg = config.documentation.markdownOutput.postProcessing?.languageListBlock;
|
|
553
|
+
const segments = md.extract(content, relPath, langListCfg ? { languageListBlock: langListCfg } : undefined);
|
|
554
|
+
const translatableCount = segments.filter((s) => s.translatable).length;
|
|
555
|
+
console.log(chalk.yellow(`š ${locale} ${relPath}: ${segments.length} segment(s) (${translatableCount} translatable)`));
|
|
556
|
+
const translations = new Map();
|
|
557
|
+
const placeholderById = new Map();
|
|
558
|
+
const originalContentByHash = new Map();
|
|
559
|
+
const toBatch = [];
|
|
560
|
+
const segmentIndicesInDoc = [];
|
|
561
|
+
let segmentsCached = 0;
|
|
562
|
+
for (let docIdx = 0; docIdx < segments.length; docIdx++) {
|
|
563
|
+
const s = segments[docIdx];
|
|
564
|
+
if (!s.translatable) {
|
|
565
|
+
continue;
|
|
566
|
+
}
|
|
567
|
+
if (!opts.force && cache && !opts.noCache) {
|
|
568
|
+
const hit = await withCacheMutex(opts.cacheMutex, () => cache.getSegmentDetails(s.hash, locale, translationFilepathMeta, s.startLine));
|
|
569
|
+
if (hit) {
|
|
570
|
+
const quality = await validateDocTranslatePair(s, hit.text);
|
|
571
|
+
if (quality.ok) {
|
|
572
|
+
const modelUsed = hit.model?.trim();
|
|
573
|
+
translations.set(s.hash, {
|
|
574
|
+
text: hit.text,
|
|
575
|
+
...(modelUsed ? { modelUsed } : {}),
|
|
576
|
+
});
|
|
577
|
+
hitKeys.add(`${s.hash}|${locale}`);
|
|
578
|
+
segmentsCached++;
|
|
579
|
+
continue;
|
|
580
|
+
}
|
|
581
|
+
if (opts.verbose) {
|
|
582
|
+
console.warn(chalk.yellow(` ā ļø ${relPath} (${locale}): cache rejected for segment (hash ${s.hash}): ${quality.errors.join("; ")}`));
|
|
583
|
+
}
|
|
584
|
+
}
|
|
585
|
+
}
|
|
586
|
+
originalContentByHash.set(s.hash, s.content);
|
|
587
|
+
const { text: protectedText, state: st } = protectSegmentForTranslation(s.content, glossary, locale, true);
|
|
588
|
+
placeholderById.set(s.id, st);
|
|
589
|
+
toBatch.push({ ...s, content: protectedText });
|
|
590
|
+
segmentIndicesInDoc.push(docIdx);
|
|
591
|
+
}
|
|
592
|
+
const batchSize = config.batchSize ?? 20;
|
|
593
|
+
const maxBatchChars = config.maxBatchChars ?? 4096;
|
|
594
|
+
const batchConcurrency = opts.batchConcurrency ?? config.batchConcurrency ?? 4;
|
|
595
|
+
const { map, inTok, outTok, cost } = await translateSegmentsBatched(toBatch, placeholderById, originalContentByHash, locale, glossary, client, opts.dryRun, opts.verbose, batchSize, maxBatchChars, "markdown", batchConcurrency, translatePromptFormatToResponseFormat(opts.promptFormat), {
|
|
596
|
+
relativePath: relPath,
|
|
597
|
+
totalSegments: segments.length,
|
|
598
|
+
segmentIndicesInDoc,
|
|
599
|
+
}, path.join(opts.cwd, config.cacheDir));
|
|
600
|
+
for (const [h, t] of map) {
|
|
601
|
+
translations.set(h, t);
|
|
602
|
+
}
|
|
603
|
+
totals.inputTokens += inTok;
|
|
604
|
+
totals.outputTokens += outTok;
|
|
605
|
+
totals.costUsd = (totals.costUsd ?? 0) + cost;
|
|
606
|
+
for (const s of segments) {
|
|
607
|
+
if (s.translatable && translations.has(s.hash)) {
|
|
608
|
+
hitKeys.add(`${s.hash}|${locale}`);
|
|
609
|
+
}
|
|
610
|
+
}
|
|
611
|
+
const merged = segments.map((s) => ({
|
|
612
|
+
...s,
|
|
613
|
+
content: s.translatable
|
|
614
|
+
? (segmentTranslationText(translations.get(s.hash)) ?? s.content)
|
|
615
|
+
: s.content,
|
|
616
|
+
}));
|
|
617
|
+
const v = await validateTranslation(segments, merged);
|
|
618
|
+
if (!opts.dryRun && (!v.valid || v.warnings.length > 0)) {
|
|
619
|
+
const parts = [...v.errors, ...v.warnings];
|
|
620
|
+
console.warn(chalk.yellow(` ā ļø ${relPath} (${locale}): ${parts.join("; ")}`));
|
|
621
|
+
}
|
|
622
|
+
let output = md.reassemble(segments, translations);
|
|
623
|
+
if (shouldRewriteFlatMarkdownLinks(config)) {
|
|
624
|
+
const mo = config.documentation.markdownOutput;
|
|
625
|
+
const docsRoot = mo.linkRewriteDocsRoot?.trim() || ".";
|
|
626
|
+
const { i18nPrefix, depthPrefix } = computeFlatLinkRewritePrefixes(opts.cwd, docsRoot, config.documentation.outputDir);
|
|
627
|
+
const parsed = matter(output);
|
|
628
|
+
const newBody = rewriteDocLinksForFlatOutput(parsed.content, locale, i18nPrefix, depthPrefix, {
|
|
629
|
+
cwd: opts.cwd,
|
|
630
|
+
config,
|
|
631
|
+
currentSourceRelPath: normalizeMarkdownRelPath(relPath),
|
|
632
|
+
translatedMarkdownRelPaths,
|
|
633
|
+
});
|
|
634
|
+
output = matterStringify(newBody, parsed.data);
|
|
635
|
+
}
|
|
636
|
+
const moPost = config.documentation.markdownOutput.postProcessing;
|
|
637
|
+
if (moPost && (moPost.languageListBlock || (moPost.regexAdjustments?.length ?? 0) > 0)) {
|
|
638
|
+
const absSource = path.join(opts.cwd, relPath);
|
|
639
|
+
const docStem = path.parse(relPath).name;
|
|
640
|
+
output = applyMarkdownPostProcessing(output, {
|
|
641
|
+
config,
|
|
642
|
+
cwd: opts.cwd,
|
|
643
|
+
relPath,
|
|
644
|
+
locale,
|
|
645
|
+
absSource,
|
|
646
|
+
absTranslated: outPath,
|
|
647
|
+
verbose: opts.verbose,
|
|
648
|
+
docStem,
|
|
649
|
+
});
|
|
650
|
+
}
|
|
651
|
+
if (config.documentation.addFrontmatter !== false && !opts.dryRun) {
|
|
652
|
+
const translationModels = collectTranslationModelsFromSegments(segments, translations);
|
|
653
|
+
output = addTranslationMetadata(output, sourceFileMtime, fileHash, locale, relPath, translationModels);
|
|
654
|
+
}
|
|
655
|
+
if (!opts.dryRun) {
|
|
656
|
+
writeAtomicUtf8(outPath, output);
|
|
657
|
+
if (cache && !opts.noCache) {
|
|
658
|
+
await withCacheMutex(opts.cacheMutex, () => {
|
|
659
|
+
cache.setFileStatus(fileTrackingKey, locale, fileHash);
|
|
660
|
+
for (const s of segments) {
|
|
661
|
+
if (!s.translatable) {
|
|
662
|
+
continue;
|
|
663
|
+
}
|
|
664
|
+
const entry = translations.get(s.hash);
|
|
665
|
+
if (entry === undefined || entry.modelUsed === undefined) {
|
|
666
|
+
continue;
|
|
667
|
+
}
|
|
668
|
+
cache.setSegment(s.hash, locale, s.content, entry.text, entry.modelUsed, translationFilepathMeta, s.startLine ?? null);
|
|
669
|
+
}
|
|
670
|
+
});
|
|
671
|
+
}
|
|
672
|
+
totals.filesWritten = 1;
|
|
673
|
+
}
|
|
674
|
+
else {
|
|
675
|
+
totals.filesWritten = 0;
|
|
676
|
+
}
|
|
677
|
+
const segmentsNew = translatableCount - segmentsCached;
|
|
678
|
+
totals.segmentsCached = segmentsCached;
|
|
679
|
+
totals.segmentsTranslated = segmentsNew;
|
|
680
|
+
totals.filesProcessed = 1;
|
|
681
|
+
logTranslateFileComplete(relPath, outPath, segmentsCached, segmentsNew, Date.now() - fileStartTime, totals.costUsd ?? 0);
|
|
682
|
+
return { skipped: false, totals };
|
|
683
|
+
}
|
|
684
|
+
export async function translateJsonFile(absSource, relPath, locale, config, cache, client, glossary, opts, hitKeys) {
|
|
685
|
+
const totals = {
|
|
686
|
+
filesWritten: 0,
|
|
687
|
+
filesSkipped: 0,
|
|
688
|
+
inputTokens: 0,
|
|
689
|
+
outputTokens: 0,
|
|
690
|
+
costUsd: 0,
|
|
691
|
+
};
|
|
692
|
+
const content = fs.readFileSync(absSource, "utf8");
|
|
693
|
+
const fileHash = hashFileContent(content);
|
|
694
|
+
/** Cwd-relative path for cache/UI and `file_tracking` (must match `resolveDocTrackingKeyToAbs` under project root). */
|
|
695
|
+
const relPathFromCwd = path.relative(opts.cwd, absSource).split(path.sep).join("/");
|
|
696
|
+
const outPath = resolveTranslatedOutputPath(config, opts.cwd, locale, relPath, "json");
|
|
697
|
+
const blockIdx = opts.documentationBlockIndex ?? 0;
|
|
698
|
+
/** Use cwd-relative path so cleanup resolves the same path as the real JSON source (json-root-only `relPath` would resolve to the wrong directory). */
|
|
699
|
+
const fileTrackingKey = documentationFileTrackingKey(blockIdx, relPathFromCwd);
|
|
700
|
+
if (opts.force && cache && !opts.noCache) {
|
|
701
|
+
await withCacheMutex(opts.cacheMutex, () => cache.clearFile(fileTrackingKey, locale));
|
|
702
|
+
if (opts.verbose) {
|
|
703
|
+
console.log(chalk.yellow(` š Force mode: cleared file tracking for ${relPath} (${locale})`));
|
|
704
|
+
}
|
|
705
|
+
}
|
|
706
|
+
const cachedFileHashJson = cache && !opts.noCache
|
|
707
|
+
? await withCacheMutex(opts.cacheMutex, () => cache.getFileHash(fileTrackingKey, locale))
|
|
708
|
+
: null;
|
|
709
|
+
if (!opts.force &&
|
|
710
|
+
!opts.forceUpdate &&
|
|
711
|
+
cache &&
|
|
712
|
+
!opts.noCache &&
|
|
713
|
+
cachedFileHashJson === fileHash &&
|
|
714
|
+
fs.existsSync(outPath)) {
|
|
715
|
+
if (opts.verbose) {
|
|
716
|
+
console.log(chalk.gray(`āļø ${timestamp()} - ${locale} ${relPath} (json, unchanged)`));
|
|
717
|
+
}
|
|
718
|
+
totals.filesSkipped = 1;
|
|
719
|
+
totals.filesProcessed = 0;
|
|
720
|
+
return { skipped: true, totals };
|
|
721
|
+
}
|
|
722
|
+
const fileStartTime = Date.now();
|
|
723
|
+
const jx = new JsonExtractor();
|
|
724
|
+
const segments = jx.extract(content, relPath);
|
|
725
|
+
const translatableCount = segments.filter((s) => s.translatable).length;
|
|
726
|
+
console.log(chalk.yellow(`š ${locale} ${relPath} (json): ${segments.length} segment(s) (${translatableCount} translatable)`));
|
|
727
|
+
const translations = new Map();
|
|
728
|
+
const placeholderByIdJson = new Map();
|
|
729
|
+
const toBatch = [];
|
|
730
|
+
const segmentIndicesInDoc = [];
|
|
731
|
+
let segmentsCached = 0;
|
|
732
|
+
for (let docIdx = 0; docIdx < segments.length; docIdx++) {
|
|
733
|
+
const s = segments[docIdx];
|
|
734
|
+
if (!s.translatable) {
|
|
735
|
+
continue;
|
|
736
|
+
}
|
|
737
|
+
if (!opts.force && cache && !opts.noCache) {
|
|
738
|
+
const hit = await withCacheMutex(opts.cacheMutex, () => cache.getSegment(s.hash, locale, relPathFromCwd));
|
|
739
|
+
if (hit) {
|
|
740
|
+
translations.set(s.hash, { text: hit });
|
|
741
|
+
hitKeys.add(`${s.hash}|${locale}`);
|
|
742
|
+
segmentsCached++;
|
|
743
|
+
continue;
|
|
744
|
+
}
|
|
745
|
+
}
|
|
746
|
+
const { text: protectedText, state: st } = protectSegmentForTranslation(s.content, glossary, locale, false);
|
|
747
|
+
placeholderByIdJson.set(s.id, st);
|
|
748
|
+
toBatch.push({ ...s, content: protectedText });
|
|
749
|
+
segmentIndicesInDoc.push(docIdx);
|
|
750
|
+
}
|
|
751
|
+
const batchSize = config.batchSize ?? 20;
|
|
752
|
+
const maxBatchChars = config.maxBatchChars ?? 4096;
|
|
753
|
+
const batchConcurrencyJson = opts.batchConcurrency ?? config.batchConcurrency ?? 4;
|
|
754
|
+
const { map, inTok, outTok, cost } = await translateSegmentsBatched(toBatch, placeholderByIdJson, new Map(), locale, glossary, client, opts.dryRun, opts.verbose, batchSize, maxBatchChars, "json", batchConcurrencyJson, translatePromptFormatToResponseFormat(opts.promptFormat), {
|
|
755
|
+
relativePath: relPath,
|
|
756
|
+
totalSegments: segments.length,
|
|
757
|
+
segmentIndicesInDoc,
|
|
758
|
+
});
|
|
759
|
+
for (const [h, t] of map) {
|
|
760
|
+
translations.set(h, t);
|
|
761
|
+
}
|
|
762
|
+
totals.inputTokens += inTok;
|
|
763
|
+
totals.outputTokens += outTok;
|
|
764
|
+
totals.costUsd = (totals.costUsd ?? 0) + cost;
|
|
765
|
+
for (const s of segments) {
|
|
766
|
+
if (s.translatable && translations.has(s.hash)) {
|
|
767
|
+
hitKeys.add(`${s.hash}|${locale}`);
|
|
768
|
+
}
|
|
769
|
+
}
|
|
770
|
+
const output = jx.reassemble(segments, translations);
|
|
771
|
+
if (!opts.dryRun) {
|
|
772
|
+
writeAtomicUtf8(outPath, output);
|
|
773
|
+
if (cache && !opts.noCache) {
|
|
774
|
+
await withCacheMutex(opts.cacheMutex, () => {
|
|
775
|
+
cache.setFileStatus(fileTrackingKey, locale, fileHash);
|
|
776
|
+
for (const s of segments) {
|
|
777
|
+
if (!s.translatable) {
|
|
778
|
+
continue;
|
|
779
|
+
}
|
|
780
|
+
const entry = translations.get(s.hash);
|
|
781
|
+
if (entry === undefined || entry.modelUsed === undefined) {
|
|
782
|
+
continue;
|
|
783
|
+
}
|
|
784
|
+
cache.setSegment(s.hash, locale, s.content, entry.text, entry.modelUsed, relPathFromCwd, null);
|
|
785
|
+
}
|
|
786
|
+
});
|
|
787
|
+
}
|
|
788
|
+
totals.filesWritten = 1;
|
|
789
|
+
}
|
|
790
|
+
const segmentsNew = translatableCount - segmentsCached;
|
|
791
|
+
totals.segmentsCached = segmentsCached;
|
|
792
|
+
totals.segmentsTranslated = segmentsNew;
|
|
793
|
+
totals.filesProcessed = 1;
|
|
794
|
+
logTranslateFileComplete(relPath, outPath, segmentsCached, segmentsNew, Date.now() - fileStartTime, totals.costUsd ?? 0);
|
|
795
|
+
return { skipped: false, totals };
|
|
796
|
+
}
|
|
797
|
+
/** Translate one SVG asset (`translate-svg`); uses `config.svg` for paths and optional `svg.svgExtractor`. */
|
|
798
|
+
export async function translateSvgAssetFile(absSource, relPathFromCwd, relPathFromSourceRoot, locale, config, cache, client, glossary, opts, hitKeys) {
|
|
799
|
+
const totals = {
|
|
800
|
+
filesWritten: 0,
|
|
801
|
+
filesSkipped: 0,
|
|
802
|
+
inputTokens: 0,
|
|
803
|
+
outputTokens: 0,
|
|
804
|
+
costUsd: 0,
|
|
805
|
+
};
|
|
806
|
+
const cacheKey = svgAssetCacheFilepath(relPathFromCwd);
|
|
807
|
+
const translationSvgFilepathMeta = svgTranslationFilepathMetadata(relPathFromCwd);
|
|
808
|
+
const outPath = resolveSvgAssetOutputPath(config, opts.cwd, locale, relPathFromCwd, relPathFromSourceRoot);
|
|
809
|
+
const forceLc = config.svg?.svgExtractor?.forceLowercase ?? false;
|
|
810
|
+
const content = fs.readFileSync(absSource, "utf8");
|
|
811
|
+
const fileHash = hashFileContent(content);
|
|
812
|
+
if (opts.force && cache && !opts.noCache) {
|
|
813
|
+
await withCacheMutex(opts.cacheMutex, () => cache.clearFile(cacheKey, locale));
|
|
814
|
+
if (opts.verbose) {
|
|
815
|
+
console.log(chalk.yellow(` š Force mode: cleared file tracking for ${relPathFromCwd} (${locale})`));
|
|
816
|
+
}
|
|
817
|
+
}
|
|
818
|
+
const cachedFileHashSvg = cache && !opts.noCache
|
|
819
|
+
? await withCacheMutex(opts.cacheMutex, () => cache.getFileHash(cacheKey, locale))
|
|
820
|
+
: null;
|
|
821
|
+
if (!opts.force &&
|
|
822
|
+
!opts.forceUpdate &&
|
|
823
|
+
cache &&
|
|
824
|
+
!opts.noCache &&
|
|
825
|
+
cachedFileHashSvg === fileHash &&
|
|
826
|
+
fs.existsSync(outPath)) {
|
|
827
|
+
if (opts.verbose) {
|
|
828
|
+
console.log(chalk.gray(`āļø ${timestamp()} - ${locale} ${relPathFromCwd} (svg, unchanged)`));
|
|
829
|
+
}
|
|
830
|
+
totals.filesSkipped = 1;
|
|
831
|
+
totals.filesProcessed = 0;
|
|
832
|
+
return { skipped: true, totals };
|
|
833
|
+
}
|
|
834
|
+
const fileStartTime = Date.now();
|
|
835
|
+
if (normalizeLocale(locale) === normalizeLocale(config.sourceLocale)) {
|
|
836
|
+
if (!opts.dryRun) {
|
|
837
|
+
ensureDirForFile(outPath);
|
|
838
|
+
fs.copyFileSync(absSource, outPath);
|
|
839
|
+
if (cache && !opts.noCache) {
|
|
840
|
+
await withCacheMutex(opts.cacheMutex, () => {
|
|
841
|
+
cache.setFileStatus(cacheKey, locale, fileHash);
|
|
842
|
+
});
|
|
843
|
+
}
|
|
844
|
+
totals.filesWritten = 1;
|
|
845
|
+
}
|
|
846
|
+
totals.filesProcessed = 1;
|
|
847
|
+
logTranslateFileComplete(relPathFromCwd, outPath, 0, 0, Date.now() - fileStartTime, 0);
|
|
848
|
+
return { skipped: false, totals };
|
|
849
|
+
}
|
|
850
|
+
const sx = new SvgExtractor({ forceLowercase: forceLc });
|
|
851
|
+
const segments = sx.extract(content, relPathFromCwd);
|
|
852
|
+
const translatableCount = segments.filter((s) => s.translatable).length;
|
|
853
|
+
console.log(chalk.yellow(`š ${locale} ${relPathFromCwd} (svg): ${segments.length} segment(s) (${translatableCount} translatable)`));
|
|
854
|
+
const translations = new Map();
|
|
855
|
+
const placeholderByIdSvg = new Map();
|
|
856
|
+
const toBatch = [];
|
|
857
|
+
const segmentIndicesInDoc = [];
|
|
858
|
+
let segmentsCached = 0;
|
|
859
|
+
for (let docIdx = 0; docIdx < segments.length; docIdx++) {
|
|
860
|
+
const s = segments[docIdx];
|
|
861
|
+
if (!s.translatable) {
|
|
862
|
+
continue;
|
|
863
|
+
}
|
|
864
|
+
if (!opts.force && cache && !opts.noCache) {
|
|
865
|
+
const hit = await withCacheMutex(opts.cacheMutex, () => cache.getSegment(s.hash, locale, translationSvgFilepathMeta));
|
|
866
|
+
if (hit) {
|
|
867
|
+
let t = hit;
|
|
868
|
+
if (forceLc) {
|
|
869
|
+
t = t.toLowerCase();
|
|
870
|
+
}
|
|
871
|
+
translations.set(s.hash, { text: t });
|
|
872
|
+
hitKeys.add(`${s.hash}|${locale}`);
|
|
873
|
+
segmentsCached++;
|
|
874
|
+
continue;
|
|
875
|
+
}
|
|
876
|
+
}
|
|
877
|
+
const { text: protectedText, state: st } = protectSegmentForTranslation(s.content, glossary, locale, false);
|
|
878
|
+
placeholderByIdSvg.set(s.id, st);
|
|
879
|
+
toBatch.push({ ...s, content: protectedText });
|
|
880
|
+
segmentIndicesInDoc.push(docIdx);
|
|
881
|
+
}
|
|
882
|
+
const batchSize = config.batchSize ?? 20;
|
|
883
|
+
const maxBatchChars = config.maxBatchChars ?? 4096;
|
|
884
|
+
const batchConcurrencySvg = opts.batchConcurrency ?? config.batchConcurrency ?? 4;
|
|
885
|
+
const { map, inTok, outTok, cost } = await translateSegmentsBatched(toBatch, placeholderByIdSvg, new Map(), locale, glossary, client, opts.dryRun, opts.verbose, batchSize, maxBatchChars, "svg", batchConcurrencySvg, translatePromptFormatToResponseFormat(opts.promptFormat), {
|
|
886
|
+
relativePath: relPathFromCwd,
|
|
887
|
+
totalSegments: segments.length,
|
|
888
|
+
segmentIndicesInDoc,
|
|
889
|
+
});
|
|
890
|
+
for (const [h, t] of map) {
|
|
891
|
+
translations.set(h, t);
|
|
892
|
+
}
|
|
893
|
+
totals.inputTokens += inTok;
|
|
894
|
+
totals.outputTokens += outTok;
|
|
895
|
+
totals.costUsd = (totals.costUsd ?? 0) + cost;
|
|
896
|
+
for (const s of segments) {
|
|
897
|
+
if (s.translatable && translations.has(s.hash)) {
|
|
898
|
+
hitKeys.add(`${s.hash}|${locale}`);
|
|
899
|
+
}
|
|
900
|
+
}
|
|
901
|
+
const output = sx.reassemble(segments, translations);
|
|
902
|
+
if (!opts.dryRun) {
|
|
903
|
+
writeAtomicUtf8(outPath, output);
|
|
904
|
+
if (cache && !opts.noCache) {
|
|
905
|
+
await withCacheMutex(opts.cacheMutex, () => {
|
|
906
|
+
cache.setFileStatus(cacheKey, locale, fileHash);
|
|
907
|
+
for (const s of segments) {
|
|
908
|
+
if (!s.translatable) {
|
|
909
|
+
continue;
|
|
910
|
+
}
|
|
911
|
+
const entry = translations.get(s.hash);
|
|
912
|
+
if (entry === undefined || entry.modelUsed === undefined) {
|
|
913
|
+
continue;
|
|
914
|
+
}
|
|
915
|
+
cache.setSegment(s.hash, locale, s.content, entry.text, entry.modelUsed, translationSvgFilepathMeta, null);
|
|
916
|
+
}
|
|
917
|
+
});
|
|
918
|
+
}
|
|
919
|
+
totals.filesWritten = 1;
|
|
920
|
+
}
|
|
921
|
+
const segmentsNew = translatableCount - segmentsCached;
|
|
922
|
+
totals.segmentsCached = segmentsCached;
|
|
923
|
+
totals.segmentsTranslated = segmentsNew;
|
|
924
|
+
totals.filesProcessed = 1;
|
|
925
|
+
logTranslateFileComplete(relPathFromCwd, outPath, segmentsCached, segmentsNew, Date.now() - fileStartTime, totals.costUsd ?? 0);
|
|
926
|
+
return { skipped: false, totals };
|
|
927
|
+
}
|
|
928
|
+
export function rewriteSourceMarkdownLanguageListBlocks(config, opts, markdownFiles) {
|
|
929
|
+
const langCfg = config.documentation.markdownOutput.postProcessing?.languageListBlock;
|
|
930
|
+
if (!langCfg || opts.dryRun || !shouldRunMarkdown(opts, config)) {
|
|
931
|
+
return 0;
|
|
932
|
+
}
|
|
933
|
+
let rewritten = 0;
|
|
934
|
+
for (const relPath of markdownFiles) {
|
|
935
|
+
if (!matchesPathFilter(relPath, opts.pathFilter)) {
|
|
936
|
+
continue;
|
|
937
|
+
}
|
|
938
|
+
const absSource = path.join(opts.cwd, relPath);
|
|
939
|
+
const input = fs.readFileSync(absSource, "utf8");
|
|
940
|
+
const docStem = path.parse(relPath).name;
|
|
941
|
+
const output = applyMarkdownLanguageListPostProcessing(input, {
|
|
942
|
+
config,
|
|
943
|
+
cwd: opts.cwd,
|
|
944
|
+
relPath,
|
|
945
|
+
absCurrentFile: absSource,
|
|
946
|
+
verbose: opts.verbose,
|
|
947
|
+
docStem,
|
|
948
|
+
missingBlockTarget: "source document",
|
|
949
|
+
});
|
|
950
|
+
if (output === input) {
|
|
951
|
+
continue;
|
|
952
|
+
}
|
|
953
|
+
writeAtomicUtf8(absSource, output);
|
|
954
|
+
rewritten++;
|
|
955
|
+
console.log(chalk.green(`ā
${relPath}: Source language lists refreshed`));
|
|
956
|
+
}
|
|
957
|
+
return rewritten;
|
|
958
|
+
}
|
|
959
|
+
/** Run translate for all enabled kinds and locales. */
|
|
960
|
+
export async function runTranslate(config, opts, files, jsonAbsRoot) {
|
|
961
|
+
const sum = {
|
|
962
|
+
filesWritten: 0,
|
|
963
|
+
filesSkipped: 0,
|
|
964
|
+
filesProcessed: 0,
|
|
965
|
+
inputTokens: 0,
|
|
966
|
+
outputTokens: 0,
|
|
967
|
+
costUsd: 0,
|
|
968
|
+
segmentsCached: 0,
|
|
969
|
+
segmentsTranslated: 0,
|
|
970
|
+
};
|
|
971
|
+
const cache = opts.noCache
|
|
972
|
+
? null
|
|
973
|
+
: new TranslationCache(path.join(opts.cwd, config.cacheDir));
|
|
974
|
+
const needsApi = !opts.dryRun && (config.features.translateMarkdown || config.features.translateJSON);
|
|
975
|
+
const client = needsApi ? new OpenRouterClient({ config }) : null;
|
|
976
|
+
const glossaryUi = config.glossary?.uiGlossary
|
|
977
|
+
? path.join(opts.cwd, config.glossary.uiGlossary)
|
|
978
|
+
: undefined;
|
|
979
|
+
const glossaryUser = config.glossary?.userGlossary
|
|
980
|
+
? path.join(opts.cwd, config.glossary.userGlossary)
|
|
981
|
+
: undefined;
|
|
982
|
+
const hitKeys = new Set();
|
|
983
|
+
const markdownHitKeys = new Set();
|
|
984
|
+
const translatedMarkdownRelPaths = new Set(files.markdown.map((r) => normalizeMarkdownRelPath(r)));
|
|
985
|
+
const locales = opts.locales.map((l) => normalizeLocale(l));
|
|
986
|
+
const glossary = new Glossary(glossaryUi, glossaryUser, locales);
|
|
987
|
+
const totalFileCount = files.markdown.length + files.json.length;
|
|
988
|
+
const models = client?.getConfiguredModels() ?? [];
|
|
989
|
+
const docDescription = config.documentation.description?.trim();
|
|
990
|
+
const translatingHeadline = docDescription
|
|
991
|
+
? `š ${docDescription}: translating ${totalFileCount} file(s) to ${locales.length} locale(s)\n`
|
|
992
|
+
: `š Translating ${totalFileCount} file(s) to ${locales.length} locale(s)\n`;
|
|
993
|
+
// Header block
|
|
994
|
+
console.log(chalk.gray("\n\n___DOCUMENTATION Translation_____________________________________________________________________________\n\n") + chalk.bold(`\n${translatingHeadline}`));
|
|
995
|
+
printModelsTryInOrder(models);
|
|
996
|
+
console.log(chalk.cyan(`Glossary terms: `) + chalk.magenta(`${glossary.size}`));
|
|
997
|
+
console.log(chalk.cyan(`Output: `) +
|
|
998
|
+
chalk.magenta(`${path.resolve(opts.cwd, config.documentation.outputDir)}`));
|
|
999
|
+
if (opts.logPath) {
|
|
1000
|
+
console.log(chalk.cyan(`Output log: `) + chalk.magenta(opts.logPath));
|
|
1001
|
+
}
|
|
1002
|
+
if (opts.dryRun) {
|
|
1003
|
+
console.log(chalk.yellow(`\nā ļø Dry run mode - no changes will be made`));
|
|
1004
|
+
}
|
|
1005
|
+
console.log("");
|
|
1006
|
+
const localeConcurrency = Math.max(1, Math.floor(opts.concurrency ?? config.concurrency ?? 3));
|
|
1007
|
+
const batchConcurrencyEffective = Math.max(1, Math.floor(opts.batchConcurrency ?? config.batchConcurrency ?? 4));
|
|
1008
|
+
console.log(chalk.cyan(`Locale concurrency: `) + chalk.magenta(`${localeConcurrency}`));
|
|
1009
|
+
console.log(chalk.cyan(`Parallel API calls per file: `) + chalk.magenta(`${batchConcurrencyEffective}`));
|
|
1010
|
+
console.log(chalk.cyan(`Batch prompt format: `) + chalk.magenta(`${opts.promptFormat ?? "xml"}`));
|
|
1011
|
+
console.log("");
|
|
1012
|
+
const wallStart = Date.now();
|
|
1013
|
+
const localeTimes = [];
|
|
1014
|
+
const cacheMutex = cache && locales.length > 1 ? new AsyncMutex() : undefined;
|
|
1015
|
+
const runOpts = {
|
|
1016
|
+
...opts,
|
|
1017
|
+
batchConcurrency: batchConcurrencyEffective,
|
|
1018
|
+
cacheMutex,
|
|
1019
|
+
};
|
|
1020
|
+
const processLocale = async (locale) => {
|
|
1021
|
+
const markdownHitKeysLocal = new Set();
|
|
1022
|
+
const partial = {
|
|
1023
|
+
filesWritten: 0,
|
|
1024
|
+
filesSkipped: 0,
|
|
1025
|
+
filesProcessed: 0,
|
|
1026
|
+
inputTokens: 0,
|
|
1027
|
+
outputTokens: 0,
|
|
1028
|
+
costUsd: 0,
|
|
1029
|
+
segmentsCached: 0,
|
|
1030
|
+
segmentsTranslated: 0,
|
|
1031
|
+
};
|
|
1032
|
+
const localeStart = Date.now();
|
|
1033
|
+
if (shouldRunMarkdown(opts, config)) {
|
|
1034
|
+
for (const rel of files.markdown) {
|
|
1035
|
+
if (!matchesPathFilter(rel, opts.pathFilter)) {
|
|
1036
|
+
continue;
|
|
1037
|
+
}
|
|
1038
|
+
const abs = path.join(opts.cwd, rel);
|
|
1039
|
+
const { skipped, totals } = await translateMarkdownFile(abs, rel, locale, config, cache, client, glossary, runOpts, markdownHitKeysLocal, translatedMarkdownRelPaths);
|
|
1040
|
+
if (skipped) {
|
|
1041
|
+
partial.filesSkipped += totals.filesSkipped;
|
|
1042
|
+
partial.filesProcessed = (partial.filesProcessed ?? 0) + (totals.filesProcessed ?? 0);
|
|
1043
|
+
}
|
|
1044
|
+
else {
|
|
1045
|
+
partial.filesWritten += totals.filesWritten;
|
|
1046
|
+
partial.filesProcessed = (partial.filesProcessed ?? 0) + (totals.filesProcessed ?? 0);
|
|
1047
|
+
partial.inputTokens += totals.inputTokens;
|
|
1048
|
+
partial.outputTokens += totals.outputTokens;
|
|
1049
|
+
partial.costUsd = (partial.costUsd ?? 0) + (totals.costUsd ?? 0);
|
|
1050
|
+
partial.segmentsCached = (partial.segmentsCached ?? 0) + (totals.segmentsCached ?? 0);
|
|
1051
|
+
partial.segmentsTranslated =
|
|
1052
|
+
(partial.segmentsTranslated ?? 0) + (totals.segmentsTranslated ?? 0);
|
|
1053
|
+
}
|
|
1054
|
+
}
|
|
1055
|
+
}
|
|
1056
|
+
if (shouldRunJson(opts, config)) {
|
|
1057
|
+
for (const rel of files.json) {
|
|
1058
|
+
if (!matchesPathFilter(rel, opts.pathFilter)) {
|
|
1059
|
+
continue;
|
|
1060
|
+
}
|
|
1061
|
+
const abs = path.join(jsonAbsRoot, rel);
|
|
1062
|
+
const { skipped, totals } = await translateJsonFile(abs, rel, locale, config, cache, client, glossary, runOpts, hitKeys);
|
|
1063
|
+
if (skipped) {
|
|
1064
|
+
partial.filesSkipped += totals.filesSkipped;
|
|
1065
|
+
partial.filesProcessed = (partial.filesProcessed ?? 0) + (totals.filesProcessed ?? 0);
|
|
1066
|
+
}
|
|
1067
|
+
else {
|
|
1068
|
+
partial.filesWritten += totals.filesWritten;
|
|
1069
|
+
partial.filesProcessed = (partial.filesProcessed ?? 0) + (totals.filesProcessed ?? 0);
|
|
1070
|
+
partial.inputTokens += totals.inputTokens;
|
|
1071
|
+
partial.outputTokens += totals.outputTokens;
|
|
1072
|
+
partial.costUsd = (partial.costUsd ?? 0) + (totals.costUsd ?? 0);
|
|
1073
|
+
partial.segmentsCached = (partial.segmentsCached ?? 0) + (totals.segmentsCached ?? 0);
|
|
1074
|
+
partial.segmentsTranslated =
|
|
1075
|
+
(partial.segmentsTranslated ?? 0) + (totals.segmentsTranslated ?? 0);
|
|
1076
|
+
}
|
|
1077
|
+
}
|
|
1078
|
+
}
|
|
1079
|
+
const localeElapsed = Date.now() - localeStart;
|
|
1080
|
+
if (localeElapsed > 0) {
|
|
1081
|
+
console.log(chalk.gray(` [${locale}] Time: ${formatElapsedMmSs(localeElapsed)}`));
|
|
1082
|
+
}
|
|
1083
|
+
return { locale, partial, markdownHitKeysLocal, localeElapsed };
|
|
1084
|
+
};
|
|
1085
|
+
const localeResults = await runMapWithConcurrency(locales, localeConcurrency, async (locale) => processLocale(locale));
|
|
1086
|
+
for (const r of localeResults) {
|
|
1087
|
+
sum.filesWritten += r.partial.filesWritten;
|
|
1088
|
+
sum.filesSkipped += r.partial.filesSkipped;
|
|
1089
|
+
sum.filesProcessed = (sum.filesProcessed ?? 0) + (r.partial.filesProcessed ?? 0);
|
|
1090
|
+
sum.inputTokens += r.partial.inputTokens;
|
|
1091
|
+
sum.outputTokens += r.partial.outputTokens;
|
|
1092
|
+
sum.costUsd = (sum.costUsd ?? 0) + (r.partial.costUsd ?? 0);
|
|
1093
|
+
sum.segmentsCached = (sum.segmentsCached ?? 0) + (r.partial.segmentsCached ?? 0);
|
|
1094
|
+
sum.segmentsTranslated = (sum.segmentsTranslated ?? 0) + (r.partial.segmentsTranslated ?? 0);
|
|
1095
|
+
localeTimes.push({
|
|
1096
|
+
locale: r.locale,
|
|
1097
|
+
elapsedMs: r.localeElapsed,
|
|
1098
|
+
});
|
|
1099
|
+
for (const k of r.markdownHitKeysLocal) {
|
|
1100
|
+
markdownHitKeys.add(k);
|
|
1101
|
+
}
|
|
1102
|
+
}
|
|
1103
|
+
if (cache) {
|
|
1104
|
+
const markdownScopeRel = shouldRunMarkdown(opts, config)
|
|
1105
|
+
? files.markdown.filter((r) => matchesPathFilter(r, opts.pathFilter))
|
|
1106
|
+
: null;
|
|
1107
|
+
const jsonScopeRel = shouldRunJson(opts, config)
|
|
1108
|
+
? files.json
|
|
1109
|
+
.filter((r) => matchesPathFilter(r, opts.pathFilter))
|
|
1110
|
+
.map((r) => path.relative(opts.cwd, path.join(jsonAbsRoot, r)).split(path.sep).join("/"))
|
|
1111
|
+
: null;
|
|
1112
|
+
if (markdownScopeRel && markdownScopeRel.length > 0 && markdownHitKeys.size > 0) {
|
|
1113
|
+
cache.resetLastHitAtForUnhitMarkdownInScope(markdownHitKeys, markdownScopeRel);
|
|
1114
|
+
}
|
|
1115
|
+
if (jsonScopeRel && jsonScopeRel.length > 0 && hitKeys.size > 0) {
|
|
1116
|
+
cache.resetLastHitAtForUnhitJsonInScope(hitKeys, jsonScopeRel);
|
|
1117
|
+
}
|
|
1118
|
+
}
|
|
1119
|
+
cache?.close();
|
|
1120
|
+
rewriteSourceMarkdownLanguageListBlocks(config, runOpts, files.markdown);
|
|
1121
|
+
const wallElapsed = Date.now() - wallStart;
|
|
1122
|
+
// Summary block (aligned with reference translate-docs)
|
|
1123
|
+
console.log(chalk.bold.green("\nā
Translation complete!\n"));
|
|
1124
|
+
console.log(chalk.bold("š Summary:"));
|
|
1125
|
+
console.log(` Total elapsed time: ${formatElapsedMmSs(wallElapsed)}`);
|
|
1126
|
+
console.log(` Total files processed: ${sum.filesProcessed ?? 0}`);
|
|
1127
|
+
console.log(` Total files skipped: ${sum.filesSkipped}`);
|
|
1128
|
+
console.log(` Segments from cache: ${sum.segmentsCached ?? 0}`);
|
|
1129
|
+
console.log(` Segments translated: ${sum.segmentsTranslated ?? 0}`);
|
|
1130
|
+
console.log(` Total tokens used: ${(sum.inputTokens + sum.outputTokens).toLocaleString()}`);
|
|
1131
|
+
if (opts.dryRun && (sum.filesWritten ?? 0) === 0 && (sum.filesProcessed ?? 0) > 0) {
|
|
1132
|
+
console.log(` Files written: 0 (dry-run)`);
|
|
1133
|
+
}
|
|
1134
|
+
else if ((sum.filesWritten ?? 0) > 0) {
|
|
1135
|
+
console.log(` Files written: ${sum.filesWritten}`);
|
|
1136
|
+
}
|
|
1137
|
+
const cost = sum.costUsd ?? 0;
|
|
1138
|
+
const segNew = sum.segmentsTranslated ?? 0;
|
|
1139
|
+
if (segNew > 0) {
|
|
1140
|
+
if (cost > 0) {
|
|
1141
|
+
console.log(` Total cost: $${cost.toFixed(6)}`);
|
|
1142
|
+
}
|
|
1143
|
+
else {
|
|
1144
|
+
console.log(` Total cost: $0.0000 (cost data not available from API)`);
|
|
1145
|
+
}
|
|
1146
|
+
}
|
|
1147
|
+
else {
|
|
1148
|
+
console.log(` Total cost: $0.0000 (all segments from cache)`);
|
|
1149
|
+
}
|
|
1150
|
+
console.log("");
|
|
1151
|
+
return sum;
|
|
1152
|
+
}
|
|
1153
|
+
//# sourceMappingURL=doc-translate.js.map
|