ai-i18n-tools 1.4.1 → 1.4.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/build-info.generated.d.ts +1 -1
- package/dist/build-info.generated.js +1 -1
- package/dist/cli/doc-translate.d.ts +10 -3
- package/dist/cli/doc-translate.d.ts.map +1 -1
- package/dist/cli/doc-translate.js +165 -72
- package/dist/cli/doc-translate.js.map +1 -1
- package/dist/cli/file-content-cache.d.ts +22 -0
- package/dist/cli/file-content-cache.d.ts.map +1 -0
- package/dist/cli/file-content-cache.js +45 -0
- package/dist/cli/file-content-cache.js.map +1 -0
- package/dist/cli/file-utils.d.ts +11 -1
- package/dist/cli/file-utils.d.ts.map +1 -1
- package/dist/cli/file-utils.js +117 -7
- package/dist/cli/file-utils.js.map +1 -1
- package/dist/cli/index.js +53 -1
- package/dist/cli/index.js.map +1 -1
- package/dist/core/cache.d.ts +11 -1
- package/dist/core/cache.d.ts.map +1 -1
- package/dist/core/cache.js +57 -0
- package/dist/core/cache.js.map +1 -1
- package/dist/core/config.js +2 -2
- package/dist/core/config.js.map +1 -1
- package/dist/core/svg-asset-paths.d.ts +5 -0
- package/dist/core/svg-asset-paths.d.ts.map +1 -1
- package/dist/core/svg-asset-paths.js +59 -4
- package/dist/core/svg-asset-paths.js.map +1 -1
- package/dist/core/types.d.ts +18 -14
- package/dist/core/types.d.ts.map +1 -1
- package/dist/core/types.js +25 -8
- package/dist/core/types.js.map +1 -1
- package/dist/core/ui-languages-catalog.d.ts +1 -0
- package/dist/core/ui-languages-catalog.d.ts.map +1 -1
- package/dist/core/ui-languages-catalog.js +12 -4
- package/dist/core/ui-languages-catalog.js.map +1 -1
- package/dist/core/ui-languages.d.ts +2 -0
- package/dist/core/ui-languages.d.ts.map +1 -1
- package/dist/core/ui-languages.js +6 -1
- package/dist/core/ui-languages.js.map +1 -1
- package/dist/extractors/svg-extractor.d.ts.map +1 -1
- package/dist/extractors/svg-extractor.js +22 -0
- package/dist/extractors/svg-extractor.js.map +1 -1
- package/docs/GETTING_STARTED.md +28 -13
- package/docs/ai-i18n-tools-context.md +36 -2
- package/package.json +1 -1
- package/translated-docs/docs/GETTING_STARTED.de.md +41 -25
- package/translated-docs/docs/GETTING_STARTED.es.md +40 -24
- package/translated-docs/docs/GETTING_STARTED.fr.md +35 -19
- package/translated-docs/docs/GETTING_STARTED.hi.md +44 -28
- package/translated-docs/docs/GETTING_STARTED.ja.md +44 -28
- package/translated-docs/docs/GETTING_STARTED.ko.md +43 -27
- package/translated-docs/docs/GETTING_STARTED.pt-BR.md +42 -26
- package/translated-docs/docs/GETTING_STARTED.zh-CN.md +39 -23
- package/translated-docs/docs/GETTING_STARTED.zh-TW.md +42 -26
|
@@ -7,6 +7,7 @@ import { Glossary } from "../glossary/glossary.js";
|
|
|
7
7
|
import { OpenRouterClient } from "../api/openrouter.js";
|
|
8
8
|
import type { DocumentBatchResponseFormat } from "../core/prompt-builder.js";
|
|
9
9
|
import { AsyncMutex } from "../utils/concurrency.js";
|
|
10
|
+
import { FileContentCache } from "./file-content-cache.js";
|
|
10
11
|
/** Same segment-extraction options as `translate-docs` for one documentation block. */
|
|
11
12
|
export declare function buildMarkdownExtractOpts(documentation: DocumentationBlock): MarkdownExtractOptions | undefined;
|
|
12
13
|
/** Batch segment prompt/response shape for `translate-docs --prompt-format`. */
|
|
@@ -60,6 +61,12 @@ export interface TranslateRunOptions {
|
|
|
60
61
|
* Default: off.
|
|
61
62
|
*/
|
|
62
63
|
debugFailed?: boolean;
|
|
64
|
+
/**
|
|
65
|
+
* Max concurrent files processed within a single locale.
|
|
66
|
+
* When > 1, files within the same locale are processed in parallel using AsyncSemaphore.
|
|
67
|
+
* Default: 1 (sequential processing).
|
|
68
|
+
*/
|
|
69
|
+
fileConcurrency?: number;
|
|
63
70
|
}
|
|
64
71
|
export interface TranslateTotals {
|
|
65
72
|
filesWritten: number;
|
|
@@ -132,15 +139,15 @@ export declare function augmentMarkdownFilesFromPathFilter(projectRoot: string,
|
|
|
132
139
|
* per-locale work so diagnostics are not repeated for every locale.
|
|
133
140
|
*/
|
|
134
141
|
export declare function scanAndRecordMarkdownSourceIssuesForTranslate(absSource: string, relPath: string, config: I18nDocTranslateConfig, cache: TranslationCache, opts: TranslateRunOptions): Promise<void>;
|
|
135
|
-
export declare function translateMarkdownFile(absSource: string, relPath: string, locale: string, config: I18nDocTranslateConfig, cache: TranslationCache | null, client: OpenRouterClient | null, glossary: Glossary, opts: TranslateRunOptions, hitKeys: Set<string>, translatedMarkdownRelPaths: ReadonlySet<string
|
|
142
|
+
export declare function translateMarkdownFile(absSource: string, relPath: string, locale: string, config: I18nDocTranslateConfig, cache: TranslationCache | null, client: OpenRouterClient | null, glossary: Glossary, opts: TranslateRunOptions, hitKeys: Set<string>, translatedMarkdownRelPaths: ReadonlySet<string>, fileContentCache?: FileContentCache): Promise<{
|
|
136
143
|
skipped: boolean;
|
|
137
144
|
totals: TranslateTotals;
|
|
138
145
|
}>;
|
|
139
|
-
export declare function translateJsonFile(absSource: string, relPath: string, locale: string, config: I18nDocTranslateConfig, cache: TranslationCache | null, client: OpenRouterClient | null, glossary: Glossary, opts: TranslateRunOptions, hitKeys: Set<string
|
|
146
|
+
export declare function translateJsonFile(absSource: string, relPath: string, locale: string, config: I18nDocTranslateConfig, cache: TranslationCache | null, client: OpenRouterClient | null, glossary: Glossary, opts: TranslateRunOptions, hitKeys: Set<string>, fileContentCache?: FileContentCache): Promise<{
|
|
140
147
|
skipped: boolean;
|
|
141
148
|
totals: TranslateTotals;
|
|
142
149
|
}>;
|
|
143
|
-
/** Translate one SVG asset (`translate-svg`); uses `config.svg` for paths and optional `svg.
|
|
150
|
+
/** Translate one SVG asset (`translate-svg`); uses `config.svg` for paths and optional `svg.forceLowercase`. */
|
|
144
151
|
export declare function translateSvgAssetFile(absSource: string, relPathFromCwd: string, relPathFromSourceRoot: string, locale: string, config: I18nConfig, cache: TranslationCache | null, client: OpenRouterClient | null, glossary: Glossary, opts: TranslateRunOptions, hitKeys: Set<string>): Promise<{
|
|
145
152
|
skipped: boolean;
|
|
146
153
|
totals: TranslateTotals;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"doc-translate.d.ts","sourceRoot":"","sources":["../../src/cli/doc-translate.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EACV,qBAAqB,EACrB,kBAAkB,EAClB,UAAU,EACV,sBAAsB,EAGvB,MAAM,kBAAkB,CAAC;AAI1B,YAAY,EAAE,qBAAqB,EAAE,CAAC;AAStC,OAAO,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAC;AAEpD,OAAO,KAAK,EAAE,sBAAsB,EAAE,MAAM,qCAAqC,CAAC;AAGlF,OAAO,EAAE,kBAAkB,EAAE,MAAM,sCAAsC,CAAC;AAM1E,OAAO,EAAE,QAAQ,EAAE,MAAM,yBAAyB,CAAC;AACnD,OAAO,EAAE,gBAAgB,EAAE,MAAM,sBAAsB,CAAC;AA+BxD,OAAO,KAAK,EACV,2BAA2B,EAE5B,MAAM,2BAA2B,CAAC;AACnC,OAAO,EAAyC,UAAU,EAAE,MAAM,yBAAyB,CAAC;
|
|
1
|
+
{"version":3,"file":"doc-translate.d.ts","sourceRoot":"","sources":["../../src/cli/doc-translate.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EACV,qBAAqB,EACrB,kBAAkB,EAClB,UAAU,EACV,sBAAsB,EAGvB,MAAM,kBAAkB,CAAC;AAI1B,YAAY,EAAE,qBAAqB,EAAE,CAAC;AAStC,OAAO,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAC;AAEpD,OAAO,KAAK,EAAE,sBAAsB,EAAE,MAAM,qCAAqC,CAAC;AAGlF,OAAO,EAAE,kBAAkB,EAAE,MAAM,sCAAsC,CAAC;AAM1E,OAAO,EAAE,QAAQ,EAAE,MAAM,yBAAyB,CAAC;AACnD,OAAO,EAAE,gBAAgB,EAAE,MAAM,sBAAsB,CAAC;AA+BxD,OAAO,KAAK,EACV,2BAA2B,EAE5B,MAAM,2BAA2B,CAAC;AACnC,OAAO,EAAyC,UAAU,EAAE,MAAM,yBAAyB,CAAC;AAC5F,OAAO,EAAE,gBAAgB,EAAE,MAAM,yBAAyB,CAAC;AAS3D,uFAAuF;AACvF,wBAAgB,wBAAwB,CACtC,aAAa,EAAE,kBAAkB,GAChC,sBAAsB,GAAG,SAAS,CAUpC;AAED,gFAAgF;AAChF,MAAM,MAAM,qBAAqB,GAAG,KAAK,GAAG,YAAY,GAAG,aAAa,CAAC;AAEzE,wBAAgB,qCAAqC,CACnD,GAAG,EAAE,qBAAqB,GAAG,SAAS,GACrC,2BAA2B,CAW7B;AAED,MAAM,WAAW,mBAAmB;IAClC,GAAG,EAAE,MAAM,CAAC;IACZ,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,MAAM,EAAE,OAAO,CAAC;IAChB,KAAK,EAAE,OAAO,CAAC;IACf;;OAEG;IACH,WAAW,EAAE,OAAO,CAAC;IACrB,OAAO,EAAE,OAAO,CAAC;IACjB,OAAO,EAAE,OAAO,CAAC;IACjB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,UAAU,CAAC,EAAE,UAAU,GAAG,MAAM,CAAC;IACjC,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,iEAAiE;IACjE,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB;;OAEG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB;;OAEG;IACH,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,qFAAqF;IACrF,UAAU,CAAC,EAAE,UAAU,CAAC;IACxB,8FAA8F;IAC9F,uBAAuB,CAAC,EAAE,MAAM,CAAC;IACjC;;;OAGG;IACH,YAAY,CAAC,EAAE,qBAAqB,CAAC;IACrC;;;OAGG;IACH,uBAAuB,CAAC,EAAE,OAAO,CAAC;IAClC;;OAEG;IACH,yBAAyB,CAAC,EAAE,OAAO,CAAC;IACpC;;;OAGG;IACH,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB;;;;OAIG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;CAC1B;AAED,MAAM,WAAW,eAAe;IAC9B,YAAY,EAAE,MAAM,CAAC;IACrB,YAAY,EAAE,MAAM,CAAC;IACrB,6EAA6E;IAC7E,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,WAAW,EAAE,MAAM,CAAC;IACpB,YAAY,EAAE,MAAM,CAAC;IACrB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,+FAA+F;IAC/F,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,sEAAsE;IACtE,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,+HAA+H;IAC/H,yBAAyB,CAAC,EAAE,MAAM,CAAC;IACnC,8HAA8H;IAC9H,6BAA6B,CAAC,EAAE,MAAM,CAAC;CACxC;AAED,KAAK,YAAY,GAAG,UAAU,CAAC,kBAAkB,CAAC,uBAAuB,CAAC,CAAC,GAAG;IAC5E,yBAAyB,CAAC,EAAE,MAAM,EAAE,CAAC;CACtC,CAAC;AAsBF,gIAAgI;AAChI,wBAAgB,4BAA4B,CAC1C,GAAG,EAAE,MAAM,EACX,QAAQ,EAAE,QAAQ,EAClB,MAAM,EAAE,MAAM,EACd,uBAAuB,EAAE,OAAO,EAChC,oBAAoB,UAAQ,GAC3B;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,YAAY,CAAA;CAAE,CA8BvC;AAgLD,wBAAgB,iBAAiB,CAC/B,IAAI,EAAE,mBAAmB,EACzB,MAAM,EAAE,sBAAsB,GAC7B,OAAO,CAWT;AAED,wBAAgB,aAAa,CAAC,IAAI,EAAE,mBAAmB,EAAE,MAAM,EAAE,sBAAsB,GAAG,OAAO,CAWhG;AAED,wBAAgB,iBAAiB,CAAC,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,SAAS,GAAG,OAAO,CAOtF;AAED;;;GAGG;AACH,wBAAgB,2BAA2B,CACzC,WAAW,EAAE,MAAM,EACnB,WAAW,EAAE,MAAM,EACnB,OAAO,EAAE,MAAM,GACd,MAAM,CAER;AAED;;;GAGG;AACH,wBAAgB,iCAAiC,CAC/C,WAAW,EAAE,MAAM,EACnB,GAAG,EAAE,MAAM,GAAG,SAAS,GACtB,MAAM,GAAG,SAAS,CAoBpB;AAID;;;GAGG;AACH,wBAAgB,iCAAiC,CAC/C,WAAW,EAAE,MAAM,EACnB,QAAQ,EAAE,MAAM,EAChB,KAAK,EAAE;IAAE,YAAY,EAAE,MAAM,EAAE,CAAA;CAAE,GAChC,OAAO,CAeT;AAED,wBAAgB,4CAA4C,CAC1D,WAAW,EAAE,MAAM,EACnB,QAAQ,EAAE,MAAM,EAChB,cAAc,EAAE,KAAK,CAAC;IAAE,YAAY,EAAE,MAAM,EAAE,CAAA;CAAE,CAAC,GAChD,OAAO,CAOT;AAED;;;GAGG;AACH,wBAAgB,+BAA+B,CAC7C,WAAW,EAAE,MAAM,EACnB,UAAU,EAAE,MAAM,GAAG,SAAS,GAC7B,MAAM,EAAE,CAoBV;AAED,MAAM,WAAW,mCAAmC;IAClD,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB,QAAQ,EAAE,MAAM,EAAE,CAAC;CACpB;AAED;;;;GAIG;AACH,wBAAgB,kCAAkC,CAChD,WAAW,EAAE,MAAM,EACnB,UAAU,EAAE,MAAM,GAAG,SAAS,EAC9B,uBAAuB,EAAE,MAAM,EAC/B,cAAc,EAAE,KAAK,CAAC;IAAE,YAAY,EAAE,MAAM,EAAE,CAAA;CAAE,CAAC,EACjD,kBAAkB,EAAE,MAAM,EAAE,GAC3B,mCAAmC,CAyDrC;AAm0BD;;;;GAIG;AACH,wBAAsB,6CAA6C,CACjE,SAAS,EAAE,MAAM,EACjB,OAAO,EAAE,MAAM,EACf,MAAM,EAAE,sBAAsB,EAC9B,KAAK,EAAE,gBAAgB,EACvB,IAAI,EAAE,mBAAmB,GACxB,OAAO,CAAC,IAAI,CAAC,CAyBf;AAED,wBAAsB,qBAAqB,CACzC,SAAS,EAAE,MAAM,EACjB,OAAO,EAAE,MAAM,EACf,MAAM,EAAE,MAAM,EACd,MAAM,EAAE,sBAAsB,EAC9B,KAAK,EAAE,gBAAgB,GAAG,IAAI,EAC9B,MAAM,EAAE,gBAAgB,GAAG,IAAI,EAC/B,QAAQ,EAAE,QAAQ,EAClB,IAAI,EAAE,mBAAmB,EACzB,OAAO,EAAE,GAAG,CAAC,MAAM,CAAC,EACpB,0BAA0B,EAAE,WAAW,CAAC,MAAM,CAAC,EAC/C,gBAAgB,CAAC,EAAE,gBAAgB,GAClC,OAAO,CAAC;IAAE,OAAO,EAAE,OAAO,CAAC;IAAC,MAAM,EAAE,eAAe,CAAA;CAAE,CAAC,CAqTxD;AAED,wBAAsB,iBAAiB,CACrC,SAAS,EAAE,MAAM,EACjB,OAAO,EAAE,MAAM,EACf,MAAM,EAAE,MAAM,EACd,MAAM,EAAE,sBAAsB,EAC9B,KAAK,EAAE,gBAAgB,GAAG,IAAI,EAC9B,MAAM,EAAE,gBAAgB,GAAG,IAAI,EAC/B,QAAQ,EAAE,QAAQ,EAClB,IAAI,EAAE,mBAAmB,EACzB,OAAO,EAAE,GAAG,CAAC,MAAM,CAAC,EACpB,gBAAgB,CAAC,EAAE,gBAAgB,GAClC,OAAO,CAAC;IAAE,OAAO,EAAE,OAAO,CAAC;IAAC,MAAM,EAAE,eAAe,CAAA;CAAE,CAAC,CAkNxD;AAED,gHAAgH;AAChH,wBAAsB,qBAAqB,CACzC,SAAS,EAAE,MAAM,EACjB,cAAc,EAAE,MAAM,EACtB,qBAAqB,EAAE,MAAM,EAC7B,MAAM,EAAE,MAAM,EACd,MAAM,EAAE,UAAU,EAClB,KAAK,EAAE,gBAAgB,GAAG,IAAI,EAC9B,MAAM,EAAE,gBAAgB,GAAG,IAAI,EAC/B,QAAQ,EAAE,QAAQ,EAClB,IAAI,EAAE,mBAAmB,EACzB,OAAO,EAAE,GAAG,CAAC,MAAM,CAAC,GACnB,OAAO,CAAC;IAAE,OAAO,EAAE,OAAO,CAAC;IAAC,MAAM,EAAE,eAAe,CAAA;CAAE,CAAC,CAmOxD;AAED,wBAAgB,uCAAuC,CACrD,MAAM,EAAE,sBAAsB,EAC9B,IAAI,EAAE,mBAAmB,EACzB,aAAa,EAAE,MAAM,EAAE,GACtB,MAAM,CAmCR;AAyDD,uDAAuD;AACvD,wBAAsB,YAAY,CAChC,MAAM,EAAE,sBAAsB,EAC9B,IAAI,EAAE,mBAAmB,EACzB,KAAK,EAAE;IACL,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB,IAAI,EAAE,MAAM,EAAE,CAAC;CAChB,EACD,WAAW,EAAE,MAAM,GAClB,OAAO,CAAC,eAAe,CAAC,CAiZ1B"}
|
|
@@ -27,6 +27,7 @@ import { ensureDirForFile, hashFileContent, resolveTranslatedOutputPath, writeAt
|
|
|
27
27
|
import { normalizeLocale } from "../core/config.js";
|
|
28
28
|
import { collectFilesByExtension } from "./file-utils.js";
|
|
29
29
|
import { runMapWithConcurrency, AsyncSemaphore, AsyncMutex } from "../utils/concurrency.js";
|
|
30
|
+
import { FileContentCache } from "./file-content-cache.js";
|
|
30
31
|
import { describeEmphasisPlaceholdersPolicy, resolveMarkdownEmphasisPlaceholders, usesAutomaticEmphasisPlaceholdersForLocale, } from "../core/markdown-emphasis-defaults.js";
|
|
31
32
|
import { collectMarkdownIssuesForSegment } from "../processors/markdown-source-diagnostics.js";
|
|
32
33
|
/** Same segment-extraction options as `translate-docs` for one documentation block. */
|
|
@@ -981,7 +982,7 @@ export async function scanAndRecordMarkdownSourceIssuesForTranslate(absSource, r
|
|
|
981
982
|
}
|
|
982
983
|
}
|
|
983
984
|
}
|
|
984
|
-
export async function translateMarkdownFile(absSource, relPath, locale, config, cache, client, glossary, opts, hitKeys, translatedMarkdownRelPaths) {
|
|
985
|
+
export async function translateMarkdownFile(absSource, relPath, locale, config, cache, client, glossary, opts, hitKeys, translatedMarkdownRelPaths, fileContentCache) {
|
|
985
986
|
const totals = {
|
|
986
987
|
filesWritten: 0,
|
|
987
988
|
filesSkipped: 0,
|
|
@@ -989,9 +990,17 @@ export async function translateMarkdownFile(absSource, relPath, locale, config,
|
|
|
989
990
|
outputTokens: 0,
|
|
990
991
|
costUsd: 0,
|
|
991
992
|
};
|
|
992
|
-
|
|
993
|
-
const
|
|
994
|
-
|
|
993
|
+
// Use file content cache if provided, otherwise fall back to direct reads
|
|
994
|
+
const fileData = fileContentCache
|
|
995
|
+
? fileContentCache.readFile(absSource)
|
|
996
|
+
: {
|
|
997
|
+
content: fs.readFileSync(absSource, "utf8"),
|
|
998
|
+
hash: hashFileContent(fs.readFileSync(absSource, "utf8")),
|
|
999
|
+
mtime: fs.statSync(absSource).mtime.toISOString(),
|
|
1000
|
+
};
|
|
1001
|
+
const content = fileData.content;
|
|
1002
|
+
const fileHash = fileData.hash;
|
|
1003
|
+
const sourceFileMtime = fileData.mtime;
|
|
995
1004
|
/** Cwd-relative posix path for `translations.filepath` metadata (aligned with JSON/SVG). */
|
|
996
1005
|
const translationFilepathMeta = relPath.split(path.sep).join("/");
|
|
997
1006
|
const outPath = resolveTranslatedOutputPath(config, opts.cwd, locale, relPath, "markdown");
|
|
@@ -1043,30 +1052,40 @@ export async function translateMarkdownFile(absSource, relPath, locale, config,
|
|
|
1043
1052
|
: undefined;
|
|
1044
1053
|
const emphasisOn = resolveMarkdownEmphasisPlaceholders(locale, config.documentation, config, opts);
|
|
1045
1054
|
let segmentsCached = 0;
|
|
1055
|
+
// Collect translatable segment hashes for batch cache lookup
|
|
1056
|
+
const translatableSegments = [];
|
|
1046
1057
|
for (let docIdx = 0; docIdx < segments.length; docIdx++) {
|
|
1047
1058
|
const s = segments[docIdx];
|
|
1048
|
-
if (
|
|
1049
|
-
|
|
1059
|
+
if (s.translatable) {
|
|
1060
|
+
translatableSegments.push({ s, docIdx });
|
|
1050
1061
|
}
|
|
1051
|
-
|
|
1052
|
-
|
|
1053
|
-
|
|
1054
|
-
|
|
1055
|
-
|
|
1056
|
-
|
|
1057
|
-
|
|
1058
|
-
|
|
1059
|
-
|
|
1060
|
-
|
|
1061
|
-
|
|
1062
|
-
|
|
1063
|
-
|
|
1064
|
-
|
|
1065
|
-
|
|
1066
|
-
|
|
1067
|
-
|
|
1062
|
+
}
|
|
1063
|
+
// Batch cache lookup: fetch all cached segments in one query
|
|
1064
|
+
let batchCacheHits;
|
|
1065
|
+
if (!opts.force && cache && !opts.noCache && translatableSegments.length > 0) {
|
|
1066
|
+
const hashes = translatableSegments.map(({ s }) => s.hash);
|
|
1067
|
+
batchCacheHits = await withCacheMutex(opts.cacheMutex, () => cache.getSegmentsBatch(hashes, locale));
|
|
1068
|
+
}
|
|
1069
|
+
// Process segments: use batch cache results and validate
|
|
1070
|
+
for (const { s, docIdx } of translatableSegments) {
|
|
1071
|
+
const cached = batchCacheHits?.get(s.hash);
|
|
1072
|
+
if (cached) {
|
|
1073
|
+
const quality = await validateDocTranslatePair(s, cached.text);
|
|
1074
|
+
if (quality.ok) {
|
|
1075
|
+
const modelUsed = cached.model?.trim();
|
|
1076
|
+
translations.set(s.hash, {
|
|
1077
|
+
text: cached.text,
|
|
1078
|
+
...(modelUsed ? { modelUsed } : {}),
|
|
1079
|
+
});
|
|
1080
|
+
hitKeys.add(`${s.hash}|${locale}`);
|
|
1081
|
+
segmentsCached++;
|
|
1082
|
+
continue;
|
|
1083
|
+
}
|
|
1084
|
+
else if (opts.verbose) {
|
|
1085
|
+
console.warn(chalk.yellow(` ⚠️ ${relPath} (${locale}): cache rejected for segment (hash ${s.hash}): ${quality.errors.join("; ")}`));
|
|
1068
1086
|
}
|
|
1069
1087
|
}
|
|
1088
|
+
// Cache miss or validation failed: prepare for translation
|
|
1070
1089
|
if (failureTracker && !clearedFailureHashes.has(s.hash)) {
|
|
1071
1090
|
await failureTracker.clearSegmentFailures(s.hash, locale);
|
|
1072
1091
|
clearedFailureHashes.add(s.hash);
|
|
@@ -1171,7 +1190,7 @@ export async function translateMarkdownFile(absSource, relPath, locale, config,
|
|
|
1171
1190
|
logTranslateFileComplete(relPath, outPath, segmentsCached, segmentsNew, Date.now() - fileStartTime, totals.costUsd ?? 0);
|
|
1172
1191
|
return { skipped: false, totals };
|
|
1173
1192
|
}
|
|
1174
|
-
export async function translateJsonFile(absSource, relPath, locale, config, cache, client, glossary, opts, hitKeys) {
|
|
1193
|
+
export async function translateJsonFile(absSource, relPath, locale, config, cache, client, glossary, opts, hitKeys, fileContentCache) {
|
|
1175
1194
|
const totals = {
|
|
1176
1195
|
filesWritten: 0,
|
|
1177
1196
|
filesSkipped: 0,
|
|
@@ -1179,8 +1198,16 @@ export async function translateJsonFile(absSource, relPath, locale, config, cach
|
|
|
1179
1198
|
outputTokens: 0,
|
|
1180
1199
|
costUsd: 0,
|
|
1181
1200
|
};
|
|
1182
|
-
|
|
1183
|
-
const
|
|
1201
|
+
// Use file content cache if provided, otherwise fall back to direct reads
|
|
1202
|
+
const fileData = fileContentCache
|
|
1203
|
+
? fileContentCache.readFile(absSource)
|
|
1204
|
+
: {
|
|
1205
|
+
content: fs.readFileSync(absSource, "utf8"),
|
|
1206
|
+
hash: hashFileContent(fs.readFileSync(absSource, "utf8")),
|
|
1207
|
+
mtime: fs.statSync(absSource).mtime.toISOString(),
|
|
1208
|
+
};
|
|
1209
|
+
const content = fileData.content;
|
|
1210
|
+
const fileHash = fileData.hash;
|
|
1184
1211
|
/** Cwd-relative path for cache/UI and `file_tracking` (must match `resolveDocTrackingKeyToAbs` under project root). */
|
|
1185
1212
|
const relPathFromCwd = path.relative(opts.cwd, absSource).split(path.sep).join("/");
|
|
1186
1213
|
const outPath = resolveTranslatedOutputPath(config, opts.cwd, locale, relPath, "json");
|
|
@@ -1301,7 +1328,7 @@ export async function translateJsonFile(absSource, relPath, locale, config, cach
|
|
|
1301
1328
|
logTranslateFileComplete(relPath, outPath, segmentsCached, segmentsNew, Date.now() - fileStartTime, totals.costUsd ?? 0);
|
|
1302
1329
|
return { skipped: false, totals };
|
|
1303
1330
|
}
|
|
1304
|
-
/** Translate one SVG asset (`translate-svg`); uses `config.svg` for paths and optional `svg.
|
|
1331
|
+
/** Translate one SVG asset (`translate-svg`); uses `config.svg` for paths and optional `svg.forceLowercase`. */
|
|
1305
1332
|
export async function translateSvgAssetFile(absSource, relPathFromCwd, relPathFromSourceRoot, locale, config, cache, client, glossary, opts, hitKeys) {
|
|
1306
1333
|
const totals = {
|
|
1307
1334
|
filesWritten: 0,
|
|
@@ -1313,7 +1340,7 @@ export async function translateSvgAssetFile(absSource, relPathFromCwd, relPathFr
|
|
|
1313
1340
|
const cacheKey = svgAssetCacheFilepath(relPathFromCwd);
|
|
1314
1341
|
const translationSvgFilepathMeta = svgTranslationFilepathMetadata(relPathFromCwd);
|
|
1315
1342
|
const outPath = resolveSvgAssetOutputPath(config, opts.cwd, locale, relPathFromCwd, relPathFromSourceRoot);
|
|
1316
|
-
const forceLc = config.svg?.
|
|
1343
|
+
const forceLc = config.svg?.forceLowercase ?? false;
|
|
1317
1344
|
const content = fs.readFileSync(absSource, "utf8");
|
|
1318
1345
|
const fileHash = hashFileContent(content);
|
|
1319
1346
|
if (opts.force && cache && !opts.noCache) {
|
|
@@ -1588,7 +1615,9 @@ export async function runTranslate(config, opts, files, jsonAbsRoot) {
|
|
|
1588
1615
|
console.log("");
|
|
1589
1616
|
const localeConcurrency = Math.max(1, Math.floor(opts.concurrency ?? config.concurrency ?? 3));
|
|
1590
1617
|
const batchConcurrencyEffective = Math.max(1, Math.floor(opts.batchConcurrency ?? config.batchConcurrency ?? 4));
|
|
1618
|
+
const fileConcurrencyEffective = Math.max(1, Math.floor(opts.fileConcurrency ?? config.fileConcurrency ?? 1));
|
|
1591
1619
|
console.log(chalk.cyan(`Locale concurrency: `) + chalk.magenta(`${localeConcurrency}`));
|
|
1620
|
+
console.log(chalk.cyan(`File concurrency per locale: `) + chalk.magenta(`${fileConcurrencyEffective}`));
|
|
1592
1621
|
console.log(chalk.cyan(`Parallel API calls per file: `) + chalk.magenta(`${batchConcurrencyEffective}`));
|
|
1593
1622
|
console.log(chalk.cyan(`Batch prompt format: `) + chalk.magenta(`${opts.promptFormat ?? "json-array"}`));
|
|
1594
1623
|
console.log(chalk.cyan(`Markdown emphasis placeholders: `) +
|
|
@@ -1597,9 +1626,11 @@ export async function runTranslate(config, opts, files, jsonAbsRoot) {
|
|
|
1597
1626
|
const wallStart = Date.now();
|
|
1598
1627
|
const localeTimes = [];
|
|
1599
1628
|
const cacheMutex = cache && locales.length > 1 ? new AsyncMutex() : undefined;
|
|
1629
|
+
const fileContentCache = new FileContentCache();
|
|
1600
1630
|
const runOpts = {
|
|
1601
1631
|
...opts,
|
|
1602
1632
|
batchConcurrency: batchConcurrencyEffective,
|
|
1633
|
+
fileConcurrency: fileConcurrencyEffective,
|
|
1603
1634
|
cacheMutex,
|
|
1604
1635
|
};
|
|
1605
1636
|
if (shouldRunMarkdown(opts, config) && cache && !opts.noCache && !opts.dryRun) {
|
|
@@ -1632,62 +1663,124 @@ export async function runTranslate(config, opts, files, jsonAbsRoot) {
|
|
|
1632
1663
|
if (usesAutomaticEmphasisPlaceholdersForLocale(locale, config.documentation, config, runOpts)) {
|
|
1633
1664
|
console.log(chalk.gray(` ${locale}: Markdown emphasis placeholders is "on". Override with documentations[].emphasisPlaceholders or CLI flags)`));
|
|
1634
1665
|
}
|
|
1635
|
-
|
|
1636
|
-
|
|
1637
|
-
continue;
|
|
1638
|
-
}
|
|
1666
|
+
const markdownFilesToProcess = files.markdown.filter((rel) => matchesPathFilter(rel, opts.pathFilter));
|
|
1667
|
+
const processMarkdownFile = async (rel) => {
|
|
1639
1668
|
const abs = path.join(opts.cwd, rel);
|
|
1640
|
-
const { skipped, totals } = await translateMarkdownFile(abs, rel, locale, config, cache, client, glossary, runOpts, markdownHitKeysLocal, translatedMarkdownRelPaths);
|
|
1641
|
-
|
|
1642
|
-
|
|
1643
|
-
|
|
1669
|
+
const { skipped, totals } = await translateMarkdownFile(abs, rel, locale, config, cache, client, glossary, runOpts, markdownHitKeysLocal, translatedMarkdownRelPaths, fileContentCache);
|
|
1670
|
+
return { skipped, totals };
|
|
1671
|
+
};
|
|
1672
|
+
if (fileConcurrencyEffective > 1) {
|
|
1673
|
+
const results = await runMapWithConcurrency(markdownFilesToProcess, fileConcurrencyEffective, processMarkdownFile);
|
|
1674
|
+
for (const { skipped, totals } of results) {
|
|
1675
|
+
if (skipped) {
|
|
1676
|
+
partial.filesSkipped += totals.filesSkipped;
|
|
1677
|
+
partial.filesProcessed = (partial.filesProcessed ?? 0) + (totals.filesProcessed ?? 0);
|
|
1678
|
+
}
|
|
1679
|
+
else {
|
|
1680
|
+
partial.filesWritten += totals.filesWritten;
|
|
1681
|
+
partial.filesProcessed = (partial.filesProcessed ?? 0) + (totals.filesProcessed ?? 0);
|
|
1682
|
+
partial.inputTokens += totals.inputTokens;
|
|
1683
|
+
partial.outputTokens += totals.outputTokens;
|
|
1684
|
+
partial.costUsd = (partial.costUsd ?? 0) + (totals.costUsd ?? 0);
|
|
1685
|
+
partial.segmentsCached = (partial.segmentsCached ?? 0) + (totals.segmentsCached ?? 0);
|
|
1686
|
+
partial.segmentsTranslated =
|
|
1687
|
+
(partial.segmentsTranslated ?? 0) + (totals.segmentsTranslated ?? 0);
|
|
1688
|
+
partial.segmentValidationFailures =
|
|
1689
|
+
(partial.segmentValidationFailures ?? 0) + (totals.segmentValidationFailures ?? 0);
|
|
1690
|
+
partial.individualSegmentTranslations =
|
|
1691
|
+
(partial.individualSegmentTranslations ?? 0) +
|
|
1692
|
+
(totals.individualSegmentTranslations ?? 0);
|
|
1693
|
+
}
|
|
1644
1694
|
}
|
|
1645
|
-
|
|
1646
|
-
|
|
1647
|
-
|
|
1648
|
-
|
|
1649
|
-
|
|
1650
|
-
|
|
1651
|
-
|
|
1652
|
-
|
|
1653
|
-
|
|
1654
|
-
|
|
1655
|
-
(partial.
|
|
1656
|
-
|
|
1657
|
-
|
|
1658
|
-
|
|
1695
|
+
}
|
|
1696
|
+
else {
|
|
1697
|
+
for (const rel of markdownFilesToProcess) {
|
|
1698
|
+
const { skipped, totals } = await processMarkdownFile(rel);
|
|
1699
|
+
if (skipped) {
|
|
1700
|
+
partial.filesSkipped += totals.filesSkipped;
|
|
1701
|
+
partial.filesProcessed = (partial.filesProcessed ?? 0) + (totals.filesProcessed ?? 0);
|
|
1702
|
+
}
|
|
1703
|
+
else {
|
|
1704
|
+
partial.filesWritten += totals.filesWritten;
|
|
1705
|
+
partial.filesProcessed = (partial.filesProcessed ?? 0) + (totals.filesProcessed ?? 0);
|
|
1706
|
+
partial.inputTokens += totals.inputTokens;
|
|
1707
|
+
partial.outputTokens += totals.outputTokens;
|
|
1708
|
+
partial.costUsd = (partial.costUsd ?? 0) + (totals.costUsd ?? 0);
|
|
1709
|
+
partial.segmentsCached = (partial.segmentsCached ?? 0) + (totals.segmentsCached ?? 0);
|
|
1710
|
+
partial.segmentsTranslated =
|
|
1711
|
+
(partial.segmentsTranslated ?? 0) + (totals.segmentsTranslated ?? 0);
|
|
1712
|
+
partial.segmentValidationFailures =
|
|
1713
|
+
(partial.segmentValidationFailures ?? 0) + (totals.segmentValidationFailures ?? 0);
|
|
1714
|
+
partial.individualSegmentTranslations =
|
|
1715
|
+
(partial.individualSegmentTranslations ?? 0) +
|
|
1716
|
+
(totals.individualSegmentTranslations ?? 0);
|
|
1717
|
+
}
|
|
1659
1718
|
}
|
|
1660
1719
|
}
|
|
1661
1720
|
}
|
|
1662
1721
|
if (shouldRunJson(opts, config)) {
|
|
1663
|
-
|
|
1722
|
+
const jsonFilesToProcess = files.json.filter((rel) => {
|
|
1664
1723
|
const jsonRelFromProjectRoot = jsonFileProjectRelativePath(opts.cwd, jsonAbsRoot, rel);
|
|
1665
|
-
|
|
1666
|
-
|
|
1667
|
-
|
|
1724
|
+
return matchesPathFilter(jsonRelFromProjectRoot, opts.pathFilter);
|
|
1725
|
+
});
|
|
1726
|
+
const processJsonFile = async (rel) => {
|
|
1668
1727
|
const abs = path.join(jsonAbsRoot, rel);
|
|
1669
|
-
const { skipped, totals } = await translateJsonFile(abs, rel, locale, config, cache, client, glossary, runOpts, hitKeys);
|
|
1670
|
-
|
|
1671
|
-
|
|
1672
|
-
|
|
1728
|
+
const { skipped, totals } = await translateJsonFile(abs, rel, locale, config, cache, client, glossary, runOpts, hitKeys, fileContentCache);
|
|
1729
|
+
return { skipped, totals };
|
|
1730
|
+
};
|
|
1731
|
+
if (fileConcurrencyEffective > 1) {
|
|
1732
|
+
const results = await runMapWithConcurrency(jsonFilesToProcess, fileConcurrencyEffective, processJsonFile);
|
|
1733
|
+
for (const { skipped, totals } of results) {
|
|
1734
|
+
if (skipped) {
|
|
1735
|
+
partial.filesSkipped += totals.filesSkipped;
|
|
1736
|
+
partial.filesProcessed = (partial.filesProcessed ?? 0) + (totals.filesProcessed ?? 0);
|
|
1737
|
+
}
|
|
1738
|
+
else {
|
|
1739
|
+
partial.filesWritten += totals.filesWritten;
|
|
1740
|
+
partial.filesProcessed = (partial.filesProcessed ?? 0) + (totals.filesProcessed ?? 0);
|
|
1741
|
+
partial.inputTokens += totals.inputTokens;
|
|
1742
|
+
partial.outputTokens += totals.outputTokens;
|
|
1743
|
+
partial.costUsd = (partial.costUsd ?? 0) + (totals.costUsd ?? 0);
|
|
1744
|
+
partial.segmentsCached = (partial.segmentsCached ?? 0) + (totals.segmentsCached ?? 0);
|
|
1745
|
+
partial.segmentsTranslated =
|
|
1746
|
+
(partial.segmentsTranslated ?? 0) + (totals.segmentsTranslated ?? 0);
|
|
1747
|
+
partial.segmentValidationFailures =
|
|
1748
|
+
(partial.segmentValidationFailures ?? 0) + (totals.segmentValidationFailures ?? 0);
|
|
1749
|
+
partial.individualSegmentTranslations =
|
|
1750
|
+
(partial.individualSegmentTranslations ?? 0) +
|
|
1751
|
+
(totals.individualSegmentTranslations ?? 0);
|
|
1752
|
+
}
|
|
1673
1753
|
}
|
|
1674
|
-
|
|
1675
|
-
|
|
1676
|
-
|
|
1677
|
-
|
|
1678
|
-
|
|
1679
|
-
|
|
1680
|
-
|
|
1681
|
-
|
|
1682
|
-
|
|
1683
|
-
|
|
1684
|
-
(partial.
|
|
1685
|
-
|
|
1686
|
-
|
|
1687
|
-
|
|
1754
|
+
}
|
|
1755
|
+
else {
|
|
1756
|
+
for (const rel of jsonFilesToProcess) {
|
|
1757
|
+
const { skipped, totals } = await processJsonFile(rel);
|
|
1758
|
+
if (skipped) {
|
|
1759
|
+
partial.filesSkipped += totals.filesSkipped;
|
|
1760
|
+
partial.filesProcessed = (partial.filesProcessed ?? 0) + (totals.filesProcessed ?? 0);
|
|
1761
|
+
}
|
|
1762
|
+
else {
|
|
1763
|
+
partial.filesWritten += totals.filesWritten;
|
|
1764
|
+
partial.filesProcessed = (partial.filesProcessed ?? 0) + (totals.filesProcessed ?? 0);
|
|
1765
|
+
partial.inputTokens += totals.inputTokens;
|
|
1766
|
+
partial.outputTokens += totals.outputTokens;
|
|
1767
|
+
partial.costUsd = (partial.costUsd ?? 0) + (totals.costUsd ?? 0);
|
|
1768
|
+
partial.segmentsCached = (partial.segmentsCached ?? 0) + (totals.segmentsCached ?? 0);
|
|
1769
|
+
partial.segmentsTranslated =
|
|
1770
|
+
(partial.segmentsTranslated ?? 0) + (totals.segmentsTranslated ?? 0);
|
|
1771
|
+
partial.segmentValidationFailures =
|
|
1772
|
+
(partial.segmentValidationFailures ?? 0) + (totals.segmentValidationFailures ?? 0);
|
|
1773
|
+
partial.individualSegmentTranslations =
|
|
1774
|
+
(partial.individualSegmentTranslations ?? 0) +
|
|
1775
|
+
(totals.individualSegmentTranslations ?? 0);
|
|
1776
|
+
}
|
|
1688
1777
|
}
|
|
1689
1778
|
}
|
|
1690
1779
|
}
|
|
1780
|
+
// Batch update last_hit_at for all cached segments in this locale
|
|
1781
|
+
if (cache && !opts.noCache && markdownHitKeysLocal.size > 0) {
|
|
1782
|
+
await withCacheMutex(opts.cacheMutex, () => cache.batchUpdateLastHitAt(Array.from(markdownHitKeysLocal)));
|
|
1783
|
+
}
|
|
1691
1784
|
}
|
|
1692
1785
|
catch (e) {
|
|
1693
1786
|
error = e;
|