@alauda/doom 2.5.4 → 2.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/cli/api.d.ts +2 -0
- package/lib/cli/api.js +147 -0
- package/lib/cli/helpers.d.ts +1 -7
- package/lib/cli/helpers.js +4 -41
- package/lib/cli/index.js +2 -0
- package/lib/cli/translate-agent.d.ts +81 -0
- package/lib/cli/translate-agent.js +450 -0
- package/lib/cli/translate-check.d.ts +17 -0
- package/lib/cli/translate-check.js +67 -0
- package/lib/cli/translate-checker.d.ts +58 -0
- package/lib/cli/translate-checker.js +112 -0
- package/lib/cli/translate-judge.d.ts +119 -0
- package/lib/cli/translate-judge.js +224 -0
- package/lib/cli/translate-limits.d.ts +43 -0
- package/lib/cli/translate-limits.js +55 -0
- package/lib/cli/translate-mask.d.ts +144 -0
- package/lib/cli/translate-mask.js +617 -0
- package/lib/cli/translate-models.d.ts +127 -0
- package/lib/cli/translate-models.js +189 -0
- package/lib/cli/translate-scratch.d.ts +142 -0
- package/lib/cli/translate-scratch.js +187 -0
- package/lib/cli/translate-terms.d.ts +22 -0
- package/lib/cli/translate-terms.js +33 -0
- package/lib/cli/translate.d.ts +15 -10
- package/lib/cli/translate.js +228 -185
- package/lib/eslint.js +22 -3
- package/lib/plugins/api/index.js +1 -1
- package/lib/plugins/api/types.d.ts +36 -1
- package/lib/plugins/global/components.d.ts +16 -0
- package/lib/plugins/global/components.js +16 -1
- package/lib/remark-lint/index.d.ts +20 -0
- package/lib/remark-lint/index.js +40 -0
- package/lib/remark-lint/no-unmatched-anchor.d.ts +10 -0
- package/lib/remark-lint/no-unmatched-anchor.js +13 -0
- package/lib/remark-lint/no-unopened-frontmatter.d.ts +2 -0
- package/lib/remark-lint/no-unopened-frontmatter.js +70 -0
- package/lib/remark-lint/no-unparsed-emphasis.d.ts +2 -0
- package/lib/remark-lint/no-unparsed-emphasis.js +53 -0
- package/lib/remark-lint/no-unresolved-api-ref.d.ts +40 -0
- package/lib/remark-lint/no-unresolved-api-ref.js +194 -0
- package/lib/remark-lint/rule-sets.d.ts +75 -0
- package/lib/remark-lint/rule-sets.js +120 -0
- package/lib/remark-lint/translation-parity/component-multiset.d.ts +17 -0
- package/lib/remark-lint/translation-parity/component-multiset.js +35 -0
- package/lib/remark-lint/translation-parity/echoed-source.d.ts +2 -0
- package/lib/remark-lint/translation-parity/echoed-source.js +37 -0
- package/lib/remark-lint/translation-parity/frontmatter-preservation.d.ts +2 -0
- package/lib/remark-lint/translation-parity/frontmatter-preservation.js +36 -0
- package/lib/remark-lint/translation-parity/heading-sequence.d.ts +12 -0
- package/lib/remark-lint/translation-parity/heading-sequence.js +25 -0
- package/lib/remark-lint/translation-parity/index.d.ts +24 -0
- package/lib/remark-lint/translation-parity/index.js +24 -0
- package/lib/remark-lint/translation-parity/jsx-attribute-parity.d.ts +14 -0
- package/lib/remark-lint/translation-parity/jsx-attribute-parity.js +47 -0
- package/lib/remark-lint/translation-parity/length-ratio.d.ts +2 -0
- package/lib/remark-lint/translation-parity/length-ratio.js +49 -0
- package/lib/remark-lint/translation-parity/link-isomorphism.d.ts +26 -0
- package/lib/remark-lint/translation-parity/link-isomorphism.js +45 -0
- package/lib/remark-lint/translation-parity/shared.d.ts +110 -0
- package/lib/remark-lint/translation-parity/shared.js +282 -0
- package/lib/remark-lint/translation-parity/terminology-adherence.d.ts +2 -0
- package/lib/remark-lint/translation-parity/terminology-adherence.js +67 -0
- package/lib/remark-lint/translation-parity/up-to-date.d.ts +11 -0
- package/lib/remark-lint/translation-parity/up-to-date.js +21 -0
- package/lib/remark-lint/translation-parity/url-residue.d.ts +2 -0
- package/lib/remark-lint/translation-parity/url-residue.js +46 -0
- package/lib/remarkrc.d.ts +1 -1
- package/lib/remarkrc.js +7 -34
- package/lib/runtime/api-paths.d.ts +20 -0
- package/lib/runtime/api-paths.js +29 -0
- package/lib/runtime/components/K8sAPI.d.ts +14 -1
- package/lib/runtime/components/K8sAPI.js +49 -10
- package/lib/runtime/components/K8sCrd.d.ts +1 -1
- package/lib/runtime/components/K8sPermissionTable.js +10 -5
- package/lib/runtime/components/OpenAPIPath.js +19 -2
- package/lib/runtime/components/_HeadingTitle.js +24 -6
- package/lib/runtime/components/_K8sAPIEndpoints.d.ts +8 -1
- package/lib/runtime/components/_K8sAPIEndpoints.js +27 -9
- package/lib/runtime/components/_K8sAPISchema.d.ts +9 -1
- package/lib/runtime/components/_K8sAPISchema.js +24 -6
- package/lib/runtime/components/_translation-policy.d.ts +23 -0
- package/lib/runtime/components/_translation-policy.js +32 -0
- package/lib/runtime/hooks/useTranslation.d.ts +1 -1
- package/lib/runtime/translation.d.ts +68 -0
- package/lib/runtime/translation.js +51 -0
- package/lib/runtime/utils.d.ts +24 -2
- package/lib/runtime/utils.js +87 -5
- package/lib/shared/constants.d.ts +9 -0
- package/lib/shared/constants.js +14 -0
- package/lib/types.d.ts +70 -0
- package/lib/utils/helpers.js +8 -2
- package/package.json +3 -1
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
import { logger } from '@rspress/core';
|
|
2
|
+
import { Language } from "../shared/index.js";
|
|
3
|
+
import { parseTerms } from "./helpers.js";
|
|
4
|
+
export const TERMS_SUPPORTED_LANGUAGES = ['en', 'zh', 'ru'];
|
|
5
|
+
export const resolveTerms = async (sourceLang, targetLang, sourceContent) => {
|
|
6
|
+
if (![sourceLang, targetLang].every((lang) => TERMS_SUPPORTED_LANGUAGES.includes(lang))) {
|
|
7
|
+
return { text: '', pairs: [] };
|
|
8
|
+
}
|
|
9
|
+
const parsedTerms = await parseTerms();
|
|
10
|
+
// Only the terms this document actually uses: the table is long, and a term
|
|
11
|
+
// the source never mentions is noise in both prompts.
|
|
12
|
+
const relevantTerms = parsedTerms.filter((term) => {
|
|
13
|
+
const sourceTranslation = term[sourceLang];
|
|
14
|
+
const targetTranslation = term[targetLang];
|
|
15
|
+
if (!sourceTranslation || !targetTranslation) {
|
|
16
|
+
return false;
|
|
17
|
+
}
|
|
18
|
+
const sourceTermRegex = new RegExp(`\\b${sourceTranslation.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}\\b`, 'i');
|
|
19
|
+
return sourceTermRegex.test(sourceContent);
|
|
20
|
+
});
|
|
21
|
+
if (relevantTerms.length === 0) {
|
|
22
|
+
logger.debug('No relevant terms found for translation');
|
|
23
|
+
return { text: '', pairs: [] };
|
|
24
|
+
}
|
|
25
|
+
const pairs = relevantTerms.map((term) => ({
|
|
26
|
+
source: term[sourceLang],
|
|
27
|
+
target: term[targetLang],
|
|
28
|
+
}));
|
|
29
|
+
const text = `- The following is a common related terminology vocabulary table (${Language[sourceLang]} <=> ${Language[targetLang]}), you should use it to translate the matched text.\n` +
|
|
30
|
+
pairs.map((pair) => ` * ${pair.source} <=> ${pair.target}`).join('\n');
|
|
31
|
+
logger.debug('Resolved terms:', text);
|
|
32
|
+
return { text, pairs };
|
|
33
|
+
};
|
package/lib/cli/translate.d.ts
CHANGED
|
@@ -1,6 +1,19 @@
|
|
|
1
1
|
import { Command } from 'commander';
|
|
2
2
|
import { Language } from '../shared/index.js';
|
|
3
|
-
|
|
3
|
+
/**
|
|
4
|
+
* `doom translate` — orchestration and the gate.
|
|
5
|
+
*
|
|
6
|
+
* Everything that decides *what* gets translated lives here: which files the
|
|
7
|
+
* globs match, which are already current by `sourceSHA`, which are copied
|
|
8
|
+
* rather than translated, how a translation's frontmatter is merged, and the
|
|
9
|
+
* deterministic rewrites (`normalizeImgSrc`, `translateCodeFile`) that are ours
|
|
10
|
+
* to make rather than a model's.
|
|
11
|
+
*
|
|
12
|
+
* Translating a document does not live here at all. That is a checked loop, in
|
|
13
|
+
* `translate-agent.ts`. There is one path through it: no flag turns it off and
|
|
14
|
+
* no single-shot fallback remains, because a protection that can be skipped is
|
|
15
|
+
* one that gets skipped.
|
|
16
|
+
*/
|
|
4
17
|
export interface I18nFrontmatter {
|
|
5
18
|
i18n?: {
|
|
6
19
|
additionalPrompts?: string;
|
|
@@ -10,18 +23,10 @@ export interface I18nFrontmatter {
|
|
|
10
23
|
title?: string;
|
|
11
24
|
description?: string;
|
|
12
25
|
}
|
|
13
|
-
export declare const TERMS_SUPPORTED_LANGUAGES: Language[];
|
|
14
|
-
export interface InternalTranslateOptions extends TranslateOptions {
|
|
15
|
-
source: Language;
|
|
16
|
-
sourceContent: string;
|
|
17
|
-
target: Language;
|
|
18
|
-
additionalPrompts?: string;
|
|
19
|
-
}
|
|
20
|
-
export declare const translate: ({ source, sourceContent, target, systemPrompt, userPrompt, additionalPrompts, }: InternalTranslateOptions) => Promise<string>;
|
|
21
26
|
export interface TranslateCommandOptions {
|
|
22
27
|
source: Language;
|
|
23
28
|
target: Language;
|
|
24
|
-
glob
|
|
29
|
+
glob?: string[];
|
|
25
30
|
copy?: boolean;
|
|
26
31
|
}
|
|
27
32
|
export declare const translateCommand: Command;
|
package/lib/cli/translate.js
CHANGED
|
@@ -1,44 +1,41 @@
|
|
|
1
1
|
import crypto from 'node:crypto';
|
|
2
2
|
import fs from 'node:fs/promises';
|
|
3
3
|
import path from 'node:path';
|
|
4
|
-
import { setTimeout } from 'node:timers/promises';
|
|
5
4
|
import { isDeepStrictEqual } from 'node:util';
|
|
6
5
|
import { logger } from '@rspress/core';
|
|
7
6
|
import { removeLeadingSlash } from '@rspress/shared';
|
|
8
7
|
import matter from '@rspress/shared/gray-matter';
|
|
9
8
|
import { Command } from 'commander';
|
|
10
9
|
import ejs from 'ejs';
|
|
11
|
-
import { OpenAI, RateLimitError } from 'openai';
|
|
12
|
-
import { pRateLimit } from 'p-ratelimit';
|
|
13
10
|
import { glob } from 'tinyglobby';
|
|
14
11
|
import { cyan, red } from 'yoctocolors';
|
|
15
12
|
import { mdProcessor, mdxProcessor, normalizeImgSrc, } from '../plugins/index.js';
|
|
16
|
-
import { Language, SUPPORTED_LANGUAGES, TITLE_TRANSLATION_MAP, } from '../shared/index.js';
|
|
17
|
-
import { pathExists } from '../utils/index.js';
|
|
18
|
-
import { escapeMarkdownHeadingIds, getMatchedDocFilePaths, parseBoolean,
|
|
13
|
+
import { DEFAULT_COPY_ONLY_DIRECTORIES, Language, SUPPORTED_LANGUAGES, TITLE_TRANSLATION_MAP, } from '../shared/index.js';
|
|
14
|
+
import { OPTIONS_FILE, STORAGE_DIR, pathExists } from '../utils/index.js';
|
|
15
|
+
import { escapeMarkdownHeadingIds, getMatchedDocFilePaths, parseBoolean, stringifyMatter, translateCodeFile, } from './helpers.js';
|
|
19
16
|
import { loadConfig } from './load-config.js';
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
17
|
+
import { translateWithAgent, } from './translate-agent.js';
|
|
18
|
+
import { checkTranslations, } from './translate-check.js';
|
|
19
|
+
import { createTranslationChecker, } from './translate-checker.js';
|
|
20
|
+
import { createJudge } from './translate-judge.js';
|
|
21
|
+
import { CONCURRENCY_ENV, DEFAULT_CONCURRENCY, DEFAULT_REQUESTS_PER_MINUTE, REQUESTS_PER_MINUTE_ENV, createLimits, positiveIntFromEnv, } from './translate-limits.js';
|
|
22
|
+
import { maskAst } from './translate-mask.js';
|
|
23
|
+
import { DEFAULT_JUDGE_REASONING_EFFORT, DEFAULT_REASONING_EFFORT, createGateway, } from './translate-models.js';
|
|
24
|
+
import { resolveTerms } from './translate-terms.js';
|
|
25
|
+
/**
|
|
26
|
+
* The translation rules.
|
|
27
|
+
*
|
|
28
|
+
* Overridable per repository through `translate.systemPrompt`, which is why
|
|
29
|
+
* the placeholder discipline is *not* here: it is part of the harness's own
|
|
30
|
+
* prompt in `translate-agent.ts`, where a repository cannot drop it by
|
|
31
|
+
* replacing this text.
|
|
32
|
+
*/
|
|
28
33
|
const DEFAULT_SYSTEM_PROMPT = `
|
|
29
|
-
You are a professional technical documentation engineer, skilled in writing high-quality technical documentation in <%= targetLang %>. Please accurately translate
|
|
34
|
+
You are a professional technical documentation engineer, skilled in writing high-quality technical documentation in <%= targetLang %>. Please accurately translate from <%= sourceLang %> to <%= targetLang %>, maintaining the style consistent with technical documentation in <%= sourceLang %>.
|
|
30
35
|
|
|
31
36
|
## Baseline Requirements
|
|
32
37
|
- Sentences should be fluent and conform to the expression habits of the <%= targetLang %> language.
|
|
33
38
|
- Input format is MDX; output format must also retain the original MDX format. Do not translate the names of jsx components such as <Overview />, and do not wrap output in unnecessary code blocks.
|
|
34
|
-
- **CRITICAL**: Do not translate or modify ANY link content in the document. This includes:
|
|
35
|
-
- URLs in markdown links: [text](URL) - keep URL exactly as is but translate the text
|
|
36
|
-
- Reference-style links: [text][ref] and [ref]: URL - keep both ref and URL unchanged
|
|
37
|
-
- Inline URLs: https://example.com - keep completely unchanged
|
|
38
|
-
- Image links:  - keep src unchanged, but alt text should be translated
|
|
39
|
-
- Anchor links: [text](#anchor) - keep #anchor unchanged
|
|
40
|
-
- Custom anchors in headings: # Heading \\{#custom-anchor} - keep \\{#custom-anchor} unchanged and translate "Heading"
|
|
41
|
-
- Any href attributes in HTML tags - keep unchanged
|
|
42
39
|
- Do not translate professional technical terms and proper nouns, including but not limited to: Kubernetes, Docker, CLI, API, REST, GraphQL, JSON, YAML, Git, GitHub, GitLab, AWS, Azure, GCP, Linux, Windows, macOS, Node.js, React, Vue, Angular, TypeScript, JavaScript, Python, Java, Go, Rust, etc. Keep these terms in their original form.
|
|
43
40
|
- The title field and description field in frontmatter should be translated, other frontmatter fields should retain and do not translate.
|
|
44
41
|
- Content within MDX components needs to be translated, whereas MDX component names and parameter keys do not.
|
|
@@ -67,7 +64,6 @@ You are a professional technical documentation engineer, skilled in writing high
|
|
|
67
64
|
## Additional Requirements
|
|
68
65
|
These are additional requirements for the translation. They should be met along with the baseline requirements, and in case of any conflict, the baseline requirements should take precedence.
|
|
69
66
|
|
|
70
|
-
The text for translation is provided below, within triple quotes:
|
|
71
67
|
"""
|
|
72
68
|
<% if (userPrompt) { %>
|
|
73
69
|
<%- userPrompt %>
|
|
@@ -79,35 +75,17 @@ The text for translation is provided below, within triple quotes:
|
|
|
79
75
|
"""
|
|
80
76
|
<% } %>
|
|
81
77
|
`.trim();
|
|
82
|
-
|
|
83
|
-
const
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
}
|
|
94
|
-
// Check if the source translation appears in the source content (case-insensitive)
|
|
95
|
-
const sourceTermRegex = new RegExp(`\\b${sourceTranslation.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}\\b`, 'i');
|
|
96
|
-
return sourceTermRegex.test(sourceContent);
|
|
97
|
-
});
|
|
98
|
-
if (relevantTerms.length === 0) {
|
|
99
|
-
logger.debug('No relevant terms found for translation');
|
|
100
|
-
return '';
|
|
101
|
-
}
|
|
102
|
-
const sourceLangName = Language[sourceLang];
|
|
103
|
-
const targetLangName = Language[targetLang];
|
|
104
|
-
const terms = `- The following is a common related terminology vocabulary table (${sourceLangName} <=> ${targetLangName}), you should use it to translate the matched text.\n` +
|
|
105
|
-
relevantTerms
|
|
106
|
-
.map((term) => ` * ${term[sourceLang]} <=> ${term[targetLang]}`)
|
|
107
|
-
.join('\n');
|
|
108
|
-
logger.debug('Resolved terms:', terms);
|
|
109
|
-
return terms;
|
|
110
|
-
};
|
|
78
|
+
/** How many times findings are fed back before a document is failed. */
|
|
79
|
+
const DEFAULT_MAX_REPAIR_ROUNDS = 3;
|
|
80
|
+
/**
|
|
81
|
+
* Total assistant turns for one document, tool calls included.
|
|
82
|
+
*
|
|
83
|
+
* Separate from the repair rounds on purpose: a long document legitimately
|
|
84
|
+
* spends many turns reading and appending before it has anything to repair, so
|
|
85
|
+
* one number cannot be both the budget for producing a translation and the
|
|
86
|
+
* budget for fixing it. This one is a runaway guard, not a quality knob.
|
|
87
|
+
*/
|
|
88
|
+
const DEFAULT_MAX_TURNS = 60;
|
|
111
89
|
function extractFirstLevelHeading(content) {
|
|
112
90
|
const lines = content.split('\n');
|
|
113
91
|
for (const line of lines) {
|
|
@@ -126,19 +104,11 @@ function getTitleTranslation(title, sourceLang, targetLang) {
|
|
|
126
104
|
}
|
|
127
105
|
return null;
|
|
128
106
|
}
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
openai = new OpenAI({
|
|
132
|
-
baseURL: process.env.ALAUDA_OPENAI_BASE_URL,
|
|
133
|
-
apiKey: process.env.ALAUDA_OPENAI_API_KEY,
|
|
134
|
-
});
|
|
135
|
-
}
|
|
107
|
+
/** Renders the translation rules for one document. */
|
|
108
|
+
const renderTranslationRules = async ({ source, target, sourceContent, options, additionalPrompts, }) => {
|
|
136
109
|
const sourceLang = Language[source];
|
|
137
110
|
const targetLang = Language[target];
|
|
138
|
-
|
|
139
|
-
if ([source, target].every((lang) => TERMS_SUPPORTED_LANGUAGES.includes(lang))) {
|
|
140
|
-
terms = await resolveTerms(source, target, sourceContent);
|
|
141
|
-
}
|
|
111
|
+
const terms = await resolveTerms(source, target, sourceContent);
|
|
142
112
|
const firstLevelHeading = extractFirstLevelHeading(sourceContent);
|
|
143
113
|
let titleTranslationPrompt = '';
|
|
144
114
|
if (firstLevelHeading) {
|
|
@@ -147,51 +117,35 @@ export const translate = async ({ source, sourceContent, target, systemPrompt, u
|
|
|
147
117
|
titleTranslationPrompt = `- The heading "${firstLevelHeading}" should be translated as "${titleTranslation}".`;
|
|
148
118
|
}
|
|
149
119
|
}
|
|
150
|
-
const
|
|
120
|
+
const rules = await ejs.render(options.systemPrompt?.trim() || DEFAULT_SYSTEM_PROMPT, {
|
|
151
121
|
sourceLang,
|
|
152
122
|
targetLang,
|
|
153
|
-
userPrompt,
|
|
154
|
-
additionalPrompts,
|
|
155
|
-
terms,
|
|
123
|
+
userPrompt: options.userPrompt ?? '',
|
|
124
|
+
additionalPrompts: additionalPrompts ?? '',
|
|
125
|
+
terms: terms.text,
|
|
156
126
|
titleTranslationPrompt,
|
|
157
127
|
}, { async: true });
|
|
158
|
-
|
|
159
|
-
const stream = await openai.chat.completions.create({
|
|
160
|
-
messages: [
|
|
161
|
-
{
|
|
162
|
-
role: 'system',
|
|
163
|
-
content: finalSystemPrompt,
|
|
164
|
-
},
|
|
165
|
-
{
|
|
166
|
-
role: 'user',
|
|
167
|
-
content: sourceContent,
|
|
168
|
-
},
|
|
169
|
-
],
|
|
170
|
-
model: openaiModel,
|
|
171
|
-
temperature: 0.2,
|
|
172
|
-
stream: true,
|
|
173
|
-
});
|
|
174
|
-
let content = '';
|
|
175
|
-
for await (const chunk of stream) {
|
|
176
|
-
content += chunk.choices[0]?.delta.content ?? '';
|
|
177
|
-
}
|
|
178
|
-
return content;
|
|
128
|
+
return { rules, terms: terms.pairs };
|
|
179
129
|
};
|
|
180
|
-
const limit = pRateLimit({
|
|
181
|
-
interval: 60_000, // 1min
|
|
182
|
-
rate: 50,
|
|
183
|
-
concurrency: 10,
|
|
184
|
-
});
|
|
185
130
|
const supportedLanguages = SUPPORTED_LANGUAGES.join(', ');
|
|
186
131
|
export const translateCommand = new Command('translate')
|
|
187
132
|
.description('Translate the documentation')
|
|
133
|
+
.enablePositionalOptions()
|
|
188
134
|
.argument('[root]', 'Root directory of the documentation')
|
|
189
135
|
.option('-s, --source <language>', `Document source language, one of ${supportedLanguages}`, 'en')
|
|
190
136
|
.option('-t, --target <language>', `Document target language, one of ${supportedLanguages}`, 'zh')
|
|
191
|
-
|
|
137
|
+
// Deliberately not `requiredOption`: commander walks up the hierarchy when it
|
|
138
|
+
// checks mandatory options, so a required option here would make every
|
|
139
|
+
// subcommand — `doom translate check` — unusable. Checked in the action.
|
|
140
|
+
.option('-g, --glob <path...>', 'Glob patterns of source dirs/files to translate')
|
|
192
141
|
.option('-C, --copy [boolean]', 'Whether to copy relative assets to the target directory instead of following links', parseBoolean, false)
|
|
193
142
|
.action(async function (root) {
|
|
194
143
|
const { source, target, glob: globs, copy, force, ...globalOptions } = this.optsWithGlobals();
|
|
144
|
+
if (!globs?.length) {
|
|
145
|
+
logger.error(`Missing required option \`${cyan('-g, --glob <path...>')}\`: which source files to translate.`);
|
|
146
|
+
process.exitCode = 1;
|
|
147
|
+
return;
|
|
148
|
+
}
|
|
195
149
|
if (!Object.hasOwn(Language, source) ||
|
|
196
150
|
!Object.hasOwn(Language, target) ||
|
|
197
151
|
source === target) {
|
|
@@ -200,6 +154,12 @@ export const translateCommand = new Command('translate')
|
|
|
200
154
|
return;
|
|
201
155
|
}
|
|
202
156
|
const { config } = await loadConfig(root, globalOptions);
|
|
157
|
+
// The pairwise lint rules find the docs root through this file. Without it
|
|
158
|
+
// they resolve against whatever the last doom command left behind, decide
|
|
159
|
+
// the document is not a translation, and report nothing at all — a check
|
|
160
|
+
// that passes because it never ran.
|
|
161
|
+
await fs.mkdir(STORAGE_DIR, { recursive: true });
|
|
162
|
+
await fs.writeFile(OPTIONS_FILE, JSON.stringify({ root, globalOptions }, null, 2));
|
|
203
163
|
const docsDir = config.root;
|
|
204
164
|
const sourceDir = path.resolve(docsDir, source);
|
|
205
165
|
const targetDir = path.resolve(docsDir, target);
|
|
@@ -223,7 +183,7 @@ export const translateCommand = new Command('translate')
|
|
|
223
183
|
allSourceFilePaths.delete(internalFilePath);
|
|
224
184
|
}
|
|
225
185
|
// Get copy-only files using glob patterns
|
|
226
|
-
const copyOnlyFilePaths = await glob(
|
|
186
|
+
const copyOnlyFilePaths = await glob(config.translate?.copyOnlyDirectories ?? DEFAULT_COPY_ONLY_DIRECTORIES, {
|
|
227
187
|
absolute: true,
|
|
228
188
|
cwd: sourceDir,
|
|
229
189
|
});
|
|
@@ -259,12 +219,53 @@ export const translateCommand = new Command('translate')
|
|
|
259
219
|
await Promise.all(toRemoveTargetFilePaths.map((file) => fs.rm(file)));
|
|
260
220
|
}
|
|
261
221
|
}
|
|
262
|
-
const
|
|
222
|
+
const translateOptions = config.translate ?? {};
|
|
223
|
+
// Built once: the provider wiring, and the processors the rules run in.
|
|
224
|
+
// Missing credentials fail here, by name, rather than once per file as a
|
|
225
|
+
// wall of unrelated stream errors.
|
|
226
|
+
const gateway = await createGateway({
|
|
227
|
+
modelId: translateOptions.model,
|
|
228
|
+
judgeModelId: translateOptions.judge?.model,
|
|
229
|
+
contextWindow: translateOptions.contextWindow,
|
|
230
|
+
maxOutputTokens: translateOptions.maxOutputTokens,
|
|
231
|
+
});
|
|
232
|
+
const checker = createTranslationChecker();
|
|
233
|
+
const scratchDir = path.resolve(STORAGE_DIR, 'translate-scratch');
|
|
234
|
+
await fs.rm(scratchDir, { recursive: true, force: true });
|
|
235
|
+
// Per-site config wins; the environment is how one gateway's budget is set
|
|
236
|
+
// for every repository that talks to it, without editing each of them.
|
|
237
|
+
const concurrency = translateOptions.concurrency ??
|
|
238
|
+
positiveIntFromEnv(CONCURRENCY_ENV, DEFAULT_CONCURRENCY);
|
|
239
|
+
const requestsPerMinute = translateOptions.requestsPerMinute ??
|
|
240
|
+
positiveIntFromEnv(REQUESTS_PER_MINUTE_ENV, DEFAULT_REQUESTS_PER_MINUTE);
|
|
241
|
+
const { modelCallLimit, documentLimit } = createLimits({
|
|
242
|
+
concurrency,
|
|
243
|
+
requestsPerMinute,
|
|
244
|
+
});
|
|
245
|
+
const maxRepairRounds = translateOptions.maxRepairRounds ?? DEFAULT_MAX_REPAIR_ROUNDS;
|
|
246
|
+
const maxTurns = translateOptions.maxTurns ?? DEFAULT_MAX_TURNS;
|
|
247
|
+
const reasoningEffort = translateOptions.reasoningEffort ?? DEFAULT_REASONING_EFFORT;
|
|
248
|
+
const judge = translateOptions.judge?.enabled === false
|
|
249
|
+
? undefined
|
|
250
|
+
: createJudge({
|
|
251
|
+
models: gateway.models,
|
|
252
|
+
model: gateway.judgeModel,
|
|
253
|
+
reasoningEffort: translateOptions.judge?.reasoningEffort ??
|
|
254
|
+
DEFAULT_JUDGE_REASONING_EFFORT,
|
|
255
|
+
draws: translateOptions.judge?.draws,
|
|
256
|
+
limit: modelCallLimit,
|
|
257
|
+
});
|
|
258
|
+
logger.info(`Translating with \`${cyan(gateway.model.id)}\` (reasoning ${cyan(reasoningEffort)}), up to ${cyan(String(maxRepairRounds))} repair round(s) per document. ` +
|
|
259
|
+
`${cyan(String(concurrency))} at a time, at most ${cyan(String(requestsPerMinute))} model request(s) a minute. ` +
|
|
260
|
+
(judge
|
|
261
|
+
? `Reviewed by \`${cyan(gateway.judgeModel.id)}\`.`
|
|
262
|
+
: `${red('Judge disabled')} — only the deterministic checks are running.`));
|
|
263
|
+
const failures = [];
|
|
264
|
+
await Promise.all([...allSourceFilePaths].map(async (sourceFilePath) => {
|
|
263
265
|
const sourceContent = await fs.readFile(sourceFilePath, 'utf-8');
|
|
264
266
|
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
|
265
267
|
const { sourceSHA: _sourceSHA, ...sourceFrontmatter } = matter(sourceContent).data;
|
|
266
268
|
if (sourceFrontmatter.i18n?.disableAutoTranslation) {
|
|
267
|
-
allSourceFilePaths.delete(sourceFilePath);
|
|
268
269
|
return;
|
|
269
270
|
}
|
|
270
271
|
const sourceSHA = crypto
|
|
@@ -272,107 +273,149 @@ export const translateCommand = new Command('translate')
|
|
|
272
273
|
.update(sourceContent)
|
|
273
274
|
.digest('hex');
|
|
274
275
|
const targetFilePath = sourceFilePath.replace(sourceDir, targetDir);
|
|
275
|
-
let targetContent;
|
|
276
|
-
let targetFrontmatter;
|
|
277
276
|
if (await pathExists(targetFilePath, 'file')) {
|
|
278
|
-
targetContent = await fs.readFile(targetFilePath, 'utf-8');
|
|
279
|
-
targetFrontmatter = matter(targetContent)
|
|
277
|
+
const targetContent = await fs.readFile(targetFilePath, 'utf-8');
|
|
278
|
+
const targetFrontmatter = matter(targetContent)
|
|
279
|
+
.data;
|
|
280
280
|
if (!force && targetFrontmatter.sourceSHA === sourceSHA) {
|
|
281
|
-
allSourceFilePaths.delete(sourceFilePath);
|
|
282
281
|
return;
|
|
283
282
|
}
|
|
284
283
|
}
|
|
285
284
|
const shouldCopyOnly = copyOnlyFilePathsSet.has(sourceFilePath);
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
285
|
+
const sourceRelativePath = path.relative(docsDir, sourceFilePath);
|
|
286
|
+
const targetRelativePath = path.relative(docsDir, targetFilePath);
|
|
287
|
+
const targetBase = path.dirname(targetFilePath);
|
|
288
|
+
/** The document exactly as it will be written, given a translated body. */
|
|
289
|
+
const compose = (restored) => {
|
|
290
|
+
const newFrontmatter = { ...sourceFrontmatter, sourceSHA };
|
|
291
|
+
delete newFrontmatter.i18n;
|
|
292
|
+
const { data, content } = matter(restored);
|
|
293
|
+
const typedData = data;
|
|
294
|
+
if (typedData.title && typeof typedData.title === 'string') {
|
|
295
|
+
newFrontmatter.title = typedData.title;
|
|
296
|
+
}
|
|
297
|
+
if (typedData.description &&
|
|
298
|
+
typeof typedData.description === 'string') {
|
|
299
|
+
newFrontmatter.description = typedData.description;
|
|
300
|
+
}
|
|
301
|
+
if (sourceFrontmatter.title) {
|
|
302
|
+
const titleTranslation = getTitleTranslation(sourceFrontmatter.title, source, target);
|
|
303
|
+
if (titleTranslation) {
|
|
304
|
+
newFrontmatter.title = titleTranslation;
|
|
305
|
+
}
|
|
306
|
+
}
|
|
307
|
+
if (typeof newFrontmatter.title !== 'string') {
|
|
308
|
+
delete newFrontmatter.title;
|
|
309
|
+
}
|
|
310
|
+
return stringifyMatter(newFrontmatter, content);
|
|
311
|
+
};
|
|
312
|
+
await documentLimit(async () => {
|
|
289
313
|
if (shouldCopyOnly) {
|
|
290
314
|
logger.info(`Copying ${cyan(sourceRelativePath)} to ${cyan(targetRelativePath)}`);
|
|
291
|
-
// For copy-only files, we still update the sourceSHA but don't translate
|
|
292
|
-
const newFrontmatter = { ...sourceFrontmatter, sourceSHA };
|
|
293
|
-
delete newFrontmatter.i18n;
|
|
294
315
|
const { content } = matter(sourceContent);
|
|
295
|
-
targetContent = stringifyMatter(newFrontmatter, content);
|
|
296
|
-
const targetBase = path.dirname(targetFilePath);
|
|
297
316
|
await fs.mkdir(targetBase, { recursive: true });
|
|
298
|
-
await fs.writeFile(targetFilePath,
|
|
299
|
-
|
|
317
|
+
await fs.writeFile(targetFilePath, compose(content));
|
|
318
|
+
return;
|
|
300
319
|
}
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
320
|
+
logger.info(`Translating ${cyan(sourceRelativePath)} to ${cyan(targetRelativePath)}`);
|
|
321
|
+
const isMdx = sourceFilePath.endsWith('.mdx');
|
|
322
|
+
const processor = isMdx ? mdxProcessor : mdProcessor;
|
|
323
|
+
const ast = processor.parse(escapeMarkdownHeadingIds(sourceContent));
|
|
324
|
+
const sourceBase = path.dirname(sourceFilePath);
|
|
325
|
+
const normalizeOptions = { sourceBase, targetBase };
|
|
326
|
+
const normalizeImgSrcOptions = {
|
|
327
|
+
...normalizeOptions,
|
|
328
|
+
localPublicBase: path.resolve(docsDir, 'public'),
|
|
329
|
+
translating: { source, target, copy },
|
|
330
|
+
};
|
|
331
|
+
const normalizedAst = {
|
|
332
|
+
...ast,
|
|
333
|
+
children: ast.children.map((it) => translateCodeFile(normalizeImgSrc(it, normalizeImgSrcOptions), normalizeOptions)),
|
|
334
|
+
};
|
|
335
|
+
// Everything the model must not author — link targets, JSX attribute
|
|
336
|
+
// values, code, anchors — becomes an opaque placeholder here, so the
|
|
337
|
+
// real value is never in the model's context to rewrite.
|
|
338
|
+
const maskEntries = maskAst(normalizedAst);
|
|
339
|
+
const maskedSource = processor.stringify(normalizedAst);
|
|
340
|
+
// Proves the pairwise rules will actually compare this document
|
|
341
|
+
// against its source before anything is spent translating it. Their
|
|
342
|
+
// failure mode is to return quietly, so "no findings" and "not
|
|
343
|
+
// checked" are the same output — and this is the difference.
|
|
344
|
+
await checker.assertCheckable(targetFilePath, compose('# probe\n'));
|
|
345
|
+
const { rules, terms } = await renderTranslationRules({
|
|
346
|
+
source,
|
|
347
|
+
target,
|
|
348
|
+
sourceContent,
|
|
349
|
+
options: translateOptions,
|
|
350
|
+
additionalPrompts: sourceFrontmatter.i18n?.additionalPrompts,
|
|
351
|
+
});
|
|
352
|
+
const result = await translateWithAgent({
|
|
353
|
+
maskedSource,
|
|
354
|
+
maskEntries,
|
|
355
|
+
processor,
|
|
356
|
+
compose,
|
|
357
|
+
targetPath: targetFilePath,
|
|
358
|
+
sourceLabel: sourceRelativePath,
|
|
359
|
+
source,
|
|
360
|
+
target,
|
|
361
|
+
translationRules: rules,
|
|
362
|
+
terms,
|
|
363
|
+
checker,
|
|
364
|
+
judge,
|
|
365
|
+
models: gateway.models,
|
|
366
|
+
model: gateway.model,
|
|
367
|
+
reasoningEffort,
|
|
368
|
+
scratchDir,
|
|
369
|
+
maxRepairRounds,
|
|
370
|
+
maxTurns,
|
|
371
|
+
limit: modelCallLimit,
|
|
372
|
+
onProgress: (message) => {
|
|
373
|
+
logger.info(message);
|
|
374
|
+
},
|
|
375
|
+
});
|
|
376
|
+
if (!result.document) {
|
|
377
|
+
failures.push({
|
|
378
|
+
file: targetRelativePath,
|
|
379
|
+
findings: result.findings,
|
|
380
|
+
result,
|
|
326
381
|
});
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
restoreCodeBlockPlaceholders(targetAst, codeBlockPlaceholders);
|
|
330
|
-
targetContent = processor.stringify(targetAst);
|
|
331
|
-
}
|
|
332
|
-
const newFrontmatter = { ...sourceFrontmatter, sourceSHA };
|
|
333
|
-
delete newFrontmatter.i18n;
|
|
334
|
-
const { data, content } = matter(targetContent);
|
|
335
|
-
const typedData = data;
|
|
336
|
-
if (typedData.title && typeof typedData.title === 'string') {
|
|
337
|
-
newFrontmatter.title = typedData.title;
|
|
338
|
-
}
|
|
339
|
-
if (typedData.description &&
|
|
340
|
-
typeof typedData.description === 'string') {
|
|
341
|
-
newFrontmatter.description = typedData.description;
|
|
342
|
-
}
|
|
343
|
-
if (sourceFrontmatter.title) {
|
|
344
|
-
const titleTranslation = getTitleTranslation(sourceFrontmatter.title, source, target);
|
|
345
|
-
if (titleTranslation) {
|
|
346
|
-
newFrontmatter.title = titleTranslation;
|
|
347
|
-
}
|
|
348
|
-
}
|
|
349
|
-
if (typeof newFrontmatter.title !== 'string') {
|
|
350
|
-
delete newFrontmatter.title;
|
|
351
|
-
}
|
|
352
|
-
targetContent = stringifyMatter(newFrontmatter, content);
|
|
353
|
-
await fs.mkdir(targetBase, { recursive: true });
|
|
354
|
-
await fs.writeFile(targetFilePath, targetContent);
|
|
355
|
-
logger.info(`${cyan(sourceRelativePath)} translated to ${cyan(targetRelativePath)}`);
|
|
382
|
+
logger.error(`${cyan(targetRelativePath)} did not pass its checks after ${result.repairRounds} repair round(s).`);
|
|
383
|
+
return;
|
|
356
384
|
}
|
|
357
|
-
|
|
385
|
+
await fs.mkdir(targetBase, { recursive: true });
|
|
386
|
+
await fs.writeFile(targetFilePath, result.document);
|
|
387
|
+
logger.info(`${cyan(sourceRelativePath)} translated to ${cyan(targetRelativePath)} (${result.turns} turn(s), ${result.modelCalls} model call(s), ${result.repairRounds} repair round(s))`);
|
|
358
388
|
});
|
|
359
389
|
}));
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
390
|
+
await fs.rm(scratchDir, { recursive: true, force: true });
|
|
391
|
+
// Fail at the end, with the whole list.
|
|
392
|
+
//
|
|
393
|
+
// Not per file, and not by falling back to the previous translation: a
|
|
394
|
+
// failed document that is written out anyway, or quietly left at its old
|
|
395
|
+
// version, is a problem nobody sees. A red pipeline is the one channel in
|
|
396
|
+
// this organisation that has ever actually reached anyone, and the build
|
|
397
|
+
// does not go red for a document that was silently skipped.
|
|
398
|
+
if (failures.length > 0) {
|
|
399
|
+
logger.error(`${red(String(failures.length))} document(s) could not be translated to a state that passes their checks. Nothing will be uploaded.\n` +
|
|
400
|
+
failures
|
|
401
|
+
.map(({ file, findings, result }) => `\n${cyan(file)} — ${findings.length} unresolved problem(s) after ${result?.repairRounds ?? 0} repair round(s):\n` +
|
|
402
|
+
findings
|
|
403
|
+
.map((finding) => ` ${red(finding.rule)}${finding.line ? ` (line ${finding.line})` : ''} ${finding.reason}`)
|
|
404
|
+
.join('\n'))
|
|
405
|
+
.join('\n'));
|
|
406
|
+
process.exitCode = 1;
|
|
407
|
+
return;
|
|
375
408
|
}
|
|
376
|
-
logger.
|
|
377
|
-
|
|
409
|
+
logger.success('All translations passed their checks.');
|
|
410
|
+
});
|
|
411
|
+
translateCommand
|
|
412
|
+
.command('check')
|
|
413
|
+
.description('Check translations that already exist against their sources — offline, with no translation model involved. Runs the same rules the translator runs in its own loop')
|
|
414
|
+
.argument('[root]', 'Root directory of the documentation')
|
|
415
|
+
.option('-g, --glob <path...>', 'Glob patterns within each language directory', ['**/*.md{,x}'])
|
|
416
|
+
.option('-t, --target <language...>', 'Languages to check (default: every translated language directory present)')
|
|
417
|
+
.action(async function (root) {
|
|
418
|
+
const globalOptions = this.optsWithGlobals();
|
|
419
|
+
const { glob: globs, target } = this.opts();
|
|
420
|
+
await checkTranslations(root, globalOptions, { glob: globs, target });
|
|
378
421
|
});
|
package/lib/eslint.js
CHANGED
|
@@ -12,6 +12,7 @@ import globals from 'globals';
|
|
|
12
12
|
import tseslint from 'typescript-eslint';
|
|
13
13
|
import { loadConfig } from './cli/load-config.js';
|
|
14
14
|
import { getGlobalComponentNames } from './plugins/global/components.js';
|
|
15
|
+
import { SUPPORTED_LANGUAGES } from './shared/index.js';
|
|
15
16
|
import { pkgResolve } from './utils/index.js';
|
|
16
17
|
const cjsRequire = createRequire(import.meta.url);
|
|
17
18
|
let remarkConfigPath;
|
|
@@ -69,14 +70,32 @@ async function doom(userConfigOrRoot) {
|
|
|
69
70
|
},
|
|
70
71
|
},
|
|
71
72
|
{
|
|
72
|
-
|
|
73
|
-
|
|
73
|
+
// Every language directory, not just the one documents are written in.
|
|
74
|
+
// The remark config is what carries doom's lint rules, so scoping it to
|
|
75
|
+
// the source language meant a translated document was parsed and then
|
|
76
|
+
// checked against nothing: a lint run over `zh/` reported success having
|
|
77
|
+
// applied no rule at all.
|
|
78
|
+
//
|
|
79
|
+
// Still scoped to language directories rather than to every markdown file
|
|
80
|
+
// in the repository — changesets, READMEs and design notes are not
|
|
81
|
+
// documentation pages and do not want documentation rules.
|
|
82
|
+
files: [
|
|
83
|
+
`**/{${SUPPORTED_LANGUAGES.join(',')}}/**/*.{js,jsx,md,mdx,ts,tsx}`,
|
|
84
|
+
],
|
|
74
85
|
languageOptions: {
|
|
75
|
-
globals: globals.browser,
|
|
76
86
|
parserOptions: {
|
|
77
87
|
...(remarkConfigPath && { remarkConfigPath }),
|
|
78
88
|
},
|
|
79
89
|
},
|
|
90
|
+
},
|
|
91
|
+
{
|
|
92
|
+
// Spell checking stays scoped to the source language: the dictionaries
|
|
93
|
+
// are for it, and running them over a translation is noise.
|
|
94
|
+
files: [`**/${config?.lang ?? 'en'}/**/*.{js,jsx,md,mdx,ts,tsx}`],
|
|
95
|
+
extends: cspellEnabled ? [cspellRecommended] : [],
|
|
96
|
+
languageOptions: {
|
|
97
|
+
globals: globals.browser,
|
|
98
|
+
},
|
|
80
99
|
rules: cspellEnabled
|
|
81
100
|
? {
|
|
82
101
|
'@cspell/spellchecker': [
|