@alauda/doom 2.6.0 → 2.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/cli/translate-call.d.ts +85 -0
- package/lib/cli/translate-call.js +149 -0
- package/lib/cli/translate-checker.d.ts +8 -0
- package/lib/cli/translate-diagnose.d.ts +45 -0
- package/lib/cli/translate-diagnose.js +118 -0
- package/lib/cli/translate-judge.d.ts +28 -3
- package/lib/cli/translate-judge.js +40 -7
- package/lib/cli/translate-mask.d.ts +13 -0
- package/lib/cli/translate-mask.js +19 -5
- package/lib/cli/translate-pipeline.d.ts +144 -0
- package/lib/cli/translate-pipeline.js +489 -0
- package/lib/cli/translate-repair-agent.d.ts +69 -0
- package/lib/cli/translate-repair-agent.js +241 -0
- package/lib/cli/translate-scratch.d.ts +9 -1
- package/lib/cli/translate-scratch.js +2 -2
- package/lib/cli/translate-segment-cache.d.ts +90 -0
- package/lib/cli/translate-segment-cache.js +301 -0
- package/lib/cli/translate-segment-check.d.ts +42 -0
- package/lib/cli/translate-segment-check.js +247 -0
- package/lib/cli/translate-segment.d.ts +307 -0
- package/lib/cli/translate-segment.js +625 -0
- package/lib/cli/translate.d.ts +14 -4
- package/lib/cli/translate.js +232 -52
- package/lib/remark-lint/no-unmatched-anchor.js +6 -2
- package/lib/remark-lint/no-unopened-frontmatter.js +5 -1
- package/lib/remark-lint/rule-sets.d.ts +1 -3
- package/lib/remark-lint/rule-sets.js +3 -9
- package/lib/remark-lint/syntax-plugins.d.ts +32 -0
- package/lib/remark-lint/syntax-plugins.js +46 -0
- package/lib/remark-lint/translation-parity/frontmatter-preservation.js +5 -3
- package/lib/remark-lint/translation-parity/link-isomorphism.d.ts +8 -0
- package/lib/remark-lint/translation-parity/link-isomorphism.js +40 -4
- package/lib/remark-lint/translation-parity/shared.js +6 -2
- package/lib/remark-lint/translation-parity/url-residue.d.ts +1 -0
- package/lib/remark-lint/translation-parity/url-residue.js +25 -4
- package/lib/runtime/hooks/useTranslation.d.ts +1 -1
- package/lib/types.d.ts +101 -6
- package/package.json +1 -1
- package/lib/cli/translate-agent.d.ts +0 -81
- package/lib/cli/translate-agent.js +0 -450
package/lib/cli/translate.js
CHANGED
|
@@ -14,20 +14,33 @@ import { DEFAULT_COPY_ONLY_DIRECTORIES, Language, SUPPORTED_LANGUAGES, TITLE_TRA
|
|
|
14
14
|
import { OPTIONS_FILE, STORAGE_DIR, pathExists } from '../utils/index.js';
|
|
15
15
|
import { escapeMarkdownHeadingIds, getMatchedDocFilePaths, parseBoolean, stringifyMatter, translateCodeFile, } from './helpers.js';
|
|
16
16
|
import { loadConfig } from './load-config.js';
|
|
17
|
-
import {
|
|
17
|
+
import { createSegmentTranslator } from './translate-call.js';
|
|
18
18
|
import { checkTranslations, } from './translate-check.js';
|
|
19
|
-
import { createTranslationChecker
|
|
19
|
+
import { createTranslationChecker } from './translate-checker.js';
|
|
20
|
+
import { createDiagnoser } from './translate-diagnose.js';
|
|
20
21
|
import { createJudge } from './translate-judge.js';
|
|
21
22
|
import { CONCURRENCY_ENV, DEFAULT_CONCURRENCY, DEFAULT_REQUESTS_PER_MINUTE, REQUESTS_PER_MINUTE_ENV, createLimits, positiveIntFromEnv, } from './translate-limits.js';
|
|
22
23
|
import { maskAst } from './translate-mask.js';
|
|
23
|
-
import { DEFAULT_JUDGE_REASONING_EFFORT, DEFAULT_REASONING_EFFORT, createGateway, } from './translate-models.js';
|
|
24
|
+
import { DEFAULT_JUDGE_REASONING_EFFORT, DEFAULT_REASONING_EFFORT, createGateway, gatewayModel, } from './translate-models.js';
|
|
25
|
+
import { DEFAULT_MAX_SEGMENT_ATTEMPTS, translateDocument, } from './translate-pipeline.js';
|
|
26
|
+
import { createRepairAgent } from './translate-repair-agent.js';
|
|
24
27
|
import { resolveTerms } from './translate-terms.js';
|
|
28
|
+
/**
|
|
29
|
+
* How many readings a segment gets, and how many must agree.
|
|
30
|
+
*
|
|
31
|
+
* Three and two, against two-and-unanimous for a whole page. A segment is a
|
|
32
|
+
* short text, and a reviewer given less to go on has more room to find
|
|
33
|
+
* something to say; the third reading buys back the precision that unanimity
|
|
34
|
+
* gives when there is a whole document to be sure about.
|
|
35
|
+
*/
|
|
36
|
+
const DEFAULT_SEGMENT_DRAWS = 3;
|
|
37
|
+
const DEFAULT_SEGMENT_VOTES = 2;
|
|
25
38
|
/**
|
|
26
39
|
* The translation rules.
|
|
27
40
|
*
|
|
28
41
|
* Overridable per repository through `translate.systemPrompt`, which is why
|
|
29
42
|
* the placeholder discipline is *not* here: it is part of the harness's own
|
|
30
|
-
* prompt in `translate-
|
|
43
|
+
* prompt in `translate-call.ts`, where a repository cannot drop it by
|
|
31
44
|
* replacing this text.
|
|
32
45
|
*/
|
|
33
46
|
const DEFAULT_SYSTEM_PROMPT = `
|
|
@@ -75,17 +88,6 @@ These are additional requirements for the translation. They should be met along
|
|
|
75
88
|
"""
|
|
76
89
|
<% } %>
|
|
77
90
|
`.trim();
|
|
78
|
-
/** How many times findings are fed back before a document is failed. */
|
|
79
|
-
const DEFAULT_MAX_REPAIR_ROUNDS = 3;
|
|
80
|
-
/**
|
|
81
|
-
* Total assistant turns for one document, tool calls included.
|
|
82
|
-
*
|
|
83
|
-
* Separate from the repair rounds on purpose: a long document legitimately
|
|
84
|
-
* spends many turns reading and appending before it has anything to repair, so
|
|
85
|
-
* one number cannot be both the budget for producing a translation and the
|
|
86
|
-
* budget for fixing it. This one is a runaway guard, not a quality knob.
|
|
87
|
-
*/
|
|
88
|
-
const DEFAULT_MAX_TURNS = 60;
|
|
89
91
|
function extractFirstLevelHeading(content) {
|
|
90
92
|
const lines = content.split('\n');
|
|
91
93
|
for (const line of lines) {
|
|
@@ -127,6 +129,66 @@ const renderTranslationRules = async ({ source, target, sourceContent, options,
|
|
|
127
129
|
}, { async: true });
|
|
128
130
|
return { rules, terms: terms.pairs };
|
|
129
131
|
};
|
|
132
|
+
/** One segment, named the way a person reading a build log needs it named. */
|
|
133
|
+
const segmentName = (outcome) => `segment ${outcome.index + 1}` +
|
|
134
|
+
(outcome.label.line || outcome.label.heading
|
|
135
|
+
? ` (${[
|
|
136
|
+
outcome.label.line ? `line ${outcome.label.line}` : undefined,
|
|
137
|
+
outcome.label.heading ? `‹${outcome.label.heading}›` : undefined,
|
|
138
|
+
]
|
|
139
|
+
.filter(Boolean)
|
|
140
|
+
.join(' ')})`
|
|
141
|
+
: '');
|
|
142
|
+
/**
|
|
143
|
+
* Why a document failed, in one line.
|
|
144
|
+
*
|
|
145
|
+
* Three ways to fail, and they are told apart on purpose: a segment nothing
|
|
146
|
+
* could translate, a whole-document check that kept failing after the segments
|
|
147
|
+
* it named were redone, and a whole-document check nobody could attribute to a
|
|
148
|
+
* segment. They call for different things from whoever reads the log, and the
|
|
149
|
+
* design this replaced reported all of them as the same wall of problems.
|
|
150
|
+
*/
|
|
151
|
+
const describeFailure = (result) => {
|
|
152
|
+
const { failure } = result;
|
|
153
|
+
switch (failure?.kind) {
|
|
154
|
+
case 'unsplittable-block': {
|
|
155
|
+
return 'A single block is too large to translate and cannot be divided.';
|
|
156
|
+
}
|
|
157
|
+
case 'segment': {
|
|
158
|
+
return `${failure.segments.length} segment(s) never passed: ${failure.segments
|
|
159
|
+
.map((index) => segmentName(result.outcomes[index]))
|
|
160
|
+
.join(', ')}.`;
|
|
161
|
+
}
|
|
162
|
+
case 'assembly': {
|
|
163
|
+
return `The assembled document still failed its checks after ${result.assemblyRounds} round(s) of sending segments back.`;
|
|
164
|
+
}
|
|
165
|
+
case 'unlocatable': {
|
|
166
|
+
return 'The assembled document failed a check of the whole page that could not be attributed to any one segment.';
|
|
167
|
+
}
|
|
168
|
+
default: {
|
|
169
|
+
return 'The document did not pass its checks.';
|
|
170
|
+
}
|
|
171
|
+
}
|
|
172
|
+
};
|
|
173
|
+
/** Everything that was wrong with one failed document, grouped by segment. */
|
|
174
|
+
const describeFailedDocument = ({ file, result, diagnosis, }) => {
|
|
175
|
+
const lines = [`\n${cyan(file)} — ${describeFailure(result)}`];
|
|
176
|
+
const failed = result.failure?.kind === 'segment'
|
|
177
|
+
? result.failure.segments.map((index) => result.outcomes[index])
|
|
178
|
+
: [];
|
|
179
|
+
for (const outcome of failed) {
|
|
180
|
+
lines.push(` ${segmentName(outcome)}, after ${outcome.attempts} attempt(s):`, ...outcome.findings.map((finding) => ` ${red(finding.rule)} ${finding.reason}`));
|
|
181
|
+
}
|
|
182
|
+
if (failed.length === 0) {
|
|
183
|
+
lines.push(...result.findings
|
|
184
|
+
.filter((finding) => finding.blocking !== false)
|
|
185
|
+
.map((finding) => ` ${red(finding.rule)}${finding.line ? ` (line ${finding.line})` : ''} ${finding.reason}`));
|
|
186
|
+
}
|
|
187
|
+
if (diagnosis) {
|
|
188
|
+
lines.push('', ` ${cyan('What this looks like')} (written by a model from the evidence above; advisory, and not part of the verdict):`, ...diagnosis.split('\n').map((line) => ` ${line}`));
|
|
189
|
+
}
|
|
190
|
+
return lines.join('\n');
|
|
191
|
+
};
|
|
130
192
|
const supportedLanguages = SUPPORTED_LANGUAGES.join(', ');
|
|
131
193
|
export const translateCommand = new Command('translate')
|
|
132
194
|
.description('Translate the documentation')
|
|
@@ -220,6 +282,25 @@ export const translateCommand = new Command('translate')
|
|
|
220
282
|
}
|
|
221
283
|
}
|
|
222
284
|
const translateOptions = config.translate ?? {};
|
|
285
|
+
// Two options were removed when the whole-document loop was, and both of
|
|
286
|
+
// them named a budget that no longer exists. Ignoring them quietly would
|
|
287
|
+
// leave a repository believing it had configured something — which is the
|
|
288
|
+
// exact failure mode this rewrite exists to remove, in miniature.
|
|
289
|
+
const removed = {
|
|
290
|
+
maxRepairRounds: '`maxSegmentAttempts` (per segment) and `maxAssemblyRounds` (per document)',
|
|
291
|
+
maxTurns: '`repairAgent.maxTurns`, which now bounds only the repair agent',
|
|
292
|
+
};
|
|
293
|
+
const usedRemoved = Object.keys(removed).filter((key) => key in translateOptions);
|
|
294
|
+
if (usedRemoved.length > 0) {
|
|
295
|
+
logger.error(`\`translate\` no longer has ${usedRemoved
|
|
296
|
+
.map((key) => `\`${cyan(key)}\``)
|
|
297
|
+
.join(' or ')}: a translation is now produced a segment at a time, so a budget for the whole document has nothing to bound.\n` +
|
|
298
|
+
usedRemoved
|
|
299
|
+
.map((key) => ` ${cyan(key)} → replaced by ${removed[key]}`)
|
|
300
|
+
.join('\n'));
|
|
301
|
+
process.exitCode = 1;
|
|
302
|
+
return;
|
|
303
|
+
}
|
|
223
304
|
// Built once: the provider wiring, and the processors the rules run in.
|
|
224
305
|
// Missing credentials fail here, by name, rather than once per file as a
|
|
225
306
|
// wall of unrelated stream errors.
|
|
@@ -242,22 +323,57 @@ export const translateCommand = new Command('translate')
|
|
|
242
323
|
concurrency,
|
|
243
324
|
requestsPerMinute,
|
|
244
325
|
});
|
|
245
|
-
const maxRepairRounds = translateOptions.maxRepairRounds ?? DEFAULT_MAX_REPAIR_ROUNDS;
|
|
246
|
-
const maxTurns = translateOptions.maxTurns ?? DEFAULT_MAX_TURNS;
|
|
247
326
|
const reasoningEffort = translateOptions.reasoningEffort ?? DEFAULT_REASONING_EFFORT;
|
|
248
|
-
const
|
|
249
|
-
|
|
250
|
-
|
|
327
|
+
const maxSegmentAttempts = translateOptions.maxSegmentAttempts ?? DEFAULT_MAX_SEGMENT_ATTEMPTS;
|
|
328
|
+
const judgeEnabled = translateOptions.judge?.enabled !== false;
|
|
329
|
+
const judgeReasoning = translateOptions.judge?.reasoningEffort ?? DEFAULT_JUDGE_REASONING_EFFORT;
|
|
330
|
+
// Two reviewers, because they are asked two different questions. The first
|
|
331
|
+
// reads one segment against its source and can stop it being frozen; the
|
|
332
|
+
// second reads the finished page for what no segment can see — a term that
|
|
333
|
+
// drifted between sections, a join that reads badly — and only ever
|
|
334
|
+
// reports.
|
|
335
|
+
const segmentJudge = judgeEnabled
|
|
336
|
+
? createJudge({
|
|
251
337
|
models: gateway.models,
|
|
252
338
|
model: gateway.judgeModel,
|
|
253
|
-
reasoningEffort:
|
|
254
|
-
|
|
339
|
+
reasoningEffort: judgeReasoning,
|
|
340
|
+
draws: translateOptions.judge?.segmentDraws ?? DEFAULT_SEGMENT_DRAWS,
|
|
341
|
+
votes: translateOptions.judge?.segmentVotes ?? DEFAULT_SEGMENT_VOTES,
|
|
342
|
+
limit: modelCallLimit,
|
|
343
|
+
})
|
|
344
|
+
: undefined;
|
|
345
|
+
const documentJudge = judgeEnabled && translateOptions.fullDocJudge !== false
|
|
346
|
+
? createJudge({
|
|
347
|
+
models: gateway.models,
|
|
348
|
+
model: gateway.judgeModel,
|
|
349
|
+
reasoningEffort: judgeReasoning,
|
|
255
350
|
draws: translateOptions.judge?.draws,
|
|
256
351
|
limit: modelCallLimit,
|
|
352
|
+
})
|
|
353
|
+
: undefined;
|
|
354
|
+
// A segment that repeated attempts could not fix goes to an agent that can
|
|
355
|
+
// only edit. It may be a stronger model than the one doing the translating:
|
|
356
|
+
// this path is rare and it is the hard one.
|
|
357
|
+
const repairModelId = translateOptions.repairAgent?.model;
|
|
358
|
+
const repairModel = repairModelId && repairModelId !== gateway.model.id
|
|
359
|
+
? gatewayModel({
|
|
360
|
+
id: repairModelId,
|
|
361
|
+
baseUrl: gateway.baseUrl,
|
|
362
|
+
contextWindow: translateOptions.contextWindow,
|
|
363
|
+
maxOutputTokens: translateOptions.maxOutputTokens,
|
|
364
|
+
})
|
|
365
|
+
: gateway.model;
|
|
366
|
+
const diagnoser = translateOptions.diagnose?.enabled === false
|
|
367
|
+
? undefined
|
|
368
|
+
: createDiagnoser({
|
|
369
|
+
models: gateway.models,
|
|
370
|
+
model: gateway.model,
|
|
371
|
+
reasoningEffort: translateOptions.diagnose?.reasoningEffort,
|
|
372
|
+
limit: modelCallLimit,
|
|
257
373
|
});
|
|
258
|
-
logger.info(`Translating with \`${cyan(gateway.model.id)}\` (reasoning ${cyan(reasoningEffort)}), up to ${cyan(String(
|
|
259
|
-
`${cyan(String(concurrency))} at a time, at most ${cyan(String(requestsPerMinute))} model request(s) a minute. ` +
|
|
260
|
-
(
|
|
374
|
+
logger.info(`Translating with \`${cyan(gateway.model.id)}\` (reasoning ${cyan(reasoningEffort)}), a segment at a time, up to ${cyan(String(maxSegmentAttempts))} attempt(s) per segment. ` +
|
|
375
|
+
`${cyan(String(concurrency))} document(s) at a time, at most ${cyan(String(requestsPerMinute))} model request(s) a minute. ` +
|
|
376
|
+
(segmentJudge
|
|
261
377
|
? `Reviewed by \`${cyan(gateway.judgeModel.id)}\`.`
|
|
262
378
|
: `${red('Judge disabled')} — only the deterministic checks are running.`));
|
|
263
379
|
const failures = [];
|
|
@@ -273,22 +389,42 @@ export const translateCommand = new Command('translate')
|
|
|
273
389
|
.update(sourceContent)
|
|
274
390
|
.digest('hex');
|
|
275
391
|
const targetFilePath = sourceFilePath.replace(sourceDir, targetDir);
|
|
392
|
+
/**
|
|
393
|
+
* The translation already on disk, when there is one.
|
|
394
|
+
*
|
|
395
|
+
* Two layers use it. `sourceSHA` skips a document nobody touched at
|
|
396
|
+
* all; below that, the segments whose source did not change are reused
|
|
397
|
+
* rather than retranslated, so a one-line edit costs one segment
|
|
398
|
+
* instead of a whole page. `--force` turns off both, which is what
|
|
399
|
+
* "translate it again" has always meant.
|
|
400
|
+
*/
|
|
401
|
+
let previous;
|
|
276
402
|
if (await pathExists(targetFilePath, 'file')) {
|
|
277
403
|
const targetContent = await fs.readFile(targetFilePath, 'utf-8');
|
|
278
|
-
const
|
|
279
|
-
|
|
404
|
+
const parsedTarget = matter(targetContent);
|
|
405
|
+
const targetFrontmatter = parsedTarget.data;
|
|
280
406
|
if (!force && targetFrontmatter.sourceSHA === sourceSHA) {
|
|
281
407
|
return;
|
|
282
408
|
}
|
|
409
|
+
if (!force && translateOptions.segmentCache !== false) {
|
|
410
|
+
previous = {
|
|
411
|
+
body: parsedTarget.content,
|
|
412
|
+
record: targetFrontmatter.i18nSegments,
|
|
413
|
+
};
|
|
414
|
+
}
|
|
283
415
|
}
|
|
284
416
|
const shouldCopyOnly = copyOnlyFilePathsSet.has(sourceFilePath);
|
|
285
417
|
const sourceRelativePath = path.relative(docsDir, sourceFilePath);
|
|
286
418
|
const targetRelativePath = path.relative(docsDir, targetFilePath);
|
|
287
419
|
const targetBase = path.dirname(targetFilePath);
|
|
288
420
|
/** The document exactly as it will be written, given a translated body. */
|
|
289
|
-
const compose = (restored) => {
|
|
421
|
+
const compose = (restored, cache) => {
|
|
290
422
|
const newFrontmatter = { ...sourceFrontmatter, sourceSHA };
|
|
291
423
|
delete newFrontmatter.i18n;
|
|
424
|
+
delete newFrontmatter.i18nSegments;
|
|
425
|
+
if (cache) {
|
|
426
|
+
newFrontmatter.i18nSegments = cache;
|
|
427
|
+
}
|
|
292
428
|
const { data, content } = matter(restored);
|
|
293
429
|
const typedData = data;
|
|
294
430
|
if (typedData.title && typeof typedData.title === 'string') {
|
|
@@ -349,42 +485,91 @@ export const translateCommand = new Command('translate')
|
|
|
349
485
|
options: translateOptions,
|
|
350
486
|
additionalPrompts: sourceFrontmatter.i18n?.additionalPrompts,
|
|
351
487
|
});
|
|
352
|
-
const
|
|
488
|
+
const repairer = translateOptions.repairAgent?.enabled === false
|
|
489
|
+
? undefined
|
|
490
|
+
: createRepairAgent({
|
|
491
|
+
models: gateway.models,
|
|
492
|
+
model: repairModel,
|
|
493
|
+
reasoningEffort: translateOptions.repairAgent?.reasoningEffort,
|
|
494
|
+
scratchDir,
|
|
495
|
+
extension: isMdx ? '.mdx' : '.md',
|
|
496
|
+
maxTurns: translateOptions.repairAgent?.maxTurns,
|
|
497
|
+
limit: modelCallLimit,
|
|
498
|
+
onProgress: (message) => {
|
|
499
|
+
logger.info(`${cyan(sourceRelativePath)}: ${message}`);
|
|
500
|
+
},
|
|
501
|
+
});
|
|
502
|
+
const translator = createSegmentTranslator({
|
|
503
|
+
models: gateway.models,
|
|
504
|
+
model: gateway.model,
|
|
505
|
+
reasoningEffort,
|
|
506
|
+
translationRules: rules,
|
|
507
|
+
sourceLanguage: Language[source],
|
|
508
|
+
targetLanguage: Language[target],
|
|
509
|
+
processor,
|
|
510
|
+
limit: modelCallLimit,
|
|
511
|
+
});
|
|
512
|
+
const result = await translateDocument({
|
|
513
|
+
tree: normalizedAst,
|
|
353
514
|
maskedSource,
|
|
354
515
|
maskEntries,
|
|
355
516
|
processor,
|
|
356
517
|
compose,
|
|
357
518
|
targetPath: targetFilePath,
|
|
358
519
|
sourceLabel: sourceRelativePath,
|
|
359
|
-
source,
|
|
360
|
-
target,
|
|
361
|
-
|
|
362
|
-
terms,
|
|
520
|
+
sourceLanguage: Language[source],
|
|
521
|
+
targetLanguage: Language[target],
|
|
522
|
+
translator,
|
|
363
523
|
checker,
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
524
|
+
segmentJudge,
|
|
525
|
+
documentJudge,
|
|
526
|
+
terms,
|
|
527
|
+
segmentCap: translateOptions.segmentCap,
|
|
528
|
+
segmentHardCap: translateOptions.segmentHardCap,
|
|
529
|
+
segmentFloor: translateOptions.segmentFloor,
|
|
530
|
+
maxSegmentAttempts,
|
|
531
|
+
maxAssemblyRounds: translateOptions.maxAssemblyRounds,
|
|
532
|
+
contextTail: translateOptions.contextTail,
|
|
533
|
+
repairer,
|
|
534
|
+
previous,
|
|
372
535
|
onProgress: (message) => {
|
|
373
536
|
logger.info(message);
|
|
374
537
|
},
|
|
375
538
|
});
|
|
376
539
|
if (!result.document) {
|
|
377
|
-
|
|
378
|
-
|
|
540
|
+
// Explained before it is reported, so the log carries the analysis
|
|
541
|
+
// next to the failure rather than in a separate place nobody
|
|
542
|
+
// correlates. Advisory throughout: it cannot change this outcome.
|
|
543
|
+
const diagnosis = await diagnoser?.diagnose({
|
|
544
|
+
sourceLabel: sourceRelativePath,
|
|
545
|
+
failure: result.failure,
|
|
546
|
+
outcomes: result.outcomes,
|
|
379
547
|
findings: result.findings,
|
|
380
|
-
result,
|
|
548
|
+
assemblyRounds: result.assemblyRounds,
|
|
381
549
|
});
|
|
382
|
-
|
|
550
|
+
failures.push({ file: targetRelativePath, result, diagnosis });
|
|
551
|
+
logger.error(`${cyan(targetRelativePath)} did not pass its checks. ${describeFailure(result)}`);
|
|
383
552
|
return;
|
|
384
553
|
}
|
|
385
554
|
await fs.mkdir(targetBase, { recursive: true });
|
|
386
555
|
await fs.writeFile(targetFilePath, result.document);
|
|
387
|
-
|
|
556
|
+
const cached = result.outcomes.filter((outcome) => outcome.status === 'cached').length;
|
|
557
|
+
const repaired = result.outcomes.filter((outcome) => outcome.status === 'repaired').length;
|
|
558
|
+
logger.info(`${cyan(sourceRelativePath)} translated to ${cyan(targetRelativePath)} ` +
|
|
559
|
+
`(${result.outcomes.length} segment(s)` +
|
|
560
|
+
(cached ? `, ${cached} reused` : '') +
|
|
561
|
+
(repaired ? `, ${repaired} repaired` : '') +
|
|
562
|
+
// Translation calls, named as such. The reviewers, the repair
|
|
563
|
+
// agent and the diagnosis all cost model calls too, and none of
|
|
564
|
+
// them are counted here: their judges are shared by every
|
|
565
|
+
// document being translated at once, so there is no per-document
|
|
566
|
+
// number to print. Calling this "model calls" made it look like
|
|
567
|
+
// there was.
|
|
568
|
+
`, ${translator.calls()} translation call(s)` +
|
|
569
|
+
(result.assemblyRounds
|
|
570
|
+
? `, ${result.assemblyRounds} assembly round(s)`
|
|
571
|
+
: '') +
|
|
572
|
+
')');
|
|
388
573
|
});
|
|
389
574
|
}));
|
|
390
575
|
await fs.rm(scratchDir, { recursive: true, force: true });
|
|
@@ -397,12 +582,7 @@ export const translateCommand = new Command('translate')
|
|
|
397
582
|
// does not go red for a document that was silently skipped.
|
|
398
583
|
if (failures.length > 0) {
|
|
399
584
|
logger.error(`${red(String(failures.length))} document(s) could not be translated to a state that passes their checks. Nothing will be uploaded.\n` +
|
|
400
|
-
failures
|
|
401
|
-
.map(({ file, findings, result }) => `\n${cyan(file)} — ${findings.length} unresolved problem(s) after ${result?.repairRounds ?? 0} repair round(s):\n` +
|
|
402
|
-
findings
|
|
403
|
-
.map((finding) => ` ${red(finding.rule)}${finding.line ? ` (line ${finding.line})` : ''} ${finding.reason}`)
|
|
404
|
-
.join('\n'))
|
|
405
|
-
.join('\n'));
|
|
585
|
+
failures.map(describeFailedDocument).join('\n'));
|
|
406
586
|
process.exitCode = 1;
|
|
407
587
|
return;
|
|
408
588
|
}
|
|
@@ -6,7 +6,7 @@ import { lintRule } from 'unified-lint-rule';
|
|
|
6
6
|
import { visit } from 'unist-util-visit';
|
|
7
7
|
import { visitParents } from 'unist-util-visit-parents';
|
|
8
8
|
import { isDoc } from "../cli/helpers.js";
|
|
9
|
-
import {
|
|
9
|
+
import { syntaxProcessor } from "./syntax-plugins.js";
|
|
10
10
|
import { getConfig } from "./utils.js";
|
|
11
11
|
const anchorsCache = new Map();
|
|
12
12
|
const astCache = new Map();
|
|
@@ -30,7 +30,11 @@ const getAnchors = (filepath) => {
|
|
|
30
30
|
const anchors = new Set();
|
|
31
31
|
let ast = astCache.get(filepath);
|
|
32
32
|
if (!ast) {
|
|
33
|
-
|
|
33
|
+
// The same stack the lint pipeline uses, so a document's anchors do not
|
|
34
|
+
// depend on whether this run happened to lint it before reading it.
|
|
35
|
+
const processor = filepath.endsWith('.mdx')
|
|
36
|
+
? syntaxProcessor.mdx
|
|
37
|
+
: syntaxProcessor.md;
|
|
34
38
|
ast = processor.parse(fs.readFileSync(filepath, 'utf-8'));
|
|
35
39
|
astCache.set(filepath, ast);
|
|
36
40
|
}
|
|
@@ -33,7 +33,10 @@ import { visit } from 'unist-util-visit';
|
|
|
33
33
|
/**
|
|
34
34
|
* The keys doom's frontmatter actually uses, counted across acp-docs rather
|
|
35
35
|
* than remembered: weight 797 · sourceSHA 194 · title 132 · i18n 125 ·
|
|
36
|
-
* queries 73 · description 21 · category 5 · author 4.
|
|
36
|
+
* queries 73 · description 21 · category 5 · author 4. `i18nSegments` is
|
|
37
|
+
* written by `doom translate` and so appears in no count taken before it
|
|
38
|
+
* existed; it is here because a translation is exactly the kind of file whose
|
|
39
|
+
* frontmatter a model can lose the opening `---` from.
|
|
37
40
|
*
|
|
38
41
|
* Deliberately a list and not `\w+:` — a heading really can begin "Note:" or
|
|
39
42
|
* "Step 1:", and reporting those would be a rule nobody keeps.
|
|
@@ -43,6 +46,7 @@ const FRONTMATTER_KEYS = [
|
|
|
43
46
|
'category',
|
|
44
47
|
'description',
|
|
45
48
|
'i18n',
|
|
49
|
+
'i18nSegments',
|
|
46
50
|
'queries',
|
|
47
51
|
'sourceSHA',
|
|
48
52
|
'title',
|
|
@@ -10,8 +10,7 @@ import type { Plugin } from 'unified';
|
|
|
10
10
|
* register it twice.
|
|
11
11
|
*/
|
|
12
12
|
type Rule = Plugin<[], never>;
|
|
13
|
-
|
|
14
|
-
export declare const SYNTAX_PLUGINS: Rule[];
|
|
13
|
+
export { SYNTAX_PLUGINS } from './syntax-plugins.ts';
|
|
15
14
|
/**
|
|
16
15
|
* Rules that compare a translation with the source it was made from.
|
|
17
16
|
*
|
|
@@ -72,4 +71,3 @@ export declare const LINT_RULES: Rule[];
|
|
|
72
71
|
* position, which some of the orderings above depend on.
|
|
73
72
|
*/
|
|
74
73
|
export declare const TRANSLATION_CHECK_RULES: Rule[];
|
|
75
|
-
export {};
|
|
@@ -1,6 +1,3 @@
|
|
|
1
|
-
import remarkDirective from 'remark-directive';
|
|
2
|
-
import remarkFrontmatter from 'remark-frontmatter';
|
|
3
|
-
import remarkGfm from 'remark-gfm';
|
|
4
1
|
// @ts-expect-error -- https://github.com/ilyatitovich/remark-lint-code-block-split-list/issues/2
|
|
5
2
|
import remarkLintCodeBlockSplitList from 'remark-lint-code-block-split-list';
|
|
6
3
|
import remarkLintHeadingIncrement from 'remark-lint-heading-increment';
|
|
@@ -11,12 +8,9 @@ import remarkLintNoChinesePunctuationInNumber from 'remark-lint-no-chinese-punct
|
|
|
11
8
|
import remarkLintNoDuplicateHeadingsInSection from 'remark-lint-no-duplicate-headings-in-section';
|
|
12
9
|
import remarkLintNoHiddenTableCell from 'remark-lint-no-hidden-table-cell';
|
|
13
10
|
import { checkDeadLinks, chineseTypographyOnlyInChinese, headingAnchorFormat, noDeepHeading, noDeepList, noLegacyOSNames, noMultiOpenAPIPaths, noParagraphIndent, noUnmatchedAnchor, noUnopenedFrontmatter, noUnparsedEmphasis, noUnresolvedApiRef, site, tableSize, titleRequired, translationComponentMultiset, translationEchoedSource, translationFrontmatterPreservation, translationHeadingSequence, translationJsxAttributeParity, translationLengthRatio, translationLinkIsomorphism, translationUpToDate, translationUrlResidue, unitCase, } from "./index.js";
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
remarkFrontmatter,
|
|
18
|
-
remarkGfm,
|
|
19
|
-
];
|
|
11
|
+
// The syntax stack lives in its own module so `translation-parity` can parse
|
|
12
|
+
// the source side with it without importing the rules that import it back.
|
|
13
|
+
export { SYNTAX_PLUGINS } from "./syntax-plugins.js";
|
|
20
14
|
/**
|
|
21
15
|
* Rules that compare a translation with the source it was made from.
|
|
22
16
|
*
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
import type { Root } from 'mdast';
|
|
2
|
+
import type { Plugin } from 'unified';
|
|
3
|
+
/**
|
|
4
|
+
* The syntax extensions a document is parsed with, in one place.
|
|
5
|
+
*
|
|
6
|
+
* This exists because the `translation-parity` rules compare two trees, and for
|
|
7
|
+
* a comparison to mean anything both trees have to have been parsed the same
|
|
8
|
+
* way. They were not: the document being linted came through the lint pipeline
|
|
9
|
+
* (these plugins), while the source it was compared against was parsed with the
|
|
10
|
+
* site's `mdxProcessor`, which has no `remark-directive`. Anything directive
|
|
11
|
+
* syntax touches — a port in `http://host:8080`, a tag in `image:1.25`, a time
|
|
12
|
+
* in `10:30` — then became a `textDirective` on one side and stayed plain text
|
|
13
|
+
* on the other, and the rules reported the difference as damage the translator
|
|
14
|
+
* could not repair, because nothing it could write would make the two sides
|
|
15
|
+
* agree.
|
|
16
|
+
*
|
|
17
|
+
* `remark-directive` is in the list because the site renders with it
|
|
18
|
+
* (`plugins/directives`), so it is what the published page actually is.
|
|
19
|
+
*/
|
|
20
|
+
export declare const SYNTAX_PLUGINS: Plugin<[], never>[];
|
|
21
|
+
/** Parse-only view of the stack — the rules never stringify the source. */
|
|
22
|
+
export interface SyntaxProcessor {
|
|
23
|
+
parse: (content: string) => Root;
|
|
24
|
+
}
|
|
25
|
+
/**
|
|
26
|
+
* Parses a document exactly as the lint pipeline does — the reference side of
|
|
27
|
+
* every `translation-parity` comparison.
|
|
28
|
+
*/
|
|
29
|
+
export declare const syntaxProcessor: {
|
|
30
|
+
md: SyntaxProcessor;
|
|
31
|
+
mdx: SyntaxProcessor;
|
|
32
|
+
};
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
import remarkDirective from 'remark-directive';
|
|
2
|
+
import remarkFrontmatter from 'remark-frontmatter';
|
|
3
|
+
import remarkGfm from 'remark-gfm';
|
|
4
|
+
import remarkMdx from 'remark-mdx';
|
|
5
|
+
import remarkParse from 'remark-parse';
|
|
6
|
+
import { unified } from 'unified';
|
|
7
|
+
/**
|
|
8
|
+
* The syntax extensions a document is parsed with, in one place.
|
|
9
|
+
*
|
|
10
|
+
* This exists because the `translation-parity` rules compare two trees, and for
|
|
11
|
+
* a comparison to mean anything both trees have to have been parsed the same
|
|
12
|
+
* way. They were not: the document being linted came through the lint pipeline
|
|
13
|
+
* (these plugins), while the source it was compared against was parsed with the
|
|
14
|
+
* site's `mdxProcessor`, which has no `remark-directive`. Anything directive
|
|
15
|
+
* syntax touches — a port in `http://host:8080`, a tag in `image:1.25`, a time
|
|
16
|
+
* in `10:30` — then became a `textDirective` on one side and stayed plain text
|
|
17
|
+
* on the other, and the rules reported the difference as damage the translator
|
|
18
|
+
* could not repair, because nothing it could write would make the two sides
|
|
19
|
+
* agree.
|
|
20
|
+
*
|
|
21
|
+
* `remark-directive` is in the list because the site renders with it
|
|
22
|
+
* (`plugins/directives`), so it is what the published page actually is.
|
|
23
|
+
*/
|
|
24
|
+
export const SYNTAX_PLUGINS = [
|
|
25
|
+
remarkDirective,
|
|
26
|
+
remarkFrontmatter,
|
|
27
|
+
remarkGfm,
|
|
28
|
+
];
|
|
29
|
+
const build = (mdx) => {
|
|
30
|
+
let processor = unified().use(remarkParse);
|
|
31
|
+
for (const plugin of SYNTAX_PLUGINS) {
|
|
32
|
+
processor = processor.use(plugin);
|
|
33
|
+
}
|
|
34
|
+
if (mdx) {
|
|
35
|
+
processor = processor.use(remarkMdx);
|
|
36
|
+
}
|
|
37
|
+
return processor.freeze();
|
|
38
|
+
};
|
|
39
|
+
/**
|
|
40
|
+
* Parses a document exactly as the lint pipeline does — the reference side of
|
|
41
|
+
* every `translation-parity` comparison.
|
|
42
|
+
*/
|
|
43
|
+
export const syntaxProcessor = {
|
|
44
|
+
md: build(false),
|
|
45
|
+
mdx: build(true),
|
|
46
|
+
};
|
|
@@ -10,9 +10,11 @@ import { currentPair } from "./shared.js";
|
|
|
10
10
|
* language and not the other.
|
|
11
11
|
*/
|
|
12
12
|
const TRANSLATED_KEYS = new Set(['title', 'description']);
|
|
13
|
-
// `sourceSHA`
|
|
14
|
-
// from the translation —
|
|
15
|
-
|
|
13
|
+
// `sourceSHA` and `i18nSegments` are written by the translator and `i18n` is
|
|
14
|
+
// deliberately dropped from the translation — none of the three is a
|
|
15
|
+
// divergence. Leaving `i18nSegments` out of this set would report every
|
|
16
|
+
// translated document in the corpus, every time.
|
|
17
|
+
const TRANSLATOR_OWNED_KEYS = new Set(['sourceSHA', 'i18nSegments', 'i18n']);
|
|
16
18
|
export const translationFrontmatterPreservation = lintRule('doom-lint:translation-frontmatter-preservation', async (tree, vfile) => {
|
|
17
19
|
const pair = await currentPair(tree, vfile);
|
|
18
20
|
if (!pair) {
|
|
@@ -1,4 +1,12 @@
|
|
|
1
1
|
import type { Root } from 'mdast';
|
|
2
|
+
export declare const partitionAbsorbed: (missing: readonly string[], extra: readonly string[]) => {
|
|
3
|
+
absorbed: {
|
|
4
|
+
target: string;
|
|
5
|
+
suffix: string;
|
|
6
|
+
}[];
|
|
7
|
+
lost: string[];
|
|
8
|
+
unmatched: string[];
|
|
9
|
+
};
|
|
2
10
|
/**
|
|
3
11
|
* A translation points at exactly the same places its source points at.
|
|
4
12
|
*
|
|
@@ -4,6 +4,38 @@ const preview = (values) => values
|
|
|
4
4
|
.slice(0, 5)
|
|
5
5
|
.map((value) => `\`${value}\``)
|
|
6
6
|
.join(', ') + (values.length > 5 ? `, … (${values.length} total)` : '');
|
|
7
|
+
/**
|
|
8
|
+
* A bare URL ends where markdown says it ends, and markdown does not know
|
|
9
|
+
* about Chinese sentence punctuation.
|
|
10
|
+
*
|
|
11
|
+
* GFM closes an autolink literal at whitespace and drops *ASCII* trailing
|
|
12
|
+
* punctuation, so `see http://h:8080.` links to `http://h:8080`. A full-width
|
|
13
|
+
* stop is not ASCII: it is absorbed, and `例如 http://h:8080。` links to
|
|
14
|
+
* `http://h:8080。`, an address that does not exist. The translator writes
|
|
15
|
+
* correct Chinese and gets a broken link for it — and the two messages it used
|
|
16
|
+
* to get, "lost this target" and "points at that one", never said which of the
|
|
17
|
+
* two spellings was the one to change, so the repair loop could only guess.
|
|
18
|
+
*
|
|
19
|
+
* Only a non-ASCII suffix is read this way. `https://x.com` becoming
|
|
20
|
+
* `https://x.com/docs` is also a prefix, and that one is a real retarget.
|
|
21
|
+
*/
|
|
22
|
+
const isAbsorbedSuffix = (suffix) => suffix.length > 0 && suffix.codePointAt(0) > 0x7f;
|
|
23
|
+
export const partitionAbsorbed = (missing, extra) => {
|
|
24
|
+
const unmatched = [...extra];
|
|
25
|
+
const lost = [];
|
|
26
|
+
const absorbed = [];
|
|
27
|
+
for (const target of missing) {
|
|
28
|
+
const index = unmatched.findIndex((candidate) => candidate.startsWith(target) &&
|
|
29
|
+
isAbsorbedSuffix(candidate.slice(target.length)));
|
|
30
|
+
if (index === -1) {
|
|
31
|
+
lost.push(target);
|
|
32
|
+
continue;
|
|
33
|
+
}
|
|
34
|
+
absorbed.push({ target, suffix: unmatched[index].slice(target.length) });
|
|
35
|
+
unmatched.splice(index, 1);
|
|
36
|
+
}
|
|
37
|
+
return { absorbed, lost, unmatched };
|
|
38
|
+
};
|
|
7
39
|
/**
|
|
8
40
|
* A translation points at exactly the same places its source points at.
|
|
9
41
|
*
|
|
@@ -36,10 +68,14 @@ export const translationLinkIsomorphism = lintRule('doom-lint:translation-link-i
|
|
|
36
68
|
const expected = collectLinkTargets(pair.sourceTree, docDirInRoot(pair.sourceLang, pair.relativePath));
|
|
37
69
|
const actual = collectLinkTargets(tree, docDirInRoot(pair.targetLang, pair.relativePath));
|
|
38
70
|
const { missing, extra } = diffMultiset(expected, actual);
|
|
39
|
-
|
|
40
|
-
|
|
71
|
+
const { absorbed, lost, unmatched } = partitionAbsorbed(missing, extra);
|
|
72
|
+
for (const { target, suffix } of absorbed) {
|
|
73
|
+
vfile.message(`Translation link \`${target}${suffix}\` swallowed the \`${suffix}\` that follows it: a bare URL runs on until a space, and markdown only leaves out trailing ASCII punctuation. Write it as \`[${target}](${target})\` so the punctuation stays outside the link.`, tree);
|
|
74
|
+
}
|
|
75
|
+
if (lost.length) {
|
|
76
|
+
vfile.message(`Translation lost ${lost.length} link target(s) the source has: ${preview(lost)}.`, tree);
|
|
41
77
|
}
|
|
42
|
-
if (
|
|
43
|
-
vfile.message(`Translation points at ${
|
|
78
|
+
if (unmatched.length) {
|
|
79
|
+
vfile.message(`Translation points at ${unmatched.length} target(s) the source does not: ${preview(unmatched)}.`, tree);
|
|
44
80
|
}
|
|
45
81
|
});
|