@alauda/doom 2.6.1 → 2.7.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/cli/translate-call.d.ts +85 -0
- package/lib/cli/translate-call.js +162 -0
- package/lib/cli/translate-checker.d.ts +8 -0
- package/lib/cli/translate-diagnose.d.ts +45 -0
- package/lib/cli/translate-diagnose.js +118 -0
- package/lib/cli/translate-judge.d.ts +28 -3
- package/lib/cli/translate-judge.js +40 -7
- package/lib/cli/translate-mask.d.ts +13 -0
- package/lib/cli/translate-mask.js +19 -5
- package/lib/cli/translate-pipeline.d.ts +156 -0
- package/lib/cli/translate-pipeline.js +501 -0
- package/lib/cli/translate-repair-agent.d.ts +69 -0
- package/lib/cli/translate-repair-agent.js +241 -0
- package/lib/cli/translate-scratch.d.ts +9 -1
- package/lib/cli/translate-scratch.js +2 -2
- package/lib/cli/translate-segment-cache.d.ts +90 -0
- package/lib/cli/translate-segment-cache.js +301 -0
- package/lib/cli/translate-segment-check.d.ts +42 -0
- package/lib/cli/translate-segment-check.js +288 -0
- package/lib/cli/translate-segment.d.ts +307 -0
- package/lib/cli/translate-segment.js +625 -0
- package/lib/cli/translate.d.ts +14 -4
- package/lib/cli/translate.js +232 -52
- package/lib/remark-lint/no-unopened-frontmatter.js +5 -1
- package/lib/remark-lint/no-unparsed-emphasis.d.ts +13 -0
- package/lib/remark-lint/no-unparsed-emphasis.js +30 -5
- package/lib/remark-lint/translation-parity/frontmatter-preservation.js +5 -3
- package/lib/runtime/hooks/useTranslation.d.ts +1 -1
- package/lib/types.d.ts +101 -6
- package/package.json +1 -1
- package/lib/cli/translate-agent.d.ts +0 -81
- package/lib/cli/translate-agent.js +0 -450
package/lib/cli/translate.js
CHANGED
|
@@ -14,20 +14,33 @@ import { DEFAULT_COPY_ONLY_DIRECTORIES, Language, SUPPORTED_LANGUAGES, TITLE_TRA
|
|
|
14
14
|
import { OPTIONS_FILE, STORAGE_DIR, pathExists } from '../utils/index.js';
|
|
15
15
|
import { escapeMarkdownHeadingIds, getMatchedDocFilePaths, parseBoolean, stringifyMatter, translateCodeFile, } from './helpers.js';
|
|
16
16
|
import { loadConfig } from './load-config.js';
|
|
17
|
-
import {
|
|
17
|
+
import { createSegmentTranslator } from './translate-call.js';
|
|
18
18
|
import { checkTranslations, } from './translate-check.js';
|
|
19
|
-
import { createTranslationChecker
|
|
19
|
+
import { createTranslationChecker } from './translate-checker.js';
|
|
20
|
+
import { createDiagnoser } from './translate-diagnose.js';
|
|
20
21
|
import { createJudge } from './translate-judge.js';
|
|
21
22
|
import { CONCURRENCY_ENV, DEFAULT_CONCURRENCY, DEFAULT_REQUESTS_PER_MINUTE, REQUESTS_PER_MINUTE_ENV, createLimits, positiveIntFromEnv, } from './translate-limits.js';
|
|
22
23
|
import { maskAst } from './translate-mask.js';
|
|
23
|
-
import { DEFAULT_JUDGE_REASONING_EFFORT, DEFAULT_REASONING_EFFORT, createGateway, } from './translate-models.js';
|
|
24
|
+
import { DEFAULT_JUDGE_REASONING_EFFORT, DEFAULT_REASONING_EFFORT, createGateway, gatewayModel, } from './translate-models.js';
|
|
25
|
+
import { DEFAULT_MAX_SEGMENT_ATTEMPTS, translateDocument, } from './translate-pipeline.js';
|
|
26
|
+
import { createRepairAgent } from './translate-repair-agent.js';
|
|
24
27
|
import { resolveTerms } from './translate-terms.js';
|
|
28
|
+
/**
|
|
29
|
+
* How many readings a segment gets, and how many must agree.
|
|
30
|
+
*
|
|
31
|
+
* Three and two, against two-and-unanimous for a whole page. A segment is a
|
|
32
|
+
* short text, and a reviewer given less to go on has more room to find
|
|
33
|
+
* something to say; the third reading buys back the precision that unanimity
|
|
34
|
+
* gives when there is a whole document to be sure about.
|
|
35
|
+
*/
|
|
36
|
+
const DEFAULT_SEGMENT_DRAWS = 3;
|
|
37
|
+
const DEFAULT_SEGMENT_VOTES = 2;
|
|
25
38
|
/**
|
|
26
39
|
* The translation rules.
|
|
27
40
|
*
|
|
28
41
|
* Overridable per repository through `translate.systemPrompt`, which is why
|
|
29
42
|
* the placeholder discipline is *not* here: it is part of the harness's own
|
|
30
|
-
* prompt in `translate-
|
|
43
|
+
* prompt in `translate-call.ts`, where a repository cannot drop it by
|
|
31
44
|
* replacing this text.
|
|
32
45
|
*/
|
|
33
46
|
const DEFAULT_SYSTEM_PROMPT = `
|
|
@@ -75,17 +88,6 @@ These are additional requirements for the translation. They should be met along
|
|
|
75
88
|
"""
|
|
76
89
|
<% } %>
|
|
77
90
|
`.trim();
|
|
78
|
-
/** How many times findings are fed back before a document is failed. */
|
|
79
|
-
const DEFAULT_MAX_REPAIR_ROUNDS = 3;
|
|
80
|
-
/**
|
|
81
|
-
* Total assistant turns for one document, tool calls included.
|
|
82
|
-
*
|
|
83
|
-
* Separate from the repair rounds on purpose: a long document legitimately
|
|
84
|
-
* spends many turns reading and appending before it has anything to repair, so
|
|
85
|
-
* one number cannot be both the budget for producing a translation and the
|
|
86
|
-
* budget for fixing it. This one is a runaway guard, not a quality knob.
|
|
87
|
-
*/
|
|
88
|
-
const DEFAULT_MAX_TURNS = 60;
|
|
89
91
|
function extractFirstLevelHeading(content) {
|
|
90
92
|
const lines = content.split('\n');
|
|
91
93
|
for (const line of lines) {
|
|
@@ -127,6 +129,66 @@ const renderTranslationRules = async ({ source, target, sourceContent, options,
|
|
|
127
129
|
}, { async: true });
|
|
128
130
|
return { rules, terms: terms.pairs };
|
|
129
131
|
};
|
|
132
|
+
/** One segment, named the way a person reading a build log needs it named. */
|
|
133
|
+
const segmentName = (outcome) => `segment ${outcome.index + 1}` +
|
|
134
|
+
(outcome.label.line || outcome.label.heading
|
|
135
|
+
? ` (${[
|
|
136
|
+
outcome.label.line ? `line ${outcome.label.line}` : undefined,
|
|
137
|
+
outcome.label.heading ? `‹${outcome.label.heading}›` : undefined,
|
|
138
|
+
]
|
|
139
|
+
.filter(Boolean)
|
|
140
|
+
.join(' ')})`
|
|
141
|
+
: '');
|
|
142
|
+
/**
|
|
143
|
+
* Why a document failed, in one line.
|
|
144
|
+
*
|
|
145
|
+
* Three ways to fail, and they are told apart on purpose: a segment nothing
|
|
146
|
+
* could translate, a whole-document check that kept failing after the segments
|
|
147
|
+
* it named were redone, and a whole-document check nobody could attribute to a
|
|
148
|
+
* segment. They call for different things from whoever reads the log, and the
|
|
149
|
+
* design this replaced reported all of them as the same wall of problems.
|
|
150
|
+
*/
|
|
151
|
+
const describeFailure = (result) => {
|
|
152
|
+
const { failure } = result;
|
|
153
|
+
switch (failure?.kind) {
|
|
154
|
+
case 'unsplittable-block': {
|
|
155
|
+
return 'A single block is too large to translate and cannot be divided.';
|
|
156
|
+
}
|
|
157
|
+
case 'segment': {
|
|
158
|
+
return `${failure.segments.length} segment(s) never passed: ${failure.segments
|
|
159
|
+
.map((index) => segmentName(result.outcomes[index]))
|
|
160
|
+
.join(', ')}.`;
|
|
161
|
+
}
|
|
162
|
+
case 'assembly': {
|
|
163
|
+
return `The assembled document still failed its checks after ${result.assemblyRounds} round(s) of sending segments back.`;
|
|
164
|
+
}
|
|
165
|
+
case 'unlocatable': {
|
|
166
|
+
return 'The assembled document failed a check of the whole page that could not be attributed to any one segment.';
|
|
167
|
+
}
|
|
168
|
+
default: {
|
|
169
|
+
return 'The document did not pass its checks.';
|
|
170
|
+
}
|
|
171
|
+
}
|
|
172
|
+
};
|
|
173
|
+
/** Everything that was wrong with one failed document, grouped by segment. */
|
|
174
|
+
const describeFailedDocument = ({ file, result, diagnosis, }) => {
|
|
175
|
+
const lines = [`\n${cyan(file)} — ${describeFailure(result)}`];
|
|
176
|
+
const failed = result.failure?.kind === 'segment'
|
|
177
|
+
? result.failure.segments.map((index) => result.outcomes[index])
|
|
178
|
+
: [];
|
|
179
|
+
for (const outcome of failed) {
|
|
180
|
+
lines.push(` ${segmentName(outcome)}, after ${outcome.attempts} attempt(s):`, ...outcome.findings.map((finding) => ` ${red(finding.rule)} ${finding.reason}`));
|
|
181
|
+
}
|
|
182
|
+
if (failed.length === 0) {
|
|
183
|
+
lines.push(...result.findings
|
|
184
|
+
.filter((finding) => finding.blocking !== false)
|
|
185
|
+
.map((finding) => ` ${red(finding.rule)}${finding.line ? ` (line ${finding.line})` : ''} ${finding.reason}`));
|
|
186
|
+
}
|
|
187
|
+
if (diagnosis) {
|
|
188
|
+
lines.push('', ` ${cyan('What this looks like')} (written by a model from the evidence above; advisory, and not part of the verdict):`, ...diagnosis.split('\n').map((line) => ` ${line}`));
|
|
189
|
+
}
|
|
190
|
+
return lines.join('\n');
|
|
191
|
+
};
|
|
130
192
|
const supportedLanguages = SUPPORTED_LANGUAGES.join(', ');
|
|
131
193
|
export const translateCommand = new Command('translate')
|
|
132
194
|
.description('Translate the documentation')
|
|
@@ -220,6 +282,25 @@ export const translateCommand = new Command('translate')
|
|
|
220
282
|
}
|
|
221
283
|
}
|
|
222
284
|
const translateOptions = config.translate ?? {};
|
|
285
|
+
// Two options were removed when the whole-document loop was, and both of
|
|
286
|
+
// them named a budget that no longer exists. Ignoring them quietly would
|
|
287
|
+
// leave a repository believing it had configured something — which is the
|
|
288
|
+
// exact failure mode this rewrite exists to remove, in miniature.
|
|
289
|
+
const removed = {
|
|
290
|
+
maxRepairRounds: '`maxSegmentAttempts` (per segment) and `maxAssemblyRounds` (per document)',
|
|
291
|
+
maxTurns: '`repairAgent.maxTurns`, which now bounds only the repair agent',
|
|
292
|
+
};
|
|
293
|
+
const usedRemoved = Object.keys(removed).filter((key) => key in translateOptions);
|
|
294
|
+
if (usedRemoved.length > 0) {
|
|
295
|
+
logger.error(`\`translate\` no longer has ${usedRemoved
|
|
296
|
+
.map((key) => `\`${cyan(key)}\``)
|
|
297
|
+
.join(' or ')}: a translation is now produced a segment at a time, so a budget for the whole document has nothing to bound.\n` +
|
|
298
|
+
usedRemoved
|
|
299
|
+
.map((key) => ` ${cyan(key)} → replaced by ${removed[key]}`)
|
|
300
|
+
.join('\n'));
|
|
301
|
+
process.exitCode = 1;
|
|
302
|
+
return;
|
|
303
|
+
}
|
|
223
304
|
// Built once: the provider wiring, and the processors the rules run in.
|
|
224
305
|
// Missing credentials fail here, by name, rather than once per file as a
|
|
225
306
|
// wall of unrelated stream errors.
|
|
@@ -242,22 +323,57 @@ export const translateCommand = new Command('translate')
|
|
|
242
323
|
concurrency,
|
|
243
324
|
requestsPerMinute,
|
|
244
325
|
});
|
|
245
|
-
const maxRepairRounds = translateOptions.maxRepairRounds ?? DEFAULT_MAX_REPAIR_ROUNDS;
|
|
246
|
-
const maxTurns = translateOptions.maxTurns ?? DEFAULT_MAX_TURNS;
|
|
247
326
|
const reasoningEffort = translateOptions.reasoningEffort ?? DEFAULT_REASONING_EFFORT;
|
|
248
|
-
const
|
|
249
|
-
|
|
250
|
-
|
|
327
|
+
const maxSegmentAttempts = translateOptions.maxSegmentAttempts ?? DEFAULT_MAX_SEGMENT_ATTEMPTS;
|
|
328
|
+
const judgeEnabled = translateOptions.judge?.enabled !== false;
|
|
329
|
+
const judgeReasoning = translateOptions.judge?.reasoningEffort ?? DEFAULT_JUDGE_REASONING_EFFORT;
|
|
330
|
+
// Two reviewers, because they are asked two different questions. The first
|
|
331
|
+
// reads one segment against its source and can stop it being frozen; the
|
|
332
|
+
// second reads the finished page for what no segment can see — a term that
|
|
333
|
+
// drifted between sections, a join that reads badly — and only ever
|
|
334
|
+
// reports.
|
|
335
|
+
const segmentJudge = judgeEnabled
|
|
336
|
+
? createJudge({
|
|
251
337
|
models: gateway.models,
|
|
252
338
|
model: gateway.judgeModel,
|
|
253
|
-
reasoningEffort:
|
|
254
|
-
|
|
339
|
+
reasoningEffort: judgeReasoning,
|
|
340
|
+
draws: translateOptions.judge?.segmentDraws ?? DEFAULT_SEGMENT_DRAWS,
|
|
341
|
+
votes: translateOptions.judge?.segmentVotes ?? DEFAULT_SEGMENT_VOTES,
|
|
342
|
+
limit: modelCallLimit,
|
|
343
|
+
})
|
|
344
|
+
: undefined;
|
|
345
|
+
const documentJudge = judgeEnabled && translateOptions.fullDocJudge !== false
|
|
346
|
+
? createJudge({
|
|
347
|
+
models: gateway.models,
|
|
348
|
+
model: gateway.judgeModel,
|
|
349
|
+
reasoningEffort: judgeReasoning,
|
|
255
350
|
draws: translateOptions.judge?.draws,
|
|
256
351
|
limit: modelCallLimit,
|
|
352
|
+
})
|
|
353
|
+
: undefined;
|
|
354
|
+
// A segment that repeated attempts could not fix goes to an agent that can
|
|
355
|
+
// only edit. It may be a stronger model than the one doing the translating:
|
|
356
|
+
// this path is rare and it is the hard one.
|
|
357
|
+
const repairModelId = translateOptions.repairAgent?.model;
|
|
358
|
+
const repairModel = repairModelId && repairModelId !== gateway.model.id
|
|
359
|
+
? gatewayModel({
|
|
360
|
+
id: repairModelId,
|
|
361
|
+
baseUrl: gateway.baseUrl,
|
|
362
|
+
contextWindow: translateOptions.contextWindow,
|
|
363
|
+
maxOutputTokens: translateOptions.maxOutputTokens,
|
|
364
|
+
})
|
|
365
|
+
: gateway.model;
|
|
366
|
+
const diagnoser = translateOptions.diagnose?.enabled === false
|
|
367
|
+
? undefined
|
|
368
|
+
: createDiagnoser({
|
|
369
|
+
models: gateway.models,
|
|
370
|
+
model: gateway.model,
|
|
371
|
+
reasoningEffort: translateOptions.diagnose?.reasoningEffort,
|
|
372
|
+
limit: modelCallLimit,
|
|
257
373
|
});
|
|
258
|
-
logger.info(`Translating with \`${cyan(gateway.model.id)}\` (reasoning ${cyan(reasoningEffort)}), up to ${cyan(String(
|
|
259
|
-
`${cyan(String(concurrency))} at a time, at most ${cyan(String(requestsPerMinute))} model request(s) a minute. ` +
|
|
260
|
-
(
|
|
374
|
+
logger.info(`Translating with \`${cyan(gateway.model.id)}\` (reasoning ${cyan(reasoningEffort)}), a segment at a time, up to ${cyan(String(maxSegmentAttempts))} attempt(s) per segment. ` +
|
|
375
|
+
`${cyan(String(concurrency))} document(s) at a time, at most ${cyan(String(requestsPerMinute))} model request(s) a minute. ` +
|
|
376
|
+
(segmentJudge
|
|
261
377
|
? `Reviewed by \`${cyan(gateway.judgeModel.id)}\`.`
|
|
262
378
|
: `${red('Judge disabled')} — only the deterministic checks are running.`));
|
|
263
379
|
const failures = [];
|
|
@@ -273,22 +389,42 @@ export const translateCommand = new Command('translate')
|
|
|
273
389
|
.update(sourceContent)
|
|
274
390
|
.digest('hex');
|
|
275
391
|
const targetFilePath = sourceFilePath.replace(sourceDir, targetDir);
|
|
392
|
+
/**
|
|
393
|
+
* The translation already on disk, when there is one.
|
|
394
|
+
*
|
|
395
|
+
* Two layers use it. `sourceSHA` skips a document nobody touched at
|
|
396
|
+
* all; below that, the segments whose source did not change are reused
|
|
397
|
+
* rather than retranslated, so a one-line edit costs one segment
|
|
398
|
+
* instead of a whole page. `--force` turns off both, which is what
|
|
399
|
+
* "translate it again" has always meant.
|
|
400
|
+
*/
|
|
401
|
+
let previous;
|
|
276
402
|
if (await pathExists(targetFilePath, 'file')) {
|
|
277
403
|
const targetContent = await fs.readFile(targetFilePath, 'utf-8');
|
|
278
|
-
const
|
|
279
|
-
|
|
404
|
+
const parsedTarget = matter(targetContent);
|
|
405
|
+
const targetFrontmatter = parsedTarget.data;
|
|
280
406
|
if (!force && targetFrontmatter.sourceSHA === sourceSHA) {
|
|
281
407
|
return;
|
|
282
408
|
}
|
|
409
|
+
if (!force && translateOptions.segmentCache !== false) {
|
|
410
|
+
previous = {
|
|
411
|
+
body: parsedTarget.content,
|
|
412
|
+
record: targetFrontmatter.i18nSegments,
|
|
413
|
+
};
|
|
414
|
+
}
|
|
283
415
|
}
|
|
284
416
|
const shouldCopyOnly = copyOnlyFilePathsSet.has(sourceFilePath);
|
|
285
417
|
const sourceRelativePath = path.relative(docsDir, sourceFilePath);
|
|
286
418
|
const targetRelativePath = path.relative(docsDir, targetFilePath);
|
|
287
419
|
const targetBase = path.dirname(targetFilePath);
|
|
288
420
|
/** The document exactly as it will be written, given a translated body. */
|
|
289
|
-
const compose = (restored) => {
|
|
421
|
+
const compose = (restored, cache) => {
|
|
290
422
|
const newFrontmatter = { ...sourceFrontmatter, sourceSHA };
|
|
291
423
|
delete newFrontmatter.i18n;
|
|
424
|
+
delete newFrontmatter.i18nSegments;
|
|
425
|
+
if (cache) {
|
|
426
|
+
newFrontmatter.i18nSegments = cache;
|
|
427
|
+
}
|
|
292
428
|
const { data, content } = matter(restored);
|
|
293
429
|
const typedData = data;
|
|
294
430
|
if (typedData.title && typeof typedData.title === 'string') {
|
|
@@ -349,42 +485,91 @@ export const translateCommand = new Command('translate')
|
|
|
349
485
|
options: translateOptions,
|
|
350
486
|
additionalPrompts: sourceFrontmatter.i18n?.additionalPrompts,
|
|
351
487
|
});
|
|
352
|
-
const
|
|
488
|
+
const repairer = translateOptions.repairAgent?.enabled === false
|
|
489
|
+
? undefined
|
|
490
|
+
: createRepairAgent({
|
|
491
|
+
models: gateway.models,
|
|
492
|
+
model: repairModel,
|
|
493
|
+
reasoningEffort: translateOptions.repairAgent?.reasoningEffort,
|
|
494
|
+
scratchDir,
|
|
495
|
+
extension: isMdx ? '.mdx' : '.md',
|
|
496
|
+
maxTurns: translateOptions.repairAgent?.maxTurns,
|
|
497
|
+
limit: modelCallLimit,
|
|
498
|
+
onProgress: (message) => {
|
|
499
|
+
logger.info(`${cyan(sourceRelativePath)}: ${message}`);
|
|
500
|
+
},
|
|
501
|
+
});
|
|
502
|
+
const translator = createSegmentTranslator({
|
|
503
|
+
models: gateway.models,
|
|
504
|
+
model: gateway.model,
|
|
505
|
+
reasoningEffort,
|
|
506
|
+
translationRules: rules,
|
|
507
|
+
sourceLanguage: Language[source],
|
|
508
|
+
targetLanguage: Language[target],
|
|
509
|
+
processor,
|
|
510
|
+
limit: modelCallLimit,
|
|
511
|
+
});
|
|
512
|
+
const result = await translateDocument({
|
|
513
|
+
tree: normalizedAst,
|
|
353
514
|
maskedSource,
|
|
354
515
|
maskEntries,
|
|
355
516
|
processor,
|
|
356
517
|
compose,
|
|
357
518
|
targetPath: targetFilePath,
|
|
358
519
|
sourceLabel: sourceRelativePath,
|
|
359
|
-
source,
|
|
360
|
-
target,
|
|
361
|
-
|
|
362
|
-
terms,
|
|
520
|
+
sourceLanguage: Language[source],
|
|
521
|
+
targetLanguage: Language[target],
|
|
522
|
+
translator,
|
|
363
523
|
checker,
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
524
|
+
segmentJudge,
|
|
525
|
+
documentJudge,
|
|
526
|
+
terms,
|
|
527
|
+
segmentCap: translateOptions.segmentCap,
|
|
528
|
+
segmentHardCap: translateOptions.segmentHardCap,
|
|
529
|
+
segmentFloor: translateOptions.segmentFloor,
|
|
530
|
+
maxSegmentAttempts,
|
|
531
|
+
maxAssemblyRounds: translateOptions.maxAssemblyRounds,
|
|
532
|
+
contextTail: translateOptions.contextTail,
|
|
533
|
+
repairer,
|
|
534
|
+
previous,
|
|
372
535
|
onProgress: (message) => {
|
|
373
536
|
logger.info(message);
|
|
374
537
|
},
|
|
375
538
|
});
|
|
376
539
|
if (!result.document) {
|
|
377
|
-
|
|
378
|
-
|
|
540
|
+
// Explained before it is reported, so the log carries the analysis
|
|
541
|
+
// next to the failure rather than in a separate place nobody
|
|
542
|
+
// correlates. Advisory throughout: it cannot change this outcome.
|
|
543
|
+
const diagnosis = await diagnoser?.diagnose({
|
|
544
|
+
sourceLabel: sourceRelativePath,
|
|
545
|
+
failure: result.failure,
|
|
546
|
+
outcomes: result.outcomes,
|
|
379
547
|
findings: result.findings,
|
|
380
|
-
result,
|
|
548
|
+
assemblyRounds: result.assemblyRounds,
|
|
381
549
|
});
|
|
382
|
-
|
|
550
|
+
failures.push({ file: targetRelativePath, result, diagnosis });
|
|
551
|
+
logger.error(`${cyan(targetRelativePath)} did not pass its checks. ${describeFailure(result)}`);
|
|
383
552
|
return;
|
|
384
553
|
}
|
|
385
554
|
await fs.mkdir(targetBase, { recursive: true });
|
|
386
555
|
await fs.writeFile(targetFilePath, result.document);
|
|
387
|
-
|
|
556
|
+
const cached = result.outcomes.filter((outcome) => outcome.status === 'cached').length;
|
|
557
|
+
const repaired = result.outcomes.filter((outcome) => outcome.status === 'repaired').length;
|
|
558
|
+
logger.info(`${cyan(sourceRelativePath)} translated to ${cyan(targetRelativePath)} ` +
|
|
559
|
+
`(${result.outcomes.length} segment(s)` +
|
|
560
|
+
(cached ? `, ${cached} reused` : '') +
|
|
561
|
+
(repaired ? `, ${repaired} repaired` : '') +
|
|
562
|
+
// Translation calls, named as such. The reviewers, the repair
|
|
563
|
+
// agent and the diagnosis all cost model calls too, and none of
|
|
564
|
+
// them are counted here: their judges are shared by every
|
|
565
|
+
// document being translated at once, so there is no per-document
|
|
566
|
+
// number to print. Calling this "model calls" made it look like
|
|
567
|
+
// there was.
|
|
568
|
+
`, ${translator.calls()} translation call(s)` +
|
|
569
|
+
(result.assemblyRounds
|
|
570
|
+
? `, ${result.assemblyRounds} assembly round(s)`
|
|
571
|
+
: '') +
|
|
572
|
+
')');
|
|
388
573
|
});
|
|
389
574
|
}));
|
|
390
575
|
await fs.rm(scratchDir, { recursive: true, force: true });
|
|
@@ -397,12 +582,7 @@ export const translateCommand = new Command('translate')
|
|
|
397
582
|
// does not go red for a document that was silently skipped.
|
|
398
583
|
if (failures.length > 0) {
|
|
399
584
|
logger.error(`${red(String(failures.length))} document(s) could not be translated to a state that passes their checks. Nothing will be uploaded.\n` +
|
|
400
|
-
failures
|
|
401
|
-
.map(({ file, findings, result }) => `\n${cyan(file)} — ${findings.length} unresolved problem(s) after ${result?.repairRounds ?? 0} repair round(s):\n` +
|
|
402
|
-
findings
|
|
403
|
-
.map((finding) => ` ${red(finding.rule)}${finding.line ? ` (line ${finding.line})` : ''} ${finding.reason}`)
|
|
404
|
-
.join('\n'))
|
|
405
|
-
.join('\n'));
|
|
585
|
+
failures.map(describeFailedDocument).join('\n'));
|
|
406
586
|
process.exitCode = 1;
|
|
407
587
|
return;
|
|
408
588
|
}
|
|
@@ -33,7 +33,10 @@ import { visit } from 'unist-util-visit';
|
|
|
33
33
|
/**
|
|
34
34
|
* The keys doom's frontmatter actually uses, counted across acp-docs rather
|
|
35
35
|
* than remembered: weight 797 · sourceSHA 194 · title 132 · i18n 125 ·
|
|
36
|
-
* queries 73 · description 21 · category 5 · author 4.
|
|
36
|
+
* queries 73 · description 21 · category 5 · author 4. `i18nSegments` is
|
|
37
|
+
* written by `doom translate` and so appears in no count taken before it
|
|
38
|
+
* existed; it is here because a translation is exactly the kind of file whose
|
|
39
|
+
* frontmatter a model can lose the opening `---` from.
|
|
37
40
|
*
|
|
38
41
|
* Deliberately a list and not `\w+:` — a heading really can begin "Note:" or
|
|
39
42
|
* "Step 1:", and reporting those would be a rule nobody keeps.
|
|
@@ -43,6 +46,7 @@ const FRONTMATTER_KEYS = [
|
|
|
43
46
|
'category',
|
|
44
47
|
'description',
|
|
45
48
|
'i18n',
|
|
49
|
+
'i18nSegments',
|
|
46
50
|
'queries',
|
|
47
51
|
'sourceSHA',
|
|
48
52
|
'title',
|
|
@@ -1,2 +1,15 @@
|
|
|
1
1
|
import type { Root } from 'mdast';
|
|
2
|
+
/** What one of these looks like to a person, wherever it is reported. */
|
|
3
|
+
export declare const explainUnparsedEmphasis: (delimiter: string, window: string) => string;
|
|
4
|
+
/**
|
|
5
|
+
* The first unparsed delimiter in a string, if there is one.
|
|
6
|
+
*
|
|
7
|
+
* `maskedText` narrows the test to `**`; see `UNPARSED_STRONG`.
|
|
8
|
+
*/
|
|
9
|
+
export declare const findUnparsedEmphasis: (value: string, { maskedText }?: {
|
|
10
|
+
maskedText?: boolean;
|
|
11
|
+
}) => {
|
|
12
|
+
delimiter: string;
|
|
13
|
+
window: string;
|
|
14
|
+
} | undefined;
|
|
2
15
|
export declare const noUnparsedEmphasis: import("unified-lint-rule").Plugin<Root, unknown>;
|
|
@@ -34,15 +34,40 @@ import { visitParents } from 'unist-util-visit-parents';
|
|
|
34
34
|
* accident.
|
|
35
35
|
*/
|
|
36
36
|
const UNPARSED_EMPHASIS = /\*\*|(?<![\p{L}\p{N}])__|__(?![\p{L}\p{N}])/u;
|
|
37
|
+
/**
|
|
38
|
+
* The `**` half of the test on its own.
|
|
39
|
+
*
|
|
40
|
+
* `doom translate` runs this check a segment at a time, before the placeholders
|
|
41
|
+
* are put back, and a masked segment is full of `__DOOM_TR_ICODE_0__`. The `__`
|
|
42
|
+
* half would report every one of them. `**` never appears in a placeholder, so
|
|
43
|
+
* the half that matters for the defect this rule was written for is also the
|
|
44
|
+
* half that is safe to run on masked text.
|
|
45
|
+
*/
|
|
46
|
+
const UNPARSED_STRONG = /\*\*/u;
|
|
47
|
+
/** What one of these looks like to a person, wherever it is reported. */
|
|
48
|
+
export const explainUnparsedEmphasis = (delimiter, window) => `\`${delimiter}\` is printed to the page here, not read as emphasis: \`${window}\`. ` +
|
|
49
|
+
`Emphasis delimiters must sit directly against the text they emphasise — \`**text**\`, not \`** text **\` and not \`**text:**more\`.`;
|
|
50
|
+
/**
|
|
51
|
+
* The first unparsed delimiter in a string, if there is one.
|
|
52
|
+
*
|
|
53
|
+
* `maskedText` narrows the test to `**`; see `UNPARSED_STRONG`.
|
|
54
|
+
*/
|
|
55
|
+
export const findUnparsedEmphasis = (value, { maskedText = false } = {}) => {
|
|
56
|
+
const match = (maskedText ? UNPARSED_STRONG : UNPARSED_EMPHASIS).exec(value);
|
|
57
|
+
return match
|
|
58
|
+
? { delimiter: match[0], window: preview(value, match.index) }
|
|
59
|
+
: undefined;
|
|
60
|
+
};
|
|
37
61
|
export const noUnparsedEmphasis = lintRule('doom-lint:no-unparsed-emphasis', (root, vfile) => {
|
|
38
62
|
visitParents(root, 'text', (node, parents) => {
|
|
39
|
-
const
|
|
40
|
-
if (!
|
|
63
|
+
const found = findUnparsedEmphasis(node.value);
|
|
64
|
+
if (!found) {
|
|
41
65
|
return;
|
|
42
66
|
}
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
67
|
+
vfile.message(explainUnparsedEmphasis(found.delimiter, found.window), {
|
|
68
|
+
ancestors: [...parents, node],
|
|
69
|
+
place: pointStart(node),
|
|
70
|
+
});
|
|
46
71
|
});
|
|
47
72
|
});
|
|
48
73
|
/** A short window around the offending delimiter, for a person reading the log. */
|
|
@@ -10,9 +10,11 @@ import { currentPair } from "./shared.js";
|
|
|
10
10
|
* language and not the other.
|
|
11
11
|
*/
|
|
12
12
|
const TRANSLATED_KEYS = new Set(['title', 'description']);
|
|
13
|
-
// `sourceSHA`
|
|
14
|
-
// from the translation —
|
|
15
|
-
|
|
13
|
+
// `sourceSHA` and `i18nSegments` are written by the translator and `i18n` is
|
|
14
|
+
// deliberately dropped from the translation — none of the three is a
|
|
15
|
+
// divergence. Leaving `i18nSegments` out of this set would report every
|
|
16
|
+
// translated document in the corpus, every time.
|
|
17
|
+
const TRANSLATOR_OWNED_KEYS = new Set(['sourceSHA', 'i18nSegments', 'i18n']);
|
|
16
18
|
export const translationFrontmatterPreservation = lintRule('doom-lint:translation-frontmatter-preservation', async (tree, vfile) => {
|
|
17
19
|
const pair = await currentPair(tree, vfile);
|
|
18
20
|
if (!pair) {
|
|
@@ -1,3 +1,3 @@
|
|
|
1
1
|
import type { Language } from '../../shared/index.js';
|
|
2
2
|
export declare const useLang: () => Language;
|
|
3
|
-
export declare const useTranslation: () => (key: "function" | "name" | "description" | "delete" | "type" | "parameters" | "required" | "toc" | "create" | "update" | "password" | "
|
|
3
|
+
export declare const useTranslation: () => (key: "function" | "name" | "description" | "delete" | "type" | "parameters" | "required" | "toc" | "create" | "update" | "password" | "close" | "collapse_all" | "expand_all" | "crd_no_schema" | "action" | "view" | "chinese" | "english" | "property" | "specification" | "api_endpoints" | "available_api_endpoints" | "http_method" | "http_responses" | "http_code" | "response_body" | "query_parameters" | "body_parameters" | "parameter" | "common_parameters" | "request_body" | "response" | "chinese_bad_cases" | "english_bad_cases" | "download_pdf" | "view_docs_as_pdf" | "ai_assistant" | "exclamation" | "semicolon" | "period" | "colon" | "user_login" | "user_login_tip1" | "user_login_tip2" | "customer_portal_global" | "customer_portal_china" | "account_id" | "username" | "captcha" | "login" | "current_field" | "field_required" | "ERR_INTERNET_DISCONNECTED" | "AccountDisabledError" | "CaptchaError" | "LoginError" | "NetworkError" | "TenantError" | "TenantNotFoundError" | "logout" | "hi_there" | "ai_assistant_tip" | "ai_assistant_placeholder" | "max_chars_exceeded_tip" | "thinking" | "thinking_process" | "referenced_doc_links" | "view_more_related_docs" | "view_less_related_docs" | "new_chat" | "show_more" | "show_less" | "products" | "all_product_documentation" | "explore_doc_all_products") => string;
|
package/lib/types.d.ts
CHANGED
|
@@ -32,15 +32,106 @@ export interface TranslateOptions {
|
|
|
32
32
|
* then to the version this was measured against.
|
|
33
33
|
*/
|
|
34
34
|
model?: string;
|
|
35
|
-
/**
|
|
35
|
+
/**
|
|
36
|
+
* Reasoning effort for translating a segment. Defaults to `low`.
|
|
37
|
+
*
|
|
38
|
+
* The main path is deliberately cheap: a segment is a small, well-specified
|
|
39
|
+
* job, and the reasoning is spent where it earns something — on repairing a
|
|
40
|
+
* segment that three attempts could not fix, and on explaining a failure.
|
|
41
|
+
*/
|
|
36
42
|
reasoningEffort?: 'minimal' | 'low' | 'medium' | 'high' | 'xhigh' | 'max';
|
|
37
43
|
/**
|
|
38
|
-
*
|
|
39
|
-
*
|
|
44
|
+
* Largest segment sent to the model, in masked characters. Defaults to 16000.
|
|
45
|
+
*
|
|
46
|
+
* The single most important number here: it bounds how much has to be right
|
|
47
|
+
* at once. Raising it trades a lower chance of any one segment passing for
|
|
48
|
+
* fewer calls; lowering it costs context and coherence.
|
|
49
|
+
*/
|
|
50
|
+
segmentCap?: number;
|
|
51
|
+
/**
|
|
52
|
+
* Size at which an indivisible block fails the document. Defaults to 48000.
|
|
53
|
+
*
|
|
54
|
+
* A block over `segmentCap` that cannot be divided — a huge table or list —
|
|
55
|
+
* becomes an oversized segment of its own. This is where that stops being
|
|
56
|
+
* reasonable, and the document has to be edited instead.
|
|
57
|
+
*/
|
|
58
|
+
segmentHardCap?: number;
|
|
59
|
+
/**
|
|
60
|
+
* Size a segment must reach before a heading may end it. Defaults to 2000.
|
|
61
|
+
*
|
|
62
|
+
* Without it, a page of one-line sections becomes a page of one-line
|
|
63
|
+
* segments, each its own model call and judge reading for no benefit.
|
|
64
|
+
*/
|
|
65
|
+
segmentFloor?: number;
|
|
66
|
+
/**
|
|
67
|
+
* How many times a segment is asked for before it is escalated to the repair
|
|
68
|
+
* agent. Defaults to 3 — the first go plus two with the findings attached.
|
|
69
|
+
*/
|
|
70
|
+
maxSegmentAttempts?: number;
|
|
71
|
+
/**
|
|
72
|
+
* How many times the assembled document may send segments back to be redone.
|
|
73
|
+
* Defaults to 2.
|
|
74
|
+
*/
|
|
75
|
+
maxAssemblyRounds?: number;
|
|
76
|
+
/**
|
|
77
|
+
* Lines of the previous segment's translation shown to the next one, so the
|
|
78
|
+
* voice and the wording carry across a boundary. Defaults to 20; `0` turns it
|
|
79
|
+
* off.
|
|
80
|
+
*/
|
|
81
|
+
contextTail?: number;
|
|
82
|
+
/**
|
|
83
|
+
* Reuse the segments whose source has not changed, from the translation
|
|
84
|
+
* already on disk. Defaults to `true`.
|
|
85
|
+
*
|
|
86
|
+
* Turning it off makes every changed document a full retranslation. Nothing
|
|
87
|
+
* is trusted unverified either way: a reused segment has to still account for
|
|
88
|
+
* exactly the content the current source expects, or it is retranslated.
|
|
89
|
+
*/
|
|
90
|
+
segmentCache?: boolean;
|
|
91
|
+
/**
|
|
92
|
+
* Read the assembled page as a whole, after the segments have passed.
|
|
93
|
+
* Defaults to `true`.
|
|
94
|
+
*
|
|
95
|
+
* Never blocking. It is there for what a segment cannot see — a term that
|
|
96
|
+
* drifted between sections, a join that reads badly — and those belong in a
|
|
97
|
+
* report, not in a gate.
|
|
98
|
+
*/
|
|
99
|
+
fullDocJudge?: boolean;
|
|
100
|
+
/**
|
|
101
|
+
* The last resort for a segment that repeated attempts could not fix.
|
|
102
|
+
*
|
|
103
|
+
* An agent with `read`, `edit` and `check` and deliberately no way to replace
|
|
104
|
+
* a file: it works inside one segment, and the reason it cannot rewrite
|
|
105
|
+
* everything is the tools it has rather than the instructions it is given.
|
|
40
106
|
*/
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
107
|
+
repairAgent?: {
|
|
108
|
+
/** Defaults to `true`. */
|
|
109
|
+
enabled?: boolean;
|
|
110
|
+
/** Defaults to the translator's model. Set it to the strongest one available. */
|
|
111
|
+
model?: string;
|
|
112
|
+
/** Defaults to `high` — this path is rare and hard, which is what reasoning is for. */
|
|
113
|
+
reasoningEffort?: 'minimal' | 'low' | 'medium' | 'high' | 'xhigh' | 'max';
|
|
114
|
+
/**
|
|
115
|
+
* Runaway guard, in agent turns. Defaults to 40.
|
|
116
|
+
*
|
|
117
|
+
* A fixed number works here and did not for whole documents: a segment's
|
|
118
|
+
* size has an upper bound, so the ratio of budget to work is a constant.
|
|
119
|
+
*/
|
|
120
|
+
maxTurns?: number;
|
|
121
|
+
};
|
|
122
|
+
/**
|
|
123
|
+
* The failure analysis attached to a document that could not be translated.
|
|
124
|
+
*
|
|
125
|
+
* One model call over the evidence already collected, written for whoever
|
|
126
|
+
* reads the build log. Purely advisory: it never changes whether the build
|
|
127
|
+
* passes.
|
|
128
|
+
*/
|
|
129
|
+
diagnose?: {
|
|
130
|
+
/** Defaults to `true`. */
|
|
131
|
+
enabled?: boolean;
|
|
132
|
+
/** Defaults to `high`. */
|
|
133
|
+
reasoningEffort?: 'minimal' | 'low' | 'medium' | 'high' | 'xhigh' | 'max';
|
|
134
|
+
};
|
|
44
135
|
/**
|
|
45
136
|
* Context the translator plans for, in tokens. Deliberately lower than what
|
|
46
137
|
* the gateway offers: planning for less costs a re-read, planning for more
|
|
@@ -76,6 +167,10 @@ export interface TranslateOptions {
|
|
|
76
167
|
reasoningEffort?: 'minimal' | 'low' | 'medium' | 'high' | 'xhigh' | 'max';
|
|
77
168
|
/** How many independent readings must agree before a finding counts. Defaults to 2. */
|
|
78
169
|
draws?: number;
|
|
170
|
+
/** Readings taken of a single segment. Defaults to 3. */
|
|
171
|
+
segmentDraws?: number;
|
|
172
|
+
/** How many of those must agree before a segment finding counts. Defaults to 2. */
|
|
173
|
+
segmentVotes?: number;
|
|
79
174
|
/**
|
|
80
175
|
* Turns the judge off.
|
|
81
176
|
*
|