@alauda/doom 2.6.0 → 2.7.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (40) hide show
  1. package/lib/cli/translate-call.d.ts +85 -0
  2. package/lib/cli/translate-call.js +149 -0
  3. package/lib/cli/translate-checker.d.ts +8 -0
  4. package/lib/cli/translate-diagnose.d.ts +45 -0
  5. package/lib/cli/translate-diagnose.js +118 -0
  6. package/lib/cli/translate-judge.d.ts +28 -3
  7. package/lib/cli/translate-judge.js +40 -7
  8. package/lib/cli/translate-mask.d.ts +13 -0
  9. package/lib/cli/translate-mask.js +19 -5
  10. package/lib/cli/translate-pipeline.d.ts +144 -0
  11. package/lib/cli/translate-pipeline.js +489 -0
  12. package/lib/cli/translate-repair-agent.d.ts +69 -0
  13. package/lib/cli/translate-repair-agent.js +241 -0
  14. package/lib/cli/translate-scratch.d.ts +9 -1
  15. package/lib/cli/translate-scratch.js +2 -2
  16. package/lib/cli/translate-segment-cache.d.ts +90 -0
  17. package/lib/cli/translate-segment-cache.js +301 -0
  18. package/lib/cli/translate-segment-check.d.ts +42 -0
  19. package/lib/cli/translate-segment-check.js +247 -0
  20. package/lib/cli/translate-segment.d.ts +307 -0
  21. package/lib/cli/translate-segment.js +625 -0
  22. package/lib/cli/translate.d.ts +14 -4
  23. package/lib/cli/translate.js +232 -52
  24. package/lib/remark-lint/no-unmatched-anchor.js +6 -2
  25. package/lib/remark-lint/no-unopened-frontmatter.js +5 -1
  26. package/lib/remark-lint/rule-sets.d.ts +1 -3
  27. package/lib/remark-lint/rule-sets.js +3 -9
  28. package/lib/remark-lint/syntax-plugins.d.ts +32 -0
  29. package/lib/remark-lint/syntax-plugins.js +46 -0
  30. package/lib/remark-lint/translation-parity/frontmatter-preservation.js +5 -3
  31. package/lib/remark-lint/translation-parity/link-isomorphism.d.ts +8 -0
  32. package/lib/remark-lint/translation-parity/link-isomorphism.js +40 -4
  33. package/lib/remark-lint/translation-parity/shared.js +6 -2
  34. package/lib/remark-lint/translation-parity/url-residue.d.ts +1 -0
  35. package/lib/remark-lint/translation-parity/url-residue.js +25 -4
  36. package/lib/runtime/hooks/useTranslation.d.ts +1 -1
  37. package/lib/types.d.ts +101 -6
  38. package/package.json +1 -1
  39. package/lib/cli/translate-agent.d.ts +0 -81
  40. package/lib/cli/translate-agent.js +0 -450
@@ -14,20 +14,33 @@ import { DEFAULT_COPY_ONLY_DIRECTORIES, Language, SUPPORTED_LANGUAGES, TITLE_TRA
14
14
  import { OPTIONS_FILE, STORAGE_DIR, pathExists } from '../utils/index.js';
15
15
  import { escapeMarkdownHeadingIds, getMatchedDocFilePaths, parseBoolean, stringifyMatter, translateCodeFile, } from './helpers.js';
16
16
  import { loadConfig } from './load-config.js';
17
- import { translateWithAgent, } from './translate-agent.js';
17
+ import { createSegmentTranslator } from './translate-call.js';
18
18
  import { checkTranslations, } from './translate-check.js';
19
- import { createTranslationChecker, } from './translate-checker.js';
19
+ import { createTranslationChecker } from './translate-checker.js';
20
+ import { createDiagnoser } from './translate-diagnose.js';
20
21
  import { createJudge } from './translate-judge.js';
21
22
  import { CONCURRENCY_ENV, DEFAULT_CONCURRENCY, DEFAULT_REQUESTS_PER_MINUTE, REQUESTS_PER_MINUTE_ENV, createLimits, positiveIntFromEnv, } from './translate-limits.js';
22
23
  import { maskAst } from './translate-mask.js';
23
- import { DEFAULT_JUDGE_REASONING_EFFORT, DEFAULT_REASONING_EFFORT, createGateway, } from './translate-models.js';
24
+ import { DEFAULT_JUDGE_REASONING_EFFORT, DEFAULT_REASONING_EFFORT, createGateway, gatewayModel, } from './translate-models.js';
25
+ import { DEFAULT_MAX_SEGMENT_ATTEMPTS, translateDocument, } from './translate-pipeline.js';
26
+ import { createRepairAgent } from './translate-repair-agent.js';
24
27
  import { resolveTerms } from './translate-terms.js';
28
+ /**
29
+ * How many readings a segment gets, and how many must agree.
30
+ *
31
+ * Three and two, against two-and-unanimous for a whole page. A segment is a
32
+ * short text, and a reviewer given less to go on has more room to find
33
+ * something to say; the third reading buys back the precision that unanimity
34
+ * gives when there is a whole document to be sure about.
35
+ */
36
+ const DEFAULT_SEGMENT_DRAWS = 3;
37
+ const DEFAULT_SEGMENT_VOTES = 2;
25
38
  /**
26
39
  * The translation rules.
27
40
  *
28
41
  * Overridable per repository through `translate.systemPrompt`, which is why
29
42
  * the placeholder discipline is *not* here: it is part of the harness's own
30
- * prompt in `translate-agent.ts`, where a repository cannot drop it by
43
+ * prompt in `translate-call.ts`, where a repository cannot drop it by
31
44
  * replacing this text.
32
45
  */
33
46
  const DEFAULT_SYSTEM_PROMPT = `
@@ -75,17 +88,6 @@ These are additional requirements for the translation. They should be met along
75
88
  """
76
89
  <% } %>
77
90
  `.trim();
78
- /** How many times findings are fed back before a document is failed. */
79
- const DEFAULT_MAX_REPAIR_ROUNDS = 3;
80
- /**
81
- * Total assistant turns for one document, tool calls included.
82
- *
83
- * Separate from the repair rounds on purpose: a long document legitimately
84
- * spends many turns reading and appending before it has anything to repair, so
85
- * one number cannot be both the budget for producing a translation and the
86
- * budget for fixing it. This one is a runaway guard, not a quality knob.
87
- */
88
- const DEFAULT_MAX_TURNS = 60;
89
91
  function extractFirstLevelHeading(content) {
90
92
  const lines = content.split('\n');
91
93
  for (const line of lines) {
@@ -127,6 +129,66 @@ const renderTranslationRules = async ({ source, target, sourceContent, options,
127
129
  }, { async: true });
128
130
  return { rules, terms: terms.pairs };
129
131
  };
132
+ /** One segment, named the way a person reading a build log needs it named. */
133
+ const segmentName = (outcome) => `segment ${outcome.index + 1}` +
134
+ (outcome.label.line || outcome.label.heading
135
+ ? ` (${[
136
+ outcome.label.line ? `line ${outcome.label.line}` : undefined,
137
+ outcome.label.heading ? `‹${outcome.label.heading}›` : undefined,
138
+ ]
139
+ .filter(Boolean)
140
+ .join(' ')})`
141
+ : '');
142
+ /**
143
+ * Why a document failed, in one line.
144
+ *
145
+ * Three ways to fail, and they are told apart on purpose: a segment nothing
146
+ * could translate, a whole-document check that kept failing after the segments
147
+ * it named were redone, and a whole-document check nobody could attribute to a
148
+ * segment. They call for different things from whoever reads the log, and the
149
+ * design this replaced reported all of them as the same wall of problems.
150
+ */
151
+ const describeFailure = (result) => {
152
+ const { failure } = result;
153
+ switch (failure?.kind) {
154
+ case 'unsplittable-block': {
155
+ return 'A single block is too large to translate and cannot be divided.';
156
+ }
157
+ case 'segment': {
158
+ return `${failure.segments.length} segment(s) never passed: ${failure.segments
159
+ .map((index) => segmentName(result.outcomes[index]))
160
+ .join(', ')}.`;
161
+ }
162
+ case 'assembly': {
163
+ return `The assembled document still failed its checks after ${result.assemblyRounds} round(s) of sending segments back.`;
164
+ }
165
+ case 'unlocatable': {
166
+ return 'The assembled document failed a check of the whole page that could not be attributed to any one segment.';
167
+ }
168
+ default: {
169
+ return 'The document did not pass its checks.';
170
+ }
171
+ }
172
+ };
173
+ /** Everything that was wrong with one failed document, grouped by segment. */
174
+ const describeFailedDocument = ({ file, result, diagnosis, }) => {
175
+ const lines = [`\n${cyan(file)} — ${describeFailure(result)}`];
176
+ const failed = result.failure?.kind === 'segment'
177
+ ? result.failure.segments.map((index) => result.outcomes[index])
178
+ : [];
179
+ for (const outcome of failed) {
180
+ lines.push(` ${segmentName(outcome)}, after ${outcome.attempts} attempt(s):`, ...outcome.findings.map((finding) => ` ${red(finding.rule)} ${finding.reason}`));
181
+ }
182
+ if (failed.length === 0) {
183
+ lines.push(...result.findings
184
+ .filter((finding) => finding.blocking !== false)
185
+ .map((finding) => ` ${red(finding.rule)}${finding.line ? ` (line ${finding.line})` : ''} ${finding.reason}`));
186
+ }
187
+ if (diagnosis) {
188
+ lines.push('', ` ${cyan('What this looks like')} (written by a model from the evidence above; advisory, and not part of the verdict):`, ...diagnosis.split('\n').map((line) => ` ${line}`));
189
+ }
190
+ return lines.join('\n');
191
+ };
130
192
  const supportedLanguages = SUPPORTED_LANGUAGES.join(', ');
131
193
  export const translateCommand = new Command('translate')
132
194
  .description('Translate the documentation')
@@ -220,6 +282,25 @@ export const translateCommand = new Command('translate')
220
282
  }
221
283
  }
222
284
  const translateOptions = config.translate ?? {};
285
+ // Two options were removed when the whole-document loop was, and both of
286
+ // them named a budget that no longer exists. Ignoring them quietly would
287
+ // leave a repository believing it had configured something — which is the
288
+ // exact failure mode this rewrite exists to remove, in miniature.
289
+ const removed = {
290
+ maxRepairRounds: '`maxSegmentAttempts` (per segment) and `maxAssemblyRounds` (per document)',
291
+ maxTurns: '`repairAgent.maxTurns`, which now bounds only the repair agent',
292
+ };
293
+ const usedRemoved = Object.keys(removed).filter((key) => key in translateOptions);
294
+ if (usedRemoved.length > 0) {
295
+ logger.error(`\`translate\` no longer has ${usedRemoved
296
+ .map((key) => `\`${cyan(key)}\``)
297
+ .join(' or ')}: a translation is now produced a segment at a time, so a budget for the whole document has nothing to bound.\n` +
298
+ usedRemoved
299
+ .map((key) => ` ${cyan(key)} → replaced by ${removed[key]}`)
300
+ .join('\n'));
301
+ process.exitCode = 1;
302
+ return;
303
+ }
223
304
  // Built once: the provider wiring, and the processors the rules run in.
224
305
  // Missing credentials fail here, by name, rather than once per file as a
225
306
  // wall of unrelated stream errors.
@@ -242,22 +323,57 @@ export const translateCommand = new Command('translate')
242
323
  concurrency,
243
324
  requestsPerMinute,
244
325
  });
245
- const maxRepairRounds = translateOptions.maxRepairRounds ?? DEFAULT_MAX_REPAIR_ROUNDS;
246
- const maxTurns = translateOptions.maxTurns ?? DEFAULT_MAX_TURNS;
247
326
  const reasoningEffort = translateOptions.reasoningEffort ?? DEFAULT_REASONING_EFFORT;
248
- const judge = translateOptions.judge?.enabled === false
249
- ? undefined
250
- : createJudge({
327
+ const maxSegmentAttempts = translateOptions.maxSegmentAttempts ?? DEFAULT_MAX_SEGMENT_ATTEMPTS;
328
+ const judgeEnabled = translateOptions.judge?.enabled !== false;
329
+ const judgeReasoning = translateOptions.judge?.reasoningEffort ?? DEFAULT_JUDGE_REASONING_EFFORT;
330
+ // Two reviewers, because they are asked two different questions. The first
331
+ // reads one segment against its source and can stop it being frozen; the
332
+ // second reads the finished page for what no segment can see — a term that
333
+ // drifted between sections, a join that reads badly — and only ever
334
+ // reports.
335
+ const segmentJudge = judgeEnabled
336
+ ? createJudge({
251
337
  models: gateway.models,
252
338
  model: gateway.judgeModel,
253
- reasoningEffort: translateOptions.judge?.reasoningEffort ??
254
- DEFAULT_JUDGE_REASONING_EFFORT,
339
+ reasoningEffort: judgeReasoning,
340
+ draws: translateOptions.judge?.segmentDraws ?? DEFAULT_SEGMENT_DRAWS,
341
+ votes: translateOptions.judge?.segmentVotes ?? DEFAULT_SEGMENT_VOTES,
342
+ limit: modelCallLimit,
343
+ })
344
+ : undefined;
345
+ const documentJudge = judgeEnabled && translateOptions.fullDocJudge !== false
346
+ ? createJudge({
347
+ models: gateway.models,
348
+ model: gateway.judgeModel,
349
+ reasoningEffort: judgeReasoning,
255
350
  draws: translateOptions.judge?.draws,
256
351
  limit: modelCallLimit,
352
+ })
353
+ : undefined;
354
+ // A segment that repeated attempts could not fix goes to an agent that can
355
+ // only edit. It may be a stronger model than the one doing the translating:
356
+ // this path is rare and it is the hard one.
357
+ const repairModelId = translateOptions.repairAgent?.model;
358
+ const repairModel = repairModelId && repairModelId !== gateway.model.id
359
+ ? gatewayModel({
360
+ id: repairModelId,
361
+ baseUrl: gateway.baseUrl,
362
+ contextWindow: translateOptions.contextWindow,
363
+ maxOutputTokens: translateOptions.maxOutputTokens,
364
+ })
365
+ : gateway.model;
366
+ const diagnoser = translateOptions.diagnose?.enabled === false
367
+ ? undefined
368
+ : createDiagnoser({
369
+ models: gateway.models,
370
+ model: gateway.model,
371
+ reasoningEffort: translateOptions.diagnose?.reasoningEffort,
372
+ limit: modelCallLimit,
257
373
  });
258
- logger.info(`Translating with \`${cyan(gateway.model.id)}\` (reasoning ${cyan(reasoningEffort)}), up to ${cyan(String(maxRepairRounds))} repair round(s) per document. ` +
259
- `${cyan(String(concurrency))} at a time, at most ${cyan(String(requestsPerMinute))} model request(s) a minute. ` +
260
- (judge
374
+ logger.info(`Translating with \`${cyan(gateway.model.id)}\` (reasoning ${cyan(reasoningEffort)}), a segment at a time, up to ${cyan(String(maxSegmentAttempts))} attempt(s) per segment. ` +
375
+ `${cyan(String(concurrency))} document(s) at a time, at most ${cyan(String(requestsPerMinute))} model request(s) a minute. ` +
376
+ (segmentJudge
261
377
  ? `Reviewed by \`${cyan(gateway.judgeModel.id)}\`.`
262
378
  : `${red('Judge disabled')} — only the deterministic checks are running.`));
263
379
  const failures = [];
@@ -273,22 +389,42 @@ export const translateCommand = new Command('translate')
273
389
  .update(sourceContent)
274
390
  .digest('hex');
275
391
  const targetFilePath = sourceFilePath.replace(sourceDir, targetDir);
392
+ /**
393
+ * The translation already on disk, when there is one.
394
+ *
395
+ * Two layers use it. `sourceSHA` skips a document nobody touched at
396
+ * all; below that, the segments whose source did not change are reused
397
+ * rather than retranslated, so a one-line edit costs one segment
398
+ * instead of a whole page. `--force` turns off both, which is what
399
+ * "translate it again" has always meant.
400
+ */
401
+ let previous;
276
402
  if (await pathExists(targetFilePath, 'file')) {
277
403
  const targetContent = await fs.readFile(targetFilePath, 'utf-8');
278
- const targetFrontmatter = matter(targetContent)
279
- .data;
404
+ const parsedTarget = matter(targetContent);
405
+ const targetFrontmatter = parsedTarget.data;
280
406
  if (!force && targetFrontmatter.sourceSHA === sourceSHA) {
281
407
  return;
282
408
  }
409
+ if (!force && translateOptions.segmentCache !== false) {
410
+ previous = {
411
+ body: parsedTarget.content,
412
+ record: targetFrontmatter.i18nSegments,
413
+ };
414
+ }
283
415
  }
284
416
  const shouldCopyOnly = copyOnlyFilePathsSet.has(sourceFilePath);
285
417
  const sourceRelativePath = path.relative(docsDir, sourceFilePath);
286
418
  const targetRelativePath = path.relative(docsDir, targetFilePath);
287
419
  const targetBase = path.dirname(targetFilePath);
288
420
  /** The document exactly as it will be written, given a translated body. */
289
- const compose = (restored) => {
421
+ const compose = (restored, cache) => {
290
422
  const newFrontmatter = { ...sourceFrontmatter, sourceSHA };
291
423
  delete newFrontmatter.i18n;
424
+ delete newFrontmatter.i18nSegments;
425
+ if (cache) {
426
+ newFrontmatter.i18nSegments = cache;
427
+ }
292
428
  const { data, content } = matter(restored);
293
429
  const typedData = data;
294
430
  if (typedData.title && typeof typedData.title === 'string') {
@@ -349,42 +485,91 @@ export const translateCommand = new Command('translate')
349
485
  options: translateOptions,
350
486
  additionalPrompts: sourceFrontmatter.i18n?.additionalPrompts,
351
487
  });
352
- const result = await translateWithAgent({
488
+ const repairer = translateOptions.repairAgent?.enabled === false
489
+ ? undefined
490
+ : createRepairAgent({
491
+ models: gateway.models,
492
+ model: repairModel,
493
+ reasoningEffort: translateOptions.repairAgent?.reasoningEffort,
494
+ scratchDir,
495
+ extension: isMdx ? '.mdx' : '.md',
496
+ maxTurns: translateOptions.repairAgent?.maxTurns,
497
+ limit: modelCallLimit,
498
+ onProgress: (message) => {
499
+ logger.info(`${cyan(sourceRelativePath)}: ${message}`);
500
+ },
501
+ });
502
+ const translator = createSegmentTranslator({
503
+ models: gateway.models,
504
+ model: gateway.model,
505
+ reasoningEffort,
506
+ translationRules: rules,
507
+ sourceLanguage: Language[source],
508
+ targetLanguage: Language[target],
509
+ processor,
510
+ limit: modelCallLimit,
511
+ });
512
+ const result = await translateDocument({
513
+ tree: normalizedAst,
353
514
  maskedSource,
354
515
  maskEntries,
355
516
  processor,
356
517
  compose,
357
518
  targetPath: targetFilePath,
358
519
  sourceLabel: sourceRelativePath,
359
- source,
360
- target,
361
- translationRules: rules,
362
- terms,
520
+ sourceLanguage: Language[source],
521
+ targetLanguage: Language[target],
522
+ translator,
363
523
  checker,
364
- judge,
365
- models: gateway.models,
366
- model: gateway.model,
367
- reasoningEffort,
368
- scratchDir,
369
- maxRepairRounds,
370
- maxTurns,
371
- limit: modelCallLimit,
524
+ segmentJudge,
525
+ documentJudge,
526
+ terms,
527
+ segmentCap: translateOptions.segmentCap,
528
+ segmentHardCap: translateOptions.segmentHardCap,
529
+ segmentFloor: translateOptions.segmentFloor,
530
+ maxSegmentAttempts,
531
+ maxAssemblyRounds: translateOptions.maxAssemblyRounds,
532
+ contextTail: translateOptions.contextTail,
533
+ repairer,
534
+ previous,
372
535
  onProgress: (message) => {
373
536
  logger.info(message);
374
537
  },
375
538
  });
376
539
  if (!result.document) {
377
- failures.push({
378
- file: targetRelativePath,
540
+ // Explained before it is reported, so the log carries the analysis
541
+ // next to the failure rather than in a separate place nobody
542
+ // correlates. Advisory throughout: it cannot change this outcome.
543
+ const diagnosis = await diagnoser?.diagnose({
544
+ sourceLabel: sourceRelativePath,
545
+ failure: result.failure,
546
+ outcomes: result.outcomes,
379
547
  findings: result.findings,
380
- result,
548
+ assemblyRounds: result.assemblyRounds,
381
549
  });
382
- logger.error(`${cyan(targetRelativePath)} did not pass its checks after ${result.repairRounds} repair round(s).`);
550
+ failures.push({ file: targetRelativePath, result, diagnosis });
551
+ logger.error(`${cyan(targetRelativePath)} did not pass its checks. ${describeFailure(result)}`);
383
552
  return;
384
553
  }
385
554
  await fs.mkdir(targetBase, { recursive: true });
386
555
  await fs.writeFile(targetFilePath, result.document);
387
- logger.info(`${cyan(sourceRelativePath)} translated to ${cyan(targetRelativePath)} (${result.turns} turn(s), ${result.modelCalls} model call(s), ${result.repairRounds} repair round(s))`);
556
+ const cached = result.outcomes.filter((outcome) => outcome.status === 'cached').length;
557
+ const repaired = result.outcomes.filter((outcome) => outcome.status === 'repaired').length;
558
+ logger.info(`${cyan(sourceRelativePath)} translated to ${cyan(targetRelativePath)} ` +
559
+ `(${result.outcomes.length} segment(s)` +
560
+ (cached ? `, ${cached} reused` : '') +
561
+ (repaired ? `, ${repaired} repaired` : '') +
562
+ // Translation calls, named as such. The reviewers, the repair
563
+ // agent and the diagnosis all cost model calls too, and none of
564
+ // them are counted here: their judges are shared by every
565
+ // document being translated at once, so there is no per-document
566
+ // number to print. Calling this "model calls" made it look like
567
+ // there was.
568
+ `, ${translator.calls()} translation call(s)` +
569
+ (result.assemblyRounds
570
+ ? `, ${result.assemblyRounds} assembly round(s)`
571
+ : '') +
572
+ ')');
388
573
  });
389
574
  }));
390
575
  await fs.rm(scratchDir, { recursive: true, force: true });
@@ -397,12 +582,7 @@ export const translateCommand = new Command('translate')
397
582
  // does not go red for a document that was silently skipped.
398
583
  if (failures.length > 0) {
399
584
  logger.error(`${red(String(failures.length))} document(s) could not be translated to a state that passes their checks. Nothing will be uploaded.\n` +
400
- failures
401
- .map(({ file, findings, result }) => `\n${cyan(file)} — ${findings.length} unresolved problem(s) after ${result?.repairRounds ?? 0} repair round(s):\n` +
402
- findings
403
- .map((finding) => ` ${red(finding.rule)}${finding.line ? ` (line ${finding.line})` : ''} ${finding.reason}`)
404
- .join('\n'))
405
- .join('\n'));
585
+ failures.map(describeFailedDocument).join('\n'));
406
586
  process.exitCode = 1;
407
587
  return;
408
588
  }
@@ -6,7 +6,7 @@ import { lintRule } from 'unified-lint-rule';
6
6
  import { visit } from 'unist-util-visit';
7
7
  import { visitParents } from 'unist-util-visit-parents';
8
8
  import { isDoc } from "../cli/helpers.js";
9
- import { mdProcessor, mdxProcessor } from "../plugins/index.js";
9
+ import { syntaxProcessor } from "./syntax-plugins.js";
10
10
  import { getConfig } from "./utils.js";
11
11
  const anchorsCache = new Map();
12
12
  const astCache = new Map();
@@ -30,7 +30,11 @@ const getAnchors = (filepath) => {
30
30
  const anchors = new Set();
31
31
  let ast = astCache.get(filepath);
32
32
  if (!ast) {
33
- const processor = filepath.endsWith('.mdx') ? mdxProcessor : mdProcessor;
33
+ // The same stack the lint pipeline uses, so a document's anchors do not
34
+ // depend on whether this run happened to lint it before reading it.
35
+ const processor = filepath.endsWith('.mdx')
36
+ ? syntaxProcessor.mdx
37
+ : syntaxProcessor.md;
34
38
  ast = processor.parse(fs.readFileSync(filepath, 'utf-8'));
35
39
  astCache.set(filepath, ast);
36
40
  }
@@ -33,7 +33,10 @@ import { visit } from 'unist-util-visit';
33
33
  /**
34
34
  * The keys doom's frontmatter actually uses, counted across acp-docs rather
35
35
  * than remembered: weight 797 · sourceSHA 194 · title 132 · i18n 125 ·
36
- * queries 73 · description 21 · category 5 · author 4.
36
+ * queries 73 · description 21 · category 5 · author 4. `i18nSegments` is
37
+ * written by `doom translate` and so appears in no count taken before it
38
+ * existed; it is here because a translation is exactly the kind of file whose
39
+ * frontmatter a model can lose the opening `---` from.
37
40
  *
38
41
  * Deliberately a list and not `\w+:` — a heading really can begin "Note:" or
39
42
  * "Step 1:", and reporting those would be a rule nobody keeps.
@@ -43,6 +46,7 @@ const FRONTMATTER_KEYS = [
43
46
  'category',
44
47
  'description',
45
48
  'i18n',
49
+ 'i18nSegments',
46
50
  'queries',
47
51
  'sourceSHA',
48
52
  'title',
@@ -10,8 +10,7 @@ import type { Plugin } from 'unified';
10
10
  * register it twice.
11
11
  */
12
12
  type Rule = Plugin<[], never>;
13
- /** Syntax plugins every rule needs before it can run. */
14
- export declare const SYNTAX_PLUGINS: Rule[];
13
+ export { SYNTAX_PLUGINS } from './syntax-plugins.ts';
15
14
  /**
16
15
  * Rules that compare a translation with the source it was made from.
17
16
  *
@@ -72,4 +71,3 @@ export declare const LINT_RULES: Rule[];
72
71
  * position, which some of the orderings above depend on.
73
72
  */
74
73
  export declare const TRANSLATION_CHECK_RULES: Rule[];
75
- export {};
@@ -1,6 +1,3 @@
1
- import remarkDirective from 'remark-directive';
2
- import remarkFrontmatter from 'remark-frontmatter';
3
- import remarkGfm from 'remark-gfm';
4
1
  // @ts-expect-error -- https://github.com/ilyatitovich/remark-lint-code-block-split-list/issues/2
5
2
  import remarkLintCodeBlockSplitList from 'remark-lint-code-block-split-list';
6
3
  import remarkLintHeadingIncrement from 'remark-lint-heading-increment';
@@ -11,12 +8,9 @@ import remarkLintNoChinesePunctuationInNumber from 'remark-lint-no-chinese-punct
11
8
  import remarkLintNoDuplicateHeadingsInSection from 'remark-lint-no-duplicate-headings-in-section';
12
9
  import remarkLintNoHiddenTableCell from 'remark-lint-no-hidden-table-cell';
13
10
  import { checkDeadLinks, chineseTypographyOnlyInChinese, headingAnchorFormat, noDeepHeading, noDeepList, noLegacyOSNames, noMultiOpenAPIPaths, noParagraphIndent, noUnmatchedAnchor, noUnopenedFrontmatter, noUnparsedEmphasis, noUnresolvedApiRef, site, tableSize, titleRequired, translationComponentMultiset, translationEchoedSource, translationFrontmatterPreservation, translationHeadingSequence, translationJsxAttributeParity, translationLengthRatio, translationLinkIsomorphism, translationUpToDate, translationUrlResidue, unitCase, } from "./index.js";
14
- /** Syntax plugins every rule needs before it can run. */
15
- export const SYNTAX_PLUGINS = [
16
- remarkDirective,
17
- remarkFrontmatter,
18
- remarkGfm,
19
- ];
11
+ // The syntax stack lives in its own module so `translation-parity` can parse
12
+ // the source side with it without importing the rules that import it back.
13
+ export { SYNTAX_PLUGINS } from "./syntax-plugins.js";
20
14
  /**
21
15
  * Rules that compare a translation with the source it was made from.
22
16
  *
@@ -0,0 +1,32 @@
1
+ import type { Root } from 'mdast';
2
+ import type { Plugin } from 'unified';
3
+ /**
4
+ * The syntax extensions a document is parsed with, in one place.
5
+ *
6
+ * This exists because the `translation-parity` rules compare two trees, and for
7
+ * a comparison to mean anything both trees have to have been parsed the same
8
+ * way. They were not: the document being linted came through the lint pipeline
9
+ * (these plugins), while the source it was compared against was parsed with the
10
+ * site's `mdxProcessor`, which has no `remark-directive`. Anything directive
11
+ * syntax touches — a port in `http://host:8080`, a tag in `image:1.25`, a time
12
+ * in `10:30` — then became a `textDirective` on one side and stayed plain text
13
+ * on the other, and the rules reported the difference as damage the translator
14
+ * could not repair, because nothing it could write would make the two sides
15
+ * agree.
16
+ *
17
+ * `remark-directive` is in the list because the site renders with it
18
+ * (`plugins/directives`), so it is what the published page actually is.
19
+ */
20
+ export declare const SYNTAX_PLUGINS: Plugin<[], never>[];
21
+ /** Parse-only view of the stack — the rules never stringify the source. */
22
+ export interface SyntaxProcessor {
23
+ parse: (content: string) => Root;
24
+ }
25
+ /**
26
+ * Parses a document exactly as the lint pipeline does — the reference side of
27
+ * every `translation-parity` comparison.
28
+ */
29
+ export declare const syntaxProcessor: {
30
+ md: SyntaxProcessor;
31
+ mdx: SyntaxProcessor;
32
+ };
@@ -0,0 +1,46 @@
1
+ import remarkDirective from 'remark-directive';
2
+ import remarkFrontmatter from 'remark-frontmatter';
3
+ import remarkGfm from 'remark-gfm';
4
+ import remarkMdx from 'remark-mdx';
5
+ import remarkParse from 'remark-parse';
6
+ import { unified } from 'unified';
7
+ /**
8
+ * The syntax extensions a document is parsed with, in one place.
9
+ *
10
+ * This exists because the `translation-parity` rules compare two trees, and for
11
+ * a comparison to mean anything both trees have to have been parsed the same
12
+ * way. They were not: the document being linted came through the lint pipeline
13
+ * (these plugins), while the source it was compared against was parsed with the
14
+ * site's `mdxProcessor`, which has no `remark-directive`. Anything directive
15
+ * syntax touches — a port in `http://host:8080`, a tag in `image:1.25`, a time
16
+ * in `10:30` — then became a `textDirective` on one side and stayed plain text
17
+ * on the other, and the rules reported the difference as damage the translator
18
+ * could not repair, because nothing it could write would make the two sides
19
+ * agree.
20
+ *
21
+ * `remark-directive` is in the list because the site renders with it
22
+ * (`plugins/directives`), so it is what the published page actually is.
23
+ */
24
+ export const SYNTAX_PLUGINS = [
25
+ remarkDirective,
26
+ remarkFrontmatter,
27
+ remarkGfm,
28
+ ];
29
+ const build = (mdx) => {
30
+ let processor = unified().use(remarkParse);
31
+ for (const plugin of SYNTAX_PLUGINS) {
32
+ processor = processor.use(plugin);
33
+ }
34
+ if (mdx) {
35
+ processor = processor.use(remarkMdx);
36
+ }
37
+ return processor.freeze();
38
+ };
39
+ /**
40
+ * Parses a document exactly as the lint pipeline does — the reference side of
41
+ * every `translation-parity` comparison.
42
+ */
43
+ export const syntaxProcessor = {
44
+ md: build(false),
45
+ mdx: build(true),
46
+ };
@@ -10,9 +10,11 @@ import { currentPair } from "./shared.js";
10
10
  * language and not the other.
11
11
  */
12
12
  const TRANSLATED_KEYS = new Set(['title', 'description']);
13
- // `sourceSHA` is written by the translator and `i18n` is deliberately dropped
14
- // from the translation — neither is a divergence.
15
- const TRANSLATOR_OWNED_KEYS = new Set(['sourceSHA', 'i18n']);
13
+ // `sourceSHA` and `i18nSegments` are written by the translator and `i18n` is
14
+ // deliberately dropped from the translation — none of the three is a
15
+ // divergence. Leaving `i18nSegments` out of this set would report every
16
+ // translated document in the corpus, every time.
17
+ const TRANSLATOR_OWNED_KEYS = new Set(['sourceSHA', 'i18nSegments', 'i18n']);
16
18
  export const translationFrontmatterPreservation = lintRule('doom-lint:translation-frontmatter-preservation', async (tree, vfile) => {
17
19
  const pair = await currentPair(tree, vfile);
18
20
  if (!pair) {
@@ -1,4 +1,12 @@
1
1
  import type { Root } from 'mdast';
2
+ export declare const partitionAbsorbed: (missing: readonly string[], extra: readonly string[]) => {
3
+ absorbed: {
4
+ target: string;
5
+ suffix: string;
6
+ }[];
7
+ lost: string[];
8
+ unmatched: string[];
9
+ };
2
10
  /**
3
11
  * A translation points at exactly the same places its source points at.
4
12
  *
@@ -4,6 +4,38 @@ const preview = (values) => values
4
4
  .slice(0, 5)
5
5
  .map((value) => `\`${value}\``)
6
6
  .join(', ') + (values.length > 5 ? `, … (${values.length} total)` : '');
7
+ /**
8
+ * A bare URL ends where markdown says it ends, and markdown does not know
9
+ * about Chinese sentence punctuation.
10
+ *
11
+ * GFM closes an autolink literal at whitespace and drops *ASCII* trailing
12
+ * punctuation, so `see http://h:8080.` links to `http://h:8080`. A full-width
13
+ * stop is not ASCII: it is absorbed, and `例如 http://h:8080。` links to
14
+ * `http://h:8080。`, an address that does not exist. The translator writes
15
+ * correct Chinese and gets a broken link for it — and the two messages it used
16
+ * to get, "lost this target" and "points at that one", never said which of the
17
+ * two spellings was the one to change, so the repair loop could only guess.
18
+ *
19
+ * Only a non-ASCII suffix is read this way. `https://x.com` becoming
20
+ * `https://x.com/docs` is also a prefix, and that one is a real retarget.
21
+ */
22
+ const isAbsorbedSuffix = (suffix) => suffix.length > 0 && suffix.codePointAt(0) > 0x7f;
23
+ export const partitionAbsorbed = (missing, extra) => {
24
+ const unmatched = [...extra];
25
+ const lost = [];
26
+ const absorbed = [];
27
+ for (const target of missing) {
28
+ const index = unmatched.findIndex((candidate) => candidate.startsWith(target) &&
29
+ isAbsorbedSuffix(candidate.slice(target.length)));
30
+ if (index === -1) {
31
+ lost.push(target);
32
+ continue;
33
+ }
34
+ absorbed.push({ target, suffix: unmatched[index].slice(target.length) });
35
+ unmatched.splice(index, 1);
36
+ }
37
+ return { absorbed, lost, unmatched };
38
+ };
7
39
  /**
8
40
  * A translation points at exactly the same places its source points at.
9
41
  *
@@ -36,10 +68,14 @@ export const translationLinkIsomorphism = lintRule('doom-lint:translation-link-i
36
68
  const expected = collectLinkTargets(pair.sourceTree, docDirInRoot(pair.sourceLang, pair.relativePath));
37
69
  const actual = collectLinkTargets(tree, docDirInRoot(pair.targetLang, pair.relativePath));
38
70
  const { missing, extra } = diffMultiset(expected, actual);
39
- if (missing.length) {
40
- vfile.message(`Translation lost ${missing.length} link target(s) the source has: ${preview(missing)}.`, tree);
71
+ const { absorbed, lost, unmatched } = partitionAbsorbed(missing, extra);
72
+ for (const { target, suffix } of absorbed) {
73
+ vfile.message(`Translation link \`${target}${suffix}\` swallowed the \`${suffix}\` that follows it: a bare URL runs on until a space, and markdown only leaves out trailing ASCII punctuation. Write it as \`[${target}](${target})\` so the punctuation stays outside the link.`, tree);
74
+ }
75
+ if (lost.length) {
76
+ vfile.message(`Translation lost ${lost.length} link target(s) the source has: ${preview(lost)}.`, tree);
41
77
  }
42
- if (extra.length) {
43
- vfile.message(`Translation points at ${extra.length} target(s) the source does not: ${preview(extra)}.`, tree);
78
+ if (unmatched.length) {
79
+ vfile.message(`Translation points at ${unmatched.length} target(s) the source does not: ${preview(unmatched)}.`, tree);
44
80
  }
45
81
  });