@alauda/doom 2.7.1 → 2.8.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -19,8 +19,9 @@ import type { Segment } from './translate-segment.ts';
19
19
  * context it has filled and had released is a model that reaches for a rewrite.
20
20
  */
21
21
  /** Retries for a refused request. Transient limits are normal at corpus scale. */
22
- export declare const DEFAULT_CALL_MAX_RETRIES = 5;
23
- export declare const DEFAULT_CALL_RETRY_DELAY_MS = 3000;
22
+ export declare const DEFAULT_CALL_MAX_RETRIES = 6;
23
+ /** 5 s, doubled per attempt: about five minutes, the same budget as the judge. See there for the measurement. */
24
+ export declare const DEFAULT_CALL_RETRY_DELAY_MS = 5000;
24
25
  /** How many lines of the previous segment's translation carry the voice forward. */
25
26
  export declare const DEFAULT_CONTEXT_TAIL = 20;
26
27
  export interface SegmentTranslationRequest {
@@ -15,8 +15,9 @@
15
15
  * context it has filled and had released is a model that reaches for a rewrite.
16
16
  */
17
17
  /** Retries for a refused request. Transient limits are normal at corpus scale. */
18
- export const DEFAULT_CALL_MAX_RETRIES = 5;
19
- export const DEFAULT_CALL_RETRY_DELAY_MS = 3_000;
18
+ export const DEFAULT_CALL_MAX_RETRIES = 6;
19
+ /** 5 s, doubled per attempt: about five minutes, the same budget as the judge. See there for the measurement. */
20
+ export const DEFAULT_CALL_RETRY_DELAY_MS = 5_000;
20
21
  /** How many lines of the previous segment's translation carry the voice forward. */
21
22
  export const DEFAULT_CONTEXT_TAIL = 20;
22
23
  const PLACEHOLDER_DISCIPLINE = `## Placeholders
@@ -129,8 +129,16 @@ export declare const DEFAULT_JUDGE_DRAWS = 2;
129
129
  * readings will meet it. Retrying is not softening the check — a reading that
130
130
  * never succeeds still fails the document.
131
131
  */
132
- export declare const DEFAULT_JUDGE_MAX_RETRIES = 4;
133
- export declare const DEFAULT_JUDGE_RETRY_DELAY_MS = 2000;
132
+ export declare const DEFAULT_JUDGE_MAX_RETRIES = 6;
133
+ /**
134
+ * 5 s, doubled per attempt: about five minutes in all.
135
+ *
136
+ * Measured on 2026-09-04 with four translation runs sharing one gateway
137
+ * account: the refusals lasted minutes, not seconds, and the previous budget
138
+ * of about thirty seconds turned a busy gateway into a failed document — and,
139
+ * before `translate.ts` learned to contain it, into a failed run.
140
+ */
141
+ export declare const DEFAULT_JUDGE_RETRY_DELAY_MS = 5000;
134
142
  /**
135
143
  * The findings enough readings agreed on.
136
144
  *
@@ -17,8 +17,16 @@ export const DEFAULT_JUDGE_DRAWS = 2;
17
17
  * readings will meet it. Retrying is not softening the check — a reading that
18
18
  * never succeeds still fails the document.
19
19
  */
20
- export const DEFAULT_JUDGE_MAX_RETRIES = 4;
21
- export const DEFAULT_JUDGE_RETRY_DELAY_MS = 2_000;
20
+ export const DEFAULT_JUDGE_MAX_RETRIES = 6;
21
+ /**
22
+ * 5 s, doubled per attempt: about five minutes in all.
23
+ *
24
+ * Measured on 2026-09-04 with four translation runs sharing one gateway
25
+ * account: the refusals lasted minutes, not seconds, and the previous budget
26
+ * of about thirty seconds turned a busy gateway into a failed document — and,
27
+ * before `translate.ts` learned to contain it, into a failed run.
28
+ */
29
+ export const DEFAULT_JUDGE_RETRY_DELAY_MS = 5_000;
22
30
  const SYSTEM_PROMPT = `You are reviewing a translation of technical documentation. You are not improving it and not rewriting it: you are saying what is wrong with it, if anything.
23
31
 
24
32
  Work in two passes, in this order.
@@ -246,8 +254,24 @@ export const createJudge = ({ models, model, reasoningEffort, draws = DEFAULT_JU
246
254
  return {
247
255
  readings: () => readings,
248
256
  async review(request) {
249
- const results = await Promise.all(Array.from({ length: draws }, () => readOnce(request)));
250
- return agreedFindings(results, votes).map((finding) => ({
257
+ // One reading first, and a clean reading is final.
258
+ //
259
+ // Measured on 2026-09-04 over 80 readings of translations the pipeline
260
+ // had already accepted: 97% found nothing, and the two that found
261
+ // something were right. The extra readings exist to confirm a finding,
262
+ // not to look for one the first reading missed — so they are taken only
263
+ // when there is something blocking to confirm, and the same vote then
264
+ // applies to all of them. What gets sent back does not change; what a
265
+ // segment that passes costs does: one reading instead of `draws`.
266
+ const first = await readOnce(request);
267
+ const confirm = first.some((finding) => BLOCKING_JUDGE_KINDS.includes(finding.kind));
268
+ const results = confirm
269
+ ? [
270
+ first,
271
+ ...(await Promise.all(Array.from({ length: Math.max(0, draws - 1) }, () => readOnce(request)))),
272
+ ]
273
+ : [first];
274
+ return agreedFindings(results, confirm ? votes : 1).map((finding) => ({
251
275
  rule: `doom-judge:${finding.kind}`,
252
276
  reason: `${finding.detail} — the source says: “${finding.source}”`,
253
277
  blocking: BLOCKING_JUDGE_KINDS.includes(finding.kind),
@@ -67,6 +67,11 @@ export type DocumentFailure =
67
67
  /** The assembled document failed a rule nothing could attribute to a segment. */
68
68
  | {
69
69
  kind: 'unlocatable';
70
+ }
71
+ /** The translation stopped on an error — a gateway that never answered, or a bug. Nothing was written. */
72
+ | {
73
+ kind: 'error';
74
+ detail: string;
70
75
  };
71
76
  export interface TranslateDocumentResult {
72
77
  /** The finished document, when it passed. */
@@ -154,3 +159,11 @@ export interface TranslateDocumentOptions {
154
159
  */
155
160
  export declare const maskFindingToTranslationFinding: (finding: MaskIntegrityError["findings"][number]) => TranslationFinding;
156
161
  export declare const translateDocument: (options: TranslateDocumentOptions) => Promise<TranslateDocumentResult>;
162
+ /**
163
+ * Whether a segment has any text a reviewer could judge.
164
+ *
165
+ * Frontmatter, headings, blank lines, lines that are only a component tag or
166
+ * a placeholder do not count: a heading is held to the source by the structure
167
+ * check, a tag by the component multiset, a placeholder by the count.
168
+ */
169
+ export declare const hasProse: (segment: Pick<Segment, "text">) => boolean;
@@ -167,6 +167,13 @@ export const translateDocument = async (options) => {
167
167
  let retry = sentBack?.length
168
168
  ? { previous: frozen[segment.index] ?? '', findings: sentBack }
169
169
  : undefined;
170
+ // A page that is a heading and an `<Overview />` has nothing for a
171
+ // reviewer to read, and the deterministic layers already hold the heading
172
+ // to the source. Half the index pages of a site are exactly that.
173
+ const judge = hasProse(segment) ? segmentJudge : undefined;
174
+ if (segmentJudge && !judge && !sentBack) {
175
+ onProgress?.(`${sourceLabel} [segment ${segment.index + 1}/${plan.segments.length}${describe(segment)}] not reviewed: nothing but headings and components`);
176
+ }
170
177
  for (let attempt = 1; attempt <= maxSegmentAttempts; attempt++) {
171
178
  outcome.attempts++;
172
179
  const translation = await translator.translate({
@@ -181,7 +188,7 @@ export const translateDocument = async (options) => {
181
188
  translation,
182
189
  processor,
183
190
  documentTokens,
184
- judge: segmentJudge,
191
+ judge,
185
192
  fragment,
186
193
  sourceLanguage,
187
194
  targetLanguage,
@@ -201,7 +208,7 @@ export const translateDocument = async (options) => {
201
208
  best[segment.index] = { text: translation, blocking: blocking.length };
202
209
  }
203
210
  retry = { previous: translation, findings };
204
- onProgress?.(`${sourceLabel} [segment ${segment.index + 1}/${plan.segments.length}${describe(segment)}] attempt ${attempt} rejected: ${summarise(blocking)}`);
211
+ onProgress?.(`${sourceLabel} [segment ${segment.index + 1}/${plan.segments.length}${describe(segment)}] attempt ${attempt} rejected: ${summarise(blocking)}${reasons(blocking)}`);
205
212
  }
206
213
  if (repairer) {
207
214
  const draft = best[segment.index]?.text ?? '';
@@ -364,7 +371,7 @@ export const translateDocument = async (options) => {
364
371
  assemblyRounds++;
365
372
  for (const [index, sentBack] of routed) {
366
373
  const segment = plan.segments[index];
367
- onProgress?.(`${sourceLabel} [segment ${index + 1}/${plan.segments.length}${describe(segment)}] sent back by the assembled document: ${summarise(sentBack)}`);
374
+ onProgress?.(`${sourceLabel} [segment ${index + 1}/${plan.segments.length}${describe(segment)}] sent back by the assembled document: ${summarise(sentBack)}${reasons(sentBack)}`);
368
375
  await workOn(segment, sentBack);
369
376
  }
370
377
  }
@@ -374,6 +381,39 @@ const summarise = (findings) => findings
374
381
  .slice(0, 3)
375
382
  .map((finding) => finding.rule)
376
383
  .join(', ') + (findings.length > 3 ? `, +${findings.length - 3} more` : '');
384
+ /**
385
+ * What the findings said, one per line under the summary.
386
+ *
387
+ * The rule names alone were what the build log used to carry, and they are
388
+ * not enough to tell a reviewer that is right from one that is not: on
389
+ * 2026-09-04 that question needed a day of probing to answer, because the
390
+ * reviewer's reasons for 29 rejections had been printed nowhere. Now they
391
+ * are in the log, where a person can read them.
392
+ */
393
+ const reasons = (findings) => findings
394
+ .slice(0, 3)
395
+ .map((finding) => `\n ${finding.rule}: ${finding.reason.length > 240 ? `${finding.reason.slice(0, 240)}…` : finding.reason}`)
396
+ .join('');
397
+ /**
398
+ * Whether a segment has any text a reviewer could judge.
399
+ *
400
+ * Frontmatter, headings, blank lines, lines that are only a component tag or
401
+ * a placeholder do not count: a heading is held to the source by the structure
402
+ * check, a tag by the component multiset, a placeholder by the count.
403
+ */
404
+ export const hasProse = (segment) => {
405
+ const body = segment.text.replace(/^---\n[\s\S]*?\n---\n?/u, '');
406
+ return body.split('\n').some((line) => {
407
+ const text = line.trim();
408
+ if (text === '' || text.startsWith('#')) {
409
+ return false;
410
+ }
411
+ if (/^<\/?[A-Za-z][^>]*>$/u.test(text) || /^\{.*\}$/u.test(text)) {
412
+ return false;
413
+ }
414
+ return /[\p{L}\p{N}]/u.test(text.replace(/__DOOM_TR_[A-Z]+_\d+__/gu, ''));
415
+ });
416
+ };
377
417
  /**
378
418
  * Rules that compare the translation with its source across the whole tree.
379
419
  *
@@ -1,6 +1,7 @@
1
1
  import { visit } from 'unist-util-visit';
2
2
  import { explainUnparsedEmphasis, findUnparsedEmphasis, } from "../remark-lint/no-unparsed-emphasis.js";
3
- import { collectComponents, collectHeadingDepths, } from "../remark-lint/translation-parity/shared.js";
3
+ import { collectComponents, collectHeadingDepths, diffMultiset, } from "../remark-lint/translation-parity/shared.js";
4
+ import { collectProseUrls } from "../remark-lint/translation-parity/url-residue.js";
4
5
  import { countPlaceholders } from "./translate-mask.js";
5
6
  import { parseAttributeElement } from "./translate-segment.js";
6
7
  /**
@@ -140,11 +141,13 @@ const collectTrailingAnchors = (tree) => {
140
141
  /**
141
142
  * The skeleton is the source's, not the model's.
142
143
  *
143
- * Headings and components are structure rather than prose: translating a page
144
- * never changes a heading's level and never adds or removes a component. Both
145
- * are already checked over the whole document by `translation-heading-sequence`
146
- * and `translation-component-multiset` — this runs the *same* collectors, one
147
- * segment at a time.
144
+ * Headings, components and URLs written in prose are structure rather than
145
+ * prose: translating a page never changes a heading's level, never adds or
146
+ * removes a component, and never drops a URL that markdown left as plain text.
147
+ * All three are already checked over the whole document by
148
+ * `translation-heading-sequence`, `translation-component-multiset` and
149
+ * `translation-url-residue` — this runs the *same* collectors, one segment at
150
+ * a time.
148
151
  *
149
152
  * That is not a duplicate of those rules, it is where they become actionable.
150
153
  * Every pairwise rule reports against the document as a whole, with no line
@@ -193,6 +196,26 @@ const structureFindings = (segment, tree, processor) => {
193
196
  : `This segment's translation invented ${got - want} \`<${name}>\` (the source has ${want}, the translation has ${got}).`,
194
197
  });
195
198
  }
199
+ // URLs markdown did not turn into links.
200
+ //
201
+ // The whole-document rule that owns this comparison,
202
+ // `translation-url-residue`, is the one that proved the note above the hard
203
+ // way: it fired on an assembled page, had no segment to be routed to, and
204
+ // failed the document with no repair round — on a false positive, and on
205
+ // every build for a day. Checked here it arrives with a segment attached, so
206
+ // a genuine loss is retranslated instead of being fatal, and the whole-page
207
+ // rule goes back to meaning what the note says it means: assembly did it.
208
+ const expectedUrls = collectProseUrls(source);
209
+ const actualUrls = collectProseUrls(tree);
210
+ const { missing: missingUrls } = diffMultiset(expectedUrls, actualUrls);
211
+ if (missingUrls.length > 0) {
212
+ findings.push({
213
+ rule: 'doom-translate:segment-url-residue',
214
+ reason: `This segment's translation dropped ${missingUrls.length} URL(s) that the source writes out in prose: ${missingUrls
215
+ .map((url) => `\`${url}\``)
216
+ .join(', ')}. They are plain text rather than links, so nothing carries them through for you — reproduce each one exactly as the source spells it.`,
217
+ });
218
+ }
196
219
  // Only meaningful while the headings still correspond one to one; when they
197
220
  // do not, the sequence finding above already says so and pairing by index
198
221
  // would invent a second, misleading one.
@@ -165,6 +165,9 @@ const describeFailure = (result) => {
165
165
  case 'unlocatable': {
166
166
  return 'The assembled document failed a check of the whole page that could not be attributed to any one segment.';
167
167
  }
168
+ case 'error': {
169
+ return `The translation stopped on an error and nothing was written: ${failure.detail}`;
170
+ }
168
171
  default: {
169
172
  return 'The document did not pass its checks.';
170
173
  }
@@ -445,7 +448,32 @@ export const translateCommand = new Command('translate')
445
448
  }
446
449
  return stringifyMatter(newFrontmatter, content);
447
450
  };
451
+ // One document's error is that document's failure, not the run's.
452
+ //
453
+ // Measured on 2026-09-04: a judge reading refused by the gateway for
454
+ // longer than its retries threw out of here, and the whole process
455
+ // exited — forty minutes of accepted translations of other documents
456
+ // still in flight went with it. The documents already written stay
457
+ // written; this one is reported with the rest at the end.
448
458
  await documentLimit(async () => {
459
+ try {
460
+ await translateOne();
461
+ }
462
+ catch (error) {
463
+ const detail = error instanceof Error ? error.message : String(error);
464
+ failures.push({
465
+ file: targetRelativePath,
466
+ result: {
467
+ findings: [{ rule: 'doom-translate:error', reason: detail }],
468
+ failure: { kind: 'error', detail },
469
+ outcomes: [],
470
+ assemblyRounds: 0,
471
+ },
472
+ });
473
+ logger.error(`${cyan(sourceRelativePath)}: translation stopped on an error and nothing was written: ${detail}`);
474
+ }
475
+ });
476
+ async function translateOne() {
449
477
  if (shouldCopyOnly) {
450
478
  logger.info(`Copying ${cyan(sourceRelativePath)} to ${cyan(targetRelativePath)}`);
451
479
  const { content } = matter(sourceContent);
@@ -570,7 +598,7 @@ export const translateCommand = new Command('translate')
570
598
  ? `, ${result.assemblyRounds} assembly round(s)`
571
599
  : '') +
572
600
  ')');
573
- });
601
+ }
574
602
  }));
575
603
  await fs.rm(scratchDir, { recursive: true, force: true });
576
604
  // Fail at the end, with the whole list.
@@ -19,11 +19,108 @@ import { currentPair, diffMultiset } from "./shared.js";
19
19
  * between prose and inline code. Those are identifiers, not links, and a guard
20
20
  * that cries wolf about them is worse than no guard at all.
21
21
  *
22
- * Honest caveat: measured exposure on this corpus is zero — all seven bare URLs
23
- * carry `https://`, which makes them link nodes, protected structurally. This
24
- * is a guard for a case nothing else covers, not a demonstrated detector.
22
+ * Honest caveat, kept because it turned out to matter: when this shipped,
23
+ * measured exposure on the corpus was zero — all seven bare URLs carried
24
+ * `https://`, which makes them link nodes, protected structurally. It was a
25
+ * guard for a case nothing else covered, not a demonstrated detector. Its
26
+ * first meeting with real prose was a false positive that took a build down
27
+ * for a day; what that cost, and what it changed, is written up on
28
+ * {@link BARE_URL} and {@link HAS_HOST}.
25
29
  */
26
- const BARE_URL = /\b(?:[a-z][a-z0-9+.-]*:\/\/|www\.)\S+/gi;
30
+ /**
31
+ * A URL runs to the first character a URL cannot contain.
32
+ *
33
+ * This used to be `\S+`, "everything up to the next space" — and Chinese does
34
+ * not write spaces. On the translation side the match therefore swallowed the
35
+ * rest of the sentence: `(https://)和路径` yielded `https://)和路径`, which
36
+ * could not equal anything the English side produced no matter how faithful
37
+ * the translation was. `[!-~]` is printable ASCII, which is what a URL is made
38
+ * of, so a match now ends at the first full-width bracket or Han character.
39
+ *
40
+ * `[^\s\P{ASCII}]` reads as a double negative because that is the only way to
41
+ * say it without an opaque code-point range: not whitespace, and not outside
42
+ * ASCII.
43
+ *
44
+ * The letters are spelled out rather than left to an `i` flag, and that is not
45
+ * style. Under `iu` the engine case-folds before testing the class, and two
46
+ * ASCII letters fold to non-ASCII ones — `s` to `ſ` (U+017F), `k` to `K`
47
+ * (U+212A) — so `\P{ASCII}` would swallow them and the negated class would
48
+ * *reject* `s` and `k`. Caught by the one test with an `s` mid-URL:
49
+ * `ftp://h/Foo_(disambiguation)` matched as far as `ftp://h/Foo_(di`.
50
+ *
51
+ * That misfire is not hypothetical and it was not cheap. `connectors-operator`
52
+ * release-1.14 was red on every run for 22 hours over one line of prose —
53
+ * `includes the correct protocol (https://) and path` — where nothing had been
54
+ * lost at all: the translation said the same thing with full-width brackets.
55
+ * And because this rule reports against the whole page, the assembled document
56
+ * could not attribute the finding to any segment and failed outright without a
57
+ * repair round. A misfire here does not cost "one finding to read".
58
+ *
59
+ * The price of the narrowing, stated plainly: an internationalised host
60
+ * written out in prose (`https://例え.jp`) is no longer detected. That is the
61
+ * right trade — the sources are English technical documentation, and a
62
+ * detector that cannot agree with its own translation is not a detector.
63
+ *
64
+ * The sibling rule made the opposite choice, and deliberately:
65
+ * `translation-link-isomorphism` *reports* the full-width stop an autolink
66
+ * absorbed rather than normalising it away, because there the punctuation ends
67
+ * up inside a real `href` and the link genuinely breaks. Prose has no `href`
68
+ * to break — `(ftp://h/x)` reads exactly as intended — so here the two
69
+ * spellings are made comparable instead of being reported.
70
+ */
71
+ const BARE_URL = /\b(?:[a-zA-Z][a-zA-Z0-9+.-]*:\/\/|[wW]{3}\.)[^\s\P{ASCII}]+/gu;
72
+ /**
73
+ * Punctuation that ends the sentence rather than the URL.
74
+ *
75
+ * `(see ftp://h/x)` and `fetch ftp://h/x.tar.` both put a character in the
76
+ * match that the writer did not mean as part of the address, and the
77
+ * translation — which repunctuates, because that is what translating prose
78
+ * does — will not have it. Trimmed the way GFM's autolink extension trims it,
79
+ * which is also the way a reader would.
80
+ *
81
+ * Only ASCII forms are listed: {@link BARE_URL} cannot match anything else, so
82
+ * a full-width bracket here would be a branch that can never be taken.
83
+ */
84
+ const TRAILING_PUNCTUATION = /[!"'*,.:;?_~]+$/u;
85
+ /**
86
+ * Closing brackets, and the opener each needs in order to belong to the URL.
87
+ *
88
+ * So `ftp://h/Foo_(disambiguation)` keeps its parenthesis — the URL opened it
89
+ * — and `(see ftp://h/x)` does not.
90
+ */
91
+ const BRACKETS = new Map([
92
+ [')', '('],
93
+ [']', '['],
94
+ ['}', '{'],
95
+ ]);
96
+ const occurrences = (value, character) => value.split(character).length - 1;
97
+ const trimSentence = (url) => {
98
+ let value = url;
99
+ for (;;) {
100
+ const before = value;
101
+ value = value.replace(TRAILING_PUNCTUATION, '');
102
+ const closing = value.at(-1);
103
+ const opening = closing == null ? undefined : BRACKETS.get(closing);
104
+ if (closing != null &&
105
+ opening != null &&
106
+ occurrences(value, closing) > occurrences(value, opening)) {
107
+ value = value.slice(0, -1);
108
+ }
109
+ if (value === before) {
110
+ return value;
111
+ }
112
+ }
113
+ };
114
+ /**
115
+ * A scheme with nothing after it is not a URL.
116
+ *
117
+ * `https://` on its own is prose — a sentence naming the protocol, as in "the
118
+ * address must include the protocol (https://)". Requiring a host is what
119
+ * tells the two apart, and it gives up nothing this rule was protecting: an
120
+ * address with no host cannot be fetched, so it cannot be a link a translation
121
+ * has to carry through.
122
+ */
123
+ const HAS_HOST = /(?::\/\/|^[wW]{3}\.)[^\s\P{ASCII}]/u;
27
124
  /**
28
125
  * Text inside a link is not prose for this purpose.
29
126
  *
@@ -49,7 +146,10 @@ export const collectProseUrls = (tree) => {
49
146
  return;
50
147
  }
51
148
  for (const [match] of node.value.matchAll(BARE_URL)) {
52
- found.push(match);
149
+ const url = trimSentence(match);
150
+ if (HAS_HOST.test(url)) {
151
+ found.push(url);
152
+ }
53
153
  }
54
154
  });
55
155
  return found;
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@alauda/doom",
3
- "version": "2.7.1",
3
+ "version": "2.8.1",
4
4
  "type": "module",
5
5
  "description": "Doctor Doom making docs.",
6
6
  "repository": {