@alauda/doom 2.6.1 → 2.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/cli/translate-call.d.ts +85 -0
- package/lib/cli/translate-call.js +149 -0
- package/lib/cli/translate-checker.d.ts +8 -0
- package/lib/cli/translate-diagnose.d.ts +45 -0
- package/lib/cli/translate-diagnose.js +118 -0
- package/lib/cli/translate-judge.d.ts +28 -3
- package/lib/cli/translate-judge.js +40 -7
- package/lib/cli/translate-mask.d.ts +13 -0
- package/lib/cli/translate-mask.js +19 -5
- package/lib/cli/translate-pipeline.d.ts +144 -0
- package/lib/cli/translate-pipeline.js +489 -0
- package/lib/cli/translate-repair-agent.d.ts +69 -0
- package/lib/cli/translate-repair-agent.js +241 -0
- package/lib/cli/translate-scratch.d.ts +9 -1
- package/lib/cli/translate-scratch.js +2 -2
- package/lib/cli/translate-segment-cache.d.ts +90 -0
- package/lib/cli/translate-segment-cache.js +301 -0
- package/lib/cli/translate-segment-check.d.ts +42 -0
- package/lib/cli/translate-segment-check.js +247 -0
- package/lib/cli/translate-segment.d.ts +307 -0
- package/lib/cli/translate-segment.js +625 -0
- package/lib/cli/translate.d.ts +14 -4
- package/lib/cli/translate.js +232 -52
- package/lib/remark-lint/no-unopened-frontmatter.js +5 -1
- package/lib/remark-lint/translation-parity/frontmatter-preservation.js +5 -3
- package/lib/runtime/hooks/useTranslation.d.ts +1 -1
- package/lib/types.d.ts +101 -6
- package/package.json +1 -1
- package/lib/cli/translate-agent.d.ts +0 -81
- package/lib/cli/translate-agent.js +0 -450
|
@@ -0,0 +1,625 @@
|
|
|
1
|
+
import crypto from 'node:crypto';
|
|
2
|
+
import { visit } from 'unist-util-visit';
|
|
3
|
+
import { isTranslatableJsxAttr } from "../runtime/components/_translation-policy.js";
|
|
4
|
+
import { countPlaceholders } from "./translate-mask.js";
|
|
5
|
+
/**
|
|
6
|
+
* Cutting one document into the units everything else works in.
|
|
7
|
+
*
|
|
8
|
+
* The unit of work, the unit of acceptance, the unit of retry and the unit of
|
|
9
|
+
* caching are all this: a segment. Before this existed they were all "the whole
|
|
10
|
+
* document", while the unit of *failure* was a single placeholder — so a
|
|
11
|
+
* document with 871 placeholders had to win 871 coin tosses at once, and losing
|
|
12
|
+
* any one of them threw away every correct one along with it.
|
|
13
|
+
*
|
|
14
|
+
* Cutting is done by code rather than by a model, and it is deterministic: the
|
|
15
|
+
* same source and the same parameters produce byte-identical segments. That is
|
|
16
|
+
* not tidiness, it is what the segment cache stands on — a segment whose source
|
|
17
|
+
* has not changed is recognised by its hash, and a hash is only meaningful if
|
|
18
|
+
* the cut is reproducible. Any change to the algorithm or its defaults must
|
|
19
|
+
* bump {@link SEGMENTER_VERSION}, which invalidates every recorded segment.
|
|
20
|
+
*/
|
|
21
|
+
/**
|
|
22
|
+
* Bumped whenever the cut changes.
|
|
23
|
+
*
|
|
24
|
+
* Recorded in each translation's `i18nSegments`; a translation recorded under a
|
|
25
|
+
* different version is retranslated whole rather than trusted.
|
|
26
|
+
*/
|
|
27
|
+
export const SEGMENTER_VERSION = 1;
|
|
28
|
+
/**
|
|
29
|
+
* How large a segment may get, in masked characters.
|
|
30
|
+
*
|
|
31
|
+
* Roughly 4k tokens of source, whose translation is far inside the 32k
|
|
32
|
+
* single-response cap. Measured against the four documents from the incident:
|
|
33
|
+
* at this cap the hardest segment carries 154 placeholders, below the 307 of
|
|
34
|
+
* the one document that reliably succeeded whole, and the median segment
|
|
35
|
+
* carries 25.
|
|
36
|
+
*/
|
|
37
|
+
export const DEFAULT_SEGMENT_CAP = 16_000;
|
|
38
|
+
/**
|
|
39
|
+
* The size past which a single indivisible block fails the document.
|
|
40
|
+
*
|
|
41
|
+
* A block that is over {@link DEFAULT_SEGMENT_CAP} and cannot be drilled into
|
|
42
|
+
* — a huge table, a huge list — becomes a segment of its own and is allowed to
|
|
43
|
+
* exceed the cap. This is where that stops. Measured over 1030 documents: the
|
|
44
|
+
* largest such block is 24.3 KB, so this is a guard rail rather than a path.
|
|
45
|
+
*/
|
|
46
|
+
export const DEFAULT_SEGMENT_HARD_CAP = 48_000;
|
|
47
|
+
/**
|
|
48
|
+
* How large a segment must already be before a heading is allowed to end it.
|
|
49
|
+
*
|
|
50
|
+
* Without a floor, "start a new segment at every `##`" fragments a
|
|
51
|
+
* section-dense page into uselessly small pieces: measured over 1030
|
|
52
|
+
* documents, one CLI reference of 11,948 masked characters — a page that fits
|
|
53
|
+
* in a single segment by size — cut into 104 segments averaging 115
|
|
54
|
+
* characters. Each of those is a model call and a judge reading.
|
|
55
|
+
*
|
|
56
|
+
* Measured across the same corpus, this floor halves the work without
|
|
57
|
+
* coarsening the documents that matter: 3645 segments become 1750, the median
|
|
58
|
+
* segment grows from 484 to 1986 characters, and the document this design was
|
|
59
|
+
* written for is cut into 13 segments either way — it has real sections, so
|
|
60
|
+
* the floor never fires on it. The most-segmented document in the corpus stops
|
|
61
|
+
* being a page of one-line headings and becomes that one.
|
|
62
|
+
*
|
|
63
|
+
* Merging adjacent small sections costs the cache almost nothing (an edit to a
|
|
64
|
+
* 300-character section retranslates 600 characters instead of 300, while the
|
|
65
|
+
* rest of the page is still reused) and helps everywhere else: more context
|
|
66
|
+
* per call, and fewer of the very short texts where a reviewer is most likely
|
|
67
|
+
* to invent a finding.
|
|
68
|
+
*/
|
|
69
|
+
export const DEFAULT_SEGMENT_FLOOR = 2_000;
|
|
70
|
+
/** A block too large to send and impossible to divide. */
|
|
71
|
+
export class UnsplittableBlockError extends Error {
|
|
72
|
+
size;
|
|
73
|
+
hardCap;
|
|
74
|
+
label;
|
|
75
|
+
constructor(size, hardCap, label) {
|
|
76
|
+
super(`This document contains a single block of ${size} characters that cannot be divided (line ${label.line ?? '?'}${label.heading ? `, under “${label.heading}”` : ''}). The limit is ${hardCap}. Split it in the source — a table or list this large is also unreadable.`);
|
|
77
|
+
this.size = size;
|
|
78
|
+
this.hardCap = hardCap;
|
|
79
|
+
this.label = label;
|
|
80
|
+
this.name = 'UnsplittableBlockError';
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
/**
|
|
84
|
+
* Raised when the segments do not add up to the document.
|
|
85
|
+
*
|
|
86
|
+
* The first line of defence against the segmenter itself losing content: if
|
|
87
|
+
* every placeholder in the masked document is not accounted for exactly once
|
|
88
|
+
* across the segments and the held-out tags, the cut is wrong, and no amount of
|
|
89
|
+
* good translation downstream would reveal it.
|
|
90
|
+
*/
|
|
91
|
+
export class SegmentConservationError extends Error {
|
|
92
|
+
divergences;
|
|
93
|
+
constructor(divergences) {
|
|
94
|
+
super(`The segmenter did not conserve the document's placeholders:\n${divergences
|
|
95
|
+
.map((line) => ` - ${line}`)
|
|
96
|
+
.join('\n')}`);
|
|
97
|
+
this.divergences = divergences;
|
|
98
|
+
this.name = 'SegmentConservationError';
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
/** Raised when a translated segment cannot be put back where it came from. */
|
|
102
|
+
export class SegmentAssemblyError extends Error {
|
|
103
|
+
constructor(message) {
|
|
104
|
+
super(message);
|
|
105
|
+
this.name = 'SegmentAssemblyError';
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
/**
|
|
109
|
+
* Node types that stand on their own as a block.
|
|
110
|
+
*
|
|
111
|
+
* A container is only drilled into when all of its children are one of these:
|
|
112
|
+
* a segment is stringified as a document, and phrasing content is not a
|
|
113
|
+
* document. `<Note>some **text**</Note>` therefore stays whole, which is what
|
|
114
|
+
* we want for something that small anyway.
|
|
115
|
+
*/
|
|
116
|
+
const FLOW_TYPES = new Set([
|
|
117
|
+
'blockquote',
|
|
118
|
+
'code',
|
|
119
|
+
'definition',
|
|
120
|
+
'footnoteDefinition',
|
|
121
|
+
'heading',
|
|
122
|
+
'html',
|
|
123
|
+
'list',
|
|
124
|
+
'mdxFlowExpression',
|
|
125
|
+
'mdxJsxFlowElement',
|
|
126
|
+
'paragraph',
|
|
127
|
+
'table',
|
|
128
|
+
'thematicBreak',
|
|
129
|
+
'yaml',
|
|
130
|
+
]);
|
|
131
|
+
const isDrillable = (node) => node.type === 'mdxJsxFlowElement' &&
|
|
132
|
+
node.children.length > 0 &&
|
|
133
|
+
node.children.every((child) => FLOW_TYPES.has(child.type));
|
|
134
|
+
/** The attributes on a tag that carry prose, and so still need translating. */
|
|
135
|
+
const translatableAttributes = (node) => node.attributes.filter((attr) => attr.type === 'mdxJsxAttribute' &&
|
|
136
|
+
typeof attr.value === 'string' &&
|
|
137
|
+
attr.value.trim() !== '' &&
|
|
138
|
+
isTranslatableJsxAttr(node.name, attr.name));
|
|
139
|
+
/** Everything a tag's attributes hold, as one string, for counting placeholders. */
|
|
140
|
+
const attributeText = (node) => node.attributes
|
|
141
|
+
.map((attr) => {
|
|
142
|
+
if (attr.type === 'mdxJsxExpressionAttribute') {
|
|
143
|
+
return attr.value;
|
|
144
|
+
}
|
|
145
|
+
if (attr.value == null) {
|
|
146
|
+
return '';
|
|
147
|
+
}
|
|
148
|
+
return typeof attr.value === 'string' ? attr.value : attr.value.value;
|
|
149
|
+
})
|
|
150
|
+
.join('\n');
|
|
151
|
+
const addCounts = (into, from) => {
|
|
152
|
+
for (const [token, count] of from) {
|
|
153
|
+
into.set(token, (into.get(token) ?? 0) + count);
|
|
154
|
+
}
|
|
155
|
+
};
|
|
156
|
+
const shaOf = (kind, text) => crypto
|
|
157
|
+
.createHash('sha256')
|
|
158
|
+
.update(`${kind}\0${text}`)
|
|
159
|
+
.digest('hex')
|
|
160
|
+
.slice(0, 12);
|
|
161
|
+
/** A heading's own words, with placeholders taken out — this is a label, not content. */
|
|
162
|
+
const headingText = (node) => {
|
|
163
|
+
if (node.type !== 'heading') {
|
|
164
|
+
return undefined;
|
|
165
|
+
}
|
|
166
|
+
const words = [];
|
|
167
|
+
const walk = (value) => {
|
|
168
|
+
if (Array.isArray(value)) {
|
|
169
|
+
value.forEach(walk);
|
|
170
|
+
return;
|
|
171
|
+
}
|
|
172
|
+
if (!value || typeof value !== 'object') {
|
|
173
|
+
return;
|
|
174
|
+
}
|
|
175
|
+
const node_ = value;
|
|
176
|
+
if (typeof node_.value === 'string' && node_.type === 'text') {
|
|
177
|
+
words.push(node_.value);
|
|
178
|
+
}
|
|
179
|
+
walk(node_.children);
|
|
180
|
+
};
|
|
181
|
+
walk(node.children);
|
|
182
|
+
const text = words.join('').trim();
|
|
183
|
+
return text === '' ? undefined : text;
|
|
184
|
+
};
|
|
185
|
+
/** The first heading a run of blocks establishes, if any. */
|
|
186
|
+
const firstHeadingIn = (children) => {
|
|
187
|
+
for (const child of children) {
|
|
188
|
+
if (child.type === 'heading' && child.depth <= 3) {
|
|
189
|
+
const text = headingText(child);
|
|
190
|
+
if (text) {
|
|
191
|
+
return { heading: text, headingLine: child.position?.start.line };
|
|
192
|
+
}
|
|
193
|
+
}
|
|
194
|
+
}
|
|
195
|
+
return undefined;
|
|
196
|
+
};
|
|
197
|
+
/** Every heading on the page, in order — the map a segment is placed on. */
|
|
198
|
+
export const documentOutline = (tree) => {
|
|
199
|
+
const headings = [];
|
|
200
|
+
const walk = (children) => {
|
|
201
|
+
for (const child of children) {
|
|
202
|
+
if (child.type === 'heading') {
|
|
203
|
+
const text = headingText(child);
|
|
204
|
+
if (text) {
|
|
205
|
+
headings.push({
|
|
206
|
+
depth: child.depth,
|
|
207
|
+
text,
|
|
208
|
+
line: child.position?.start.line,
|
|
209
|
+
});
|
|
210
|
+
}
|
|
211
|
+
}
|
|
212
|
+
if ('children' in child && Array.isArray(child.children)) {
|
|
213
|
+
walk(child.children);
|
|
214
|
+
}
|
|
215
|
+
}
|
|
216
|
+
};
|
|
217
|
+
walk(tree.children);
|
|
218
|
+
return headings;
|
|
219
|
+
};
|
|
220
|
+
/**
|
|
221
|
+
* Cuts a masked document into segments.
|
|
222
|
+
*
|
|
223
|
+
* Three rules, applied in order, all of them deterministic:
|
|
224
|
+
*
|
|
225
|
+
* 1. **an `##` heading starts a new segment**, once the open one has reached
|
|
226
|
+
* {@link DEFAULT_SEGMENT_FLOOR}, so segments line up with the sections an
|
|
227
|
+
* author edits — which is what makes the cache hit after an ordinary edit,
|
|
228
|
+
* and what makes a failure report nameable — without a page of one-line
|
|
229
|
+
* sections turning into a page of one-line segments;
|
|
230
|
+
* 2. **the cap closes a segment**, greedily;
|
|
231
|
+
* 3. **a container larger than the cap is drilled into** rather than sent
|
|
232
|
+
* whole. The 24.9 KB `<Tabs>` in the document that failed is four ordinary
|
|
233
|
+
* segments under this rule. Its tag is held by the assembler; the prose on
|
|
234
|
+
* the tag — a `<Tab label>` — becomes a segment of its own, because
|
|
235
|
+
* otherwise it would be the one piece of the document nobody translates.
|
|
236
|
+
*
|
|
237
|
+
* The frontmatter is never a segment of its own: it joins the segment that
|
|
238
|
+
* follows it, so the model sees `title` and `description` in the context of the
|
|
239
|
+
* page they belong to.
|
|
240
|
+
*/
|
|
241
|
+
export const planSegments = ({ tree, processor, cap = DEFAULT_SEGMENT_CAP, hardCap = DEFAULT_SEGMENT_HARD_CAP, floor = Math.min(DEFAULT_SEGMENT_FLOOR, cap), }) => {
|
|
242
|
+
const segments = [];
|
|
243
|
+
const heldOut = new Map();
|
|
244
|
+
const stringifyBlocks = (children) => processor.stringify({ type: 'root', children: [...children] });
|
|
245
|
+
const push = (address, text, label) => {
|
|
246
|
+
segments.push({
|
|
247
|
+
index: segments.length,
|
|
248
|
+
address,
|
|
249
|
+
text,
|
|
250
|
+
expected: countPlaceholders(text),
|
|
251
|
+
sha: shaOf(address.kind, text),
|
|
252
|
+
label,
|
|
253
|
+
});
|
|
254
|
+
};
|
|
255
|
+
/** The section heading in force, carried down into drilled containers. */
|
|
256
|
+
const section = {};
|
|
257
|
+
const walk = (children, container) => {
|
|
258
|
+
let start = -1;
|
|
259
|
+
let size = 0;
|
|
260
|
+
/** Whether the open group holds anything but frontmatter. */
|
|
261
|
+
let hasContent = false;
|
|
262
|
+
let label = {};
|
|
263
|
+
const flush = (end) => {
|
|
264
|
+
if (start < 0 || end <= start) {
|
|
265
|
+
return;
|
|
266
|
+
}
|
|
267
|
+
const slice = children.slice(start, end);
|
|
268
|
+
push({ kind: 'blocks', container, start, end }, stringifyBlocks(slice),
|
|
269
|
+
// A segment that opened before any heading was in effect — the one
|
|
270
|
+
// holding the frontmatter — takes the first heading it contains. Its
|
|
271
|
+
// section is the page's own title, and without this it is the one
|
|
272
|
+
// segment with no place in the outline.
|
|
273
|
+
label.heading ? label : { ...label, ...firstHeadingIn(slice) });
|
|
274
|
+
start = -1;
|
|
275
|
+
size = 0;
|
|
276
|
+
hasContent = false;
|
|
277
|
+
};
|
|
278
|
+
for (const [index, child] of children.entries()) {
|
|
279
|
+
const heading = headingText(child);
|
|
280
|
+
if (heading && child.type === 'heading' && child.depth <= 3) {
|
|
281
|
+
section.heading = heading;
|
|
282
|
+
section.headingLine = child.position?.start.line;
|
|
283
|
+
}
|
|
284
|
+
const labelHere = {
|
|
285
|
+
line: child.position?.start.line,
|
|
286
|
+
heading: section.heading,
|
|
287
|
+
headingLine: section.headingLine,
|
|
288
|
+
};
|
|
289
|
+
const text = stringifyBlocks([child]);
|
|
290
|
+
if (text.length > cap && isDrillable(child)) {
|
|
291
|
+
flush(index);
|
|
292
|
+
const path = [...container, index];
|
|
293
|
+
// The tag stays behind: its placeholders are never asked for back, so
|
|
294
|
+
// they are recorded as held out rather than quietly dropped.
|
|
295
|
+
addCounts(heldOut, countPlaceholders(attributeText(child)));
|
|
296
|
+
const attrs = translatableAttributes(child);
|
|
297
|
+
if (attrs.length > 0) {
|
|
298
|
+
push({ kind: 'attributes', container: path }, stringifyBlocks([attributeElement(child, attrs)]), labelHere);
|
|
299
|
+
}
|
|
300
|
+
walk(child.children, path);
|
|
301
|
+
continue;
|
|
302
|
+
}
|
|
303
|
+
if (text.length > hardCap) {
|
|
304
|
+
throw new UnsplittableBlockError(text.length, hardCap, labelHere);
|
|
305
|
+
}
|
|
306
|
+
if (hasContent) {
|
|
307
|
+
if (child.type === 'heading' && child.depth === 2 && size >= floor) {
|
|
308
|
+
flush(index);
|
|
309
|
+
}
|
|
310
|
+
else if (size + text.length > cap) {
|
|
311
|
+
flush(index);
|
|
312
|
+
}
|
|
313
|
+
}
|
|
314
|
+
if (start < 0) {
|
|
315
|
+
start = index;
|
|
316
|
+
label = labelHere;
|
|
317
|
+
}
|
|
318
|
+
size += text.length;
|
|
319
|
+
if (child.type !== 'yaml') {
|
|
320
|
+
hasContent = true;
|
|
321
|
+
}
|
|
322
|
+
// A block that is over the cap on its own — an indivisible table or list
|
|
323
|
+
// — closes its segment immediately rather than dragging the next one in.
|
|
324
|
+
if (size > cap && hasContent) {
|
|
325
|
+
flush(index + 1);
|
|
326
|
+
}
|
|
327
|
+
}
|
|
328
|
+
flush(children.length);
|
|
329
|
+
};
|
|
330
|
+
walk(tree.children, []);
|
|
331
|
+
assertConservation({ segments, heldOut, cap, version: SEGMENTER_VERSION }, processor.stringify(tree));
|
|
332
|
+
return { segments, heldOut, cap, version: SEGMENTER_VERSION };
|
|
333
|
+
};
|
|
334
|
+
/** A tag reduced to the prose on it — a standalone element the model can translate. */
|
|
335
|
+
const attributeElement = (node, attrs) => ({
|
|
336
|
+
type: 'mdxJsxFlowElement',
|
|
337
|
+
name: node.name,
|
|
338
|
+
attributes: attrs.map((attr) => ({
|
|
339
|
+
type: 'mdxJsxAttribute',
|
|
340
|
+
name: attr.name,
|
|
341
|
+
value: attr.value,
|
|
342
|
+
})),
|
|
343
|
+
children: [],
|
|
344
|
+
});
|
|
345
|
+
/**
|
|
346
|
+
* Proves the cut lost nothing.
|
|
347
|
+
*
|
|
348
|
+
* Counts every placeholder in the whole masked document and every placeholder
|
|
349
|
+
* across the segments plus the held-out tags, and requires them to be equal.
|
|
350
|
+
* A segmenter that dropped a block, double-counted one, or cut a placeholder in
|
|
351
|
+
* half fails here — before a single model call is spent, and long before the
|
|
352
|
+
* damage could be mistaken for a bad translation.
|
|
353
|
+
*/
|
|
354
|
+
const assertConservation = (plan, maskedSource) => {
|
|
355
|
+
const expected = countPlaceholders(maskedSource);
|
|
356
|
+
const actual = new Map(plan.heldOut);
|
|
357
|
+
for (const segment of plan.segments) {
|
|
358
|
+
addCounts(actual, segment.expected);
|
|
359
|
+
}
|
|
360
|
+
const divergences = [];
|
|
361
|
+
for (const [token, count] of expected) {
|
|
362
|
+
const found = actual.get(token) ?? 0;
|
|
363
|
+
if (found !== count) {
|
|
364
|
+
divergences.push(`${token}: the document has ${count}, the segments account for ${found}`);
|
|
365
|
+
}
|
|
366
|
+
}
|
|
367
|
+
for (const [token, count] of actual) {
|
|
368
|
+
if (!expected.has(token)) {
|
|
369
|
+
divergences.push(`${token}: the segments account for ${count}, the document has none`);
|
|
370
|
+
}
|
|
371
|
+
}
|
|
372
|
+
if (divergences.length > 0) {
|
|
373
|
+
throw new SegmentConservationError(divergences);
|
|
374
|
+
}
|
|
375
|
+
};
|
|
376
|
+
export const definitionContext = (tree, processor) => {
|
|
377
|
+
const definitions = [];
|
|
378
|
+
visit(tree, ['definition', 'footnoteDefinition'], (node) => {
|
|
379
|
+
definitions.push(node);
|
|
380
|
+
});
|
|
381
|
+
if (definitions.length === 0) {
|
|
382
|
+
return undefined;
|
|
383
|
+
}
|
|
384
|
+
const text = processor.stringify({ type: 'root', children: definitions });
|
|
385
|
+
let blocks;
|
|
386
|
+
try {
|
|
387
|
+
blocks = processor.parse(text).children.length;
|
|
388
|
+
}
|
|
389
|
+
catch {
|
|
390
|
+
return undefined;
|
|
391
|
+
}
|
|
392
|
+
return blocks > 0 ? { text, blocks } : undefined;
|
|
393
|
+
};
|
|
394
|
+
/**
|
|
395
|
+
* Parses one segment's worth of markdown with the document's definitions in
|
|
396
|
+
* scope, and gives back only the segment's own blocks.
|
|
397
|
+
*
|
|
398
|
+
* Shared by the two places that take a segment out of a document and parse it
|
|
399
|
+
* on its own — assembly, which parses each translated segment before splicing
|
|
400
|
+
* it back, and the segment cache, which parses the previous translation's
|
|
401
|
+
* blocks to check whether they can be reused. Both were losing references the
|
|
402
|
+
* same way, so they resolve them the same way.
|
|
403
|
+
*/
|
|
404
|
+
export const parseWithDefinitions = (text, context, processor) => {
|
|
405
|
+
const alone = processor.parse(text).children;
|
|
406
|
+
if (!context) {
|
|
407
|
+
return alone;
|
|
408
|
+
}
|
|
409
|
+
const withContext = processor.parse(`${text}\n\n${context.text}`).children;
|
|
410
|
+
// Anything but an exact match means the appended context merged with the
|
|
411
|
+
// text's own last block, and slicing would eat part of the segment. Parsing
|
|
412
|
+
// it alone is then the safe answer — no worse than not having tried.
|
|
413
|
+
// Documents with no definitions never reach here, so the second parse is paid
|
|
414
|
+
// only by the documents it is for.
|
|
415
|
+
return withContext.length === alone.length + context.blocks
|
|
416
|
+
? withContext.slice(0, alone.length)
|
|
417
|
+
: alone;
|
|
418
|
+
};
|
|
419
|
+
export const assemble = ({ tree, plan, processor, translations, }) => {
|
|
420
|
+
const blocksAt = new Map();
|
|
421
|
+
const attrsAt = new Map();
|
|
422
|
+
const drilled = new Set();
|
|
423
|
+
for (const segment of plan.segments) {
|
|
424
|
+
const { address } = segment;
|
|
425
|
+
if (address.kind === 'attributes') {
|
|
426
|
+
attrsAt.set(address.container.join('.'), segment);
|
|
427
|
+
continue;
|
|
428
|
+
}
|
|
429
|
+
blocksAt.set(`${address.container.join('.')}#${address.start}`, segment);
|
|
430
|
+
}
|
|
431
|
+
// A container is drilled into if any segment addresses something inside it.
|
|
432
|
+
for (const segment of plan.segments) {
|
|
433
|
+
const path = segment.address.container;
|
|
434
|
+
for (let depth = 1; depth <= path.length; depth++) {
|
|
435
|
+
drilled.add(path.slice(0, depth).join('.'));
|
|
436
|
+
}
|
|
437
|
+
}
|
|
438
|
+
const placed = new Map();
|
|
439
|
+
const context = definitionContext(tree, processor);
|
|
440
|
+
const parseBlocks = (segment) => {
|
|
441
|
+
const translation = translations[segment.index];
|
|
442
|
+
if (typeof translation !== 'string') {
|
|
443
|
+
throw new SegmentAssemblyError(`Segment ${segment.index} has no accepted translation.`);
|
|
444
|
+
}
|
|
445
|
+
try {
|
|
446
|
+
return parseWithDefinitions(translation, context, processor);
|
|
447
|
+
}
|
|
448
|
+
catch (error) {
|
|
449
|
+
throw new SegmentAssemblyError(`Segment ${segment.index}'s translation does not parse: ${error instanceof Error ? error.message : String(error)}`);
|
|
450
|
+
}
|
|
451
|
+
};
|
|
452
|
+
const rebuild = (children, sourcePath, targetPath) => {
|
|
453
|
+
const out = [];
|
|
454
|
+
let index = 0;
|
|
455
|
+
while (index < children.length) {
|
|
456
|
+
const segment = blocksAt.get(`${sourcePath.join('.')}#${index}`);
|
|
457
|
+
if (segment) {
|
|
458
|
+
const blocks = parseBlocks(segment);
|
|
459
|
+
placed.set(segment.index, {
|
|
460
|
+
kind: 'blocks',
|
|
461
|
+
container: targetPath,
|
|
462
|
+
start: out.length,
|
|
463
|
+
end: out.length + blocks.length,
|
|
464
|
+
});
|
|
465
|
+
out.push(...blocks);
|
|
466
|
+
index = segment.address.end;
|
|
467
|
+
continue;
|
|
468
|
+
}
|
|
469
|
+
const child = children[index];
|
|
470
|
+
const path = [...sourcePath, index];
|
|
471
|
+
if (drilled.has(path.join('.')) && isDrillable(child)) {
|
|
472
|
+
const here = [...targetPath, out.length];
|
|
473
|
+
const attrs = attrsAt.get(path.join('.'));
|
|
474
|
+
const clone = {
|
|
475
|
+
...child,
|
|
476
|
+
attributes: [...child.attributes],
|
|
477
|
+
children: [],
|
|
478
|
+
};
|
|
479
|
+
if (attrs) {
|
|
480
|
+
applyAttributes(clone, translations[attrs.index], processor);
|
|
481
|
+
placed.set(attrs.index, { kind: 'attributes', container: here });
|
|
482
|
+
}
|
|
483
|
+
clone.children = rebuild(child.children, path, here);
|
|
484
|
+
out.push(clone);
|
|
485
|
+
index++;
|
|
486
|
+
continue;
|
|
487
|
+
}
|
|
488
|
+
throw new SegmentAssemblyError(`No segment covers child ${index} of container [${sourcePath.join('.')}] — the plan does not cover the document.`);
|
|
489
|
+
}
|
|
490
|
+
return out;
|
|
491
|
+
};
|
|
492
|
+
const assembled = { ...tree, children: rebuild(tree.children, [], []) };
|
|
493
|
+
const text = processor.stringify(assembled);
|
|
494
|
+
const shift = assembled.children[0]?.type === 'yaml' ? 1 : 0;
|
|
495
|
+
const toBody = (address) => address.kind === 'attributes'
|
|
496
|
+
? { ...address, container: shiftPath(address.container, shift) }
|
|
497
|
+
: {
|
|
498
|
+
...address,
|
|
499
|
+
container: shiftPath(address.container, shift),
|
|
500
|
+
start: address.container.length === 0
|
|
501
|
+
? Math.max(0, address.start - shift)
|
|
502
|
+
: address.start,
|
|
503
|
+
end: address.container.length === 0
|
|
504
|
+
? Math.max(0, address.end - shift)
|
|
505
|
+
: address.end,
|
|
506
|
+
};
|
|
507
|
+
const placements = [];
|
|
508
|
+
const records = [];
|
|
509
|
+
for (const segment of plan.segments) {
|
|
510
|
+
const address = placed.get(segment.index);
|
|
511
|
+
if (!address) {
|
|
512
|
+
continue;
|
|
513
|
+
}
|
|
514
|
+
const body = toBody(address);
|
|
515
|
+
placements.push({ segment: segment.index, address: body });
|
|
516
|
+
const carriesFrontmatter = shift === 1 &&
|
|
517
|
+
address.kind === 'blocks' &&
|
|
518
|
+
address.container.length === 0 &&
|
|
519
|
+
address.start === 0;
|
|
520
|
+
if (!carriesFrontmatter) {
|
|
521
|
+
records.push({ sha: segment.sha, address: body });
|
|
522
|
+
}
|
|
523
|
+
}
|
|
524
|
+
return { text, placements, records };
|
|
525
|
+
};
|
|
526
|
+
/** The frontmatter block only shifts the top level; nested containers are unaffected. */
|
|
527
|
+
const shiftPath = (path, shift) => path.length === 0 ? path : [path[0] - shift, ...path.slice(1)];
|
|
528
|
+
/**
|
|
529
|
+
* The children a placement addresses, in a parsed translation.
|
|
530
|
+
*
|
|
531
|
+
* The one place that knows how a recorded address is read back — used both to
|
|
532
|
+
* pull a cached segment out of the previous translation and to work out which
|
|
533
|
+
* lines of the composed document a segment occupies. `shift` accounts for the
|
|
534
|
+
* frontmatter, which is block 0 of a whole file and absent from a body.
|
|
535
|
+
*/
|
|
536
|
+
export const resolvePlacement = (tree, address, shift = 0) => {
|
|
537
|
+
let children = tree.children;
|
|
538
|
+
const [first, ...rest] = address.container;
|
|
539
|
+
if (address.container.length > 0) {
|
|
540
|
+
// `.at` rather than `[]`: an address can point past the end of a document
|
|
541
|
+
// that has been edited since it was written, and that has to come back as
|
|
542
|
+
// "not there" rather than as `undefined` typed as a node.
|
|
543
|
+
const node = children.at(first + shift);
|
|
544
|
+
if (!node || !('children' in node) || !Array.isArray(node.children)) {
|
|
545
|
+
return undefined;
|
|
546
|
+
}
|
|
547
|
+
children = node.children;
|
|
548
|
+
for (const index of rest) {
|
|
549
|
+
const next = children.at(index);
|
|
550
|
+
if (!next || !('children' in next) || !Array.isArray(next.children)) {
|
|
551
|
+
return undefined;
|
|
552
|
+
}
|
|
553
|
+
children = next.children;
|
|
554
|
+
}
|
|
555
|
+
}
|
|
556
|
+
if (address.kind === 'attributes') {
|
|
557
|
+
// An attribute placement names a node, not a range — see
|
|
558
|
+
// {@link resolveAttributeNode}.
|
|
559
|
+
return undefined;
|
|
560
|
+
}
|
|
561
|
+
const offset = address.container.length === 0 ? shift : 0;
|
|
562
|
+
const nodes = children.slice(address.start + offset, address.end + offset);
|
|
563
|
+
return nodes.length === address.end - address.start
|
|
564
|
+
? { parent: children, nodes }
|
|
565
|
+
: undefined;
|
|
566
|
+
};
|
|
567
|
+
/** The node an `attributes` placement addresses. */
|
|
568
|
+
export const resolveAttributeNode = (tree, address, shift = 0) => {
|
|
569
|
+
if (address.kind !== 'attributes') {
|
|
570
|
+
return undefined;
|
|
571
|
+
}
|
|
572
|
+
let node;
|
|
573
|
+
let children = tree.children;
|
|
574
|
+
for (const [depth, index] of address.container.entries()) {
|
|
575
|
+
node = children.at(index + (depth === 0 ? shift : 0));
|
|
576
|
+
if (!node) {
|
|
577
|
+
return undefined;
|
|
578
|
+
}
|
|
579
|
+
children =
|
|
580
|
+
'children' in node && Array.isArray(node.children) ? node.children : [];
|
|
581
|
+
}
|
|
582
|
+
return node?.type === 'mdxJsxFlowElement' ? node : undefined;
|
|
583
|
+
};
|
|
584
|
+
/**
|
|
585
|
+
* Copies the translated prose back onto a tag.
|
|
586
|
+
*
|
|
587
|
+
* Strict about shape: the translation of an attribute segment is one JSX
|
|
588
|
+
* element, and its attributes are the ones that were sent. Anything else is an
|
|
589
|
+
* assembly error rather than a silent fallback to the source language, which is
|
|
590
|
+
* the failure this whole design exists to stop happening quietly.
|
|
591
|
+
*/
|
|
592
|
+
export const applyAttributes = (node, translation, processor) => {
|
|
593
|
+
if (typeof translation !== 'string') {
|
|
594
|
+
throw new SegmentAssemblyError(`The attributes of \`<${node.name ?? ''}>\` have no accepted translation.`);
|
|
595
|
+
}
|
|
596
|
+
const parsed = parseAttributeElement(translation, processor);
|
|
597
|
+
if (!parsed) {
|
|
598
|
+
throw new SegmentAssemblyError(`The translation of \`<${node.name ?? ''}>\`'s attributes is not a single JSX element.`);
|
|
599
|
+
}
|
|
600
|
+
const translated = new Map(parsed.attributes
|
|
601
|
+
.filter((attr) => attr.type === 'mdxJsxAttribute' && typeof attr.value === 'string')
|
|
602
|
+
.map((attr) => [attr.name, attr.value]));
|
|
603
|
+
node.attributes = node.attributes.map((attr) => {
|
|
604
|
+
if (attr.type !== 'mdxJsxAttribute' || !translated.has(attr.name)) {
|
|
605
|
+
return attr;
|
|
606
|
+
}
|
|
607
|
+
return { ...attr, value: translated.get(attr.name) };
|
|
608
|
+
});
|
|
609
|
+
};
|
|
610
|
+
/** The single JSX element an attribute segment must be, or nothing. */
|
|
611
|
+
export const parseAttributeElement = (translation, processor) => {
|
|
612
|
+
let root;
|
|
613
|
+
try {
|
|
614
|
+
root = processor.parse(translation);
|
|
615
|
+
}
|
|
616
|
+
catch {
|
|
617
|
+
return undefined;
|
|
618
|
+
}
|
|
619
|
+
const blocks = root.children.filter((child) => child.type !== 'yaml' && !isBlank(child));
|
|
620
|
+
const only = blocks.at(0);
|
|
621
|
+
return blocks.length === 1 && only?.type === 'mdxJsxFlowElement'
|
|
622
|
+
? only
|
|
623
|
+
: undefined;
|
|
624
|
+
};
|
|
625
|
+
const isBlank = (node) => node.type === 'paragraph' && node.children.length === 0;
|
package/lib/cli/translate.d.ts
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { Command } from 'commander';
|
|
2
2
|
import { Language } from '../shared/index.js';
|
|
3
|
+
import type { SegmentCacheRecord } from './translate-segment-cache.js';
|
|
3
4
|
/**
|
|
4
5
|
* `doom translate` — orchestration and the gate.
|
|
5
6
|
*
|
|
@@ -9,10 +10,10 @@ import { Language } from '../shared/index.js';
|
|
|
9
10
|
* deterministic rewrites (`normalizeImgSrc`, `translateCodeFile`) that are ours
|
|
10
11
|
* to make rather than a model's.
|
|
11
12
|
*
|
|
12
|
-
* Translating a document does not live here at all. That is a
|
|
13
|
-
* `translate-
|
|
14
|
-
* no single-shot fallback remains, because a protection that can be
|
|
15
|
-
* one that gets skipped.
|
|
13
|
+
* Translating a document does not live here at all. That is a segment pipeline,
|
|
14
|
+
* in `translate-pipeline.ts`. There is one path through it: no flag turns it
|
|
15
|
+
* off and no single-shot fallback remains, because a protection that can be
|
|
16
|
+
* skipped is one that gets skipped.
|
|
16
17
|
*/
|
|
17
18
|
export interface I18nFrontmatter {
|
|
18
19
|
i18n?: {
|
|
@@ -20,6 +21,15 @@ export interface I18nFrontmatter {
|
|
|
20
21
|
disableAutoTranslation?: boolean;
|
|
21
22
|
};
|
|
22
23
|
sourceSHA?: string;
|
|
24
|
+
/**
|
|
25
|
+
* Where each of this translation's segments came from and where it went.
|
|
26
|
+
*
|
|
27
|
+
* The translator's own bookkeeping, like `sourceSHA` beside it: it lets the
|
|
28
|
+
* next run reuse the segments whose source has not changed instead of
|
|
29
|
+
* retranslating a whole page because one line of it moved. Nothing renders
|
|
30
|
+
* it, and `translation-frontmatter-preservation` knows it is ours.
|
|
31
|
+
*/
|
|
32
|
+
i18nSegments?: SegmentCacheRecord;
|
|
23
33
|
title?: string;
|
|
24
34
|
description?: string;
|
|
25
35
|
}
|