@alauda/doom 2.6.0 → 2.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/cli/translate-call.d.ts +85 -0
- package/lib/cli/translate-call.js +149 -0
- package/lib/cli/translate-checker.d.ts +8 -0
- package/lib/cli/translate-diagnose.d.ts +45 -0
- package/lib/cli/translate-diagnose.js +118 -0
- package/lib/cli/translate-judge.d.ts +28 -3
- package/lib/cli/translate-judge.js +40 -7
- package/lib/cli/translate-mask.d.ts +13 -0
- package/lib/cli/translate-mask.js +19 -5
- package/lib/cli/translate-pipeline.d.ts +144 -0
- package/lib/cli/translate-pipeline.js +489 -0
- package/lib/cli/translate-repair-agent.d.ts +69 -0
- package/lib/cli/translate-repair-agent.js +241 -0
- package/lib/cli/translate-scratch.d.ts +9 -1
- package/lib/cli/translate-scratch.js +2 -2
- package/lib/cli/translate-segment-cache.d.ts +90 -0
- package/lib/cli/translate-segment-cache.js +301 -0
- package/lib/cli/translate-segment-check.d.ts +42 -0
- package/lib/cli/translate-segment-check.js +247 -0
- package/lib/cli/translate-segment.d.ts +307 -0
- package/lib/cli/translate-segment.js +625 -0
- package/lib/cli/translate.d.ts +14 -4
- package/lib/cli/translate.js +232 -52
- package/lib/remark-lint/no-unmatched-anchor.js +6 -2
- package/lib/remark-lint/no-unopened-frontmatter.js +5 -1
- package/lib/remark-lint/rule-sets.d.ts +1 -3
- package/lib/remark-lint/rule-sets.js +3 -9
- package/lib/remark-lint/syntax-plugins.d.ts +32 -0
- package/lib/remark-lint/syntax-plugins.js +46 -0
- package/lib/remark-lint/translation-parity/frontmatter-preservation.js +5 -3
- package/lib/remark-lint/translation-parity/link-isomorphism.d.ts +8 -0
- package/lib/remark-lint/translation-parity/link-isomorphism.js +40 -4
- package/lib/remark-lint/translation-parity/shared.js +6 -2
- package/lib/remark-lint/translation-parity/url-residue.d.ts +1 -0
- package/lib/remark-lint/translation-parity/url-residue.js +25 -4
- package/lib/runtime/hooks/useTranslation.d.ts +1 -1
- package/lib/types.d.ts +101 -6
- package/package.json +1 -1
- package/lib/cli/translate-agent.d.ts +0 -81
- package/lib/cli/translate-agent.js +0 -450
|
@@ -0,0 +1,301 @@
|
|
|
1
|
+
import { isTranslatableJsxAttr } from "../runtime/components/_translation-policy.js";
|
|
2
|
+
import { countPlaceholders, maskAst, } from "./translate-mask.js";
|
|
3
|
+
import { definitionContext, parseWithDefinitions, resolveAttributeNode, resolvePlacement, } from "./translate-segment.js";
|
|
4
|
+
/**
|
|
5
|
+
* One record, as one short string.
|
|
6
|
+
*
|
|
7
|
+
* `<sha> <start>-<end>` at the top level, `<sha> <path>/<start>-<end>` inside a
|
|
8
|
+
* container, `<sha> <path>:attrs` for a tag's own words. Compact because it
|
|
9
|
+
* lives in every translated document's frontmatter, and readable because the
|
|
10
|
+
* first thing anyone does with an unexpected diff is read it.
|
|
11
|
+
*/
|
|
12
|
+
export const encodeRecord = ({ sha, address }) => {
|
|
13
|
+
const container = address.container.join('/');
|
|
14
|
+
if (address.kind === 'attributes') {
|
|
15
|
+
return `${sha} ${container}:attrs`;
|
|
16
|
+
}
|
|
17
|
+
const range = `${address.start}-${address.end}`;
|
|
18
|
+
return `${sha} ${container ? `${container}/${range}` : range}`;
|
|
19
|
+
};
|
|
20
|
+
export const decodeRecord = (entry) => {
|
|
21
|
+
const [sha, locator] = entry.trim().split(/\s+/u);
|
|
22
|
+
if (!sha || !locator || !/^[0-9a-f]{12}$/u.test(sha)) {
|
|
23
|
+
return undefined;
|
|
24
|
+
}
|
|
25
|
+
if (locator.endsWith(':attrs')) {
|
|
26
|
+
const path = locator.slice(0, -':attrs'.length);
|
|
27
|
+
const container = parsePath(path);
|
|
28
|
+
return container && container.length > 0
|
|
29
|
+
? { sha, address: { kind: 'attributes', container } }
|
|
30
|
+
: undefined;
|
|
31
|
+
}
|
|
32
|
+
const cut = locator.lastIndexOf('/');
|
|
33
|
+
const path = cut < 0 ? '' : locator.slice(0, cut);
|
|
34
|
+
const range = cut < 0 ? locator : locator.slice(cut + 1);
|
|
35
|
+
const match = /^(\d+)-(\d+)$/u.exec(range);
|
|
36
|
+
const container = parsePath(path);
|
|
37
|
+
if (!match || !container) {
|
|
38
|
+
return undefined;
|
|
39
|
+
}
|
|
40
|
+
const start = Number(match[1]);
|
|
41
|
+
const end = Number(match[2]);
|
|
42
|
+
return end < start
|
|
43
|
+
? undefined
|
|
44
|
+
: { sha, address: { kind: 'blocks', container, start, end } };
|
|
45
|
+
};
|
|
46
|
+
const parsePath = (path) => {
|
|
47
|
+
if (path === '') {
|
|
48
|
+
return [];
|
|
49
|
+
}
|
|
50
|
+
const parts = path.split('/').map(Number);
|
|
51
|
+
return parts.every((part) => Number.isInteger(part) && part >= 0)
|
|
52
|
+
? parts
|
|
53
|
+
: undefined;
|
|
54
|
+
};
|
|
55
|
+
export const encodeCacheRecord = (plan, records) => records.length === 0
|
|
56
|
+
? undefined
|
|
57
|
+
: { v: plan.version, cap: plan.cap, segs: records.map(encodeRecord) };
|
|
58
|
+
/**
|
|
59
|
+
* Which of this document's segments can be taken from the previous translation.
|
|
60
|
+
*
|
|
61
|
+
* The two hash sequences are matched with a longest-common-subsequence, so a
|
|
62
|
+
* segment inserted in the middle of a page shifts nothing after it and two
|
|
63
|
+
* identical segments cannot swap places. Every match is then verified, and only
|
|
64
|
+
* what survives is offered.
|
|
65
|
+
*/
|
|
66
|
+
export const matchCachedSegments = ({ plan, maskEntries, processor, record, previousBody, }) => {
|
|
67
|
+
const reuse = new Map();
|
|
68
|
+
if (!record) {
|
|
69
|
+
return { reuse, reason: 'the previous translation has no segment record' };
|
|
70
|
+
}
|
|
71
|
+
if (record.v !== plan.version) {
|
|
72
|
+
return {
|
|
73
|
+
reuse,
|
|
74
|
+
reason: `the previous translation was cut by segmenter v${record.v}, this run is v${plan.version}`,
|
|
75
|
+
};
|
|
76
|
+
}
|
|
77
|
+
if (record.cap !== plan.cap) {
|
|
78
|
+
return {
|
|
79
|
+
reuse,
|
|
80
|
+
reason: `the previous translation was cut at ${record.cap} characters, this run cuts at ${plan.cap}`,
|
|
81
|
+
};
|
|
82
|
+
}
|
|
83
|
+
const previousRecords = record.segs
|
|
84
|
+
.map(decodeRecord)
|
|
85
|
+
.filter((entry) => !!entry);
|
|
86
|
+
if (previousRecords.length !== record.segs.length) {
|
|
87
|
+
return { reuse, reason: 'the previous segment record is malformed' };
|
|
88
|
+
}
|
|
89
|
+
let tree;
|
|
90
|
+
try {
|
|
91
|
+
tree = processor.parse(previousBody);
|
|
92
|
+
}
|
|
93
|
+
catch {
|
|
94
|
+
return { reuse, reason: 'the previous translation does not parse' };
|
|
95
|
+
}
|
|
96
|
+
const entriesByToken = new Map(maskEntries.map((entry) => [entry.placeholder.toLowerCase(), entry]));
|
|
97
|
+
// Computed once for the document, not once per segment: it walks the whole
|
|
98
|
+
// previous translation, and every segment is asked about the same one.
|
|
99
|
+
const context = definitionContext(tree, processor);
|
|
100
|
+
for (const [index, previousIndex] of alignBySha(plan.segments.map((segment) => segment.sha), previousRecords.map((entry) => entry.sha))) {
|
|
101
|
+
const segment = plan.segments[index];
|
|
102
|
+
const candidate = extract({
|
|
103
|
+
tree,
|
|
104
|
+
address: previousRecords[previousIndex].address,
|
|
105
|
+
processor,
|
|
106
|
+
context,
|
|
107
|
+
});
|
|
108
|
+
if (candidate == null) {
|
|
109
|
+
continue;
|
|
110
|
+
}
|
|
111
|
+
const remasked = remask({
|
|
112
|
+
candidate,
|
|
113
|
+
segment,
|
|
114
|
+
entriesByToken,
|
|
115
|
+
processor,
|
|
116
|
+
});
|
|
117
|
+
if (remasked != null) {
|
|
118
|
+
reuse.set(index, remasked);
|
|
119
|
+
}
|
|
120
|
+
}
|
|
121
|
+
return { reuse };
|
|
122
|
+
};
|
|
123
|
+
/**
|
|
124
|
+
* The longest run of segments, in order, whose hashes agree.
|
|
125
|
+
*
|
|
126
|
+
* A plain two-pointer walk loses alignment for the rest of the page the moment
|
|
127
|
+
* one segment is inserted or removed. This is the standard subsequence instead:
|
|
128
|
+
* pairs never cross, so two identical segments cannot be matched to each
|
|
129
|
+
* other's positions, and an edit only costs the segment it touched.
|
|
130
|
+
*/
|
|
131
|
+
export const alignBySha = (current, previous) => {
|
|
132
|
+
const rows = current.length;
|
|
133
|
+
const columns = previous.length;
|
|
134
|
+
const lengths = Array.from({ length: rows + 1 }, () => Array.from({ length: columns + 1 }, () => 0));
|
|
135
|
+
for (let row = rows - 1; row >= 0; row--) {
|
|
136
|
+
for (let column = columns - 1; column >= 0; column--) {
|
|
137
|
+
lengths[row][column] =
|
|
138
|
+
current[row] === previous[column]
|
|
139
|
+
? lengths[row + 1][column + 1] + 1
|
|
140
|
+
: Math.max(lengths[row + 1][column], lengths[row][column + 1]);
|
|
141
|
+
}
|
|
142
|
+
}
|
|
143
|
+
const pairs = [];
|
|
144
|
+
let row = 0;
|
|
145
|
+
let column = 0;
|
|
146
|
+
while (row < rows && column < columns) {
|
|
147
|
+
if (current[row] === previous[column]) {
|
|
148
|
+
pairs.push([row, column]);
|
|
149
|
+
row++;
|
|
150
|
+
column++;
|
|
151
|
+
}
|
|
152
|
+
else if (lengths[row + 1][column] >= lengths[row][column + 1]) {
|
|
153
|
+
row++;
|
|
154
|
+
}
|
|
155
|
+
else {
|
|
156
|
+
column++;
|
|
157
|
+
}
|
|
158
|
+
}
|
|
159
|
+
return pairs;
|
|
160
|
+
};
|
|
161
|
+
/** The previous translation of one segment, as it sits on disk: restored, not masked. */
|
|
162
|
+
const extract = ({ tree, address, processor, context, }) => {
|
|
163
|
+
if (address.kind === 'attributes') {
|
|
164
|
+
const node = resolveAttributeNode(tree, address);
|
|
165
|
+
if (!node) {
|
|
166
|
+
return undefined;
|
|
167
|
+
}
|
|
168
|
+
const attrs = node.attributes.filter((attr) => attr.type === 'mdxJsxAttribute' &&
|
|
169
|
+
typeof attr.value === 'string' &&
|
|
170
|
+
isTranslatableJsxAttr(node.name, attr.name));
|
|
171
|
+
return attrs.length === 0
|
|
172
|
+
? undefined
|
|
173
|
+
: {
|
|
174
|
+
type: 'root',
|
|
175
|
+
children: [
|
|
176
|
+
{
|
|
177
|
+
type: 'mdxJsxFlowElement',
|
|
178
|
+
name: node.name,
|
|
179
|
+
attributes: attrs.map((attr) => ({ ...attr })),
|
|
180
|
+
children: [],
|
|
181
|
+
},
|
|
182
|
+
],
|
|
183
|
+
};
|
|
184
|
+
}
|
|
185
|
+
const resolved = resolvePlacement(tree, address);
|
|
186
|
+
if (!resolved) {
|
|
187
|
+
return undefined;
|
|
188
|
+
}
|
|
189
|
+
// Re-parsed rather than reused: masking mutates, and the tree these nodes
|
|
190
|
+
// came from is the caller's.
|
|
191
|
+
try {
|
|
192
|
+
return {
|
|
193
|
+
type: 'root',
|
|
194
|
+
children: parseWithDefinitions(processor.stringify({ type: 'root', children: resolved.nodes }), context, processor),
|
|
195
|
+
};
|
|
196
|
+
}
|
|
197
|
+
catch {
|
|
198
|
+
return undefined;
|
|
199
|
+
}
|
|
200
|
+
};
|
|
201
|
+
/**
|
|
202
|
+
* Puts this document's placeholders back into a translation that has none.
|
|
203
|
+
*
|
|
204
|
+
* The candidate is masked with {@link maskAst}, which numbers from zero for
|
|
205
|
+
* whatever it is given; those numbers mean nothing here. What does mean
|
|
206
|
+
* something is *what* was masked, so each candidate entry is matched to a
|
|
207
|
+
* segment entry of the same kind holding the same value, and the candidate's
|
|
208
|
+
* tokens are rewritten accordingly — in one pass, so a token that is both a
|
|
209
|
+
* source and a target of the rewrite cannot be replaced twice.
|
|
210
|
+
*
|
|
211
|
+
* Anything that does not line up exactly returns nothing, and the segment is
|
|
212
|
+
* translated again. That covers a hand-edited translation, a stale record, a
|
|
213
|
+
* link somebody fixed by hand in the target language — every way a cache entry
|
|
214
|
+
* can be quietly wrong.
|
|
215
|
+
*/
|
|
216
|
+
const remask = ({ candidate, segment, entriesByToken, processor, }) => {
|
|
217
|
+
const wanted = [...segment.expected.keys()]
|
|
218
|
+
.map((token) => entriesByToken.get(token))
|
|
219
|
+
.filter((entry) => !!entry);
|
|
220
|
+
if (wanted.length !== segment.expected.size) {
|
|
221
|
+
return undefined;
|
|
222
|
+
}
|
|
223
|
+
let candidateEntries;
|
|
224
|
+
let masked;
|
|
225
|
+
try {
|
|
226
|
+
candidateEntries = maskAst(candidate);
|
|
227
|
+
masked = processor.stringify(candidate);
|
|
228
|
+
}
|
|
229
|
+
catch {
|
|
230
|
+
// Includes the case where the previous translation already contains a
|
|
231
|
+
// reserved token, which is not something to paper over.
|
|
232
|
+
return undefined;
|
|
233
|
+
}
|
|
234
|
+
const available = new Map();
|
|
235
|
+
for (const entry of wanted) {
|
|
236
|
+
const key = valueKey(entry);
|
|
237
|
+
const list = available.get(key);
|
|
238
|
+
if (list) {
|
|
239
|
+
list.push(entry.placeholder);
|
|
240
|
+
}
|
|
241
|
+
else {
|
|
242
|
+
available.set(key, [entry.placeholder]);
|
|
243
|
+
}
|
|
244
|
+
}
|
|
245
|
+
const rewrite = new Map();
|
|
246
|
+
for (const entry of candidateEntries) {
|
|
247
|
+
const list = available.get(valueKey(entry));
|
|
248
|
+
const replacement = list?.shift();
|
|
249
|
+
if (!replacement) {
|
|
250
|
+
// The translation holds something this segment's source does not — it has
|
|
251
|
+
// been edited, or the record points somewhere else entirely.
|
|
252
|
+
return undefined;
|
|
253
|
+
}
|
|
254
|
+
rewrite.set(entry.placeholder.toLowerCase(), replacement);
|
|
255
|
+
}
|
|
256
|
+
for (const list of available.values()) {
|
|
257
|
+
if (list.length > 0) {
|
|
258
|
+
return undefined;
|
|
259
|
+
}
|
|
260
|
+
}
|
|
261
|
+
const remasked = masked.replace(/__DOOM_TR_[A-Z]+_\d+__/giu, (token) => {
|
|
262
|
+
return rewrite.get(token.toLowerCase()) ?? token;
|
|
263
|
+
});
|
|
264
|
+
// The last word is the same measurement the segment check would make.
|
|
265
|
+
const counts = countPlaceholders(remasked);
|
|
266
|
+
if (counts.size !== segment.expected.size) {
|
|
267
|
+
return undefined;
|
|
268
|
+
}
|
|
269
|
+
for (const [token, expected] of segment.expected) {
|
|
270
|
+
if (counts.get(token) !== expected) {
|
|
271
|
+
return undefined;
|
|
272
|
+
}
|
|
273
|
+
}
|
|
274
|
+
return remasked;
|
|
275
|
+
};
|
|
276
|
+
/** What a mask entry actually protects, as a comparable string. */
|
|
277
|
+
const valueKey = (entry) => {
|
|
278
|
+
switch (entry.kind) {
|
|
279
|
+
case 'DEF':
|
|
280
|
+
case 'IMG':
|
|
281
|
+
case 'LINK': {
|
|
282
|
+
return `${entry.kind}\0${entry.url}`;
|
|
283
|
+
}
|
|
284
|
+
case 'FNID':
|
|
285
|
+
case 'REFID': {
|
|
286
|
+
return `${entry.kind}\0${entry.identifier.toLowerCase()}`;
|
|
287
|
+
}
|
|
288
|
+
case 'CODE': {
|
|
289
|
+
return `CODE\0${entry.node.lang ?? ''}\0${entry.node.meta ?? ''}\0${entry.node.value}`;
|
|
290
|
+
}
|
|
291
|
+
case 'URL': {
|
|
292
|
+
return `URL\0${entry.node.url}`;
|
|
293
|
+
}
|
|
294
|
+
case 'JSXATTR': {
|
|
295
|
+
return `JSXATTR\0${entry.attrKind}\0${typeof entry.value === 'string' ? entry.value : entry.value.value}`;
|
|
296
|
+
}
|
|
297
|
+
default: {
|
|
298
|
+
return `${entry.kind}\0${entry.value}`;
|
|
299
|
+
}
|
|
300
|
+
}
|
|
301
|
+
};
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
import type { TranslationFinding } from './translate-checker.ts';
|
|
2
|
+
import type { Judge } from './translate-judge.ts';
|
|
3
|
+
import { type MaskProcessor } from './translate-mask.ts';
|
|
4
|
+
import { type Segment } from './translate-segment.ts';
|
|
5
|
+
import type { TermPair } from './translate-terms.ts';
|
|
6
|
+
export interface CheckSegmentOptions {
|
|
7
|
+
segment: Segment;
|
|
8
|
+
/** What the model returned for it. */
|
|
9
|
+
translation: string;
|
|
10
|
+
processor: MaskProcessor;
|
|
11
|
+
/**
|
|
12
|
+
* Every placeholder the document issued.
|
|
13
|
+
*
|
|
14
|
+
* Lets a token that belongs to a *different* segment be reported as what it
|
|
15
|
+
* is — content dragged in from a neighbour — rather than as an invented one.
|
|
16
|
+
* The two have different causes and different fixes.
|
|
17
|
+
*/
|
|
18
|
+
documentTokens?: ReadonlySet<string>;
|
|
19
|
+
/** The semantic check. Absent in tests that are about control flow. */
|
|
20
|
+
judge?: Judge;
|
|
21
|
+
/**
|
|
22
|
+
* Whether this segment is one piece of a longer page.
|
|
23
|
+
*
|
|
24
|
+
* It decides which prompt the reviewer gets, and the two differ in what
|
|
25
|
+
* counts as an omission: a fragment has no beginning and no end, so a missing
|
|
26
|
+
* introduction or a section that runs past the cut is a property of the cut
|
|
27
|
+
* rather than of the translation, and the fragment prompt says so.
|
|
28
|
+
*
|
|
29
|
+
* That relaxation is right for a piece and wrong for a whole page. Half of
|
|
30
|
+
* this corpus is one segment, and on those pages the segment reviewer is the
|
|
31
|
+
* *only* blocking semantic check there is — the whole-page one is advisory
|
|
32
|
+
* and is not even asked when there is a single segment, because it would be
|
|
33
|
+
* reading the same text twice. Telling it "this is a fragment" there would
|
|
34
|
+
* quietly excuse exactly the defect it exists to catch: a how-to whose last
|
|
35
|
+
* section, "what to do next", never made it into the translation.
|
|
36
|
+
*/
|
|
37
|
+
fragment?: boolean;
|
|
38
|
+
sourceLanguage: string;
|
|
39
|
+
targetLanguage: string;
|
|
40
|
+
terms?: readonly TermPair[];
|
|
41
|
+
}
|
|
42
|
+
export declare const checkSegment: ({ segment, translation, processor, documentTokens, judge, fragment, sourceLanguage, targetLanguage, terms, }: CheckSegmentOptions) => Promise<TranslationFinding[]>;
|
|
@@ -0,0 +1,247 @@
|
|
|
1
|
+
import { visit } from 'unist-util-visit';
|
|
2
|
+
import { collectComponents, collectHeadingDepths, } from "../remark-lint/translation-parity/shared.js";
|
|
3
|
+
import { countPlaceholders } from "./translate-mask.js";
|
|
4
|
+
import { parseAttributeElement } from "./translate-segment.js";
|
|
5
|
+
/**
|
|
6
|
+
* Whether one translated segment is good enough to freeze.
|
|
7
|
+
*
|
|
8
|
+
* This is the gate the whole design rests on. A segment that passes here is
|
|
9
|
+
* never touched again — not by a retry, not by a repair agent, not by a later
|
|
10
|
+
* assembly round — so everything that was right stays right. The incident this
|
|
11
|
+
* replaced had no such gate: a document one problem away from finished was
|
|
12
|
+
* handed back to a model that rewrote it whole, and 870 correct placeholders
|
|
13
|
+
* became 1000 problems with no way back.
|
|
14
|
+
*
|
|
15
|
+
* Four layers, cheapest first, and a failing layer stops the ones after it.
|
|
16
|
+
* Three of the four are free; only the last one costs a model call, and it is
|
|
17
|
+
* not spent on a segment the free ones have already faulted.
|
|
18
|
+
*
|
|
19
|
+
* The same function is what a repair agent calls to see its own work, so "the
|
|
20
|
+
* agent is checked against the rules it checks itself with" is true because
|
|
21
|
+
* there is one function, not because two of them were kept in step.
|
|
22
|
+
*/
|
|
23
|
+
/** How much of a mismatched segment to quote back. Enough to act on, not enough to drown in. */
|
|
24
|
+
const PREVIEW = 80;
|
|
25
|
+
const preview = (value) => value.length > PREVIEW ? `${value.slice(0, PREVIEW - 1)}…` : value;
|
|
26
|
+
export const checkSegment = async ({ segment, translation, processor, documentTokens, judge, fragment = false, sourceLanguage, targetLanguage, terms, }) => {
|
|
27
|
+
if (translation.trim() === '') {
|
|
28
|
+
return [
|
|
29
|
+
{
|
|
30
|
+
rule: 'doom-translate:nothing-written',
|
|
31
|
+
reason: 'The model returned nothing for this segment.',
|
|
32
|
+
},
|
|
33
|
+
];
|
|
34
|
+
}
|
|
35
|
+
// ------------------------------------------------------------ 1. it parses
|
|
36
|
+
if (segment.address.kind === 'attributes') {
|
|
37
|
+
return checkAttributes(segment, translation, processor);
|
|
38
|
+
}
|
|
39
|
+
let tree;
|
|
40
|
+
try {
|
|
41
|
+
tree = processor.parse(translation);
|
|
42
|
+
}
|
|
43
|
+
catch (error) {
|
|
44
|
+
return [
|
|
45
|
+
{
|
|
46
|
+
rule: 'doom-translate:segment-unparseable',
|
|
47
|
+
reason: `This segment's translation is not valid markdown/MDX: ${error instanceof Error ? error.message : String(error)}`,
|
|
48
|
+
},
|
|
49
|
+
];
|
|
50
|
+
}
|
|
51
|
+
// --------------------------------------------------- 2. the placeholders
|
|
52
|
+
const findings = placeholderFindings(segment, translation, documentTokens);
|
|
53
|
+
if (findings.length > 0) {
|
|
54
|
+
return findings;
|
|
55
|
+
}
|
|
56
|
+
// ----------------------------------------------------- 3. the structure
|
|
57
|
+
findings.push(...structureFindings(segment, tree, processor));
|
|
58
|
+
if (findings.length > 0) {
|
|
59
|
+
return findings;
|
|
60
|
+
}
|
|
61
|
+
// --------------------------------------------------------- 4. the meaning
|
|
62
|
+
if (judge) {
|
|
63
|
+
findings.push(...(await judge.review({
|
|
64
|
+
// The masked pair, as everywhere else: a finding that quoted an
|
|
65
|
+
// unmasked link target would put that target back in the model's reach,
|
|
66
|
+
// which is the one thing masking exists to prevent.
|
|
67
|
+
sourceText: segment.text,
|
|
68
|
+
translationText: translation,
|
|
69
|
+
sourceLanguage,
|
|
70
|
+
targetLanguage,
|
|
71
|
+
terms,
|
|
72
|
+
fragment,
|
|
73
|
+
})));
|
|
74
|
+
}
|
|
75
|
+
return findings;
|
|
76
|
+
};
|
|
77
|
+
/**
|
|
78
|
+
* Every placeholder back, exactly as often as it went out.
|
|
79
|
+
*
|
|
80
|
+
* Pure string counting — no restore, no tree walk, nothing to configure — which
|
|
81
|
+
* is why it can run on every attempt of every segment without thinking about
|
|
82
|
+
* cost. It is also where the failure that ended the incident is now caught: in
|
|
83
|
+
* a few kilobytes, against a table of a few dozen tokens, instead of in 78 KB
|
|
84
|
+
* against 871.
|
|
85
|
+
*/
|
|
86
|
+
const placeholderFindings = (segment, translation, documentTokens) => {
|
|
87
|
+
const found = countPlaceholders(translation);
|
|
88
|
+
const findings = [];
|
|
89
|
+
for (const [token, expected] of segment.expected) {
|
|
90
|
+
const actual = found.get(token) ?? 0;
|
|
91
|
+
found.delete(token);
|
|
92
|
+
if (actual === expected) {
|
|
93
|
+
continue;
|
|
94
|
+
}
|
|
95
|
+
findings.push({
|
|
96
|
+
rule: actual < expected
|
|
97
|
+
? 'doom-translate:missing-placeholder'
|
|
98
|
+
: 'doom-translate:duplicate-placeholder',
|
|
99
|
+
reason: actual < expected
|
|
100
|
+
? `\`${token.toUpperCase()}\` is not in this segment's translation${expected > 1 ? ` as often as it should be (expected ${expected}, found ${actual})` : ''}. It stands for content you must reproduce exactly — a link target, some code, an identifier. Either it has not been translated yet, or it was dropped.`
|
|
101
|
+
: `\`${token.toUpperCase()}\` appears ${actual} times but must appear ${expected}.`,
|
|
102
|
+
});
|
|
103
|
+
}
|
|
104
|
+
for (const [token, actual] of found) {
|
|
105
|
+
const fromElsewhere = documentTokens?.has(token) ?? false;
|
|
106
|
+
findings.push({
|
|
107
|
+
rule: fromElsewhere
|
|
108
|
+
? 'doom-translate:out-of-segment-placeholder'
|
|
109
|
+
: 'doom-translate:unregistered-placeholder',
|
|
110
|
+
reason: fromElsewhere
|
|
111
|
+
? `\`${token.toUpperCase()}\` belongs to a different part of the document and appears here ${actual} time(s). Translate only the segment you were given; do not carry text in from anywhere else.`
|
|
112
|
+
: `\`${token.toUpperCase()}\` was never issued — do not invent placeholders.`,
|
|
113
|
+
});
|
|
114
|
+
}
|
|
115
|
+
return findings;
|
|
116
|
+
};
|
|
117
|
+
const ANCHOR_CARRIER = /^__DOOM_TR_ANCHOR_\d+__$/iu;
|
|
118
|
+
/**
|
|
119
|
+
* The anchor carrier a heading ends with, if it ends with one.
|
|
120
|
+
*
|
|
121
|
+
* `maskAst` puts a heading's `{#id}` into an `inlineCode` carrier appended as
|
|
122
|
+
* the heading's last child, because that is the only place the id exists:
|
|
123
|
+
* `HEADING_ANCHOR_PATTERN` is anchored to the end of the heading, so `{#id}`
|
|
124
|
+
* anywhere else is literal text.
|
|
125
|
+
*/
|
|
126
|
+
const trailingAnchor = (heading) => {
|
|
127
|
+
const last = heading.children.at(-1);
|
|
128
|
+
return last?.type === 'inlineCode' && ANCHOR_CARRIER.test(last.value)
|
|
129
|
+
? last.value
|
|
130
|
+
: undefined;
|
|
131
|
+
};
|
|
132
|
+
const collectTrailingAnchors = (tree) => {
|
|
133
|
+
const anchors = [];
|
|
134
|
+
visit(tree, 'heading', (node) => {
|
|
135
|
+
anchors.push(trailingAnchor(node));
|
|
136
|
+
});
|
|
137
|
+
return anchors;
|
|
138
|
+
};
|
|
139
|
+
/**
|
|
140
|
+
* The skeleton is the source's, not the model's.
|
|
141
|
+
*
|
|
142
|
+
* Headings and components are structure rather than prose: translating a page
|
|
143
|
+
* never changes a heading's level and never adds or removes a component. Both
|
|
144
|
+
* are already checked over the whole document by `translation-heading-sequence`
|
|
145
|
+
* and `translation-component-multiset` — this runs the *same* collectors, one
|
|
146
|
+
* segment at a time.
|
|
147
|
+
*
|
|
148
|
+
* That is not a duplicate of those rules, it is where they become actionable.
|
|
149
|
+
* Every pairwise rule reports against the document as a whole, with no line
|
|
150
|
+
* number to route by, so a finding from one of them at assembly time names a
|
|
151
|
+
* defect but not a place. Checked per segment, the same defect arrives with the
|
|
152
|
+
* segment attached, inside a blast radius of a few kilobytes, and can simply be
|
|
153
|
+
* retranslated. When one of those rules does fire at assembly time afterwards,
|
|
154
|
+
* it means something assembly itself did — which is worth failing loudly for
|
|
155
|
+
* rather than retrying blindly.
|
|
156
|
+
*/
|
|
157
|
+
const structureFindings = (segment, tree, processor) => {
|
|
158
|
+
let source;
|
|
159
|
+
try {
|
|
160
|
+
source = processor.parse(segment.text);
|
|
161
|
+
}
|
|
162
|
+
catch {
|
|
163
|
+
// The segment came from a parsed tree, so this cannot normally happen; if
|
|
164
|
+
// it somehow does, it is not the translation's fault.
|
|
165
|
+
return [];
|
|
166
|
+
}
|
|
167
|
+
const findings = [];
|
|
168
|
+
const expectedDepths = collectHeadingDepths(source);
|
|
169
|
+
const actualDepths = collectHeadingDepths(tree);
|
|
170
|
+
if (expectedDepths.length !== actualDepths.length ||
|
|
171
|
+
expectedDepths.some((depth, index) => depth !== actualDepths[index])) {
|
|
172
|
+
findings.push({
|
|
173
|
+
rule: 'doom-translate:segment-heading-sequence',
|
|
174
|
+
reason: `This segment's headings changed: the source is ${expectedDepths.join('-') || '(none)'}, the translation is ${actualDepths.join('-') || '(none)'}. Translate the text of a heading; never its level, and never add or drop one.`,
|
|
175
|
+
});
|
|
176
|
+
}
|
|
177
|
+
const expectedComponents = collectComponents(source);
|
|
178
|
+
const actualComponents = collectComponents(tree);
|
|
179
|
+
for (const name of new Set([
|
|
180
|
+
...expectedComponents.keys(),
|
|
181
|
+
...actualComponents.keys(),
|
|
182
|
+
])) {
|
|
183
|
+
const want = expectedComponents.get(name) ?? 0;
|
|
184
|
+
const got = actualComponents.get(name) ?? 0;
|
|
185
|
+
if (want === got) {
|
|
186
|
+
continue;
|
|
187
|
+
}
|
|
188
|
+
findings.push({
|
|
189
|
+
rule: 'doom-translate:segment-component-multiset',
|
|
190
|
+
reason: got < want
|
|
191
|
+
? `This segment's translation dropped ${want - got} \`<${name}>\` (the source has ${want}, the translation has ${got}) — content that was in the source is missing.`
|
|
192
|
+
: `This segment's translation invented ${got - want} \`<${name}>\` (the source has ${want}, the translation has ${got}).`,
|
|
193
|
+
});
|
|
194
|
+
}
|
|
195
|
+
// Only meaningful while the headings still correspond one to one; when they
|
|
196
|
+
// do not, the sequence finding above already says so and pairing by index
|
|
197
|
+
// would invent a second, misleading one.
|
|
198
|
+
if (expectedDepths.length === actualDepths.length) {
|
|
199
|
+
const expectedAnchors = collectTrailingAnchors(source);
|
|
200
|
+
const actualAnchors = collectTrailingAnchors(tree);
|
|
201
|
+
expectedAnchors.forEach((placeholder, index) => {
|
|
202
|
+
if (!placeholder || actualAnchors[index] === placeholder) {
|
|
203
|
+
return;
|
|
204
|
+
}
|
|
205
|
+
findings.push({
|
|
206
|
+
rule: 'doom-translate:segment-heading-anchor-moved',
|
|
207
|
+
reason: `This segment's translation moved \`${placeholder}\` out of the end of its heading. A heading's \`{#id}\` is an id only as the last thing in the heading; anywhere else it is literal text, so the heading would render the id as visible punctuation and be given a generated one instead. Word order may change; keep the anchor last.`,
|
|
208
|
+
});
|
|
209
|
+
});
|
|
210
|
+
}
|
|
211
|
+
return findings;
|
|
212
|
+
};
|
|
213
|
+
/**
|
|
214
|
+
* A tag's prose, checked as the tag it has to become.
|
|
215
|
+
*
|
|
216
|
+
* An attribute segment is one JSX element carrying the words on a tag — a
|
|
217
|
+
* `<Tab label>` inside a container too large to send whole. It goes back into
|
|
218
|
+
* the document as attribute values, so what matters is that it is still one
|
|
219
|
+
* element and still has the same attributes; a model that answered with a
|
|
220
|
+
* sentence instead would otherwise leave that label silently untranslated.
|
|
221
|
+
*/
|
|
222
|
+
const checkAttributes = (segment, translation, processor) => {
|
|
223
|
+
const expected = parseAttributeElement(segment.text, processor);
|
|
224
|
+
const actual = parseAttributeElement(translation, processor);
|
|
225
|
+
if (!actual) {
|
|
226
|
+
return [
|
|
227
|
+
{
|
|
228
|
+
rule: 'doom-translate:segment-attributes',
|
|
229
|
+
reason: `Answer with the element itself and nothing else, exactly as it was given — \`${preview(segment.text.trim())}\` — translating only the attribute values.`,
|
|
230
|
+
},
|
|
231
|
+
];
|
|
232
|
+
}
|
|
233
|
+
const names = (element) => element.attributes
|
|
234
|
+
.map((attr) => (attr.type === 'mdxJsxAttribute' ? attr.name : '...'))
|
|
235
|
+
.sort();
|
|
236
|
+
const want = expected ? names(expected) : [];
|
|
237
|
+
const got = names(actual);
|
|
238
|
+
if (actual.name !== expected?.name || want.join() !== got.join()) {
|
|
239
|
+
return [
|
|
240
|
+
{
|
|
241
|
+
rule: 'doom-translate:segment-attributes',
|
|
242
|
+
reason: `The element must stay \`<${expected?.name ?? ''}>\` with the attributes ${want.map((name) => `\`${name}\``).join(', ')}; the translation has \`<${actual.name ?? ''}>\` with ${got.map((name) => `\`${name}\``).join(', ') || '(none)'}. Translate the values, not the names.`,
|
|
243
|
+
},
|
|
244
|
+
];
|
|
245
|
+
}
|
|
246
|
+
return [];
|
|
247
|
+
};
|