@alauda/doom 2.6.0 → 2.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/cli/translate-call.d.ts +85 -0
- package/lib/cli/translate-call.js +149 -0
- package/lib/cli/translate-checker.d.ts +8 -0
- package/lib/cli/translate-diagnose.d.ts +45 -0
- package/lib/cli/translate-diagnose.js +118 -0
- package/lib/cli/translate-judge.d.ts +28 -3
- package/lib/cli/translate-judge.js +40 -7
- package/lib/cli/translate-mask.d.ts +13 -0
- package/lib/cli/translate-mask.js +19 -5
- package/lib/cli/translate-pipeline.d.ts +144 -0
- package/lib/cli/translate-pipeline.js +489 -0
- package/lib/cli/translate-repair-agent.d.ts +69 -0
- package/lib/cli/translate-repair-agent.js +241 -0
- package/lib/cli/translate-scratch.d.ts +9 -1
- package/lib/cli/translate-scratch.js +2 -2
- package/lib/cli/translate-segment-cache.d.ts +90 -0
- package/lib/cli/translate-segment-cache.js +301 -0
- package/lib/cli/translate-segment-check.d.ts +42 -0
- package/lib/cli/translate-segment-check.js +247 -0
- package/lib/cli/translate-segment.d.ts +307 -0
- package/lib/cli/translate-segment.js +625 -0
- package/lib/cli/translate.d.ts +14 -4
- package/lib/cli/translate.js +232 -52
- package/lib/remark-lint/no-unmatched-anchor.js +6 -2
- package/lib/remark-lint/no-unopened-frontmatter.js +5 -1
- package/lib/remark-lint/rule-sets.d.ts +1 -3
- package/lib/remark-lint/rule-sets.js +3 -9
- package/lib/remark-lint/syntax-plugins.d.ts +32 -0
- package/lib/remark-lint/syntax-plugins.js +46 -0
- package/lib/remark-lint/translation-parity/frontmatter-preservation.js +5 -3
- package/lib/remark-lint/translation-parity/link-isomorphism.d.ts +8 -0
- package/lib/remark-lint/translation-parity/link-isomorphism.js +40 -4
- package/lib/remark-lint/translation-parity/shared.js +6 -2
- package/lib/remark-lint/translation-parity/url-residue.d.ts +1 -0
- package/lib/remark-lint/translation-parity/url-residue.js +25 -4
- package/lib/runtime/hooks/useTranslation.d.ts +1 -1
- package/lib/types.d.ts +101 -6
- package/package.json +1 -1
- package/lib/cli/translate-agent.d.ts +0 -81
- package/lib/cli/translate-agent.js +0 -450
|
@@ -0,0 +1,144 @@
|
|
|
1
|
+
import type { Root } from 'mdast';
|
|
2
|
+
import type { SegmentTranslator } from './translate-call.ts';
|
|
3
|
+
import { type TranslationChecker, type TranslationFinding } from './translate-checker.ts';
|
|
4
|
+
import type { Judge } from './translate-judge.ts';
|
|
5
|
+
import { MaskIntegrityError, type MaskEntry, type MaskProcessor } from './translate-mask.ts';
|
|
6
|
+
import type { SegmentCacheRecord } from './translate-segment-cache.ts';
|
|
7
|
+
import { type Segment, type SegmentRecord } from './translate-segment.ts';
|
|
8
|
+
import type { TermPair } from './translate-terms.ts';
|
|
9
|
+
/**
|
|
10
|
+
* Translating one document, a segment at a time.
|
|
11
|
+
*
|
|
12
|
+
* The shape of this file is the whole proposal in one place: cut the document
|
|
13
|
+
* up, translate each piece on its own, **freeze what passes**, ask again only
|
|
14
|
+
* for what did not, put the pieces back, and check the result as a whole.
|
|
15
|
+
*
|
|
16
|
+
* The freezing is the part that matters. In the design this replaces, the unit
|
|
17
|
+
* of work, of acceptance and of retry were all "the document", while the unit
|
|
18
|
+
* of failure was one placeholder out of 871 — so every retry put everything
|
|
19
|
+
* that was already right back at risk, and one of them took a translation that
|
|
20
|
+
* was one problem from done and turned it into a thousand. Here, a segment that
|
|
21
|
+
* passes is never touched again by anything: not a retry, not a repair, not a
|
|
22
|
+
* later assembly round. Work only ever accumulates.
|
|
23
|
+
*/
|
|
24
|
+
/** How many times a segment is asked for before it is escalated. */
|
|
25
|
+
export declare const DEFAULT_MAX_SEGMENT_ATTEMPTS = 3;
|
|
26
|
+
/** How many times the assembled document may send segments back. */
|
|
27
|
+
export declare const DEFAULT_MAX_ASSEMBLY_ROUNDS = 2;
|
|
28
|
+
export type SegmentStatus = 'translated' | 'cached' | 'repaired' | 'failed';
|
|
29
|
+
export interface SegmentOutcome {
|
|
30
|
+
index: number;
|
|
31
|
+
status: SegmentStatus;
|
|
32
|
+
label: Segment['label'];
|
|
33
|
+
/** Model calls spent asking for this segment, across every round. */
|
|
34
|
+
attempts: number;
|
|
35
|
+
/** What was wrong on each rejected attempt, oldest first. For the diagnosis. */
|
|
36
|
+
history: TranslationFinding[][];
|
|
37
|
+
/** What was still wrong when it was given up on. */
|
|
38
|
+
findings: TranslationFinding[];
|
|
39
|
+
}
|
|
40
|
+
export type DocumentFailure =
|
|
41
|
+
/** A single block too large to send and impossible to divide. */
|
|
42
|
+
{
|
|
43
|
+
kind: 'unsplittable-block';
|
|
44
|
+
detail: string;
|
|
45
|
+
}
|
|
46
|
+
/** One or more segments never passed. */
|
|
47
|
+
| {
|
|
48
|
+
kind: 'segment';
|
|
49
|
+
segments: readonly number[];
|
|
50
|
+
}
|
|
51
|
+
/** The assembled document failed a whole-document rule that was routed back and still failed. */
|
|
52
|
+
| {
|
|
53
|
+
kind: 'assembly';
|
|
54
|
+
}
|
|
55
|
+
/** The assembled document failed a rule nothing could attribute to a segment. */
|
|
56
|
+
| {
|
|
57
|
+
kind: 'unlocatable';
|
|
58
|
+
};
|
|
59
|
+
export interface TranslateDocumentResult {
|
|
60
|
+
/** The finished document, when it passed. */
|
|
61
|
+
document?: string;
|
|
62
|
+
/**
|
|
63
|
+
* What was still wrong when the run ended.
|
|
64
|
+
*
|
|
65
|
+
* `document` is set exactly when none of these block. A passing document can
|
|
66
|
+
* still carry advisory findings — readability notes, and the whole-document
|
|
67
|
+
* review — which are worth printing and not worth failing a build over.
|
|
68
|
+
*/
|
|
69
|
+
findings: TranslationFinding[];
|
|
70
|
+
failure?: DocumentFailure;
|
|
71
|
+
outcomes: SegmentOutcome[];
|
|
72
|
+
assemblyRounds: number;
|
|
73
|
+
/** Where each segment landed, for the next run's cache. Only when it passed. */
|
|
74
|
+
records?: SegmentRecord[];
|
|
75
|
+
}
|
|
76
|
+
/** A last resort for a segment repeated attempts could not fix. */
|
|
77
|
+
export interface SegmentRepairer {
|
|
78
|
+
repair(request: {
|
|
79
|
+
segment: Segment;
|
|
80
|
+
/** The best rejected attempt — the one with the fewest blocking findings. */
|
|
81
|
+
draft: string;
|
|
82
|
+
history: readonly TranslationFinding[][];
|
|
83
|
+
/**
|
|
84
|
+
* The acceptance check for this segment, so the agent sees exactly the
|
|
85
|
+
* verdict that will decide its work.
|
|
86
|
+
*
|
|
87
|
+
* Handed in rather than rebuilt: an agent checking itself against anything
|
|
88
|
+
* but the real gate would pass its own check and then be rejected, which is
|
|
89
|
+
* a loop with no exit.
|
|
90
|
+
*/
|
|
91
|
+
check: (translation: string) => Promise<TranslationFinding[]>;
|
|
92
|
+
}): Promise<string | undefined>;
|
|
93
|
+
}
|
|
94
|
+
export interface TranslateDocumentOptions {
|
|
95
|
+
/** The masked tree. Segments address it, assembly rebuilds it. */
|
|
96
|
+
tree: Root;
|
|
97
|
+
/** The masked source as text, for the whole-document review. */
|
|
98
|
+
maskedSource: string;
|
|
99
|
+
maskEntries: readonly MaskEntry[];
|
|
100
|
+
processor: MaskProcessor;
|
|
101
|
+
/** Merges a restored translation with the frontmatter it will be written with. */
|
|
102
|
+
compose: (restored: string, cache?: SegmentCacheRecord) => string;
|
|
103
|
+
/** Absolute path the translation will be written to. */
|
|
104
|
+
targetPath: string;
|
|
105
|
+
/** Path of the source, relative to the docs root — for logs only. */
|
|
106
|
+
sourceLabel: string;
|
|
107
|
+
/** Language names, as the reviewer is told them — `English`, not `en`. */
|
|
108
|
+
sourceLanguage: string;
|
|
109
|
+
targetLanguage: string;
|
|
110
|
+
translator: SegmentTranslator;
|
|
111
|
+
checker: TranslationChecker;
|
|
112
|
+
/** Reads each segment against its source. */
|
|
113
|
+
segmentJudge?: Judge;
|
|
114
|
+
/** Reads the assembled document. Advisory only — it never fails a build. */
|
|
115
|
+
documentJudge?: Judge;
|
|
116
|
+
terms?: readonly TermPair[];
|
|
117
|
+
segmentCap?: number;
|
|
118
|
+
segmentHardCap?: number;
|
|
119
|
+
segmentFloor?: number;
|
|
120
|
+
maxSegmentAttempts?: number;
|
|
121
|
+
maxAssemblyRounds?: number;
|
|
122
|
+
contextTail?: number;
|
|
123
|
+
repairer?: SegmentRepairer;
|
|
124
|
+
/**
|
|
125
|
+
* The translation already on disk, for reusing the segments that did not
|
|
126
|
+
* change. Absent on a first translation, and when `--force` is given.
|
|
127
|
+
*
|
|
128
|
+
* What may be reused is decided in `translate-segment-cache.ts`, which
|
|
129
|
+
* verifies every candidate against this document's own mask table before
|
|
130
|
+
* offering it.
|
|
131
|
+
*/
|
|
132
|
+
previous?: {
|
|
133
|
+
body: string;
|
|
134
|
+
record?: SegmentCacheRecord;
|
|
135
|
+
};
|
|
136
|
+
onProgress?: (message: string) => void;
|
|
137
|
+
}
|
|
138
|
+
/**
|
|
139
|
+
* A mask finding, said in the vocabulary the rest of the pipeline speaks.
|
|
140
|
+
*
|
|
141
|
+
* Exported because the repair agent reports the same things.
|
|
142
|
+
*/
|
|
143
|
+
export declare const maskFindingToTranslationFinding: (finding: MaskIntegrityError["findings"][number]) => TranslationFinding;
|
|
144
|
+
export declare const translateDocument: (options: TranslateDocumentOptions) => Promise<TranslateDocumentResult>;
|
|
@@ -0,0 +1,489 @@
|
|
|
1
|
+
import { renderOutline, tailOf } from "./translate-call.js";
|
|
2
|
+
import { blockingFindings, } from "./translate-checker.js";
|
|
3
|
+
import { MaskIntegrityError, restoreMaskedContent, } from "./translate-mask.js";
|
|
4
|
+
import { encodeCacheRecord, matchCachedSegments, } from "./translate-segment-cache.js";
|
|
5
|
+
import { checkSegment } from "./translate-segment-check.js";
|
|
6
|
+
import { UnsplittableBlockError, assemble, documentOutline, planSegments, resolveAttributeNode, resolvePlacement, } from "./translate-segment.js";
|
|
7
|
+
/**
|
|
8
|
+
* Translating one document, a segment at a time.
|
|
9
|
+
*
|
|
10
|
+
* The shape of this file is the whole proposal in one place: cut the document
|
|
11
|
+
* up, translate each piece on its own, **freeze what passes**, ask again only
|
|
12
|
+
* for what did not, put the pieces back, and check the result as a whole.
|
|
13
|
+
*
|
|
14
|
+
* The freezing is the part that matters. In the design this replaces, the unit
|
|
15
|
+
* of work, of acceptance and of retry were all "the document", while the unit
|
|
16
|
+
* of failure was one placeholder out of 871 — so every retry put everything
|
|
17
|
+
* that was already right back at risk, and one of them took a translation that
|
|
18
|
+
* was one problem from done and turned it into a thousand. Here, a segment that
|
|
19
|
+
* passes is never touched again by anything: not a retry, not a repair, not a
|
|
20
|
+
* later assembly round. Work only ever accumulates.
|
|
21
|
+
*/
|
|
22
|
+
/** How many times a segment is asked for before it is escalated. */
|
|
23
|
+
export const DEFAULT_MAX_SEGMENT_ATTEMPTS = 3;
|
|
24
|
+
/** How many times the assembled document may send segments back. */
|
|
25
|
+
export const DEFAULT_MAX_ASSEMBLY_ROUNDS = 2;
|
|
26
|
+
/**
|
|
27
|
+
* A mask finding, said in the vocabulary the rest of the pipeline speaks.
|
|
28
|
+
*
|
|
29
|
+
* Exported because the repair agent reports the same things.
|
|
30
|
+
*/
|
|
31
|
+
export const maskFindingToTranslationFinding = (finding) => ({
|
|
32
|
+
rule: `doom-translate:${finding.code}`,
|
|
33
|
+
reason: finding.code === 'missing-placeholder'
|
|
34
|
+
? `\`${finding.placeholder}\` is not in the translation. It stands for a ${finding.kind}${finding.detail ? ` (${finding.detail})` : ''}. Either that part has not been translated yet, or it was dropped.`
|
|
35
|
+
: finding.code === 'duplicate-placeholder'
|
|
36
|
+
? `\`${finding.placeholder}\` appears ${finding.actual} times but must appear ${finding.expected}.`
|
|
37
|
+
: finding.code === 'unregistered-placeholder'
|
|
38
|
+
? `\`${finding.placeholder}\` was never issued — do not invent placeholders.`
|
|
39
|
+
: finding.code === 'unrestored-placeholder'
|
|
40
|
+
? `\`${finding.placeholder}\` ended up somewhere it does not belong${finding.detail ? ` — ${finding.detail}` : ''}.`
|
|
41
|
+
: (finding.detail ??
|
|
42
|
+
`the placeholder round trip failed (${finding.code})`),
|
|
43
|
+
placeholder: finding.placeholder,
|
|
44
|
+
});
|
|
45
|
+
export const translateDocument = async (options) => {
|
|
46
|
+
const { tree, maskedSource, maskEntries, processor, compose, targetPath, sourceLabel, sourceLanguage, targetLanguage, translator, checker, segmentJudge, documentJudge, terms, segmentCap, segmentHardCap, segmentFloor, maxSegmentAttempts = DEFAULT_MAX_SEGMENT_ATTEMPTS, maxAssemblyRounds = DEFAULT_MAX_ASSEMBLY_ROUNDS, contextTail, repairer, previous, onProgress, } = options;
|
|
47
|
+
let plan;
|
|
48
|
+
try {
|
|
49
|
+
plan = planSegments({
|
|
50
|
+
tree,
|
|
51
|
+
processor,
|
|
52
|
+
cap: segmentCap,
|
|
53
|
+
hardCap: segmentHardCap,
|
|
54
|
+
floor: segmentFloor,
|
|
55
|
+
});
|
|
56
|
+
}
|
|
57
|
+
catch (error) {
|
|
58
|
+
if (error instanceof UnsplittableBlockError) {
|
|
59
|
+
// Not a translation failure: nothing a model could have done differently.
|
|
60
|
+
// The document has to change.
|
|
61
|
+
return {
|
|
62
|
+
findings: [
|
|
63
|
+
{ rule: 'doom-translate:unsplittable-block', reason: error.message },
|
|
64
|
+
],
|
|
65
|
+
failure: { kind: 'unsplittable-block', detail: error.message },
|
|
66
|
+
outcomes: [],
|
|
67
|
+
assemblyRounds: 0,
|
|
68
|
+
};
|
|
69
|
+
}
|
|
70
|
+
throw error;
|
|
71
|
+
}
|
|
72
|
+
const { reuse, reason: noReuse } = previous
|
|
73
|
+
? matchCachedSegments({
|
|
74
|
+
plan,
|
|
75
|
+
maskEntries,
|
|
76
|
+
processor,
|
|
77
|
+
previousBody: previous.body,
|
|
78
|
+
record: previous.record,
|
|
79
|
+
})
|
|
80
|
+
: { reuse: new Map(), reason: undefined };
|
|
81
|
+
if (previous && reuse.size === 0 && noReuse) {
|
|
82
|
+
onProgress?.(`${sourceLabel}: nothing reused — ${noReuse}`);
|
|
83
|
+
}
|
|
84
|
+
const documentTokens = new Set(maskEntries.map((entry) => entry.placeholder.toLowerCase()));
|
|
85
|
+
/**
|
|
86
|
+
* Whether the reviewer is looking at a piece of a page or at the page.
|
|
87
|
+
*
|
|
88
|
+
* A one-segment document is not a fragment, and must not be reviewed as one:
|
|
89
|
+
* the fragment prompt stops counting a missing beginning or end as an
|
|
90
|
+
* omission, and on a page that *is* the segment those are the omissions.
|
|
91
|
+
* Half of this corpus is a single segment, and there the segment reviewer is
|
|
92
|
+
* the only blocking semantic check there is.
|
|
93
|
+
*/
|
|
94
|
+
const fragment = plan.segments.length > 1;
|
|
95
|
+
/**
|
|
96
|
+
* The page's headings, with the segment's own section marked.
|
|
97
|
+
*
|
|
98
|
+
* Rendered per segment rather than once: the list is the same, but the mark
|
|
99
|
+
* is what makes it useful — the prompt tells the model this is "so you know
|
|
100
|
+
* where the segment sits", and an unmarked list does not say that.
|
|
101
|
+
*/
|
|
102
|
+
const headings = documentOutline(tree);
|
|
103
|
+
const outlines = new Map();
|
|
104
|
+
const outlineFor = (segment) => {
|
|
105
|
+
let rendered = outlines.get(segment.index);
|
|
106
|
+
if (rendered == null) {
|
|
107
|
+
rendered = renderOutline(headings, segment.label.headingLine);
|
|
108
|
+
outlines.set(segment.index, rendered);
|
|
109
|
+
}
|
|
110
|
+
return rendered;
|
|
111
|
+
};
|
|
112
|
+
/** The accepted translation of each segment. Only ever written on a pass. */
|
|
113
|
+
const frozen = [];
|
|
114
|
+
/** The least-bad rejected attempt, for a repairer to start from. */
|
|
115
|
+
const best = [];
|
|
116
|
+
const outcomes = plan.segments.map((segment) => ({
|
|
117
|
+
index: segment.index,
|
|
118
|
+
status: 'failed',
|
|
119
|
+
label: segment.label,
|
|
120
|
+
attempts: 0,
|
|
121
|
+
history: [],
|
|
122
|
+
findings: [],
|
|
123
|
+
}));
|
|
124
|
+
/**
|
|
125
|
+
* The non-blocking notes each segment came back with, by segment.
|
|
126
|
+
*
|
|
127
|
+
* Kept per segment and replaced rather than appended to, because a segment
|
|
128
|
+
* can be worked on more than once — a later assembly round can send it back
|
|
129
|
+
* — and a note from the attempt that was superseded is not a note about the
|
|
130
|
+
* document that shipped. Appending printed the same readability remark twice
|
|
131
|
+
* for every segment that was ever revisited.
|
|
132
|
+
*/
|
|
133
|
+
const advisoryBySegment = [];
|
|
134
|
+
/** Notes about the assembled page, which belong to no segment. */
|
|
135
|
+
const documentAdvisory = [];
|
|
136
|
+
const advisory = () => [...advisoryBySegment.flat(), ...documentAdvisory];
|
|
137
|
+
const previousTail = (segment) => {
|
|
138
|
+
if (contextTail === 0) {
|
|
139
|
+
return undefined;
|
|
140
|
+
}
|
|
141
|
+
const before = frozen[segment.index - 1];
|
|
142
|
+
return before == null ? undefined : tailOf(before, contextTail);
|
|
143
|
+
};
|
|
144
|
+
/**
|
|
145
|
+
* One pass at a segment: ask, check, and freeze only on a pass.
|
|
146
|
+
*
|
|
147
|
+
* The ratchet lives in this function and nowhere else — `frozen` is assigned
|
|
148
|
+
* exactly once here, on the branch where nothing blocks. Every other outcome
|
|
149
|
+
* leaves whatever was already accepted alone, which is what makes "the
|
|
150
|
+
* document uses the version of each segment that passed" true by
|
|
151
|
+
* construction rather than by care.
|
|
152
|
+
*/
|
|
153
|
+
const workOn = async (segment, sentBack) => {
|
|
154
|
+
const outcome = outcomes[segment.index];
|
|
155
|
+
let retry = sentBack?.length
|
|
156
|
+
? { previous: frozen[segment.index] ?? '', findings: sentBack }
|
|
157
|
+
: undefined;
|
|
158
|
+
for (let attempt = 1; attempt <= maxSegmentAttempts; attempt++) {
|
|
159
|
+
outcome.attempts++;
|
|
160
|
+
const translation = await translator.translate({
|
|
161
|
+
segment,
|
|
162
|
+
outline: outlineFor(segment),
|
|
163
|
+
previousTail: previousTail(segment),
|
|
164
|
+
attempt,
|
|
165
|
+
retry,
|
|
166
|
+
});
|
|
167
|
+
const findings = await checkSegment({
|
|
168
|
+
segment,
|
|
169
|
+
translation,
|
|
170
|
+
processor,
|
|
171
|
+
documentTokens,
|
|
172
|
+
judge: segmentJudge,
|
|
173
|
+
fragment,
|
|
174
|
+
sourceLanguage,
|
|
175
|
+
targetLanguage,
|
|
176
|
+
terms,
|
|
177
|
+
});
|
|
178
|
+
const blocking = blockingFindings(findings);
|
|
179
|
+
if (blocking.length === 0) {
|
|
180
|
+
frozen[segment.index] = translation;
|
|
181
|
+
outcome.status = 'translated';
|
|
182
|
+
outcome.findings = [];
|
|
183
|
+
advisoryBySegment[segment.index] = findings;
|
|
184
|
+
return true;
|
|
185
|
+
}
|
|
186
|
+
outcome.history.push(findings);
|
|
187
|
+
const previousBest = best[segment.index];
|
|
188
|
+
if (!previousBest || blocking.length < previousBest.blocking) {
|
|
189
|
+
best[segment.index] = { text: translation, blocking: blocking.length };
|
|
190
|
+
}
|
|
191
|
+
retry = { previous: translation, findings };
|
|
192
|
+
onProgress?.(`${sourceLabel} [segment ${segment.index + 1}/${plan.segments.length}${describe(segment)}] attempt ${attempt} rejected: ${summarise(blocking)}`);
|
|
193
|
+
}
|
|
194
|
+
if (repairer) {
|
|
195
|
+
const draft = best[segment.index]?.text ?? '';
|
|
196
|
+
onProgress?.(`${sourceLabel} [segment ${segment.index + 1}/${plan.segments.length}${describe(segment)}] escalating to the repair agent`);
|
|
197
|
+
const repaired = await repairer.repair({
|
|
198
|
+
segment,
|
|
199
|
+
draft,
|
|
200
|
+
history: outcome.history,
|
|
201
|
+
check: (translation) => checkSegment({
|
|
202
|
+
segment,
|
|
203
|
+
translation,
|
|
204
|
+
processor,
|
|
205
|
+
documentTokens,
|
|
206
|
+
judge: segmentJudge,
|
|
207
|
+
fragment,
|
|
208
|
+
sourceLanguage,
|
|
209
|
+
targetLanguage,
|
|
210
|
+
terms,
|
|
211
|
+
}),
|
|
212
|
+
});
|
|
213
|
+
if (repaired != null) {
|
|
214
|
+
const findings = await checkSegment({
|
|
215
|
+
segment,
|
|
216
|
+
translation: repaired,
|
|
217
|
+
processor,
|
|
218
|
+
documentTokens,
|
|
219
|
+
judge: segmentJudge,
|
|
220
|
+
fragment,
|
|
221
|
+
sourceLanguage,
|
|
222
|
+
targetLanguage,
|
|
223
|
+
terms,
|
|
224
|
+
});
|
|
225
|
+
if (blockingFindings(findings).length === 0) {
|
|
226
|
+
frozen[segment.index] = repaired;
|
|
227
|
+
outcome.status = 'repaired';
|
|
228
|
+
outcome.findings = [];
|
|
229
|
+
advisoryBySegment[segment.index] = findings;
|
|
230
|
+
onProgress?.(`${sourceLabel} [segment ${segment.index + 1}/${plan.segments.length}${describe(segment)}] repaired`);
|
|
231
|
+
return true;
|
|
232
|
+
}
|
|
233
|
+
outcome.history.push(findings);
|
|
234
|
+
}
|
|
235
|
+
}
|
|
236
|
+
// Nothing passed. Whatever was frozen before — from an earlier round —
|
|
237
|
+
// stays frozen: a failed attempt never costs a segment its best accepted
|
|
238
|
+
// version.
|
|
239
|
+
outcome.findings = outcome.history.at(-1) ?? [];
|
|
240
|
+
if (frozen[segment.index] == null) {
|
|
241
|
+
outcome.status = 'failed';
|
|
242
|
+
return false;
|
|
243
|
+
}
|
|
244
|
+
return true;
|
|
245
|
+
};
|
|
246
|
+
// ------------------------------------------------ every segment, in order
|
|
247
|
+
for (const segment of plan.segments) {
|
|
248
|
+
const reused = reuse.get(segment.index);
|
|
249
|
+
if (reused != null) {
|
|
250
|
+
frozen[segment.index] = reused;
|
|
251
|
+
outcomes[segment.index].status = 'cached';
|
|
252
|
+
continue;
|
|
253
|
+
}
|
|
254
|
+
await workOn(segment);
|
|
255
|
+
}
|
|
256
|
+
if (reuse.size > 0) {
|
|
257
|
+
onProgress?.(`${sourceLabel}: reused ${reuse.size} of ${plan.segments.length} segment(s) from the previous translation`);
|
|
258
|
+
}
|
|
259
|
+
const unfinished = plan.segments.filter((segment) => frozen[segment.index] == null);
|
|
260
|
+
if (unfinished.length > 0) {
|
|
261
|
+
for (const segment of unfinished) {
|
|
262
|
+
onProgress?.(`${sourceLabel} [segment ${segment.index + 1}/${plan.segments.length}${describe(segment)}] gave up after ${outcomes[segment.index].attempts} attempt(s)`);
|
|
263
|
+
}
|
|
264
|
+
return {
|
|
265
|
+
findings: unfinished.flatMap((segment) => outcomes[segment.index].findings),
|
|
266
|
+
failure: {
|
|
267
|
+
kind: 'segment',
|
|
268
|
+
segments: unfinished.map((segment) => segment.index),
|
|
269
|
+
},
|
|
270
|
+
outcomes,
|
|
271
|
+
assemblyRounds: 0,
|
|
272
|
+
};
|
|
273
|
+
}
|
|
274
|
+
// ------------------------------------------------------------- assembling
|
|
275
|
+
let assemblyRounds = 0;
|
|
276
|
+
for (;;) {
|
|
277
|
+
const assembled = assemble({
|
|
278
|
+
tree,
|
|
279
|
+
plan,
|
|
280
|
+
processor,
|
|
281
|
+
translations: frozen,
|
|
282
|
+
});
|
|
283
|
+
let restored;
|
|
284
|
+
let findings = [];
|
|
285
|
+
try {
|
|
286
|
+
restored = restoreMaskedContent(assembled.text, maskEntries, processor);
|
|
287
|
+
}
|
|
288
|
+
catch (error) {
|
|
289
|
+
if (!(error instanceof MaskIntegrityError)) {
|
|
290
|
+
throw error;
|
|
291
|
+
}
|
|
292
|
+
findings = error.findings.map(maskFindingToTranslationFinding);
|
|
293
|
+
}
|
|
294
|
+
const document = restored == null
|
|
295
|
+
? undefined
|
|
296
|
+
: compose(restored, encodeCacheRecord(plan, assembled.records));
|
|
297
|
+
if (document != null) {
|
|
298
|
+
findings = await checker.check(targetPath, document);
|
|
299
|
+
}
|
|
300
|
+
const blocking = blockingFindings(findings);
|
|
301
|
+
if (blocking.length === 0 && document != null) {
|
|
302
|
+
if (documentJudge && plan.segments.length > 1) {
|
|
303
|
+
// Advisory, always: this reads the whole page for the things a segment
|
|
304
|
+
// cannot see — a term that drifted between sections, a join that reads
|
|
305
|
+
// badly — and those are worth telling someone about and not worth
|
|
306
|
+
// failing a build over. A page that is one segment has already been
|
|
307
|
+
// read by exactly this check, so it is not read twice.
|
|
308
|
+
const whole = await documentJudge.review({
|
|
309
|
+
sourceText: maskedSource,
|
|
310
|
+
translationText: assembled.text,
|
|
311
|
+
sourceLanguage,
|
|
312
|
+
targetLanguage,
|
|
313
|
+
terms,
|
|
314
|
+
});
|
|
315
|
+
documentAdvisory.push(...whole.map((finding) => ({ ...finding, blocking: false })));
|
|
316
|
+
}
|
|
317
|
+
return {
|
|
318
|
+
document,
|
|
319
|
+
findings: [...findings, ...advisory()],
|
|
320
|
+
outcomes,
|
|
321
|
+
assemblyRounds,
|
|
322
|
+
records: assembled.records,
|
|
323
|
+
};
|
|
324
|
+
}
|
|
325
|
+
if (assemblyRounds >= maxAssemblyRounds) {
|
|
326
|
+
return {
|
|
327
|
+
findings: [...blocking, ...advisory()],
|
|
328
|
+
failure: { kind: 'assembly' },
|
|
329
|
+
outcomes,
|
|
330
|
+
assemblyRounds,
|
|
331
|
+
};
|
|
332
|
+
}
|
|
333
|
+
const routed = route({
|
|
334
|
+
findings: blocking,
|
|
335
|
+
plan,
|
|
336
|
+
placements: assembled.placements,
|
|
337
|
+
document,
|
|
338
|
+
processor,
|
|
339
|
+
});
|
|
340
|
+
if (routed.size === 0) {
|
|
341
|
+
// Deliberately not "redo every segment": that is the whole-document blast
|
|
342
|
+
// radius this design exists to remove. A finding nobody can attribute is
|
|
343
|
+
// reported as it is, which is already an order of magnitude better than
|
|
344
|
+
// the 873 undifferentiated problems this replaced.
|
|
345
|
+
return {
|
|
346
|
+
findings: [...blocking, ...advisory()],
|
|
347
|
+
failure: { kind: 'unlocatable' },
|
|
348
|
+
outcomes,
|
|
349
|
+
assemblyRounds,
|
|
350
|
+
};
|
|
351
|
+
}
|
|
352
|
+
assemblyRounds++;
|
|
353
|
+
for (const [index, sentBack] of routed) {
|
|
354
|
+
const segment = plan.segments[index];
|
|
355
|
+
onProgress?.(`${sourceLabel} [segment ${index + 1}/${plan.segments.length}${describe(segment)}] sent back by the assembled document: ${summarise(sentBack)}`);
|
|
356
|
+
await workOn(segment, sentBack);
|
|
357
|
+
}
|
|
358
|
+
}
|
|
359
|
+
};
|
|
360
|
+
const describe = (segment) => segment.label.heading ? ` ‹${segment.label.heading}›` : '';
|
|
361
|
+
const summarise = (findings) => findings
|
|
362
|
+
.slice(0, 3)
|
|
363
|
+
.map((finding) => finding.rule)
|
|
364
|
+
.join(', ') + (findings.length > 3 ? `, +${findings.length - 3} more` : '');
|
|
365
|
+
/**
|
|
366
|
+
* Rules that compare the translation with its source across the whole tree.
|
|
367
|
+
*
|
|
368
|
+
* Every one of them reports on the root node, so `line` is 1 no matter where
|
|
369
|
+
* the defect is. Treating that 1 as a location is how a page without
|
|
370
|
+
* frontmatter used to send its first segment back for something that segment
|
|
371
|
+
* had nothing to do with.
|
|
372
|
+
*/
|
|
373
|
+
const PAIRWISE_RULE_PREFIX = 'doom-lint:translation-';
|
|
374
|
+
/**
|
|
375
|
+
* Which segment each whole-document finding belongs to.
|
|
376
|
+
*
|
|
377
|
+
* Two routes, because the findings arrive in two shapes:
|
|
378
|
+
*
|
|
379
|
+
* - **a placeholder** names its segments exactly — the token belongs to those
|
|
380
|
+
* segments' tables and no others. This is the important one: it covers every
|
|
381
|
+
* way the masked round trip can fail. Reference and footnote labels
|
|
382
|
+
* (`REFID`/`FNID`) are shared linkage keys, so a definition and its uses can
|
|
383
|
+
* sit in different segments; all of them are asked again, because which one
|
|
384
|
+
* dropped the label is exactly what the finding does not say;
|
|
385
|
+
* - **a line number** is looked up in the composed document, whose block
|
|
386
|
+
* structure is the one assembly recorded. When several segments' spans
|
|
387
|
+
* contain the line — a drilled container's tag segment covers every line of
|
|
388
|
+
* its children — the **narrowest** one is the owner. A child is always
|
|
389
|
+
* narrower than the container it sits in.
|
|
390
|
+
*
|
|
391
|
+
* The pairwise rules — `translation-component-multiset`,
|
|
392
|
+
* `translation-heading-sequence` and the rest — report against the document as
|
|
393
|
+
* a whole and always land on line 1, which is not a location: on a page with
|
|
394
|
+
* frontmatter that line belongs to nobody, and on a page without one it belongs
|
|
395
|
+
* to the first segment by accident. So they are recognised by name and never
|
|
396
|
+
* routed by line, which is what makes the same defect fail the same way on both
|
|
397
|
+
* kinds of page. That is also why the same comparisons are made per segment
|
|
398
|
+
* during acceptance, where they arrive with a segment attached. One of them
|
|
399
|
+
* firing *here*, after every segment passed the same comparison, means assembly
|
|
400
|
+
* itself did something — and that is worth failing loudly for rather than
|
|
401
|
+
* retrying blindly.
|
|
402
|
+
*/
|
|
403
|
+
const route = ({ findings, plan, placements, document, processor, }) => {
|
|
404
|
+
const routed = new Map();
|
|
405
|
+
const add = (index, finding) => {
|
|
406
|
+
const list = routed.get(index);
|
|
407
|
+
if (list) {
|
|
408
|
+
list.push(finding);
|
|
409
|
+
}
|
|
410
|
+
else {
|
|
411
|
+
routed.set(index, [finding]);
|
|
412
|
+
}
|
|
413
|
+
};
|
|
414
|
+
const byPlaceholder = new Map();
|
|
415
|
+
for (const segment of plan.segments) {
|
|
416
|
+
for (const token of segment.expected.keys()) {
|
|
417
|
+
const owners = byPlaceholder.get(token);
|
|
418
|
+
if (owners) {
|
|
419
|
+
owners.push(segment.index);
|
|
420
|
+
}
|
|
421
|
+
else {
|
|
422
|
+
byPlaceholder.set(token, [segment.index]);
|
|
423
|
+
}
|
|
424
|
+
}
|
|
425
|
+
}
|
|
426
|
+
const spans = document ? lineSpans(document, placements, processor) : [];
|
|
427
|
+
for (const finding of findings) {
|
|
428
|
+
const token = finding.placeholder?.toLowerCase();
|
|
429
|
+
const owners = token ? byPlaceholder.get(token) : undefined;
|
|
430
|
+
if (owners?.length) {
|
|
431
|
+
for (const owner of owners) {
|
|
432
|
+
add(owner, finding);
|
|
433
|
+
}
|
|
434
|
+
continue;
|
|
435
|
+
}
|
|
436
|
+
if (finding.line == null ||
|
|
437
|
+
finding.line <= 0 ||
|
|
438
|
+
finding.rule.startsWith(PAIRWISE_RULE_PREFIX)) {
|
|
439
|
+
continue;
|
|
440
|
+
}
|
|
441
|
+
const line = finding.line;
|
|
442
|
+
let owner;
|
|
443
|
+
for (const span of spans) {
|
|
444
|
+
if (line < span.start || line > span.end) {
|
|
445
|
+
continue;
|
|
446
|
+
}
|
|
447
|
+
if (!owner || span.end - span.start < owner.end - owner.start) {
|
|
448
|
+
owner = span;
|
|
449
|
+
}
|
|
450
|
+
}
|
|
451
|
+
if (owner) {
|
|
452
|
+
add(owner.segment, finding);
|
|
453
|
+
}
|
|
454
|
+
}
|
|
455
|
+
return routed;
|
|
456
|
+
};
|
|
457
|
+
/** Which lines of the composed document each segment occupies. */
|
|
458
|
+
const lineSpans = (document, placements, processor) => {
|
|
459
|
+
let tree;
|
|
460
|
+
try {
|
|
461
|
+
tree = processor.parse(document);
|
|
462
|
+
}
|
|
463
|
+
catch {
|
|
464
|
+
return [];
|
|
465
|
+
}
|
|
466
|
+
const shift = tree.children[0]?.type === 'yaml' ? 1 : 0;
|
|
467
|
+
const spans = [];
|
|
468
|
+
for (const { segment, address } of placements) {
|
|
469
|
+
if (address.kind === 'attributes') {
|
|
470
|
+
const node = resolveAttributeNode(tree, address, shift);
|
|
471
|
+
const position = node?.position;
|
|
472
|
+
if (position) {
|
|
473
|
+
spans.push({
|
|
474
|
+
segment,
|
|
475
|
+
start: position.start.line,
|
|
476
|
+
end: position.end.line,
|
|
477
|
+
});
|
|
478
|
+
}
|
|
479
|
+
continue;
|
|
480
|
+
}
|
|
481
|
+
const resolved = resolvePlacement(tree, address, shift);
|
|
482
|
+
const first = resolved?.nodes.at(0)?.position?.start.line;
|
|
483
|
+
const last = resolved?.nodes.at(-1)?.position?.end.line;
|
|
484
|
+
if (first != null && last != null) {
|
|
485
|
+
spans.push({ segment, start: first, end: last });
|
|
486
|
+
}
|
|
487
|
+
}
|
|
488
|
+
return spans;
|
|
489
|
+
};
|