@alauda/doom 2.6.0 → 2.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/cli/translate-call.d.ts +85 -0
- package/lib/cli/translate-call.js +149 -0
- package/lib/cli/translate-checker.d.ts +8 -0
- package/lib/cli/translate-diagnose.d.ts +45 -0
- package/lib/cli/translate-diagnose.js +118 -0
- package/lib/cli/translate-judge.d.ts +28 -3
- package/lib/cli/translate-judge.js +40 -7
- package/lib/cli/translate-mask.d.ts +13 -0
- package/lib/cli/translate-mask.js +19 -5
- package/lib/cli/translate-pipeline.d.ts +144 -0
- package/lib/cli/translate-pipeline.js +489 -0
- package/lib/cli/translate-repair-agent.d.ts +69 -0
- package/lib/cli/translate-repair-agent.js +241 -0
- package/lib/cli/translate-scratch.d.ts +9 -1
- package/lib/cli/translate-scratch.js +2 -2
- package/lib/cli/translate-segment-cache.d.ts +90 -0
- package/lib/cli/translate-segment-cache.js +301 -0
- package/lib/cli/translate-segment-check.d.ts +42 -0
- package/lib/cli/translate-segment-check.js +247 -0
- package/lib/cli/translate-segment.d.ts +307 -0
- package/lib/cli/translate-segment.js +625 -0
- package/lib/cli/translate.d.ts +14 -4
- package/lib/cli/translate.js +232 -52
- package/lib/remark-lint/no-unmatched-anchor.js +6 -2
- package/lib/remark-lint/no-unopened-frontmatter.js +5 -1
- package/lib/remark-lint/rule-sets.d.ts +1 -3
- package/lib/remark-lint/rule-sets.js +3 -9
- package/lib/remark-lint/syntax-plugins.d.ts +32 -0
- package/lib/remark-lint/syntax-plugins.js +46 -0
- package/lib/remark-lint/translation-parity/frontmatter-preservation.js +5 -3
- package/lib/remark-lint/translation-parity/link-isomorphism.d.ts +8 -0
- package/lib/remark-lint/translation-parity/link-isomorphism.js +40 -4
- package/lib/remark-lint/translation-parity/shared.js +6 -2
- package/lib/remark-lint/translation-parity/url-residue.d.ts +1 -0
- package/lib/remark-lint/translation-parity/url-residue.js +25 -4
- package/lib/runtime/hooks/useTranslation.d.ts +1 -1
- package/lib/types.d.ts +101 -6
- package/package.json +1 -1
- package/lib/cli/translate-agent.d.ts +0 -81
- package/lib/cli/translate-agent.js +0 -450
|
@@ -0,0 +1,307 @@
|
|
|
1
|
+
import type { Root, RootContent } from 'mdast';
|
|
2
|
+
import type { MdxJsxFlowElement } from 'mdast-util-mdx';
|
|
3
|
+
import { type MaskProcessor } from './translate-mask.ts';
|
|
4
|
+
/**
|
|
5
|
+
* Cutting one document into the units everything else works in.
|
|
6
|
+
*
|
|
7
|
+
* The unit of work, the unit of acceptance, the unit of retry and the unit of
|
|
8
|
+
* caching are all this: a segment. Before this existed they were all "the whole
|
|
9
|
+
* document", while the unit of *failure* was a single placeholder — so a
|
|
10
|
+
* document with 871 placeholders had to win 871 coin tosses at once, and losing
|
|
11
|
+
* any one of them threw away every correct one along with it.
|
|
12
|
+
*
|
|
13
|
+
* Cutting is done by code rather than by a model, and it is deterministic: the
|
|
14
|
+
* same source and the same parameters produce byte-identical segments. That is
|
|
15
|
+
* not tidiness, it is what the segment cache stands on — a segment whose source
|
|
16
|
+
* has not changed is recognised by its hash, and a hash is only meaningful if
|
|
17
|
+
* the cut is reproducible. Any change to the algorithm or its defaults must
|
|
18
|
+
* bump {@link SEGMENTER_VERSION}, which invalidates every recorded segment.
|
|
19
|
+
*/
|
|
20
|
+
/**
|
|
21
|
+
* Bumped whenever the cut changes.
|
|
22
|
+
*
|
|
23
|
+
* Recorded in each translation's `i18nSegments`; a translation recorded under a
|
|
24
|
+
* different version is retranslated whole rather than trusted.
|
|
25
|
+
*/
|
|
26
|
+
export declare const SEGMENTER_VERSION = 1;
|
|
27
|
+
/**
|
|
28
|
+
* How large a segment may get, in masked characters.
|
|
29
|
+
*
|
|
30
|
+
* Roughly 4k tokens of source, whose translation is far inside the 32k
|
|
31
|
+
* single-response cap. Measured against the four documents from the incident:
|
|
32
|
+
* at this cap the hardest segment carries 154 placeholders, below the 307 of
|
|
33
|
+
* the one document that reliably succeeded whole, and the median segment
|
|
34
|
+
* carries 25.
|
|
35
|
+
*/
|
|
36
|
+
export declare const DEFAULT_SEGMENT_CAP = 16000;
|
|
37
|
+
/**
|
|
38
|
+
* The size past which a single indivisible block fails the document.
|
|
39
|
+
*
|
|
40
|
+
* A block that is over {@link DEFAULT_SEGMENT_CAP} and cannot be drilled into
|
|
41
|
+
* — a huge table, a huge list — becomes a segment of its own and is allowed to
|
|
42
|
+
* exceed the cap. This is where that stops. Measured over 1030 documents: the
|
|
43
|
+
* largest such block is 24.3 KB, so this is a guard rail rather than a path.
|
|
44
|
+
*/
|
|
45
|
+
export declare const DEFAULT_SEGMENT_HARD_CAP = 48000;
|
|
46
|
+
/**
|
|
47
|
+
* How large a segment must already be before a heading is allowed to end it.
|
|
48
|
+
*
|
|
49
|
+
* Without a floor, "start a new segment at every `##`" fragments a
|
|
50
|
+
* section-dense page into uselessly small pieces: measured over 1030
|
|
51
|
+
* documents, one CLI reference of 11,948 masked characters — a page that fits
|
|
52
|
+
* in a single segment by size — cut into 104 segments averaging 115
|
|
53
|
+
* characters. Each of those is a model call and a judge reading.
|
|
54
|
+
*
|
|
55
|
+
* Measured across the same corpus, this floor halves the work without
|
|
56
|
+
* coarsening the documents that matter: 3645 segments become 1750, the median
|
|
57
|
+
* segment grows from 484 to 1986 characters, and the document this design was
|
|
58
|
+
* written for is cut into 13 segments either way — it has real sections, so
|
|
59
|
+
* the floor never fires on it. The most-segmented document in the corpus stops
|
|
60
|
+
* being a page of one-line headings and becomes that one.
|
|
61
|
+
*
|
|
62
|
+
* Merging adjacent small sections costs the cache almost nothing (an edit to a
|
|
63
|
+
* 300-character section retranslates 600 characters instead of 300, while the
|
|
64
|
+
* rest of the page is still reused) and helps everywhere else: more context
|
|
65
|
+
* per call, and fewer of the very short texts where a reviewer is most likely
|
|
66
|
+
* to invent a finding.
|
|
67
|
+
*/
|
|
68
|
+
export declare const DEFAULT_SEGMENT_FLOOR = 2000;
|
|
69
|
+
/**
|
|
70
|
+
* Where a segment lives in the tree.
|
|
71
|
+
*
|
|
72
|
+
* `container` is the path of ancestor child-indices to the node whose
|
|
73
|
+
* `children` the segment addresses — empty for the top level. `blocks` names a
|
|
74
|
+
* half-open range of those children; `attributes` names the container node
|
|
75
|
+
* itself, and covers the prose carried by its own tag rather than by its
|
|
76
|
+
* children.
|
|
77
|
+
*/
|
|
78
|
+
export type SegmentAddress = {
|
|
79
|
+
kind: 'blocks';
|
|
80
|
+
container: number[];
|
|
81
|
+
start: number;
|
|
82
|
+
end: number;
|
|
83
|
+
} | {
|
|
84
|
+
kind: 'attributes';
|
|
85
|
+
container: number[];
|
|
86
|
+
};
|
|
87
|
+
export interface SegmentLabel {
|
|
88
|
+
/** Line the segment starts on, in the source. */
|
|
89
|
+
line?: number;
|
|
90
|
+
/** The section it falls under. Both are for people reading a build log. */
|
|
91
|
+
heading?: string;
|
|
92
|
+
/**
|
|
93
|
+
* Line that heading is on, in the source.
|
|
94
|
+
*
|
|
95
|
+
* The text alone does not identify a section: `Procedure`, `Prerequisites`
|
|
96
|
+
* and `Verification` repeat under every top-level heading of a real page, so
|
|
97
|
+
* marking the outline by text puts the mark on the first one of that name
|
|
98
|
+
* rather than on this segment's. The line is what tells them apart.
|
|
99
|
+
*/
|
|
100
|
+
headingLine?: number;
|
|
101
|
+
}
|
|
102
|
+
export interface Segment {
|
|
103
|
+
/** Position in {@link SegmentPlan.segments}; how a translation is matched back. */
|
|
104
|
+
index: number;
|
|
105
|
+
address: SegmentAddress;
|
|
106
|
+
/** The masked text handed to the model — a standalone, parseable document. */
|
|
107
|
+
text: string;
|
|
108
|
+
/**
|
|
109
|
+
* How many times each placeholder must come back, keyed by the canonical
|
|
110
|
+
* lower-case token.
|
|
111
|
+
*
|
|
112
|
+
* Measured from {@link text} with the same function that will measure the
|
|
113
|
+
* translation, so the comparison is between two measurements of the same
|
|
114
|
+
* kind rather than between a text count and a tree count.
|
|
115
|
+
*/
|
|
116
|
+
expected: Map<string, number>;
|
|
117
|
+
/** `sha256(kind + text)`, first 12 hex digits. The segment cache's key. */
|
|
118
|
+
sha: string;
|
|
119
|
+
label: SegmentLabel;
|
|
120
|
+
}
|
|
121
|
+
export interface SegmentPlan {
|
|
122
|
+
segments: Segment[];
|
|
123
|
+
cap: number;
|
|
124
|
+
version: number;
|
|
125
|
+
/**
|
|
126
|
+
* Placeholders that belong to no segment.
|
|
127
|
+
*
|
|
128
|
+
* When a container is drilled into, its own opening and closing tags are held
|
|
129
|
+
* by the assembler and never sent to a model — so the placeholders masked
|
|
130
|
+
* into its attributes are never asked for back. They are counted here so the
|
|
131
|
+
* conservation check stays exact rather than being loosened to accommodate
|
|
132
|
+
* them.
|
|
133
|
+
*/
|
|
134
|
+
heldOut: Map<string, number>;
|
|
135
|
+
}
|
|
136
|
+
/** A block too large to send and impossible to divide. */
|
|
137
|
+
export declare class UnsplittableBlockError extends Error {
|
|
138
|
+
readonly size: number;
|
|
139
|
+
readonly hardCap: number;
|
|
140
|
+
readonly label: SegmentLabel;
|
|
141
|
+
constructor(size: number, hardCap: number, label: SegmentLabel);
|
|
142
|
+
}
|
|
143
|
+
/**
|
|
144
|
+
* Raised when the segments do not add up to the document.
|
|
145
|
+
*
|
|
146
|
+
* The first line of defence against the segmenter itself losing content: if
|
|
147
|
+
* every placeholder in the masked document is not accounted for exactly once
|
|
148
|
+
* across the segments and the held-out tags, the cut is wrong, and no amount of
|
|
149
|
+
* good translation downstream would reveal it.
|
|
150
|
+
*/
|
|
151
|
+
export declare class SegmentConservationError extends Error {
|
|
152
|
+
readonly divergences: readonly string[];
|
|
153
|
+
constructor(divergences: readonly string[]);
|
|
154
|
+
}
|
|
155
|
+
/** Raised when a translated segment cannot be put back where it came from. */
|
|
156
|
+
export declare class SegmentAssemblyError extends Error {
|
|
157
|
+
constructor(message: string);
|
|
158
|
+
}
|
|
159
|
+
export interface OutlineHeading {
|
|
160
|
+
depth: number;
|
|
161
|
+
text: string;
|
|
162
|
+
/** Where it is in the source — how a segment says which of two same-named sections it is in. */
|
|
163
|
+
line?: number;
|
|
164
|
+
}
|
|
165
|
+
/** Every heading on the page, in order — the map a segment is placed on. */
|
|
166
|
+
export declare const documentOutline: (tree: Root) => OutlineHeading[];
|
|
167
|
+
export interface PlanSegmentsOptions {
|
|
168
|
+
/** The masked tree, exactly as it will be sent and assembled. */
|
|
169
|
+
tree: Root;
|
|
170
|
+
processor: MaskProcessor;
|
|
171
|
+
cap?: number;
|
|
172
|
+
hardCap?: number;
|
|
173
|
+
floor?: number;
|
|
174
|
+
}
|
|
175
|
+
/**
|
|
176
|
+
* Cuts a masked document into segments.
|
|
177
|
+
*
|
|
178
|
+
* Three rules, applied in order, all of them deterministic:
|
|
179
|
+
*
|
|
180
|
+
* 1. **an `##` heading starts a new segment**, once the open one has reached
|
|
181
|
+
* {@link DEFAULT_SEGMENT_FLOOR}, so segments line up with the sections an
|
|
182
|
+
* author edits — which is what makes the cache hit after an ordinary edit,
|
|
183
|
+
* and what makes a failure report nameable — without a page of one-line
|
|
184
|
+
* sections turning into a page of one-line segments;
|
|
185
|
+
* 2. **the cap closes a segment**, greedily;
|
|
186
|
+
* 3. **a container larger than the cap is drilled into** rather than sent
|
|
187
|
+
* whole. The 24.9 KB `<Tabs>` in the document that failed is four ordinary
|
|
188
|
+
* segments under this rule. Its tag is held by the assembler; the prose on
|
|
189
|
+
* the tag — a `<Tab label>` — becomes a segment of its own, because
|
|
190
|
+
* otherwise it would be the one piece of the document nobody translates.
|
|
191
|
+
*
|
|
192
|
+
* The frontmatter is never a segment of its own: it joins the segment that
|
|
193
|
+
* follows it, so the model sees `title` and `description` in the context of the
|
|
194
|
+
* page they belong to.
|
|
195
|
+
*/
|
|
196
|
+
export declare const planSegments: ({ tree, processor, cap, hardCap, floor, }: PlanSegmentsOptions) => SegmentPlan;
|
|
197
|
+
/** Where one segment's translation ended up, so the next run can find it again. */
|
|
198
|
+
export interface SegmentRecord {
|
|
199
|
+
sha: string;
|
|
200
|
+
address: SegmentAddress;
|
|
201
|
+
}
|
|
202
|
+
export interface AssembleOptions {
|
|
203
|
+
/** The masked source tree the plan was made from. */
|
|
204
|
+
tree: Root;
|
|
205
|
+
plan: SegmentPlan;
|
|
206
|
+
processor: MaskProcessor;
|
|
207
|
+
/** Each segment's accepted translation, by segment index. */
|
|
208
|
+
translations: readonly string[];
|
|
209
|
+
}
|
|
210
|
+
export interface SegmentPlacement {
|
|
211
|
+
/** Index into {@link SegmentPlan.segments}. */
|
|
212
|
+
segment: number;
|
|
213
|
+
address: SegmentAddress;
|
|
214
|
+
}
|
|
215
|
+
export interface AssembleResult {
|
|
216
|
+
/** The whole masked translation, ready to restore. */
|
|
217
|
+
text: string;
|
|
218
|
+
/**
|
|
219
|
+
* Where every segment landed, addressed in the *translation*.
|
|
220
|
+
*
|
|
221
|
+
* Top-level indices are relative to the body — the document without its
|
|
222
|
+
* frontmatter — because that is the coordinate system every later reader
|
|
223
|
+
* works in: both the next run, which parses the written file, and the finding
|
|
224
|
+
* router, which parses the composed document.
|
|
225
|
+
*/
|
|
226
|
+
placements: SegmentPlacement[];
|
|
227
|
+
/**
|
|
228
|
+
* The subset of {@link placements} worth recording for the next run.
|
|
229
|
+
*
|
|
230
|
+
* The segment carrying the frontmatter is left out: `compose` rewrites
|
|
231
|
+
* frontmatter on the way out, so it is the one place where the assembled tree
|
|
232
|
+
* and the written file can disagree about what block 0 is, and one segment is
|
|
233
|
+
* not worth that risk.
|
|
234
|
+
*/
|
|
235
|
+
records: SegmentRecord[];
|
|
236
|
+
}
|
|
237
|
+
/**
|
|
238
|
+
* Puts the translated segments back into the shape of the source.
|
|
239
|
+
*
|
|
240
|
+
* Assembly is structural, not textual: each translation is parsed into blocks
|
|
241
|
+
* and spliced into the position its segment came from, and the whole tree is
|
|
242
|
+
* stringified once. Segments therefore cannot be lost, duplicated or reordered
|
|
243
|
+
* — not because a check says so afterwards, but because there is no step in
|
|
244
|
+
* which it could happen.
|
|
245
|
+
*/
|
|
246
|
+
/**
|
|
247
|
+
* The document's reference definitions, rendered so a segment can be parsed
|
|
248
|
+
* with them in scope.
|
|
249
|
+
*
|
|
250
|
+
* `[text][label]` and `[^1]` are only links and footnotes if the definition
|
|
251
|
+
* they name exists — otherwise markdown reads them as ordinary brackets. A
|
|
252
|
+
* definition lives once, wherever the author put it, so any *other* segment
|
|
253
|
+
* holding a reference to it parses standalone as literal text, and stringifying
|
|
254
|
+
* that text escapes the brackets and the underscores in the label: a masked
|
|
255
|
+
* `[manual][__DOOM_TR_REFID_0__]` comes back out as
|
|
256
|
+
* `\[manual]\[**DOOM\_TR\_REFID\_0**]` and the placeholder is destroyed.
|
|
257
|
+
*
|
|
258
|
+
* So every segment is parsed with all of the document's definitions appended,
|
|
259
|
+
* and the appended blocks are dropped again. Duplicating a definition a segment
|
|
260
|
+
* already has is harmless — markdown keeps the first — and dropping by count is
|
|
261
|
+
* exact because the context is parsed once on its own to learn how many blocks
|
|
262
|
+
* it is.
|
|
263
|
+
*/
|
|
264
|
+
export interface DefinitionContext {
|
|
265
|
+
/** The definitions, rendered. Appended to a segment before parsing it. */
|
|
266
|
+
text: string;
|
|
267
|
+
/** How many blocks that renders to, so the appended part can be dropped exactly. */
|
|
268
|
+
blocks: number;
|
|
269
|
+
}
|
|
270
|
+
export declare const definitionContext: (tree: Root, processor: MaskProcessor) => DefinitionContext | undefined;
|
|
271
|
+
/**
|
|
272
|
+
* Parses one segment's worth of markdown with the document's definitions in
|
|
273
|
+
* scope, and gives back only the segment's own blocks.
|
|
274
|
+
*
|
|
275
|
+
* Shared by the two places that take a segment out of a document and parse it
|
|
276
|
+
* on its own — assembly, which parses each translated segment before splicing
|
|
277
|
+
* it back, and the segment cache, which parses the previous translation's
|
|
278
|
+
* blocks to check whether they can be reused. Both were losing references the
|
|
279
|
+
* same way, so they resolve them the same way.
|
|
280
|
+
*/
|
|
281
|
+
export declare const parseWithDefinitions: (text: string, context: DefinitionContext | undefined, processor: MaskProcessor) => RootContent[];
|
|
282
|
+
export declare const assemble: ({ tree, plan, processor, translations, }: AssembleOptions) => AssembleResult;
|
|
283
|
+
/**
|
|
284
|
+
* The children a placement addresses, in a parsed translation.
|
|
285
|
+
*
|
|
286
|
+
* The one place that knows how a recorded address is read back — used both to
|
|
287
|
+
* pull a cached segment out of the previous translation and to work out which
|
|
288
|
+
* lines of the composed document a segment occupies. `shift` accounts for the
|
|
289
|
+
* frontmatter, which is block 0 of a whole file and absent from a body.
|
|
290
|
+
*/
|
|
291
|
+
export declare const resolvePlacement: (tree: Root, address: SegmentAddress, shift?: number) => {
|
|
292
|
+
parent: RootContent[];
|
|
293
|
+
nodes: RootContent[];
|
|
294
|
+
} | undefined;
|
|
295
|
+
/** The node an `attributes` placement addresses. */
|
|
296
|
+
export declare const resolveAttributeNode: (tree: Root, address: SegmentAddress, shift?: number) => MdxJsxFlowElement | undefined;
|
|
297
|
+
/**
|
|
298
|
+
* Copies the translated prose back onto a tag.
|
|
299
|
+
*
|
|
300
|
+
* Strict about shape: the translation of an attribute segment is one JSX
|
|
301
|
+
* element, and its attributes are the ones that were sent. Anything else is an
|
|
302
|
+
* assembly error rather than a silent fallback to the source language, which is
|
|
303
|
+
* the failure this whole design exists to stop happening quietly.
|
|
304
|
+
*/
|
|
305
|
+
export declare const applyAttributes: (node: MdxJsxFlowElement, translation: string | undefined, processor: MaskProcessor) => void;
|
|
306
|
+
/** The single JSX element an attribute segment must be, or nothing. */
|
|
307
|
+
export declare const parseAttributeElement: (translation: string, processor: MaskProcessor) => MdxJsxFlowElement | undefined;
|