@alauda/doom 2.5.4 → 2.6.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/cli/api.d.ts +2 -0
- package/lib/cli/api.js +147 -0
- package/lib/cli/helpers.d.ts +1 -7
- package/lib/cli/helpers.js +4 -41
- package/lib/cli/index.js +2 -0
- package/lib/cli/translate-agent.d.ts +81 -0
- package/lib/cli/translate-agent.js +450 -0
- package/lib/cli/translate-check.d.ts +17 -0
- package/lib/cli/translate-check.js +67 -0
- package/lib/cli/translate-checker.d.ts +58 -0
- package/lib/cli/translate-checker.js +112 -0
- package/lib/cli/translate-judge.d.ts +119 -0
- package/lib/cli/translate-judge.js +224 -0
- package/lib/cli/translate-limits.d.ts +43 -0
- package/lib/cli/translate-limits.js +55 -0
- package/lib/cli/translate-mask.d.ts +144 -0
- package/lib/cli/translate-mask.js +617 -0
- package/lib/cli/translate-models.d.ts +127 -0
- package/lib/cli/translate-models.js +189 -0
- package/lib/cli/translate-scratch.d.ts +142 -0
- package/lib/cli/translate-scratch.js +187 -0
- package/lib/cli/translate-terms.d.ts +22 -0
- package/lib/cli/translate-terms.js +33 -0
- package/lib/cli/translate.d.ts +15 -10
- package/lib/cli/translate.js +228 -185
- package/lib/eslint.js +22 -3
- package/lib/plugins/api/index.js +1 -1
- package/lib/plugins/api/types.d.ts +36 -1
- package/lib/plugins/global/components.d.ts +16 -0
- package/lib/plugins/global/components.js +16 -1
- package/lib/remark-lint/index.d.ts +20 -0
- package/lib/remark-lint/index.js +40 -0
- package/lib/remark-lint/no-unmatched-anchor.d.ts +10 -0
- package/lib/remark-lint/no-unmatched-anchor.js +19 -2
- package/lib/remark-lint/no-unopened-frontmatter.d.ts +2 -0
- package/lib/remark-lint/no-unopened-frontmatter.js +70 -0
- package/lib/remark-lint/no-unparsed-emphasis.d.ts +2 -0
- package/lib/remark-lint/no-unparsed-emphasis.js +53 -0
- package/lib/remark-lint/no-unresolved-api-ref.d.ts +40 -0
- package/lib/remark-lint/no-unresolved-api-ref.js +194 -0
- package/lib/remark-lint/rule-sets.d.ts +73 -0
- package/lib/remark-lint/rule-sets.js +114 -0
- package/lib/remark-lint/syntax-plugins.d.ts +32 -0
- package/lib/remark-lint/syntax-plugins.js +46 -0
- package/lib/remark-lint/translation-parity/component-multiset.d.ts +17 -0
- package/lib/remark-lint/translation-parity/component-multiset.js +35 -0
- package/lib/remark-lint/translation-parity/echoed-source.d.ts +2 -0
- package/lib/remark-lint/translation-parity/echoed-source.js +37 -0
- package/lib/remark-lint/translation-parity/frontmatter-preservation.d.ts +2 -0
- package/lib/remark-lint/translation-parity/frontmatter-preservation.js +36 -0
- package/lib/remark-lint/translation-parity/heading-sequence.d.ts +12 -0
- package/lib/remark-lint/translation-parity/heading-sequence.js +25 -0
- package/lib/remark-lint/translation-parity/index.d.ts +24 -0
- package/lib/remark-lint/translation-parity/index.js +24 -0
- package/lib/remark-lint/translation-parity/jsx-attribute-parity.d.ts +14 -0
- package/lib/remark-lint/translation-parity/jsx-attribute-parity.js +47 -0
- package/lib/remark-lint/translation-parity/length-ratio.d.ts +2 -0
- package/lib/remark-lint/translation-parity/length-ratio.js +49 -0
- package/lib/remark-lint/translation-parity/link-isomorphism.d.ts +34 -0
- package/lib/remark-lint/translation-parity/link-isomorphism.js +81 -0
- package/lib/remark-lint/translation-parity/shared.d.ts +110 -0
- package/lib/remark-lint/translation-parity/shared.js +286 -0
- package/lib/remark-lint/translation-parity/terminology-adherence.d.ts +2 -0
- package/lib/remark-lint/translation-parity/terminology-adherence.js +67 -0
- package/lib/remark-lint/translation-parity/up-to-date.d.ts +11 -0
- package/lib/remark-lint/translation-parity/up-to-date.js +21 -0
- package/lib/remark-lint/translation-parity/url-residue.d.ts +3 -0
- package/lib/remark-lint/translation-parity/url-residue.js +67 -0
- package/lib/remarkrc.d.ts +1 -1
- package/lib/remarkrc.js +7 -34
- package/lib/runtime/api-paths.d.ts +20 -0
- package/lib/runtime/api-paths.js +29 -0
- package/lib/runtime/components/K8sAPI.d.ts +14 -1
- package/lib/runtime/components/K8sAPI.js +49 -10
- package/lib/runtime/components/K8sCrd.d.ts +1 -1
- package/lib/runtime/components/K8sPermissionTable.js +10 -5
- package/lib/runtime/components/OpenAPIPath.js +19 -2
- package/lib/runtime/components/_HeadingTitle.js +24 -6
- package/lib/runtime/components/_K8sAPIEndpoints.d.ts +8 -1
- package/lib/runtime/components/_K8sAPIEndpoints.js +27 -9
- package/lib/runtime/components/_K8sAPISchema.d.ts +9 -1
- package/lib/runtime/components/_K8sAPISchema.js +24 -6
- package/lib/runtime/components/_translation-policy.d.ts +23 -0
- package/lib/runtime/components/_translation-policy.js +32 -0
- package/lib/runtime/hooks/useTranslation.d.ts +1 -1
- package/lib/runtime/translation.d.ts +68 -0
- package/lib/runtime/translation.js +51 -0
- package/lib/runtime/utils.d.ts +24 -2
- package/lib/runtime/utils.js +87 -5
- package/lib/shared/constants.d.ts +9 -0
- package/lib/shared/constants.js +14 -0
- package/lib/types.d.ts +70 -0
- package/lib/utils/helpers.js +8 -2
- package/package.json +3 -1
|
@@ -0,0 +1,450 @@
|
|
|
1
|
+
import path from 'node:path';
|
|
2
|
+
import { Language } from "../shared/index.js";
|
|
3
|
+
import { blockingFindings, } from "./translate-checker.js";
|
|
4
|
+
import { MaskIntegrityError, restoreMaskedContent, } from "./translate-mask.js";
|
|
5
|
+
import { createScratch, loadPi } from "./translate-scratch.js";
|
|
6
|
+
/** How many findings to put in front of the model at once. */
|
|
7
|
+
const MAX_REPORTED_FINDINGS = 25;
|
|
8
|
+
/** Retries for a refused request. Transient limits are normal at corpus scale. */
|
|
9
|
+
const MODEL_MAX_RETRIES = 5;
|
|
10
|
+
const MODEL_RETRY_DELAY_MS = 3_000;
|
|
11
|
+
const SCRATCH_SOURCE_NAME = 'source';
|
|
12
|
+
const SCRATCH_TRANSLATION_NAME = 'translation';
|
|
13
|
+
const maskFindingToTranslationFinding = (finding) => ({
|
|
14
|
+
rule: `doom-translate:${finding.code}`,
|
|
15
|
+
reason: finding.code === 'missing-placeholder'
|
|
16
|
+
? `\`${finding.placeholder}\` is not in the translation. It stands for a ${finding.kind}${finding.detail ? ` (${finding.detail})` : ''}. Either that part has not been translated yet, or it was dropped.`
|
|
17
|
+
: finding.code === 'duplicate-placeholder'
|
|
18
|
+
? `\`${finding.placeholder}\` appears ${finding.actual} times but must appear ${finding.expected}.`
|
|
19
|
+
: finding.code === 'unregistered-placeholder'
|
|
20
|
+
? `\`${finding.placeholder}\` was never issued — do not invent placeholders.`
|
|
21
|
+
: finding.code === 'unrestored-placeholder'
|
|
22
|
+
? `\`${finding.placeholder}\` ended up somewhere it does not belong${finding.detail ? ` — ${finding.detail}` : ''}.`
|
|
23
|
+
: (finding.detail ??
|
|
24
|
+
`the placeholder round trip failed (${finding.code})`),
|
|
25
|
+
});
|
|
26
|
+
const renderFindings = (findings) => {
|
|
27
|
+
const shown = findings.slice(0, MAX_REPORTED_FINDINGS);
|
|
28
|
+
const lines = shown.map((finding) => `- ${finding.rule}${finding.line ? ` (line ${finding.line})` : ''}: ${finding.reason}`);
|
|
29
|
+
if (findings.length > shown.length) {
|
|
30
|
+
lines.push(`- …and ${findings.length - shown.length} more of the same kind. Fix these first and check again.`);
|
|
31
|
+
}
|
|
32
|
+
return lines.join('\n');
|
|
33
|
+
};
|
|
34
|
+
export const translateWithAgent = async (options) => {
|
|
35
|
+
const { maskedSource, maskEntries, processor, compose, targetPath, sourceLabel, source, target, translationRules, terms = [], checker, judge, models, model, reasoningEffort, scratchDir, maxRepairRounds, maxTurns, limit, maxModelRetries = MODEL_MAX_RETRIES, modelRetryDelayMs = MODEL_RETRY_DELAY_MS, onProgress, } = options;
|
|
36
|
+
const [pi, { Type }] = await Promise.all([
|
|
37
|
+
loadPi(),
|
|
38
|
+
import('@earendil-works/pi-ai'),
|
|
39
|
+
]);
|
|
40
|
+
const { createReadTool, createWriteTool, createEditTool, runAgentLoop, estimateContextTokens, shouldCompact, DEFAULT_COMPACTION_SETTINGS, } = pi.core;
|
|
41
|
+
const extension = path.extname(targetPath) || '.mdx';
|
|
42
|
+
const scratch = await createScratch({
|
|
43
|
+
parentDir: scratchDir,
|
|
44
|
+
label: path.basename(targetPath, extension),
|
|
45
|
+
maskedSource,
|
|
46
|
+
extension,
|
|
47
|
+
});
|
|
48
|
+
const sourceName = `${SCRATCH_SOURCE_NAME}${extension}`;
|
|
49
|
+
const translationName = `${SCRATCH_TRANSLATION_NAME}${extension}`;
|
|
50
|
+
try {
|
|
51
|
+
const inspectDraft = async () => {
|
|
52
|
+
const draft = await scratch.readTranslation();
|
|
53
|
+
if (draft.trim() === '') {
|
|
54
|
+
return {
|
|
55
|
+
findings: [
|
|
56
|
+
{
|
|
57
|
+
rule: 'doom-translate:nothing-written',
|
|
58
|
+
reason: `\`${translationName}\` is still empty. Write the translation there.`,
|
|
59
|
+
},
|
|
60
|
+
],
|
|
61
|
+
};
|
|
62
|
+
}
|
|
63
|
+
let restored;
|
|
64
|
+
try {
|
|
65
|
+
restored = restoreMaskedContent(draft, maskEntries, processor);
|
|
66
|
+
}
|
|
67
|
+
catch (error) {
|
|
68
|
+
if (error instanceof MaskIntegrityError) {
|
|
69
|
+
return {
|
|
70
|
+
findings: error.findings.map(maskFindingToTranslationFinding),
|
|
71
|
+
};
|
|
72
|
+
}
|
|
73
|
+
throw error;
|
|
74
|
+
}
|
|
75
|
+
const document = compose(restored);
|
|
76
|
+
const findings = await checker.check(targetPath, document);
|
|
77
|
+
if (findings.length > 0) {
|
|
78
|
+
// The judge is the expensive check and the only one that reads. Asking
|
|
79
|
+
// it about a draft the deterministic rules have already faulted spends
|
|
80
|
+
// a reading on a document that is going back for repair regardless —
|
|
81
|
+
// and asks it to judge the meaning of a half-written page.
|
|
82
|
+
return { findings };
|
|
83
|
+
}
|
|
84
|
+
if (judge) {
|
|
85
|
+
findings.push(...(await judge.review({
|
|
86
|
+
// The masked pair, not the restored one. Judge findings are fed
|
|
87
|
+
// back into the translator's context, and a finding quoting an
|
|
88
|
+
// unmasked link target would put that target back within the
|
|
89
|
+
// model's reach. Placeholders are stable on both sides, so nothing
|
|
90
|
+
// about judging prose is lost.
|
|
91
|
+
sourceText: maskedSource,
|
|
92
|
+
translationText: draft,
|
|
93
|
+
sourceLanguage: Language[source],
|
|
94
|
+
targetLanguage: Language[target],
|
|
95
|
+
terms,
|
|
96
|
+
})));
|
|
97
|
+
}
|
|
98
|
+
return blockingFindings(findings).length > 0
|
|
99
|
+
? { findings }
|
|
100
|
+
: { findings, document };
|
|
101
|
+
};
|
|
102
|
+
// ---------------------------------------------------------------- tools
|
|
103
|
+
const toolContext = { env: scratch.env };
|
|
104
|
+
const bind = (tool) => ({
|
|
105
|
+
...tool,
|
|
106
|
+
execute: (toolCallId, params, signal, onUpdate) => tool.execute(toolCallId, params, signal, onUpdate, toolContext),
|
|
107
|
+
});
|
|
108
|
+
const textResult = (text, details) => ({
|
|
109
|
+
content: [{ type: 'text', text }],
|
|
110
|
+
details,
|
|
111
|
+
});
|
|
112
|
+
/**
|
|
113
|
+
* Appending.
|
|
114
|
+
*
|
|
115
|
+
* pi's `write` replaces a file, and using `edit` to add to the end means
|
|
116
|
+
* quoting the end as an anchor. Neither suits a translation produced a
|
|
117
|
+
* section at a time, and `FileSystem` already has `appendFile`.
|
|
118
|
+
*/
|
|
119
|
+
const appendTool = {
|
|
120
|
+
label: 'Append',
|
|
121
|
+
name: 'append',
|
|
122
|
+
description: 'Add text to the end of a file, keeping what is already there. Use this for each section of the translation after the first.',
|
|
123
|
+
parameters: Type.Object({
|
|
124
|
+
path: Type.String({ description: 'File to append to.' }),
|
|
125
|
+
text: Type.String({ description: 'Text to add at the end.' }),
|
|
126
|
+
}),
|
|
127
|
+
execute: async (_toolCallId, params) => {
|
|
128
|
+
const { path: target, text } = params;
|
|
129
|
+
const appended = await scratch.env.appendFile(target, text);
|
|
130
|
+
if (!appended.ok) {
|
|
131
|
+
throw new Error(`Could not append to ${target}: ${appended.error.message}`);
|
|
132
|
+
}
|
|
133
|
+
const info = await scratch.env.fileInfo(target);
|
|
134
|
+
return textResult(`Appended ${text.length} characters.${info.ok ? ` \`${path.basename(target)}\` is now ${info.value.size} bytes.` : ''}`, undefined);
|
|
135
|
+
},
|
|
136
|
+
};
|
|
137
|
+
/**
|
|
138
|
+
* Reading by byte range.
|
|
139
|
+
*
|
|
140
|
+
* pi's `read` pages by line, and when one line is longer than its byte
|
|
141
|
+
* limit it tells the model to fall back to `sed` through a shell. There is
|
|
142
|
+
* no shell here, on purpose — which would leave that content unreachable,
|
|
143
|
+
* and a document unfinishable for a reason no repair round could clear. So
|
|
144
|
+
* the escape hatch is a tool instead of dead text; `afterToolCall` below
|
|
145
|
+
* rewrites the message that points at it.
|
|
146
|
+
*/
|
|
147
|
+
const readBytesTool = {
|
|
148
|
+
label: 'Read bytes',
|
|
149
|
+
name: 'read_bytes',
|
|
150
|
+
description: 'Read a byte range of a file. Use this when `read` reports a line too long to show.',
|
|
151
|
+
parameters: Type.Object({
|
|
152
|
+
path: Type.String({ description: 'File to read.' }),
|
|
153
|
+
offset: Type.Number({ description: 'First byte to read, from 0.' }),
|
|
154
|
+
limit: Type.Number({ description: 'How many bytes to read.' }),
|
|
155
|
+
}),
|
|
156
|
+
execute: async (_toolCallId, params) => {
|
|
157
|
+
const { path: target, offset, limit: byteLimit, } = params;
|
|
158
|
+
const read = await scratch.env.readBinaryFile(target);
|
|
159
|
+
if (!read.ok) {
|
|
160
|
+
throw new Error(`Could not read ${target}: ${read.error.message}`);
|
|
161
|
+
}
|
|
162
|
+
const slice = Buffer.from(read.value).subarray(Math.max(0, offset), Math.max(0, offset) + Math.max(0, byteLimit));
|
|
163
|
+
const total = read.value.byteLength;
|
|
164
|
+
const end = Math.min(total, Math.max(0, offset) + Math.max(0, byteLimit));
|
|
165
|
+
return textResult(`${slice.toString('utf8')}\n\n[bytes ${offset}–${end} of ${total}]`, undefined);
|
|
166
|
+
},
|
|
167
|
+
};
|
|
168
|
+
let checkCalls = 0;
|
|
169
|
+
/**
|
|
170
|
+
* The agent's view of its own work.
|
|
171
|
+
*
|
|
172
|
+
* It returns findings and nothing else — never the restored document. The
|
|
173
|
+
* restored text holds the real link targets and identifiers, and handing
|
|
174
|
+
* them back would put in the model's context exactly what masking took out
|
|
175
|
+
* of it.
|
|
176
|
+
*/
|
|
177
|
+
const checkTool = {
|
|
178
|
+
label: 'Check',
|
|
179
|
+
name: 'check',
|
|
180
|
+
description: 'Compare your translation with the source and list what is wrong. Reports nothing when the translation is complete and correct.',
|
|
181
|
+
parameters: Type.Object({}),
|
|
182
|
+
execute: async () => {
|
|
183
|
+
checkCalls++;
|
|
184
|
+
const report = await inspectDraft();
|
|
185
|
+
if (report.findings.length === 0) {
|
|
186
|
+
return textResult('No problems found. The translation is complete and matches the source.', undefined);
|
|
187
|
+
}
|
|
188
|
+
return textResult(`${report.findings.length} problem(s):\n${renderFindings(report.findings)}`, undefined);
|
|
189
|
+
},
|
|
190
|
+
};
|
|
191
|
+
/**
|
|
192
|
+
* pi's `read`, with its one untrue sentence corrected.
|
|
193
|
+
*
|
|
194
|
+
* When a line is longer than its byte limit, the tool tells the model to
|
|
195
|
+
* fall back to `sed` through a shell. There is no shell here, on purpose,
|
|
196
|
+
* so that advice is dead text — and a document containing such a line would
|
|
197
|
+
* be unfinishable for a reason no repair round could clear. Rewritten to
|
|
198
|
+
* point at `read_bytes` instead, next to the tool it is about rather than
|
|
199
|
+
* in a hook somewhere else.
|
|
200
|
+
*/
|
|
201
|
+
const baseReadTool = bind(createReadTool());
|
|
202
|
+
const readTool = {
|
|
203
|
+
...baseReadTool,
|
|
204
|
+
execute: async (toolCallId, params, signal, onUpdate) => {
|
|
205
|
+
const result = await baseReadTool.execute(toolCallId, params, signal, onUpdate);
|
|
206
|
+
return {
|
|
207
|
+
...result,
|
|
208
|
+
content: result.content.map((part) => part.type === 'text' && part.text.includes('Use bash:')
|
|
209
|
+
? {
|
|
210
|
+
...part,
|
|
211
|
+
text: part.text.replace(/Use bash: sed -n '\d+p' (\S+) \| head -c \d+/, 'Use read_bytes on $1 to read it a byte range at a time'),
|
|
212
|
+
}
|
|
213
|
+
: part),
|
|
214
|
+
};
|
|
215
|
+
},
|
|
216
|
+
};
|
|
217
|
+
const tools = [
|
|
218
|
+
readTool,
|
|
219
|
+
bind(createWriteTool()),
|
|
220
|
+
bind(createEditTool()),
|
|
221
|
+
appendTool,
|
|
222
|
+
readBytesTool,
|
|
223
|
+
checkTool,
|
|
224
|
+
];
|
|
225
|
+
// -------------------------------------------------------------- prompts
|
|
226
|
+
const systemPrompt = [
|
|
227
|
+
`You are translating one document, on your own, until it is correct.`,
|
|
228
|
+
'',
|
|
229
|
+
`## Your working directory`,
|
|
230
|
+
'',
|
|
231
|
+
`It is everything you can reach. There is no other file, and no shell.`,
|
|
232
|
+
'',
|
|
233
|
+
`- \`${sourceName}\` — the prepared source document.`,
|
|
234
|
+
`- \`${translationName}\` — where your translation goes. It starts empty.`,
|
|
235
|
+
'',
|
|
236
|
+
`## How to work`,
|
|
237
|
+
'',
|
|
238
|
+
`1. \`read\` \`${sourceName}\`. Long documents come back a page at a time; keep reading with \`offset\` until you have seen all of it, and translate it section by section rather than holding it all at once.`,
|
|
239
|
+
`2. \`write\` the first part of the translation to \`${translationName}\`, then \`append\` each following part. \`write\` replaces the file, so use it once.`,
|
|
240
|
+
`3. When the whole document has been translated, call \`check\`. It compares your translation against the source and lists what is wrong.`,
|
|
241
|
+
`4. Repair with \`edit\` and \`check\` again. Prefer \`edit\` over rewriting: a targeted change is one fewer chance to damage what was already right.`,
|
|
242
|
+
'',
|
|
243
|
+
`Anything you released from your context is still on disk — \`read\` it again rather than working from memory.`,
|
|
244
|
+
'',
|
|
245
|
+
`## When you are finished`,
|
|
246
|
+
'',
|
|
247
|
+
`You are finished when \`check\` reports nothing. Saying so yourself does not end the task: the same check runs again when you stop, and anything it finds comes back to you as another turn.`,
|
|
248
|
+
'',
|
|
249
|
+
`## Placeholders`,
|
|
250
|
+
'',
|
|
251
|
+
`Tokens shaped like \`__DOOM_TR_LINK_3__\` stand for content you must not author — link targets, code, identifiers, component attribute values. Reproduce each one verbatim and exactly once, in the position it appears. Never translate, reformat, split, renumber, remove, duplicate or invent one. The text around a placeholder is translated as usual.`,
|
|
252
|
+
'',
|
|
253
|
+
`## Translation rules`,
|
|
254
|
+
'',
|
|
255
|
+
translationRules,
|
|
256
|
+
].join('\n');
|
|
257
|
+
const initialPrompt = {
|
|
258
|
+
role: 'user',
|
|
259
|
+
content: `Translate \`${sourceName}\` and write the translation to \`${translationName}\`. The source is ${maskedSource.length} characters long and contains ${maskEntries.length} placeholder(s), every one of which must appear in your translation.`,
|
|
260
|
+
timestamp: Date.now(),
|
|
261
|
+
};
|
|
262
|
+
// ------------------------------------------------------- context release
|
|
263
|
+
/**
|
|
264
|
+
* Releasing context, deterministically.
|
|
265
|
+
*
|
|
266
|
+
* What is dropped is chosen by a rule, not by a model: the bodies of
|
|
267
|
+
* `read`/`read_bytes` results that are no longer the recent ones. pi ships
|
|
268
|
+
* a compaction that summarises the conversation with an LLM, and that is
|
|
269
|
+
* the one thing this cannot use — the context holds source and translation
|
|
270
|
+
* text that has to survive byte for byte, and summarising it hands the
|
|
271
|
+
* bytes to the process they are being protected from. Only pi's
|
|
272
|
+
* *measurement* is borrowed, to decide when to release.
|
|
273
|
+
*
|
|
274
|
+
* Releasing is safe because nothing is lost: every released page is still
|
|
275
|
+
* in the scratch directory, and the agent is told to read it again.
|
|
276
|
+
*/
|
|
277
|
+
const KEEP_RECENT_MESSAGES = 6;
|
|
278
|
+
const RELEASABLE_TOOLS = new Set(['read', 'read_bytes']);
|
|
279
|
+
const wordingAlreadyUsed = (produced) => terms.filter((term) => produced.includes(term.target));
|
|
280
|
+
const transformContext = async (messages) => {
|
|
281
|
+
const estimate = estimateContextTokens(messages);
|
|
282
|
+
if (!shouldCompact(estimate.tokens, model.contextWindow, DEFAULT_COMPACTION_SETTINGS)) {
|
|
283
|
+
return messages;
|
|
284
|
+
}
|
|
285
|
+
const cutoff = messages.length - KEEP_RECENT_MESSAGES;
|
|
286
|
+
let released = 0;
|
|
287
|
+
const trimmed = messages.map((message, index) => {
|
|
288
|
+
if (index >= cutoff) {
|
|
289
|
+
return message;
|
|
290
|
+
}
|
|
291
|
+
if (!('role' in message) ||
|
|
292
|
+
message.role !== 'toolResult' ||
|
|
293
|
+
!RELEASABLE_TOOLS.has(message.toolName)) {
|
|
294
|
+
return message;
|
|
295
|
+
}
|
|
296
|
+
released++;
|
|
297
|
+
return {
|
|
298
|
+
...message,
|
|
299
|
+
content: [
|
|
300
|
+
{
|
|
301
|
+
type: 'text',
|
|
302
|
+
text: `[released from context to make room. Read \`${sourceName}\` again if you need this part.]`,
|
|
303
|
+
},
|
|
304
|
+
],
|
|
305
|
+
};
|
|
306
|
+
});
|
|
307
|
+
if (released === 0) {
|
|
308
|
+
return messages;
|
|
309
|
+
}
|
|
310
|
+
// Carried forward as a fact, not a preference: these are the words this
|
|
311
|
+
// document has already used, and the released pages are where the agent
|
|
312
|
+
// would otherwise have seen them.
|
|
313
|
+
const produced = await scratch.readTranslation();
|
|
314
|
+
const used = wordingAlreadyUsed(produced);
|
|
315
|
+
if (used.length === 0) {
|
|
316
|
+
return trimmed;
|
|
317
|
+
}
|
|
318
|
+
return [
|
|
319
|
+
...trimmed,
|
|
320
|
+
{
|
|
321
|
+
role: 'user',
|
|
322
|
+
content: `For consistency with what you have already written in this document: ${used
|
|
323
|
+
.map((term) => `"${term.source}" → "${term.target}"`)
|
|
324
|
+
.join(', ')}.`,
|
|
325
|
+
timestamp: Date.now(),
|
|
326
|
+
},
|
|
327
|
+
];
|
|
328
|
+
};
|
|
329
|
+
// ------------------------------------------------------------- the loop
|
|
330
|
+
let turns = 0;
|
|
331
|
+
let repairRounds = 0;
|
|
332
|
+
let modelCalls = 0;
|
|
333
|
+
const failure = {};
|
|
334
|
+
// Cleared through a call, not an assignment: an assignment here narrows the
|
|
335
|
+
// property to `undefined` for the rest of the block, and the sink that sets
|
|
336
|
+
// it is a callback the compiler cannot follow.
|
|
337
|
+
const clearFailure = () => {
|
|
338
|
+
failure.message = undefined;
|
|
339
|
+
};
|
|
340
|
+
const streamFn = (streamModel, context, streamOptions) => {
|
|
341
|
+
modelCalls++;
|
|
342
|
+
return limit(() => Promise.resolve(models.streamSimple(streamModel, context, streamOptions)));
|
|
343
|
+
};
|
|
344
|
+
const config = {
|
|
345
|
+
model,
|
|
346
|
+
reasoning: reasoningEffort,
|
|
347
|
+
// The gateway refuses requests when a user is over its per-minute limit,
|
|
348
|
+
// and a corpus run is thousands of calls. Measured: a translation that
|
|
349
|
+
// met a 429 ended the whole document, because nothing retried it. The
|
|
350
|
+
// rate limiter keeps us under the limit in the normal case; this is what
|
|
351
|
+
// happens when something else is also using the budget.
|
|
352
|
+
maxRetries: MODEL_MAX_RETRIES,
|
|
353
|
+
convertToLlm: (messages) => messages,
|
|
354
|
+
transformContext,
|
|
355
|
+
// The runaway guard. pi checks this *before* it polls for follow-ups, so
|
|
356
|
+
// it wins over the exit condition below: a loop that will not converge
|
|
357
|
+
// ends here, and the caller finds out it ended failed.
|
|
358
|
+
shouldStopAfterTurn: () => ++turns >= maxTurns,
|
|
359
|
+
// The exit condition. pi calls this when the agent has run out of tool
|
|
360
|
+
// calls and would stop; anything returned goes into the context and the
|
|
361
|
+
// agent takes another turn. So the agent finishes exactly when the
|
|
362
|
+
// harness's own check finds nothing — it never sees this call and cannot
|
|
363
|
+
// route around it.
|
|
364
|
+
getFollowUpMessages: async () => {
|
|
365
|
+
const report = await inspectDraft();
|
|
366
|
+
// Advisory findings — the judge's readability notes — travel with the
|
|
367
|
+
// feedback when there is feedback, and never cause a round of their
|
|
368
|
+
// own. Meeting the standard is the bar; "cannot be improved" is not a
|
|
369
|
+
// bar a loop can reach.
|
|
370
|
+
const blocking = blockingFindings(report.findings);
|
|
371
|
+
if (blocking.length === 0) {
|
|
372
|
+
return [];
|
|
373
|
+
}
|
|
374
|
+
if (repairRounds >= maxRepairRounds) {
|
|
375
|
+
return [];
|
|
376
|
+
}
|
|
377
|
+
repairRounds++;
|
|
378
|
+
onProgress?.(`${sourceLabel}: ${blocking.length} problem(s) after round ${repairRounds}, sending them back`);
|
|
379
|
+
return [
|
|
380
|
+
{
|
|
381
|
+
role: 'user',
|
|
382
|
+
content: `The translation is not finished. Checking it against the source reports ${report.findings.length} problem(s):\n\n${renderFindings(report.findings)}\n\nFix them in \`${translationName}\`, then call \`check\` again.`,
|
|
383
|
+
timestamp: Date.now(),
|
|
384
|
+
},
|
|
385
|
+
];
|
|
386
|
+
},
|
|
387
|
+
};
|
|
388
|
+
const watchForStreamFailure = (event) => {
|
|
389
|
+
if (event.type === 'turn_end' &&
|
|
390
|
+
'stopReason' in event.message &&
|
|
391
|
+
(event.message.stopReason === 'error' ||
|
|
392
|
+
event.message.stopReason === 'aborted')) {
|
|
393
|
+
failure.message =
|
|
394
|
+
('errorMessage' in event.message
|
|
395
|
+
? event.message.errorMessage
|
|
396
|
+
: undefined) ?? event.message.stopReason;
|
|
397
|
+
}
|
|
398
|
+
};
|
|
399
|
+
/**
|
|
400
|
+
* A gateway that refuses is not a translation that failed.
|
|
401
|
+
*
|
|
402
|
+
* Measured on the real gateway while translating three documents: "429
|
|
403
|
+
* user requests-per-minute limit exceeded" and "Upstream service
|
|
404
|
+
* temporarily unavailable", both of which pi surfaces as a stream error
|
|
405
|
+
* that ends the run. A corpus is thousands of calls; meeting one of those
|
|
406
|
+
* is routine, and failing the document over it would fail the wrong thing
|
|
407
|
+
* — loudly, and in a way that makes the gate look unreliable rather than
|
|
408
|
+
* the gateway.
|
|
409
|
+
*
|
|
410
|
+
* Retrying is cheap here because the draft is on disk. The agent starts
|
|
411
|
+
* with a fresh context and is told to carry on from what it has already
|
|
412
|
+
* written, so a refusal costs the turns since the last write rather than
|
|
413
|
+
* the whole document.
|
|
414
|
+
*/
|
|
415
|
+
for (let attempt = 0;; attempt++) {
|
|
416
|
+
clearFailure();
|
|
417
|
+
const prompt = attempt === 0
|
|
418
|
+
? initialPrompt
|
|
419
|
+
: {
|
|
420
|
+
role: 'user',
|
|
421
|
+
content: `Continue translating \`${sourceName}\` into \`${translationName}\`. Some of it may already be written: \`read\` \`${translationName}\` first, then call \`check\` to see what is still missing, and carry on from there.`,
|
|
422
|
+
timestamp: Date.now(),
|
|
423
|
+
};
|
|
424
|
+
await runAgentLoop([prompt], { systemPrompt, messages: [], tools }, config, watchForStreamFailure, undefined, streamFn);
|
|
425
|
+
if (!failure.message) {
|
|
426
|
+
break;
|
|
427
|
+
}
|
|
428
|
+
if (attempt >= maxModelRetries || turns >= maxTurns) {
|
|
429
|
+
throw new Error(`The translation model failed while translating ${sourceLabel}: ${failure.message}`);
|
|
430
|
+
}
|
|
431
|
+
onProgress?.(`${sourceLabel}: the gateway refused (${failure.message}) — retrying from what is already written`);
|
|
432
|
+
await new Promise((resolve) => globalThis.setTimeout(resolve, modelRetryDelayMs * 2 ** attempt));
|
|
433
|
+
}
|
|
434
|
+
// The loop can end two ways — the check passed, or the turn cap fired
|
|
435
|
+
// before it did. Only one of them means the document is good, so it is
|
|
436
|
+
// decided here rather than inferred from how the loop exited.
|
|
437
|
+
const final = await inspectDraft();
|
|
438
|
+
void checkCalls;
|
|
439
|
+
return {
|
|
440
|
+
document: final.document,
|
|
441
|
+
findings: final.findings,
|
|
442
|
+
turns,
|
|
443
|
+
repairRounds,
|
|
444
|
+
modelCalls,
|
|
445
|
+
};
|
|
446
|
+
}
|
|
447
|
+
finally {
|
|
448
|
+
await scratch.dispose();
|
|
449
|
+
}
|
|
450
|
+
};
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
import type { GlobalCliOptions } from '../types.ts';
|
|
2
|
+
/**
|
|
3
|
+
* `doom translate check` — run the translation checks over translations that
|
|
4
|
+
* already exist, without calling a translation model at all.
|
|
5
|
+
*
|
|
6
|
+
* Two things this is for. It is the offline way to survey the ~40 repositories
|
|
7
|
+
* that consume doom and find out which translated documents are damaged, at no
|
|
8
|
+
* token cost, before deciding what is worth re-translating. And it is the same
|
|
9
|
+
* rule set the translator runs in its own loop — deliberately the same, because
|
|
10
|
+
* a second set of criteria is a second thing to keep in step, and the first one
|
|
11
|
+
* to fall out of it.
|
|
12
|
+
*/
|
|
13
|
+
export interface TranslateCheckOptions {
|
|
14
|
+
glob: string[];
|
|
15
|
+
target?: string[];
|
|
16
|
+
}
|
|
17
|
+
export declare const checkTranslations: (root: string | undefined, globalOptions: GlobalCliOptions, { glob: globs, target }: TranslateCheckOptions) => Promise<void>;
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
import fs from 'node:fs/promises';
|
|
2
|
+
import path from 'node:path';
|
|
3
|
+
import { logger } from '@rspress/core';
|
|
4
|
+
import { glob } from 'tinyglobby';
|
|
5
|
+
import { cyan, red } from 'yoctocolors';
|
|
6
|
+
import { SUPPORTED_LANGUAGES } from "../shared/index.js";
|
|
7
|
+
import { OPTIONS_FILE, STORAGE_DIR } from "../utils/index.js";
|
|
8
|
+
import { loadConfig } from "./load-config.js";
|
|
9
|
+
import { createTranslationChecker } from "./translate-checker.js";
|
|
10
|
+
export const checkTranslations = async (root, globalOptions, { glob: globs, target }) => {
|
|
11
|
+
await fs.mkdir(STORAGE_DIR, { recursive: true });
|
|
12
|
+
await fs.writeFile(OPTIONS_FILE, JSON.stringify({ root, globalOptions }, null, 2));
|
|
13
|
+
const { config } = await loadConfig(root, globalOptions);
|
|
14
|
+
const docsDir = config.root;
|
|
15
|
+
const sourceLang = config.lang ?? 'en';
|
|
16
|
+
const targets = [];
|
|
17
|
+
if (target?.length) {
|
|
18
|
+
targets.push(...target);
|
|
19
|
+
}
|
|
20
|
+
else {
|
|
21
|
+
for (const lang of SUPPORTED_LANGUAGES) {
|
|
22
|
+
if (lang === sourceLang) {
|
|
23
|
+
continue;
|
|
24
|
+
}
|
|
25
|
+
const present = await fs
|
|
26
|
+
.stat(path.resolve(docsDir, lang))
|
|
27
|
+
.then((stat) => stat.isDirectory())
|
|
28
|
+
.catch(() => false);
|
|
29
|
+
if (present) {
|
|
30
|
+
targets.push(lang);
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
if (targets.length === 0) {
|
|
35
|
+
logger.error(`No translated language directories found under \`${cyan(docsDir)}\`. Nothing to check.`);
|
|
36
|
+
process.exitCode = 1;
|
|
37
|
+
return;
|
|
38
|
+
}
|
|
39
|
+
const files = (await Promise.all(targets.map((lang) => glob(globs.map((pattern) => `${lang}/${pattern}`), { absolute: true, cwd: docsDir }))))
|
|
40
|
+
.flat()
|
|
41
|
+
.sort();
|
|
42
|
+
if (files.length === 0) {
|
|
43
|
+
logger.error(`No documents matched in ${targets.map((lang) => `\`${cyan(lang)}\``).join(', ')}. Refusing to report "no problems" for a search that found nothing.`);
|
|
44
|
+
process.exitCode = 1;
|
|
45
|
+
return;
|
|
46
|
+
}
|
|
47
|
+
logger.start(`Checking ${cyan(String(files.length))} translated document(s) in ${targets.map((lang) => `\`${cyan(lang)}\``).join(', ')}...`);
|
|
48
|
+
const checker = createTranslationChecker();
|
|
49
|
+
let problems = 0;
|
|
50
|
+
for (const file of files) {
|
|
51
|
+
const value = await fs.readFile(file, 'utf8');
|
|
52
|
+
const findings = await checker.check(file, value);
|
|
53
|
+
if (findings.length === 0) {
|
|
54
|
+
continue;
|
|
55
|
+
}
|
|
56
|
+
problems += findings.length;
|
|
57
|
+
logger.error(`${cyan(path.relative(docsDir, file))}\n${findings
|
|
58
|
+
.map((finding) => ` ${red(finding.rule)}${finding.line ? ` (line ${finding.line})` : ''} ${finding.reason}`)
|
|
59
|
+
.join('\n')}`);
|
|
60
|
+
}
|
|
61
|
+
if (problems > 0) {
|
|
62
|
+
logger.error(`\`doom translate check\` found ${red(String(problems))} problem(s) in ${files.length} document(s).`);
|
|
63
|
+
process.exitCode = 1;
|
|
64
|
+
return;
|
|
65
|
+
}
|
|
66
|
+
logger.success(`\`doom translate check\` passed: ${files.length} document(s), no problems.`);
|
|
67
|
+
};
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Checking one translated document — the single implementation, used from all
|
|
3
|
+
* three places that need it.
|
|
4
|
+
*
|
|
5
|
+
* 1. `doom translate check`, over translations that are already on disk;
|
|
6
|
+
* 2. the `check` tool the translating agent calls to see its own work;
|
|
7
|
+
* 3. the harness's own verification, which is what lets a translation finish.
|
|
8
|
+
*
|
|
9
|
+
* They share a function rather than a convention. "The agent is checked
|
|
10
|
+
* against the same rules it checks itself with" is then true in the literal
|
|
11
|
+
* sense — there is one rule set because there is one call.
|
|
12
|
+
*/
|
|
13
|
+
export interface TranslationFinding {
|
|
14
|
+
/** Rule id, e.g. `doom-lint:translation-component-multiset`. */
|
|
15
|
+
rule: string;
|
|
16
|
+
reason: string;
|
|
17
|
+
line?: number;
|
|
18
|
+
column?: number;
|
|
19
|
+
/**
|
|
20
|
+
* Whether this finding stops the document from shipping.
|
|
21
|
+
*
|
|
22
|
+
* Everything a rule finds does — the rules only report things that are
|
|
23
|
+
* wrong. The judge is the exception: it also reports readability, which is
|
|
24
|
+
* worth telling the translator about and is not worth failing a build over.
|
|
25
|
+
* "Meets the standard" is the bar, not "cannot be improved".
|
|
26
|
+
*
|
|
27
|
+
* Absent means blocking.
|
|
28
|
+
*/
|
|
29
|
+
blocking?: boolean;
|
|
30
|
+
}
|
|
31
|
+
/** Findings that stop a document from shipping. */
|
|
32
|
+
export declare const blockingFindings: (findings: readonly TranslationFinding[]) => TranslationFinding[];
|
|
33
|
+
/**
|
|
34
|
+
* Raised when a document cannot be checked at all.
|
|
35
|
+
*
|
|
36
|
+
* This is not a finding: a finding says the translation is wrong, this says
|
|
37
|
+
* nothing was examined. The distinction matters because the failure mode of
|
|
38
|
+
* the pairwise rules is to return quietly — a translation whose path or
|
|
39
|
+
* frontmatter is not what they expect is not a translation as far as they are
|
|
40
|
+
* concerned, and every rule passes. Silence and success look identical, so the
|
|
41
|
+
* caller has to be told which one it got.
|
|
42
|
+
*/
|
|
43
|
+
export declare class TranslationNotCheckableError extends Error {
|
|
44
|
+
readonly targetPath: string;
|
|
45
|
+
constructor(targetPath: string, reason: string);
|
|
46
|
+
}
|
|
47
|
+
export interface TranslationChecker {
|
|
48
|
+
/** Every problem the rules find in `content`, judged as the document at `targetPath`. */
|
|
49
|
+
check(targetPath: string, content: string): Promise<TranslationFinding[]>;
|
|
50
|
+
/**
|
|
51
|
+
* Confirms the pairwise rules will actually compare this document against a
|
|
52
|
+
* source, and throws {@link TranslationNotCheckableError} when they will not.
|
|
53
|
+
*/
|
|
54
|
+
assertCheckable(targetPath: string, content: string): Promise<void>;
|
|
55
|
+
}
|
|
56
|
+
export declare const createTranslationChecker: () => TranslationChecker;
|
|
57
|
+
/** One finding, rendered for a person reading a build log. */
|
|
58
|
+
export declare const describeFinding: (finding: TranslationFinding) => string;
|