@alauda/doom 2.6.0 → 2.7.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (40) hide show
  1. package/lib/cli/translate-call.d.ts +85 -0
  2. package/lib/cli/translate-call.js +149 -0
  3. package/lib/cli/translate-checker.d.ts +8 -0
  4. package/lib/cli/translate-diagnose.d.ts +45 -0
  5. package/lib/cli/translate-diagnose.js +118 -0
  6. package/lib/cli/translate-judge.d.ts +28 -3
  7. package/lib/cli/translate-judge.js +40 -7
  8. package/lib/cli/translate-mask.d.ts +13 -0
  9. package/lib/cli/translate-mask.js +19 -5
  10. package/lib/cli/translate-pipeline.d.ts +144 -0
  11. package/lib/cli/translate-pipeline.js +489 -0
  12. package/lib/cli/translate-repair-agent.d.ts +69 -0
  13. package/lib/cli/translate-repair-agent.js +241 -0
  14. package/lib/cli/translate-scratch.d.ts +9 -1
  15. package/lib/cli/translate-scratch.js +2 -2
  16. package/lib/cli/translate-segment-cache.d.ts +90 -0
  17. package/lib/cli/translate-segment-cache.js +301 -0
  18. package/lib/cli/translate-segment-check.d.ts +42 -0
  19. package/lib/cli/translate-segment-check.js +247 -0
  20. package/lib/cli/translate-segment.d.ts +307 -0
  21. package/lib/cli/translate-segment.js +625 -0
  22. package/lib/cli/translate.d.ts +14 -4
  23. package/lib/cli/translate.js +232 -52
  24. package/lib/remark-lint/no-unmatched-anchor.js +6 -2
  25. package/lib/remark-lint/no-unopened-frontmatter.js +5 -1
  26. package/lib/remark-lint/rule-sets.d.ts +1 -3
  27. package/lib/remark-lint/rule-sets.js +3 -9
  28. package/lib/remark-lint/syntax-plugins.d.ts +32 -0
  29. package/lib/remark-lint/syntax-plugins.js +46 -0
  30. package/lib/remark-lint/translation-parity/frontmatter-preservation.js +5 -3
  31. package/lib/remark-lint/translation-parity/link-isomorphism.d.ts +8 -0
  32. package/lib/remark-lint/translation-parity/link-isomorphism.js +40 -4
  33. package/lib/remark-lint/translation-parity/shared.js +6 -2
  34. package/lib/remark-lint/translation-parity/url-residue.d.ts +1 -0
  35. package/lib/remark-lint/translation-parity/url-residue.js +25 -4
  36. package/lib/runtime/hooks/useTranslation.d.ts +1 -1
  37. package/lib/types.d.ts +101 -6
  38. package/package.json +1 -1
  39. package/lib/cli/translate-agent.d.ts +0 -81
  40. package/lib/cli/translate-agent.js +0 -450
@@ -1,450 +0,0 @@
1
- import path from 'node:path';
2
- import { Language } from "../shared/index.js";
3
- import { blockingFindings, } from "./translate-checker.js";
4
- import { MaskIntegrityError, restoreMaskedContent, } from "./translate-mask.js";
5
- import { createScratch, loadPi } from "./translate-scratch.js";
6
- /** How many findings to put in front of the model at once. */
7
- const MAX_REPORTED_FINDINGS = 25;
8
- /** Retries for a refused request. Transient limits are normal at corpus scale. */
9
- const MODEL_MAX_RETRIES = 5;
10
- const MODEL_RETRY_DELAY_MS = 3_000;
11
- const SCRATCH_SOURCE_NAME = 'source';
12
- const SCRATCH_TRANSLATION_NAME = 'translation';
13
- const maskFindingToTranslationFinding = (finding) => ({
14
- rule: `doom-translate:${finding.code}`,
15
- reason: finding.code === 'missing-placeholder'
16
- ? `\`${finding.placeholder}\` is not in the translation. It stands for a ${finding.kind}${finding.detail ? ` (${finding.detail})` : ''}. Either that part has not been translated yet, or it was dropped.`
17
- : finding.code === 'duplicate-placeholder'
18
- ? `\`${finding.placeholder}\` appears ${finding.actual} times but must appear ${finding.expected}.`
19
- : finding.code === 'unregistered-placeholder'
20
- ? `\`${finding.placeholder}\` was never issued — do not invent placeholders.`
21
- : finding.code === 'unrestored-placeholder'
22
- ? `\`${finding.placeholder}\` ended up somewhere it does not belong${finding.detail ? ` — ${finding.detail}` : ''}.`
23
- : (finding.detail ??
24
- `the placeholder round trip failed (${finding.code})`),
25
- });
26
- const renderFindings = (findings) => {
27
- const shown = findings.slice(0, MAX_REPORTED_FINDINGS);
28
- const lines = shown.map((finding) => `- ${finding.rule}${finding.line ? ` (line ${finding.line})` : ''}: ${finding.reason}`);
29
- if (findings.length > shown.length) {
30
- lines.push(`- …and ${findings.length - shown.length} more of the same kind. Fix these first and check again.`);
31
- }
32
- return lines.join('\n');
33
- };
34
- export const translateWithAgent = async (options) => {
35
- const { maskedSource, maskEntries, processor, compose, targetPath, sourceLabel, source, target, translationRules, terms = [], checker, judge, models, model, reasoningEffort, scratchDir, maxRepairRounds, maxTurns, limit, maxModelRetries = MODEL_MAX_RETRIES, modelRetryDelayMs = MODEL_RETRY_DELAY_MS, onProgress, } = options;
36
- const [pi, { Type }] = await Promise.all([
37
- loadPi(),
38
- import('@earendil-works/pi-ai'),
39
- ]);
40
- const { createReadTool, createWriteTool, createEditTool, runAgentLoop, estimateContextTokens, shouldCompact, DEFAULT_COMPACTION_SETTINGS, } = pi.core;
41
- const extension = path.extname(targetPath) || '.mdx';
42
- const scratch = await createScratch({
43
- parentDir: scratchDir,
44
- label: path.basename(targetPath, extension),
45
- maskedSource,
46
- extension,
47
- });
48
- const sourceName = `${SCRATCH_SOURCE_NAME}${extension}`;
49
- const translationName = `${SCRATCH_TRANSLATION_NAME}${extension}`;
50
- try {
51
- const inspectDraft = async () => {
52
- const draft = await scratch.readTranslation();
53
- if (draft.trim() === '') {
54
- return {
55
- findings: [
56
- {
57
- rule: 'doom-translate:nothing-written',
58
- reason: `\`${translationName}\` is still empty. Write the translation there.`,
59
- },
60
- ],
61
- };
62
- }
63
- let restored;
64
- try {
65
- restored = restoreMaskedContent(draft, maskEntries, processor);
66
- }
67
- catch (error) {
68
- if (error instanceof MaskIntegrityError) {
69
- return {
70
- findings: error.findings.map(maskFindingToTranslationFinding),
71
- };
72
- }
73
- throw error;
74
- }
75
- const document = compose(restored);
76
- const findings = await checker.check(targetPath, document);
77
- if (findings.length > 0) {
78
- // The judge is the expensive check and the only one that reads. Asking
79
- // it about a draft the deterministic rules have already faulted spends
80
- // a reading on a document that is going back for repair regardless —
81
- // and asks it to judge the meaning of a half-written page.
82
- return { findings };
83
- }
84
- if (judge) {
85
- findings.push(...(await judge.review({
86
- // The masked pair, not the restored one. Judge findings are fed
87
- // back into the translator's context, and a finding quoting an
88
- // unmasked link target would put that target back within the
89
- // model's reach. Placeholders are stable on both sides, so nothing
90
- // about judging prose is lost.
91
- sourceText: maskedSource,
92
- translationText: draft,
93
- sourceLanguage: Language[source],
94
- targetLanguage: Language[target],
95
- terms,
96
- })));
97
- }
98
- return blockingFindings(findings).length > 0
99
- ? { findings }
100
- : { findings, document };
101
- };
102
- // ---------------------------------------------------------------- tools
103
- const toolContext = { env: scratch.env };
104
- const bind = (tool) => ({
105
- ...tool,
106
- execute: (toolCallId, params, signal, onUpdate) => tool.execute(toolCallId, params, signal, onUpdate, toolContext),
107
- });
108
- const textResult = (text, details) => ({
109
- content: [{ type: 'text', text }],
110
- details,
111
- });
112
- /**
113
- * Appending.
114
- *
115
- * pi's `write` replaces a file, and using `edit` to add to the end means
116
- * quoting the end as an anchor. Neither suits a translation produced a
117
- * section at a time, and `FileSystem` already has `appendFile`.
118
- */
119
- const appendTool = {
120
- label: 'Append',
121
- name: 'append',
122
- description: 'Add text to the end of a file, keeping what is already there. Use this for each section of the translation after the first.',
123
- parameters: Type.Object({
124
- path: Type.String({ description: 'File to append to.' }),
125
- text: Type.String({ description: 'Text to add at the end.' }),
126
- }),
127
- execute: async (_toolCallId, params) => {
128
- const { path: target, text } = params;
129
- const appended = await scratch.env.appendFile(target, text);
130
- if (!appended.ok) {
131
- throw new Error(`Could not append to ${target}: ${appended.error.message}`);
132
- }
133
- const info = await scratch.env.fileInfo(target);
134
- return textResult(`Appended ${text.length} characters.${info.ok ? ` \`${path.basename(target)}\` is now ${info.value.size} bytes.` : ''}`, undefined);
135
- },
136
- };
137
- /**
138
- * Reading by byte range.
139
- *
140
- * pi's `read` pages by line, and when one line is longer than its byte
141
- * limit it tells the model to fall back to `sed` through a shell. There is
142
- * no shell here, on purpose — which would leave that content unreachable,
143
- * and a document unfinishable for a reason no repair round could clear. So
144
- * the escape hatch is a tool instead of dead text; `afterToolCall` below
145
- * rewrites the message that points at it.
146
- */
147
- const readBytesTool = {
148
- label: 'Read bytes',
149
- name: 'read_bytes',
150
- description: 'Read a byte range of a file. Use this when `read` reports a line too long to show.',
151
- parameters: Type.Object({
152
- path: Type.String({ description: 'File to read.' }),
153
- offset: Type.Number({ description: 'First byte to read, from 0.' }),
154
- limit: Type.Number({ description: 'How many bytes to read.' }),
155
- }),
156
- execute: async (_toolCallId, params) => {
157
- const { path: target, offset, limit: byteLimit, } = params;
158
- const read = await scratch.env.readBinaryFile(target);
159
- if (!read.ok) {
160
- throw new Error(`Could not read ${target}: ${read.error.message}`);
161
- }
162
- const slice = Buffer.from(read.value).subarray(Math.max(0, offset), Math.max(0, offset) + Math.max(0, byteLimit));
163
- const total = read.value.byteLength;
164
- const end = Math.min(total, Math.max(0, offset) + Math.max(0, byteLimit));
165
- return textResult(`${slice.toString('utf8')}\n\n[bytes ${offset}–${end} of ${total}]`, undefined);
166
- },
167
- };
168
- let checkCalls = 0;
169
- /**
170
- * The agent's view of its own work.
171
- *
172
- * It returns findings and nothing else — never the restored document. The
173
- * restored text holds the real link targets and identifiers, and handing
174
- * them back would put in the model's context exactly what masking took out
175
- * of it.
176
- */
177
- const checkTool = {
178
- label: 'Check',
179
- name: 'check',
180
- description: 'Compare your translation with the source and list what is wrong. Reports nothing when the translation is complete and correct.',
181
- parameters: Type.Object({}),
182
- execute: async () => {
183
- checkCalls++;
184
- const report = await inspectDraft();
185
- if (report.findings.length === 0) {
186
- return textResult('No problems found. The translation is complete and matches the source.', undefined);
187
- }
188
- return textResult(`${report.findings.length} problem(s):\n${renderFindings(report.findings)}`, undefined);
189
- },
190
- };
191
- /**
192
- * pi's `read`, with its one untrue sentence corrected.
193
- *
194
- * When a line is longer than its byte limit, the tool tells the model to
195
- * fall back to `sed` through a shell. There is no shell here, on purpose,
196
- * so that advice is dead text — and a document containing such a line would
197
- * be unfinishable for a reason no repair round could clear. Rewritten to
198
- * point at `read_bytes` instead, next to the tool it is about rather than
199
- * in a hook somewhere else.
200
- */
201
- const baseReadTool = bind(createReadTool());
202
- const readTool = {
203
- ...baseReadTool,
204
- execute: async (toolCallId, params, signal, onUpdate) => {
205
- const result = await baseReadTool.execute(toolCallId, params, signal, onUpdate);
206
- return {
207
- ...result,
208
- content: result.content.map((part) => part.type === 'text' && part.text.includes('Use bash:')
209
- ? {
210
- ...part,
211
- text: part.text.replace(/Use bash: sed -n '\d+p' (\S+) \| head -c \d+/, 'Use read_bytes on $1 to read it a byte range at a time'),
212
- }
213
- : part),
214
- };
215
- },
216
- };
217
- const tools = [
218
- readTool,
219
- bind(createWriteTool()),
220
- bind(createEditTool()),
221
- appendTool,
222
- readBytesTool,
223
- checkTool,
224
- ];
225
- // -------------------------------------------------------------- prompts
226
- const systemPrompt = [
227
- `You are translating one document, on your own, until it is correct.`,
228
- '',
229
- `## Your working directory`,
230
- '',
231
- `It is everything you can reach. There is no other file, and no shell.`,
232
- '',
233
- `- \`${sourceName}\` — the prepared source document.`,
234
- `- \`${translationName}\` — where your translation goes. It starts empty.`,
235
- '',
236
- `## How to work`,
237
- '',
238
- `1. \`read\` \`${sourceName}\`. Long documents come back a page at a time; keep reading with \`offset\` until you have seen all of it, and translate it section by section rather than holding it all at once.`,
239
- `2. \`write\` the first part of the translation to \`${translationName}\`, then \`append\` each following part. \`write\` replaces the file, so use it once.`,
240
- `3. When the whole document has been translated, call \`check\`. It compares your translation against the source and lists what is wrong.`,
241
- `4. Repair with \`edit\` and \`check\` again. Prefer \`edit\` over rewriting: a targeted change is one fewer chance to damage what was already right.`,
242
- '',
243
- `Anything you released from your context is still on disk — \`read\` it again rather than working from memory.`,
244
- '',
245
- `## When you are finished`,
246
- '',
247
- `You are finished when \`check\` reports nothing. Saying so yourself does not end the task: the same check runs again when you stop, and anything it finds comes back to you as another turn.`,
248
- '',
249
- `## Placeholders`,
250
- '',
251
- `Tokens shaped like \`__DOOM_TR_LINK_3__\` stand for content you must not author — link targets, code, identifiers, component attribute values. Reproduce each one verbatim and exactly once, in the position it appears. Never translate, reformat, split, renumber, remove, duplicate or invent one. The text around a placeholder is translated as usual.`,
252
- '',
253
- `## Translation rules`,
254
- '',
255
- translationRules,
256
- ].join('\n');
257
- const initialPrompt = {
258
- role: 'user',
259
- content: `Translate \`${sourceName}\` and write the translation to \`${translationName}\`. The source is ${maskedSource.length} characters long and contains ${maskEntries.length} placeholder(s), every one of which must appear in your translation.`,
260
- timestamp: Date.now(),
261
- };
262
- // ------------------------------------------------------- context release
263
- /**
264
- * Releasing context, deterministically.
265
- *
266
- * What is dropped is chosen by a rule, not by a model: the bodies of
267
- * `read`/`read_bytes` results that are no longer the recent ones. pi ships
268
- * a compaction that summarises the conversation with an LLM, and that is
269
- * the one thing this cannot use — the context holds source and translation
270
- * text that has to survive byte for byte, and summarising it hands the
271
- * bytes to the process they are being protected from. Only pi's
272
- * *measurement* is borrowed, to decide when to release.
273
- *
274
- * Releasing is safe because nothing is lost: every released page is still
275
- * in the scratch directory, and the agent is told to read it again.
276
- */
277
- const KEEP_RECENT_MESSAGES = 6;
278
- const RELEASABLE_TOOLS = new Set(['read', 'read_bytes']);
279
- const wordingAlreadyUsed = (produced) => terms.filter((term) => produced.includes(term.target));
280
- const transformContext = async (messages) => {
281
- const estimate = estimateContextTokens(messages);
282
- if (!shouldCompact(estimate.tokens, model.contextWindow, DEFAULT_COMPACTION_SETTINGS)) {
283
- return messages;
284
- }
285
- const cutoff = messages.length - KEEP_RECENT_MESSAGES;
286
- let released = 0;
287
- const trimmed = messages.map((message, index) => {
288
- if (index >= cutoff) {
289
- return message;
290
- }
291
- if (!('role' in message) ||
292
- message.role !== 'toolResult' ||
293
- !RELEASABLE_TOOLS.has(message.toolName)) {
294
- return message;
295
- }
296
- released++;
297
- return {
298
- ...message,
299
- content: [
300
- {
301
- type: 'text',
302
- text: `[released from context to make room. Read \`${sourceName}\` again if you need this part.]`,
303
- },
304
- ],
305
- };
306
- });
307
- if (released === 0) {
308
- return messages;
309
- }
310
- // Carried forward as a fact, not a preference: these are the words this
311
- // document has already used, and the released pages are where the agent
312
- // would otherwise have seen them.
313
- const produced = await scratch.readTranslation();
314
- const used = wordingAlreadyUsed(produced);
315
- if (used.length === 0) {
316
- return trimmed;
317
- }
318
- return [
319
- ...trimmed,
320
- {
321
- role: 'user',
322
- content: `For consistency with what you have already written in this document: ${used
323
- .map((term) => `"${term.source}" → "${term.target}"`)
324
- .join(', ')}.`,
325
- timestamp: Date.now(),
326
- },
327
- ];
328
- };
329
- // ------------------------------------------------------------- the loop
330
- let turns = 0;
331
- let repairRounds = 0;
332
- let modelCalls = 0;
333
- const failure = {};
334
- // Cleared through a call, not an assignment: an assignment here narrows the
335
- // property to `undefined` for the rest of the block, and the sink that sets
336
- // it is a callback the compiler cannot follow.
337
- const clearFailure = () => {
338
- failure.message = undefined;
339
- };
340
- const streamFn = (streamModel, context, streamOptions) => {
341
- modelCalls++;
342
- return limit(() => Promise.resolve(models.streamSimple(streamModel, context, streamOptions)));
343
- };
344
- const config = {
345
- model,
346
- reasoning: reasoningEffort,
347
- // The gateway refuses requests when a user is over its per-minute limit,
348
- // and a corpus run is thousands of calls. Measured: a translation that
349
- // met a 429 ended the whole document, because nothing retried it. The
350
- // rate limiter keeps us under the limit in the normal case; this is what
351
- // happens when something else is also using the budget.
352
- maxRetries: MODEL_MAX_RETRIES,
353
- convertToLlm: (messages) => messages,
354
- transformContext,
355
- // The runaway guard. pi checks this *before* it polls for follow-ups, so
356
- // it wins over the exit condition below: a loop that will not converge
357
- // ends here, and the caller finds out it ended failed.
358
- shouldStopAfterTurn: () => ++turns >= maxTurns,
359
- // The exit condition. pi calls this when the agent has run out of tool
360
- // calls and would stop; anything returned goes into the context and the
361
- // agent takes another turn. So the agent finishes exactly when the
362
- // harness's own check finds nothing — it never sees this call and cannot
363
- // route around it.
364
- getFollowUpMessages: async () => {
365
- const report = await inspectDraft();
366
- // Advisory findings — the judge's readability notes — travel with the
367
- // feedback when there is feedback, and never cause a round of their
368
- // own. Meeting the standard is the bar; "cannot be improved" is not a
369
- // bar a loop can reach.
370
- const blocking = blockingFindings(report.findings);
371
- if (blocking.length === 0) {
372
- return [];
373
- }
374
- if (repairRounds >= maxRepairRounds) {
375
- return [];
376
- }
377
- repairRounds++;
378
- onProgress?.(`${sourceLabel}: ${blocking.length} problem(s) after round ${repairRounds}, sending them back`);
379
- return [
380
- {
381
- role: 'user',
382
- content: `The translation is not finished. Checking it against the source reports ${report.findings.length} problem(s):\n\n${renderFindings(report.findings)}\n\nFix them in \`${translationName}\`, then call \`check\` again.`,
383
- timestamp: Date.now(),
384
- },
385
- ];
386
- },
387
- };
388
- const watchForStreamFailure = (event) => {
389
- if (event.type === 'turn_end' &&
390
- 'stopReason' in event.message &&
391
- (event.message.stopReason === 'error' ||
392
- event.message.stopReason === 'aborted')) {
393
- failure.message =
394
- ('errorMessage' in event.message
395
- ? event.message.errorMessage
396
- : undefined) ?? event.message.stopReason;
397
- }
398
- };
399
- /**
400
- * A gateway that refuses is not a translation that failed.
401
- *
402
- * Measured on the real gateway while translating three documents: "429
403
- * user requests-per-minute limit exceeded" and "Upstream service
404
- * temporarily unavailable", both of which pi surfaces as a stream error
405
- * that ends the run. A corpus is thousands of calls; meeting one of those
406
- * is routine, and failing the document over it would fail the wrong thing
407
- * — loudly, and in a way that makes the gate look unreliable rather than
408
- * the gateway.
409
- *
410
- * Retrying is cheap here because the draft is on disk. The agent starts
411
- * with a fresh context and is told to carry on from what it has already
412
- * written, so a refusal costs the turns since the last write rather than
413
- * the whole document.
414
- */
415
- for (let attempt = 0;; attempt++) {
416
- clearFailure();
417
- const prompt = attempt === 0
418
- ? initialPrompt
419
- : {
420
- role: 'user',
421
- content: `Continue translating \`${sourceName}\` into \`${translationName}\`. Some of it may already be written: \`read\` \`${translationName}\` first, then call \`check\` to see what is still missing, and carry on from there.`,
422
- timestamp: Date.now(),
423
- };
424
- await runAgentLoop([prompt], { systemPrompt, messages: [], tools }, config, watchForStreamFailure, undefined, streamFn);
425
- if (!failure.message) {
426
- break;
427
- }
428
- if (attempt >= maxModelRetries || turns >= maxTurns) {
429
- throw new Error(`The translation model failed while translating ${sourceLabel}: ${failure.message}`);
430
- }
431
- onProgress?.(`${sourceLabel}: the gateway refused (${failure.message}) — retrying from what is already written`);
432
- await new Promise((resolve) => globalThis.setTimeout(resolve, modelRetryDelayMs * 2 ** attempt));
433
- }
434
- // The loop can end two ways — the check passed, or the turn cap fired
435
- // before it did. Only one of them means the document is good, so it is
436
- // decided here rather than inferred from how the loop exited.
437
- const final = await inspectDraft();
438
- void checkCalls;
439
- return {
440
- document: final.document,
441
- findings: final.findings,
442
- turns,
443
- repairRounds,
444
- modelCalls,
445
- };
446
- }
447
- finally {
448
- await scratch.dispose();
449
- }
450
- };