@alauda/doom 2.6.1 → 2.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/cli/translate-call.d.ts +85 -0
- package/lib/cli/translate-call.js +149 -0
- package/lib/cli/translate-checker.d.ts +8 -0
- package/lib/cli/translate-diagnose.d.ts +45 -0
- package/lib/cli/translate-diagnose.js +118 -0
- package/lib/cli/translate-judge.d.ts +28 -3
- package/lib/cli/translate-judge.js +40 -7
- package/lib/cli/translate-mask.d.ts +13 -0
- package/lib/cli/translate-mask.js +19 -5
- package/lib/cli/translate-pipeline.d.ts +144 -0
- package/lib/cli/translate-pipeline.js +489 -0
- package/lib/cli/translate-repair-agent.d.ts +69 -0
- package/lib/cli/translate-repair-agent.js +241 -0
- package/lib/cli/translate-scratch.d.ts +9 -1
- package/lib/cli/translate-scratch.js +2 -2
- package/lib/cli/translate-segment-cache.d.ts +90 -0
- package/lib/cli/translate-segment-cache.js +301 -0
- package/lib/cli/translate-segment-check.d.ts +42 -0
- package/lib/cli/translate-segment-check.js +247 -0
- package/lib/cli/translate-segment.d.ts +307 -0
- package/lib/cli/translate-segment.js +625 -0
- package/lib/cli/translate.d.ts +14 -4
- package/lib/cli/translate.js +232 -52
- package/lib/remark-lint/no-unopened-frontmatter.js +5 -1
- package/lib/remark-lint/translation-parity/frontmatter-preservation.js +5 -3
- package/lib/runtime/hooks/useTranslation.d.ts +1 -1
- package/lib/types.d.ts +101 -6
- package/package.json +1 -1
- package/lib/cli/translate-agent.d.ts +0 -81
- package/lib/cli/translate-agent.js +0 -450
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
import type { Model, Models, ThinkingLevel } from '@earendil-works/pi-ai';
|
|
2
|
+
import type { TranslationFinding } from './translate-checker.ts';
|
|
3
|
+
import type { MaskProcessor } from './translate-mask.ts';
|
|
4
|
+
import type { Segment } from './translate-segment.ts';
|
|
5
|
+
/**
|
|
6
|
+
* Translating one segment: a single ordinary model call.
|
|
7
|
+
*
|
|
8
|
+
* Not an agent. There is no loop here, no tool, no scratch directory and no
|
|
9
|
+
* bookkeeping — a segment is small enough to hand over, and what comes back is
|
|
10
|
+
* either accepted or asked for again. Autonomy is kept for the place it
|
|
11
|
+
* actually wins (a hard repair, in `translate-repair-agent.ts`); on the path
|
|
12
|
+
* that runs thousands of times it only adds variance.
|
|
13
|
+
*
|
|
14
|
+
* The context a segment needs is small and fixed: the page's headings so the
|
|
15
|
+
* model knows where it is, the tail of the previous segment so the voice
|
|
16
|
+
* carries, and the segment itself. That is a few hundred tokens on top of the
|
|
17
|
+
* segment — which is the whole point. The old design put every byte of a
|
|
18
|
+
* document and its translation through one context, and a model working in a
|
|
19
|
+
* context it has filled and had released is a model that reaches for a rewrite.
|
|
20
|
+
*/
|
|
21
|
+
/** Retries for a refused request. Transient limits are normal at corpus scale. */
|
|
22
|
+
export declare const DEFAULT_CALL_MAX_RETRIES = 5;
|
|
23
|
+
export declare const DEFAULT_CALL_RETRY_DELAY_MS = 3000;
|
|
24
|
+
/** How many lines of the previous segment's translation carry the voice forward. */
|
|
25
|
+
export declare const DEFAULT_CONTEXT_TAIL = 20;
|
|
26
|
+
export interface SegmentTranslationRequest {
|
|
27
|
+
segment: Segment;
|
|
28
|
+
/** Every heading on the page, so the model knows where this segment sits. */
|
|
29
|
+
outline?: string;
|
|
30
|
+
/** The end of the previous segment's accepted translation. */
|
|
31
|
+
previousTail?: string;
|
|
32
|
+
/** 1 for the first go; higher when the segment is being asked for again. */
|
|
33
|
+
attempt: number;
|
|
34
|
+
/** The answer that was rejected, and why. Absent on the first attempt. */
|
|
35
|
+
retry?: {
|
|
36
|
+
previous: string;
|
|
37
|
+
findings: readonly TranslationFinding[];
|
|
38
|
+
};
|
|
39
|
+
}
|
|
40
|
+
export interface SegmentTranslator {
|
|
41
|
+
translate(request: SegmentTranslationRequest): Promise<string>;
|
|
42
|
+
/** Model calls made so far, for the run summary. */
|
|
43
|
+
calls(): number;
|
|
44
|
+
}
|
|
45
|
+
export interface CreateSegmentTranslatorOptions {
|
|
46
|
+
models: Models;
|
|
47
|
+
model: Model<'openai-completions'>;
|
|
48
|
+
reasoningEffort: ThinkingLevel;
|
|
49
|
+
/** The rendered translation rules — the same prompt body the whole document used. */
|
|
50
|
+
translationRules: string;
|
|
51
|
+
sourceLanguage: string;
|
|
52
|
+
targetLanguage: string;
|
|
53
|
+
processor: MaskProcessor;
|
|
54
|
+
/** Applied to every model call, so segments count against the same budget as everything else. */
|
|
55
|
+
limit: <T>(run: () => Promise<T>) => Promise<T>;
|
|
56
|
+
maxRetries?: number;
|
|
57
|
+
retryDelayMs?: number;
|
|
58
|
+
}
|
|
59
|
+
export declare const createSegmentTranslator: ({ models, model, reasoningEffort, translationRules, sourceLanguage, targetLanguage, processor, limit, maxRetries, retryDelayMs, }: CreateSegmentTranslatorOptions) => SegmentTranslator;
|
|
60
|
+
export declare const buildUserPrompt: ({ segment, outline, previousTail, attempt, retry, }: SegmentTranslationRequest) => string;
|
|
61
|
+
/**
|
|
62
|
+
* Takes off a code fence the model wrapped the whole answer in.
|
|
63
|
+
*
|
|
64
|
+
* Asked not to, models still sometimes do. Stripping by pattern would be
|
|
65
|
+
* wrong — a segment can legitimately consist of a fenced block — so the test is
|
|
66
|
+
* structural: the answer parses to exactly one fenced block while the source
|
|
67
|
+
* segment does not. Then, and only then, what is inside the fence is the answer.
|
|
68
|
+
*/
|
|
69
|
+
export declare const unwrap: (reply: string, segment: Segment, processor: MaskProcessor) => string;
|
|
70
|
+
/**
|
|
71
|
+
* Every heading on the page, indented by level, with the current segment marked.
|
|
72
|
+
*
|
|
73
|
+
* Marked by the heading's **line**, not by its text. A real page repeats
|
|
74
|
+
* `Procedure`, `Prerequisites` and `Verification` under every top-level
|
|
75
|
+
* heading, and matching on text put the mark on the first one of that name — so
|
|
76
|
+
* a segment in the CLI section was told it was in the console section, which is
|
|
77
|
+
* worse than not marking anything.
|
|
78
|
+
*/
|
|
79
|
+
export declare const renderOutline: (headings: readonly {
|
|
80
|
+
depth: number;
|
|
81
|
+
text: string;
|
|
82
|
+
line?: number;
|
|
83
|
+
}[], currentLine?: number) => string;
|
|
84
|
+
/** The last few lines of a translation, for the next segment to continue from. */
|
|
85
|
+
export declare const tailOf: (translation: string, lines?: number) => string | undefined;
|
|
@@ -0,0 +1,149 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Translating one segment: a single ordinary model call.
|
|
3
|
+
*
|
|
4
|
+
* Not an agent. There is no loop here, no tool, no scratch directory and no
|
|
5
|
+
* bookkeeping — a segment is small enough to hand over, and what comes back is
|
|
6
|
+
* either accepted or asked for again. Autonomy is kept for the place it
|
|
7
|
+
* actually wins (a hard repair, in `translate-repair-agent.ts`); on the path
|
|
8
|
+
* that runs thousands of times it only adds variance.
|
|
9
|
+
*
|
|
10
|
+
* The context a segment needs is small and fixed: the page's headings so the
|
|
11
|
+
* model knows where it is, the tail of the previous segment so the voice
|
|
12
|
+
* carries, and the segment itself. That is a few hundred tokens on top of the
|
|
13
|
+
* segment — which is the whole point. The old design put every byte of a
|
|
14
|
+
* document and its translation through one context, and a model working in a
|
|
15
|
+
* context it has filled and had released is a model that reaches for a rewrite.
|
|
16
|
+
*/
|
|
17
|
+
/** Retries for a refused request. Transient limits are normal at corpus scale. */
|
|
18
|
+
export const DEFAULT_CALL_MAX_RETRIES = 5;
|
|
19
|
+
export const DEFAULT_CALL_RETRY_DELAY_MS = 3_000;
|
|
20
|
+
/** How many lines of the previous segment's translation carry the voice forward. */
|
|
21
|
+
export const DEFAULT_CONTEXT_TAIL = 20;
|
|
22
|
+
const PLACEHOLDER_DISCIPLINE = `## Placeholders
|
|
23
|
+
|
|
24
|
+
Tokens shaped like \`__DOOM_TR_LINK_3__\` stand for content you must not author — link targets, code, identifiers, component attribute values. Reproduce each one verbatim and exactly once, in the position it appears. Never translate, reformat, split, renumber, remove, duplicate or invent one. The text around a placeholder is translated as usual.`;
|
|
25
|
+
const SEGMENT_TASK = `Answer with the translated segment and nothing else: no explanation, no preamble, no summary, and no code fence wrapped around your whole answer.
|
|
26
|
+
|
|
27
|
+
The segment is one piece of a longer page. It begins and ends mid-document, and that is correct — do not add a title, an introduction or a conclusion, and do not finish a section that carries on past the end of what you were given.
|
|
28
|
+
|
|
29
|
+
Keep its structure exactly as it is: the same headings at the same levels, the same lists, tables and components, in the same order. Translate the words.`;
|
|
30
|
+
const ATTRIBUTE_TASK = `You are given one component tag, carrying the words it shows to a reader.
|
|
31
|
+
|
|
32
|
+
Answer with that same element and nothing else — the same component name and the same attributes, in the same order — translating only the attribute values. Do not translate attribute names, do not add or remove attributes, and do not give it any children.`;
|
|
33
|
+
export const createSegmentTranslator = ({ models, model, reasoningEffort, translationRules, sourceLanguage, targetLanguage, processor, limit, maxRetries = DEFAULT_CALL_MAX_RETRIES, retryDelayMs = DEFAULT_CALL_RETRY_DELAY_MS, }) => {
|
|
34
|
+
let calls = 0;
|
|
35
|
+
const systemPrompt = (kind) => [
|
|
36
|
+
`You are a professional technical documentation translator, translating from ${sourceLanguage} to ${targetLanguage}.`,
|
|
37
|
+
'',
|
|
38
|
+
kind === 'attributes' ? ATTRIBUTE_TASK : SEGMENT_TASK,
|
|
39
|
+
'',
|
|
40
|
+
PLACEHOLDER_DISCIPLINE,
|
|
41
|
+
'',
|
|
42
|
+
'## Translation rules',
|
|
43
|
+
'',
|
|
44
|
+
translationRules,
|
|
45
|
+
].join('\n');
|
|
46
|
+
return {
|
|
47
|
+
calls: () => calls,
|
|
48
|
+
async translate(request) {
|
|
49
|
+
const { contentText } = await import('@earendil-works/pi-ai');
|
|
50
|
+
const context = {
|
|
51
|
+
systemPrompt: systemPrompt(request.segment.address.kind),
|
|
52
|
+
messages: [
|
|
53
|
+
{
|
|
54
|
+
role: 'user',
|
|
55
|
+
content: buildUserPrompt(request),
|
|
56
|
+
timestamp: Date.now(),
|
|
57
|
+
},
|
|
58
|
+
],
|
|
59
|
+
};
|
|
60
|
+
let lastError = 'unknown';
|
|
61
|
+
for (let attempt = 0; attempt <= maxRetries; attempt++) {
|
|
62
|
+
calls++;
|
|
63
|
+
const reply = await limit(() => models.completeSimple(model, context, {
|
|
64
|
+
reasoning: reasoningEffort,
|
|
65
|
+
maxRetries,
|
|
66
|
+
}));
|
|
67
|
+
if (reply.stopReason !== 'error' && reply.stopReason !== 'aborted') {
|
|
68
|
+
return unwrap(contentText(reply.content), request.segment, processor);
|
|
69
|
+
}
|
|
70
|
+
lastError = reply.errorMessage ?? reply.stopReason;
|
|
71
|
+
// A gateway that refuses is not a translation that failed, so this does
|
|
72
|
+
// not spend one of the segment's attempts. Running out still throws.
|
|
73
|
+
if (attempt < maxRetries) {
|
|
74
|
+
await new Promise((resolve) => globalThis.setTimeout(resolve, retryDelayMs * 2 ** attempt));
|
|
75
|
+
}
|
|
76
|
+
}
|
|
77
|
+
throw new Error(`The translation model failed: ${lastError}`);
|
|
78
|
+
},
|
|
79
|
+
};
|
|
80
|
+
};
|
|
81
|
+
export const buildUserPrompt = ({ segment, outline, previousTail, attempt, retry, }) => {
|
|
82
|
+
const lines = [];
|
|
83
|
+
if (outline && segment.address.kind !== 'attributes') {
|
|
84
|
+
lines.push('For context, these are the headings of the whole page. Do not translate this list; it is only so you know where the segment sits.', '', outline, '');
|
|
85
|
+
}
|
|
86
|
+
// The tail is dropped the moment a segment has to be asked for again. It is
|
|
87
|
+
// there for voice, and it is also the one thing in the prompt that contains
|
|
88
|
+
// another segment's placeholders — which is exactly what a model copies from
|
|
89
|
+
// when it is trying to satisfy a complaint it has not understood.
|
|
90
|
+
if (previousTail && attempt === 1 && segment.address.kind !== 'attributes') {
|
|
91
|
+
lines.push('The previous segment of this page ended like this. Continue in the same voice and use the same wording for the same things; do not repeat any of it.', '', '<<<PREVIOUS', previousTail, 'PREVIOUS', '');
|
|
92
|
+
}
|
|
93
|
+
if (retry) {
|
|
94
|
+
lines.push('You already translated this segment, and the result was rejected:', '', '<<<REJECTED', retry.previous, 'REJECTED', '', `What is wrong with it (${retry.findings.length}):`, ...retry.findings.map((finding) => `- ${finding.rule}${finding.line ? ` (line ${finding.line})` : ''}: ${finding.reason}`), '', 'Answer with the corrected translation of the whole segment — not a diff, not just the part that was wrong.', '');
|
|
95
|
+
}
|
|
96
|
+
lines.push(retry ? 'The segment, again:' : 'Translate this segment:', '', '<<<SEGMENT', segment.text, 'SEGMENT');
|
|
97
|
+
return lines.join('\n');
|
|
98
|
+
};
|
|
99
|
+
/**
|
|
100
|
+
* Takes off a code fence the model wrapped the whole answer in.
|
|
101
|
+
*
|
|
102
|
+
* Asked not to, models still sometimes do. Stripping by pattern would be
|
|
103
|
+
* wrong — a segment can legitimately consist of a fenced block — so the test is
|
|
104
|
+
* structural: the answer parses to exactly one fenced block while the source
|
|
105
|
+
* segment does not. Then, and only then, what is inside the fence is the answer.
|
|
106
|
+
*/
|
|
107
|
+
export const unwrap = (reply, segment, processor) => {
|
|
108
|
+
const trimmed = reply.trim();
|
|
109
|
+
if (!trimmed.startsWith('```')) {
|
|
110
|
+
return trimmed;
|
|
111
|
+
}
|
|
112
|
+
const onlyFence = (content) => {
|
|
113
|
+
try {
|
|
114
|
+
const { children } = processor.parse(content);
|
|
115
|
+
return children.length === 1 && children[0].type === 'code'
|
|
116
|
+
? children[0]
|
|
117
|
+
: undefined;
|
|
118
|
+
}
|
|
119
|
+
catch {
|
|
120
|
+
return undefined;
|
|
121
|
+
}
|
|
122
|
+
};
|
|
123
|
+
const wrapper = onlyFence(trimmed);
|
|
124
|
+
return wrapper && !onlyFence(segment.text) ? wrapper.value : trimmed;
|
|
125
|
+
};
|
|
126
|
+
/**
|
|
127
|
+
* Every heading on the page, indented by level, with the current segment marked.
|
|
128
|
+
*
|
|
129
|
+
* Marked by the heading's **line**, not by its text. A real page repeats
|
|
130
|
+
* `Procedure`, `Prerequisites` and `Verification` under every top-level
|
|
131
|
+
* heading, and matching on text put the mark on the first one of that name — so
|
|
132
|
+
* a segment in the CLI section was told it was in the console section, which is
|
|
133
|
+
* worse than not marking anything.
|
|
134
|
+
*/
|
|
135
|
+
export const renderOutline = (headings, currentLine) => headings
|
|
136
|
+
.map(({ depth, text, line }) => {
|
|
137
|
+
const mark = currentLine != null && line === currentLine ? ' ← you are here' : '';
|
|
138
|
+
return `${' '.repeat(Math.max(0, depth - 1))}- ${text}${mark}`;
|
|
139
|
+
})
|
|
140
|
+
.join('\n');
|
|
141
|
+
/** The last few lines of a translation, for the next segment to continue from. */
|
|
142
|
+
export const tailOf = (translation, lines = DEFAULT_CONTEXT_TAIL) => {
|
|
143
|
+
if (lines <= 0) {
|
|
144
|
+
return undefined;
|
|
145
|
+
}
|
|
146
|
+
const all = translation.trimEnd().split('\n');
|
|
147
|
+
const tail = all.slice(Math.max(0, all.length - lines)).join('\n');
|
|
148
|
+
return tail.trim() === '' ? undefined : tail;
|
|
149
|
+
};
|
|
@@ -16,6 +16,14 @@ export interface TranslationFinding {
|
|
|
16
16
|
reason: string;
|
|
17
17
|
line?: number;
|
|
18
18
|
column?: number;
|
|
19
|
+
/**
|
|
20
|
+
* The placeholder this finding is about, when it is about one.
|
|
21
|
+
*
|
|
22
|
+
* Carried so a whole-document mask failure can be routed back to the one
|
|
23
|
+
* segment that owns the token, rather than to the document that contains it.
|
|
24
|
+
* Rules never set it; only the mask assertions do.
|
|
25
|
+
*/
|
|
26
|
+
placeholder?: string;
|
|
19
27
|
/**
|
|
20
28
|
* Whether this finding stops the document from shipping.
|
|
21
29
|
*
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
import type { Model, Models, ThinkingLevel } from '@earendil-works/pi-ai';
|
|
2
|
+
import type { TranslationFinding } from './translate-checker.ts';
|
|
3
|
+
import type { DocumentFailure, SegmentOutcome } from './translate-pipeline.ts';
|
|
4
|
+
/**
|
|
5
|
+
* Explaining a failure to the person who has to do something about it.
|
|
6
|
+
*
|
|
7
|
+
* Around forty repositories run `doom translate` unattended, and the only
|
|
8
|
+
* channel any of them has to a human is a red pipeline. Working out what a red
|
|
9
|
+
* one means used to take a specialist an afternoon — the incident behind this
|
|
10
|
+
* design needed two rounds of log-gathering before anyone could say which of
|
|
11
|
+
* two unrelated causes it was.
|
|
12
|
+
*
|
|
13
|
+
* So when a document fails, the evidence that was collected on the way is
|
|
14
|
+
* handed to a model with the reasoning turned up, and what comes back is a
|
|
15
|
+
* paragraph in the build log saying what went wrong and what to try. One call,
|
|
16
|
+
* not an agent: every fact is already in hand, and there is nothing to go and
|
|
17
|
+
* look up.
|
|
18
|
+
*
|
|
19
|
+
* **It is advisory and it is inert.** It does not change the exit code, it is
|
|
20
|
+
* not a finding, nothing branches on it, and when it fails to produce anything
|
|
21
|
+
* the build says exactly that and carries on failing for the reason it already
|
|
22
|
+
* had. A diagnosis that could change a verdict would be a gate made of prose.
|
|
23
|
+
*/
|
|
24
|
+
export declare const DEFAULT_DIAGNOSE_REASONING_EFFORT: ThinkingLevel;
|
|
25
|
+
export interface DiagnoseRequest {
|
|
26
|
+
/** The document, as a person would name it. */
|
|
27
|
+
sourceLabel: string;
|
|
28
|
+
failure?: DocumentFailure;
|
|
29
|
+
outcomes: readonly SegmentOutcome[];
|
|
30
|
+
/** What was still wrong when the run ended. */
|
|
31
|
+
findings: readonly TranslationFinding[];
|
|
32
|
+
assemblyRounds: number;
|
|
33
|
+
}
|
|
34
|
+
export interface Diagnoser {
|
|
35
|
+
/** A paragraph for the build log, or nothing when it could not be produced. */
|
|
36
|
+
diagnose(request: DiagnoseRequest): Promise<string | undefined>;
|
|
37
|
+
}
|
|
38
|
+
export interface CreateDiagnoserOptions {
|
|
39
|
+
models: Models;
|
|
40
|
+
model: Model<'openai-completions'>;
|
|
41
|
+
reasoningEffort?: ThinkingLevel;
|
|
42
|
+
limit: <T>(run: () => Promise<T>) => Promise<T>;
|
|
43
|
+
}
|
|
44
|
+
export declare const buildEvidence: ({ sourceLabel, failure, outcomes, findings, assemblyRounds, }: DiagnoseRequest) => string;
|
|
45
|
+
export declare const createDiagnoser: ({ models, model, reasoningEffort, limit, }: CreateDiagnoserOptions) => Diagnoser;
|
|
@@ -0,0 +1,118 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Explaining a failure to the person who has to do something about it.
|
|
3
|
+
*
|
|
4
|
+
* Around forty repositories run `doom translate` unattended, and the only
|
|
5
|
+
* channel any of them has to a human is a red pipeline. Working out what a red
|
|
6
|
+
* one means used to take a specialist an afternoon — the incident behind this
|
|
7
|
+
* design needed two rounds of log-gathering before anyone could say which of
|
|
8
|
+
* two unrelated causes it was.
|
|
9
|
+
*
|
|
10
|
+
* So when a document fails, the evidence that was collected on the way is
|
|
11
|
+
* handed to a model with the reasoning turned up, and what comes back is a
|
|
12
|
+
* paragraph in the build log saying what went wrong and what to try. One call,
|
|
13
|
+
* not an agent: every fact is already in hand, and there is nothing to go and
|
|
14
|
+
* look up.
|
|
15
|
+
*
|
|
16
|
+
* **It is advisory and it is inert.** It does not change the exit code, it is
|
|
17
|
+
* not a finding, nothing branches on it, and when it fails to produce anything
|
|
18
|
+
* the build says exactly that and carries on failing for the reason it already
|
|
19
|
+
* had. A diagnosis that could change a verdict would be a gate made of prose.
|
|
20
|
+
*/
|
|
21
|
+
export const DEFAULT_DIAGNOSE_REASONING_EFFORT = 'high';
|
|
22
|
+
/** How many rejected attempts of one segment to show. The pattern is in the first few. */
|
|
23
|
+
const MAX_ATTEMPTS_SHOWN = 4;
|
|
24
|
+
/** How many segments to describe in full. */
|
|
25
|
+
const MAX_SEGMENTS_SHOWN = 12;
|
|
26
|
+
const SYSTEM_PROMPT = `You are explaining, to a documentation engineer reading a failed build log, why one document could not be translated.
|
|
27
|
+
|
|
28
|
+
You are given the evidence the pipeline collected: how the document was cut into segments, what happened to each one, and what each rejected attempt was rejected for.
|
|
29
|
+
|
|
30
|
+
How the pipeline works, so you can read the evidence:
|
|
31
|
+
- a document is cut into segments by code, and each segment is translated by a single model call;
|
|
32
|
+
- a segment is accepted only when it parses, reproduces every placeholder exactly, keeps its source's headings and components, and passes a semantic review;
|
|
33
|
+
- a segment that is accepted is frozen and never touched again;
|
|
34
|
+
- a segment that is rejected is asked for again, with the findings attached; after several attempts it goes to a repair agent that can only edit, not rewrite;
|
|
35
|
+
- once every segment is accepted they are assembled and the whole page is checked again.
|
|
36
|
+
- tokens shaped like \`__DOOM_TR_ICODE_3__\` are placeholders standing in for content kept out of translation — link targets, code, identifiers.
|
|
37
|
+
|
|
38
|
+
Write at most 200 words, as plain prose, covering:
|
|
39
|
+
1. which segment or segments failed and what the failure actually was;
|
|
40
|
+
2. whether the findings got better, worse, or stayed the same across attempts — and what that suggests;
|
|
41
|
+
3. the most useful next step for a person: something about the source document, something about the configuration, or "this looks like a model or gateway problem, retry".
|
|
42
|
+
|
|
43
|
+
Be concrete and be honest about uncertainty. Do not invent evidence, do not repeat the findings verbatim as a list, and do not pad. If the evidence does not support a conclusion, say which further evidence would settle it.`;
|
|
44
|
+
const describeFinding = (finding) => ` - ${finding.rule}${finding.line ? ` (line ${finding.line})` : ''}: ${finding.reason}`;
|
|
45
|
+
export const buildEvidence = ({ sourceLabel, failure, outcomes, findings, assemblyRounds, }) => {
|
|
46
|
+
const lines = [
|
|
47
|
+
`Document: ${sourceLabel}`,
|
|
48
|
+
`Cut into ${outcomes.length} segment(s). Assembly rounds used: ${assemblyRounds}.`,
|
|
49
|
+
`Failure: ${failure?.kind === 'unsplittable-block'
|
|
50
|
+
? `a single block could not be divided — ${failure.detail}`
|
|
51
|
+
: failure?.kind === 'segment'
|
|
52
|
+
? `segment(s) ${failure.segments.map((index) => index + 1).join(', ')} were never accepted`
|
|
53
|
+
: failure?.kind === 'assembly'
|
|
54
|
+
? 'the assembled page kept failing a check of the whole document'
|
|
55
|
+
: failure?.kind === 'unlocatable'
|
|
56
|
+
? 'the assembled page failed a check of the whole document that could not be attributed to any segment'
|
|
57
|
+
: 'unknown'}`,
|
|
58
|
+
'',
|
|
59
|
+
];
|
|
60
|
+
const interesting = outcomes.filter((outcome) => outcome.history.length > 0 || outcome.status === 'failed');
|
|
61
|
+
const shown = (interesting.length > 0 ? interesting : outcomes).slice(0, MAX_SEGMENTS_SHOWN);
|
|
62
|
+
const summary = new Map();
|
|
63
|
+
for (const outcome of outcomes) {
|
|
64
|
+
summary.set(outcome.status, (summary.get(outcome.status) ?? 0) + 1);
|
|
65
|
+
}
|
|
66
|
+
lines.push(`Segment outcomes: ${[...summary]
|
|
67
|
+
.map(([status, count]) => `${count} ${status}`)
|
|
68
|
+
.join(', ')}`, '');
|
|
69
|
+
for (const outcome of shown) {
|
|
70
|
+
lines.push(`Segment ${outcome.index + 1}${outcome.label.heading ? ` — under “${outcome.label.heading}”` : ''}${outcome.label.line ? `, source line ${outcome.label.line}` : ''}: ${outcome.status}, ${outcome.attempts} attempt(s)`);
|
|
71
|
+
for (const [round, attemptFindings] of outcome.history
|
|
72
|
+
.slice(0, MAX_ATTEMPTS_SHOWN)
|
|
73
|
+
.entries()) {
|
|
74
|
+
lines.push(` attempt ${round + 1} rejected with ${attemptFindings.length} finding(s):`, ...attemptFindings.slice(0, 8).map(describeFinding), ...(attemptFindings.length > 8
|
|
75
|
+
? [` - …and ${attemptFindings.length - 8} more of the same kinds`]
|
|
76
|
+
: []));
|
|
77
|
+
}
|
|
78
|
+
if (outcome.history.length > MAX_ATTEMPTS_SHOWN) {
|
|
79
|
+
lines.push(` …and ${outcome.history.length - MAX_ATTEMPTS_SHOWN} further rejected attempt(s)`);
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
if (interesting.length > shown.length) {
|
|
83
|
+
lines.push('', `…and ${interesting.length - shown.length} further segment(s).`);
|
|
84
|
+
}
|
|
85
|
+
const blocking = findings.filter((finding) => finding.blocking !== false);
|
|
86
|
+
if (blocking.length > 0) {
|
|
87
|
+
lines.push('', 'What was still wrong when the run ended:', ...blocking.slice(0, 12).map(describeFinding));
|
|
88
|
+
}
|
|
89
|
+
return lines.join('\n');
|
|
90
|
+
};
|
|
91
|
+
export const createDiagnoser = ({ models, model, reasoningEffort = DEFAULT_DIAGNOSE_REASONING_EFFORT, limit, }) => ({
|
|
92
|
+
async diagnose(request) {
|
|
93
|
+
try {
|
|
94
|
+
const { contentText } = await import('@earendil-works/pi-ai');
|
|
95
|
+
const reply = await limit(() => models.completeSimple(model, {
|
|
96
|
+
systemPrompt: SYSTEM_PROMPT,
|
|
97
|
+
messages: [
|
|
98
|
+
{
|
|
99
|
+
role: 'user',
|
|
100
|
+
content: buildEvidence(request),
|
|
101
|
+
timestamp: Date.now(),
|
|
102
|
+
},
|
|
103
|
+
],
|
|
104
|
+
}, { reasoning: reasoningEffort, maxRetries: 1 }));
|
|
105
|
+
if (reply.stopReason === 'error' || reply.stopReason === 'aborted') {
|
|
106
|
+
return undefined;
|
|
107
|
+
}
|
|
108
|
+
const text = contentText(reply.content).trim();
|
|
109
|
+
return text === '' ? undefined : text;
|
|
110
|
+
}
|
|
111
|
+
catch {
|
|
112
|
+
// Never allowed to affect the run. The document has already failed for a
|
|
113
|
+
// reason that is on record; failing to explain it changes nothing about
|
|
114
|
+
// that, and must not turn one failure into two.
|
|
115
|
+
return undefined;
|
|
116
|
+
}
|
|
117
|
+
},
|
|
118
|
+
});
|
|
@@ -67,6 +67,15 @@ export interface JudgeRequest {
|
|
|
67
67
|
source: string;
|
|
68
68
|
target: string;
|
|
69
69
|
}[];
|
|
70
|
+
/**
|
|
71
|
+
* Whether this is one segment of a longer document rather than a whole page.
|
|
72
|
+
*
|
|
73
|
+
* A fragment begins and ends mid-document on purpose. Without being told, a
|
|
74
|
+
* reviewer reads the missing introduction as an omission and the section that
|
|
75
|
+
* continues past the end as truncation — reporting, every time, the one thing
|
|
76
|
+
* that is certainly not wrong.
|
|
77
|
+
*/
|
|
78
|
+
fragment?: boolean;
|
|
70
79
|
}
|
|
71
80
|
export interface Judge {
|
|
72
81
|
review(request: JudgeRequest): Promise<TranslationFinding[]>;
|
|
@@ -85,6 +94,15 @@ export interface CreateJudgeOptions {
|
|
|
85
94
|
* answers — so it needs no parameter to create it.
|
|
86
95
|
*/
|
|
87
96
|
draws?: number;
|
|
97
|
+
/**
|
|
98
|
+
* How many of those readings must agree before a finding counts.
|
|
99
|
+
*
|
|
100
|
+
* Defaults to all of them, which is what two draws have always meant.
|
|
101
|
+
* Segments take three draws and require two: a short text gives a reviewer
|
|
102
|
+
* less to go on and more room to find something to say, so the extra reading
|
|
103
|
+
* buys back the precision that unanimity gives on a whole page.
|
|
104
|
+
*/
|
|
105
|
+
votes?: number;
|
|
88
106
|
/** Applied to every model call, so judge readings count against the same budget as translation. */
|
|
89
107
|
limit: <T>(run: () => Promise<T>) => Promise<T>;
|
|
90
108
|
/** How many times a failed reading is retried before the document fails. */
|
|
@@ -113,7 +131,14 @@ export declare const DEFAULT_JUDGE_DRAWS = 2;
|
|
|
113
131
|
*/
|
|
114
132
|
export declare const DEFAULT_JUDGE_MAX_RETRIES = 4;
|
|
115
133
|
export declare const DEFAULT_JUDGE_RETRY_DELAY_MS = 2000;
|
|
116
|
-
/**
|
|
117
|
-
|
|
134
|
+
/**
|
|
135
|
+
* The findings enough readings agreed on.
|
|
136
|
+
*
|
|
137
|
+
* `votes` defaults to every reading, which is what this has always done: with
|
|
138
|
+
* two draws, unanimity. Asking for two of three is the same idea with one more
|
|
139
|
+
* sample — a false positive still has to happen twice to reach anyone, and a
|
|
140
|
+
* real defect no longer has to be seen by every reader to count.
|
|
141
|
+
*/
|
|
142
|
+
export declare const agreedFindings: (draws: readonly JudgeFinding[][], votes?: number) => JudgeFinding[];
|
|
118
143
|
export declare const parseJudgeFindings: (reply: string) => JudgeFinding[];
|
|
119
|
-
export declare const createJudge: ({ models, model, reasoningEffort, draws, limit, maxRetries, retryDelayMs, onDraw, }: CreateJudgeOptions) => Judge;
|
|
144
|
+
export declare const createJudge: ({ models, model, reasoningEffort, draws, votes, limit, maxRetries, retryDelayMs, onDraw, }: CreateJudgeOptions) => Judge;
|
|
@@ -52,6 +52,18 @@ Answer with a JSON array and nothing else. Each element:
|
|
|
52
52
|
{"kind": "omission" | "addition" | "mistranslation" | "fluency", "source": "<the exact passage from the source>", "detail": "<what is wrong, in one sentence>"}
|
|
53
53
|
|
|
54
54
|
An empty array means the translation is faithful. That is the expected answer for a good translation — do not look for something to say.`;
|
|
55
|
+
/**
|
|
56
|
+
* What changes when the pair is one piece of a page.
|
|
57
|
+
*
|
|
58
|
+
* Only the coverage pass needs adjusting: a fragment has no beginning and no
|
|
59
|
+
* end, so "the introduction is missing" and "it stops mid-section" are
|
|
60
|
+
* properties of the cut, not of the translation. Everything else — what counts
|
|
61
|
+
* as a mistranslation, what placeholders mean — is unchanged.
|
|
62
|
+
*/
|
|
63
|
+
const FRAGMENT_PROMPT = `
|
|
64
|
+
IMPORTANT: what you are given is one FRAGMENT of a longer document, not a whole page. It starts and stops in the middle on purpose.
|
|
65
|
+
|
|
66
|
+
So: a missing introduction, a missing conclusion, a heading whose section clearly continues past the end, or a sentence that refers to something you cannot see are NOT omissions. Report an omission only for a block that is present in the SOURCE fragment in front of you and has nothing corresponding to it in the TRANSLATION fragment in front of you. Judge only what you can see.`;
|
|
55
67
|
/**
|
|
56
68
|
* Drops the frontmatter block.
|
|
57
69
|
*
|
|
@@ -165,22 +177,43 @@ const sameFinding = (a, b) => {
|
|
|
165
177
|
}
|
|
166
178
|
return left.includes(right) || right.includes(left);
|
|
167
179
|
};
|
|
168
|
-
/**
|
|
169
|
-
|
|
180
|
+
/**
|
|
181
|
+
* The findings enough readings agreed on.
|
|
182
|
+
*
|
|
183
|
+
* `votes` defaults to every reading, which is what this has always done: with
|
|
184
|
+
* two draws, unanimity. Asking for two of three is the same idea with one more
|
|
185
|
+
* sample — a false positive still has to happen twice to reach anyone, and a
|
|
186
|
+
* real defect no longer has to be seen by every reader to count.
|
|
187
|
+
*/
|
|
188
|
+
export const agreedFindings = (draws, votes = draws.length) => {
|
|
170
189
|
if (draws.length === 0) {
|
|
171
190
|
return [];
|
|
172
191
|
}
|
|
173
|
-
const
|
|
174
|
-
|
|
192
|
+
const required = Math.max(1, Math.min(votes, draws.length));
|
|
193
|
+
const agreed = [];
|
|
194
|
+
for (const draw of draws) {
|
|
195
|
+
for (const finding of draw) {
|
|
196
|
+
if (agreed.some((kept) => sameFinding(kept, finding))) {
|
|
197
|
+
continue;
|
|
198
|
+
}
|
|
199
|
+
const support = draws.filter((other) => other.some((candidate) => sameFinding(finding, candidate))).length;
|
|
200
|
+
if (support >= required) {
|
|
201
|
+
agreed.push(finding);
|
|
202
|
+
}
|
|
203
|
+
}
|
|
204
|
+
}
|
|
205
|
+
return agreed;
|
|
175
206
|
};
|
|
176
207
|
export const parseJudgeFindings = parseFindings;
|
|
177
|
-
export const createJudge = ({ models, model, reasoningEffort, draws = DEFAULT_JUDGE_DRAWS, limit, maxRetries = DEFAULT_JUDGE_MAX_RETRIES, retryDelayMs = DEFAULT_JUDGE_RETRY_DELAY_MS, onDraw, }) => {
|
|
208
|
+
export const createJudge = ({ models, model, reasoningEffort, draws = DEFAULT_JUDGE_DRAWS, votes, limit, maxRetries = DEFAULT_JUDGE_MAX_RETRIES, retryDelayMs = DEFAULT_JUDGE_RETRY_DELAY_MS, onDraw, }) => {
|
|
178
209
|
let readings = 0;
|
|
179
210
|
const readOnce = async (request) => {
|
|
180
211
|
readings++;
|
|
181
212
|
const { contentText } = await import('@earendil-works/pi-ai');
|
|
182
213
|
const context = {
|
|
183
|
-
systemPrompt:
|
|
214
|
+
systemPrompt: request.fragment
|
|
215
|
+
? `${SYSTEM_PROMPT}\n${FRAGMENT_PROMPT}`
|
|
216
|
+
: SYSTEM_PROMPT,
|
|
184
217
|
messages: [
|
|
185
218
|
{
|
|
186
219
|
role: 'user',
|
|
@@ -214,7 +247,7 @@ export const createJudge = ({ models, model, reasoningEffort, draws = DEFAULT_JU
|
|
|
214
247
|
readings: () => readings,
|
|
215
248
|
async review(request) {
|
|
216
249
|
const results = await Promise.all(Array.from({ length: draws }, () => readOnce(request)));
|
|
217
|
-
return agreedFindings(results).map((finding) => ({
|
|
250
|
+
return agreedFindings(results, votes).map((finding) => ({
|
|
218
251
|
rule: `doom-judge:${finding.kind}`,
|
|
219
252
|
reason: `${finding.detail} — the source says: “${finding.source}”`,
|
|
220
253
|
blocking: BLOCKING_JUDGE_KINDS.includes(finding.kind),
|
|
@@ -39,6 +39,19 @@ export type MaskKind =
|
|
|
39
39
|
| 'REFID'
|
|
40
40
|
/** a bare URL written in prose (a GFM autolink literal), masked whole */
|
|
41
41
|
| 'URL';
|
|
42
|
+
/**
|
|
43
|
+
* How many times each placeholder appears in a raw document, keyed by the
|
|
44
|
+
* canonical lower-case token.
|
|
45
|
+
*
|
|
46
|
+
* Counted in the *text* rather than in the tree, and tolerant of markdown's
|
|
47
|
+
* escaping, because this is what the segment-level check has to work with: a
|
|
48
|
+
* segment's translation is a string, and counting it against the same string
|
|
49
|
+
* measurement taken from the segment's source is the only comparison that is
|
|
50
|
+
* apples to apples. {@link collectPlaceholderCounts} counts the same tokens in
|
|
51
|
+
* a parsed tree, for the whole-document assertion — the two are deliberately
|
|
52
|
+
* different measurements of the same thing.
|
|
53
|
+
*/
|
|
54
|
+
export declare const countPlaceholders: (content: string) => Map<string, number>;
|
|
42
55
|
export interface MaskEntryBase {
|
|
43
56
|
kind: MaskKind;
|
|
44
57
|
placeholder: string;
|
|
@@ -19,16 +19,30 @@ const PLACEHOLDER_PATTERN = /__DOOM_TR_[A-Z]+_\d+__/gi;
|
|
|
19
19
|
* response text (with escapes stripped) rather than in the parsed AST.
|
|
20
20
|
*/
|
|
21
21
|
const LOOSE_PLACEHOLDER_PATTERN = /_{0,2}DOOM_TR_([A-Z]+)_(\d+)_{0,2}/gi;
|
|
22
|
-
/**
|
|
23
|
-
|
|
24
|
-
|
|
22
|
+
/**
|
|
23
|
+
* How many times each placeholder appears in a raw document, keyed by the
|
|
24
|
+
* canonical lower-case token.
|
|
25
|
+
*
|
|
26
|
+
* Counted in the *text* rather than in the tree, and tolerant of markdown's
|
|
27
|
+
* escaping, because this is what the segment-level check has to work with: a
|
|
28
|
+
* segment's translation is a string, and counting it against the same string
|
|
29
|
+
* measurement taken from the segment's source is the only comparison that is
|
|
30
|
+
* apples to apples. {@link collectPlaceholderCounts} counts the same tokens in
|
|
31
|
+
* a parsed tree, for the whole-document assertion — the two are deliberately
|
|
32
|
+
* different measurements of the same thing.
|
|
33
|
+
*/
|
|
34
|
+
export const countPlaceholders = (content) => {
|
|
35
|
+
const counts = new Map();
|
|
25
36
|
for (const match of content
|
|
26
37
|
.replace(/\\/g, '')
|
|
27
38
|
.matchAll(LOOSE_PLACEHOLDER_PATTERN)) {
|
|
28
|
-
|
|
39
|
+
const token = placeholderOf(match[1].toUpperCase(), +match[2]).toLowerCase();
|
|
40
|
+
counts.set(token, (counts.get(token) ?? 0) + 1);
|
|
29
41
|
}
|
|
30
|
-
return
|
|
42
|
+
return counts;
|
|
31
43
|
};
|
|
44
|
+
/** Every placeholder-ish token in a raw document, in canonical lower case. */
|
|
45
|
+
const collectLoosePlaceholders = (content) => new Set(countPlaceholders(content).keys());
|
|
32
46
|
const matchesPlaceholder = (value) => {
|
|
33
47
|
PLACEHOLDER_PATTERN.lastIndex = 0;
|
|
34
48
|
return PLACEHOLDER_PATTERN.test(value);
|