@alauda/doom 2.5.4 → 2.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/cli/api.d.ts +2 -0
- package/lib/cli/api.js +147 -0
- package/lib/cli/helpers.d.ts +1 -7
- package/lib/cli/helpers.js +4 -41
- package/lib/cli/index.js +2 -0
- package/lib/cli/translate-agent.d.ts +81 -0
- package/lib/cli/translate-agent.js +450 -0
- package/lib/cli/translate-check.d.ts +17 -0
- package/lib/cli/translate-check.js +67 -0
- package/lib/cli/translate-checker.d.ts +58 -0
- package/lib/cli/translate-checker.js +112 -0
- package/lib/cli/translate-judge.d.ts +119 -0
- package/lib/cli/translate-judge.js +224 -0
- package/lib/cli/translate-limits.d.ts +43 -0
- package/lib/cli/translate-limits.js +55 -0
- package/lib/cli/translate-mask.d.ts +144 -0
- package/lib/cli/translate-mask.js +617 -0
- package/lib/cli/translate-models.d.ts +127 -0
- package/lib/cli/translate-models.js +189 -0
- package/lib/cli/translate-scratch.d.ts +142 -0
- package/lib/cli/translate-scratch.js +187 -0
- package/lib/cli/translate-terms.d.ts +22 -0
- package/lib/cli/translate-terms.js +33 -0
- package/lib/cli/translate.d.ts +15 -10
- package/lib/cli/translate.js +228 -185
- package/lib/eslint.js +22 -3
- package/lib/plugins/api/index.js +1 -1
- package/lib/plugins/api/types.d.ts +36 -1
- package/lib/plugins/global/components.d.ts +16 -0
- package/lib/plugins/global/components.js +16 -1
- package/lib/remark-lint/index.d.ts +20 -0
- package/lib/remark-lint/index.js +40 -0
- package/lib/remark-lint/no-unmatched-anchor.d.ts +10 -0
- package/lib/remark-lint/no-unmatched-anchor.js +13 -0
- package/lib/remark-lint/no-unopened-frontmatter.d.ts +2 -0
- package/lib/remark-lint/no-unopened-frontmatter.js +70 -0
- package/lib/remark-lint/no-unparsed-emphasis.d.ts +2 -0
- package/lib/remark-lint/no-unparsed-emphasis.js +53 -0
- package/lib/remark-lint/no-unresolved-api-ref.d.ts +40 -0
- package/lib/remark-lint/no-unresolved-api-ref.js +194 -0
- package/lib/remark-lint/rule-sets.d.ts +75 -0
- package/lib/remark-lint/rule-sets.js +120 -0
- package/lib/remark-lint/translation-parity/component-multiset.d.ts +17 -0
- package/lib/remark-lint/translation-parity/component-multiset.js +35 -0
- package/lib/remark-lint/translation-parity/echoed-source.d.ts +2 -0
- package/lib/remark-lint/translation-parity/echoed-source.js +37 -0
- package/lib/remark-lint/translation-parity/frontmatter-preservation.d.ts +2 -0
- package/lib/remark-lint/translation-parity/frontmatter-preservation.js +36 -0
- package/lib/remark-lint/translation-parity/heading-sequence.d.ts +12 -0
- package/lib/remark-lint/translation-parity/heading-sequence.js +25 -0
- package/lib/remark-lint/translation-parity/index.d.ts +24 -0
- package/lib/remark-lint/translation-parity/index.js +24 -0
- package/lib/remark-lint/translation-parity/jsx-attribute-parity.d.ts +14 -0
- package/lib/remark-lint/translation-parity/jsx-attribute-parity.js +47 -0
- package/lib/remark-lint/translation-parity/length-ratio.d.ts +2 -0
- package/lib/remark-lint/translation-parity/length-ratio.js +49 -0
- package/lib/remark-lint/translation-parity/link-isomorphism.d.ts +26 -0
- package/lib/remark-lint/translation-parity/link-isomorphism.js +45 -0
- package/lib/remark-lint/translation-parity/shared.d.ts +110 -0
- package/lib/remark-lint/translation-parity/shared.js +282 -0
- package/lib/remark-lint/translation-parity/terminology-adherence.d.ts +2 -0
- package/lib/remark-lint/translation-parity/terminology-adherence.js +67 -0
- package/lib/remark-lint/translation-parity/up-to-date.d.ts +11 -0
- package/lib/remark-lint/translation-parity/up-to-date.js +21 -0
- package/lib/remark-lint/translation-parity/url-residue.d.ts +2 -0
- package/lib/remark-lint/translation-parity/url-residue.js +46 -0
- package/lib/remarkrc.d.ts +1 -1
- package/lib/remarkrc.js +7 -34
- package/lib/runtime/api-paths.d.ts +20 -0
- package/lib/runtime/api-paths.js +29 -0
- package/lib/runtime/components/K8sAPI.d.ts +14 -1
- package/lib/runtime/components/K8sAPI.js +49 -10
- package/lib/runtime/components/K8sCrd.d.ts +1 -1
- package/lib/runtime/components/K8sPermissionTable.js +10 -5
- package/lib/runtime/components/OpenAPIPath.js +19 -2
- package/lib/runtime/components/_HeadingTitle.js +24 -6
- package/lib/runtime/components/_K8sAPIEndpoints.d.ts +8 -1
- package/lib/runtime/components/_K8sAPIEndpoints.js +27 -9
- package/lib/runtime/components/_K8sAPISchema.d.ts +9 -1
- package/lib/runtime/components/_K8sAPISchema.js +24 -6
- package/lib/runtime/components/_translation-policy.d.ts +23 -0
- package/lib/runtime/components/_translation-policy.js +32 -0
- package/lib/runtime/hooks/useTranslation.d.ts +1 -1
- package/lib/runtime/translation.d.ts +68 -0
- package/lib/runtime/translation.js +51 -0
- package/lib/runtime/utils.d.ts +24 -2
- package/lib/runtime/utils.js +87 -5
- package/lib/shared/constants.d.ts +9 -0
- package/lib/shared/constants.js +14 -0
- package/lib/types.d.ts +70 -0
- package/lib/utils/helpers.js +8 -2
- package/package.json +3 -1
|
@@ -0,0 +1,112 @@
|
|
|
1
|
+
import remarkMdx from 'remark-mdx';
|
|
2
|
+
import remarkParse from 'remark-parse';
|
|
3
|
+
import remarkStringify from 'remark-stringify';
|
|
4
|
+
import { unified } from 'unified';
|
|
5
|
+
import { VFile } from 'vfile';
|
|
6
|
+
import doomLint from "../remark-lint/index.js";
|
|
7
|
+
import { forgetDocumentAnchors } from "../remark-lint/no-unmatched-anchor.js";
|
|
8
|
+
import { SYNTAX_PLUGINS, TRANSLATION_CHECK_RULES, } from "../remark-lint/rule-sets.js";
|
|
9
|
+
import { resolveTranslation } from "../remark-lint/translation-parity/shared.js";
|
|
10
|
+
/** Findings that stop a document from shipping. */
|
|
11
|
+
export const blockingFindings = (findings) => findings.filter((finding) => finding.blocking !== false);
|
|
12
|
+
/**
|
|
13
|
+
* Raised when a document cannot be checked at all.
|
|
14
|
+
*
|
|
15
|
+
* This is not a finding: a finding says the translation is wrong, this says
|
|
16
|
+
* nothing was examined. The distinction matters because the failure mode of
|
|
17
|
+
* the pairwise rules is to return quietly — a translation whose path or
|
|
18
|
+
* frontmatter is not what they expect is not a translation as far as they are
|
|
19
|
+
* concerned, and every rule passes. Silence and success look identical, so the
|
|
20
|
+
* caller has to be told which one it got.
|
|
21
|
+
*/
|
|
22
|
+
export class TranslationNotCheckableError extends Error {
|
|
23
|
+
targetPath;
|
|
24
|
+
constructor(targetPath, reason) {
|
|
25
|
+
super(`Cannot check \`${targetPath}\`: ${reason}`);
|
|
26
|
+
this.targetPath = targetPath;
|
|
27
|
+
this.name = 'TranslationNotCheckableError';
|
|
28
|
+
}
|
|
29
|
+
}
|
|
30
|
+
const buildProcessor = (mdx) => {
|
|
31
|
+
let processor = unified()
|
|
32
|
+
.use(remarkParse)
|
|
33
|
+
.use(remarkStringify);
|
|
34
|
+
for (const plugin of SYNTAX_PLUGINS) {
|
|
35
|
+
processor = processor.use(plugin);
|
|
36
|
+
}
|
|
37
|
+
if (mdx) {
|
|
38
|
+
processor = processor.use(remarkMdx);
|
|
39
|
+
}
|
|
40
|
+
// Message control is what makes `<!-- lint disable -->` work. Included so a
|
|
41
|
+
// document says the same thing here as it does under `doom lint`.
|
|
42
|
+
processor = processor.use(doomLint);
|
|
43
|
+
for (const rule of TRANSLATION_CHECK_RULES) {
|
|
44
|
+
processor = processor.use(rule);
|
|
45
|
+
}
|
|
46
|
+
return processor.freeze();
|
|
47
|
+
};
|
|
48
|
+
export const createTranslationChecker = () => {
|
|
49
|
+
const mdxProcessor = buildProcessor(true);
|
|
50
|
+
const mdProcessor = buildProcessor(false);
|
|
51
|
+
const processorFor = (targetPath) => targetPath.endsWith('.mdx') ? mdxProcessor : mdProcessor;
|
|
52
|
+
const check = async (targetPath, content) => {
|
|
53
|
+
// A lint run over a directory sees each path once, so rules are free to
|
|
54
|
+
// cache what a path contained. This does not: the same path holds different
|
|
55
|
+
// content on every repair turn, and a rule reading last turn's copy would
|
|
56
|
+
// report a problem the repair already fixed. `no-unmatched-anchor` is the
|
|
57
|
+
// one rule that caches today — it is not in this rule set any more, but the
|
|
58
|
+
// invariant is a property of the checker rather than of that rule.
|
|
59
|
+
forgetDocumentAnchors(targetPath);
|
|
60
|
+
let vfile;
|
|
61
|
+
try {
|
|
62
|
+
vfile = await processorFor(targetPath).process(new VFile({ path: targetPath, value: content }));
|
|
63
|
+
}
|
|
64
|
+
catch (error) {
|
|
65
|
+
return [
|
|
66
|
+
{
|
|
67
|
+
rule: 'doom-lint:unparseable',
|
|
68
|
+
reason: `the document does not parse: ${error instanceof Error ? error.message : String(error)}`,
|
|
69
|
+
},
|
|
70
|
+
];
|
|
71
|
+
}
|
|
72
|
+
return vfile.messages.map((message) => ({
|
|
73
|
+
rule: message.ruleId ?? 'unknown',
|
|
74
|
+
reason: message.reason,
|
|
75
|
+
line: message.line ?? undefined,
|
|
76
|
+
column: message.column ?? undefined,
|
|
77
|
+
}));
|
|
78
|
+
};
|
|
79
|
+
return {
|
|
80
|
+
check,
|
|
81
|
+
async assertCheckable(targetPath, content) {
|
|
82
|
+
const processor = processorFor(targetPath);
|
|
83
|
+
let tree;
|
|
84
|
+
try {
|
|
85
|
+
tree = processor.parse(new VFile({ path: targetPath, value: content }));
|
|
86
|
+
}
|
|
87
|
+
catch (error) {
|
|
88
|
+
throw new TranslationNotCheckableError(targetPath, `it does not parse: ${error instanceof Error ? error.message : String(error)}`);
|
|
89
|
+
}
|
|
90
|
+
const status = await resolveTranslation(tree, new VFile({ path: targetPath, value: content }));
|
|
91
|
+
switch (status.kind) {
|
|
92
|
+
case 'current': {
|
|
93
|
+
return;
|
|
94
|
+
}
|
|
95
|
+
case 'not-a-translation': {
|
|
96
|
+
throw new TranslationNotCheckableError(targetPath, 'the pairwise rules do not see it as a translation — the path is not `<docs root>/<language>/…`, or the language is the source language');
|
|
97
|
+
}
|
|
98
|
+
case 'unmanaged': {
|
|
99
|
+
throw new TranslationNotCheckableError(targetPath, 'it carries no `sourceSHA`, so nothing pairs it with a source');
|
|
100
|
+
}
|
|
101
|
+
case 'source-missing': {
|
|
102
|
+
throw new TranslationNotCheckableError(targetPath, `its source \`${status.sourcePath}\` does not exist`);
|
|
103
|
+
}
|
|
104
|
+
case 'stale': {
|
|
105
|
+
throw new TranslationNotCheckableError(targetPath, `its \`sourceSHA\` names a different version of \`${status.sourcePath}\` than the one on disk`);
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
},
|
|
109
|
+
};
|
|
110
|
+
};
|
|
111
|
+
/** One finding, rendered for a person reading a build log. */
|
|
112
|
+
export const describeFinding = (finding) => `${finding.rule}${finding.line ? ` (line ${finding.line})` : ''}: ${finding.reason}`;
|
|
@@ -0,0 +1,119 @@
|
|
|
1
|
+
import type { Model, Models, ThinkingLevel } from '@earendil-works/pi-ai';
|
|
2
|
+
import type { TranslationFinding } from './translate-checker.ts';
|
|
3
|
+
/**
|
|
4
|
+
* The semantic check — the one that reads.
|
|
5
|
+
*
|
|
6
|
+
* Every deterministic check in this pipeline asks a structural question: is
|
|
7
|
+
* every placeholder back, does the component count match, do the headings line
|
|
8
|
+
* up. All of them pass a translation that is fluent, well-formed, correctly
|
|
9
|
+
* linked, and about something else. That blind spot is not a gap in the rules,
|
|
10
|
+
* it is what rules are: `state.md` §7 records two documents that lost whole
|
|
11
|
+
* sentences and were caught only because the missing sentence happened to
|
|
12
|
+
* contain a `<Term>`.
|
|
13
|
+
*
|
|
14
|
+
* So a second model reads both documents and says what the translation lost,
|
|
15
|
+
* added, or got wrong. Three things keep that from becoming a coin toss:
|
|
16
|
+
*
|
|
17
|
+
* 1. **Only three kinds block.** Omission, addition and mistranslation are
|
|
18
|
+
* defects. Readability is an opinion, and it is reported without failing
|
|
19
|
+
* anything — a translation that meets the standard passes, rather than being
|
|
20
|
+
* iterated until nobody can improve it.
|
|
21
|
+
* 2. **A finding must be drawn twice.** Two independent readings, and only what
|
|
22
|
+
* both report counts. A false positive has to happen twice to reach anyone.
|
|
23
|
+
* 3. **It quotes.** Every finding names the passage it is about, so a person —
|
|
24
|
+
* or the translator — can go and look instead of taking its word.
|
|
25
|
+
*
|
|
26
|
+
* It reads the *masked* documents, the same ones the translator sees. Its
|
|
27
|
+
* findings are fed back into the translator's context, and an unmasked link
|
|
28
|
+
* target quoted in a finding would be a link target back in the model's reach
|
|
29
|
+
* — the one thing masking exists to prevent. Placeholders are stable tokens on
|
|
30
|
+
* both sides, so nothing about judging prose is lost by leaving them in.
|
|
31
|
+
*/
|
|
32
|
+
export type JudgeFindingKind =
|
|
33
|
+
/** the translation dropped something the source says */
|
|
34
|
+
'omission'
|
|
35
|
+
/** the translation says something the source does not */
|
|
36
|
+
| 'addition'
|
|
37
|
+
/** the translation says something different from the source */
|
|
38
|
+
| 'mistranslation'
|
|
39
|
+
/** the translation is awkward or unidiomatic — reported, never blocking */
|
|
40
|
+
| 'fluency';
|
|
41
|
+
/** The kinds that stop a document from shipping. */
|
|
42
|
+
export declare const BLOCKING_JUDGE_KINDS: readonly JudgeFindingKind[];
|
|
43
|
+
export interface JudgeFinding {
|
|
44
|
+
kind: JudgeFindingKind;
|
|
45
|
+
/** The passage in the source the finding is about. Required: a finding nobody can locate is not actionable. */
|
|
46
|
+
source: string;
|
|
47
|
+
/** What is wrong with the translation of that passage. */
|
|
48
|
+
detail: string;
|
|
49
|
+
}
|
|
50
|
+
export interface JudgeRequest {
|
|
51
|
+
/** The masked source, as the translator saw it. */
|
|
52
|
+
sourceText: string;
|
|
53
|
+
/** The masked translation, as the translator wrote it. */
|
|
54
|
+
translationText: string;
|
|
55
|
+
sourceLanguage: string;
|
|
56
|
+
targetLanguage: string;
|
|
57
|
+
/**
|
|
58
|
+
* The product's established translations for terms the source uses.
|
|
59
|
+
*
|
|
60
|
+
* The same table the translator is given. Without it the reviewer reports
|
|
61
|
+
* the house translation of a product term as a mistranslation — measured:
|
|
62
|
+
* four of the ten false reds in the first calibration were one term,
|
|
63
|
+
* "workload cluster" → "业务集群", which is simply what these documents call
|
|
64
|
+
* it.
|
|
65
|
+
*/
|
|
66
|
+
terms?: readonly {
|
|
67
|
+
source: string;
|
|
68
|
+
target: string;
|
|
69
|
+
}[];
|
|
70
|
+
}
|
|
71
|
+
export interface Judge {
|
|
72
|
+
review(request: JudgeRequest): Promise<TranslationFinding[]>;
|
|
73
|
+
/** Readings taken so far. For the run summary — the judge is the expensive part. */
|
|
74
|
+
readings(): number;
|
|
75
|
+
}
|
|
76
|
+
export interface CreateJudgeOptions {
|
|
77
|
+
models: Models;
|
|
78
|
+
model: Model<'openai-completions'>;
|
|
79
|
+
reasoningEffort: ThinkingLevel;
|
|
80
|
+
/**
|
|
81
|
+
* How many independent readings must agree.
|
|
82
|
+
*
|
|
83
|
+
* Two by default. The variety comes from the model's own non-determinism —
|
|
84
|
+
* measured on this gateway, `temperature: 0` does not produce identical
|
|
85
|
+
* answers — so it needs no parameter to create it.
|
|
86
|
+
*/
|
|
87
|
+
draws?: number;
|
|
88
|
+
/** Applied to every model call, so judge readings count against the same budget as translation. */
|
|
89
|
+
limit: <T>(run: () => Promise<T>) => Promise<T>;
|
|
90
|
+
/** How many times a failed reading is retried before the document fails. */
|
|
91
|
+
maxRetries?: number;
|
|
92
|
+
/** First backoff, doubled per attempt. */
|
|
93
|
+
retryDelayMs?: number;
|
|
94
|
+
/**
|
|
95
|
+
* Called with each raw reading, before the draws are compared.
|
|
96
|
+
*
|
|
97
|
+
* Only calibration uses this. Agreement throws most of what a single reading
|
|
98
|
+
* says away, which is the point — but it also means "no findings" and "the
|
|
99
|
+
* judge said nothing at all" produce the same output, and those are very
|
|
100
|
+
* different situations to be looking at when deciding whether this can be a
|
|
101
|
+
* gate.
|
|
102
|
+
*/
|
|
103
|
+
onDraw?: (findings: readonly JudgeFinding[]) => void;
|
|
104
|
+
}
|
|
105
|
+
export declare const DEFAULT_JUDGE_DRAWS = 2;
|
|
106
|
+
/**
|
|
107
|
+
* Transient refusals are normal at corpus scale.
|
|
108
|
+
*
|
|
109
|
+
* Measured on this gateway: a handful of concurrent full-document readings gets
|
|
110
|
+
* "Concurrency limit exceeded for user, please retry later". Thousands of
|
|
111
|
+
* readings will meet it. Retrying is not softening the check — a reading that
|
|
112
|
+
* never succeeds still fails the document.
|
|
113
|
+
*/
|
|
114
|
+
export declare const DEFAULT_JUDGE_MAX_RETRIES = 4;
|
|
115
|
+
export declare const DEFAULT_JUDGE_RETRY_DELAY_MS = 2000;
|
|
116
|
+
/** The findings every reading reported. */
|
|
117
|
+
export declare const agreedFindings: (draws: readonly JudgeFinding[][]) => JudgeFinding[];
|
|
118
|
+
export declare const parseJudgeFindings: (reply: string) => JudgeFinding[];
|
|
119
|
+
export declare const createJudge: ({ models, model, reasoningEffort, draws, limit, maxRetries, retryDelayMs, onDraw, }: CreateJudgeOptions) => Judge;
|
|
@@ -0,0 +1,224 @@
|
|
|
1
|
+
/** The kinds that stop a document from shipping. */
|
|
2
|
+
export const BLOCKING_JUDGE_KINDS = [
|
|
3
|
+
'omission',
|
|
4
|
+
'addition',
|
|
5
|
+
'mistranslation',
|
|
6
|
+
];
|
|
7
|
+
const ALL_KINDS = [
|
|
8
|
+
...BLOCKING_JUDGE_KINDS,
|
|
9
|
+
'fluency',
|
|
10
|
+
];
|
|
11
|
+
export const DEFAULT_JUDGE_DRAWS = 2;
|
|
12
|
+
/**
|
|
13
|
+
* Transient refusals are normal at corpus scale.
|
|
14
|
+
*
|
|
15
|
+
* Measured on this gateway: a handful of concurrent full-document readings gets
|
|
16
|
+
* "Concurrency limit exceeded for user, please retry later". Thousands of
|
|
17
|
+
* readings will meet it. Retrying is not softening the check — a reading that
|
|
18
|
+
* never succeeds still fails the document.
|
|
19
|
+
*/
|
|
20
|
+
export const DEFAULT_JUDGE_MAX_RETRIES = 4;
|
|
21
|
+
export const DEFAULT_JUDGE_RETRY_DELAY_MS = 2_000;
|
|
22
|
+
const SYSTEM_PROMPT = `You are reviewing a translation of technical documentation. You are not improving it and not rewriting it: you are saying what is wrong with it, if anything.
|
|
23
|
+
|
|
24
|
+
Work in two passes, in this order.
|
|
25
|
+
|
|
26
|
+
PASS 1 — coverage. Go through the SOURCE one block at a time: every paragraph, every list item, every table row, every heading. For each block, find what corresponds to it in the translation. A block with nothing corresponding to it is an "omission" — report it and quote the block.
|
|
27
|
+
|
|
28
|
+
Do this pass first and do it completely. A dropped sentence or a dropped bullet is what this review exists to catch: it leaves the document well-formed, correctly linked and fluent, so nothing else in the pipeline can see it. Counting is part of the job — if a list has six items in the source and five in the translation, one was dropped, and saying which is the finding.
|
|
29
|
+
|
|
30
|
+
PASS 2 — the blocks that do have a counterpart. Report "mistranslation" only when a reader would come away with a different fact or a different instruction: a changed number, a reversed relationship stated in words, a different component, a step that now says something else. Judge the relationship a sentence states, not the sequence its parts appear in. A wording that is merely less precise than the source, or that you would have phrased better, is "fluency" — not a defect. If you find yourself explaining that a word "should rather be" some other word, it is fluency.
|
|
31
|
+
|
|
32
|
+
Report "addition" when the translation states a fact the source does not.
|
|
33
|
+
|
|
34
|
+
Report only what you can point at. Every finding quotes the passage of the SOURCE it is about.
|
|
35
|
+
|
|
36
|
+
Tokens shaped like \`__DOOM_TR_ICODE_3__\` are placeholders. They stand in for content deliberately kept out of translation — link targets, code, identifiers — so you cannot see what any one of them means, and nothing but its number tells two of them apart.
|
|
37
|
+
|
|
38
|
+
Their order therefore carries no information. Languages reorder freely: "manage Y through X" is written "through X, manage Y" in Chinese and in Russian, and the placeholders travel with the words that moved. That is the target language working, not a swap.
|
|
39
|
+
|
|
40
|
+
So never report a finding that rests on a placeholder — which one appears, where it appears, or in what order relative to another. Whether they are all present, and whether each stands in the node kind it was issued for, is checked exactly, by code, elsewhere.
|
|
41
|
+
|
|
42
|
+
Before you answer, apply this test to every finding you are about to report: **would it still stand if the placeholders were renumbered?** If it would not, it is not a finding — drop it.
|
|
43
|
+
|
|
44
|
+
That holds however you refer to them — by number, or as "the first item", "the second one", "the third". Describing which thing comes before which is the same finding in other words, and it is still not one.
|
|
45
|
+
|
|
46
|
+
Differences that are not findings:
|
|
47
|
+
- word order — including a different order of placeholders — and sentence splitting or merging that keeps the meaning;
|
|
48
|
+
- a technical term, product name, UI label or command left in the source language;
|
|
49
|
+
- formatting, whitespace, or markdown structure.
|
|
50
|
+
|
|
51
|
+
Answer with a JSON array and nothing else. Each element:
|
|
52
|
+
{"kind": "omission" | "addition" | "mistranslation" | "fluency", "source": "<the exact passage from the source>", "detail": "<what is wrong, in one sentence>"}
|
|
53
|
+
|
|
54
|
+
An empty array means the translation is faithful. That is the expected answer for a good translation — do not look for something to say.`;
|
|
55
|
+
/**
|
|
56
|
+
* Drops the frontmatter block.
|
|
57
|
+
*
|
|
58
|
+
* The translator is *supposed* to change it: `sourceSHA` is written by the
|
|
59
|
+
* translator, `i18n` is dropped, `title` is translated. Showing that to a
|
|
60
|
+
* reviewer asked to find additions guarantees it reports one, every time, on
|
|
61
|
+
* every document — which is what the first calibration run measured. Whether
|
|
62
|
+
* the rest of the frontmatter survived is counted exactly by
|
|
63
|
+
* `translation-frontmatter-preservation`.
|
|
64
|
+
*/
|
|
65
|
+
const withoutFrontmatter = (document) => {
|
|
66
|
+
if (!document.startsWith('---')) {
|
|
67
|
+
return document;
|
|
68
|
+
}
|
|
69
|
+
const end = document.indexOf('\n---', 3);
|
|
70
|
+
return end < 0 ? document : document.slice(end + 4).replace(/^\n+/u, '');
|
|
71
|
+
};
|
|
72
|
+
const buildUserPrompt = ({ sourceText, translationText, sourceLanguage, targetLanguage, terms = [], }) => [
|
|
73
|
+
...(terms.length > 0
|
|
74
|
+
? [
|
|
75
|
+
"These are this product's established translations. Using them is correct by definition — never report one as a mistranslation:",
|
|
76
|
+
...terms.map((term) => ` ${term.source} => ${term.target}`),
|
|
77
|
+
'',
|
|
78
|
+
]
|
|
79
|
+
: []),
|
|
80
|
+
`The source document (${sourceLanguage}):`,
|
|
81
|
+
'<<<SOURCE',
|
|
82
|
+
withoutFrontmatter(sourceText),
|
|
83
|
+
'SOURCE',
|
|
84
|
+
'',
|
|
85
|
+
`Its translation (${targetLanguage}):`,
|
|
86
|
+
'<<<TRANSLATION',
|
|
87
|
+
withoutFrontmatter(translationText),
|
|
88
|
+
'TRANSLATION',
|
|
89
|
+
].join('\n');
|
|
90
|
+
/**
|
|
91
|
+
* Pulls the array out of a reply that may be wrapped in prose or a code fence.
|
|
92
|
+
*
|
|
93
|
+
* Lenient on purpose: a reply that is unusable is treated as a reading that
|
|
94
|
+
* found nothing, because the alternative — failing the document because the
|
|
95
|
+
* judge was hard to parse — fails the wrong thing. A judge that cannot be read
|
|
96
|
+
* cannot report a defect either, and the deterministic checks are unaffected.
|
|
97
|
+
*/
|
|
98
|
+
const parseFindings = (reply) => {
|
|
99
|
+
const start = reply.indexOf('[');
|
|
100
|
+
const end = reply.lastIndexOf(']');
|
|
101
|
+
if (start < 0 || end <= start) {
|
|
102
|
+
return [];
|
|
103
|
+
}
|
|
104
|
+
let parsed;
|
|
105
|
+
try {
|
|
106
|
+
parsed = JSON.parse(reply.slice(start, end + 1));
|
|
107
|
+
}
|
|
108
|
+
catch {
|
|
109
|
+
return [];
|
|
110
|
+
}
|
|
111
|
+
if (!Array.isArray(parsed)) {
|
|
112
|
+
return [];
|
|
113
|
+
}
|
|
114
|
+
const findings = [];
|
|
115
|
+
for (const entry of parsed) {
|
|
116
|
+
if (!entry || typeof entry !== 'object') {
|
|
117
|
+
continue;
|
|
118
|
+
}
|
|
119
|
+
const { kind, source, detail } = entry;
|
|
120
|
+
if (typeof kind !== 'string' ||
|
|
121
|
+
!ALL_KINDS.includes(kind) ||
|
|
122
|
+
typeof source !== 'string' ||
|
|
123
|
+
source.trim() === '' ||
|
|
124
|
+
typeof detail !== 'string') {
|
|
125
|
+
continue;
|
|
126
|
+
}
|
|
127
|
+
if (isAboutPlaceholdersOnly(source)) {
|
|
128
|
+
continue;
|
|
129
|
+
}
|
|
130
|
+
findings.push({
|
|
131
|
+
kind: kind,
|
|
132
|
+
source: source.trim(),
|
|
133
|
+
detail: detail.trim(),
|
|
134
|
+
});
|
|
135
|
+
}
|
|
136
|
+
return findings;
|
|
137
|
+
};
|
|
138
|
+
/**
|
|
139
|
+
* A quote with nothing in it but placeholders.
|
|
140
|
+
*
|
|
141
|
+
* The prompt tells the judge that placeholders are not its business, and it
|
|
142
|
+
* mostly listens. When it does not, the finding is about something the
|
|
143
|
+
* deterministic checks already count exactly — so it is dropped here rather
|
|
144
|
+
* than sent to the translator, who would be asked to fix a token it is
|
|
145
|
+
* required to reproduce verbatim.
|
|
146
|
+
*/
|
|
147
|
+
const isAboutPlaceholdersOnly = (source) => source.replace(/__DOOM_TR_[A-Z]+_\d+__/gu, '').trim() === '';
|
|
148
|
+
/** Whitespace and case removed, so two readings quoting the same passage look the same. */
|
|
149
|
+
const normalise = (value) => value.toLowerCase().replace(/\s+/gu, ' ').trim();
|
|
150
|
+
/**
|
|
151
|
+
* Whether two readings are talking about the same thing.
|
|
152
|
+
*
|
|
153
|
+
* Same kind, and one quote contains the other. Requiring the quotes to match
|
|
154
|
+
* exactly would make agreement depend on where each reading chose to start and
|
|
155
|
+
* stop, which is not what "both saw it" should mean.
|
|
156
|
+
*/
|
|
157
|
+
const sameFinding = (a, b) => {
|
|
158
|
+
if (a.kind !== b.kind) {
|
|
159
|
+
return false;
|
|
160
|
+
}
|
|
161
|
+
const left = normalise(a.source);
|
|
162
|
+
const right = normalise(b.source);
|
|
163
|
+
if (left === '' || right === '') {
|
|
164
|
+
return false;
|
|
165
|
+
}
|
|
166
|
+
return left.includes(right) || right.includes(left);
|
|
167
|
+
};
|
|
168
|
+
/** The findings every reading reported. */
|
|
169
|
+
export const agreedFindings = (draws) => {
|
|
170
|
+
if (draws.length === 0) {
|
|
171
|
+
return [];
|
|
172
|
+
}
|
|
173
|
+
const [first, ...rest] = draws;
|
|
174
|
+
return first.filter((finding) => rest.every((draw) => draw.some((other) => sameFinding(finding, other))));
|
|
175
|
+
};
|
|
176
|
+
export const parseJudgeFindings = parseFindings;
|
|
177
|
+
export const createJudge = ({ models, model, reasoningEffort, draws = DEFAULT_JUDGE_DRAWS, limit, maxRetries = DEFAULT_JUDGE_MAX_RETRIES, retryDelayMs = DEFAULT_JUDGE_RETRY_DELAY_MS, onDraw, }) => {
|
|
178
|
+
let readings = 0;
|
|
179
|
+
const readOnce = async (request) => {
|
|
180
|
+
readings++;
|
|
181
|
+
const { contentText } = await import('@earendil-works/pi-ai');
|
|
182
|
+
const context = {
|
|
183
|
+
systemPrompt: SYSTEM_PROMPT,
|
|
184
|
+
messages: [
|
|
185
|
+
{
|
|
186
|
+
role: 'user',
|
|
187
|
+
content: buildUserPrompt(request),
|
|
188
|
+
timestamp: Date.now(),
|
|
189
|
+
},
|
|
190
|
+
],
|
|
191
|
+
};
|
|
192
|
+
let lastError = 'unknown';
|
|
193
|
+
for (let attempt = 0; attempt <= maxRetries; attempt++) {
|
|
194
|
+
const reply = await limit(() => models.completeSimple(model, context, {
|
|
195
|
+
reasoning: reasoningEffort,
|
|
196
|
+
maxRetries,
|
|
197
|
+
}));
|
|
198
|
+
if (reply.stopReason !== 'error' && reply.stopReason !== 'aborted') {
|
|
199
|
+
const drawn = parseFindings(contentText(reply.content));
|
|
200
|
+
onDraw?.(drawn);
|
|
201
|
+
return drawn;
|
|
202
|
+
}
|
|
203
|
+
lastError = reply.errorMessage ?? reply.stopReason;
|
|
204
|
+
if (attempt < maxRetries) {
|
|
205
|
+
// A whole corpus is thousands of readings, and a gateway under load
|
|
206
|
+
// refuses some of them. Backing off and asking again is not softening
|
|
207
|
+
// the gate: running out of attempts still throws.
|
|
208
|
+
await new Promise((resolve) => globalThis.setTimeout(resolve, retryDelayMs * 2 ** attempt));
|
|
209
|
+
}
|
|
210
|
+
}
|
|
211
|
+
throw new Error(`The judge model failed: ${lastError}`);
|
|
212
|
+
};
|
|
213
|
+
return {
|
|
214
|
+
readings: () => readings,
|
|
215
|
+
async review(request) {
|
|
216
|
+
const results = await Promise.all(Array.from({ length: draws }, () => readOnce(request)));
|
|
217
|
+
return agreedFindings(results).map((finding) => ({
|
|
218
|
+
rule: `doom-judge:${finding.kind}`,
|
|
219
|
+
reason: `${finding.detail} — the source says: “${finding.source}”`,
|
|
220
|
+
blocking: BLOCKING_JUDGE_KINDS.includes(finding.kind),
|
|
221
|
+
}));
|
|
222
|
+
},
|
|
223
|
+
};
|
|
224
|
+
};
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* How hard a translation run is allowed to drive the gateway.
|
|
3
|
+
*
|
|
4
|
+
* Its own module rather than part of the command: the command reads config and
|
|
5
|
+
* walks files, while this is one small unit with an exact contract, and the
|
|
6
|
+
* only part of the pair worth testing on its own.
|
|
7
|
+
*/
|
|
8
|
+
export declare const CONCURRENCY_ENV = "ALAUDA_OPENAI_CONCURRENCY";
|
|
9
|
+
export declare const REQUESTS_PER_MINUTE_ENV = "ALAUDA_OPENAI_REQUESTS_PER_MINUTE";
|
|
10
|
+
/** How many documents are translated at once. */
|
|
11
|
+
export declare const DEFAULT_CONCURRENCY = 2;
|
|
12
|
+
/** How many model requests a minute the gateway is asked to take. */
|
|
13
|
+
export declare const DEFAULT_REQUESTS_PER_MINUTE = 25;
|
|
14
|
+
/**
|
|
15
|
+
* Reads a positive integer from the environment, or fails naming the variable.
|
|
16
|
+
*
|
|
17
|
+
* Falling back to the default on a malformed value would be the worst of the
|
|
18
|
+
* three outcomes: the run is not what was asked for, and nothing says so. A
|
|
19
|
+
* gateway budget set to `twenty` is a mistake worth stopping for.
|
|
20
|
+
*/
|
|
21
|
+
export declare const positiveIntFromEnv: (name: string, fallback: number) => number;
|
|
22
|
+
/**
|
|
23
|
+
* The two limits a run is given, and the one number each of them means.
|
|
24
|
+
*
|
|
25
|
+
* `concurrency` bounds documents in flight *and* model calls in flight. They
|
|
26
|
+
* are one knob rather than two because "how many translations are running" is
|
|
27
|
+
* what a gateway feels, and two numbers that mean nearly the same thing drift
|
|
28
|
+
* apart. The cost is internal queueing — the judge's two draws take their turn
|
|
29
|
+
* instead of going out together — which slows a document down without changing
|
|
30
|
+
* what it produces.
|
|
31
|
+
*
|
|
32
|
+
* `requestsPerMinute` is the budget the gateway is protected by. It counts
|
|
33
|
+
* calls, not documents: the loop makes as many as a document needs, and the
|
|
34
|
+
* extra turns an agent takes count against the same budget rather than
|
|
35
|
+
* slipping past it.
|
|
36
|
+
*/
|
|
37
|
+
export declare const createLimits: ({ concurrency, requestsPerMinute, }: {
|
|
38
|
+
concurrency: number;
|
|
39
|
+
requestsPerMinute: number;
|
|
40
|
+
}) => {
|
|
41
|
+
modelCallLimit: <T>(fn: () => Promise<T>) => Promise<T>;
|
|
42
|
+
documentLimit: <T>(fn: () => Promise<T>) => Promise<T>;
|
|
43
|
+
};
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
import { pRateLimit } from 'p-ratelimit';
|
|
2
|
+
/**
|
|
3
|
+
* How hard a translation run is allowed to drive the gateway.
|
|
4
|
+
*
|
|
5
|
+
* Its own module rather than part of the command: the command reads config and
|
|
6
|
+
* walks files, while this is one small unit with an exact contract, and the
|
|
7
|
+
* only part of the pair worth testing on its own.
|
|
8
|
+
*/
|
|
9
|
+
export const CONCURRENCY_ENV = 'ALAUDA_OPENAI_CONCURRENCY';
|
|
10
|
+
export const REQUESTS_PER_MINUTE_ENV = 'ALAUDA_OPENAI_REQUESTS_PER_MINUTE';
|
|
11
|
+
/** How many documents are translated at once. */
|
|
12
|
+
export const DEFAULT_CONCURRENCY = 2;
|
|
13
|
+
/** How many model requests a minute the gateway is asked to take. */
|
|
14
|
+
export const DEFAULT_REQUESTS_PER_MINUTE = 25;
|
|
15
|
+
/**
|
|
16
|
+
* Reads a positive integer from the environment, or fails naming the variable.
|
|
17
|
+
*
|
|
18
|
+
* Falling back to the default on a malformed value would be the worst of the
|
|
19
|
+
* three outcomes: the run is not what was asked for, and nothing says so. A
|
|
20
|
+
* gateway budget set to `twenty` is a mistake worth stopping for.
|
|
21
|
+
*/
|
|
22
|
+
export const positiveIntFromEnv = (name, fallback) => {
|
|
23
|
+
const raw = process.env[name];
|
|
24
|
+
if (raw == null || raw === '') {
|
|
25
|
+
return fallback;
|
|
26
|
+
}
|
|
27
|
+
const value = Number(raw);
|
|
28
|
+
if (!Number.isInteger(value) || value < 1) {
|
|
29
|
+
throw new Error(`\`${name}\` must be a whole number of at least 1, and is \`${raw}\`.`);
|
|
30
|
+
}
|
|
31
|
+
return value;
|
|
32
|
+
};
|
|
33
|
+
/**
|
|
34
|
+
* The two limits a run is given, and the one number each of them means.
|
|
35
|
+
*
|
|
36
|
+
* `concurrency` bounds documents in flight *and* model calls in flight. They
|
|
37
|
+
* are one knob rather than two because "how many translations are running" is
|
|
38
|
+
* what a gateway feels, and two numbers that mean nearly the same thing drift
|
|
39
|
+
* apart. The cost is internal queueing — the judge's two draws take their turn
|
|
40
|
+
* instead of going out together — which slows a document down without changing
|
|
41
|
+
* what it produces.
|
|
42
|
+
*
|
|
43
|
+
* `requestsPerMinute` is the budget the gateway is protected by. It counts
|
|
44
|
+
* calls, not documents: the loop makes as many as a document needs, and the
|
|
45
|
+
* extra turns an agent takes count against the same budget rather than
|
|
46
|
+
* slipping past it.
|
|
47
|
+
*/
|
|
48
|
+
export const createLimits = ({ concurrency, requestsPerMinute, }) => ({
|
|
49
|
+
modelCallLimit: pRateLimit({
|
|
50
|
+
interval: 60_000, // 1min
|
|
51
|
+
rate: requestsPerMinute,
|
|
52
|
+
concurrency,
|
|
53
|
+
}),
|
|
54
|
+
documentLimit: pRateLimit({ concurrency }),
|
|
55
|
+
});
|