@alauda/doom 2.6.0 → 2.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/cli/translate-call.d.ts +85 -0
- package/lib/cli/translate-call.js +149 -0
- package/lib/cli/translate-checker.d.ts +8 -0
- package/lib/cli/translate-diagnose.d.ts +45 -0
- package/lib/cli/translate-diagnose.js +118 -0
- package/lib/cli/translate-judge.d.ts +28 -3
- package/lib/cli/translate-judge.js +40 -7
- package/lib/cli/translate-mask.d.ts +13 -0
- package/lib/cli/translate-mask.js +19 -5
- package/lib/cli/translate-pipeline.d.ts +144 -0
- package/lib/cli/translate-pipeline.js +489 -0
- package/lib/cli/translate-repair-agent.d.ts +69 -0
- package/lib/cli/translate-repair-agent.js +241 -0
- package/lib/cli/translate-scratch.d.ts +9 -1
- package/lib/cli/translate-scratch.js +2 -2
- package/lib/cli/translate-segment-cache.d.ts +90 -0
- package/lib/cli/translate-segment-cache.js +301 -0
- package/lib/cli/translate-segment-check.d.ts +42 -0
- package/lib/cli/translate-segment-check.js +247 -0
- package/lib/cli/translate-segment.d.ts +307 -0
- package/lib/cli/translate-segment.js +625 -0
- package/lib/cli/translate.d.ts +14 -4
- package/lib/cli/translate.js +232 -52
- package/lib/remark-lint/no-unmatched-anchor.js +6 -2
- package/lib/remark-lint/no-unopened-frontmatter.js +5 -1
- package/lib/remark-lint/rule-sets.d.ts +1 -3
- package/lib/remark-lint/rule-sets.js +3 -9
- package/lib/remark-lint/syntax-plugins.d.ts +32 -0
- package/lib/remark-lint/syntax-plugins.js +46 -0
- package/lib/remark-lint/translation-parity/frontmatter-preservation.js +5 -3
- package/lib/remark-lint/translation-parity/link-isomorphism.d.ts +8 -0
- package/lib/remark-lint/translation-parity/link-isomorphism.js +40 -4
- package/lib/remark-lint/translation-parity/shared.js +6 -2
- package/lib/remark-lint/translation-parity/url-residue.d.ts +1 -0
- package/lib/remark-lint/translation-parity/url-residue.js +25 -4
- package/lib/runtime/hooks/useTranslation.d.ts +1 -1
- package/lib/types.d.ts +101 -6
- package/package.json +1 -1
- package/lib/cli/translate-agent.d.ts +0 -81
- package/lib/cli/translate-agent.js +0 -450
|
@@ -5,8 +5,8 @@ import picomatch from 'picomatch';
|
|
|
5
5
|
import { visit } from 'unist-util-visit';
|
|
6
6
|
import { parse as parseYaml } from 'yaml';
|
|
7
7
|
import { escapeMarkdownHeadingIds } from "../../cli/helpers.js";
|
|
8
|
-
import { mdProcessor, mdxProcessor } from "../../plugins/index.js";
|
|
9
8
|
import { DEFAULT_COPY_ONLY_DIRECTORIES, SUPPORTED_LANGUAGES, } from "../../shared/index.js";
|
|
9
|
+
import { syntaxProcessor } from "../syntax-plugins.js";
|
|
10
10
|
import { getConfig } from "../utils.js";
|
|
11
11
|
const matcherCache = new Map();
|
|
12
12
|
/** Memoised, because it is asked once per document and the patterns never change. */
|
|
@@ -36,7 +36,11 @@ const loadSource = async (sourcePath) => {
|
|
|
36
36
|
if (cached?.sha === sha) {
|
|
37
37
|
return cached;
|
|
38
38
|
}
|
|
39
|
-
|
|
39
|
+
// The same syntax stack the document being linted came through: a
|
|
40
|
+
// comparison between two differently-parsed trees measures the parsers.
|
|
41
|
+
const processor = sourcePath.endsWith('.mdx')
|
|
42
|
+
? syntaxProcessor.mdx
|
|
43
|
+
: syntaxProcessor.md;
|
|
40
44
|
// The same escaping `doom translate` applies before parsing: without it a
|
|
41
45
|
// custom heading anchor is an MDX expression and the parse fails outright.
|
|
42
46
|
const entry = {
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { lintRule } from 'unified-lint-rule';
|
|
2
|
-
import { visit } from 'unist-util-visit';
|
|
2
|
+
import { SKIP, visit } from 'unist-util-visit';
|
|
3
3
|
import { currentPair, diffMultiset } from "./shared.js";
|
|
4
4
|
/**
|
|
5
5
|
* URLs written in prose that markdown does not turn into links still survive
|
|
@@ -24,9 +24,30 @@ import { currentPair, diffMultiset } from "./shared.js";
|
|
|
24
24
|
* is a guard for a case nothing else covers, not a demonstrated detector.
|
|
25
25
|
*/
|
|
26
26
|
const BARE_URL = /\b(?:[a-z][a-z0-9+.-]*:\/\/|www\.)\S+/gi;
|
|
27
|
-
|
|
27
|
+
/**
|
|
28
|
+
* Text inside a link is not prose for this purpose.
|
|
29
|
+
*
|
|
30
|
+
* The rule's whole premise is that a URL markdown *did* turn into a link is
|
|
31
|
+
* protected structurally and belongs to `link-isomorphism`. The first
|
|
32
|
+
* implementation still walked into link nodes, so an autolink counted twice:
|
|
33
|
+
* once as a target and once as the label text that repeats it. That is not
|
|
34
|
+
* merely redundant. A translator that writes `[http://host:8080](http://host:8080)`
|
|
35
|
+
* — the one spelling of a bare URL that Chinese sentence punctuation cannot
|
|
36
|
+
* corrupt — had its label read as prose, found the URL missing from it (the
|
|
37
|
+
* port is a `textDirective` there), and was told it had lost a URL it had in
|
|
38
|
+
* fact preserved exactly. No spelling it could reach satisfied both rules, so
|
|
39
|
+
* the repair loop could only run out of rounds.
|
|
40
|
+
*/
|
|
41
|
+
const IS_LINK = new Set(['link', 'linkReference', 'definition']);
|
|
42
|
+
export const collectProseUrls = (tree) => {
|
|
28
43
|
const found = [];
|
|
29
|
-
visit(tree,
|
|
44
|
+
visit(tree, (node) => {
|
|
45
|
+
if (IS_LINK.has(node.type)) {
|
|
46
|
+
return SKIP;
|
|
47
|
+
}
|
|
48
|
+
if (node.type !== 'text') {
|
|
49
|
+
return;
|
|
50
|
+
}
|
|
30
51
|
for (const [match] of node.value.matchAll(BARE_URL)) {
|
|
31
52
|
found.push(match);
|
|
32
53
|
}
|
|
@@ -38,7 +59,7 @@ export const translationUrlResidue = lintRule('doom-lint:translation-url-residue
|
|
|
38
59
|
if (!pair) {
|
|
39
60
|
return;
|
|
40
61
|
}
|
|
41
|
-
const { missing } = diffMultiset(
|
|
62
|
+
const { missing } = diffMultiset(collectProseUrls(pair.sourceTree), collectProseUrls(tree));
|
|
42
63
|
if (!missing.length) {
|
|
43
64
|
return;
|
|
44
65
|
}
|
|
@@ -1,3 +1,3 @@
|
|
|
1
1
|
import type { Language } from '../../shared/index.js';
|
|
2
2
|
export declare const useLang: () => Language;
|
|
3
|
-
export declare const useTranslation: () => (key: "function" | "name" | "description" | "delete" | "type" | "parameters" | "required" | "toc" | "create" | "update" | "password" | "
|
|
3
|
+
export declare const useTranslation: () => (key: "function" | "name" | "description" | "delete" | "type" | "parameters" | "required" | "toc" | "create" | "update" | "password" | "close" | "collapse_all" | "expand_all" | "crd_no_schema" | "action" | "view" | "chinese" | "english" | "property" | "specification" | "api_endpoints" | "available_api_endpoints" | "http_method" | "http_responses" | "http_code" | "response_body" | "query_parameters" | "body_parameters" | "parameter" | "common_parameters" | "request_body" | "response" | "chinese_bad_cases" | "english_bad_cases" | "download_pdf" | "view_docs_as_pdf" | "ai_assistant" | "exclamation" | "semicolon" | "period" | "colon" | "user_login" | "user_login_tip1" | "user_login_tip2" | "customer_portal_global" | "customer_portal_china" | "account_id" | "username" | "captcha" | "login" | "current_field" | "field_required" | "ERR_INTERNET_DISCONNECTED" | "AccountDisabledError" | "CaptchaError" | "LoginError" | "NetworkError" | "TenantError" | "TenantNotFoundError" | "logout" | "hi_there" | "ai_assistant_tip" | "ai_assistant_placeholder" | "max_chars_exceeded_tip" | "thinking" | "thinking_process" | "referenced_doc_links" | "view_more_related_docs" | "view_less_related_docs" | "new_chat" | "show_more" | "show_less" | "products" | "all_product_documentation" | "explore_doc_all_products") => string;
|
package/lib/types.d.ts
CHANGED
|
@@ -32,15 +32,106 @@ export interface TranslateOptions {
|
|
|
32
32
|
* then to the version this was measured against.
|
|
33
33
|
*/
|
|
34
34
|
model?: string;
|
|
35
|
-
/**
|
|
35
|
+
/**
|
|
36
|
+
* Reasoning effort for translating a segment. Defaults to `low`.
|
|
37
|
+
*
|
|
38
|
+
* The main path is deliberately cheap: a segment is a small, well-specified
|
|
39
|
+
* job, and the reasoning is spent where it earns something — on repairing a
|
|
40
|
+
* segment that three attempts could not fix, and on explaining a failure.
|
|
41
|
+
*/
|
|
36
42
|
reasoningEffort?: 'minimal' | 'low' | 'medium' | 'high' | 'xhigh' | 'max';
|
|
37
43
|
/**
|
|
38
|
-
*
|
|
39
|
-
*
|
|
44
|
+
* Largest segment sent to the model, in masked characters. Defaults to 16000.
|
|
45
|
+
*
|
|
46
|
+
* The single most important number here: it bounds how much has to be right
|
|
47
|
+
* at once. Raising it trades a lower chance of any one segment passing for
|
|
48
|
+
* fewer calls; lowering it costs context and coherence.
|
|
49
|
+
*/
|
|
50
|
+
segmentCap?: number;
|
|
51
|
+
/**
|
|
52
|
+
* Size at which an indivisible block fails the document. Defaults to 48000.
|
|
53
|
+
*
|
|
54
|
+
* A block over `segmentCap` that cannot be divided — a huge table or list —
|
|
55
|
+
* becomes an oversized segment of its own. This is where that stops being
|
|
56
|
+
* reasonable, and the document has to be edited instead.
|
|
57
|
+
*/
|
|
58
|
+
segmentHardCap?: number;
|
|
59
|
+
/**
|
|
60
|
+
* Size a segment must reach before a heading may end it. Defaults to 2000.
|
|
61
|
+
*
|
|
62
|
+
* Without it, a page of one-line sections becomes a page of one-line
|
|
63
|
+
* segments, each its own model call and judge reading for no benefit.
|
|
64
|
+
*/
|
|
65
|
+
segmentFloor?: number;
|
|
66
|
+
/**
|
|
67
|
+
* How many times a segment is asked for before it is escalated to the repair
|
|
68
|
+
* agent. Defaults to 3 — the first go plus two with the findings attached.
|
|
69
|
+
*/
|
|
70
|
+
maxSegmentAttempts?: number;
|
|
71
|
+
/**
|
|
72
|
+
* How many times the assembled document may send segments back to be redone.
|
|
73
|
+
* Defaults to 2.
|
|
74
|
+
*/
|
|
75
|
+
maxAssemblyRounds?: number;
|
|
76
|
+
/**
|
|
77
|
+
* Lines of the previous segment's translation shown to the next one, so the
|
|
78
|
+
* voice and the wording carry across a boundary. Defaults to 20; `0` turns it
|
|
79
|
+
* off.
|
|
80
|
+
*/
|
|
81
|
+
contextTail?: number;
|
|
82
|
+
/**
|
|
83
|
+
* Reuse the segments whose source has not changed, from the translation
|
|
84
|
+
* already on disk. Defaults to `true`.
|
|
85
|
+
*
|
|
86
|
+
* Turning it off makes every changed document a full retranslation. Nothing
|
|
87
|
+
* is trusted unverified either way: a reused segment has to still account for
|
|
88
|
+
* exactly the content the current source expects, or it is retranslated.
|
|
89
|
+
*/
|
|
90
|
+
segmentCache?: boolean;
|
|
91
|
+
/**
|
|
92
|
+
* Read the assembled page as a whole, after the segments have passed.
|
|
93
|
+
* Defaults to `true`.
|
|
94
|
+
*
|
|
95
|
+
* Never blocking. It is there for what a segment cannot see — a term that
|
|
96
|
+
* drifted between sections, a join that reads badly — and those belong in a
|
|
97
|
+
* report, not in a gate.
|
|
98
|
+
*/
|
|
99
|
+
fullDocJudge?: boolean;
|
|
100
|
+
/**
|
|
101
|
+
* The last resort for a segment that repeated attempts could not fix.
|
|
102
|
+
*
|
|
103
|
+
* An agent with `read`, `edit` and `check` and deliberately no way to replace
|
|
104
|
+
* a file: it works inside one segment, and the reason it cannot rewrite
|
|
105
|
+
* everything is the tools it has rather than the instructions it is given.
|
|
40
106
|
*/
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
107
|
+
repairAgent?: {
|
|
108
|
+
/** Defaults to `true`. */
|
|
109
|
+
enabled?: boolean;
|
|
110
|
+
/** Defaults to the translator's model. Set it to the strongest one available. */
|
|
111
|
+
model?: string;
|
|
112
|
+
/** Defaults to `high` — this path is rare and hard, which is what reasoning is for. */
|
|
113
|
+
reasoningEffort?: 'minimal' | 'low' | 'medium' | 'high' | 'xhigh' | 'max';
|
|
114
|
+
/**
|
|
115
|
+
* Runaway guard, in agent turns. Defaults to 40.
|
|
116
|
+
*
|
|
117
|
+
* A fixed number works here and did not for whole documents: a segment's
|
|
118
|
+
* size has an upper bound, so the ratio of budget to work is a constant.
|
|
119
|
+
*/
|
|
120
|
+
maxTurns?: number;
|
|
121
|
+
};
|
|
122
|
+
/**
|
|
123
|
+
* The failure analysis attached to a document that could not be translated.
|
|
124
|
+
*
|
|
125
|
+
* One model call over the evidence already collected, written for whoever
|
|
126
|
+
* reads the build log. Purely advisory: it never changes whether the build
|
|
127
|
+
* passes.
|
|
128
|
+
*/
|
|
129
|
+
diagnose?: {
|
|
130
|
+
/** Defaults to `true`. */
|
|
131
|
+
enabled?: boolean;
|
|
132
|
+
/** Defaults to `high`. */
|
|
133
|
+
reasoningEffort?: 'minimal' | 'low' | 'medium' | 'high' | 'xhigh' | 'max';
|
|
134
|
+
};
|
|
44
135
|
/**
|
|
45
136
|
* Context the translator plans for, in tokens. Deliberately lower than what
|
|
46
137
|
* the gateway offers: planning for less costs a re-read, planning for more
|
|
@@ -76,6 +167,10 @@ export interface TranslateOptions {
|
|
|
76
167
|
reasoningEffort?: 'minimal' | 'low' | 'medium' | 'high' | 'xhigh' | 'max';
|
|
77
168
|
/** How many independent readings must agree before a finding counts. Defaults to 2. */
|
|
78
169
|
draws?: number;
|
|
170
|
+
/** Readings taken of a single segment. Defaults to 3. */
|
|
171
|
+
segmentDraws?: number;
|
|
172
|
+
/** How many of those must agree before a segment finding counts. Defaults to 2. */
|
|
173
|
+
segmentVotes?: number;
|
|
79
174
|
/**
|
|
80
175
|
* Turns the judge off.
|
|
81
176
|
*
|
package/package.json
CHANGED
|
@@ -1,81 +0,0 @@
|
|
|
1
|
-
import type { Model, Models, ThinkingLevel } from '@earendil-works/pi-ai';
|
|
2
|
-
import { Language } from '../shared/index.ts';
|
|
3
|
-
import { type TranslationChecker, type TranslationFinding } from './translate-checker.ts';
|
|
4
|
-
import type { Judge } from './translate-judge.ts';
|
|
5
|
-
import { type MaskEntry, type MaskProcessor } from './translate-mask.ts';
|
|
6
|
-
import type { TermPair } from './translate-terms.ts';
|
|
7
|
-
/**
|
|
8
|
-
* Translating one document, as a loop that ends when the document passes its
|
|
9
|
-
* checks.
|
|
10
|
-
*
|
|
11
|
-
* The loop is pi's. What is ours is its boundary: the tools the agent has, the
|
|
12
|
-
* directory it can reach, what is released from its context, and — the part
|
|
13
|
-
* everything else rests on — the condition under which it is allowed to stop.
|
|
14
|
-
*
|
|
15
|
-
* `getFollowUpMessages` is pi's "the agent would finish here" hook: whatever it
|
|
16
|
-
* returns is appended to the context and the agent takes another turn. Putting
|
|
17
|
-
* the harness's own check there means the agent cannot finish while the
|
|
18
|
-
* document still fails it. Not "we check afterwards and start it again if it
|
|
19
|
-
* failed" — the check *is* the exit condition, and the agent has no name for
|
|
20
|
-
* it, no way to call it, and no way around it.
|
|
21
|
-
*
|
|
22
|
-
* The turn cap is enforced separately, in `shouldStopAfterTurn`, which pi
|
|
23
|
-
* consults *before* polling for follow-ups. So a run that will not converge
|
|
24
|
-
* ends, and ends failed: there is no third exit where a document is written out
|
|
25
|
-
* anyway.
|
|
26
|
-
*/
|
|
27
|
-
export interface TranslateAgentOptions {
|
|
28
|
-
/** The masked source, exactly as the model will see it. */
|
|
29
|
-
maskedSource: string;
|
|
30
|
-
/** The mask table, for restoring and for counting what came back. */
|
|
31
|
-
maskEntries: readonly MaskEntry[];
|
|
32
|
-
/** Parser/stringifier matching the document's flavour. */
|
|
33
|
-
processor: MaskProcessor;
|
|
34
|
-
/** Merges the restored translation with the frontmatter it will be written with. */
|
|
35
|
-
compose: (restored: string) => string;
|
|
36
|
-
/** Absolute path the translation will be written to. Rules read the language and find the source from it. */
|
|
37
|
-
targetPath: string;
|
|
38
|
-
/** Path of the source, relative to the docs root — for logs only. */
|
|
39
|
-
sourceLabel: string;
|
|
40
|
-
source: Language;
|
|
41
|
-
target: Language;
|
|
42
|
-
/** The translation rules, rendered — the same prompt body the single-shot translator used. */
|
|
43
|
-
translationRules: string;
|
|
44
|
-
/** Terms the source matched, used to keep one document's wording consistent across segments. */
|
|
45
|
-
terms?: readonly TermPair[];
|
|
46
|
-
checker: TranslationChecker;
|
|
47
|
-
/** The semantic check. Absent only in tests that are about control flow. */
|
|
48
|
-
judge?: Judge;
|
|
49
|
-
models: Models;
|
|
50
|
-
model: Model<'openai-completions'>;
|
|
51
|
-
reasoningEffort: ThinkingLevel;
|
|
52
|
-
/** Directory scratch directories are created under. */
|
|
53
|
-
scratchDir: string;
|
|
54
|
-
/** How many times findings may be fed back before the document is failed. */
|
|
55
|
-
maxRepairRounds: number;
|
|
56
|
-
/** Runaway guard: total assistant turns, tool calls included. */
|
|
57
|
-
maxTurns: number;
|
|
58
|
-
/** Applied to every model call, so an agent's extra turns count against the same budget. */
|
|
59
|
-
limit: <T>(run: () => Promise<T>) => Promise<T>;
|
|
60
|
-
/** How many times a refused request is retried before the document fails. */
|
|
61
|
-
maxModelRetries?: number;
|
|
62
|
-
/** First backoff after a refused request, doubled per attempt. */
|
|
63
|
-
modelRetryDelayMs?: number;
|
|
64
|
-
onProgress?: (message: string) => void;
|
|
65
|
-
}
|
|
66
|
-
export interface TranslateAgentResult {
|
|
67
|
-
/** The finished document, when it passed. */
|
|
68
|
-
document?: string;
|
|
69
|
-
/**
|
|
70
|
-
* What was still wrong when the run ended.
|
|
71
|
-
*
|
|
72
|
-
* `document` is set exactly when none of these block. A passing document can
|
|
73
|
-
* still carry advisory findings — the judge's readability notes — which are
|
|
74
|
-
* worth printing and are not worth failing a build over.
|
|
75
|
-
*/
|
|
76
|
-
findings: TranslationFinding[];
|
|
77
|
-
turns: number;
|
|
78
|
-
repairRounds: number;
|
|
79
|
-
modelCalls: number;
|
|
80
|
-
}
|
|
81
|
-
export declare const translateWithAgent: (options: TranslateAgentOptions) => Promise<TranslateAgentResult>;
|