@alauda/doom 1.3.0 → 1.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/cli/translate.d.ts +3 -1
- package/lib/cli/translate.js +55 -2
- package/package.json +1 -1
package/lib/cli/translate.d.ts
CHANGED
|
@@ -16,8 +16,10 @@ export interface InternalTranslateOptions extends TranslateOptions {
|
|
|
16
16
|
sourceContent: string;
|
|
17
17
|
target: Language;
|
|
18
18
|
additionalPrompts?: string;
|
|
19
|
+
isChunk?: boolean;
|
|
19
20
|
}
|
|
20
|
-
export declare const
|
|
21
|
+
export declare const translateWithChunks: (options: InternalTranslateOptions) => Promise<string>;
|
|
22
|
+
export declare const translate: ({ source, sourceContent, target, systemPrompt, userPrompt, additionalPrompts, isChunk, }: InternalTranslateOptions) => Promise<string>;
|
|
21
23
|
export interface TranslateCommandOptions {
|
|
22
24
|
source: Language;
|
|
23
25
|
target: Language;
|
package/lib/cli/translate.js
CHANGED
|
@@ -61,6 +61,10 @@ You are a professional technical documentation engineer, skilled in writing high
|
|
|
61
61
|
<% if (terms) { %>
|
|
62
62
|
<%- terms %>
|
|
63
63
|
<% } %>
|
|
64
|
+
<% if (isChunk) { %>
|
|
65
|
+
## Chunk Translation Notice
|
|
66
|
+
This is part of a larger document that has been split into smaller chunks for translation. Please translate this chunk as if it's part of a continuous document, maintaining consistency with the overall document style and context.
|
|
67
|
+
<% } %>
|
|
64
68
|
|
|
65
69
|
<% if (userPrompt || additionalPrompts) { %>
|
|
66
70
|
## Additional Requirements
|
|
@@ -147,7 +151,55 @@ function getTitleTranslation(title, sourceLang, targetLang) {
|
|
|
147
151
|
}
|
|
148
152
|
return null;
|
|
149
153
|
}
|
|
150
|
-
|
|
154
|
+
function splitContentIntoChunks(content, maxChunkSize) {
|
|
155
|
+
const lines = content.split('\n');
|
|
156
|
+
const chunks = [];
|
|
157
|
+
let currentChunk = [];
|
|
158
|
+
let currentSize = 0;
|
|
159
|
+
for (const line of lines) {
|
|
160
|
+
const lineSize = Buffer.byteLength(line + '\n', 'utf8');
|
|
161
|
+
// If adding this line would exceed the chunk size, and we have content in current chunk
|
|
162
|
+
if (currentSize + lineSize > maxChunkSize && currentChunk.length > 0) {
|
|
163
|
+
chunks.push(currentChunk.join('\n'));
|
|
164
|
+
currentChunk = [line];
|
|
165
|
+
currentSize = lineSize;
|
|
166
|
+
}
|
|
167
|
+
else {
|
|
168
|
+
currentChunk.push(line);
|
|
169
|
+
currentSize += lineSize;
|
|
170
|
+
}
|
|
171
|
+
}
|
|
172
|
+
// Add the last chunk if it has content
|
|
173
|
+
if (currentChunk.length > 0) {
|
|
174
|
+
chunks.push(currentChunk.join('\n'));
|
|
175
|
+
}
|
|
176
|
+
return chunks;
|
|
177
|
+
}
|
|
178
|
+
export const translateWithChunks = async (options) => {
|
|
179
|
+
const { sourceContent } = options;
|
|
180
|
+
const maxChunkSize = 60 * 1024;
|
|
181
|
+
const contentSize = Buffer.byteLength(sourceContent, 'utf8');
|
|
182
|
+
if (contentSize <= maxChunkSize) {
|
|
183
|
+
return translate(options);
|
|
184
|
+
}
|
|
185
|
+
logger.info(`Content size (${Math.round(contentSize / 1024)}KB) exceeds limit, splitting into chunks...`);
|
|
186
|
+
const chunks = splitContentIntoChunks(sourceContent, maxChunkSize);
|
|
187
|
+
logger.info(`Split content into ${chunks.length} chunks`);
|
|
188
|
+
const translatedChunks = [];
|
|
189
|
+
for (let i = 0; i < chunks.length; i++) {
|
|
190
|
+
logger.info(`Translating chunk ${i + 1}/${chunks.length}...`);
|
|
191
|
+
const translatedChunk = await translate({
|
|
192
|
+
...options,
|
|
193
|
+
sourceContent: chunks[i],
|
|
194
|
+
isChunk: true,
|
|
195
|
+
});
|
|
196
|
+
translatedChunks.push(translatedChunk);
|
|
197
|
+
}
|
|
198
|
+
const result = translatedChunks.join('\n');
|
|
199
|
+
logger.info(`Successfully translated ${chunks.length} chunks`);
|
|
200
|
+
return result;
|
|
201
|
+
};
|
|
202
|
+
export const translate = async ({ source, sourceContent, target, systemPrompt, userPrompt = '', additionalPrompts = '', isChunk = false, }) => {
|
|
151
203
|
if (!openai) {
|
|
152
204
|
openai = new AzureOpenAI({
|
|
153
205
|
endpoint: process.env.AZURE_OPENAI_ENDPOINT ||
|
|
@@ -178,6 +230,7 @@ export const translate = async ({ source, sourceContent, target, systemPrompt, u
|
|
|
178
230
|
additionalPrompts: additionalPrompts,
|
|
179
231
|
terms,
|
|
180
232
|
titleTranslationPrompt,
|
|
233
|
+
isChunk,
|
|
181
234
|
}, { async: true });
|
|
182
235
|
logger.debug('Final system prompt:\n', finalSystemPrompt);
|
|
183
236
|
const { choices } = await openai.chat.completions.parse({
|
|
@@ -337,7 +390,7 @@ export const translateCommand = new Command('translate')
|
|
|
337
390
|
...ast,
|
|
338
391
|
children: ast.children.map((it) => normalizeImgSrc(it, normalizeImgSrcOptions)),
|
|
339
392
|
});
|
|
340
|
-
targetContent = await
|
|
393
|
+
targetContent = await translateWithChunks({
|
|
341
394
|
...config.translate,
|
|
342
395
|
source,
|
|
343
396
|
sourceContent: normalizedSourceContent,
|