glotto 2.9.0 → 3.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +86 -55
- package/esm/cli.js +47 -39
- package/esm/deno.d.ts +0 -1
- package/esm/deno.js +1 -2
- package/esm/src/contants.d.ts +2 -2
- package/esm/src/contants.d.ts.map +1 -1
- package/esm/src/contants.js +22 -12
- package/esm/src/file.d.ts +2 -7
- package/esm/src/file.d.ts.map +1 -1
- package/esm/src/file.js +1 -108
- package/esm/src/providers/anthropic.d.ts +6 -11
- package/esm/src/providers/anthropic.d.ts.map +1 -1
- package/esm/src/providers/anthropic.js +14 -108
- package/esm/src/providers/gemini.d.ts +6 -11
- package/esm/src/providers/gemini.d.ts.map +1 -1
- package/esm/src/providers/gemini.js +13 -114
- package/esm/src/providers/openai.d.ts +6 -11
- package/esm/src/providers/openai.d.ts.map +1 -1
- package/esm/src/providers/openai.js +10 -109
- package/esm/src/translator.d.ts +8 -0
- package/esm/src/translator.d.ts.map +1 -0
- package/esm/src/translator.js +200 -0
- package/esm/src/types.d.ts +28 -11
- package/esm/src/types.d.ts.map +1 -1
- package/esm/src/utilites.d.ts +0 -6
- package/esm/src/utilites.d.ts.map +1 -1
- package/esm/src/utilites.js +18 -132
- package/package.json +1 -1
- package/script/cli.js +45 -37
- package/script/deno.d.ts +0 -1
- package/script/deno.js +1 -2
- package/script/src/contants.d.ts +2 -2
- package/script/src/contants.d.ts.map +1 -1
- package/script/src/contants.js +23 -13
- package/script/src/file.d.ts +2 -7
- package/script/src/file.d.ts.map +1 -1
- package/script/src/file.js +2 -114
- package/script/src/providers/anthropic.d.ts +6 -11
- package/script/src/providers/anthropic.d.ts.map +1 -1
- package/script/src/providers/anthropic.js +13 -107
- package/script/src/providers/gemini.d.ts +6 -11
- package/script/src/providers/gemini.d.ts.map +1 -1
- package/script/src/providers/gemini.js +12 -113
- package/script/src/providers/openai.d.ts +6 -11
- package/script/src/providers/openai.d.ts.map +1 -1
- package/script/src/providers/openai.js +9 -108
- package/script/src/translator.d.ts +8 -0
- package/script/src/translator.d.ts.map +1 -0
- package/script/src/translator.js +209 -0
- package/script/src/types.d.ts +28 -11
- package/script/src/types.d.ts.map +1 -1
- package/script/src/utilites.d.ts +0 -6
- package/script/src/utilites.d.ts.map +1 -1
- package/script/src/utilites.js +19 -136
- package/esm/deps/jsr.io/@std/encoding/1.0.10/_common16.d.ts +0 -23
- package/esm/deps/jsr.io/@std/encoding/1.0.10/_common16.d.ts.map +0 -1
- package/esm/deps/jsr.io/@std/encoding/1.0.10/_common16.js +0 -51
- package/esm/deps/jsr.io/@std/encoding/1.0.10/_common32.d.ts +0 -35
- package/esm/deps/jsr.io/@std/encoding/1.0.10/_common32.d.ts.map +0 -1
- package/esm/deps/jsr.io/@std/encoding/1.0.10/_common32.js +0 -192
- package/esm/deps/jsr.io/@std/encoding/1.0.10/_common64.d.ts +0 -35
- package/esm/deps/jsr.io/@std/encoding/1.0.10/_common64.d.ts.map +0 -1
- package/esm/deps/jsr.io/@std/encoding/1.0.10/_common64.js +0 -113
- package/esm/deps/jsr.io/@std/encoding/1.0.10/_common_detach.d.ts +0 -4
- package/esm/deps/jsr.io/@std/encoding/1.0.10/_common_detach.d.ts.map +0 -1
- package/esm/deps/jsr.io/@std/encoding/1.0.10/_common_detach.js +0 -13
- package/esm/deps/jsr.io/@std/encoding/1.0.10/_types.d.ts +0 -9
- package/esm/deps/jsr.io/@std/encoding/1.0.10/_types.d.ts.map +0 -1
- package/esm/deps/jsr.io/@std/encoding/1.0.10/_types.js +0 -2
- package/esm/deps/jsr.io/@std/encoding/1.0.10/_validate_binary_like.d.ts +0 -2
- package/esm/deps/jsr.io/@std/encoding/1.0.10/_validate_binary_like.d.ts.map +0 -1
- package/esm/deps/jsr.io/@std/encoding/1.0.10/_validate_binary_like.js +0 -26
- package/esm/deps/jsr.io/@std/encoding/1.0.10/ascii85.d.ts +0 -61
- package/esm/deps/jsr.io/@std/encoding/1.0.10/ascii85.d.ts.map +0 -1
- package/esm/deps/jsr.io/@std/encoding/1.0.10/ascii85.js +0 -152
- package/esm/deps/jsr.io/@std/encoding/1.0.10/base32.d.ts +0 -40
- package/esm/deps/jsr.io/@std/encoding/1.0.10/base32.d.ts.map +0 -1
- package/esm/deps/jsr.io/@std/encoding/1.0.10/base32.js +0 -87
- package/esm/deps/jsr.io/@std/encoding/1.0.10/base58.d.ts +0 -40
- package/esm/deps/jsr.io/@std/encoding/1.0.10/base58.d.ts.map +0 -1
- package/esm/deps/jsr.io/@std/encoding/1.0.10/base58.js +0 -131
- package/esm/deps/jsr.io/@std/encoding/1.0.10/base64.d.ts +0 -40
- package/esm/deps/jsr.io/@std/encoding/1.0.10/base64.d.ts.map +0 -1
- package/esm/deps/jsr.io/@std/encoding/1.0.10/base64.js +0 -82
- package/esm/deps/jsr.io/@std/encoding/1.0.10/base64url.d.ts +0 -40
- package/esm/deps/jsr.io/@std/encoding/1.0.10/base64url.d.ts.map +0 -1
- package/esm/deps/jsr.io/@std/encoding/1.0.10/base64url.js +0 -72
- package/esm/deps/jsr.io/@std/encoding/1.0.10/hex.d.ts +0 -39
- package/esm/deps/jsr.io/@std/encoding/1.0.10/hex.d.ts.map +0 -1
- package/esm/deps/jsr.io/@std/encoding/1.0.10/hex.js +0 -87
- package/esm/deps/jsr.io/@std/encoding/1.0.10/mod.d.ts +0 -98
- package/esm/deps/jsr.io/@std/encoding/1.0.10/mod.d.ts.map +0 -1
- package/esm/deps/jsr.io/@std/encoding/1.0.10/mod.js +0 -99
- package/esm/deps/jsr.io/@std/encoding/1.0.10/varint.d.ts +0 -120
- package/esm/deps/jsr.io/@std/encoding/1.0.10/varint.d.ts.map +0 -1
- package/esm/deps/jsr.io/@std/encoding/1.0.10/varint.js +0 -205
- package/script/deps/jsr.io/@std/encoding/1.0.10/_common16.d.ts +0 -23
- package/script/deps/jsr.io/@std/encoding/1.0.10/_common16.d.ts.map +0 -1
- package/script/deps/jsr.io/@std/encoding/1.0.10/_common16.js +0 -57
- package/script/deps/jsr.io/@std/encoding/1.0.10/_common32.d.ts +0 -35
- package/script/deps/jsr.io/@std/encoding/1.0.10/_common32.d.ts.map +0 -1
- package/script/deps/jsr.io/@std/encoding/1.0.10/_common32.js +0 -198
- package/script/deps/jsr.io/@std/encoding/1.0.10/_common64.d.ts +0 -35
- package/script/deps/jsr.io/@std/encoding/1.0.10/_common64.d.ts.map +0 -1
- package/script/deps/jsr.io/@std/encoding/1.0.10/_common64.js +0 -119
- package/script/deps/jsr.io/@std/encoding/1.0.10/_common_detach.d.ts +0 -4
- package/script/deps/jsr.io/@std/encoding/1.0.10/_common_detach.d.ts.map +0 -1
- package/script/deps/jsr.io/@std/encoding/1.0.10/_common_detach.js +0 -16
- package/script/deps/jsr.io/@std/encoding/1.0.10/_types.d.ts +0 -9
- package/script/deps/jsr.io/@std/encoding/1.0.10/_types.d.ts.map +0 -1
- package/script/deps/jsr.io/@std/encoding/1.0.10/_types.js +0 -3
- package/script/deps/jsr.io/@std/encoding/1.0.10/_validate_binary_like.d.ts +0 -2
- package/script/deps/jsr.io/@std/encoding/1.0.10/_validate_binary_like.d.ts.map +0 -1
- package/script/deps/jsr.io/@std/encoding/1.0.10/_validate_binary_like.js +0 -29
- package/script/deps/jsr.io/@std/encoding/1.0.10/ascii85.d.ts +0 -61
- package/script/deps/jsr.io/@std/encoding/1.0.10/ascii85.d.ts.map +0 -1
- package/script/deps/jsr.io/@std/encoding/1.0.10/ascii85.js +0 -156
- package/script/deps/jsr.io/@std/encoding/1.0.10/base32.d.ts +0 -40
- package/script/deps/jsr.io/@std/encoding/1.0.10/base32.d.ts.map +0 -1
- package/script/deps/jsr.io/@std/encoding/1.0.10/base32.js +0 -91
- package/script/deps/jsr.io/@std/encoding/1.0.10/base58.d.ts +0 -40
- package/script/deps/jsr.io/@std/encoding/1.0.10/base58.d.ts.map +0 -1
- package/script/deps/jsr.io/@std/encoding/1.0.10/base58.js +0 -135
- package/script/deps/jsr.io/@std/encoding/1.0.10/base64.d.ts +0 -40
- package/script/deps/jsr.io/@std/encoding/1.0.10/base64.d.ts.map +0 -1
- package/script/deps/jsr.io/@std/encoding/1.0.10/base64.js +0 -86
- package/script/deps/jsr.io/@std/encoding/1.0.10/base64url.d.ts +0 -40
- package/script/deps/jsr.io/@std/encoding/1.0.10/base64url.d.ts.map +0 -1
- package/script/deps/jsr.io/@std/encoding/1.0.10/base64url.js +0 -76
- package/script/deps/jsr.io/@std/encoding/1.0.10/hex.d.ts +0 -39
- package/script/deps/jsr.io/@std/encoding/1.0.10/hex.d.ts.map +0 -1
- package/script/deps/jsr.io/@std/encoding/1.0.10/hex.js +0 -91
- package/script/deps/jsr.io/@std/encoding/1.0.10/mod.d.ts +0 -98
- package/script/deps/jsr.io/@std/encoding/1.0.10/mod.d.ts.map +0 -1
- package/script/deps/jsr.io/@std/encoding/1.0.10/mod.js +0 -115
- package/script/deps/jsr.io/@std/encoding/1.0.10/varint.d.ts +0 -120
- package/script/deps/jsr.io/@std/encoding/1.0.10/varint.d.ts.map +0 -1
- package/script/deps/jsr.io/@std/encoding/1.0.10/varint.js +0 -211
|
@@ -1,29 +1,7 @@
|
|
|
1
1
|
import Anthropic from '@anthropic-ai/sdk';
|
|
2
|
-
import {
|
|
3
|
-
import { logger } from '../logger.js';
|
|
4
|
-
import { mergeInputs, writeTemp } from '../file.js';
|
|
5
|
-
import { BASE_RETRY_DELAY_MS, DEFAULT_MODELS, INTER_CHUNK_DELAY_MS, MAX_RETRIES } from '../contants.js';
|
|
6
|
-
const decoder = new TextDecoder();
|
|
2
|
+
import { DEFAULT_MODELS } from '../contants.js';
|
|
7
3
|
class AnthropicModel {
|
|
8
|
-
constructor(key,
|
|
9
|
-
Object.defineProperty(this, "chunks", {
|
|
10
|
-
enumerable: true,
|
|
11
|
-
configurable: true,
|
|
12
|
-
writable: true,
|
|
13
|
-
value: void 0
|
|
14
|
-
});
|
|
15
|
-
Object.defineProperty(this, "from", {
|
|
16
|
-
enumerable: true,
|
|
17
|
-
configurable: true,
|
|
18
|
-
writable: true,
|
|
19
|
-
value: void 0
|
|
20
|
-
});
|
|
21
|
-
Object.defineProperty(this, "to", {
|
|
22
|
-
enumerable: true,
|
|
23
|
-
configurable: true,
|
|
24
|
-
writable: true,
|
|
25
|
-
value: void 0
|
|
26
|
-
});
|
|
4
|
+
constructor(key, baseUrl, modelName, options = { noLimit: false, noTimeout: false }) {
|
|
27
5
|
Object.defineProperty(this, "client", {
|
|
28
6
|
enumerable: true,
|
|
29
7
|
configurable: true,
|
|
@@ -36,96 +14,24 @@ class AnthropicModel {
|
|
|
36
14
|
writable: true,
|
|
37
15
|
value: void 0
|
|
38
16
|
});
|
|
39
|
-
this.chunks = chunks;
|
|
40
|
-
this.from = from;
|
|
41
|
-
this.to = to;
|
|
42
17
|
this.model = modelName ?? DEFAULT_MODELS.anthropic;
|
|
43
18
|
this.client = new Anthropic({
|
|
44
19
|
apiKey: key,
|
|
45
20
|
baseURL: baseUrl,
|
|
21
|
+
timeout: options.noTimeout ? 0 : undefined,
|
|
46
22
|
});
|
|
47
23
|
}
|
|
48
|
-
async
|
|
49
|
-
const
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
model: this.model,
|
|
60
|
-
max_tokens: 4096,
|
|
61
|
-
messages: [
|
|
62
|
-
{
|
|
63
|
-
role: 'user',
|
|
64
|
-
content: `${systemPrompt}\n\n${userPrompt}\n\nSOURCE_JSON:\n${sourceJson}`,
|
|
65
|
-
},
|
|
66
|
-
],
|
|
67
|
-
});
|
|
68
|
-
const textParts = message.content
|
|
69
|
-
.filter((part) => part.type === 'text')
|
|
70
|
-
.map((part) => part.text);
|
|
71
|
-
const text = textParts.join('').trim();
|
|
72
|
-
if (!text) {
|
|
73
|
-
logger.error(`${chunkLabel} Empty response from Anthropic`);
|
|
74
|
-
continue;
|
|
75
|
-
}
|
|
76
|
-
const cleanedText = stripJsonMarkdown(text);
|
|
77
|
-
const tempJsonFileName = `chunk_${chunk.index + 1}.json`;
|
|
78
|
-
await writeTemp(this.to, tempJsonFileName, cleanedText);
|
|
79
|
-
if (!isValidJson(cleanedText)) {
|
|
80
|
-
if (attempt === MAX_RETRIES) {
|
|
81
|
-
logger.error(`${chunkLabel} Failed after ${MAX_RETRIES + 1} attempts: Invalid JSON response`);
|
|
82
|
-
return null;
|
|
83
|
-
}
|
|
84
|
-
logger.error(`${chunkLabel} Invalid JSON response, saved to ${tempJsonFileName}`);
|
|
85
|
-
continue;
|
|
86
|
-
}
|
|
87
|
-
logger.info(`${chunkLabel} Translated successfully (${formatBytes(chunk.byteSize)}, ${chunk.keyCount} keys)`);
|
|
88
|
-
return cleanedText;
|
|
89
|
-
}
|
|
90
|
-
catch (error) {
|
|
91
|
-
const isLastAttempt = attempt === MAX_RETRIES;
|
|
92
|
-
const errorMessage = error instanceof Error ? error.message : String(error);
|
|
93
|
-
if (isLastAttempt) {
|
|
94
|
-
logger.error(`${chunkLabel} Failed after ${MAX_RETRIES + 1} attempts: ${errorMessage}`);
|
|
95
|
-
return null;
|
|
96
|
-
}
|
|
97
|
-
logger.warn(`${chunkLabel} Attempt ${attempt + 1} failed: ${errorMessage}`);
|
|
98
|
-
}
|
|
99
|
-
}
|
|
100
|
-
return null;
|
|
101
|
-
}
|
|
102
|
-
async translate() {
|
|
103
|
-
const { systemPrompt, userPrompt } = generatePrompts(this.from, this.to);
|
|
104
|
-
const totalChunks = this.chunks.length;
|
|
105
|
-
logger.info(`Starting translation: ${totalChunks} chunk(s) to process`);
|
|
106
|
-
const results = [];
|
|
107
|
-
const failedChunks = [];
|
|
108
|
-
for (let i = 0; i < totalChunks; i++) {
|
|
109
|
-
const chunk = this.chunks[i];
|
|
110
|
-
if (i > 0) {
|
|
111
|
-
logger.info(`Waiting ${INTER_CHUNK_DELAY_MS}ms before next chunk (rate limit protection)...`);
|
|
112
|
-
await delay(INTER_CHUNK_DELAY_MS);
|
|
113
|
-
}
|
|
114
|
-
const result = await this.translateChunk(chunk, systemPrompt, userPrompt);
|
|
115
|
-
if (result) {
|
|
116
|
-
results.push(result);
|
|
117
|
-
}
|
|
118
|
-
else {
|
|
119
|
-
failedChunks.push(chunk.index + 1);
|
|
120
|
-
}
|
|
121
|
-
}
|
|
122
|
-
if (failedChunks.length > 0) {
|
|
123
|
-
throw new Error(`Translation failed for chunk(s): ${failedChunks.join(', ')}. Check temp files for partial results.`);
|
|
124
|
-
}
|
|
125
|
-
logger.info(`All ${totalChunks} chunk(s) translated successfully, merging results...`);
|
|
126
|
-
const jsonInputs = results.map((r) => JSON.parse(r));
|
|
127
|
-
const mergedContent = mergeInputs(jsonInputs);
|
|
128
|
-
return JSON.stringify(mergedContent, null, 2);
|
|
24
|
+
async translate(prompt) {
|
|
25
|
+
const message = await this.client.messages.create({
|
|
26
|
+
model: this.model,
|
|
27
|
+
max_tokens: 8192,
|
|
28
|
+
temperature: 0.2,
|
|
29
|
+
messages: [{ role: 'user', content: prompt }],
|
|
30
|
+
});
|
|
31
|
+
return message.content
|
|
32
|
+
.filter((part) => part.type === 'text')
|
|
33
|
+
.map((part) => part.text)
|
|
34
|
+
.join('');
|
|
129
35
|
}
|
|
130
36
|
}
|
|
131
37
|
export default AnthropicModel;
|
|
@@ -1,14 +1,9 @@
|
|
|
1
|
-
import {
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
genAI: GoogleGenAI;
|
|
8
|
-
model: string;
|
|
9
|
-
constructor(key: string, chunks: ChunkInfo[], from: string, to: string, modelName?: string);
|
|
10
|
-
private translateChunk;
|
|
11
|
-
translate(): Promise<string>;
|
|
1
|
+
import type { TextTranslator, TranslateOptions } from '../types.js';
|
|
2
|
+
declare class Gemini implements TextTranslator {
|
|
3
|
+
private genAI;
|
|
4
|
+
private model;
|
|
5
|
+
constructor(key: string, modelName?: string, options?: TranslateOptions);
|
|
6
|
+
translate(prompt: string): Promise<string>;
|
|
12
7
|
}
|
|
13
8
|
export default Gemini;
|
|
14
9
|
//# sourceMappingURL=gemini.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"gemini.d.ts","sourceRoot":"","sources":["../../../src/src/providers/gemini.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"gemini.d.ts","sourceRoot":"","sources":["../../../src/src/providers/gemini.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,cAAc,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAC;AAGpE,cAAM,MAAO,YAAW,cAAc;IACpC,OAAO,CAAC,KAAK,CAAc;IAC3B,OAAO,CAAC,KAAK,CAAS;gBAGpB,GAAG,EAAE,MAAM,EACX,SAAS,CAAC,EAAE,MAAM,EAClB,OAAO,GAAE,gBAAuD;IAS5D,SAAS,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;CAWjD;AAED,eAAe,MAAM,CAAC"}
|
|
@@ -1,29 +1,7 @@
|
|
|
1
1
|
import { GoogleGenAI } from '@google/genai';
|
|
2
|
-
import {
|
|
3
|
-
import { delay, formatBytes, generatePrompts, isValidJson, stripJsonMarkdown } from '../utilites.js';
|
|
4
|
-
import { logger } from '../logger.js';
|
|
5
|
-
import { mergeInputs, writeTemp } from '../file.js';
|
|
6
|
-
import { BASE_RETRY_DELAY_MS, DEFAULT_MODELS, INTER_CHUNK_DELAY_MS, MAX_RETRIES } from '../contants.js';
|
|
2
|
+
import { DEFAULT_MODELS } from '../contants.js';
|
|
7
3
|
class Gemini {
|
|
8
|
-
constructor(key,
|
|
9
|
-
Object.defineProperty(this, "chunks", {
|
|
10
|
-
enumerable: true,
|
|
11
|
-
configurable: true,
|
|
12
|
-
writable: true,
|
|
13
|
-
value: void 0
|
|
14
|
-
});
|
|
15
|
-
Object.defineProperty(this, "from", {
|
|
16
|
-
enumerable: true,
|
|
17
|
-
configurable: true,
|
|
18
|
-
writable: true,
|
|
19
|
-
value: void 0
|
|
20
|
-
});
|
|
21
|
-
Object.defineProperty(this, "to", {
|
|
22
|
-
enumerable: true,
|
|
23
|
-
configurable: true,
|
|
24
|
-
writable: true,
|
|
25
|
-
value: void 0
|
|
26
|
-
});
|
|
4
|
+
constructor(key, modelName, options = { noLimit: false, noTimeout: false }) {
|
|
27
5
|
Object.defineProperty(this, "genAI", {
|
|
28
6
|
enumerable: true,
|
|
29
7
|
configurable: true,
|
|
@@ -36,101 +14,22 @@ class Gemini {
|
|
|
36
14
|
writable: true,
|
|
37
15
|
value: void 0
|
|
38
16
|
});
|
|
39
|
-
this.chunks = chunks;
|
|
40
|
-
this.from = from;
|
|
41
|
-
this.to = to;
|
|
42
17
|
this.model = modelName ?? DEFAULT_MODELS.gemini;
|
|
43
18
|
this.genAI = new GoogleGenAI({
|
|
44
19
|
apiKey: key,
|
|
20
|
+
httpOptions: options.noTimeout ? { timeout: 2_147_483_647 } : undefined,
|
|
45
21
|
});
|
|
46
22
|
}
|
|
47
|
-
async
|
|
48
|
-
const
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
const part = {
|
|
58
|
-
inlineData: {
|
|
59
|
-
data: fileBase64,
|
|
60
|
-
mimeType: 'text/plain',
|
|
61
|
-
},
|
|
62
|
-
};
|
|
63
|
-
const { text } = await this.genAI.models.generateContent({
|
|
64
|
-
model: this.model,
|
|
65
|
-
contents: [systemPrompt, userPrompt, part],
|
|
66
|
-
config: {
|
|
67
|
-
systemInstruction: systemPrompt,
|
|
68
|
-
candidateCount: 1,
|
|
69
|
-
responseMimeType: 'application/json',
|
|
70
|
-
responseLogprobs: false,
|
|
71
|
-
temperature: 0.3,
|
|
72
|
-
topP: 0.7,
|
|
73
|
-
topK: 20,
|
|
74
|
-
maxOutputTokens: Number.POSITIVE_INFINITY,
|
|
75
|
-
},
|
|
76
|
-
});
|
|
77
|
-
if (!text) {
|
|
78
|
-
logger.error(`${chunkLabel} Empty response from Gemini AI`);
|
|
79
|
-
continue;
|
|
80
|
-
}
|
|
81
|
-
const cleanedText = stripJsonMarkdown(text);
|
|
82
|
-
const tempJsonFileName = `chunk_${chunk.index + 1}.json`;
|
|
83
|
-
await writeTemp(this.to, tempJsonFileName, cleanedText);
|
|
84
|
-
if (!isValidJson(cleanedText)) {
|
|
85
|
-
if (attempt === MAX_RETRIES) {
|
|
86
|
-
logger.error(`${chunkLabel} Failed after ${MAX_RETRIES + 1} attempts: Invalid JSON response`);
|
|
87
|
-
return null;
|
|
88
|
-
}
|
|
89
|
-
logger.error(`${chunkLabel} Invalid JSON response, saved to ${tempJsonFileName}`);
|
|
90
|
-
continue;
|
|
91
|
-
}
|
|
92
|
-
logger.info(`${chunkLabel} Translated successfully (${formatBytes(chunk.byteSize)}, ${chunk.keyCount} keys)`);
|
|
93
|
-
return cleanedText;
|
|
94
|
-
}
|
|
95
|
-
catch (error) {
|
|
96
|
-
const isLastAttempt = attempt === MAX_RETRIES;
|
|
97
|
-
const errorMessage = error instanceof Error ? error.message : String(error);
|
|
98
|
-
if (isLastAttempt) {
|
|
99
|
-
logger.error(`${chunkLabel} Failed after ${MAX_RETRIES + 1} attempts: ${errorMessage}`);
|
|
100
|
-
return null;
|
|
101
|
-
}
|
|
102
|
-
logger.warn(`${chunkLabel} Attempt ${attempt + 1} failed: ${errorMessage}`);
|
|
103
|
-
}
|
|
104
|
-
}
|
|
105
|
-
return null;
|
|
106
|
-
}
|
|
107
|
-
async translate() {
|
|
108
|
-
const { systemPrompt, userPrompt } = generatePrompts(this.from, this.to);
|
|
109
|
-
const totalChunks = this.chunks.length;
|
|
110
|
-
logger.info(`Starting translation: ${totalChunks} chunk(s) to process`);
|
|
111
|
-
const results = [];
|
|
112
|
-
const failedChunks = [];
|
|
113
|
-
for (let i = 0; i < totalChunks; i++) {
|
|
114
|
-
const chunk = this.chunks[i];
|
|
115
|
-
if (i > 0) {
|
|
116
|
-
logger.info(`Waiting ${INTER_CHUNK_DELAY_MS}ms before next chunk (rate limit protection)...`);
|
|
117
|
-
await delay(INTER_CHUNK_DELAY_MS);
|
|
118
|
-
}
|
|
119
|
-
const result = await this.translateChunk(chunk, systemPrompt, userPrompt);
|
|
120
|
-
if (result) {
|
|
121
|
-
results.push(result);
|
|
122
|
-
}
|
|
123
|
-
else {
|
|
124
|
-
failedChunks.push(chunk.index + 1);
|
|
125
|
-
}
|
|
126
|
-
}
|
|
127
|
-
if (failedChunks.length > 0) {
|
|
128
|
-
throw new Error(`Translation failed for chunk(s): ${failedChunks.join(', ')}. Check temp files for partial results.`);
|
|
129
|
-
}
|
|
130
|
-
logger.info(`All ${totalChunks} chunk(s) translated successfully, merging results...`);
|
|
131
|
-
const jsonInputs = results.map((r) => JSON.parse(r));
|
|
132
|
-
const mergedContent = mergeInputs(jsonInputs);
|
|
133
|
-
return JSON.stringify(mergedContent, null, 2);
|
|
23
|
+
async translate(prompt) {
|
|
24
|
+
const { text } = await this.genAI.models.generateContent({
|
|
25
|
+
model: this.model,
|
|
26
|
+
contents: prompt,
|
|
27
|
+
config: {
|
|
28
|
+
candidateCount: 1,
|
|
29
|
+
temperature: 0.2,
|
|
30
|
+
},
|
|
31
|
+
});
|
|
32
|
+
return text ?? '';
|
|
134
33
|
}
|
|
135
34
|
}
|
|
136
35
|
export default Gemini;
|
|
@@ -1,14 +1,9 @@
|
|
|
1
|
-
import
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
client: OpenAI;
|
|
8
|
-
model: string;
|
|
9
|
-
constructor(key: string, chunks: ChunkInfo[], from: string, to: string, baseUrl?: string, modelName?: string);
|
|
10
|
-
private translateChunk;
|
|
11
|
-
translate(): Promise<string>;
|
|
1
|
+
import type { TextTranslator, TranslateOptions } from '../types.js';
|
|
2
|
+
declare class OpenAIModel implements TextTranslator {
|
|
3
|
+
private client;
|
|
4
|
+
private model;
|
|
5
|
+
constructor(key: string, baseUrl?: string, modelName?: string, options?: TranslateOptions);
|
|
6
|
+
translate(prompt: string): Promise<string>;
|
|
12
7
|
}
|
|
13
8
|
export default OpenAIModel;
|
|
14
9
|
//# sourceMappingURL=openai.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"openai.d.ts","sourceRoot":"","sources":["../../../src/src/providers/openai.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"openai.d.ts","sourceRoot":"","sources":["../../../src/src/providers/openai.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,cAAc,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAC;AAGpE,cAAM,WAAY,YAAW,cAAc;IACzC,OAAO,CAAC,MAAM,CAAS;IACvB,OAAO,CAAC,KAAK,CAAS;gBAGpB,GAAG,EAAE,MAAM,EACX,OAAO,CAAC,EAAE,MAAM,EAChB,SAAS,CAAC,EAAE,MAAM,EAClB,OAAO,GAAE,gBAAuD;IAU5D,SAAS,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;CAQjD;AAED,eAAe,WAAW,CAAC"}
|
|
@@ -1,29 +1,7 @@
|
|
|
1
1
|
import OpenAI from 'openai';
|
|
2
|
-
import {
|
|
3
|
-
import { logger } from '../logger.js';
|
|
4
|
-
import { mergeInputs, writeTemp } from '../file.js';
|
|
5
|
-
import { BASE_RETRY_DELAY_MS, DEFAULT_MODELS, INTER_CHUNK_DELAY_MS, MAX_RETRIES } from '../contants.js';
|
|
6
|
-
const decoder = new TextDecoder();
|
|
2
|
+
import { DEFAULT_MODELS } from '../contants.js';
|
|
7
3
|
class OpenAIModel {
|
|
8
|
-
constructor(key,
|
|
9
|
-
Object.defineProperty(this, "chunks", {
|
|
10
|
-
enumerable: true,
|
|
11
|
-
configurable: true,
|
|
12
|
-
writable: true,
|
|
13
|
-
value: void 0
|
|
14
|
-
});
|
|
15
|
-
Object.defineProperty(this, "from", {
|
|
16
|
-
enumerable: true,
|
|
17
|
-
configurable: true,
|
|
18
|
-
writable: true,
|
|
19
|
-
value: void 0
|
|
20
|
-
});
|
|
21
|
-
Object.defineProperty(this, "to", {
|
|
22
|
-
enumerable: true,
|
|
23
|
-
configurable: true,
|
|
24
|
-
writable: true,
|
|
25
|
-
value: void 0
|
|
26
|
-
});
|
|
4
|
+
constructor(key, baseUrl, modelName, options = { noLimit: false, noTimeout: false }) {
|
|
27
5
|
Object.defineProperty(this, "client", {
|
|
28
6
|
enumerable: true,
|
|
29
7
|
configurable: true,
|
|
@@ -36,97 +14,20 @@ class OpenAIModel {
|
|
|
36
14
|
writable: true,
|
|
37
15
|
value: void 0
|
|
38
16
|
});
|
|
39
|
-
this.chunks = chunks;
|
|
40
|
-
this.from = from;
|
|
41
|
-
this.to = to;
|
|
42
17
|
this.model = modelName ?? DEFAULT_MODELS.openai;
|
|
43
18
|
this.client = new OpenAI({
|
|
44
19
|
apiKey: key,
|
|
45
20
|
baseURL: baseUrl,
|
|
21
|
+
timeout: options.noTimeout ? 2_147_483_647 : undefined,
|
|
46
22
|
});
|
|
47
23
|
}
|
|
48
|
-
async
|
|
49
|
-
const
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
logger.warn(`${chunkLabel} Retry ${attempt}/${MAX_RETRIES} after ${backoffMs}ms backoff...`);
|
|
56
|
-
await delay(backoffMs);
|
|
57
|
-
}
|
|
58
|
-
const response = await this.client.responses.create({
|
|
59
|
-
model: this.model,
|
|
60
|
-
input: [
|
|
61
|
-
{
|
|
62
|
-
role: 'user',
|
|
63
|
-
content: [
|
|
64
|
-
{
|
|
65
|
-
type: 'input_text',
|
|
66
|
-
text: `${systemPrompt}\n\n${userPrompt}\n\nSOURCE_JSON:\n${sourceJson}`,
|
|
67
|
-
},
|
|
68
|
-
],
|
|
69
|
-
},
|
|
70
|
-
],
|
|
71
|
-
});
|
|
72
|
-
const text = response.output_text;
|
|
73
|
-
if (!text) {
|
|
74
|
-
logger.error(`${chunkLabel} Empty response from OpenAI`);
|
|
75
|
-
continue;
|
|
76
|
-
}
|
|
77
|
-
const cleanedText = stripJsonMarkdown(text);
|
|
78
|
-
const tempJsonFileName = `chunk_${chunk.index + 1}.json`;
|
|
79
|
-
await writeTemp(this.to, tempJsonFileName, cleanedText);
|
|
80
|
-
if (!isValidJson(cleanedText)) {
|
|
81
|
-
if (attempt === MAX_RETRIES) {
|
|
82
|
-
logger.error(`${chunkLabel} Failed after ${MAX_RETRIES + 1} attempts: Invalid JSON response`);
|
|
83
|
-
return null;
|
|
84
|
-
}
|
|
85
|
-
logger.error(`${chunkLabel} Invalid JSON response, saved to ${tempJsonFileName}`);
|
|
86
|
-
continue;
|
|
87
|
-
}
|
|
88
|
-
logger.info(`${chunkLabel} Translated successfully (${formatBytes(chunk.byteSize)}, ${chunk.keyCount} keys)`);
|
|
89
|
-
return cleanedText;
|
|
90
|
-
}
|
|
91
|
-
catch (error) {
|
|
92
|
-
const isLastAttempt = attempt === MAX_RETRIES;
|
|
93
|
-
const errorMessage = error instanceof Error ? error.message : String(error);
|
|
94
|
-
if (isLastAttempt) {
|
|
95
|
-
logger.error(`${chunkLabel} Failed after ${MAX_RETRIES + 1} attempts: ${errorMessage}`);
|
|
96
|
-
return null;
|
|
97
|
-
}
|
|
98
|
-
logger.warn(`${chunkLabel} Attempt ${attempt + 1} failed: ${errorMessage}`);
|
|
99
|
-
}
|
|
100
|
-
}
|
|
101
|
-
return null;
|
|
102
|
-
}
|
|
103
|
-
async translate() {
|
|
104
|
-
const { systemPrompt, userPrompt } = generatePrompts(this.from, this.to);
|
|
105
|
-
const totalChunks = this.chunks.length;
|
|
106
|
-
logger.info(`Starting translation: ${totalChunks} chunk(s) to process`);
|
|
107
|
-
const results = [];
|
|
108
|
-
const failedChunks = [];
|
|
109
|
-
for (let i = 0; i < totalChunks; i++) {
|
|
110
|
-
const chunk = this.chunks[i];
|
|
111
|
-
if (i > 0) {
|
|
112
|
-
logger.info(`Waiting ${INTER_CHUNK_DELAY_MS}ms before next chunk (rate limit protection)...`);
|
|
113
|
-
await delay(INTER_CHUNK_DELAY_MS);
|
|
114
|
-
}
|
|
115
|
-
const result = await this.translateChunk(chunk, systemPrompt, userPrompt);
|
|
116
|
-
if (result) {
|
|
117
|
-
results.push(result);
|
|
118
|
-
}
|
|
119
|
-
else {
|
|
120
|
-
failedChunks.push(chunk.index + 1);
|
|
121
|
-
}
|
|
122
|
-
}
|
|
123
|
-
if (failedChunks.length > 0) {
|
|
124
|
-
throw new Error(`Translation failed for chunk(s): ${failedChunks.join(', ')}. Check temp files for partial results.`);
|
|
125
|
-
}
|
|
126
|
-
logger.info(`All ${totalChunks} chunk(s) translated successfully, merging results...`);
|
|
127
|
-
const jsonInputs = results.map((r) => JSON.parse(r));
|
|
128
|
-
const mergedContent = mergeInputs(jsonInputs);
|
|
129
|
-
return JSON.stringify(mergedContent, null, 2);
|
|
24
|
+
async translate(prompt) {
|
|
25
|
+
const completion = await this.client.chat.completions.create({
|
|
26
|
+
model: this.model,
|
|
27
|
+
messages: [{ role: 'user', content: prompt }],
|
|
28
|
+
temperature: 0.2,
|
|
29
|
+
});
|
|
30
|
+
return completion.choices[0]?.message?.content ?? '';
|
|
130
31
|
}
|
|
131
32
|
}
|
|
132
33
|
export default OpenAIModel;
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
import type { Batch, JsonValue, Leaf, TextTranslator, TranslateOptions } from './types.js';
|
|
2
|
+
export declare const extractLeaves: (data: JsonValue) => Leaf[];
|
|
3
|
+
export declare const groupIntoBatches: (leaves: Leaf[], maxBytes: number) => Batch[];
|
|
4
|
+
export declare const buildBatchPrompt: (from: string, to: string, leaves: Leaf[]) => string;
|
|
5
|
+
export declare const decodeResponse: (text: string) => Map<number, string>;
|
|
6
|
+
export declare const reconstruct: (allLeaves: Leaf[], translations: Map<number, string>) => JsonValue;
|
|
7
|
+
export declare const runBatches: (batches: Batch[], translator: TextTranslator, from: string, to: string, options: TranslateOptions) => Promise<Map<number, string>>;
|
|
8
|
+
//# sourceMappingURL=translator.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"translator.d.ts","sourceRoot":"","sources":["../../src/src/translator.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,KAAK,EAAE,SAAS,EAAE,IAAI,EAAQ,cAAc,EAAE,gBAAgB,EAAE,MAAM,YAAY,CAAC;AAQjG,eAAO,MAAM,aAAa,GAAI,MAAM,SAAS,KAAG,IAAI,EAiCnD,CAAC;AAQF,eAAO,MAAM,gBAAgB,GAAI,QAAQ,IAAI,EAAE,EAAE,UAAU,MAAM,KAAG,KAAK,EA0BxE,CAAC;AAEF,eAAO,MAAM,gBAAgB,GAAI,MAAM,MAAM,EAAE,IAAI,MAAM,EAAE,QAAQ,IAAI,EAAE,KAAG,MAkB3E,CAAC;AAQF,eAAO,MAAM,cAAc,GAAI,MAAM,MAAM,KAAG,GAAG,CAAC,MAAM,EAAE,MAAM,CAS/D,CAAC;AAgBF,eAAO,MAAM,WAAW,GAAI,WAAW,IAAI,EAAE,EAAE,cAAc,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,KAAG,SAalF,CAAC;AAEF,eAAO,MAAM,UAAU,GACrB,SAAS,KAAK,EAAE,EAChB,YAAY,cAAc,EAC1B,MAAM,MAAM,EACZ,IAAI,MAAM,EACV,SAAS,gBAAgB,KACxB,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAgF7B,CAAC"}
|