glotto 3.0.0 → 3.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +112 -47
- package/esm/cli.js +136 -32
- package/esm/deno.d.ts +5 -0
- package/esm/deno.js +18 -6
- package/esm/src/config.d.ts +4 -0
- package/esm/src/config.d.ts.map +1 -0
- package/esm/src/config.js +95 -0
- package/esm/src/contants.d.ts +4 -0
- package/esm/src/contants.d.ts.map +1 -1
- package/esm/src/contants.js +14 -5
- package/esm/src/diff.d.ts +4 -0
- package/esm/src/diff.d.ts.map +1 -0
- package/esm/src/diff.js +53 -0
- package/esm/src/file.d.ts +4 -3
- package/esm/src/file.d.ts.map +1 -1
- package/esm/src/file.js +23 -5
- package/esm/src/providers/anthropic.d.ts +2 -2
- package/esm/src/providers/anthropic.d.ts.map +1 -1
- package/esm/src/providers/anthropic.js +9 -1
- package/esm/src/providers/gemini.d.ts +2 -2
- package/esm/src/providers/gemini.d.ts.map +1 -1
- package/esm/src/providers/gemini.js +10 -2
- package/esm/src/providers/openai.d.ts +2 -2
- package/esm/src/providers/openai.d.ts.map +1 -1
- package/esm/src/providers/openai.js +9 -1
- package/esm/src/translator.d.ts +14 -7
- package/esm/src/translator.d.ts.map +1 -1
- package/esm/src/translator.js +161 -77
- package/esm/src/types.d.ts +31 -3
- package/esm/src/types.d.ts.map +1 -1
- package/esm/src/utilites.d.ts +3 -4
- package/esm/src/utilites.d.ts.map +1 -1
- package/esm/src/utilites.js +40 -16
- package/package.json +20 -6
- package/schema/glotto.schema.json +87 -0
- package/script/cli.js +134 -30
- package/script/deno.d.ts +5 -0
- package/script/deno.js +18 -6
- package/script/src/config.d.ts +4 -0
- package/script/src/config.d.ts.map +1 -0
- package/script/src/config.js +132 -0
- package/script/src/contants.d.ts +4 -0
- package/script/src/contants.d.ts.map +1 -1
- package/script/src/contants.js +15 -6
- package/script/src/diff.d.ts +4 -0
- package/script/src/diff.d.ts.map +1 -0
- package/script/src/diff.js +57 -0
- package/script/src/file.d.ts +4 -3
- package/script/src/file.d.ts.map +1 -1
- package/script/src/file.js +28 -10
- package/script/src/providers/anthropic.d.ts +2 -2
- package/script/src/providers/anthropic.d.ts.map +1 -1
- package/script/src/providers/anthropic.js +9 -1
- package/script/src/providers/gemini.d.ts +2 -2
- package/script/src/providers/gemini.d.ts.map +1 -1
- package/script/src/providers/gemini.js +10 -2
- package/script/src/providers/openai.d.ts +2 -2
- package/script/src/providers/openai.d.ts.map +1 -1
- package/script/src/providers/openai.js +9 -1
- package/script/src/translator.d.ts +14 -7
- package/script/src/translator.d.ts.map +1 -1
- package/script/src/translator.js +168 -83
- package/script/src/types.d.ts +31 -3
- package/script/src/types.d.ts.map +1 -1
- package/script/src/utilites.d.ts +3 -4
- package/script/src/utilites.d.ts.map +1 -1
- package/script/src/utilites.js +43 -20
package/script/src/translator.js
CHANGED
|
@@ -1,15 +1,24 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.
|
|
3
|
+
exports.extractLeaves = extractLeaves;
|
|
4
|
+
exports.groupIntoBatches = groupIntoBatches;
|
|
5
|
+
exports.buildBatchPrompt = buildBatchPrompt;
|
|
6
|
+
exports.buildSinglePrompt = buildSinglePrompt;
|
|
7
|
+
exports.decodeResponse = decodeResponse;
|
|
8
|
+
exports.setPath = setPath;
|
|
9
|
+
exports.reconstruct = reconstruct;
|
|
10
|
+
exports.runBatches = runBatches;
|
|
4
11
|
const logger_js_1 = require("./logger.js");
|
|
5
12
|
const utilites_js_1 = require("./utilites.js");
|
|
6
13
|
const contants_js_1 = require("./contants.js");
|
|
7
14
|
const encoder = new TextEncoder();
|
|
8
|
-
|
|
9
|
-
|
|
15
|
+
function byteSize(s) {
|
|
16
|
+
return encoder.encode(s).byteLength;
|
|
17
|
+
}
|
|
18
|
+
function extractLeaves(data) {
|
|
10
19
|
const out = [];
|
|
11
20
|
let nextId = 0;
|
|
12
|
-
|
|
21
|
+
function walk(node, path) {
|
|
13
22
|
if (node === null || typeof node !== 'object') {
|
|
14
23
|
const isString = typeof node === 'string';
|
|
15
24
|
out.push({
|
|
@@ -33,18 +42,24 @@ const extractLeaves = (data) => {
|
|
|
33
42
|
out.push({ id: ++nextId, path, value: node, translatable: false });
|
|
34
43
|
return;
|
|
35
44
|
}
|
|
36
|
-
for (const [k, v] of entries)
|
|
45
|
+
for (const [k, v] of entries) {
|
|
37
46
|
walk(v, [...path, k]);
|
|
38
|
-
|
|
47
|
+
}
|
|
48
|
+
}
|
|
39
49
|
walk(data, []);
|
|
40
50
|
return out;
|
|
41
|
-
}
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
51
|
+
}
|
|
52
|
+
function tagOpen(id) {
|
|
53
|
+
return `≪${id}≫`;
|
|
54
|
+
}
|
|
55
|
+
function tagClose(id) {
|
|
56
|
+
return `≪/${id}≫`;
|
|
57
|
+
}
|
|
45
58
|
const TAG_OVERHEAD_CHARS = 8;
|
|
46
|
-
|
|
47
|
-
|
|
59
|
+
function wrapEntry(leaf) {
|
|
60
|
+
return `${tagOpen(leaf.id)}${String(leaf.value)}${tagClose(leaf.id)}`;
|
|
61
|
+
}
|
|
62
|
+
function groupIntoBatches(leaves, maxBytes) {
|
|
48
63
|
const translatable = leaves.filter((l) => l.translatable);
|
|
49
64
|
const batches = [];
|
|
50
65
|
let current = [];
|
|
@@ -66,9 +81,8 @@ const groupIntoBatches = (leaves, maxBytes) => {
|
|
|
66
81
|
batches.push({ index: batches.length, leaves: current, byteSize: currentSize });
|
|
67
82
|
}
|
|
68
83
|
return batches;
|
|
69
|
-
}
|
|
70
|
-
|
|
71
|
-
const buildBatchPrompt = (from, to, leaves) => {
|
|
84
|
+
}
|
|
85
|
+
function buildBatchPrompt(from, to, leaves) {
|
|
72
86
|
const entries = leaves.map(wrapEntry).join('\n');
|
|
73
87
|
return `Translate each tagged entry below from ${from} to ${to}.
|
|
74
88
|
|
|
@@ -86,26 +100,32 @@ What to write back:
|
|
|
86
100
|
|
|
87
101
|
Now translate the following entries:
|
|
88
102
|
${entries}`;
|
|
89
|
-
}
|
|
90
|
-
|
|
103
|
+
}
|
|
104
|
+
function buildSinglePrompt(from, to, value) {
|
|
105
|
+
return `You are a professional ${from} to ${to} translator. Your goal is to accurately convey the meaning and nuances of the original ${from} text while adhering to ${to} grammar, vocabulary, and cultural sensitivities. Preserve every variable ({{name}}, {name}, __VAR__, $t(...), %s, %d), HTML tag, markdown token, escape sequence, and special character exactly as given.
|
|
106
|
+
Produce only the ${to} translation, without any additional explanations, quotes, labels, or commentary. Please translate the following ${from} text into ${to}:
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
${value}`;
|
|
110
|
+
}
|
|
91
111
|
const TAG_PATTERNS = [
|
|
92
112
|
/≪\s*(\d+)\s*≫([\s\S]*?)≪\s*\/\s*\1\s*≫/g,
|
|
93
113
|
/<<\s*(\d+)\s*>>([\s\S]*?)<<\s*\/\s*\1\s*>>/g,
|
|
94
114
|
/«\s*(\d+)\s*»([\s\S]*?)«\s*\/\s*\1\s*»/g,
|
|
95
115
|
];
|
|
96
|
-
|
|
116
|
+
function decodeResponse(text) {
|
|
97
117
|
const map = new Map();
|
|
98
118
|
for (const pattern of TAG_PATTERNS) {
|
|
99
119
|
for (const match of text.matchAll(pattern)) {
|
|
100
120
|
const id = parseInt(match[1], 10);
|
|
101
|
-
if (!map.has(id))
|
|
121
|
+
if (!map.has(id)) {
|
|
102
122
|
map.set(id, match[2].trim());
|
|
123
|
+
}
|
|
103
124
|
}
|
|
104
125
|
}
|
|
105
126
|
return map;
|
|
106
|
-
}
|
|
107
|
-
|
|
108
|
-
const setPath = (root, path, value) => {
|
|
127
|
+
}
|
|
128
|
+
function setPath(root, path, value) {
|
|
109
129
|
let node = root;
|
|
110
130
|
for (let i = 0; i < path.length - 1; i++) {
|
|
111
131
|
const key = path[i];
|
|
@@ -117,10 +137,11 @@ const setPath = (root, path, value) => {
|
|
|
117
137
|
node = container[key];
|
|
118
138
|
}
|
|
119
139
|
node[path[path.length - 1]] = value;
|
|
120
|
-
}
|
|
121
|
-
|
|
122
|
-
if (allLeaves.length === 0)
|
|
140
|
+
}
|
|
141
|
+
function reconstruct(allLeaves, translations) {
|
|
142
|
+
if (allLeaves.length === 0) {
|
|
123
143
|
return {};
|
|
144
|
+
}
|
|
124
145
|
const firstStep = allLeaves[0].path[0];
|
|
125
146
|
if (firstStep === undefined) {
|
|
126
147
|
const only = allLeaves[0];
|
|
@@ -132,11 +153,115 @@ const reconstruct = (allLeaves, translations) => {
|
|
|
132
153
|
setPath(root, leaf.path, value);
|
|
133
154
|
}
|
|
134
155
|
return root;
|
|
135
|
-
}
|
|
136
|
-
|
|
137
|
-
|
|
156
|
+
}
|
|
157
|
+
function accumulateUsage(target, source) {
|
|
158
|
+
if (!source) {
|
|
159
|
+
return;
|
|
160
|
+
}
|
|
161
|
+
target.inputTokens += source.inputTokens;
|
|
162
|
+
target.outputTokens += source.outputTokens;
|
|
163
|
+
}
|
|
164
|
+
async function translateOne(translator, from, to, value, usage, callCounter) {
|
|
165
|
+
for (let attempt = 0; attempt <= contants_js_1.MAX_RETRIES; attempt++) {
|
|
166
|
+
if (attempt > 0) {
|
|
167
|
+
await (0, utilites_js_1.delay)(contants_js_1.BASE_RETRY_DELAY_MS * Math.pow(2, attempt - 1));
|
|
168
|
+
}
|
|
169
|
+
try {
|
|
170
|
+
callCounter.count += 1;
|
|
171
|
+
const response = await translator.translate(buildSinglePrompt(from, to, value));
|
|
172
|
+
accumulateUsage(usage, response.usage);
|
|
173
|
+
const trimmed = response.text.trim();
|
|
174
|
+
if (trimmed.length > 0) {
|
|
175
|
+
return trimmed;
|
|
176
|
+
}
|
|
177
|
+
}
|
|
178
|
+
catch {
|
|
179
|
+
if (attempt === contants_js_1.MAX_RETRIES) {
|
|
180
|
+
return null;
|
|
181
|
+
}
|
|
182
|
+
}
|
|
183
|
+
}
|
|
184
|
+
return null;
|
|
185
|
+
}
|
|
186
|
+
async function runBatchAttempts(batchLeaves, translator, from, to, label, translations, usage, callCounter) {
|
|
187
|
+
let remaining = [...batchLeaves];
|
|
188
|
+
let firstAttempt = true;
|
|
189
|
+
for (let attempt = 0; attempt <= contants_js_1.MAX_RETRIES; attempt++) {
|
|
190
|
+
if (attempt > 0) {
|
|
191
|
+
const backoffMs = contants_js_1.BASE_RETRY_DELAY_MS * Math.pow(2, attempt - 1);
|
|
192
|
+
logger_js_1.logger.warn(`${label} Retry ${attempt}/${contants_js_1.MAX_RETRIES} after ${backoffMs}ms backoff (${remaining.length} entries)...`);
|
|
193
|
+
await (0, utilites_js_1.delay)(backoffMs);
|
|
194
|
+
}
|
|
195
|
+
try {
|
|
196
|
+
const prompt = buildBatchPrompt(from, to, remaining);
|
|
197
|
+
callCounter.count += 1;
|
|
198
|
+
const response = await translator.translate(prompt);
|
|
199
|
+
accumulateUsage(usage, response.usage);
|
|
200
|
+
const text = response.text;
|
|
201
|
+
if (!text || text.trim().length === 0) {
|
|
202
|
+
if (attempt === contants_js_1.MAX_RETRIES) {
|
|
203
|
+
return remaining;
|
|
204
|
+
}
|
|
205
|
+
logger_js_1.logger.warn(`${label} Attempt ${attempt + 1} returned empty response`);
|
|
206
|
+
continue;
|
|
207
|
+
}
|
|
208
|
+
const decoded = decodeResponse(text);
|
|
209
|
+
const stillMissing = [];
|
|
210
|
+
for (const leaf of remaining) {
|
|
211
|
+
const value = decoded.get(leaf.id);
|
|
212
|
+
if (!value || value.trim().length === 0) {
|
|
213
|
+
stillMissing.push(leaf);
|
|
214
|
+
}
|
|
215
|
+
else {
|
|
216
|
+
translations.set(leaf.id, value);
|
|
217
|
+
}
|
|
218
|
+
}
|
|
219
|
+
if (stillMissing.length === 0) {
|
|
220
|
+
return [];
|
|
221
|
+
}
|
|
222
|
+
if (firstAttempt && stillMissing.length >= remaining.length * contants_js_1.PER_LEAF_FALLBACK_RATIO) {
|
|
223
|
+
logger_js_1.logger.warn(`${label} Batch mode unreliable for this model (${stillMissing.length}/${remaining.length} missing on first attempt), switching to per-entry mode`);
|
|
224
|
+
return stillMissing;
|
|
225
|
+
}
|
|
226
|
+
logger_js_1.logger.warn(`${label} ${stillMissing.length}/${remaining.length} entries missing, will retry`);
|
|
227
|
+
firstAttempt = false;
|
|
228
|
+
remaining = stillMissing;
|
|
229
|
+
}
|
|
230
|
+
catch (error) {
|
|
231
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
232
|
+
if (attempt === contants_js_1.MAX_RETRIES) {
|
|
233
|
+
logger_js_1.logger.warn(`${label} Batch attempt failed after retries: ${message}`);
|
|
234
|
+
return remaining;
|
|
235
|
+
}
|
|
236
|
+
logger_js_1.logger.warn(`${label} Attempt ${attempt + 1} failed: ${message}`);
|
|
237
|
+
}
|
|
238
|
+
}
|
|
239
|
+
return remaining;
|
|
240
|
+
}
|
|
241
|
+
async function runPerLeafFallback(leaves, translator, from, to, options, label, translations, usage, callCounter) {
|
|
242
|
+
logger_js_1.logger.info(`${label} Per-entry fallback for ${leaves.length} entries`);
|
|
243
|
+
const failed = [];
|
|
244
|
+
for (let i = 0; i < leaves.length; i++) {
|
|
245
|
+
if (i > 0 && !options.noLimit) {
|
|
246
|
+
await (0, utilites_js_1.delay)(contants_js_1.INTER_LEAF_DELAY_MS);
|
|
247
|
+
}
|
|
248
|
+
const leaf = leaves[i];
|
|
249
|
+
const result = await translateOne(translator, from, to, String(leaf.value), usage, callCounter);
|
|
250
|
+
if (result) {
|
|
251
|
+
translations.set(leaf.id, result);
|
|
252
|
+
}
|
|
253
|
+
else {
|
|
254
|
+
logger_js_1.logger.warn(`${label} Per-entry failed for id=${leaf.id}`);
|
|
255
|
+
failed.push(leaf);
|
|
256
|
+
}
|
|
257
|
+
}
|
|
258
|
+
return failed;
|
|
259
|
+
}
|
|
260
|
+
async function runBatches(batches, translator, from, to, options) {
|
|
138
261
|
const translations = new Map();
|
|
139
262
|
const failedIds = [];
|
|
263
|
+
const usage = { inputTokens: 0, outputTokens: 0 };
|
|
264
|
+
const callCounter = { count: 0 };
|
|
140
265
|
for (let i = 0; i < batches.length; i++) {
|
|
141
266
|
const batch = batches[i];
|
|
142
267
|
const label = `[Batch ${batch.index + 1}/${batches.length}]`;
|
|
@@ -144,58 +269,19 @@ const runBatches = async (batches, translator, from, to, options) => {
|
|
|
144
269
|
logger_js_1.logger.info(`Waiting ${contants_js_1.INTER_BATCH_DELAY_MS}ms before next batch (rate limit protection)...`);
|
|
145
270
|
await (0, utilites_js_1.delay)(contants_js_1.INTER_BATCH_DELAY_MS);
|
|
146
271
|
}
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
for (const leaf of remaining)
|
|
161
|
-
failedIds.push(leaf.id);
|
|
162
|
-
break;
|
|
163
|
-
}
|
|
164
|
-
logger_js_1.logger.warn(`${label} Attempt ${attempt + 1} returned empty response`);
|
|
165
|
-
continue;
|
|
166
|
-
}
|
|
167
|
-
const decoded = (0, exports.decodeResponse)(response);
|
|
168
|
-
const stillMissing = [];
|
|
169
|
-
for (const leaf of remaining) {
|
|
170
|
-
const value = decoded.get(leaf.id);
|
|
171
|
-
if (!value || value.trim().length === 0) {
|
|
172
|
-
stillMissing.push(leaf);
|
|
173
|
-
continue;
|
|
174
|
-
}
|
|
175
|
-
translations.set(leaf.id, value);
|
|
176
|
-
}
|
|
177
|
-
if (stillMissing.length === 0) {
|
|
178
|
-
logger_js_1.logger.info(`${label} Translated successfully (${(0, utilites_js_1.formatBytes)(batch.byteSize)}, ${batch.leaves.length} entries)`);
|
|
179
|
-
break;
|
|
180
|
-
}
|
|
181
|
-
if (attempt === contants_js_1.MAX_RETRIES) {
|
|
182
|
-
logger_js_1.logger.error(`${label} Failed after ${contants_js_1.MAX_RETRIES + 1} attempts: ${stillMissing.length} entries missing`);
|
|
183
|
-
for (const leaf of stillMissing)
|
|
184
|
-
failedIds.push(leaf.id);
|
|
185
|
-
break;
|
|
186
|
-
}
|
|
187
|
-
logger_js_1.logger.warn(`${label} ${stillMissing.length}/${remaining.length} entries missing, will retry`);
|
|
188
|
-
remaining = stillMissing;
|
|
189
|
-
}
|
|
190
|
-
catch (error) {
|
|
191
|
-
const message = error instanceof Error ? error.message : String(error);
|
|
192
|
-
if (attempt === contants_js_1.MAX_RETRIES) {
|
|
193
|
-
logger_js_1.logger.error(`${label} Failed after ${contants_js_1.MAX_RETRIES + 1} attempts: ${message}`);
|
|
194
|
-
for (const leaf of remaining)
|
|
195
|
-
failedIds.push(leaf.id);
|
|
196
|
-
break;
|
|
197
|
-
}
|
|
198
|
-
logger_js_1.logger.warn(`${label} Attempt ${attempt + 1} failed: ${message}`);
|
|
272
|
+
const missingAfterBatch = await runBatchAttempts(batch.leaves, translator, from, to, label, translations, usage, callCounter);
|
|
273
|
+
let stillFailed = [];
|
|
274
|
+
if (missingAfterBatch.length > 0) {
|
|
275
|
+
stillFailed = await runPerLeafFallback(missingAfterBatch, translator, from, to, options, label, translations, usage, callCounter);
|
|
276
|
+
}
|
|
277
|
+
const succeeded = batch.leaves.length - stillFailed.length;
|
|
278
|
+
if (stillFailed.length === 0) {
|
|
279
|
+
logger_js_1.logger.info(`${label} Translated successfully (${(0, utilites_js_1.formatBytes)(batch.byteSize)}, ${succeeded} entries)`);
|
|
280
|
+
}
|
|
281
|
+
else {
|
|
282
|
+
logger_js_1.logger.error(`${label} ${stillFailed.length}/${batch.leaves.length} entries failed permanently`);
|
|
283
|
+
for (const leaf of stillFailed) {
|
|
284
|
+
failedIds.push(leaf.id);
|
|
199
285
|
}
|
|
200
286
|
}
|
|
201
287
|
}
|
|
@@ -204,6 +290,5 @@ const runBatches = async (batches, translator, from, to, options) => {
|
|
|
204
290
|
const more = failedIds.length > 20 ? `, …(+${failedIds.length - 20})` : '';
|
|
205
291
|
throw new Error(`Translation failed for ${failedIds.length} entry/entries (ids: ${preview}${more})`);
|
|
206
292
|
}
|
|
207
|
-
return translations;
|
|
208
|
-
}
|
|
209
|
-
exports.runBatches = runBatches;
|
|
293
|
+
return { translations, usage, calls: callCounter.count };
|
|
294
|
+
}
|
package/script/src/types.d.ts
CHANGED
|
@@ -12,6 +12,9 @@ export type TranslateArgs = {
|
|
|
12
12
|
from?: string;
|
|
13
13
|
to?: string;
|
|
14
14
|
url?: string;
|
|
15
|
+
config?: string;
|
|
16
|
+
stats?: boolean;
|
|
17
|
+
incremental?: boolean;
|
|
15
18
|
'no-limit'?: boolean;
|
|
16
19
|
'no-timeout'?: boolean;
|
|
17
20
|
'max-batch-size'?: string;
|
|
@@ -21,13 +24,30 @@ export type ValidatedTranslateArgs = {
|
|
|
21
24
|
provider: Provider;
|
|
22
25
|
model?: string;
|
|
23
26
|
input: string;
|
|
24
|
-
output: string;
|
|
27
|
+
output: string[];
|
|
25
28
|
from: string;
|
|
26
|
-
to: string;
|
|
29
|
+
to: string[];
|
|
27
30
|
url?: string;
|
|
28
31
|
noLimit: boolean;
|
|
29
32
|
noTimeout: boolean;
|
|
30
33
|
maxBatchBytes: number;
|
|
34
|
+
stats: boolean;
|
|
35
|
+
incremental: boolean;
|
|
36
|
+
};
|
|
37
|
+
export type GlottoConfig = {
|
|
38
|
+
key?: string;
|
|
39
|
+
provider?: Provider;
|
|
40
|
+
model?: string;
|
|
41
|
+
input?: string;
|
|
42
|
+
output?: string | string[];
|
|
43
|
+
from?: string;
|
|
44
|
+
to?: string | string[];
|
|
45
|
+
url?: string;
|
|
46
|
+
noLimit?: boolean;
|
|
47
|
+
noTimeout?: boolean;
|
|
48
|
+
maxBatchSize?: number;
|
|
49
|
+
stats?: boolean;
|
|
50
|
+
incremental?: boolean;
|
|
31
51
|
};
|
|
32
52
|
export type JsonValue = string | number | boolean | null | JsonObject | JsonArray;
|
|
33
53
|
export interface JsonObject {
|
|
@@ -47,7 +67,15 @@ export type Batch = {
|
|
|
47
67
|
leaves: Leaf[];
|
|
48
68
|
byteSize: number;
|
|
49
69
|
};
|
|
70
|
+
export type TranslateUsage = {
|
|
71
|
+
inputTokens: number;
|
|
72
|
+
outputTokens: number;
|
|
73
|
+
};
|
|
74
|
+
export type TranslateResult = {
|
|
75
|
+
text: string;
|
|
76
|
+
usage?: TranslateUsage;
|
|
77
|
+
};
|
|
50
78
|
export interface TextTranslator {
|
|
51
|
-
translate(prompt: string): Promise<
|
|
79
|
+
translate(prompt: string): Promise<TranslateResult>;
|
|
52
80
|
}
|
|
53
81
|
//# sourceMappingURL=types.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/src/types.ts"],"names":[],"mappings":"AAAA,MAAM,MAAM,QAAQ,GAAG,QAAQ,GAAG,QAAQ,GAAG,WAAW,CAAC;AAEzD,MAAM,MAAM,gBAAgB,GAAG;IAC7B,OAAO,EAAE,OAAO,CAAC;IACjB,SAAS,EAAE,OAAO,CAAC;CACpB,CAAC;AAEF,MAAM,MAAM,aAAa,GAAG;IAC1B,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB,gBAAgB,CAAC,EAAE,MAAM,CAAC;CAC3B,CAAC;AAEF,MAAM,MAAM,sBAAsB,GAAG;IACnC,GAAG,EAAE,MAAM,CAAC;IACZ,QAAQ,EAAE,QAAQ,CAAC;IACnB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/src/types.ts"],"names":[],"mappings":"AAAA,MAAM,MAAM,QAAQ,GAAG,QAAQ,GAAG,QAAQ,GAAG,WAAW,CAAC;AAEzD,MAAM,MAAM,gBAAgB,GAAG;IAC7B,OAAO,EAAE,OAAO,CAAC;IACjB,SAAS,EAAE,OAAO,CAAC;CACpB,CAAC;AAEF,MAAM,MAAM,aAAa,GAAG;IAC1B,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB,gBAAgB,CAAC,EAAE,MAAM,CAAC;CAC3B,CAAC;AAEF,MAAM,MAAM,sBAAsB,GAAG;IACnC,GAAG,EAAE,MAAM,CAAC;IACZ,QAAQ,EAAE,QAAQ,CAAC;IACnB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,IAAI,EAAE,MAAM,CAAC;IACb,EAAE,EAAE,MAAM,EAAE,CAAC;IACb,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,OAAO,CAAC;IACjB,SAAS,EAAE,OAAO,CAAC;IACnB,aAAa,EAAE,MAAM,CAAC;IACtB,KAAK,EAAE,OAAO,CAAC;IACf,WAAW,EAAE,OAAO,CAAC;CACtB,CAAC;AAEF,MAAM,MAAM,YAAY,GAAG;IACzB,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,QAAQ,CAAC,EAAE,QAAQ,CAAC;IACpB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,GAAG,MAAM,EAAE,CAAC;IAC3B,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,EAAE,CAAC,EAAE,MAAM,GAAG,MAAM,EAAE,CAAC;IACvB,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,WAAW,CAAC,EAAE,OAAO,CAAC;CACvB,CAAC;AAEF,MAAM,MAAM,SAAS,GAAG,MAAM,GAAG,MAAM,GAAG,OAAO,GAAG,IAAI,GAAG,UAAU,GAAG,SAAS,CAAC;AAClF,MAAM,WAAW,UAAU;IACzB,CAAC,GAAG,EAAE,MAAM,GAAG,SAAS,CAAC;CAC1B;AACD,MAAM,MAAM,SAAS,GAAG,SAAS,EAAE,CAAC;AAEpC,MAAM,MAAM,WAAW,GAAG,MAAM,GAAG,MAAM,CAAC;AAC1C,MAAM,MAAM,IAAI,GAAG,WAAW,EAAE,CAAC;AAEjC,MAAM,MAAM,IAAI,GAAG;IACjB,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,IAAI,CAAC;IACX,KAAK,EAAE,SAAS,CAAC;IACjB,YAAY,EAAE,OAAO,CAAC;CACvB,CAAC;AAEF,MAAM,MAAM,KAAK,GAAG;IAClB,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,IAAI,EAAE,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC;CAClB,CAAC;AAEF,MAAM,MAAM,cAAc,GAAG;IAC3B,WAAW,EAAE,MAAM,CAAC;IACpB,YAAY,EAAE,MAAM,CAAC;CACtB,CAAC;AAEF,MAAM,MAAM,eAAe,GAAG;IAC5B,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,CAAC,EAAE,cAAc,CAAC;CACxB,CAAC;AAEF,MAAM,WAAW,cAAc;IAC7B,SAAS,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,eAAe,CAAC,CAAC;CACrD"}
|
package/script/src/utilites.d.ts
CHANGED
|
@@ -1,6 +1,5 @@
|
|
|
1
1
|
import type { TranslateArgs, ValidatedTranslateArgs } from './types.js';
|
|
2
|
-
|
|
3
|
-
export declare
|
|
4
|
-
export declare
|
|
5
|
-
export declare const delay: (ms: number) => Promise<void>;
|
|
2
|
+
export declare function validateArgs(args: TranslateArgs): ValidatedTranslateArgs;
|
|
3
|
+
export declare function formatBytes(bytes: number): string;
|
|
4
|
+
export declare function delay(ms: number): Promise<void>;
|
|
6
5
|
//# sourceMappingURL=utilites.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"utilites.d.ts","sourceRoot":"","sources":["../../src/src/utilites.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAY,aAAa,EAAE,sBAAsB,EAAE,MAAM,YAAY,CAAC;
|
|
1
|
+
{"version":3,"file":"utilites.d.ts","sourceRoot":"","sources":["../../src/src/utilites.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAY,aAAa,EAAE,sBAAsB,EAAE,MAAM,YAAY,CAAC;AAOlF,wBAAgB,YAAY,CAAC,IAAI,EAAE,aAAa,GAAG,sBAAsB,CA0DxE;AAED,wBAAgB,WAAW,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAQjD;AAED,wBAAgB,KAAK,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAE/C"}
|
package/script/src/utilites.js
CHANGED
|
@@ -1,20 +1,42 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.
|
|
3
|
+
exports.validateArgs = validateArgs;
|
|
4
|
+
exports.formatBytes = formatBytes;
|
|
5
|
+
exports.delay = delay;
|
|
4
6
|
const contants_js_1 = require("./contants.js");
|
|
5
|
-
|
|
6
|
-
|
|
7
|
+
function splitCsv(value) {
|
|
8
|
+
return value.split(',').map((s) => s.trim()).filter((s) => s.length > 0);
|
|
9
|
+
}
|
|
10
|
+
function validateArgs(args) {
|
|
11
|
+
if (!args.key) {
|
|
7
12
|
throw new Error('AI Key is required');
|
|
8
|
-
|
|
13
|
+
}
|
|
14
|
+
if (!args.provider) {
|
|
9
15
|
throw new Error('Provider parameter is required');
|
|
10
|
-
|
|
16
|
+
}
|
|
17
|
+
if (!args.input) {
|
|
11
18
|
throw new Error('Input parameter is required');
|
|
12
|
-
|
|
19
|
+
}
|
|
20
|
+
if (!args.output) {
|
|
13
21
|
throw new Error('Output parameter is required');
|
|
14
|
-
|
|
22
|
+
}
|
|
23
|
+
if (!args.from) {
|
|
15
24
|
throw new Error('Source language (from) parameter is required');
|
|
16
|
-
|
|
25
|
+
}
|
|
26
|
+
if (!args.to) {
|
|
17
27
|
throw new Error('Target language (to) parameter is required');
|
|
28
|
+
}
|
|
29
|
+
const toList = splitCsv(args.to);
|
|
30
|
+
const outputList = splitCsv(args.output);
|
|
31
|
+
if (toList.length === 0) {
|
|
32
|
+
throw new Error('Target language (to) must contain at least one value');
|
|
33
|
+
}
|
|
34
|
+
if (outputList.length === 0) {
|
|
35
|
+
throw new Error('Output must contain at least one value');
|
|
36
|
+
}
|
|
37
|
+
if (toList.length !== outputList.length) {
|
|
38
|
+
throw new Error(`--to (${toList.length}) and --output (${outputList.length}) must have the same number of values`);
|
|
39
|
+
}
|
|
18
40
|
let maxBatchBytes = contants_js_1.DEFAULT_MAX_BATCH_BYTES;
|
|
19
41
|
const rawMaxBatchSize = args['max-batch-size'];
|
|
20
42
|
if (rawMaxBatchSize !== undefined) {
|
|
@@ -29,25 +51,26 @@ const validateArgs = (args) => {
|
|
|
29
51
|
provider: args.provider,
|
|
30
52
|
model: args.model,
|
|
31
53
|
input: args.input,
|
|
32
|
-
output:
|
|
54
|
+
output: outputList,
|
|
33
55
|
from: args.from,
|
|
34
|
-
to:
|
|
56
|
+
to: toList,
|
|
35
57
|
url: args.url,
|
|
36
58
|
noLimit: args['no-limit'] ?? false,
|
|
37
59
|
noTimeout: args['no-timeout'] ?? false,
|
|
38
60
|
maxBatchBytes,
|
|
61
|
+
stats: args.stats ?? false,
|
|
62
|
+
incremental: args.incremental ?? false,
|
|
39
63
|
};
|
|
40
|
-
}
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
if (bytes < 1024)
|
|
64
|
+
}
|
|
65
|
+
function formatBytes(bytes) {
|
|
66
|
+
if (bytes < 1024) {
|
|
44
67
|
return `${bytes} B`;
|
|
45
|
-
|
|
68
|
+
}
|
|
69
|
+
if (bytes < 1024 * 1024) {
|
|
46
70
|
return `${(bytes / 1024).toFixed(1)} KB`;
|
|
71
|
+
}
|
|
47
72
|
return `${(bytes / (1024 * 1024)).toFixed(2)} MB`;
|
|
48
|
-
}
|
|
49
|
-
|
|
50
|
-
const delay = (ms) => {
|
|
73
|
+
}
|
|
74
|
+
function delay(ms) {
|
|
51
75
|
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
52
|
-
}
|
|
53
|
-
exports.delay = delay;
|
|
76
|
+
}
|