@gringow/gringow 0.1.4 → 0.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +31 -36
- package/dist/config.d.mts +4 -6
- package/dist/config.mjs +30 -20
- package/dist/config.test.mjs +42 -30
- package/dist/constants/default-config.mjs +3 -3
- package/dist/create-cache-id.test.mjs +5 -5
- package/dist/data/check-cache.mjs +31 -63
- package/dist/data/clear-cache.mjs +30 -20
- package/dist/data/create-cache-dir.mjs +30 -20
- package/dist/data/delete-cache-item.mjs +30 -20
- package/dist/data/get-cache.mjs +30 -20
- package/dist/data/get-or-create-cache.mjs +30 -20
- package/dist/data/md5.test.mjs +5 -5
- package/dist/data/save-cache.mjs +30 -20
- package/dist/data/write-cache-to-disk.mjs +30 -20
- package/dist/flat-template-string.test.mjs +5 -5
- package/dist/gringow.mjs +82 -491
- package/dist/index.mjs +82 -491
- package/dist/io/ask-llm.d.mts +3 -1
- package/dist/io/ask-llm.mjs +75 -449
- package/dist/logger.test.mjs +5 -5
- package/dist/schemas/config.shcema.d.mts +23 -35
- package/dist/schemas/config.shcema.mjs +25 -15
- package/dist/types/index.d.mts +3 -25
- package/gringow.config.js.example +2 -4
- package/gringow.config.json.example +4 -2
- package/package.json +2 -5
- package/dist/io/anthropic/ask.d.mts +0 -11
- package/dist/io/anthropic/ask.mjs +0 -192
- package/dist/io/anthropic/commit-response.d.mts +0 -21
- package/dist/io/anthropic/commit-response.mjs +0 -10
- package/dist/io/anthropic/index.d.mts +0 -14
- package/dist/io/anthropic/index.mjs +0 -210
- package/dist/io/anthropic/parse.d.mts +0 -20
- package/dist/io/anthropic/parse.mjs +0 -10
- package/dist/io/gemini/ask.d.mts +0 -11
- package/dist/io/gemini/ask.mjs +0 -234
- package/dist/io/gemini/commit-response.d.mts +0 -11
- package/dist/io/gemini/commit-response.mjs +0 -7
- package/dist/io/gemini/index.d.mts +0 -14
- package/dist/io/gemini/index.mjs +0 -250
- package/dist/io/gemini/parse.d.mts +0 -20
- package/dist/io/gemini/parse.mjs +0 -11
- package/dist/io/huggingface/ask.d.mts +0 -13
- package/dist/io/huggingface/ask.mjs +0 -407
- package/dist/io/huggingface/commit-response.d.mts +0 -39
- package/dist/io/huggingface/commit-response.mjs +0 -14
- package/dist/io/huggingface/index.d.mts +0 -13
- package/dist/io/huggingface/index.mjs +0 -428
- package/dist/io/huggingface/languages-map.d.mts +0 -191
- package/dist/io/huggingface/languages-map.mjs +0 -187
- package/dist/io/huggingface/parse.d.mts +0 -20
- package/dist/io/huggingface/parse.mjs +0 -9
package/dist/gringow.mjs
CHANGED
|
@@ -4,9 +4,7 @@ import { z } from 'zod';
|
|
|
4
4
|
import fs from 'fs/promises';
|
|
5
5
|
import { dirname, resolve, parse } from 'path';
|
|
6
6
|
import { mkdir, stat, statSync, mkdirSync } from 'fs';
|
|
7
|
-
import
|
|
8
|
-
import { GoogleGenerativeAI } from '@google/generative-ai';
|
|
9
|
-
import { env, pipeline } from '@huggingface/transformers';
|
|
7
|
+
import { igniteEngine, Message } from 'multi-llm-ts';
|
|
10
8
|
|
|
11
9
|
// src/config.ts
|
|
12
10
|
|
|
@@ -22,20 +20,34 @@ var defaultConfig = {
|
|
|
22
20
|
prefix: "grw",
|
|
23
21
|
write: true
|
|
24
22
|
},
|
|
25
|
-
keys: {},
|
|
26
23
|
sourceLanguage: "en-US",
|
|
27
24
|
defaultLanguage: "pt-BR",
|
|
28
25
|
languages: ["pt-BR", "fr-CA"],
|
|
29
26
|
llm: {
|
|
30
|
-
|
|
31
|
-
|
|
27
|
+
provider: "openai",
|
|
28
|
+
apiKey: void 0,
|
|
29
|
+
model: "gpt-4o-mini",
|
|
32
30
|
temperature: 1,
|
|
33
31
|
sleep: 0
|
|
34
32
|
},
|
|
35
33
|
globby: "./src/**/*.{js,jsx,ts,tsx}"
|
|
36
34
|
};
|
|
37
35
|
var languageSchema = z.string().regex(/^[a-z]{2,3}(-[A-Z]{2})?$/);
|
|
38
|
-
var
|
|
36
|
+
var llmProviderSchema = z.enum([
|
|
37
|
+
"anthropic",
|
|
38
|
+
"gemini",
|
|
39
|
+
"openai",
|
|
40
|
+
"groq",
|
|
41
|
+
"mistral",
|
|
42
|
+
"ollama",
|
|
43
|
+
"xai",
|
|
44
|
+
"deepseek",
|
|
45
|
+
"openrouter",
|
|
46
|
+
"cerebras",
|
|
47
|
+
"lmstudio",
|
|
48
|
+
"azure",
|
|
49
|
+
"meta"
|
|
50
|
+
]);
|
|
39
51
|
var configSchema = z.object({
|
|
40
52
|
debug: z.boolean().optional(),
|
|
41
53
|
log: z.object({
|
|
@@ -47,15 +59,13 @@ var configSchema = z.object({
|
|
|
47
59
|
prefix: z.string(),
|
|
48
60
|
write: z.boolean().optional()
|
|
49
61
|
}),
|
|
50
|
-
keys: z.object({
|
|
51
|
-
anthropicApiKey: z.string().optional(),
|
|
52
|
-
geminiApiKey: z.string().optional()
|
|
53
|
-
}),
|
|
54
62
|
sourceLanguage: languageSchema,
|
|
55
63
|
defaultLanguage: languageSchema,
|
|
56
64
|
languages: z.array(languageSchema),
|
|
57
65
|
llm: z.object({
|
|
58
|
-
|
|
66
|
+
provider: llmProviderSchema,
|
|
67
|
+
apiKey: z.string().optional(),
|
|
68
|
+
baseURL: z.string().optional(),
|
|
59
69
|
model: z.string().nullable(),
|
|
60
70
|
temperature: z.number().optional(),
|
|
61
71
|
sleep: z.number().optional(),
|
|
@@ -69,17 +79,15 @@ var configSchema = z.object({
|
|
|
69
79
|
globby: z.string().regex(/^\.\/.*/).optional()
|
|
70
80
|
}).refine(
|
|
71
81
|
(data) => {
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
if (data.llm.choice === "anthropic" && !data.keys.anthropicApiKey) {
|
|
76
|
-
return false;
|
|
82
|
+
const noApiKeyRequired = ["ollama", "lmstudio"];
|
|
83
|
+
if (noApiKeyRequired.includes(data.llm.provider)) {
|
|
84
|
+
return true;
|
|
77
85
|
}
|
|
78
|
-
return
|
|
86
|
+
return !!data.llm.apiKey;
|
|
79
87
|
},
|
|
80
88
|
{
|
|
81
|
-
message: "The API key
|
|
82
|
-
path: ["
|
|
89
|
+
message: "The API key must be present in llm.apiKey for the chosen provider.",
|
|
90
|
+
path: ["llm", "apiKey"]
|
|
83
91
|
}
|
|
84
92
|
);
|
|
85
93
|
|
|
@@ -115,7 +123,7 @@ Your task is to translate a given text and output **only one valid JSON object**
|
|
|
115
123
|
|
|
116
124
|
Follow these rules strictly:
|
|
117
125
|
|
|
118
|
-
1. The original text is written in English.
|
|
126
|
+
1. The original text is written in English or Spanish.
|
|
119
127
|
2. You will receive additional user parameters under the variable 'extraInstructions'. These may specify tone, persona, style, context, or any relevant translation guideline.
|
|
120
128
|
- Examples: \u201Ctranslate as if you were a Shakespearean narrator\u201D or \u201Cuse a formal business tone suitable for a corporate presentation.\u201D
|
|
121
129
|
- Apply these creatively but **without altering the meaning or punctuation** of the original text.
|
|
@@ -148,10 +156,10 @@ Translate the following text faithfully:
|
|
|
148
156
|
function defaultPostPrompt(_config) {
|
|
149
157
|
return "";
|
|
150
158
|
}
|
|
151
|
-
function prePrompt(config,
|
|
159
|
+
function prePrompt(config, prepend, replace) {
|
|
152
160
|
return [
|
|
153
161
|
// ...(append ? ['[SPECIFIC_INPUTS]', String(append), '[/SPECIFIC_INPUTS]'] : []),
|
|
154
|
-
replace ? String(replace) : defaultPrePrompt(config,
|
|
162
|
+
replace ? String(replace) : defaultPrePrompt(config, prepend)
|
|
155
163
|
].join(" ").trim();
|
|
156
164
|
}
|
|
157
165
|
function postPrompt(config, prepend, replace) {
|
|
@@ -468,53 +476,11 @@ async function getOrCreateCache() {
|
|
|
468
476
|
return rawCache ? JSON.parse(rawCache?.toString()) : {};
|
|
469
477
|
}
|
|
470
478
|
|
|
471
|
-
// src/logger.ts
|
|
472
|
-
var Dim = "\x1B[2m";
|
|
473
|
-
var FgGreen = "\x1B[32m";
|
|
474
|
-
var FgYellow = "\x1B[33m";
|
|
475
|
-
var FgCyan = "\x1B[36m";
|
|
476
|
-
var FgWhite = "\x1B[37m";
|
|
477
|
-
var BgRed = "\x1B[41m";
|
|
478
|
-
var Clear = "\x1B[0m";
|
|
479
|
-
var Colors = {
|
|
480
|
-
debug: `${Dim}${FgGreen}`,
|
|
481
|
-
trace: `${Dim}${FgCyan}`,
|
|
482
|
-
info: `${FgCyan}`,
|
|
483
|
-
error: `${BgRed}${FgWhite}`,
|
|
484
|
-
warn: `${FgYellow}`,
|
|
485
|
-
verbose: `${FgWhite}`
|
|
486
|
-
};
|
|
487
|
-
var isProd = process.env.NODE_ENV === "production";
|
|
488
|
-
function log(level, ...params) {
|
|
489
|
-
console.log(`${!isProd ? Colors[level] : ""}${String(level).toUpperCase()} `, ...params, `${!isProd ? Clear : ""}`);
|
|
490
|
-
}
|
|
491
|
-
var logger = {
|
|
492
|
-
debug(...params) {
|
|
493
|
-
return log("debug", ...params);
|
|
494
|
-
},
|
|
495
|
-
trace(...params) {
|
|
496
|
-
return log("trace", ...params);
|
|
497
|
-
},
|
|
498
|
-
info(...params) {
|
|
499
|
-
return log("info", ...params);
|
|
500
|
-
},
|
|
501
|
-
warn(...params) {
|
|
502
|
-
return log("warn", ...params);
|
|
503
|
-
},
|
|
504
|
-
error(...params) {
|
|
505
|
-
return log("error", ...params);
|
|
506
|
-
},
|
|
507
|
-
verbose(...params) {
|
|
508
|
-
return log("verbose", ...params);
|
|
509
|
-
}
|
|
510
|
-
};
|
|
511
|
-
var logger_default = logger;
|
|
512
|
-
|
|
513
479
|
// src/data/check-cache.ts
|
|
514
480
|
async function checkCache(cacheId) {
|
|
515
481
|
const cache = await getOrCreateCache();
|
|
516
482
|
if (cache?.[cacheId]) {
|
|
517
|
-
|
|
483
|
+
console.log(`Cache hit, id:${cacheId}`);
|
|
518
484
|
return cache[cacheId];
|
|
519
485
|
}
|
|
520
486
|
return void 0;
|
|
@@ -542,445 +508,62 @@ function flatTemplateString(givenStrings, givenValues) {
|
|
|
542
508
|
}, "");
|
|
543
509
|
return flatted;
|
|
544
510
|
}
|
|
545
|
-
|
|
546
|
-
|
|
547
|
-
|
|
548
|
-
|
|
549
|
-
|
|
550
|
-
|
|
551
|
-
|
|
552
|
-
|
|
553
|
-
|
|
554
|
-
|
|
555
|
-
|
|
556
|
-
|
|
557
|
-
|
|
558
|
-
|
|
559
|
-
|
|
560
|
-
|
|
561
|
-
|
|
562
|
-
|
|
563
|
-
${postPrompt2}`).trim();
|
|
564
|
-
const response = await anthropic.messages.create({
|
|
565
|
-
model: config.llm.model,
|
|
566
|
-
max_tokens: 1024,
|
|
567
|
-
messages: [
|
|
568
|
-
{
|
|
569
|
-
role: "user",
|
|
570
|
-
content: fullPrompt
|
|
571
|
-
}
|
|
572
|
-
]
|
|
573
|
-
});
|
|
574
|
-
return response;
|
|
575
|
-
}
|
|
576
|
-
|
|
577
|
-
// src/io/anthropic/commit-response.ts
|
|
578
|
-
function commitResponseAnthropic(response) {
|
|
579
|
-
const result2 = response.content.reduce((acc, item) => {
|
|
580
|
-
const nextAcc = acc + (item.type === "text" ? item.text : "");
|
|
581
|
-
return nextAcc;
|
|
582
|
-
}, "");
|
|
583
|
-
return result2;
|
|
584
|
-
}
|
|
585
|
-
|
|
586
|
-
// src/io/anthropic/parse.ts
|
|
587
|
-
function parseAnthropic(unparsed) {
|
|
588
|
-
if (typeof unparsed !== "string") {
|
|
589
|
-
throw new Error("CommitResponse should be a string here.");
|
|
590
|
-
}
|
|
591
|
-
const parsed = JSON.parse(unparsed);
|
|
511
|
+
var providerEngineMap = {
|
|
512
|
+
anthropic: "anthropic",
|
|
513
|
+
gemini: "google",
|
|
514
|
+
openai: "openai",
|
|
515
|
+
groq: "groq",
|
|
516
|
+
mistral: "mistralai",
|
|
517
|
+
ollama: "ollama",
|
|
518
|
+
xai: "xai",
|
|
519
|
+
deepseek: "deepseek",
|
|
520
|
+
openrouter: "openrouter",
|
|
521
|
+
cerebras: "cerebras",
|
|
522
|
+
lmstudio: "lmstudio",
|
|
523
|
+
azure: "azure",
|
|
524
|
+
meta: "meta"
|
|
525
|
+
};
|
|
526
|
+
function parseResponse(responseText) {
|
|
527
|
+
const cleaned = responseText.replace(/```json/g, "").replace(/```/g, "").trim();
|
|
528
|
+
const parsed = JSON.parse(cleaned);
|
|
592
529
|
return parsed;
|
|
593
530
|
}
|
|
594
|
-
async function
|
|
595
|
-
try {
|
|
596
|
-
if (!config.keys.geminiApiKey) {
|
|
597
|
-
throw new Error("Gemini API key is not set in the config");
|
|
598
|
-
}
|
|
599
|
-
const generativeAIClient = new GoogleGenerativeAI(config.keys.geminiApiKey);
|
|
600
|
-
const model = generativeAIClient.getGenerativeModel({
|
|
601
|
-
model: config.llm.model,
|
|
602
|
-
generationConfig: {
|
|
603
|
-
temperature: config.llm.temperature ?? 0.8
|
|
604
|
-
}
|
|
605
|
-
});
|
|
606
|
-
if (config.debug) {
|
|
607
|
-
console.log("Gemini model prompt:", JSON.stringify({ prompt }));
|
|
608
|
-
}
|
|
609
|
-
const response = await model.generateContent([prompt]);
|
|
610
|
-
return response;
|
|
611
|
-
} catch (err) {
|
|
612
|
-
const error = err;
|
|
613
|
-
console.log("Error details from Gemini model", JSON.stringify({ error }, null, 2));
|
|
614
|
-
const retryDelay = error?.errorDetails?.find(
|
|
615
|
-
(detail) => detail?.["@type"] === "type.googleapis.com/google.rpc.RetryInfo"
|
|
616
|
-
)?.retryDelay ?? void 0;
|
|
617
|
-
if (error.status === 429 && retryDelay !== void 0) {
|
|
618
|
-
const sleepTime = config.llm.sleep ?? 3;
|
|
619
|
-
console.warn(
|
|
620
|
-
`Rate limit exceeded when generating response from Gemini model, retrying in... ${retryDelay} + ${sleepTime / 1e3}s`
|
|
621
|
-
);
|
|
622
|
-
await new Promise((resolve2) => setTimeout(resolve2, (parseInt(retryDelay, 10) + sleepTime / 1e3) * 1e3));
|
|
623
|
-
return generateResponse(prompt, config);
|
|
624
|
-
}
|
|
625
|
-
console.error("Error generating response from Gemini model", { error });
|
|
626
|
-
throw error;
|
|
627
|
-
}
|
|
628
|
-
}
|
|
629
|
-
async function askGemini(prompt, prePrompt2, postPrompt2) {
|
|
531
|
+
async function askLLM(prompt, prePrompt2, postPrompt2) {
|
|
630
532
|
const config = await awaitedConfig;
|
|
631
|
-
|
|
632
|
-
|
|
533
|
+
const provider = config.llm.provider;
|
|
534
|
+
const engineName = providerEngineMap[provider];
|
|
535
|
+
if (!engineName) {
|
|
536
|
+
throw new Error(`Unknown LLM provider: ${provider}`);
|
|
633
537
|
}
|
|
634
|
-
if (!config.
|
|
635
|
-
throw new Error(
|
|
538
|
+
if (!config.llm.apiKey) {
|
|
539
|
+
throw new Error(`API key for ${provider} is not configured in llm.apiKey`);
|
|
636
540
|
}
|
|
637
|
-
if (config.
|
|
638
|
-
|
|
541
|
+
if (config.llm.sleep && config.llm.sleep > 0) {
|
|
542
|
+
await new Promise((resolve2) => setTimeout(resolve2, config.llm.sleep));
|
|
639
543
|
}
|
|
640
|
-
const
|
|
544
|
+
const engine = igniteEngine(engineName, {
|
|
545
|
+
apiKey: config.llm.apiKey,
|
|
546
|
+
...config.llm.baseURL ? { baseURL: config.llm.baseURL } : {}
|
|
547
|
+
});
|
|
548
|
+
const fullPrompt = `${prePrompt2}
|
|
641
549
|
|
|
642
550
|
|
|
643
551
|
${prompt}
|
|
644
552
|
|
|
645
553
|
|
|
646
|
-
${postPrompt2}
|
|
647
|
-
|
|
648
|
-
|
|
649
|
-
|
|
650
|
-
|
|
651
|
-
if (responseString.startsWith("{") && responseString.endsWith("}")) {
|
|
652
|
-
response = maybeResponse;
|
|
653
|
-
break;
|
|
654
|
-
}
|
|
655
|
-
if (config.debug) {
|
|
656
|
-
console.log("Gemini model response is not valid, contains no JSON:", JSON.stringify({ maybeResponse }));
|
|
657
|
-
}
|
|
658
|
-
}
|
|
659
|
-
if (config.debug) {
|
|
660
|
-
console.log("Gemini model response:", JSON.stringify({ response }));
|
|
661
|
-
}
|
|
662
|
-
if (!response) {
|
|
663
|
-
throw new Error("Gemini model response is not valid");
|
|
664
|
-
}
|
|
665
|
-
return response;
|
|
666
|
-
}
|
|
667
|
-
|
|
668
|
-
// src/io/gemini/commit-response.ts
|
|
669
|
-
function commitResponseGemini(response) {
|
|
670
|
-
const result2 = response.response.text();
|
|
671
|
-
return result2;
|
|
672
|
-
}
|
|
673
|
-
|
|
674
|
-
// src/io/gemini/parse.ts
|
|
675
|
-
function parseGemini(unparsed) {
|
|
676
|
-
if (typeof unparsed !== "string") {
|
|
677
|
-
throw new Error("CommitResponse should be a string here.");
|
|
678
|
-
}
|
|
679
|
-
const cleaned = unparsed.replace(/```json/g, "").replace(/```/g, "");
|
|
680
|
-
const parsed = JSON.parse(cleaned);
|
|
681
|
-
return parsed;
|
|
682
|
-
}
|
|
683
|
-
|
|
684
|
-
// src/io/huggingface/languages-map.ts
|
|
685
|
-
var languagesMap = {
|
|
686
|
-
"en-US": "eng_Latn",
|
|
687
|
-
"en-GB": "eng_Latn",
|
|
688
|
-
"pt-BR": "por_Latn",
|
|
689
|
-
"pt-PT": "por_Latn",
|
|
690
|
-
"es-ES": "spa_Latn",
|
|
691
|
-
"es-MX": "spa_Latn",
|
|
692
|
-
"fr-FR": "fra_Latn",
|
|
693
|
-
"fr-CA": "fra_Latn",
|
|
694
|
-
"de-DE": "deu_Latn",
|
|
695
|
-
"it-IT": "ita_Latn",
|
|
696
|
-
"nl-NL": "nld_Latn",
|
|
697
|
-
"pl-PL": "pol_Latn",
|
|
698
|
-
"ru-RU": "rus_Cyrl",
|
|
699
|
-
"uk-UA": "ukr_Cyrl",
|
|
700
|
-
"bg-BG": "bul_Cyrl",
|
|
701
|
-
"hr-HR": "hrv_Latn",
|
|
702
|
-
"cs-CZ": "ces_Latn",
|
|
703
|
-
"sk-SK": "slk_Latn",
|
|
704
|
-
"hu-HU": "hun_Latn",
|
|
705
|
-
"ro-RO": "ron_Latn",
|
|
706
|
-
"fi-FI": "fin_Latn",
|
|
707
|
-
"sv-SE": "swe_Latn",
|
|
708
|
-
"da-DK": "dan_Latn",
|
|
709
|
-
"no-NO": "nob_Latn",
|
|
710
|
-
"is-IS": "isl_Latn",
|
|
711
|
-
"et-EE": "est_Latn",
|
|
712
|
-
"lv-LV": "lav_Latn",
|
|
713
|
-
"lt-LT": "lit_Latn",
|
|
714
|
-
"mt-MT": "mlt_Latn",
|
|
715
|
-
"ga-IE": "gle_Latn",
|
|
716
|
-
"cy-GB": "cym_Latn",
|
|
717
|
-
"eu-ES": "eus_Latn",
|
|
718
|
-
"ca-ES": "cat_Latn",
|
|
719
|
-
"gl-ES": "glg_Latn",
|
|
720
|
-
"oc-FR": "oci_Latn",
|
|
721
|
-
"br-FR": "bre_Latn",
|
|
722
|
-
"gd-GB": "gla_Latn",
|
|
723
|
-
"kw-GB": "cor_Latn",
|
|
724
|
-
"fur-IT": "fur_Latn",
|
|
725
|
-
"rm-CH": "roh_Latn",
|
|
726
|
-
"lb-LU": "ltz_Latn",
|
|
727
|
-
"fy-NL": "fry_Latn",
|
|
728
|
-
"af-ZA": "afr_Latn",
|
|
729
|
-
"zh-CN": "zho_Hans",
|
|
730
|
-
"zh-TW": "zho_Hant",
|
|
731
|
-
"zh-HK": "zho_Hant",
|
|
732
|
-
"ja-JP": "jpn_Jpan",
|
|
733
|
-
"ko-KR": "kor_Hang",
|
|
734
|
-
"th-TH": "tha_Thai",
|
|
735
|
-
"vi-VN": "vie_Latn",
|
|
736
|
-
"id-ID": "ind_Latn",
|
|
737
|
-
"ms-MY": "zsm_Latn",
|
|
738
|
-
"tl-PH": "tgl_Latn",
|
|
739
|
-
"my-MM": "mya_Mymr",
|
|
740
|
-
"km-KH": "khm_Khmr",
|
|
741
|
-
"lo-LA": "lao_Laoo",
|
|
742
|
-
"bn-BD": "ben_Beng",
|
|
743
|
-
"bn-IN": "ben_Beng",
|
|
744
|
-
"hi-IN": "hin_Deva",
|
|
745
|
-
"ur-PK": "urd_Arab",
|
|
746
|
-
"ur-IN": "urd_Arab",
|
|
747
|
-
"pa-IN": "pan_Guru",
|
|
748
|
-
"gu-IN": "guj_Gujr",
|
|
749
|
-
"or-IN": "ory_Orya",
|
|
750
|
-
"as-IN": "asm_Beng",
|
|
751
|
-
"ml-IN": "mal_Mlym",
|
|
752
|
-
"ta-IN": "tam_Tamil",
|
|
753
|
-
"te-IN": "tel_Telu",
|
|
754
|
-
"kn-IN": "kan_Knda",
|
|
755
|
-
"si-LK": "sin_Sinh",
|
|
756
|
-
"ne-NP": "nep_Deva",
|
|
757
|
-
"dz-BT": "dzo_Tibt",
|
|
758
|
-
"bo-CN": "bod_Tibt",
|
|
759
|
-
"mn-MN": "khk_Cyrl",
|
|
760
|
-
"ka-GE": "kat_Geor",
|
|
761
|
-
"hy-AM": "hye_Armn",
|
|
762
|
-
"az-AZ": "azj_Latn",
|
|
763
|
-
"tr-TR": "tur_Latn",
|
|
764
|
-
"fa-IR": "pes_Arab",
|
|
765
|
-
"fa-AF": "pes_Arab",
|
|
766
|
-
"ps-AF": "pbt_Arab",
|
|
767
|
-
"ku-TR": "kmr_Latn",
|
|
768
|
-
"he-IL": "heb_Hebr",
|
|
769
|
-
"ar-SA": "arb_Arab",
|
|
770
|
-
"ar-EG": "arb_Arab",
|
|
771
|
-
"ar-MA": "arb_Arab",
|
|
772
|
-
"ar-DZ": "arb_Arab",
|
|
773
|
-
"ar-TN": "arb_Arab",
|
|
774
|
-
"ar-LY": "arb_Arab",
|
|
775
|
-
"ar-SD": "arb_Arab",
|
|
776
|
-
"ar-IQ": "arb_Arab",
|
|
777
|
-
"ar-SY": "arb_Arab",
|
|
778
|
-
"ar-JO": "arb_Arab",
|
|
779
|
-
"ar-LB": "arb_Arab",
|
|
780
|
-
"ar-KW": "arb_Arab",
|
|
781
|
-
"ar-AE": "arb_Arab",
|
|
782
|
-
"ar-QA": "arb_Arab",
|
|
783
|
-
"ar-BH": "arb_Arab",
|
|
784
|
-
"ar-OM": "arb_Arab",
|
|
785
|
-
"ar-YE": "arb_Arab",
|
|
786
|
-
"ar-SO": "arb_Arab",
|
|
787
|
-
"ar-DJ": "arb_Arab",
|
|
788
|
-
"ar-KM": "arb_Arab",
|
|
789
|
-
"ar-TD": "arb_Arab",
|
|
790
|
-
"ar-ER": "arb_Arab",
|
|
791
|
-
"ar-IL": "arb_Arab",
|
|
792
|
-
"ar-PS": "arb_Arab",
|
|
793
|
-
"ar-001": "arb_Arab",
|
|
794
|
-
"sw-KE": "swh_Latn",
|
|
795
|
-
"sw-TZ": "swh_Latn",
|
|
796
|
-
"am-ET": "amh_Ethi",
|
|
797
|
-
"om-ET": "gaz_Latn",
|
|
798
|
-
"so-SO": "som_Latn",
|
|
799
|
-
"ha-NG": "hau_Latn",
|
|
800
|
-
"yo-NG": "yor_Latn",
|
|
801
|
-
"ig-NG": "ibo_Latn",
|
|
802
|
-
"zu-ZA": "zul_Latn",
|
|
803
|
-
"xh-ZA": "xho_Latn",
|
|
804
|
-
"rw-RW": "kin_Latn",
|
|
805
|
-
"lg-UG": "lug_Latn",
|
|
806
|
-
"ak-GH": "aka_Latn",
|
|
807
|
-
"tw-GH": "twi_Latn",
|
|
808
|
-
"ee-GH": "ewe_Latn",
|
|
809
|
-
"ff-SN": "fuv_Latn",
|
|
810
|
-
"wo-SN": "wol_Latn",
|
|
811
|
-
"sn-ZW": "sna_Latn",
|
|
812
|
-
"nd-ZW": "nde_Latn",
|
|
813
|
-
"ny-MW": "nya_Latn",
|
|
814
|
-
"st-ZA": "sot_Latn",
|
|
815
|
-
"tn-BW": "tsn_Latn",
|
|
816
|
-
"ts-ZA": "tso_Latn",
|
|
817
|
-
"ve-ZA": "ven_Latn",
|
|
818
|
-
"ss-ZA": "ssw_Latn",
|
|
819
|
-
"nr-ZA": "nbl_Latn",
|
|
820
|
-
"qu-PE": "quy_Latn",
|
|
821
|
-
"qu-BO": "quy_Latn",
|
|
822
|
-
"qu-EC": "quy_Latn",
|
|
823
|
-
"ay-BO": "aym_Latn",
|
|
824
|
-
"gn-PY": "grn_Latn",
|
|
825
|
-
"ht-HT": "hat_Latn",
|
|
826
|
-
"jv-ID": "jav_Latn",
|
|
827
|
-
"su-ID": "sun_Latn",
|
|
828
|
-
"ceb-PH": "ceb_Latn",
|
|
829
|
-
"ilo-PH": "ilo_Latn",
|
|
830
|
-
"war-PH": "war_Latn",
|
|
831
|
-
"bjn-ID": "bjn_Latn",
|
|
832
|
-
"ban-ID": "ban_Latn",
|
|
833
|
-
"mad-ID": "mad_Latn",
|
|
834
|
-
"min-ID": "min_Latn",
|
|
835
|
-
"ace-ID": "ace_Latn",
|
|
836
|
-
"bug-ID": "bug_Latn",
|
|
837
|
-
"gor-ID": "gor_Latn",
|
|
838
|
-
"mak-ID": "mak_Latn",
|
|
839
|
-
"mdr-ID": "mdr_Latn",
|
|
840
|
-
"sas-ID": "sas_Latn",
|
|
841
|
-
"shn-MM": "shn_Mymr",
|
|
842
|
-
"kac-MM": "kac_Latn",
|
|
843
|
-
"lzh-CN": "lzh_Hant",
|
|
844
|
-
"wuu-CN": "wuu_Hans",
|
|
845
|
-
"yue-CN": "yue_Hant",
|
|
846
|
-
"nan-TW": "nan_Hant",
|
|
847
|
-
"hak-TW": "hak_Hant",
|
|
848
|
-
"cjy-CN": "cjy_Hans",
|
|
849
|
-
"cmn-CN": "cmn_Hans",
|
|
850
|
-
"cmn-TW": "cmn_Hant",
|
|
851
|
-
"cmn-SG": "cmn_Hans",
|
|
852
|
-
"cmn-HK": "cmn_Hant",
|
|
853
|
-
"cmn-MO": "cmn_Hant",
|
|
854
|
-
"cmn-001": "cmn_Hans",
|
|
855
|
-
"haw-US": "haw_Latn",
|
|
856
|
-
"mi-NZ": "mri_Latn",
|
|
857
|
-
"sm-WS": "smo_Latn",
|
|
858
|
-
"to-TO": "ton_Latn",
|
|
859
|
-
"fj-FJ": "fij_Latn",
|
|
860
|
-
"ty-PF": "tah_Latn",
|
|
861
|
-
"co-FR": "cos_Latn",
|
|
862
|
-
"eu-FR": "eus_Latn",
|
|
863
|
-
"kk-KZ": "kaz_Cyrl",
|
|
864
|
-
"ky-KG": "kir_Cyrl",
|
|
865
|
-
"uz-UZ": "uzn_Latn",
|
|
866
|
-
"tg-TJ": "tgk_Cyrl",
|
|
867
|
-
"tk-TM": "tuk_Latn"
|
|
868
|
-
};
|
|
869
|
-
|
|
870
|
-
// src/io/huggingface/ask.ts
|
|
871
|
-
function getLanguageCode(language) {
|
|
872
|
-
return languagesMap[language];
|
|
873
|
-
}
|
|
874
|
-
function progressCallback(progress) {
|
|
875
|
-
process.stdout.clearLine(0);
|
|
876
|
-
process.stdout.cursorTo(0);
|
|
877
|
-
if (progress.status === "download") {
|
|
878
|
-
process.stdout.write("Downloading...");
|
|
879
|
-
} else if (progress.status === "initiate") {
|
|
880
|
-
process.stdout.write("Initializing...");
|
|
881
|
-
} else if (progress.status === "done") {
|
|
882
|
-
process.stdout.write("Done!");
|
|
883
|
-
} else if (progress.status === "progress" && "progress" in progress && "loaded" in progress && "total" in progress) {
|
|
884
|
-
process.stdout.write(
|
|
885
|
-
`Downloading: ${Number(progress.progress).toFixed(2)}%, loaded: ${progress.loaded}/${progress.total}`
|
|
886
|
-
);
|
|
887
|
-
}
|
|
888
|
-
}
|
|
889
|
-
async function pipeLanguage(pipe, original, srcLanguage, targetLanguage) {
|
|
890
|
-
const response = await pipe(original, {
|
|
891
|
-
src_lang: srcLanguage,
|
|
892
|
-
tgt_lang: targetLanguage,
|
|
893
|
-
temperature: 1
|
|
554
|
+
${postPrompt2}`.trim();
|
|
555
|
+
const messages = [new Message("user", fullPrompt)];
|
|
556
|
+
const model = config.llm.model ?? "gpt-4o-mini";
|
|
557
|
+
const response = await engine.complete(engine.buildModel(model), messages, {
|
|
558
|
+
temperature: config.llm.temperature
|
|
894
559
|
});
|
|
895
|
-
|
|
896
|
-
}
|
|
897
|
-
async function askHuggingface(original) {
|
|
898
|
-
const config = await awaitedConfig;
|
|
899
|
-
const cacheDir = `${config.cache.dir}/.cache`;
|
|
900
|
-
env.cacheDir = cacheDir;
|
|
901
|
-
const pipe = await pipeline("translation", config.llm.model, {
|
|
902
|
-
progress_callback: progressCallback,
|
|
903
|
-
cache_dir: cacheDir
|
|
904
|
-
});
|
|
905
|
-
const { languages } = config;
|
|
906
|
-
let responses = [];
|
|
907
|
-
for await (const language of languages) {
|
|
908
|
-
const response = await pipeLanguage(
|
|
909
|
-
pipe,
|
|
910
|
-
original,
|
|
911
|
-
getLanguageCode(config.sourceLanguage),
|
|
912
|
-
getLanguageCode(language)
|
|
913
|
-
);
|
|
914
|
-
if (Array.isArray(response)) {
|
|
915
|
-
responses = responses.concat(
|
|
916
|
-
response.map((r) => ({
|
|
917
|
-
...r,
|
|
918
|
-
targetLanguage: language
|
|
919
|
-
}))
|
|
920
|
-
);
|
|
921
|
-
} else {
|
|
922
|
-
responses.push({
|
|
923
|
-
...response,
|
|
924
|
-
targetLanguage: language
|
|
925
|
-
});
|
|
926
|
-
}
|
|
927
|
-
}
|
|
928
|
-
return { responses, original };
|
|
929
|
-
}
|
|
930
|
-
|
|
931
|
-
// src/io/huggingface/commit-response.ts
|
|
932
|
-
function commitResponseHuggingface(response) {
|
|
933
|
-
const { responses, original } = response;
|
|
934
|
-
const translation = responses.reduce((acc, curr) => {
|
|
935
|
-
acc[curr.targetLanguage] = curr.translation_text;
|
|
936
|
-
return acc;
|
|
937
|
-
}, {});
|
|
938
|
-
return {
|
|
939
|
-
translation,
|
|
940
|
-
original
|
|
941
|
-
};
|
|
942
|
-
}
|
|
943
|
-
|
|
944
|
-
// src/io/huggingface/parse.ts
|
|
945
|
-
function parseHuggingface(unparsed) {
|
|
946
|
-
if (typeof unparsed === "string") {
|
|
947
|
-
throw new Error("CommitResponse should not be a string here.");
|
|
948
|
-
}
|
|
949
|
-
return unparsed;
|
|
950
|
-
}
|
|
951
|
-
|
|
952
|
-
// src/io/ask-llm.ts
|
|
953
|
-
var llmChoices = {
|
|
954
|
-
anthropic: {
|
|
955
|
-
ask: askAnthropic,
|
|
956
|
-
commitResponse: commitResponseAnthropic,
|
|
957
|
-
parse: parseAnthropic
|
|
958
|
-
},
|
|
959
|
-
gemini: {
|
|
960
|
-
ask: askGemini,
|
|
961
|
-
commitResponse: commitResponseGemini,
|
|
962
|
-
parse: parseGemini
|
|
963
|
-
},
|
|
964
|
-
huggingface: {
|
|
965
|
-
ask: askHuggingface,
|
|
966
|
-
commitResponse: commitResponseHuggingface,
|
|
967
|
-
parse: parseHuggingface
|
|
560
|
+
if (config.debug) {
|
|
561
|
+
console.log("LLM response:", JSON.stringify({ provider, model: config.llm.model, response: response.content }));
|
|
968
562
|
}
|
|
969
|
-
|
|
970
|
-
|
|
971
|
-
const response = await llmChoice.ask(prompt, prePrompt2, postPrompt2);
|
|
972
|
-
const unparsed = llmChoice.commitResponse(response);
|
|
973
|
-
const parsed = llmChoice.parse(unparsed);
|
|
974
|
-
return parsed;
|
|
975
|
-
}
|
|
976
|
-
async function askLLM(prompt, prePrompt2, postPrompt2) {
|
|
977
|
-
const config = await awaitedConfig;
|
|
978
|
-
const llmChoiceName = config.llm.choice;
|
|
979
|
-
const llmChoice = llmChoices[llmChoiceName];
|
|
980
|
-
if (!llmChoice) {
|
|
981
|
-
throw new Error("LLMChoice not found");
|
|
563
|
+
if (!response.content) {
|
|
564
|
+
throw new Error("LLM response content is empty");
|
|
982
565
|
}
|
|
983
|
-
return
|
|
566
|
+
return parseResponse(response.content);
|
|
984
567
|
}
|
|
985
568
|
|
|
986
569
|
// src/gringow.ts
|
|
@@ -998,6 +581,10 @@ async function rebuild(cacheId) {
|
|
|
998
581
|
const cached = await checkCache(cacheId);
|
|
999
582
|
if (!cached) throw new Error("Cache item not found");
|
|
1000
583
|
const { prepend, append } = getPrependAndAppend(config, cached);
|
|
584
|
+
if (config.debug) {
|
|
585
|
+
console.log(`rebuild::prepend ${cacheId}:`, { prepend });
|
|
586
|
+
console.log(`rebuild::append ${cacheId}:`, { append });
|
|
587
|
+
}
|
|
1001
588
|
const translation = await askLLM(cached.original, prepend, append);
|
|
1002
589
|
const output = {
|
|
1003
590
|
...translation,
|
|
@@ -1023,6 +610,10 @@ async function Gringow(strings, ...values) {
|
|
|
1023
610
|
return { cacheId, translation: cached };
|
|
1024
611
|
}
|
|
1025
612
|
const { prepend, append } = getPrependAndAppend(config, cached);
|
|
613
|
+
if (config.debug) {
|
|
614
|
+
console.log(`asking::prepend ${cacheId}:`, { prepend });
|
|
615
|
+
console.log(`asking::append ${cacheId}:`, { append });
|
|
616
|
+
}
|
|
1026
617
|
const translation = await askLLM(flatten, prepend, append);
|
|
1027
618
|
const output = {
|
|
1028
619
|
...translation,
|