@gringow/gringow 0.0.1 → 0.0.4

package/src/index.ts

@@ -0,0 +1,37 @@
+import '@/config'
+
+export type GringowTranslation = {
+  [key: string]: string
+}
+
+export type GringowCacheItem = {
+  original: string
+  translation: GringowTranslation
+}
+
+export type GringowCache = {
+  [key: string]: GringowCacheItem
+}
+
+export type GringowResponse = {
+  cacheId: string
+  translation: GringowCacheItem
+}
+
+export type GringowError = Error & {
+  code?: string
+  cause?: Error
+}
+
+export { awaitedConfig, getConfigSource, isLocalConfigFound } from '@/config'
+export { checkCache } from '@/data/check-cache'
+export { clearCache } from '@/data/clear-cache'
+export { createCacheId } from '@/data/create-cache-id'
+export { deleteCacheItem } from '@/data/delete-cache-item'
+export { getCache } from '@/data/get-cache'
+export { getOrCreateCache } from '@/data/get-or-create-cache'
+export { writeCacheToDisk } from '@/data/write-cache-to-disk'
+export { flatTemplateString } from '@/flat-template-string'
+export { Gringow as default } from '@/gringow'
+
+export type { GringowConfig } from '@/default-config'

package/src/io/anthropic/ask.ts

@@ -0,0 +1,31 @@
+import Anthropic from '@anthropic-ai/sdk'
+import { awaitedConfig } from '@/config'
+
+export async function askAnthropic(message: string) {
+  const config = await awaitedConfig
+  if (Number(config.llm.sleep)) {
+    await new Promise((resolve) => setTimeout(resolve, config.llm.sleep ?? 1))
+  }
+
+  if (!config.keys.anthropicApiKey) {
+    throw new Error('Anthropic API key is not set in the config')
+  }
+
+  const anthropic = new Anthropic({
+    apiKey: config.keys.anthropicApiKey,
+    dangerouslyAllowBrowser: true,
+  })
+
+  const response = await anthropic.messages.create({
+    model: config.llm.model as Anthropic.Model,
+    max_tokens: 1024,
+    messages: [
+      {
+        role: 'user',
+        content: `${config.llm.prePrompt}\n\n\n${message}\n\n\n${config.llm.postPrompt}`,
+      },
+    ],
+  })
+
+  return response as Anthropic.Messages.Message
+}

package/src/io/anthropic/commit-response.ts

@@ -0,0 +1,11 @@
+import type { Anthropic } from '@anthropic-ai/sdk'
+import type { CommitResponse } from '@/types'
+
+export function commitResponseAnthropic(response: Anthropic.Messages.Message) {
+  const result = response.content.reduce((acc: string, item: Anthropic.Messages.ContentBlock) => {
+    const nextAcc = acc + (item.type === 'text' ? item.text : '')
+    return nextAcc
+  }, '')
+
+  return result as CommitResponse
+}

package/src/io/anthropic/index.ts

@@ -0,0 +1,3 @@
+export * from './ask'
+export * from './commit-response'
+export * from './parse'

package/src/io/anthropic/parse.ts

@@ -0,0 +1,11 @@
+import type { GringowCacheItem } from '@/index'
+import type { CommitResponse } from '@/types'
+
+export function parseAnthropic(unparsed: CommitResponse): GringowCacheItem {
+  if (typeof unparsed !== 'string') {
+    throw new Error('CommitResponse should be a string here.')
+  }
+
+  const parsed = JSON.parse(unparsed) as GringowCacheItem
+  return parsed
+}

package/src/io/ask-llm.ts

@@ -0,0 +1,44 @@
+import { awaitedConfig } from '@/config'
+import { askAnthropic, commitResponseAnthropic, parseAnthropic } from './anthropic'
+import { askGemini, commitResponseGemini, parseGemini } from './gemini'
+import { askHuggingface, commitResponseHuggingface, parseHuggingface } from './huggingface'
+
+import type { GringowCacheItem } from '@/index'
+import type { LLMChoice, LLMChoices, LLMOptions } from '@/types'
+
+const llmChoices: LLMChoices = {
+  anthropic: {
+    ask: askAnthropic,
+    commitResponse: commitResponseAnthropic,
+    parse: parseAnthropic,
+  },
+  gemini: {
+    ask: askGemini,
+    commitResponse: commitResponseGemini,
+    parse: parseGemini,
+  },
+  huggingface: {
+    ask: askHuggingface,
+    commitResponse: commitResponseHuggingface,
+    parse: parseHuggingface,
+  },
+}
+
+async function processChoice<T = { test: number }>(choice: LLMChoice<T>, prompt: string): Promise<GringowCacheItem> {
+  const response = await choice.ask(prompt)
+  const unparsed = choice.commitResponse(response)
+  const parsed = choice.parse(unparsed)
+  return parsed
+}
+
+export async function askLLM(prompt: string): Promise<GringowCacheItem> {
+  const config = await awaitedConfig
+  const llmChoiceName = config.llm.choice as LLMOptions
+  const llmChoice = llmChoices[llmChoiceName]
+
+  if (!llmChoice) {
+    throw new Error('LLMChoice not found')
+  }
+
+  return processChoice<Awaited<ReturnType<typeof llmChoice.ask>>>(llmChoice, prompt)
+}

package/src/io/gemini/ask.ts

@@ -0,0 +1,65 @@
+import { type GenerateContentResult, GoogleGenerativeAI } from '@google/generative-ai'
+
+import { awaitedConfig } from '@/config'
+
+async function generateResponse(message: string, temperature: number = 1.0) {
+  const config = await awaitedConfig
+  if (!config.keys.geminiApiKey) {
+    throw new Error('Gemini API key is not set in the config')
+  }
+
+  const generativeAIClient = new GoogleGenerativeAI(config.keys.geminiApiKey)
+  const model = generativeAIClient.getGenerativeModel({
+    model: config.llm.model as string,
+    generationConfig: {
+      temperature,
+    },
+  })
+
+  const prompt = String(`${config.llm.prePrompt}\n\n\n${message}\n\n\n${config.llm.postPrompt}`).trim()
+  if (config.debug) {
+    console.log('Gemini model prompt:', JSON.stringify({ prompt }))
+  }
+
+  const response = await model.generateContent([prompt])
+
+  return response
+}
+
+export async function askGemini(message: string) {
+  const config = await awaitedConfig
+  if (Number(config.llm.sleep)) {
+    await new Promise((resolve) => setTimeout(resolve, config.llm.sleep ?? 1))
+  }
+
+  if (!config.keys.geminiApiKey) {
+    throw new Error('Gemini API key is not set in the config')
+  }
+
+  if (config.debug) {
+    console.log('\nAsking Gemini model:', JSON.stringify({ model: config.llm.model, message }))
+  }
+
+  let response: GenerateContentResult | undefined
+  for (let i = 0; i < 3; i++) {
+    const maybeResponse = await generateResponse(message, config.llm.temperature)
+    if (!String(maybeResponse.response.candidates?.[0]?.content?.parts?.[0]?.text ?? '').includes('```json')) {
+      response = maybeResponse
+      break
+    }
+
+    if (config.debug) {
+      console.log('Gemini model response is not valid, contains no JSON:', JSON.stringify({ maybeResponse }))
+    }
+  }
+
+  if (config.debug) {
+    console.log('Gemini model response:', JSON.stringify({ response }))
+  }
+
+  if (!response) {
+    throw new Error('Gemini model response is not valid')
+  }
+
+  return response
+}

package/src/io/gemini/commit-response.ts

@@ -0,0 +1,6 @@
+import type { GenerateContentResult } from '@google/generative-ai'
+
+export function commitResponseGemini(response: GenerateContentResult): string {
+  const result = response.response.text()
+  return result
+}

package/src/io/gemini/index.ts

@@ -0,0 +1,3 @@
+export * from './ask'
+export * from './commit-response'
+export * from './parse'

package/src/io/gemini/parse.ts

@@ -0,0 +1,12 @@
+import type { GringowCacheItem } from '@/index'
+import type { CommitResponse } from '@/types'
+
+export function parseGemini(unparsed: CommitResponse): GringowCacheItem {
+  if (typeof unparsed !== 'string') {
+    throw new Error('CommitResponse should be a string here.')
+  }
+
+  const cleaned = unparsed.replace(/```json/g, '').replace(/```/g, '')
+  const parsed = JSON.parse(cleaned) as GringowCacheItem
+  return parsed
+}

package/src/io/huggingface/ask.ts

@@ -0,0 +1,82 @@
+import {
+  env,
+  type ProgressInfo,
+  pipeline,
+  type TranslationPipeline,
+  type TranslationSingle,
+} from '@huggingface/transformers'
+import 'node:fs'
+
+import { awaitedConfig } from '@/config'
+import { languagesMap } from './languagesMap'
+
+import type { AskHuggingfaceResponse, AskHuggingfaceResponseTranslation } from '@/io/huggingface'
+
+function getLanguageCode(language: string) {
+  return languagesMap[language as keyof typeof languagesMap]
+}
+
+function progressCallback(progress: ProgressInfo) {
+  process.stdout.clearLine(0)
+  process.stdout.cursorTo(0)
+  if (progress.status === 'download') {
+    process.stdout.write('Downloading...')
+  } else if (progress.status === 'initiate') {
+    process.stdout.write('Initializing...')
+  } else if (progress.status === 'done') {
+    process.stdout.write('Done!')
+  } else if (progress.status === 'progress' && 'progress' in progress && 'loaded' in progress && 'total' in progress) {
+    process.stdout.write(
+      `Downloading: ${Number(progress.progress).toFixed(2)}%, loaded: ${progress.loaded}/${progress.total}`
+    )
+  }
+}
+
+async function pipeLanguage(pipe: TranslationPipeline, original: string, srcLanguage: string, targetLanguage: string) {
+  const response = await pipe(original, {
+    src_lang: srcLanguage,
+    tgt_lang: targetLanguage,
+    temperature: 1.0,
+  } as Parameters<TranslationPipeline['call']>[1])
+
+  return response
+}
+
+export async function askHuggingface(original: string) {
+  const config = await awaitedConfig
+  const cacheDir = `${config.cache.dir}/.cache`
+  env.cacheDir = cacheDir
+
+  const pipe = await pipeline('translation', config.llm.model as string, {
+    progress_callback: progressCallback,
+    cache_dir: cacheDir,
+  })
+
+  const { languages } = config
+
+  let responses: AskHuggingfaceResponseTranslation[] = []
+  for await (const language of languages) {
+    const response = await pipeLanguage(
+      pipe,
+      original,
+      getLanguageCode(config.sourceLanguage),
+      getLanguageCode(language)
+    )
+
+    if (Array.isArray(response)) {
+      responses = responses.concat(
+        response.map((r) => ({
+          ...r,
+          targetLanguage: language,
+        })) as AskHuggingfaceResponseTranslation[]
+      )
+    } else {
+      responses.push({
+        ...(response as TranslationSingle),
+        targetLanguage: language,
+      } as AskHuggingfaceResponseTranslation)
+    }
+  }
+
+  return { responses, original } as AskHuggingfaceResponse
+}

package/src/io/huggingface/commit-response.ts

@@ -0,0 +1,16 @@
+import type { GringowTranslation } from '@/index'
+import type { AskHuggingfaceResponse } from '@/io/huggingface'
+import type { CommitResponse } from '@/types'
+
+export function commitResponseHuggingface(response: AskHuggingfaceResponse) {
+  const { responses, original } = response
+  const translation = responses.reduce((acc, curr) => {
+    acc[curr.targetLanguage] = curr.translation_text
+    return acc
+  }, {} as GringowTranslation)
+
+  return {
+    translation,
+    original,
+  } as CommitResponse
+}

package/src/io/huggingface/index.ts

@@ -0,0 +1,16 @@
+import type { TranslationSingle } from '@huggingface/transformers'
+
+export type AskHuggingfaceResponseTranslation = TranslationSingle & {
+  targetLanguage: string
+}
+
+export type AskHuggingfaceResponse = {
+  responses: AskHuggingfaceResponseTranslation[]
+  original: string
+}
+
+export type AskHuggingfaceResult = string
+
+export * from './ask'
+export * from './commit-response'
+export * from './parse'

package/src/io/huggingface/languagesMap.ts

@@ -0,0 +1,189 @@
+export const languagesMap = {
+  'en-US': 'eng_Latn',
+  'en-GB': 'eng_Latn',
+  'pt-BR': 'por_Latn',
+  'pt-PT': 'por_Latn',
+  'es-ES': 'spa_Latn',
+  'es-MX': 'spa_Latn',
+  'fr-FR': 'fra_Latn',
+  'fr-CA': 'fra_Latn',
+  'de-DE': 'deu_Latn',
+  'it-IT': 'ita_Latn',
+  'nl-NL': 'nld_Latn',
+  'pl-PL': 'pol_Latn',
+  'ru-RU': 'rus_Cyrl',
+  'uk-UA': 'ukr_Cyrl',
+  'bg-BG': 'bul_Cyrl',
+  'hr-HR': 'hrv_Latn',
+  'cs-CZ': 'ces_Latn',
+  'sk-SK': 'slk_Latn',
+  'hu-HU': 'hun_Latn',
+  'ro-RO': 'ron_Latn',
+  'fi-FI': 'fin_Latn',
+  'sv-SE': 'swe_Latn',
+  'da-DK': 'dan_Latn',
+  'no-NO': 'nob_Latn',
+  'is-IS': 'isl_Latn',
+  'et-EE': 'est_Latn',
+  'lv-LV': 'lav_Latn',
+  'lt-LT': 'lit_Latn',
+  'mt-MT': 'mlt_Latn',
+  'ga-IE': 'gle_Latn',
+  'cy-GB': 'cym_Latn',
+  'eu-ES': 'eus_Latn',
+  'ca-ES': 'cat_Latn',
+  'gl-ES': 'glg_Latn',
+  'oc-FR': 'oci_Latn',
+  'br-FR': 'bre_Latn',
+  'gd-GB': 'gla_Latn',
+  'kw-GB': 'cor_Latn',
+  'fur-IT': 'fur_Latn',
+  'rm-CH': 'roh_Latn',
+  'lb-LU': 'ltz_Latn',
+  'fy-NL': 'fry_Latn',
+  'af-ZA': 'afr_Latn',
+
+  'zh-CN': 'zho_Hans',
+  'zh-TW': 'zho_Hant',
+  'zh-HK': 'zho_Hant',
+  'ja-JP': 'jpn_Jpan',
+  'ko-KR': 'kor_Hang',
+  'th-TH': 'tha_Thai',
+  'vi-VN': 'vie_Latn',
+  'id-ID': 'ind_Latn',
+  'ms-MY': 'zsm_Latn',
+  'tl-PH': 'tgl_Latn',
+  'my-MM': 'mya_Mymr',
+  'km-KH': 'khm_Khmr',
+  'lo-LA': 'lao_Laoo',
+  'bn-BD': 'ben_Beng',
+  'bn-IN': 'ben_Beng',
+  'hi-IN': 'hin_Deva',
+  'ur-PK': 'urd_Arab',
+  'ur-IN': 'urd_Arab',
+  'pa-IN': 'pan_Guru',
+  'gu-IN': 'guj_Gujr',
+  'or-IN': 'ory_Orya',
+  'as-IN': 'asm_Beng',
+  'ml-IN': 'mal_Mlym',
+  'ta-IN': 'tam_Tamil',
+  'te-IN': 'tel_Telu',
+  'kn-IN': 'kan_Knda',
+  'si-LK': 'sin_Sinh',
+  'ne-NP': 'nep_Deva',
+  'dz-BT': 'dzo_Tibt',
+  'bo-CN': 'bod_Tibt',
+  'mn-MN': 'khk_Cyrl',
+  'ka-GE': 'kat_Geor',
+  'hy-AM': 'hye_Armn',
+  'az-AZ': 'azj_Latn',
+  'tr-TR': 'tur_Latn',
+  'fa-IR': 'pes_Arab',
+  'fa-AF': 'pes_Arab',
+  'ps-AF': 'pbt_Arab',
+  'ku-TR': 'kmr_Latn',
+  'he-IL': 'heb_Hebr',
+  'ar-SA': 'arb_Arab',
+  'ar-EG': 'arb_Arab',
+  'ar-MA': 'arb_Arab',
+  'ar-DZ': 'arb_Arab',
+  'ar-TN': 'arb_Arab',
+  'ar-LY': 'arb_Arab',
+  'ar-SD': 'arb_Arab',
+  'ar-IQ': 'arb_Arab',
+  'ar-SY': 'arb_Arab',
+  'ar-JO': 'arb_Arab',
+  'ar-LB': 'arb_Arab',
+  'ar-KW': 'arb_Arab',
+  'ar-AE': 'arb_Arab',
+  'ar-QA': 'arb_Arab',
+  'ar-BH': 'arb_Arab',
+  'ar-OM': 'arb_Arab',
+  'ar-YE': 'arb_Arab',
+  'ar-SO': 'arb_Arab',
+  'ar-DJ': 'arb_Arab',
+  'ar-KM': 'arb_Arab',
+  'ar-TD': 'arb_Arab',
+  'ar-ER': 'arb_Arab',
+  'ar-IL': 'arb_Arab',
+  'ar-PS': 'arb_Arab',
+  'ar-001': 'arb_Arab',
+
+  'sw-KE': 'swh_Latn',
+  'sw-TZ': 'swh_Latn',
+  'am-ET': 'amh_Ethi',
+  'om-ET': 'gaz_Latn',
+  'so-SO': 'som_Latn',
+  'ha-NG': 'hau_Latn',
+  'yo-NG': 'yor_Latn',
+  'ig-NG': 'ibo_Latn',
+  'zu-ZA': 'zul_Latn',
+  'xh-ZA': 'xho_Latn',
+  'rw-RW': 'kin_Latn',
+  'lg-UG': 'lug_Latn',
+  'ak-GH': 'aka_Latn',
+  'tw-GH': 'twi_Latn',
+  'ee-GH': 'ewe_Latn',
+  'ff-SN': 'fuv_Latn',
+  'wo-SN': 'wol_Latn',
+  'sn-ZW': 'sna_Latn',
+  'nd-ZW': 'nde_Latn',
+  'ny-MW': 'nya_Latn',
+  'st-ZA': 'sot_Latn',
+  'tn-BW': 'tsn_Latn',
+  'ts-ZA': 'tso_Latn',
+  've-ZA': 'ven_Latn',
+  'ss-ZA': 'ssw_Latn',
+  'nr-ZA': 'nbl_Latn',
+
+  'qu-PE': 'quy_Latn',
+  'qu-BO': 'quy_Latn',
+  'qu-EC': 'quy_Latn',
+  'ay-BO': 'aym_Latn',
+  'gn-PY': 'grn_Latn',
+  'ht-HT': 'hat_Latn',
+  'jv-ID': 'jav_Latn',
+  'su-ID': 'sun_Latn',
+  'ceb-PH': 'ceb_Latn',
+  'ilo-PH': 'ilo_Latn',
+  'war-PH': 'war_Latn',
+  'bjn-ID': 'bjn_Latn',
+  'ban-ID': 'ban_Latn',
+  'mad-ID': 'mad_Latn',
+  'min-ID': 'min_Latn',
+  'ace-ID': 'ace_Latn',
+  'bug-ID': 'bug_Latn',
+  'gor-ID': 'gor_Latn',
+  'mak-ID': 'mak_Latn',
+  'mdr-ID': 'mdr_Latn',
+  'sas-ID': 'sas_Latn',
+  'shn-MM': 'shn_Mymr',
+  'kac-MM': 'kac_Latn',
+  'lzh-CN': 'lzh_Hant',
+  'wuu-CN': 'wuu_Hans',
+  'yue-CN': 'yue_Hant',
+  'nan-TW': 'nan_Hant',
+  'hak-TW': 'hak_Hant',
+  'cjy-CN': 'cjy_Hans',
+  'cmn-CN': 'cmn_Hans',
+  'cmn-TW': 'cmn_Hant',
+  'cmn-SG': 'cmn_Hans',
+  'cmn-HK': 'cmn_Hant',
+  'cmn-MO': 'cmn_Hant',
+  'cmn-001': 'cmn_Hans',
+
+  'haw-US': 'haw_Latn',
+  'mi-NZ': 'mri_Latn',
+  'sm-WS': 'smo_Latn',
+  'to-TO': 'ton_Latn',
+  'fj-FJ': 'fij_Latn',
+  'ty-PF': 'tah_Latn',
+  'co-FR': 'cos_Latn',
+  'eu-FR': 'eus_Latn',
+
+  'kk-KZ': 'kaz_Cyrl',
+  'ky-KG': 'kir_Cyrl',
+  'uz-UZ': 'uzn_Latn',
+  'tg-TJ': 'tgk_Cyrl',
+  'tk-TM': 'tuk_Latn',
+}

package/src/io/huggingface/parse.ts

@@ -0,0 +1,10 @@
+import type { GringowCacheItem } from '@/index'
+import type { CommitResponse } from '@/types'
+
+export function parseHuggingface(unparsed: CommitResponse): GringowCacheItem {
+  if (typeof unparsed === 'string') {
+    throw new Error('CommitResponse should not be a string here.')
+  }
+
+  return unparsed
+}

package/src/logger.ts

@@ -0,0 +1,25 @@
+// "error", "warn", "info", "debug", "trace"
+const levels = {
+  error: 0,
+  warn: 1,
+  info: 2,
+  debug: 3,
+  trace: 4,
+}
+
+type Level = keyof typeof levels
+
+const currentLevel: Level = 'info'
+const maxLevel = levels[currentLevel]
+
+const noop = () => void 0
+
+const logger = {
+  error: maxLevel >= levels.error ? console.error : noop,
+  warn: maxLevel >= levels.warn ? console.warn : noop,
+  info: maxLevel >= levels.info ? console.info : noop,
+  debug: maxLevel >= levels.debug ? console.debug : noop,
+  trace: maxLevel >= levels.trace ? console.trace : noop,
+}
+
+export default logger

package/src/schemas/config.shcema.ts

@@ -0,0 +1,58 @@
+import { z } from 'zod'
+
+export const languageSchema = z.string().regex(/^[a-z]{2,3}(-[A-Z]{2})?$/)
+
+export const llmOptionsSchema = z.enum(['gemini', 'anthropic', 'huggingface'])
+
+export const configSchema = z
+  .object({
+    debug: z.boolean().optional(),
+    log: z.object({
+      level: z.enum(['debug', 'info', 'warn', 'error', 'trace']),
+    }),
+    cache: z.object({
+      dir: z.string(),
+      file: z.string(),
+      prefix: z.string(),
+      write: z.boolean().optional(),
+    }),
+    keys: z.object({
+      anthropicApiKey: z.string().optional(),
+      geminiApiKey: z.string().optional(),
+    }),
+    sourceLanguage: languageSchema,
+    languages: z.array(languageSchema),
+    llm: z.object({
+      choice: llmOptionsSchema,
+      model: z.string().nullable(),
+      temperature: z.number().optional(),
+      sleep: z.number().optional(),
+      prePrompt: z.string().optional(),
+      postPrompt: z.string().optional(),
+      prePromptReplace: z.string().optional(),
+      postPromptReplace: z.string().optional(),
+      prePromptAppend: z.string().optional(),
+      postPromptAppend: z.string().optional(),
+    }),
+    globby: z
+      .string()
+      .regex(/^\.\/.*/)
+      .optional(),
+  })
+  .refine(
+    (data) => {
+      if (data.llm.choice === 'gemini' && !data.keys.geminiApiKey) {
+        return false
+      }
+
+      if (data.llm.choice === 'anthropic' && !data.keys.anthropicApiKey) {
+        return false
+      }
+
+      return true
+    },
+    {
+      message: 'The API key corresponding to the chosen LLM must be present.',
+      path: ['keys'],
+    }
+  )

package/src/types/index.ts

@@ -0,0 +1,22 @@
+import type { z } from 'zod'
+import type { GringowCacheItem, GringowTranslation } from '@/index'
+import type { llmOptionsSchema } from '@/schemas/config.shcema'
+
+export type CommitResponse = string | { translation: GringowTranslation; original: string }
+
+export type Asker<T> = (message: string) => Promise<T>
+export type Commiter<T> = (response: T) => CommitResponse
+export type Parser = (unparsed: CommitResponse) => GringowCacheItem
+
+export type LLMOptions = z.infer<typeof llmOptionsSchema>
+
+export type LLMChoice<T> = {
+  ask: Asker<T>
+  commitResponse: Commiter<T>
+  parse: Parser
+}
+
+export type LLMChoices = {
+  // biome-ignore lint/suspicious/noExplicitAny: LLMChoice can receive any type
+  [K in LLMOptions]: LLMChoice<any>
+}