@drone1/alt 0.4.2 → 0.7.1

package/src/commands/list-models.js

@@ -0,0 +1,6 @@
+import { loadTranslationProvider } from '../lib/provider.js'
+
+export async function runListModels({ appState, options, log }) {
+	const { apiKey, api } = await loadTranslationProvider({ __dirname: appState.__dirname, providerName: options.provider, log })
+	return log.I(await api.listModels(apiKey))
+}

package/src/{translate.js → commands/translate.js}

@@ -2,23 +2,22 @@ import * as path from 'path'
 import axios from 'axios'
 import { fileURLToPath } from 'url'
 import { Listr } from 'listr2'
-import { localize, localizeFormatted } from './localizer/localize.js'
+import { localize, localizeFormatted } from '../localizer/localize.js'
 import {
-	DEFAULT_CACHE_FILENAME,
+	DEFAULT_CACHE_FILENAME, DEFAULT_LLM_MODELS,
 	OVERLOADED_BACKOFF_INTERVAL_MS,
 	TRANSLATION_FAILED_RESPONSE_TEXT,
 	VALID_TRANSLATION_PROVIDERS
-} from './consts.js'
-import { assertValidPath } from './assert.js'
-import { copyFileToTempAndEnsureExtension, importJsFile, mkTmpDir, normalizeOutputPath, readFileAsText, readJsonFile, rmDir, writeJsonFile } from './io.js'
-import { calculateHash, normalizeData, sleep } from './utils.js'
-import { formatContextKeyFromKey, isContextKey } from './context-keys.js'
-import { loadConfig } from './config.js'
-import { loadTranslationProvider } from './provider.js'
-import { loadCache } from './cache.js'
-import { shutdown } from './shutdown.js'
-
-const __dirname = path.dirname(fileURLToPath(import.meta.url))
+} from '../lib/consts.js'
+import { assertValidPath } from '../lib/assert.js'
+import { mkTmpDir, normalizeOutputPath, readFileAsText, readJsonFile, writeJsonFile } from '../lib/io.js'
+import { calculateHash, normalizeData, sleep } from '../lib/utils.js'
+import { formatContextKeyFromKey, isContextKey } from '../lib/context-keys.js'
+import { loadConfig } from '../lib/config.js'
+import { loadTranslationProvider } from '../lib/provider.js'
+import { loadCache } from '../lib/cache.js'
+import { shutdown } from '../shutdown.js'
+import { loadReferenceFile } from '../lib/reference-loader.js'
 
 export async function runTranslation({ appState, options, log }) {
 	let exitCode = 0
@@ -70,15 +69,9 @@ export async function runTranslation({ appState, options, log }) {
 		appState.tmpDir = tmpDir
 
 		// Copy to a temp location first so we can ensure it has an .mjs extension
-		const tmpReferencePath = await copyFileToTempAndEnsureExtension({
-			filePath: options.referenceFile,
-			tmpDir,
-			ext: 'mjs',
-		})
-		const referenceContent = normalizeData(JSON.parse(JSON.stringify(await importJsFile(tmpReferencePath))), log)  // TODO: Don't do this
-		const referenceData = referenceContent[options.referenceVarName]
+		const referenceData = await loadReferenceFile({ appLang: appState.lang, options, tmpDir, log })
 		if (!referenceData) {
-			log.E(`No reference data found in variable "${options.referenceVarName}" in ${options.referenceFile}`)
+			log.E(`No reference data found in variable "${options.referenceExportedVarName}" in ${options.referenceFile}`)
 			process.exit(2)
 		}
 
@@ -98,7 +91,7 @@ export async function runTranslation({ appState, options, log }) {
 		assertValidPath(cacheFilePath)
 		appState.filesToWrite[cacheFilePath] = writableCache
 
-		const { apiKey, api: translationProvider } = await loadTranslationProvider({ __dirname, providerName, log })
+		const { apiKey, api: translationProvider } = await loadTranslationProvider({ __dirname: appState.__dirname, providerName, log })
 		log.V(`translation provider "${providerName}" loaded`)
 
 		const addContextToTranslation = options.lookForContextData || config.lookForContextData
@@ -153,7 +146,7 @@ export async function runTranslation({ appState, options, log }) {
 					suffix: contextSuffix
 				})
 				log.T(`contextKey=${contextKey}`)
-				const storedHashForReferenceValue = readOnlyCache?.referenceKeyHashes?.[key]
+				const storedHashForReferenceValue = readOnlyCache?.referenceKeyHashes?.[targetLang]?.[key]	// See https://github.com/drone1/alt/issues/1
 				const storedHashForTargetLangAndValue = readOnlyCache.state[targetLang]?.keyHashes?.[key]
 				const refValue = referenceData[key]
 				const refContextValue = (contextKey in referenceData) ? referenceData[contextKey] : null
@@ -161,8 +154,28 @@ export async function runTranslation({ appState, options, log }) {
 				const curValue = (key in outputData) ? outputData[key] : null
 
 				// Skip non-string values (objects, arrays, etc.)
-				if (typeof refValue !== 'string') {
-					errors.push(`Value for reference key "${key}" was not a string! Skipping...`)
+				const refValueType = typeof refValue
+				if (refValueType !== 'string') {
+					if (refValueType === 'undefined') {
+						// This can happen if a user specifies a key explicitly via --keys
+						errors.push(
+							localizeFormatted({
+								token: 'error-value-not-in-reference-data',
+								data: { key },
+								lang: appState.lang,
+								log
+							})
+						)
+					} else {
+						errors.push(
+							localizeFormatted({
+								token: 'error-value-not-a-string',
+								data: { key, type: refValueType },
+								lang: appState.lang,
+								log
+							})
+						)
+					}
 					continue
 				}
 
@@ -249,7 +262,6 @@ export async function runTranslation({ appState, options, log }) {
 			}
 		}
 
-		let nextTaskDelayMs = 0
 		let totalTasks = workQueue.length
 		let errorsEncountered = 0
 		for (const taskInfoIdx in workQueue) {
@@ -266,18 +278,14 @@ export async function runTranslation({ appState, options, log }) {
 						log
 					}),
 					task: async (ctx, task) => {
-						ctx.nextTaskDelayMs = nextTaskDelayMs
-
 						return task.newListr([
 							{
 								title: localize({ token: 'msg-translating', lang: appState.lang, log }),
 								task: async (_, task) => {
 									const translationResult = await processTranslationTask({
-										appState, taskInfo, listrTask: task, listrCtx: ctx, options, log
+										appState, taskInfo, listrTask: task, options, log
 									})
 
-									nextTaskDelayMs = translationResult.nextTaskDelayMs
-
 									if (translationResult.error) {
 										++errorsEncountered
 										throw new Error(translationResult.error)
@@ -321,7 +329,7 @@ export async function runTranslation({ appState, options, log }) {
 			log.I(`\x1B[38;2;44;190;78m✔\x1B[0m ${localize({ token: 'msg-nothing-to-do', lang: appState.lang, log })}`)
 		}
 	} catch (error) {
-		log.E('Error:', error)
+		log.E(error)
 		exitCode = 2
 	}
 
@@ -332,50 +340,27 @@ export async function runTranslation({ appState, options, log }) {
 	}
 }
 
-export async function processTranslationTask({ appState, taskInfo, listrTask, listrCtx, options, log }) {
+export async function processTranslationTask({ appState, taskInfo, listrTask, options, log }) {
 	const { key, sourceLang, targetLang, reasonsForTranslationMap, outputData, outputFilePath, writableCache, cacheFilePath, state } = taskInfo
-	const {
-		translationProvider,
-		apiKey,
-		appContextMessage,
-		storedHashForReferenceValue,
-		refValue,
-		refContextValue,
-		referenceValueHash,
-		userMissingReferenceValueHash,
-		userModifiedReferenceValue,
-		curValue,
-		currentValueHash,
-		storedHashForTargetLangAndValue
-	} = state
-
-	let reasons = Object.keys(reasonsForTranslationMap)
+	const { referenceValueHash } = state
+
+	listrTask.output = Object.keys(reasonsForTranslationMap)
 		.map(k => localize({ token: `msg-translation-reason-${k}`, lang: appState.lang, log }))
 		.join(', ')
-	listrTask.output = reasons
 
 	const {
 		success,
 		translated,
-		nextTaskDelayMs,
 		newValue,
 		error
 	} = await translateKeyForLanguage({
 		appState,
 		listrTask,
-		ctx: listrCtx,
-		translationProvider,
-		apiKey,
-		referenceValueHash,
-		storedHashForReferenceValue,
-		storedHashForTargetLangAndValue,
 		sourceLang,
 		targetLang,
 		key,
-		refValue,
-		refContextValue,
-		curValue,
 		options,
+		state,
 		log
 	})
 
@@ -398,7 +383,8 @@ export async function processTranslationTask({ appState, taskInfo, listrTask, li
 			listrTask.output = localizeFormatted({ token: 'msg-show-translation-result', data: { key, newValue }, lang: appState.lang, log })
 
 			// Update the hash for the reference key, so we can monitor if the user changed a specific key
-			writableCache.referenceKeyHashes[key] = referenceValueHash
+			writableCache.referenceKeyHashes[targetLang] = writableCache.referenceKeyHashes[targetLang] || {}
+			writableCache.referenceKeyHashes[targetLang][key] = referenceValueHash
 
 			// Update state file every time, in case the user kills the process
 			if (options.realtimeWrites) {
@@ -420,57 +406,43 @@ export async function processTranslationTask({ appState, taskInfo, listrTask, li
 		appState.filesToWrite[outputFilePath] = outputData
 	}
 
-	return { nextTaskDelayMs, error }
+	return { error }
 }
 
 async function translateKeyForLanguage({
 																				 appState,
 																				 listrTask,
-																				 ctx,
-																				 translationProvider,
-																				 apiKey,
-																				 appContextMessage,
 																				 sourceLang,
 																				 targetLang,
+																				 state,
 																				 key,
-																				 refValue,
-																				 refContextValue,
-																				 options: { maxRetries },
+																				 options: { maxRetries, model },
 																				 log
 																			 }) {
-	const result = { success: false, translated: false, newValue: null, nextTaskDelayMs: 0, error: null }
+	const { translationProvider, apiKey, appContextMessage, refValue, refContextValue } = state
+	const result = { success: false, translated: false, newValue: null, error: null }
+
+	const providerName = translationProvider.name().toLowerCase()
+	model = model ?? DEFAULT_LLM_MODELS[providerName]
+	if (!model?.length) {
+		throw new Error(
+			localizeFormatted({ token: 'error-invalid-llm-model', data: { model }, lang: appState.lang, log })
+		)
+	}
 
 	// Call translation provider
 	log.D(`[${targetLang}] Translating "${key}"...`)
 	listrTask.output = localizeFormatted({ token: 'msg-translating-key', data: { key }, lang: appState.lang, log })
 
-	const providerName = translationProvider.name()
-	let translated = null
 	let newValue
 
-	// Because of the simple (naive) way we handle being rate-limited and backing off, we kind of want to retry forever but not forever.
-	// A single key may need to retry many times, since the algorithm is quite simple: if a task is told to retry after 10s,
-	// any subsequent tasks that run will delay 10s also, then those concurrent remaining tasks will all hammer at once, some
-	// will complete (maybe), then we'll wait again, then hammer again. A more proper solution may or may not be forthcoming...
-	for (let attempt = 0; !newValue && attempt <= maxRetries; ++attempt) {
+	for (let attempt = 0; !newValue?.length && attempt <= maxRetries; ++attempt) {
 		const attemptStr = attempt > 0 ? ` [Attempt: ${attempt + 1}]` : ''
 		log.D(`[translate] attempt=${attempt}`)
 
-		log.D('next task delay', ctx.nextTaskDelayMs)
-		if (ctx.nextTaskDelayMs > 0) {
-			const msg = localizeFormatted({
-				token: 'msg-rate-limited-sleeping',
-				data: { interval: Math.floor(ctx.nextTaskDelayMs / 1000), attemptStr }, lang: appState.lang, log
-			})
-			listrTask.output = msg
-			log.D(msg)
-			await sleep(ctx.nextTaskDelayMs)
-		}
-
 		const translateResult = await translate({
 			appState,
 			listrTask,
-			ctx,
 			provider: translationProvider,
 			text: refValue,
 			context: refContextValue,
@@ -478,16 +450,23 @@ async function translateKeyForLanguage({
 			targetLang,
 			appContextMessage,
 			apiKey,
+			model,
 			maxRetries: maxRetries,
 			attemptStr,
 			log
 		})
 
-		if (translateResult.backoffInterval > 0) {
-			log.D(`backing off... interval: ${translateResult.backoffInterval}`)
-			//listrTask.output = 'Rate limited'
-			log.D(`ctx.nextTaskDelayMs=${ctx.nextTaskDelayMs}`)
-			result.nextTaskDelayMs = Math.max(ctx.nextTaskDelayMs, translateResult.backoffInterval)
+		const { backoffInterval } = translateResult
+		if (backoffInterval > 0) {
+			log.D(`backing off... interval: ${backoffInterval}`)
+
+			if (backoffInterval > 0) {
+				listrTask.output = localizeFormatted({
+					token: 'msg-rate-limited-sleeping',
+					data: { interval: Math.floor(backoffInterval / 1000), attemptStr }, lang: appState.lang, log
+				})
+				await sleep(backoffInterval)
+			}
 		} else {
 			newValue = translateResult.translated
 			result.success = true
@@ -499,9 +478,40 @@ async function translateKeyForLanguage({
 		result.translated = true
 		result.newValue = newValue
 	} else {
-		result.error = `Translation was empty; target language=${targetLang}; key=${key}; text=${refValue}`
+		result.error = localizeFormatted({ token: 'error-translation-failed', data: { targetLang, key, refValue }, lang: appState.lang, log })
+	}
+
+	return result
+}
+
+async function translate({
+													 appState,
+													 listrTask,
+													 provider,
+													 appContextMessage,
+													 text,
+													 context,
+													 sourceLang,
+													 targetLang,
+													 apiKey,
+													 model,
+													 attemptStr,
+													 log
+												 }) {
+	log.D(`[translate] sourceLang=${sourceLang}; targetLang=${targetLang}; text=${text}`)
+	const result = { translated: null, backoffInterval: 0 }
+
+	if (sourceLang === targetLang) {
+		log.D(`Using reference value since source & target language are the same`)
+		result.translated = text
+	} else {
+		await translateTextViaProvider({
+			appState, provider, listrTask, sourceLang, targetLang, appContextMessage, context, text, log, apiKey, model, attemptStr, providerName: provider.name(), outResult: result
+		})
 	}
 
+	log.D(`[translate] `, result)
+
 	return result
 }
 
@@ -516,9 +526,10 @@ async function translateTextViaProvider({
 																					text,
 																					log,
 																					apiKey,
+																					model,
 																					attemptStr,
-																					result,
-																					providerName
+																					providerName,
+																					outResult
 																				}) {
 	try {
 		const providerName = provider.name()
@@ -541,7 +552,7 @@ async function translateTextViaProvider({
 			+ `\n\n${text}`
 		)
 		log.D(`prompt: `, messages)
-		const { url, params, config } = provider.getTranslationRequestDetails({ messages, apiKey, log })
+		const { url, params, config } = provider.getTranslationRequestDetails({ model, messages, apiKey, log })
 		log.T('url: ', url, 'params: ', params, 'config: ', config)
 		listrTask.output = localizeFormatted({ token: 'msg-hitting-provider-endpoint', data: { providerName, attemptStr }, lang: appState.lang, log })
 		const response = await axios.post(url, params, config)
@@ -550,53 +561,27 @@ async function translateTextViaProvider({
 		if (!translated?.length) throw new Error(`${providerName} translated text to empty string. You may need to top up your credits.`)
 		log.D(`${translated}`)
 		if (translated === TRANSLATION_FAILED_RESPONSE_TEXT) throw new Error(`${providerName} failed to translate string to ${targetLang}; string: ${text}`)
-		result.translated = translated
+		outResult.translated = translated
 	} catch (error) {
+		let errorHandled = false
+
 		const response = error?.response
 		if (response) {
 			if (response.status === 429) {
-				result.backoffInterval = provider.getSleepInterval(response.headers, log)
-				log.D(`Rate limited; retrying in ${result.backoffInterval}`)
-			} else if (error.response.status === 529) { // Unofficial 'overloaded' code
-				result.backoffInterval = OVERLOADED_BACKOFF_INTERVAL_MS
-				log.D(`Overloaded; retrying in ${result.backoffInterval}`)
-				listrTask.output = `${providerName} overloaded; retrying in ${result.backoffInterval / 1000}s `
+				outResult.backoffInterval = provider.getSleepInterval(response.headers, log)
+				log.D(`Rate limited; retrying in ${outResult.backoffInterval}`)
+				errorHandled = true
+			} else if (response.status === 529) { // Unofficial 'overloaded' code
+				outResult.backoffInterval = OVERLOADED_BACKOFF_INTERVAL_MS
+				log.D(`Overloaded; retrying in ${outResult.backoffInterval}`)
+				listrTask.output = `${providerName} overloaded; retrying in ${outResult.backoffInterval / 1000}s `
+				errorHandled = true
 			}
-		} else {
-			log.W(`API failed. Error:`, error.message)
 		}
-	}
-}
-
-async function translate({
-													 appState,
-													 listrTask,
-													 ctx,
-													 provider,
-													 appContextMessage,
-													 text,
-													 context,
-													 sourceLang,
-													 targetLang,
-													 apiKey,
-													 attemptStr,
-													 log
-												 }) {
-	log.D(`[translate] sourceLang=${sourceLang}; targetLang=${targetLang}; text=${text}`)
-	const result = { translated: null, backoffInterval: 0 }
-
-	const providerName = provider.name()
 
-	if (sourceLang === targetLang) {
-		log.D(`Using reference value since source & target language are the same`)
-		result.translated = text
-	} else {
-		await translateTextViaProvider({
-			appState, provider, listrTask, sourceLang, targetLang, appContextMessage, context, text, log, apiKey, attemptStr, result, providerName
-		})
+		if (!errorHandled) {
+			log.W(`${providerName} API failed. Error:`, error?.message ?? error)
+		}
 	}
-
-	log.D(`[translate] `, result)
-
-	return result
 }
+

package/src/{consts.js → lib/consts.js}

@@ -21,3 +21,15 @@ export const DEFAULT_CONFIG_FILENAME = 'config.json'
 export const OVERLOADED_BACKOFF_INTERVAL_MS = 30 * 1000
 export const CWD = process.cwd()
 
+export const SUPPORTED_REFERENCE_FILE_EXTENSIONS = [
+	'js',
+	'mjs',
+	'json',
+	'jsonc'
+]
+
+export const DEFAULT_LLM_MODELS = {
+	anthropic: 'claude-3-7-sonnet-20250219',
+	openai: 'gpt-4-turbo'
+}
+

package/src/{io.js → lib/io.js}

@@ -33,7 +33,7 @@ export async function readFileAsText(filePath) {
 
 export async function readJsonFile(filePath, isJSONComments = false) {
 	let content = await readFileAsText(filePath)
-	if (isJSONComments) content = stripJsonComments.stripJsonComments(content)
+	if (isJSONComments) content = stripJsonComments(content)
 	return parseJson(content)
 }
 

package/src/{logging.js → lib/logging.js}

@@ -1,12 +1,12 @@
-export function createLog() {
+export function createLog(isDevMode) {
 	return {
 		// T/D/V blackholed until program options are parsed
 		T: () => { },
 		D: () => { },
 		V: () => { },
 
-		E: function(...args) {
-			console.error(...args)
+		E: function(args) {
+			console.error(isDevMode ? args : (args?.message ?? args))
 		},
 		W: function(...args) {
 			console.warn(...args)

package/src/{options.js → lib/options.js}

@@ -1,4 +1,4 @@
-import { isBcp47LanguageTagValid } from './localizer/localize.js'
+import { isBcp47LanguageTagValid } from '../localizer/localize.js'
 import { unique } from './utils.js'
 
 // Helper function to parse comma-separated list

package/src/lib/reference-loader.js

@@ -0,0 +1,91 @@
+import { copyFileToTempAndEnsureExtension, importJsFile, readJsonFile } from './io.js'
+import { getFileExtension } from './utils.js'
+import { SUPPORTED_REFERENCE_FILE_EXTENSIONS } from './consts.js'
+import { localizeFormatted } from '../localizer/localize.js'
+
+export async function loadReferenceFile({ appLang, options: { referenceFile, referenceExportedVarName }, tmpDir, log }) {
+	const ext = getFileExtension(referenceFile)?.toLowerCase()
+	if (!SUPPORTED_REFERENCE_FILE_EXTENSIONS.includes(ext)) {
+		throw new Error(
+			localizeFormatted({
+				token: 'error-bad-reference-file-ext',
+				data: { ext },
+				lang: appLang,
+				log
+			})
+		)
+	}
+
+	let content
+	let useRefVar
+	switch(ext) {
+		case 'js': {
+			log.D(`Reading JS file "${referenceFile}"...`)
+
+			// For .js, we need to copy to a temp location as an .mjs so we can dynamically import
+			const tmpReferencePath = await copyFileToTempAndEnsureExtension({
+				filePath: referenceFile,
+				tmpDir,
+				ext: 'mjs',
+			})
+			//const referenceContent = normalizeData(await importJsFile(tmpReferencePath), log)
+			content = await importJsFile(tmpReferencePath)
+			useRefVar = true
+			break
+		}
+
+		case 'mjs': {
+			// We can dynamically import an .mjs directly from its actual path
+			log.D(`Reading MJS file "${referenceFile}"...`)
+			content = await importJsFile(referenceFile)
+			useRefVar = true
+			break
+		}
+
+		case 'json': {
+			log.D(`Reading JSON file "${referenceFile}"...`)
+			content = await readJsonFile(referenceFile, false)
+			useRefVar = false
+			break
+		}
+
+		case 'jsonc': {
+			log.D(`Reading JSONC file "${referenceFile}"...`)
+			content = await readJsonFile(referenceFile, true)
+			useRefVar = false
+			break
+		}
+	}
+
+	if (!content) {
+		throw new Error(
+			localizeFormatted({
+				token: 'error-reference-file-load-failed',
+				data: { referenceFile },
+				lang: appLang,
+				log
+			})
+		)
+	}
+
+	let result
+	if (useRefVar && referenceExportedVarName?.length) {
+		log.D(`[loadReferenceFile] useRefVar: ${useRefVar}`)
+		if (!(referenceExportedVarName in content)) {
+			throw new Error(
+				localizeFormatted({
+					token: 'error-reference-var-not-found-in-data',
+					data: { referenceExportedVarName, referenceFile, possibleKeys: Object.keys(content) },
+					lang: appLang,
+					log
+				})
+			)
+		}
+
+		result = content[referenceExportedVarName]
+	} else {
+		result = content
+	}
+
+	return result
+}

package/src/{utils.js → lib/utils.js}

@@ -106,3 +106,18 @@ export function normalizeData(data) {
 	}
 	return normalizedData
 }
+
+export function pick(o, ...props) {
+	return Object.assign(
+		{},
+		...props.filter(prop => o[prop] !== undefined)
+			.map(prop => ({ [prop]: o[prop] }))
+	)
+}
+
+// Returns the extension without the '.'
+export function getFileExtension(path) {
+	if (path.indexOf('.') < 0) return null
+	return path.split('.').pop()
+}
+

package/src/localizer/localize.js

@@ -1,9 +1,8 @@
-import * as fsp from 'fs/promises'
 import * as locale from 'locale-codes'
 import * as path from 'path'
-import { obj2Str, replaceStringVarsWithObjectValues } from '../utils.js'
-import { fileExists, readJsonFile } from '../io.js'
-import { LANGTAG_DEFAULT } from '../consts.js'
+import { obj2Str, replaceStringVarsWithObjectValues } from '../lib/utils.js'
+import { readJsonFile } from '../lib/io.js'
+import { LANGTAG_DEFAULT } from '../lib/consts.js'
 
 const LocalizationMap = {}