@drone1/alt 0.4.0

package/package.json

@@ -0,0 +1,48 @@
+{
+  "type": "module",
+  "name": "@drone1/alt",
+  "version": "0.4.0",
+  "description": "An AI-powered localization tool",
+  "main": "src/index.mjs",
+  "bin": {
+    "alt": "./bin.mjs"
+  },
+  "scripts": {
+    "localize-display-strings": "node ./bin.mjs --reference-file localization/reference.js "
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/drone1/ai-localization-tool.git"
+  },
+  "keywords": [
+    "localization",
+    "i18n",
+    "translation",
+    "cli",
+    "ai",
+    "claude",
+    "openai",
+    "gemini",
+    "google",
+    "anthropic"
+  ],
+  "author": "drone1",
+  "license": "MIT",
+  "bugs": {
+    "url": "https://github.com/drone1/ai-localization-tool/issues"
+  },
+  "homepage": "https://github.com/drone1/ai-localization-tool#readme",
+  "dependencies": {
+    "axios": "^1.8.4",
+    "chalk": "^5.4.1",
+    "commander": "^11.1.0",
+    "figlet": "^1.8.0",
+    "gradient-string": "^3.0.0",
+    "listr2": "^8.2.5",
+    "locale-codes": "^1.3.1",
+    "strip-json-comments": "^5.0.1"
+  },
+  "engines": {
+    "node": ">=14.0.0"
+  }
+}

package/src/assert.js

@@ -0,0 +1,19 @@
+export function assert(b, msg) {
+	if (!b) {
+		debugger
+		throw new Error(msg || `Assertion failed`)
+	}
+}
+
+export function assertIsNonEmptyString(s, msg) {
+	assert(s?.length, msg || `parameter was not a non-empty string`)
+}
+
+export function assertValidPath(path, msg) {
+	assertIsNonEmptyString(path, msg || `parameter was not a valid path`)
+}
+
+export function assertIsObj(x, msg) {
+	assert(typeof x === 'object', msg || `parameter was not an object`)
+}
+

package/src/cache.js

@@ -0,0 +1,12 @@
+import { readJsonFile } from './io.js'
+
+export async function loadCache(path) {
+	const storedCache = await readJsonFile(path)
+	return {
+		referenceHash: storedCache?.referenceHash ?? '',
+		referenceKeyHashes: storedCache?.referenceKeyHashes ?? {},
+		state: storedCache?.state ?? {},
+		lastRun: storedCache?.lastRun ?? null,
+	}
+}
+

package/src/config.js

@@ -0,0 +1,26 @@
+import * as path from 'path'
+import { DEFAULT_CONFIG_FILENAME } from './consts.js'
+import { readJsonFile } from './io.js'
+
+export async function loadConfig({ configFile, refFileDir, log }) {
+	let configFilePath
+	if (configFile?.length) {
+		log.V(`Using config file specified by --config-file "${configFile}"...`)
+		configFilePath = configFile
+	} else {
+		log.V(`Using config file path based on reference file dir, "${refFileDir}"...`)
+		configFilePath = path.resolve(refFileDir, DEFAULT_CONFIG_FILENAME)
+	}
+
+	log.V(`Attempting to load config file from "${configFilePath}"`)
+	return await readJsonFile(configFilePath) || {
+		provider: null,
+		targetLanguages: [],
+		lookForContextData: true,
+		contextPrefix: '',
+		contextSuffix: '',
+		referenceLanguage: null,
+		normalizeOutputFilenames: false
+	}
+}
+

package/src/consts.js

@@ -0,0 +1,23 @@
+import * as path from 'path'
+
+export const TRANSLATION_FAILED_RESPONSE_TEXT = '<<<TRANSLATION_FAILED>>>'
+export const LANGTAG_ENGLISH = 'en'
+export const LANGTAG_DEFAULT = LANGTAG_ENGLISH
+
+export const VALID_TRANSLATION_PROVIDERS = [
+	'anthropic',
+	'openai'
+]
+
+export const ENV_VARS = [
+	{ name: 'ANTHROPIC_API_KEY', description: 'Your Anthropic API key' },
+	{ name: 'OPENAI_API_KEY', description: 'Your OpenAI API key' },
+	{ name: 'ALT_LANGUAGE', description: 'POSIX locale used for display' }
+]
+
+export const LOCALIZATION_SRC_DIR = path.resolve('localization')
+export const DEFAULT_CACHE_FILENAME = '.localization.cache.json'
+export const DEFAULT_CONFIG_FILENAME = 'config.json'
+export const OVERLOADED_BACKOFF_INTERVAL_MS = 30 * 1000
+export const CWD = process.cwd()
+

package/src/context-keys.js

@@ -0,0 +1,10 @@
+export function isContextKey({ key, contextPrefix, contextSuffix }) {
+	if (contextPrefix?.length) return key.startsWith(contextPrefix)
+	if (contextSuffix?.length) return key.endsWith(contextSuffix)
+	throw new Error(`Either the context prefix or context suffix must be defined`)
+}
+
+export function formatContextKeyFromKey({ key, prefix, suffix }) {
+	return `${prefix}${key}${suffix}`
+}
+

package/src/io.js

@@ -0,0 +1,129 @@
+import * as os from 'os'
+import * as fs from 'fs'
+import * as fsp from 'fs/promises'
+import * as path from 'path'
+import stripJsonComments from 'strip-json-comments'
+import { ensureExtension, normalizeKey } from './utils.js'
+import { assertIsObj, assertValidPath } from './assert.js'
+import { pathToFileURL } from 'url'
+import { CWD } from './consts.js'
+
+export async function mkTmpDir() {
+	return await fsp.mkdtemp(path.join(os.tmpdir(), 'alt-'))
+}
+
+export function parseJson(s) {
+	try {
+		return JSON.parse(s)
+	} catch (e) {
+		return null
+	}
+}
+
+export async function readFileAsText(filePath) {
+	try {
+		return await fsp.readFile(filePath, 'utf8')
+	} catch (error) {
+		if (error.code === 'ENOENT') {
+			return null
+		}
+		throw error
+	}
+}
+
+export async function readJsonFile(filePath, isJSONComments = false) {
+	let content = await readFileAsText(filePath)
+	if (isJSONComments) content = stripJsonComments.stripJsonComments(content)
+	return parseJson(content)
+}
+
+export async function fileExists(path) {
+	try {
+		await fsp.access(path)
+		return true
+	} catch {
+		return false
+	}
+}
+
+export function rmDir(dir, log) {
+	try {
+		fs.rmSync(dir, { recursive: true, force: true })
+		log.D(`Removed dir ${dir}`)
+	} catch (error) {
+		log.E(`Error cleaning up temp directory "${dir}": ${error.message}`)
+		throw error
+	}
+}
+
+// This is basically so that we can dynamically import .js files by copying them to temp .mjs files, to avoid errors from node
+export async function copyFileToTempAndEnsureExtension({ filePath, tmpDir, ext }) {
+	try {
+		const fileName = ensureExtension(path.basename(filePath), ext)
+		const destPath = path.join(tmpDir, fileName)
+		await fsp.copyFile(filePath, destPath)
+		return destPath
+	} catch (error) {
+		log.E(`Error copying file to temp directory: ${error.message}`)
+		throw error
+	}
+}
+
+// Dynamically imports the javascript file at filePath, which can be relative or absolute
+export async function importJsFile(filePath) {
+	if (!path.isAbsolute(filePath)) {
+		filePath = path.resolve(CWD, filePath)
+	}
+	// Convert the file path to a proper URL
+	const fileUrl = pathToFileURL(filePath)
+	return await import(fileUrl)
+}
+
+export function normalizeOutputPath({ dir, filename, normalize }) {
+	return path.join(dir, normalize ? filename.toLowerCase() : filename)
+}
+
+export function writeJsonFile(filePath, data, log) {
+	assertValidPath(filePath)
+	assertIsObj(data)
+	log.V(`Preparing to write ${filePath}...`)
+
+	// Create normalized version of data with consistent key encoding
+	log.D(`Normalizing data...`)
+	const normalizedData = {}
+	for (const [ key, value ] of Object.entries(data)) {
+		normalizedData[normalizeKey(key)] = value
+	}
+	log.D(`Done.`)
+
+	try {
+		const dir = path.dirname(filePath)
+		log.D(`Ensuring directory ${dir} exists...`)
+		if (!dirExists(dir, log)) {
+			log.D(`Directory ${dir} did not exist; creating...`)
+			fs.mkdirSync(dir, { recursive: true })
+		}
+		log.D(`Done.`)
+	} catch (err) {
+		log.E(err)
+	}
+
+	log.V(`Writing ${filePath}...`)
+	try {
+		fs.writeFileSync(filePath, JSON.stringify(normalizedData, null, 2), 'utf8')
+	} catch (err) {
+		log.E(err)
+	}
+	log.D(`Done.`)
+}
+
+export function dirExists(dir, log) {
+	try {
+		log.D(`fetching stats for ${dir}...`)
+		return fs.statSync(dir).isDirectory()
+	} catch (error) {
+		log.E(error)
+		return false
+	}
+}
+

package/src/localizer/localize.js

@@ -0,0 +1,135 @@
+import * as fsp from 'fs/promises'
+import * as locale from 'locale-codes'
+import * as path from 'path'
+import { obj2Str, replaceStringVarsWithObjectValues } from '../utils.js'
+import { fileExists, readJsonFile } from '../io.js'
+import { LANGTAG_DEFAULT } from '../consts.js'
+
+const LocalizationMap = {}
+
+export async function initLocalizer({ defaultAppLanguage, appLanguage, srcDir, log }) {
+	let lang
+
+	// Always load the default language, as a fallback
+	await addLocalizationDataForLanguage({ lang: defaultAppLanguage, srcPath: path.resolve(srcDir, `${defaultAppLanguage}.json`), log })
+
+	if (appLanguage?.length && appLanguage !== defaultAppLanguage) {
+		try {
+			if (!isBcp47LanguageTagValid(appLanguage)) {
+				log.W(`"${appLanguage}" is not a valid BCP47 language`)
+				throw new Error()	// Print another warning and use default language
+			}
+			await addLocalizationDataForLanguage({ lang: appLanguage, srcPath: path.resolve(srcDir, `${appLanguage}.json`), log })
+			lang = appLanguage
+		} catch (err) {
+			log.W(`No localization data found for language "${appLanguage}"; falling back to "${defaultAppLanguage}"...`)
+			lang = defaultAppLanguage
+		}
+	} else {
+		lang = defaultAppLanguage
+	}
+
+	return lang
+}
+
+export function isBcp47LanguageTagValid(tag) {
+	return locale.getByTag(tag)
+}
+
+export function isDefaultLanguage(tag) {
+	return tag === LANGTAG_DEFAULT
+}
+
+export function localizeFormatted({ token, data, lang, fallbackToken, log }) {
+	//assertIsObj(data)
+	//assertIsNonEmptyString(lang)
+	//assertValidBcp47LanguageTag(lang)
+
+	// We don't use fallbackToken directly in our call to localize(), because we don't want to inadvertently format in odd ways
+	// But let's see how we end up using this function; for now we just directly localize the fallback below, if localize() fails here
+	const str = localize({ token, lang, log })
+
+	// Directly attempt to localize the fallback (or returns '')
+	if (!str?.length) {
+		const fallbackResult = localize({ token: fallbackToken, lang, log })
+		if (fallbackResult.indexOf('%%') >= 0) {
+			log.W(`'fallbackToken' should not include formatting variables; not currently supported; string="${fallbackResult}"`)
+		}
+		return fallbackResult
+	}
+
+	const warnings = []
+	const result = replaceStringVarsWithObjectValues({
+		format: str, data, outWarningsArray: warnings
+	})
+
+	if (warnings.length) warnings.forEach(w => log.W(`[localizeFormatted] warning: ${w.message} (code=${w.code})`))
+
+	return result
+}
+
+export function localize({ token, lang, fallbackToken, log }) {
+	log.D(`[localize] ${obj2Str({ token, lang, fallbackToken })}`)
+
+	if (!isBcp47LanguageTagValid(lang)) {
+		log.D(`[localize] Invalid language "${lang}" passed; falling back to default language...`)
+		lang = LANGTAG_DEFAULT
+	}
+
+	if (!token) token = ''
+
+	if (token.startsWith('#')) {
+		log.D(`[localize] removed leading '#' character`)
+		token = token.substring(1)
+	}
+
+	if (!token.length && fallbackToken) {
+		log.D(`[localize] token was empty; trying fallback...`)
+		return localize({ token: fallbackToken, lang, log })
+	}
+
+	if (!(lang in LocalizationMap)) {
+		log.W(`[localize] lang "${lang}" was not in LocalizationMap`)
+
+		if (!isDefaultLanguage(lang)) {
+			return localize({ token, lang: LANGTAG_DEFAULT, log })
+		}
+		return ''
+	} else if (!(token in LocalizationMap[lang])) {
+		log.D(`[localize] token "${token}" was not in LocalizationMap[${lang}]`)
+
+		// If it didn't exist, check the fallback in the same language
+		if (fallbackToken) {
+			log.D(`[localize] falling back to token ${fallbackToken}`)
+			return localize({ token: fallbackToken, lang, log })
+		}
+
+		// If no fallback in the same language, attempt to find an english version of 'token'
+		if (!isDefaultLanguage(lang)) {
+			log.D(`[localize] attempting to fall back to english`)
+			// This will get the fallback in english if not found in the requested language
+			return localize({ token, lang: LANGTAG_DEFAULT, fallbackToken, log })
+		}
+
+		log.W(`Failed to find localization string for language="${lang}", token="${token}"`)
+
+		return ''
+	}
+
+	const result = LocalizationMap[lang][token]
+	log.D(`[localize] success; found localization string; ${obj2Str({ token, lang, result })}`)
+
+	return result
+}
+
+async function addLocalizationDataForLanguage({ lang, srcPath, log }) {
+	//assertValidBcp47LanguageTag(lang)
+	//assertIsObj(data)
+	//assertIsObj(LocalizationMap)
+
+	const data = await readJsonFile(srcPath)
+	if (!data) throw new Error(`Failed to load localization file "${srcPath}"`)
+
+	LocalizationMap[lang] = LocalizationMap.lang ?? {}
+	LocalizationMap[lang] = { ...LocalizationMap.lang, ...data }
+}

package/src/logging.js

@@ -0,0 +1,35 @@
+export function createLog() {
+	return {
+		// T/D/V blackholed until program options are parsed
+		T: () => { },
+		D: () => { },
+		V: () => { },
+
+		E: function(...args) {
+			console.error(...args)
+		},
+		W: function(...args) {
+			console.warn(...args)
+		},
+		I: function(...args) {
+			console.log(...args)
+		},
+	}
+}
+
+export function initLogFromOptions({ options, log }) {
+	// Init optional logging functions
+	log.V = (options.trace || options.debug || options.verbose) ? function(...args) {
+		console.log(...args)
+	} : () => {
+	}
+	log.D = (options.trace || options.debug) ? function(...args) {
+		console.debug(...args)
+	} : () => {
+	}
+	log.T = options.trace ? function(...args) {
+		console.debug(...args)
+	} : () => {
+	}
+}
+

package/src/logo.js

@@ -0,0 +1,22 @@
+import figlet from 'figlet'
+import gradient from 'gradient-string'
+import * as path from 'path'
+import * as fsp from 'fs/promises'
+
+export async function printLogo({ fontsSrcDir, tagline, log }) {
+	const fontName = 'THIS.flf'
+	const fontPath = path.resolve(fontsSrcDir, fontName)
+	const fontData = await fsp.readFile(fontPath, 'utf8')
+	figlet.parseFont(fontName, fontData)
+	const asciiTitle = figlet.textSync('ALT', {
+		font: fontName,
+		horizontalLayout: 'full',
+		verticalLayout: 'default',
+	})
+
+	log.I(`\n${gradient([
+		'#000FFF',
+		'#ed00b1'
+	])(asciiTitle)}\n`)
+}
+

package/src/main.mjs

@@ -0,0 +1,103 @@
+import * as path from 'path'
+import { program } from 'commander'
+import { fileURLToPath } from 'url'
+import { initLocalizer } from './localizer/localize.js'
+import {
+	DEFAULT_CONFIG_FILENAME,
+	ENV_VARS,
+	LANGTAG_DEFAULT,
+	LOCALIZATION_SRC_DIR,
+} from './consts.js'
+import { readJsonFile } from './io.js'
+import { printLogo } from './logo.js'
+import { createLog, initLogFromOptions } from './logging.js'
+import { keyList, languageList } from './options.js'
+import { runTranslation } from './translate.js'
+import { registerSignalHandlers } from './shutdown.js'
+
+const __dirname = path.dirname(fileURLToPath(import.meta.url))
+
+// Main function
+export async function run() {
+	const log = createLog()
+
+	const appState = {
+		lang: null,	// The app language, for output display (unrelated to translator)
+		tmpDir: null,
+		filesToWrite: {},	// Map of file path => JSON data to write
+		errors: [],
+		log
+	}
+
+	try {
+		registerSignalHandlers(appState)
+
+		appState.lang = await initLocalizer({
+			defaultAppLanguage: LANGTAG_DEFAULT,
+			appLanguage: process.env?.ALT_LANGUAGE,
+			srcDir: path.resolve(__dirname, LOCALIZATION_SRC_DIR),
+			log
+		})
+
+		const p = await readJsonFile(path.resolve(__dirname, '../package.json'))
+		if (!p) throw new Error(`Couldn't read 'package.json'`)
+
+		// Define CLI options
+		program
+			.version(p.version)
+			.description(p.description)
+			.on('--help', () => {
+				log.I()
+				log.I('Environment variables:')
+				ENV_VARS.forEach(v => {
+					log.I(`  ${v.name.padEnd(37)} ${v.description}`)
+				})
+			})
+			.requiredOption('-r, --reference-file <path>', 'Path to reference JSONC file (default language)')
+			.option('-rl, --reference-language <language>', `The reference file's language; overrides any 'referenceLanguage' config setting`)
+			.option('-p, --provider <name>', `AI provider to use for translations (anthropic, openai); overrides any 'provider' config setting`)
+			.option('-o, --output-dir <path>', 'Output directory for localized files')
+			.option('-l, --target-languages <list>', `Comma-separated list of language codes; overrides any 'targetLanguages' config setting`, value => languageList(value, log))
+			.option('-k, --keys <list>', 'Comma-separated list of keys to process', keyList)
+			.option('-j, --reference-var-name <var name>', `The exported variable in the reference file, e.g. export default = {...} you'd use 'default'`, 'default')
+			.option('-f, --force', 'Force regeneration of all translations', false)
+			.option('-rtw, --realtime-writes', 'Write updates to disk immediately, rather than on shutdown', false)
+			.option('-m, --app-context-message <message>', `Description of your app to give context. Passed with each translation request; overrides any 'appContextMessage' config setting`)
+			.option('-y, --tty', 'Use tty/simple renderer; useful for CI', false)
+			.option('-c, --config-file <path>', `Path to config file; defaults to <output dir>/${DEFAULT_CONFIG_FILENAME}`)
+			.option('-x, --max-retries <integer>', 'Maximum retries on failure', 3)
+			.option('-n, --normalize-output-filenames', `Normalizes output filenames (to all lower-case); overrides any 'normalizeOutputFilenames' in config setting`, false)
+			.option('-v, --verbose', `Enables verbose spew`, false)
+			.option('-d, --debug', `Enables debug spew`, false)
+			.option('-t, --trace', `Enables trace spew`, false)
+			.option('--context-prefix <value>', `String to be prefixed to all keys to search for additional context, which are passed along to the AI for context`)
+			.option('--context-suffix <value>', `String to be suffixed to all keys to search for additional context, which are passed along to the AI for context`)
+			.option('--look-for-context-data', `If specified, ALT will pass any context data specified in the reference file to the AI provider for translation. At least one of --contextPrefix or --contextSuffix must be specified`, false)
+			.hook('preAction', (thisCommand) => {
+				const opts = thisCommand.opts()
+				if (opts.lookForContextData && !(opts.contextPrefix?.length || opts.contextSuffix?.length)) {
+					thisCommand.error('--lookForContextData requires at least 1 of --contextPrefix or --contextSuffix be defined and non-empty')
+				}
+			})
+			.action(async (thisCommand) => {
+			})	// Dummy action() for preAction
+
+		program
+			.command('translate', { isDefault: true }) // This makes it the default command
+			.action(async () => {
+				const options = program.opts()
+				initLogFromOptions({ options, log })
+				await runTranslation({ appState, options, log })
+			})
+
+		program.parse(process.argv)
+
+		await printLogo({
+			fontsSrcDir: path.resolve(__dirname, '../assets/figlet-fonts/'),
+			tagline: p.description,
+			log
+		})
+	} catch (error) {
+		log.E(error)
+	}
+}

package/src/options.js

@@ -0,0 +1,18 @@
+import { isBcp47LanguageTagValid } from './localizer/localize.js'
+import { unique } from './utils.js'
+
+// Helper function to parse comma-separated list
+export function languageList(value, log) {
+	const languages = unique(value.split(',').map(item => item.trim()))
+	const invalid = languages.filter(tag => !isBcp47LanguageTagValid(tag))
+	if (invalid.length) {
+		log.E(`Found invalid language(s): ${invalid.join(', ')}`)
+		process.exit(1)
+	}
+	return languages
+}
+
+export function keyList(value) {
+	return value.split(',').map(item => item.trim())
+}
+

package/src/provider.js

@@ -0,0 +1,17 @@
+import * as path from 'path'
+import { importJsFile } from './io.js'
+
+export async function loadTranslationProvider({__dirname, providerName, log}) {
+	const apiKeyName = `${providerName.toUpperCase()}_API_KEY`
+	const apiKey = process.env[apiKeyName]
+	if (!apiKey?.length) {
+		log.E(`${apiKeyName} environment variable is not set`)
+		process.exit(1)
+	}
+
+	return {
+		apiKey,
+		api: await importJsFile(path.resolve(__dirname, `providers/${providerName}.mjs`)),
+	}
+}
+

package/src/providers/anthropic.mjs

@@ -0,0 +1,39 @@
+export function name() {
+	return 'Anthropic'
+}
+
+export function getTranslationRequestDetails({ messages, apiKey, log }) {
+	return {
+		url: 'https://api.anthropic.com/v1/messages',
+		params: {
+			model: 'claude-3-7-sonnet-20250219',
+			max_tokens: 1024,
+			messages: messages.map(m => ({ role: 'user', content: m })),
+		},
+		config: {
+			headers: {
+				'Content-Type': 'application/json',
+				'x-api-key': apiKey,
+				'anthropic-version': '2023-06-01',
+			},
+		},
+	}
+}
+
+export function getResult(response, log) {
+	return response.data.content[0].text.trim()
+}
+
+function getHeader(headers, name) {
+	return headers[name] || headers.get?.(name)
+}
+
+export function getSleepInterval(headers, log) {
+	log.T(headers)
+	if (getHeader(headers, 'x-should-retry') !== 'true')
+		return 0
+
+	const retryAfter = parseInt(getHeader(headers, 'retry-after'))
+	log.D('retryAfter', retryAfter)
+	return 1000 * retryAfter + 200
+}

package/src/providers/openai.mjs

@@ -0,0 +1,30 @@
+export function name() {
+	return 'OpenAI'
+}
+
+export function getTranslationRequestDetails({ messages, apiKey, log }) {
+	return {
+		url: 'https://api.openai.com/v1/chat/completions',
+		params: {
+			model: 'gpt-4-turbo',
+			messages: messages.map((m, idx) => {
+				return {
+					role: idx === messages.length - 1 ? 'user' : 'system',
+					content: m,
+				}
+			}),
+			temperature: 0.3,
+			max_tokens: 1024,
+		},
+		config: {
+			headers: {
+				'Content-Type': 'application/json',
+				'Authorization': `Bearer ${apiKey}`,
+			},
+		},
+	}
+}
+
+export function getResult(response) {
+	return response.data.choices[0].message.content.trim()
+}