echogarden 1.3.0 → 1.3.2

package/src/text-translation/NLLBTextTranslation.ts

@@ -0,0 +1,252 @@
+import { Logger } from '../utilities/Logger.js'
+import { loadPackage } from '../utilities/PackageManager.js'
+
+export async function translateText(sourceText: string, sourceLanguage: string, targetLanguage: string) {
+	const logger = new Logger()
+
+	const { AutoTokenizer,  M2M100ForConditionalGeneration } = await import('@echogarden/transformers-nodejs-lite')
+
+	const modelPath = await loadPackage(`xenova-nllb-200-distilled-600M-quantized`)
+
+	const tokenizer = await AutoTokenizer.from_pretrained(modelPath)
+	const model = await M2M100ForConditionalGeneration.from_pretrained(modelPath)
+
+	const config = {
+		src_lang: 'eng_Latn',
+		tgt_lang: 'fra_Latn'
+	}
+
+	const inputs = (tokenizer as any)._build_translation_inputs(sourceText, {
+		padding: true,
+		truncation: true,
+	}, config)
+
+	const result = await model.generate(inputs.input_ids, config)
+
+	logger.log(tokenizer.model.convert_ids_to_tokens(result[0]))
+
+	const inputTokens = tokenizer.model.convert_ids_to_tokens(Array.from(inputs.input_ids.data))
+	const embeddingResult = await model(inputs)
+
+	const lastHiddenState = embeddingResult.last_hidden_state
+
+	const tokenCount = lastHiddenState.dims[1]
+	const embeddingSize = lastHiddenState.dims[2]
+
+	for (let i = 0; i < tokenCount; i++) {
+		const tokenEmbedding = lastHiddenState.data.slice(i * embeddingSize, (i + 1) * embeddingSize)
+
+		const tokenId = inputTokens[i]
+
+		logger.log(`Token ${i} (${tokenId}):`, tokenEmbedding);
+	}
+
+	logger.log(inputTokens)
+}
+
+const languageNameToNLLBCode: Record<string, string> = {
+	'Acehnese (Arabic script)': 'ace_Arab',
+	'Acehnese (Latin script)': 'ace_Latn',
+	'Afrikaans': 'afr_Latn',
+	'Akan': 'aka_Latn',
+	'Amharic': 'amh_Ethi',
+	'Armenian': 'hye_Armn',
+	'Assamese': 'asm_Beng',
+	'Asturian': 'ast_Latn',
+	'Awadhi': 'awa_Deva',
+	'Ayacucho Quechua': 'quy_Latn',
+	'Balinese': 'ban_Latn',
+	'Bambara': 'bam_Latn',
+	'Banjar (Arabic script)': 'bjn_Arab',
+	'Banjar (Latin script)': 'bjn_Latn',
+	'Bashkir': 'bak_Cyrl',
+	'Basque': 'eus_Latn',
+	'Belarusian': 'bel_Cyrl',
+	'Bemba': 'bem_Latn',
+	'Bengali': 'ben_Beng',
+	'Bhojpuri': 'bho_Deva',
+	'Bosnian': 'bos_Latn',
+	'Buginese': 'bug_Latn',
+	'Bulgarian': 'bul_Cyrl',
+	'Burmese': 'mya_Mymr',
+	'Catalan': 'cat_Latn',
+	'Cebuano': 'ceb_Latn',
+	'Central Atlas Tamazight': 'tzm_Tfng',
+	'Central Aymara': 'ayr_Latn',
+	'Central Kanuri (Arabic script)': 'knc_Arab',
+	'Central Kanuri (Latin script)': 'knc_Latn',
+	'Central Kurdish': 'ckb_Arab',
+	'Chhattisgarhi': 'hne_Deva',
+	'Chinese (Simplified)': 'zho_Hans',
+	'Chinese (Traditional)': 'zho_Hant',
+	'Chokwe': 'cjk_Latn',
+	'Crimean Tatar': 'crh_Latn',
+	'Croatian': 'hrv_Latn',
+	'Czech': 'ces_Latn',
+	'Danish': 'dan_Latn',
+	'Dari': 'prs_Arab',
+	'Dutch': 'nld_Latn',
+	'Dyula': 'dyu_Latn',
+	'Dzongkha': 'dzo_Tibt',
+	'Eastern Panjabi': 'pan_Guru',
+	'Eastern Yiddish': 'ydd_Hebr',
+	'Egyptian Arabic': 'arz_Arab',
+	'English': 'eng_Latn',
+	'Esperanto': 'epo_Latn',
+	'Estonian': 'est_Latn',
+	'Ewe': 'ewe_Latn',
+	'Faroese': 'fao_Latn',
+	'Fijian': 'fij_Latn',
+	'Finnish': 'fin_Latn',
+	'Fon': 'fon_Latn',
+	'French': 'fra_Latn',
+	'Friulian': 'fur_Latn',
+	'Galician': 'glg_Latn',
+	'Ganda': 'lug_Latn',
+	'Georgian': 'kat_Geor',
+	'German': 'deu_Latn',
+	'Greek': 'ell_Grek',
+	'Guarani': 'grn_Latn',
+	'Gujarati': 'guj_Gujr',
+	'Haitian Creole': 'hat_Latn',
+	'Halh Mongolian': 'khk_Cyrl',
+	'Hausa': 'hau_Latn',
+	'Hebrew': 'heb_Hebr',
+	'Hindi': 'hin_Deva',
+	'Hungarian': 'hun_Latn',
+	'Icelandic': 'isl_Latn',
+	'Igbo': 'ibo_Latn',
+	'Ilocano': 'ilo_Latn',
+	'Indonesian': 'ind_Latn',
+	'Irish': 'gle_Latn',
+	'Italian': 'ita_Latn',
+	'Japanese': 'jpn_Jpan',
+	'Javanese': 'jav_Latn',
+	'Jingpho': 'kac_Latn',
+	'Kabiyè': 'kbp_Latn',
+	'Kabuverdianu': 'kea_Latn',
+	'Kabyle': 'kab_Latn',
+	'Kamba': 'kam_Latn',
+	'Kannada': 'kan_Knda',
+	'Kashmiri (Arabic script)': 'kas_Arab',
+	'Kashmiri (Devanagari script)': 'kas_Deva',
+	'Kazakh': 'kaz_Cyrl',
+	'Khmer': 'khm_Khmr',
+	'Kikongo': 'kon_Latn',
+	'Kikuyu': 'kik_Latn',
+	'Kimbundu': 'kmb_Latn',
+	'Kinyarwanda': 'kin_Latn',
+	'Korean': 'kor_Hang',
+	'Kyrgyz': 'kir_Cyrl',
+	'Lao': 'lao_Laoo',
+	'Latgalian': 'ltg_Latn',
+	'Ligurian': 'lij_Latn',
+	'Limburgish': 'lim_Latn',
+	'Lingala': 'lin_Latn',
+	'Lithuanian': 'lit_Latn',
+	'Lombard': 'lmo_Latn',
+	'Luba-Kasai': 'lua_Latn',
+	'Luo': 'luo_Latn',
+	'Luxembourgish': 'ltz_Latn',
+	'Macedonian': 'mkd_Cyrl',
+	'Magahi': 'mag_Deva',
+	'Maithili': 'mai_Deva',
+	'Malayalam': 'mal_Mlym',
+	'Maltese': 'mlt_Latn',
+	'Maori': 'mri_Latn',
+	'Marathi': 'mar_Deva',
+	'Meitei (Bengali script)': 'mni_Beng',
+	'Mesopotamian Arabic': 'acm_Arab',
+	'Minangkabau (Arabic script)': 'min_Arab',
+	'Minangkabau (Latin script)': 'min_Latn',
+	'Mizo': 'lus_Latn',
+	'Modern Standard Arabic (Romanized)': 'arb_Latn',
+	'Modern Standard Arabic': 'arb_Arab',
+	'Moroccan Arabic': 'ary_Arab',
+	'Mossi': 'mos_Latn',
+	'Najdi Arabic': 'ars_Arab',
+	'Nepali': 'npi_Deva',
+	'Nigerian Fulfulde': 'fuv_Latn',
+	'North Azerbaijani': 'azj_Latn',
+	'North Levantine Arabic': 'apc_Arab',
+	'Northern Kurdish': 'kmr_Latn',
+	'Northern Sotho': 'nso_Latn',
+	'Northern Uzbek': 'uzn_Latn',
+	'Norwegian Bokmål': 'nob_Latn',
+	'Norwegian Nynorsk': 'nno_Latn',
+	'Nuer': 'nus_Latn',
+	'Nyanja': 'nya_Latn',
+	'Occitan': 'oci_Latn',
+	'Odia': 'ory_Orya',
+	'Pangasinan': 'pag_Latn',
+	'Papiamento': 'pap_Latn',
+	'Plateau Malagasy': 'plt_Latn',
+	'Polish': 'pol_Latn',
+	'Portuguese': 'por_Latn',
+	'Romanian': 'ron_Latn',
+	'Rundi': 'run_Latn',
+	'Russian': 'rus_Cyrl',
+	'Samoan': 'smo_Latn',
+	'Sango': 'sag_Latn',
+	'Sanskrit': 'san_Deva',
+	'Santali': 'sat_Olck',
+	'Sardinian': 'srd_Latn',
+	'Scottish Gaelic': 'gla_Latn',
+	'Serbian': 'srp_Cyrl',
+	'Shan': 'shn_Mymr',
+	'Shona': 'sna_Latn',
+	'Sicilian': 'scn_Latn',
+	'Silesian': 'szl_Latn',
+	'Sindhi': 'snd_Arab',
+	'Sinhala': 'sin_Sinh',
+	'Slovak': 'slk_Latn',
+	'Slovenian': 'slv_Latn',
+	'Somali': 'som_Latn',
+	'South Azerbaijani': 'azb_Arab',
+	'South Levantine Arabic': 'ajp_Arab',
+	'Southern Pashto': 'pbt_Arab',
+	'Southern Sotho': 'sot_Latn',
+	'Southwestern Dinka': 'dik_Latn',
+	'Spanish': 'spa_Latn',
+	'Standard Latvian': 'lvs_Latn',
+	'Standard Malay': 'zsm_Latn',
+	'Standard Tibetan': 'bod_Tibt',
+	'Sundanese': 'sun_Latn',
+	'Swahili': 'swh_Latn',
+	'Swati': 'ssw_Latn',
+	'Swedish': 'swe_Latn',
+	'Tagalog': 'tgl_Latn',
+	'Tajik': 'tgk_Cyrl',
+	'Tamasheq (Latin script)': 'taq_Latn',
+	'Tamasheq (Tifinagh script)': 'taq_Tfng',
+	'Tamil': 'tam_Taml',
+	'Tatar': 'tat_Cyrl',
+	'Ta’izzi-Adeni Arabic': 'acq_Arab',
+	'Telugu': 'tel_Telu',
+	'Thai': 'tha_Thai',
+	'Tigrinya': 'tir_Ethi',
+	'Tok Pisin': 'tpi_Latn',
+	'Tosk Albanian': 'als_Latn',
+	'Tsonga': 'tso_Latn',
+	'Tswana': 'tsn_Latn',
+	'Tumbuka': 'tum_Latn',
+	'Tunisian Arabic': 'aeb_Arab',
+	'Turkish': 'tur_Latn',
+	'Turkmen': 'tuk_Latn',
+	'Twi': 'twi_Latn',
+	'Ukrainian': 'ukr_Cyrl',
+	'Umbundu': 'umb_Latn',
+	'Urdu': 'urd_Arab',
+	'Uyghur': 'uig_Arab',
+	'Venetian': 'vec_Latn',
+	'Vietnamese': 'vie_Latn',
+	'Waray': 'war_Latn',
+	'Welsh': 'cym_Latn',
+	'West Central Oromo': 'gaz_Latn',
+	'Western Persian': 'pes_Arab',
+	'Wolof': 'wol_Latn',
+	'Xhosa': 'xho_Latn',
+	'Yoruba': 'yor_Latn',
+	'Yue Chinese': 'yue_Hant',
+	'Zulu': 'zul_Latn',
+}

package/src/utilities/BinaryArrayConversion.ts

@@ -1,17 +1,30 @@
 // Typed arrays to Buffer (little endian) conversions
+//
+// The faster conversion methods (other than the methods for int8) would only work correctly
+// on little-endian architectures, since they assume the byte order of the underlying architecture.
+//
+// Since Echogarden only supports little-endian architectures, this shouldn't matter.
 
 // int8 <-> bufferLE
-export function int8ToBuffer(ints: Int8Array) {
-	const buffer = Buffer.alloc(ints.length)
+export function int8ToBuffer(int8s: Int8Array) {
+	return Buffer.copyBytesFrom(int8s)
+}
+
+export function int8ToBuffer_Slow(int8s: Int8Array) {
+	const buffer = Buffer.alloc(int8s.length)
 
-	for (let i = 0; i < ints.length; i++) {
-		buffer[i] = ints[i] + 128
+	for (let i = 0; i < int8s.length; i++) {
+		buffer[i] = int8s[i] + 128
 	}
 
 	return buffer
 }
 
 export function bufferToInt8(buffer: Buffer) {
+	return new Int8Array(buffer.buffer, buffer.byteOffset, buffer.byteLength)
+}
+
+export function bufferToInt8_Slow(buffer: Buffer) {
 	const result = new Int8Array(buffer.length)
 
 	for (let i = 0; i < result.length; i++) {
@@ -22,17 +35,25 @@ export function bufferToInt8(buffer: Buffer) {
 }
 
 // int16 <-> bufferLE
-export function int16ToBufferLE(ints: Int16Array) {
-	const buffer = Buffer.alloc(ints.length * 2)
+export function int16ToBufferLE(int16s: Int16Array) {
+	return Buffer.copyBytesFrom(int16s)
+}
+
+export function int16ToBufferLE_Slow(int16s: Int16Array) {
+	const buffer = Buffer.alloc(int16s.length * 2)
 
-	for (let i = 0; i < ints.length; i++) {
-		buffer.writeInt16LE(ints[i], i * 2)
+	for (let i = 0; i < int16s.length; i++) {
+		buffer.writeInt16LE(int16s[i], i * 2)
 	}
 
 	return buffer
 }
 
 export function bufferLEToInt16(buffer: Buffer) {
+	return new Int16Array(buffer.buffer, buffer.byteOffset, buffer.byteLength / 2)
+}
+
+export function bufferLEToInt16_Slow(buffer: Buffer) {
 	const result = new Int16Array(buffer.length / 2)
 
 	for (let i = 0; i < result.length; i++) {
@@ -43,11 +64,11 @@ export function bufferLEToInt16(buffer: Buffer) {
 }
 
 // int24 <-> bufferLE (uses int32 for storage)
-export function int24ToBufferLE(ints: Int32Array) {
-	const buffer = Buffer.alloc(ints.length * 3)
+export function int24ToBufferLE(int24s: Int32Array) {
+	const buffer = Buffer.alloc(int24s.length * 3)
 
-	for (let i = 0; i < ints.length; i++) {
-		const val = ints[i]
+	for (let i = 0; i < int24s.length; i++) {
+		const val = int24s[i]
 		const encodedVal = val < 0 ? val + 0x1000000 : val
 
 		buffer[(i * 3) + 0] = (encodedVal >> 0) & 0xff
@@ -74,17 +95,25 @@ export function bufferLEToInt24(buffer: Buffer) {
 }
 
 // int32 <-> bufferLE
-export function int32ToBufferLE(ints: Int32Array) {
-	const buffer = Buffer.alloc(ints.length * 4)
+export function int32ToBufferLE(int32s: Int32Array) {
+	return Buffer.copyBytesFrom(int32s)
+}
 
-	for (let i = 0; i < ints.length; i++) {
-		buffer.writeInt32LE(ints[i], i * 4)
+export function int32ToBufferLE_Slow(int32s: Int32Array) {
+	const buffer = Buffer.alloc(int32s.length * 4)
+
+	for (let i = 0; i < int32s.length; i++) {
+		buffer.writeInt32LE(int32s[i], i * 4)
 	}
 
 	return buffer
 }
 
 export function bufferLEToInt32(buffer: Buffer) {
+	return new Int32Array(buffer.buffer, buffer.byteOffset, buffer.byteLength / 4)
+}
+
+export function bufferLEToInt32_Slow(buffer: Buffer) {
 	const result = new Int32Array(buffer.length / 4)
 
 	for (let i = 0; i < result.length; i++) {
@@ -95,17 +124,25 @@ export function bufferLEToInt32(buffer: Buffer) {
 }
 
 // float32 <-> bufferLE
-export function float32ToBufferLE(floats: Float32Array) {
-	const buffer = Buffer.alloc(floats.length * 4)
+export function float32ToBufferLE(float32s: Float32Array) {
+	return Buffer.copyBytesFrom(float32s)
+}
+
+export function float32ToBufferLE_Slow(float32s: Float32Array) {
+	const buffer = Buffer.alloc(float32s.length * 4)
 
-	for (let i = 0; i < floats.length; i++) {
-		buffer.writeFloatLE(floats[i], i * 4)
+	for (let i = 0; i < float32s.length; i++) {
+		buffer.writeFloatLE(float32s[i], i * 4)
 	}
 
 	return buffer
 }
 
 export function bufferLEToFloat32(buffer: Buffer) {
+	return new Float32Array(buffer.buffer, buffer.byteOffset, buffer.byteLength / 4)
+}
+
+export function bufferLEToFloat32_Slow(buffer: Buffer) {
 	const result = new Float32Array(buffer.length / 4)
 
 	for (let i = 0; i < result.length; i++) {
@@ -116,17 +153,25 @@ export function bufferLEToFloat32(buffer: Buffer) {
 }
 
 // float64 <-> bufferLE
-export function float64ToBufferLE(floats: Float64Array) {
-	const buffer = Buffer.alloc(floats.length * 8)
+export function float64ToBufferLE(float64s: Float64Array) {
+	return Buffer.copyBytesFrom(float64s)
+}
+
+export function float64ToBufferLE_Slow(float64s: Float64Array) {
+	const buffer = Buffer.alloc(float64s.length * 8)
 
-	for (let i = 0; i < floats.length; i++) {
-		buffer.writeDoubleLE(floats[i], i * 8)
+	for (let i = 0; i < float64s.length; i++) {
+		buffer.writeDoubleLE(float64s[i], i * 8)
 	}
 
 	return buffer
 }
 
 export function bufferLEToFloat64(buffer: Buffer) {
+	return new Float64Array(buffer.buffer, buffer.byteOffset, buffer.byteLength / 8)
+}
+
+export function bufferLEToFloat64_Slow(buffer: Buffer) {
 	const result = new Float64Array(buffer.length / 8)
 
 	for (let i = 0; i < result.length; i++) {
@@ -137,23 +182,10 @@ export function bufferLEToFloat64(buffer: Buffer) {
 }
 
 // float64 <-> float32
-export function float64Tofloat32(doubles: Float64Array) {
-	const floats = new Float32Array(doubles.length)
-
-	for (let i = 0; i < doubles.length; i++) {
-		floats[i] = doubles[i]
-	}
-
-	return floats
+export function float64Tofloat32(float64s: Float64Array) {
+	return Float32Array.from(float64s)
 }
 
-export function float32Tofloat64(floats: Float32Array) {
-	const doubles = new Float64Array(floats.length)
-
-	for (let i = 0; i < floats.length; i++) {
-		doubles[i] = floats[i]
-	}
-
-	return doubles
+export function float32Tofloat64(float32s: Float32Array) {
+	return Float64Array.from(float32s)
 }
-

package/src/utilities/PackageManager.ts

@@ -182,4 +182,7 @@ const packageVersionTagResolutionLookup: { [packageName: string]: string } = {
 
 	'whisper.cpp-binaries-windows-x64-cublas-12.4.0-latest-patched': '20240409',
 	'whisper.cpp-binaries-windows-x64-cublas-11.8.0-latest-patched': '20240409',
+
+	'xenova-multilingual-e5-small-quantized': '20240504',
+	'xenova-nllb-200-distilled-600M-quantized': '20240505',
 }