@qvac/translation-nmtcpp 0.1.0

package/third-party/indic-processor-deps/indicnlp/sinhala_transliterator.js

@@ -0,0 +1,197 @@
+/**
+ *
+ *  Copyright (c) 2013-present, Anoop Kunchukuttan
+ *  All rights reserved.
+ *
+ *  This source code is licensed under the MIT license found in the
+ *  INDIC_NPL_LICENCE file in the indicnlp directory of this source tree.
+ *
+ *  This code is a ported version of the sacremoses library. Please refer to NOTICE
+ *  file in the root directory of this source tree.
+ */
+
+/**
+ * A Devanagari to Sinhala transliterator based on explicit Unicode Mapping
+ */
+class SinhalaDevanagariTransliterator {
+  /**
+   * Map of Sinhala to Devanagari Unicode characters
+   */
+  static sinhalaDevnagMap = {
+    '\u0d82': '\u0902',
+    '\u0d83': '\u0903',
+    '\u0d84': '\u0904',
+    අ: '\u0905',
+    ආ: '\u0906',
+    ඇ: '\u090d',
+    ඈ: '\u090d',
+    ඉ: '\u0907',
+    ඊ: '\u0908',
+    උ: '\u0909',
+    ඌ: '\u090a',
+    ඍ: '\u090b',
+    ඏ: '\u090c',
+    එ: '\u090e',
+    ඒ: '\u090f',
+    ඓ: '\u0910',
+    ඔ: '\u0912',
+    ඕ: '\u0913',
+    ඖ: '\u0914',
+    ක: '\u0915',
+    ඛ: '\u0916',
+    ග: '\u0917',
+    ඝ: '\u0918',
+    ඞ: '\u0919',
+    ඟ: '\u0919',
+    ච: '\u091a',
+    ඡ: '\u091b',
+    ජ: '\u091c',
+    ඣ: '\u091d',
+    ඤ: '\u091e',
+    ඥ: '\u091e',
+    ඦ: '\u091e',
+    ට: '\u091f',
+    ඨ: '\u0920',
+    ඩ: '\u0921',
+    ඪ: '\u0922',
+    ණ: '\u0923',
+    ඬ: '\u0923',
+    ත: '\u0924',
+    ථ: '\u0925',
+    ද: '\u0926',
+    ධ: '\u0927',
+    න: '\u0928',
+    '\u0db2': '\u0928',
+    ඳ: '\u0928',
+    ප: '\u092a',
+    ඵ: '\u092b',
+    බ: '\u092c',
+    භ: '\u092d',
+    ම: '\u092e',
+    ය: '\u092f',
+    ර: '\u0930',
+    ල: '\u0932',
+    ළ: '\u0933',
+    ව: '\u0935',
+    ශ: '\u0936',
+    ෂ: '\u0937',
+    ස: '\u0938',
+    හ: '\u0939',
+    '\u0dcf': '\u093e',
+    '\u0dd0': '\u0949',
+    '\u0dd1': '\u0949',
+    '\u0dd2': '\u093f',
+    '\u0dd3': '\u0940',
+    '\u0dd4': '\u0941',
+    '\u0dd6': '\u0942',
+    '\u0dd8': '\u0943',
+    '\u0dd9': '\u0946',
+    '\u0dda': '\u0947',
+    '\u0ddb': '\u0948',
+    '\u0ddc': '\u094a',
+    '\u0ddd': '\u094b',
+    '\u0dde': '\u094c',
+    '\u0dca': '\u094d'
+  }
+
+  /**
+   * Map of Devanagari to Sinhala Unicode characters
+   */
+  static devnagSinhalaMap = {
+    '\u0900': '\u0d82',
+    '\u0901': '\u0d82',
+    '\u0902': '\u0d82',
+    '\u0903': '\u0d83',
+    ऄ: '\u0d84',
+    अ: '\u0d85',
+    आ: '\u0d86',
+    इ: '\u0d89',
+    ई: '\u0d8a',
+    उ: '\u0d8b',
+    ऊ: '\u0d8c',
+    ऋ: '\u0d8d',
+    ऌ: '\u0d8f',
+    ऍ: '\u0d88',
+    ऎ: '\u0d91',
+    ए: '\u0d92',
+    ऐ: '\u0d93',
+    ऒ: '\u0d94',
+    ओ: '\u0d95',
+    औ: '\u0d96',
+    क: '\u0d9a',
+    ख: '\u0d9b',
+    ग: '\u0d9c',
+    घ: '\u0d9d',
+    ङ: '\u0d9e',
+    च: '\u0da0',
+    छ: '\u0da1',
+    ज: '\u0da2',
+    झ: '\u0da3',
+    ञ: '\u0da4',
+    ट: '\u0da7',
+    ठ: '\u0da8',
+    ड: '\u0da9',
+    ढ: '\u0daa',
+    ण: '\u0dab',
+    त: '\u0dad',
+    थ: '\u0dae',
+    द: '\u0daf',
+    ध: '\u0db0',
+    न: '\u0db1',
+    ऩ: '\u0db1',
+    प: '\u0db4',
+    फ: '\u0db5',
+    ब: '\u0db6',
+    भ: '\u0db7',
+    म: '\u0db8',
+    य: '\u0dba',
+    र: '\u0dbb',
+    ल: '\u0dbd',
+    ळ: '\u0dc5',
+    व: '\u0dc0',
+    श: '\u0dc1',
+    ष: '\u0dc2',
+    स: '\u0dc3',
+    ह: '\u0dc4',
+    '\u093e': '\u0dcf',
+    '\u0949': '\u0dd1',
+    '\u093f': '\u0dd2',
+    '\u0940': '\u0dd3',
+    '\u0941': '\u0dd4',
+    '\u0942': '\u0dd6',
+    '\u0943': '\u0dd8',
+    '\u0946': '\u0dd9',
+    '\u0947': '\u0dda',
+    '\u0948': '\u0ddb',
+    '\u094a': '\u0ddc',
+    '\u094b': '\u0ddd',
+    '\u094c': '\u0dde',
+    '\u094d': '\u0dca'
+  }
+
+  /**
+   * Transliterate Devanagari text to Sinhala
+   * @param {string} text - Input Devanagari text
+   * @returns {string} - Transliterated Sinhala text
+   */
+  static devanagariToSinhala (text) {
+    return Array.from(text)
+      .map((c) => SinhalaDevanagariTransliterator.devnagSinhalaMap[c] || c)
+      .join('')
+  }
+
+  /**
+   * Transliterate Sinhala text to Devanagari
+   * @param {string} text - Input Sinhala text
+   * @returns {string} - Transliterated Devanagari text
+   */
+  static sinhalaToDevanagari (text) {
+    return Array.from(text)
+      .map((c) => SinhalaDevanagariTransliterator.sinhalaDevnagMap[c] || c)
+      .join('')
+  }
+}
+
+module.exports = {
+  SinhalaDevanagariTransliterator
+}

package/third-party/indic-processor-deps/indicnlp/unicode_transliterator.js

@@ -0,0 +1,120 @@
+/**
+ *
+ *  Copyright (c) 2013-present, Anoop Kunchukuttan
+ *  All rights reserved.
+ *
+ *  This source code is licensed under the MIT license found in the
+ *  INDIC_NPL_LICENCE file in the indicnlp directory of this source tree.
+ *
+ *  This code is a ported version of the sacremoses library. Please refer to NOTICE
+ *  file in the root directory of this source tree.
+ */
+
+const langinfo = require('./langinfo')
+const sdt = require('./sinhala_transliterator').SinhalaDevanagariTransliterator
+
+/**
+ * Base class for rule-based transliteration among Indian languages.
+ *
+ * Script pair specific transliterators should derive from this class and override the transliterate() method.
+ * They can call the super class 'transliterate()' method to avail of the common transliteration.
+ */
+class UnicodeIndicTransliterator {
+  /**
+   * Handle missing unaspirated and voiced plosives in Tamil script
+   * Replace by unvoiced, unaspirated plosives
+   * @param {number} offset - Character offset
+   * @returns {number} - Corrected offset
+   * @private
+   */
+  static _correctTamilMapping (offset) {
+    // For first 4 consonant rows of varnamala
+    // Exception: ja has a mapping in Tamil
+    if (
+      offset >= 0x15 &&
+      offset <= 0x28 &&
+      offset !== 0x1c &&
+      !((offset - 0x15) % 5 === 0 || (offset - 0x15) % 5 === 4)
+    ) {
+      const substChar = Math.floor((offset - 0x15) / 5)
+      offset = 0x15 + 5 * substChar
+    }
+
+    // For 5th consonant row of varnamala
+    if (offset === 0x2b || offset === 0x2c || offset === 0x2d) {
+      offset = 0x2a
+    }
+
+    // 'sh' becomes 'Sh'
+    if (offset === 0x36) {
+      offset = 0x37
+    }
+
+    return offset
+  }
+
+  /**
+   * Convert the source language script (lang1) to target language script (lang2)
+   * @param {string} text - Text to transliterate
+   * @param {string} lang1Code - Source language code
+   * @param {string} lang2Code - Target language code
+   * @returns {string} - Transliterated text
+   */
+  static transliterate (text, lang1Code, lang2Code) {
+    if (
+      lang1Code in langinfo.SCRIPT_RANGES &&
+      lang2Code in langinfo.SCRIPT_RANGES
+    ) {
+      // If Sinhala is source, do a mapping to Devanagari first
+      if (lang1Code === 'si') {
+        text = sdt.sinhalaToDevanagari(text)
+        lang1Code = 'hi'
+      }
+
+      // If Sinhala is target, make Devanagari the intermediate target
+      let orgLang2Code = ''
+      if (lang2Code === 'si') {
+        orgLang2Code = 'si'
+        lang2Code = 'hi'
+      }
+
+      const transLitText = []
+      for (const c of text) {
+        let newc = c
+        const offset = c.charCodeAt(0) - langinfo.SCRIPT_RANGES[lang1Code][0]
+
+        if (
+          offset >= langinfo.COORDINATED_RANGE_START_INCLUSIVE &&
+          offset <= langinfo.COORDINATED_RANGE_END_INCLUSIVE &&
+          c !== '\u0964' &&
+          c !== '\u0965'
+        ) {
+          let correctedOffset = offset
+          if (lang2Code === 'ta') {
+            // Tamil exceptions
+            correctedOffset =
+              UnicodeIndicTransliterator._correctTamilMapping(offset)
+          }
+
+          newc = String.fromCharCode(
+            langinfo.SCRIPT_RANGES[lang2Code][0] + correctedOffset
+          )
+        }
+
+        transLitText.push(newc)
+      }
+
+      // If Sinhala is target, convert from Devanagari
+      if (orgLang2Code === 'si') {
+        return sdt.devanagariToSinhala(transLitText.join(''))
+      }
+      return transLitText.join('')
+    } else {
+      return text
+    }
+  }
+}
+
+module.exports = {
+  UnicodeIndicTransliterator
+}

package/third-party/indic-processor-deps/sacremoses/SACREMOSES_LICENCE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2020 alvations
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/third-party/indic-processor-deps/sacremoses/cjk.js

@@ -0,0 +1,202 @@
+/**
+ * This module provides functions to detect CJK (Chinese, Japanese, Korean) characters
+ * based on their Unicode code points.
+ */
+
+/**
+ * CJKChars class that enumerates the code points of CJK characters
+ * as listed on http://en.wikipedia.org/wiki/Basic_Multilingual_Plane#Basic_Multilingual_Plane
+ */
+class CJKChars {
+  constructor () {
+    // Hangul Jamo (1100–11FF)
+    this.Hangul_Jamo = [0x1100, 0x11ff]
+
+    // CJK Radicals Supplement (2E80–2EFF)
+    // Kangxi Radicals (2F00–2FDF)
+    // Ideographic Description Characters (2FF0–2FFF)
+    // CJK Symbols and Punctuation (3000–303F)
+    // Hiragana (3040–309F)
+    // Katakana (30A0–30FF)
+    // Bopomofo (3100–312F)
+    // Hangul Compatibility Jamo (3130–318F)
+    // Kanbun (3190–319F)
+    // Bopomofo Extended (31A0–31BF)
+    // CJK Strokes (31C0–31EF)
+    // Katakana Phonetic Extensions (31F0–31FF)
+    // Enclosed CJK Letters and Months (3200–32FF)
+    // CJK Compatibility (3300–33FF)
+    // CJK Unified Ideographs Extension A (3400–4DBF)
+    // Yijing Hexagram Symbols (4DC0–4DFF)
+    // CJK Unified Ideographs (4E00–9FFF)
+    // Yi Syllables (A000–A48F)
+    // Yi Radicals (A490–A4CF)
+    this.CJK_Radicals = [0x2e80, 0xa4cf]
+
+    // Phags-pa (A840–A87F)
+    this.Phags_Pa = [0xa840, 0xa87f]
+
+    // Hangul Syllables (AC00–D7AF)
+    this.Hangul_Syllables = [0xac00, 0xd7af]
+
+    // CJK Compatibility Ideographs (F900–FAFF)
+    this.CJK_Compatibility_Ideographs = [0xf900, 0xfaff]
+
+    // CJK Compatibility Forms (FE30–FE4F)
+    this.CJK_Compatibility_Forms = [0xfe30, 0xfe4f]
+
+    // Range U+FF65–FFDC encodes halfwidth forms, of Katakana and Hangul characters
+    this.Katakana_Hangul_Halfwidth = [0xff65, 0xffdc]
+
+    // Ideographic Symbols and Punctuation (16FE0–16FFF)
+    this.Ideographic_Symbols_And_Punctuation = [0x16fe0, 0x16fff]
+
+    // Tangut (17000-187FF)
+    // Tangut Components (18800-18AFF)
+    this.Tangut = [0x17000, 0x18aff]
+
+    // Kana Supplement (1B000-1B0FF)
+    // Kana Extended-A (1B100-1B12F)
+    this.Kana_Supplement = [0x1b000, 0x1b12f]
+
+    // Nushu (1B170-1B2FF)
+    this.Nushu = [0x1b170, 0x1b2ff]
+
+    // Supplementary Ideographic Plane (20000–2FFFF)
+    this.Supplementary_Ideographic_Plane = [0x20000, 0x2ffff]
+
+    // Collect all ranges in a single array
+    this.ranges = [
+      this.Hangul_Jamo,
+      this.CJK_Radicals,
+      this.Phags_Pa,
+      this.Hangul_Syllables,
+      this.CJK_Compatibility_Ideographs,
+      this.CJK_Compatibility_Forms,
+      this.Katakana_Hangul_Halfwidth,
+      this.Ideographic_Symbols_And_Punctuation,
+      this.Tangut,
+      this.Kana_Supplement,
+      this.Nushu,
+      this.Supplementary_Ideographic_Plane
+    ]
+  }
+}
+
+// Create a singleton instance of CJKChars
+const cjkCharsRanges = new CJKChars().ranges
+
+/**
+ * Checks if a character is a CJK (Chinese, Japanese, Korean) character
+ *
+ * @param {string} character - The character to check (must be a single character)
+ * @returns {boolean} - True if the character is a CJK character, false otherwise
+ *
+ * @example
+ * isCJK('世'); // true
+ * isCJK('A'); // false
+ * isCJK('ひ'); // true (Hiragana)
+ * isCJK('カ'); // true (Katakana)
+ * isCJK('한'); // true (Hangul)
+ */
+function isCJK (character) {
+  // Check that we've got a single character
+  if (character.length !== 1) {
+    throw new Error('isCJK requires a single character as input')
+  }
+
+  // Get the Unicode code point of the character
+  const codePoint = character.codePointAt(0)
+
+  // Check if the code point falls within any of the CJK ranges
+  for (const [start, end] of cjkCharsRanges) {
+    if (codePoint < end) {
+      return codePoint >= start
+    }
+  }
+
+  return false
+}
+
+/**
+ * Checks if a string contains any CJK characters
+ *
+ * @param {string} text - The string to check
+ * @returns {boolean} - True if the string contains any CJK characters, false otherwise
+ *
+ * @example
+ * containsCJK('Hello 世界'); // true
+ * containsCJK('Hello world'); // false
+ */
+function containsCJK (text) {
+  for (let i = 0; i < text.length; i++) {
+    // Handle surrogate pairs correctly for characters outside BMP
+    const char = text.charAt(i)
+
+    // Skip high surrogate if it's part of a surrogate pair
+    if (
+      i < text.length - 1 &&
+      char.charCodeAt(0) >= 0xd800 &&
+      char.charCodeAt(0) <= 0xdbff
+    ) {
+      const pair = text.substring(i, i + 2)
+      if (isCJK(pair)) {
+        return true
+      }
+      i++ // Skip the low surrogate in the next iteration
+      continue
+    }
+
+    if (isCJK(char)) {
+      return true
+    }
+  }
+
+  return false
+}
+
+/**
+ * Counts the number of CJK characters in a string
+ *
+ * @param {string} text - The string to check
+ * @returns {number} - The count of CJK characters in the string
+ *
+ * @example
+ * countCJKChars('Hello 世界'); // 2
+ * countCJKChars('Hello world'); // 0
+ */
+function countCJKChars (text) {
+  let count = 0
+
+  for (let i = 0; i < text.length; i++) {
+    // Handle surrogate pairs correctly for characters outside BMP
+    const char = text.charAt(i)
+
+    // Skip high surrogate if it's part of a surrogate pair
+    if (
+      i < text.length - 1 &&
+      char.charCodeAt(0) >= 0xd800 &&
+      char.charCodeAt(0) <= 0xdbff
+    ) {
+      const pair = text.substring(i, i + 2)
+      if (isCJK(pair)) {
+        count++
+      }
+      i++ // Skip the low surrogate in the next iteration
+      continue
+    }
+
+    if (isCJK(char)) {
+      count++
+    }
+  }
+
+  return count
+}
+
+module.exports = {
+  CJKChars,
+  isCJK,
+  containsCJK,
+  countCJKChars
+}

package/third-party/indic-processor-deps/sacremoses/data/nonbreaking_prefixes/README.txt

@@ -0,0 +1,8 @@
+The language suffix can be found here:
+
+http://www.loc.gov/standards/iso639-2/php/code_list.php
+
+This code includes data from Daniel Naber's Language Tools (czech abbreviations).
+This code includes data from czech wiktionary (also czech abbreviations).
+
+

package/third-party/indic-processor-deps/sacremoses/data/nonbreaking_prefixes/nonbreaking_prefix.as

@@ -0,0 +1,65 @@
+#Anything in this file, followed by a period (and an upper-case word), does NOT indicate an end-of-sentence marker.
+
+#common exceptions
+# Dr
+ড
+
+#others
+
+
+#phonetics
+# A
+এ
+# B
+বি
+# C
+সি
+# D
+ডি
+# E
+ই
+# F
+এফ
+# G
+জি
+# H
+এইচ
+# I
+আম
+# J
+জে
+# K
+কে
+# L
+এল
+# M
+এম
+# N
+এন
+# O
+হে
+# P
+পি
+# Q
+কিউ
+# R
+আর
+# S
+এস
+# T
+টি
+# U
+ইউ
+# V
+ভি 
+# W
+ডব্লু
+# X
+এক্স
+# Y
+ওয়াই
+# Z
+জেড
+
+#consonants
+

package/third-party/indic-processor-deps/sacremoses/data/nonbreaking_prefixes/nonbreaking_prefix.bn

@@ -0,0 +1,65 @@
+#Anything in this file, followed by a period (and an upper-case word), does NOT indicate an end-of-sentence marker.
+
+#common exceptions
+# Dr
+ড
+
+#others
+
+
+#phonetics
+# A
+এ
+# B
+বি
+# C
+সি
+# D
+ডি
+# E
+ই
+# F
+এফ
+# G
+জি
+# H
+এইচ
+# I
+আম
+# J
+জে
+# K
+কে
+# L
+এল
+# M
+এম
+# N
+এন
+# O
+হে
+# P
+পি
+# Q
+কিউ
+# R
+আর
+# S
+এস
+# T
+টি
+# U
+ইউ
+# V
+ভি 
+# W
+ডব্লু
+# X
+এক্স
+# Y
+ওয়াই
+# Z
+জেড
+
+#consonants
+