interscript-maps 2.2.1 → 2.3.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7770cf9ef9d575be661e1ab0f7890d757e28dbb31493f88ea09fcccb05efbaa1
-  data.tar.gz: 4c9e662f53f5d77a27666823c4d8aec0c050fdbb6dada2ec77df78cb5a275998
+  metadata.gz: ea020ba190accdefcf21f2d09e0ffd3b0c67561f36df93961f692152a356547f
+  data.tar.gz: 17fb0c0f6ef53c620f261caa215c5b20c1401e98949b9f191208d2df81c17ba5
 SHA512:
-  metadata.gz: 02d5302ba2245cc7197fb6e4025436f3337c210e7211c20dd5553ae7757f495c31e6a1f1b746c71c4dc6e8d0913b9d6c1686f5345c850423ec4abade560cf1e3
-  data.tar.gz: 0b0087d68fd3f363ecec221ce3edf1a1e5f99a14088bee500553250eccb0a48a059a97475903d765b1ec7c8d875fdbb21db4002d10d9e351e8e31f88e687bcf7
+  metadata.gz: b5a546886594fedd2bfccf0d6aa91e768959439b0989cba8e2f0d4a4f3f494aaac79020e9c15d1cba64973fcfb90f05c5cf0d09697f530027833703c64dad282
+  data.tar.gz: 3d1eed5cff23dc6ad83ca6620f5c7c86faef60616a927c549d6f49bc2bd4e4437c78be9aeb4a9a0eaefbb4bc7a87090605c6f425ca38c31a4949403716fe2bf6

data/interscript-maps.gemspec

@@ -1,4 +1,4 @@
-INTERSCRIPT_MAPS_VERSION="2.2.1"
+INTERSCRIPT_MAPS_VERSION="2.3.2"
 
 Gem::Specification.new do |spec|
   spec.name          = "interscript-maps"

data/interscript-maps.yaml

@@ -16,6 +16,36 @@ staging:
 secryst-models:
   - https://raw.githubusercontent.com/secryst/data-thai-interscript/master/index.yaml
 
+# Here we describe Rababa configs for the maps that are required
+rababa-configs:
+  "200":
+    model: https://github.com/secryst/rababa-models/releases/download/0.1/diacritization_model_max_len_200.onnx
+    config:
+      session_name: base
+      text_encoder: ArabicEncoderWithStartSymbol
+      text_cleaner: valid_arabic_cleaners
+      max_len: 200
+      batch_size: 32
+      
+# This instruction denotes that the following maps are incompatible with certain
+# compilers and platforms and shouldn't be tested or compiled in.
+skip:
+  Interscript::Compiler::Javascript:
+    - var-ara-Arab-Arab-rababa
+  # It works for detection, but takes too much time and doesn't make sense for
+  # the main usecase of Detector
+  Interscript::Detector:
+    - var-ara-Arab-Arab-rababa
+  # Windows currently fails tests for rababa
+  mswin64:
+    - var-ara-Arab-Arab-rababa
+  mswin32:
+    - var-ara-Arab-Arab-rababa
+  mingw32:
+    - var-ara-Arab-Arab-rababa
+  cygwin:
+    - var-ara-Arab-Arab-rababa
+
 # This part registers aliases. Those were previously a part of maps themselves,
 # moving this here saves us from a burden of having to load all the maps to
 # locate one that we are trying to access.

data/maps/alalc-div-Thaa-Latn-2012.imp

@@ -0,0 +1,74 @@
+metadata {
+  authority_id: alalc
+  id: 2012
+  language: iso-639-2:div
+  source_script: Thaa
+  destination_script: Latn
+  name: Romanization Table -- Divehi (2012)
+  url: https://www.loc.gov/catdir/cpso/romanization/divehi.pdf
+  creation_date: 2012
+  description: |
+    ALA-Library of Congress Divehi Romanization 2012 System
+
+  notes:
+
+    - |
+      Romanize ށް as ḫ when it doubles the following consonant or is used as a glottal stop.
+        aḫvana	އަށްވަނަ
+        maśaḫ	މަށަށް
+    - |
+      When used in medial position without ް (sukūn), romanize ނ as ṁ.
+        aṁga	އަނގަ
+        haṁdu	ހަނދު
+    - |
+      Romanization of އ.
+      (a) When used in the initial position with any vowel sign, do not romanize.
+        ata	އަތަ
+        idu	އިދު
+        umuru	އުމުރު
+        egahugi	އެގަހުގި
+      (b) When used in the medial position with any vowel sign, romanize as ’.
+        ha’hūnu	ހައިހޫނު
+        fa’isa	ފައިސަ
+        k’īn	ކްއީން
+      (c) When a consonant follows އް in medial position, double it in romanization.
+        cappalu	ޗައްޕަލު
+        appacci	އައްޕައްޗި
+      (d) When used in final position with ް (sukūn), romanize as h.
+        boh	ބޮއް
+        biheh	ބިހެއް
+    - |
+      Romanize ތް followed by another ތ as  t̤ .
+       at̤teri   އަތްތެރި
+    - |
+      Only the vowel forms that appear at the beginning of a syllable are listed.
+      When the vowels follow a consonant, އ is not used and the vowel signs are added to the consonant forms.
+      Do not distinguish between the two in romanization.
+    - |
+      ް (called sukūn) generally indicates omission of an inherent vowel associated with a consonant.
+      For its other uses, see Notes 1, 3, and 4.
+}
+
+tests {
+  test "މަށަށް", "maśaḫ"
+  test "އަނގަ", "aṁga"
+  test "ހަނދު", "haṁdu"
+  test "އަތަ", "ata"
+  test "އިދު", "idu"
+  test "އުމުރު", "umuru"
+  test "އެގަހުގި", "egahugi"
+  test "ފައިސަ", "faʼisa"
+  test "ބޮއް", "boh"
+  test "ބިހެއް", "biheh"
+  test "އަތްތެރި", "at̤teri"
+  test "ޗައްޕަލު", "cappalu"
+  test "އައްޕައްޗި", "appacci"
+}
+
+dependency "alalc-div-Thaa-Latn-1997", as: thaalatn
+
+stage {
+
+  run map.thaalatn.stage.main
+
+}

data/maps/alalc-kat-Geok-Latn-1997.imp

@@ -5,12 +5,11 @@ metadata {
   source_script: Geok
   destination_script: Latn
   name: Romanization Table -- Georgian Khutsuri (1997)
-  url: https://www.loc.gov/catdir/cpso/romanization/georgian.pdf
+  url: http://catdir.loc.gov/catdir/cpso/romanization/georgian.pdf
   creation_date: 1997
   confirmation_date: 1997
   description: |
     Values are shown for the Khutsuri alphabet.
-
   notes:
 }
 

data/maps/alalc-kat-Geok-Latn-2011.imp

@@ -0,0 +1,31 @@
+metadata {
+  authority_id: alalc
+  id: 2011
+  language: iso-639-2:kat
+  source_script: Geok
+  destination_script: Latn
+  name: Romanization Table -- Georgian Khutsuri (2011)
+  url: https://www.loc.gov/catdir/cpso/romanization/georgian.pdf
+  creation_date: 2011
+  confirmation_date: 2011
+  description: |
+    Values are shown for the Khutsuri alphabet.
+  notes:
+}
+
+tests {
+  test "ႼႨႢႬႨ", "CIGNI"
+  test "ⴜⴈⴂⴌⴈ", "cigni"
+  test "ႱႭႪႭႫႭႬ", "SOLOMON"
+  test "ⴑⴍⴊⴍⴋⴍⴌ", "solomon"
+  test "ႠႡႰႠჀႠႫ", "ABRAHAM"
+}
+
+
+dependency "alalc-kat-Geok-Latn-1997", as: geoklatn
+
+stage {
+
+  run map.geoklatn.stage.main
+
+}

data/maps/alalc-kat-Geor-Latn-1997.imp

@@ -5,13 +5,12 @@ metadata {
   source_script: Geor
   destination_script: Latn
   name: Romanization Table -- Georgian Mkhedruli (1997)
-  url: https://www.loc.gov/catdir/cpso/romanization/georgian.pdf
+  url: http://catdir.loc.gov/catdir/cpso/romanization/georgian.pdf
   creation_date: 1997
   confirmation_date: 1997
   description: |
     Values are shown for the older Khutsuri and the modern Mkhedruli alphabets.
     There are no upper case letters in Mkhedruli.
-
   notes:
 }
 

data/maps/alalc-kat-Geor-Latn-2011.imp

@@ -0,0 +1,52 @@
+metadata {
+  authority_id: alalc
+  id: 2011
+  language: iso-639-2:kat
+  source_script: Geor
+  destination_script: Latn
+  name: Romanization Table -- Georgian Mkhedruli (2011)
+  url: https://www.loc.gov/catdir/cpso/romanization/georgian.pdf
+  creation_date: 2011
+  confirmation_date: 2011
+  description: |
+    Values are shown for the older Khutsuri and the modern Mkhedruli alphabets.
+    There are no upper case letters in Mkhedruli.
+  notes:
+}
+
+tests {
+  test "ხაოფსე", "xaopʻse"
+  test "ჭლოუ", "člou"
+  test "ჩოხულდი", "čʻoxuldi"
+  test "ქვემო ლინდა", "kʻvemo linda"
+  test "ტამკვაჩ იგვავერა", "tamkvačʻ igvavera"
+  test "სვანეთი", "svanetʻi"
+  test "საცხვარისი", "sacʻxvarisi"
+  test "მუხრან-თელეთი", "muxran-tʻeletʻi"
+  test "მუცდი", "mucʻdi"
+  test "ლეჩხუმი", "lečʻxumi"
+  test "ვერხნაია მწარა", "verxnaia mcara"
+  test "ეგრისის ქედი", "egrisis kʻedi"
+  test "დოჩარიფშა", "dočʻaripʻša"
+  test "ბოლოკო", "boloko"
+  test "აჭანდარა", "ačandara"
+  test "აუალიცა", "aualicʻa"
+  test "აკალამრა", "akalamra"
+  test "ლასილი", "lasili"
+  test "გუბაზეული", "gubazeuli"
+  test "ბაყაყი", "baqaqi"
+  test "ძროხა", "żroxa"
+  test "ჰაერი", "haeri"
+  test "ჟოლო", "žolo"
+  test "ჯართი", "jartʻi"
+  test "ღრმაღელე", "ġrmaġele"
+}
+
+dependency "alalc-kat-Geor-Latn-1997", as: georlatn
+
+stage {
+
+  run map.georlatn.stage.main
+
+}
+

data/maps/alalc-mal-Mlym-Latn-2012.imp

@@ -4,7 +4,7 @@ metadata {
   language: iso-639-2:mal
   source_script: Mlym
   destination_script: Latn
-  name: Malayalam Romanization, 2012
+  name: Romanization Table -- Malayalam (2012)
   url: https://www.loc.gov/catdir/cpso/romanization/malayalam.pdf
   creation_date: 2012
   description: |

data/maps/bas-rus-Cyrl-Latn-2017-bss.imp

@@ -4,7 +4,7 @@ metadata {
   language: iso-639-2:rus
   source_script: Cyrl
   destination_script: Latn
-  name: Streamlined Romanization of Russian Cyrillic -- Basic Streamlined System
+  name: Streamlined Romanization of Russian Cyrillic (Basic Streamlined System)
   url: https://www.researchgate.net/publication/318402098
   creation_date: 2017-07
   description: |

data/maps/bas-rus-Cyrl-Latn-2017-oss.imp

@@ -4,7 +4,7 @@ metadata {
   language: iso-639-2:rus
   source_script: Cyrl
   destination_script: Latn
-  name: Streamlined Romanization of Russian Cyrillic -- Optimized Streamlined System
+  name: Streamlined Romanization of Russian Cyrillic (Optimized Streamlined System)
   url: https://www.researchgate.net/publication/318402098
   creation_date: 2017-07
   description: |

data/maps/bgn-jpn-Hrkt-Latn-1962.imp

@@ -4,7 +4,7 @@ metadata {
   language: iso-639-2:jpn
   source_script: Hrkt
   destination_script: Latn
-  name: BGN Modified Hepburn Japanese transliteration system (1962)
+  name: Japanese Modified Hepburn (1962)
   url:
   creation_date: 1930
   adoption_date: 1962

data/maps/bgn-kor-Hang-Latn-1943.imp

@@ -4,7 +4,7 @@ metadata {
   language: iso-639-2:kor
   source_script: Hang
   destination_script: Latn
-  name: BGN Agreement -- Korean McCune-Reischauer System (Hangul to Latin) (1943)
+  name: Korean McCune-Reischauer (Hangul to Latin) (1943 Agreement)
   url:
   creation_date: 1943
   adoption_date:

data/maps/bgn-kor-Kore-Latn-1943.imp

@@ -4,7 +4,7 @@ metadata {
   language: iso-639-2:kor
   source_script: Kore
   destination_script: Latn
-  name: BGN Agreement -- Korean McCune-Reischauer System (Korean to Latin) (1943)
+  name: Korean McCune-Reischauer (Korean to Latin) (1943 Agreement)
   url:
   creation_date: 1943
   adoption_date:

data/maps/bgnpcgn-bak-Cyrl-Latn-2007.imp

@@ -4,7 +4,7 @@ metadata {
   language: iso-639-2:rus
   source_script: Cyrl
   destination_script: Latn
-  name: BASHKIR TABLE OF CORRESPONDENCES CYRILLIC-ROMAN BGN/PCGN 2007 Agreement
+  name: Table of Correspondences for Bashkir (Cyrillic-Roman) (2007 Agreement)
   url: https://assets.publishing.service.gov.uk/government/uploads/system/uploads/attachment_data/file/829203/TABLE_OF_CORRESPONDENCES__FOR_BASHKIR.pdf
   creation_date: 2007
   confirmation_date: 2019

data/maps/bgnpcgn-bel-Cyrl-Latn-1979.imp

@@ -4,7 +4,7 @@ metadata {
   language: iso-639-2:bel
   source_script: Cyrl
   destination_script: Latn
-  name: United States Board on Geographic Names Foreign Names Committee Staff, 1994. Romanization Systems and Roman-Script Spelling Conventions, p. 23.
+  name: Romanization of Belarusian (1979)
   url: https://assets.publishing.service.gov.uk/government/uploads/system/uploads/attachment_data/file/811510/ROMANIZATION_OF_BELARUSIAN.pdf
   creation_date: 1979
   description: |
@@ -12,6 +12,9 @@ metadata {
     romanizing names written in the Belarusian Cyrillic alphabet. The Belarusian alphabet contains three
     characters not present in the Russian alphabet: і, ў, and ’.
 
+    United States Board on Geographic Names Foreign Names Committee Staff, 1994.
+    Romanization Systems and Roman-Script Spelling Conventions, p. 23.
+
   notes:
    - The character sequences зг, кг, сг, тс and цг and may be romanized z·h, k·h, s·h, t·s and ts·h in order to differentiate those romanizations from the digraphs zh, kh, sh, ts, and the letter sequence tsh, which are used to render the characters ж, x, ш, ц, and the character sequence тш
    - All apostrophes appearing in romanization are Unicode encoding 2019.

data/maps/bgnpcgn-che-Cyrl-Latn-2008.imp

@@ -4,7 +4,8 @@ metadata {
   language: iso-639-2:che
   source_script: Cyrl
   destination_script: Latn
-  name: https://assets.publishing.service.gov.uk/government/uploads/system/uploads/attachment_data/file/835782/TABLE_OF_CORRESPONDENCES_FOR_CHECHEN.pdf
+  name: Table of Correspondences for Chechen (Cyrillic-Roman) (2008 Agreement)
+  url: https://assets.publishing.service.gov.uk/government/uploads/system/uploads/attachment_data/file/835782/TABLE_OF_CORRESPONDENCES_FOR_CHECHEN.pdf
   creation_date: 2008
   confirmation_date: 2019-07
   description: |

data/maps/bgnpcgn-kor-Hang-Latn-kn-1945.imp

@@ -4,7 +4,7 @@ metadata {
   language: iso-639-2:kor
   source_script: Hang
   destination_script: Latn
-  name: Romanization of Korean -- For DPRK (Hangul to Latin) (1945 Agreement)
+  name: Romanization of Korean for DPRK (Hangul-Latin) (1945 Agreement)
   url: https://assets.publishing.service.gov.uk/government/uploads/system/uploads/attachment_data/file/693725/ROMANIZATION_OF_KOREAN-_MR_for_DPRK.pdf
   creation_date: 1945
   adoption_date:

data/maps/bgnpcgn-kor-Hang-Latn-rok-2011.imp

@@ -4,14 +4,14 @@ metadata {
   language: iso-639-2:kor
   source_script: Hang
   destination_script: Latn
-  name: Romanization of Korean -- For the ROK (2011 Agreement, Ministry of Culture and Tourism System (2000))
+  name: Romanization of Korean for ROK (Hangul-Latin) (2011 Agreement, MOCT System 2000)
   url: https://assets.publishing.service.gov.uk/government/uploads/system/uploads/attachment_data/file/693724/ROMANIZATION_OF_KOREAN-__MOCT_for_ROK.pdf
   creation_date: 2011
   adoption_date: 2011
   description:
 
   notes:
-    BGN/PCGN 2011 Agreement
+  - BGN/PCGN 2011 Agreement
 }
 
 tests {

data/maps/bgnpcgn-kor-Kore-Latn-rok-2011.imp

@@ -4,9 +4,8 @@ metadata {
   language: iso-639-2:kor
   source_script: Kore
   destination_script: Latn
-  name: Romanization of Korean -- For the ROK -- Korean to Latin (2011 Agreement, Ministry of Culture and Tourism System (2000))
+  name: Romanization of Korean for ROK (Korean-Latin) (2011 Agreement, MOCT System 2000)
   url: https://assets.publishing.service.gov.uk/government/uploads/system/uploads/attachment_data/file/693724/ROMANIZATION_OF_KOREAN-__MOCT_for_ROK.pdf
-  url: ""
   creation_date: 2011
   adoption_date: 2011
   description: |
@@ -14,7 +13,7 @@ metadata {
     Intended to provide a means for international communication of written documents.
 
   notes:
-    - BGN/PCGN 2011 Agreement
+  - BGN/PCGN 2011 Agreement
 }
 
 tests {

data/maps/bis-asm-Beng-Latn-13194-1991.imp

@@ -4,7 +4,7 @@ metadata {
   language: iso-639-2:asm
   source_script: Beng
   destination_script: Latn
-  name: Indian script code for information interchange - ISCII - Assamese Romanization
+  name: Indian script code for information interchange (ISCII) -- Assamese Romanization
 
   #url:
   creation_date: 1991

data/maps/bis-ben-Beng-Latn-13194-1991.imp

@@ -4,7 +4,7 @@ metadata {
   language: iso-639-2:ben
   source_script: Beng
   destination_script: Latn
-  name: Indian script code for information interchange - ISCII - Bengali Romanization
+  name: Indian script code for information interchange (ISCII) -- Bengali Romanization
 
   #url:
   creation_date: 1991

data/maps/bis-dev-Deva-Latn-13194-1991.imp

@@ -4,7 +4,7 @@ metadata {
   language: iso-639-2:dev
   source_script: Deva
   destination_script: Latn
-  name: Indian script code for information interchange - ISCII - Devanagri Romanization
+  name: Indian script code for information interchange (ISCII) -- Devanagri Romanization
 
   #url:
   creation_date: 1991

data/maps/bis-guj-Gujr-Latn-13194-1991.imp

@@ -4,7 +4,7 @@ metadata {
   language: iso-639-2:guj
   source_script: Gujr
   destination_script: Latn
-  name: Indian script code for information interchange - ISCII - Gujrati Romanization
+  name: Indian script code for information interchange (ISCII) -- Gujrati Romanization
 
   #url:
   creation_date: 1991

data/maps/bis-kan-Kana-Latn-13194-1991.imp

@@ -4,7 +4,7 @@ metadata {
   language: iso-639-2:kan
   source_script: Kana
   destination_script: Latn
-  name: Indian script code for information interchange - ISCII - Kannada Romanization
+  name: Indian script code for information interchange (ISCII) -- Kannada Romanization
 
   #url:
   creation_date: 1991