interscript 0.1.5 → 0.1.6

data/maps/odni-kat-Geor-Latn-2015.yaml

@@ -85,4 +85,3 @@ map:
     '\u10ee' : 'kh' # ხ
     '\u10ef' : 'j' # ჯ
     '\u10f0' : 'h' # ჰ
-

data/maps/odni-rus-Cyrl-Latn-2015.yaml

@@ -35,7 +35,7 @@ description: |
   time been listed in the Factbook or Chiefs of State directory but who no longer appears in those resources
   (i.e. is no longer a government official), the IC Standard spelling should appear first, with the spelling, if
   known, as it previously appeared in those resources listed within parentheses at the first usage.
-  
+
   The primary goal is to produce a consistent Romanized transcription of names that is specifically readable
   to the English-speaking non-specialist. The system uses the 26 letters of the standard (English) Roman
   alphabet. Some ambiguities in the Romanized form will occur without the use of diacritics. However,

data/maps/odni-srp-Cyrl-Latn-2015.yaml

@@ -35,14 +35,14 @@ description: |
   longer appears in those resources (i.e. is no longer a government official), the IC Standard
   spelling should appear first, with the spelling, if known, as it previously appeared in those
   resources listed within parentheses at the first usage.
-  
+
   The primary goal is to produce a consistent Romanized transcription of names that is specifically
   readable to the English-speaking non-specialist. The system uses the 26 letters of the standard
   (English) Roman alphabet. Some ambiguities in the Romanized form will occur without the use
   of diacritics. However, within the context of a report, where additional information about the
   individual is provided, the referent will be clearly identified. This system will be used in
   conjunction with on-line tools, name dictionaries, and lists containing conventional spellings of
-  names of well-known individuals. 
+  names of well-known individuals.
 
 notes:
 

data/maps/odni-urd-Arab-Latn-2015.yaml

@@ -0,0 +1,221 @@
+---
+authority_id: odni
+id: 2015
+language: urd
+source_script: Arab
+destination_script: Latn
+name: Office of the Director Of National Intelligence Hindi Urdu Personal Names 2015 System, ICS-630-01 Annex F
+#url:
+creation_date: 2015
+confirmation_date: 2015
+description: |
+  This system is the Intelligence Community (IC) standard for the transliteration of names
+  from Hindi and Urdu that will be applied to all final written reports and products for IC
+  consumers. It is not intended to eliminate variations of a name that can contribute
+  forensic information. Rather, it is to provide an IC standard Romanized (English)
+  transliteration from standard Hindi and Urdu that can then be linked to forensic
+  information in ways that will help identify the referent of the name.
+
+  There are typically a number of ways that names can be Romanized from either
+  Devanagari (Hindi) or modified Arabic (Urdu) scripts. Ambiguities can result from the
+  Romanization of Hindi and Urdu names for several reasons, including the fact that some
+  sounds in South Asian languages (e.g., retroflex consonants, voiced aspirates) have no
+  equivalent in English or other European languages. In the case of Urdu, as in the original
+  Arabic source of many Islamic names, short vowel markings, double consonant marks
+  and other diacritics that would clearly distinguish the name are almost always omitted
+  from standard written texts. And many Islamic names of Arabic or Persian origin reflect
+  spelling distinctions from those languages that are lost in modern Urdu pronunciation
+  (e.g., three distinct Arabic letters all represent the identical sound [s] for Urdu speakers);
+  transliterations might either maintain those spelling distinctions or ignore them entirely.
+  And, as in the Arabic source, names containing the Arabic definite article ‘al’ (‘ul’) show
+  anticipatory assimilation in pronunciation (e.g., Shams al Din > Shamsuddin);
+  transliterations may either reflect spelling or pronunciation in such cases.
+
+  Because Hindi and Urdu overlap so extensively, it is desirable to correlate Hindi and
+  Urdu transliterations as much as possible. In the area of names, this can become
+  problematic when the Urdu spelling accurately reflects original Arabic spelling while the
+  Hindi spelling was phonetically-based, in essence, already a transliteration. This would
+  argue for ignoring Arabic/Urdu spelling distinctions not reflected in pronunciation in
+  either Urdu or Hindi (e.g., letter sin versus letter sad) while maintaining Arabic/Urdu
+  spelling distinctions like Hindi also maintains (e.g., qaf as in Qutubbin versus kaf)).
+  However, this is not always possible (see Hindi va versus Urdu wau).
+
+  In cases where an individual’s name has already been transliterated, that is to be indicated
+  – as found – in parentheses immediately following its rendition in the transliteration
+  standard (e.g., Muhammad Khulud (Mohamed Khulood)). In addition, if the original
+  Devanagari or Arabic-script spelling is known, that spelling should also appear in
+  parentheses following the name, if possible, following best practices of the issuing
+  organization and taking into consideration information system capabilities. This
+  convention is designed to ensure that vital forensic information is not lost.
+
+  For names of persons who are known to not be part of the Hindi- or Urdu-speaking
+  community, use the relevant IC transliteration standard for names from that language
+  (e.g., Mikhail, Yitzhak). A translator’s note may be used to clarify the known origin of
+  the person. Spell names of individuals from languages that are written in Roman letters as
+  they are spelled in those languages (e.g., George Clooney, Jorge Garcia, Georges
+  Pompidou).
+
+  In the case of active senior government officials in the on-line CIA World Factbook and
+  the on-line directory of Chiefs of State and Cabinet Members of Foreign Governments,
+  the spellings given in these on-line reference works should be used in place of the IC
+  Standard. For any individual who has at one time been listed in the Factbook or Chiefs of
+  State directory but who no longer appears in those resources (i.e. is no longer a
+  government official), the IC Standard spelling should appear first, with the spelling, if
+  known, as it previously appeared in those resources listed within parentheses at the first
+  usage.
+
+  The primary goal of this system is to produce a consistent Romanized transcription of the
+  name that is readable to the non-specialist. The system uses the 26 letters of the standard
+  (English) Roman alphabet. Some ambiguities in the Romanized form will occur without
+  the use of diacritics. However, within the context of a report, where additional
+  information about the individual is provided, the referent will be clearly identified.
+  Additionally, this system will be used in conjunction with on-line tools, name dictionaries
+  and lists containing conventional spellings of names of well-known individuals
+
+notes:
+
+  - |
+    Long/Short Vowels: Long and short vowels are not distinguished in the system:
+    The borrowed Arabic name Samir could represent two distinct names, one with a
+    long /a/ (Saamir) and one with a long /i/ (Samiir). One solution would be to use
+    /ee/ to stand for the long /i/, as is often done (Sameer). The IC Standard will not
+    distinguish between these.
+
+  - |
+    No distinction is made between: retroflex and non-retroflex consonants; and
+    nasalized vowels and vowels followed by /n/.
+  - |
+    No distinction is made between the several Arabic letters with the same phonetic
+    value in Urdu: e.g., letters sin/svad, zal/ze/zoe.
+  - |
+    A distinction is drawn between Urdu letters qaf and kaf (and correspondingly,
+    Hindi qa and ka).
+  - |
+    A distinction is drawn between aspirated (e.g., /d/) and nonaspirated consonants
+    (e.g., /dh/), with the exception of ch/chh, both represented by /ch/.
+  - |
+    Double consonants: Double consonants represented by the tashdid (shaddah) are
+    shown in most cases (e.g., Hassan, Muhammad). Exceptions: consonants
+    represented by digraphs are not doubled (e.g., Mubashir [not Mubashshir]).
+  - |
+    Hamzah (glottal stop) and ayn: Unlike in the Arabic IC Standard, these are not
+    represented in the IC standard.
+  - |
+    Digraphs: No distinction is made between digraphs such as /sh/ and single
+    contiguous letters such as /s/ followed by /h/.
+  - |
+    Arabic definite article “al” (‘the’): Shows sun letter assimilation in the
+    Romanized form (e.g., Abdur Rahman rather than Abdal Rahman, rather than
+    Arabic IC standard 'Abd al-Rahman).
+  - |
+    Special Rules
+    - |
+      Hyphens: Hyphens (-) are NOT used to connect name elements within a name:
+      Abdur Rahman. The single exception to this is the izafat (i.e., linking vowel in
+      noun-link-modifier construction of Persian origin), which does show a hyphen
+      before the /e/ and a following space: Koh-e Nur (‘mountain of light’), “Jaish-e
+      xx” (‘Army of xx’ construction).
+    - |
+      Names incorporating “din” are written as one unit: Azermuddin, Badruddin,
+    Faizuddin, Salahuddin.
+    - |
+      Names that incorporate Allah as part of the name show the Arabic grammatical
+      marker /u/ rather than the /a/ of Allah: Abdullah (not Abdallah).
+    - |
+      Inherent short vowel /a/ in Devanagari is represented with an /a/ in Roman. Final
+      consonants are assumed not to have a short /a/ (e.g., masc. name Ram Lal, not
+      Rama Lala).
+    - |
+      As a general rule, Devanagari va is transcribed as a /v/: Vijay, Vishal, etc.
+      Exception: /sw/ combination: Saraswati, Krishnaswami. Urdu wau, however, is
+      transcribed as /w/: Wasim, Walid.
+
+tests:
+  - source: "وشال ٹھاکر"
+    expected: "ishal tsakr"
+  - source: "حسن محمود"
+    expected: "hasn mhamid"
+  - source: "یوسف خان"
+    expected: "iisf khan"
+  - source: "معین خان"
+    expected: "mein khan"
+  - source: "سعید اجمل"
+    expected: "seid ajml"
+  - source: "انضمام الحق"
+    expected: "anzmam alhaq"
+  - source: "فرہاد رضا"
+    expected: "frahad rza"
+  - source: "وسیم اکرام"
+    expected: "isim akram"
+  - source: "شکیب الحسن"
+    expected: "shkib alhasn"
+  - source: "حسن عبد اللہ"
+    expected: "hasn ebd allah"
+
+map:
+
+  characters:
+
+
+    '\u0622': 'a' # ARABIC ALEF W/MADDA ABOVE
+    '\u0627': 'a' # or u ARABIC ALEF
+    '\u06cc': 'i' # or ai or u ARABIC LETTER WAW
+    '\u0648': 'i' # or o,au ARABIC LETTER FARSI YEH
+    '\u06d2': 'e' # ARABIC LETTER YEH BARREE
+    '\u0650': 'e' # ARABIC ARABIC KASRA
+    '\u0639': 'e' # ARABIC LETTER AIN
+    '\u06c1': 'ah' # ARABIC LETTER HEH GOAL
+    '\u062d': 'ha' # ARABIC LETTER HAH
+    '\u0648\u0670': 'a' # ARABIC LETTER FARSI YEH + ARABIC LETTER SUPERSCRIPT ALEF
+    '\u0670': 'a' # ARABIC LETTER SUPERSCRIPT ALEF
+    '\u0627\u0650': 'i' # ARABIC ALEF + ARABIC KASRA
+    '\u0639\u0650': 'i' # ARABIC LETTER AIN + ARABIC KASRA
+    '\u064f': 'i' # ARABIC DAMMA
+    '\u06d3': 'i' #  ARABIC LETTER YEH BARREE WITH HAMZA ABOVE
+    '\u06c2': '-e' #  ARABIC LETTER HEH GOAL WITH HAMZA ABOVE
+    '\u06c1\u0650': 'o' #  ARABIC LETTER YEH  WITH HAMZA ABOVE
+    '\u0624': 'au' #  ARABIC LETTER WAW WITH HAMZA ABOVE
+
+
+
+    '\u06a9': 'k' #   ARABIC LETTER KEHEH
+    '\u0642': 'q' #   ARABIC LETTER QAF
+    '\u062e': 'kh' #   ARABIC LETTER KHAH
+    '\u06af': 'g' #   ARABIC LETTER GAF
+    '\u063a': 'gh' #   ARABIC LETTER GHAIN
+    '\u0686': 'ch' #   ARABIC LETTER TCHEH
+    '\u062c': 'j' #   Arabic Letter Jeem
+
+    '\u0632': 'z' #   Arabic Letter ZAIN
+    '\u0630': 'z' #   Arabic Letter THAL
+    '\u0636': 'z' #   Arabic Letter DAD
+    '\u0638': 'z' #   Arabic Letter ZAH
+    '\u0698': 'zh' #   Arabic Letter JEH
+
+
+    '\u0679': 't' #   ARABIC LETTER TTEH
+    '\u0688': 'd' #   Arabic Letter DDAL
+    '\u062f': 'd' #   Arabic Letter DAL
+    '\u0691': 'r' #   Arabic Letter RREH
+    '\u062a': 't' #   Arabic Letter TEH
+    '\u0637': 't' #   Arabic Letter TAH
+    '\u0646': 'n' #   Arabic Letter NOON
+    '\u06ba': 'n' #   Arabic Letter NOON GHUNNA
+
+
+    '\u067e': 'p' #   Arabic Letter PE
+    '\u0641': 'f' #   Arabic Letter FEH
+    '\u0628': 'b' #   Arabic Letter BEH
+    '\u0645': 'm' #   Arabic Letter MEEM
+    '\u0631': 'r' #   Arabic Letter REH
+    '\u0644': 'l' #   Arabic Letter LAM
+    '\u0634': 'sh' #   Arabic Letter SHEEN
+    '\u0633': 's' #   Arabic Letter SHEEN
+    '\u062b': 's' #   Arabic Letter THEH
+    '\u0635': 's' #   Arabic Letter SAD
+    '\u0621': '' #   Arabic Letter HAMZA
+    '\u06be': 's' #   Arabic Letter HEH DOACHASHMEE
+    '\u064b': 'n' #   Arabic Letter FATHATAN
+    '\u064e': 'n' #   Arabic sign FATHATAN
+    '\u0652': '' #   Arabic Letter SUKUN
+    '\u0651': '' #   Arabic Letter SHADDA

data/maps/odni-uzb-Cyrl-Latn-2015.yaml

@@ -36,7 +36,7 @@ description: |
   longer appears in those resources (i.e. is no longer a government official), the IC Standard
   spelling should appear first, with the spelling, if known, as it previously appeared in those
   resources listed within parentheses at the first usage.
-  
+
   The primary goal is to produce a consistent Romanized transcription of names that is specifically
   readable to the English-speaking non-specialist. The system uses the 26 letters of the standard
   (English) Roman alphabet. Some ambiguities in the Romanized form will occur without the use
@@ -164,4 +164,3 @@ map:
     '\u042c': ''     # Ь
     '\u044a': ''     # ъ
     '\u044c': ''     # ь
-

data/maps/royin-tha-Thai-Latn-1939-generic.yaml

@@ -7,11 +7,11 @@ destination_script: Latn
 name: Royal Thai General System of Transcription (1939) Generic
 url: http://www.siamese-heritage.org/jsspdf/1941/JSS_033_1d_RoyalInstituteTranscriptionOfThaiIntoRomanCharacters.pdf
 creation_date: 1939
-adoption_date: 
+adoption_date:
 description: |
   This map loads two external maps to convert Thai text first into phonemic Thai,
   and then into IPA transcription.
-  
+
   The IPA transcription will then be handled by this map, and converted into
   Royal Thai General System of Transcription (1939)
 
@@ -37,7 +37,7 @@ chain: ["var-tha-Thai-Thai-phonemic" ,"var-tha-Thai-Zsym-ipa"]
 
 map:
   title-case: false
-  word_separator: " "  
+  word_separator: " "
 
   rules:
     - pattern: '[˩˨˧˦˥]'
@@ -62,7 +62,7 @@ map:
     'ŋ': 'ng'
     'j': 'y'
     'ɔ': 'o̦'
-    'ɤ': 'œ' 
+    'ɤ': 'œ'
     'ɛ': 'æ'
     'ɯ': 'ư'
     'ʰ': 'h'

data/maps/royin-tha-Thai-Latn-1968.yaml

@@ -7,11 +7,11 @@ destination_script: Latn
 name: Royal Thai General System of Transcription (1968)
 url: http://www.royin.go.th/wp-content/uploads/royin-ebook/276/FileUpload/758_6484.pdf
 creation_date: 1968
-adoption_date: 
+adoption_date:
 description: |
   This map loads two external maps to convert Thai text first into phonemic Thai,
   and then into IPA transcription.
-  
+
   The IPA transcription will then be handled by this map, and converted into
   Royal Thai General System of Transcription (1968).
 
@@ -124,7 +124,7 @@ chain: ["var-tha-Thai-Thai-phonemic" ,"var-tha-Thai-Zsym-ipa"]
 
 map:
   title-case: false
-  word_separator: " "  
+  word_separator: " "
 
   rules:
     - pattern: '[˩˨˧˦˥]'
@@ -151,7 +151,7 @@ map:
     'ŋ': 'ng'
     'j': 'y'
     'ɔ': 'o'
-    'ɤ': 'oe' 
+    'ɤ': 'oe'
     'ɛ': 'ae'
     'ɯ': 'u'
     'ʰ': 'h'

data/maps/royin-tha-Thai-Latn-1999-chained.yaml

@@ -7,11 +7,11 @@ destination_script: Latn
 name: Royal Thai General System of Transcription (1999)
 url: http://www.royin.go.th/wp-content/uploads/royin-ebook/276/FileUpload/758_6484.pdf
 creation_date: 1999
-adoption_date: 
+adoption_date:
 description: |
   This map loads two external maps to convert Thai text first into phonemic Thai,
   and then into IPA transcription.
-  
+
   The IPA transcription will then be handled by this map, and converted into
   Royal Thai General System of Transcription (1999)
 
@@ -125,7 +125,7 @@ chain: ["var-tha-Thai-Thai-phonemic" ,"var-tha-Thai-Zsym-ipa"]
 
 map:
   title-case: false
-  word_separator: " "  
+  word_separator: " "
 
   rules:
     - pattern: '[˩˨˧˦˥]'
@@ -152,7 +152,7 @@ map:
     'ŋ': 'ng'
     'j': 'y'
     'ɔ': 'o'
-    'ɤ': 'oe' 
+    'ɤ': 'oe'
     'ɛ': 'ae'
     'ɯ': 'ue' # New spelling, was u in 1968
     'ʰ': 'h'

data/maps/royin-tha-Thai-Latn-1999.yaml

@@ -7,7 +7,7 @@ destination_script: Latn
 name: Royal Thai General System of Transcription (1999)
 url: http://www.royin.go.th/wp-content/uploads/royin-ebook/276/FileUpload/758_6484.pdf
 creation_date: 1999
-adoption_date: 
+adoption_date:
 description:
 
 notes: |
@@ -48,7 +48,7 @@ tests:
 
 map:
   title-case: false
-  word_separator: " "  
+  word_separator: " "
   transcription: "sequitur.pythainlp_lexicon"
 
   rules:
@@ -68,7 +68,7 @@ map:
     'j': 'i'
     'w': 'o'
     'ɔ': 'o'
-    'ɤ': 'oe' 
+    'ɤ': 'oe'
     'ɛ': 'ae'
     'ɯ': 'ue'
     'ʔ' : ''

data/maps/{ses-ara-arab-latn-1930.yaml → ses-ara-Arab-Latn-1930.yaml}

@@ -85,12 +85,16 @@ tests:
   # https://unstats.un.org/unsd/geoinfo/geonames/
 
   - source: شَرم الشَيْخ
-    expected: sharm el-sheikh
+    expected: Sharm el-Sheikh
 
   - source: الكَفر القَدِيم
-    expected: el-kafr el-qadîm
+    expected: El-Kafr el-Qadîm
 map:
   inherit: "un-ara-Arab-Latn-2017"
+  postrules:
+    - pattern : ' El-'  # الص
+      result: ' el-'
+
   characters:
 
 
@@ -117,7 +121,7 @@ map:
       - 'â'
       - 'a'
 
-    # ta' marboota
+    # ta' marboota  in iso-233-1984 is all the same `a`
     '\u0629$' : 'a'
     '(?<=\b\u0627\u0644[\u0600-\u06ff]{2})\u0629' : 'a'
     '(?<=\b\u0627\u0644[\u0600-\u06ff]{3})\u0629' : 'a'

data/maps/un-ara-Arab-Latn-1971.yaml

@@ -72,16 +72,29 @@ tests:
   # page 31 (38 digital)
 
   - source: خَيبَر
-    expected: k͟haybar
+    expected: K͟haybar
 
   - source: ظَهران
-    expected: z͟hahrān
+    expected: Z͟hahrān
 
   - source: القُدس
-    expected: al quds
+    expected: Al Quds
+
+  - source: شَرم الشَيْخ
+    expected: S͟harm as͟h S͟hayk͟h
 
 map:
   inherit: "un-ara-Arab-Latn-2017"
+  map:
+  postrules:
+    - pattern : ' At͟h T͟h'  # الث
+      result: ' at͟h T͟h'
+    - pattern : ' Ad͟h D͟h'  # الذ
+      result: ' ad͟h D͟h'
+    - pattern : ' As͟h S͟h' # الش
+      result: ' as͟h S͟h'
+    - pattern : ' Az͟h Z͟h' # الظ
+      result: ' az͟h Z͟h'
   characters:
 
     # sun letters
@@ -124,4 +137,3 @@ map:
     '\ufec7' : 'z͟h'  # ﻇ
     '\ufec8' : 'z͟h'  # ﻈ
     '\ufec6' : 'z͟h'  # ﻆ
-