interscript 0.1.1 → 0.1.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: def03b1856a5b9d98e397f378e9958627dfe19077b90293797cc6fc6b81bce3e
-  data.tar.gz: d105d03d4c918bfb5a478a4928a8d15cd59dd0be4a04c990bae0b38e52df7a23
+  metadata.gz: 7557ea1ca381562c61be7dbbeab2ea4adc42ef57ce857ef86acd62a08e5ce588
+  data.tar.gz: 5099d4a7bf07817155d620db716af452607b8565f4e08d37aa805f486556e0e1
 SHA512:
-  metadata.gz: 8c8e1535bc0d688b54da2ebcf29e8c846e07f315c5fe3cc456f918f15b9d46e3ff87ebfb3dc0b40efcc6769f236002b3784fc9e1febfe133b2e81f7a7c5a09a0
-  data.tar.gz: 6c11004204682f6b476d4764ac68cb5bbba8a1b53ac6afc268edd2204d3746e953ffbef51ab4bb8c5661ff959bb51c05447622a1e79fa7e012c40a2e77830d0e
+  metadata.gz: f8a738a34aba269c0a01b4d123f01cc423a3c296541f302f65a6043bd2b618ef2c702f32d469866ee7c488624985b45ed2eee2ad9bbe2955f230b5b45472c364
+  data.tar.gz: a3a66af7fcb9d8c82bcf927b17ea68686298bccec2f6e8dfac25796e7013b36f68fa9f130bd2fb9e7bc214a528896cafdd41a2f464886f09c504f376c19065d8

data/README.adoc

@@ -1,48 +1,263 @@
-= Interscript: Interoperable Script Conversion Systems and a Ruby implementation
+= Interscript: Interoperable Script Conversion Systems, with a Ruby implementation
 
-== Introducation
+image:https://github.com/interscript/interscript/workflows/test/badge.svg["Ruby build status", link="https://github.com/interscript/interscript/actions?workflow=test"]
+image:https://github.com/interscript/interscript/workflows/js/badge.svg["JavaScript build status", link="https://github.com/interscript/interscript/actions?workflow=js"]
 
-This repository contains a number of transliteration schemes from:
+== Introduction
 
+This repository contains interoperable transliteration schemes from:
+
+* ALA-LC
 * BGN/PCGN
 * ICAO
 * ISO
 * UN (by UNGEGN)
+* Many, many other script conversion system authorities.
 
 The goal is to achieve interoperable transliteration schemes allowing quality comparisons.
 
 
-== STATUS (work in progress!)
 
-These transliteration systems currently work:
+== Demonstration
+
+These transliteration systems are used in the demo:
 
 `bgnpcgn-rus-Cyrl-Latn-1947`:: BGN/PCGN Romanization of Russian
-`iso-rus-Cyrl-Latn-iso9`::     ISO 9 Romanization of Russian
+`iso-rus-Cyrl-Latn-9-1995`::     ISO 9 Romanization of Russian
 `icao-rus-Cyrl-Latn-9303`::    ICAO MRZ Romanization of Russian
-`bas-rus-Cyrl-Latn-bss`::      Bulgaria Academy of Science Streamlined System for Russian
+`bas-rus-Cyrl-Latn-2017-bss`::      Bulgaria Academy of Science Streamlined System for Russian
+
+image:demo/20191118-interscript-demo-cast.gif["interscript screencast"]
+
+
+== Installation
+
+=== Prerequisites
+
+Linux:
+
+[source,sh]
+----
+apt-get install swig python3-setuptools
+----
+
+Windows:
 
+[source,sh]
+----
+choco install --no-progress swig
+----
+
+Interscript depends on Python and the https://github.com/sequitur-g2p/sequitur-g2p[`sequitur-g2p`] module
+
+[source,sh]
+----
+pip3 install setuptools numpy
+curl -sSL -o sequitur-g2p.zip https://github.com/sequitur-g2p/sequitur-g2p/archive/806273f.zip
+pip3 install sequitur-g2p.zip
+----
+
+Interscript depends on Ruby. Once you manage to install Ruby, it's easy.
+
+[source,sh]
+----
+gem install interscript
+----
 
 == Usage
 
+Assume you have a file ready in the source script like this:
+
+[source,sh]
+----
+cat <<EOT > rus-Cyrl.txt
+Эх, тройка! птица тройка, кто тебя выдумал? знать, у бойкого народа ты
+могла только родиться, в той земле, что не любит шутить, а
+ровнем-гладнем разметнулась на полсвета, да и ступай считать версты,
+пока не зарябит тебе в очи. И не хитрый, кажись, дорожный снаряд, не
+железным схвачен винтом, а наскоро живьём с одним топором да долотом
+снарядил и собрал тебя ярославский расторопный мужик. Не в немецких
+ботфортах ямщик: борода да рукавицы, и сидит чёрт знает на чём; а
+привстал, да замахнулся, да затянул песню — кони вихрем, спицы в
+колесах смешались в один гладкий круг, только дрогнула дорога, да
+вскрикнул в испуге остановившийся пешеход — и вон она понеслась,
+понеслась, понеслась!
+
+Н.В. Гоголь
+EOT
+----
+
+You can run `interscript` on this text using different transliteration systems.
+
+[source,sh]
+----
+interscript rus-Cyrl.txt \
+  --system=bgnpcgn-rus-Cyrl-Latn-1947 \
+  --output=bgnpcgn-rus-Latn.txt
+
+interscript rus-Cyrl.txt \
+  --system=iso-rus-Cyrl-Latn-9-1995 \
+  --output=iso-rus-Latn.txt
+
+interscript rus-Cyrl.txt \
+  --system=icao-rus-Cyrl-Latn-9303 \
+  --output=icao-rus-Latn.txt
+
+interscript rus-Cyrl.txt \
+  --system=bas-rus-Cyrl-Latn-2017-bss \
+  --output=bas-rus-Latn.txt
+----
+
+It is then easy to see the exact differences in rendering between the systems.
+
+[source,sh]
+----
+diff bgnpcgn-rus-Latn.txt bas-rus-Latn.txt
+----
+
+== Adding transliteration system
+
+Transliteration systems stored in a `maps/` directory as YAML files.
+You can create a new file and add it to the directory.
+
+The file should be named as `<system-code>.yaml`, where `system-code`
+is in accordance with
+http://calconnect.gitlab.io/tc-localization/csd-transcription-systems[ISO/CC 24229].
+
+=== File structure
+
+[source,yaml]
+----
+authority_id: bgnpcgn
+id: 1947
+language: rus
+source_script: Cyrl
+destination_script: Latn
+name: ROMANIZATION OF RUSSIAN, BGN/PCGN 1947 System
+url: https://assets.publishing.service.gov.uk/government/uploads/system/uploads/attachment_data/file/807920/ROMANIZATION_OF_RUSSIAN.pdf
+creation_date: 1947
+confirmation_date: 2019-06
+description: The BGN/PCGN system for Russian was adopted ...
+
+notes:
+  - The character e should be romanized ye initially, after the vowel ...
+
+tests:
+  - source: ДЛИННОЕ ПОКРЫВАЛО
+    expected: DLINNOYE POKRYVALO
+  - source: Еловая шишка
+    expected: Yelovaya shishka
+
+map:
+  rules:
+    - pattern: (?<=[АаЕеЁёИиОоУуЫыЭэЮюЯяЙйЪъЬь])\u0415 # Е after a, e, ё, и, о, у, ы, э, ю, я, й, ъ, ь
+      result: Ye
+    - pattern: \b\u0415 # Е initially
+      result: Ye
+
+  characters:
+    "\u0410": "A"
+    "\u0411": "B"
+    "\u0412": "V"
+----
+
+
+=== Rules
+
+The subsection `rules` is placed under the `map` key. All rules are applied in order they are placed before the subsection `characters` applying. Rules apply to an original text, not to a result of previous rules applying.
+
+Each rule has `pattern` and `result` elements.
+
+Pattern is a regex expression. It should be representing as a string without `//` or `%r{}` parentheses. For example `\b\u0415`. In case a rule is depend on previous or next content, lookahead or lookbehind could be used. For example a rule with the pattern `(?<=[АаЕеЁёИиОоУуЫыЭэЮюЯяЙйЪъЬь])\u0415` find every Е after upper or lower case symbols a, e, ё, и, о, у, ы, э, ю, я, й, ъ, ь.
+
+Result is a replacement a for pattern's match. It can contain a string, an Unicode characters specified by a hexadecimal number, a captured group reference. String with hexadecimal number or captured group reference should be double quoted. For example `"Y\u00eb"` or `"\\1\u00b7\\2"`. Captured group are referred by double backslash and group's number.
+
+Because rules are applied in order, multiple rules applicable to the same segment of a string can be addressed by rule ordering, and rules can be used as priority over characters. For example:
+
+[source,yaml]
+----
+map:
+  rules:
+    - pattern: \u03B3\u03B3    # γ (before Γ, Ξ, Χ)
+      result: ng
+    - pattern: (?<![Γγ])\u03B3(?=[ΕεέΗηήΙιίΥυύ])    # γ (before front vowels)
+      result: y
+----
+
+(γι maps to `yi`; but γγ maps to `ng`. In the case of γγι, the first rule takes priority, and the transliteration is `ngi`: it makes the second rule impossible.)
+
+[source,yaml]
+----
+map:
+  rules:
+    - pattern: (?<=\b)\u03BC[πΠ]  # μπ (initially)
+      result: b
+    - pattern: \u03BC[πΠ]         # μπ (medially)
+      result: mb
+----
+
+(The first rule applies at the start of a word; the second rule does not specify a context, as it applies in all other cases not covered by the first rule.)
+
+[source,yaml]
+----
+map:
+  rules:
+    - pattern: ";"
+      result: "?"
+
+  characters
+    "\u00B7": ";
+----
+
+(This guarantees that any `;` are converted to `?` before any new `;` are introduced; because all three are Latin script, they could be mixed up in ordering.)
+
+Normally rules "`bleed`" each other: once a rule applies to a segment, that segment cannot trigger other rules, because it is already converted to Roman. Exceptionally, it will be necessary to have a rule add or remove characters in the original script, rather than transliterate them, so that the same context can be invoked by two rules in succession:
+
+[source,yaml]
+----
+map:
+  rules:
+    - pattern: (?<=[АаЕеЁёИиОоУуЫыЭэЮюЯя])\u042b # Ы after any vowel character
+      result: "\u00b7Ы"
+    - pattern: \u042b(?=[АаУуЫыЭэ])              # Ы before а, у, ы, or э
+      result: "Ы\u00b7"
+----
+
+(If the result were `\u00B7Y`, the second rule could not be applied afterwards; but we want ОЫУ to transliterate as `O·Y·U`. In order to make that happen, we preserve the Ы during the rules phase, resulting in О·Ы·У; we only convert the letters to Roman script in the `characters` phase.)
+
+=== Testing transliteration systems
+
+To test all transliteration systems in the `maps/` directory, run:
 
 [source,sh]
 ----
-# Transliterating Russian Cyrillic to Latin using the Streamlined System for Russian
-interscript samples/rus-Cyrl.txt --system=bas-rus-Cyrl-Latn-bss --output=rus-Latn.txt
+bundle exec rspec
+----
+
+The command takes `source` texts from the `test` section, transforms
+them using `rules` and `charmaps` from the `map` key, and compares the
+results with `expected:` text from the `source:` section.
+
+To test a specific transliteration system, set the environment variable
+`TRANSLIT_SYSTEM` to the system code of the desired system
+(i.e. the "`basename`" of the system's YAML file):
 
-# Transliterating Russian Cyrillic to Latin using the BGN/PCGN Romanization of Russian
-interscript samples/rus-Cyrl.txt --system=bgnpcgn-rus-Cyrl-Latn-1947 --output=rus-Latn.txt
+[source,sh]
+----
+TRANSLIT_SYSTEM=bgnpcgn-rus-Cyrl-Latn-1947 bundle exec rspec
 ----
 
 
 == ISCS system codes
 
-The system code identifying a script conversion system has a few components:
+In accordance with
+http://calconnect.gitlab.io/tc-localization/csd-transcription-systems[ISO/CC 24229],
+the system code identifying a script conversion system has the following components:
 
-e.g. `bgnpcgn-rus-Cyrl-Latn-1947`
+e.g. `bgnpcgn-rus-Cyrl-Latn-1947`:
 
 `bgnpcgn`:: the authority identifier
-`rus`:: an ISO 639-2 3-letter language code that this system applies to
+`rus`:: an ISO 639-{1,2,3,5} language code that this system applies to (For 639-2, use (T) code)
 `Cyrl`:: an ISO 15924 script code, identifying the source script
 `Latn`:: an ISO 15924 script code, identifying the target script
 `1947`:: an identifier unit within the authority to identify this system
@@ -53,13 +268,31 @@ e.g. `bgnpcgn-rus-Cyrl-Latn-1947`
 Currently the schemes cover Cyrillic, Armenian, Greek, Arabic and Hebrew.
 
 
-== Sources
+== Samples to play with
 
 * `rus-Cyrl-1.txt`: Copied from the XLS output from http://www.primorsk.vybory.izbirkom.ru/region/primorsk?action=show&global=true&root=254017025&tvd=4254017212287&vrn=100100067795849&prver=0&pronetvd=0&region=25&sub_region=25&type=242&vibid=4254017212287
 
 * `rus-Cyrl-2.txt`: Copied from the XLS output from http://www.yaroslavl.vybory.izbirkom.ru/region/yaroslavl?action=show&root=764013001&tvd=4764013188704&vrn=4764013188693&prver=0&pronetvd=0&region=76&sub_region=76&type=426&vibid=4764013188704
 
 
-== Credits
+== References
+
+Reference documents are located at the
+https://github.com/interscript/interscript-references[interscript-references repository].
+Some specifications that have distribution limitations may not be reproduced there.
+
+
+== Links to system definitions
+
+* https://www.iso.org/committee/48750.html[ISO/TC 46 (see standards published by WG 3)]
+* http://geonames.nga.mil/gns/html/romanization.html[BGN/PCGN and BGN Romanization systems (BGN)]
+* https://www.gov.uk/government/publications/romanization-systems[BGN/PCGN Romanization systems (PCGN)]
+* https://www.loc.gov/catdir/cpso/roman.html[ALA-LC Romanization systems in current use]
+* http://catdir.loc.gov/catdir/cpso/roman.html[ALA-LC Romanization systems from 1997]
+* http://www.eki.ee/wgrs/[UN Romanization systems]
+* http://www.eki.ee/knab/kblatyl2.htm[EKI KNAB systems]
+
+== Copyright and license
+
+This is a Ribose project. Copyright Ribose.
 
-This is a Ribose project.

data/bin/interscript

@@ -1,20 +1,41 @@
 #!/usr/bin/env ruby
+
 require 'rubygems'
-require_relative '../lib/interscript'
-
-if ARGV.empty?
-  puts "write source file, source format, and output file"
-else
-  args = Hash[ARGV.flat_map { |s| s.scan(/--?([^=\s]+)(?:=(\S+))?/) }]
-  input = ARGV[0]
-  system_code = args["system"]
-  output_file = args["output"]
-
-  raise "Please enter the system code with --system={system_code}" unless system_code
-
-  if output_file
-    Interscript.transliterate_file(system_code, input, output_file)
-  else
-    puts Interscript.transliterate(system_code, IO.read(input))
-  end
+
+# resolve bin path, ignoring symlinks
+require 'pathname'
+bin_file = Pathname.new(__FILE__).realpath
+
+# add self to libpath
+$LOAD_PATH.unshift File.expand_path("../../lib", bin_file)
+
+# Fixes https://github.com/rubygems/rubygems/issues/1420
+require 'rubygems/specification'
+
+class Gem::Specification
+  def this; self; end
 end
+
+require 'interscript/command'
+
+if ARGV.any? && !Interscript::Command.all_tasks.key?(ARGV.first)
+  ARGV.unshift :translit
+end
+Interscript::Command.start ARGV
+
+# if ARGV.empty?
+#   puts "write source file, source format, and output file"
+# else
+#   args = Hash[ARGV.flat_map { |s| s.scan(/--?([^=\s]+)(?:=(\S+))?/) }]
+#   input = ARGV[0]
+#   system_code = args["system"]
+#   output_file = args["output"]
+
+#   raise "Please enter the system code with --system={system_code}" unless system_code
+
+#   if output_file
+#     Interscript.transliterate_file(system_code, input, output_file)
+#   else
+#     puts Interscript.transliterate(system_code, IO.read(input))
+#   end
+# end

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/lib/g2pwrapper.py

@@ -0,0 +1,34 @@
+import g2p, SequiturTool
+import numpy
+
+def transliterate(model, word):
+
+  class Struct:
+      def __init__(self, **entries):
+          self.__dict__.update(entries)
+
+  model_path = {
+    'pythainlp_lexicon': './lib/model-7', 
+    'wiktionary_phonemic': './lib/tha-pt-b-7'
+  }
+
+  connector_dict = {
+    'pythainlp_lexicon': '', 
+    'wiktionary_phonemic': '-'
+  }
+
+
+  modelFile = model_path[model]
+  connector = connector_dict[model]
+
+  options = Struct(**{'profile': None, 'resource_usage': None, 'psyco': None, 'tempdir': None, 'trainSample': None, 'develSample': None, 'testSample': None, 'checkpoint': None, 'resume_from_checkpoint': None, 'shouldTranspose': None, 'modelFile': modelFile , 'newModelFile': None, 'shouldTestContinuously': None, 'shouldSelfTest': None, 'lengthConstraints': None, 'shouldSuppressNewMultigrams': None, 'viterbi': None, 'shouldRampUp': None, 'shouldWipeModel': None, 'shouldInitializeWithCounts': None, 'minIterations': 20, 'maxIterations': 100, 'eager_discount_adjustment': None, 'fixed_discount': None, 'encoding': 'UTF-8', 'phoneme_to_phoneme': None, 'test_segmental': None, 'testResult': None, 'applySample': None, 'applyWord': word, 'variants_mass': None, 'variants_number': None, 'fakeTranslator': None, 'stack_limit': None})
+
+  loadSample = g2p.loadG2PSample
+
+  model = SequiturTool.procureModel(options, loadSample)
+  if not model:
+      return 1
+  translator = g2p.Translator(model)
+  del model
+
+  return connector.join(translator(tuple(word)))

data/lib/interscript-opal.rb

@@ -0,0 +1,2 @@
+require "opal"
+require "interscript"

data/lib/interscript.rb

@@ -1,39 +1,157 @@
 # frozen_string_literal: true
 
-require 'yaml'
+require "interscript/opal/maps" if RUBY_ENGINE == "opal"
+require "interscript/mapping"
 
 # Transliteration
 module Interscript
-  SYSTEM_DEFINITIONS_PATH = File.expand_path('../maps', __dir__)
+
+  class InvalidSystemError < StandardError; end
+  class ExternalProcessNotRecognizedError < StandardError; end
+  class ExternalProcessUnavailableError < StandardError; end
+
+  if RUBY_ENGINE == 'opal'
+    require "interscript/opal"
+    extend Opal
+  else
+    require "interscript/fs"
+    extend Fs
+  end
 
   class << self
-    def transliterate_file(system_code, input_file, output_file)
-      input = File.read(input_file)
-      output = transliterate(system_code, input)
 
-      File.open(output_file, "w") do |f|
-        f.puts(output)
+    def transliterate(system_code, string, maps={})
+      unless maps.has_key? system_code
+        maps[system_code] = Interscript::Mapping.for(system_code)
+      end
+      # mapping = Interscript::Mapping.for(system_code)
+      mapping = maps[system_code]
+
+      # First, apply chained transliteration as specified in the list `chain`
+      chain = mapping.chain.dup
+      while chain.length > 0
+        string = transliterate(chain.shift, string, maps)
+      end
+
+      # Then, apply the rest of the map
+      separator = mapping.character_separator || ""
+      word_separator = mapping.word_separator || ""
+      title_case = mapping.title_case
+      downcase = mapping.downcase
+
+      # charmap = mapping.characters&.sort_by { |k, _v| k.size }&.reverse&.to_h
+      # dictmap = mapping.dictionary&.sort_by { |k, _v| k.size }&.reverse&.to_h
+      charmap = mapping.characters_hash
+      dictmap = mapping.dictionary_hash
+      trie = mapping.dictionary_trie
+
+      string = external_processing(mapping, string)
+
+      pos = 0
+      while pos < string.to_s.size
+        m = 0
+        wordmatch = ""
+
+        # Using Trie, find the longest matching substring
+        while (pos + m < string.to_s.size) && (trie.partial_word?string[pos..pos+m])
+          wordmatch = string[pos..pos+m] if trie.word?string[pos..pos+m]
+          m += 1
+        end
+
+        m = wordmatch.length
+        if m > 0
+          repl = dictmap[string[pos..pos+m-1]]
+          string = sub_replace(string, pos, m, repl)
+          pos += repl.length
+        else
+          pos += 1
+        end
+      end
+
+      output = string.clone
+      offsets = Array.new string.to_s.size, 1
+
+      # mapping.rules.each do |r|
+      #   string.to_s.scan(/#{r['pattern']}/) do |matches|
+      #     match = Regexp.last_match
+      #     pos = match.offset(0).first
+      #     result = r['result'].clone
+      #     matches.each.with_index { |v, i| result.sub!(/\\#{i + 1}/, v) } if matches.is_a? Array
+      #     result.upcase! if up_case_around?(string, pos)
+      #     output[offsets[0...pos].sum, match[0].size] = result
+      #     offsets[pos] += result.size - match[0].size
+      #   end
+      # end
+
+      mapping.rules.each do |r|
+        next unless output
+        re = mkregexp(r["pattern"])
+        output = output.gsub(re, r["result"])
+      end
+
+      charmap.each do |k, v|
+        while (match = output&.match(/#{k}/))
+          pos = match.offset(0).first
+          result = !downcase && up_case_around?(output, pos) ? v.upcase : v
+
+          # if more than one, choose the first one
+          result = result[0] if result.is_a?(Array)
+
+          output = sub_replace(
+            output,
+            pos,
+            match[0].size,
+            add_separator(separator, pos, result)
+          )
+        end
+      end
+
+      mapping.postrules.each do |r|
+        next unless output
+        re = mkregexp(r["pattern"])
+        output = if r["result"] == "upcase"
+                   output.gsub(re, &:upcase)
+                 else
+                   output.gsub(re, r["result"])
+                 end
       end
-      puts "Output written to: #{output_file}"
+
+      return unless output
+
+      output = output.sub(/^(.)/, &:upcase) if title_case
+      if word_separator != ''
+        output = output.gsub(/#{word_separator}#{separator}/u, word_separator)
+
+        if title_case
+          output = output.gsub(/#{word_separator}(.)/u, &:upcase)
+        end
+      end
+
+      output.unicode_normalize
     end
 
-    def load_system_definition(system_code)
-      YAML.load_file(File.join(SYSTEM_DEFINITIONS_PATH, "#{system_code}.yaml"))
+    private
+
+    def add_separator(separator, pos, result)
+      pos == 0 ? result : separator + result
     end
 
-    def transliterate(system_code, string)
-      system = load_system_definition(system_code)
+    def up_case_around?(string, pos)
+      return false if string[pos] == string[pos].downcase
 
-      rules = system["map"]["rules"] || []
-      charmap = system["map"]["characters"] || {}
+      i = pos - 1
+      i -= 1 while i.positive? && string[i] !~ Regexp.new(ALPHA_REGEXP)
+      before = i >= 0 && i < pos ? string[i].to_s.strip : ''
 
-      rules.each do |r|
-        string.gsub! %r{#{r["pattern"]}}, r["result"]
-      end
+      i = pos + 1
+      i += 1 while i < string.size - 1 && string[i] !~ Regexp.new(ALPHA_REGEXP)
+      after = i > pos ? string[i].to_s.strip : ''
 
-      string.split('').map do |char|
-        charmap[char] || char
-      end.join('')
+      before_uc = !before.empty? && before == before.upcase
+      after_uc = !after.empty? && after == after.upcase
+      # before_uc && (after.empty? || after_uc) || after_uc && (before.empty? || before_uc)
+      before_uc || after_uc
     end
+
   end
 end