interscript 0.1.4 → 0.1.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6228d56d873603190ed290757f3e8056258cb88761b93405e15f06d25a08cba5
-  data.tar.gz: 0ceb25cb5862530d2c4798f921c4eaa5e9d999844266e8eab1d6bcd3d4b96842
+  metadata.gz: 3eb5c858ad7332995b5f8ed1c7766305f6be5fb5524e91de7df1d0d3d770a3ee
+  data.tar.gz: 80be952635dd65cf1e23107f70c44557ec9711590759e610a042f2c746f67d4e
 SHA512:
-  metadata.gz: 301342d3cc50849fcb7f14c2b8b42404929cef1ed47192bc9cda236333b7b147c55d7b98648becc9c84d1b8790e6dbeffa30ecb27f472adf803463416e9babdc
-  data.tar.gz: cf95797a136ff95dfd3edc66378c0b6320f6a87325206d56c5e9acda381733b4168d8f651cb15a30c8a5b40add43bd3b2eeafde24bc19a959de0eb6983ad8359
+  metadata.gz: bdb63b2ea8e6337a31879e7c1331f6e9a95dd205580bc8656cdb806e624b7cf946c17d300125470494c1643332b3f5f1a3d713a040bd6ee907dcba5d189db95e
+  data.tar.gz: 1aa1b254b253f148b4a5cc316027d3e67ca1ba68e866ebedbc025dc9c5f5c78680b12c24bc52177d65b2c9a6dd262ee97e34f52b4d32f83dfc9c232dc2c2d0de

data/README.adoc

@@ -1,6 +1,7 @@
 = Interscript: Interoperable Script Conversion Systems, with a Ruby implementation
 
-image:https://github.com/interscript/interscript/workflows/test/badge.svg["Build Status", link="https://github.com/interscript/interscript/actions?workflow=test"]
+image:https://github.com/interscript/interscript/workflows/test/badge.svg["Ruby build status", link="https://github.com/interscript/interscript/actions?workflow=test"]
+image:https://github.com/interscript/interscript/workflows/js/badge.svg["JavaScript build status", link="https://github.com/interscript/interscript/actions?workflow=js"]
 
 == Introduction
 
@@ -22,9 +23,9 @@ The goal is to achieve interoperable transliteration schemes allowing quality co
 These transliteration systems are used in the demo:
 
 `bgnpcgn-rus-Cyrl-Latn-1947`:: BGN/PCGN Romanization of Russian
-`iso-rus-Cyrl-Latn-iso9`::     ISO 9 Romanization of Russian
+`iso-rus-Cyrl-Latn-9-1995`::     ISO 9 Romanization of Russian
 `icao-rus-Cyrl-Latn-9303`::    ICAO MRZ Romanization of Russian
-`bas-rus-Cyrl-Latn-bss`::      Bulgaria Academy of Science Streamlined System for Russian
+`bas-rus-Cyrl-Latn-2017-bss`::      Bulgaria Academy of Science Streamlined System for Russian
 
 image:demo/20191118-interscript-demo-cast.gif["interscript screencast"]
 
@@ -95,7 +96,7 @@ interscript rus-Cyrl.txt \
   --output=bgnpcgn-rus-Latn.txt
 
 interscript rus-Cyrl.txt \
-  --system=iso-rus-Cyrl-Latn-iso9 \
+  --system=iso-rus-Cyrl-Latn-9-1995 \
   --output=iso-rus-Latn.txt
 
 interscript rus-Cyrl.txt \
@@ -103,7 +104,7 @@ interscript rus-Cyrl.txt \
   --output=icao-rus-Latn.txt
 
 interscript rus-Cyrl.txt \
-  --system=bas-rus-Cyrl-Latn-bss \
+  --system=bas-rus-Cyrl-Latn-2017-bss \
   --output=bas-rus-Latn.txt
 ----
 
@@ -149,7 +150,7 @@ tests:
 
 map:
   rules:
-    - pattern: (?<=[АаЕеЁёИиОоУуЫыЭэЮюЯяЙйЪъЬь])\u0415 # Е after a, e, ё, и, о, у, ы, э, ю, я, й, ъ, ь
+    - pattern: (?<=[АаЕеЁёИиОоУуЫыЭэЮюЯяЙйЪъЬь])\u0415 # Е after a, e, ё, и, о, у, ы, э, ю, я, й, ъ, ь
       result: Ye
     - pattern: \b\u0415 # Е initially
       result: Ye
@@ -167,7 +168,7 @@ The subsection `rules` is placed under the `map` key. All rules are applied in o
 
 Each rule has `pattern` and `result` elements.
 
-Pattern is a regex expression. It should be representing as a string without `//` or `%r{}` parentheses. For example `\b\u0415`. In case a rule is depend on previous or next content, lookahead or lookbehind could be used. For example a rule with the pattern `(?<=[АаЕеЁёИиОоУуЫыЭэЮюЯяЙйЪъЬь])\u0415` find every Е after upper or lower case symbols a, e, ё, и, о, у, ы, э, ю, я, й, ъ, ь.
+Pattern is a regex expression. It should be representing as a string without `//` or `%r{}` parentheses. For example `\b\u0415`. In case a rule is depend on previous or next content, lookahead or lookbehind could be used. For example a rule with the pattern `(?<=[АаЕеЁёИиОоУуЫыЭэЮюЯяЙйЪъЬь])\u0415` find every Е after upper or lower case symbols a, e, ё, и, о, у, ы, э, ю, я, й, ъ, ь.
 
 Result is a replacement a for pattern's match. It can contain a string, an Unicode characters specified by a hexadecimal number, a captured group reference. String with hexadecimal number or captured group reference should be double quoted. For example `"Y\u00eb"` or `"\\1\u00b7\\2"`. Captured group are referred by double backslash and group's number.
 
@@ -256,7 +257,7 @@ the system code identifying a script conversion system has the following compone
 e.g. `bgnpcgn-rus-Cyrl-Latn-1947`:
 
 `bgnpcgn`:: the authority identifier
-`rus`:: an ISO 639-2 3-letter language code that this system applies to
+`rus`:: an ISO 639-{1,2,3,5} language code that this system applies to (For 639-2, use (T) code)
 `Cyrl`:: an ISO 15924 script code, identifying the source script
 `Latn`:: an ISO 15924 script code, identifying the target script
 `1947`:: an identifier unit within the authority to identify this system

data/lib/interscript-opal.rb

@@ -0,0 +1,2 @@
+require "opal"
+require "interscript"

data/lib/interscript.rb

@@ -1,56 +1,32 @@
 # frozen_string_literal: true
 
-require "yaml"
+require "interscript/opal/maps" if RUBY_ENGINE == "opal"
 require "interscript/mapping"
 
 # Transliteration
 module Interscript
 
-  class << self
-    def root_path
-      @root_path ||= Pathname.new(File.dirname(__dir__))
-    end
-
-    def transliterate_file(system_code, input_file, output_file, maps)
-      input = File.read(input_file)
-      output = transliterate(system_code, input, maps)
-
-      File.open(output_file, 'w') do |f|
-        f.puts(output)
-      end
-      puts "Output written to: #{output_file}"
-    end
+  class InvalidSystemError < StandardError; end
+  class ExternalProcessNotRecognizedError < StandardError; end
+  class ExternalProcessUnavailableError < StandardError; end
 
-    def import_python_modules
-      begin
-        pyimport :g2pwrapper
-      rescue
-        pyimport :sys
-        sys.path.append(root_path.to_s+"/lib/")
-        pyimport :g2pwrapper
-      end
-    end
+  if RUBY_ENGINE == 'opal'
+    require "interscript/opal"
+    extend Opal
+  else
+    require "interscript/fs"
+    extend Fs
+  end
 
-    def external_process(process_name, string)
-      import_python_modules
-      case process_name
-        when 'sequitur.pythainlp_lexicon'
-          return g2pwrapper.transliterate('pythainlp_lexicon', string)
-        when 'sequitur.wiktionary_phonemic'
-          return g2pwrapper.transliterate('wiktionary_phonemic', string)
-      else
-        puts "Invalid Process"
-      end
-    end
+  class << self
 
     def transliterate(system_code, string, maps={})
-      if (!maps.has_key?system_code)
+      unless maps.has_key? system_code
         maps[system_code] = Interscript::Mapping.for(system_code)
       end
       # mapping = Interscript::Mapping.for(system_code)
       mapping = maps[system_code]
 
-
       # First, apply chained transliteration as specified in the list `chain`
       chain = mapping.chain.dup
       while chain.length > 0
@@ -69,11 +45,7 @@ module Interscript
       dictmap = mapping.dictionary_hash
       trie = mapping.dictionary_trie
 
-      # Segmentation
-      string = external_process(mapping.segmentation, string) if mapping.segmentation
-
-      # Transliteration/Transcription
-      string = external_process(mapping.transcription, string) if mapping.transcription
+      string = external_processing(mapping, string)
 
       pos = 0
       while pos < string.to_s.size
@@ -81,14 +53,15 @@ module Interscript
         wordmatch = ""
 
         # Using Trie, find the longest matching substring
-        while (pos + m < string.to_s.size) && (trie.partial_word?string[pos..pos+m]) 
+        while (pos + m < string.to_s.size) && (trie.partial_word?string[pos..pos+m])
           wordmatch = string[pos..pos+m] if trie.word?string[pos..pos+m]
           m += 1
         end
+
         m = wordmatch.length
         if m > 0
           repl = dictmap[string[pos..pos+m-1]]
-          string[pos..pos+m-1] = repl
+          string = sub_replace(string, pos, m, repl)
           pos += repl.length
         else
           pos += 1
@@ -109,32 +82,48 @@ module Interscript
       #     offsets[pos] += result.size - match[0].size
       #   end
       # end
+
       mapping.rules.each do |r|
-        output.gsub!(/#{r['pattern']}/, r['result'])
+        next unless output
+        re = mkregexp(r["pattern"])
+        output = output.gsub(re, r["result"])
       end
 
       charmap.each do |k, v|
         while (match = output&.match(/#{k}/))
           pos = match.offset(0).first
           result = !downcase && up_case_around?(output, pos) ? v.upcase : v
-          result = result[0] if result.is_a?(Array) # if more than one, choose the first one
-          output[pos, match[0].size] = add_separator(separator, pos, result)
+
+          # if more than one, choose the first one
+          result = result[0] if result.is_a?(Array)
+
+          output = sub_replace(
+            output,
+            pos,
+            match[0].size,
+            add_separator(separator, pos, result)
+          )
         end
       end
 
       mapping.postrules.each do |r|
-        output.gsub!(/#{r['pattern']}/, r['result'])
+        next unless output
+        re = mkregexp(r["pattern"])
+        output = output.gsub(re, r["result"])
       end
 
-      if output
-        output.sub!(/^(.)/,  &:upcase) if title_case
-        if word_separator != ''
-          output.gsub!(/#{word_separator}#{separator}/,word_separator)
-          output.gsub!(/#{word_separator}(.)/, &:upcase) if title_case
+      return unless output
+
+      output = output.sub(/^(.)/, &:upcase) if title_case
+      if word_separator != ''
+        output = output.gsub(/#{word_separator}#{separator}/u, word_separator)
+
+        if title_case
+          output = output.gsub(/#{word_separator}(.)/u, &:upcase)
         end
       end
 
-      output ? output.unicode_normalize : output
+      output.unicode_normalize
     end
 
     private
@@ -147,11 +136,11 @@ module Interscript
       return false if string[pos] == string[pos].downcase
 
       i = pos - 1
-      i -= 1 while i.positive? && string[i] !~ /[[:alpha:]]/
+      i -= 1 while i.positive? && string[i] !~ Regexp.new(ALPHA_REGEXP)
       before = i >= 0 && i < pos ? string[i].to_s.strip : ''
 
       i = pos + 1
-      i += 1 while i < string.size - 1 && string[i] !~ /[[:alpha:]]/
+      i += 1 while i < string.size - 1 && string[i] !~ Regexp.new(ALPHA_REGEXP)
       after = i > pos ? string[i].to_s.strip : ''
 
       before_uc = !before.empty? && before == before.upcase
@@ -159,5 +148,6 @@ module Interscript
       # before_uc && (after.empty? || after_uc) || after_uc && (before.empty? || before_uc)
       before_uc || after_uc
     end
+
   end
 end

data/lib/interscript/command.rb

@@ -1,16 +1,17 @@
 require 'thor'
 require 'interscript'
-
+require 'json'
 module Interscript
   # Command line interface
   class Command < Thor
     desc '<file>', 'Transliterate text'
     option :system, aliases: '-s', required: true, desc: 'Transliteration system'
     option :output, aliases: '-o', required: false, desc: 'Output file'
+    option :map, aliases: '-m', required: false, default: "{}", desc: 'Transliteration mapping json'
 
     def translit(input)
       if options[:output]
-        Interscript.transliterate_file(options[:system], input, options[:output])
+        Interscript.transliterate_file(options[:system], input, options[:output], JSON.parse(options[:map]))
       else
         puts Interscript.transliterate(options[:system], IO.read(input))
       end

data/lib/interscript/fs.rb

@@ -0,0 +1,69 @@
+module Interscript
+  module Fs
+    ALPHA_REGEXP = '[[:alpha:]]'
+
+    def sub_replace(string, pos, size, repl)
+      string[pos..pos + size - 1] = repl
+      string
+    end
+
+    def root_path
+      @root_path ||= Pathname.new(File.join(File.dirname(__dir__), ".."))
+    end
+
+    def transliterate_file(system_code, input_file, output_file, maps={})
+      input = File.read(input_file)
+      output = transliterate(system_code, input, maps)
+
+      File.open(output_file, 'w') do |f|
+        f.puts(output)
+      end
+
+      puts "Output written to: #{output_file}"
+      output_file
+    end
+
+    def import_python_modules
+      begin
+        pyimport :g2pwrapper
+      rescue
+        pyimport :sys
+        sys.path.append(root_path.to_s + "/lib/")
+        pyimport :g2pwrapper
+      end
+    end
+
+    def external_process(process_name, string)
+      import_python_modules
+
+      case process_name
+      when 'sequitur.pythainlp_lexicon'
+        return g2pwrapper.transliterate('pythainlp_lexicon', string)
+      when 'sequitur.wiktionary_phonemic'
+        return g2pwrapper.transliterate('wiktionary_phonemic', string)
+      else
+        raise ExternalProcessNotRecognizedError.new
+      end
+
+    rescue
+      raise ExternalProcessUnavailableError.new
+    end
+
+    def external_processing(mapping, string)
+      # Segmentation
+      string = external_process(mapping.segmentation, string) if mapping.segmentation
+
+      # Transliteration/Transcription
+      string = external_process(mapping.transcription, string) if mapping.transcription
+
+      string
+    end
+
+    private
+
+    def mkregexp(regexpstring)
+      /#{regexpstring}/u
+    end
+
+  end
+end

data/lib/interscript/mapping.rb

@@ -1,7 +1,8 @@
 require 'rambling-trie'
+require 'yaml'
+require 'json'
 
 module Interscript
-  class InvalidSystemError < StandardError; end
 
   class Mapping
     attr_reader(
@@ -35,7 +36,10 @@ module Interscript
     def initialize(system_code, options = {})
       @system_code = system_code
       @depth = options.fetch(:depth, 0).to_i
-      @system_path = options.fetch(:system_code, default_path)
+
+      unless RUBY_ENGINE == 'opal'
+        @system_path = options.fetch(:system_code, default_path)
+      end
 
       load_and_serialize_system_mappings
     end
@@ -45,10 +49,10 @@ module Interscript
     end
 
     def load_and_serialize_system_mappings
-      if depth < 5
-        mappings = load_system_mappings
-        serialize_system_mappings(mappings)
-      end
+      return if depth >= 5
+
+      mappings = load_system_mappings
+      serialize_system_mappings(mappings)
     end
 
     private
@@ -64,6 +68,18 @@ module Interscript
     end
 
     def load_system_mappings
+      if RUBY_ENGINE == 'opal'
+        load_opal_mappings
+      else
+        load_fs_mappings
+      end
+    end
+
+    def load_opal_mappings
+      JSON.parse(`InterscriptMaps[#{system_code}]`)
+    end
+
+    def load_fs_mappings
       YAML.load_file(system_path.join(system_code_file))
     rescue Errno::ENOENT
       raise Interscript::InvalidSystemError.new("No system mappings found")
@@ -100,26 +116,27 @@ module Interscript
 
     def include_inherited_mappings(mappings)
       inherit_systems = [].push(mappings["map"]["inherit"]).flatten
-      for inherit_system in inherit_systems do
-        if (inherit_system)
-          inherited_mapping = Mapping.for(inherit_system, depth: depth + 1)
-
-          @rules = [inherited_mapping.rules, rules].flatten
-          @postrules = [inherited_mapping.postrules, postrules].flatten
-          @characters = (inherited_mapping.characters|| {}).merge(characters)
-          @dictionary = (inherited_mapping.dictionary|| {}).merge(dictionary)
-        end
+
+      inherit_systems.each do |inherit_system|
+        next unless inherit_system
+
+        inherited_mapping = Mapping.for(inherit_system, depth: depth + 1)
+
+        @rules = [inherited_mapping.rules, rules].flatten
+        @postrules = [inherited_mapping.postrules, postrules].flatten
+        @characters = (inherited_mapping.characters|| {}).merge(characters)
+        @dictionary = (inherited_mapping.dictionary|| {}).merge(dictionary)
       end
     end
 
-    def build_hashes()
+    def build_hashes
       @characters_hash = characters&.sort_by { |k, _v| k.size }&.reverse&.to_h
       @dictionary_hash = dictionary&.sort_by { |k, _v| k.size }&.reverse&.to_h
     end
 
-    def build_trie()
+    def build_trie
       @dictionary_trie = Rambling::Trie.create
       dictionary_trie.concat dictionary.keys
     end
   end
-end
+end

data/lib/interscript/opal.rb

@@ -0,0 +1,23 @@
+module Interscript
+  module Opal
+    ALPHA_REGEXP = '\p{L}'
+
+    def mkregexp(regexpstring)
+      flags = 'u'
+      if regexpstring.include? "(?i)"
+        regexpstring = regexpstring.gsub("(?i)", "").gsub("(?-i)", "")
+        flags = 'ui'
+      end
+      Regexp.new("/#{regexpstring}/#{flags}")
+    end
+
+    def sub_replace(string, pos, size, repl)
+      string[0, pos] + repl + string[pos + size..-1]
+    end
+
+    def external_processing(mapping, string)
+      string
+    end
+
+  end
+end