interscript 0.1.0 → 0.1.5

data/lib/interscript-opal.rb

@@ -0,0 +1,2 @@
+require "opal"
+require "interscript"

data/lib/interscript.rb

@@ -1,53 +1,153 @@
-require 'yaml'
-require 'singleton'
+# frozen_string_literal: true
 
-class Interscript
-  include Singleton
-  
-  SYSTEM_DEFINITIONS_PATH = File.expand_path('../../maps', __FILE__)
+require "interscript/opal/maps" if RUBY_ENGINE == "opal"
+require "interscript/mapping"
 
-  def initialize
-    @systems = {}
-  end
+# Transliteration
+module Interscript
 
-  def transliterate_file(system_code, input_file, output_file)
-    input = File.read(input_file)
-    output = transliterate(system_code, input)
+  class InvalidSystemError < StandardError; end
+  class ExternalProcessNotRecognizedError < StandardError; end
+  class ExternalProcessUnavailableError < StandardError; end
 
-    File.open(output_file, "w") do |f|
-      f.puts(output)
-    end
-    puts "Output written to: #{output_file}"
+  if RUBY_ENGINE == 'opal'
+    require "interscript/opal"
+    extend Opal
+  else
+    require "interscript/fs"
+    extend Fs
   end
 
-  def load_system_definition(system_code)
-    @systems[system_code] ||= YAML.load_file(File.join(SYSTEM_DEFINITIONS_PATH, "#{system_code}.yaml"))
-  end
+  class << self
 
-  def get_system(system_code)
-    @systems[system_code]
-  end
+    def transliterate(system_code, string, maps={})
+      unless maps.has_key? system_code
+        maps[system_code] = Interscript::Mapping.for(system_code)
+      end
+      # mapping = Interscript::Mapping.for(system_code)
+      mapping = maps[system_code]
 
-  def system_char_map(system_code)
-    get_system(system_code)["map"]["characters"]
-  end
+      # First, apply chained transliteration as specified in the list `chain`
+      chain = mapping.chain.dup
+      while chain.length > 0
+        string = transliterate(chain.shift, string, maps)
+      end
 
-  def system_rules(system_code)
-    get_system(system_code)["map"]["rules"]
-  end
+      # Then, apply the rest of the map
+      separator = mapping.character_separator || ""
+      word_separator = mapping.word_separator || ""
+      title_case = mapping.title_case
+      downcase = mapping.downcase
 
-  def transliterate(system_code, string)
-    load_system_definition(system_code)
+      # charmap = mapping.characters&.sort_by { |k, _v| k.size }&.reverse&.to_h
+      # dictmap = mapping.dictionary&.sort_by { |k, _v| k.size }&.reverse&.to_h
+      charmap = mapping.characters_hash
+      dictmap = mapping.dictionary_hash
+      trie = mapping.dictionary_trie
 
-    # TODO: also need to support regular expressions via system_rules(system_code), before system_char_map
+      string = external_processing(mapping, string)
 
-    character_map = system_char_map(system_code)
+      pos = 0
+      while pos < string.to_s.size
+        m = 0
+        wordmatch = ""
 
-    string.split('').map do |char|
-      converted_char = character_map[char] ? character_map[char] : char
-      string[char] = converted_char
-    end.join('')
-  end
+        # Using Trie, find the longest matching substring
+        while (pos + m < string.to_s.size) && (trie.partial_word?string[pos..pos+m])
+          wordmatch = string[pos..pos+m] if trie.word?string[pos..pos+m]
+          m += 1
+        end
 
-end
+        m = wordmatch.length
+        if m > 0
+          repl = dictmap[string[pos..pos+m-1]]
+          string = sub_replace(string, pos, m, repl)
+          pos += repl.length
+        else
+          pos += 1
+        end
+      end
+
+      output = string.clone
+      offsets = Array.new string.to_s.size, 1
+
+      # mapping.rules.each do |r|
+      #   string.to_s.scan(/#{r['pattern']}/) do |matches|
+      #     match = Regexp.last_match
+      #     pos = match.offset(0).first
+      #     result = r['result'].clone
+      #     matches.each.with_index { |v, i| result.sub!(/\\#{i + 1}/, v) } if matches.is_a? Array
+      #     result.upcase! if up_case_around?(string, pos)
+      #     output[offsets[0...pos].sum, match[0].size] = result
+      #     offsets[pos] += result.size - match[0].size
+      #   end
+      # end
+
+      mapping.rules.each do |r|
+        next unless output
+        re = mkregexp(r["pattern"])
+        output = output.gsub(re, r["result"])
+      end
+
+      charmap.each do |k, v|
+        while (match = output&.match(/#{k}/))
+          pos = match.offset(0).first
+          result = !downcase && up_case_around?(output, pos) ? v.upcase : v
+
+          # if more than one, choose the first one
+          result = result[0] if result.is_a?(Array)
+
+          output = sub_replace(
+            output,
+            pos,
+            match[0].size,
+            add_separator(separator, pos, result)
+          )
+        end
+      end
+
+      mapping.postrules.each do |r|
+        next unless output
+        re = mkregexp(r["pattern"])
+        output = output.gsub(re, r["result"])
+      end
 
+      return unless output
+
+      output = output.sub(/^(.)/, &:upcase) if title_case
+      if word_separator != ''
+        output = output.gsub(/#{word_separator}#{separator}/u, word_separator)
+
+        if title_case
+          output = output.gsub(/#{word_separator}(.)/u, &:upcase)
+        end
+      end
+
+      output.unicode_normalize
+    end
+
+    private
+
+    def add_separator(separator, pos, result)
+      pos == 0 ? result : separator + result
+    end
+
+    def up_case_around?(string, pos)
+      return false if string[pos] == string[pos].downcase
+
+      i = pos - 1
+      i -= 1 while i.positive? && string[i] !~ Regexp.new(ALPHA_REGEXP)
+      before = i >= 0 && i < pos ? string[i].to_s.strip : ''
+
+      i = pos + 1
+      i += 1 while i < string.size - 1 && string[i] !~ Regexp.new(ALPHA_REGEXP)
+      after = i > pos ? string[i].to_s.strip : ''
+
+      before_uc = !before.empty? && before == before.upcase
+      after_uc = !after.empty? && after == after.upcase
+      # before_uc && (after.empty? || after_uc) || after_uc && (before.empty? || before_uc)
+      before_uc || after_uc
+    end
+
+  end
+end

data/lib/interscript/command.rb

@@ -0,0 +1,28 @@
+require 'thor'
+require 'interscript'
+require 'json'
+module Interscript
+  # Command line interface
+  class Command < Thor
+    desc '<file>', 'Transliterate text'
+    option :system, aliases: '-s', required: true, desc: 'Transliteration system'
+    option :output, aliases: '-o', required: false, desc: 'Output file'
+    option :map, aliases: '-m', required: false, default: "{}", desc: 'Transliteration mapping json'
+
+    def translit(input)
+      if options[:output]
+        Interscript.transliterate_file(options[:system], input, options[:output], JSON.parse(options[:map]))
+      else
+        puts Interscript.transliterate(options[:system], IO.read(input))
+      end
+    end
+
+    desc 'list', 'Prints allowed transliteration systems'
+    def list
+      dir = File.expand_path '../../maps/*.yaml', __dir__
+      Dir[dir].each do |path|
+        puts File.basename path, '.yaml'
+      end
+    end
+  end
+end

data/lib/interscript/fs.rb

@@ -0,0 +1,69 @@
+module Interscript
+  module Fs
+    ALPHA_REGEXP = '[[:alpha:]]'
+
+    def sub_replace(string, pos, size, repl)
+      string[pos..pos + size - 1] = repl
+      string
+    end
+
+    def root_path
+      @root_path ||= Pathname.new(File.join(File.dirname(__dir__), ".."))
+    end
+
+    def transliterate_file(system_code, input_file, output_file, maps={})
+      input = File.read(input_file)
+      output = transliterate(system_code, input, maps)
+
+      File.open(output_file, 'w') do |f|
+        f.puts(output)
+      end
+
+      puts "Output written to: #{output_file}"
+      output_file
+    end
+
+    def import_python_modules
+      begin
+        pyimport :g2pwrapper
+      rescue
+        pyimport :sys
+        sys.path.append(root_path.to_s + "/lib/")
+        pyimport :g2pwrapper
+      end
+    end
+
+    def external_process(process_name, string)
+      import_python_modules
+
+      case process_name
+      when 'sequitur.pythainlp_lexicon'
+        return g2pwrapper.transliterate('pythainlp_lexicon', string)
+      when 'sequitur.wiktionary_phonemic'
+        return g2pwrapper.transliterate('wiktionary_phonemic', string)
+      else
+        raise ExternalProcessNotRecognizedError.new
+      end
+
+    rescue
+      raise ExternalProcessUnavailableError.new
+    end
+
+    def external_processing(mapping, string)
+      # Segmentation
+      string = external_process(mapping.segmentation, string) if mapping.segmentation
+
+      # Transliteration/Transcription
+      string = external_process(mapping.transcription, string) if mapping.transcription
+
+      string
+    end
+
+    private
+
+    def mkregexp(regexpstring)
+      /#{regexpstring}/u
+    end
+
+  end
+end

data/lib/interscript/mapping.rb

@@ -0,0 +1,142 @@
+require 'rambling-trie'
+require 'yaml'
+require 'json'
+
+module Interscript
+
+  class Mapping
+    attr_reader(
+      :id,
+      :url,
+      :name,
+      :notes,
+      :rules,
+      :tests,
+      :language,
+      :postrules,
+      :characters,
+      :description,
+      :authority_id,
+      :creation_date,
+      :source_script,
+      :destination_script,
+      :chain,
+      :character_separator,
+      :word_separator,
+      :title_case,
+      :downcase,
+      :dictionary,
+      :characters_hash,
+      :dictionary_hash,
+      :segmentation,
+      :transcription,
+      :dictionary_trie
+    )
+
+    def initialize(system_code, options = {})
+      @system_code = system_code
+      @depth = options.fetch(:depth, 0).to_i
+
+      unless RUBY_ENGINE == 'opal'
+        @system_path = options.fetch(:system_code, default_path)
+      end
+
+      load_and_serialize_system_mappings
+    end
+
+    def self.for(system_code, options = {})
+      new(system_code, options)
+    end
+
+    def load_and_serialize_system_mappings
+      return if depth >= 5
+
+      mappings = load_system_mappings
+      serialize_system_mappings(mappings)
+    end
+
+    private
+
+    attr_reader :depth, :system_code, :system_path
+
+    def system_code_file
+      [system_code, "yaml"].join(".")
+    end
+
+    def default_path
+      @default_path ||= Interscript.root_path.join("maps")
+    end
+
+    def load_system_mappings
+      if RUBY_ENGINE == 'opal'
+        load_opal_mappings
+      else
+        load_fs_mappings
+      end
+    end
+
+    def load_opal_mappings
+      JSON.parse(`InterscriptMaps[#{system_code}]`)
+    end
+
+    def load_fs_mappings
+      YAML.load_file(system_path.join(system_code_file))
+    rescue Errno::ENOENT
+      raise Interscript::InvalidSystemError.new("No system mappings found")
+    end
+
+    def serialize_system_mappings(mappings)
+      @id = mappings.fetch("id", nil)
+      @url = mappings.fetch("url", nil)
+      @name = mappings.fetch("name", nil)
+      @notes = mappings.fetch("notes", nil)
+      @tests = mappings.fetch("tests", [])
+      @language = mappings.fetch("language", nil)
+      @description = mappings.fetch("description", nil)
+      @authority_id = mappings.fetch("authority_id", nil)
+      @creation_date = mappings.fetch("creation_date", nil)
+      @source_script = mappings.fetch("source_script", nil)
+      @destination_script = mappings.fetch("destination_script", nil)
+      @chain = mappings.fetch("chain", [])
+      @character_separator = mappings["map"]["character_separator"] || nil
+      @word_separator = mappings["map"]["word_separator"] || nil
+      @title_case = mappings["map"]["title_case"] || false
+      @downcase = mappings["map"]["downcase"] || false
+      @rules = mappings["map"]["rules"] || []
+      @postrules = mappings["map"]["postrules"] || []
+      @characters = mappings["map"]["characters"] || {}
+      @dictionary = mappings["map"]["dictionary"] || {}
+      @segmentation = mappings["map"]["segementation"] || nil
+      @transcription = mappings["map"]["transcription"] || nil
+
+      include_inherited_mappings(mappings)
+      build_hashes
+      build_trie
+    end
+
+    def include_inherited_mappings(mappings)
+      inherit_systems = [].push(mappings["map"]["inherit"]).flatten
+
+      inherit_systems.each do |inherit_system|
+        next unless inherit_system
+
+        inherited_mapping = Mapping.for(inherit_system, depth: depth + 1)
+
+        @rules = [inherited_mapping.rules, rules].flatten
+        @postrules = [inherited_mapping.postrules, postrules].flatten
+        @characters = (inherited_mapping.characters|| {}).merge(characters)
+        @dictionary = (inherited_mapping.dictionary|| {}).merge(dictionary)
+      end
+    end
+
+    def build_hashes
+      @characters_hash = characters&.sort_by { |k, _v| k.size }&.reverse&.to_h
+      @dictionary_hash = dictionary&.sort_by { |k, _v| k.size }&.reverse&.to_h
+    end
+
+    def build_trie
+      @dictionary_trie = Rambling::Trie.create
+      dictionary_trie.concat dictionary.keys
+    end
+  end
+end

data/lib/interscript/opal.rb

@@ -0,0 +1,23 @@
+module Interscript
+  module Opal
+    ALPHA_REGEXP = '\p{L}'
+
+    def mkregexp(regexpstring)
+      flags = 'u'
+      if regexpstring.include? "(?i)"
+        regexpstring = regexpstring.gsub("(?i)", "").gsub("(?-i)", "")
+        flags = 'ui'
+      end
+      Regexp.new("/#{regexpstring}/#{flags}")
+    end
+
+    def sub_replace(string, pos, size, repl)
+      string[0, pos] + repl + string[pos + size..-1]
+    end
+
+    def external_processing(mapping, string)
+      string
+    end
+
+  end
+end