interscript 0.1.4 → 2.0.5

data/exe/interscript

@@ -0,0 +1,6 @@
+require 'interscript/command'
+
+if ARGV.any? && !Interscript::Command.all_tasks.key?(ARGV.first)
+  ARGV.unshift :translit
+end
+Interscript::Command.start ARGV

data/interscript.gemspec

@@ -0,0 +1,31 @@
+require_relative 'lib/interscript/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "interscript"
+  spec.version       = Interscript::VERSION
+  spec.summary       = %q{Interoperable script conversion systems}
+  spec.description   = %q{Interoperable script conversion systems}
+  spec.authors       = ["Ribose Inc."]
+  spec.email         = ["open.source@ribose.com"]
+
+  spec.date = %q{2019-11-17}
+  spec.homepage      = "https://www.interscript.com"
+  spec.license       = "BSD-2-Clause"
+
+  spec.required_ruby_version = Gem::Requirement.new(">= 2.3.0")
+
+  spec.metadata["homepage_uri"] = spec.homepage
+  spec.metadata["source_code_uri"] = "https://github.com/interscript/interscript"
+
+  # Specify which files should be added to the gem when it is released.
+  # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
+  spec.files         = Dir.chdir(File.expand_path('..', __FILE__)) do
+    `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  end
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency "thor"
+  spec.add_dependency "interscript-maps"
+end

data/lib/interscript.rb

@@ -1,163 +1,111 @@
-# frozen_string_literal: true
-
+require "interscript/version"
 require "yaml"
-require "interscript/mapping"
 
-# Transliteration
 module Interscript
+  class MapNotFoundError < StandardError; end
 
   class << self
-    def root_path
-      @root_path ||= Pathname.new(File.dirname(__dir__))
+    def load_path
+      @load_path ||= ['.', *Interscript.map_locations]
     end
 
-    def transliterate_file(system_code, input_file, output_file, maps)
-      input = File.read(input_file)
-      output = transliterate(system_code, input, maps)
+    def locate map_name
+      map_name = map_aliases[map_name] if map_aliases.include? map_name
 
-      File.open(output_file, 'w') do |f|
-        f.puts(output)
+      load_path.each do |i|
+        # iml is an extension for a library, imp for a map
+        ["iml", "imp"].each do |ext|
+          f = File.expand_path("#{map_name}.#{ext}", i)
+          return f if File.exist?(f)
+        end
       end
-      puts "Output written to: #{output_file}"
+      raise MapNotFoundError, "Couldn't locate #{map_name}"
     end
 
-    def import_python_modules
-      begin
-        pyimport :g2pwrapper
-      rescue
-        pyimport :sys
-        sys.path.append(root_path.to_s+"/lib/")
-        pyimport :g2pwrapper
-      end
+    def parse(map_name)
+      Interscript::DSL.parse(map_name)
     end
 
-    def external_process(process_name, string)
-      import_python_modules
-      case process_name
-        when 'sequitur.pythainlp_lexicon'
-          return g2pwrapper.transliterate('pythainlp_lexicon', string)
-        when 'sequitur.wiktionary_phonemic'
-          return g2pwrapper.transliterate('wiktionary_phonemic', string)
-      else
-        puts "Invalid Process"
-      end
+    def load(system_code, maps={}, compiler: Interscript::Interpreter)
+      maps[[system_code, compiler.name]] ||= compiler.(system_code)
     end
 
-    def transliterate(system_code, string, maps={})
-      if (!maps.has_key?system_code)
-        maps[system_code] = Interscript::Mapping.for(system_code)
-      end
-      # mapping = Interscript::Mapping.for(system_code)
-      mapping = maps[system_code]
+    # Transliterates the string.
+    def transliterate(system_code, string, maps={}, compiler: Interscript::Interpreter)
+      # The current best implementation is Interpreter
+      load(system_code, maps, compiler: compiler).(string)
+    end
 
+    # Gives each possible value of the transliteration.
+    def transliterate_each(system_code, string, maps={}, &block)
+      load(system_code, maps).(string, each: true, &block)
+    end
 
-      # First, apply chained transliteration as specified in the list `chain`
-      chain = mapping.chain.dup
-      while chain.length > 0
-        string = transliterate(chain.shift, string, maps)
-      end
+    def transliterate_file(system_code, input_file, output_file, maps={})
+      input = File.read(input_file)
+      output = transliterate(system_code, input, maps)
 
-      # Then, apply the rest of the map
-      separator = mapping.character_separator || ""
-      word_separator = mapping.word_separator || ""
-      title_case = mapping.title_case
-      downcase = mapping.downcase
-
-      # charmap = mapping.characters&.sort_by { |k, _v| k.size }&.reverse&.to_h
-      # dictmap = mapping.dictionary&.sort_by { |k, _v| k.size }&.reverse&.to_h
-      charmap = mapping.characters_hash
-      dictmap = mapping.dictionary_hash
-      trie = mapping.dictionary_trie
-
-      # Segmentation
-      string = external_process(mapping.segmentation, string) if mapping.segmentation
-
-      # Transliteration/Transcription
-      string = external_process(mapping.transcription, string) if mapping.transcription
-
-      pos = 0
-      while pos < string.to_s.size
-        m = 0
-        wordmatch = ""
-
-        # Using Trie, find the longest matching substring
-        while (pos + m < string.to_s.size) && (trie.partial_word?string[pos..pos+m]) 
-          wordmatch = string[pos..pos+m] if trie.word?string[pos..pos+m]
-          m += 1
-        end
-        m = wordmatch.length
-        if m > 0
-          repl = dictmap[string[pos..pos+m-1]]
-          string[pos..pos+m-1] = repl
-          pos += repl.length
-        else
-          pos += 1
-        end
+      File.open(output_file, 'w') do |f|
+        f.puts(output)
       end
 
-      output = string.clone
-      offsets = Array.new string.to_s.size, 1
-
-      # mapping.rules.each do |r|
-      #   string.to_s.scan(/#{r['pattern']}/) do |matches|
-      #     match = Regexp.last_match
-      #     pos = match.offset(0).first
-      #     result = r['result'].clone
-      #     matches.each.with_index { |v, i| result.sub!(/\\#{i + 1}/, v) } if matches.is_a? Array
-      #     result.upcase! if up_case_around?(string, pos)
-      #     output[offsets[0...pos].sum, match[0].size] = result
-      #     offsets[pos] += result.size - match[0].size
-      #   end
-      # end
-      mapping.rules.each do |r|
-        output.gsub!(/#{r['pattern']}/, r['result'])
-      end
+      puts "Output written to: #{output_file}"
+      output_file
+    end
 
-      charmap.each do |k, v|
-        while (match = output&.match(/#{k}/))
-          pos = match.offset(0).first
-          result = !downcase && up_case_around?(output, pos) ? v.upcase : v
-          result = result[0] if result.is_a?(Array) # if more than one, choose the first one
-          output[pos, match[0].size] = add_separator(separator, pos, result)
-        end
-      end
+    def map_gems
+      @map_gems ||= Gem.find_latest_files('interscript-maps.yaml').map do |i|
+        [i, YAML.load_file(i)]
+      end.to_h
+    end
 
-      mapping.postrules.each do |r|
-        output.gsub!(/#{r['pattern']}/, r['result'])
-      end
+    def map_locations
+      @map_locations ||= map_gems.map do |i,v|
+        paths = v["paths"].dup
+        paths += v["staging"] if ENV["INTERSCRIPT_STAGING"] && v["staging"]
 
-      if output
-        output.sub!(/^(.)/,  &:upcase) if title_case
-        if word_separator != ''
-          output.gsub!(/#{word_separator}#{separator}/,word_separator)
-          output.gsub!(/#{word_separator}(.)/, &:upcase) if title_case
+        paths.map do |j|
+          File.expand_path(j, File.dirname(i))
         end
-      end
+      end.flatten
+    end
 
-      output ? output.unicode_normalize : output
+    def secryst_index_locations
+      @secryst_index_locations ||= map_gems.map do |i,v|
+        v["secryst-models"]
+      end.compact.flatten
     end
 
-    private
+    def map_aliases
+      return @map_aliases if @map_aliases
 
-    def add_separator(separator, pos, result)
-      pos == 0 ? result : separator + result
+      @map_aliases = {}
+      map_gems.each do |i,v|
+        (v["aliases"] || {}).each do |code, value|
+          value.each do |al, map|
+            @map_aliases[al] = map["alias_to"]
+          end
+        end
+      end
+      @map_aliases
     end
 
-    def up_case_around?(string, pos)
-      return false if string[pos] == string[pos].downcase
+    # List all possible maps to use
+    def maps(basename: true, load_path: false, select: "*", libraries: false)
+      paths = load_path ? Interscript.load_path : Interscript.map_locations
+      ext = libraries ? "iml" : "imp"
 
-      i = pos - 1
-      i -= 1 while i.positive? && string[i] !~ /[[:alpha:]]/
-      before = i >= 0 && i < pos ? string[i].to_s.strip : ''
+      imps = paths.map { |i| Dir["#{i}/#{select}.#{ext}"] }.flatten
 
-      i = pos + 1
-      i += 1 while i < string.size - 1 && string[i] !~ /[[:alpha:]]/
-      after = i > pos ? string[i].to_s.strip : ''
-
-      before_uc = !before.empty? && before == before.upcase
-      after_uc = !after.empty? && after == after.upcase
-      # before_uc && (after.empty? || after_uc) || after_uc && (before.empty? || before_uc)
-      before_uc || after_uc
+      basename ? imps.map { |j| File.basename(j, ".#{ext}") } : imps
     end
   end
 end
+
+require 'interscript/stdlib'
+
+require "interscript/compiler"
+require "interscript/interpreter"
+
+require 'interscript/dsl'
+require 'interscript/node'

data/lib/interscript/command.rb

@@ -1,16 +1,18 @@
 require 'thor'
 require 'interscript'
-
+require 'json'
 module Interscript
   # Command line interface
   class Command < Thor
     desc '<file>', 'Transliterate text'
     option :system, aliases: '-s', required: true, desc: 'Transliteration system'
     option :output, aliases: '-o', required: false, desc: 'Output file'
+    # Was this option really well thought out? The last parameter is a cache, isn't it?
+    #option :map, aliases: '-m', required: false, default: "{}", desc: 'Transliteration mapping json'
 
     def translit(input)
       if options[:output]
-        Interscript.transliterate_file(options[:system], input, options[:output])
+        Interscript.transliterate_file(options[:system], input, options[:output]) #, JSON.parse(options[:map]))
       else
         puts Interscript.transliterate(options[:system], IO.read(input))
       end
@@ -18,9 +20,8 @@ module Interscript
 
     desc 'list', 'Prints allowed transliteration systems'
     def list
-      dir = File.expand_path '../../maps/*.yaml', __dir__
-      Dir[dir].each do |path|
-        puts File.basename path, '.yaml'
+      Interscript.maps(load_path: true).each do |path|
+        puts path
       end
     end
   end

data/lib/interscript/compiler.rb

@@ -0,0 +1,22 @@
+# An Interscript compiler interface
+class Interscript::Compiler
+  attr_accessor :code
+
+  def self.call(map, **kwargs)
+    if String === map
+      map = Interscript::DSL.parse(map)
+    end
+    compiler = new
+    compiler.compile(map, **kwargs)
+    compiler
+  end
+
+  def compile(map)
+    raise NotImplementedError, "Compile method on #{self.class} is not implemented"
+  end
+
+  # Execute a map
+  def call
+    raise NotImplementedError, "Call class on #{self.class} is not implemented"
+  end
+end

data/lib/interscript/compiler/javascript.rb

@@ -0,0 +1,292 @@
+begin
+  require 'mini_racer'
+rescue LoadError
+  # Ignore loading error
+end
+require 'json'
+
+class Interscript::Compiler::Javascript < Interscript::Compiler
+  def compile(map, debug: false)
+    @map = map
+    @parallel_trees = {}
+    @parallel_regexps = {}
+    @debug = debug
+    c = "var map = function(Interscript) {"
+    c << "Interscript.define_map(#{map.name.inspect}, function(Interscript, map) {\n";
+    c << "map.dependencies = #{map.dependencies.map(&:full_name).to_json};\n"
+    c
+
+    map.aliases.each do |name, value|
+      val = compile_item(value.data, map, :str)
+      c << "map.aliases.#{name} = #{val};\n"
+      val = '"'+compile_item(value.data, map, :re)+'"'
+      c << "map.aliases_re.#{name} = #{val};\n"
+    end
+
+    map.stages.each do |_, stage|
+      c << compile_rule(stage, @map, true)
+    end
+    @parallel_trees.each do |k,v|
+      c << "map.cache.PTREE_#{k} = #{v.to_json};\n"
+    end
+    @parallel_regexps.each do |k,v|
+      v = "[\"#{v[0]}\", #{v[1].to_json}]"
+      c << "map.cache.PRE_#{k} = #{v};\n"
+    end
+
+    c << "});"
+    c << "};"
+    c << "if (typeof module !== 'undefined') { module.exports = map; }"
+    c << "else if (typeof Interscript !== 'undefined') { map(Interscript); }"
+    c << 'else { throw "We couldn\'t dispatch Interscript from a map!"; }'
+    @code = c
+  end
+
+  def parallel_regexp_compile(subs_hash)
+    # puts subs_hash.inspect
+    regexp = subs_hash.each_with_index.map do |p,i|
+      "(?<_%d>%s)" % [i,p[0]]
+    end.join("|")
+    subs_regexp = regexp
+    # puts subs_regexp.inspect
+  end
+
+  def compile_rule(r, map = @map, wrapper = false)
+    c = ""
+    case r
+    when Interscript::Node::Stage
+      c += "map.stages.#{r.name} = function(s) {\n"
+      c += "globalThis.map_debug = globalThis.map_debug || [];\n" if @debug
+      r.children.each do |t|
+        comp = compile_rule(t, map)
+        c += comp
+        c += %{globalThis.map_debug.push([s, #{@map.name.to_s.to_json}, #{r.name.to_s.to_json}, #{t.inspect.to_json}, #{comp.to_json}]);\n} if @debug
+      end
+      c += "return s;\n"
+      c += "};\n"
+    when Interscript::Node::Group::Parallel
+      begin
+        # Try to build a tree
+        a = []
+        r.children.each do |i|
+          raise ArgumentError, "Can't parallelize #{i.class}" unless Interscript::Node::Rule::Sub === i
+          raise ArgumentError, "Can't parallelize rules with :before" if i.before
+          raise ArgumentError, "Can't parallelize rules with :after" if i.after
+          raise ArgumentError, "Can't parallelize rules with :not_before" if i.not_before
+          raise ArgumentError, "Can't parallelize rules with :not_after" if i.not_after
+
+          a << [compile_item(i.from, map, :par), compile_item(i.to, map, :parstr)]
+        end
+        ah = a.hash.abs
+        unless @parallel_trees.include? ah
+          tree = Interscript::Stdlib.parallel_replace_compile_tree(a)
+          @parallel_trees[ah] = tree
+        end
+        c += "s = Interscript.parallel_replace_tree(s, map.cache.PTREE_#{ah});\n"
+      rescue
+        # Otherwise let's build a megaregexp
+        a = []
+        Interscript::Stdlib.deterministic_sort_by_max_length(r.children).each do |i|
+          raise ArgumentError, "Can't parallelize #{i.class}" unless Interscript::Node::Rule::Sub === i
+
+          a << [build_regexp(i, map), compile_item(i.to, map, :parstr)]
+        end
+        ah = a.hash.abs
+        unless @parallel_regexps.include? ah
+          re = parallel_regexp_compile(a)
+          @parallel_regexps[ah] = [re, a.map(&:last)]
+        end
+        c += "s = Interscript.parallel_regexp_gsub(s, map.cache.PRE_#{ah});\n"
+      end
+    when Interscript::Node::Rule::Sub
+      from = %{"#{build_regexp(r, map).gsub("/", "\\\\/")}"}
+      if r.to == :upcase
+        to = 'function(a){return a.toUpperCase();}'
+      else
+        to = compile_item(r.to, map, :str)
+      end
+      c += "s = Interscript.gsub(s, #{from}, #{to});\n"
+    when Interscript::Node::Rule::Funcall
+      c += "s = Interscript.functions.#{r.name}(s, #{r.kwargs.to_json});\n"
+    when Interscript::Node::Rule::Run
+      if r.stage.map
+        doc = map.dep_aliases[r.stage.map].document
+        stage = doc.imported_stages[r.stage.name]
+      else
+        stage = map.imported_stages[r.stage.name]
+      end
+      c += "s = Interscript.transliterate(#{stage.doc_name.to_json}, s, #{stage.name.to_json});\n"
+    else
+      raise ArgumentError, "Can't compile unhandled #{r.class}"
+    end
+    c
+  end
+
+  def build_regexp(r, map=@map)
+    from = compile_item(r.from, map, :re)
+    before = compile_item(r.before, map, :re) if r.before
+    after = compile_item(r.after, map, :re) if r.after
+    not_before = compile_item(r.not_before, map, :re) if r.not_before
+    not_after = compile_item(r.not_after, map, :re) if r.not_after
+
+    re = ""
+    re += "(?<=#{before})" if before
+    re += "(?<!#{not_before})" if not_before
+    re += from
+    re += "(?!#{not_after})" if not_after
+    re += "(?=#{after})" if after
+
+    re
+  end
+
+  def compile_item i, doc=@map, target=nil
+    i = i.first_string if %i[str parstr].include? target
+    i = Interscript::Node::Item.try_convert(i)
+    if target == :parstr
+      parstr = true
+      target = :par
+    end
+
+    out = case i
+    when Interscript::Node::Item::Alias
+      astr = if i.map
+        d = doc.dep_aliases[i.map].document
+        a = d.imported_aliases[i.name]
+        raise ArgumentError, "Alias #{i.name} of #{i.stage.map} not found" unless a
+        "Interscript.get_alias_ALIASTYPE(#{a.doc_name.to_json}, #{a.name.to_json})"
+      elsif Interscript::Stdlib::ALIASES.include?(i.name)
+        if target != :re && Interscript::Stdlib.re_only_alias?(i.name)
+          raise ArgumentError, "Can't use #{i.name} in a #{target} context"
+        end
+        stdlib_alias = true
+        "Interscript.aliases.#{i.name}"
+      else
+        a = doc.imported_aliases[i.name]
+        raise ArgumentError, "Alias #{i.name} not found" unless a
+
+        "Interscript.get_alias_ALIASTYPE(#{a.doc_name.to_json}, #{a.name.to_json})"
+      end
+
+      if target == :str
+        astr = astr.sub("_ALIASTYPE(", "(")
+      elsif target == :re
+        astr = %{"+#{astr.sub("_ALIASTYPE(", "_re(")}+"}
+      elsif parstr && stdlib_alias
+        astr = Interscript::Stdlib::ALIASES[i.name]
+      elsif target == :par
+        # raise NotImplementedError, "Can't use aliases in parallel mode yet"
+        astr = Interscript::Stdlib::ALIASES[i.name]
+      end
+    when Interscript::Node::Item::String
+      if target == :str
+        # Replace $1 with \$1, this is weird, but it works!
+        i.data.gsub("$", "\\\\$").to_json
+      elsif target == :par
+        i.data
+      elsif target == :re
+        Regexp.escape(i.data)
+      end
+    when Interscript::Node::Item::Group
+      if target == :par
+        i.children.map do |j|
+          compile_item(j, doc, target)
+        end.reduce([""]) do |j,k|
+          Array(j).product(Array(k)).map(&:join)
+        end
+      elsif target == :str
+        i.children.map { |j| compile_item(j, doc, target) }.join("+")
+      elsif target == :re
+        i.children.map { |j| compile_item(j, doc, target) }.join
+      end
+    when Interscript::Node::Item::CaptureGroup
+      if target != :re
+        raise ArgumentError, "Can't use a CaptureGroup in a #{target} context"
+      end
+      "(" + compile_item(i.data, doc, target) + ")"
+    when Interscript::Node::Item::Maybe,
+         Interscript::Node::Item::MaybeSome,
+         Interscript::Node::Item::Some
+
+      resuffix = { Interscript::Node::Item::Maybe     => "?" ,
+                   Interscript::Node::Item::Some      => "+" ,
+                   Interscript::Node::Item::MaybeSome => "*" }[i.class]
+
+      if target == :par
+        raise ArgumentError, "Can't use a MaybeSome in a #{target} context"
+      end
+      if Interscript::Node::Item::String === i.data && i.data.data.length != 1
+        "(?:" + compile_item(i.data, doc, target) + ")" + resuffix
+      else
+        compile_item(i.data, doc, target) + resuffix
+      end
+    when Interscript::Node::Item::CaptureRef
+      if target == :par
+        raise ArgumentError, "Can't use CaptureRef in parallel mode"
+      elsif target == :re
+        "\\\\#{i.id}"
+      elsif target == :str
+        "\"$#{i.id}\""
+      end
+    when Interscript::Node::Item::Any
+      if target == :str
+        raise ArgumentError, "Can't use Any in a string context" # A linter could find this!
+      elsif target == :par
+        i.data.map(&:data)
+      elsif target == :re
+        case i.value
+        when Array
+          data = i.data.map { |j| compile_item(j, doc, target) }
+          "(?:"+data.join("|")+")"
+        when String
+          "[#{Regexp.escape(i.value)}]"
+        when Range
+          "[#{Regexp.escape(i.value.first)}-#{Regexp.escape(i.value.last)}]"
+        end
+      end
+    end
+  end
+
+  @maps_loaded = {}
+  @ctx = nil
+  class << self
+    attr_accessor :maps_loaded
+    attr_accessor :ctx
+  end
+
+  def load
+    if !self.class.maps_loaded[@map.name]
+      @map.dependencies.each do |dep|
+        dep = dep.full_name
+        if !self.class.maps_loaded[dep]
+          Interscript.load(dep, compiler: self.class).load
+        end
+      end
+
+      ctx = self.class.ctx
+      unless ctx
+        ctx = MiniRacer::Context.new
+        ctx.eval File.read(__dir__+"/../../../../js/test-compiler/xregexp.js")
+        # Compatibility with Safari: will come later
+        #ctx.eval File.read(__dir__+"/../../../js/xregexp-oniguruma.js")
+        ctx.eval File.read(__dir__+"/../../../../js/src/stdlib.js")
+        self.class.ctx = ctx
+      end
+      #puts @code
+      ctx.eval @code
+      self.class.maps_loaded[@map.name] = true
+    end
+  end
+
+  def call(str, stage=:main)
+    load
+    self.class.ctx.eval "Interscript.transliterate(#{@map.name.to_json}, #{str.to_json}, #{stage.to_json})"
+  end
+
+  def self.read_debug_data
+    self.ctx.eval "globalThis.map_debug || []"
+  end
+
+  def self.reset_debug_data
+    self.ctx.eval "globalThis.map_debug = []"
+  end
+end