interscript 0.1.6 → 2.1.0a9

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7557ea1ca381562c61be7dbbeab2ea4adc42ef57ce857ef86acd62a08e5ce588
-  data.tar.gz: 5099d4a7bf07817155d620db716af452607b8565f4e08d37aa805f486556e0e1
+  metadata.gz: 863d75eaef5d110e3460efa16357dd784120a11a119d5e33d6551727f4c9f33f
+  data.tar.gz: 91d2a59dc9b03f9c7e8123f0cc0980e7bcde8dc0277f0e77732d80b0cd0e3508
 SHA512:
-  metadata.gz: f8a738a34aba269c0a01b4d123f01cc423a3c296541f302f65a6043bd2b618ef2c702f32d469866ee7c488624985b45ed2eee2ad9bbe2955f230b5b45472c364
-  data.tar.gz: a3a66af7fcb9d8c82bcf927b17ea68686298bccec2f6e8dfac25796e7013b36f68fa9f130bd2fb9e7bc214a528896cafdd41a2f464886f09c504f376c19065d8
+  metadata.gz: dd8d30f472a7e3a049c8df13537c280d2b577956ce9439aeecc3164d533453c31bf2b66a662c9a8d09f43a610d3029e699113573c64bb005704b620c8bbb218f
+  data.tar.gz: e002aa79cfd6bf3c07403a8a87342383e575b00ffb8e9cc96271a9e0a1390214d13a01a6cdb5665fbadc881eaefbbf915520ee18a12baac1d70bfae1eb37bbd3

data/.gitignore

@@ -0,0 +1,11 @@
+/.bundle/
+/.yardoc
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+
+# rspec failure tracking
+.rspec_status

data/.rspec

@@ -0,0 +1,3 @@
+--format documentation
+--color
+--require spec_helper

data/Gemfile

@@ -0,0 +1,29 @@
+source "https://rubygems.org"
+
+# Specify your gem's dependencies in interscript.gemspec
+gemspec
+
+gem "rake", "~> 12.0"
+gem "rspec", "~> 3.0"
+
+gem "interscript-maps", path: "../maps"
+
+unless Gem::Version.new(RUBY_VERSION) < Gem::Version.new("2.7")
+  group :secryst do
+    if File.exist? "../../secryst"
+      gem "secryst", path: "../../secryst"
+    else
+      gem "secryst"
+    end
+  end
+end
+
+gem 'regexp_parser'
+
+unless ENV["SKIP_JS"]
+  group :jsexec do
+    gem 'mini_racer'
+  end
+end
+
+gem 'simplecov', require: false, group: :test

data/LICENSE.adoc

@@ -0,0 +1,31 @@
+= Licenses & Copyright
+
+This license file adheres to the formatting guidelines of
+https://github.com/nevir/readable-licenses[readable-licenses].
+
+
+== Ribose BSD 2-Clause License
+
+Copyright (c) 2019-, https://www.ribose.com[Ribose Inc].
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification,
+are permitted provided that the following conditions are met:
+
+1.  Redistributions of source code must retain the above copyright notice,
+    this list of conditions and the following disclaimer.
+
+2.  Redistributions in binary form must reproduce the above copyright notice,
+    this list of conditions and the following disclaimer in the documentation
+    and/or other materials provided with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

data/README.md

@@ -0,0 +1,3 @@
+# Interscript
+
+Documentation is available directory higher in file README.adoc.

data/Rakefile

@@ -0,0 +1,53 @@
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new(:spec)
+
+task :compile, [:compiler, :target] do |t, args|
+  require "bundler/setup"
+  require "interscript"
+
+  compiler, ext = case args[:compiler]
+  when "ruby"
+    require "interscript/compiler/ruby"
+    [Interscript::Compiler::Ruby, "rb"]
+  when "javascript"
+    require "interscript/compiler/javascript"
+    [Interscript::Compiler::Javascript, "js"]
+  end
+
+  FileUtils.mkdir_p(args[:target])
+
+  maplist = {}
+
+  Interscript.maps.each do |map|
+    code = compiler.(map).code
+    File.write(args[:target] + "/" + map + "." + ext, code)
+    maplist[map] = nil
+  end
+
+  Interscript.maps(libraries: true).each do |map|
+    code = compiler.(map).code
+    File.write(args[:target] + "/" + map + "." + ext, code)
+  end
+
+  File.write(args[:target] + "/index.json", maplist.to_json)
+end
+
+task :version, [:ver] do |t, ver|
+  ver = ver[:ver]
+
+  rubyver = File.read(rubyfile = __dir__+"/lib/interscript/version.rb")
+  jsver   = File.read(jsfile   = __dir__+"/../js/package.json")
+  mapsver = File.read(mapsfile = __dir__+"/../maps/interscript-maps.gemspec")
+
+  rubyver = rubyver.gsub(/(VERSION = ")([0-9a-z.-]*)(")/,                "\\1#{ver}\\3")
+  jsver   = jsver.gsub(/("version": ")([0-9a-z.-]*)(")/,                 "\\1#{ver}\\3")
+  mapsver = mapsver.gsub(/(INTERSCRIPT_MAPS_VERSION=")([0-9a-z.-]*)(")/, "\\1#{ver}\\3")
+
+  File.write(rubyfile, rubyver)
+  File.write(jsfile,   jsver)
+  File.write(mapsfile, mapsver)
+end
+
+task :default => :spec

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "interscript"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start(__FILE__)

data/bin/interscript

@@ -1,41 +1,5 @@
 #!/usr/bin/env ruby
 
-require 'rubygems'
-
-# resolve bin path, ignoring symlinks
-require 'pathname'
-bin_file = Pathname.new(__FILE__).realpath
-
-# add self to libpath
-$LOAD_PATH.unshift File.expand_path("../../lib", bin_file)
-
-# Fixes https://github.com/rubygems/rubygems/issues/1420
-require 'rubygems/specification'
-
-class Gem::Specification
-  def this; self; end
-end
-
-require 'interscript/command'
-
-if ARGV.any? && !Interscript::Command.all_tasks.key?(ARGV.first)
-  ARGV.unshift :translit
-end
-Interscript::Command.start ARGV
-
-# if ARGV.empty?
-#   puts "write source file, source format, and output file"
-# else
-#   args = Hash[ARGV.flat_map { |s| s.scan(/--?([^=\s]+)(?:=(\S+))?/) }]
-#   input = ARGV[0]
-#   system_code = args["system"]
-#   output_file = args["output"]
-
-#   raise "Please enter the system code with --system={system_code}" unless system_code
-
-#   if output_file
-#     Interscript.transliterate_file(system_code, input, output_file)
-#   else
-#     puts Interscript.transliterate(system_code, IO.read(input))
-#   end
-# end
+require 'bundler/setup'
+$LOAD_PATH << "#{__dir__}/../lib"
+load "#{__dir__}/../exe/interscript"

data/bin/maps_analyze_staging

@@ -0,0 +1,168 @@
+#!/usr/bin/env ruby
+ENV["INTERSCRIPT_STAGING"] = "1"
+require "bundler/setup"
+require "interscript"
+require "interscript/compiler/ruby"
+
+$map_name = ARGV[0]
+
+# surpress warnings
+def warn(*)
+end
+
+
+def debug_single_map(from, expected, result, result2)
+  puts ""
+  # puts $subs_array.inspect
+  # puts $subs_matches.inspect
+  puts from.inspect
+  puts from.chars.inspect
+  puts from.codepoints.map{|cp| '\\u%04x' % cp}.join(' ')
+  puts expected
+  puts result
+  puts result2
+  return nil unless $subs_matches
+  first_mismatch = $subs_matches.select{|match_data|
+    exp = expected[match_data[:begin]...match_data[:end]]
+    res = result[match_data[:begin]...match_data[:end]]
+    exp != res
+  }.first
+
+  # puts $subs_matches.inspect
+  # puts first_mismatch.inspect
+  output_string = ''
+  last_end = 0
+  $subs_matches.each do |match_data|
+    substr = from[match_data[:begin]...match_data[:end]]
+    if match_data[:begin] != last_end
+      output_string << '_' * (match_data[:begin] - last_end)
+    end
+    output_string << match_data[:result]
+
+    puts ''
+    puts "ERROR " if match_data == first_mismatch
+    indent = ' ' * match_data[:begin]
+    # puts indent + substr
+
+
+
+    matched_rule = $subs_array[match_data[:idx]]
+
+    if match_data == first_mismatch
+      puts indent + matched_rule.inspect + "   (#{matched_rule[0].codepoints.map{|cp| '\\u%04x' % cp}.join(' ')})"
+    else
+      puts indent + matched_rule.inspect + "    (#{matched_rule[0].codepoints.map{|cp| '\\u%04x' % cp}.join(' ')})"
+    end
+
+    puts from[0,match_data[:end]]
+
+    puts expected[0,output_string.size]
+    puts output_string
+    last_end = match_data[:end]
+
+
+    #return if match_data == first_mismatch
+  end
+  puts ''
+end
+
+def analyze_maps( filelist )
+  cache = {}
+
+  good_total = 0
+  almost_total = 0
+  bad_total = 0
+  passed_total = 0
+  for i in filelist
+    # used at the bottom
+    raw_lines = File.open(i).read.split("\n")
+    rules_line = raw_lines.index "  # RULES"
+    characters_line = raw_lines.index "  # CHARACTERS"
+    postrules_line = raw_lines.index "  # POSTRULES"
+
+    system_name = File.basename(i, ".imp")
+
+    print "#{system_name}: "
+
+    exceptions = []
+
+    # begin
+      system = Interscript.parse(system_name)
+    # rescue Exception => e
+    #   exceptions << e
+    # end
+    # puts $using_tree.inspect
+    if exceptions.length == 0 && system.tests && system.tests.data && system.tests.data.length > 0
+      good = 0
+      bad = 0
+      almost = 0
+
+      system.tests.data.each do |from,expected|
+        testname = from[0...300].gsub("\n", " / ")
+
+        result = Interscript.transliterate(system_name, from, cache, compiler: Interscript::Interpreter)
+        result2 = Interscript.transliterate(system_name, from, cache, compiler: Interscript::Compiler::Ruby)
+        if result == nil or result2 == nil
+          print '0'
+          bad += 1
+        elsif result != result2
+          # Compiler / Interpreter bug!
+          print "!"
+          bad += 1
+        elsif result == expected
+          print "."
+          good += 1
+        elsif result.unicode_normalize == expected.unicode_normalize
+          print ","
+          almost += 1
+        elsif result.unicode_normalize.chomp == expected.unicode_normalize.chomp
+          print ";"
+          almost += 1
+        else
+          print "#"
+          debug_single_map(from, expected, result, result2) if $map_name
+          bad += 1
+        end
+      rescue Exception => e
+        print "%"
+        bad += 1
+        exceptions << e
+      end
+
+      if bad == 0
+        passed_total += 1
+        puts " all tests passed!"
+        `git mv #{i} #{i.sub("-staging", "")}`   if i.include? '-staging'
+      else
+        puts " (#{good}/#{good+almost+bad})"
+      end
+      good_total += good
+      almost_total += almost
+      bad_total += bad
+    elsif exceptions.length != 0
+    else
+      puts "no tests!"
+    end
+    puts "warning: postrules before rules\n\n" if (postrules_line||10000000) < (rules_line||0)
+    puts "warning: postrules before characters\n\n" if (postrules_line||10000000) < (characters_line||0)
+
+    Array(exceptions.first).compact.each do |e|
+      puts e.full_message
+    end
+  end
+  puts " #{good_total} (#{good_total+almost_total})/#{good_total+almost_total+bad_total}"
+  puts " #{passed_total} passed!"
+end
+
+#analyze_maps Dir[__dir__+"/../../maps/maps/*.imp"].sort
+if $map_name
+  $DEBUG_RE = true
+  analyze_maps Dir[__dir__+"/../../maps/maps/#{$map_name}.imp"].sort
+  analyze_maps Dir[__dir__+"/../../maps/maps-staging/#{$map_name}.imp"].sort
+else
+  analyze_maps Dir[__dir__+"/../../maps/maps-staging/*.imp"].sort
+  if ENV['NONSTAGING']
+    puts "\n" * 5
+    analyze_maps Dir[__dir__+"/../../maps/maps/*.imp"].sort
+  end
+end

data/bin/maps_debug_compilers

@@ -0,0 +1,58 @@
+#!/usr/bin/env ruby
+require "bundler/setup"
+require "interscript"
+require "interscript/compiler/ruby"
+require "interscript/compiler/javascript"
+
+# Differing maps as of writing this code:
+# - alalc-ara-Arab-Latn-1997
+# - alalc-aze-Arab-Latn-1997
+# - alalc-ell-Grek-Latn-2010
+# - bgnpcgn-bal-Arab-Latn-2008
+# - bgnpcgn-per-Arab-Latn-1958
+# - bgnpcgn-prs-Arab-Latn-2007
+# - bgnpcgn-pus-Arab-Latn-1968
+# - bgnpcgn-ukr-Cyrl-Latn-1965
+# - bgnpcgn-urd-Arab-Latn-2007
+# - odni-ara-Arab-Latn-2004
+# - odni-ara-Arab-Latn-2015
+# - stategeocadastre-ukr-Cyrl-Latn-1993
+# - ua-ukr-Cyrl-Latn-1996
+# - un-ara-Arab-Latn-2017
+# - un-ell-Grek-Latn-1987-phonetic
+# - un-prs-Arab-Latn-1967
+# - un-urd-Arab-Latn-1972
+
+map = ARGV[0]
+
+m = Interscript.parse(map)
+
+cr = Interscript::Compiler::Ruby
+mr = cr.(map, debug: true)
+cj = Interscript::Compiler::Javascript
+mj = cj.(map, debug: true)
+
+m.tests.data.each do |from, expected|
+  r = mr.(from)
+  j = mj.(from)
+
+  next if r == j
+
+  dr = cr.read_debug_data
+  dj = cj.read_debug_data
+
+  dr.zip(dj).each do |er, ej|
+    if er[0] != ej[0]
+      puts "First differing rule in map: #{er[1]}, stage #{er[2]}"
+      puts "Ruby: #{er[0]}, Javascript: #{ej[0]}"
+      puts "Rule:\n#{er[3].gsub(/^/, "  ")}"
+      puts "Generated Ruby code:\n#{er[4].gsub(/^/, "  ")}"
+      puts "Generated Javascript code:\n#{ej[4].gsub(/^/, "  ")}"
+      puts
+      break
+    end
+  end
+
+  cr.reset_debug_data
+  cj.reset_debug_data
+end

data/bin/maps_debug_ordering

@@ -0,0 +1,88 @@
+#!/usr/bin/env ruby
+ENV["INTERSCRIPT_STAGING"] = "1"
+require "bundler/setup"
+require "interscript"
+require "interscript/compiler/ruby"
+
+$map_name = ARGV[0]
+
+if $map_name
+  filelist  = [ __dir__+"/../../maps/maps-staging/#{$map_name}.imp" ]
+else
+  filelist = Dir[__dir__+"/../../maps/maps-staging/*.imp"].sort
+end
+
+
+# levenshtein distance algorithm for comparing string similarity
+def ld(s, t)
+  v0 = (0..t.length).to_a
+  v1 = []
+  #p v0
+
+  s.chars.each_with_index do |s_ch, i|
+    v1[0] = i + 1
+
+    t.chars.each_with_index do |t_ch, j|
+      cost = s_ch == t_ch ? 0 : 1
+      v1[j + 1] = [v1[j] + 1, v0[j + 1] + 1, v0[j] + cost].min
+    end
+    v0 = v1.dup
+    #p v1
+  end
+
+  v0[t.length]
+end
+
+
+def score_reordering( system, source, target )
+  interpreter = Marshal.load( Marshal.dump(  $interpreter ))
+  parallel = interpreter.map.stages[:main].children.select{|x| Interscript::Node::Group::Parallel === x}[0]
+  parallel.reorder_children(source,target)
+  interpreter.map.stages[:main].children[$parallel_idx].children = parallel.children.compact  #.reorder_children(source,target)
+  delta_sum = 0
+  system.tests.data.each do |from, expected|
+    result = interpreter.(from)
+    delta = ld(expected, result)
+    delta_sum += delta
+  end;
+  delta_sum
+end
+
+
+for i in filelist
+
+  begin
+    system_name = File.basename(i, ".imp")
+    puts "\ndebugging #{system_name}"
+
+    system = Interscript.parse(system_name);
+    if system.tests && system.tests.data && system.tests.data.length > 0
+
+    $interpreter = Interscript::Interpreter.new.compile(system);
+    $orig_parallel = $interpreter.map.stages[:main].children.select{|x| Interscript::Node::Group::Parallel === x}[0].dup;
+    $parallel_idx = $interpreter.map.stages[:main].children.each_with_index.select{|x,i| Interscript::Node::Group::Parallel === x}.map{|x,i| i}[0]
+    next if !$parallel_idx
+    starting_score = score_reordering(system, 0,0)
+    parallel_size = $interpreter.map.stages[:main].children.select{|x| Interscript::Node::Group::Parallel === x}[0].children.size
+
+    puts "starting_score = #{starting_score}"
+    source = parallel_size
+    while true
+      break if source ==0
+      target = 0
+      curr_score = score_reordering(system, source, target)
+      print "#{source} <-> #{target} = #{curr_score}; "
+
+      if curr_score < starting_score
+        puts ''
+        puts "#{source} <-> #{target}  changes score  from #{starting_score} to #{curr_score}" 
+        pp system.stages[:main].children[$parallel_idx].children[source]
+      end
+      source = source - 1
+    end
+    end
+  rescue Exception => e
+    puts e
+  end
+
+end