interscript 0.1.9 → 2.0.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7813dba0d0cc7493ed3b9279c61283c8d305f1e05584a44aa700e9b72acb2f06
-  data.tar.gz: f4c87e24d7c2719b4f358198967d55e0c17d8aaac354311cb9eecc800a592b2d
+  metadata.gz: 4838c7e99a2ba57c43e7a1e7d67330e14e6e5b56eba6840ed980429ce88259a4
+  data.tar.gz: 07d690a917b58a06e1cb22395eb7f3c18e3e0e323ba778503c13dd2abe4750e3
 SHA512:
-  metadata.gz: 8e23000fe8fb016dec9351241787608c892d7af48682259c10345ce417f94a626ea06a5bc8b1e7a3f084da8c0502d13b96bd06c9a53da31530f5c487b06fe4e9
-  data.tar.gz: b2d8cb122b2c1bbb2d989d832802a007a60ecc6d0f2984e323a983ec94ae9505664b98e0d9c33e0b2711f5fa31f4e657a720648dcb572915f69006a1719a9610
+  metadata.gz: 40356145a903228279aeafb9146dfcd3312c32b6a40a84de7d1c6140c99be3fcfb44825f05f9873fd7b3a4aef85047f5efb8434bae0a6cb229734823a1eb1a71
+  data.tar.gz: 20d872918cc9e02333d988f2f502e9a18176fc1e038e1ce5514fd6a47c8572f0f97c6a88fec9a95e1fa08530f87a26cfc7df92cb6c2a9e1414b50aedf8ee077e

data/.gitignore

@@ -0,0 +1,11 @@
+/.bundle/
+/.yardoc
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+
+# rspec failure tracking
+.rspec_status

data/.rspec

@@ -0,0 +1,3 @@
+--format documentation
+--color
+--require spec_helper

data/Gemfile

@@ -0,0 +1,29 @@
+source "https://rubygems.org"
+
+# Specify your gem's dependencies in interscript.gemspec
+gemspec
+
+gem "rake", "~> 12.0"
+gem "rspec", "~> 3.0"
+
+gem "interscript-maps", path: "../maps"
+
+unless Gem::Version.new(RUBY_VERSION) < Gem::Version.new("2.7")
+  group :secryst do
+    if File.exist? "../../secryst"
+      gem "secryst", path: "../../secryst"
+    else
+      gem "secryst"
+    end
+  end
+end
+
+gem 'regexp_parser'
+
+unless ENV["SKIP_JS"]
+  group :jsexec do
+    gem 'mini_racer'
+  end
+end
+
+gem 'simplecov', require: false, group: :test

data/LICENSE.adoc

@@ -0,0 +1,31 @@
+= Licenses & Copyright
+
+This license file adheres to the formatting guidelines of
+https://github.com/nevir/readable-licenses[readable-licenses].
+
+
+== Ribose BSD 2-Clause License
+
+Copyright (c) 2019-, https://www.ribose.com[Ribose Inc].
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification,
+are permitted provided that the following conditions are met:
+
+1.  Redistributions of source code must retain the above copyright notice,
+    this list of conditions and the following disclaimer.
+
+2.  Redistributions in binary form must reproduce the above copyright notice,
+    this list of conditions and the following disclaimer in the documentation
+    and/or other materials provided with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

data/README.md

@@ -0,0 +1,3 @@
+# Interscript
+
+Documentation is available directory higher in file README.adoc.

data/Rakefile

@@ -0,0 +1,53 @@
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new(:spec)
+
+task :compile, [:compiler, :target] do |t, args|
+  require "bundler/setup"
+  require "interscript"
+
+  compiler, ext = case args[:compiler]
+  when "ruby"
+    require "interscript/compiler/ruby"
+    [Interscript::Compiler::Ruby, "rb"]
+  when "javascript"
+    require "interscript/compiler/javascript"
+    [Interscript::Compiler::Javascript, "js"]
+  end
+
+  FileUtils.mkdir_p(args[:target])
+
+  maplist = {}
+
+  Interscript.maps.each do |map|
+    code = compiler.(map).code
+    File.write(args[:target] + "/" + map + "." + ext, code)
+    maplist[map] = nil
+  end
+
+  Interscript.maps(libraries: true).each do |map|
+    code = compiler.(map).code
+    File.write(args[:target] + "/" + map + "." + ext, code)
+  end
+
+  File.write(args[:target] + "/index.json", maplist.to_json)
+end
+
+task :version, [:ver] do |t, ver|
+  ver = ver[:ver]
+
+  rubyver = File.read(rubyfile = __dir__+"/lib/interscript/version.rb")
+  jsver   = File.read(jsfile   = __dir__+"/../js/package.json")
+  mapsver = File.read(mapsfile = __dir__+"/../maps/interscript-maps.gemspec")
+
+  rubyver = rubyver.gsub(/(VERSION = ")([0-9a-z.-]*)(")/,                "\\1#{ver}\\3")
+  jsver   = jsver.gsub(/("version": ")([0-9a-z.-]*)(")/,                 "\\1#{ver}\\3")
+  mapsver = mapsver.gsub(/(INTERSCRIPT_MAPS_VERSION=")([0-9a-z.-]*)(")/, "\\1#{ver}\\3")
+
+  File.write(rubyfile, rubyver)
+  File.write(jsfile,   jsver)
+  File.write(mapsfile, mapsver)
+end
+
+task :default => :spec

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "interscript"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start(__FILE__)

data/bin/interscript

@@ -0,0 +1,5 @@
+#!/usr/bin/env ruby
+
+require 'bundler/setup'
+$LOAD_PATH << "#{__dir__}/../lib"
+load "#{__dir__}/../exe/interscript"

data/bin/maps_analyze_staging

@@ -0,0 +1,168 @@
+#!/usr/bin/env ruby
+ENV["INTERSCRIPT_STAGING"] = "1"
+require "bundler/setup"
+require "interscript"
+require "interscript/compiler/ruby"
+
+$map_name = ARGV[0]
+
+# surpress warnings
+def warn(*)
+end
+
+
+def debug_single_map(from, expected, result, result2)
+  puts ""
+  # puts $subs_array.inspect
+  # puts $subs_matches.inspect
+  puts from.inspect
+  puts from.chars.inspect
+  puts from.codepoints.map{|cp| '\\u%04x' % cp}.join(' ')
+  puts expected
+  puts result
+  puts result2
+  return nil unless $subs_matches
+  first_mismatch = $subs_matches.select{|match_data|
+    exp = expected[match_data[:begin]...match_data[:end]]
+    res = result[match_data[:begin]...match_data[:end]]
+    exp != res
+  }.first
+
+  # puts $subs_matches.inspect
+  # puts first_mismatch.inspect
+  output_string = ''
+  last_end = 0
+  $subs_matches.each do |match_data|
+    substr = from[match_data[:begin]...match_data[:end]]
+    if match_data[:begin] != last_end
+      output_string << '_' * (match_data[:begin] - last_end)
+    end
+    output_string << match_data[:result]
+
+    puts ''
+    puts "ERROR " if match_data == first_mismatch
+    indent = ' ' * match_data[:begin]
+    # puts indent + substr
+
+
+
+    matched_rule = $subs_array[match_data[:idx]]
+
+    if match_data == first_mismatch
+      puts indent + matched_rule.inspect + "   (#{matched_rule[0].codepoints.map{|cp| '\\u%04x' % cp}.join(' ')})"
+    else
+      puts indent + matched_rule.inspect + "    (#{matched_rule[0].codepoints.map{|cp| '\\u%04x' % cp}.join(' ')})"
+    end
+
+    puts from[0,match_data[:end]]
+
+    puts expected[0,output_string.size]
+    puts output_string
+    last_end = match_data[:end]
+
+
+    #return if match_data == first_mismatch
+  end
+  puts ''
+end
+
+def analyze_maps( filelist )
+  cache = {}
+
+  good_total = 0
+  almost_total = 0
+  bad_total = 0
+  passed_total = 0
+  for i in filelist
+    # used at the bottom
+    raw_lines = File.open(i).read.split("\n")
+    rules_line = raw_lines.index "  # RULES"
+    characters_line = raw_lines.index "  # CHARACTERS"
+    postrules_line = raw_lines.index "  # POSTRULES"
+
+    system_name = File.basename(i, ".imp")
+
+    print "#{system_name}: "
+
+    exceptions = []
+
+    # begin
+      system = Interscript.parse(system_name)
+    # rescue Exception => e
+    #   exceptions << e
+    # end
+    # puts $using_tree.inspect
+    if exceptions.length == 0 && system.tests && system.tests.data && system.tests.data.length > 0
+      good = 0
+      bad = 0
+      almost = 0
+
+      system.tests.data.each do |from,expected|
+        testname = from[0...300].gsub("\n", " / ")
+
+        result = Interscript.transliterate(system_name, from, cache, compiler: Interscript::Interpreter)
+        result2 = Interscript.transliterate(system_name, from, cache, compiler: Interscript::Compiler::Ruby)
+        if result == nil or result2 == nil
+          print '0'
+          bad += 1
+        elsif result != result2
+          # Compiler / Interpreter bug!
+          print "!"
+          bad += 1
+        elsif result == expected
+          print "."
+          good += 1
+        elsif result.unicode_normalize == expected.unicode_normalize
+          print ","
+          almost += 1
+        elsif result.unicode_normalize.chomp == expected.unicode_normalize.chomp
+          print ";"
+          almost += 1
+        else
+          print "#"
+          debug_single_map(from, expected, result, result2) if $map_name
+          bad += 1
+        end
+      rescue Exception => e
+        print "%"
+        bad += 1
+        exceptions << e
+      end
+
+      if bad == 0
+        passed_total += 1
+        puts " all tests passed!"
+        `git mv #{i} #{i.sub("-staging", "")}`   if i.include? '-staging'
+      else
+        puts " (#{good}/#{good+almost+bad})"
+      end
+      good_total += good
+      almost_total += almost
+      bad_total += bad
+    elsif exceptions.length != 0
+    else
+      puts "no tests!"
+    end
+    puts "warning: postrules before rules\n\n" if (postrules_line||10000000) < (rules_line||0)
+    puts "warning: postrules before characters\n\n" if (postrules_line||10000000) < (characters_line||0)
+
+    Array(exceptions.first).compact.each do |e|
+      puts e.full_message
+    end
+  end
+  puts " #{good_total} (#{good_total+almost_total})/#{good_total+almost_total+bad_total}"
+  puts " #{passed_total} passed!"
+end
+
+#analyze_maps Dir[__dir__+"/../../maps/maps/*.imp"].sort
+if $map_name
+  $DEBUG_RE = true
+  analyze_maps Dir[__dir__+"/../../maps/maps/#{$map_name}.imp"].sort
+  analyze_maps Dir[__dir__+"/../../maps/maps-staging/#{$map_name}.imp"].sort
+else
+  analyze_maps Dir[__dir__+"/../../maps/maps-staging/*.imp"].sort
+  if ENV['NONSTAGING']
+    puts "\n" * 5
+    analyze_maps Dir[__dir__+"/../../maps/maps/*.imp"].sort
+  end
+end

data/bin/maps_debug_compilers

@@ -0,0 +1,58 @@
+#!/usr/bin/env ruby
+require "bundler/setup"
+require "interscript"
+require "interscript/compiler/ruby"
+require "interscript/compiler/javascript"
+
+# Differing maps as of writing this code:
+# - alalc-ara-Arab-Latn-1997
+# - alalc-aze-Arab-Latn-1997
+# - alalc-ell-Grek-Latn-2010
+# - bgnpcgn-bal-Arab-Latn-2008
+# - bgnpcgn-per-Arab-Latn-1958
+# - bgnpcgn-prs-Arab-Latn-2007
+# - bgnpcgn-pus-Arab-Latn-1968
+# - bgnpcgn-ukr-Cyrl-Latn-1965
+# - bgnpcgn-urd-Arab-Latn-2007
+# - odni-ara-Arab-Latn-2004
+# - odni-ara-Arab-Latn-2015
+# - stategeocadastre-ukr-Cyrl-Latn-1993
+# - ua-ukr-Cyrl-Latn-1996
+# - un-ara-Arab-Latn-2017
+# - un-ell-Grek-Latn-1987-phonetic
+# - un-prs-Arab-Latn-1967
+# - un-urd-Arab-Latn-1972
+
+map = ARGV[0]
+
+m = Interscript.parse(map)
+
+cr = Interscript::Compiler::Ruby
+mr = cr.(map, debug: true)
+cj = Interscript::Compiler::Javascript
+mj = cj.(map, debug: true)
+
+m.tests.data.each do |from, expected|
+  r = mr.(from)
+  j = mj.(from)
+
+  next if r == j
+
+  dr = cr.read_debug_data
+  dj = cj.read_debug_data
+
+  dr.zip(dj).each do |er, ej|
+    if er[0] != ej[0]
+      puts "First differing rule in map: #{er[1]}, stage #{er[2]}"
+      puts "Ruby: #{er[0]}, Javascript: #{ej[0]}"
+      puts "Rule:\n#{er[3].gsub(/^/, "  ")}"
+      puts "Generated Ruby code:\n#{er[4].gsub(/^/, "  ")}"
+      puts "Generated Javascript code:\n#{ej[4].gsub(/^/, "  ")}"
+      puts
+      break
+    end
+  end
+
+  cr.reset_debug_data
+  cj.reset_debug_data
+end

data/bin/maps_debug_ordering

@@ -0,0 +1,88 @@
+#!/usr/bin/env ruby
+ENV["INTERSCRIPT_STAGING"] = "1"
+require "bundler/setup"
+require "interscript"
+require "interscript/compiler/ruby"
+
+$map_name = ARGV[0]
+
+if $map_name
+  filelist  = [ __dir__+"/../../maps/maps-staging/#{$map_name}.imp" ]
+else
+  filelist = Dir[__dir__+"/../../maps/maps-staging/*.imp"].sort
+end
+
+
+# levenshtein distance algorithm for comparing string similarity
+def ld(s, t)
+  v0 = (0..t.length).to_a
+  v1 = []
+  #p v0
+
+  s.chars.each_with_index do |s_ch, i|
+    v1[0] = i + 1
+
+    t.chars.each_with_index do |t_ch, j|
+      cost = s_ch == t_ch ? 0 : 1
+      v1[j + 1] = [v1[j] + 1, v0[j + 1] + 1, v0[j] + cost].min
+    end
+    v0 = v1.dup
+    #p v1
+  end
+
+  v0[t.length]
+end
+
+
+def score_reordering( system, source, target )
+  interpreter = Marshal.load( Marshal.dump(  $interpreter ))
+  parallel = interpreter.map.stages[:main].children.select{|x| Interscript::Node::Group::Parallel === x}[0]
+  parallel.reorder_children(source,target)
+  interpreter.map.stages[:main].children[$parallel_idx].children = parallel.children.compact  #.reorder_children(source,target)
+  delta_sum = 0
+  system.tests.data.each do |from, expected|
+    result = interpreter.(from)
+    delta = ld(expected, result)
+    delta_sum += delta
+  end;
+  delta_sum
+end
+
+
+for i in filelist
+
+  begin
+    system_name = File.basename(i, ".imp")
+    puts "\ndebugging #{system_name}"
+
+    system = Interscript.parse(system_name);
+    if system.tests && system.tests.data && system.tests.data.length > 0
+
+    $interpreter = Interscript::Interpreter.new.compile(system);
+    $orig_parallel = $interpreter.map.stages[:main].children.select{|x| Interscript::Node::Group::Parallel === x}[0].dup;
+    $parallel_idx = $interpreter.map.stages[:main].children.each_with_index.select{|x,i| Interscript::Node::Group::Parallel === x}.map{|x,i| i}[0]
+    next if !$parallel_idx
+    starting_score = score_reordering(system, 0,0)
+    parallel_size = $interpreter.map.stages[:main].children.select{|x| Interscript::Node::Group::Parallel === x}[0].children.size
+
+    puts "starting_score = #{starting_score}"
+    source = parallel_size
+    while true
+      break if source ==0
+      target = 0
+      curr_score = score_reordering(system, source, target)
+      print "#{source} <-> #{target} = #{curr_score}; "
+
+      if curr_score < starting_score
+        puts ''
+        puts "#{source} <-> #{target}  changes score  from #{starting_score} to #{curr_score}" 
+        pp system.stages[:main].children[$parallel_idx].children[source]
+      end
+      source = source - 1
+    end
+    end
+  rescue Exception => e
+    puts e
+  end
+
+end