interscript 0.1.7 → 2.1.0b1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 82fae2b248d9c86139b7f188da2ac72699696c9768e54a4510a6f1af2b933dc9
-  data.tar.gz: 3e2cc24b8d33f5a8ed0f8b475e4d109049439221274ecda9ee1b9c7743896e07
+  metadata.gz: 5fc3d229c727e71662e0fd33a1e40424015f2c4caeea2b592a464554a3610ba1
+  data.tar.gz: f254317afa4d3d2372572df7b4751edcfb0fe2f24acc2ffc61978b613d062656
 SHA512:
-  metadata.gz: 5f9925a97d17f0433446a898f63d18869a73e92f9975d8259c916dae242fc5b15ff93dd6d3c28ca2ff5bcbda29489d4ea59af26d4e16f2e1416d701354a6f6e2
-  data.tar.gz: 77321c4a1001cabda8cc057950682037b637176fd05637ce78a6ea698d8c55e238e6bbcc850dda97828c719afbdae1192a8e7a93fd2307a9d52196342fb015f5
+  metadata.gz: 69bf1648abe50c5a27dd3a1c3d96edfd011c790b35fcfdba78ea0dd85c2339497c44add107571b05de4c042889f4b1fe9edb766a31a0613bd1b7d526bc4257de
+  data.tar.gz: 155a553e164a7ce32cf13ead6beaffdeb449937087d86fb605ec7813f818a1985fdc36624db8669f86b2e0bea5377ea853d6e960ed821c811cc016416b857537

data/.gitignore

@@ -0,0 +1,11 @@
+/.bundle/
+/.yardoc
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+
+# rspec failure tracking
+.rspec_status

data/.rspec

@@ -0,0 +1,3 @@
+--format documentation
+--color
+--require spec_helper

data/Gemfile

@@ -0,0 +1,29 @@
+source "https://rubygems.org"
+
+# Specify your gem's dependencies in interscript.gemspec
+gemspec
+
+gem "rake", "~> 12.0"
+gem "rspec", "~> 3.0"
+
+gem "interscript-maps", path: "../maps"
+
+unless Gem::Version.new(RUBY_VERSION) < Gem::Version.new("2.7")
+  group :secryst do
+    if File.exist? "../../secryst"
+      gem "secryst", path: "../../secryst"
+    else
+      gem "secryst"
+    end
+  end
+end
+
+gem 'regexp_parser'
+
+unless ENV["SKIP_JS"]
+  group :jsexec do
+    gem 'mini_racer'
+  end
+end
+
+gem 'simplecov', require: false, group: :test

data/LICENSE.adoc

@@ -0,0 +1,31 @@
+= Licenses & Copyright
+
+This license file adheres to the formatting guidelines of
+https://github.com/nevir/readable-licenses[readable-licenses].
+
+
+== Ribose BSD 2-Clause License
+
+Copyright (c) 2019-, https://www.ribose.com[Ribose Inc].
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification,
+are permitted provided that the following conditions are met:
+
+1.  Redistributions of source code must retain the above copyright notice,
+    this list of conditions and the following disclaimer.
+
+2.  Redistributions in binary form must reproduce the above copyright notice,
+    this list of conditions and the following disclaimer in the documentation
+    and/or other materials provided with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

data/README.md

@@ -0,0 +1,3 @@
+# Interscript
+
+Documentation is available directory higher in file README.adoc.

data/Rakefile

@@ -0,0 +1,116 @@
+require "bundler/setup"
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new(:spec)
+
+task :compile, [:compiler, :target] do |t, args|
+  require "interscript"
+
+  compiler, ext = case args[:compiler]
+  when "ruby"
+    require "interscript/compiler/ruby"
+    [Interscript::Compiler::Ruby, "rb"]
+  when "javascript"
+    require "interscript/compiler/javascript"
+    [Interscript::Compiler::Javascript, "js"]
+  end
+
+  FileUtils.mkdir_p(args[:target])
+
+  maplist = {}
+
+  Interscript.maps.each do |map|
+    code = compiler.(map).code
+    File.write(args[:target] + "/" + map + "." + ext, code)
+    maplist[map] = nil
+  end
+
+  Interscript.maps(libraries: true).each do |map|
+    code = compiler.(map).code
+    File.write(args[:target] + "/" + map + "." + ext, code)
+  end
+
+  File.write(args[:target] + "/index.json", maplist.to_json)
+end
+
+task :version, [:ver] do |t, ver|
+  ver = ver[:ver]
+
+  rubyver = File.read(rubyfile = __dir__+"/lib/interscript/version.rb")
+  jsver   = File.read(jsfile   = __dir__+"/../js/package.json")
+  mapsver = File.read(mapsfile = __dir__+"/../maps/interscript-maps.gemspec")
+
+  rubyver = rubyver.gsub(/(VERSION = ")([0-9a-z.-]*)(")/,                "\\1#{ver}\\3")
+  jsver   = jsver.gsub(/("version": ")([0-9a-z.-]*)(")/,                 "\\1#{ver}\\3")
+  mapsver = mapsver.gsub(/(INTERSCRIPT_MAPS_VERSION=")([0-9a-z.-]*)(")/, "\\1#{ver}\\3")
+
+  File.write(rubyfile, rubyver)
+  File.write(jsfile,   jsver)
+  File.write(mapsfile, mapsver)
+end
+
+task :generate_visualization_html do
+  require "fileutils"
+  require "interscript"
+  require "interscript/visualize"
+
+  FileUtils.rm_rf(dir = __dir__+"/visualizations/")
+  FileUtils.mkdir_p(dir)
+
+  Interscript.maps.each do |map|
+    html = Interscript::Visualize.(map)
+    File.write(dir+map+".html", html)
+  end
+end
+
+task :generate_metadata_json do
+  require "fileutils"
+  require "json"
+  require "interscript"
+
+  FileUtils.rm_rf(file = __dir__+"/metadata.json")
+
+  hash = Interscript.maps.map do |map|
+    parsed_map = Interscript.parse(map)
+    md = parsed_map.metadata.to_hash
+    md["test"] = parsed_map.tests&.data&.first
+    [map, md]
+  end.to_h
+
+  File.write(file, JSON.pretty_generate(hash))
+end
+
+task :generate_json do
+  require "fileutils"
+  require "json"
+  require "interscript"
+
+  FileUtils.rm_rf(dir = __dir__+"/json/")
+  FileUtils.mkdir_p(dir)
+
+  (Interscript.maps + Interscript.maps(libraries: true)).each do |map|
+    json = JSON.pretty_generate(Interscript.parse(map).to_hash)
+    File.write(dir+map+".json", json)
+  end
+end
+
+task :generate_visualization_json do
+  require "fileutils"
+  require "interscript"
+  require "json"
+  require "interscript/visualize"
+
+  FileUtils.rm_rf(dir = __dir__+"/vis_json/")
+  FileUtils.mkdir_p(dir)
+
+  (Interscript.maps + Interscript.maps(libraries: true)).each do |map_name|
+    map = Interscript.parse(map_name)
+    map.stages.each do |stage_name, stage|
+      json = JSON.pretty_generate(stage.to_visualization_array(map))
+      File.write(dir+map_name+"_#{stage_name}.json", json)
+    end
+  end
+end
+
+task :default => :spec

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "interscript"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start(__FILE__)

data/bin/interscript

@@ -0,0 +1,5 @@
+#!/usr/bin/env ruby
+
+require 'bundler/setup'
+$LOAD_PATH << "#{__dir__}/../lib"
+load "#{__dir__}/../exe/interscript"

data/bin/maps_analyze_staging

@@ -0,0 +1,168 @@
+#!/usr/bin/env ruby
+ENV["INTERSCRIPT_STAGING"] = "1"
+require "bundler/setup"
+require "interscript"
+require "interscript/compiler/ruby"
+
+$map_name = ARGV[0]
+
+# surpress warnings
+def warn(*)
+end
+
+
+def debug_single_map(from, expected, result, result2)
+  puts ""
+  # puts $subs_array.inspect
+  # puts $subs_matches.inspect
+  puts from.inspect
+  puts from.chars.inspect
+  puts from.codepoints.map{|cp| '\\u%04x' % cp}.join(' ')
+  puts expected
+  puts result
+  puts result2
+  return nil unless $subs_matches
+  first_mismatch = $subs_matches.select{|match_data|
+    exp = expected[match_data[:begin]...match_data[:end]]
+    res = result[match_data[:begin]...match_data[:end]]
+    exp != res
+  }.first
+
+  # puts $subs_matches.inspect
+  # puts first_mismatch.inspect
+  output_string = ''
+  last_end = 0
+  $subs_matches.each do |match_data|
+    substr = from[match_data[:begin]...match_data[:end]]
+    if match_data[:begin] != last_end
+      output_string << '_' * (match_data[:begin] - last_end)
+    end
+    output_string << match_data[:result]
+
+    puts ''
+    puts "ERROR " if match_data == first_mismatch
+    indent = ' ' * match_data[:begin]
+    # puts indent + substr
+
+
+
+    matched_rule = $subs_array[match_data[:idx]]
+
+    if match_data == first_mismatch
+      puts indent + matched_rule.inspect + "   (#{matched_rule[0].codepoints.map{|cp| '\\u%04x' % cp}.join(' ')})"
+    else
+      puts indent + matched_rule.inspect + "    (#{matched_rule[0].codepoints.map{|cp| '\\u%04x' % cp}.join(' ')})"
+    end
+
+    puts from[0,match_data[:end]]
+
+    puts expected[0,output_string.size]
+    puts output_string
+    last_end = match_data[:end]
+
+
+    #return if match_data == first_mismatch
+  end
+  puts ''
+end
+
+def analyze_maps( filelist )
+  cache = {}
+
+  good_total = 0
+  almost_total = 0
+  bad_total = 0
+  passed_total = 0
+  for i in filelist
+    # used at the bottom
+    raw_lines = File.open(i).read.split("\n")
+    rules_line = raw_lines.index "  # RULES"
+    characters_line = raw_lines.index "  # CHARACTERS"
+    postrules_line = raw_lines.index "  # POSTRULES"
+
+    system_name = File.basename(i, ".imp")
+
+    print "#{system_name}: "
+
+    exceptions = []
+
+    # begin
+      system = Interscript.parse(system_name)
+    # rescue Exception => e
+    #   exceptions << e
+    # end
+    # puts $using_tree.inspect
+    if exceptions.length == 0 && system.tests && system.tests.data && system.tests.data.length > 0
+      good = 0
+      bad = 0
+      almost = 0
+
+      system.tests.data.each do |from,expected|
+        testname = from[0...300].gsub("\n", " / ")
+
+        result = Interscript.transliterate(system_name, from, cache, compiler: Interscript::Interpreter)
+        result2 = Interscript.transliterate(system_name, from, cache, compiler: Interscript::Compiler::Ruby)
+        if result == nil or result2 == nil
+          print '0'
+          bad += 1
+        elsif result != result2
+          # Compiler / Interpreter bug!
+          print "!"
+          bad += 1
+        elsif result == expected
+          print "."
+          good += 1
+        elsif result.unicode_normalize == expected.unicode_normalize
+          print ","
+          almost += 1
+        elsif result.unicode_normalize.chomp == expected.unicode_normalize.chomp
+          print ";"
+          almost += 1
+        else
+          print "#"
+          debug_single_map(from, expected, result, result2) if $map_name
+          bad += 1
+        end
+      rescue Exception => e
+        print "%"
+        bad += 1
+        exceptions << e
+      end
+
+      if bad == 0
+        passed_total += 1
+        puts " all tests passed!"
+        `git mv #{i} #{i.sub("-staging", "")}`   if i.include? '-staging'
+      else
+        puts " (#{good}/#{good+almost+bad})"
+      end
+      good_total += good
+      almost_total += almost
+      bad_total += bad
+    elsif exceptions.length != 0
+    else
+      puts "no tests!"
+    end
+    puts "warning: postrules before rules\n\n" if (postrules_line||10000000) < (rules_line||0)
+    puts "warning: postrules before characters\n\n" if (postrules_line||10000000) < (characters_line||0)
+
+    Array(exceptions.first).compact.each do |e|
+      puts e.full_message
+    end
+  end
+  puts " #{good_total} (#{good_total+almost_total})/#{good_total+almost_total+bad_total}"
+  puts " #{passed_total} passed!"
+end
+
+#analyze_maps Dir[__dir__+"/../../maps/maps/*.imp"].sort
+if $map_name
+  $DEBUG_RE = true
+  analyze_maps Dir[__dir__+"/../../maps/maps/#{$map_name}.imp"].sort
+  analyze_maps Dir[__dir__+"/../../maps/maps-staging/#{$map_name}.imp"].sort
+else
+  analyze_maps Dir[__dir__+"/../../maps/maps-staging/*.imp"].sort
+  if ENV['NONSTAGING']
+    puts "\n" * 5
+    analyze_maps Dir[__dir__+"/../../maps/maps/*.imp"].sort
+  end
+end

data/bin/maps_debug_compilers

@@ -0,0 +1,58 @@
+#!/usr/bin/env ruby
+require "bundler/setup"
+require "interscript"
+require "interscript/compiler/ruby"
+require "interscript/compiler/javascript"
+
+# Differing maps as of writing this code:
+# - alalc-ara-Arab-Latn-1997
+# - alalc-aze-Arab-Latn-1997
+# - alalc-ell-Grek-Latn-2010
+# - bgnpcgn-bal-Arab-Latn-2008
+# - bgnpcgn-per-Arab-Latn-1958
+# - bgnpcgn-prs-Arab-Latn-2007
+# - bgnpcgn-pus-Arab-Latn-1968
+# - bgnpcgn-ukr-Cyrl-Latn-1965
+# - bgnpcgn-urd-Arab-Latn-2007
+# - odni-ara-Arab-Latn-2004
+# - odni-ara-Arab-Latn-2015
+# - stategeocadastre-ukr-Cyrl-Latn-1993
+# - ua-ukr-Cyrl-Latn-1996
+# - un-ara-Arab-Latn-2017
+# - un-ell-Grek-Latn-1987-phonetic
+# - un-prs-Arab-Latn-1967
+# - un-urd-Arab-Latn-1972
+
+map = ARGV[0]
+
+m = Interscript.parse(map)
+
+cr = Interscript::Compiler::Ruby
+mr = cr.(map, debug: true)
+cj = Interscript::Compiler::Javascript
+mj = cj.(map, debug: true)
+
+m.tests.data.each do |from, expected|
+  r = mr.(from)
+  j = mj.(from)
+
+  next if r == j
+
+  dr = cr.read_debug_data
+  dj = cj.read_debug_data
+
+  dr.zip(dj).each do |er, ej|
+    if er[0] != ej[0]
+      puts "First differing rule in map: #{er[1]}, stage #{er[2]}"
+      puts "Ruby: #{er[0]}, Javascript: #{ej[0]}"
+      puts "Rule:\n#{er[3].gsub(/^/, "  ")}"
+      puts "Generated Ruby code:\n#{er[4].gsub(/^/, "  ")}"
+      puts "Generated Javascript code:\n#{ej[4].gsub(/^/, "  ")}"
+      puts
+      break
+    end
+  end
+
+  cr.reset_debug_data
+  cj.reset_debug_data
+end