interscript 2.4.5 → 3.0.0

data/.github/workflows/release.yml

@@ -1,68 +0,0 @@
-name: release
-
-on:
-  push:
-    tags:
-      - 'v*'
-
-jobs:
-  release:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v2
-        with:
-          repository: interscript/interscript
-
-      - name: Run bootstrap script
-        run: ruby bootstrap.rb
-
-      - uses: ruby/setup-ruby@v1
-        with:
-          ruby-version: '3.0'
-
-      - uses: actions/setup-node@v1
-        with:
-          node-version: 14.x
-
-      # For now let's install without secryst, as we don't necessarily need it.
-      # We may need to change it once we start to depend on secryst maps.
-      - name: Install bundle
-        working-directory: ./ruby
-        run: bundle install --jobs 4 --retry 3 --with jsexec --without secryst
-
-      - name: Test Ruby package
-        working-directory: ./ruby
-        run: bundle exec rake
-
-      - name: Test JS package
-        working-directory: ./js
-        run: npm install && npm run prepareMaps && npm test
-
-      - name: Publish to rubygems.org
-        env:
-          RUBYGEMS_API_KEY: ${{secrets.INTERSCRIPT_RUBYGEMS_API_KEY}}
-        run: |
-          gem install gem-release
-          mkdir -p ~/.gem
-          touch ~/.gem/credentials
-          cat > ~/.gem/credentials << EOF
-          ---
-          :rubygems_api_key: ${RUBYGEMS_API_KEY}
-          EOF
-          chmod 0600 ~/.gem/credentials
-          pushd ruby
-          git status
-          gem release
-          popd
-
-      # Let's keep it commented out for now. Please uncomment it once you are ready with
-      # interscript-api to support Interscript v2.
-      #
-      #- name: Trigger interscript-api
-      #  uses: peter-evans/repository-dispatch@v1
-      #  with:
-      #    token: ${{ secrets.INTERSCRIPT_CI_PAT_TOKEN }}
-      #    repository: interscript/interscript-api
-      #    event-type: ${{ github.repository }}
-      #    client-payload: '{ "ref": "${{ github.ref }}" }'

data/.gitignore

@@ -1,79 +0,0 @@
-*.gem
-*.rbc
-/.config
-/coverage/
-/InstalledFiles
-/pkg/
-/spec/reports/
-/spec/examples.txt
-/test/tmp/
-/test/version_tmp/
-/tmp/
-/Gemfile.lock
-
-# Used by dotenv library to load environment variables.
-# .env
-
-# Ignore Byebug command history file.
-.byebug_history
-
-## Specific to RubyMotion:
-.dat*
-.repl_history
-/build/
-*.bridgesupport
-/build-iPhoneOS/
-/build-iPhoneSimulator/
-
-## Specific to RubyMotion (use of CocoaPods):
-#
-# We recommend against adding the Pods directory to your .gitignore. However
-# you should judge for yourself, the pros and cons are mentioned at:
-# https://guides.cocoapods.org/using/using-cocoapods.html#should-i-check-the-pods-directory-into-source-control
-#
-# vendor/Pods/
-
-## Documentation cache and generated files:
-/.yardoc/
-/_yardoc/
-/doc/
-/rdoc/
-
-## Environment normalization:
-/.bundle/
-/vendor/bundle
-/lib/bundler/man/
-
-# for a library or gem, you might want to ignore these files since the code is
-# intended to run in multiple environments; otherwise, check them in:
-# Gemfile.lock
-# .ruby-version
-# .ruby-gemset
-
-# unless supporting rvm < 1.11.0 or doing something fancy, ignore this:
-/.rvmrc
-
-# Don't bundle generated files
-/visualizations
-/json
-/vis_json
-/auth_json
-/metadata.json
-/compiled
-
-# Used by RuboCop. Remote config files pulled in from inherit_from directive.
-# .rubocop-https?--*
-
-#misic
-.idea
-.bundle/.bundle/
-/.yardoc
-/_yardoc/
-/coverage/
-/doc/
-/pkg/
-/spec/reports/
-/tmp/
-
-# rspec failure tracking
-.rspec_status

data/.rspec

@@ -1,3 +0,0 @@
---format documentation
---color
---require spec_helper

data/Gemfile

@@ -1,38 +0,0 @@
-source "https://rubygems.org"
-
-# Specify your gem's dependencies in interscript.gemspec
-gemspec
-
-gem "rake", "~> 12.0"
-gem "rspec", "~> 3.0"
-
-gem "interscript-maps", path: "../maps"
-
-unless Gem::Version.new(RUBY_VERSION) < Gem::Version.new("2.7")
-  group :secryst do
-    if File.exist? "../../secryst"
-      gem "secryst", path: "../../secryst"
-    else
-      gem "secryst"
-    end
-  end
-end
-
-gem 'regexp_parser'
-
-unless ENV["SKIP_JS"]
-  group :jsexec do
-    gem 'mini_racer'
-  end
-end
-
-group :rababa do
-  gem 'rababa', "~> 0.1.1"
-end
-
-gem 'pry'
-
-gem 'iso-639-data'
-gem 'iso-15924'
-
-gem 'simplecov', require: false, group: :test

data/Rakefile

@@ -1,139 +0,0 @@
-require "bundler/setup"
-require "bundler/gem_tasks"
-require "rspec/core/rake_task"
-
-RSpec::Core::RakeTask.new(:spec)
-
-task :compile, [:compiler, :target] do |t, args|
-  require "interscript"
-
-  compiler, ext = case args[:compiler]
-  when "ruby"
-    require "interscript/compiler/ruby"
-    [Interscript::Compiler::Ruby, "rb"]
-  when "javascript"
-    require "interscript/compiler/javascript"
-    [Interscript::Compiler::Javascript, "js"]
-  end
-
-  FileUtils.mkdir_p(args[:target])
-
-  maplist = {}
-
-  maps = Interscript.maps
-  maps = Interscript.exclude_maps(maps, compiler: compiler, platform: false)
-  
-  maps.each do |map|
-    code = compiler.(map).code
-    File.write(args[:target] + "/" + map + "." + ext, code)
-    maplist[map] = nil
-  end
-
-  Interscript.maps(libraries: true).each do |map|
-    code = compiler.(map).code
-    File.write(args[:target] + "/" + map + "." + ext, code)
-  end
-
-  File.write(args[:target] + "/index.json", maplist.to_json)
-end
-
-task :generate_visualization_html do
-  require "fileutils"
-  require "interscript"
-  require "interscript/visualize"
-
-  FileUtils.rm_rf(dir = __dir__+"/visualizations/")
-  FileUtils.mkdir_p(dir)
-
-  Interscript.maps.each do |map|
-    html = Interscript::Visualize.(map)
-    File.write(dir+map+".html", html)
-  end
-end
-
-task :generate_metadata_json do
-  require "fileutils"
-  require "json"
-  require "interscript"
-  require "interscript/compiler/javascript"
-
-  FileUtils.rm_rf(file = __dir__+"/metadata.json")
-
-  hash = Interscript.maps.map do |map|
-    parsed_map = Interscript.parse(map)
-    md = parsed_map.metadata.to_hash
-    md["test"] = parsed_map.tests&.data&.first
-    md["skip_js"] = Interscript.exclude_maps([map],
-                      compiler: Interscript::Compiler::Javascript,
-                      platform: false,
-                    ).empty?
-    [map, md]
-  end.to_h
-
-  File.write(file, JSON.pretty_generate(hash))
-end
-
-task :generate_json do
-  require "fileutils"
-  require "json"
-  require "interscript"
-
-  FileUtils.rm_rf(dir = __dir__+"/json/")
-  FileUtils.mkdir_p(dir)
-
-  (Interscript.maps + Interscript.maps(libraries: true)).each do |map|
-    json = JSON.pretty_generate(Interscript.parse(map).to_hash)
-    File.write(dir+map+".json", json)
-  end
-end
-
-task :generate_visualization_json do
-  require "fileutils"
-  require "interscript"
-  require "json"
-  require "interscript/visualize"
-
-  FileUtils.rm_rf(dir = __dir__+"/vis_json/")
-  FileUtils.mkdir_p(dir)
-
-  (Interscript.maps + Interscript.maps(libraries: true)).each do |map_name|
-    map = Interscript.parse(map_name)
-    map.stages.each do |stage_name, stage|
-      json = JSON.pretty_generate(stage.to_visualization_array(map))
-      File.write(dir+map_name+"_#{stage_name}.json", json)
-    end
-  end
-end
-
-task :generate_authority_json do
-  require "interscript"
-  require "json"
-  require "iso-639-data"
-
-  FileUtils.rm_rf(dir = __dir__+"/auth_json/")
-  FileUtils.mkdir_p(dir)
-
-  %w[iso icao din].each do |auth|
-    out = Interscript.maps.select do |map_name|
-      map_name.start_with? "#{auth}-"
-    end.sort.map do |map_name|
-      map = Interscript.parse(map_name)
-      tests = map.tests&.data&.first(2)&.transpose || []
-      std, lang = map.metadata.data[:language].split(':')
-
-      {
-        lang: std.end_with?("-3") ? Iso639Data.iso_639_3[lang]['Ref_Name'] : Iso639Data.iso_639_2[lang]['eng'],
-        isoName: map.metadata.data[:name],
-        systemName: map_name,
-        samples: tests[0] || [],
-        english: [],
-        result: []
-      }
-    end
-
-    json = JSON.pretty_generate(out)
-    File.write(dir+auth+".json", json)
-  end
-end
-
-task :default => :spec

data/bin/console

@@ -1,10 +0,0 @@
-#!/usr/bin/env ruby
-
-require "bundler/setup"
-require "interscript"
-
-require "interscript/utils/helpers"
-include Interscript::Utils::Helpers
-
-require "pry"
-Pry.start

data/bin/interscript

@@ -1,5 +0,0 @@
-#!/usr/bin/env ruby
-
-require 'bundler/setup'
-$LOAD_PATH << "#{__dir__}/../lib"
-load "#{__dir__}/../exe/interscript"

data/bin/maps_analyze_staging

@@ -1,168 +0,0 @@
-#!/usr/bin/env ruby
-ENV["INTERSCRIPT_STAGING"] = "1"
-require "bundler/setup"
-require "interscript"
-require "interscript/compiler/ruby"
-
-$map_name = ARGV[0]
-
-# surpress warnings
-def warn(*)
-end
-
-
-def debug_single_map(from, expected, result, result2)
-  puts ""
-  # puts $subs_array.inspect
-  # puts $subs_matches.inspect
-  puts from.inspect
-  puts from.chars.inspect
-  puts from.codepoints.map{|cp| '\\u%04x' % cp}.join(' ')
-  puts expected
-  puts result
-  puts result2
-  return nil unless $subs_matches
-  first_mismatch = $subs_matches.select{|match_data|
-    exp = expected[match_data[:begin]...match_data[:end]]
-    res = result[match_data[:begin]...match_data[:end]]
-    exp != res
-  }.first
-
-  # puts $subs_matches.inspect
-  # puts first_mismatch.inspect
-  output_string = ''
-  last_end = 0
-  $subs_matches.each do |match_data|
-    substr = from[match_data[:begin]...match_data[:end]]
-    if match_data[:begin] != last_end
-      output_string << '_' * (match_data[:begin] - last_end)
-    end
-    output_string << match_data[:result]
-
-    puts ''
-    puts "ERROR " if match_data == first_mismatch
-    indent = ' ' * match_data[:begin]
-    # puts indent + substr
-
-
-
-    matched_rule = $subs_array[match_data[:idx]]
-
-    if match_data == first_mismatch
-      puts indent + matched_rule.inspect + "   (#{matched_rule[0].codepoints.map{|cp| '\\u%04x' % cp}.join(' ')})"
-    else
-      puts indent + matched_rule.inspect + "    (#{matched_rule[0].codepoints.map{|cp| '\\u%04x' % cp}.join(' ')})"
-    end
-
-    puts from[0,match_data[:end]]
-
-    puts expected[0,output_string.size]
-    puts output_string
-    last_end = match_data[:end]
-
-
-    #return if match_data == first_mismatch
-  end
-  puts ''
-end
-
-def analyze_maps( filelist )
-  cache = {}
-
-  good_total = 0
-  almost_total = 0
-  bad_total = 0
-  passed_total = 0
-  for i in filelist
-    # used at the bottom
-    raw_lines = File.open(i).read.split("\n")
-    rules_line = raw_lines.index "  # RULES"
-    characters_line = raw_lines.index "  # CHARACTERS"
-    postrules_line = raw_lines.index "  # POSTRULES"
-
-    system_name = File.basename(i, ".imp")
-
-    print "#{system_name}: "
-
-    exceptions = []
-
-    # begin
-      system = Interscript.parse(system_name)
-    # rescue Exception => e
-    #   exceptions << e
-    # end
-    # puts $using_tree.inspect
-    if exceptions.length == 0 && system.tests && system.tests.data && system.tests.data.length > 0
-      good = 0
-      bad = 0
-      almost = 0
-
-      system.tests.data.each do |from,expected|
-        testname = from[0...300].gsub("\n", " / ")
-
-        result = Interscript.transliterate(system_name, from, cache, compiler: Interscript::Interpreter)
-        result2 = Interscript.transliterate(system_name, from, cache, compiler: Interscript::Compiler::Ruby)
-        if result == nil or result2 == nil
-          print '0'
-          bad += 1
-        elsif result != result2
-          # Compiler / Interpreter bug!
-          print "!"
-          bad += 1
-        elsif result == expected
-          print "."
-          good += 1
-        elsif result.unicode_normalize == expected.unicode_normalize
-          print ","
-          almost += 1
-        elsif result.unicode_normalize.chomp == expected.unicode_normalize.chomp
-          print ";"
-          almost += 1
-        else
-          print "#"
-          debug_single_map(from, expected, result, result2) if $map_name
-          bad += 1
-        end
-      rescue Exception => e
-        print "%"
-        bad += 1
-        exceptions << e
-      end
-
-      if bad == 0
-        passed_total += 1
-        puts " all tests passed!"
-        `git mv #{i} #{i.sub("-staging", "")}`   if i.include? '-staging'
-      else
-        puts " (#{good}/#{good+almost+bad})"
-      end
-      good_total += good
-      almost_total += almost
-      bad_total += bad
-    elsif exceptions.length != 0
-    else
-      puts "no tests!"
-    end
-    puts "warning: postrules before rules\n\n" if (postrules_line||10000000) < (rules_line||0)
-    puts "warning: postrules before characters\n\n" if (postrules_line||10000000) < (characters_line||0)
-
-    Array(exceptions.first).compact.each do |e|
-      puts e.full_message
-    end
-  end
-  puts " #{good_total} (#{good_total+almost_total})/#{good_total+almost_total+bad_total}"
-  puts " #{passed_total} passed!"
-end
-
-#analyze_maps Dir[__dir__+"/../../maps/maps/*.imp"].sort
-if $map_name
-  $DEBUG_RE = true
-  analyze_maps Dir[__dir__+"/../../maps/maps/#{$map_name}.imp"].sort
-  analyze_maps Dir[__dir__+"/../../maps/maps-staging/#{$map_name}.imp"].sort
-else
-  analyze_maps Dir[__dir__+"/../../maps/maps-staging/*.imp"].sort
-  if ENV['NONSTAGING']
-    puts "\n" * 5
-    analyze_maps Dir[__dir__+"/../../maps/maps/*.imp"].sort
-  end
-end

data/bin/maps_debug_compilers

@@ -1,58 +0,0 @@
-#!/usr/bin/env ruby
-require "bundler/setup"
-require "interscript"
-require "interscript/compiler/ruby"
-require "interscript/compiler/javascript"
-
-# Differing maps as of writing this code:
-# - alalc-ara-Arab-Latn-1997
-# - alalc-aze-Arab-Latn-1997
-# - alalc-ell-Grek-Latn-2010
-# - bgnpcgn-bal-Arab-Latn-2008
-# - bgnpcgn-per-Arab-Latn-1958
-# - bgnpcgn-prs-Arab-Latn-2007
-# - bgnpcgn-pus-Arab-Latn-1968
-# - bgnpcgn-ukr-Cyrl-Latn-1965
-# - bgnpcgn-urd-Arab-Latn-2007
-# - odni-ara-Arab-Latn-2004
-# - odni-ara-Arab-Latn-2015
-# - stategeocadastre-ukr-Cyrl-Latn-1993
-# - ua-ukr-Cyrl-Latn-1996
-# - un-ara-Arab-Latn-2017
-# - un-ell-Grek-Latn-1987-phonetic
-# - un-prs-Arab-Latn-1967
-# - un-urd-Arab-Latn-1972
-
-map = ARGV[0]
-
-m = Interscript.parse(map)
-
-cr = Interscript::Compiler::Ruby
-mr = cr.(map, debug: true)
-cj = Interscript::Compiler::Javascript
-mj = cj.(map, debug: true)
-
-m.tests.data.each do |from, expected|
-  r = mr.(from)
-  j = mj.(from)
-
-  next if r == j
-
-  dr = cr.read_debug_data
-  dj = cj.read_debug_data
-
-  dr.zip(dj).each do |er, ej|
-    if er[0] != ej[0]
-      puts "First differing rule in map: #{er[1]}, stage #{er[2]}"
-      puts "Ruby: #{er[0]}, Javascript: #{ej[0]}"
-      puts "Rule:\n#{er[3].gsub(/^/, "  ")}"
-      puts "Generated Ruby code:\n#{er[4].gsub(/^/, "  ")}"
-      puts "Generated Javascript code:\n#{ej[4].gsub(/^/, "  ")}"
-      puts
-      break
-    end
-  end
-
-  cr.reset_debug_data
-  cj.reset_debug_data
-end

data/bin/maps_debug_ordering

@@ -1,88 +0,0 @@
-#!/usr/bin/env ruby
-ENV["INTERSCRIPT_STAGING"] = "1"
-require "bundler/setup"
-require "interscript"
-require "interscript/compiler/ruby"
-
-$map_name = ARGV[0]
-
-if $map_name
-  filelist  = [ __dir__+"/../../maps/maps-staging/#{$map_name}.imp" ]
-else
-  filelist = Dir[__dir__+"/../../maps/maps-staging/*.imp"].sort
-end
-
-
-# levenshtein distance algorithm for comparing string similarity
-def ld(s, t)
-  v0 = (0..t.length).to_a
-  v1 = []
-  #p v0
-
-  s.chars.each_with_index do |s_ch, i|
-    v1[0] = i + 1
-
-    t.chars.each_with_index do |t_ch, j|
-      cost = s_ch == t_ch ? 0 : 1
-      v1[j + 1] = [v1[j] + 1, v0[j + 1] + 1, v0[j] + cost].min
-    end
-    v0 = v1.dup
-    #p v1
-  end
-
-  v0[t.length]
-end
-
-
-def score_reordering( system, source, target )
-  interpreter = Marshal.load( Marshal.dump(  $interpreter ))
-  parallel = interpreter.map.stages[:main].children.select{|x| Interscript::Node::Group::Parallel === x}[0]
-  parallel.reorder_children(source,target)
-  interpreter.map.stages[:main].children[$parallel_idx].children = parallel.children.compact  #.reorder_children(source,target)
-  delta_sum = 0
-  system.tests.data.each do |from, expected|
-    result = interpreter.(from)
-    delta = ld(expected, result)
-    delta_sum += delta
-  end;
-  delta_sum
-end
-
-
-for i in filelist
-
-  begin
-    system_name = File.basename(i, ".imp")
-    puts "\ndebugging #{system_name}"
-
-    system = Interscript.parse(system_name);
-    if system.tests && system.tests.data && system.tests.data.length > 0
-
-    $interpreter = Interscript::Interpreter.new.compile(system);
-    $orig_parallel = $interpreter.map.stages[:main].children.select{|x| Interscript::Node::Group::Parallel === x}[0].dup;
-    $parallel_idx = $interpreter.map.stages[:main].children.each_with_index.select{|x,i| Interscript::Node::Group::Parallel === x}.map{|x,i| i}[0]
-    next if !$parallel_idx
-    starting_score = score_reordering(system, 0,0)
-    parallel_size = $interpreter.map.stages[:main].children.select{|x| Interscript::Node::Group::Parallel === x}[0].children.size
-
-    puts "starting_score = #{starting_score}"
-    source = parallel_size
-    while true
-      break if source ==0
-      target = 0
-      curr_score = score_reordering(system, source, target)
-      print "#{source} <-> #{target} = #{curr_score}; "
-
-      if curr_score < starting_score
-        puts ''
-        puts "#{source} <-> #{target}  changes score  from #{starting_score} to #{curr_score}" 
-        pp system.stages[:main].children[$parallel_idx].children[source]
-      end
-      source = source - 1
-    end
-    end
-  rescue Exception => e
-    puts e
-  end
-
-end

data/bin/maps_debug_ruby_compile

@@ -1,24 +0,0 @@
-#!/usr/bin/env ruby
-ENV["INTERSCRIPT_STAGING"] = "1"
-require "bundler/setup"
-require "interscript"
-require "interscript/compiler/ruby"
-
-# Compile a given map with the Ruby compiler for debugging purposes
-
-if ARGV[0] == '-b'
-  require 'base64'
-  $b64 = true
-  ARGV.shift
-end
-
-map = ARGV[0]
-m = Interscript.parse(map)
-cr = Interscript::Compiler::Ruby
-mr = cr.(map)
-
-if $b64
-  puts Base64.encode64(mr.code)
-else
-  puts mr.code
-end