interscript 2.4.5 → 3.0.0

data/bin/maps_debug_step_by_step

@@ -1,44 +0,0 @@
-#!/usr/bin/env ruby
-ENV["INTERSCRIPT_STAGING"] = "1"
-require "bundler/setup"
-require "interscript"
-require "interscript/compiler/ruby"
-
-# This script has been written because there are some differences between platforms
-# (ie. windows vs linux) that we wish to find out more about
-
-if ARGV[0] == '-b'
-  require 'base64'
-  $b64 = []
-  ARGV.shift
-end
-
-map = ARGV[0]
-m = Interscript.parse(map)
-cr = Interscript::Compiler::Ruby
-mr = cr.(map, debug: true)
-
-m.tests.data.each_with_index do |(from, expected), idx|
-  r = mr.(from)
-
-  unless ARGV[1] && ARGV[1].split(",").any? { |i| i.to_i == idx }
-    if r == expected
-      cr.reset_debug_data
-      next
-    end
-  end
-
-  dr = cr.read_debug_data
-
-  if $b64
-    $b64 << [idx, dr]
-  else
-    pp [idx, dr]
-  end
-
-  cr.reset_debug_data
-end
-
-if $b64
-  puts Base64.encode64($b64.inspect)
-end

data/bin/maps_optimize_order

@@ -1,112 +0,0 @@
-#!/usr/bin/env ruby
-ENV["INTERSCRIPT_STAGING"] = "1"
-require "bundler/setup"
-require "interscript"
-require "interscript/compiler/ruby"
-
-$map_name = ARGV[0]
-
-if $map_name
-  filelist  = [ __dir__+"/../../maps/maps-staging/#{$map_name}.imp" ]
-else
-  filelist = Dir[__dir__+"/../../maps/maps-staging/*.imp"].sort
-end
-
-
-# levenshtein distance algorithm for comparing string similarity
-def ld(s, t)
-  v0 = (0..t.length).to_a
-  v1 = []
-  #p v0
-
-  s.chars.each_with_index do |s_ch, i|
-    v1[0] = i + 1
-
-    t.chars.each_with_index do |t_ch, j|
-      cost = s_ch == t_ch ? 0 : 1
-      v1[j + 1] = [v1[j] + 1, v0[j + 1] + 1, v0[j] + cost].min
-    end
-    v0 = v1.dup
-    #p v1
-  end
-
-  v0[t.length]
-end
-
-
-def score_order( system, order )
-  interpreter = Marshal.load( Marshal.dump(  $interpreter ))
-  parallel = interpreter.map.stages[:main].children.select{|x| Interscript::Node::Group::Parallel === x}[0]
-  parallel.apply_order(order)
-  interpreter.map.stages[:main].children[$parallel_idx].children = parallel.children.compact  #.reorder_children(source,target)
-  delta_sum = 0
-  errors = []
-  system.tests.data.each do |from, expected|
-    result = interpreter.(from)
-    delta = ld(expected, result)
-    errors << [expected, result] if delta != 0
-    delta_sum += delta
-  end;
-  [delta_sum, errors]
-end
-
-
-def mutate_order(order)
-  order2 = order.dup
-  a = rand(order2.size)
-  b = rand(order2.size)
-  order2[a], order2[b] = order2[b], order2[a]
-  order2
-end
-
-for i in filelist
-
-  begin
-    system_name = File.basename(i, ".imp")
-    puts "\ndebugging #{system_name}"
-
-    system = Interscript.parse(system_name);
-    if system.tests && system.tests.data && system.tests.data.length > 0
-
-    $interpreter = Interscript::Interpreter.new.compile(system);
-    $orig_parallel = $interpreter.map.stages[:main].children.select{|x| Interscript::Node::Group::Parallel === x}[0].dup;
-    $parallel_idx = $interpreter.map.stages[:main].children.each_with_index.select{|x,i| Interscript::Node::Group::Parallel === x}.map{|x,i| i}[0]
-    next if !$parallel_idx
-    starting_score, starting_errors = score_order(system,  $orig_parallel.children.size.times.to_a)
-    parallel_size = $interpreter.map.stages[:main].children.select{|x| Interscript::Node::Group::Parallel === x}[0].children.size
-
-    puts "starting_score = #{starting_score}"
-    best_score = starting_score
-    best_errors = starting_errors
-    curr_order = $orig_parallel.children.size.times.to_a.shuffle
-    #curr_order = [28, 308, 61, 87, 29, 147, 124, 22, 373, 186, 336, 19, 405, 387, 215, 209, 131, 30, 60, 343, 34, 380, 189, 53, 300, 286, 162, 31, 33, 218, 90, 51, 293, 226, 56, 305, 142, 102, 346, 222, 126, 338, 50, 52, 363, 144, 136, 16, 388, 221, 267, 63, 352, 365, 251, 78, 68, 328, 69, 12, 67, 317, 334, 94, 366, 412, 302, 243, 311, 318, 281, 274, 143, 236, 386, 135, 280, 167, 173, 291, 271, 309, 73, 20, 157, 331, 43, 242, 65, 351, 134, 151, 0, 285, 211, 417, 220, 179, 91, 353, 255, 141, 23, 104, 413, 409, 256, 326, 180, 140, 24, 348, 261, 5, 99, 47, 35, 358, 177, 123, 277, 396, 114, 213, 116, 188, 217, 249, 419, 120, 289, 330, 110, 118, 176, 113, 278, 127, 313, 55, 370, 48, 364, 171, 244, 407, 57, 371, 128, 196, 103, 202, 294, 239, 283, 299, 237, 394, 81, 230, 97, 46, 109, 337, 355, 240, 195, 100, 204, 389, 146, 153, 121, 183, 137, 159, 254, 231, 3, 101, 290, 323, 148, 359, 250, 25, 40, 219, 119, 169, 378, 282, 377, 238, 130, 279, 385, 58, 41, 115, 197, 382, 193, 225, 199, 6, 59, 208, 93, 138, 11, 15, 37, 38, 27, 354, 175, 411, 83, 89, 368, 216, 301, 168, 401, 84, 235, 333, 246, 284, 372, 155, 105, 339, 228, 342, 122, 161, 316, 145, 272, 321, 80, 315, 163, 107, 288, 227, 191, 306, 310, 76, 85, 132, 2, 320, 36, 13, 74, 233, 72, 381, 269, 70, 402, 86, 95, 111, 8, 383, 314, 10, 200, 203, 292, 241, 212, 374, 234, 369, 422, 42, 357, 18, 49, 214, 9, 156, 129, 258, 259, 190, 79, 367, 414, 201, 166, 270, 319, 332, 4, 184, 187, 164, 395, 325, 88, 245, 185, 71, 400, 275, 312, 324, 1, 224, 45, 205, 404, 260, 392, 253, 273, 416, 96, 408, 112, 349, 393, 345, 152, 329, 420, 410, 14, 361, 7, 257, 207, 194, 298, 17, 98, 340, 391, 399, 397, 82, 263, 376, 158, 327, 406, 265, 418, 322, 77, 92, 266, 262, 44, 360, 172, 403, 350, 66, 384, 247, 139, 181, 198, 248, 232, 32, 295, 106, 160, 287, 379, 341, 344, 421, 182, 375, 307, 415, 64, 75, 297, 125, 276, 223, 149, 26, 398, 303, 154, 133, 210, 150, 206, 174, 62, 170, 390, 54, 347, 39, 229, 178, 296, 108, 21, 165, 268, 264, 356, 304, 192, 252, 117, 335, 362]
-
-    best_order = curr_order.dup
-    while true
-      
-      curr_score, curr_errors = score_order(system, curr_order)
-      #print "#{source} <-> #{target} = #{curr_score}; "
-      puts Time.now.inspect
-      puts best_order.inspect
-      puts curr_score
-      puts best_errors.inspect
-      puts best_score
-      puts ''
-
-      if curr_score < best_score
-        puts ''
-        best_score = curr_score.dup
-        best_order = curr_order.dup
-        best_errors = curr_errors.dup
-      end
-      curr_order = mutate_order(best_order)
-    end
-    end
-  rescue Exception => e
-    puts e
-  end
-
-end
-# for this code to work sorting by max_length in interpreter.rb line 46  needs to be disabled
-#            #r.children.each do |i|
-#            r.children.sort_by{ |rule| -rule.max_length }.each do |i|

data/bin/maps_v1_analyze_regexps

@@ -1,45 +0,0 @@
-#!/usr/bin/env ruby
-
-require 'yaml'
-require 'fileutils'
-require 'regexp_parser'
-
-old_maps = Dir["../../interscript/maps/*.yaml"]
-
-$expr_classes = []
-$quantifiers = []
-old_maps.each do |old_map|
-  old_map_name = File.basename(old_map, ".yaml")
-  puts old_map
-  f = File.read(old_map)
-  yaml = YAML.load(f)
-  map_keys = yaml['map'].keys
-  puts map_keys
-  rs = yaml['map']['postrules']&.map{|h| h['pattern']} || []
-  rs += yaml['map']['characters']&.keys || []
-  rs.each do |regexp|
-    tree = Regexp::Parser.parse( regexp )
-
-    arr = tree.expressions
-    while true
-      new_arr = arr.map do |elem|
-        if elem.respond_to? :quantifier and elem.quantifier
-          $expr_classes << elem.quantifier
-        end
-        el = el.class == Class ? el : el.class
-        if elem.respond_to?(:expressions)
-          [el, elem.expressions]
-        else
-          el
-        end
-      end.flatten
-      break if new_arr == arr
-      arr = new_arr
-    end
-    $expr_classes += arr
-  end
-end;
-# $expressions.map{|elem|  elem.class.to_s=="Class" ? elem : elem.class}.tally
-
-pp $expr_classes.tally.sort_by{|k,v| -v}
-pp $quanitifiers.map{|q| q.text}.tally.sort_by{|k,v| -v}

data/bin/maps_v1_to_v2

@@ -1,426 +0,0 @@
-#!/usr/bin/env ruby
-# This is a helper script for porting Interscript v1 maps to v2 format. It won't
-# ever be able to port them completely, but it should help bootstrap the process.
-
-require 'bundler/setup'
-
-require 'yaml'
-require 'fileutils'
-
-Dir.chdir(__dir__ + "/../")
-FileUtils.rm_rf(Dir.glob("../maps/maps-staging/*"))
-#FileUtils.mkdir_p("../maps/maps-staging/")
-
-#old_maps = []
-old_maps = Dir["../../interscript/maps/*.yaml"]
-#old_maps = Dir["../../interscript/maps/alalc-aze-Arab-Latn-1997.yaml"]
-#old_maps = Dir["../../interscript/maps/mofa-jpn-Hrkt-Latn-1989.yaml"]
-
-
-ex_maps = Dir["../maps/maps/*.imp"]
-ex_map_names = ex_maps.map { |i| File.basename(i, ".imp") }
-
-
-require 'regexp_parser'
-
-require 'interscript/utils/regexp_converter.rb'
-
-
-def process_line( kkk, vvv, indent: 0)
-
-    parse_kkk = Regexp::Parser.parse(kkk, 'ruby/2.1')
-    tokens_kkk = process(parse_kkk)
-    # pp conv
-    root_hash = process_root(tokens_kkk)
-    # pp root_hash
-    # puts "vvv = #{vvv.inspect}"
-    if vvv.class == String
-      if vvv == '' or vvv =='""' or vvv == nil or vvv.include? '~'
-        root_hash[:to] = '""'
-      elsif vvv == '"'
-        root_hash[:to] = '"\""'
-      else
-        if vvv == "?" #alalc-ell-Grek-Latn-1997.imp un-ell-Grek-Latn-1987-phonetic have to "?"
-          root_hash[:to] = "?".inspect
-        #if root_hash[:from].to_s.include?('capture') or root_hash[:to] =~ /\\\\([0-9]+)/
-        else
-          parse_vvv = Regexp::Parser.parse(vvv)
-          tokens_vvv = process(parse_vvv)
-          string_vvv = stringify(tokens_vvv)
-          # puts string_vvv
-          root_hash[:to] = string_vvv
-          root_hash[:to] = 'upcase' if string_vvv.include? 'upcase'
-        end
-      end
-    elsif vvv.class == Array
-      root_hash[:to] = "any(#{vvv.inspect})"
-    else
-      root_hash[:to] = "\"\""
-      # puts "unknown class #{vvv.inspect}"
-    end
-
-    str = stringify_root(root_hash, indent: indent)
-    # puts str
-    # puts ""
-    str
-end
-
-
-old_maps.sort.each do |old_map|
-  old_map_name = File.basename(old_map, ".yaml")
-
-  if ex_map_names.include? old_map_name
-    puts "* Skipping #{old_map_name} as it's already ported"
-    next
-  end
-
-  print "* Converting #{old_map_name}."
-
-  f = File.read(old_map)
-  fl = f.split("\n")
-
-  md = []
-  tests = []
-  map = []
-  chain = nil
-
-  cur = md
-
-  bugnotes = false
-
-  fl.each do |i|
-    if i == '---'
-      # skip the first line
-    elsif i =~ /\A\s+|\A\z/
-      # continuation
-      if bugnotes
-        i = "#{i}"
-        md << i
-      else
-        cur << i
-      end
-    else
-      cmt = nil
-      i = i.sub(/(#.*?)\z/) do |j|
-        cmt = j
-        ""
-      end
-
-      # block begin or md
-      case i.strip
-      when "tests:"
-        cur = tests
-        bugnotes = false
-      when "map:"
-        cur = map
-        bugnotes = false
-      when "notes:"
-        md << "notes:"
-        bugnotes = true
-      when /\Achain:/
-        chain = i
-      else
-        cur << i
-      end
-
-      cur << cmt if cmt
-    end
-  end
-
-  print "."
-
-  newmd = []
-  aliasff = false
-  md.each do |i|
-    if i.strip == "alias:"
-      aliasff = true
-    elsif i !~ /\A\s+/
-      aliasff = false
-    end
-    newmd << i unless aliasff
-  end
-  md = newmd
-
-  print "."
-
-  newmd = md.map(&"  ".method(:+))
-            .join("\n")
-            .gsub("  note:", "  notes:")
-            .gsub("confirmation date:", "confirmation_date:")
-            #.gsub("special_rules:", '  - "special rules:"')
-            #.gsub("original_description:", "  # original description:")
-            #.gsub("original_notes:", '  - "original notes:"')
-            #.gsub("implementation_notes:", '  - "implementation notes:"')
-            .rstrip
-
-  new = "metadata {\n"
-  new << newmd
-  new << "\n}\n\n"
-
-  class MultilineError < StandardError; end
-
-  if tests.length > 0
-    new << "tests {\n"
-    cmt = ""
-
-    iter = 0
-    while iter < tests.length; begin
-      test = tests[iter]
-
-      if test =~ /\A\s*#/
-        new << "  " << test.strip << "\n"
-        iter += 1
-        next
-      end
-
-      re_source = /\A(?:  ){0,2}- source: (.*?)(\s*#.*?)?\z/m
-      re_expect = /\A(?:  ){0,3}expected:[ \t](.*?)(\s*#.*?)?\z/m
-
-      if test.rstrip.end_with?("|") ||
-         (test =~ /"/ && !test.rstrip.end_with?('"'))
-        while iter < tests.length
-          xtest = tests[iter+1]
-          break if xtest =~ re_source || xtest =~ re_expect
-          test << "\n" << (xtest||"")
-          iter += 1
-        end
-      end
-
-      case test
-      when re_source
-        new << "  test #{YAML.load($1).inspect}, "
-        cmt = $2 if $2
-      when re_expect
-        new << "#{YAML.load($1).inspect}".unicode_normalize
-        new << cmt
-        new << $2 if $2
-        new << "\n"
-        cmt = ""
-      when /\A\s*\z/
-        # empty line, ignore
-      else
-        new << "\n# BUG: #{test}\n"
-      end
-
-      iter += 1
-    rescue Psych::SyntaxError
-      p test
-    end; end
-
-    new << "}\n\n"
-  end
-
-  print "."
-
-  new << "# This map has been partially converted by the bin/maps_v1_to_v2 script\n"
-  new << "# The section below requires human attention. Remember to remove this\n"
-  new << "# comment and move the converted map to 'maps/' directory. Please also\n"
-  new << "# take note that the maps-staging directory will be cleaned up whenever\n"
-  new << "# you run the bin/maps_v1_to_v2 script. You should particularly be\n"
-  new << "# concerned about any regular expressions found in this file and about\n"
-  new << "# advanced expressions in parallel {} parts, and also about the order\n"
-  new << "# of particular parts of the stage.\n\n"
-
-  transcription = nil
-  title_case = nil
-  downcase = nil
-  inherit = nil
-
-  characters, rules, dictionary, postrules = [], [], [], []
-
-  cur = nil
-  indent = 0
-  stagedone = false
-  efini = proc do
-    if inherit
-      new << "\n"
-      inherit.each do |i|
-        new << "  run map.#{i}.stage.main\n"
-      end
-    end
-    new << "\n"
-    efini=proc{}
-  end
-  fini = proc{}
-  begn = proc { new << "stage {\n"; stagedone = true; begn=proc{} }
-
-  iter = -1
-  while iter+1 < map.length
-    iter += 1
-    i = map[iter]
-
-    cmt = nil
-    if i =~ /\A  [^\s#]/ || i =~ /\A    inherit:/
-      i.sub(/(#.*?)\z/) { cmt = $1 }
-      if cmt
-        new << " "*indent << cmt << "\n"
-      end
-
-      maybe_val = YAML.load(i.split(":").last)
-      case i.split(":").first.strip
-      when "inherit"
-        inherit = Array(maybe_val)
-        inherit = inherit.map do |inh|
-          short = inh.split("-")[2..3].join.downcase
-          new << "dependency #{inh.inspect}, as: #{short}\n"
-          short
-        end
-        new << "\n"
-        raise "Duplicate items" unless inherit.length == inherit.uniq.length
-      when "dictionary"
-        begn.()
-        fini.()
-        new << "  # DICTIONARY\n"
-        new << "  parallel {\n"
-        indent = 4
-        cur = dictionary
-        fini = proc{new << "  }\n\n";indent = 2}
-      when "rules"
-        begn.()
-        fini.()
-        efini.()
-        new << "  # RULES\n"
-        indent = 2
-        cur = rules
-        fini = proc{new << "\n"}
-      when "characters"
-        begn.()
-        fini.()
-        efini.()
-        new << "  # CHARACTERS\n"
-        new << "  parallel {\n"
-        indent = 4
-        cur = characters
-        fini = proc{new << "  }\n\n";indent = 2}
-      when "postrules"
-        begn.()
-        fini.()
-        efini.()
-        new << "  # POSTRULES\n"
-        indent = 2
-        cur = postrules
-        fini = proc{new << "\n"}
-      when "downcase"
-        downcase = maybe_val
-      when "title_case"
-        title_case = maybe_val
-      when "transcription"
-        transcription = maybe_val
-      # Those we will ignore for now
-      when "word_separator", "segmentation", "character_separator", "map"
-      # Those are bugs
-      when "title-case"
-      else
-        p i
-      end
-    else
-      cmt = ""
-      i = i.sub(/(#.*?)\z/) { cmt << $1; "" }
-      #new << " "*indent << i.strip << "\n"
-      if i.strip == ""
-        new << " "*indent << cmt << "\n"
-        next
-      end
-
-      case cur.object_id
-      when nil.object_id
-        raise "Unexpected line #{i}"
-      when characters.object_id, dictionary.object_id
-        k,v = i.split(":", 2).map(&:strip)
-        if !v || v == ""
-          v = ""
-          # Load array
-          iter2 = iter + 1
-          while iter2 < map.length
-            i2 = map[iter2]
-            break unless i2.strip =~ /\A-/
-            i2 = i2.sub(/(#.*?)\z/) { cmt << $1; "" }
-            v << "\n" << i2
-            iter2 += 1
-          end
-          iter = iter2 - 1
-        end
-
-        kk,vv = YAML.load(k), YAML.load(v)
-
-        kkk,vvv = kk.inspect, vv.inspect
-        if vv.class == Array
-          if vv.all? { |z| z.length == 1 }
-            vvv = "any(" + vv.join.inspect + ")"
-          else
-            vvv = "any(" + vv.inspect + ")"
-          end
-        elsif vv.class == NilClass
-          vvv = "none"
-        end
-        kkk,vvv = kk.dup,vv.dup
-        # This worked due to use of regexps... it should remove 1 slash.
-        kkk = kkk.gsub("\\\\u", "\\\\u")
-        kkk = kkk.gsub("\\\\U", "\\\\u")
-
-        new << process_line( kkk,vvv, indent: indent )
-      when rules.object_id, postrules.object_id
-        if i.strip =~ /\A- pattern\s*:/
-          _, k = i.split(":", 2)
-          ii = map[iter+1]
-          ii = ii.sub(/(#.*?)\z/) { cmt << $1; "" }
-          if ii.strip == ""
-            iter += 1
-            ii = map[iter+1]
-            ii = ii.sub(/(#.*?)\z/) { cmt << $1; "" }
-          end
-          if ii.strip =~ /\Aresult\s*:/
-            _, v = ii.split(":", 2)
-          else
-            raise "Unexpected(2): #{ii.strip.inspect}"
-          end
-          iter += 1
-        else
-          raise "Unexpected(1): #{i.strip.inspect}"
-        end
-
-        kk,vv = YAML.load(k), YAML.load(v)
-        # kkk,vvv = kk.inspect, vv.inspect
-        kkk,vvv = kk.dup,vv.dup
-
-        new << process_line( kkk,vvv, indent: indent )
-      end
-
-      if cmt != ""
-        new << " " << cmt << "\n"
-      else
-        new << "\n"
-      end
-    end
-  end
-
-  fini.()
-
-  unless stagedone
-    new << "stage {\n"
-    efini.()
-  end
-
-  new << "  title_case\n" if title_case
-  new << "  downcase\n" if downcase
-
-  new << "}\n\n"
-
-  if chain
-    new << "# This map is chained and probably depends on seq2seq:\n"
-    new << "# #{chain}\n\n"
-  end
-
-  if transcription
-    new << "# This map contains transcription and probably depends on seq2seq:\n"
-    new << "# transcription: #{transcription}\n\n"
-  end
-
-  new = new.gsub(/ +$/, '') # Cleanup trailing whitespaces
-
-  #new << map.join("\n")
-
-  File.write("../maps/maps-staging/#{old_map_name}.imp", new)
-
-  puts " done!"
-end

data/bin/set_version

@@ -1,16 +0,0 @@
-#!/usr/bin/env ruby
-ver = ARGV[0]
-part = ARGV[1]
-
-rubyver = File.read(rubyfile = __dir__+"/../lib/interscript/version.rb")
-jsver   = File.read(jsfile   = __dir__+"/../../js/package.json")
-mapsver = File.read(mapsfile = __dir__+"/../../maps/interscript-maps.gemspec")
-
-rubyver = rubyver.gsub(/(VERSION = ")([0-9a-z.-]*)(")/,                "\\1#{ver}\\3")
-jsver   = jsver.gsub(/("version": ")([0-9a-z.-]*)(")/,                 "\\1#{ver}\\3")
-mapsver = mapsver.gsub(/(INTERSCRIPT_MAPS_VERSION=")([0-9a-z.-]*)(")/, "\\1#{ver}\\3")
-
-File.write(rubyfile, rubyver) if %w[all ruby].include? part
-File.write(jsfile,   jsver) if %w[all js].include? part
-File.write(mapsfile, mapsver) if %w[all maps].include? part
-

data/bin/setup

@@ -1,8 +0,0 @@
-#!/usr/bin/env bash
-set -euo pipefail
-IFS=$'\n\t'
-set -vx
-
-bundle install
-
-# Do any other automated setup that you need to do here

data/requirements.txt

@@ -1 +0,0 @@
-torch