interscript 0.1.4 → 2.0.5

data/bin/maps_debug_ruby_compile

@@ -0,0 +1,24 @@
+#!/usr/bin/env ruby
+ENV["INTERSCRIPT_STAGING"] = "1"
+require "bundler/setup"
+require "interscript"
+require "interscript/compiler/ruby"
+
+# Compile a given map with the Ruby compiler for debugging purposes
+
+if ARGV[0] == '-b'
+  require 'base64'
+  $b64 = true
+  ARGV.shift
+end
+
+map = ARGV[0]
+m = Interscript.parse(map)
+cr = Interscript::Compiler::Ruby
+mr = cr.(map)
+
+if $b64
+  puts Base64.encode64(mr.code)
+else
+  puts mr.code
+end

data/bin/maps_debug_step_by_step

@@ -0,0 +1,44 @@
+#!/usr/bin/env ruby
+ENV["INTERSCRIPT_STAGING"] = "1"
+require "bundler/setup"
+require "interscript"
+require "interscript/compiler/ruby"
+
+# This script has been written because there are some differences between platforms
+# (ie. windows vs linux) that we wish to find out more about
+
+if ARGV[0] == '-b'
+  require 'base64'
+  $b64 = []
+  ARGV.shift
+end
+
+map = ARGV[0]
+m = Interscript.parse(map)
+cr = Interscript::Compiler::Ruby
+mr = cr.(map, debug: true)
+
+m.tests.data.each_with_index do |(from, expected), idx|
+  r = mr.(from)
+
+  unless ARGV[1] && ARGV[1].split(",").any? { |i| i.to_i == idx }
+    if r == expected
+      cr.reset_debug_data
+      next
+    end
+  end
+
+  dr = cr.read_debug_data
+
+  if $b64
+    $b64 << [idx, dr]
+  else
+    pp [idx, dr]
+  end
+
+  cr.reset_debug_data
+end
+
+if $b64
+  puts Base64.encode64($b64.inspect)
+end

data/bin/maps_optimize_order

@@ -0,0 +1,112 @@
+#!/usr/bin/env ruby
+ENV["INTERSCRIPT_STAGING"] = "1"
+require "bundler/setup"
+require "interscript"
+require "interscript/compiler/ruby"
+
+$map_name = ARGV[0]
+
+if $map_name
+  filelist  = [ __dir__+"/../../maps/maps-staging/#{$map_name}.imp" ]
+else
+  filelist = Dir[__dir__+"/../../maps/maps-staging/*.imp"].sort
+end
+
+
+# levenshtein distance algorithm for comparing string similarity
+def ld(s, t)
+  v0 = (0..t.length).to_a
+  v1 = []
+  #p v0
+
+  s.chars.each_with_index do |s_ch, i|
+    v1[0] = i + 1
+
+    t.chars.each_with_index do |t_ch, j|
+      cost = s_ch == t_ch ? 0 : 1
+      v1[j + 1] = [v1[j] + 1, v0[j + 1] + 1, v0[j] + cost].min
+    end
+    v0 = v1.dup
+    #p v1
+  end
+
+  v0[t.length]
+end
+
+
+def score_order( system, order )
+  interpreter = Marshal.load( Marshal.dump(  $interpreter ))
+  parallel = interpreter.map.stages[:main].children.select{|x| Interscript::Node::Group::Parallel === x}[0]
+  parallel.apply_order(order)
+  interpreter.map.stages[:main].children[$parallel_idx].children = parallel.children.compact  #.reorder_children(source,target)
+  delta_sum = 0
+  errors = []
+  system.tests.data.each do |from, expected|
+    result = interpreter.(from)
+    delta = ld(expected, result)
+    errors << [expected, result] if delta != 0
+    delta_sum += delta
+  end;
+  [delta_sum, errors]
+end
+
+
+def mutate_order(order)
+  order2 = order.dup
+  a = rand(order2.size)
+  b = rand(order2.size)
+  order2[a], order2[b] = order2[b], order2[a]
+  order2
+end
+
+for i in filelist
+
+  begin
+    system_name = File.basename(i, ".imp")
+    puts "\ndebugging #{system_name}"
+
+    system = Interscript.parse(system_name);
+    if system.tests && system.tests.data && system.tests.data.length > 0
+
+    $interpreter = Interscript::Interpreter.new.compile(system);
+    $orig_parallel = $interpreter.map.stages[:main].children.select{|x| Interscript::Node::Group::Parallel === x}[0].dup;
+    $parallel_idx = $interpreter.map.stages[:main].children.each_with_index.select{|x,i| Interscript::Node::Group::Parallel === x}.map{|x,i| i}[0]
+    next if !$parallel_idx
+    starting_score, starting_errors = score_order(system,  $orig_parallel.children.size.times.to_a)
+    parallel_size = $interpreter.map.stages[:main].children.select{|x| Interscript::Node::Group::Parallel === x}[0].children.size
+
+    puts "starting_score = #{starting_score}"
+    best_score = starting_score
+    best_errors = starting_errors
+    curr_order = $orig_parallel.children.size.times.to_a.shuffle
+    #curr_order = [28, 308, 61, 87, 29, 147, 124, 22, 373, 186, 336, 19, 405, 387, 215, 209, 131, 30, 60, 343, 34, 380, 189, 53, 300, 286, 162, 31, 33, 218, 90, 51, 293, 226, 56, 305, 142, 102, 346, 222, 126, 338, 50, 52, 363, 144, 136, 16, 388, 221, 267, 63, 352, 365, 251, 78, 68, 328, 69, 12, 67, 317, 334, 94, 366, 412, 302, 243, 311, 318, 281, 274, 143, 236, 386, 135, 280, 167, 173, 291, 271, 309, 73, 20, 157, 331, 43, 242, 65, 351, 134, 151, 0, 285, 211, 417, 220, 179, 91, 353, 255, 141, 23, 104, 413, 409, 256, 326, 180, 140, 24, 348, 261, 5, 99, 47, 35, 358, 177, 123, 277, 396, 114, 213, 116, 188, 217, 249, 419, 120, 289, 330, 110, 118, 176, 113, 278, 127, 313, 55, 370, 48, 364, 171, 244, 407, 57, 371, 128, 196, 103, 202, 294, 239, 283, 299, 237, 394, 81, 230, 97, 46, 109, 337, 355, 240, 195, 100, 204, 389, 146, 153, 121, 183, 137, 159, 254, 231, 3, 101, 290, 323, 148, 359, 250, 25, 40, 219, 119, 169, 378, 282, 377, 238, 130, 279, 385, 58, 41, 115, 197, 382, 193, 225, 199, 6, 59, 208, 93, 138, 11, 15, 37, 38, 27, 354, 175, 411, 83, 89, 368, 216, 301, 168, 401, 84, 235, 333, 246, 284, 372, 155, 105, 339, 228, 342, 122, 161, 316, 145, 272, 321, 80, 315, 163, 107, 288, 227, 191, 306, 310, 76, 85, 132, 2, 320, 36, 13, 74, 233, 72, 381, 269, 70, 402, 86, 95, 111, 8, 383, 314, 10, 200, 203, 292, 241, 212, 374, 234, 369, 422, 42, 357, 18, 49, 214, 9, 156, 129, 258, 259, 190, 79, 367, 414, 201, 166, 270, 319, 332, 4, 184, 187, 164, 395, 325, 88, 245, 185, 71, 400, 275, 312, 324, 1, 224, 45, 205, 404, 260, 392, 253, 273, 416, 96, 408, 112, 349, 393, 345, 152, 329, 420, 410, 14, 361, 7, 257, 207, 194, 298, 17, 98, 340, 391, 399, 397, 82, 263, 376, 158, 327, 406, 265, 418, 322, 77, 92, 266, 262, 44, 360, 172, 403, 350, 66, 384, 247, 139, 181, 198, 248, 232, 32, 295, 106, 160, 287, 379, 341, 344, 421, 182, 375, 307, 415, 64, 75, 297, 125, 276, 223, 149, 26, 398, 303, 154, 133, 210, 150, 206, 174, 62, 170, 390, 54, 347, 39, 229, 178, 296, 108, 21, 165, 268, 264, 356, 304, 192, 252, 117, 335, 362]
+
+    best_order = curr_order.dup
+    while true
+      
+      curr_score, curr_errors = score_order(system, curr_order)
+      #print "#{source} <-> #{target} = #{curr_score}; "
+      puts Time.now.inspect
+      puts best_order.inspect
+      puts curr_score
+      puts best_errors.inspect
+      puts best_score
+      puts ''
+
+      if curr_score < best_score
+        puts ''
+        best_score = curr_score.dup
+        best_order = curr_order.dup
+        best_errors = curr_errors.dup
+      end
+      curr_order = mutate_order(best_order)
+    end
+    end
+  rescue Exception => e
+    puts e
+  end
+
+end
+# for this code to work sorting by max_length in interpreter.rb line 46  needs to be disabled
+#            #r.children.each do |i|
+#            r.children.sort_by{ |rule| -rule.max_length }.each do |i|

data/bin/maps_v1_analyze_regexps

@@ -0,0 +1,45 @@
+#!/usr/bin/env ruby
+
+require 'yaml'
+require 'fileutils'
+require 'regexp_parser'
+
+old_maps = Dir["../../interscript/maps/*.yaml"]
+
+$expr_classes = []
+$quantifiers = []
+old_maps.each do |old_map|
+  old_map_name = File.basename(old_map, ".yaml")
+  puts old_map
+  f = File.read(old_map)
+  yaml = YAML.load(f)
+  map_keys = yaml['map'].keys
+  puts map_keys
+  rs = yaml['map']['postrules']&.map{|h| h['pattern']} || []
+  rs += yaml['map']['characters']&.keys || []
+  rs.each do |regexp|
+    tree = Regexp::Parser.parse( regexp )
+
+    arr = tree.expressions
+    while true
+      new_arr = arr.map do |elem|
+        if elem.respond_to? :quantifier and elem.quantifier
+          $expr_classes << elem.quantifier
+        end
+        el = el.class == Class ? el : el.class
+        if elem.respond_to?(:expressions)
+          [el, elem.expressions]
+        else
+          el
+        end
+      end.flatten
+      break if new_arr == arr
+      arr = new_arr
+    end
+    $expr_classes += arr
+  end
+end;
+# $expressions.map{|elem|  elem.class.to_s=="Class" ? elem : elem.class}.tally
+
+pp $expr_classes.tally.sort_by{|k,v| -v}
+pp $quanitifiers.map{|q| q.text}.tally.sort_by{|k,v| -v}

data/bin/maps_v1_to_v2

@@ -0,0 +1,426 @@
+#!/usr/bin/env ruby
+# This is a helper script for porting Interscript v1 maps to v2 format. It won't
+# ever be able to port them completely, but it should help bootstrap the process.
+
+require 'bundler/setup'
+
+require 'yaml'
+require 'fileutils'
+
+Dir.chdir(__dir__ + "/../")
+FileUtils.rm_rf(Dir.glob("../maps/maps-staging/*"))
+#FileUtils.mkdir_p("../maps/maps-staging/")
+
+#old_maps = []
+old_maps = Dir["../../interscript/maps/*.yaml"]
+#old_maps = Dir["../../interscript/maps/alalc-aze-Arab-Latn-1997.yaml"]
+#old_maps = Dir["../../interscript/maps/mofa-jpn-Hrkt-Latn-1989.yaml"]
+
+
+ex_maps = Dir["../maps/maps/*.imp"]
+ex_map_names = ex_maps.map { |i| File.basename(i, ".imp") }
+
+
+require 'regexp_parser'
+
+require 'interscript/utils/regexp_converter.rb'
+
+
+def process_line( kkk, vvv, indent: 0)
+
+    parse_kkk = Regexp::Parser.parse(kkk, 'ruby/2.1')
+    tokens_kkk = process(parse_kkk)
+    # pp conv
+    root_hash = process_root(tokens_kkk)
+    # pp root_hash
+    # puts "vvv = #{vvv.inspect}"
+    if vvv.class == String
+      if vvv == '' or vvv =='""' or vvv == nil or vvv.include? '~'
+        root_hash[:to] = '""'
+      elsif vvv == '"'
+        root_hash[:to] = '"\""'
+      else
+        if vvv == "?" #alalc-ell-Grek-Latn-1997.imp un-ell-Grek-Latn-1987-phonetic have to "?"
+          root_hash[:to] = "?".inspect
+        #if root_hash[:from].to_s.include?('capture') or root_hash[:to] =~ /\\\\([0-9]+)/
+        else
+          parse_vvv = Regexp::Parser.parse(vvv)
+          tokens_vvv = process(parse_vvv)
+          string_vvv = stringify(tokens_vvv)
+          # puts string_vvv
+          root_hash[:to] = string_vvv
+          root_hash[:to] = 'upcase' if string_vvv.include? 'upcase'
+        end
+      end
+    elsif vvv.class == Array
+      root_hash[:to] = "any(#{vvv.inspect})"
+    else
+      root_hash[:to] = "\"\""
+      # puts "unknown class #{vvv.inspect}"
+    end
+
+    str = stringify_root(root_hash, indent: indent)
+    # puts str
+    # puts ""
+    str
+end
+
+
+old_maps.sort.each do |old_map|
+  old_map_name = File.basename(old_map, ".yaml")
+
+  if ex_map_names.include? old_map_name
+    puts "* Skipping #{old_map_name} as it's already ported"
+    next
+  end
+
+  print "* Converting #{old_map_name}."
+
+  f = File.read(old_map)
+  fl = f.split("\n")
+
+  md = []
+  tests = []
+  map = []
+  chain = nil
+
+  cur = md
+
+  bugnotes = false
+
+  fl.each do |i|
+    if i == '---'
+      # skip the first line
+    elsif i =~ /\A\s+|\A\z/
+      # continuation
+      if bugnotes
+        i = "#{i}"
+        md << i
+      else
+        cur << i
+      end
+    else
+      cmt = nil
+      i = i.sub(/(#.*?)\z/) do |j|
+        cmt = j
+        ""
+      end
+
+      # block begin or md
+      case i.strip
+      when "tests:"
+        cur = tests
+        bugnotes = false
+      when "map:"
+        cur = map
+        bugnotes = false
+      when "notes:"
+        md << "notes:"
+        bugnotes = true
+      when /\Achain:/
+        chain = i
+      else
+        cur << i
+      end
+
+      cur << cmt if cmt
+    end
+  end
+
+  print "."
+
+  newmd = []
+  aliasff = false
+  md.each do |i|
+    if i.strip == "alias:"
+      aliasff = true
+    elsif i !~ /\A\s+/
+      aliasff = false
+    end
+    newmd << i unless aliasff
+  end
+  md = newmd
+
+  print "."
+
+  newmd = md.map(&"  ".method(:+))
+            .join("\n")
+            .gsub("  note:", "  notes:")
+            .gsub("confirmation date:", "confirmation_date:")
+            #.gsub("special_rules:", '  - "special rules:"')
+            #.gsub("original_description:", "  # original description:")
+            #.gsub("original_notes:", '  - "original notes:"')
+            #.gsub("implementation_notes:", '  - "implementation notes:"')
+            .rstrip
+
+  new = "metadata {\n"
+  new << newmd
+  new << "\n}\n\n"
+
+  class MultilineError < StandardError; end
+
+  if tests.length > 0
+    new << "tests {\n"
+    cmt = ""
+
+    iter = 0
+    while iter < tests.length; begin
+      test = tests[iter]
+
+      if test =~ /\A\s*#/
+        new << "  " << test.strip << "\n"
+        iter += 1
+        next
+      end
+
+      re_source = /\A(?:  ){0,2}- source: (.*?)(\s*#.*?)?\z/m
+      re_expect = /\A(?:  ){0,3}expected:[ \t](.*?)(\s*#.*?)?\z/m
+
+      if test.rstrip.end_with?("|") ||
+         (test =~ /"/ && !test.rstrip.end_with?('"'))
+        while iter < tests.length
+          xtest = tests[iter+1]
+          break if xtest =~ re_source || xtest =~ re_expect
+          test << "\n" << (xtest||"")
+          iter += 1
+        end
+      end
+
+      case test
+      when re_source
+        new << "  test #{YAML.load($1).inspect}, "
+        cmt = $2 if $2
+      when re_expect
+        new << "#{YAML.load($1).inspect}".unicode_normalize
+        new << cmt
+        new << $2 if $2
+        new << "\n"
+        cmt = ""
+      when /\A\s*\z/
+        # empty line, ignore
+      else
+        new << "\n# BUG: #{test}\n"
+      end
+
+      iter += 1
+    rescue Psych::SyntaxError
+      p test
+    end; end
+
+    new << "}\n\n"
+  end
+
+  print "."
+
+  new << "# This map has been partially converted by the bin/maps_v1_to_v2 script\n"
+  new << "# The section below requires human attention. Remember to remove this\n"
+  new << "# comment and move the converted map to 'maps/' directory. Please also\n"
+  new << "# take note that the maps-staging directory will be cleaned up whenever\n"
+  new << "# you run the bin/maps_v1_to_v2 script. You should particularly be\n"
+  new << "# concerned about any regular expressions found in this file and about\n"
+  new << "# advanced expressions in parallel {} parts, and also about the order\n"
+  new << "# of particular parts of the stage.\n\n"
+
+  transcription = nil
+  title_case = nil
+  downcase = nil
+  inherit = nil
+
+  characters, rules, dictionary, postrules = [], [], [], []
+
+  cur = nil
+  indent = 0
+  stagedone = false
+  efini = proc do
+    if inherit
+      new << "\n"
+      inherit.each do |i|
+        new << "  run map.#{i}.stage.main\n"
+      end
+    end
+    new << "\n"
+    efini=proc{}
+  end
+  fini = proc{}
+  begn = proc { new << "stage {\n"; stagedone = true; begn=proc{} }
+
+  iter = -1
+  while iter+1 < map.length
+    iter += 1
+    i = map[iter]
+
+    cmt = nil
+    if i =~ /\A  [^\s#]/ || i =~ /\A    inherit:/
+      i.sub(/(#.*?)\z/) { cmt = $1 }
+      if cmt
+        new << " "*indent << cmt << "\n"
+      end
+
+      maybe_val = YAML.load(i.split(":").last)
+      case i.split(":").first.strip
+      when "inherit"
+        inherit = Array(maybe_val)
+        inherit = inherit.map do |inh|
+          short = inh.split("-")[2..3].join.downcase
+          new << "dependency #{inh.inspect}, as: #{short}\n"
+          short
+        end
+        new << "\n"
+        raise "Duplicate items" unless inherit.length == inherit.uniq.length
+      when "dictionary"
+        begn.()
+        fini.()
+        new << "  # DICTIONARY\n"
+        new << "  parallel {\n"
+        indent = 4
+        cur = dictionary
+        fini = proc{new << "  }\n\n";indent = 2}
+      when "rules"
+        begn.()
+        fini.()
+        efini.()
+        new << "  # RULES\n"
+        indent = 2
+        cur = rules
+        fini = proc{new << "\n"}
+      when "characters"
+        begn.()
+        fini.()
+        efini.()
+        new << "  # CHARACTERS\n"
+        new << "  parallel {\n"
+        indent = 4
+        cur = characters
+        fini = proc{new << "  }\n\n";indent = 2}
+      when "postrules"
+        begn.()
+        fini.()
+        efini.()
+        new << "  # POSTRULES\n"
+        indent = 2
+        cur = postrules
+        fini = proc{new << "\n"}
+      when "downcase"
+        downcase = maybe_val
+      when "title_case"
+        title_case = maybe_val
+      when "transcription"
+        transcription = maybe_val
+      # Those we will ignore for now
+      when "word_separator", "segmentation", "character_separator", "map"
+      # Those are bugs
+      when "title-case"
+      else
+        p i
+      end
+    else
+      cmt = ""
+      i = i.sub(/(#.*?)\z/) { cmt << $1; "" }
+      #new << " "*indent << i.strip << "\n"
+      if i.strip == ""
+        new << " "*indent << cmt << "\n"
+        next
+      end
+
+      case cur.object_id
+      when nil.object_id
+        raise "Unexpected line #{i}"
+      when characters.object_id, dictionary.object_id
+        k,v = i.split(":", 2).map(&:strip)
+        if !v || v == ""
+          v = ""
+          # Load array
+          iter2 = iter + 1
+          while iter2 < map.length
+            i2 = map[iter2]
+            break unless i2.strip =~ /\A-/
+            i2 = i2.sub(/(#.*?)\z/) { cmt << $1; "" }
+            v << "\n" << i2
+            iter2 += 1
+          end
+          iter = iter2 - 1
+        end
+
+        kk,vv = YAML.load(k), YAML.load(v)
+
+        kkk,vvv = kk.inspect, vv.inspect
+        if vv.class == Array
+          if vv.all? { |z| z.length == 1 }
+            vvv = "any(" + vv.join.inspect + ")"
+          else
+            vvv = "any(" + vv.inspect + ")"
+          end
+        elsif vv.class == NilClass
+          vvv = "none"
+        end
+        kkk,vvv = kk.dup,vv.dup
+        # This worked due to use of regexps... it should remove 1 slash.
+        kkk = kkk.gsub("\\\\u", "\\\\u")
+        kkk = kkk.gsub("\\\\U", "\\\\u")
+
+        new << process_line( kkk,vvv, indent: indent )
+      when rules.object_id, postrules.object_id
+        if i.strip =~ /\A- pattern\s*:/
+          _, k = i.split(":", 2)
+          ii = map[iter+1]
+          ii = ii.sub(/(#.*?)\z/) { cmt << $1; "" }
+          if ii.strip == ""
+            iter += 1
+            ii = map[iter+1]
+            ii = ii.sub(/(#.*?)\z/) { cmt << $1; "" }
+          end
+          if ii.strip =~ /\Aresult\s*:/
+            _, v = ii.split(":", 2)
+          else
+            raise "Unexpected(2): #{ii.strip.inspect}"
+          end
+          iter += 1
+        else
+          raise "Unexpected(1): #{i.strip.inspect}"
+        end
+
+        kk,vv = YAML.load(k), YAML.load(v)
+        # kkk,vvv = kk.inspect, vv.inspect
+        kkk,vvv = kk.dup,vv.dup
+
+        new << process_line( kkk,vvv, indent: indent )
+      end
+
+      if cmt != ""
+        new << " " << cmt << "\n"
+      else
+        new << "\n"
+      end
+    end
+  end
+
+  fini.()
+
+  unless stagedone
+    new << "stage {\n"
+    efini.()
+  end
+
+  new << "  title_case\n" if title_case
+  new << "  downcase\n" if downcase
+
+  new << "}\n\n"
+
+  if chain
+    new << "# This map is chained and probably depends on seq2seq:\n"
+    new << "# #{chain}\n\n"
+  end
+
+  if transcription
+    new << "# This map contains transcription and probably depends on seq2seq:\n"
+    new << "# transcription: #{transcription}\n\n"
+  end
+
+  new = new.gsub(/ +$/, '') # Cleanup trailing whitespaces
+
+  #new << map.join("\n")
+
+  File.write("../maps/maps-staging/#{old_map_name}.imp", new)
+
+  puts " done!"
+end