interscript 2.1.0 → 2.3.1a

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f92a8060d3eee8e0eb4815795294c045ccc7ad341c92a6ba48cb825be4598b95
-  data.tar.gz: 6b977f41c3b1ba80bf25ef1d120666a228bcfd050f31f0a87a9c9ceeb86f5007
+  metadata.gz: e2a42db10c2cd99083f404f74ab10be34229e8393d8b29a7f6b8f839eb857839
+  data.tar.gz: 5a15755052db0086a4703ddb88a402ad8fa23ed59302f11206ddf0d122510477
 SHA512:
-  metadata.gz: 2fc32770358443ca0baae5416a918032b4a169fdc98b32d493fe1a487599fbdf9c5005fb53b98a25e5962f18f2ff52a1667010bda48394e891998437b2a2f1b6
-  data.tar.gz: a96e1cc807ebc83c9630b1a78f5b9c26558f771954b55622ec55d0f0b0ff17601a27a5753ab407dc49ec0db8d960940c811148c2872ba9c56e1b19baa3a7ccb7
+  metadata.gz: fd85bca889dba2bb86e028fdc5498b3bfac060771c428f6b27c65729fb38f03bf455fc8e6d7f41d69a954568426a6b6c8e9d5b635406e8b6f89ee5459389d151
+  data.tar.gz: c09d42b2531d92ab16dfa36d710e4f4a1cb68ba358f3c99336a99677ac786e38825faa92fdfe3fcda7eadb099c68f544c80888520007eedc7ee10d049f36002a

data/Gemfile

@@ -26,4 +26,10 @@ unless ENV["SKIP_JS"]
   end
 end
 
+group :rababa do
+  gem 'rababa', "~> 0.1.1"
+end
+
+gem 'pry'
+
 gem 'simplecov', require: false, group: :test

data/Rakefile

@@ -20,7 +20,10 @@ task :compile, [:compiler, :target] do |t, args|
 
   maplist = {}
 
-  Interscript.maps.each do |map|
+  maps = Interscript.maps
+  maps = Interscript.exclude_maps(maps, compiler: compiler, platform: false)
+  
+  maps.each do |map|
     code = compiler.(map).code
     File.write(args[:target] + "/" + map + "." + ext, code)
     maplist[map] = nil
@@ -68,6 +71,7 @@ task :generate_metadata_json do
   require "fileutils"
   require "json"
   require "interscript"
+  require "interscript/compiler/javascript"
 
   FileUtils.rm_rf(file = __dir__+"/metadata.json")
 
@@ -75,6 +79,10 @@ task :generate_metadata_json do
     parsed_map = Interscript.parse(map)
     md = parsed_map.metadata.to_hash
     md["test"] = parsed_map.tests&.data&.first
+    md["skip_js"] = Interscript.exclude_maps([map],
+                      compiler: Interscript::Compiler::Javascript,
+                      platform: false,
+                    ).empty?
     [map, md]
   end.to_h
 

data/bin/console

@@ -3,12 +3,8 @@
 require "bundler/setup"
 require "interscript"
 
-# You can add fixtures and/or initialization code here to make experimenting
-# with your gem easier. You can also use a different console, if you like.
+require "interscript/utils/helpers"
+include Interscript::Utils::Helpers
 
-# (If you use this, don't forget to add pry to your Gemfile!)
-# require "pry"
-# Pry.start
-
-require "irb"
-IRB.start(__FILE__)
+require "pry"
+Pry.start

data/interscript.gemspec

@@ -27,5 +27,6 @@ Gem::Specification.new do |spec|
   spec.require_paths = ["lib"]
 
   spec.add_dependency "thor"
-  spec.add_dependency "interscript-maps"
+  spec.add_dependency "interscript-maps", "~> #{Interscript::VERSION.split('.')[0,2].join(".")}.0a"
+  spec.add_dependency "text"
 end

data/lib/interscript.rb

@@ -53,6 +53,16 @@ module Interscript
       output_file
     end
 
+    # Detects the transliteration that gives the most close approximation
+    # of transliterating source into destination.
+    #
+    # Set multiple: true to get a full report.
+    def detect(source, destination, **kwargs)
+      detector = Detector.new
+      detector.set_from_kwargs(**kwargs)
+      detector.(source, destination)
+    end
+
     def map_gems
       @map_gems ||= Gem.find_latest_files('interscript-maps.yaml').map do |i|
         [i, YAML.load_file(i)]
@@ -76,6 +86,56 @@ module Interscript
       end.compact.flatten
     end
 
+    def rababa_configs
+      @rababa_configs ||= map_gems.map do |i,v|
+        v["rababa-configs"]
+      end.compact.inject({}) do |a,b|
+        a.merge(b)
+      end
+    end
+
+    # This code is borrowed from Secryst and should end up in Rababa, but for now,
+    # let's keep it here.
+    def rababa_provision(model_name, model_uri)
+      require 'fileutils'
+      require 'open-uri'
+
+      # We provision the environment in the following way:
+      # First, we try the RABABA_DATA environment variable. If that's available,
+      # we use it to store the Rababa data we need. Otherwise, we try the following
+      # paths:
+
+      possible_paths = [
+        "/var/lib/rababa",
+        "/usr/local/share/rababa",
+        "/usr/share/rababa",
+        File.join(Dir.home, ".local/share/rababa")
+      ]
+
+      # We find the first writable path
+
+      write_path = nil
+
+      ([ENV["RABABA_DATA"]] + possible_paths).compact.each do |path|
+        FileUtils.mkdir_p(path)
+        write_path = path unless write_path
+      rescue
+      end
+
+      raise StandardError, "Can't find a writable path for Rababa. Consider setting a RABABA_DATA environment variable" unless write_path
+
+      model_path = "#{write_path}/model-#{model_name}.onnx"
+
+      # Redownload every hour
+      if File.exist?(model_path) && File.mtime(model_path) + 3600 >= Time.now
+        return model_path
+      else
+        data = URI.open(model_uri).read
+        File.write(model_path, data)
+        return model_path
+      end
+    end
+
     def map_aliases
       return @map_aliases if @map_aliases
 
@@ -99,6 +159,22 @@ module Interscript
 
       basename ? imps.map { |j| File.basename(j, ".#{ext}") } : imps
     end
+
+    # Removes the excluded maps for a given compiler and RUBY_PLATFORM.
+    # To be used by tests
+    # and builders. It uses the `skip` directive in interscript-maps.yaml
+    def exclude_maps(maps, compiler:, platform: true)
+      map_gems.each do |i,v|
+        [compiler.name, (Gem::Platform.local.os if platform)].compact.each do |name|
+          skips = v.dig('skip', name) || []
+          skips.each do |skip|
+            skip_re = /#{Regexp.escape(skip).gsub("\\*", ".*?")}/
+            maps = maps.grep_v(skip_re)
+          end
+        end
+      end
+      maps
+    end
   end
 end
 
@@ -109,3 +185,5 @@ require "interscript/interpreter"
 
 require 'interscript/dsl'
 require 'interscript/node'
+
+require 'interscript/detector'

data/lib/interscript/compiler/javascript.rb

@@ -53,6 +53,7 @@ class Interscript::Compiler::Javascript < Interscript::Compiler
 
   def compile_rule(r, map = @map, wrapper = false)
     c = ""
+    return c if r.reverse_run == true
     case r
     when Interscript::Node::Stage
       c += "map.stages.#{r.name} = function(s) {\n"
@@ -75,6 +76,7 @@ class Interscript::Compiler::Javascript < Interscript::Compiler
           raise ArgumentError, "Can't parallelize rules with :not_before" if i.not_before
           raise ArgumentError, "Can't parallelize rules with :not_after" if i.not_after
 
+          next if i.reverse_run == true
           a << [compile_item(i.from, map, :par), compile_item(i.to, map, :parstr)]
         end
         ah = a.hash.abs
@@ -88,7 +90,8 @@ class Interscript::Compiler::Javascript < Interscript::Compiler
         a = []
         Interscript::Stdlib.deterministic_sort_by_max_length(r.children).each do |i|
           raise ArgumentError, "Can't parallelize #{i.class}" unless Interscript::Node::Rule::Sub === i
-
+          
+          next if i.reverse_run == true
           a << [build_regexp(i, map), compile_item(i.to, map, :parstr)]
         end
         ah = a.hash.abs
@@ -102,6 +105,8 @@ class Interscript::Compiler::Javascript < Interscript::Compiler
       from = %{"#{build_regexp(r, map).gsub("/", "\\\\/")}"}
       if r.to == :upcase
         to = 'function(a){return a.toUpperCase();}'
+      elsif r.to == :downcase
+        to = 'function(a){return a.toLowerCase();}'
       else
         to = compile_item(r.to, map, :str)
       end

data/lib/interscript/compiler/ruby.rb

@@ -42,6 +42,7 @@ class Interscript::Compiler::Ruby < Interscript::Compiler
 
   def compile_rule(r, map = @map, wrapper = false)
     c = ""
+    return c if r.reverse_run == true
     case r
     when Interscript::Node::Stage
       c += "Interscript::Maps.add_map_stage \"#{@map.name}\", #{r.name.inspect} do |s|\n"
@@ -65,6 +66,7 @@ class Interscript::Compiler::Ruby < Interscript::Compiler
           raise ArgumentError, "Can't parallelize rules with :not_before" if i.not_before
           raise ArgumentError, "Can't parallelize rules with :not_after" if i.not_after
 
+          next if i.reverse_run == true
           a << [compile_item(i.from, map, :par), compile_item(i.to, map, :parstr)]
         end
         ah = a.hash.abs
@@ -79,6 +81,7 @@ class Interscript::Compiler::Ruby < Interscript::Compiler
         Interscript::Stdlib.deterministic_sort_by_max_length(r.children).each do |i|
           raise ArgumentError, "Can't parallelize #{i.class}" unless Interscript::Node::Rule::Sub === i
 
+          next if i.reverse_run == true
           a << [build_regexp(i, map), compile_item(i.to, map, :parstr)]
         end
         ah = a.hash.abs
@@ -92,6 +95,8 @@ class Interscript::Compiler::Ruby < Interscript::Compiler
       from = "/#{build_regexp(r, map).gsub("/", "\\\\/")}/"
       if r.to == :upcase
         to = '&:upcase'
+      elsif r.to == :downcase
+        to = '&:downcase'
       else
         to = compile_item(r.to, map, :str)
       end

data/lib/interscript/detector.rb

@@ -0,0 +1,62 @@
+require "text"
+
+class Interscript::Detector
+  attr_accessor :compiler
+  attr_accessor :distance_computer
+  attr_accessor :map_pattern
+
+  # TODO: use transliterate_each
+  attr_accessor :each
+
+  attr_accessor :load_path
+  attr_accessor :cache
+
+  # Returns a summary of all detected transliterations
+  attr_accessor :multiple
+
+  def initialize
+    @compiler = Interscript::Interpreter
+    @distance_computer = DistanceComputer::Levenshtein
+    @map_pattern = "*"
+
+    @each = false
+
+    @load_path = false
+    @cache = CACHE
+  end
+
+  def set_from_kwargs(**kwargs)
+    kwargs.each do |k,v|
+      self.public_send(:"#{k}=", v)
+    end
+  end
+
+  def call(source, destination)
+    maps = Interscript.maps(select: @map_pattern, load_path: @load_path)
+    maps = Interscript.exclude_maps(maps, compiler: self.class)
+    maps = Interscript.exclude_maps(maps, compiler: @compiler)
+
+    summary = maps.map do |map|
+      try_dest = Interscript.transliterate(map, source, compiler: @compiler)
+
+      [map, try_dest]
+    end.map do |map, try_dest|
+      dist = @distance_computer.(try_dest, destination)
+      
+      [map, dist]
+    end.sort_by(&:last).to_h
+
+    if @multiple
+      summary.to_h
+    else
+      summary.first.first
+    end
+  end
+
+  CACHE = {}
+
+  # A DistanceComputer needs to respond to #call(source, destination)
+  module DistanceComputer
+    Levenshtein = Text::Levenshtein.method(:distance)
+  end
+end

data/lib/interscript/dsl.rb

@@ -2,12 +2,45 @@ require "yaml"
 
 module Interscript::DSL
   @cache = {}
-  def self.parse(map_name)
+  def self.parse(map_name, reverse: true)
     # map name aliases? here may be a place to wrap it
 
     return @cache[map_name] if @cache[map_name]
-    path = Interscript.locate(map_name)
+
+    # This is a composition, so let's make a new virtual map
+    # that calls all maps in a sequence.
+    if map_name.include? "|"
+      map_parts = map_name.split("|").map(&:strip)
+
+      doc = Interscript::DSL::Document.new(map_name) do
+        map_parts.each_with_index do |i, idx|
+          dependency i, as: :"part#{idx}"
+        end
+
+        stage {
+          map_parts.each_with_index do |i, idx|
+            run map[:"part#{idx}"].stage.main
+          end
+        }
+      end.node
+
+      return @cache[map_name] = doc
+    end
+
+    path = begin
+      Interscript.locate(map_name)
+    rescue Interscript::MapNotFoundError => e
+      # But maybe we called the map in a reversed fashion?
+      begin
+        raise e if reverse == false # Protect from an infinite loop
+        reverse_name = Interscript::Node::Document.reverse_name(map_name)
+        return @cache[map_name] = parse(reverse_name, reverse: false).reverse
+      rescue Interscript::MapNotFoundError
+        raise e
+      end
+    end
     library = path.end_with?(".iml")
+
     map_name = File.basename(path, ".imp")
     map_name = File.basename(map_name, ".iml")
 

data/lib/interscript/dsl/document.rb

@@ -37,10 +37,11 @@ class Interscript::DSL::Document
     @node.dep_aliases[dep.name] = dep if dep.name
   end
 
-  def stage(name = :main, &block)
+  def stage(name = :main, dont_reverse: false, &block)
     puts "stage(#{name}) from #{self.inspect}" if $DEBUG
     stage = Interscript::DSL::Stage.new(name, &block)
     stage.node.doc_name = @node.name
+    stage.node.dont_reverse = dont_reverse
     @node.stages[name] = stage.node
   end
 end

data/lib/interscript/dsl/group.rb

@@ -8,16 +8,16 @@ class Interscript::DSL::Group
     self.instance_exec(&block)
   end
 
-  def run(stage)
+  def run(stage, **kwargs)
     if stage.class != Interscript::Node::Item::Stage
       raise TypeError, "I::Node::Item::Stage expected, got #{stage.class}"
     end
-    @node.children << Interscript::Node::Rule::Run.new(stage)
+    @node.children << Interscript::Node::Rule::Run.new(stage, **kwargs)
   end
 
   def sub(from, to, **kwargs, &block)
-    puts "sub(#{from.inspect},#{to}, kargs = #{
-      kargs.inspect
+    puts "sub(#{from.inspect},#{to}, kwargs = #{
+      kwargs.inspect
     }) from #{self.inspect}" if $DEBUG
 
     rule = Interscript::Node::Rule::Sub.new(from, to, **kwargs)
@@ -25,6 +25,7 @@ class Interscript::DSL::Group
   end
 
   def upcase; :upcase; end
+  def downcase; :downcase; end
 
   Interscript::Stdlib.available_functions.each do |fun|
     define_method fun do |**kwargs|
@@ -35,9 +36,9 @@ class Interscript::DSL::Group
     end
   end
 
-  def parallel(&block)
+  def parallel(**kwargs, &block)
     puts "parallel(#{chars.inspect}) from #{self.inspect}" if $DEBUG
-    group = Interscript::DSL::Group::Parallel.new(&block)
+    group = Interscript::DSL::Group::Parallel.new(**kwargs, &block)
     @node.children << group.node
   end
 end

data/lib/interscript/dsl/group/parallel.rb

@@ -1,6 +1,6 @@
 class Interscript::DSL::Group::Parallel < Interscript::DSL::Group
-  def initialize(&block)
-    @node = Interscript::Node::Group::Parallel.new
+  def initialize(reverse_run: nil, &block)
+    @node = Interscript::Node::Group::Parallel.new(reverse_run: reverse_run)
     self.instance_exec(&block)
   end
 end

data/lib/interscript/dsl/tests.rb

@@ -6,7 +6,7 @@ class Interscript::DSL::Tests
     self.instance_exec(&block)
   end
 
-  def test(from,to)
-    @node << [from, to]
+  def test(from, to, reverse_run: nil)
+    @node << [from, to, reverse_run]
   end
 end

data/lib/interscript/interpreter.rb

@@ -76,6 +76,7 @@ class Interscript::Interpreter < Interscript::Compiler
     end
 
     def execute_rule r
+      return if r.reverse_run == true
       case r
       when Interscript::Node::Group::Parallel
         if r.cached_tree
@@ -96,6 +97,7 @@ class Interscript::Interpreter < Interscript::Compiler
               raise ArgumentError, "Can't parallelize rules with :after" if i.after
               raise ArgumentError, "Can't parallelize rules with :not_before" if i.not_before
               raise ArgumentError, "Can't parallelize rules with :not_after" if i.not_after
+              next if i.reverse_run == true
               subs_array << [build_item(i.from, :par), build_item(i.to, :parstr)]
             end
             tree = Interscript::Stdlib.parallel_replace_compile_tree(subs_array) #.sort_by{|k,v| -k.length})
@@ -108,7 +110,7 @@ class Interscript::Interpreter < Interscript::Compiler
             subs_array = []
             Interscript::Stdlib.deterministic_sort_by_max_length(r.children).each do |i|  # rule.from.max_length gives somewhat better test results, why is that
               raise ArgumentError, "Can't parallelize #{i.class}" unless Interscript::Node::Rule::Sub === i
-
+              next if i.reverse_run == true
               subs_array << [build_regexp(i), build_item(i.to, :parstr)]
             end
             r.subs_regexp = Interscript::Stdlib.parallel_regexp_compile(subs_array)
@@ -129,6 +131,8 @@ class Interscript::Interpreter < Interscript::Compiler
       when Interscript::Node::Rule::Sub
         if r.to == :upcase
           @str = @str.gsub(Regexp.new(build_regexp(r)), &:upcase)
+        elsif r.to == :downcase
+          @str = @str.gsub(Regexp.new(build_regexp(r)), &:downcase)
         else
           @str = @str.gsub(Regexp.new(build_regexp(r)), build_item(r.to, :str))
         end