interscript 0.1.7 → 2.1.0b1

data/lib/interscript/compiler/ruby.rb

@@ -0,0 +1,262 @@
+$main_binding = binding
+
+class Interscript::Compiler::Ruby < Interscript::Compiler
+  def compile(map, debug: false)
+    @map = map
+    @debug = debug
+    @parallel_trees = {}
+    @parallel_regexps = {}
+    c = "require 'interscript/stdlib'\n"
+    c << "if !defined?(Interscript::Maps); module Interscript; module Maps\n"
+    c << "module Cache; end\n"
+    c << "class Map < Struct.new(:stages, :aliases, :aliases_re); end\n"
+    c << "@maps = Hash.new { |h,id| h[id] = Map.new({},{},{}) }\n"
+    c << "def self.has_map?(map);                      @maps.include?(map); end\n"
+    c << "def self.add_map_alias(map,name,value)       @maps[map].aliases[name] = value; end\n"
+    c << "def self.add_map_alias_re(map,name,value)    @maps[map].aliases_re[name] = value; end\n"
+    c << "def self.add_map_stage(map,stage,&block);    @maps[map].stages[stage] = block; end\n"
+    c << "def self.get_alias(map,name);                @maps[map].aliases[name]; end\n"
+    c << "def self.get_alias_re(map,name);             @maps[map].aliases_re[name]; end\n"
+    c << "def self.transliterate(map,string,stage=:main); @maps[map].stages[stage].(string); end\n"
+    c << "end; end; end\n"
+    c
+
+    map.aliases.each do |name, value|
+      val = compile_item(value.data, map, :str)
+      c << "Interscript::Maps.add_map_alias(#{map.name.inspect}, #{name.inspect}, #{val})\n"
+      val = '/'+compile_item(value.data, map, :re).gsub('/', '\\\\/')+'/'
+      c << "Interscript::Maps.add_map_alias_re(#{map.name.inspect}, #{name.inspect}, #{val})\n"
+    end
+
+    map.stages.each do |_, stage|
+      c << compile_rule(stage, @map, true)
+    end
+    @parallel_trees.each do |k,v|
+      c << "Interscript::Maps::Cache::PTREE_#{k} ||= #{v.inspect}\n"
+    end
+    @parallel_regexps.each do |k,v|
+      c << "Interscript::Maps::Cache::PRE_#{k} ||= #{v.inspect}\n"
+    end
+    @code = c
+  end
+
+  def compile_rule(r, map = @map, wrapper = false)
+    c = ""
+    case r
+    when Interscript::Node::Stage
+      c += "Interscript::Maps.add_map_stage \"#{@map.name}\", #{r.name.inspect} do |s|\n"
+      c += "$map_debug ||= []\n" if @debug
+      c += "s = s.dup\n"
+      r.children.each do |t|
+        comp = compile_rule(t, map)
+        c += comp
+        c += %{$map_debug << [s.dup, #{@map.name.to_s.inspect}, #{r.name.to_s.inspect}, #{t.inspect.inspect}, #{comp.inspect}]\n} if @debug
+      end
+      c += "s\n"
+      c += "end\n"
+    when Interscript::Node::Group::Parallel
+      begin
+        # Try to build a tree
+        a = []
+        r.children.each do |i|
+          raise ArgumentError, "Can't parallelize #{i.class}" unless Interscript::Node::Rule::Sub === i
+          raise ArgumentError, "Can't parallelize rules with :before" if i.before
+          raise ArgumentError, "Can't parallelize rules with :after" if i.after
+          raise ArgumentError, "Can't parallelize rules with :not_before" if i.not_before
+          raise ArgumentError, "Can't parallelize rules with :not_after" if i.not_after
+
+          a << [compile_item(i.from, map, :par), compile_item(i.to, map, :parstr)]
+        end
+        ah = a.hash.abs
+        unless @parallel_trees.include? ah
+          tree = Interscript::Stdlib.parallel_replace_compile_tree(a)
+          @parallel_trees[ah] = tree
+        end
+        c += "s = Interscript::Stdlib.parallel_replace_tree(s, Interscript::Maps::Cache::PTREE_#{ah})\n"
+      rescue
+        # Otherwise let's build a megaregexp
+        a = []
+        Interscript::Stdlib.deterministic_sort_by_max_length(r.children).each do |i|
+          raise ArgumentError, "Can't parallelize #{i.class}" unless Interscript::Node::Rule::Sub === i
+
+          a << [build_regexp(i, map), compile_item(i.to, map, :parstr)]
+        end
+        ah = a.hash.abs
+        unless @parallel_regexps.include? ah
+          re = Interscript::Stdlib.parallel_regexp_compile(a)
+          @parallel_regexps[ah] = [re, a.map(&:last)]
+        end
+        c += "s = Interscript::Stdlib.parallel_regexp_gsub(s, *Interscript::Maps::Cache::PRE_#{ah})\n"
+      end
+    when Interscript::Node::Rule::Sub
+      from = "/#{build_regexp(r, map).gsub("/", "\\\\/")}/"
+      if r.to == :upcase
+        to = '&:upcase'
+      else
+        to = compile_item(r.to, map, :str)
+      end
+      c += "s.gsub!(#{from}, #{to})\n"
+    when Interscript::Node::Rule::Funcall
+      c += "s = Interscript::Stdlib::Functions.#{r.name}(s, #{r.kwargs.inspect[1..-2]})\n"
+    when Interscript::Node::Rule::Run
+      if r.stage.map
+        doc = map.dep_aliases[r.stage.map].document
+        stage = doc.imported_stages[r.stage.name]
+      else
+        stage = map.imported_stages[r.stage.name]
+      end
+      c += "s = Interscript::Maps.transliterate(#{stage.doc_name.inspect}, s, #{stage.name.inspect})\n"
+    else
+      raise ArgumentError, "Can't compile unhandled #{r.class}"
+    end
+    c
+  end
+
+  def build_regexp(r, map=@map)
+    from = compile_item(r.from, map, :re)
+    before = compile_item(r.before, map, :re) if r.before
+    after = compile_item(r.after, map, :re) if r.after
+    not_before = compile_item(r.not_before, map, :re) if r.not_before
+    not_after = compile_item(r.not_after, map, :re) if r.not_after
+
+    re = ""
+    re += "(?<=#{before})" if before
+    re += "(?<!#{not_before})" if not_before
+    re += from
+    re += "(?!#{not_after})" if not_after
+    re += "(?=#{after})" if after
+    re
+  end
+
+  def compile_item i, doc=@map, target=nil
+    i = i.first_string if %i[str parstr].include? target
+    i = Interscript::Node::Item.try_convert(i)
+    if target == :parstr
+      parstr = true
+      target = :par
+    end
+
+    out = case i
+    when Interscript::Node::Item::Alias
+      astr = if i.map
+        d = doc.dep_aliases[i.map].document
+        a = d.imported_aliases[i.name]
+        raise ArgumentError, "Alias #{i.name} of #{i.stage.map} not found" unless a
+        "Interscript::Maps.get_alias_ALIASTYPE(#{a.doc_name.inspect}, #{a.name.inspect})"
+      elsif Interscript::Stdlib::ALIASES.include?(i.name)
+        if target != :re && Interscript::Stdlib.re_only_alias?(i.name)
+          raise ArgumentError, "Can't use #{i.name} in a #{target} context"
+        end
+        stdlib_alias = true
+        "Interscript::Stdlib::ALIASES[#{i.name.inspect}]"
+      else
+        a = doc.imported_aliases[i.name]
+        raise ArgumentError, "Alias #{i.name} not found" unless a
+
+        "Interscript::Maps.get_alias_ALIASTYPE(#{a.doc_name.inspect}, #{a.name.inspect})"
+      end
+
+      if target == :str
+        astr = astr.sub("_ALIASTYPE(", "(")
+      elsif target == :re
+        astr = "\#{#{astr.sub("_ALIASTYPE(", "_re(")}}"
+      elsif parstr && stdlib_alias
+        astr = Interscript::Stdlib::ALIASES[i.name]
+      elsif target == :par
+        # raise NotImplementedError, "Can't use aliases in parallel mode yet"
+        astr = Interscript::Stdlib::ALIASES[i.name]
+      end
+    when Interscript::Node::Item::String
+      if target == :str
+        # Replace \1 with \\1, this is weird, but it works!
+        i.data.gsub("\\", "\\\\\\\\").inspect
+      elsif target == :par
+        i.data
+      elsif target == :re
+        Regexp.escape(i.data)
+      end
+    when Interscript::Node::Item::Group
+      if target == :par
+        i.children.map do |j|
+          compile_item(j, doc, target)
+        end.reduce([""]) do |j,k|
+          Array(j).product(Array(k)).map(&:join)
+        end
+      elsif target == :str
+        i.children.map { |j| compile_item(j, doc, target) }.join("+")
+      elsif target == :re
+        i.children.map { |j| compile_item(j, doc, target) }.join
+      end
+    when Interscript::Node::Item::CaptureGroup
+      if target != :re
+        raise ArgumentError, "Can't use a CaptureGroup in a #{target} context"
+      end
+      "(" + compile_item(i.data, doc, target) + ")"
+    when Interscript::Node::Item::Maybe,
+         Interscript::Node::Item::MaybeSome,
+         Interscript::Node::Item::Some
+
+      resuffix = { Interscript::Node::Item::Maybe     => "?" ,
+                   Interscript::Node::Item::Some      => "+" ,
+                   Interscript::Node::Item::MaybeSome => "*" }[i.class]
+
+      if target == :par
+        raise ArgumentError, "Can't use a Maybe in a #{target} context"
+      end
+      if Interscript::Node::Item::String === i.data && i.data.data.length != 1
+        "(?:" + compile_item(i.data, doc, target) + ")" + resuffix
+      else
+        compile_item(i.data, doc, target) + resuffix
+      end
+    when Interscript::Node::Item::CaptureRef
+      if target == :par
+        raise ArgumentError, "Can't use CaptureRef in parallel mode"
+      elsif target == :re
+        "\\#{i.id}"
+      elsif target == :str
+        "\"\\\\#{i.id}\""
+      end
+    when Interscript::Node::Item::Any
+      if target == :str
+        raise ArgumentError, "Can't use Any in a string context" # A linter could find this!
+      elsif target == :par
+        i.data.map(&:data)
+      elsif target == :re
+        case i.value
+        when Array
+          data = i.data.map { |j| compile_item(j, doc, target) }
+          "(?:"+data.join("|")+")"
+        when String
+          "[#{Regexp.escape(i.value)}]"
+        when Range
+          "[#{Regexp.escape(i.value.first)}-#{Regexp.escape(i.value.last)}]"
+        end
+      end
+    end
+  end
+
+  def load
+    if !defined?(Interscript::Maps) || !Interscript::Maps.has_map?(@map.name)
+      @map.dependencies.each do |dep|
+        dep = dep.full_name
+        if !defined?(Interscript::Maps) || !Interscript::Maps.has_map?(dep)
+          Interscript.load(dep, compiler: self.class).load
+        end
+      end
+      eval(@code, $main_binding)
+    end
+  end
+
+  def call(str, stage=:main)
+    load
+    Interscript::Maps.transliterate(@map.name, str, stage)
+  end
+
+  def self.read_debug_data
+    $map_debug || []
+  end
+
+  def self.reset_debug_data
+    $map_debug = []
+  end
+end

data/lib/interscript/dsl.rb

@@ -0,0 +1,68 @@
+require "yaml"
+
+module Interscript::DSL
+  @cache = {}
+  def self.parse(map_name)
+    # map name aliases? here may be a place to wrap it
+
+    return @cache[map_name] if @cache[map_name]
+    path = Interscript.locate(map_name)
+    library = path.end_with?(".iml")
+    map_name = File.basename(path, ".imp")
+    map_name = File.basename(map_name, ".iml")
+
+    ruby = []
+    yaml = []
+
+    file = File.read(path).split("\n")
+    exc_fname = File.expand_path(path, Dir.pwd)
+
+    md_reading = false
+    md_indent = nil
+    md_inner_indent = nil
+    file.each do |l|
+      if md_reading && l =~ /\A#{md_indent}\}\s*\z/
+        md_reading = false
+      elsif md_reading
+        ruby << ""
+        yaml << l
+      elsif l =~ /\A(\s*)metadata\s*\{\z/
+        md_indent = $1
+        md_reading = true
+      else
+        yaml << ""
+        ruby << l
+      end
+    end
+    raise ArgumentError, "metadata stage isn't terminated" if md_reading
+    ruby, yaml = ruby.join("\n"), yaml.join("\n")
+
+    obj = Interscript::DSL::Document.new(map_name)
+    obj.instance_eval ruby, exc_fname, 1
+
+    yaml = if yaml =~ /\A\s*\z/
+      {}
+    else
+      YAML.load(yaml, exc_fname)
+    end
+
+    md = Interscript::DSL::Metadata.new(yaml: true, map_name: map_name, library: library) do
+      yaml.each do |k,v|
+        public_send(k.to_sym, v)
+      end
+    end
+    obj.node.metadata = md.node
+
+    @cache[map_name] = obj.node
+  end
+end
+
+require 'interscript/dsl/symbol_mm'
+require 'interscript/dsl/items'
+
+require 'interscript/dsl/document'
+require 'interscript/dsl/group'
+require 'interscript/dsl/stage'
+require 'interscript/dsl/metadata'
+require 'interscript/dsl/tests'
+require 'interscript/dsl/aliases'

data/lib/interscript/dsl/aliases.rb

@@ -0,0 +1,23 @@
+class Interscript::DSL::Aliases
+  include Interscript::DSL::Items
+
+  attr_accessor :node
+
+  def initialize(&block)
+    @node = {}
+    self.instance_exec(&block)
+  end
+
+  def def_alias(name, value)
+    if Interscript::Node::Item::Alias === name
+      name = name.name
+    end
+
+    unless Symbol === name
+      raise TypeError, "Alias name must be a Symbol, given #{name.class}"
+    end
+
+    puts "def_alias(#{name.inspect}, #{thing.inspect})" if $DEBUG
+    @node[name] = Interscript::Node::AliasDef.new(name, value)
+  end
+end

data/lib/interscript/dsl/document.rb

@@ -0,0 +1,46 @@
+class Interscript::DSL::Document
+  include Interscript::DSL::SymbolMM
+
+  attr_accessor :node
+
+  def initialize(name = nil, &block)
+    @node = Interscript::Node::Document.new
+    @node.name = name if name
+    self.instance_exec &block if block_given?
+  end
+
+  def metadata(&block)
+    metadata = Interscript::DSL::Metadata.new(&block)
+    @node.metadata = metadata.node
+  end
+
+  def tests(&block)
+    tests = Interscript::DSL::Tests.new(&block)
+    @node.tests = tests.node
+  end
+
+  def aliases(&block)
+    aliases = Interscript::DSL::Aliases.new(&block)
+    @node.aliases = aliases.node
+    @node.aliases.transform_values { |v| v.doc_name = @node.name; v }
+  end
+
+  def dependency(full_name, **kargs)
+    puts "dependency(#{name.inspect}, #{kargs.inspect}" if $DEBUG
+    dep = Interscript::Node::Dependency.new
+    dep.name = kargs[:as]
+    dep.full_name = full_name
+    dep.import = kargs[:import] || false
+
+    dep.document = Interscript::DSL.parse(full_name)
+    @node.dependencies << dep
+    @node.dep_aliases[dep.name] = dep if dep.name
+  end
+
+  def stage(name = :main, &block)
+    puts "stage(#{name}) from #{self.inspect}" if $DEBUG
+    stage = Interscript::DSL::Stage.new(name, &block)
+    stage.node.doc_name = @node.name
+    @node.stages[name] = stage.node
+  end
+end

data/lib/interscript/dsl/group.rb

@@ -0,0 +1,45 @@
+class Interscript::DSL::Group
+  include Interscript::DSL::Items
+
+  attr_accessor :node
+
+  def initialize(&block)
+    @node = Interscript::Node::Group.new
+    self.instance_exec(&block)
+  end
+
+  def run(stage)
+    if stage.class != Interscript::Node::Item::Stage
+      raise TypeError, "I::Node::Item::Stage expected, got #{stage.class}"
+    end
+    @node.children << Interscript::Node::Rule::Run.new(stage)
+  end
+
+  def sub(from, to, **kwargs, &block)
+    puts "sub(#{from.inspect},#{to}, kargs = #{
+      kargs.inspect
+    }) from #{self.inspect}" if $DEBUG
+
+    rule = Interscript::Node::Rule::Sub.new(from, to, **kwargs)
+    @node.children << rule
+  end
+
+  def upcase; :upcase; end
+
+  Interscript::Stdlib.available_functions.each do |fun|
+    define_method fun do |**kwargs|
+      puts "funcall(#{fun}, #{kwargs.inspect}) from #{self.inspect}" if $DEBUG
+
+      rule = Interscript::Node::Rule::Funcall.new(fun, **kwargs)
+      @node.children << rule
+    end
+  end
+
+  def parallel(&block)
+    puts "parallel(#{chars.inspect}) from #{self.inspect}" if $DEBUG
+    group = Interscript::DSL::Group::Parallel.new(&block)
+    @node.children << group.node
+  end
+end
+
+require 'interscript/dsl/group/parallel'