interscript 0.1.7 → 2.1.0b1

data/lib/interscript/utils/regexp_converter.rb

@@ -0,0 +1,283 @@
+require 'regexp_parser'
+
+
+def process(node)
+  children = if node.respond_to?(:expressions) && node.expressions
+               children = node.expressions.map.each { |expr| process(expr) }
+             end
+  # puts node.inspect
+  out = case node
+        when Regexp::Expression::Root
+          children
+        when Regexp::Expression::Assertion::Lookbehind
+          [:lookbehind_start, children, :lookbehind_stop]
+        when Regexp::Expression::Assertion::NegativeLookbehind
+          [:negative_lookbehind_start, children, :negative_lookbehind_stop]
+        when Regexp::Expression::Assertion::Lookahead
+          [:lookahead_start, children, :lookahead_stop]
+        when Regexp::Expression::Assertion::NegativeLookahead
+          [:negative_lookahead_start, children, :negative_lookahead_stop]
+        when Regexp::Expression::Group::Capture
+          [:capture_start, children, :capture_stop]
+        when Regexp::Expression::CharacterSet
+          # puts children.inspect
+          if children.flatten.include? (:range_start) #or children.size > 1
+            [:characterset_start, :array_start, children, :array_stop, :characterset_stop]
+          else
+            [:characterset_start, children, :characterset_stop]
+          end
+        when Regexp::Expression::Alternation
+          [:alternation_start, children, :alternation_stop]
+        when Regexp::Expression::Alternative
+          [:alternative_start, children, :alternative_stop]
+        when Regexp::Expression::CharacterSet::Range
+          lit1 = node.expressions[0].text
+          lit2 = node.expressions[1].text
+          [:range_start, lit1, :range_mid, lit2, :range_stop]
+        when Regexp::Expression::Anchor::WordBoundary
+          :boundary
+        when Regexp::Expression::Anchor::NonWordBoundary
+          :non_word_boundary
+        when Regexp::Expression::EscapeSequence::Backspace
+          :boundary # most probably boundary
+        when Regexp::Expression::CharacterType::Space
+          :space
+        when Regexp::Expression::Anchor::BeginningOfLine
+          :line_start
+        when Regexp::Expression::Anchor::EndOfLine
+          :line_end
+        when Regexp::Expression::CharacterType::Any
+          :any_character
+        when Regexp::Expression::Literal
+          node.text
+        when Regexp::Expression::EscapeSequence::Literal
+          node.text
+        when Regexp::Expression::EscapeSequence::Codepoint
+          node.text
+        when Regexp::Expression::PosixClass
+          '[' + node.text + ']'
+        when Regexp::Expression::UnicodeProperty::Script
+          node.text
+        when Regexp::Expression::Backreference::Number # why is there a space before after node.number?
+          [:backref_num_start, node.number, :backref_num_stop]
+        else
+          out = [:missing, node.class]
+
+          out << children if node.respond_to? :expressions
+           if node.respond_to? :quantifier and node.quantifier
+            # TODO add quantifier support
+            pp node
+            # out << process(node.quantifier)
+          end
+          out
+        end
+  if node.respond_to?(:quantifier) && node.quantifier&.token.to_s == "interval" && node.quantifier.max == node.quantifier.min
+    out = [out] * node.quantifier.max
+  elsif node.respond_to?(:quantifier) && node.quantifier
+    qname = node.quantifier.token.to_s
+    out = ["#{qname}_start".to_sym, [out], "#{qname}_stop".to_sym]
+  end
+  out
+end
+
+def process_root(node)
+  node2 = node.dup
+  root = {}
+  if before = node.select { |x| x[0] == :lookbehind_start }
+    # root[:before] = before[1]
+    # node2.delete(before)
+    if before.size == 1
+      root[:before] = before[0][1]
+      node2.delete(before[0])
+    elsif before.size >1
+      # pp not_before
+
+      a = [:alternation_start]
+      a << before.map{|x| [:alternative_start, x[1], :alternative_stop] }
+      a << [:alternation_stop]
+      root[:before] = a
+      # pp root[:not_before]
+      before.each{|n| node2.delete(n)}
+    end
+
+  end
+  if not_before = node.select { |x| x[0] == :negative_lookbehind_start }
+    # root[:not_before] = not_before[1]
+    # node2.delete(not_before)
+
+    if not_before.size == 1
+      root[:not_before] = not_before[0][1]
+      node2.delete(not_before[0])
+    elsif not_before.size >1
+      # pp not_before
+
+      a = [:alternation_start]
+      a << not_before.map{|x| [:alternative_start, x[1], :alternative_stop] }
+      a << [:alternation_stop]
+      root[:not_before] = a
+      # pp root[:not_before]
+      not_before.each{|n| node2.delete(n)}
+    end
+  end
+  if after = node.select { |x| x[0] == :lookahead_start }
+    # root[:after] = after[1]
+    # node2.delete(after)
+
+    if after.size == 1
+      root[:after] = after[0][1]
+      node2.delete(after[0])
+    elsif after.size >1
+      # pp not_before
+
+      a = [:alternation_start]
+      a << after.map{|x| [:alternative_start, x[1], :alternative_stop] }
+      a << [:alternation_stop]
+      root[:after] = a
+      # pp root[:not_before]
+      after.each{|n| node2.delete(n)}
+    end
+
+  end
+  if not_after = node.select { |x| x[0] == :negative_lookahead_start }
+    # root[:not_after] = not_after[1]
+    # node2.delete(not_after)
+    if not_after.size == 1
+      root[:not_after] = not_after[0][1]
+      node2.delete(not_after[0])
+    elsif not_after.size >1
+      # pp not_after
+
+      a = [:alternation_start]
+      a << not_after.map{|x| [:alternative_start, x[1], :alternative_stop] }
+      a << [:alternation_stop]
+      root[:not_after] = a
+      # pp root[:not_after]
+      not_after.each{|n| node2.delete(n)}
+    end
+
+  end
+  root[:from] = node2
+  root
+end
+
+def stringify(node)
+  tokens = node.flatten
+  subs = {
+    characterset_start: 'any(',
+    characterset_stop: ')',
+    array_start: '[',
+    array_stop: ']',
+    capture_start: 'capture(',
+    capture_stop: ')',
+    zero_or_one_start: 'maybe(',
+    zero_or_one_stop: ')',
+    zero_or_more_start: 'maybe_some(',
+    zero_or_more_stop: ')',
+    one_or_more_start: 'some(',
+    one_or_more_stop: ')',
+    alternation_start: 'any([',
+    alternation_stop: '])',
+    alternative_start: '',
+    alternative_stop: '',
+    boundary: 'boundary',
+    non_word_boundary: 'non_word_boundary',
+    space: 'space',
+    line_start: 'line_start',
+    line_end: 'line_end',
+    any_character: 'any_character',
+    range_start: 'any(',
+    range_mid: '..',
+    range_stop: ')',
+    backref_num_start: 'ref(',
+    backref_num_stop: ')'
+  }
+
+  str = []
+  tokens.each_with_index do |token, idx|
+    prev = tokens[idx - 1] if idx > 0
+    left_side = %i[characterset_stop capture_stop
+           zero_or_one_stop zero_or_more_stop one_or_more_stop
+           boundary non_word_boundary
+           line_start any_character range_stop space
+                                           backref_num_stop]
+    right_side = %i[characterset_start capture_start
+            zero_or_one_start zero_or_more_start one_or_more_start
+            boundary non_word_boundary
+            line_end any_character range_start space
+                                       backref_num_start]
+    #if prev==:range_stop and token==:range_start
+    #  str << ' :adding_ranges '
+    #end
+    if (prev.instance_of?(String) && right_side.include?(token)) or
+      (left_side.include?(prev) && token.instance_of?(String)) or
+      (left_side.include?(prev) && right_side.include?(token))
+      str << ' + '
+    end
+    str << ', ' if prev == :alternative_stop and token == :alternative_start
+    # str << '[' if prev == :characterset_start and token == :range_start
+    # str << ']' if prev == :range_stop and token ==:characterset_stop
+    if subs.include? token
+      str << subs[token]
+    elsif token.instance_of?(String)
+      if prev.instance_of?(String)
+        str[-1] = "#{str[-1][0..-2]}#{token}\""
+      else
+        str << "\"#{token}\""
+      end
+    else
+      str << " #{token.inspect} "
+    end
+    # puts [idx, token].inspect
+    # puts str.inspect
+  end
+  str.join.gsub('\\\\u', '\\u')
+end
+
+def stringify_root(root, indent: 0)
+  warning = ''
+  root[:from] = [""] if root[:from] == []
+  str = " "*indent+"sub #{stringify(root[:from])}, #{root[:to]}"
+  [:before, :not_before, :after, :not_after].each do |look|
+    # puts "#{look.inspect} = #{root[look]}"
+    next unless root[look]
+    str_look = stringify(root[look])
+    str_look = "\"\"" if root[look] == [] || root[look] == nil
+    #if str_look.empty?  #apparently it is empty sometimes. iso-mal-Mlym-Latn for example
+    #  warning << "warning: #{look} is empty string;"
+    #else
+      str << ", #{look}: #{str_look}"
+    #end
+  end
+  str = " "*indent+"# #{str} # warning: :" if str =~ /[^\[]:[^ \]]/
+  str = " "*indent+"# #{str} # #{warning}" if !warning.empty?
+
+  str = " "*indent+"# #{str} # warning: :missing unimplemented" if str.include?(':missing')
+  str = " "*indent+"# #{str} # warning: :interval unimplemented" if str.include?(':interval')
+  str = " "*indent+"# #{str} # warning: :adding_ranges unimplemented" if str.include?(':adding_ranges')
+  if str.include?('zero_or_one')
+    str = " "*indent+"# #{str} # warning: zero_or_one"
+    puts "str.includes 'zero_or_one'"
+    pp root
+  end
+  # str = " "*indent+"# #{str} # warning: one_or_more" if str.include?('one_or_more')
+   str = " "*indent+"# #{str} # warning: :lookahead_start" if str.include?(':lookahead_start')
+  # str += " # original: #{root[:from]}"
+  str
+end
+
+if __FILE__ == $0
+  rs = File.open(__dir__+"/../../docs/utils/regexp_examples.txt").read.gsub(/([^\\^])\\u/, '\\1\\\\u').gsub(/\\\\b/, '\b')
+  rs = rs.split("\n")
+  rs.each do |r|
+    puts r
+    tree = Regexp::Parser.parse(r, 'ruby/2.1')
+    conv = process(tree)
+    pp conv
+    root = process_root(conv)
+    pp root
+    root[:to] = ['X']
+    str = stringify_root(root)
+    puts str
+    puts "\n\n"
+  end
+end

data/lib/interscript/version.rb

@@ -1,3 +1,3 @@
 module Interscript
-  VERSION = "0.1.7"
+  VERSION = "2.1.0b1"
 end

data/lib/interscript/visualize.rb

@@ -0,0 +1,61 @@
+require 'erb'
+require 'interscript/visualize/nodes'
+require 'interscript/visualize/json'
+
+def h(str)
+  str.to_s.gsub("&", "&amp;").gsub("<", "&lt;").gsub(">", "&gt;").gsub('"', "&quot;")
+end
+
+class Interscript::Visualize
+  def self.def_template(template)
+    @template = ERB.new(File.read(__dir__+"/visualize/#{template}.html.erb"))
+  end
+  def get_binding; binding; end
+
+  def self.call(*args)
+    return Map.(*args) if self == Interscript::Visualize
+
+    tplctx = self.new(*args)
+    @template.result(tplctx.get_binding)
+  end
+
+  class Map < self
+    def_template :map
+
+    def initialize(map_name)
+      @map = Interscript.parse(map_name)
+    end
+
+    attr_reader :map
+
+    def render_stage(map_name, stage)
+      Stage.(map_name, stage)
+    end
+  end
+
+  class Group < self
+    def_template :group
+
+    def initialize(map, group, style=nil)
+      @map = map
+      @group = group
+      @style = style
+    end
+
+    attr_reader :map, :group
+
+    def render_group(map, group, style=nil)
+      Group.(map, group, style)
+    end
+  end
+
+  class Stage < Group
+    def_template :group
+
+    def initialize(map_name, stage_name, style=nil)
+      @map = Interscript.parse(map_name)
+      @group = map.stages[stage_name]
+      @style = style
+    end
+  end  
+end

data/lib/interscript/visualize/group.html.erb

@@ -0,0 +1,59 @@
+<table style='<%= @style %>'>
+  <% group.children.each do |rule| %>
+    <tr>
+      <% case rule
+         when Interscript::Node::Rule::Sub %>
+        <td>
+          Replace
+        <td>
+          <%= rule.from.to_html(map) %>
+        <td>
+          <%= Symbol === rule.to ? rule.to : rule.to.to_html(map) %>
+        <td>
+          <%=
+            out = []
+            out << "before: #{rule.before.to_html(map)}" if rule.before
+            out << "after: #{rule.after.to_html(map)}" if rule.after
+            out << "<nobr>not before:</nobr> #{rule.not_before.to_html(map)}" if rule.not_before
+            out << "<nobr>not after:</nobr> #{rule.not_after.to_html(map)}" if rule.not_after
+            out.join(", ")
+          %>
+      <% when Interscript::Node::Group::Parallel %>
+        <td>
+          Parallel
+        <td colspan='3'>
+          <%= render_group map, rule %>
+        </td>
+      <% when Interscript::Node::Rule::Funcall %>
+        <td>
+          <%= h rule.name.to_s.gsub("_", " ").gsub(/^(.)/, &:upcase) %>
+        <td>
+          <%=
+            rule.kwargs.map do |k,v|
+              "#{k.to_s.gsub("_", " ")}: #{v}"
+            end.join(", ")
+          %>
+      <% when Interscript::Node::Rule::Run %>
+        <td>
+          <nobr>Run</nobr>
+        <td>
+          <a href='#' onclick='this.parentNode.lastElementChild.style.display=this.parentNode.lastElementChild.style.display=="none"?"table":"none";return!1'>
+            <%= rule.stage.to_html(map) %>
+          </a>
+
+          <%=
+            if rule.stage.map
+              doc = map.dep_aliases[rule.stage.map].document
+              stage = doc.imported_stages[rule.stage.name]
+            else
+              doc = map
+              stage = map.imported_stages[rule.stage.name]
+            end
+
+            render_group doc, stage, "display: none"
+          %>
+      <% else %>
+        <td colspan='4'><pre><%= h rule.inspect %></pre>
+      <% end %>
+  <% end %>
+</table>

data/lib/interscript/visualize/json.rb

@@ -0,0 +1,57 @@
+class Interscript::Node::Group
+  def to_visualization_array(map=self)
+    out = []
+
+    self.children.each do |rule|
+      case rule
+      when Interscript::Node::Rule::Sub
+        more = []
+        more << "before: #{rule.before.to_html(map)}" if rule.before
+        more << "after: #{rule.after.to_html(map)}" if rule.after
+        more << "<nobr>not before:</nobr> #{rule.not_before.to_html(map)}" if rule.not_before
+        more << "<nobr>not after:</nobr> #{rule.not_after.to_html(map)}" if rule.not_after
+        more = more.join(", ")
+
+        out << {
+          type: "Replace",
+          from: rule.from.to_html(map),
+          to: Symbol === rule.to ? rule.to : rule.to.to_html(map),
+          more: more
+        }
+      when Interscript::Node::Group::Parallel
+        out << {
+          type: "Parallel",
+          children: rule.to_visualization_array(map)
+        }
+      when Interscript::Node::Rule::Funcall
+        out << {
+          type: rule.name.to_s.gsub("_", " ").gsub(/^(.)/, &:upcase),
+          more: rule.kwargs.map do |k,v|
+            "#{k.to_s.gsub("_", " ")}: #{v}"
+          end.join(", ")
+        }
+      when Interscript::Node::Rule::Run
+        if rule.stage.map
+          doc = map.dep_aliases[rule.stage.map].document
+          stage = rule.stage.name
+        else
+          doc = map
+          stage = rule.stage.name
+        end
+
+        out << {
+          type: "Run",
+          doc: doc.name,
+          stage: stage
+        }
+      else
+        out << {
+          type: "Unknown",
+          more: "<pre>#{h rule.inspect}</pre>"
+        }
+      end
+    end
+
+    out
+  end
+end