interscript 0.1.5 → 2.1.0a8

data/lib/interscript/node/metadata.rb

@@ -0,0 +1,18 @@
+class Interscript::Node::MetaData < Interscript::Node
+  attr_accessor :data
+  def initialize data={}
+    @data = data
+  end
+
+  def []=(k,v)
+    @data[k] = v
+  end
+  def [](k)
+    @data[k]
+  end
+
+  def to_hash
+    {:class => self.class.to_s,
+      :data => @data}
+  end
+end

data/lib/interscript/node/rule.rb

@@ -0,0 +1,6 @@
+class Interscript::Node::Rule < Interscript::Node
+end
+
+require "interscript/node/rule/sub"
+require "interscript/node/rule/run"
+require "interscript/node/rule/funcall"

data/lib/interscript/node/rule/funcall.rb

@@ -0,0 +1,18 @@
+class Interscript::Node::Rule::Funcall < Interscript::Node::Rule
+  attr_accessor :name, :kwargs
+  def initialize name, **kwargs
+    @name = name
+    @kwargs = kwargs
+  end
+
+  def to_hash
+    { :class => self.class.to_s,
+      :name => self.name,
+      :kwargs => self.kwargs
+    }
+  end
+
+  def inspect
+    "#{@name} #{kwargs.inspect[1..-2]}"
+  end
+end

data/lib/interscript/node/rule/run.rb

@@ -0,0 +1,15 @@
+class Interscript::Node::Rule::Run < Interscript::Node::Rule
+  attr_accessor :stage
+  def initialize stage
+    @stage = stage
+  end
+
+  def to_hash
+    { :class => self.class.to_s,
+      :stage => self.stage.to_hash }
+  end
+
+  def inspect
+    "run #{@stage.inspect}"
+  end
+end

data/lib/interscript/node/rule/sub.rb

@@ -0,0 +1,65 @@
+class Interscript::Node::Rule::Sub < Interscript::Node::Rule
+  attr_accessor :from, :to
+  attr_accessor :before, :not_before, :after, :not_after
+  attr_accessor :priority
+
+  def initialize from, to, before: nil, not_before: nil, after: nil, not_after: nil, priority: nil
+    self.from = Interscript::Node::Item.try_convert from
+    if to == :upcase
+      self.to = :upcase
+    else
+      self.to = Interscript::Node::Item.try_convert to
+    end
+
+    self.priority = priority
+
+    #raise TypeError, "Can't supply both before and not_before" if before && not_before
+    #raise TypeError, "Can't supply both after and not_after" if after && not_after
+
+    self.before = Interscript::Node::Item.try_convert(before) if before
+    self.after = Interscript::Node::Item.try_convert(after) if after
+    self.not_before = Interscript::Node::Item.try_convert(not_before) if not_before
+    self.not_after = Interscript::Node::Item.try_convert(not_after) if not_after
+  end
+
+  def max_length
+    len = self.from.max_length
+    len += self.before.max_length if self.before
+    len += self.after.max_length if self.after
+    len += self.not_before.max_length if self.not_before
+    len += self.not_after.max_length if self.not_after
+    len += self.priority if self.priority
+    len
+  end
+
+  def to_hash
+    puts self.from.inspect if $DEBUG
+    puts params.inspect if $DEBUG
+    { :class => self.class.to_s,
+      :from => self.from.to_hash,
+      :to => self.to == :upcase ? :upcase : self.to.to_hash,
+      :before => self.before&.to_hash,
+      :not_before => self.not_before&.to_hash,
+      :after => self.after&.to_hash,
+      :not_after => self.not_after&.to_hash,
+      :priority => self.priority
+    }
+  end
+
+  def inspect
+    out = "sub "
+    params = []
+    params << @from.inspect
+    if @to == :upcase
+      params << "upcase"
+    else
+      params << @to.inspect
+    end
+    params << "before: #{@before.inspect}" if @before
+    params << "after: #{@after.inspect}" if @after
+    params << "not_before: #{@not_before.inspect}" if @not_before
+    params << "not_after: #{@not_after.inspect}" if @not_after
+    params << "priority: #{@priority.inspect}" if @priority
+    out << params.join(", ")
+  end
+end

data/lib/interscript/node/stage.rb

@@ -0,0 +1,19 @@
+class Interscript::Node::Stage < Interscript::Node::Group::Sequential
+  attr_accessor :name, :doc_name
+
+  def initialize name = :main
+    @name = name
+    super()
+  end
+
+  def to_hash
+    { :class => self.class.to_s,
+      :name => name,
+      :children => @children.map{|x| x.to_hash} }
+  end
+
+  def inspect
+    name = "(#{@name})" if @name != :main
+    "stage#{name} {\n#{super}\n}"
+  end
+end

data/lib/interscript/node/tests.rb

@@ -0,0 +1,15 @@
+class Interscript::Node::Tests < Interscript::Node
+  attr_accessor :data
+  def initialize data=[]
+    @data = data
+  end
+
+  def <<(pair)
+    @data << pair
+  end
+
+  def to_hash
+    { :class => self.class.to_s,
+      :data => @data }
+  end
+end

data/lib/interscript/stdlib.rb

@@ -0,0 +1,211 @@
+class Interscript::Stdlib
+  ALIASES = {
+    any_character: '.',
+    none: "",
+    space: " ",
+    whitespace: "[\\b \\t\\0\\r\\n]",
+    boundary: "\\b",
+    non_word_boundary: "\\B",
+    word: "\\w",
+    not_word: "\\W",
+    alpha: "[a-zA-Z]",
+    not_alpha: "[^a-zA-Z]",
+    digit: "\\d",
+    not_digit: "\\D",
+    line_start: "^",
+    line_end: "$",
+    string_start: "\\A",
+    string_end: "\\z"
+  }
+
+  def self.re_only_alias?(a)
+    ! %i[none space].include?(a)
+  end
+
+  @treecache = {}
+
+  def self.parallel_regexp_compile(subs_hash)
+    # puts subs_hash.inspect
+    regexp = subs_hash.each_with_index.map do |p,i|
+      "(?<_%d>%s)" % [i,p[0]]
+    end.join("|")
+    subs_regexp = Regexp.compile(regexp)
+    # puts subs_regexp.inspect
+  end
+
+  def self.parallel_regexp_gsub(string, subs_regexp, subs_hash)
+    string.gsub(subs_regexp) do |match|
+      lm = Regexp.last_match
+      # Extract the match name
+      idx = lm.named_captures.compact.keys.first[1..-1].to_i
+      subs_hash[idx]
+    end
+  end
+
+  def self.parallel_regexp_gsub_debug(string, subs_regexp, subs_array)
+    # only gathering debug info, test data is available in maps_analyze_staging
+    $subs_matches = []
+    $subs_regexp = subs_regexp
+    #$subs_array = subs_array
+    string.gsub(subs_regexp) do |match|
+      lm = Regexp.last_match
+      # puts lm.inspect
+      # Extract the match name
+      matched = lm.named_captures.compact.keys.first
+      # puts matched.inspect
+      # puts [lm.begin(matched), lm.end(matched)].inspect
+      idx = matched[1..-1].to_i
+      debug_info = {begin: lm.begin(matched), end: lm.end(matched), idx: idx, result: subs_array[idx]}
+      $subs_matches << debug_info
+      subs_array[idx]
+    end
+  end
+
+
+  def self.parallel_replace_compile_hash(a)
+    h = {}
+    a.each do |from,to|
+      h[from] = to
+    end
+    h
+  end
+
+  def self.parallel_replace_hash(str,h)
+    newstr = ""
+    len = str.length
+    max_key_len = h.keys.map(&:length).max
+    i = 0
+    while i < len
+      max_key_len.downto(1).each do |checked_len|
+        substr = str[i,checked_len]
+        if h[substr]
+          newstr << h[substr]
+          i += substr.length
+        elsif checked_len==1
+          newstr << str[i,1]
+          i += 1
+        end
+      end
+    end
+    newstr
+  end
+
+  # hash can be either a hash or a hash-like array
+  def self.parallel_replace_compile_tree(hash)
+    hh = hash.hash
+    if @treecache[hh]
+      tree = @treecache[hh]
+    else
+      tree = {}
+      hash.each do |from, to|
+        from = Array(from)
+        from.each do |f|
+          branch = tree
+          chars = f.split("")
+          chars[0..-2].each do |c|
+            branch[c.ord] ||= {}
+            branch = branch[c.ord]
+          end
+          branch[chars.last.ord] ||= {}
+          branch[chars.last.ord][nil] = to
+        end
+      end
+      @treecache[hh] = tree
+    end
+  end
+
+  def self.parallel_replace_tree(str, tree)
+    newstr = ""
+    len = str.length
+    i = 0
+    while i < len
+      c = str[i]
+
+      sub = ""
+      branch = tree
+      match, repl = nil, nil
+
+      j = 0
+      while j < len-i
+        cc = str[i+j]
+        if branch.include? cc.ord
+          branch = branch[cc.ord]
+          sub << cc
+          if branch.include? nil
+            match = sub.dup
+            repl = branch[nil]
+          end
+          j += 1
+        else
+          break
+        end
+      end
+
+      if match
+        i += match.length
+        newstr << repl
+      else
+        newstr << c
+        i += 1
+      end
+    end
+    newstr
+  end
+
+  def self.parallel_replace(str, hash)
+    tree = parallel_replace_compile_tree(hash)
+    parallel_replace_tree(str, tree)
+  end
+
+  # On Windows at least, sort_by is non-deterministic. Let's add some determinism
+  # to our efforts.
+  def self.deterministic_sort_by_max_length(ary)
+    # Deterministic on Linux:
+    # ary.sort_by{ |rule| -rule.max_length }
+
+    ary.each_with_index.sort_by{ |rule,idx| -rule.max_length*100000 + idx }.map(&:first)
+  end
+
+  def self.available_functions
+    %i[title_case downcase compose decompose separate secryst]
+  end
+
+  module Functions
+    def self.title_case(output, word_separator: " ")
+      output = output.gsub(/^(.)/, &:upcase)
+      output = output.gsub(/#{word_separator}(.)/, &:upcase) unless word_separator == ''
+      output
+    end
+
+    def self.downcase(output, _:nil)
+      output.downcase
+    end
+
+    def self.compose(output, _:nil)
+      output.unicode_normalize(:nfc)
+    end
+
+    def self.decompose(output, _:nil)
+      output.unicode_normalize(:nfd)
+    end
+
+    def self.separate(output, separator: " ")
+      output.split("").join(separator)
+    end
+
+    @secryst_models = {}
+    def self.secryst(output, model:)
+      require "secryst" rescue nil # Try to load secryst, but don't fail hard if not possible.
+      unless defined? Secryst
+        raise StandardError, "Secryst is not loaded. Please read docs/Usage_with_Secryst.adoc"
+      end
+      Interscript.secryst_index_locations.each do |remote|
+        Secryst::Provisioning.add_remote(remote)
+      end
+      @secryst_models[model] ||= Secryst::Translator.new(model_file: model)
+      output.split("\n").map(&:chomp).map do |i|
+        @secryst_models[model].translate(i)
+      end.join("\n")
+    end
+  end
+end

data/lib/interscript/utils/regexp_converter.rb

@@ -0,0 +1,283 @@
+require 'regexp_parser'
+
+
+def process(node)
+  children = if node.respond_to?(:expressions) && node.expressions
+               children = node.expressions.map.each { |expr| process(expr) }
+             end
+  # puts node.inspect
+  out = case node
+        when Regexp::Expression::Root
+          children
+        when Regexp::Expression::Assertion::Lookbehind
+          [:lookbehind_start, children, :lookbehind_stop]
+        when Regexp::Expression::Assertion::NegativeLookbehind
+          [:negative_lookbehind_start, children, :negative_lookbehind_stop]
+        when Regexp::Expression::Assertion::Lookahead
+          [:lookahead_start, children, :lookahead_stop]
+        when Regexp::Expression::Assertion::NegativeLookahead
+          [:negative_lookahead_start, children, :negative_lookahead_stop]
+        when Regexp::Expression::Group::Capture
+          [:capture_start, children, :capture_stop]
+        when Regexp::Expression::CharacterSet
+          # puts children.inspect
+          if children.flatten.include? (:range_start) #or children.size > 1
+            [:characterset_start, :array_start, children, :array_stop, :characterset_stop]
+          else
+            [:characterset_start, children, :characterset_stop]
+          end
+        when Regexp::Expression::Alternation
+          [:alternation_start, children, :alternation_stop]
+        when Regexp::Expression::Alternative
+          [:alternative_start, children, :alternative_stop]
+        when Regexp::Expression::CharacterSet::Range
+          lit1 = node.expressions[0].text
+          lit2 = node.expressions[1].text
+          [:range_start, lit1, :range_mid, lit2, :range_stop]
+        when Regexp::Expression::Anchor::WordBoundary
+          :boundary
+        when Regexp::Expression::Anchor::NonWordBoundary
+          :non_word_boundary
+        when Regexp::Expression::EscapeSequence::Backspace
+          :boundary # most probably boundary
+        when Regexp::Expression::CharacterType::Space
+          :space
+        when Regexp::Expression::Anchor::BeginningOfLine
+          :line_start
+        when Regexp::Expression::Anchor::EndOfLine
+          :line_end
+        when Regexp::Expression::CharacterType::Any
+          :any_character
+        when Regexp::Expression::Literal
+          node.text
+        when Regexp::Expression::EscapeSequence::Literal
+          node.text
+        when Regexp::Expression::EscapeSequence::Codepoint
+          node.text
+        when Regexp::Expression::PosixClass
+          '[' + node.text + ']'
+        when Regexp::Expression::UnicodeProperty::Script
+          node.text
+        when Regexp::Expression::Backreference::Number # why is there a space before after node.number?
+          [:backref_num_start, node.number, :backref_num_stop]
+        else
+          out = [:missing, node.class]
+
+          out << children if node.respond_to? :expressions
+           if node.respond_to? :quantifier and node.quantifier
+            # TODO add quantifier support
+            pp node
+            # out << process(node.quantifier)
+          end
+          out
+        end
+  if node.respond_to?(:quantifier) && node.quantifier&.token.to_s == "interval" && node.quantifier.max == node.quantifier.min
+    out = [out] * node.quantifier.max
+  elsif node.respond_to?(:quantifier) && node.quantifier
+    qname = node.quantifier.token.to_s
+    out = ["#{qname}_start".to_sym, [out], "#{qname}_stop".to_sym]
+  end
+  out
+end
+
+def process_root(node)
+  node2 = node.dup
+  root = {}
+  if before = node.select { |x| x[0] == :lookbehind_start }
+    # root[:before] = before[1]
+    # node2.delete(before)
+    if before.size == 1
+      root[:before] = before[0][1]
+      node2.delete(before[0])
+    elsif before.size >1
+      # pp not_before
+
+      a = [:alternation_start]
+      a << before.map{|x| [:alternative_start, x[1], :alternative_stop] }
+      a << [:alternation_stop]
+      root[:before] = a
+      # pp root[:not_before]
+      before.each{|n| node2.delete(n)}
+    end
+
+  end
+  if not_before = node.select { |x| x[0] == :negative_lookbehind_start }
+    # root[:not_before] = not_before[1]
+    # node2.delete(not_before)
+
+    if not_before.size == 1
+      root[:not_before] = not_before[0][1]
+      node2.delete(not_before[0])
+    elsif not_before.size >1
+      # pp not_before
+
+      a = [:alternation_start]
+      a << not_before.map{|x| [:alternative_start, x[1], :alternative_stop] }
+      a << [:alternation_stop]
+      root[:not_before] = a
+      # pp root[:not_before]
+      not_before.each{|n| node2.delete(n)}
+    end
+  end
+  if after = node.select { |x| x[0] == :lookahead_start }
+    # root[:after] = after[1]
+    # node2.delete(after)
+
+    if after.size == 1
+      root[:after] = after[0][1]
+      node2.delete(after[0])
+    elsif after.size >1
+      # pp not_before
+
+      a = [:alternation_start]
+      a << after.map{|x| [:alternative_start, x[1], :alternative_stop] }
+      a << [:alternation_stop]
+      root[:after] = a
+      # pp root[:not_before]
+      after.each{|n| node2.delete(n)}
+    end
+
+  end
+  if not_after = node.select { |x| x[0] == :negative_lookahead_start }
+    # root[:not_after] = not_after[1]
+    # node2.delete(not_after)
+    if not_after.size == 1
+      root[:not_after] = not_after[0][1]
+      node2.delete(not_after[0])
+    elsif not_after.size >1
+      # pp not_after
+
+      a = [:alternation_start]
+      a << not_after.map{|x| [:alternative_start, x[1], :alternative_stop] }
+      a << [:alternation_stop]
+      root[:not_after] = a
+      # pp root[:not_after]
+      not_after.each{|n| node2.delete(n)}
+    end
+
+  end
+  root[:from] = node2
+  root
+end
+
+def stringify(node)
+  tokens = node.flatten
+  subs = {
+    characterset_start: 'any(',
+    characterset_stop: ')',
+    array_start: '[',
+    array_stop: ']',
+    capture_start: 'capture(',
+    capture_stop: ')',
+    zero_or_one_start: 'maybe(',
+    zero_or_one_stop: ')',
+    zero_or_more_start: 'maybe_some(',
+    zero_or_more_stop: ')',
+    one_or_more_start: 'some(',
+    one_or_more_stop: ')',
+    alternation_start: 'any([',
+    alternation_stop: '])',
+    alternative_start: '',
+    alternative_stop: '',
+    boundary: 'boundary',
+    non_word_boundary: 'non_word_boundary',
+    space: 'space',
+    line_start: 'line_start',
+    line_end: 'line_end',
+    any_character: 'any_character',
+    range_start: 'any(',
+    range_mid: '..',
+    range_stop: ')',
+    backref_num_start: 'ref(',
+    backref_num_stop: ')'
+  }
+
+  str = []
+  tokens.each_with_index do |token, idx|
+    prev = tokens[idx - 1] if idx > 0
+    left_side = %i[characterset_stop capture_stop
+           zero_or_one_stop zero_or_more_stop one_or_more_stop
+           boundary non_word_boundary
+           line_start any_character range_stop space
+                                           backref_num_stop]
+    right_side = %i[characterset_start capture_start
+            zero_or_one_start zero_or_more_start one_or_more_start
+            boundary non_word_boundary
+            line_end any_character range_start space
+                                       backref_num_start]
+    #if prev==:range_stop and token==:range_start
+    #  str << ' :adding_ranges '
+    #end
+    if (prev.instance_of?(String) && right_side.include?(token)) or
+      (left_side.include?(prev) && token.instance_of?(String)) or
+      (left_side.include?(prev) && right_side.include?(token))
+      str << ' + '
+    end
+    str << ', ' if prev == :alternative_stop and token == :alternative_start
+    # str << '[' if prev == :characterset_start and token == :range_start
+    # str << ']' if prev == :range_stop and token ==:characterset_stop
+    if subs.include? token
+      str << subs[token]
+    elsif token.instance_of?(String)
+      if prev.instance_of?(String)
+        str[-1] = "#{str[-1][0..-2]}#{token}\""
+      else
+        str << "\"#{token}\""
+      end
+    else
+      str << " #{token.inspect} "
+    end
+    # puts [idx, token].inspect
+    # puts str.inspect
+  end
+  str.join.gsub('\\\\u', '\\u')
+end
+
+def stringify_root(root, indent: 0)
+  warning = ''
+  root[:from] = [""] if root[:from] == []
+  str = " "*indent+"sub #{stringify(root[:from])}, #{root[:to]}"
+  [:before, :not_before, :after, :not_after].each do |look|
+    # puts "#{look.inspect} = #{root[look]}"
+    next unless root[look]
+    str_look = stringify(root[look])
+    str_look = "\"\"" if root[look] == [] || root[look] == nil
+    #if str_look.empty?  #apparently it is empty sometimes. iso-mal-Mlym-Latn for example
+    #  warning << "warning: #{look} is empty string;"
+    #else
+      str << ", #{look}: #{str_look}"
+    #end
+  end
+  str = " "*indent+"# #{str} # warning: :" if str =~ /[^\[]:[^ \]]/
+  str = " "*indent+"# #{str} # #{warning}" if !warning.empty?
+
+  str = " "*indent+"# #{str} # warning: :missing unimplemented" if str.include?(':missing')
+  str = " "*indent+"# #{str} # warning: :interval unimplemented" if str.include?(':interval')
+  str = " "*indent+"# #{str} # warning: :adding_ranges unimplemented" if str.include?(':adding_ranges')
+  if str.include?('zero_or_one')
+    str = " "*indent+"# #{str} # warning: zero_or_one"
+    puts "str.includes 'zero_or_one'"
+    pp root
+  end
+  # str = " "*indent+"# #{str} # warning: one_or_more" if str.include?('one_or_more')
+   str = " "*indent+"# #{str} # warning: :lookahead_start" if str.include?(':lookahead_start')
+  # str += " # original: #{root[:from]}"
+  str
+end
+
+if __FILE__ == $0
+  rs = File.open(__dir__+"/../../docs/utils/regexp_examples.txt").read.gsub(/([^\\^])\\u/, '\\1\\\\u').gsub(/\\\\b/, '\b')
+  rs = rs.split("\n")
+  rs.each do |r|
+    puts r
+    tree = Regexp::Parser.parse(r, 'ruby/2.1')
+    conv = process(tree)
+    pp conv
+    root = process_root(conv)
+    pp root
+    root[:to] = ['X']
+    str = stringify_root(root)
+    puts str
+    puts "\n\n"
+  end
+end