regextest 0.1.2

data/lib/pre-case-folding.rb

@@ -0,0 +1,101 @@
+# encoding: utf-8
+
+# Copyright (C) 2016 Mikio Ikoma
+
+require "pp"
+
+# A script for generating case-folding of Unicode
+# This uses tables of Unicode.org, i.e.  
+
+class RegextestPreCaseFolding
+  def self.generate(input_file, output_file)
+    # Get valid casefoldings from unicode table
+    case_foldings = read_unicode_case_folding("./contrib/unicode/CaseFolding.txt")
+    puts_unicode_case_folding('lib/regextest/front/case-folding.rb', case_foldings)
+  end
+
+  # Get list of case-folding pairs from Unicode.org table
+  def self.read_unicode_case_folding(file)
+    case_foldings = {}
+    read_unicode_line(file) do | line |
+      if md = line.match(/^(\h{4,6});\s*([CFST]); ([ \h]+);/)
+        code_point = md[1].to_i(16)
+        mapping = md[3].split(" ").map{|elem| elem.to_i(16)}
+        code_point_string = [code_point].pack("U*")
+        mapping_string = mapping.map{|elem| [elem].pack("U*")}.join("")
+        if /(?ai:#{code_point_string})/.match(mapping_string)
+          case_foldings[[code_point]] ||= []
+          case_foldings[[code_point]].push mapping
+          case_foldings[mapping] ||= []
+          case_foldings[mapping].push [code_point]
+        else
+          # puts "code=#{code_point_string}, map=#{mapping_string}"
+        end
+      else
+        raise "not matched line: #{line}"
+      end
+    end
+    # case_foldings.each do | key, value |
+    #   value.each do | elem |
+    #     puts "#{key.pack("U*")} #{key}: #{elem.pack("U*") } #{elem}"
+    #   end
+    # end
+    case_foldings
+  end
+
+  # common process for parsing tables of Unicode.org
+  def self.read_unicode_line(file)
+    content = open(file, 'r:BOM|UTF-8') {|f| f.read}  # ignore BOM header
+    content.split(/\r?\n/).each do | line |
+      next if(line.length == 0 || line[0..0] == '#')
+      yield(line)
+    end
+  end
+
+  # puts source to unicode.rb
+  def self.puts_unicode_case_folding(case_folding_file, case_folding)
+    
+    template =<<"    END_OF_TEMPLATE"
+      # encoding: utf-8
+      # DO NOT Modify This File Since Automatically Generated
+
+      # Range of Unicode
+      class Regextest::Front::CaseFolding
+        # return case foldings
+        def self.ignore_case(letter_array)
+          CASE_FOLDING_HASH[letter_array]
+        end
+        
+        # case folding hash [codepoint] => [[mapping_1], ...]
+        CASE_FOLDING_HASH =
+#{case_folding.inspect}
+      end
+
+      # Test suite (execute when this file is specified in command line)
+      if __FILE__ == $0 
+      end
+    END_OF_TEMPLATE
+    template.gsub!(/^      /, "")
+    File.open(case_folding_file, "w") do |fp|
+      fp.puts template
+    end
+    
+  end
+end
+
+input_file  = "./contrib/unicode/CaseFolding.txt"
+output_file = "./lib/regextest/front/case-folding.rb"
+
+
+RegextestPreCaseFolding.generate(input_file, output_file)
+
+# test code
+require "regextest"
+require "#{output_file}"
+
+if Regextest::Front::CaseFolding.ignore_case([65]) == [[97]]
+  puts "OK"
+else
+  puts "NG"
+  exit(1)
+end

data/lib/pre-posix-char-class.rb

@@ -0,0 +1,150 @@
+# encoding: utf-8
+
+# Copyright (C) 2016 Mikio Ikoma
+
+# CURRENTLY NOT USED
+
+require "pp"
+
+# 鬼雲のマニュアルからPOSIX文字クラスの定義を得る
+def get_onigmo_posix_char_class(file, hash)
+  content = IO.read(file)
+  if(!md = content.match(/\r?\n\d\.\s+Character\s+class.+?   Unicode Case:(.+?)\r?\n\r?\n\r?\n/m))
+    raise "#{file} format is unmatched"
+  end
+  posix_def = md[1]
+  posix_def.gsub!(/\r?\n+/m, "\n")
+  posix_def.gsub!(/\|\r?\n/m, "|")
+  posix_def.split(/\r?\n/).each do | line |
+    elems = line.split(/\s+/)
+    if(elems[1] && elems[1].match(/^\w+$/) && elems[2])
+      raise "Duplicated symbol #{elems[1]}" if hash[elems[1]]
+      hash[elems[1]] = elems[2..-1].join("")
+    end
+  end
+end
+
+# 鬼雲のマニュアルからUnicode文字クラスの一覧を得る
+def get_onigmo_unicode_propety_class(file, hash)
+  content = IO.read(file)
+  class_name = nil
+  content.split(/\r?\n/).each do | line |
+    if(line[0..0] == "*")
+      class_name = line[2..-1].gsub(/\W+/, "_")
+      class_name.chop! if(class_name[-1..-1] == "_")
+      next
+    end
+    next if(!class_name || line.length == 0)
+    prop_name = line.gsub(/^\s+/, "")
+    raise "Duplicated symbol #{prop_name}" if hash[prop_name]
+    hash[prop_name] = class_name.to_sym
+  end
+end
+
+hash = {}
+get_onigmo_posix_char_class("../contrib/onigmo/RE.txt", hash)
+# get_onigmo_unicode_propety_class("../contrib/onigmo/UnicodeProps.txt", hash)
+pp hash
+exit
+
+
+
+
+
+# Unicode定義ファイルの共通文法の処理
+def read_unicode_line(file)
+  content = IO.read(file)
+  content.split(/\r?\n/).each do | line |
+    next if(line.length == 0 || line[0..0] == '#')
+    yield(line)
+  end
+end
+
+# スクリプトファイルの読み込み
+def read_scripts(scripts_file, ranges)
+  read_unicode_line(scripts_file) do | line |
+    if(md = line.match(/^(\h{4,6})(?:\.\.(\h{4,6}))?\s+;\s+(\w+)\s+#\s+(\S+)\s+/))
+      range_start = md[1].hex
+      range_end   = (md[2])?(md[2].hex):(range_start)
+
+      script1 = md[3]
+      script2 = md[4]
+      script2 = "LC" if(script2 == "L&")
+      script3 = script2[0..0]
+
+      #puts "range: [#{range_start}:#{range_end}]\t#{script1}\t#{script2}"
+      [script1, script2, script3].each do | script |
+        if(ranges[script])
+          if(range_start == ranges[script][-1][1] + 1)
+            ranges[script][-1][1] = range_end
+          else
+            ranges[script].push [range_start, range_end]
+          end
+        else
+          ranges[script] = [[range_start, range_end]]
+        end
+      end
+        
+    else
+      raise "syntax error: #{line}"
+    end
+  end
+end
+
+# ブロックファイルの読み込み
+def read_blocks(blocks_file, ranges)
+  read_unicode_line(blocks_file) do | line |
+    if(md = line.match(/^(\h{4,6})\.\.(\h{4,6})\s*;\s+(.+)$/))
+      range_start = md[1].hex
+      range_end   = md[2].hex
+      block_name = "In_" + md[3].gsub(/\W/, "_")
+      if ranges[block_name]
+        raise "block name #{block_name} is already used"
+      else
+        ranges[block_name] = [[range_start, range_end]]
+      end
+    end
+  end
+end
+
+# Unicodeのスクリプト、ブロックに対応したTRangeのRubyソースの出力
+def puts_unicode_ranges(unicode_file, ranges)
+  ranges_source = ranges.keys.map { |class_name|
+    (" "*12) +
+    "hash[\"#{class_name}\"] = CharClass.new([" +
+    ( ranges[class_name].map{|range| "TRange.new(#{range[0]}, #{range[1]})"}.join(", ") ) +
+    "])"
+  }.join("\n")
+  
+  template =<<"  END_OF_TEMPLATE"
+    # encoding: utf-8
+    # DO NOT Modify This File Since Automatically Generated
+
+    # Unicodeのレンジ
+    module Regextest::Front::ParseUnicode
+      class Unicode
+        # ハッシュの生成
+        def self.ranges()
+          hash = {}
+#{ranges_source}
+          hash
+        end
+      end
+    end
+
+    # Test suite (execute when this file is specified in command line)
+    if __FILE__ == $0 
+    end
+  END_OF_TEMPLATE
+  template.gsub!(/^    /, "")
+  File.open(unicode_file, "w") do |fp|
+    fp.puts template
+  end
+  
+end
+
+ranges = {}
+read_scripts("./unicode/Scripts.txt", ranges)
+read_blocks("./unicode/Blocks.txt", ranges)
+puts_unicode_ranges('tst-reg-parse-unicode', ranges)
+# pp ranges

data/lib/pre-unicode.rb

@@ -0,0 +1,116 @@
+# encoding: utf-8
+
+# Copyright (C) 2016 Mikio Ikoma
+
+require "pp"
+
+# A script for generating character class of Unicode
+# It does not use tables of Unicode.org, 
+# but use result of Ruby Regexp execution
+
+class RegextestPreUnicode
+  def self.generate
+    # Get valid properties of Ruby
+    onig_properties = read_onig_properties("./contrib/Onigmo/UnicodeProps.txt")
+    ranges = get_ranges_of_properties(onig_properties)
+    puts_unicode_ranges('lib/regextest/front/unicode.rb', ranges)
+  end
+
+  # Get list of Unicode classes from Onigmo manual
+  def self.read_onig_properties(file)
+    content = IO.read(file)
+    class_name = nil
+    properties = {}
+    content.split(/\r?\n/).each_with_index do | line, i |
+      # Type or property
+      if(line[0..0] == "*")
+        class_name = line[2..-1].gsub(/\W+/, "_")
+        class_name.chop! if(class_name[-1..-1] == "_")
+        next
+      end
+      next if(!class_name || line.length == 0)
+      prop_name = line.gsub(/^\s+/, "").downcase
+      raise "Duplicated symbol #{prop_name}" if properties[prop_name]
+      begin
+        properties[prop_name] = { class: class_name, reg: /\p{#{prop_name}}+/ , ranges: []}
+      rescue RegexpError
+        # Somehow some property name fails. ignore as for now
+        warn "Regexp error at /\\p{#{prop_name}}/"
+      end
+      
+      # for debugging
+      # break if(i > 10)
+    end
+    properties
+  end
+
+  # output ruby source (using TRange) corresponding to scripts/blocks of Unicode
+  def self.get_ranges_of_properties(properties)
+    puts "\nGenerating Unicode table. It takes 1-2 minutes."
+    ranges = {}
+    
+    # form whole letter to array, then join all letters
+    # (concatinating string cause performance problem)
+    whole_letters_array = []
+    0.step(0x10ffff).each do | codepoint |
+      # skip surrogate part
+      next if (codepoint >= 0xd800 && codepoint <= 0xdfff)
+      whole_letters_array.push  [codepoint].pack("U*")
+    end
+    whole_letters = whole_letters_array.join("")
+
+    # scan string generated for each class
+    properties.each do | prop_name, value |
+      whole_letters.scan(value[:reg]) do | matched |
+        
+        value[:ranges].push (matched[0].unpack("U*")[0]..matched[-1].unpack("U*")[0])
+      end
+      # puts "#{prop_name}: #{value}"
+      ranges[prop_name] = value[:ranges]
+    end
+    ranges
+  end
+
+  # puts source to unicode.rb
+  def self.puts_unicode_ranges(unicode_file, ranges)
+    ranges_source = ranges.keys.map { |prop_name|
+      (" "*14) + "when \"#{prop_name}\"\n" +
+      (" "*16) + "([" +
+      ( ranges[prop_name].map{|range| "[#{range.begin}, #{range.end}]"}.join(", ") ) +
+      "])"
+    }.join("\n")
+    
+    template =<<"    END_OF_TEMPLATE"
+      # encoding: utf-8
+      # DO NOT Modify This File Since Automatically Generated
+
+      # Range of Unicode
+      class Regextest::Front::Unicode
+        # Generate hash of properties
+        def self.property(class_name)
+          case class_name.downcase
+#{ranges_source}
+          else
+            raise "Internal error. Class name (#\{class_name\}) not found"
+          end
+        end
+        
+        # enumerate char-set
+        def self.enumerate(class_name)
+          self.property(class_name).inject([]){|result,elem| result += (elem[0]..elem[1]).to_a}
+        end
+      end
+
+      # Test suite (execute when this file is specified in command line)
+      if __FILE__ == $0 
+      end
+    END_OF_TEMPLATE
+    template.gsub!(/^      /, "")
+    File.open(unicode_file, "w") do |fp|
+      fp.puts template
+    end
+    
+  end
+end
+
+RegextestPreUnicode.generate

data/lib/regextest.rb

@@ -0,0 +1,268 @@
+# encoding: utf-8
+
+# Copyright (C) 2016 Mikio Ikoma
+
+# This routine defines Regextest class
+class Regextest; end
+
+# Required classes
+require 'regextest/version'
+require 'regextest/common'
+require 'regextest/front'
+require 'regextest/regex-option'
+require 'regextest/back'
+require 'regextest/regexp'
+require 'timeout'
+
+class Regextest
+  include Regextest::Common
+
+  # exceptions
+  class RegextestError             < RuntimeError; end
+  class RegextestFailedToGenerate  < RuntimeError; end
+  class RegextestTimeout           < RuntimeError; end
+
+  # Constructor of Regextest class
+  # @param [String|Regexp] regex regular expression object (or string)
+  # @param [Hash] options parameters for generating
+  # @option options [Regextest::RegexOption] :reg_options Regex option parameter
+  # @option options [Fixnum] :seed seed for randomization
+  # @option options [TrueClass] :verification specify true (or not speficy) to verify generated string using ruby Regexp.
+  # @option options [FalseClass] :verification specify false if skip to verify generated string.
+  # @return [Regextest] constructed object
+  def initialize(regex, options = {})
+    @@parse_options = options
+    @@parse_options[:reg_options] ||= Regextest::RegexOption.new
+    @verification = (options && options[:verification] == false)?false:true
+    @reg_string = nil
+    @reg_exp = nil
+    
+    # Set seed for randomizing
+    @seed = set_seed_for_randomizing(@@parse_options[:seed])
+
+    # Covert to source string if necessary
+    set_regex(regex)
+
+    # Parse string
+    @front_end = Regextest::Front.new(@reg_string, @@parse_options)
+    
+    # To json (use json format for backend)
+    @json_obj = @front_end.get_json_obj
+    
+    # Prepare back-end process. (use generate method for generating string)
+    @back_end = Regextest::Back.new(@json_obj)
+    
+    @result = nil
+    @reason = nil
+  end
+  
+  # @!attribute [r] reason
+  #   Reason if failed to generate
+  #   @return [hash] return reasons if failed to generate
+  #   @return [nil] return nil unless error
+  attr_reader :reason
+  
+  # @!attribute [r] seed
+  #   Seed for randomization
+  #   @return [Fixnum] return seed for randomization
+  #   @return [nil] return nil if no seed provided
+  attr_reader :seed
+  
+  # Genetate string matched with specified regular expression
+  # @return [MatchData] if matched and verified.
+  # @return [String] if matched without verification (i.e. return unverified matched string).
+  # @return [nil] nil if failed to generate
+  # @raise [RuntimeError] if something wrong...
+  # @raise [Regextest::RegextestTimeout] if detected timeout while verification. Option 'verification: false' may be workaround.
+  def generate
+    TstConstRetryMax.times do
+    
+      # generate string
+      reset_random_called
+      @result = @back_end.generate
+      if !@result
+        TstLog "NG: Failed to generate"
+        @reason = :failed_to_generate
+        if !is_random?
+          raise(RegextestError, "It is impossible to generate sample string of #{@reg_string}.")
+        end
+        next
+      end
+
+      result_string = @result.pre_match + @result.match + @result.post_match
+      
+      # verify generated string
+      if @verification
+        @result = verify(result_string)    # returns a match-object
+        if !@result
+          TstLog "NG: Failed to verify"
+          @reason = :failed_to_verify
+          next
+        end
+        # break if @result is verified
+      else
+        @result = result_string            # returns a string
+      end
+      break
+    end
+    
+    if !@result
+      raise(RegextestFailedToGenerate, "Regextest failed to generate sample string of #{@reg_string}.")
+    end
+    @result
+  end
+  
+  # Get parsed result as JSON string
+  # @return [String] parsed result as JSON string
+  def to_json
+    @front_end.get_json_string
+  end
+  #---------------#
+  private
+  
+  # Set seed for randomizing
+  def set_seed_for_randomizing(seed)
+    if seed
+      raise "Invalid seed (#{seed}: #{seed.class}) specified" if !(Integer === seed)
+      srand seed
+      seed
+    else
+      srand   # return preset seed
+    end
+  end
+  
+  # Covert to source string if necessary
+  def set_regex(param)
+    case param
+    when String
+      if md = param.match(/^\/(.*)\/([imx]*)$/)
+        @reg_exp = eval(param)
+        @reg_string = @reg_exp.source
+      else
+        new_param = check_builtin(param)
+        @reg_string = new_param
+        @reg_exp = /#{@reg_string}/
+      end
+      @@parse_options[:reg_source] = @reg_string
+    when Regexp
+      @reg_exp = param
+      @@parse_options[:reg_options].set(@reg_exp.options)   # inner regex options have priorty
+      @reg_string = @reg_exp.source
+    else
+      raise "Error: string or regular expression required"
+    end
+  end
+
+  # add built-in functions if any
+  def check_builtin(param)
+    builtin_functions = {}
+    param.scan(/\\g[\<\'](_\w+_)[\>\']/) do | func_name |
+      builtin_functions[func_name[0]] = true
+    end
+    if builtin_functions.keys.size > 0
+      require 'regextest/front/builtin-functions'
+      functions = Regextest::Front::BuiltinFunctions.new
+      builtin_functions.keys.each do | func_name |
+        if func_string = functions.find_func(func_name)
+          param = param + func_string
+        else
+          raise "invalid built-in function name (#{func_name})"
+        end
+      end
+    end
+    param
+  end
+  
+  # Verifies the result
+  def verify(result_string)
+    md = nil
+    begin
+      timeout(TstConstTimeout){
+        md = @reg_exp.match(result_string)
+      }
+    rescue Timeout::Error => ex
+      raise(RegextestTimeout,
+            "Timeout(#{TstConstTimeout} sec) detected while verifying string(#{result_string}) matched with regex(#{@reg_exp}).")
+    end
+    
+    if(md)
+      # matched string sometime differs from expected one...
+      if(md.pre_match  != @result.pre_match || 
+         md.to_a[0]    != @result.match ||
+         md.post_match != @result.post_match)
+        @reason = :invalid_match_string
+        TstLog "WARN: Invalid matched string, expected <--> actual"
+        TstLog "  proc: #{md.pre_match.inspect}  <-->  #{@result.pre_match.inspect}"
+        TstLog "  body: #{md.to_a[0].inspect}  <-->  #{@result.match.inspect}"
+        TstLog "  succ: #{md.post_match.inspect}  <-->  #{@result.post_match.inspect}"
+      end
+    else
+      @reason = { rc: :not_matched, string: result_string}
+      raise("failed to generate. Not matched regex(#{@reg_string}) string(#{result_string.inspect})")
+    end
+    md
+  end
+end
+
+# Test program
+if __FILE__ == $0
+  # ruby regextest.rb 'regular-expression'    =>  regular-expression
+  # ruby regextest.rb '[ab]'                  =>  a
+  include Regextest::Common
+
+  begin
+    
+    regex = ARGV[0] || $<
+    if(regex == "reg")
+      regex = /ab # comment
+      [a-z]{5,10}
+      cd	   /ix
+    end
+    if(regex == "reg2")
+      regex = %r(
+      (?<name> [a-zA-Z_:]+ ){0}
+      (?<stag> < \g<name>  > ){0}
+      (?<content> ||\w+|\w+|\w+ (\g<element> | \w+)* ){0}
+      (?<etag> </ \k<name+1> >){0}
+      (?<element> \g<stag> \g<content>* \g<etag> ){0}
+      \g<element>
+      )x
+    end
+    
+    begin
+      if ARGV[1]
+        reg = eval "/#{regex}/#{ARGV[1]}"
+      else
+        reg = regex
+      end
+    rescue SyntaxError => ex
+      warn "Ruby Regexp: Syntax error: " + ex.message
+      reg = regex
+    end
+    
+    prog = Regextest.new(reg)
+    
+    10.times do
+      if(md = prog.generate)
+        puts "  " + TstMdPrint(md)     # md.string.inspect
+      else
+        puts "Failed to generate regex(#{reg})"
+      end
+    end
+  
+  rescue RegexpError => ex
+    $stderr.puts "Parse error. #{ex.message}"
+    exit(1)
+
+  rescue Regextest::RegextestTimeout => ex
+    $stderr.puts ex.message
+    exit(1)
+    
+  rescue RuntimeError => ex
+    # Error process. put error message and exit
+    $stderr.puts ex.message
+    exit(1)
+  end
+
+end
+