regexp_parser 0.1.6 → 0.2.0

data/Rakefile

@@ -18,7 +18,9 @@ Bundler::GemHelper.install_tasks
 task :default => [:test]
 
 Rake::TestTask.new('test') do |t|
-  t.description = "Run all unit tests under the test directory"
+  if t.respond_to?(:description)
+    t.description = "Run all unit tests under the test directory"
+  end
 
   t.libs << "test"
   t.test_files = FileList['test/test_all.rb']
@@ -27,7 +29,9 @@ end
 namespace :test do
   %w{scanner lexer parser expression syntax}.each do |component|
     Rake::TestTask.new(component) do |t|
-      t.description = "Run all #{component} unit tests under the test/#{component} directory"
+      if t.respond_to?(:description)
+        t.description = "Run all #{component} unit tests under the test/#{component} directory"
+      end
 
       t.libs << "test"
       t.test_files = ["test/#{component}/test_all.rb"]
@@ -35,7 +39,9 @@ namespace :test do
   end
 
   Rake::TestTask.new('full' => 'ragel:rb') do |t|
-    t.description = "Regenerate the scanner and run all unit tests under the test directory"
+    if t.respond_to?(:description)
+      t.description = "Regenerate the scanner and run all unit tests under the test directory"
+    end
 
     t.libs << "test"
     t.test_files = FileList['test/test_all.rb']

data/lib/regexp_parser.rb

@@ -1,12 +1,8 @@
-require 'yaml'
+# encoding: utf-8
 
-class Regexp
-  module Parser
-    VERFILE = File.expand_path('../../VERSION.yml', __FILE__)
-    VERSION = YAML.load(File.read(VERFILE)).values.compact.join('.')
-  end
-end
-
-%w{token ctype scanner syntax lexer parser}.each do |file|
-  require File.expand_path("../regexp_parser/#{file}", __FILE__)
-end
+require 'regexp_parser/version'
+require 'regexp_parser/token'
+require 'regexp_parser/scanner'
+require 'regexp_parser/syntax'
+require 'regexp_parser/lexer'
+require 'regexp_parser/parser'

data/lib/regexp_parser/expression.rb

@@ -2,18 +2,21 @@ module Regexp::Expression
 
   class Base
     attr_accessor :type, :token
-    attr_accessor :level, :text, :ts
+    attr_accessor :text, :ts
+    attr_accessor :level, :set_level, :conditional_level
 
     attr_accessor :quantifier
     attr_accessor :options
 
     def initialize(token)
-      @type         = token.type
-      @token        = token.token
-      @text         = token.text
-      @ts           = token.ts
-      @level        = token.level
-      @options      = nil
+      @type               = token.type
+      @token              = token.token
+      @text               = token.text
+      @ts                 = token.ts
+      @level              = token.level
+      @set_level          = token.set_level
+      @conditional_level  = token.conditional_level
+      @options            = nil
     end
 
     def clone
@@ -106,6 +109,47 @@ module Regexp::Expression
     end
     alias :x? :free_spacing?
     alias :extended? :free_spacing?
+
+    if RUBY_VERSION >= '2.0'
+      def default_classes?
+        (@options and @options[:d]) ? true : false
+      end
+      alias :d? :default_classes?
+
+      def ascii_classes?
+        (@options and @options[:a]) ? true : false
+      end
+      alias :a? :ascii_classes?
+
+      def unicode_classes?
+        (@options and @options[:u]) ? true : false
+      end
+      alias :u? :unicode_classes?
+    end
+
+    def matches?(string)
+      Regexp.new(to_s) =~ string ? true : false
+    end
+
+    def match(string, offset)
+      Regexp.new(to_s).match(string, offset)
+    end
+    alias :=~ :match
+
+    def to_h
+      {
+        :type               => @type,
+        :token              => @token,
+        :text               => to_s(:base),
+        :starts_at          => @ts,
+        :length             => full_length,
+        :level              => @level,
+        :set_level          => @set_level,
+        :conditional_level  => @conditional_level,
+        :options            => @options,
+        :quantifier         => quantified? ? @quantifier.to_h : nil
+      }
+    end
   end
 
   def self.parsed(exp)
@@ -125,10 +169,24 @@ module Regexp::Expression
 
 end # module Regexp::Expression
 
-
-[ # Order is important
-  '/expression/*.rb',
-  '/expression/classes/*.rb',
-].each do |path|
-  Dir[File.join(File.dirname(__FILE__), path)].each {|f| require f }
-end
+require 'regexp_parser/expression/methods/tests'
+require 'regexp_parser/expression/methods/traverse'
+require 'regexp_parser/expression/methods/strfregexp'
+
+require 'regexp_parser/expression/quantifier'
+require 'regexp_parser/expression/subexpression'
+require 'regexp_parser/expression/sequence'
+
+require 'regexp_parser/expression/classes/alternation'
+require 'regexp_parser/expression/classes/anchor'
+require 'regexp_parser/expression/classes/backref'
+require 'regexp_parser/expression/classes/conditional'
+require 'regexp_parser/expression/classes/escape'
+require 'regexp_parser/expression/classes/free_space'
+require 'regexp_parser/expression/classes/group'
+require 'regexp_parser/expression/classes/keep'
+require 'regexp_parser/expression/classes/literal'
+require 'regexp_parser/expression/classes/property'
+require 'regexp_parser/expression/classes/root'
+require 'regexp_parser/expression/classes/set'
+require 'regexp_parser/expression/classes/type'

data/lib/regexp_parser/expression/classes/alternation.rb

@@ -12,7 +12,7 @@ module Regexp::Expression
     end
 
     def alternative(exp = nil)
-      @expressions << (exp ? exp : Sequence.new)
+      @expressions << (exp ? exp : Alternative.new(level, set_level, conditional_level))
     end
 
     def alternatives
@@ -28,20 +28,7 @@ module Regexp::Expression
     end
   end
 
-  # A sequence of expressions, used by alternations as one alternative.
-  # TODO: perhaps rename this to Alternative?
-  class Sequence < Regexp::Expression::Subexpression
-    def initialize
-      super Regexp::Token.new(:expression, :sequence, '')
-    end
-
-    def starts_at
-      @expressions.first.starts_at
-    end
-
-    def quantify(token, text, min = nil, max = nil, mode = :greedy)
-      last.quantify(token, text, min, max, mode)
-    end
-  end
+  # A sequence of expressions, used by Alternation as one of its alternative.
+  class Alternative < Regexp::Expression::Sequence; end
 
 end

data/lib/regexp_parser/expression/classes/conditional.rb

@@ -0,0 +1,57 @@
+module Regexp::Expression
+
+  module Conditional
+    class TooManyBranches < StandardError
+      def initialize
+        super('The conditional expression has more than 2 branches')
+      end
+    end
+
+    class Condition < Regexp::Expression::Base; end
+    class Branch    < Regexp::Expression::Sequence; end
+
+    class Expression < Regexp::Expression::Subexpression
+      def initialize(token)
+        super(token)
+
+        @condition = nil
+        @branches  = []
+      end
+
+      def condition(exp = nil)
+        return @condition unless exp
+        @condition = exp
+        @expressions << exp
+      end
+
+      def <<(exp)
+        @expressions.last << exp
+      end
+
+      def branch(exp = nil)
+        raise TooManyBranches.new if @branches.length == 2
+
+        sequence = Branch.new(level, set_level, conditional_level + 1)
+
+        @expressions << sequence
+        @branches << @expressions.last
+      end
+
+      def branches
+        @branches
+      end
+
+      def quantify(token, text, min = nil, max = nil, mode = :greedy)
+        branches.last.last.quantify(token, text, min, max, mode)
+      end
+
+      def to_s
+        s = @text.dup
+        s << @condition.text
+        s << branches.map{|e| e.to_s}.join('|')
+        s << ')'
+      end
+    end
+  end
+
+end

data/lib/regexp_parser/expression/classes/free_space.rb

@@ -0,0 +1,17 @@
+module Regexp::Expression
+
+  class FreeSpace < Regexp::Expression::Base
+    def quantify(token, text, min = nil, max = nil, mode = :greedy)
+      raise "Can not quantify a free space object"
+    end
+  end
+
+  class Comment < Regexp::Expression::FreeSpace; end
+
+  class WhiteSpace < Regexp::Expression::FreeSpace
+    def merge(exp)
+      @text << exp.text
+    end
+  end
+
+end

data/lib/regexp_parser/expression/classes/keep.rb

@@ -0,0 +1,7 @@
+module Regexp::Expression
+
+  module Keep
+    class Mark          < Regexp::Expression::Base; end
+  end
+
+end

data/lib/regexp_parser/expression/classes/set.rb

@@ -72,6 +72,33 @@ module Regexp::Expression
       @closed
     end
 
+    # Returns an array of the members with any shorthand members like \d and \W
+    # expanded to either traditional form or unicode properties.
+    def expand_members(use_properties = false)
+      @members.map do |member|
+        case member
+        when "\\d"
+          use_properties ? '\p{Digit}'  : '0-9'
+        when "\\D"
+          use_properties ? '\P{Digit}'  : '^0-9'
+        when "\\w"
+          use_properties ? '\p{Word}'   : 'A-Za-z0-9_'
+        when "\\W"
+          use_properties ? '\P{Word}'   : '^A-Za-z0-9_'
+        when "\\s"
+          use_properties ? '\p{Space}'  : ' \t\f\v\n\r'
+        when "\\S"
+          use_properties ? '\P{Space}'  : '^ \t\f\v\n\r'
+        when "\\h"
+          use_properties ? '\p{Xdigit}' : '0-9A-Fa-f'
+        when "\\H"
+          use_properties ? '\P{Xdigit}' : '^0-9A-Fa-f'
+        else
+          member
+        end
+      end
+    end
+
     def to_s(format = :full)
       s = ''
 
@@ -80,18 +107,12 @@ module Regexp::Expression
       s << @members.join
       s << ']'
 
-      case format
-      when :base
-      else
+      unless format == :base
         s << @quantifier.to_s if quantified?
       end
 
       s
     end
-
-    def matches?(input)
-      input =~ /#{to_s}/ ? true : false
-    end
   end
 
   class CharacterSubSet < CharacterSet

data/lib/regexp_parser/expression/methods/strfregexp.rb

@@ -0,0 +1,113 @@
+module Regexp::Expression
+
+  class Base
+
+    #   %l  Level (depth) of the expression. Returns 'root' for the root
+    #       expression, returns zero or higher for all others.
+    #
+    #   %>  Indentation at expression's level.
+    #
+    #   %x  Index of the expression at its depth. Available when using
+    #       the sprintf_tree method only.
+    #
+    #   %s  Start offset within the whole expression.
+    #   %e  End offset within the whole expression.
+    #   %S  Length of expression.
+    #
+    #   %o  Coded offset and length, same as '@%s+%S'
+    #
+    #   %y  Type of expression.
+    #   %k  Token of expression.
+    #   %i  ID, same as '%y:%k'
+    #   %c  Class name
+    #
+    #   %q  Quantifier info, as {m[,M]}
+    #   %Q  Quantifier text
+    #
+    #   %z  Quantifier min
+    #   %Z  Quantifier max
+    #
+    #   %t  Base text of the expression (excludes quantifier, if any)
+    #   %~t Full text if the expression is terminal, otherwise %i
+    #   %T  Full text of the expression (includes quantifier, if any)
+    #
+    #   %b  Basic info, same as '%o %i'
+    #   %m  Most info, same as '%b %q'
+    #   %a  All info, same as '%m %t'
+    #
+    def strfregexp(format = '%a', indent_offset = 0, index = nil)
+      have_index    = index ? true : false
+
+      part = {}
+
+      # Order is important! Fields that use other fields in their
+      # definition must appear before the fields they use.
+      part_keys = %w{a m b o i l x s e S y k c q Q z Z t ~t T >}
+      part.keys.each {|k| part[k] = "<?#{k}?>"}
+
+      part['>'] = level ? ('  ' * (level + indent_offset)) : ''
+
+      part['l'] = level ? "#{'%d' % level}" : 'root'
+      part['x'] = "#{'%d' % index}" if have_index
+
+      part['s'] = starts_at
+      part['S'] = full_length
+      part['e'] = starts_at + full_length
+      part['o'] = coded_offset
+
+      part['k'] = token
+      part['y'] = type
+      part['i'] = '%y:%k'
+      part['c'] = self.class.name
+
+      if quantified?
+        if quantifier.max == -1
+          part['q'] = "{#{quantifier.min}, or-more}"
+        else
+          part['q'] = "{#{quantifier.min}, #{quantifier.max}}"
+        end
+
+        part['Q'] = quantifier.text
+        part['z'] = quantifier.min
+        part['Z'] = quantifier.max
+      else
+        part['q'] = '{1}'
+        part['Q'] = ''
+        part['z'] = '1'
+        part['Z'] = '1'
+      end
+
+      part['t'] = to_s(:base)
+      part['~t'] = terminal? ? to_s : "#{type}:#{token}"
+      part['T'] = to_s(:full)
+
+      part['b'] = '%o %i'
+      part['m'] = '%b %q'
+      part['a'] = '%m %t'
+
+      out = format.dup
+
+      part_keys.each do |k|
+        out.gsub!(/%#{k}/, part[k].to_s)
+      end
+
+      out
+    end
+
+    alias :strfre :strfregexp
+  end
+
+  class Subexpression < Regexp::Expression::Base
+    def strfregexp_tree(format = '%a', include_self = true, separator = "\n")
+      output = include_self ? [self.strfregexp(format)] : []
+
+      output += map {|exp, index|
+        exp.strfregexp(format, (include_self ? 1 : 0), index)
+      }
+
+      output.join(separator)
+    end
+
+    alias :strfre_tree :strfregexp_tree
+  end
+end