regexp_parser 0.1.0

data/lib/regexp_parser/syntax.rb

@@ -0,0 +1,172 @@
+module Regexp::Syntax
+  require File.expand_path('../syntax/tokens', __FILE__)
+
+  class SyntaxError < StandardError
+    def initialize(what)
+      super what
+    end
+  end
+
+  class UnknownSyntaxNameError < SyntaxError
+    def initialize(name)
+      super "Unknown syntax name '#{name}'"
+    end
+  end
+
+  class MissingSyntaxSpecError < SyntaxError
+    def initialize(name)
+      super "Missing syntax specification file for '#{name}'"
+    end
+  end
+
+  class NotImplementedError < SyntaxError
+    def initialize(syntax, type, token)
+      super "#{syntax.class.name} does not implement: [#{type}:#{token}]"
+    end
+  end
+
+  SYNTAX_SPEC_ROOT = File.expand_path('../syntax', __FILE__)
+
+  # Loads, and instantiates an instance of the syntax specification class for
+  # the given syntax flavor name. The special names 'any' and '*' returns a
+  # instance of Syntax::Any. See below for more details.
+  def self.new(name)
+    return Regexp::Syntax::Any.new if
+      ['*', 'any'].include?( name.to_s )
+
+    self.load(name)
+
+    case name
+      when 'ruby/1.8.6';  syntax = Regexp::Syntax::Ruby::V186.new
+      when 'ruby/1.8.7';  syntax = Regexp::Syntax::Ruby::V187.new
+
+      # alias for the latest 1.8 implementation
+      when 'ruby/1.8';    syntax = Regexp::Syntax::Ruby::V18.new
+
+      when 'ruby/1.9.1';  syntax = Regexp::Syntax::Ruby::V191.new
+      when 'ruby/1.9.2';  syntax = Regexp::Syntax::Ruby::V192.new
+      when 'ruby/1.9.3';  syntax = Regexp::Syntax::Ruby::V193.new
+
+      # alias for the latest 1.9 implementation
+      when 'ruby/1.9';    syntax = Regexp::Syntax::Ruby::V19.new
+
+      else
+        raise UnknownSyntaxError.new(name)
+    end
+  end
+
+  # Checks if the named syntax has a specification class file, and requires
+  # it if it does. Downcases names, and adds the .rb extension if omitted.
+  def self.load(name)
+    full = "#{SYNTAX_SPEC_ROOT}/#{name.downcase}"
+    full = (full[-1, 3] == '.rb') ? full : "#{full}.rb"
+
+    raise MissingSyntaxSpecError.new(name) unless File.exist?(full)
+    require full
+  end
+
+  # A lookup map of supported types and tokens in a given syntax
+  class Base
+    def initialize
+      @implements = {}
+
+      implements :literal, [:literal]
+    end
+
+    def implementation
+      @implements
+    end
+
+    def implements(type, tokens)
+      if @implements[type]
+        @implements[type] = (@implements[type] + tokens).uniq
+      else
+        @implements[type] = tokens
+      end
+    end
+
+    # removes
+    def excludes(type, tokens)
+      if tokens
+        tokens = [tokens] unless tokens.is_a?(Array)
+      end
+
+      if @implements[type]
+        if tokens
+          @implements[type] = @implements[type] - tokens
+          @implements[type] = nil if @implements[type].empty?
+        else
+          @implements[type] = nil
+        end
+      end
+    end
+
+    def implements?(type, token)
+      return true if @implements[type] and @implements[type].include?(token)
+      false
+    end
+    alias :check? :implements?
+
+    def implements!(type, token)
+      raise NotImplementedError.new(self, type, token) unless
+        implements?(type, token)
+    end
+    alias :check! :implements!
+
+    def normalize(type, token)
+      case type
+      when :group
+        normalize_group(type, token)
+      when :backref
+        normalize_backref(type, token)
+      else
+        [type, token]
+      end
+    end
+
+    def normalize_group(type, token)
+      case token
+      when :named_ab, :named_sq
+        [:group, :named]
+      else
+        [type, token]
+      end
+    end
+
+    def normalize_backref(type, token)
+      case token
+      when :name_ref_ab, :name_ref_sq
+        [:backref, :name_ref]
+      when :name_call_ab, :name_call_sq
+        [:backref, :name_call]
+      when :name_nest_ref_ab, :name_nest_ref_sq
+        [:backref, :name_nest_ref]
+      when :number_ref_ab, :number_ref_sq
+        [:backref, :number_ref]
+      when :number_call_ab, :number_call_sq
+        [:backref, :number_call]
+      when :number_rel_ref_ab, :number_rel_ref_sq
+        [:backref, :number_rel_ref]
+      when :number_rel_call_ab, :number_rel_call_sq
+        [:backref, :number_rel_call]
+      when :number_nest_ref_ab, :number_nest_ref_sq
+        [:backref, :number_nest_ref]
+      else
+        [type, token]
+      end
+    end
+  end
+
+  # A syntax that always returns true, passing all tokens as implemented. This
+  # is useful during development, testing, and should be useful for some types
+  # of transformations as well.
+  class Any < Base
+    def initialize
+      @implements = { :* => [:*] }
+    end
+
+    def implements?(type, token) true end
+    def implements!(type, token) true end
+  end
+
+end

data/lib/regexp_parser.rb

@@ -0,0 +1,45 @@
+class Regexp
+
+  module Parser
+    VERSION = '0.0.1'
+  end
+
+  TOKEN_KEYS = [:type, :token, :text, :ts, :te, :depth, :set_depth].freeze
+  Token = Struct.new(*TOKEN_KEYS) do
+    def offset
+      [self.ts, self.te]
+    end
+
+    def length
+      self.te - self.ts
+    end
+
+    def to_h
+      hash = {}
+      members.each do |member|
+        hash[member.to_sym] = self.send(member.to_sym)
+      end; hash
+    end
+
+    def next(exp = nil)
+      if exp
+        @next = exp
+      else
+        @next
+      end
+    end
+
+    def previous(exp = nil)
+      if exp
+        @previous = exp
+      else
+        @previous
+      end
+    end
+  end
+
+end
+
+%w{ctype scanner syntax lexer parser}.each do |file|
+  require File.expand_path("../regexp_parser/#{file}", __FILE__)
+end

data/test/helpers.rb

@@ -0,0 +1,8 @@
+require "test/unit"
+require File.expand_path("../../lib/regexp_parser", __FILE__)
+
+RS = Regexp::Scanner
+RL = Regexp::Lexer
+RP = Regexp::Parser
+
+include Regexp::Expression

data/test/lexer/test_all.rb

@@ -0,0 +1,26 @@
+require File.expand_path("../../helpers", __FILE__)
+
+%w{
+  literals nesting refcalls
+}.each do|tc|
+  require File.expand_path("../test_#{tc}", __FILE__)
+end
+
+class TestRegexpLexer < Test::Unit::TestCase
+
+  def test_lexer_returns_an_array
+    assert_instance_of( Array, RL.scan('abc'))
+  end
+
+  def test_lexer_returns_tokens
+    tokens = RL.scan('^abc+[^one]{2,3}\b\d\\\C-C$')
+    assert( tokens.all?{|token| token.kind_of?(Regexp::Token)},
+          "Not all array members are tokens")
+  end
+
+  def test_lexer_token_count
+    tokens = RL.scan(/^(one|two){2,3}([^d\]efm-qz\,\-]*)(ghi)+$/i)
+    assert_equal( 26, tokens.length )
+  end
+
+end

data/test/lexer/test_literals.rb

@@ -0,0 +1,120 @@
+# -*- encoding: utf-8 -*-
+
+require File.expand_path("../../helpers", __FILE__)
+
+class LexerLiterals < Test::Unit::TestCase
+
+  tests = {
+    # ascii, single byte characters
+    'a' => {
+      0     => [:literal,     :literal,       'a',        0, 1, 0, 0],
+    },
+
+    'ab+' => {
+      0     => [:literal,     :literal,       'a',        0, 1, 0, 0],
+      1     => [:literal,     :literal,       'b',        1, 2, 0, 0],
+      2     => [:quantifier,  :one_or_more,   '+',        2, 3, 0, 0],
+    },
+
+
+    # 2 byte wide characters, Arabic
+    'ا' => {
+      0     => [:literal,     :literal,       'ا',        0, 2, 0, 0],
+    },
+
+    'aاbبcت' => {
+      0     => [:literal,     :literal,       'aاbبcت',   0, 9, 0, 0],
+    },
+
+    'aاbبت?' => {
+      0     => [:literal,     :literal,       'aاbب',     0, 6, 0, 0],
+      1     => [:literal,     :literal,       'ت',        6, 8, 0, 0],
+      2     => [:quantifier,  :zero_or_one,   '?',        8, 9, 0, 0],
+    },
+
+    'aا?bبcت+' => {
+      0     => [:literal,     :literal,       'a',        0, 1, 0, 0],
+      1     => [:literal,     :literal,       'ا',        1, 3, 0, 0],
+      2     => [:quantifier,  :zero_or_one,   '?',        3, 4, 0, 0],
+      3     => [:literal,     :literal,       'bبc',      4, 8, 0, 0],
+      4     => [:literal,     :literal,       'ت',        8, 10, 0, 0],
+      5     => [:quantifier,  :one_or_more,   '+',        10, 11, 0, 0],
+    },
+
+    'a(اbب+)cت?' => {
+      0     => [:literal,     :literal,       'a',        0, 1, 0, 0],
+      1     => [:group,       :capture,       '(',        1, 2, 0, 0],
+      2     => [:literal,     :literal,       'اb',       2, 5, 1, 0],
+      3     => [:literal,     :literal,       'ب',        5, 7, 1, 0],
+      4     => [:quantifier,  :one_or_more,   '+',        7, 8, 1, 0],
+      5     => [:group,       :close,         ')',        8, 9, 0, 0],
+      6     => [:literal,     :literal,       'c',        9, 10, 0, 0],
+      7     => [:literal,     :literal,       'ت',        10, 12, 0, 0],
+      8     => [:quantifier,  :zero_or_one,   '?',        12, 13, 0, 0],
+    },
+
+
+    # 3 byte wide characters, Japanese
+    'ab?れます+cd' => {
+      0     => [:literal,     :literal,       'a',        0, 1, 0, 0],
+      1     => [:literal,     :literal,       'b',        1, 2, 0, 0],
+      2     => [:quantifier,  :zero_or_one,   '?',        2, 3, 0, 0],
+      3     => [:literal,     :literal,       'れま',     3, 9, 0, 0],
+      4     => [:literal,     :literal,       'す',       9, 12, 0, 0],
+      5     => [:quantifier,  :one_or_more,   '+',        12, 13, 0, 0],
+      6     => [:literal,     :literal,       'cd',       13, 15, 0, 0],
+    },
+
+
+    # 4 byte wide characters, Osmanya
+    '𐒀𐒁?𐒂ab+𐒃' => {
+      0     => [:literal,     :literal,       '𐒀',        0, 4, 0, 0],
+      1     => [:literal,     :literal,       '𐒁',        4, 8, 0, 0],
+      2     => [:quantifier,  :zero_or_one,   '?',        8, 9, 0, 0],
+      3     => [:literal,     :literal,       '𐒂a',       9, 14, 0, 0],
+      4     => [:literal,     :literal,       'b',        14, 15, 0, 0],
+      5     => [:quantifier,  :one_or_more,   '+',        15, 16, 0, 0],
+      6     => [:literal,     :literal,       '𐒃',        16, 20, 0, 0],
+    },
+
+    'mu𝄞?si*𝄫c+' => {
+      0     => [:literal,     :literal,       'mu',       0, 2, 0, 0],
+      1     => [:literal,     :literal,       '𝄞',        2, 6, 0, 0],
+      2     => [:quantifier,  :zero_or_one,   '?',        6, 7, 0, 0],
+      3     => [:literal,     :literal,       's',        7, 8, 0, 0],
+      4     => [:literal,     :literal,       'i',        8, 9, 0, 0],
+      5     => [:quantifier,  :zero_or_more,  '*',        9, 10, 0, 0],
+      6     => [:literal,     :literal,       '𝄫',        10, 14, 0, 0],
+      7     => [:literal,     :literal,       'c',        14, 15, 0, 0],
+      8     => [:quantifier,  :one_or_more,   '+',        15, 16, 0, 0],
+    },
+  }
+
+  count = 0
+  tests.each do |pattern, checks|
+    define_method "test_lex_literal_runs_#{count+=1}" do
+
+      tokens = RL.scan(pattern)
+      checks.each do |offset, token|
+        assert_equal( token, tokens[offset].to_a )
+      end
+
+    end
+  end
+
+  def test_lex_single_2_byte_char
+    tokens = RL.scan('ا+')
+    assert_equal( 2, tokens.length )
+  end
+
+  def test_lex_single_3_byte_char
+    tokens = RL.scan('れ+')
+    assert_equal( 2, tokens.length )
+  end
+
+  def test_lex_single_4_byte_char
+    tokens = RL.scan('𝄞+')
+    assert_equal( 2, tokens.length )
+  end
+
+end

data/test/lexer/test_nesting.rb

@@ -0,0 +1,107 @@
+require File.expand_path("../../helpers", __FILE__)
+
+class LexerNesting < Test::Unit::TestCase
+
+  tests = {
+    '(((b)))' => {
+      0     => [:group,       :capture,       '(',      0,  1, 0, 0],
+      1     => [:group,       :capture,       '(',      1,  2, 1, 0],
+      2     => [:group,       :capture,       '(',      2,  3, 2, 0],
+      3     => [:literal,     :literal,       'b',      3,  4, 3, 0],
+      4     => [:group,       :close,         ')',      4,  5, 2, 0],
+      5     => [:group,       :close,         ')',      5,  6, 1, 0],
+      6     => [:group,       :close,         ')',      6,  7, 0, 0],
+    },
+
+    '(\((b)\))' => {
+      0     => [:group,       :capture,       '(',      0,  1, 0, 0],
+      1     => [:escape,      :group_open,    '\(',     1,  3, 1, 0],
+      2     => [:group,       :capture,       '(',      3,  4, 1, 0],
+      3     => [:literal,     :literal,       'b',      4,  5, 2, 0],
+      4     => [:group,       :close,         ')',      5,  6, 1, 0],
+      5     => [:escape,      :group_close,   '\)',     6,  8, 1, 0],
+      6     => [:group,       :close,         ')',      8,  9, 0, 0],
+    },
+
+    '(?>a(?>b(?>c)))' => {
+      0     => [:group,       :atomic,        '(?>',    0,  3, 0, 0],
+      2     => [:group,       :atomic,        '(?>',    4,  7, 1, 0],
+      4     => [:group,       :atomic,        '(?>',    8, 11, 2, 0],
+      6     => [:group,       :close,         ')',     12, 13, 2, 0],
+      7     => [:group,       :close,         ')',     13, 14, 1, 0],
+      8     => [:group,       :close,         ')',     14, 15, 0, 0],
+    },
+
+    '(?:a(?:b(?:c)))' => {
+      0     => [:group,       :passive,       '(?:',    0,  3, 0, 0],
+      2     => [:group,       :passive,       '(?:',    4,  7, 1, 0],
+      4     => [:group,       :passive,       '(?:',    8, 11, 2, 0],
+      6     => [:group,       :close,         ')',     12, 13, 2, 0],
+      7     => [:group,       :close,         ')',     13, 14, 1, 0],
+      8     => [:group,       :close,         ')',     14, 15, 0, 0],
+    },
+
+    '(?=a(?!b(?<=c(?<!d))))' => {
+      0     => [:assertion,   :lookahead,     '(?=',    0,  3, 0, 0],
+      2     => [:assertion,   :nlookahead,    '(?!',    4,  7, 1, 0],
+      4     => [:assertion,   :lookbehind,    '(?<=',   8, 12, 2, 0],
+      6     => [:assertion,   :nlookbehind,   '(?<!',  13, 17, 3, 0],
+      8     => [:group,       :close,         ')',     18, 19, 3, 0],
+      9     => [:group,       :close,         ')',     19, 20, 2, 0],
+      10    => [:group,       :close,         ')',     20, 21, 1, 0],
+      11    => [:group,       :close,         ')',     21, 22, 0, 0],
+    },
+
+    '((?#a)b(?#c)d(?#e))' => {
+      0     => [:group,       :capture,       '(',      0,  1, 0, 0],
+      1     => [:group,       :comment,       '(?#a)',  1,  6, 1, 0],
+      3     => [:group,       :comment,       '(?#c)',  7, 12, 1, 0],
+      5     => [:group,       :comment,       '(?#e)', 13, 18, 1, 0],
+      6     => [:group,       :close,         ')',     18, 19, 0, 0],
+    },
+
+    'a[b-e]f' => {
+      1     => [:set,         :open,          '[',      1,  2, 0, 0],
+      2     => [:set,         :range,         'b-e',    2,  5, 0, 1],
+      3     => [:set,         :close,         ']',      5,  6, 0, 0],
+    },
+
+    '[a-w&&[^c-g]z]' => {
+      0     => [:set,         :open,          '[',      0,  1, 0, 0],
+      2     => [:set,         :intersection,  '&&',     4,  6, 0, 1],
+      3     => [:subset,      :open,          '[',      6,  7, 0, 1],
+      4     => [:subset,      :negate,        '^',      7,  8, 0, 2],
+      5     => [:subset,      :range,         'c-g',    8, 11, 0, 2],
+      6     => [:subset,      :close,         ']',     11, 12, 0, 1],
+      8     => [:set,         :close,         ']',     13, 14, 0, 0],
+    },
+
+    '[a[b[c[d-g]]]]' => {
+      0     => [:set,         :open,          '[',      0,  1, 0, 0],
+      1     => [:set,         :member,        'a',      1,  2, 0, 1],
+      2     => [:subset,      :open,          '[',      2,  3, 0, 1],
+      3     => [:subset,      :member,        'b',      3,  4, 0, 2],
+      4     => [:subset,      :open,          '[',      4,  5, 0, 2],
+      5     => [:subset,      :member,        'c',      5,  6, 0, 3],
+      6     => [:subset,      :open,          '[',      6,  7, 0, 3],
+      7     => [:subset,      :range,         'd-g',    7, 10, 0, 4],
+      8     => [:subset,      :close,         ']',     10, 11, 0, 3],
+      9     => [:subset,      :close,         ']',     11, 12, 0, 2],
+     10     => [:subset,      :close,         ']',     12, 13, 0, 1],
+     11     => [:set,         :close,         ']',     13, 14, 0, 0],
+    },
+  }
+
+  count = 0
+  tests.each do |pattern, checks|
+    define_method "test_lex_nesting_#{count+=1}" do
+
+      tokens = RL.scan(pattern, 'ruby/1.9')
+      checks.each do |offset, token|
+        assert_equal( token, tokens[offset].to_a )
+      end
+
+    end
+  end
+
+end