regexador 0.4.5

data/lib/regexador.rb

@@ -0,0 +1,79 @@
+class Regexador 
+  # Only a skeleton...
+end
+
+require_relative './regexador_parser'
+require_relative './regexador_xform'
+
+require 'parslet/convenience'
+
+class Regexador
+  def initialize(str, debug=false)
+    @code = str
+    if debug
+      puts
+      puts "---- Code: ------"
+      puts str
+      puts "-----------------"
+    end
+
+    @parser = Parser.new
+    meth = debug ? :parse_with_debug : :parse
+    @tree = @parser.send(meth, str)
+
+    xform = Transform.new
+    if debug
+      puts "\n\nParser gives:"
+      pp @tree
+    end
+
+    @regex_tree = xform.apply(@tree)
+    @regex_str  = @regex_tree.to_s
+    if debug
+      puts "\n\nTransform gives:"
+      pp @regex_tree
+    end
+
+    @regex = Regexp.compile(@regex_tree.to_s)
+  end
+
+  def to_regex
+    @regex
+  end
+
+  def match(str, hash={})
+    hash.each_pair do |var, val|
+      @regex_str.gsub!(/\(#{var}\)\{0\}/, val)
+    end
+    @regex = Regexp.compile(@regex_str) unless hash.empty?
+    result = @regex.match(str)
+    return nil if result.nil?
+
+    # Logic below may change...
+
+    names = result.names
+    obj = Object.new
+    klass = obj.singleton_class
+    names.each {|name| klass.class_eval { define_method(name) { result[name] } } }
+    klass.class_eval { define_method(:[]) {|*args| args.map {|cvar| result[name] } } }
+    obj
+  end
+
+  def match?(str, hash={})
+    !!match(str, hash)  # Return Boolean
+  end
+
+  def =~(other)
+    other = stringify(other)
+    raise ArgumentError unless String === other
+    match(other)
+  end
+
+  private
+ 
+  def stringify(obj)
+    return obj if String === obj
+    return obj.to_str if obj.respond_to?(:to_str)
+    return obj
+  end
+end

data/lib/regexador_parser.rb

@@ -0,0 +1,113 @@
+require 'parslet'
+
+abort "Require out of order" if ! defined? Regexador
+
+class Regexador::Parser < Parslet::Parser
+end
+
+require_relative './chars'    # These three files 
+require_relative './predefs'  #   reopen the class
+require_relative './keywords' #     Regexador::Parser
+
+class Regexador::Parser
+  rule(:space)         { match[" \t"].repeat(1) }
+  rule(:space?)        { space.maybe }
+  rule(:white)         { (endofline | match("\s")).repeat(1) }
+  rule(:white?)        { white.maybe }
+
+  rule(:lower)         { match('[a-z]') }
+  rule(:upper)         { match('[A-Z]') }
+
+  rule(:comment)       { cHASH >> space >> (cNEWLINE.absent? >> any).repeat(0) }
+  rule(:endofline)     { space? >> comment.maybe >> cNEWLINE }
+
+  rule(:digit)         { match('[0-9]') }
+  rule(:digits)        { digit.repeat(1) }
+  rule(:hexdigit)      { digit | match("[abcdef]") }
+  rule(:quoted)        { match('[^"]').repeat(0) }
+  rule(:single_quoted) { match("[^']").repeat(0) }
+  rule(:graph_char)    { match ("[[:graph:]]") }   # { match('[!-~]') }
+  rule(:name)          { keyword.absent? >> lower >> (lower | cUNDERSCORE | digit).repeat(0) }
+
+  rule(:variable)      { name.as(:var) }
+  rule(:capture_var)   { (cAT >> name.as(:cvar)) }
+  rule(:parameter)     { (cCOLON >> name.as(:param)) }
+
+  rule(:posix_class)   { cPERCENT >> name.as(:pclass) }
+
+  rule(:string)        { cQUOTE >> quoted.as(:string) >> cQUOTE }
+
+  rule(:simple_class)  { cSQUOTE >> single_quoted.as(:char_class) >> cSQUOTE }
+  rule(:negated_class) { cTILDE >> cSQUOTE >> single_quoted.as(:neg_class) >> cSQUOTE }
+  rule(:char_class)    { simple_class | negated_class }
+
+  rule(:number)        { digits }
+  rule(:numeric)       { number | variable | parameter }
+
+  rule(:codepoint)     { cAMPERSAND >> (hexdigit >> hexdigit >> hexdigit >> hexdigit).as(:unicode) }
+
+  rule(:char)          { (cTICK >> graph_char.as(:char)) | codepoint }
+
+  rule(:simple_range)  { char.as(:c1) >> cHYPHEN >> char.as(:c2) }
+  rule(:negated_range) { char.as(:nr1) >> cTILDE  >> char.as(:nr2) }
+  rule(:range)         { negated_range | simple_range }
+
+  rule(:negated_char)  { cTILDE  >> char.as(:nchar) }   #    ~`x means /[^x]/
+
+  rule(:capture)       { capture_var.as(:lhs) >> space? >> (cEQUAL >> space? >> pattern.as(:rhs)).maybe }
+
+  rule(:simple_pattern) { predef | range | negated_char | posix_class | string | 
+                        # X        `a-`c   ~`a            %name         "abc"    
+                          char_class | char | parameter | variable | capture }
+                        # 'abc'        `a     :param      xyz        @xyz = ...
+
+  rule(:qualifier)     { (kANY | kMANY | kMAYBE | kNOCASE | kWITHIN | kESCAPING).as(:qualifier) >> 
+                         fancy_pattern.as(:match_item) }
+
+# FIXME above: within and escaping can't really take an arbitrary pattern
+
+###
+  rule(:pos_lookahead) { kFIND >> space >> simple_pattern.as(:findpat_ahead) >> space >> 
+                         kWITH >> space >> simple_pattern.as(:pospat) }
+  rule(:neg_lookahead) { kFIND >> space >> simple_pattern.as(:findpat_ahead) >> space >> 
+                         kWITHOUT >> space >> simple_pattern.as(:negpat) }
+  rule(:pos_lookbehind) { kWITH >> space >> simple_pattern.as(:pospat) >> space >>
+                          kFIND >> space >> simple_pattern.as(:findpat_behind) }
+  rule(:neg_lookbehind) { kWITHOUT >> space >> simple_pattern.as(:negpat) >> space >>
+                          kFIND >> space >> simple_pattern.as(:findpat_behind) }
+  rule(:lookaround)     { pos_lookahead | neg_lookahead | pos_lookbehind | neg_lookbehind }
+###
+
+  rule(:repeat1)       { numeric.as(:num1) }
+  rule(:repeat2)       { repeat1 >> cCOMMA >> numeric.as(:num2) }
+  rule(:repetition)    { (repeat2 | repeat1) >> space? >> cTIMES >> space? >> fancy_pattern.as(:match_item) }
+
+  rule(:parenthesized) { cLPAREN >> space? >> pattern >> space? >> cRPAREN }
+
+  rule(:fancy_pattern) { space? >> (repetition | simple_pattern | qualifier | lookaround | parenthesized) >> space? }
+                       #            num          `~"'             keyword     find/with    (
+
+  rule(:concat)        { (fancy_pattern >> (space? >> fancy_pattern).repeat(0)).as(:sequence) }
+ 
+  rule(:pattern)       { (concat >> space? >> (cBAR >> space? >> concat).repeat(0)).as(:alternation) }
+
+  rule(:rvalue)        { pattern | numeric }   # a string is-a pattern
+
+  rule(:assignment)    { space? >> name.as(:var) >> space? >> cEQUAL >> space? >> rvalue.as(:rvalue) }
+
+  rule(:definitions)   { (endofline | assignment >> endofline).repeat(0) }
+
+  rule(:oneline_clause)   { space? >> kMATCH >> space? >> pattern >> kEND >> endofline.maybe }
+
+  rule(:single_line)      { endofline | space? >> pattern >> endofline }
+
+  rule(:multiline_clause) { space? >> kMATCH >> endofline >> single_line.repeat(1).as(:lines) >> space? >>
+                            kEND >> endofline.maybe }
+
+  rule(:match_clause)  { multiline_clause | oneline_clause }
+
+  rule(:program)       { definitions.as(:definitions) >> match_clause.as(:match) >> endofline.repeat(0) }
+
+  root(:program)
+end
+

data/lib/regexador_xform.rb

@@ -0,0 +1,180 @@
+require 'parslet'
+
+abort "Require out of order" if ! defined? Regexador
+
+class Regexador::Transform < Parslet::Transform
+  class Node
+    def self.make(*fields, &block)
+      klass = ::Class.new(self) do
+        fields.each {|field| attr_accessor field }
+        define_method(:fields) { fields.dup }
+        define_method(:to_s, &block)
+      end
+      klass
+    end
+
+    def initialize *values
+      fields.zip(values) {|f,v| self.send("#{f}=", v) }
+    end
+
+    def to_s
+      raise NotImplementedError, 
+            "Please implement #to_s for #{short_name}."
+    end
+
+    def to_str
+      to_s
+    end
+
+    def short_name
+      str = self.class.name
+      str[str.rindex('::')+2..-1]
+    end
+
+    def inspect
+      data = fields.map {|f| "#{f}=#{self.send(f).inspect}" }.join(', ')
+      short_name + "(" + data + ")"
+    end
+  end
+
+  # Later: Remember escaping for chars (char, c1, c2, nchar, ...)
+
+  XChar        = Node.make(:char) do 
+    Regexp.escape(char)
+  end
+
+  CharRange    = Node.make(:c1, :c2)    { "[#@c1-#@c2]" }
+  NegatedRange = Node.make(:nr1, :nr2)  { "[^#@nr1-#@nr2]" }
+  NegatedChar  = Node.make(:nchar)      { "[^#@nchar]" }    # More like a range really
+  POSIXClass   = Node.make(:pclass)     { "[[:#@pclass:]]" }
+  CharClass    = Node.make(:char_class) { "[#@char_class]" }
+  NegatedClass = Node.make(:neg_class)  { "[^#@neg_class]" }
+  Predefined   = Node.make(:pre) do 
+    sym = "p#@pre".to_sym
+    str = Regexador::Parser::Predef2Regex[sym]
+    raise "#@pre is not handled yet" if str.nil?
+    str
+  end
+
+  StringNode = Node.make(:string)                   { Regexp.escape(string.to_s) }
+  Repeat1    = Node.make(:num1, :match_item)        { "(#@match_item){#@num1}" }
+  Repeat2    = Node.make(:num1, :num2, :match_item) { "(#@match_item){#@num1,#@num2}" }
+  Any        = Node.make(:match_item)               { "(#@match_item)*" }
+  Many       = Node.make(:match_item)               { "(#@match_item)+" }
+  Maybe      = Node.make(:match_item)               { "(#@match_item)?" }
+  Nocase     = Node.make(:match_item)               { "((?i)#@match_item)" }
+
+  FindWith    = Node.make(:findpat_ahead, :pospat)  { "((?=#@findpat_ahead#@pospat)#@findpat_ahead)" }
+  FindWithout = Node.make(:findpat_ahead, :negpat)  { "((?!#@findpat#@negpat)#@findpat)" }
+  WithFind    = Node.make(:pospat, :findpat_behind) { "((?<=#@pospat)#@findpat)" }
+  WithoutFind = Node.make(:negpat, :findpat_behind) { "((?<!#@negpat)#@pospat)" }
+
+  Within     = Node.make(:delim)                    { "(#@delim.*?#@delim)" }   # /x[^y]*?y/ 
+  Escaping   = Node.make(:delim)                    { "\\#@delim|[^#@delim]*?#@delim" }
+# escaping `"         # /"(\\"|[^"])*?"/
+
+  Sequence    = Node.make(:elements) { elements.map(&:to_s).join }
+  Alternation = Node.make(:elements) { '(' + elements.map(&:to_s).join('|') + ')' }
+
+  Assignment = Node.make(:var, :rvalue)  { "" }  # Doesn't actually translate directly.
+  Usage      = Node.make(:var)           { Assignment.bindings[var.to_s].to_s }
+
+  Program    = Node.make(:definitions, :match) do 
+    # NOTE Since we're using to_s for conversion to regular expression, 
+    # debugging cannot be done using string interpolation, otherwise we 
+    # call things out of order just by debug-printing them! 
+    # 
+    # puts "In Program: #{match}"          # Don't do this
+    # puts "In Program: #{match.inspect}"  # But this is OK
+    definitions.each {|d| d.store }
+    match.to_s 
+  end
+
+  class Assignment < Node    # For clarity: Really already is-a Node
+    class << self
+      attr_accessor :bindings
+    end
+
+    def store
+      # puts "Storing #@var = #{@rvalue.inspect}"
+      hash = self.class.bindings ||= {}
+
+      hash[@var.to_s] = @rvalue          # Late binding
+      # hash[@var.to_s] = @rvalue.to_s   # Early binding
+      # Think about the difference... :)
+    end
+  end
+
+  Captured = Node.make(:cname, :pattern) { "(?<#@cname>#@pattern)" }
+  Backref = Node.make(:name) { "\\k<#@name>" }
+
+  Parameter = Node.make(:param) { "(#{param}){0}" }
+
+  PosAhead  = Node.make(:pla1, :pla2)  { "(?=#@pla1#@pla2)#@pla1" }
+  NegAhead  = Node.make(:nla1, :nla2)  { "(?!#@nla1#@nla2)#@nla1" }
+  PosBehind = Node.make(:plb1, :plb2)  { "(?<=#@plb1)#@plb2" }
+  NegBehind = Node.make(:nlb1, :nlb2)  { "(?<!#@nlb1)#@nlb2" }
+
+  # Actual transformation rules
+
+  rule(:char => simple(:ch))        { XChar.new(ch) }
+  rule(:unicode => simple(:hex4))   { StringNode.new("" << Integer("0x#{hex4}")) }
+
+  rule(:string => simple(:string))  { StringNode.new(string) }
+  # When the string is empty, parslet returns an empty array for lack of content. 
+  # Map that to the empty string node.
+  rule(:string => sequence(:string))  { StringNode.new('') }
+
+  rule(:c1 => simple(:c1), :c2 => simple(:c2)) { CharRange.new(c1, c2) }
+
+  rule(:nr1 => simple(:nr1), :nr2 => simple(:nr2)) { NegatedRange.new(nr1, nr2) }
+  rule(:nchar => simple(:nchar))  { NegatedChar.new(nchar) } # Don't forget escaping
+
+  rule(:pclass => simple(:pclass)) { POSIXClass.new(pclass) }
+
+  rule(:char_class => simple(:char_class)) { CharClass.new(char_class) }
+  rule(:neg_class => simple(:neg_class))   { NegatedClass.new(neg_class) }
+
+  rule(:predef => simple(:content)) { Predefined.new(content) }
+
+  rule(:num1 => simple(:num1), :match_item => simple(:match_item)) { Repeat1.new(num1, match_item) }
+  
+  rule(:num1 => simple(:num1), :num2 => simple(:num2), :match_item => simple(:match_item)) { Repeat2.new(num1, num2, match_item) }
+
+  rule(:qualifier => 'any',    :match_item => simple(:match_item)) { Any.new(match_item) }
+  rule(:qualifier => 'many',   :match_item => simple(:match_item)) { Many.new(match_item) }
+  rule(:qualifier => 'maybe',  :match_item => simple(:match_item)) { Maybe.new(match_item) }
+  rule(:qualifier => 'nocase', :match_item => simple(:match_item)) { Nocase.new(match_item) }
+  rule(:qualifier => 'within', :match_item => simple(:match_item)) { Within.new(match_item) }
+  rule(:qualifier => 'escaping', :match_item => simple(:match_item)) { Escaping.new(match_item) }
+
+  rule(:findpat_ahead => simple(:pla1), :pospat => simple(:pla2))  { PosAhead.new(pla1, pla2) }
+  rule(:findpat_ahead => simple(:nla1), :negpat => simple(:nla2))  { NegAhead.new(nla1, nla2) }
+  rule(:pospat => simple(:plb1), :findpat_behind => simple(:plb2)) { PosBehind.new(plb1, plb2) }
+  rule(:negpat => simple(:nlb1), :findpat_behind => simple(:nlb2)) { NegBehind.new(nlb1, nlb2) }
+
+  rule(:var => simple(:var), :rvalue => simple(:rvalue)) { Assignment.new(@var, @rvalue) }
+
+  rule(:param => simple(:param)) { Parameter.new(param) }
+
+  rule(:alternation => simple(:pattern))        { pattern }
+  rule(:alternation => sequence(:alternatives)) { Alternation.new(alternatives) }
+
+  rule(:sequence => simple(:element))    { element }
+  rule(:sequence => sequence(:elements)) { Sequence.new(elements) }
+
+  # A series of statements on different lines is also a sequence.
+  rule(:lines => sequence(:lines)) { Sequence.new(lines) }
+  
+  rule(:var => simple(:name)) { Usage.new(name) }
+
+  rule(:definitions => sequence(:definitions), :match => simple(:match)) { Program.new(definitions, match) }
+  rule(:definitions => sequence(:definitions), :match => sequence(:match)) { Program.new(definitions, match) }
+
+  # An expression of the form '@variable'
+  rule(:lhs => {:cvar => simple(:backref)}) { Backref.new(backref) }
+
+  # An expression of the form '@variable = expr'
+  rule(:lhs => {:cvar => simple(:cname)}, :rhs => simple(:pattern)) { Captured.new(cname, pattern) }
+end
+

data/spec/parsing_spec.rb

@@ -0,0 +1,174 @@
+# Encoding: UTF-8
+require './spec/testing'
+
+class Object
+  def succeeds
+    self.should_not == nil 
+  end
+end
+
+describe Regexador do
+  before(:all) do
+    @parser = Regexador::Parser.new 
+    @pattern = @parser.pattern
+  end
+  
+  describe "A special character" do
+    it "can be matched correctly" do
+      @parser.cSQUOTE.parse_with_debug("'").succeeds
+      @parser.cHASH.parse('#').succeeds
+      @parser.cNEWLINE.parse("\n").succeeds
+      @parser.cEQUAL.parse('=').succeeds
+    end
+  end
+
+  describe "An international character" do
+    it "can follow a backtick" do  # 
+      @parser.char.parse_with_debug("`æ").succeeds
+      @parser.char.parse("`ß").succeeds
+      @parser.char.parse("`ç").succeeds
+      @parser.char.parse("`ö").succeeds
+      @parser.char.parse("`ñ").succeeds
+    end
+  end
+
+  describe "A Unicode codepoint expression" do
+    it "can be matched" do
+      @parser.codepoint.parse_with_debug("&1234").succeeds
+      @parser.codepoint.parse('&beef').succeeds
+    end
+  end
+
+  describe "A predefined token" do
+    %w(BOS EOS START END).each do |token|
+      describe token do
+        it 'matches using pattern' do
+          @parser.pattern.parse_with_debug(token).succeeds
+        end
+      end
+    end
+  end
+  
+  describe "An assignment" do
+    it "can be parsed" do
+      @parser.assignment.parse("a = 5").succeeds
+      @parser.assignment.parse("a= 5").succeeds
+      @parser.assignment.parse("a =5").succeeds
+      @parser.assignment.parse("a=5").succeeds
+      @parser.assignment.parse("myvar = 'xyz'").succeeds
+      @parser.assignment.parse('var2 = "hello"').succeeds
+      @parser.assignment.parse('this_var = `x-`z').succeeds
+      @parser.assignment.parse_with_debug('pat = maybe many `x-`z').succeeds
+    end
+  end
+  
+  describe "A keyword used as a variable name" do
+    it "will not parse" do
+      @parser.assignment.should_not parse("end = 'hello'")
+  #   @parser.assignment.parse("endx = 'hello'")
+    end
+  end
+  
+  describe "A definition section" do
+    it "can be parsed" do
+      defs1 = "a = 5\nstr = \"hello\"\n"
+      @parser.definitions.parse_with_debug(defs1).succeeds
+      defs2 = <<-EOF
+        a = 5
+        # comment...
+        pat = maybe many `a-`c
+        # empty line follows:
+  
+        str = "hello"
+        # another comment...
+      EOF
+      @parser.definitions.parse_with_debug(defs2).succeeds
+    end
+  end
+
+  describe "A capture variable" do
+    it "can be parsed" do
+      str1 = "@myvar"
+      @parser.capture_var.parse(str1).succeeds
+    end
+  end
+
+  describe "A captured pattern" do
+    let(:prog) { "@myvar = maybe 'abc'" }
+
+    it "can be parsed (#capture)" do
+      @parser.capture.parse(prog).succeeds
+    end
+    it "can be parsed (#program)" do
+      @parser.parse("match #{prog} end").succeeds
+    end
+  end
+  describe "A back reference" do
+    let(:prog) { '@myvar' }
+
+    it 'can be parsed (#capture)' do
+      @parser.capture.parse(prog).succeeds
+    end
+    it 'can be parsed' do
+      @parser.parse("match #{prog} end").succeeds
+    end
+  end
+  
+  
+  describe "A one-line match clause" do
+    it "can be parsed" do
+      mc1 = <<-EOF
+        match `a~`x end
+      EOF
+      @parser.match_clause.parse_with_debug(mc1).succeeds
+    end
+  end
+  
+  describe "A multiline match clause" do
+    it "can be parsed" do
+      mc2 = <<-EOF
+        match 
+          `< "tag" WB 
+          any ~`>
+          # blah blah blah
+          "</" "tag" `> 
+        end
+      EOF
+      @parser.multiline_clause.parse_with_debug(mc2).succeeds
+    end
+  end
+  
+  describe "An entire one-line program" do
+    it "can be parsed" do
+      prog = "match `a-`f end"
+      @parser.parse_with_debug(prog).succeeds
+    end
+  end
+  
+  
+  describe "An entire program" do
+    it "can be parsed" do
+      prog1 = <<-EOF
+        dot = "."
+        num = "25" D5 | `2 D4 D | maybe D1 1,2*D
+        match WB num dot num dot num dot num WB end
+      EOF
+      @parser.program.parse_with_debug(prog1).succeeds
+  
+      prog2 = <<-EOF
+        # Warning: This one likely has errors!
+    
+        visa     = `4 12*D maybe 3*D
+        mc       = `5 D5 14*D
+        amex     = `3 '47' 13*D
+        diners   = `3 (`0 D5 | '68' D) 11*D
+        discover = `6 ("011" | `5 2*D) 12*D
+        jcb      = ("2131"|"1800"|"35" 3*D) 11*D
+    
+        match visa | mc | amex | diners | discover | jcb end
+      EOF
+      @parser.program.parse_with_debug(prog2).succeeds
+    end
+  end
+end
+