fabulator-grammar 0.0.1 → 0.0.3

data/lib/fabulator/grammar/actions/rule.rb

@@ -0,0 +1,51 @@
+module Fabulator
+  module Grammar
+    module Actions
+      class Rule < Fabulator::Structural
+
+        namespace GRAMMAR_NS
+
+        attribute :name, :static => true
+        attribute :mode, :default => "default", :static => true
+
+        contains :when, :as => :choices
+
+        has_actions
+
+        def initialize(g = nil)
+          @grammar = g
+          @choices = [ ]
+        end
+
+        def parse(cursor)
+          # try each when...
+          best_attempt = nil
+          @choices.each do |choice|
+            cursor.attempt do |c|
+              ret = choice.parse(c)
+              if !ret.nil?
+                score = choice.score(cursor.context, ret)
+                if best_attempt.nil? || best_attempt[:score] < score
+                  best_attempt = {
+                    :score => score,
+                    :choice => choice,
+                    :ret => ret
+                  }
+                end
+              end
+            end
+          end
+          return nil if best_attempt.nil?
+          choice = best_attempt[:choice]
+          ret = best_attempt[:ret]
+          if choice.has_actions?
+            ctx = cursor.context.with_root(cursor.context.root.anon_node(nil))
+            ctx.merge_data(ret)
+            choice.run(ctx)
+          end
+          return ret
+        end
+      end
+    end
+  end
+end

data/lib/fabulator/grammar/actions/token.rb

@@ -0,0 +1,27 @@
+module Fabulator
+  module Grammar
+    module Actions
+      class Token < Fabulator::Structural
+
+        namespace GRAMMAR_NS
+
+        attribute :name, :static => true
+        attribute :mode, :default => "default", :static => true
+        attribute :matches, :static => true
+
+        def compile_xml(xml, ctx = nil)
+          super
+
+          parser = Fabulator::Grammar::TokenParser.new
+
+          # parse @matches
+          @c_matches = parser.parse(self.matches)
+        end
+
+        def parse(cursor)
+          @c_matches.parse(cursor)
+        end
+      end
+    end
+  end
+end

data/lib/fabulator/grammar/actions/when.rb

@@ -0,0 +1,35 @@
+module Fabulator
+  module Grammar
+    module Actions
+      class When < Fabulator::Structural
+
+        namespace GRAMMAR_NS
+
+        attribute :matches, :static => true
+        attribute :score, :eval => true
+
+        has_actions
+
+        def compile_xml(xml, ctx = nil)
+          super
+
+          parser = Fabulator::Grammar::RuleParser.new
+
+          # parse @matches
+          @c_matches = parser.parse(self.matches)
+        end
+
+        def parse(cursor)
+          @c_matches.parse(cursor)
+        end
+
+        def score(context, data)
+          return 0 if @score.nil?
+          ctx = context.with_root(context.root.anon_node(nil))
+          ctx.merge_data(data)
+          (self.score(ctx).value rescue 0)
+        end
+      end
+    end
+  end
+end

data/lib/fabulator/grammar/cursor.rb

@@ -0,0 +1,118 @@
+module Fabulator
+  module Grammar
+    class Cursor
+      attr_accessor :mode, :skip
+
+      def initialize(g,ctx,s)
+        @source = s
+        @grammar = g
+        @curpos = 0
+        @end = @source.length-1
+        @line = 0
+        @col = 0
+        @anchored = false
+        @mode = :default
+        @skip = nil
+        @context = ctx.with_root(ctx.root.anon_node(nil))
+      end
+
+      def context
+        @context
+      end
+
+      def pos
+        @curpos
+      end
+
+      def resync(pat)
+        until self.eof || @source[@curpos..@source.length-1] =~ %r{^#{pat}}
+          @curpos += 1
+        end
+      end
+
+      def eof
+        @curpos > @end
+      end
+
+      def advance_position(i)
+        @curpos += i if i > 0
+      end
+
+      def anchored
+        @anchored
+      end
+
+      def anchored=(t)
+        @anchored = t
+      end
+
+      def grammar
+        @grammar
+      end
+
+      def point
+        { :curpos => @curpos, :line => @line, :col => @col, :root => @context.root, :mode => @mode, :anchored => @anchored, :skip => @skip }
+      end
+
+      def point=(p)
+        @curpos = p[:curpos]
+        @line = p[:line]
+        @col = p[:col]
+        @mode = p[:mode]
+        @anchored = p[:anchored]
+        @skip = p[:skip]
+        @context.root = p[:root]
+      end
+
+      def attempt(&block)
+        saved = self.point
+        ret = yield self
+        if ret.nil?
+          self.point = saved
+          return nil
+        end
+        
+        return ret
+      end
+
+      def find_rule(nom)
+        r = @grammar.get_rule(@mode, nom)
+        if r.nil? && @mode.to_s != 'default'
+          r = @grammar.get_rule('default', nom)
+        end
+        r
+      end
+
+      def data
+        @context
+      end
+
+      def do_skip
+        if !@skip.nil?
+          my_skip = @skip
+          new_pos = @curpos
+          self.attempt do |cursor|
+            cursor.skip = nil
+            cursor.anchored
+            r = my_skip.parse(cursor)
+            while !r.nil?
+              r = my_skip.parse(cursor)
+            end
+            new_pos = cursor.pos
+          end
+          @curpos = new_pos
+        end
+      end
+
+      def match_token(regex)
+        res = nil
+        do_skip
+        if @source[@curpos .. @end] =~ %r{^(#{regex})}
+          res = $1.to_s
+          @curpos += res.length
+        end
+        res
+      end
+    end
+  end
+end

data/lib/fabulator/grammar/expr/anchor.rb

@@ -0,0 +1,28 @@
+module Fabulator
+  module Grammar
+    module Expr
+      class Anchor
+        def initialize(t)
+          @anchor = t
+        end
+
+        def name
+          nil
+        end
+
+        def parse(source)
+          ret = nil
+          case @anchor
+            when :start_of_string: 
+              ret = source.pos == 0 ? {} : nil
+            when :start_of_line:
+            when :end_of_string:
+              ret = source.eof ? {} : nil
+            when :end_of_line:
+          end
+          ret
+        end
+      end
+    end
+  end
+end

data/lib/fabulator/grammar/expr/char_set.rb

@@ -1,45 +1,94 @@
+require 'bitset'
+
 module Fabulator::Grammar::Expr
   class CharSet
-    def initialize(cs)
-      chars = ""
-      ranges = ""
+    def initialize(cs = "")
+      @set = BitSet.new
       if cs[0..0] == '-'
-        chars = '-'
+        @set.on(('-')[0])
         cs = cs[1..cs.length-1]
       end
       bits = cs.split(/-/) # to pull out ranges
       if bits.size == 1
-        chars = bits[0]
-      else
+        bits[0].each_char{ |c|
+          @set.on(c[0])
+        }
+      elsif bits.size > 1
         if bits[0].size > 1
-          chars += b[0..0]
+          @set.on(bits[0][0])
         end
         while(bits.size > 1)
           b = bits.shift
           if b.size > 2
-            chars += b[1..b.size-2]
+            b[1..b.size-2].each_char { |c| @set.on(c[0]) }
           end
-          ranges += Regexp.quote(b[b.size-1 .. b.size-1]) + '-' + Regexp.quote(bits[0][0..0])
+          @set.on(b[b.size-1] .. bits[0][0])
         end
         if bits[0].size > 1
-          chars += bits[0][1..bits[0].size-1]
+          bits[0][1..bits[0].size-2].each_char { |c|
+            @set.on(c[0])
+          }
         end
       end
-      chars = chars.collect{ |cc| Regexp.quote(cc) }.join('')
-      @set = chars + ranges
-      @inverted = false
     end
 
-    def inverted
-      @inverted = true
+    def set
+      @set
+    end
+
+    def or(c)
+      @set = @set | c.set
+      self
+    end
+
+    def but_not(c)
+      @set = @set - c.set
+      self
+    end
+
+# for now, we restrict ourselves to 8-bit characters
+    def universal
+      @set.on(0..0xff)
     end
 
     def to_regex
-      if @set != ''
-        Regexp.compile("[" + (@inverted ? '^' : '') + @set + "]")
+      # want a compact set of ranges for the regex
+      set_def = ''
+      @set.to_ary.each do |r|
+        if r.is_a?(Range)
+          set_def += Regexp.quote(r.begin.to_i.chr) + '-' + Regexp.quote(r.end.to_i.chr)
+        else
+          set_def += Regexp.quote(r.to_i.chr)
+        end
+      end
+      if set_def == ''
+        return %r{.}
       else
-        %r{}
+        %r{[#{set_def}]}
       end
     end
   end
+
+  class CharClass < CharSet
+    @@charsets = {
+      'alnum' => [ 0x30 .. 0x39, 0x41 .. 0x5a, 0x61 .. 0x7a ],
+      'alpha' => [ 0x41 .. 0x5a, 0x61 .. 0x7a ],
+      'ascii' => [ 0x00 .. 0x7f ],
+      'blank' => [ 0x0b, 0x20 ], # \t + space
+      'cntrl' => [ 0x00 .. 0x1f, 0x7f ],
+      'digit' => [ 0x30 .. 0x39 ],
+      'graph' => [ 0x21 .. 0x7e ],
+      'lower' => [ 0x61 .. 0x7a ],
+      'print' => [ 0x20 .. 0x7e ],
+      'space' => [ 0x0a, 0x0b, 0x0c, 0x0f, 0x20 ], # \t\r\n\v\f + space
+      'upper' => [ 0x41 .. 0x5a ],
+      'word'  => [ 0x30 .. 0x39, 0x41 .. 0x5a, 0x61 .. 0x7a, '_'[0] ],
+      'xdigit'=> [ 0x30 .. 0x39, 0x41 .. 0x46, 0x61 .. 0x66 ],
+      'nl'    => [ 0x0a, 0x0c ]
+    }
+
+    def initialize(cs)
+      @set = BitSet.new.on(@@charsets[cs.downcase] || [])
+    end
+  end
 end

data/lib/fabulator/grammar/expr/look_ahead.rb

@@ -0,0 +1,44 @@
+module Fabulator
+  module Grammar
+    module Expr
+      class LookAhead
+        def initialize(sequence)
+          @sequence = sequence
+        end
+
+        def name
+          nil
+        end
+
+        def parse(source)
+          ret = nil
+          source.attempt do |c|
+            ret = @sequence.parse(c)
+            nil
+          end
+          ret.nil? ? nil : {}
+        end
+      end
+
+      class NegLookAhead   
+        def initialize(sequence)
+          @sequence = sequence
+        end
+ 
+        def name
+          nil  
+        end
+ 
+        def parse(source)
+          ret = nil
+          source.attempt do |c|
+            ret = @sequence.parse(c)
+            nil
+          end
+          ret.nil? ? {} : nil
+        end
+      end
+
+    end
+  end
+end

data/lib/fabulator/grammar/expr/rule.rb

@@ -1,35 +1,40 @@
-module Fabulator::Grammar::Expr
-  class Rule
-    def initialize
-      @sequences = [ ]
-      @anchor_start = false
-      @anchor_end = false
-    end
-
-    def anchor_start
-      @anchor_start = true
-    end
+module Fabulator
+  module Grammar
+    module Expr
+      class Rule
+        attr_accessor :name
 
-    def anchor_end
-      @anchor_end = true
-    end
+        def initialize
+          @alternatives = [ ]
+        end
 
-    def add_sequence(s)
-      @sequences << s
-    end
+        def add_alternative(a)
+          @alternatives << a
+        end
 
-    def to_regex
-      r = %r{#{@sequences.collect{ |s| s.to_regex }}}
-      if @anchor_start
-        if @anchor_end
-          %r{^#{r}$}
-        else
-          %r{^#{r}}
+        def parse(s)
+          if s.anchored
+            @alternatives.each do |alternative|
+              ret = s.attempt { |cursor| 
+                cursor.anchored = true 
+                alternative.parse(cursor) 
+              }
+              return ret unless ret.nil?
+            end
+          else
+            while !s.eof
+              @alternatives.each do |alternative|
+                ret = s.attempt { |cursor| 
+                  cursor.anchored = true 
+                  alternative.parse(cursor) 
+                }
+                return ret unless ret.nil?
+              end
+              s.advance_position(1)
+            end
+          end
+          return nil
         end
-      elsif @anchor_end
-        %r{#{r}$}
-      else
-        r
       end
     end
   end