fabulator-grammar 0.0.1 → 0.0.3

data/rules.racc

@@ -0,0 +1,249 @@
+class Fabulator::Grammar::RuleParser
+
+  start rules
+
+rule
+
+  rules: rule { result = Fabulator::Grammar::Expr::Rule.new; result.add_alternative(val[0]) }
+       | rules PIPE rule { result = val[0]; result.add_alternative(val[2]) }
+
+  rule: { result = Fabulator::Grammar::Expr::RuleAlternative.new; }
+      | rule rule_bit { result = val[0]; result.add_sequence(val[1]) }
+
+  rule_bit: directives
+      | sequence
+
+  directives: directive { result = [ val[0] ] }
+      | directives directive { result = val[0] + [ val[1] ] }
+
+  directive: LLB MODE NCNAME RB { result = Fabulator::Grammar::Expr::RuleMode.new(val[2]) }
+      | LLB COMMIT RB
+      | LLB UNCOMMIT RB
+      | LLB REJECT RB
+      | LLB SKIP opt_separator RB { result = Fabulator::Grammar::Expr::SetSkip.new(val[2]) }
+      | LLB RESYNC opt_separator RB { result = Fabulator::Grammar::Expr::Resync.new(val[2]) }
+      | DOT_DOT_DOT sequence { result = Fabulator::Grammar::Expr::LookAhead.new(val[1]) }
+      | DOT_DOT_DOT_BANG sequence { result = Fabulator::Grammar::Expr::NegLookAhead.new(val[1]) }
+      | CARET { result = Fabulator::Grammar::Expr::Anchor.new(:start_of_line) }
+      | CARET_CARET { result = Fabulator::Grammar::Expr::Anchor.new(:start_of_string) }
+      | DOLLAR { result = Fabulator::Grammar::Expr::Anchor.new(:end_of_line) }
+      | DOLLAR_DOLLAR { result = Fabulator::Grammar::Expr::Anchor.new(:end_of_string) }
+
+  sequence: atom sequence_qualifiers { result = Fabulator::Grammar::Expr::RuleSequence.new(nil, val[0], val[1]) }
+          | atom { result = Fabulator::Grammar::Expr::RuleSequence.new(nil, val[0]) }
+          | hypothetical atom sequence_qualifiers { result = Fabulator::Grammar::Expr::RuleSequence.new(val[0], val[1], val[2]) }
+          | hypothetical atom { result = Fabulator::Grammar::Expr::RuleSequence.new(val[0], val[1]) }
+
+  hypothetical: NCNAME COLON_EQUAL { result = val[0] }
+
+  # /\((\?|s|s\?|\d+(\.\.(\d+)?)?|\.\.\d+)(\s+ncname)?\)/
+  # and with no prior space -- and ncname should be specifiable as
+  #   a text constant
+  # the ncname specifies a token that separates instances
+  # shorthand: if we see '(s[ )]', '(s?[ )]', '(?)', '(\d', then we have
+  # something for the following instead of LP rules RP
+  sequence_qualifiers: LLP QUESTION RP { result = [ '?'.to_sym ] }
+          | LLP S opt_separator RP { result = [ :s, val[2] ] }
+          | LLP S QUESTION opt_separator RP { result = [ 's?'.to_sym, val[3] ] }
+          | LLP INTEGER opt_separator RP { result = [ :count, val[1], val[2] ] }
+          | LLP INTEGER DOT_DOT INTEGER opt_separator RP { result = [ :range, [ val[1], val[3] ], val[4] ] }
+          | LLP DOT_DOT INTEGER opt_separator RP { result = [ :upto, val[2], val[3] ] }
+          | LLP INTEGER DOT_DOT opt_separator RP { result = [ :atleast, val[1],val[3] ] }
+
+  opt_separator:
+      | atom
+
+  atom: LITERAL { result = Fabulator::Grammar::Expr::Text.new(val[0]) }
+      | LP rules RP { result = val[1] }
+      | NCNAME opt_params { result = Fabulator::Grammar::Expr::RuleRef.new(val[0]) }
+
+  opt_params:
+      | LB params RB
+
+  params: param
+      | params COMMA param
+      | params COMMA
+
+  param: hypothetical relative_path
+      | hypothetical LITERAL
+
+  relative_path: NCNAME
+      | NCNAME SLASH NCNAME
+
+
+---- inner
+  require 'fabulator/grammar'
+
+  def parse(t)
+    @source = t
+    @curpos = 0
+    @line = 0
+    @col = 0
+
+    @in_quantifier = false
+       
+    @yydebug = true
+
+    @last_token = nil
+       
+    do_parse
+  end
+
+  def on_error(*args)
+    raise Fabulator::Grammar::ParserError.new("unable to parse '#{args[1]}' near line #{@line + 1}, column #{@col}")
+  end
+
+  @@ops = {
+    ':=' => :COLON_EQUAL,
+    '['  => :LB,
+    ']'  => :RB,
+    '('  => :LP,
+    ')'  => :RP,
+    '{'  => :LC,
+    '}'  => :RC,
+    '?'  => :QUESTION,
+    '.'  => :DOT,
+    '..' => :DOT_DOT,
+    '...'=> :DOT_DOT_DOT,
+    '...!'=> :DOT_DOT_DOT_BANG,
+    '|'  => :PIPE,
+    ','  => :COMMA,
+    ':'  => :COLON,
+    '^'  => :CARET,
+    '^^' => :CARET_CARET,
+    '$'  => :DOLLAR,
+    '$$' => :DOLLAR_DOLLAR,
+    '/'  => :SLASH,
+  }
+
+  @@regex = {
+    :simple_tokens => %r{^(#{Regexp.union(@@ops.keys.sort_by{|a| a.length}.reverse.collect{ |k| k })})},
+    :ncname => %r{(?:[a-zA-Z_][-a-zA-Z0-9_.]*)},
+    :integer => %r{(\d+)},
+    :literal => %r{((?:"(?:[^\\"]*(?:\\.[^\\"]*)*)")|(?:'(?:[^\\']*(?:\\.[^\\']*)*)'))},
+  }
+
+  @@regex[:general] = Regexp.compile(%{^(#{@@regex[:ncname]})|#{@@regex[:integer]}|#{@@regex[:literal]}})
+
+  def next_token
+    @token = nil  
+    white_space = 0
+    new_line = 0
+    while @curpos < @source.length && @source[@curpos..@curpos] =~ /\s/ do
+      if @source[@curpos..@curpos] =~ /\n/
+        new_line = new_line + 1
+        @line = @line + 1
+        @col = 0
+      else
+        @col = @col + 1
+      end   
+      @curpos = @curpos + 1
+      white_space = white_space + 1
+    end
+    
+    # skip comments delimited by (:  :)
+    # comments can be nested
+    # these are XPath 2.0 comments
+    #
+
+    if @curpos < @source.length && @source[@curpos..@curpos+1] == '(:'
+      comment_depth = 1
+      @curpos = @curpos + 2
+      @col = @col + 2
+      while comment_depth > 0 && @curpos < @source.length
+        if @source[@curpos..@curpos+1] == '(:'
+          comment_depth = comment_depth + 1
+          @curpos = @curpos + 1
+          @col = @col + 1
+        end
+        if @source[@curpos..@curpos+1] == ':)'
+          comment_depth = comment_depth - 1
+          @curpos = @curpos + 1
+          @col = @col + 1
+        end
+        @curpos = @curpos + 1
+        @col = @col + 1
+      end
+      white_space = white_space + 1
+    end
+    
+    while @curpos < @source.length && @source[@curpos..@curpos] =~ /\s/ do
+      if @source[@curpos..@curpos] =~ /\n/
+        new_line = new_line + 1
+        @line = @line + 1
+        @col = 0
+      else
+        @col = @col + 1
+      end   
+      @curpos = @curpos + 1
+      white_space = white_space + 1
+    end
+    
+    if @curpos >= @source.length
+      @last_token = nil
+      return [ false, false ]
+    end
+
+    res = @@regex[:simple_tokens].match(@source[@curpos..@source.length-1])
+    if !res.nil?
+      if !res[1].nil?  
+        @token = [ @@ops[res[1]], res[1] ]
+      end
+    end
+  
+    if @token.nil?
+      res = @@regex[:general].match(@source[@curpos..@source.length-1])
+      if res.nil?
+        raise "Failed to parse '#{@source}' at #{@curpos}': #{@source[@curpos..@source.length-1]}"
+      end
+  #ncname, integer, literal
+      if !res[1].nil?
+        @token = [:NCNAME, res[1].to_s]
+      elsif !res[2].nil?
+        @token = [:INTEGER, res[2].to_s]
+      elsif !res[3].nil?
+        @token = [:LITERAL, res[3].to_s]
+        @token[1] = @token[1][1..@token[1].size-2]
+        @col += 2
+        @curpos += 2
+      end
+    end
+
+
+    if @token.nil?
+      puts "Uh oh... we don't know what to do: #{@source[@curpos .. @source.length-1]}"
+      return [ nil, nil ]
+    else
+      @curpos += @token[1].length
+      @col += @token[1].length
+    end
+
+    if !@token.nil? && @token[0] == :LP
+       # shorthand: if we see '(s[ )]', '(s?[ )]', '(?)', '(\d', then we have
+       if @curpos > 1 && ![' ', '('].include?(@source[@curpos-2 .. @curpos-2])
+         @token[0] = :LLP
+         @in_quantifier = true
+       end
+    elsif @in_quantifier
+      @in_quantifier = false
+      if @token[0] == :NCNAME
+        @token[0] = case @token[1]
+          when 's': :S
+          else :NCNAME
+        end
+      end
+    end
+
+    if !@token.nil? && @token[0] == :LB
+      if @curpos == 1 || @source[@curpos-2 .. @curpos - 2] == ' '
+        @token[0] = :LLB
+        @in_directive = true
+      end
+    elsif @in_directive && @token[0] == :NCNAME
+      @token[0] = @token[1].upcase.to_sym
+      @in_directive = false
+    end
+
+  #  puts "token: #{@token.join(' => ')}"
+    return @token
+  end

data/tokens.racc

@@ -0,0 +1,257 @@
+class Fabulator::Grammar::TokenParser
+
+  start rules
+
+rule
+  rules: anchored_rule { result = Fabulator::Grammar::Expr::Token.new; result.add_alternative(val[0]) }
+       | rules PIPE anchored_rule { result = val[0]; result.add_alternative(val[2]) }
+
+  anchored_rule: rule { result = val[0] }
+       | left_anchor rule { result = val[1]; result.anchor_start(val[0]) }
+       | rule right_anchor { result = val[0]; result.anchor_end(val[1]) }
+       | left_anchor rule right_anchor { result = val[1]; result.anchor_start(val[0]); result.anchor_end(val[2]) }
+
+  left_anchor: CARET { result = '^' }
+       | CARET CARET { result = '^^' }
+
+  right_anchor: DOLLAR { result = '$' }
+       | DOLLAR DOLLAR { result = '$$' }
+
+  rule: { result = Fabulator::Grammar::Expr::TokenAlternative.new; }
+      | rule sequence { result = val[0]; result.add_sequence(val[1]); }
+
+  sequence: atom sequence_qualifiers { result = Fabulator::Grammar::Expr::Sequence.new(nil, val[0], val[1]) }
+          | atom { result = Fabulator::Grammar::Expr::Sequence.new(nil, val[0]) }
+
+  atom: text { result = Fabulator::Grammar::Expr::Text.new(val[0]) }
+      | DOT { result = Fabulator::Grammar::Expr::Any.new }
+      | LP rules RP { result = val[1] }
+      | LB atom_expr RB { result = val[1] }
+
+  atom_expr: char_set_expr
+
+ #{ result = Fabulator::Grammar::Expr::CharSet.new; result.universal }
+  char_set_expr: char_set { result = val[0] }
+      | MINUS char_set { result = Fabulator::Grammar::Expr::CharSet.new; result.universal; result.but_not(val[1]) }
+      | char_set_expr PLUS char_set { result = val[0].or(val[2]) }
+      | char_set_expr MINUS char_set { result = val[0].but_not(val[2]) }
+
+  char_set: LB char_set_text RB { result = Fabulator::Grammar::Expr::CharSet.new(val[1]) }
+      | COLON NCNAME COLON { result = Fabulator::Grammar::Expr::CharClass.new(val[1]) }
+      | LP char_set_expr RP { result = val[1] }
+
+  char_set_text: { result = '' }
+      | char_set_text CHAR_TEXT { result = val[0] + val[1] }
+      | char_set_text MINUS CHAR_TEXT { result = val[0] + '-' + val[2] }
+      | char_set_text PLUS CHAR_TEXT { result = val[0] + '+' + val[2] }
+      
+  text: qname { result = val[0] }
+      | TEXT { result = val[0] }
+      | INTEGER { result = val[0] }
+      | COMMA { result = val[0] }
+
+  qname: NCNAME { result = val[0] }
+       | NCNAME COLON NCNAME { result = val[0] + ':' + val[2] }
+
+  sequence_qualifiers: STAR { result = [ :zero_or_more ] }
+      | STAR QUESTION { result = [ :zero_or_more, :min ] }
+      | PLUS { result = [ :one_or_more ] }
+      | PLUS QUESTION { result = [ :one_or_more, :min ] }
+      | QUESTION { result = [ :zero_or_one ] }
+      | QUESTION QUESTION { result = [ :zero_or_one, :min ] }
+      | LLB INTEGER RB { result = [ :exact, val[1].to_i ] }
+      | LLB INTEGER COMMA INTEGER RB { result = [ :range, val[1].to_i, val[3].to_i ] }
+      | LLB INTEGER COMMA RB { result = [ :range, val[1], '' ] }
+      | LLB INTEGER COMMA RB QUESTION { result = [ :min, :range, val[1], '' ] }
+      | LLB INTEGER COMMA INTEGER RB QUESTION { result = [ :min, :range, val[1].to_i, val[3].to_i ] }
+      
+
+---- inner
+  require 'fabulator/grammar'
+
+  def parse(t)
+    @source = t
+    @curpos = 0 
+    @col = 0
+    @line = 0   
+
+    @yydebug = true
+      
+    @last_token = nil
+
+    @brackets = 0
+  
+    do_parse
+  end
+    
+  def on_error(*args)
+    raise Fabulator::Grammar::ParserError.new("unable to parse '#{args[1]}' near line #{@line + 1}, column #{@col}")
+  end
+
+  @@ops = {
+    #'[{' => :LB_LC,
+    #'}]' => :RC_RB,
+    #'[[' => :LB_LB,
+    #']]' => :RB_RB,
+    '['  => :LB,
+    ']'  => :RB,
+    '('  => :LP,
+    ')'  => :RP,
+    #'{'  => :LC,
+    #'}'  => :RC,
+    #'#'  => :HASH,
+    '$'  => :DOLLAR,
+    '^'  => :CARET,
+    #'&'  => :AND,
+    '*'  => :STAR,
+    '+'  => :PLUS,
+    '-'  => :MINUS,
+    '?'  => :QUESTION,
+    '.'  => :DOT,
+    '|'  => :PIPE,
+    ','  => :COMMA,
+    ':'  => :COLON
+  }
+
+
+  @@regex = {
+     :simple_tokens => %r{^(#{Regexp.union(@@ops.keys.sort_by{|a| a.length}.reverse.collect{ |k| k })})},
+    :ncname => %r{(?:[a-zA-Z_][-a-zA-Z0-9_.]*)}
+  }
+
+ #puts @@regex[:simple_tokens]
+
+  @@regex[:qname] = %r{((?:#{@@regex[:ncname]}:)?#{@@regex[:ncname]})}
+  @@regex[:general] = Regexp.compile(%{^#{@@regex[:qname]}|#{@@regex[:simple_tokens]}})
+
+  def next_token
+    @token = nil
+    white_space = 0
+    new_line = 0
+    while @curpos < @source.length && @source[@curpos..@curpos] =~ /\s/ do
+      if @source[@curpos..@curpos] =~ /\n/
+        new_line = new_line + 1
+        @line = @line + 1
+        @col = 0
+      else
+        @col = @col + 1
+      end
+      @curpos = @curpos + 1
+      white_space = white_space + 1   
+    end
+
+    # skip comments delimited by (:  :)
+    # comments can be nested
+    # these are XPath 2.0 comments
+    #
+    if @curpos < @source.length && @source[@curpos..@curpos+1] == '(:'
+      comment_depth = 1
+      @curpos = @curpos + 2
+      @col = @col + 2
+      while comment_depth > 0 && @curpos < @source.length
+        if @source[@curpos..@curpos+1] == '(:'
+          comment_depth = comment_depth + 1
+          @curpos = @curpos + 1
+          @col = @col + 1
+        end
+        if @source[@curpos..@curpos+1] == ':)'
+          comment_depth = comment_depth - 1
+          @curpos = @curpos + 1
+          @col = @col + 1
+        end
+        @curpos = @curpos + 1
+        @col = @col + 1
+      end
+      white_space = white_space + 1
+    end
+
+    while @curpos < @source.length && @source[@curpos..@curpos] =~ /\s/ do
+      if @source[@curpos..@curpos] =~ /\n/
+        new_line = new_line + 1
+        @line = @line + 1
+        @col = 0
+      else
+        @col = @col + 1
+      end
+      @curpos = @curpos + 1
+      white_space = white_space + 1
+    end
+
+    if @curpos >= @source.length
+      @last_token = nil
+      return [ false, false ]
+    end
+
+#    case @source[@curpos..@curpos]
+#      when '<': @token = [ :LT, '<' ]
+#      when '>': @token = [ :GT, '>' ]
+#      when '[': @token = [ :LB, '[' ]
+#      when ']': @token = [ :RB, ']' ]
+#      when '(': @token = [ :LP, '(' ]
+#      when ')': @token = [ :RP, ')' ]
+#      when '{': @token = [ :LC, '{' ]
+#      when '}': @token = [ :RC, '}' ]
+#      when ':': @token = [ :COLON, ':' ]
+#      when ',': @token = [ :COMMA, ',' ]
+#      when '|': @token = [ :PIPE, '|' ]
+#      when '*': @token = [ :STAR, '*' ]
+#      when '+': @token = [ :PLUS, '+' ]
+#      when '.': @token = [ :DOT,  '.' ]
+#      when '?': @token = [ :QUESTION, '?' ]
+#      when '$': @token = [ :DOLLAR, '$' ]
+#      when '^': @token = [ :CARET, '^' ]
+#    end
+
+    res = @@regex[:simple_tokens].match(@source[@curpos..@source.length-1])
+    if !res.nil?
+      if !res[1].nil?
+        @token = [ @@ops[res[1]], res[1] ]
+      end
+    end
+
+    if @token.nil?
+      # get longest sequence of non-special characters
+      # if it's all digits, report INTEGER
+      # if it's a qname, report QNAME
+      # otherwise, report TEXT
+      @source[@curpos..@source.length-1] =~ /^(((\\.)|[^ \$\^\[\]\{\}\(\):,|*+.?])+)*/
+      text = $1
+      bits = text.split(/\\/)
+      text = bits.join('')
+      @curpos += bits.size - 1
+      @col += bits.size - 1
+      if text.length > 0
+        if @source[@curpos+text.length .. @curpos+text.length] =~ /[*?+\{]/
+# TODO: make sure we backtrack properly if the last character is escaped
+          text = text[0..text.length-2]
+          @token = [ :TEXT, text ]
+        else
+          case text
+            when /^\d+$/: @token = [ :INTEGER, text ]
+            when /^#{@@regex[:ncname]}$/: @token = [ :NCNAME, text ]
+            else @token = [ :TEXT, text ]
+          end
+        end
+      end
+    end
+
+    if @token.nil?
+      puts "Uh oh... we don't know what to do: #{@source[@curpos .. @source.length-1]}"
+    else
+      @curpos += @token[1].length
+      @col += @token[1].length
+    end
+
+    if @token[0] == :LB
+      if @brackets == 0 && @source[@curpos..@source.length-1] =~ /^\s*\d/
+        @token[0] = :LLB
+      end
+      @brackets += 1
+    elsif @token[0] == :RB
+      @brackets -= 1
+    elsif @brackets > 1
+      @token[0] = :CHAR_TEXT
+    end
+
+    return @token
+  end