ebnf 1.1.1 → 2.1.0

data/lib/ebnf/ll1/scanner.rb

@@ -3,7 +3,7 @@ require 'strscan'    unless defined?(StringScanner)
 
 module EBNF::LL1
   ##
-  # Overload StringScanner with file operations
+  # Overload StringScanner with file operations and line counting
   #
   # * Reloads scanner as required until EOF.
   # * Loads to a high-water and reloads when remaining size reaches a low-water.
@@ -14,25 +14,14 @@ module EBNF::LL1
     LOW_WATER  = 4 * 1024
 
     ##
-    # @return [IO, StringIO]
+    # @return [String, IO, StringIO]
     attr_reader :input
 
     ##
-    # If we don't have an IO input, simply use StringScanner directly
-    # @private
-    def self.new(input, options = {})
-      input ||= ""
-      if input.respond_to?(:read)
-        scanner = self.allocate
-        scanner.send(:initialize, input, options)
-      else
-        if input.encoding != Encoding::UTF_8
-          input = input.dup if input.frozen?
-          input.force_encoding(Encoding::UTF_8)
-        end
-        StringScanner.new(input)
-      end
-    end
+    # The current line number (one-based).
+    #
+    # @return [Integer]
+    attr_accessor :lineno
 
     ##
     # Create a scanner, from an IO
@@ -42,35 +31,26 @@ module EBNF::LL1
     # @option options[Integer] :high_water (HIGH_WATER)
     # @option options[Integer] :low_water (LOW_WATER)
     # @return [Scanner]
-    def initialize(input, options = {})
+    def initialize(input, **options)
       @options = options.merge(high_water: HIGH_WATER, low_water: LOW_WATER)
 
-      @input = input
-      super("")
+      @previous_lineno = @lineno = 1
+      @input = input.is_a?(String) ? encode_utf8(input) : input
+      super(input.is_a?(String) ? @input : "")
       feed_me
       self
     end
 
     ##
-    # Returns the "rest" of the line, or the next line if at EOL (i.e. everything after the scan pointer).
-    # If there is no more data (eos? = true), it returns "".
-    #
-    # @return [String]
-    def rest
-      feed_me
-      encode_utf8 super
-    end
-    
-    ##
-    # Attempts to skip over the given `pattern` beginning with the scan pointer.
-    # If it matches, the scan pointer is advanced to the end of the match,
-    # and the length of the match is returned. Otherwise, `nil` is returned.
-    #
-    # similar to `scan`, but without returning the matched string.
-    # @param [Regexp] pattern
-    def skip(pattern)
-      feed_me
-      super
+    # Ensures that the input buffer is full to the high water mark, or end of file. Useful when matching tokens that may be longer than the low water mark
+    def ensure_buffer_full
+      # Read up to high-water mark ensuring we're at an end of line
+      if @input.respond_to?(:eof?) && !@input.eof?
+        diff = @options[:high_water] - rest_size
+        string = encode_utf8(@input.read(diff))
+        string << encode_utf8(@input.gets) unless @input.eof?
+        self << string if string
+      end
     end
 
     ##
@@ -83,10 +63,14 @@ module EBNF::LL1
     end
 
     ##
-    # Set the scan pointer to the end of the string and clear matching data
-    def terminate
+    # Returns the "rest" of the line, or the next line if at EOL (i.e. everything after the scan pointer).
+    # If there is no more data (eos? = true), it returns "".
+    #
+    # @return [String]
+    def rest
       feed_me
-      super
+      @lineno += 1 if eos?
+      encode_utf8 super
     end
 
     ##
@@ -108,19 +92,68 @@ module EBNF::LL1
     # @return [String]
     def scan(pattern)
       feed_me
-      encode_utf8 super
+      @previous_lineno = @lineno
+      if matched = encode_utf8(super)
+        @lineno += matched.count("\n")
+      end
+      matched
     end
 
     ##
-    # Ensures that the input buffer is full to the high water mark, or end of file. Useful when matching tokens that may be longer than the low water mark
-    def ensure_buffer_full
-      # Read up to high-water mark ensuring we're at an end of line
-      if @input && !@input.eof?
-        diff = @options[:high_water] - rest_size
-        string = encode_utf8(@input.read(diff))
-        string << encode_utf8(@input.gets) unless @input.eof?
-        self << string if string
+    # Scans the string until the pattern is matched. Returns the substring up to and including the end of the match, advancing the scan pointer to that location. If there is no match, nil is returned.
+    #
+    # @example
+    #     s = StringScanner.new("Fri Dec 12 1975 14:39")
+    #     s.scan_until(/1/)        # -> "Fri Dec 1"
+    #     s.pre_match              # -> "Fri Dec "
+    #     s.scan_until(/XYZ/)      # -> nil
+    #
+    # @param [Regexp] pattern
+    # @return [String]
+    def scan_until(pattern)
+      feed_me
+      @previous_lineno = @lineno
+      if matched = encode_utf8(super)
+        @lineno += matched.count("\n")
       end
+      matched
+    end
+
+    ##
+    # Attempts to skip over the given `pattern` beginning with the scan pointer.
+    # If it matches, the scan pointer is advanced to the end of the match,
+    # and the length of the match is returned. Otherwise, `nil` is returned.
+    #
+    # similar to `scan`, but without returning the matched string.
+    # @param [Regexp] pattern
+    def skip(pattern)
+      scan(pattern)
+      nil
+    end
+
+    ##
+    # Advances the scan pointer until pattern is matched and consumed. Returns the number of bytes advanced, or nil if no match was found.
+    #
+    # Look ahead to match pattern, and advance the scan pointer to the end of the match. Return the number of characters advanced, or nil if the match was unsuccessful.
+    #
+    # It’s similar to scan_until, but without returning the intervening string.
+    # @param [Regexp] pattern
+    def skip_until(pattern)
+      (matched = scan_until(pattern)) && matched.length
+    end
+
+    ##
+    # Sets the scan pointer to the previous position. Only one previous position is remembered, and it changes with each scanning operation.
+    def unscan
+      @lineno = @previous_lineno
+      super
+    end
+
+    ##
+    # Set the scan pointer to the end of the string and clear matching data
+    def terminate
+      feed_me
+      super
     end
 
   private

data/lib/ebnf/native.rb

@@ -0,0 +1,320 @@
+module EBNF
+  module Native
+    ##
+    # Native parser for EBNF; less accurate, but appropriate when changing EBNF grammar, itself.
+    #
+    # Iterate over rule strings.
+    # a line that starts with '\[' or '@' starts a new rule
+    #
+    # @param [StringScanner] scanner
+    # @yield rule_string
+    # @yieldparam [String] rule_string
+    def eachRule(scanner)
+      cur_lineno = 1
+      r = ''
+      until scanner.eos?
+        case
+        when s = scanner.scan(%r(\s+)m)
+          # Eat whitespace
+          cur_lineno += s.count("\n")
+          #debug("eachRule(ws)") { "[#{cur_lineno}] #{s.inspect}" }
+        when s = scanner.scan(%r(/\*([^\*]|\*[^\/])*\*/)m)
+          # Eat comments /* .. */
+          cur_lineno += s.count("\n")
+          debug("eachRule(comment)") { "[#{cur_lineno}] #{s.inspect}" }
+        when s = scanner.scan(%r(\(\*([^\*]|\*[^\)])*\*\))m)
+          # Eat comments (* .. *)
+          cur_lineno += s.count("\n")
+          debug("eachRule(comment)") { "[#{cur_lineno}] #{s.inspect}" }
+        when s = scanner.scan(%r((#(?!x)|//).*$))
+          # Eat comments // & #
+          cur_lineno += s.count("\n")
+          debug("eachRule(comment)") { "[#{cur_lineno}] #{s.inspect}" }
+        when s = scanner.scan(/\A["']/)
+          # Found a quote, scan until end of matching quote
+          s += scanner.scan_until(/#{scanner.matched}|$/)
+          r += s
+        when s = scanner.scan(%r(^@terminals))
+          #debug("eachRule(@terminals)") { "[#{cur_lineno}] #{s.inspect}" }
+          yield(r) unless r.empty?
+          @lineno = cur_lineno
+          yield(s)
+          r = ''
+        when s = scanner.scan(/@pass/)
+          # Found rule start, if we've already collected a rule, yield it
+          #debug("eachRule(@pass)") { "[#{cur_lineno}] #{s.inspect}" }
+          yield r unless r.empty?
+          @lineno = cur_lineno
+          r = s
+        when s = scanner.scan(EBNF::Terminals::LHS)
+          # Found rule start, if we've already collected a rule, yield it
+          yield r unless r.empty?
+          #debug("eachRule(rule)") { "[#{cur_lineno}] #{s.inspect}" }
+          @lineno = cur_lineno
+          r = s
+        else
+          # Collect until end of line, or start of comment or quote
+          s = scanner.scan_until(%r{(?:[/\(]\*)|#(?!x)|//|["']|$})
+          if scanner.matched.length > 0
+            # Back up scan head before ending match
+            scanner.pos = scanner.pos - scanner.matched.length
+
+            # Remove matched from end of string
+            s = s[0..-(scanner.matched.length+1)]
+          end
+          cur_lineno += s.count("\n")
+          #debug("eachRule(rest)") { "[#{cur_lineno}] #{s.inspect}" }
+          r += s
+        end
+      end
+      yield r unless r.empty?
+    end
+  
+    ##
+    # Parse a rule into an optional rule number, a symbol and an expression
+    #
+    # @param [String] rule
+    # @return [Rule]
+    def ruleParts(rule)
+      num_sym, expr = rule.split('::=', 2).map(&:strip)
+      num, sym = num_sym.split(']', 2).map(&:strip)
+      num, sym = "", num if sym.nil?
+      num = num[1..-1]
+      r = Rule.new(sym && sym.to_sym, num, expression(expr).first, ebnf: self)
+      debug("ruleParts") { r.inspect }
+      r
+    end
+
+    ##
+    # Parse a string into an expression tree and a remaining string
+    #
+    # @example
+    #     >>> expression("a b c")
+    #     ((seq a b c) '')
+    #     
+    #     >>> expression("a? b+ c*")
+    #     ((seq (opt a) (plus b) (star c)) '')
+    #     
+    #     >>> expression(" | x xlist")
+    #     ((alt (seq) (seq x xlist)) '')
+    #     
+    #     >>> expression("a | (b - c)")
+    #     ((alt a (diff b c)) '')
+    #     
+    #     >>> expression("a b | c d")
+    #     ((alt (seq a b) (seq c d)) '')
+    #     
+    #     >>> expression("a | b | c")
+    #     ((alt a b c) '')
+    #     
+    #     >>> expression("a) b c")
+    #     (a ' b c')
+    #     
+    #     >>> expression("BaseDecl? PrefixDecl*")
+    #     ((seq (opt BaseDecl) (star PrefixDecl)) '')
+    #     
+    #     >>> expression("NCCHAR1 | diff | [0-9] | #x00B7 | [#x0300-#x036F] | \[#x203F-#x2040\]")
+    #     ((alt NCCHAR1 diff
+    #           (range '0-9')
+    #           (hex '#x00B7')
+    #           (range '#x0300-#x036F')
+    #           (range, '#x203F-#x2040')) '')
+    #     
+    # @param [String] s
+    # @return [Array]
+    def expression(s)
+      debug("expression") {"(#{s.inspect})"}
+      e, s = depth {alt(s)}
+      debug {"=> alt returned #{[e, s].inspect}"}
+      unless s.to_s.empty?
+        t, ss = depth {terminal(s)}
+        debug {"=> terminal returned #{[t, ss].inspect}"}
+        return [e, ss] if t.is_a?(Array) && t.first == :")"
+      end
+      [e, s]
+    end
+  
+    ##
+    # Parse alt
+    #     >>> alt("a | b | c")
+    #     ((alt a b c) '')
+    # @param [String] s
+    # @return [Array]
+    def alt(s)
+      debug("alt") {"(#{s.inspect})"}
+      args = []
+      while !s.to_s.empty?
+        e, s = depth {seq(s)}
+        debug {"=> seq returned #{[e, s].inspect}"}
+        if e.to_s.empty?
+          break unless args.empty?
+          e = [:seq, []] # empty sequence
+        end
+        args << e
+        unless s.to_s.empty?
+          t, ss = depth {terminal(s)}
+          break unless t[0] == :alt
+          s = ss
+        end
+      end
+      args.length > 1 ? [args.unshift(:alt), s] : [e, s]
+    end
+  
+    ##
+    # parse seq
+    #
+    #     >>> seq("a b c")
+    #     ((seq a b c) '')
+    #     
+    #     >>> seq("a b? c")
+    #     ((seq a (opt b) c) '')
+    def seq(s)
+      debug("seq") {"(#{s.inspect})"}
+      args = []
+      while !s.to_s.empty?
+        e, ss = depth {diff(s)}
+        debug {"=> diff returned #{[e, ss].inspect}"}
+        unless e.to_s.empty?
+          args << e
+          s = ss
+        else
+          break;
+        end
+      end
+      if args.length > 1
+        [args.unshift(:seq), s]
+      elsif args.length == 1
+        args + [s]
+      else
+        ["", s]
+      end
+    end
+  
+    ##
+    # parse diff
+    # 
+    #     >>> diff("a - b")
+    #     ((diff a b) '')
+    def diff(s)
+      debug("diff") {"(#{s.inspect})"}
+      e1, s = depth {postfix(s)}
+      debug {"=> postfix returned #{[e1, s].inspect}"}
+      unless e1.to_s.empty?
+        unless s.to_s.empty?
+          t, ss = depth {terminal(s)}
+          debug {"diff #{[t, ss].inspect}"}
+          if t.is_a?(Array) && t.first == :diff
+            s = ss
+            e2, s = primary(s)
+            unless e2.to_s.empty?
+              return [[:diff, e1, e2], s]
+            else
+              error("diff", "Syntax Error")
+              raise SyntaxError, "diff missing second operand"
+            end
+          end
+        end
+      end
+      [e1, s]
+    end
+  
+    ##
+    # parse postfix
+    # 
+    #     >>> postfix("a b c")
+    #     (a ' b c')
+    #     
+    #     >>> postfix("a? b c")
+    #     ((opt, a) ' b c')
+    def postfix(s)
+      debug("postfix") {"(#{s.inspect})"}
+      e, s = depth {primary(s)}
+      debug {"=> primary returned #{[e, s].inspect}"}
+      return ["", s] if e.to_s.empty?
+      if !s.to_s.empty?
+        t, ss = depth {terminal(s)}
+        debug {"=> #{[t, ss].inspect}"}
+        if t.is_a?(Array) && [:opt, :star, :plus].include?(t.first)
+          return [[t.first, e], ss]
+        end
+      end
+      [e, s]
+    end
+
+    ##
+    # parse primary
+    # 
+    #     >>> primary("a b c")
+    #     (a ' b c')
+    def primary(s)
+      debug("primary") {"(#{s.inspect})"}
+      t, s = depth {terminal(s)}
+      debug {"=> terminal returned #{[t, s].inspect}"}
+      if t.is_a?(Symbol) || t.is_a?(String)
+        [t, s]
+      elsif %w(range hex).map(&:to_sym).include?(t.first)
+        [t, s]
+      elsif t.first == :"("
+        e, s = depth {expression(s)}
+        debug {"=> expression returned #{[e, s].inspect}"}
+        [e, s]
+      else
+        ["", s]
+      end
+    end
+  
+    ##
+    # parse one terminal; return the terminal and the remaining string
+    # 
+    # A terminal is represented as a tuple whose 1st item gives the type;
+    # some types have additional info in the tuple.
+    # 
+    # @example
+    #     >>> terminal("'abc' def")
+    #     ('abc' ' def')
+    #     
+    #     >>> terminal("[0-9]")
+    #     ((range '0-9') '')
+    #     >>> terminal("#x00B7")
+    #     ((hex '#x00B7') '')
+    #     >>> terminal ("\[#x0300-#x036F\]")
+    #     ((range '#x0300-#x036F') '')
+    #     >>> terminal("\[^<>'{}|^`\]-\[#x00-#x20\]")
+    #     ((range "^<>'{}|^`") '-\[#x00-#x20\]')
+    def terminal(s)
+      s = s.strip
+      #STDERR.puts s.inspect
+      case m = s[0,1]
+      when '"', "'" # STRING1 or STRING2
+        l, s = s[1..-1].split(m.rstrip, 2)
+        [LL1::Lexer.unescape_string(l), s]
+      when '[' # RANGE, O_RANGE
+        l, s = s[1..-1].split(/(?<=[^\\])\]/, 2)
+        [[:range, LL1::Lexer.unescape_string(l)], s]
+      when '#' # HEX
+        s.match(/(#x\h+)(.*)$/)
+        l, s = $1, $2
+        [[:hex, l], s]
+      when /[\w\.]/ # SYMBOL
+        s.match(/([\w\.]+)(.*)$/)
+        l, s = $1, $2
+        [l.to_sym, s]
+      when '-'
+        [[:diff], s[1..-1]]
+      when '?'
+        [[:opt], s[1..-1]]
+      when '|'
+        [[:alt], s[1..-1]]
+      when '+'
+        [[:plus], s[1..-1]]
+      when '*'
+        [[:star], s[1..-1]]
+      when /[\(\)]/ # '(' or ')'
+        [[m.to_sym], s[1..-1]]
+      else
+        error("terminal", "unrecognized terminal: #{s.inspect}")
+        raise SyntaxError, "unrecognized terminal: #{s.inspect}"
+      end
+    end
+  end
+end