ebnf 0.2.2 → 0.2.3

checksums.yaml

@@ -1,15 +1,15 @@
 ---
 !binary "U0hBMQ==":
   metadata.gz: !binary |-
-    MmVlMjA5ODQ5YTk5OWUxZDgwYzc2Y2RjYjEyOTU2NTAzN2U0MjQ5Yw==
+    NDg4MzMwZTA0Yzg3MzY5Y2UyZDMwMWI2ZWRiODNjZWZjZmU5NGRlZQ==
   data.tar.gz: !binary |-
-    MWE3ZGJmYTYwNzdlYmQxZGEzZDdkY2U4NDc4NjhhYTI2MGYxZDhlNg==
+    NzA2ODg3ZTI1NzIwNjVjNmE1YWJjMDYzNTEyNjI1NGIzZWI5OTg1ZQ==
 !binary "U0hBNTEy":
   metadata.gz: !binary |-
-    MGMyZGRlM2I5Zjc4NzQ1YTRlMDJjYmY3M2NjNDc3Y2VmOTRjYmUyMGIzMGM3
-    ZjUzMzVkMmM2ODBmNGE3OGQ3ZDcxNjM4ODk0MmQ3YWZhYTllOTI4N2JkMmU0
-    MDI4MjIxM2FiMDE1MjhjMzdlNmVmNWZjN2ExOWY2ZDVhMDAyMzQ=
+    N2FjYTRhNzI5NDFkZWI3OTNiOWFkNTNmYWY5NjUwMjA4YTM0ZmNiZTQ5NDYy
+    MjFjODYxODc1MzRjMTZjNGM0N2U4NDk2NTM2ZjM0MmY0ZGI3ZDE3OTg3OWFi
+    NjQwYTdjNzcxYTc3ZWJmYzcyMzlmOWJiZDlmZjc3YTJiOWVkODA=
   data.tar.gz: !binary |-
-    YWQwMDk1Y2U2MzdlMTYzNGVjY2VlOGQ1ZGIyYTNmYzI4YzkwYjUzNTczMWYy
-    YzExNTE4NDkyNjk5NzQ1N2Y1MmY5NTkwZjI3NjE4Zjk0NmViZmU0NjM2NzYx
-    Mzk1MTMzZDQyN2Y2YWI5ODMyYjE1YTdmM2I1OThjOTQ1OTUwMDI=
+    ODVmZDZlMjVkMjY0ZDkwYWFiZGY4OGE3ZTRhNDQzZWRiZDlkZTQzODQ4ZWMw
+    MTZkZGMzNWYzZTMxNDc0MDc3YWFkNmU2NjExNWM2ZTkzOTJlZWE0MDQ1Yzdm
+    N2RmOTU5NGFhZTVlZDEwYzhlMjJjMDNhYmIwNDkyOTc3ZGVkNDQ=

data/VERSION

@@ -1 +1 @@
-0.2.2
+0.2.3

data/lib/ebnf/ll1/lexer.rb

@@ -135,10 +135,7 @@ module EBNF::LL1
       raise Error, "Terminal patterns not defined" unless @terminals && @terminals.length > 0
 
       @lineno = 1
-      @scanner = Scanner.new(input) do |string|
-        string.force_encoding(Encoding::UTF_8) if string.respond_to?(:force_encoding)
-        string
-      end
+      @scanner = Scanner.new(input)
     end
 
     ##

data/lib/ebnf/ll1/parser.rb

@@ -193,6 +193,8 @@ module EBNF::LL1
     #   Show progress of parser productions
     # @option options [Boolean] :debug
     #   Detailed debug output
+    # @option options [Boolean] :reset_on_start
+    #   Reset the parser state if the start token set with `prod` is found in a production. This reduces the production stack depth growth, which is appropriate for some grammars.
     # @yield [context, *data]
     #   Yields for to return data to parser
     # @yieldparam [:statement, :trace] context
@@ -227,6 +229,15 @@ module EBNF::LL1
           todo_stack.last[:terms] = []
           cur_prod = todo_stack.last[:prod]
 
+          # If cur_prod is the starting production, we can reset the stack
+          # to the beginning to avoid excessive growth in the production
+          # stack
+          if options[:reset_on_start] && cur_prod == prod
+            todo_stack = [{:prod => prod, :terms => []}]
+            @productions = []
+            @prod_data = [{}]
+          end
+
           # Get this first valid token appropriate for the stacked productions,
           # skipping invalid tokens until either a valid token is found (from @first),
           # or a token appearing in @follow appears.
@@ -239,11 +250,11 @@ module EBNF::LL1
             "prod #{cur_prod.inspect}, " + 
             "depth #{depth}"
           end
-          
+
           # Got an opened production
           onStart(cur_prod)
           break if token.nil?
-          
+
           if prod_branch = @branch[cur_prod]
             @recovering = false
             sequence = prod_branch[token.representation]
@@ -304,7 +315,6 @@ module EBNF::LL1
                 (@recovering && @follow.fetch(terms.last, []).none? {|t| (token || :_eps) == t}))
           debug("parse(pop)", :level => 2) {"todo #{todo_stack.last.inspect}, depth #{depth}"}
           if terms.empty?
-            prod = todo_stack.last[:prod]
             todo_stack.pop
             onFinish
           else
@@ -483,21 +493,6 @@ module EBNF::LL1
       token
     end
 
-    ##
-    # @param [String] node Relevant location associated with message
-    # @param [String] message Error string
-    # @param [Hash] options
-    # @option options [URI, #to_s] :production
-    # @option options [Token] :token
-    def error(node, message, options = {})
-      message += ", found #{options[:token].representation.inspect}" if options[:token]
-      message += " at line #{@lineno}" if @lineno
-      message += ", production = #{options[:production].inspect}" if options[:production]
-      @error_log << message unless @recovering
-      @recovering = true
-      debug(node, message, options.merge(:level => 0))
-    end
-
     ##
     # Return the next token, entering error recovery if the token is invalid
     #
@@ -521,6 +516,21 @@ module EBNF::LL1
       token
     end
 
+    ##
+    # @param [String] node Relevant location associated with message
+    # @param [String] message Error string
+    # @param [Hash] options
+    # @option options [URI, #to_s] :production
+    # @option options [Token] :token
+    def error(node, message, options = {})
+      message += ", found #{options[:token].representation.inspect}" if options[:token]
+      message += " at line #{@lineno}" if @lineno
+      message += ", production = #{options[:production].inspect}" if options[:production]
+      @error_log << message unless @recovering
+      @recovering = true
+      debug(node, message, options.merge(:level => 0))
+    end
+
     ##
     # Progress output when parsing
     #   param [String] node Relevant location associated with message
@@ -535,35 +545,57 @@ module EBNF::LL1
       message = args.join(",")
       depth = options[:depth] || self.depth
       message += yield.to_s if block_given?
-      if @options[:debug]
-        return debug(node, message, {:level => 0}.merge(options))
-      else
-        $stderr.puts("[#{@lineno}]#{' ' * depth}#{node}: #{message}")
-      end
+      debug(node, message, options.merge(:level => 1))
     end
 
     ##
-    # Progress output when debugging
-    # @param [String] node Relevant location associated with message
-    # @param [String] message ("")
-    # @param [Hash] options
-    # @option options [Integer] :depth
-    #   Recursion depth for indenting output
+    # Progress output when debugging.
+    # Captures output to `@options[:debug]` if it is an array.
+    # Otherwise, if `@options[:debug]` is set, or
+    # `@options[:progress]` is set and `:level` <= 1, or
+    # `@options[:validate]` is set and `:level` == 0 output
+    # to standard error.
+    # 
+    # @overload debug(node, message)
+    #   @param [String] node Relevant location associated with message
+    #   @param [String] message ("")
+    #   @param [Hash] options
+    #   @option options [Integer] :depth
+    #     Recursion depth for indenting output
+    #   @option options [Integer] :level
+    #     Debug level, `0` for errors, `1` for progress, anything else
+    #     for debug output.
+    #
+    # @overload debug(message)
+    #   @param [String] node Relevant location associated with message
+    #   @param [Hash] options
+    #   @option options [Integer] :depth
+    #     Recursion depth for indenting output
+    #   @option options [Integer] :level
+    #     Debug level, `0` for errors, `1` for progress, anything else
+    #     for debug output.
     # @yieldreturn [String] added to message
-    def debug(node, message = "", options = {})
-      return unless @options[:debug]
-      debug_level = options.fetch(:level, 1)
-      return unless debug_level <= DEBUG_LEVEL
+    def debug(*args)
+      options = args.last.is_a?(Hash) ? args.pop : {}
+      debug_level = options.fetch(:level, 2)
+      return unless @options[:debug]  && debug_level <= DEBUG_LEVEL ||
+                    @options[:progress] && debug_level <= 1 ||
+                    @options[:validate] && debug_level == 0
       depth = options[:depth] || self.depth
-      message += yield if block_given?
-      str = "[#{@lineno}](#{debug_level})#{' ' * depth}#{node}: #{message}"
-      case @options[:debug]
-      when Array
-        @options[:debug] << str
-      when TrueClass
-        $stderr.puts str
-      when :yield
+      d_str = depth > 20 ? ' ' * 20 + '+' : ' ' * depth
+      args << yield if block_given?
+      message = "#{args.join(': ')}"
+      str = "[#{@lineno}](#{debug_level})#{d_str}#{message}"
+      @options[:debug] << str if @options[:debug].is_a?(Array)
+      case
+      when @options[:yield]
         @parse_callback.call(:trace, node, message, options)
+      when @options[:debug] == true
+        $stderr.puts str
+      when @options[:progress] && debug_level <= 1
+        $stderr.puts str
+      when @options[:validate] && debug_level == 0
+        $stderr.puts str
       end
     end
 

data/lib/ebnf/ll1/scanner.rb

@@ -24,12 +24,8 @@ module EBNF::LL1
     # @param [Hash{Symbol => Object}] options
     # @option options[Integer] :high_water (HIGH_WATER)
     # @option options[Integer] :low_water (LOW_WATER)
-    # @yield [string]
-    # @yieldparam [String] string data read from input file
-    # @yieldreturn [String] replacement read data, useful for decoding escapes.
     # @return [Scanner]
-    def initialize(input, options = {}, &block)
-      @block = block
+    def initialize(input, options = {})
       @options = options.merge(:high_water => HIGH_WATER, :low_water => LOW_WATER)
 
       if input.respond_to?(:read)
@@ -48,7 +44,7 @@ module EBNF::LL1
     # @return [String]
     def rest
       feed_me
-      super
+      encode_utf8 super
     end
     
     ##
@@ -82,7 +78,7 @@ module EBNF::LL1
     # @return [String]
     def scan(pattern)
       feed_me
-      super
+      encode_utf8 super
     end
     
   private
@@ -91,11 +87,15 @@ module EBNF::LL1
       if rest_size < @options[:low_water] && @input && !@input.eof?
         # Read up to high-water mark ensuring we're at an end of line
         diff = @options[:high_water] - rest_size
-        string = @input.read(diff)
-        string << @input.gets unless @input.eof?
-        string = @block.call(string) if @block
+        string = encode_utf8(@input.read(diff))
+        string << encode_utf8(@input.gets) unless @input.eof?
         self << string if string
       end
     end
+
+    # Perform UTF-8 encoding of input
+    def encode_utf8(string)
+      string.respond_to?(:force_encoding) ? string.force_encoding(Encoding::UTF_8) : string
+    end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ebnf
 version: !ruby/object:Gem::Version
-  version: 0.2.2
+  version: 0.2.3
 platform: ruby
 authors:
 - Gregg Kellogg
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-03-22 00:00:00.000000000 Z
+date: 2013-03-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: sxp