ebnf 1.0.2 → 1.1.0

data/lib/ebnf/ll1/parser.rb

@@ -226,8 +226,9 @@ module EBNF::LL1
       when @options[:validate] then 1
       end
       @branch  = options[:branch]
-      @first  = options[:first] ||= {}
+      @first   = options[:first] ||= {}
       @follow  = options[:follow] ||= {}
+      @cleanup = options[:cleanup] ||= {}
       @lexer   = input.is_a?(Lexer) ? input : Lexer.new(input, self.class.patterns, @options)
       @productions = []
       @parse_callback = block
@@ -241,7 +242,7 @@ module EBNF::LL1
 
       @prod_data = [{}]
       start = start.split('#').last.to_sym unless start.is_a?(Symbol)
-      todo_stack = [{:prod => start, :terms => nil}]
+      todo_stack = [{prod: start, terms: nil}]
 
       while !todo_stack.empty?
         begin
@@ -255,7 +256,7 @@ module EBNF::LL1
             # to the beginning to avoid excessive growth in the production
             # stack
             if options[:reset_on_start] && cur_prod == start
-              todo_stack = [{:prod => start, :terms => []}]
+              todo_stack = [{prod: start, terms: []}]
               @productions = []
               @prod_data = [{}]
             end
@@ -320,7 +321,7 @@ module EBNF::LL1
                 break
               else
                 # Push term onto stack
-                todo_stack << {:prod => term, :terms => nil}
+                todo_stack << {prod: term, terms: nil}
                 debug("parse(push)") {"term #{term.inspect}, depth #{depth}"}
                 pushed = true
                 break
@@ -348,9 +349,9 @@ module EBNF::LL1
           end
 
           # Get the list of follows for this sequence, this production and the stacked productions.
-          debug("recovery", "stack follows:", :level => 4)
+          debug("recovery", "stack follows:", level: 4)
           todo_stack.reverse.each do |todo|
-            debug("recovery", :level => 4) {"  #{todo[:prod]}: #{@follow[todo[:prod]].inspect}"}
+            debug("recovery", level: 4) {"  #{todo[:prod]}: #{@follow[todo[:prod]].inspect}"}
           end
 
           # Find all follows to the top of the stack
@@ -403,7 +404,7 @@ module EBNF::LL1
         end
       end
 
-      error("parse(eof)", "Finished processing before end of file", :token => @lexer.first) if @lexer.first
+      error("parse(eof)", "Finished processing before end of file", token: @lexer.first) if @lexer.first
 
       # Continue popping contexts off of the stack
       while !todo_stack.empty?
@@ -475,7 +476,7 @@ module EBNF::LL1
       m += ", production = #{options[:production].inspect}" if options[:production]
       @error_log << m unless @recovering
       @recovering = true
-      debug(node, m, options.merge(:level => 0))
+      debug(node, m, options.merge(level: 0))
       if options[:raise] || @options[:validate]
         raise Error.new(m, lineno: lineno, token: options[:token], production: options[:production])
       end
@@ -497,7 +498,7 @@ module EBNF::LL1
       m += " (found #{options[:token].inspect})" if options[:token]
       m += ", production = #{options[:production].inspect}" if options[:production]
       @error_log << m unless @recovering
-      debug(node, m, options.merge(:level => 1))
+      debug(node, m, options.merge(level: 1))
     end
 
     ##
@@ -569,10 +570,14 @@ module EBNF::LL1
             handler.call(@prod_data.last, data, @parse_callback)
           }
         rescue ArgumentError, Error => e
-          error("start", "#{e.class}: #{e.message}", :production => prod)
+          error("start", "#{e.class}: #{e.message}", production: prod)
           @recovering = false
         end
         @prod_data << data
+      elsif [:merge, :star].include?(@cleanup[prod])
+        # Save current data to merge later
+        @prod_data << {}
+        progress("#{prod}(:start}:#{@prod_data.length}:cleanup:#{@cleanup[prod]}") { get_token.inspect + (@recovering ? ' recovering' : '')}
       else
         # Make sure we push as many was we pop, even if there is no
         # explicit start handler
@@ -595,10 +600,23 @@ module EBNF::LL1
             handler.call(@prod_data.last, data, @parse_callback)
           }
         rescue ArgumentError, Error => e
-          error("finish", "#{e.class}: #{e.message}", :production => prod)
+          error("finish", "#{e.class}: #{e.message}", production: prod)
           @recovering = false
         end
         progress("#{prod}(:finish):#{@prod_data.length}") {@prod_data.last}
+      elsif [:merge, :star].include?(@cleanup[prod])
+        data = @prod_data.pop
+        input = @prod_data.last
+
+        # Append every element in data to last prod_data
+        data.each do |k, v|
+          input[k] = case input[k]
+          when nil then v.is_a?(Hash) ? v : Array(v)
+          when Hash then input[k].merge!(v)
+          else Array(input[k]) + Array(v)
+          end
+        end
+        progress("#{prod}(:finish):#{@prod_data.length} cleanup:#{@cleanup[prod]}") {@prod_data.last}
       else
         progress("#{prod}(:finish):#{@prod_data.length}") { "recovering" if @recovering }
       end
@@ -618,15 +636,15 @@ module EBNF::LL1
               handler.call(parentProd, token, @prod_data.last, @parse_callback)
             }
           rescue ArgumentError, Error => e
-            error("terminal", "#{e.class}: #{e.message}", :token => token, :production => prod)
+            error("terminal", "#{e.class}: #{e.message}", token: token, production: prod)
             @recovering = false
           end
-          progress("#{prod}(:terminal)", "", :depth => (depth + 1)) {"#{token}: #{@prod_data.last}"}
+          progress("#{prod}(:terminal)", "", depth: (depth + 1)) {"#{token}: #{@prod_data.last}"}
         else
-          progress("#{prod}(:terminal)", "", :depth => (depth + 1)) {token.to_s}
+          progress("#{prod}(:terminal)", "", depth: (depth + 1)) {token.to_s}
         end
       else
-        error("#{parentProd}(:terminal)", "Terminal has no parent production", :token => token, :production => prod)
+        error("#{parentProd}(:terminal)", "Terminal has no parent production", token: token, production: prod)
       end
     end
 
@@ -703,7 +721,7 @@ module EBNF::LL1
     # @example Raising a parser error
     #   raise Error.new(
     #     "invalid token '%' on line 10",
-    #     :token => '%', :lineno => 9, :production => :turtleDoc)
+    #     token: '%', lineno: 9, production: :turtleDoc)
     #
     # @see http://ruby-doc.org/core/classes/StandardError.html
     class Error < StandardError

data/lib/ebnf/ll1/scanner.rb

@@ -18,7 +18,24 @@ module EBNF::LL1
     attr_reader :input
 
     ##
-    # Create a scanner, from an IO or String
+    # If we don't have an IO input, simply use StringScanner directly
+    # @private
+    def self.new(input, options = {})
+      input ||= ""
+      if input.respond_to?(:read)
+        scanner = self.allocate
+        scanner.send(:initialize, input, options)
+      else
+        if input.encoding != Encoding::UTF_8
+          input = input.dup if input.frozen?
+          input.force_encoding(Encoding::UTF_8)
+        end
+        StringScanner.new(input)
+      end
+    end
+
+    ##
+    # Create a scanner, from an IO
     #
     # @param [String, IO, #read] input
     # @param [Hash{Symbol => Object}] options
@@ -28,13 +45,10 @@ module EBNF::LL1
     def initialize(input, options = {})
       @options = options.merge(high_water: HIGH_WATER, low_water: LOW_WATER)
 
-      if input.respond_to?(:read)
-        @input = input
-        super("")
-        feed_me
-      else
-        super(encode_utf8 input.to_s)
-      end
+      @input = input
+      super("")
+      feed_me
+      self
     end
 
     ##

data/lib/ebnf/parser.rb

@@ -78,7 +78,7 @@ module EBNF
       num, sym = num_sym.split(']', 2).map(&:strip)
       num, sym = "", num if sym.nil?
       num = num[1..-1]
-      r = Rule.new(sym && sym.to_sym, num, expression(expr).first, :ebnf => self)
+      r = Rule.new(sym && sym.to_sym, num, expression(expr).first, ebnf: self)
       debug("ruleParts") { r.inspect }
       r
     end

data/lib/ebnf/rule.rb

@@ -54,6 +54,9 @@ module EBNF
     # @return [Boolean]
     attr_accessor :start
 
+    # Determines preparation and cleanup rules for reconstituting EBNF ? * + from BNF
+    attr_accessor :cleanup
+
     # @param [Integer] id
     # @param [Symbol] sym
     # @param [Array] expr
@@ -71,6 +74,7 @@ module EBNF
       @first = options[:first]
       @follow = options[:follow]
       @start = options[:start]
+      @cleanup = options[:cleanup]
       @kind = case
       when options[:kind] then options[:kind]
       when sym.to_s == sym.to_s.upcase then :terminal
@@ -97,10 +101,12 @@ module EBNF
       first = first[1..-1] if first
       follow = sxp.detect {|e| e.is_a?(Array) && e.first.to_sym == :follow}
       follow = follow[1..-1] if follow
+      cleanup = sxp.detect {|e| e.is_a?(Array) && e.first.to_sym == :cleanup}
+      cleanup = cleanup[1..-1] if cleanup
       start = sxp.any? {|e| e.is_a?(Array) && e.first.to_sym == :start}
       sym = sxp[1] if sxp[1].is_a?(Symbol)
       id = sxp[2] if sxp[2].is_a?(String)
-      Rule.new(sym, id, expr, :kind => sxp.first, :first => first, :follow => follow, :start => start)
+      Rule.new(sym, id, expr, kind: sxp.first, first: first, follow: follow, cleanup: cleanup, start: start)
     end
 
     # Build a new rule creating a symbol and numbering from the current rule
@@ -113,9 +119,10 @@ module EBNF
     def build(expr, options = {})
       new_sym, new_id = (@top_rule ||self).send(:make_sym_id)
       Rule.new(new_sym, new_id, expr, {
-        :kind => options[:kind],
-        :ebnf => @ebnf,
-        :top_rule => @top_rule || self,
+        kind: options[:kind],
+        ebnf: @ebnf,
+        top_rule: @top_rule || self,
+        cleanup: options[:cleanup],
       }.merge(options))
     end
 
@@ -127,6 +134,7 @@ module EBNF
       elements << [:start, true] if start
       elements << first.sort_by(&:to_s).unshift(:first) if first
       elements << follow.sort_by(&:to_s).unshift(:follow) if follow
+      elements << [:cleanup, cleanup] if cleanup
       elements << expr
       elements
     end
@@ -196,16 +204,19 @@ module EBNF
         this = dup
         #   * Transform (a rule (opt b)) into (a rule (alt _empty b))
         this.expr = [:alt, :_empty, expr.last]
+        this.cleanup = :opt
         new_rules = this.to_bnf
       elsif expr.first == :star
         #   * Transform (a rule (star b)) into (a rule (alt _empty (seq b a)))
         this = dup
-        new_rule = this.build([:seq, expr.last, this.sym])
+        this.cleanup = :star
+        new_rule = this.build([:seq, expr.last, this.sym], cleanup: :merge)
         this.expr = [:alt, :_empty, new_rule.sym]
         new_rules = [this] + new_rule.to_bnf
       elsif expr.first == :plus
         #   * Transform (a rule (plus b)) into (a rule (seq b (star b)
         this = dup
+        this.cleanup = :plus
         this.expr = [:seq, expr.last, [:star, expr.last]]
         new_rules = this.to_bnf
       elsif [:alt, :seq].include?(expr.first)
@@ -340,7 +351,7 @@ module EBNF
 
     def inspect
       "#<EBNF::Rule:#{object_id} " +
-      {:sym => sym, :id => id, :kind => kind, :expr => expr}.inspect +
+      {sym: sym, id: id, kind: kind, expr: expr}.inspect +
       ">"
     end
 
@@ -387,7 +398,7 @@ module EBNF
     private
     def ttl_expr(expr, pfx, depth, is_obj = true)
       indent = '  ' * depth
-      @ebnf.debug("ttl_expr", :depth => depth) {expr.inspect}
+      @ebnf.debug("ttl_expr", depth: depth) {expr.inspect}
       op = expr.shift if expr.is_a?(Array)
       statements = []
       
@@ -424,7 +435,7 @@ module EBNF
       end
       
       statements.last << " ." unless is_obj
-      @ebnf.debug("statements", :depth => depth) {statements.join("\n")}
+      @ebnf.debug("statements", depth: depth) {statements.join("\n")}
       statements
     end
     
@@ -461,10 +472,11 @@ module EBNF
     end
 
     # Make a new symbol/number combination
-    def make_sym_id
+    # @param [String] variation added to symbol to aid reconstitution from BNF to EBNF
+    def make_sym_id(variation = nil)
       @id_seq ||= 0
       @id_seq += 1
-      ["_#{@sym}_#{@id_seq}".to_sym, "#{@id}.#{@id_seq}"]
+      ["_#{@sym}_#{@id_seq}#{variation}".to_sym, "#{@id}.#{@id_seq}#{variation}"]
     end
   end
 end

data/lib/ebnf/writer.rb

@@ -94,7 +94,7 @@ module EBNF
         buffer = if rule.pass?
           "%-#{lhs_length-2}s" % "@pass"
         else
-          lhs_fmt % {:id => "[#{rule.id}]", :sym => rule.sym}
+          lhs_fmt % {id: "[#{rule.id}]", sym: rule.sym}
         end
         formatted_expr = format(rule.expr)
         if formatted_expr.length > rhs_length

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ebnf
 version: !ruby/object:Gem::Version
-  version: 1.0.2
+  version: 1.1.0
 platform: ruby
 authors:
 - Gregg Kellogg
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-11-23 00:00:00.000000000 Z
+date: 2016-12-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: sxp
@@ -169,7 +169,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.5.1
+rubygems_version: 2.6.8
 signing_key: 
 specification_version: 4
 summary: EBNF parser and parser generator.