ebnf 0.3.2 → 0.3.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 3c3b6668384bfe503728e795abf4350ec4a327c9
-  data.tar.gz: d2b8ed718f483766f3e7756dd2feaac637d29dbe
+  metadata.gz: 1e418c3276c9f6252edf2fb8e89b815a95731629
+  data.tar.gz: 1f9fa2b8690bb7d8e5491a94b0f7f0fcc781af99
 SHA512:
-  metadata.gz: 0271fdc7808467c76a0c95a66982e6d97efdd9bcac52b5bcfde66fb1b763fab7a82eea344a98bd4fd81ef95de5a7865b88a2b912876ef69ed4814ba05cdbe635
-  data.tar.gz: 47e4b1314eb8068da0a4802d04206d47cfa2d52fe46b8b740cc00cd6187fea9b3e658441af3c2848d3b1a3bccfc614b31f95ffa3130f88cc42ade09aca0e86df
+  metadata.gz: 597dcc0e0f174981fc7d8285238f1b74bec9c0e8b4e3294de792d555af36fac9ae4983e112b31534d42f8a24a05fb250535a5c0594d425c15a662d4e97a93187
+  data.tar.gz: b1df195a3f03dada05472fb222a8df83236c12de389e8566d914b847f85b1be66dd5b8e3e6efcf5c52420794336b185f8fb4e8af9663be455f34f7a325a0c078

data/README.md

@@ -84,6 +84,8 @@ to each `terminal` and `production`. A trivial parser loop can be described as f
 The [EBNF][] variant used here is based on [W3C](http://w3.org/) [EBNF][] (see {file:etc/ebnf.ebnf EBNF grammar}) as defined in the
 [XML 1.0 recommendation](http://www.w3.org/TR/REC-xml/), with minor extensions.
 
+Parsing this grammar yields an S-Expression version: {file:etc/ebnf.ll1.sxp}.
+
 ## Example parser
 For an example parser built using this gem, see {file:examples/ebnf-parser/README EBNF Parser example}. This example creates a parser for the [EBNF][] grammar which generates the same Abstract Syntax Tree as the built-in parser in the gem.
 

data/VERSION

@@ -1 +1 @@
-0.3.2
+0.3.3

data/bin/ebnf

@@ -48,7 +48,7 @@ OPT_ARGS = [
   ["--bnf",             GetoptLong::NO_ARGUMENT,      "Transform EBNF to BNF"],
   ["--evaluate","-e",   GetoptLong::REQUIRED_ARGUMENT,"Evaluate argument as a JSON-LD document"],
   ["--ll1",             GetoptLong::REQUIRED_ARGUMENT,"Generate First/Follow rules, argument is start symbol"],
-  ["--format", "-f",    GetoptLong::REQUIRED_ARGUMENT,"Specify output format one of ttl, sxp, or rb"],
+  ["--format", "-f",    GetoptLong::REQUIRED_ARGUMENT,"Specify output format one of ebnf, ttl, sxp, or rb"],
   ["--input-format",    GetoptLong::REQUIRED_ARGUMENT,"Specify input format one of ebnf or sxp"],
   ["--mod-name",        GetoptLong::REQUIRED_ARGUMENT,"Module name used when creating ruby tables"],
   ["--output", "-o",    GetoptLong::REQUIRED_ARGUMENT,"Output to the specified file path"],
@@ -104,6 +104,7 @@ ebnf.make_bnf if options[:bnf] || options[:ll1]
 ebnf.first_follow(options[:ll1]) if options[:ll1]
 
 res = case options[:output_format]
+when :ebnf    then ebnf.to_s
 when :sxp     then ebnf.to_sxp
 when :ttl     then ebnf.to_ttl(options[:prefix], options[:namespace])
 when :rb      then dump_tables(ARGV[0], ebnf, out, options)

data/etc/ebnf.ebnf

@@ -32,32 +32,34 @@
 
     @terminals
 
-    [11] LHS        ::= ENUM SYMBOL "::="
+    [11] LHS        ::= ENUM? SYMBOL "::="
 
     [12] SYMBOL     ::= ([a-z] | [A-Z] | [0-9] | "_" | ".")+
 
     [13] HEX        ::= '#x' ([0-9]|[a-f]|[A-F]) ([0-9]|[a-f]|[A-F]) 
 
-    [14] RANGE      ::= '[' CHAR '-' CHAR ']'
+    [14] RANGE      ::= '[' R_CHAR '-' R_CHAR ']'
 
-    [15] ENUM       ::= '[' CHAR+ ']'
+    [15] ENUM       ::= '[' R_CHAR+ ']'
 
-    [16] O_RANGE    ::= '[^' CHAR '-' CHAR ']'
+    [16] O_RANGE    ::= '[^' R_CHAR '-' R_CHAR ']'
 
-    [17] O_ENUM     ::= '[^' CHAR+ ']'
+    [17] O_ENUM     ::= '[^' R_CHAR+ ']'
 
-    [18] STRING1    ::= '"' (CHAR | [\t\'\[\]\(\)\-])* '"'
+    [18] STRING1    ::= '"' ((CHAR - '"') | "\t")* '"'
 
-    [19] STRING2    ::= "'" (CHAR | [\t\"\[\]\(\)\-])* "'"
+    [19] STRING2    ::= "'" ((CHAR - "'") | "\t")* "'"
 
     [20] CHAR       ::= HEX
-                    |   ('\\' [\\trn\'\"\[\]\(\)\-])
-                    |   [^\t\r\n\'\"\[\]\(\)\-]
+                      | ('\\' [\\trn"'])
+                      | [^\t\r\n]
+
+    [21] R_CHAR     ::= CHAR - "]"
 
     # Should be able to do this inline, but not until terminal regular expressions are created automatically
-    [21] POSTFIX    ::= [?*+]
+    [22] POSTFIX    ::= [?*+]
 
-    [22] PASS       ::= ( [#x20\t\r\n]
+    [23] PASS       ::= ( [#x20\t\r\n]
                         | ( '#' | '//' ) [^\r\n]*
                         | '/*' (( '*' [^/] )? | [^*] )* '*/'
                         )+

data/etc/ebnf.ll1.sxp

@@ -141,22 +141,22 @@
   (first "(" ENUM HEX O_ENUM O_RANGE RANGE STRING1 STRING2 SYMBOL)
   (follow "@pass" "@terminals" LHS _eof)
   (seq expression))
- (terminal LHS "11" (seq ENUM SYMBOL "::="))
+ (terminal LHS "11" (seq (opt ENUM) SYMBOL "::="))
  (terminal SYMBOL "12" (plus (alt (range "a-z") (range "A-Z") (range "0-9") "_" ".")))
  (terminal HEX "13"
   (seq "#x"
    (alt (range "0-9") (range "a-f") (range "A-F"))
    (alt (range "0-9") (range "a-f") (range "A-F"))) )
- (terminal RANGE "14" (seq "[" CHAR "-" CHAR "]"))
- (terminal ENUM "15" (seq "[" (plus CHAR) "]"))
- (terminal O_RANGE "16" (seq "[^" CHAR "-" CHAR "]"))
- (terminal O_ENUM "17" (seq "[^" (plus CHAR) "]"))
- (terminal STRING1 "18" (seq "\"" (star (alt CHAR (range "\t'[]()-"))) "\""))
- (terminal STRING2 "19" (seq "'" (star (alt CHAR (range "\t\"[]()-"))) "'"))
- (terminal CHAR "20"
-  (alt HEX (seq "\\" (range "\\trn'\"[]()-")) (range "^\t\r\n'\"[]()-")))
- (terminal POSTFIX "21" (range "?*+"))
- (terminal PASS "22"
+ (terminal RANGE "14" (seq "[" R_CHAR "-" R_CHAR "]"))
+ (terminal ENUM "15" (seq "[" (plus R_CHAR) "]"))
+ (terminal O_RANGE "16" (seq "[^" R_CHAR "-" R_CHAR "]"))
+ (terminal O_ENUM "17" (seq "[^" (plus R_CHAR) "]"))
+ (terminal STRING1 "18" (seq "\"" (star (alt (diff CHAR "\"") "\t")) "\""))
+ (terminal STRING2 "19" (seq "'" (star (alt (diff CHAR "'") "\t")) "'"))
+ (terminal CHAR "20" (alt HEX (seq "\\" (range "\\trn\"'")) (range "^\t\r\n")))
+ (terminal R_CHAR "21" (diff CHAR "]"))
+ (terminal POSTFIX "22" (range "?*+"))
+ (terminal PASS "23"
   (plus
    (alt
     (range "#x20\t\r\n")

data/etc/ebnf.sxp

@@ -11,22 +11,22 @@
  (rule primary "9"
   (alt HEX SYMBOL RANGE ENUM O_RANGE O_ENUM STRING1 STRING2 (seq "(" expression ")")))
  (rule pass "10" (seq "@pass" expression))
- (terminal LHS "11" (seq ENUM SYMBOL "::="))
+ (terminal LHS "11" (seq (opt ENUM) SYMBOL "::="))
  (terminal SYMBOL "12" (plus (alt (range "a-z") (range "A-Z") (range "0-9") "_" ".")))
  (terminal HEX "13"
   (seq "#x"
    (alt (range "0-9") (range "a-f") (range "A-F"))
    (alt (range "0-9") (range "a-f") (range "A-F"))) )
- (terminal RANGE "14" (seq "[" CHAR "-" CHAR "]"))
- (terminal ENUM "15" (seq "[" (plus CHAR) "]"))
- (terminal O_RANGE "16" (seq "[^" CHAR "-" CHAR "]"))
- (terminal O_ENUM "17" (seq "[^" (plus CHAR) "]"))
- (terminal STRING1 "18" (seq "\"" (star (alt CHAR (range "\t'[]()-"))) "\""))
- (terminal STRING2 "19" (seq "'" (star (alt CHAR (range "\t\"[]()-"))) "'"))
- (terminal CHAR "20"
-  (alt HEX (seq "\\" (range "\\trn'\"[]()-")) (range "^\t\r\n'\"[]()-")))
- (terminal POSTFIX "21" (range "?*+"))
- (terminal PASS "22"
+ (terminal RANGE "14" (seq "[" R_CHAR "-" R_CHAR "]"))
+ (terminal ENUM "15" (seq "[" (plus R_CHAR) "]"))
+ (terminal O_RANGE "16" (seq "[^" R_CHAR "-" R_CHAR "]"))
+ (terminal O_ENUM "17" (seq "[^" (plus R_CHAR) "]"))
+ (terminal STRING1 "18" (seq "\"" (star (alt (diff CHAR "\"") "\t")) "\""))
+ (terminal STRING2 "19" (seq "'" (star (alt (diff CHAR "'") "\t")) "'"))
+ (terminal CHAR "20" (alt HEX (seq "\\" (range "\\trn\"'")) (range "^\t\r\n")))
+ (terminal R_CHAR "21" (diff CHAR "]"))
+ (terminal POSTFIX "22" (range "?*+"))
+ (terminal PASS "23"
   (plus
    (alt
     (range "#x20\t\r\n")

data/lib/ebnf.rb

@@ -4,6 +4,7 @@ module EBNF
   autoload :LL1,      "ebnf/ll1"
   autoload :Parser,   "ebnf/parser"
   autoload :Rule,     "ebnf/rule"
+  autoload :Writer,   "ebnf/writer"
   autoload :VERSION,  "ebnf/version"
 
   ##

data/lib/ebnf/base.rb

@@ -175,7 +175,13 @@ module EBNF
       require 'sxp' unless defined?(SXP)
       SXP::Generator.string(ast.sort_by{|r| r.id.to_f}.map(&:for_sxp))
     end
-    def to_s; to_sxp; end
+
+    ##
+    # Output formatted EBNF
+    # @return [String]
+    def to_s
+      Writer.string(*ast)
+    end
 
     def dup
       new_obj = super

data/lib/ebnf/ll1/lexer.rb

@@ -507,12 +507,5 @@ module EBNF::LL1
         super(message.to_s)
       end
     end # class Error
-    
-    unless "".respond_to?(:force_encoding)
-      # Compatibility with 1.9 Encoding
-      module Encoding
-        class CompatibilityError < StandardError; end
-      end
-    end
   end # class Lexer
 end # module EBNF

data/lib/ebnf/ll1/parser.rb

@@ -281,8 +281,8 @@ module EBNF::LL1
               end
             elsif prod_branch = @branch[cur_prod]
               sequence = prod_branch.fetch(token.representation) do
-                raise Error.new("#{token.inspect} does not match production #{cur_prod.inspect}",
-                  :production => cur_prod)
+                raise Error.new("Expected one of #{@first[cur_prod].inspect}",
+                  :token => token, :production => cur_prod)
               end
               debug("parse(production)") do
                 "token #{token.representation.inspect} " +
@@ -292,7 +292,7 @@ module EBNF::LL1
               end
               todo_stack.last[:terms] += sequence
             else
-              raise Error.new("No branches found for #{cur_prod.inspect}",
+              raise Error.new("Unexpected",
                 :production => cur_prod, :token => token)
             end
           end
@@ -309,8 +309,8 @@ module EBNF::LL1
             elsif terminals.include?(term) 
               # If term is a terminal, then it is an error if token does not
               # match it
-              raise Error.new("#{get_token.inspect} does not match terminal #{term.inspect}",
-                :production => cur_prod)
+              raise Error.new("Expected #{term.inspect}",
+                :token => get_token, :production => cur_prod)
             else
               token = get_token
 
@@ -339,12 +339,12 @@ module EBNF::LL1
           if e.is_a?(Lexer::Error)
             # Skip to the next valid terminal
             @lexer.recover
-            error("parse(#{e.class})", "With input '#{e.input}': #{e.message}, skipped to #{(get_token(:recover) || :eof).inspect}",
+            error("parse(#{e.class})", "With input '#{e.input}': #{e.message}",
                   :production => @productions.last)
           else
             # Otherwise, the terminal is fine, just not for this production.
             @lexer.shift
-            error("parse(#{e.class})", "#{e.message}, skipped to #{(get_token(:recover) || :eof).inspect}",
+            error("parse(#{e.class})", "#{e.message}",
                   :production => @productions.last, :token => e.token)
           end
 
@@ -422,7 +422,7 @@ module EBNF::LL1
 
       # When all is said and done, raise the error log
       unless @error_log.empty?
-        raise Error, @error_log.join("\n\t")
+        raise Error, @error_log.join("\n")
       end
     end
 
@@ -468,12 +468,14 @@ module EBNF::LL1
     # @option options [Token] :token
     # @see {#debug}
     def error(node, message, options = {})
-      message += ", found #{options[:token].inspect}" if options[:token]
-      message += " at line #{@lineno}" if @lineno
-      message += ", production = #{options[:production].inspect}" if options[:production]
-      @error_log << message unless @recovering
+      m = "ERROR "
+      m += "[line: #{@lineno}] " if @lineno
+      m += message
+      m += " (found #{options[:token].inspect})" if options[:token]
+      m += ", production = #{options[:production].inspect}" if options[:production]
+      @error_log << m unless @recovering
       @recovering = true
-      debug(node, message, options.merge(:level => 0))
+      debug(node, m, options.merge(:level => 0))
     end
 
     ##
@@ -486,11 +488,13 @@ module EBNF::LL1
     # @option options [Token] :token
     # @see {#debug}
     def warn(node, message, options = {})
-      message += ", with token #{options[:token].inspect}" if options[:token]
-      message += " at line #{@lineno}" if @lineno
-      message += ", production = #{options[:production].inspect}" if options[:production]
-      @error_log << message unless @recovering
-      debug(node, message, options.merge(:level => 1))
+      m = "WARNING "
+      m += "[line: #{@lineno}] " if @lineno
+      m += message
+      m += " (found #{options[:token].inspect})" if options[:token]
+      m += ", production = #{options[:production].inspect}" if options[:production]
+      @error_log << m unless @recovering
+      debug(node, m, options.merge(:level => 1))
     end
 
     ##
@@ -570,7 +574,7 @@ module EBNF::LL1
         # Make sure we push as many was we pop, even if there is no
         # explicit start handler
         @prod_data << {} if self.class.production_handlers[prod]
-        progress("#{prod}(:start)") { get_token.inspect + (@recovering ? ' recovering' : '')}
+        progress("#{prod}(:start:#{@prod_data.length})") { get_token.inspect + (@recovering ? ' recovering' : '')}
       end
       #puts "prod_data(s): " + @prod_data.inspect
     end
@@ -593,7 +597,7 @@ module EBNF::LL1
         end
         progress("#{prod}(:finish):#{@prod_data.length}") {@prod_data.last}
       else
-        progress("#{prod}(:finish)") { "recovering" if @recovering }
+        progress("#{prod}(:finish):#{@prod_data.length}") { "recovering" if @recovering }
       end
       @productions.pop
     end
@@ -611,7 +615,7 @@ module EBNF::LL1
               handler.call(parentProd, token, @prod_data.last, @parse_callback)
             }
           rescue ArgumentError, Error => e
-            error("terminal", "#{e.class}: #{e.message}", :production => prod)
+            error("terminal", "#{e.class}: #{e.message}", :token => token, :production => prod)
             @recovering = false
           end
           progress("#{prod}(:terminal)", "", :depth => (depth + 1)) {"#{token}: #{@prod_data.last}"}
@@ -619,7 +623,7 @@ module EBNF::LL1
           progress("#{prod}(:terminal)", "", :depth => (depth + 1)) {token.to_s}
         end
       else
-        error("#{parentProd}(:terminal)", "Terminal has no parent production", :production => prod)
+        error("#{parentProd}(:terminal)", "Terminal has no parent production", :token => token, :production => prod)
       end
     end
 
@@ -729,7 +733,7 @@ module EBNF::LL1
       def initialize(message, options = {})
         @production = options[:production]
         @token      = options[:token]
-        @lineno     = options[:lineno]
+        @lineno     = options[:lineno] || (@token.lineno if @token.respond_to?(:lineno))
         super(message.to_s)
       end
     end # class Error

data/lib/ebnf/parser.rb

@@ -35,7 +35,7 @@ module EBNF
           yield r unless r.empty?
           @lineno = cur_lineno
           r = s
-        when s = scanner.scan(/\[(?=[\w\.]+\])/)
+        when s = scanner.scan(/(?:\[[\w\.]+\])\s*[\w\.]+\s*::=/)
           # Found rule start, if we've already collected a rule, yield it
           yield r unless r.empty?
           #debug("eachRule(rule)") { "[#{cur_lineno}] #{s.inspect}" }
@@ -53,13 +53,14 @@ module EBNF
     end
   
     ##
-    # Parse a rule into a rule number, a symbol and an expression
+    # Parse a rule into an optional rule number, a symbol and an expression
     #
     # @param [String] rule
     # @return [Rule]
     def ruleParts(rule)
       num_sym, expr = rule.split('::=', 2).map(&:strip)
       num, sym = num_sym.split(']', 2).map(&:strip)
+      num, sym = "", num if sym.nil?
       num = num[1..-1]
       r = Rule.new(sym && sym.to_sym, num, expression(expr).first, :ebnf => self)
       debug("ruleParts") { r.inspect }

data/lib/ebnf/rule.rb

@@ -122,7 +122,8 @@ module EBNF
     # Return representation for building S-Expressions
     # @return [Array]
     def for_sxp
-      elements = [kind, sym, id]
+      elements = [kind, sym]
+      elements << id if id
       elements << [:start, true] if start
       elements << first.sort_by(&:to_s).unshift(:first) if first
       elements << follow.sort_by(&:to_s).unshift(:follow) if follow

data/lib/ebnf/writer.rb

@@ -0,0 +1,125 @@
+# -*- encoding: utf-8 -*-
+require 'rdf'
+
+##
+# Serialize ruleset back to EBNF
+module EBNF
+  class Writer
+    LINE_LENGTH = 80
+
+    ##
+    # Format rules to a String
+    #
+    # @param  [Array<Rule>] rules
+    # @return [Object]
+    def self.string(*rules)
+      require 'stringio' unless defined?(StringIO)
+      buf = StringIO.new
+      write(buf, *rules)
+      buf.string
+    end
+
+    ##
+    # Format rules to $stdout
+    #
+    # @param  [Array<Rule>] rules
+    # @return [Object]
+    def self.print(*rules)
+      write($stdout, *rules)
+    end
+
+    ##
+    # Write formatted rules to an IO like object
+    #
+    # @param  [Object] out
+    # @param  [Array<Rule>] rules
+    # @return [Object]
+    def self.write(out, *rules)
+      Writer.new(rules, out: out)
+    end
+
+    ##
+    # @param [Array<Rule>] rules
+    # @param [Hash{Symbol => Object}] options
+    # @option options [Symbol] :format
+    # @option options [#write] :out ($stdout)
+    def initialize(rules, options = {})
+      out = options.fetch(:out, $stdio)
+      #fmt = options.fetch(:format, :ebnf)
+
+      # Determine max LHS length
+      max_id = rules.max_by {|r| r.id.to_s.length}.id.to_s.length
+      max_sym = rules.max_by {|r| r.sym.to_s.length}.sym.to_s.length
+      lhs_length = max_sym + 3
+      lhs_fmt = "%-#{max_sym}{sym} ::= "
+      if max_id > 0
+        lhs_fmt = "%-#{max_id+2}{id} " + lhs_fmt
+        lhs_length += max_id + 3
+      end
+      rhs_length = LINE_LENGTH - lhs_length
+
+      # Format each rule, considering the available rhs size
+      rules.each do |rule|
+        buffer = if rule.pass?
+          "%-#{lhs_length-2}s" % "@pass"
+        else
+          lhs_fmt % {:id => "[#{rule.id}]", :sym => rule.sym}
+        end
+        formatted_expr = format(rule.expr)
+        if formatted_expr.length > rhs_length
+          buffer << format(rule.expr, ("\n" + " " * lhs_length))
+        else
+          buffer << formatted_expr
+        end
+        out.puts(buffer)
+      end
+    end
+
+    protected
+    # Format the expression part of a rule
+    def format(expr, sep = nil)
+      return expr.to_s if expr.is_a?(Symbol)
+      return %("#{escape(expr)}") if expr.is_a?(String)
+
+      case expr.first
+      when :alt, :diff
+        this_sep = (sep ? sep : " ") + {alt: "| ", diff: "- "}[expr.first.to_sym]
+        expr[1..-1].map {|e| format(e)}.join(this_sep)
+      when :star, :plus, :opt
+        raise "Expected star expression to have a single operand" unless expr.length == 2
+        char = {star: "*", plus: "+", opt: "?"}[expr.first.to_sym]
+        r = format(expr[1])
+        (r.start_with?("(") || Array(expr[1]).length == 1) ? "#{r}#{char}" : "(#{r})#{char}"
+      when :range
+        parts = expr.last.split(/(?!\\)-/, 2)
+        "[" + parts.map {|e| format(e)[1..-2]}.join("-") + "]"
+      when :seq
+        this_sep = (sep ? sep : " ")
+        expr[1..-1].map {|e| r = format(e); Array(e).length > 2 ? "(#{r})" : r}.join(this_sep)
+      else
+        raise "Unknown operator: #{expr.first}"
+      end
+    end
+
+    def escape(string)
+      buffer = ""
+      string.each_char do |c|
+        buffer << case c.to_s
+        when "\t" then "\\t"
+        when "\n" then "\\n"
+        when "\r" then "\\r"
+        when "\\" then "\\\\"
+        #when "("  then "\\("
+        #when ")"  then "\\)"
+        #when "["  then "\\["
+        #when "]"  then "\\]"
+        #when "-"  then "\\\\-"
+        when "'"  then "\\'"
+        when '"'  then "\\\""
+        else c
+        end
+      end
+      buffer
+    end
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ebnf
 version: !ruby/object:Gem::Version
-  version: 0.3.2
+  version: 0.3.3
 platform: ruby
 authors:
 - Gregg Kellogg
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-12-10 00:00:00.000000000 Z
+date: 2013-12-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: sxp
@@ -102,6 +102,7 @@ files:
 - lib/ebnf/parser.rb
 - lib/ebnf/rule.rb
 - lib/ebnf/version.rb
+- lib/ebnf/writer.rb
 - lib/ebnf.rb
 - etc/doap.ttl
 - etc/ebnf.ebnf
@@ -125,7 +126,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - '>='
     - !ruby/object:Gem::Version
-      version: 1.8.7
+      version: 1.9.2
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - '>='