ebnf 1.0.2 → 1.1.0

data/etc/sparql.rb

@@ -45629,5 +45629,145 @@ module Branch
       :DECIMAL_NEGATIVE,
       :DOUBLE_NEGATIVE],
   }.freeze
+  CLEANUP = {
+    :_Add_1 => :opt,
+    :_AdditiveExpression_1 => :star,
+    :_AdditiveExpression_3 => :merge,
+    :_AdditiveExpression_8 => :opt,
+    :_Aggregate_10 => :opt,
+    :_Aggregate_11 => :opt,
+    :_Aggregate_12 => :opt,
+    :_Aggregate_13 => :opt,
+    :_Aggregate_14 => :opt,
+    :_Aggregate_15 => :opt,
+    :_Aggregate_16 => :opt,
+    :_Aggregate_8 => :opt,
+    :_ArgList_2 => :opt,
+    :_ArgList_3 => :star,
+    :_ArgList_5 => :merge,
+    :_AskQuery_1 => :star,
+    :_AskQuery_2 => :merge,
+    :_Clear_1 => :opt,
+    :_CollectionPath_1 => :plus,
+    :_CollectionPath_2 => :star,
+    :_CollectionPath_3 => :merge,
+    :_Collection_1 => :plus,
+    :_Collection_2 => :star,
+    :_Collection_3 => :merge,
+    :_ConditionalAndExpression_1 => :star,
+    :_ConditionalAndExpression_3 => :merge,
+    :_ConditionalOrExpression_1 => :star,
+    :_ConditionalOrExpression_3 => :merge,
+    :_ConstructQuery_4 => :star,
+    :_ConstructQuery_5 => :merge,
+    :_ConstructQuery_6 => :star,
+    :_ConstructQuery_7 => :opt,
+    :_ConstructQuery_8 => :merge,
+    :_ConstructTemplate_1 => :opt,
+    :_ConstructTriples_1 => :opt,
+    :_ConstructTriples_3 => :opt,
+    :_Copy_1 => :opt,
+    :_Create_1 => :opt,
+    :_DescribeQuery_2 => :star,
+    :_DescribeQuery_3 => :opt,
+    :_DescribeQuery_4 => :plus,
+    :_DescribeQuery_5 => :star,
+    :_DescribeQuery_6 => :merge,
+    :_DescribeQuery_7 => :merge,
+    :_Drop_1 => :opt,
+    :_ExpressionList_2 => :star,
+    :_ExpressionList_4 => :merge,
+    :_GraphOrDefault_2 => :opt,
+    :_GroupClause_1 => :plus,
+    :_GroupClause_2 => :star,
+    :_GroupClause_3 => :merge,
+    :_GroupCondition_2 => :opt,
+    :_GroupGraphPatternSub_1 => :opt,
+    :_GroupGraphPatternSub_2 => :star,
+    :_GroupGraphPatternSub_4 => :merge,
+    :_GroupGraphPatternSub_5 => :opt,
+    :_GroupGraphPatternSub_6 => :opt,
+    :_GroupOrUnionGraphPattern_1 => :star,
+    :_GroupOrUnionGraphPattern_3 => :merge,
+    :_HavingClause_1 => :plus,
+    :_HavingClause_2 => :star,
+    :_HavingClause_3 => :merge,
+    :_InlineDataFull_10 => :merge,
+    :_InlineDataFull_2 => :star,
+    :_InlineDataFull_4 => :star,
+    :_InlineDataFull_5 => :merge,
+    :_InlineDataFull_7 => :merge,
+    :_InlineDataFull_9 => :star,
+    :_InlineDataOneVar_1 => :star,
+    :_InlineDataOneVar_2 => :merge,
+    :_LimitOffsetClauses_3 => :opt,
+    :_LimitOffsetClauses_4 => :opt,
+    :_Load_1 => :opt,
+    :_Load_2 => :opt,
+    :_Modify_1 => :opt,
+    :_Modify_3 => :star,
+    :_Modify_6 => :opt,
+    :_Modify_7 => :merge,
+    :_Move_1 => :opt,
+    :_MultiplicativeExpression_1 => :star,
+    :_MultiplicativeExpression_3 => :merge,
+    :_ObjectListPath_1 => :star,
+    :_ObjectListPath_3 => :merge,
+    :_ObjectList_1 => :star,
+    :_ObjectList_3 => :merge,
+    :_OrderClause_1 => :plus,
+    :_OrderClause_2 => :star,
+    :_OrderClause_3 => :merge,
+    :_PathAlternative_1 => :star,
+    :_PathAlternative_3 => :merge,
+    :_PathElt_1 => :opt,
+    :_PathNegatedPropertySet_2 => :opt,
+    :_PathNegatedPropertySet_4 => :star,
+    :_PathNegatedPropertySet_6 => :merge,
+    :_PathSequence_1 => :star,
+    :_PathSequence_3 => :merge,
+    :Prologue => :star,
+    :_Prologue_2 => :merge,
+    :PropertyList => :opt,
+    :_PropertyListNotEmptyPath_2 => :star,
+    :_PropertyListNotEmptyPath_4 => :merge,
+    :_PropertyListNotEmptyPath_5 => :opt,
+    :_PropertyListNotEmpty_1 => :star,
+    :_PropertyListNotEmpty_3 => :merge,
+    :_PropertyListNotEmpty_4 => :opt,
+    :PropertyListPath => :opt,
+    :_QuadsNotTriples_1 => :opt,
+    :_Quads_1 => :opt,
+    :_Quads_2 => :star,
+    :_Quads_4 => :merge,
+    :_Quads_5 => :opt,
+    :_Quads_6 => :opt,
+    :QueryUnit => :opt,
+    :_RDFLiteral_1 => :opt,
+    :_RegexExpression_1 => :opt,
+    :_RelationalExpression_1 => :opt,
+    :_SelectClause_1 => :opt,
+    :_SelectClause_4 => :plus,
+    :_SelectClause_6 => :star,
+    :_SelectClause_7 => :merge,
+    :_SelectQuery_1 => :star,
+    :_SelectQuery_2 => :merge,
+    :_ServiceGraphPattern_1 => :opt,
+    :_SolutionModifier_1 => :opt,
+    :_SolutionModifier_2 => :opt,
+    :_SolutionModifier_3 => :opt,
+    :_SolutionModifier_4 => :opt,
+    :_StrReplaceExpression_1 => :opt,
+    :_SubstringExpression_1 => :opt,
+    :_TriplesBlock_1 => :opt,
+    :_TriplesBlock_3 => :opt,
+    :_TriplesTemplate_1 => :opt,
+    :_TriplesTemplate_3 => :opt,
+    :_Update_1 => :opt,
+    :_Update_3 => :opt,
+    :ValuesClause => :opt,
+    :_WhereClause_1 => :opt,
+    :_iriOrFunction_1 => :opt,
+  }.freeze
 end
 

data/etc/turtle.ll1.sxp

@@ -5,11 +5,13 @@
   (first "(" "@base" "@prefix" ANON BLANK_NODE_LABEL IRIREF PNAME_LN PNAME_NS
    SPARQL_BASE SPARQL_PREFIX "[" _eps )
   (follow _eof)
+  (cleanup star)
   (alt _empty _turtleDoc_1))
  (rule _turtleDoc_1 "1.1"
   (first "(" "@base" "@prefix" ANON BLANK_NODE_LABEL IRIREF PNAME_LN PNAME_NS
    SPARQL_BASE SPARQL_PREFIX "[" )
   (follow _eof)
+  (cleanup merge)
   (seq statement turtleDoc))
  (rule _turtleDoc_2 "1.2"
   (first "(" "@base" "@prefix" ANON BLANK_NODE_LABEL IRIREF PNAME_LN PNAME_NS
@@ -69,6 +71,7 @@
  (rule _triples_3 "6.3"
   (first IRIREF PNAME_LN PNAME_NS _eps "a")
   (follow ".")
+  (cleanup opt)
   (alt _empty predicateObjectList))
  (rule _triples_4 "6.4"
   (first IRIREF PNAME_LN PNAME_NS "a")
@@ -85,6 +88,7 @@
  (rule _predicateObjectList_1 "7.1"
   (first ";" _eps)
   (follow "." "]")
+  (cleanup star)
   (alt _empty _predicateObjectList_3))
  (rule _predicateObjectList_2 "7.2"
   (first ";")
@@ -93,10 +97,12 @@
  (rule _predicateObjectList_3 "7.3"
   (first ";")
   (follow "." "]")
+  (cleanup merge)
   (seq _predicateObjectList_2 _predicateObjectList_1))
  (rule _predicateObjectList_4 "7.4"
   (first IRIREF PNAME_LN PNAME_NS _eps "a")
   (follow "." ";" "]")
+  (cleanup opt)
   (alt _empty _predicateObjectList_5))
  (rule _predicateObjectList_5 "7.5"
   (first IRIREF PNAME_LN PNAME_NS "a")
@@ -131,11 +137,13 @@
  (rule _objectList_1 "8.1"
   (first "," _eps)
   (follow "." ";" "]")
+  (cleanup star)
   (alt _empty _objectList_3))
  (rule _objectList_2 "8.2" (first ",") (follow "," "." ";" "]") (seq "," object))
  (rule _objectList_3 "8.3"
   (first ",")
   (follow "." ";" "]")
+  (cleanup merge)
   (seq _objectList_2 _objectList_1))
  (rule _objectList_4 "8.4" (first "," _eps) (follow "." ";" "]") (seq _objectList_1))
  (rule _objectList_5 "8.5" (first "," _eps) (follow "." ";" "]") (seq _objectList_1))
@@ -213,12 +221,14 @@
    PNAME_NS STRING_LITERAL_LONG_QUOTE STRING_LITERAL_LONG_SINGLE_QUOTE
    STRING_LITERAL_QUOTE STRING_LITERAL_SINGLE_QUOTE "[" _eps "false" "true" )
   (follow ")")
+  (cleanup star)
   (alt _empty _collection_2))
  (rule _collection_2 "15.2"
   (first "(" ANON BLANK_NODE_LABEL DECIMAL DOUBLE INTEGER IRIREF PNAME_LN
    PNAME_NS STRING_LITERAL_LONG_QUOTE STRING_LITERAL_LONG_SINGLE_QUOTE
    STRING_LITERAL_QUOTE STRING_LITERAL_SINGLE_QUOTE "[" "false" "true" )
   (follow ")")
+  (cleanup merge)
   (seq object _collection_1))
  (rule _collection_3 "15.3"
   (first "(" ")" ANON BLANK_NODE_LABEL DECIMAL DOUBLE INTEGER IRIREF PNAME_LN
@@ -279,6 +289,11 @@
   (seq "\"\"\"" (seq (opt (alt "\"" "\"\"")) (range "^\"] | ECHAR | UCHAR ))* '\"\"\"'"))))
  (terminal UCHAR "26"
   (alt (seq "u" HEX HEX HEX HEX) (seq "U" HEX HEX HEX HEX HEX HEX HEX HEX)))
+ (rule _sparqlPrefix_1 "28s.1"
+  (first PNAME_NS)
+  (follow "(" "@base" "@prefix" ANON BLANK_NODE_LABEL IRIREF PNAME_LN PNAME_NS
+   SPARQL_BASE SPARQL_PREFIX "[" _eof )
+  (seq PNAME_NS IRIREF))
  (rule sparqlPrefix "28s"
   (first SPARQL_PREFIX)
   (follow "(" "@base" "@prefix" ANON BLANK_NODE_LABEL IRIREF PNAME_LN PNAME_NS
@@ -286,28 +301,24 @@
   (seq SPARQL_PREFIX PNAME_NS IRIREF))
  (terminal SPARQL_PREFIX "28t"
   (seq (range "Pp") (range "Rr") (range "Ee") (range "Ff") (range "Ii") (range "Xx")))
- (rule _sparqlPrefix_1 "28s.1"
-  (first PNAME_NS)
+ (terminal SPARQL_BASE "29t" (seq (range "Bb") (range "Aa") (range "Ss") (range "Ee")))
+ (rule _sparqlBase_1 "29s.1"
+  (first IRIREF)
   (follow "(" "@base" "@prefix" ANON BLANK_NODE_LABEL IRIREF PNAME_LN PNAME_NS
    SPARQL_BASE SPARQL_PREFIX "[" _eof )
-  (seq PNAME_NS IRIREF))
+  (seq IRIREF))
  (rule sparqlBase "29s"
   (first SPARQL_BASE)
   (follow "(" "@base" "@prefix" ANON BLANK_NODE_LABEL IRIREF PNAME_LN PNAME_NS
    SPARQL_BASE SPARQL_PREFIX "[" _eof )
   (seq SPARQL_BASE IRIREF))
- (terminal SPARQL_BASE "29t" (seq (range "Bb") (range "Aa") (range "Ss") (range "Ee")))
- (rule _sparqlBase_1 "29s.1"
-  (first IRIREF)
-  (follow "(" "@base" "@prefix" ANON BLANK_NODE_LABEL IRIREF PNAME_LN PNAME_NS
-   SPARQL_BASE SPARQL_PREFIX "[" _eof )
-  (seq IRIREF))
  (rule _RDFLiteral_1 "128s.1"
   (first LANGTAG "^^" _eps)
   (follow "(" ")" "," "." ";" ANON BLANK_NODE_LABEL DECIMAL DOUBLE INTEGER
    IRIREF PNAME_LN PNAME_NS STRING_LITERAL_LONG_QUOTE
    STRING_LITERAL_LONG_SINGLE_QUOTE STRING_LITERAL_QUOTE
    STRING_LITERAL_SINGLE_QUOTE "[" "]" "false" "true" )
+  (cleanup opt)
   (alt _empty _RDFLiteral_2))
  (rule _RDFLiteral_3 "128s.3"
   (first "^^")
@@ -316,13 +327,6 @@
    STRING_LITERAL_LONG_SINGLE_QUOTE STRING_LITERAL_QUOTE
    STRING_LITERAL_SINGLE_QUOTE "[" "]" "false" "true" )
   (seq "^^" iri))
- (rule _RDFLiteral_4 "128s.4"
-  (first LANGTAG "^^" _eps)
-  (follow "(" ")" "," "." ";" ANON BLANK_NODE_LABEL DECIMAL DOUBLE INTEGER
-   IRIREF PNAME_LN PNAME_NS STRING_LITERAL_LONG_QUOTE
-   STRING_LITERAL_LONG_SINGLE_QUOTE STRING_LITERAL_QUOTE
-   STRING_LITERAL_SINGLE_QUOTE "[" "]" "false" "true" )
-  (seq _RDFLiteral_1))
  (rule _RDFLiteral_2 "128s.2"
   (first LANGTAG "^^")
   (follow "(" ")" "," "." ";" ANON BLANK_NODE_LABEL DECIMAL DOUBLE INTEGER
@@ -338,6 +342,13 @@
    STRING_LITERAL_LONG_SINGLE_QUOTE STRING_LITERAL_QUOTE
    STRING_LITERAL_SINGLE_QUOTE "[" "]" "false" "true" )
   (seq String _RDFLiteral_1))
+ (rule _RDFLiteral_4 "128s.4"
+  (first LANGTAG "^^" _eps)
+  (follow "(" ")" "," "." ";" ANON BLANK_NODE_LABEL DECIMAL DOUBLE INTEGER
+   IRIREF PNAME_LN PNAME_NS STRING_LITERAL_LONG_QUOTE
+   STRING_LITERAL_LONG_SINGLE_QUOTE STRING_LITERAL_QUOTE
+   STRING_LITERAL_SINGLE_QUOTE "[" "]" "false" "true" )
+  (seq _RDFLiteral_1))
  (rule BooleanLiteral "133s"
   (first "false" "true")
   (follow "(" ")" "," "." ";" ANON BLANK_NODE_LABEL DECIMAL DOUBLE INTEGER

data/etc/turtle.rb

@@ -1466,5 +1466,18 @@ module Branch
       :STRING_LITERAL_LONG_SINGLE_QUOTE,
       :STRING_LITERAL_LONG_QUOTE],
   }.freeze
+  CLEANUP = {
+    :_RDFLiteral_1 => :opt,
+    :_collection_1 => :star,
+    :_collection_2 => :merge,
+    :_objectList_1 => :star,
+    :_objectList_3 => :merge,
+    :_predicateObjectList_1 => :star,
+    :_predicateObjectList_3 => :merge,
+    :_predicateObjectList_4 => :opt,
+    :_triples_3 => :opt,
+    :turtleDoc => :star,
+    :_turtleDoc_1 => :merge,
+  }.freeze
 end
 

data/lib/ebnf/base.rb

@@ -122,7 +122,7 @@ module EBNF
     # @option options [Symbol] :format (:ebnf)
     #   Format of input, one of :ebnf, or :sxp
     def initialize(input, options = {})
-      @options = {:format => :ebnf}.merge(options)
+      @options = {format: :ebnf}.merge(options)
       @lineno, @depth, @errors = 1, 0, []
       terminal = false
       @ast = []
@@ -144,7 +144,7 @@ module EBNF
             terminal = true
           when /^@pass\s*(.*)$/m
             expr = expression($1).first
-            rule = Rule.new(nil, nil, expr, :kind => :pass)
+            rule = Rule.new(nil, nil, expr, kind: :pass)
             rule.orig = expr
             @ast << rule
           else
@@ -213,6 +213,7 @@ module EBNF
       self.outputTable(output, 'TERMINALS', self.terminals, 1)
       self.outputTable(output, 'FIRST', self.first, 1)
       self.outputTable(output, 'FOLLOW', self.follow, 1)
+      self.outputTable(output, 'CLEANUP', self.cleanup, 1)
       self.outputTable(output, 'PASS', [self.pass], 1) if self.pass
       unless output == STDOUT
         output.puts "end"

data/lib/ebnf/bnf.rb

@@ -8,7 +8,7 @@ module EBNF
     # @return [ENBF] self
     def make_bnf
       progress("make_bnf") {"Start: #{@ast.length} rules"}
-      new_ast = [Rule.new(:_empty, "0", [:seq], :kind => :rule)]
+      new_ast = [Rule.new(:_empty, "0", [:seq], kind: :rule)]
 
       ast.each do |rule|
         debug("make_bnf") {"expand from: #{rule.inspect}"}

data/lib/ebnf/ll1.rb

@@ -12,14 +12,21 @@ module EBNF
 
     # First table
     #
-    # @return [Hash{Symbol, String => Symbol}]
+    # @return [Hash{Symbol => Array<String, Symbol>}]
     attr_reader :first
 
     # Follow table
     #
-    # @return [Hash{Symbol, String => Symbol}]
+    # @return [Hash{Symbol => Array<String, Symbol>}]
     attr_reader :follow
 
+    # EBNF Cleanup table
+    #
+    # The list of terminals used in the grammar.
+    #
+    # @return [Hash{Symbol => Symbol}]
+    attr_reader :cleanup
+
     # Terminal table
     #
     # The list of terminals used in the grammar.
@@ -175,18 +182,20 @@ module EBNF
       @first = ast.
         select(&:first).
         inject({}) {|memo, r|
-          memo[r.sym] = r.first if r.first
-          memo
+          memo.merge(r.sym => r.first)
         }
       @follow = ast.
         select(&:follow).
         inject({}) {|memo, r|
-          memo[r.sym] = r.follow if r.follow
-          memo
+          memo.merge(r.sym => r.follow)
         }
+
+      @cleanup = ast.
+        select(&:cleanup).
+        inject({}) {|memo, r| memo.merge(r.sym => r.cleanup)}
+
       @terminals = ast.map {|r| Array(r.first) + Array(r.follow)}.flatten.uniq
       @terminals = (@terminals - [:_eps, :_eof]).sort_by{|t| t.to_s.sub(/^_/, '')}
-
       # FIXME: assumes that this is a (seq :PASS), or similar
       if pass = ast.detect {|r| r.pass?}
         @pass = pass.expr.last
@@ -228,7 +237,7 @@ module EBNF
     # @param [IO, StringIO] io
     # @param [String] name of the table constant
     # @param [String] table
-    #   to output, one of {#branch}, {#first}, {#follow}, or {#terminals}
+    #   to output, one of {#branch}, {#first}, {#follow}, {#cleanup} or {#terminals}
     # @param [Integer] indent = 0
     def outputTable(io, name, table, indent = 0)
       ind0 = '  ' * indent
@@ -239,6 +248,8 @@ module EBNF
         io.puts "#{ind0}#{name} = {"
         table.keys.sort_by{|t| t.to_s.sub(/^_/, '')}.each do |prod|
           case table[prod]
+          when Symbol, String
+            io.puts "#{ind1}#{prod.inspect} => #{table[prod].inspect},"
           when Array
             list = table[prod].map(&:inspect).join(",\n#{ind2}")
             io.puts "#{ind1}#{prod.inspect} => [\n#{ind2}#{list}],"
@@ -306,7 +317,6 @@ module EBNF
               # A First/First conflict appears when there are two rules having
               # the same first, so the parser can't know which one to choose.
               if branchDict.has_key?(f)
-                #require 'byebug'; byebug
                 error("First/First Conflict: #{f.inspect} is the condition for both #{prod_rule.sym} and #{branchDict[f].first}")
               end
 

data/lib/ebnf/ll1/lexer.rb

@@ -180,27 +180,28 @@ module EBNF::LL1
     ##
     # Returns first token in input stream
     #
+    # @param [Array[Symbol]] types Optional set of types for restricting terminals examined
     # @return [Token]
-    def first
+    def first(*types)
       return nil unless scanner
 
       @first ||= begin
         {} while !scanner.eos? && skip_whitespace
         return @scanner = nil if scanner.eos?
 
-        token = match_token
+        token = match_token(*types)
 
         if token.nil?
           lexme = (scanner.rest.split(@whitespace || /\s/).first rescue nil) || scanner.rest
           raise Error.new("Invalid token #{lexme[0..100].inspect}",
-            :input => scanner.rest[0..100], :token => lexme, :lineno => lineno)
+            input: scanner.rest[0..100], token: lexme, lineno: lineno)
         end
 
         token
       end
     rescue ArgumentError, Encoding::CompatibilityError => e
       raise Error.new(e.message,
-        :input => (scanner.rest[0..100] rescue '??'), :token => lexme, :lineno => lineno)
+        input: (scanner.rest[0..100] rescue '??'), token: lexme, lineno: lineno)
     rescue Error
       raise
     rescue
@@ -221,9 +222,10 @@ module EBNF::LL1
     ##
     # Skip input until a token is matched
     #
+    # @param [Array[Symbol]] types Optional set of types for restricting terminals examined
     # @return [Token]
-    def recover
-       until scanner.eos? || tok = match_token
+    def recover(*types)
+       until scanner.eos? || tok = match_token(*types)
         if scanner.skip_until(@whitespace || /\s/m).nil? # Skip past current "token"
           # No whitespace at the end, must be and end of string
           scanner.terminate
@@ -259,11 +261,13 @@ module EBNF::LL1
     # track this with the resulting {Token}, so that comparisons
     # with that token are also case insensitive
     #
+    # @param [Array[Symbol]] types Optional set of types for restricting terminals examined
     # @return [Token]
-    def match_token
+    def match_token(*types)
       @terminals.each do |term|
+        next unless types.empty? || types.include?(term.type)
         #STDERR.puts "match[#{term.type}] #{scanner.rest[0..100].inspect} against #{term.regexp.inspect}" #if term.type == :STRING_LITERAL_SINGLE_QUOTE
-        if term.partial_regexp && scanner.match?(term.partial_regexp) && !scanner.match?(term.regexp)
+        if term.partial_regexp && scanner.match?(term.partial_regexp) && !scanner.match?(term.regexp) && scanner.respond_to?(:ensure_buffer_full)
           scanner.ensure_buffer_full
         end
 
@@ -350,7 +354,7 @@ module EBNF::LL1
     # @param  [Hash{Symbol => Object}] options
     # @return [Token]
     def token(type, value, options = {})
-      Token.new(type, value, options.merge(:lineno => lineno))
+      Token.new(type, value, options.merge(lineno: lineno))
     end
 
     ##
@@ -442,7 +446,7 @@ module EBNF::LL1
       #
       # @return [Hash]
       def to_hash
-        {:type => @type, :value => @value}
+        {type: @type, value: @value}
       end
       
       ##
@@ -480,7 +484,7 @@ module EBNF::LL1
     # @example Raising a lexer error
     #   raise EBNF::LL1::Lexer::Error.new(
     #     "invalid token '%' on line 10",
-    #     :input => query, :token => '%', :lineno => 9)
+    #     input: query, token: '%', lineno: 9)
     #
     # @see http://ruby-doc.org/core/classes/StandardError.html
     class Error < StandardError