rley 0.7.06 → 0.8.01

data/lib/rley/notation/grouping_node.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+require_relative 'sequence_node'
+
+module Rley
+  module Notation
+    # A syntax node representing an expression bracketed by parentheses.
+    class GroupingNode < SequenceNode      
+      # @param aPosition [Rley::Lexical::Position] Start position.
+      # @param sequence [Array<ASTNode>] sequence of AST nodes
+      # @param theRepetition [Symbol] indicates how many times the symbol can be repeated
+      def initialize(aPosition, sequence, theRepetition = nil)
+        super(aPosition, sequence, theRepetition)
+      end
+
+      # Part of the 'visitee' role in Visitor design pattern.
+      # @param visitor [Notation::ASTVisitor] the visitor
+      def accept(visitor)
+        visitor.visit_grouping_node(self)
+      end
+    end # class
+  end # module
+end # module

data/lib/rley/notation/parser.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+require_relative 'tokenizer'
+require_relative 'grammar'
+require_relative 'ast_builder'
+
+module Rley
+  module Notation
+    # A Lox parser that produce concrete parse trees.
+    # Concrete parse trees are the default kind of parse tree
+    # generated by the Rley library.
+    # They consist of two node types only:
+    # - NonTerminalNode
+    # - TerminalNode
+    # A NonTerminalNode has zero or more child nodes (called subnodes)
+    # A TerminalNode is leaf node, that is, it has no child node.
+    # While concrete parse tree nodes can be generated out of the box,
+    # they have the following drawbacks:
+    # - Generic node classes that aren't always suited for the needs of
+    #     the language being processing.
+    # - Concrete parse tree tend to be deeply nested, which may complicate
+    #   further processing.
+    class Parser
+      # @return [Rley::Engine] A facade object for the Rley parsing library
+      attr_reader(:engine)
+
+      def initialize
+        # Create a Rley facade object
+        @engine = Rley::Engine.new do |cfg|
+          cfg.diagnose = true
+          cfg.repr_builder = Notation::ASTBuilder
+        end
+
+        # Step 1. Load RGN grammar
+        @engine.use_grammar(Rley::Notation::RGNGrammar)
+      end
+
+      # Parse the given Lox program into a parse tree.
+      # @param source [String] Lox program to parse
+      # @return [Rley::ParseTree] A parse tree equivalent to the Lox input.
+      def parse(source)
+        lexer = Tokenizer.new(source)
+        result = engine.parse(lexer.tokens)
+
+        unless result.success?
+          # Stop if the parse failed...
+          line1 = "Parsing failed\n"
+          line2 = "Reason: #{result.failure_reason.message}"
+          raise SyntaxError, line1 + line2
+        end
+
+        return engine.convert(result) # engine.to_ptree(result)
+      end
+    end # class
+  end # module
+end # module

data/lib/rley/notation/sequence_node.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+require_relative 'ast_node'
+
+module Rley
+  module Notation
+    # A syntax node for a sequence of AST nodes
+    class SequenceNode < ASTNode
+      # @return [Array<ASTNode>]
+      attr_reader :subnodes
+      
+      attr_accessor :constraints
+      
+      # @param aPosition [Rley::Lexical::Position] Start position.
+      # @param sequence [Array<ASTNode>] sequence of AST nodes
+      # @param theRepetition [Symbol] indicates how many times the symbol can be repeated
+      def initialize(aPosition, sequence, theRepetition = nil)
+        super(aPosition)
+        @subnodes = sequence
+        repetition=(theRepetition) if theRepetition
+        @constraints = []
+      end
+
+      def size
+        subnodes.size
+      end
+
+      # Part of the 'visitee' role in Visitor design pattern.
+      # @param visitor [Notation::ASTVisitor] the visitor
+      def accept(visitor)
+        visitor.visit_sequence_node(self)
+      end
+    end # class
+  end # module
+end # module

data/lib/rley/notation/symbol_node.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+require_relative 'ast_node'
+
+module Rley
+  module Notation
+    # A syntax node for a grammar symbol occurring in rhs of a rule
+    class SymbolNode < ASTNode
+      # @return [String] name of grammar symbol
+      attr_reader :name
+      
+      # @param aPosition [Rley::Lexical::Position] Position of the entry in the input stream.
+      # @param aName [String] name of grammar symbol
+      # @param theRepetition [Symbol] indicates how many times the symbol can be repeated
+      def initialize(aPosition, aName, theRepetition = nil)
+        super(aPosition)
+        @name = aName
+        repetition=(theRepetition) if theRepetition
+      end
+
+      # Abstract method (must be overriden in subclasses).
+      # Part of the 'visitee' role in Visitor design pattern.
+      # @param _visitor [LoxxyTreeVisitor] the visitor
+      def accept(visitor)
+        visitor.visit_symbol_node(self)
+      end
+    end # class
+  end # module
+end # module

data/lib/rley/notation/tokenizer.rb

@@ -0,0 +1,192 @@
+# frozen_string_literal: true
+
+require 'strscan'
+require_relative '../lexical/token'
+
+module Rley
+  module Notation
+    # A tokenizer for the Rley notation language.
+    # Responsibility: break input into a sequence of token objects.
+    # The tokenizer should recognize:
+    # Identifiers,
+    # Number literals including single digit
+    # String literals (quote delimited)
+    # Delimiters: e.g. parentheses '(',  ')'
+    # Separators: e.g. comma
+    class Tokenizer
+      # @return [StringScanner] Low-level input scanner
+      attr_reader(:scanner)
+
+      # @return [Integer] The current line number
+      attr_reader(:lineno)
+
+      # @return [Integer] Position of last start of line in the input
+      attr_reader(:line_start)
+
+      # One or two special character tokens.
+      @@lexeme2name = {
+        '(' => 'LEFT_PAREN',
+        ')' => 'RIGHT_PAREN',
+        '{' => 'LEFT_BRACE',
+        '}' => 'RIGHT_BRACE',
+        ',' => 'COMMA',
+        '+' => 'PLUS',
+        '?' => 'QUESTION_MARK',
+        '*' => 'STAR',
+        '..' => 'ELLIPSIS'
+      }.freeze
+      
+      # Here are all the implemented Rley notation keywords
+      @@keywords = %w[
+        match_closest repeat
+      ].map { |x| [x, x] }.to_h      
+
+      # Constructor. Initialize a tokenizer for Lox input.
+      # @param source [String] Lox text to tokenize.
+      def initialize(source = nil)
+        @scanner = StringScanner.new('')
+        start_with(source) if source
+      end
+
+      # Reset the tokenizer and make the given text, the current input.
+      # @param source [String] Lox text to tokenize.
+      def start_with(source)
+        @scanner.string = source
+        @lineno = 1
+        @line_start = 0
+      end
+
+      # Scan the source and return an array of tokens.
+      # @return [Array<Rley::Lexical::Token>] | Returns a sequence of tokens
+      def tokens
+        tok_sequence = []
+        until @scanner.eos?
+          token = _next_token
+          tok_sequence << token unless token.nil?
+        end
+
+        return tok_sequence
+      end
+
+      private
+
+      def _next_token
+        pos_before = scanner.pos
+        skip_intertoken_spaces
+        ws_found = true if scanner.pos > pos_before
+        curr_ch = scanner.peek(1)
+        return nil if curr_ch.nil? || curr_ch.empty?
+
+        token = nil
+
+        if '(){},'.include? curr_ch
+          # Single delimiter, separator or character
+          token = build_token(@@lexeme2name[curr_ch], scanner.getch)
+        elsif '?*+,'.include? curr_ch # modifier character
+          # modifiers without prefix text are symbols
+          symb = ws_found ? 'SYMBOL' : @@lexeme2name[curr_ch] 
+          token = build_token(symb, scanner.getch)
+        elsif (lexeme = scanner.scan(/\.\./))
+          # One or two special character tokens
+          token = build_token(@@lexeme2name[lexeme], lexeme)
+        elsif scanner.check(/"|'/) # Start of string detected...
+          token = build_string_token
+        elsif (lexeme = scanner.scan(/\d+/))
+          token = build_token('INT_LIT', lexeme)
+        elsif (lexeme = scanner.scan(/[a-zA-Z_][a-zA-Z_0-9]*:/))
+          keyw = @@keywords[lexeme.chop!]
+          token = build_token('KEY', lexeme) if keyw
+          # ... error case
+        elsif (lexeme = scanner.scan(/[^?*+,:(){}\s]+/))
+           token = build_token('SYMBOL', lexeme)
+        else # Unknown token
+          col = scanner.pos - @line_start + 1
+          _erroneous = curr_ch.nil? ? '' : scanner.scan(/./)
+          raise ScanError, "Error: [line #{lineno}:#{col}]: Unexpected character."
+        end
+
+        return token
+      end
+
+      def build_token(aSymbolName, aLexeme)
+        begin
+          lex_length = aLexeme ? aLexeme.size : 0
+          col = scanner.pos - lex_length - @line_start + 1
+          pos = Rley::Lexical::Position.new(@lineno, col)
+          token = Rley::Lexical::Token.new(aLexeme.dup, aSymbolName, pos)
+
+        rescue StandardError => e
+          puts "Failing with '#{aSymbolName}' and '#{aLexeme}'"
+          raise e
+        end
+
+        return token
+      end
+
+      # precondition: current position at leading quote
+      def build_string_token
+        delimiter = scanner.scan(/./)
+        scan_pos = scanner.pos
+        line = @lineno
+        column_start = scan_pos - @line_start
+        literal = +''
+        loop do
+          substr = scanner.scan(/[^"'\\\r\n]*/)
+          if scanner.eos?
+            pos_start = "line #{line}:#{column_start}"
+            raise ScanError, "Error: [#{pos_start}]: Unterminated string."
+          else
+            literal << substr
+            special = scanner.scan(/["'\\\r\n]/)
+            case special
+            when delimiter # Terminating quote found
+              break
+            when "\r"
+              next_line
+              special << scanner.scan(/./) if scanner.match?(/\n/)
+              literal << special
+            when "\n"
+              next_line
+              literal << special
+            # when '\\'
+              # ch = scanner.scan(/./)
+              # next unless ch
+
+              # escaped = @@escape_chars[ch]
+              # if escaped
+                # literal << escaped
+              # else
+                # literal << ch
+              # end
+            end
+          end
+        end
+        pos = Rley::Lexical::Position.new(line, column_start)
+        lexeme = scanner.string[scan_pos - 1..scanner.pos - 1]
+        Rley::Lexical::Token.new(literal, 'STR_LIT', pos)
+      end
+
+      # Skip non-significant whitespaces and comments.
+      # Advance the scanner until something significant is found.
+      def skip_intertoken_spaces
+        loop do
+          ws_found = scanner.skip(/[ \t\f]+/) ? true : false
+          nl_found = scanner.skip(/(?:\r\n)|\r|\n/)
+          if nl_found
+            ws_found = true
+            next_line
+          end
+
+          break unless ws_found
+        end
+
+        scanner.pos
+      end
+
+      def next_line
+        @lineno += 1
+        @line_start = scanner.pos
+      end
+    end # class
+  end # module
+end # module

data/lib/rley/parse_forest_visitor.rb

@@ -15,8 +15,8 @@ module Rley # This module is used as a namespace
         @signatures = subnodes.map { |_| prime_enumerator.next }
       end
 
-      def signature_exist?()
-        @signatures.nil? ? false : true
+      def signature_exist?
+        !@signatures.nil?
       end
     end # class
   end # module
@@ -69,7 +69,7 @@ module Rley # This module is used as a namespace
     end
 
     # The signal to begin the visit of the parse forest.
-    def start()
+    def start
       pforest.accept(self)
     end
 
@@ -169,7 +169,7 @@ module Rley # This module is used as a namespace
     def broadcast(msg, *args)
       subscribers.each do |subscr|
         next unless subscr.respond_to?(msg) || subscr.respond_to?(:accept_all)
-        
+
         subscr.send(msg, *args)
       end
     end
@@ -190,7 +190,7 @@ module Rley # This module is used as a namespace
 
     def pop_node
       return if legs.empty?
-      
+
       legs.pop
     end
   end # class

data/lib/rley/parse_rep/ast_base_builder.rb

@@ -20,14 +20,14 @@ module Rley # This module is used as a namespace
       #   Returned hash contains pairs of the form:
       #   terminal name => Class implementing the terminal tokens
       #   terminal name => Hash with pairs: production name => Class
-      def terminal2node()
+      def terminal2node
         raise NotImplementedError
       end
 
       # Method to override in subclass.
       # Default class for representing terminal nodes.
       # @return [Class]
-      def terminalnode_class()
+      def terminalnode_class
         PTree::TerminalNode
       end
 
@@ -37,7 +37,7 @@ module Rley # This module is used as a namespace
       # @param aProductionName [String]
       # @return [String]
       def method_name(aProductionName)
-        return 'reduce_' + aProductionName
+        "reduce_#{aProductionName}"
       end
 
       # Utility method.
@@ -46,7 +46,7 @@ module Rley # This module is used as a namespace
       # @param _tokens [Array<Lexical::Token>]
       # @param theChildren [Array<Object>]
       def return_first_child(_range, _tokens, theChildren)
-        return theChildren[0]
+        theChildren[0]
       end
 
       # Utility method.
@@ -55,7 +55,7 @@ module Rley # This module is used as a namespace
       # @param _tokens [Array<Lexical::Token>]
       # @param theChildren [Array<Object>]
       def return_second_child(_range, _tokens, theChildren)
-        return theChildren[1]
+        theChildren[1]
       end
 
       # Utility method.
@@ -64,7 +64,7 @@ module Rley # This module is used as a namespace
       # @param _tokens [Array<Lexical::Token>]
       # @param theChildren [Array<Object>]
       def return_last_child(_range, _tokens, theChildren)
-        return theChildren[-1]
+        theChildren[-1]
       end
 
       # Simply return an epsilon symbol
@@ -72,7 +72,7 @@ module Rley # This module is used as a namespace
       # @param _tokens [Array<Lexical::Token>]
       # @param _children [Array<Object>]
       def return_epsilon(_range, _tokens, _children)
-        return nil
+        nil
       end
 
       protected
@@ -81,7 +81,7 @@ module Rley # This module is used as a namespace
       # Create a parse tree object with given
       # node as root node.
       def create_tree(aRootNode)
-        return Rley::PTree::ParseTree.new(aRootNode)
+        Rley::PTree::ParseTree.new(aRootNode)
       end
 
       # Factory method for creating a node object for the given
@@ -96,9 +96,7 @@ module Rley # This module is used as a namespace
           # Lexical ambiguity...
           klass = klass.fetch(aProduction.name)
         end
-        node = klass.new(aToken, aTokenPosition)
-
-        return node
+        klass.new(aToken, aTokenPosition)
       end
 
       # Method to override.
@@ -125,6 +123,45 @@ module Rley # This module is used as a namespace
         end
         return node
       end
+
+      # Standard method for handling one or more modifier: symbol+
+      # rule('symbol_plus' => 'symbol_plus symbol')
+      # def reduce_base_plus_more(_production, _range, _tokens, theChildren)
+        # theChildren[0] << theChildren[1]
+      # end
+
+      # Standard rule method handling one or more modifier: symbol+
+      # rule('symbol_plus' => 'symbol')
+      # def reduce_base_plus_last(_production, _range, _tokens, theChildren)
+        # [theChildren[0]]
+      # end
+      
+      # Implicit rule generated for * modifier
+      # rule('X') => 'X item'.as '_star_more'
+      def reduce__star_more(_production, _range, _tokens, theChildren)
+        theChildren[0]  << theChildren[1]
+        theChildren[0]
+      end
+
+      # Implicit rule generated for * modifier
+      # rule('X') => ''.as '_star_none'
+      def reduce__star_none(_production, _range, _tokens, theChildren)
+        []
+      end
+
+      # Implicit rule generated for + modifier
+      # rule('X') => 'X item'.as '_plus_more'
+      def reduce__plus_more(_production, _range, _tokens, theChildren)
+        theChildren[0]  << theChildren[1]
+        theChildren[0]
+      end
+
+      # Implicit rule generated for + modifier
+      # rule('X') => 'item'.as '_plus_one'
+      def reduce__plus_one(_production, _range, _tokens, theChildren)
+        [theChildren[0]]
+      end      
+
     end # class
   end # module
 end # module