rley 0.5.01 → 0.5.02

data/examples/general/calc_iter2/calc_ast_nodes.rb

@@ -0,0 +1,151 @@
+# Classes that implement nodes of Abstract Syntax Trees (AST) representing
+# calculator parse results.
+
+
+CalcTerminalNode = Struct.new(:token, :value, :position) do
+  def initialize(aToken, aPosition)
+    self.token = aToken
+    self.position = aPosition
+    init_value(aToken.lexeme)
+  end
+
+  # This method can be overriden
+  def init_value(aLiteral)
+    self.value = aLiteral.dup
+  end
+
+  def symbol()
+    self.token.terminal
+  end
+
+  def to_ruby()
+    return value
+  end
+  
+  # Part of the 'visitee' role in Visitor design pattern.
+  # @param aVisitor[ParseTreeVisitor] the visitor
+  def accept(aVisitor)
+    aVisitor.visit_terminal(self)
+  end  
+end
+
+class CalcNumberNode  < CalcTerminalNode
+  def init_value(aLiteral)
+    case aLiteral
+      when /^[+-]?\d+$/
+        self.value = aLiteral.to_i
+
+      when /^[+-]?\d+(\.\d+)?([eE][+-]?\d+)?$/
+        self.value = aLiteral.to_f
+    end
+  end
+end
+
+class CalcCompositeNode
+  attr_accessor(:children)
+  attr_accessor(:symbol)
+
+  def initialize(aSymbol)
+    @symbol = aSymbol
+    @children = []    
+  end
+
+  # Part of the 'visitee' role in Visitor design pattern.
+  # @param aVisitor[ParseTreeVisitor] the visitor
+  def accept(aVisitor)
+    aVisitor.visit_nonterminal(self)
+  end
+  
+  alias subnodes children
+
+end # class
+
+class CalcUnaryOpNode < CalcCompositeNode
+  def initialize(aSymbol)
+    super(aSymbol)
+  end
+
+  # Convert this tree node in a simpler Ruby representation.
+  # Basically a Calc object corresponds to a Ruhy Hash
+  def to_ruby()
+    rep = {}
+    members.each do |pair|
+      rep[pair.name.to_ruby] = pair.value.to_ruby
+    end
+
+    return rep
+  end
+
+  alias members children
+end # class
+
+class CalcNegateNode CalcUnaryOpNode
+end # class
+
+class CalcBinaryOpNode < CalcCompositeNode
+  def initialize(aSymbol)
+    super(aSymbol)
+  end
+
+end # class
+
+class CalcAddNode < CalcBinaryOpNode
+
+  # Convert this tree node in a simpler Ruby representation.
+  # Basically a Calc object corresponds to a Ruhy Hash
+  def to_ruby()
+    rep = []
+    children.each do |child|
+      rep << child.to_ruby
+    end
+
+    return rep
+  end
+end # class
+
+
+class CalcSubtractNode < CalcBinaryOpNode
+
+  # Convert this tree node in a simpler Ruby representation.
+  # Basically a Calc object corresponds to a Ruhy Hash
+  def to_ruby()
+    rep = []
+    children.each do |child|
+      rep << child.to_ruby
+    end
+
+    return rep
+  end
+end # class
+
+class CalcMultiplyNode < CalcBinaryOpNode
+
+  # Convert this tree node in a simpler Ruby representation.
+  # Basically a Calc object corresponds to a Ruhy Hash
+  def to_ruby()
+    rep = []
+    children.each do |child|
+      rep << child.to_ruby
+    end
+
+    return rep
+  end
+end # class
+
+class CalcDivideNode < CalcBinaryOpNode
+
+  # Convert this tree node in a simpler Ruby representation.
+  # Basically a Calc object corresponds to a Ruhy Hash
+  def to_ruby()
+    rep = []
+    children.each do |child|
+      rep << child.to_ruby
+    end
+
+    return rep
+  end
+end # class
+
+
+
+

data/examples/general/{calc → calc_iter2}/calc_demo.rb

@@ -5,14 +5,15 @@ parser = CalcParser.new
 
 # Parse the input expression in command-line
 if ARGV.empty?
+  my_name = File.basename(__FILE__)
   msg = <<-END_MSG
 Command-line symtax:
-  ruby #{__FILE__} "arithmetic expression"
+  ruby #{my_name} "arithmetic expression"
   where:
     the arithmetic expression is enclosed between double quotes (")
 
   Example:
-  ruby #{__FILE__} "2 * 3 + (4 - 1)"
+  ruby #{my_name} "2 * 3 + (4 - 1)"
 END_MSG
   puts msg
   exit(1)

data/examples/general/calc_iter2/calc_lexer.rb

@@ -0,0 +1,81 @@
+# File: calc_lexer.rb
+# Lexer for a basic arithmetical expression parser
+require 'strscan'
+require 'rley' # Load the gem
+
+
+class CalcLexer
+  attr_reader(:scanner)
+  attr_reader(:lineno)
+  attr_reader(:line_start)
+  attr_reader(:name2symbol)
+
+  @@lexeme2name = {
+    '(' => 'LPAREN',
+    ')' => 'RPAREN',
+    '+' => 'PLUS',
+    '-' => 'MINUS',
+    '*' => 'STAR',
+    '/' => 'DIVIDE',
+    '**' => 'POWER'
+  }.freeze
+
+  class ScanError < StandardError; end
+
+  def initialize(source, aGrammar)
+    @scanner = StringScanner.new(source)
+    @name2symbol = aGrammar.name2symbol
+    @lineno = 1
+  end
+
+  def tokens()
+    tok_sequence = []
+    until @scanner.eos?
+      token = _next_token
+      tok_sequence << token unless token.nil?
+    end
+
+    return tok_sequence
+  end
+
+  private
+
+  def _next_token()
+    skip_whitespaces
+    curr_ch = scanner.peek(1)
+    return nil if curr_ch.nil?
+    
+    token = nil
+
+    if '()+/'.include? curr_ch
+      # Single character token
+      token = build_token(@@lexeme2name[curr_ch], scanner.getch)
+      
+    elsif lexeme = scanner.scan(/\*\*/)
+      token = build_token(@@lexeme2name[lexeme], lexeme)
+    elsif lexeme = scanner.scan(/\*/)
+      token = build_token(@@lexeme2name[lexeme], lexeme)
+    elsif lexeme = scanner.scan(/-?[0-9]+(\.[0-9]+)?([eE][-+]?[0-9])?/)
+      token = build_token('NUMBER', lexeme)
+    elsif lexeme = scanner.scan(/-/)
+      token = build_token(@@lexeme2name[curr_ch], lexeme)
+    else # Unknown token
+      erroneous = curr_ch.nil? ? '' : curr_ch
+      sequel = scanner.scan(/.{1,20}/)
+      erroneous += sequel unless sequel.nil?
+      raise ScanError.new("Unknown token #{erroneous}")
+    end
+
+    return token
+  end
+  
+  def build_token(aSymbolName, aLexeme)
+    token_type = name2symbol[aSymbolName]
+    return Rley::Tokens::Token.new(aLexeme, token_type)    
+  end
+
+  def skip_whitespaces()
+    scanner.scan(/[ \t\f\n\r]+/)
+  end
+
+end # class

data/examples/general/calc_iter2/calc_parser.rb

@@ -0,0 +1,24 @@
+# Purpose: to demonstrate how to build and render a parse tree for JSON
+# language
+require_relative 'calc_lexer'
+require_relative 'calc_grammar'
+
+# A parser for arithmetic expressions
+class CalcParser < Rley::Parser::GFGEarleyParser
+  attr_reader(:source_file)
+
+  # Constructor
+  def initialize()
+    # Builder the Earley parser with the calculator grammar
+    super(CalcGrammar)
+  end
+
+  def parse_expression(aText)
+    lexer = CalcLexer.new(aText, grammar)
+    result = parse(lexer.tokens)
+
+    return result
+  end
+end # class
+
+# End of file

data/lib/rley.rb

@@ -6,6 +6,7 @@ require_relative './rley/constants'
 require_relative './rley/syntax/grammar_builder'
 require_relative './rley/tokens/token'
 require_relative './rley/parser/gfg_earley_parser'
+require_relative './rley/parser/parse_tree_builder'
 require_relative './rley/parse_tree_visitor'
 require_relative './rley/formatter/debug'
 require_relative './rley/formatter/json'

data/lib/rley/constants.rb

@@ -3,7 +3,7 @@
 
 module Rley # Module used as a namespace
   # The version number of the gem.
-  Version = '0.5.01'.freeze
+  Version = '0.5.02'.freeze
 
   # Brief description of the gem.
   Description = "Ruby implementation of the Earley's parsing algorithm".freeze

data/lib/rley/parser/cst_builder.rb

@@ -1,26 +1,10 @@
-require_relative '../tokens/token_range'
-require_relative '../syntax/terminal'
-require_relative '../syntax/non_terminal'
-require_relative '../gfg/end_vertex'
-require_relative '../gfg/item_vertex'
-require_relative '../gfg/start_vertex'
+require_relative 'parse_tree_builder'
 require_relative '../ptree/non_terminal_node'
 require_relative '../ptree/terminal_node'
 require_relative '../ptree/parse_tree'
 
 module Rley # This module is used as a namespace
   module Parser # This module is used as a namespace
-    # Structure used internally by CSTBuilder class.
-    CSTRawNode = Struct.new(:range, :symbol, :children) do
-      def initialize(aRange, aSymbol)
-        super
-        self.range = aRange
-        self.symbol = aSymbol
-        self.children = nil
-      end
-    end # Struct
-
-
     # The purpose of a CSTBuilder is to build piece by piece a CST
     # (Concrete Syntax Tree) from a sequence of input tokens and
     # visit events produced by walking over a GFGParsing object.
@@ -28,185 +12,16 @@ module Rley # This module is used as a namespace
     # The Builder pattern creates a complex object
     # (say, a parse tree) from simpler objects (terminal and non-terminal
     # nodes) and using a step by step approach.
-    class CSTBuilder
-      # @return [Array<Token>] The sequence of input tokens
-      attr_reader(:tokens)
-
-      # Link to CST object (being) built.
-      attr_reader(:result)
-
-
-      # Create a new builder instance.
-      # @param theTokens [Array<Token>] The sequence of input tokens.
-      def initialize(theTokens)
-        @tokens = theTokens
-        @stack = []
-      end
-
-      # Receive events resulting from a visit of GFGParsing object.
-      # These events are produced by a specialized Enumerator created
-      # with a ParseWalkerFactory instance.
-      # @param anEvent [Symbol] Kind of visit event. Should be: :visit
-      # @param anEntry [ParseEntry] The entry being visited
-      # @param anIndex [anIndex] The token index associated with anEntry
-      def receive_event(anEvent, anEntry, anIndex)
-        # puts "Event: #{anEvent} #{anEntry} #{anIndex}"
-        if anEntry.dotted_entry? # A N => alpha . beta pattern?
-          process_item_entry(anEvent, anEntry, anIndex)
-        elsif anEntry.start_entry? # A .N pattern?
-          process_start_entry(anEvent, anEntry, anIndex)
-        elsif anEntry.end_entry? # A N. pattern?
-          process_end_entry(anEvent, anEntry, anIndex)
-        else
-          raise NotImplementedError
-        end
-
-        @last_visitee = anEntry
-      end
+    class CSTBuilder < ParseTreeBuilder
 
       protected
-
-      # Return the stack
-      def stack()
-        return @stack
-      end
-
-      private
-
-      # Return the top of stack element.
-      def tos()
-        return @stack.last
-      end
-
-      # Handler for visit events for ParseEntry matching N. pattern
-      # @param anEvent [Symbol] Kind of visit event. Should be: :visit
-      # @param anEntry [ParseEntry] The entry being visited
-      # @param anIndex [anIndex] The token index at end of anEntry
-      def process_end_entry(anEvent, anEntry, anIndex)
-        case anEvent
-          when :visit
-            range = { low: anEntry.origin, high: anIndex }
-            non_terminal = entry2nonterm(anEntry)
-            # Create raw node and push onto stack
-            push_raw_node(range, non_terminal)
-          else
-            raise NotImplementedError
-        end
-      end
-
-      # Handler for visit events for ParseEntry matching .N pattern
-      # @param anEvent [Symbol] Kind of visit event. Should be: :visit
-      # @param anEntry [ParseEntry] The entry being visited
-      # @param anIndex [anIndex] The token index at end of anEntry
-      def process_start_entry(anEvent, anEntry, anIndex)
-        raise NotImplementedError unless [:visit, :revisit].include?(anEvent)
-      end
-
-      # Handler for visit events for ParseEntry matching N => alpha* . beta*
-      # @param anEvent [Symbol] Kind of visit event. Should be: :visit
-      # @param anEntry [ParseEntry] The entry being visited
-      # @param anIndex [anIndex] The token index at end of anEntry
-      def process_item_entry(anEvent, anEntry, anIndex)
-        # TODO: what if rhs is empty?
-        case anEvent
-          when :visit, :revisit
-            dot_pos = anEntry.vertex.dotted_item.position
-            if dot_pos.zero? || dot_pos < 0
-              # Check for pattern: N => alpha* .
-              process_exit_entry(anEntry, anIndex) if anEntry.exit_entry?
-
-              # Check for pattern: N => . alpha*
-              process_entry_entry(anEntry, anIndex) if anEntry.entry_entry?
-            else
-              # (pattern: N => alpha+ . beta+)
-              process_middle_entry(anEntry, anIndex)
-            end
-          else
-            $stderr.puts "waiko '#{anEvent}'"
-            raise NotImplementedError
-        end
-      end
-
-      # @param anEntry [ParseEntry] Entry matching (pattern: N => alpha* .)
-      # @param anIndex [anIndex] The token index at end of anEntry
-      def process_exit_entry(anEntry, anIndex)
-        production =  anEntry.vertex.dotted_item.production
-        count_rhs = production.rhs.members.size
-        init_TOS_children(count_rhs) # Create placeholders for children
-        build_terminal(anEntry, anIndex) if terminal_before_dot?(anEntry)
-      end
-
-      # @param anEntry [ParseEntry] Entry matching pattern: N => alpha+ . beta+
-      # @param anIndex [anIndex] The token index at end of anEntry
-      def process_middle_entry(anEntry, anIndex)
-        build_terminal(anEntry, anIndex) if terminal_before_dot?(anEntry)
-      end
-
-
-
-      # @param anEntry [ParseEntry] Entry matching (pattern: N => . alpha)
-      # @param anIndex [anIndex] The token index at end of anEntry
-      def process_entry_entry(anEntry, anIndex)
-        dotted_item = anEntry.vertex.dotted_item
-        rule = dotted_item.production
-        previous_tos = stack.pop
-        non_terminal = entry2nonterm(anEntry)
-        # For debugging purposes
-        raise StandardError if previous_tos.symbol != non_terminal
-        
-        new_node = new_parent_node(rule, previous_tos.range,
-                                     tokens, previous_tos.children)
-        if stack.empty?
-          @result = create_tree(new_node)
-        else
-          place_TOS_child(new_node, nil)
-        end
-      end
-
-      # Create a raw node with given range
-      # and push it on top of stack.
-      def push_raw_node(aRange, aSymbol)
-        raw_node = CSTRawNode.new(Tokens::TokenRange.new(aRange), aSymbol)
-        stack.push(raw_node)
-      end
-
-      # Initialize children array of TOS with nil placeholders.
-      # The number of elements equals the number of symbols at rhs.
-      def init_TOS_children(aCount)
-        tos.children = Array.new(aCount)
-      end
-
-      # Does the position on the left side of the dot correspond
-      # a terminal symbol?
-      # @param anEntry [ParseEntry] The entry being visited
-      def terminal_before_dot?(anEntry)
-        prev_symbol = anEntry.prev_symbol
-        return prev_symbol && prev_symbol.terminal?
-      end
-
-      # A terminal symbol was detected at left of dot.
-      # Build a raw node for that terminal and make it
-      # a child of TOS.
-      # @param anEntry [ParseEntry] The entry being visited
-      # @param anIndex [anIndex] The token index at end of anEntry
-      def build_terminal(anEntry, anIndex)
-        # First, build node for terminal...
-        term_symbol = anEntry.prev_symbol
-        token_position = anIndex - 1
-        token = tokens[token_position]
-        term_node = new_leaf_node(term_symbol, token_position, token)
-
-        # Second make it a child of TOS...
-        pos = anEntry.vertex.dotted_item.prev_position # position in rhs of rule
-        place_TOS_child(term_node, pos)
-      end
-
+      
       # Method to override
       # Create a parse tree object with given
       # node as root node.
       def create_tree(aRootNode)
         return Rley::PTree::ParseTree.new(aRootNode)
-      end
+      end      
 
       # Method to override
       # Factory method for creating a node object for the given
@@ -214,7 +29,7 @@ module Rley # This module is used as a namespace
       # @param _terminal [Terminal] Terminal symbol associated with the token
       # @param aTokenPosition [Integer] Position of token in the input stream
       # @param aToken [Token] The input token
-      def new_leaf_node(_terminal, aTokenPosition, aToken)
+      def new_leaf_node(_production, _terminal, aTokenPosition, aToken)
         PTree::TerminalNode.new(aToken, aTokenPosition)
       end
 
@@ -229,41 +44,6 @@ module Rley # This module is used as a namespace
         theChildren.reverse_each { |child| node.add_subnode(child) }
         return node
       end
-
-      # Place the given node object as one of the children of the TOS
-      # (TOS = Top Of Stack).
-      # Each child has a position that is dictated by the position of the
-      # related grammar symbol in the right-handed side (RHS) of the grammar
-      # rule.
-      # @param aNode [TerminalNode, NonTerminalNode] Node object to be placed
-      # @param aRHSPos [Integer, NilClass] Position in RHS of rule.
-      # If the position is provided, then the node will placed in the children
-      # array at that position.
-      # If the position is nil, then the node will be placed at the position of
-      # the rightmost nil element in children array.
-      def place_TOS_child(aNode, aRHSPos)
-        if aRHSPos.nil?
-          # Retrieve index of most rightmost nil child...
-          pos = tos.children.rindex { |child| child.nil? }
-          raise StandardError, 'Internal error' if pos.nil?
-        else
-          pos = aRHSPos
-        end
-
-        tos.children[pos] = aNode
-      end
-      
-      # Retrieve non-terminal symbol of given parse entry
-      def entry2nonterm(anEntry)
-        case anEntry.vertex
-        when GFG::StartVertex, GFG::EndVertex
-          non_terminal = anEntry.vertex.non_terminal
-        when GFG::ItemVertex
-          non_terminal = anEntry.vertex.lhs
-        end
-
-        return non_terminal
-      end
     end # class
   end # module
 end # module