rley 0.5.01 → 0.5.02

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: fe660cf0bb663508b71cc2e40461b571d4caa6ed
-  data.tar.gz: 5217bf9f6757f849bdd0fe471b8d5aa0c2283cb5
+  metadata.gz: 638178eefcf2e673a801018aeee17ff0ef7ed16b
+  data.tar.gz: 894a767a871ba958b61047369e989783ae5fe88d
 SHA512:
-  metadata.gz: 67fa745c86ccb6e14fe8689f4a0b94fc756b1a9f9dd8be50b77e5506671038f49a85f65bbef4e986f27f8d709afb6ad02896157b0b455b580c37aa6f8d011a47
-  data.tar.gz: 37df7d6e3519396a4198ecedc42d9e45d98bc71cedf4aaeba9fdff8f88ec13891da0168ace662c23e551de3cc80fefef004ed4b7be8bf6f97c2fb9452c60a334
+  metadata.gz: 23571e5b06645c427ee3ac06e662d93a3aca87d64d1cb7b57ea133ee14fd9f6a0ac0d4bebde44dd8d70cefcbb8e4614d3e49752c76e4545ed8e0923c2cbdb1dc
+  data.tar.gz: 47bdc2e63a242463221143098758c0cfc2ff65b64789c9b46ca6e5073d5dc9a8b07317b29a6edac79056cea7c3c41d334d9a5ad7b697780e4ba17258624a7526

data/CHANGELOG.md

@@ -1,3 +1,8 @@
+### 0.5.02 / 2017-10-08
+* [NEW] Addedsupport for ASTs (Abstract Syntax Tree)
+* [CHANGE] File `examples/data_formats/JSON/JSON_demo.rb` Added New command-line switches for details use --help option
+* [CHANGE] File `examples/general/calc_iter1/calc_demo.rb` Calculator now computes the value of a parsed expression
+
 ### 0.5.01 / 2017-08-20
 * [CHANGE] File `appveyor.yml`: Added Ruby 2.4 build environment in Appveyor CI
 

data/examples/data_formats/JSON/cli_options.rb

@@ -7,12 +7,13 @@ class CLIOptions < Hash
   # parse trees from LBN output.
   def initialize(progName, progVersion, args)
     super()
-    
+
     # Default values
     self[:prog_name] = progName
     self[:prog_version] = progVersion
     self[:format] = :ascii_tree
-    
+    self[:rep] = :cst
+
     options = build_option_parser
     options.parse!(args)
   end
@@ -23,29 +24,44 @@ class CLIOptions < Hash
     OptionParser.new do |opts|
       opts.banner = <<-END_BANNER
 #{self[:prog_name]}: a demo utility that parses a JSON file
-and renders its parse tree to the standard output 
+and renders its parse tree to the standard output
 in the format specified in the command-line.
 
 Usage: json_demo.rb [options] FILE
 
 Examples:
 json_demo --format ascii_tree sample01.json
+json_demo --rep ast --format ruby sample01.json
 END_BANNER
 
       opts.separator ''
 
       format_help = <<-END_TEXT
 Select the output format (default: ascii_tree). Available formats:
-  ascii_tree  Simple text representation of parse trees
-  minify      Strip all unnecessary whitespace in the input json file
-  labelled    Labelled square notation (LBN)
-              Use online tools (e.g. http://yohasebe.com/rsyntaxtree/) 
-              to visualize parse trees from LBN output.
+        ascii_tree  [cst, ast] Simple text representation of parse trees
+        minify      [cst] Strip all unnecessary whitespace in input json file
+        labelled    [cst, ast] Labelled square notation (LBN)
+                    Use online tools (e.g. http://yohasebe.com/rsyntaxtree/)
+                    to visualize parse trees from LBN output.
+        ruby        [ast] A Ruby representation of the JSON input.
 END_TEXT
-      formats = %i[ascii_tree labelled minify]
+      formats = %i[ascii_tree labelled minify ruby]
       opts.on('-f', '--format FORMAT', formats, format_help) do |frm|
         self[:format] = frm
       end
+      opts.separator ''
+
+      rep_help = <<-END_TEXT
+Set the parse tree representation (default: cst)
+        cst         Concrete Syntax Tree. The out-of-the-box parse tree 
+                    representation.
+        ast         Abstract Syntaxt Tree. A customized parse tree for JSON.
+                    It is a more compact and practical representation.
+END_TEXT
+      representations = %i[cst ast]
+      opts.on('-r', '--rep REP', representations, rep_help) do |rep|
+        self[:rep] = rep
+      end
 
       opts.separator ''
       opts.separator '  **** Utility ****'

data/examples/data_formats/JSON/json_ast_builder.rb

@@ -0,0 +1,152 @@
+require_relative 'json_ast_nodes'
+
+# The purpose of a JSONASTBuilder is to build piece by piece an AST
+# (Abstract Syntax Tree) from a sequence of input tokens and
+# visit events produced by walking over a GFGParsing object.
+# Uses the Builder GoF pattern.
+# The Builder pattern creates a complex object
+# (say, a parse tree) from simpler objects (terminal and non-terminal
+# nodes) and using a step by step approach.
+class JSONASTBuilder < Rley::Parser::ParseTreeBuilder 
+  Terminal2NodeClass = {
+  'false' => JSONBooleanNode,
+  'true' => JSONBooleanNode,
+  'null' => JSONNullNode,
+  'string' => JSONStringNode,
+  'number' => JSONNumberNode,
+  }
+
+  protected
+  
+  def return_first_child(_range, _tokens, theChildren)
+    return theChildren[0]
+  end
+
+  def return_second_child(_range, _tokens, theChildren)
+    return theChildren[1]
+  end      
+
+  # Overriding method.
+  # Create a parse tree object with given
+  # node as root node.
+  def create_tree(aRootNode)
+    return Rley::PTree::ParseTree.new(aRootNode)
+  end
+
+  # Overriding method.
+  # Factory method for creating a node object for the given
+  # input token.
+  # @param terminal [Terminal] Terminal symbol associated with the token
+  # @param aTokenPosition [Integer] Position of token in the input stream
+  # @param aToken [Token] The input token
+  def new_leaf_node(terminal, aTokenPosition, aToken)
+    klass = Terminal2NodeClass.fetch(terminal.name, JSONTerminalNode)
+    return klass.new(aToken, aTokenPosition)
+  end
+
+
+  # Method to override.
+  # Factory method for creating a parent node object.
+  # @param aProduction [Production] Production rule
+  # @param aRange [Range] Range of tokens matched by the rule
+  # @param theTokens [Array] The input tokens
+  # @param theChildren [Array] Children nodes (one per rhs symbol)
+  def new_parent_node(aProduction, aRange, theTokens, theChildren)
+    node = case aProduction.name
+      when 'JSON-text[0]' # rule 'JSON-text' => 'value'
+        return_first_child(aRange, theTokens, theChildren)
+
+      when /value\[\d\]/
+        return_first_child(aRange, theTokens, theChildren)
+
+      when 'object[0]'
+        reduce_object_0(aProduction, aRange, theTokens, theChildren)
+
+      when 'object[1]'
+        reduce_object_1(aRange, theTokens, theChildren)
+
+      when 'member-list[0]'
+        reduce_member_list_0(aRange, theTokens, theChildren)
+
+      when 'member-list[1]'
+        reduce_member_list_1(aProduction, aRange, theTokens, theChildren)
+
+      when 'member[0]'
+        reduce_member_0(aProduction, aRange, theTokens, theChildren)
+
+      when 'array[0]'
+        reduce_array_0(aProduction, aRange, theTokens, theChildren)
+
+      when 'array[1]'
+        reduce_array_1(aRange, theTokens, theChildren)
+
+      when 'array-items[0]'
+        reduce_array_items_0(aRange, theTokens, theChildren)
+
+      when 'array-items[1]'
+        reduce_array_items_1(aProduction, aRange, theTokens, theChildren)
+      else
+        raise StandardError, "Don't know production #{aProduction.name}"
+    end
+
+    return node
+  end
+  
+  # rule 'object' => %w[begin-object member-list end-object]
+  def reduce_object_0(aProduction, aRange, theTokens, theChildren)
+    second_child = theChildren[1]
+    second_child.symbol = aProduction.lhs
+    return second_child
+  end
+
+  # rule 'object' => %w[begin-object end-object]
+  def reduce_object_1(aRange, theTokens, theChildren)
+    return JSONObjectNode.new(aProduction.lhs)
+  end
+
+  # rule 'member-list' => %w[member-list value-separator member]
+  def reduce_member_list_0(aRange, theTokens, theChildren)
+    node = theChildren[0]
+    node.members << theChildren.last
+    return node
+  end
+
+  # rule 'member-list' => 'member'
+  def reduce_member_list_1(aProduction, aRange, theTokens, theChildren)
+    node = JSONObjectNode.new(aProduction.lhs)
+    node.members << theChildren[0]
+    return node
+  end
+
+  # rule 'member' => %w[string name-separator value]
+  def reduce_member_0(aProduction, aRange, theTokens, theChildren)
+    return JSONPair.new(theChildren[0], theChildren[2], aProduction.lhs)
+  end
+
+  # rule 'object' => %w[begin-object member-list end-object]
+  def reduce_array_0(aProduction, aRange, theTokens, theChildren)
+    second_child = theChildren[1]
+    second_child.symbol = aProduction.lhs
+    return second_child  
+  end
+
+
+  # rule 'array' => %w[begin-array end-array]
+  def reduce_array_1(aRange, theTokens, theChildren)
+    return JSONArrayNode.new
+  end
+
+  # rule 'array-items' => %w[array-items value-separator value]
+  def reduce_array_items_0(aRange, theTokens, theChildren)
+    node = theChildren[0]
+    node.children << theChildren[2]
+    return node
+  end
+  
+  #   rule 'array-items' => %w[value]
+  def reduce_array_items_1(aProduction, aRange, theTokens, theChildren)
+    node = JSONArrayNode.new(aProduction.lhs)
+    node.children << theChildren[0]
+    return node
+  end
+end # class

data/examples/data_formats/JSON/json_ast_nodes.rb

@@ -0,0 +1,141 @@
+# Classes that implement nodes of Abstract Syntax Trees (AST) representing
+# JSON parse results.
+
+
+JSONTerminalNode = Struct.new(:token, :value, :position) do
+  def initialize(aToken, aPosition)
+    self.token = aToken
+    self.position = aPosition
+    init_value(aToken.lexeme)
+  end
+
+  # This method can be overriden
+  def init_value(aLiteral)
+    self.value = aLiteral.dup
+  end
+
+  def symbol()
+    self.token.terminal
+  end
+
+  def to_ruby()
+    return value
+  end
+  
+  # Part of the 'visitee' role in Visitor design pattern.
+  # @param aVisitor[ParseTreeVisitor] the visitor
+  def accept(aVisitor)
+    aVisitor.visit_terminal(self)
+  end  
+end
+
+
+class JSONNullNode < JSONTerminalNode
+  def init_value(_aLiteral)
+    self.value = nil
+  end
+end
+
+class JSONBooleanNode < JSONTerminalNode
+  def init_value(aLiteral)
+    self.value = aLiteral == 'true'
+  end
+end
+
+class JSONStringNode < JSONTerminalNode
+end
+
+class JSONNumberNode  < JSONTerminalNode
+  def init_value(aLiteral)
+    case aLiteral
+      when /^[+-]?\d+$/
+        self.value = aLiteral.to_i
+
+      when /^[+-]?\d+(\.\d+)?([eE][+-]?\d+)?$/
+        self.value = aLiteral.to_f
+    end
+  end
+end
+
+class JSONCompositeNode
+  attr_accessor(:children)
+  attr_accessor(:symbol)
+
+  def initialize(aSymbol)
+    @symbol = aSymbol
+    @children = []    
+  end
+
+  # Part of the 'visitee' role in Visitor design pattern.
+  # @param aVisitor[ParseTreeVisitor] the visitor
+  def accept(aVisitor)
+    aVisitor.visit_nonterminal(self)
+  end
+  
+  alias subnodes children
+
+end # class
+
+
+class JSONArrayNode < JSONCompositeNode
+  def initialize(aSymbol)
+    super(aSymbol)
+  end
+
+  # Convert this tree node in a simpler Ruby representation.
+  # Basically a JSON object corresponds to a Ruhy Hash
+  def to_ruby()
+    rep = []
+    children.each do |child|
+      rep << child.to_ruby
+    end
+
+    return rep
+  end
+end # class
+
+class JSONPair
+  attr_reader(:name)
+  attr_reader(:value)
+  attr_reader(:symbol)
+
+  def initialize(aName, aValue, aSymbol)
+    @name = aName
+    @value = aValue
+    @symbol = aSymbol
+  end
+
+  def children()
+    return [name, value]
+  end
+  
+  alias subnodes children
+
+  # Part of the 'visitee' role in Visitor design pattern.
+  # @param aVisitor[ParseTreeVisitor] the visitor
+  def accept(aVisitor)
+    aVisitor.visit_nonterminal(self)
+  end
+
+end # class
+
+class JSONObjectNode < JSONCompositeNode
+  def initialize(aSymbol)
+    super(aSymbol)
+  end
+
+  # Convert this tree node in a simpler Ruby representation.
+  # Basically a JSON object corresponds to a Ruhy Hash
+  def to_ruby()
+    rep = {}
+    members.each do |pair|
+      rep[pair.name.to_ruby] = pair.value.to_ruby
+    end
+
+    return rep
+  end
+
+  alias members children
+end # class
+
+

data/examples/data_formats/JSON/json_demo.rb

@@ -1,9 +1,10 @@
 require_relative 'cli_options'
 require_relative 'json_parser'
 require_relative 'json_minifier'
+require_relative 'json_ast_builder'
 
 prog_name = 'json_demo'
-prog_version = '0.2.0'
+prog_version = '0.3.0'
 
 cli_options = CLIOptions.new(prog_name, prog_version, ARGV)
 if ARGV.empty?
@@ -24,8 +25,8 @@ unless result.success?
   exit(1)
 end
 
-# Generate a parse tree from the parse result
-ptree = result.parse_tree
+tree_rep = cli_options[:rep]
+renderer = nil
 
 # Select the output format
 case cli_options[:format]
@@ -34,12 +35,27 @@ case cli_options[:format]
   when :labelled
     renderer = Rley::Formatter::BracketNotation.new($stdout)
   when :minify
-    renderer = JSONMinifier.new($stdout)    
+    msg = "minify format works for 'cst' representation only"
+    raise StandardError, msg if tree_rep == :ast
+    renderer = JSONMinifier.new($stdout)
+  when :ruby
+    msg = "ruby format works for 'ast' representation only"
+    raise StandardError, msg if tree_rep == :cst
 end
 
-# Let's create a parse tree visitor
-visitor = Rley::ParseTreeVisitor.new(ptree)
+tree_builder = (tree_rep == :ast)? JSONASTBuilder : nil
+
+# Generate a parse tree from the parse result
+ptree = result.parse_tree(tree_builder)
 
-# Now output formatted parse tree
-renderer.render(visitor)
+if renderer
+  # Let's create a parse tree visitor
+  visitor = Rley::ParseTreeVisitor.new(ptree)
+
+  # Now output formatted parse tree
+  renderer.render(visitor)
+else
+  root = ptree.root
+  p(root.to_ruby) # Output the Ruby representation of the JSON input
+end
 # End of file

data/examples/general/calc_iter1/calc_ast_builder.rb

@@ -0,0 +1,142 @@
+require_relative 'calc_ast_nodes'
+
+# The purpose of a CalcASTBuilder is to build piece by piece an AST
+# (Abstract Syntax Tree) from a sequence of input tokens and
+# visit events produced by walking over a GFGParsing object.
+# Uses the Builder GoF pattern.
+# The Builder pattern creates a complex object
+# (say, a parse tree) from simpler objects (terminal and non-terminal
+# nodes) and using a step by step approach.
+class CalcASTBuilder < Rley::Parser::ParseTreeBuilder
+  Terminal2NodeClass = {
+    'NUMBER' => CalcNumberNode
+  }
+
+  protected
+
+  def return_first_child(_range, _tokens, theChildren)
+    return theChildren[0]
+  end
+
+  def return_second_child(_range, _tokens, theChildren)
+    return theChildren[1]
+  end
+
+  def return_last_child(_range, _tokens, theChildren)
+    return theChildren[-1]
+  end
+
+  # Overriding method.
+  # Create a parse tree object with given
+  # node as root node.
+  def create_tree(aRootNode)
+    return Rley::PTree::ParseTree.new(aRootNode)
+  end
+
+  # Overriding method.
+  # Factory method for creating a node object for the given
+  # input token.
+  # @param aTerminal [Terminal] Terminal symbol associated with the token
+  # @param aTokenPosition [Integer] Position of token in the input stream
+  # @param aToken [Token] The input token
+  def new_leaf_node(aProduction, aTerminal, aTokenPosition, aToken)
+    klass = Terminal2NodeClass.fetch(aTerminal.name, CalcTerminalNode)
+    if klass
+      node = klass.new(aToken, aTokenPosition)
+    else
+      node = PTree::TerminalNode.new(aToken, aTokenPosition)
+    end
+    
+    return node
+  end
+
+
+  # Method to override.
+  # Factory method for creating a parent node object.
+  # @param aProduction [Production] Production rule
+  # @param aRange [Range] Range of tokens matched by the rule
+  # @param theTokens [Array] The input tokens
+  # @param theChildren [Array] Children nodes (one per rhs symbol)
+  def new_parent_node(aProduction, aRange, theTokens, theChildren)
+    node = case aProduction.name
+      when 'expression[0]' # rule 'expression' => 'simple_expression'
+        return_first_child(aRange, theTokens, theChildren)
+
+      when 'simple_expression[0]' # rule 'simple_expression' => 'term'
+        return_first_child(aRange, theTokens, theChildren)
+
+      when 'simple_expression[1]'
+        # rule 'simple_expression' => %w[simple_expression add_operator term]
+        reduce_simple_expression_1(aProduction, aRange, theTokens, theChildren)
+
+      when 'term[0]' # rule 'term' => 'factor'
+        return_first_child(aRange, theTokens, theChildren)
+
+      when 'term[1]' # rule 'term' => %w[term mul_operator factor]
+        reduce_term_1(aProduction, aRange, theTokens, theChildren)
+
+      when 'factor[0]' # rule 'factor' => 'NUMBER'
+        return_first_child(aRange, theTokens, theChildren)
+
+      when 'factor[1]' # rule 'factor' => %w[LPAREN expression RPAREN]
+        return_second_child(aRange, theTokens, theChildren)
+
+      when 'add_operator[0]' # rule 'add_operator' => 'PLUS'
+        reduce_add_operator_0(aProduction, aRange, theTokens, theChildren)
+        
+      when 'add_operator[1]' # rule 'add_operator' => 'MINUS'
+        reduce_add_operator_1(aProduction, aRange, theTokens, theChildren)        
+
+      when 'mul_operator[0]' # rule 'mul_operator' => 'STAR'
+         reduce_mul_operator_0(aProduction, aRange, theTokens, theChildren)
+         
+      when 'mul_operator[1]' # rule 'mul_operator' =>  'DIVIDE'
+         reduce_mul_operator_1(aProduction, aRange, theTokens, theChildren)         
+
+      else
+        raise StandardError, "Don't know production #{aProduction.name}"
+    end
+
+    return node
+  end
+  
+  def reduce_binary_operator(theChildren)
+    operator_node = theChildren[1]
+    operator_node.children << theChildren[0]
+    operator_node.children << theChildren[2]
+    return operator_node  
+  end
+
+  # rule 'simple_expression' => %w[simple_expression add_operator term]
+  def reduce_simple_expression_1(aProduction, aRange, theTokens, theChildren)
+    reduce_binary_operator(theChildren)
+  end
+
+
+  # rule 'term' => %w[term mul_operator factor]
+  def reduce_term_1(aProduction, aRange, theTokens, theChildren)
+    reduce_binary_operator(theChildren)
+  end
+  
+  # rule 'add_operator' => 'PLUS'
+  def reduce_add_operator_0(aProduction, aRange, theTokens, theChildren)
+    return CalcAddNode.new(theChildren[0].symbol)
+  end
+  
+  # rule 'add_operator' => 'MINUS'
+  def reduce_add_operator_1(aProduction, aRange, theTokens, theChildren)
+    return CalcSubtractNode.new(theChildren[0].symbol)
+  end
+  
+  # rule 'mul_operator' => 'STAR'
+  def reduce_mul_operator_0(aProduction, aRange, theTokens, theChildren)
+    return CalcMultiplyNode.new(theChildren[0].symbol)
+  end
+  
+  # rule 'mul_operator' => 'DIVIDE'
+  def reduce_mul_operator_1(aProduction, aRange, theTokens, theChildren)
+    return CalcDivideNode.new(theChildren[0].symbol)
+  end  
+  
+
+end # class