loxxy 0.0.5 → 0.0.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5db00fee0e8e7ef89a48cacc4e9ad0baf7ae5ee2cb419ba76964199bd7552199
-  data.tar.gz: ddb8e9c7ac267f32bed3f77ada05a6af38a872b5f892452a496b17522b608977
+  metadata.gz: de9d3c24db15863c6ea99c2a9c8a2e413b2a243ddf79de28da5583d1944fc4e4
+  data.tar.gz: a5f6dacfd67b6833e3c051273765e1d94c95d1d1700a974a09a3caf0519a1570
 SHA512:
-  metadata.gz: 44e0049b0bc01e5d35bed7b91c32a7a3b977e6ba4e806439958bad2dc8aae7ead017a310c8523bd4cb628c14951946981bee42f4001d68b701159fcdaff53419
-  data.tar.gz: c90236a82f0a4872b209b8462d5f13377815fd4fc41babf256ff4894ea6900e343c6f192194e1b713cb9f8a8d2bffc326e7ea4f00726193e0a792dc9e1457e41
+  metadata.gz: 3c4bd62b1a9cd960ee97d807a5f44a26cc360df4344975210ca967117c62585a2138fef382ffb5ea7d79f002b9a9a951ba77622cb8d799973c0ad8a24ffbae52
+  data.tar.gz: 5b14b5c4a6ae98f5e1d9a819160393393654cc3d817e36f1da68cd12fcfedfbce47097463dc6babcf36b5582e970ea41b4cef5ebceb768b962c8bd7b02bcb553

data/CHANGELOG.md

@@ -1,3 +1,17 @@
+## [0.0.6] - 2021-01-03
+- First iteration of a parser with AST generation.
+
+## Added
+- Class `Parser` this one generates AST's (Abstract Syntax Tree)
+- Class `AST::ASTBuilder` default code to generate an AST.
+
+## Changed
+- File `spec_helper.rb`: removed Bundler dependency
+- File `README.md` Added example with AST visualization.
+
+## Fixed
+- File `grammar.rb` ensure that the constant `Grammar` is created once only.
+
 ## [0.0.5] - 2021-01-02
 - Improved example in `README.md`, code re-styling to please `Rubocop` 1.7
 

data/README.md

@@ -12,16 +12,19 @@ a simple language used in Bob Nystrom's online book [Crafting Interpreters](http
 ## Roadmap
 - [DONE] Scanner (tokenizer)
 - [DONE] Raw parser. It parses Lox programs and generates a parse tree.
-- [TODO] Tailored parser for generating AST (Abstract Syntax Tree)
-- [TODO] Custom AST builder class  
+- [DONE] Tailored parser for generating AST (Abstract Syntax Tree)
+- [STARTED] Custom AST builder class  
 - [TODO] Hierarchy classes for representing Lox expressions in AST  
 - [TODO] Interpreter or transpiler
 
-
 ## Example
-At this, stage, the raw parser is able to recognize Lox input and to generate
+The __loxxy__ hosts two distinct parsers classes (`RawParser` and `Parser`).  
+- A `RawParser` instance is able to recognize Lox input and to generate
 a concrete parse tree from it.
+- A `Parser` instance can also parse Lox source code but will generate an AST
+(Abstract Syntax Tree) that will be used by the future tree-walking interpreter.
 
+### Example using RawParser class
 ```ruby
 require 'loxxy'
 
@@ -74,9 +77,50 @@ program
 +-- EOF: ''
 ```
 
-Soon, the `loxxy` will host another, tailored parser, that will generate
-abstract syntax tree which much more convenient for further processing 
-(like implementing an interpreter).
+### Example using Parser class
+Except for the `Parser` class name, this example is identical 
+to the previous one.
+
+```ruby
+require 'loxxy'
+
+
+lox_input = <<-LOX_END
+  // Your first Lox program!
+  print "Hello, world!";
+LOX_END
+
+# Show that the parser accepts the above program
+parser = Loxxy::FrontEnd::Parser.new
+
+# Now parse the input into an abstract parse tree...
+ptree = parser.parse(lox_input)
+
+# Display the parse tree thanks to Rley utility classes...
+visitor = Rley::ParseTreeVisitor.new(ptree)
+tree_formatter = Rley::Formatter::Asciitree.new($stdout)
+tree_formatter.render(visitor)
+```
+
+However the output differs significantly:
+```
+program
++-- declaration_star
+|   +-- printStmt
+|       +-- PRINT: 'print'
+|       +-- logic_or
+|       |   +-- logic_and
+|       |       +-- equality
+|       |           +-- comparison
+|       |               +-- term
+|       |                   +-- factor
+|       |                       +-- call
+|       |                           +-- STRING: '"Hello, world!"'
+|       +-- SEMICOLON: ';'
++-- EOF: ''
+```
+
+
 
 ## Installation
 

data/lib/loxxy.rb

@@ -2,6 +2,7 @@
 
 require_relative 'loxxy/version'
 require_relative 'loxxy/front_end/raw_parser'
+require_relative 'loxxy/front_end/parser'
 
 module Loxxy
   class Error < StandardError; end

data/lib/loxxy/ast/ast_builder.rb

@@ -0,0 +1,74 @@
+# frozen_string_literal: true
+
+require_relative '../datatype/all_datatypes'
+
+module Loxxy
+  module Ast
+    # The purpose of ASTBuilder is to build piece by piece an AST
+    # (Abstract Syntax Tree) from a sequence of input tokens and
+    # visit events produced by walking over a GFGParsing object.
+    class ASTBuilder < Rley::ParseRep::ASTBaseBuilder
+      # Terminal2NodeClass = {
+      #   'FALSE' => Datatype::False,
+      #   'NIL' => Datatype::Nil,
+      #   'NUMBER' => Datatype::Number,
+      #   'STRING' => Datatype::LXString,
+      #   'TRUE' => Datatype::True
+      # }.freeze
+
+      attr_reader :strict
+
+      # Create a new AST builder instance.
+      # @param theTokens [Array<Rley::Lexical::Token>] The sequence of input tokens.
+      def initialize(theTokens)
+        super(theTokens)
+        @strict = false
+      end
+
+      protected
+
+      def terminal2node
+        Terminal2NodeClass
+      end
+
+      # Method override
+      def new_leaf_node(_production, _terminal, aTokenPosition, aToken)
+        Rley::PTree::TerminalNode.new(aToken, aTokenPosition)
+      end
+
+      # Factory method for creating a parent node object.
+      # @param aProduction [Production] Production rule
+      # @param aRange [Range] Range of tokens matched by the rule
+      # @param theTokens [Array] The input tokens
+      # @param theChildren [Array] Children nodes (one per rhs symbol)
+      def new_parent_node(aProduction, aRange, theTokens, theChildren)
+        mth_name = method_name(aProduction.name)
+        if respond_to?(mth_name, true)
+          node = send(mth_name, aProduction, aRange, theTokens, theChildren)
+        else
+          # Default action...
+          node = case aProduction.rhs.size
+                 when 0
+                   return_epsilon(aRange, theTokens, theChildren)
+                 when 1
+                   return_first_child(aRange, theTokens, theChildren)
+                 else
+                   if strict
+                     msg = "Don't know production '#{aProduction.name}'"
+                     raise StandardError, msg
+                   else
+                     node = Rley::PTree::NonTerminalNode.new(aProduction.lhs, aRange)
+                     theChildren&.reverse_each do |child|
+                       node.add_subnode(child) if child
+                     end
+
+                     node
+                   end
+                 end
+        end
+
+        node
+      end
+    end # class
+  end # module
+end # module

data/lib/loxxy/front_end/grammar.rb

@@ -144,8 +144,10 @@ module Loxxy
       rule('arguments' => 'expression')
     end
 
-    # And now build the grammar and make it accessible via a constant
-    # @return [Rley::Syntax::Grammar]
-    Grammar = builder.grammar
+    unless defined?(Grammar)
+      # And now build the grammar and make it accessible via a constant
+      # @return [Rley::Syntax::Grammar]
+      Grammar = builder.grammar
+    end
   end # module
 end # module

data/lib/loxxy/front_end/parser.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+require_relative 'scanner'
+require_relative 'grammar'
+require_relative '../ast/ast_builder'
+
+module Loxxy
+  module FrontEnd
+    # A Lox parser that produce concrete parse trees.
+    # Concrete parse trees are the default kind of parse tree
+    # generated by the Rley library.
+    # They consist of two node types only:
+    # - NonTerminalNode
+    # - TerminalNode
+    # A NonTerminalNode has zero or more child nodes (called subnodes)
+    # A TerminalNode is leaf node, that is, it has no child node.
+    # While concrete parse tree nodes can be generated out of the box,
+    # they have the following drawbacks:
+    # - Generic node classes that aren't always suited for the needs of
+    #     the language being processing.
+    # - Concrete parse tree tend to be deeply nested, which may complicate
+    #   further processing.
+    class Parser
+      # @return [Rley::Engine] A facade object for the Rley parsing library
+      attr_reader(:engine)
+
+      def initialize
+        # Create a Rley facade object
+        @engine = Rley::Engine.new do |cfg|
+          cfg.diagnose = true
+          cfg.repr_builder = Ast::ASTBuilder
+        end
+
+        # Step 1. Load Lox grammar
+        @engine.use_grammar(Loxxy::FrontEnd::Grammar)
+      end
+
+      # Parse the given Lox program into a parse tree.
+      # @param source [String] Lox program to parse
+      # @return [Rley::ParseTree] A parse tree equivalent to the Lox input.
+      def parse(source)
+        lexer = Scanner.new(source)
+        result = engine.parse(lexer.tokens)
+
+        unless result.success?
+          # Stop if the parse failed...
+          line1 = "Parsing failed\n"
+          line2 = "Reason: #{result.failure_reason.message}"
+          raise StandardError, line1 + line2
+        end
+
+        return engine.convert(result) # engine.to_ptree(result)
+      end
+    end # class
+  end # module
+end # module

data/lib/loxxy/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Loxxy
-  VERSION = '0.0.5'
+  VERSION = '0.0.6'
 end

data/spec/front_end/parser_spec.rb

@@ -0,0 +1,96 @@
+# frozen_string_literal: true
+
+require_relative '../spec_helper' # Use the RSpec framework
+
+# Load the class under test
+require_relative '../../lib/loxxy/front_end/parser'
+
+module Loxxy
+  module FrontEnd
+    describe Parser do
+      subject { Parser.new }
+
+      context 'Initialization:' do
+        it 'should be initialized without argument' do
+          expect { Parser.new }.not_to raise_error
+        end
+
+        it 'should have its parse engine initialized' do
+          expect(subject.engine).to be_kind_of(Rley::Engine)
+        end
+      end # context
+
+      context 'Parsing blank files:' do
+        def check_empty_input_result(aParseTree)
+          # Parse results MUST to comply to grammar rule:
+          #   program => declaration_star EOF
+          #   where the declaration_star MUST be empty
+          expect(aParseTree.root.symbol.name).to eq('program')
+          expect(aParseTree.root.subnodes.size).to eq(1)
+          eof = aParseTree.root.subnodes[0]
+          expect(eof).to be_kind_of(Rley::PTree::TerminalNode)
+          expect(eof.symbol.name).to eq('EOF')
+        end
+
+        it 'should cope with an empty input' do
+          ptree = subject.parse('')
+          check_empty_input_result(ptree)
+        end
+
+        it 'should cope with whitespaces only input' do
+          ptree = subject.parse(' ' * 80 + "\n" * 20)
+          check_empty_input_result(ptree)
+        end
+
+        it 'should cope with comments only input' do
+          input = +''
+          %w[First Second Third].each do |ordinal|
+            input << "// #{ordinal} comment line\r\n"
+          end
+          ptree = subject.parse(input)
+          check_empty_input_result(ptree)
+        end
+      end # context
+
+      context 'Parsing expressions:' do
+        # Utility method to walk towards deeply nested node
+        # @param aNTNode [Rley::PTree::NonTerminalNode]
+        # @param subnodePath[Array<Integer>] An Array of subnode indices
+        def walk_subnodes(aNTNode, subnodePath)
+          curr_node = aNTNode
+          subnodePath.each do |index|
+            curr_node = curr_node.subnodes[index]
+          end
+
+          curr_node
+        end
+        it 'should parse a hello world program' do
+          program = <<-LOX_END
+            // Your first Lox program!
+            print "Hello, world!";
+LOX_END
+          ptree = subject.parse(program)
+          root = ptree.root
+          expect(root.symbol.name).to eq('program')
+          (decls, eof) = root.subnodes
+          expect(decls).to be_kind_of(Rley::PTree::NonTerminalNode)
+          expect(decls.symbol.name).to eq('declaration_star')
+          prnt_stmt = decls.subnodes[0]
+          expect(prnt_stmt).to be_kind_of(Rley::PTree::NonTerminalNode)
+          expect(prnt_stmt.subnodes.size).to eq(3)
+          expect(prnt_stmt.subnodes[0]).to be_kind_of(Rley::PTree::TerminalNode)
+          expect(prnt_stmt.subnodes[0].symbol.name).to eq('PRINT')
+          expect(prnt_stmt.subnodes[1]).to be_kind_of(Rley::PTree::NonTerminalNode)
+          # leaf_node = walk_subnodes(prnt_stmt, [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])
+          # expect(leaf_node).to be_kind_of(Rley::PTree::TerminalNode)
+          # expect(leaf_node.symbol.name).to eq('STRING')
+          # expect(leaf_node.token.value).to eq('Hello, world!')
+          expect(prnt_stmt.subnodes[2]).to be_kind_of(Rley::PTree::TerminalNode)
+          expect(prnt_stmt.subnodes[2].symbol.name).to eq('SEMICOLON')
+          expect(eof).to be_kind_of(Rley::PTree::TerminalNode)
+          expect(eof.symbol.name).to eq('EOF')
+        end
+      end
+    end # describe
+  end # module
+end # module

data/spec/spec_helper.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-require 'bundler/setup'
+require 'rspec' # Use the RSpec framework
 require 'loxxy'
 
 RSpec.configure do |config|

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: loxxy
 version: !ruby/object:Gem::Version
-  version: 0.0.5
+  version: 0.0.6
 platform: ruby
 authors:
 - Dimitri Geshef
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2021-01-02 00:00:00.000000000 Z
+date: 2021-01-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rley
@@ -84,6 +84,7 @@ files:
 - README.md
 - Rakefile
 - lib/loxxy.rb
+- lib/loxxy/ast/ast_builder.rb
 - lib/loxxy/datatype/all_datatypes.rb
 - lib/loxxy/datatype/boolean.rb
 - lib/loxxy/datatype/builtin_datatype.rb
@@ -94,11 +95,13 @@ files:
 - lib/loxxy/datatype/true.rb
 - lib/loxxy/front_end/grammar.rb
 - lib/loxxy/front_end/literal.rb
+- lib/loxxy/front_end/parser.rb
 - lib/loxxy/front_end/raw_parser.rb
 - lib/loxxy/front_end/scanner.rb
 - lib/loxxy/version.rb
 - loxxy.gemspec
 - spec/datatype/lx_string_spec.rb
+- spec/front_end/parser_spec.rb
 - spec/front_end/raw_parser_spec.rb
 - spec/front_end/scanner_spec.rb
 - spec/loxxy_spec.rb
@@ -129,6 +132,7 @@ specification_version: 4
 summary: An implementation of the Lox programming language. WIP
 test_files:
 - spec/datatype/lx_string_spec.rb
+- spec/front_end/parser_spec.rb
 - spec/front_end/raw_parser_spec.rb
 - spec/front_end/scanner_spec.rb
 - spec/loxxy_spec.rb