rley 0.7.06 → 0.8.01

data/spec/rley/notation/parser_spec.rb

@@ -0,0 +1,184 @@
+# frozen_string_literal: true
+
+require_relative '../../spec_helper' # Use the RSpec framework
+
+require_relative '../../../lib/rley/notation/ast_builder'
+# Load the class under test
+require_relative '../../../lib/rley/notation/parser'
+
+module Rley
+  module Notation
+    describe Parser do
+      subject { Parser.new }
+
+      # Utility method to walk towards deeply nested node
+      # @param aNTNode [Rley::PTree::NonTerminalNode]
+      # @param subnodePath[Array<Integer>] An Array of subnode indices
+      def walk_subnodes(aNTNode, subnodePath)
+        curr_node = aNTNode
+        subnodePath.each do |index|
+          curr_node = curr_node.subnodes[index]
+        end
+
+        curr_node
+      end
+
+      context 'Initialization:' do
+        it 'should be initialized without argument' do
+          expect { Parser.new }.not_to raise_error
+        end
+
+        it 'should have its parse engine initialized' do
+          expect(subject.engine).to be_kind_of(Rley::Engine)
+        end
+      end # context
+     
+      context 'Parsing into CST:' do
+        subject do
+          instance = Parser.new
+          instance.engine.configuration.repr_builder = Rley::ParseRep::CSTBuilder
+
+          instance
+        end
+
+        it 'should parse single symbol names' do
+          samples = %w[IF ifCondition statement]
+
+          # One drawback od CSTs: they have a deeply nested structure
+          samples.each do |source|
+            ptree = subject.parse(source)
+            expect(ptree.root).to be_kind_of(Rley::PTree::NonTerminalNode)
+            expect(ptree.root.symbol.name).to eq('notation')
+            expect(ptree.root.subnodes[0]).to be_kind_of(Rley::PTree::NonTerminalNode)
+            expect(ptree.root.subnodes[0].symbol.name).to eq('rhs')
+            expect(ptree.root.subnodes[0].subnodes[0]).to be_kind_of(Rley::PTree::NonTerminalNode)
+            member_seq = ptree.root.subnodes[0].subnodes[0]
+            expect(member_seq.symbol.name).to eq('member_seq')
+            expect(member_seq.subnodes[0]).to be_kind_of(Rley::PTree::NonTerminalNode)
+            expect(member_seq.subnodes[0].symbol.name).to eq('member')
+            expect(member_seq.subnodes[0].subnodes[0]).to be_kind_of(Rley::PTree::NonTerminalNode)
+            expect(member_seq.subnodes[0].subnodes[0].symbol.name).to eq('strait_member')
+            strait_member = member_seq.subnodes[0].subnodes[0]
+            expect(strait_member.subnodes[0]).to be_kind_of(Rley::PTree::NonTerminalNode)
+            expect(strait_member.subnodes[0].symbol.name).to eq('base_member')
+            expect(strait_member.subnodes[0].subnodes[0]).to be_kind_of(Rley::PTree::TerminalNode)
+            expect(strait_member.subnodes[0].subnodes[0].token.lexeme).to eq(source)
+          end
+        end
+
+      end # context
+
+      context 'Parsing into AST:' do
+        subject do
+          instance = Parser.new
+          instance.engine.configuration.repr_builder = ASTBuilder
+
+          instance
+        end
+
+        it 'should parse single symbol names' do
+          samples = %w[IF ifCondition statement]
+
+          samples.each do |source|
+            ptree = subject.parse(source)
+            expect(ptree.root).to be_kind_of(SymbolNode)
+            expect(ptree.root.name).to eq(source)
+            expect(ptree.root.repetition).to eq(:exactly_one)
+          end
+        end
+
+        it 'should parse a sequence of symbols' do
+          sequence = 'INT_LIT ELLIPSIS INT_LIT'
+
+          ptree = subject.parse(sequence)
+          expect(ptree.root).to be_kind_of(SequenceNode)
+          expect(ptree.root.subnodes[0]).to be_kind_of(SymbolNode)
+          expect(ptree.root.subnodes[0].name).to eq('INT_LIT')
+          expect(ptree.root.subnodes[1]).to be_kind_of(SymbolNode)
+          expect(ptree.root.subnodes[1].name).to eq('ELLIPSIS')
+          expect(ptree.root.subnodes[2]).to be_kind_of(SymbolNode)
+          expect(ptree.root.subnodes[2].name).to eq('INT_LIT')
+        end
+
+        it 'should parse an optional symbol' do
+          optional = 'member_seq?'
+
+          ptree = subject.parse(optional)
+          expect(ptree.root).to be_kind_of(SymbolNode)
+          expect(ptree.root.name).to eq('member_seq')
+          expect(ptree.root.repetition).to eq(:zero_or_one)
+        end
+
+        it 'should parse a symbol with a + modifier' do
+          one_or_more = 'member+'
+
+          ptree = subject.parse(one_or_more)
+          expect(ptree.root).to be_kind_of(SymbolNode)
+          expect(ptree.root.name).to eq('member')
+          expect(ptree.root.repetition).to eq(:one_or_more)
+        end
+
+        it 'should parse a symbol with a * modifier' do
+          zero_or_more = 'declaration* EOF'
+
+          ptree = subject.parse(zero_or_more)
+          expect(ptree.root).to be_kind_of(SequenceNode)
+          expect(ptree.root.subnodes[0]).to be_kind_of(SymbolNode)
+          expect(ptree.root.subnodes[0].name).to eq('declaration')
+          expect(ptree.root.subnodes[0].repetition).to eq(:zero_or_more)
+          expect(ptree.root.subnodes[1]).to be_kind_of(SymbolNode)
+          expect(ptree.root.subnodes[1].name).to eq('EOF')
+          expect(ptree.root.subnodes[1].repetition).to eq(:exactly_one)
+        end
+
+        it 'should parse a grouping with a modifier' do
+          input = "IF ifCondition statement (ELSE statement)?"
+
+          ptree = subject.parse(input)
+          expect(ptree.root).to be_kind_of(SequenceNode)
+          expect(ptree.root.subnodes[0]).to be_kind_of(SymbolNode)
+          expect(ptree.root.subnodes[0].name).to eq('IF')
+          expect(ptree.root.subnodes[1]).to be_kind_of(SymbolNode)
+          expect(ptree.root.subnodes[1].name).to eq('ifCondition')
+          expect(ptree.root.subnodes[2]).to be_kind_of(SymbolNode)
+          expect(ptree.root.subnodes[2].name).to eq('statement')
+          expect(ptree.root.subnodes[3]).to be_kind_of(SequenceNode)
+          expect(ptree.root.subnodes[3].repetition).to eq(:zero_or_one)
+          expect(ptree.root.subnodes[3].subnodes[0]).to be_kind_of(SymbolNode)
+          expect(ptree.root.subnodes[3].subnodes[0].name).to eq('ELSE')
+          expect(ptree.root.subnodes[3].subnodes[1]).to be_kind_of(SymbolNode)
+          expect(ptree.root.subnodes[3].subnodes[1].name).to eq('statement')
+        end
+
+        it 'should parse an annotated symbol' do
+          optional = 'member_seq{repeat: 0..1}'
+
+          ptree = subject.parse(optional)
+          expect(ptree.root).to be_kind_of(SymbolNode)
+          expect(ptree.root.name).to eq('member_seq')
+          expect(ptree.root.repetition).to eq(:zero_or_one)
+        end
+
+        it 'should parse a grouping with embedded annotation' do
+          if_stmt = "IF ifCondition statement ( ELSE { match_closest: 'IF' } statement )?"
+
+          ptree = subject.parse(if_stmt)
+          expect(ptree.root).to be_kind_of(SequenceNode)
+          expect(ptree.root.subnodes[0]).to be_kind_of(SymbolNode)
+          expect(ptree.root.subnodes[0].name).to eq('IF')
+          expect(ptree.root.subnodes[1]).to be_kind_of(SymbolNode)
+          expect(ptree.root.subnodes[1].name).to eq('ifCondition')
+          expect(ptree.root.subnodes[2]).to be_kind_of(SymbolNode)
+          expect(ptree.root.subnodes[2].name).to eq('statement')
+          optional = ptree.root.subnodes[3]
+          expect(optional).to be_kind_of(SequenceNode)
+          expect(optional.repetition).to eq(:zero_or_one)
+          expect(optional.subnodes[0]).to be_kind_of(SymbolNode)
+          expect(optional.subnodes[0].name).to eq('ELSE')
+          expect(optional.subnodes[0].annotation).to eq({'match_closest' => 'IF'})
+          expect(optional.subnodes[1].name).to eq('statement')
+        end
+      end # context
+    end # describe
+  end # module
+end # module

data/spec/rley/notation/tokenizer_spec.rb

@@ -0,0 +1,370 @@
+# frozen_string_literal: true
+
+require_relative '../../spec_helper'
+
+# Load the class under test
+require_relative '../../../lib/rley/notation/tokenizer'
+
+module Rley # Open this namespace to avoid module qualifier prefixes
+  module Notation # Open this namespace to avoid module qualifier prefixes
+    describe Tokenizer do
+      # Utility method for comparing actual and expected token
+      # sequence. The final EOF is removed from the input sequence.
+      def match_expectations(aScanner, theExpectations)
+        tokens = aScanner.tokens
+
+        tokens.each_with_index do |token, i|
+          terminal, lexeme = theExpectations[i]
+          expect(token.terminal).to eq(terminal)
+          expect(token.lexeme).to eq(lexeme)
+        end
+      end
+      
+      context 'Initialization:' do
+      let(:sample_text) { 'begin-object member-list end-object' }
+      subject { Tokenizer.new }
+
+        it 'could be initialized with a text to tokenize or...' do
+          expect { Tokenizer.new(sample_text) }.not_to raise_error
+        end
+
+        it 'could be initialized without argument...' do
+          expect { Tokenizer.new }.not_to raise_error
+        end
+
+        it 'should have its scanner initialized' do
+          expect(subject.scanner).to be_kind_of(StringScanner)
+        end
+      end # context
+      
+      context 'Input tokenization:' do
+        it 'should recognize single special character token' do
+          input = '(){}?*+,'
+          subject.start_with(input)
+          expectations = [
+            # [token lexeme]
+            %w[LEFT_PAREN (],
+            %w[RIGHT_PAREN )],
+            %w[LEFT_BRACE {],
+            %w[RIGHT_BRACE }],
+            %w[QUESTION_MARK ?],            
+            %w[STAR *],             
+            %w[PLUS +],             
+            %w[COMMA ,]
+          ]
+          match_expectations(subject, expectations)
+        end
+
+        it 'should recognize one or two special character tokens' do
+          input = '..'
+          subject.start_with(input)
+          expectations = [
+            # [token lexeme]
+            %w[ELLIPSIS ..]
+          ]
+          match_expectations(subject, expectations)
+        end
+        
+        it 'should treat ? * + as symbols if they occur as suffix' do
+          input = 'a+ + b* * 3 ?'
+          subject.start_with(input)
+          expectations = [
+            # [token lexeme]
+            %w[SYMBOL a],
+            %w[PLUS +],
+            %w[SYMBOL +],
+            %w[SYMBOL b],
+            %w[STAR *],
+            %w[SYMBOL *],
+            %w[INT_LIT 3],
+            %w[SYMBOL ?]
+          ]
+          match_expectations(subject, expectations)
+        end        
+
+        it 'should recognize annotation keywords' do
+          keywords = 'match_closest: repeat:'
+          subject.start_with(keywords)
+          expectations = [
+            # [token lexeme]
+            %w[KEY match_closest],
+            %w[KEY repeat]
+          ]
+          match_expectations(subject, expectations)
+        end
+
+        it 'should recognize ordinal integer values' do
+          input = <<-RLEY_END
+            3      123
+            987654 0
+          RLEY_END
+
+          expectations = [
+            ['3', 3],
+            ['123', 123],
+            ['987654', 987654],
+            ['0', 0]
+          ]
+
+          subject.start_with(input)
+          subject.tokens[0..-2].each_with_index do |tok, i|
+            expect(tok).to be_kind_of(Rley::Lexical::Token)
+            expect(tok.terminal).to eq('INT_LIT')
+            (lexeme, val) = expectations[i]
+            expect(tok.lexeme).to eq(lexeme)
+          end
+        end
+
+        it 'should recognize string literals' do
+          input = <<-RLEY_END
+            ""
+            "string"
+            "123"
+            ''
+            'string'
+            '123'
+          RLEY_END
+
+          expectations = [
+            '',
+            'string',
+            '123'
+          ] * 2
+
+          subject.start_with(input)
+          subject.tokens.each_with_index do |str, i|
+            expect(str).to be_kind_of(Rley::Lexical::Token)
+            expect(str.terminal).to eq('STR_LIT')
+            (lexeme, val) = expectations[i]
+            expect(str.lexeme).to eq(lexeme)
+          end
+        end
+
+        it 'should recognize a sequence of symbols' do
+          input = "IF ifCondition statement ELSE statement"
+          expectations = [
+            'IF',
+            'ifCondition',
+            'statement',
+            'ELSE',
+            'statement'
+          ]
+
+          subject.start_with(input)
+          subject.tokens.each_with_index do |str, i|
+            expect(str).to be_kind_of(Rley::Lexical::Token)
+            expect(str.terminal).to eq('SYMBOL')
+            (lexeme, val) = expectations[i]
+            expect(str.lexeme).to eq(lexeme)
+          end
+        end
+
+        it 'should recognize an optional symbol' do
+          input = "RETURN expression? SEMICOLON"
+          expectations = [
+            ['RETURN', 'SYMBOL'],
+            ['expression', 'SYMBOL'],
+            ['?', 'QUESTION_MARK'],
+            ['SEMICOLON', 'SYMBOL'],
+          ]
+
+          subject.start_with(input)
+          subject.tokens.each_with_index do |str, i|
+            expect(str).to be_kind_of(Rley::Lexical::Token)
+            (lexeme, token) = expectations[i]
+            expect(str.lexeme).to eq(lexeme)
+            expect(str.terminal).to eq(token)
+          end
+        end
+
+        it 'should recognize a symbol with a star quantifier' do
+          input = "declaration* EOF"
+          expectations = [
+            ['declaration', 'SYMBOL'],
+            ['*', 'STAR'],
+            ['EOF', 'SYMBOL'],
+          ]
+
+          subject.start_with(input)
+          subject.tokens.each_with_index do |str, i|
+            expect(str).to be_kind_of(Rley::Lexical::Token)
+            (lexeme, token) = expectations[i]
+            expect(str.lexeme).to eq(lexeme)
+            expect(str.terminal).to eq(token)
+          end
+        end
+
+        it 'should recognize a symbol with a plus quantifier' do
+          input = "declaration+ EOF"
+          expectations = [
+            ['declaration', 'SYMBOL'],
+            ['+', 'PLUS'],
+            ['EOF', 'SYMBOL'],
+          ]
+
+          subject.start_with(input)
+          subject.tokens.each_with_index do |str, i|
+            expect(str).to be_kind_of(Rley::Lexical::Token)
+            (lexeme, token) = expectations[i]
+            expect(str.lexeme).to eq(lexeme)
+            expect(str.terminal).to eq(token)
+          end
+        end
+
+        it 'should recognize a grouping with a quantifier' do
+          input = "IF ifCondition statement (ELSE statement)?"
+          expectations = [
+            ['IF', 'SYMBOL'],
+            ['ifCondition', 'SYMBOL'],
+            ['statement', 'SYMBOL'],
+            ['(', 'LEFT_PAREN'],
+            ['ELSE', 'SYMBOL'],
+            ['statement', 'SYMBOL'],
+            [')', 'RIGHT_PAREN'],
+            ['?', 'QUESTION_MARK']
+          ]
+
+          subject.start_with(input)
+          subject.tokens.each_with_index do |str, i|
+            expect(str).to be_kind_of(Rley::Lexical::Token)
+            (lexeme, token) = expectations[i]
+            expect(str.lexeme).to eq(lexeme)
+            expect(str.terminal).to eq(token)
+          end
+        end
+
+        it 'should recognize a match closest constraint' do
+          input = "IF ifCondition statement ELSE { match_closest: 'IF' } statement"
+          expectations = [
+            ['IF', 'SYMBOL'],
+            ['ifCondition', 'SYMBOL'],
+            ['statement', 'SYMBOL'],
+            ['ELSE', 'SYMBOL'],
+            ['{', 'LEFT_BRACE'],
+            ['match_closest', 'KEY'],
+            ['IF', 'STR_LIT'],
+            ['}', 'RIGHT_BRACE'],
+            ['statement', 'SYMBOL']
+          ]
+
+          subject.start_with(input)
+          subject.tokens.each_with_index do |str, i|
+            expect(str).to be_kind_of(Rley::Lexical::Token)
+            (lexeme, token) = expectations[i]
+            expect(str.lexeme).to eq(lexeme)
+            expect(str.terminal).to eq(token)
+          end
+        end
+
+        it 'should recognize a repeat constraint' do
+          input = "IF ifCondition statement { repeat: 1 }  ELSE statement"
+          expectations = [
+            ['IF', 'SYMBOL'],
+            ['ifCondition', 'SYMBOL'],
+            ['statement', 'SYMBOL'],
+            ['{', 'LEFT_BRACE'],
+            ['repeat', 'KEY'],
+            ['1', 'INT_LIT'],
+            ['}', 'RIGHT_BRACE'],
+            ['ELSE', 'SYMBOL'],
+            ['statement', 'SYMBOL']
+          ]
+
+          subject.start_with(input)
+          subject.tokens.each_with_index do |str, i|
+            expect(str).to be_kind_of(Rley::Lexical::Token)
+            (lexeme, token) = expectations[i]
+            expect(str.lexeme).to eq(lexeme)
+            expect(str.terminal).to eq(token)
+          end
+        end
+
+        it 'should recognize a grouping with a repeat constraint' do
+          input = "IF ifCondition statement ( ELSE statement ){ repeat: 0..1 }"
+          expectations = [
+            ['IF', 'SYMBOL'],
+            ['ifCondition', 'SYMBOL'],
+            ['statement', 'SYMBOL'],
+            ['(', 'LEFT_PAREN'],
+            ['ELSE', 'SYMBOL'],
+            ['statement', 'SYMBOL'],
+            [')', 'RIGHT_PAREN'],
+            ['{', 'LEFT_BRACE'],
+            ['repeat', 'KEY'],
+            ['0', 'INT_LIT'],
+            ['..', 'ELLIPSIS'],
+            ['1', 'INT_LIT'],
+            ['}', 'RIGHT_BRACE']
+          ]
+
+          subject.start_with(input)
+          subject.tokens.each_with_index do |str, i|
+            expect(str).to be_kind_of(Rley::Lexical::Token)
+            (lexeme, token) = expectations[i]
+            expect(str.lexeme).to eq(lexeme)
+            expect(str.terminal).to eq(token)
+          end
+        end
+
+        it 'should recognize a combination of constraints' do
+          input = "IF ifCondition statement ELSE { repeat: 1, match_closest: 'IF' } statement"
+          expectations = [
+            ['IF', 'SYMBOL'],
+            ['ifCondition', 'SYMBOL'],
+            ['statement', 'SYMBOL'],
+            ['ELSE', 'SYMBOL'],
+            ['{', 'LEFT_BRACE'],
+            ['repeat', 'KEY'],
+            ['1', 'INT_LIT'],
+            [',', 'COMMA'],
+            ['match_closest', 'KEY'],
+            ['IF', 'STR_LIT'],
+            ['}', 'RIGHT_BRACE'],
+            ['statement', 'SYMBOL']
+          ]
+
+          subject.start_with(input)
+          subject.tokens.each_with_index do |str, i|
+            expect(str).to be_kind_of(Rley::Lexical::Token)
+            (lexeme, token) = expectations[i]
+            expect(str.lexeme).to eq(lexeme)
+            expect(str.terminal).to eq(token)
+          end
+        end
+
+        it 'should recognize a grouping with a nested constraint' do
+          input = "IF ifCondition statement ( ELSE { match_closest: 'IF' } statement ){ repeat: 0..1 }"
+          expectations = [
+            ['IF', 'SYMBOL'],
+            ['ifCondition', 'SYMBOL'],
+            ['statement', 'SYMBOL'],
+            ['(', 'LEFT_PAREN'],
+            ['ELSE', 'SYMBOL'],
+            ['{', 'LEFT_BRACE'],
+            ['match_closest', 'KEY'],
+            ['IF', 'STR_LIT'],
+            ['}', 'RIGHT_BRACE'],
+            ['statement', 'SYMBOL'],
+            [')', 'RIGHT_PAREN'],
+            ['{', 'LEFT_BRACE'],
+            ['repeat', 'KEY'],
+            ['0', 'INT_LIT'],
+            ['..', 'ELLIPSIS'],
+            ['1', 'INT_LIT'],
+            ['}', 'RIGHT_BRACE']
+          ]
+
+          subject.start_with(input)
+          subject.tokens.each_with_index do |str, i|
+            expect(str).to be_kind_of(Rley::Lexical::Token)
+            (lexeme, token) = expectations[i]
+            expect(str.lexeme).to eq(lexeme)
+            expect(str.terminal).to eq(token)
+          end
+        end
+      end # context
+    end # describe
+  end # module
+end # module
+
+# End of file