rley 0.5.01 → 0.5.02

data/spec/rley/support/grammar_arr_int_helper.rb

@@ -1,3 +1,5 @@
+require 'strscan'
+
 # Load the builder class
 require_relative '../../../lib/rley/syntax/grammar_builder'
 require_relative '../../../lib/rley/tokens/token'
@@ -14,7 +16,7 @@ module GrammarArrIntHelper
       rule 'arr' => %w( [ sequence ] )
       rule 'sequence' => ['list']
       rule 'sequence' => []
-      rule 'list' => %w[list , integer]
+      rule 'list' => %w[list , integer]   # Right-recursive rule
       rule 'list' => 'integer'
     end
     builder
@@ -22,17 +24,25 @@ module GrammarArrIntHelper
 
   # Basic tokenizer for array of integers
   def arr_int_tokenizer(aText, aGrammar)
-    tokens = aText.scan(/\S+/).map do |lexeme|
-      case lexeme
-        when '[', ']', ','
-          terminal = aGrammar.name2symbol[lexeme]
-        when /^[-+]?\d+$/
-          terminal = aGrammar.name2symbol['integer']
-        else
-          msg = "Unknown input text '#{lexeme}'"
-          raise StandardError, msg
+    tokens = []
+    scanner = StringScanner.new(aText)
+
+    until scanner.eos? do
+      scanner.skip(/\s+/)
+      lexeme = scanner.scan(/[\[,\]]/)
+      if lexeme
+        terminal = aGrammar.name2symbol[lexeme]
+        tokens << Rley::Tokens::Token.new(lexeme, terminal)
+        next
+      end
+      lexeme = scanner.scan(/^[-+]?\d+/)
+      if lexeme
+        terminal = aGrammar.name2symbol['integer']
+        tokens << Rley::Tokens::Token.new(lexeme, terminal)
+      else
+        msg = "Unknown input text '#{lexeme}'"
+        raise StandardError, msg
       end
-      Rley::Tokens::Token.new(lexeme, terminal)
     end
 
     return tokens

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rley
 version: !ruby/object:Gem::Version
-  version: 0.5.01
+  version: 0.5.02
 platform: ruby
 authors:
 - Dimitri Geshef
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-08-20 00:00:00.000000000 Z
+date: 2017-10-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: coveralls
@@ -132,15 +132,26 @@ files:
 - appveyor.yml
 - examples/NLP/mini_en_demo.rb
 - examples/data_formats/JSON/cli_options.rb
+- examples/data_formats/JSON/json_ast_builder.rb
+- examples/data_formats/JSON/json_ast_nodes.rb
 - examples/data_formats/JSON/json_demo.rb
 - examples/data_formats/JSON/json_grammar.rb
 - examples/data_formats/JSON/json_lexer.rb
 - examples/data_formats/JSON/json_minifier.rb
 - examples/data_formats/JSON/json_parser.rb
-- examples/general/calc/calc_demo.rb
-- examples/general/calc/calc_grammar.rb
-- examples/general/calc/calc_lexer.rb
-- examples/general/calc/calc_parser.rb
+- examples/general/calc_iter1/calc_ast_builder.rb
+- examples/general/calc_iter1/calc_ast_nodes.rb
+- examples/general/calc_iter1/calc_demo.rb
+- examples/general/calc_iter1/calc_grammar.rb
+- examples/general/calc_iter1/calc_lexer.rb
+- examples/general/calc_iter1/calc_parser.rb
+- examples/general/calc_iter1/spec/calculator_spec.rb
+- examples/general/calc_iter2/calc_ast_builder.rb
+- examples/general/calc_iter2/calc_ast_nodes.rb
+- examples/general/calc_iter2/calc_demo.rb
+- examples/general/calc_iter2/calc_grammar.rb
+- examples/general/calc_iter2/calc_lexer.rb
+- examples/general/calc_iter2/calc_parser.rb
 - lib/rley.rb
 - lib/rley/constants.rb
 - lib/rley/formatter/asciitree.rb
@@ -226,6 +237,7 @@ files:
 - spec/rley/parse_forest_visitor_spec.rb
 - spec/rley/parse_tree_visitor_spec.rb
 - spec/rley/parser/ambiguous_parse_spec.rb
+- spec/rley/parser/ast_builder_spec.rb
 - spec/rley/parser/cst_builder_spec.rb
 - spec/rley/parser/dotted_item_spec.rb
 - spec/rley/parser/error_reason_spec.rb
@@ -240,7 +252,6 @@ files:
 - spec/rley/parser/parse_forest_factory_spec.rb
 - spec/rley/parser/parse_state_spec.rb
 - spec/rley/parser/parse_tracer_spec.rb
-- spec/rley/parser/parse_tree_builder_spec.rb
 - spec/rley/parser/parse_tree_factory_spec.rb
 - spec/rley/parser/parse_walker_factory_spec.rb
 - spec/rley/parser/state_set_spec.rb
@@ -296,7 +307,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.6.7
+rubygems_version: 2.6.13
 signing_key: 
 specification_version: 4
 summary: Ruby implementation of the Earley's parsing algorithm
@@ -318,6 +329,7 @@ test_files:
 - spec/rley/gfg/start_vertex_spec.rb
 - spec/rley/gfg/vertex_spec.rb
 - spec/rley/parser/ambiguous_parse_spec.rb
+- spec/rley/parser/ast_builder_spec.rb
 - spec/rley/parser/cst_builder_spec.rb
 - spec/rley/parser/dotted_item_spec.rb
 - spec/rley/parser/error_reason_spec.rb
@@ -332,7 +344,6 @@ test_files:
 - spec/rley/parser/parse_forest_factory_spec.rb
 - spec/rley/parser/parse_state_spec.rb
 - spec/rley/parser/parse_tracer_spec.rb
-- spec/rley/parser/parse_tree_builder_spec.rb
 - spec/rley/parser/parse_tree_factory_spec.rb
 - spec/rley/parser/parse_walker_factory_spec.rb
 - spec/rley/parser/state_set_spec.rb

data/examples/general/calc/calc_lexer.rb

@@ -1,90 +0,0 @@
-# File: calc_lexer.rb
-# Lexer for a basic arithmetical expression parser
-require 'strscan'
-require 'rley' # Load the gem
-
-
-class CalcLexer
-  attr_reader(:scanner)
-  attr_reader(:lineno)
-  attr_reader(:line_start)
-  attr_reader(:name2symbol)
-
-  @@lexeme2name = {
-    '(' => 'LPAREN',
-    ')' => 'RPAREN',
-    '+' => 'PLUS',
-    '-' => 'MINUS',
-    '*' => 'STAR',
-    '/' => 'DIVIDE'
-  }.freeze
-
-  class ScanError < StandardError; end
-
-  def initialize(source, aGrammar)
-    @scanner = StringScanner.new(source)
-    @name2symbol = aGrammar.name2symbol
-    @lineno = 1
-  end
-
-  def tokens()
-    tok_sequence = []
-    until @scanner.eos?
-      token = _next_token
-      tok_sequence << token unless token.nil?
-    end
-
-    return tok_sequence
-  end
-
-  private
-
-  def _next_token()
-    token = nil
-    skip_whitespaces
-    curr_ch = scanner.getch # curr_ch is at start of token or eof reached...
-
-    loop do
-      break if curr_ch.nil?
-
-      case curr_ch
-        when '(', ')', '+', '-', '*', '/'
-          type_name = @@lexeme2name[curr_ch]
-          token_type = name2symbol[type_name]
-          token = Rley::Tokens::Token.new(curr_ch, token_type)
-
-        # LITERALS
-        when /[-0-9]/ # Start character of number literal found
-          @scanner.pos = scanner.pos - 1 # Simulate putback
-          value = scanner.scan(/-?[0-9]+(\.[0-9]+)?([eE][-+]?[0-9])?/)
-          token_type = name2symbol['NUMBER']
-          token = Rley::Tokens::Token.new(value, token_type)
-
-
-        else # Unknown token
-          erroneous = curr_ch.nil? ? '' : curr_ch
-          sequel = scanner.scan(/.{1,20}/)
-          erroneous += sequel unless sequel.nil?
-          raise ScanError.new("Unknown token #{erroneous}")
-      end # case
-
-      break unless token.nil? && (curr_ch = scanner.getch)
-    end 
-
-    return token
-  end
-
-  def skip_whitespaces()
-    matched = scanner.scan(/[ \t\f\n\r]+/)
-    return if matched.nil?
-
-    newline_count = 0
-    matched.scan(/\n\r?|\r/) { |_| newline_count += 1 }
-    newline_detected(newline_count)
-  end
-
-  def newline_detected(count)
-    @lineno += count
-    @line_start = scanner.pos
-  end
-end # class

data/spec/rley/parser/parse_tree_builder_spec.rb

@@ -1,249 +0,0 @@
-require_relative '../../spec_helper'
-
-require_relative '../../../lib/rley/parser/gfg_earley_parser'
-require_relative '../../../lib/rley/parser/parse_walker_factory'
-
-require_relative '../support/expectation_helper'
-require_relative '../support/grammar_b_expr_helper'
-
-# Load the class under test
-require_relative '../../../lib/rley/parser/parse_tree_builder'
-
-module Rley # Open this namespace to avoid module qualifier prefixes
-  module Parser
-    describe ParseTreeBuilder do
-      include ExpectationHelper # Mix-in with expectation on parse entry sets
-      include GrammarBExprHelper # Mix-in for basic arithmetic language
-
-      let(:sample_grammar) do
-          builder = grammar_expr_builder
-          builder.grammar
-      end
-
-      let(:sample_tokens) do
-        expr_tokenizer('2 + 3 * 4', sample_grammar)
-      end
-
-      let(:sample_result) do
-        parser = Parser::GFGEarleyParser.new(sample_grammar)
-        parser.parse(sample_tokens)
-      end
-
-      subject { ParseTreeBuilder.new(sample_tokens) }
-
-      # Emit a text representation of the current path.
-      def path_to_s()
-        text_parts = subject.curr_path.map(&:to_s)
-        return text_parts.join('/')
-      end
-
-      def next_event(eventType, anEntryText)
-        event = @walker.next
-        subject.receive_event(*event)
-        expect(event[0]).to eq(eventType)
-        expect(event[1].to_s).to eq(anEntryText)
-      end
-
-      def expected_curr_parent(anExpectation)
-        expect(subject.curr_parent.to_string(0)).to eq(anExpectation)
-      end
-
-      def expected_curr_path(anExpectation)
-        expect(path_to_s).to eq(anExpectation)
-      end
-
-      def expected_first_child(anExpectation)
-          child = subject.curr_parent.subnodes.first
-          expect(child.to_string(0)).to eq(anExpectation)
-      end
-
-      context 'Initialization:' do
-        it 'should be created with a sequence of tokens' do
-          expect { ParseTreeBuilder.new(sample_tokens) }.not_to raise_error
-        end
-
-        it 'should know the input tokens' do
-          expect(subject.tokens).to eq(sample_tokens)
-        end
-
-        it 'should have an empty path' do
-          expect(subject.curr_path).to be_empty
-        end
-      end # context
-
-      context 'Parse tree construction:' do
-        before(:each) do
-          factory = ParseWalkerFactory.new
-          accept_entry = sample_result.accepting_entry
-          accept_index = sample_result.chart.last_index
-          @walker = factory.build_walker(accept_entry, accept_index)
-        end
-
-        it 'should initialize the root node' do
-          next_event(:visit, 'P. | 0')
-          tree = subject.result
-
-          expect(tree.root.to_string(0)).to eq('P[0, 5]')
-          expected_curr_path('P[0, 5]')
-        end
-
-        it 'should initialize the first child of the root node' do
-          next_event(:visit, 'P. | 0') # Event 1
-          next_event(:visit, 'P => S . | 0') # Event 2
-          next_event(:visit, 'S. | 0') # Event 3
-          next_event(:visit, 'S => S + M . | 0') # Event 4
-          expected_curr_path('P[0, 5]/S[0, 5]')
-          next_event(:visit, 'M. | 2') # Event 5
-          expected_curr_path('P[0, 5]/S[0, 5]/M[2, 5]')
-          next_event(:visit, 'M => M * T . | 2') # Event 6
-          next_event(:visit, 'T. | 4') # Event 7
-          expected_curr_path('P[0, 5]/S[0, 5]/M[2, 5]/T[4, 5]')
-          next_event(:visit, 'T => integer . | 4') # Event 8
-        end
-
-        it 'should build token node when scan edge was detected' do
-          8.times do
-            event = @walker.next
-            subject.receive_event(*event)
-          end
-
-          next_event(:visit, 'T => . integer | 4') # Event 9
-          expected_curr_path('P[0, 5]/S[0, 5]/M[2, 5]/T[4, 5]')
-          expected_first_child("integer[4, 5]: '4'")
-          expect(subject.curr_parent.subnodes.size).to eq(1)
-        end
-
-        it 'should handle the remaining events' do
-          9.times do
-            event = @walker.next
-            subject.receive_event(*event)
-          end
-
-          next_event(:visit, '.T | 4') # Event 10
-          expected_curr_path('P[0, 5]/S[0, 5]/M[2, 5]')
-
-          next_event(:visit, 'M => M * . T | 2') # Event 11
-
-          next_event(:visit, 'M => M . * T | 2') # Event 12
-          expected_curr_path('P[0, 5]/S[0, 5]/M[2, 5]')
-          expect(subject.curr_parent.subnodes.size).to eq(2)
-          expected_first_child("*[3, 4]: '*'")
-
-          next_event(:visit, 'M. | 2') # Event 13
-          expected_curr_path('P[0, 5]/S[0, 5]/M[2, 5]/M[2, 3]')
-
-          next_event(:visit, 'M => T . | 2') # Event 14
-          expected_curr_path('P[0, 5]/S[0, 5]/M[2, 5]/M[2, 3]')
-
-          next_event(:visit, 'T. | 2') # Event 15
-          expected_curr_path('P[0, 5]/S[0, 5]/M[2, 5]/M[2, 3]/T[2, 3]')
-
-          next_event(:visit, 'T => integer . | 2') # Event 16
-          expected_curr_path('P[0, 5]/S[0, 5]/M[2, 5]/M[2, 3]/T[2, 3]')
-          expect(subject.curr_parent.subnodes.size).to eq(1)
-          expected_first_child("integer[2, 3]: '3'")
-
-          next_event(:visit, 'T => . integer | 2') # Event 17
-
-          next_event(:visit, '.T | 2') # Event 18
-          expected_curr_path('P[0, 5]/S[0, 5]/M[2, 5]/M[2, 3]')
-
-          next_event(:visit, 'M => . T | 2') # Event 19
-          expected_curr_path('P[0, 5]/S[0, 5]/M[2, 5]/M[2, 3]')
-
-          next_event(:visit, '.M | 2') # Event 20
-          expected_curr_path('P[0, 5]/S[0, 5]/M[2, 5]')
-
-          next_event(:visit, 'M => . M * T | 2') # Event 21
-          expected_curr_path('P[0, 5]/S[0, 5]/M[2, 5]')
-
-          next_event(:revisit, '.M | 2') # Revisit Event 22
-          expected_curr_path('P[0, 5]/S[0, 5]')
-
-          next_event(:visit, 'S => S + . M | 0') # Event 23
-          expected_curr_path('P[0, 5]/S[0, 5]')
-
-          next_event(:visit, 'S => S . + M | 0') # Event 24
-          expected_curr_path('P[0, 5]/S[0, 5]')
-          expect(subject.curr_parent.subnodes.size).to eq(2)
-          expected_first_child("+[1, 2]: '+'")
-
-          next_event(:visit, 'S. | 0') # Event 25
-          expected_curr_path('P[0, 5]/S[0, 5]/S[0, 1]')
-
-          next_event(:visit, 'S => M . | 0') # Event 26
-          expected_curr_path('P[0, 5]/S[0, 5]/S[0, 1]')
-
-          next_event(:visit, 'M. | 0') # Event 27
-          expected_curr_path('P[0, 5]/S[0, 5]/S[0, 1]/M[0, 1]')
-
-          next_event(:visit, 'M => T . | 0') # Event 28
-          expected_curr_path('P[0, 5]/S[0, 5]/S[0, 1]/M[0, 1]')
-
-          next_event(:visit, 'T. | 0') # Event 29
-          expected_curr_path('P[0, 5]/S[0, 5]/S[0, 1]/M[0, 1]/T[0, 1]')
-
-          next_event(:visit, 'T => integer . | 0') # Event 30
-          expected_curr_path('P[0, 5]/S[0, 5]/S[0, 1]/M[0, 1]/T[0, 1]')
-
-          next_event(:visit, 'T => . integer | 0') # Event 31
-          expected_curr_path('P[0, 5]/S[0, 5]/S[0, 1]/M[0, 1]/T[0, 1]')
-          expect(subject.curr_parent.subnodes.size).to eq(1)
-          expected_first_child("integer[0, 1]: '2'")
-
-          next_event(:visit, '.T | 0') # Event 32
-          expected_curr_path('P[0, 5]/S[0, 5]/S[0, 1]/M[0, 1]')
-
-          next_event(:visit, 'M => . T | 0') # Event 33
-          expected_curr_path('P[0, 5]/S[0, 5]/S[0, 1]/M[0, 1]')
-
-          next_event(:visit, '.M | 0') # Event 34
-          expected_curr_path('P[0, 5]/S[0, 5]/S[0, 1]')
-
-          next_event(:visit, 'S => . M | 0') # Event 35
-          expected_curr_path('P[0, 5]/S[0, 5]/S[0, 1]')
-
-          next_event(:visit, '.S | 0') # Event 36
-          expected_curr_path('P[0, 5]/S[0, 5]')
-
-          next_event(:visit, 'S => . S + M | 0') # Event 37
-          expected_curr_path('P[0, 5]/S[0, 5]')
-
-          next_event(:revisit, '.S | 0') # Event 38
-          expected_curr_path('P[0, 5]')
-
-          next_event(:visit, 'P => . S | 0') # Event 39
-          expected_curr_path('P[0, 5]')
-
-          next_event(:visit, '.P | 0') # Event 39
-          expect(path_to_s).to be_empty
-        end
-
-        it 'should build parse trees' do
-          loop do
-            event = @walker.next
-            subject.receive_event(*event)
-            break if path_to_s.empty?
-          end
-
-          # Lightweight sanity check
-          expect(subject.result).not_to be_nil
-          expect(subject.result).to be_kind_of(PTree::ParseTree)
-          expect(subject.result.root.to_s).to eq('P[0, 5]')
-          expect(subject.result.root.subnodes.size).to eq(1)
-          child_node = subject.result.root.subnodes[0]
-          expect(child_node.to_s).to eq('S[0, 5]')
-          
-          expect(child_node.subnodes.size).to eq(3)
-          first_grandchild = child_node.subnodes[0]
-          expect(first_grandchild.to_s).to eq('S[0, 1]')
-          second_grandchild = child_node.subnodes[1]
-          expect(second_grandchild.to_s).to eq("+[1, 2]: '+'")
-          third_grandchild = child_node.subnodes[2]
-          expect(third_grandchild.to_s).to eq('M[2, 5]')
-        end
-      end # context
-    end # describe
-  end # module
-end # module
-# End of file