dendroid 0.0.9 → 0.0.11

data/lib/dendroid/syntax/grm_symbol.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 module Dendroid
-  # The namespace for all classes used to build a grammar.
+  # This module contains for all classes representing elements of .
   module Syntax
     # Abstract class for grammar symbols.
     # A grammar symbol is an element that appears in grammar rules.

data/lib/dendroid/utils/base_tokenizer.rb

@@ -5,44 +5,67 @@ require_relative '../lexical/token_position'
 require_relative '../lexical/literal'
 
 module Dendroid
+  # This module contains helper classes (e.g. a tokenizer generator)
   module Utils
+    # A basic tokenizer.
+    # Responsibility: break input into a sequence of token objects.
+    # This class defines a simple DSL to build a tokenizer.
     class BaseTokenizer
+      # @return [StringScanner] Low-level input scanner
       attr_reader :scanner
+
+      # @return [Integer] The current line number
       attr_reader :lineno
+
+      # @return [Integer] Position of last start of line in the input string
       attr_reader :line_start
+
+      # @return [Hash{Symbol, Array<Regexp>}]
       attr_reader :actions
 
+      # Constructor
+      # @param aBlock [Proc]
       def initialize(&aBlock)
         @scanner = StringScanner.new('')
         @actions = { skip: [], scan_verbatim: [], scan_value: [] }
         defaults
+        return unless block_given?
 
-        if block_given?
-          instance_exec(&aBlock)
-          # grammar_complete!
-        end
+        instance_exec(&aBlock)
       end
 
+      # Reset the tokenizer and set new text to tokenize
+      # @param source [String]
       def input=(source)
-        scanner.string = source
         reset
+        scanner.string = source
       end
 
+      # Reset the tokenizer
       def reset
         @lineno = 1
         @line_start = 0
+        scanner.reset
       end
 
       # action, pattern, terminal?, conversion?
       # action = skip, skip_nl, scan
+
+      # Associate the provided pattern to the action of skipping a newline and
+      # incrementing the line counter.
+      # @param pattern [Regexp]
       def skip_nl(pattern)
         actions[:skip_nl] = pattern
       end
 
+      # Associate the provided pattern with the action to skip whitespace(s).
+      # @param pattern [Regexp]
       def skip_ws(pattern)
         actions[:skip_ws] = pattern
       end
 
+      # Associate the provided pattern with the action to skip the matching text.
+      # @param pattern [Regexp]
       def skip(pattern)
         if actions[:skip].empty?
           actions[:skip] = pattern
@@ -52,6 +75,8 @@ module Dendroid
         end
       end
 
+      # Associate the provided pattern with the action to tokenize the matching text
+      # @param pattern [Regexp]
       def scan_verbatim(pattern)
         patt = normalize_pattern(pattern)
         if actions[:scan_verbatim].empty?
@@ -62,9 +87,15 @@ module Dendroid
         end
       end
 
-      def scan_value(pattern, terminal, convertion)
+      # Associate the provided pattern with the action to tokenize the matching text
+      # as an instance of the given terminal symbol and convert the matching text into
+      # a value by using the given conversion.
+      # @param pattern [Regexp]
+      # @param terminal [Dendroid::Syntax::Terminal]
+      # @param conversion [Proc] a Proc (lambda) that takes a String as argument and return a value.
+      def scan_value(pattern, terminal, conversion)
         patt = normalize_pattern(pattern)
-        tuple = [patt, terminal, convertion]
+        tuple = [patt, terminal, conversion]
         if actions[:scan_value].empty?
           actions[:scan_value] = [tuple]
         else
@@ -72,10 +103,16 @@ module Dendroid
         end
       end
 
+      # Set the mapping between a verbatim text to its corresponding terminal symbol name
+      # @param mapping [Hash{String, String}]
       def map_verbatim2terminal(mapping)
         @verbatim2terminal = mapping
       end
 
+      # rubocop: disable Metrics/AbcSize
+
+      # Return the next token (if any) from the input stream.
+      # @return [Dendroid::Lexical::Token, NilClass]
       def next_token
         token = nil
 
@@ -93,7 +130,7 @@ module Dendroid
             break
           end
 
-          tuple = actions[:scan_value].find do |(pattern, terminal, conversion)|
+          tuple = actions[:scan_value].find do |(pattern, _terminal, _conversion)|
             scanner.check(pattern)
           end
           if tuple
@@ -106,18 +143,20 @@ module Dendroid
           # Unknown token
           col = scanner.pos - line_start + 1
           erroneous = scanner.peek(1).nil? ? '' : scanner.scan(/./)
-          raise Exception, "Error: [line #{lineno}:#{col}]: Unexpected character #{erroneous}."
+          raise StandardError, "Error: [line #{lineno}:#{col}]: Unexpected character #{erroneous}."
         end
 
         token
       end
 
+      # rubocop: enable Metrics/AbcSize
+
       protected
 
       def defaults
         # Defaults
-        skip_nl /(?:\r\n)|\r|\n/  # Skip newlines
-        skip_ws /[ \t\f]+/ # Skip blanks
+        skip_nl(/(?:\r\n)|\r|\n/) # Skip newlines
+        skip_ws(/[ \t\f]+/) # Skip blanks
       end
 
       private
@@ -146,7 +185,7 @@ module Dendroid
           col = scanner.pos - lex_length - @line_start + 1
           pos = Lexical::TokenPosition.new(@lineno, col)
           token = Lexical::Token.new(text, pos, symbol_name)
-        rescue Exception => e
+        rescue StandardError => e
           puts "Failing with '#{symbol_name}' and '#{text}'"
           raise e
         end
@@ -154,15 +193,15 @@ module Dendroid
         token
       end
 
-      def value_scanned(aText, aSymbolName, convertion)
-        value = convertion.call(aText)
+      def value_scanned(aText, aSymbolName, conversion)
+        value = conversion.call(aText)
         lex_length = aText ? aText.size : 0
         col = scanner.pos - lex_length - @line_start + 1
         build_literal(aSymbolName, value, aText, col)
       end
 
       def build_literal(aSymbolName, aValue, aText, aPosition)
-        pos = if aPosition.kind_of?(Integer)
+        pos = if aPosition.is_a?(Integer)
                 col = aPosition
                 Lexical::TokenPosition.new(@lineno, col)
               else

data/spec/dendroid/grm_analysis/grm_analyzer_spec.rb

@@ -1,79 +1,9 @@
 # frozen_string_literal: true
 
 require_relative '../../spec_helper'
-require_relative '../../../lib/dendroid/grm_dsl/base_grm_builder'
+require_relative '../support/sample_grammars'
 require_relative '../../../lib/dendroid/grm_analysis/grm_analyzer'
 
-module SampleGrammars
-  def grammar_l1
-    builder = Dendroid::GrmDSL::BaseGrmBuilder.new do
-      # Grammar inspired from Wikipedia entry on Earley parsing
-      declare_terminals('PLUS', 'STAR', 'INTEGER')
-
-      rule('p' => 's')
-      rule('s' => ['s PLUS m', 'm'])
-      rule('m' => ['m STAR t', 't'])
-      rule('t' => 'INTEGER')
-    end
-
-    builder.grammar
-  end
-
-  def tokenizer_l1
-    Utils::BaseTokenizer.new do
-      map_verbatim2terminal({ '+' => :PLUS, '*' => :STAR })
-
-      scan_verbatim(['+', '*'])
-      scan_value(/\d+/, :INTEGER, ->(txt) { txt.to_i })
-    end
-  end
-
-  def grammar_l2
-    builder = GrmDSL::BaseGrmBuilder.new do
-      # Grammar inspired from Loup Vaillant's example
-      # https://loup-vaillant.fr/tutorials/earley-parsing/recogniser
-      declare_terminals('PLUS', 'MINUS',  'STAR', 'SLASH')
-      declare_terminals('LPAREN', 'RPAREN', 'NUMBER')
-
-      rule('p' => 'sum')
-      rule('sum' => ['sum PLUS product', 'sum MINUS product', 'product'])
-      rule('product' => ['product STAR factor', 'product SLASH factor', 'factor'])
-      rule('factor' => ['LPAREN sum RPAREN', 'NUMBER'])
-    end
-
-    builder.grammar
-  end
-
-  def tokenizer_l2
-    Utils::BaseTokenizer.new do
-      map_verbatim2terminal({
-                              '+' => :PLUS,
-                              '-' => :MINUS,
-                              '*' => :STAR,
-                              '/' => :SLASH,
-                              '(' => :LPAREN,
-                              ')' => :RPAREN })
-
-      scan_verbatim(['+', '-', '*', '/', '(', ')'])
-      scan_value(/\d+/, :NUMBER, ->(txt) { txt.to_i })
-    end
-  end
-
-  def grammar_l3
-    builder = Dendroid::GrmDSL::BaseGrmBuilder.new do
-      # Grammar inspired from Andrew Appel's example
-      # Modern Compiler Implementation in Java
-      declare_terminals('a', 'c', 'd')
-
-      rule('Z' => ['d', 'X Y Z'])
-      rule('Y' => ['', 'c'])
-      rule('X' => ['Y', 'a'])
-    end
-
-    builder.grammar
-  end
-end # module
-
 describe Dendroid::GrmAnalysis::GrmAnalyzer do
   include SampleGrammars
   let(:grammar) { grammar_l1 }
@@ -91,7 +21,7 @@ describe Dendroid::GrmAnalysis::GrmAnalyzer do
 
     it 'knows the dotted items' do
       item_count = subject.grammar.rules.reduce(0) do |count, prod|
-        count += prod.items.flatten.size
+        count + prod.items.flatten.size
       end
       expect(subject.items.size).to eq(item_count)
       expected_items = [
@@ -116,7 +46,7 @@ describe Dendroid::GrmAnalysis::GrmAnalyzer do
     end
 
     it 'knows the item that follows a given dotted item' do
-      first_item = subject.items.find { |itm| itm.to_s ==  'm => . m STAR t' }
+      first_item = subject.items.find { |itm| itm.to_s == 'm => . m STAR t' }
       second = subject.next_item(first_item)
       expect(second.to_s).to eq('m => m . STAR t')
       third = subject.next_item(second)
@@ -134,9 +64,9 @@ describe Dendroid::GrmAnalysis::GrmAnalyzer do
         'a' => ['a'],
         'c' => ['c'],
         'd' => ['d'],
-        'X' => ['a', 'c'], # Add epsilon
+        'X' => %w[a c], # Add epsilon
         'Y' => ['c'], # Add epsilon
-        'Z' => ['a', 'c', 'd']
+        'Z' => %w[a c d]
       }
       expectations.each_pair do |sym_name, first_names|
         symb = subject.grammar.name2symbol[sym_name]
@@ -149,8 +79,8 @@ describe Dendroid::GrmAnalysis::GrmAnalyzer do
     it 'constructs the FOLLOW sets for non-terminal symbols' do
       expectations = {
         'Z' => [], # Add $$
-        'Y' => ['a', 'c', 'd'],
-        'X' => ['a', 'c', 'd']
+        'Y' => %w[a c d],
+        'X' => %w[a c d]
       }
       subject.send(:build_follow_sets)
       expectations.each_pair do |sym_name, follow_names|

data/spec/dendroid/lexical/token_position_spec.rb

@@ -28,4 +28,4 @@ describe Dendroid::Lexical::TokenPosition do
       expect(subject.to_s).to eq("#{ex_lineno}:#{ex_column}")
     end
   end # context
-end # describe
+end # describe

data/spec/dendroid/recognizer/chart_spec.rb

@@ -0,0 +1,2 @@
+# frozen_string_literal: true
+

data/spec/dendroid/recognizer/e_item_spec.rb

@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+require_relative '../../spec_helper'
+require_relative '../../../lib/dendroid/syntax/terminal'
+require_relative '../../../lib/dendroid/syntax/non_terminal'
+require_relative '../../../lib/dendroid/syntax/symbol_seq'
+require_relative '../../../lib/dendroid/syntax/production'
+require_relative '../../../lib/dendroid/grm_analysis/dotted_item'
+require_relative '../../../lib/dendroid/recognizer/e_item'
+
+describe Dendroid::Recognizer::EItem do
+  let(:num_symb) { Dendroid::Syntax::Terminal.new('NUMBER') }
+  let(:plus_symb) { Dendroid::Syntax::Terminal.new('PLUS') }
+  let(:expr_symb) { Dendroid::Syntax::NonTerminal.new('expression') }
+  let(:rhs) { Dendroid::Syntax::SymbolSeq.new([num_symb, plus_symb, num_symb]) }
+  let(:empty_body) { Dendroid::Syntax::SymbolSeq.new([]) }
+  let(:prod) { Dendroid::Syntax::Production.new(expr_symb, rhs) }
+  let(:empty_prod) { Dendroid::Syntax::Production.new(expr_symb, empty_body) }
+  let(:sample_dotted) { Dendroid::GrmAnalysis::DottedItem.new(prod, 1) }
+  let(:other_dotted) { Dendroid::GrmAnalysis::DottedItem.new(empty_prod, 0) }
+  let(:sample_origin) { 3 }
+
+  subject { described_class.new(sample_dotted, sample_origin) }
+
+  context 'Initialization:' do
+    it 'is initialized with a dotted item and an origin position' do
+      expect { described_class.new(sample_dotted, sample_origin) }.not_to raise_error
+    end
+
+    it 'knows its related dotted item' do
+      expect(subject.dotted_item).to eq(sample_dotted)
+    end
+
+    it 'knows its origin value' do
+      expect(subject.origin).to eq(sample_origin)
+    end
+  end # context
+
+  context 'Provided service:' do
+    it 'knows the lhs of related production' do
+      expect(subject.lhs).to eq(expr_symb)
+    end # context
+
+    it 'can compare with another EItem' do
+      expect(subject == subject).to be_truthy
+      expect(subject == described_class.new(sample_dotted, sample_origin)).to be_truthy
+      expect(subject == described_class.new(sample_dotted, 2)).to be_falsey
+      expect(subject == described_class.new(other_dotted, sample_origin)).to be_falsey
+    end
+
+    it 'can renders a String representation of itself' do
+      expect(subject.to_s).to eq("#{sample_dotted} @ #{sample_origin}")
+    end
+  end # context
+end # describe

data/spec/dendroid/recognizer/item_set_spec.rb

@@ -0,0 +1,63 @@
+# frozen_string_literal: true
+
+require_relative '../../spec_helper'
+require_relative '../../../lib/dendroid/syntax/terminal'
+require_relative '../../../lib/dendroid/syntax/non_terminal'
+require_relative '../../../lib/dendroid/syntax/symbol_seq'
+require_relative '../../../lib/dendroid/syntax/production'
+require_relative '../../../lib/dendroid/grm_analysis/dotted_item'
+require_relative '../../../lib/dendroid/recognizer/e_item'
+require_relative '../../../lib/dendroid/recognizer/item_set'
+
+describe Dendroid::Recognizer::ItemSet do
+  let(:num_symb) { Dendroid::Syntax::Terminal.new('NUMBER') }
+  let(:plus_symb) { Dendroid::Syntax::Terminal.new('PLUS') }
+  let(:expr_symb) { Dendroid::Syntax::NonTerminal.new('expression') }
+  let(:rhs) { Dendroid::Syntax::SymbolSeq.new([num_symb, plus_symb, num_symb]) }
+  let(:empty_body) { Dendroid::Syntax::SymbolSeq.new([]) }
+  let(:prod) { Dendroid::Syntax::Production.new(expr_symb, rhs) }
+  let(:empty_prod) { Dendroid::Syntax::Production.new(expr_symb, empty_body) }
+  let(:sample_dotted) { Dendroid::GrmAnalysis::DottedItem.new(prod, 1) }
+  let(:sample_origin) { 3 }
+  let(:other_dotted) { Dendroid::GrmAnalysis::DottedItem.new(empty_prod, 0) }
+  let(:first_element) { Dendroid::Recognizer::EItem.new(sample_dotted, sample_origin) }
+  let(:second_element) { Dendroid::Recognizer::EItem.new(other_dotted, 5) }
+
+  subject { described_class.new }
+
+  context 'Initialization:' do
+    it 'is initialized without argument' do
+      expect { described_class.new }.not_to raise_error
+    end
+
+    it 'is empty at creation' do
+      expect(subject).to be_empty
+    end
+  end # context
+
+  context 'Provided services:' do
+    it 'adds a new element' do
+      subject.add_item(first_element)
+      expect(subject.size).to eq(1)
+
+      # Trying a second time, doesn't change the set
+      subject.add_item(first_element)
+      expect(subject.size).to eq(1)
+
+      subject.add_item(second_element)
+      expect(subject.size).to eq(2)
+    end
+
+    it 'can render a String representation of itself' do
+      subject.add_item(first_element)
+      subject.add_item(second_element)
+
+      expectations = [
+        'expression => NUMBER . PLUS NUMBER @ 3',
+        'expression => . @ 5'
+      ].join("\n")
+
+      expect(subject.to_s).to eq(expectations)
+    end
+  end # context
+end # describe

data/spec/dendroid/recognizer/recognizer_spec.rb

@@ -0,0 +1,186 @@
+# frozen_string_literal: true
+
+require_relative '../../spec_helper'
+require_relative '../support/sample_grammars'
+require_relative '../../../lib/dendroid/recognizer/recognizer'
+
+describe Dendroid::Recognizer::Recognizer do
+  include SampleGrammars
+  let(:grammar1) { grammar_l1 }
+
+  # Implements a dotted item: expression => NUMBER . PLUS NUMBER
+  subject { described_class.new(grammar1, tokenizer_l1) }
+
+  context 'Initialization:' do
+    it 'is initialized with a grammar' do
+      expect { described_class.new(grammar1, tokenizer_l1) }.not_to raise_error
+    end
+
+    it 'knows its grammar analyzer' do
+      expect(subject.grm_analysis).to be_kind_of(Dendroid::GrmAnalysis::GrmAnalyzer)
+      expect(subject.grm_analysis.grammar).to eq(grammar1)
+    end
+
+    it 'knows its tokenizer' do
+      expect(subject.grm_analysis).to be_kind_of(Dendroid::GrmAnalysis::GrmAnalyzer)
+      expect(subject.grm_analysis.grammar).to eq(grammar1)
+    end
+  end # context
+
+  context 'Recognizer at work:' do
+    it 'can recognize example from Wikipedia' do
+      chart = subject.run('2 + 3 * 4')
+      expect(chart).to be_successful
+
+      set0 = [ # . 2 + 3 * 4'
+        'p => . s @ 0',
+        's => . s PLUS m @ 0',
+        's => . m @ 0',
+        'm => . m STAR t @ 0',
+        'm => . t @ 0',
+        't => . INTEGER @ 0'
+      ]
+      set1 = [ # 2 . + 3 * 4'
+        't => INTEGER . @ 0',
+        'm => t . @ 0',
+        's => m . @ 0',
+        #'m => m . STAR t @ 0',
+        'p => s . @ 0',
+        's => s . PLUS m @ 0'
+      ]
+      set2 = [ # 2 + . 3 * 4'
+        's => s PLUS . m @ 0',
+        'm => . m STAR t @ 2',
+        'm => . t @ 2',
+        't => . INTEGER @ 2'
+      ]
+      set3 = [ # 2 + 3 . * 4'
+        't => INTEGER . @ 2',
+        'm => t . @ 2',
+        's => s PLUS m . @ 0',
+        'm => m . STAR t @ 2',
+        'p => s . @ 0',
+      # 's => s . PLUS m @ 0'
+      ]
+      set4 = [ # 2 + 3 * . 4'
+        'm => m STAR . t @ 2',
+        't => . INTEGER @ 4'
+      ]
+      set5 = [ # 2 + 3 * 4 .'
+        't => INTEGER . @ 4',
+        'm => m STAR t . @ 2',
+        's => s PLUS m . @ 0',
+        # 'm => m . STAR t @ 2',
+        'p => s . @ 0'
+      # 's => s . PLUS m @ 0'
+      ]
+      [set0, set1, set2, set3, set4, set5].each_with_index do |set, rank|
+        expect(chart[rank].to_s).to eq(set.join("\n"))
+      end
+    end
+
+    it 'can recognize example for L2 language' do
+      recognizer = described_class.new(grammar_l2, tokenizer_l2)
+      chart = recognizer.run('1 + (2 * 3 - 4)')
+      expect(chart).to be_successful
+
+      set0 = [ # . 1 + (2 * 3 - 4)
+        'p => . sum @ 0',
+        'sum => . sum PLUS product @ 0',
+        'sum => . sum MINUS product @ 0',
+        'sum => . product @ 0',
+        'product => . product STAR factor @ 0',
+        'product => . product SLASH factor @ 0',
+        'product => . factor @ 0',
+        # 'factor => . LPAREN sum RPAREN @ 0',
+        'factor => . NUMBER @ 0'
+      ]
+      set1 = [ # 1 . + (2 * 3 - 4)
+        'factor => NUMBER . @ 0',
+        'product => factor . @ 0',
+        'sum => product . @ 0',
+        # 'product => product . STAR factor @ 0',
+        # 'product => product . SLASH factor @ 0',
+        'p => sum . @ 0',
+        'sum => sum . PLUS product @ 0',
+      # 'sum => sum . MINUS product @ 0'
+      ]
+      set2 = [ # 1 + . (2 * 3 - 4)
+        'sum => sum PLUS . product @ 0',
+        'product => . product STAR factor @ 2',
+        'product => . product SLASH factor @ 2',
+        'product => . factor @ 2',
+        'factor => . LPAREN sum RPAREN @ 2',
+      # 'factor => . NUMBER @ 2'
+      ]
+      set3 = [ # 1 + (. 2 * 3 - 4)
+        'factor => LPAREN . sum RPAREN @ 2',
+        'sum => . sum PLUS product @ 3',
+        'sum => . sum MINUS product @ 3',
+        'sum => . product @ 3',
+        'product => . product STAR factor @ 3',
+        'product => . product SLASH factor @ 3',
+        'product => . factor @ 3',
+        # 'factor => . LPAREN sum RPAREN @ 3',
+        'factor => . NUMBER @ 3'
+      ]
+      set4 = [ # 1 + (2 . * 3 - 4)
+        'factor => NUMBER . @ 3',
+        'product => factor . @ 3',
+        'sum => product . @ 3',
+        'product => product . STAR factor @ 3',
+      # 'product => product . SLASH factor @ 3',
+      # 'factor => LPAREN sum . RPAREN @ 2',
+      # 'sum => sum . PLUS product @ 3',
+      # 'sum => sum . MINUS product @ 3'
+      ]
+      set5 = [ # 1 + (2 * . 3 - 4)
+        'product => product STAR . factor @ 3',
+        # 'factor => . LPAREN sum RPAREN @ 5',
+        'factor => . NUMBER @ 5'
+      ]
+      set6 = [ # 1 + (2 * 3 . - 4)
+        'factor => NUMBER . @ 5',
+        'product => product STAR factor . @ 3',
+        'sum => product . @ 3',
+        # 'product => product . STAR factor @ 3',
+        # 'product => product . SLASH factor @ 3',
+        # 'factor => LPAREN sum . RPAREN @ 2',
+        # 'sum => sum . PLUS product @ 3',
+        'sum => sum . MINUS product @ 3'
+      ]
+      set7 = [ # 1 + (2 * 3  - . 4)
+        'sum => sum MINUS . product @ 3',
+        'product => . product STAR factor @ 7',
+        'product => . product SLASH factor @ 7',
+        'product => . factor @ 7',
+        # 'factor => . LPAREN sum RPAREN @ 7',
+        'factor => . NUMBER @ 7'
+      ]
+      set8 = [ # 1 + (2 * 3 - 4 .)
+        'factor => NUMBER . @ 7',
+        'product => factor . @ 7',
+        'sum => sum MINUS product . @ 3',
+        # 'product => product . STAR factor @ 7',
+        # 'product => product . SLASH factor @ 7',
+        'factor => LPAREN sum . RPAREN @ 2',
+      # 'sum => sum . PLUS product @ 3',
+      # 'sum => sum . MINUS product @ 3'
+      ]
+      set9 = [ # 1 + (2 * 3 - 4 ).
+        'factor => LPAREN sum RPAREN . @ 2',
+        'product => factor . @ 2',
+        'sum => sum PLUS product . @ 0',
+        # 'product => product . STAR factor @ 2',
+        # 'product => product . SLASH factor @ 2',
+        'p => sum . @ 0',
+      # 'sum => sum . PLUS product @ 0',
+      # 'sum => sum . MINUS product @ 0'
+      ]
+      expectations = [set0, set1, set2, set3, set4, set5, set6, set7, set8, set9]
+      expectations.each_with_index do |set, rank|
+        expect(chart[rank].to_s).to eq(set.join("\n"))
+      end
+    end
+  end # context
+end # describe