dendroid 0.0.11 → 0.1.00

data/spec/dendroid/support/sample_grammars.rb

@@ -10,9 +10,7 @@ module SampleGrammars
       declare_terminals('PLUS', 'STAR', 'INTEGER')
       rule('p' => 's')
       rule('s' => ['s PLUS m', 'm'])
-      # rule('s' => 'm')
       rule('m' => ['m STAR t', 't'])
-      # rule('m' => 't')
       rule('t' => 'INTEGER')
     end
 
@@ -28,7 +26,6 @@ module SampleGrammars
     end
   end
 
-
   def grammar_l2
     builder = Dendroid::GrmDSL::BaseGrmBuilder.new do
       # Grammar inspired from Loup Vaillant's example
@@ -47,8 +44,7 @@ module SampleGrammars
 
   def tokenizer_l2
     Dendroid::Utils::BaseTokenizer.new do
-      map_verbatim2terminal({
-                              '+' => :PLUS,
+      map_verbatim2terminal({ '+' => :PLUS,
                               '-' => :MINUS,
                               '*' => :STAR,
                               '/' => :SLASH,
@@ -68,9 +64,256 @@ module SampleGrammars
 
       rule('Z' => ['d', 'X Y Z'])
       rule('Y' => ['', 'c'])
-      rule('X' => ['Y', 'a'])
+      rule('X' => %w[Y a])
+    end
+
+    builder.grammar
+  end
+
+  def grammar_l31
+    builder = Dendroid::GrmDSL::BaseGrmBuilder.new do
+      # Ambiguous arithmetical expression language
+      # This language is compatible with tokenizer L1
+      declare_terminals('PLUS', 'STAR', 'INTEGER')
+      rule('p' => 's')
+      rule('s' => ['s PLUS s', 's STAR s', 'INTEGER'])
+    end
+
+    builder.grammar
+  end
+
+  def grammar_l4
+    builder = Dendroid::GrmDSL::BaseGrmBuilder.new do
+      # (based on an example from Fisher and LeBlanc: "Crafting a Compiler")
+      declare_terminals('plus', 'id')
+
+      rule 'S' => 'E'
+      rule 'E' => ['E plus E', 'id']
+    end
+
+    builder.grammar
+  end
+
+  def tokenizer_l4
+    Dendroid::Utils::BaseTokenizer.new do
+      map_verbatim2terminal({ '+' => :plus })
+
+      scan_verbatim(['+'])
+      scan_value(/[_A-Za-z][_A-Za-z0-9]*/, :id, ->(txt) { txt })
+    end
+  end
+
+  def grammar_l5
+    builder = Dendroid::GrmDSL::BaseGrmBuilder.new do
+      # (based on example in N. Wirth "Compiler Construction" book, p. 6)
+      declare_terminals('a', 'b', 'c')
+
+      rule 'S' => 'A'
+      rule 'A' => ['a A c', 'b']
     end
 
     builder.grammar
   end
+
+  def tokenizer_l5
+    Dendroid::Utils::BaseTokenizer.new do
+      map_verbatim2terminal({ 'a' => :a, 'b' => :b, 'c' => :c })
+
+      scan_verbatim(%w[a b c])
+    end
+  end
+
+  def grammar_l6
+    builder = Dendroid::GrmDSL::BaseGrmBuilder.new do
+      # Grammar to illustrate the dangling else ambiguity
+      # (based on grammar G5 from Douglas Thain "Introduction to Compiler and Language Design" book, p. 6)
+      declare_terminals('if', 'then', 'else', 'E', 'other')
+
+      rule 'P' => 'S'
+      rule 'S' => ['if E then S', 'if E then S else S', 'other']
+    end
+
+    builder.grammar
+  end
+
+  def tokenizer_l6
+    Dendroid::Utils::BaseTokenizer.new do
+      map_verbatim2terminal({ 'if' => :if,
+                              'then' => :then,
+                              'else' => :else,
+                              'E' => :E,
+                              'other' => :other })
+
+      scan_verbatim(%w[if then else E other])
+    end
+  end
+
+  def grammar_l7
+    builder = Dendroid::GrmDSL::BaseGrmBuilder.new do
+      # (based on grammar G1 from paper Elizabeth Scott, Adrian Johnstone "Recognition
+      # is not parsing SPPF-style parsing from cubic recognisers")
+      declare_terminals('a')
+
+      rule 'S' => ['S T', 'a']
+      rule 'B' => ''
+      rule 'T' => ['a B', 'a']
+    end
+
+    builder.grammar
+  end
+
+  def tokenizer_l7
+    Dendroid::Utils::BaseTokenizer.new do
+      map_verbatim2terminal({ 'a' => :a })
+
+      scan_verbatim(['a'])
+    end
+  end
+
+  def grammar_l8
+    builder = Dendroid::GrmDSL::BaseGrmBuilder.new do
+      # (based on grammar G2 from paper Masaru Tomita "An Efficient Context-Free Parsing Algorithm
+      #   for Natural Languages")
+      declare_terminals('x')
+
+      rule 'S' => ['S S', 'x']
+    end
+
+    builder.grammar
+  end
+
+  def tokenizer_l8
+    Dendroid::Utils::BaseTokenizer.new do
+      map_verbatim2terminal({ 'x' => :x })
+
+      scan_verbatim(['x'])
+    end
+  end
+
+  def grammar_l9
+    builder = Dendroid::GrmDSL::BaseGrmBuilder.new do
+      # (based on "infinite ambiguity" grammar from paper Masaru Tomita "An Efficient Context-Free Parsing Algorithm
+      #   for Natural Languages")
+      declare_terminals('x')
+
+      rule 'S' => ['S S', '', 'x']
+    end
+
+    builder.grammar
+  end
+
+  def tokenizer_l9
+    Dendroid::Utils::BaseTokenizer.new do
+      map_verbatim2terminal({ 'x' => :x })
+
+      scan_verbatim(['x'])
+    end
+  end
+
+  def grammar_l10
+    builder = Dendroid::GrmDSL::BaseGrmBuilder.new do
+      declare_terminals('a')
+
+      rule 'A' => ['A a', '']
+    end
+
+    builder.grammar
+  end
+
+  def tokenizer_l10
+    Dendroid::Utils::BaseTokenizer.new do
+      map_verbatim2terminal({ 'a' => :a })
+
+      scan_verbatim(['a'])
+    end
+  end
+
+  def grammar_l11
+    builder = Dendroid::GrmDSL::BaseGrmBuilder.new do
+      declare_terminals('a')
+
+      rule 'A' => ['a A', '']
+    end
+
+    builder.grammar
+  end
+
+  def tokenizer_l11
+    Dendroid::Utils::BaseTokenizer.new do
+      map_verbatim2terminal({ 'a' => :a })
+
+      scan_verbatim(['a'])
+    end
+  end
+
+  def grammar_l12
+    builder = Dendroid::GrmDSL::BaseGrmBuilder.new do
+      # (based on grammar Example 3 from paper Elizabeth Scott "SPPF-Style Parsing
+      #   from Earley Recognisers")
+      # Grammar with hidden left recursion and a cycle
+      declare_terminals('a', 'b')
+
+      rule 'S' => ['A T', 'a T']
+      # rule 'S' => 'a T'
+      rule 'A' => ['a', 'B A']
+      # rule 'A' => 'B A'
+      rule 'B' => ''
+      rule 'T' => 'b b b'
+    end
+
+    builder.grammar
+  end
+
+  def tokenizer_l12
+    Dendroid::Utils::BaseTokenizer.new do
+      map_verbatim2terminal({ 'a' => :a, 'b' => :b })
+
+      scan_verbatim(%w[a b])
+    end
+  end
+
+  def grammar_l13
+    builder = Dendroid::GrmDSL::BaseGrmBuilder.new do
+      # Grammar based on example RR from Sylvie Billot, Bernard Lang "The Structure of Shared Forests
+      #   in Ambiguous Parsing"
+      declare_terminals('x')
+
+      rule 'A' => ['x A', 'x']
+      # rule 'A' => 'x'
+    end
+
+    builder.grammar
+  end
+
+  def tokenizer_l13
+    Dendroid::Utils::BaseTokenizer.new do
+      map_verbatim2terminal({ 'x' => :x })
+
+      scan_verbatim(['x'])
+    end
+  end
+
+  def grammar_l14
+    builder = Dendroid::GrmDSL::BaseGrmBuilder.new do
+      # Grammar 4: A grammar with nullable nonterminal
+      # based on example from "Parsing Techniques" book, p. 216
+      # (D. Grune, C. Jabobs)
+      declare_terminals('a', 'star', 'slash')
+
+      rule 'S' => 'E'
+      rule 'E' => ['E Q F', 'F']
+      rule 'F' => 'a'
+      rule 'Q' => ['star', 'slash', '']
+    end
+
+    builder.grammar
+  end
+
+  def tokenizer_l14
+    Dendroid::Utils::BaseTokenizer.new do
+      map_verbatim2terminal({ 'a' => :a, '*' => :star, '/' => :slash })
+
+      scan_verbatim(['a', '*', '/'])
+    end
+  end
 end # module

data/spec/dendroid/syntax/grammar_spec.rb

@@ -7,6 +7,7 @@ require_relative '..\..\..\lib\dendroid\syntax\symbol_seq'
 require_relative '..\..\..\lib\dendroid\syntax\production'
 require_relative '..\..\..\lib\dendroid\syntax\choice'
 require_relative '..\..\..\lib\dendroid\syntax\grammar'
+require_relative '..\..\..\lib\dendroid\grm_dsl\base_grm_builder'
 
 describe Dendroid::Syntax::Grammar do
   let(:int_symb) { build_terminal('INTEGER') }
@@ -57,8 +58,8 @@ describe Dendroid::Syntax::Grammar do
       expect(subject.symbols).to eq(all_terminals)
     end
 
-    it 'ignores about productions after initialization' do
-      expect(subject.rules).to be_nil
+    it 'does not have rules after initialization' do
+      expect(subject.rules).to be_empty
     end
 
     it 'maps a terminal name to one GrmSymbol object' do
@@ -108,14 +109,15 @@ describe Dendroid::Syntax::Grammar do
       end
     end
 
-    it 'maps every non-terminal to its defining productions' do
+    it 'maps every non-terminal to its defining production' do
       rules = build_all_rules
       rules.each { |rl| subject.add_rule(rl) }
       %i[p s m t].each do |symb_name|
         symb = subject.name2symbol[symb_name]
         expected_prods = subject.rules.select { |prd| prd.head == symb }
-        related_prods = subject.nonterm2productions[symb]
-        expect(related_prods).to eq(expected_prods)
+        expect(expected_prods.size).to eq(1)
+        related_prod = subject.nonterm2production[symb]
+        expect(related_prod).to eq(expected_prods[0])
       end
     end
   end # context
@@ -200,4 +202,162 @@ describe Dendroid::Syntax::Grammar do
       expect(nonproductive).to eq([nterm_D, nterm_F])
     end
   end # context
+
+  context 'Errors with terminal symbols' do
+    def grm_terminal_in_lhs
+      builder = Dendroid::GrmDSL::BaseGrmBuilder.new do
+        declare_terminals('a', 'b', 'c')
+
+        rule 'S' => 'A'
+        rule 'a' => 'a A c' # Wrong: terminal 'a' in lhs
+        rule 'A' => 'b'
+      end
+
+      builder.grammar
+    end
+
+    def grm_cyclic
+      builder = Dendroid::GrmDSL::BaseGrmBuilder.new do
+        declare_terminals('x')
+
+        rule 'S' => %w[S x] # Wrong: cyclic production (lhs and rhs are the same)
+      end
+
+      builder.grammar
+    end
+
+    def grm_no_terminal
+      builder = Dendroid::GrmDSL::BaseGrmBuilder.new do
+        # No terminal symbol explicitly declared => all symbols are non-terminals
+
+        rule 'S' => 'A'
+        rule 'A' => ['a A c', 'b']
+      end
+
+      builder.grammar
+    end
+
+    def unused_terminals
+      builder = Dendroid::GrmDSL::BaseGrmBuilder.new do
+        declare_terminals('a', 'b', 'c', 'd', 'e')
+
+        # # Wrong: terminals 'd' and 'e' never appear in rules
+        rule 'S' => 'A'
+        rule 'A' => ['a A c', 'b']
+      end
+
+      builder.grammar
+    end
+
+    it 'raises an error if there is no terminal symbol' do
+      err_msg = "Grammar doesn't contain any terminal symbol."
+      expect { grm_no_terminal }.to raise_error(StandardError, err_msg)
+    end
+
+    it 'raises an error if a terminal is in lhs of production' do
+      err_msg = "Terminal symbol 'a' may not be on left-side of a rule."
+      expect { grm_terminal_in_lhs }.to raise_error(StandardError, err_msg)
+    end
+
+    it 'raises an error if a terminal never appear in rules' do
+      err_msg = "Terminal symbols 'd', 'e' never appear in production rules."
+      expect { unused_terminals }.to raise_error(StandardError, err_msg)
+    end
+
+    it 'raises an error if a production is cyclic' do
+      err_msg = 'Cyclic rules of the kind S => S are not allowed.'
+      expect { grm_cyclic }.to raise_error(StandardError, err_msg)
+    end
+  end # context
+
+  context 'Errors with non-terminal symbols' do
+    def grm_undefined_nterm
+      builder = Dendroid::GrmDSL::BaseGrmBuilder.new do
+        declare_terminals('x')
+
+        rule 'S' => %w[x A] # Wrong: A is never defined
+      end
+
+      builder.grammar
+    end
+
+    def grm_multiple_defs
+      builder = Dendroid::GrmDSL::BaseGrmBuilder.new do
+        declare_terminals('a b c')
+
+        rule 'A' => %w[a B]
+        rule 'B' => ['b', '']
+        rule 'A' => 'c'
+      end
+
+      builder.grammar
+    end
+
+    def duplicate_production
+      builder = Dendroid::GrmDSL::BaseGrmBuilder.new do
+        declare_terminals('a')
+
+        rule 'S' => 'A'
+        rule 'A' => %w[a a] # Duplicate alternatives
+      end
+
+      builder.grammar
+    end
+
+    def grm_unreachable_symbols
+      builder = Dendroid::GrmDSL::BaseGrmBuilder.new do
+        declare_terminals('a', 'b', 'c')
+
+        rule 'S' => 'A'
+        rule 'A' => ['a A c', 'b']
+        rule 'Z' => 'a Z X'
+        rule 'X' => 'b b'
+      end
+
+      builder.grammar
+    end
+
+    def nonproductive_symbols
+      builder = Dendroid::GrmDSL::BaseGrmBuilder.new do
+        declare_terminals('a', 'b', 'c', 'd', 'e', 'f')
+
+        # # Wrong: terminals 'D' and 'F' are non-productive (they never reduce to a string of terminals)
+        rule 'S' => ['A B', 'D E']
+        rule 'A' => 'a'
+        rule 'B' => 'b C'
+        rule 'C' => 'c'
+        rule 'D' => 'd F'
+        rule 'E' => 'e'
+        rule 'F' => 'f D'
+      end
+
+      builder.grammar
+    end
+
+    it 'raises an error when a non-terminal is never defined' do
+      err_msg = "Non-terminal symbols 'A' never appear in head of any production rule."
+      expect { grm_undefined_nterm }.to raise_error(StandardError, err_msg)
+    end
+
+    it 'raises an error when a non-terminal is defined multiple times' do
+      err_msg = "Non-terminal 'A' is on left-hand side of more than one rule."
+      expect { grm_multiple_defs }.to raise_error(StandardError, err_msg)
+    end
+
+    it 'raises an error when a production is duplicated' do
+      err_msg = 'Duplicate alternatives: A => a'
+      expect { duplicate_production }.to raise_error(StandardError, err_msg)
+    end
+
+    it 'raises an error when a non-terminal is unreachable' do
+      # err_msg = "Symbols 'Z', 'X' are unreachable from start symbol."
+      err_msg = "Symbols 'Z' are non-productive."
+      expect { grm_unreachable_symbols }.to raise_error(StandardError, err_msg)
+    end
+
+    it 'raises an error when a non-terminal is non-productive' do
+      err_msg = "Symbols 'D', 'F' are non-productive."
+      expect { nonproductive_symbols }.to raise_error(StandardError, err_msg)
+    end
+  end # contex
 end # describe

data/version.txt

@@ -1 +1 @@
-0.0.11
+0.1.00

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: dendroid
 version: !ruby/object:Gem::Version
-  version: 0.0.11
+  version: 0.1.00
 platform: ruby
 authors:
 - Dimitri Geshef
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-11-02 00:00:00.000000000 Z
+date: 2023-11-03 00:00:00.000000000 Z
 dependencies: []
 description: WIP. A Ruby implementation of an Earley parser
 email: famished.tiger@yahoo.com