aurum 0.1.1 → 0.2.0

data/spec/grammar/builder/parsing_table/digraph_traverser_spec.rb

@@ -0,0 +1,42 @@
+helper_dir = File.join(File.dirname(__FILE__), '..', '..', '..')
+$:.unshift(helper_dir) unless $:.include?(helper_dir)
+
+require 'spec_helper'
+
+def construct_automata grammar, start
+  syntax_rules = grammar.instance_eval {@syntax_rules}
+  augmented_grammar = Aurum::Builder::AugmentedGrammar.new(syntax_rules, Aurum::Grammar.nonterminal(start))  
+  Aurum::Builder::ParsingTableBuilder.new(augmented_grammar, nil).instance_eval do
+    construct_automata
+    @augmented_grammar.compute_nullables
+    @augmented_grammar.compute_first_sets
+    @states
+  end
+end
+
+describe Aurum::Builder::DigraphTraverser, '(follow set calculator)' do
+  before :each do
+    @follow_set_calculator = Aurum::Builder::DigraphTraverser.new do |config|
+      (config.symbol == Aurum::Builder::StartSymbol) ? [false, [Aurum::Grammar::EOF], 65535] : [true, nil, nil]
+    end        
+  end
+	
+  it 'should calculate follow set for grammar 4.11 in dragon book' do
+    @states = construct_automata(Grammar411InDragonBook, 'expression')
+    right = Aurum::Grammar.terminal('$literal_)')
+    plus, mul = Aurum::Grammar.terminal('$literal_+'), Aurum::Grammar.terminal('$literal_*')
+    follow_set(Aurum::Grammar.nonterminal('expression')).should == [right, Aurum::Grammar::EOF].to_set
+    follow_set(Aurum::Grammar.nonterminal('expression1')).should == [right, Aurum::Grammar::EOF].to_set
+    follow_set(Aurum::Grammar.nonterminal('term')).should == [plus, right, Aurum::Grammar::EOF].to_set
+    follow_set(Aurum::Grammar.nonterminal('term1')).should == [plus, right, Aurum::Grammar::EOF].to_set
+    follow_set(Aurum::Grammar.nonterminal('factor')).should == [plus, mul, right, Aurum::Grammar::EOF].to_set
+  end
+	
+  def follow_set symbol
+    follow_set = [].to_set
+    for state in @states
+      follow_set |= @follow_set_calculator.traverse(Aurum::Builder::Configuration.new(state, symbol))
+    end
+    follow_set
+  end
+end

data/spec/grammar/builder/parsing_table/item_spec.rb

@@ -0,0 +1,51 @@
+helper_dir = File.join(File.dirname(__FILE__), '..', '..', '..')
+$:.unshift(helper_dir) unless $:.include?(helper_dir)
+
+require 'spec_helper'
+
+describe Aurum::Builder::LRItem do
+  before :all do
+    @E, @T, @id = Aurum::Grammar.nonterminal('E'), Aurum::Grammar.nonterminal('T'), Aurum::Grammar.nonterminal('id')
+    @production = Aurum::Grammar.production(@E, [@T, @id])
+  end
+
+  it 'dot symbol of [E -> .T id] should be T' do
+    Aurum::Builder::LRItem.new(@production, 0).dot_symbol.should == @T
+  end
+  it 'dot symbol of [E -> T .id] should be id' do
+    Aurum::Builder::LRItem.new(@production, 1).dot_symbol.should == @id
+  end
+  it 'dot symbol of [E -> T id.] should be nil' do
+    Aurum::Builder::LRItem.new(@production, 2).dot_symbol.should be_nil
+  end
+  it '[E -> .T id] should not be kernel item' do
+    Aurum::Builder::LRItem.new(@production, 0).should_not be_kernel
+  end
+  it '[E -> T .id] should be kernel item' do
+    Aurum::Builder::LRItem.new(@production, 1).should be_kernel
+  end
+  it '[E -> T id.] should be kernel item' do
+    Aurum::Builder::LRItem.new(@production, 2).should be_kernel
+  end
+  it '[E -> .] should be kernel item' do
+    Aurum::Builder::LRItem.new(Aurum::Grammar.production(@E, []), 0).should be_kernel
+  end
+  it '[E -> T id.] should be handle item' do
+    Aurum::Builder::LRItem.new(@production, 2).should be_handle
+  end
+  it '[E -> .] should be handle item' do
+    Aurum::Builder::LRItem.new(Aurum::Grammar.production(@E, []), 0).should be_handle
+  end
+  it '[$start -> .E] should be kernel item' do
+    Aurum::Builder::LRItem.new(Aurum::Grammar.production(Aurum::Builder::StartSymbol, [@E]), 0).should be_kernel
+  end
+  it 'remaining part of [E -> .T id] should be [T id]' do
+    Aurum::Builder::LRItem.new(@production, 0).remaining.should == [@T, @id]
+  end
+  it 'remaining part of [E -> T .id] should be [id]' do
+    Aurum::Builder::LRItem.new(@production, 1).remaining.should == [@id]
+  end
+  it 'remaining part of [E -> T id .] should be []' do
+    Aurum::Builder::LRItem.new(@production, 2).remaining.should == []
+  end
+end

data/spec/grammar/builder/parsing_table/sources_spec.rb

@@ -0,0 +1,66 @@
+helper_dir = File.join(File.dirname(__FILE__), '..', '..', '..')
+$:.unshift(helper_dir) unless $:.include?(helper_dir)
+
+require 'spec_helper'
+
+def determine_lookaheads grammar, start
+  syntax_rules = grammar.instance_eval {@syntax_rules}
+  @augmented_grammar = Aurum::Builder::AugmentedGrammar.new(syntax_rules, Aurum::Grammar.nonterminal(start))  
+  Aurum::Builder::ParsingTableBuilder.new(@augmented_grammar, nil).instance_eval do
+    construct_automata
+    unless @inconsistent_states.empty?
+      @augmented_grammar.compute_nullables
+      @augmented_grammar.compute_first_sets
+      @conflict_states = [].to_set
+      @follow_set_calculator = Aurum::Builder::DigraphTraverser.new do |config|
+        (config.symbol == Aurum::Builder::StartSymbol) ? [false, [Aurum::Grammar::EOF], 65535] : [true, nil, nil]
+      end
+      for inconsistent_state in @inconsistent_states do
+        determine_lookaheads_for(inconsistent_state)
+        @conflict_states << inconsistent_state if inconsistent_state.conflict?
+      end
+      @states
+    end
+  end
+end
+
+describe Aurum::Builder::Sources, ' for conflict state' do
+  before :all do
+    @slist, @rule = Aurum::Grammar.nonterminal('slist'), Aurum::Grammar.nonterminal('rule')
+    @s, @generate = Aurum::Grammar.terminal('_s'), Aurum::Grammar.terminal('$literal_->')
+    @states = determine_lookaheads(BnfGrammarInCharlesThesis, 'bnf')
+  end
+
+  it 'should use current state as configuration stack for read reduce action' do
+    state = @states.find {|state| state.conflict?}
+    sources = Aurum::Builder::Sources.new(state, @s)
+    stacks = sources[Aurum::ParsingTable::Action.new(:read_reduce, Aurum::Grammar.production(@slist, [@slist, @s]))]
+    stacks.should == [[@states.last]].to_set
+  end
+
+  it 'should use follow source as configuration stack for reduce action' do
+    state = @states.find {|state| state.conflict?}
+    sources = Aurum::Builder::Sources.new(state, @s)
+    stacks = sources[Aurum::ParsingTable::Action.new(:reduce, Aurum::Grammar.production(@rule, [@s, @generate, @slist]))]
+    stacks.should == [[@states[1]]].to_set
+  end
+end
+
+describe Aurum::Builder::Sources, ' for lookahead state' do
+  before :each do
+    @x, @w = Aurum::Grammar.terminal('_x'), Aurum::Grammar.terminal('_w')
+    @B, @C = Aurum::Grammar.nonterminal('b'), Aurum::Grammar.nonterminal('c')
+    @states = determine_lookaheads(LALR4Grammar, 's')
+  end
+
+  it 'should construct sources using previous one' do
+    previous_sources = Aurum::Builder::Sources.new(@states[4], @x)
+    lookahead = Aurum::Builder::State.new(@augmented_grammar, [])
+    lookahead.actions[@x] = @states[4].actions[@x]
+    sources = Aurum::Builder::Sources.new(lookahead, @x, previous_sources, @x)
+    action = Aurum::ParsingTable::Action.new(:reduce, Aurum::Grammar.production(@B, [@w]))
+    sources[action].should == [[@states[14]]].to_set
+    action = Aurum::ParsingTable::Action.new(:reduce, Aurum::Grammar.production(@C, [@w]))
+    sources[action].should == [[@states[10]]].to_set
+  end
+end

data/spec/grammar/builder/parsing_table/state_spec.rb

@@ -0,0 +1,82 @@
+helper_dir = File.join(File.dirname(__FILE__), '..', '..', '..')
+$:.unshift(helper_dir) unless $:.include?(helper_dir)
+
+require 'spec_helper'
+
+describe Aurum::Builder::State do
+  
+  before :all do
+    @syntax_rules = Grammar419InDragonBook.instance_eval {@syntax_rules}
+    @E = Aurum::Grammar.nonterminal('expression')
+    @e_productions = @syntax_rules.productions(@E).to_a
+    @f_productions = @syntax_rules.productions(Aurum::Grammar.nonterminal('factor')).to_a
+    @augmented_grammar = Aurum::Builder::AugmentedGrammar.new(@syntax_rules, @E)
+    @production = @augmented_grammar.instance_eval {@start_production}
+  end
+  
+  it 'should be represented by kernel items' do
+    kernel_item = Aurum::Builder::LRItem.new(@production)
+    non_kernel_item = Aurum::Builder::LRItem.new(@e_productions[0])
+    Aurum::Builder::State.new(@augmented_grammar, [kernel_item, non_kernel_item]).should == [kernel_item]
+  end
+
+  it 'should goto new state on X when A -> a.Xb in current state' do
+    state = Aurum::Builder::State.new(@augmented_grammar, [Aurum::Builder::LRItem.new(@production)])
+    state.goto(@E).inspect.should == '[$start -> expression ., expression -> expression .+ term ]'
+  end
+  
+  it 'should return nil when try to goto on X but A -> a.Xb not in current state' do
+    state = Aurum::Builder::State.new(@augmented_grammar, [Aurum::Builder::LRItem.new(@production)])
+    state.goto(Aurum::Grammar.terminal('other')).should be_nil
+  end
+  
+  it 'should be consistent if contains one and only one handle item' do
+    state = Aurum::Builder::State.new(@augmented_grammar, [Aurum::Builder::LRItem.new(@production, 1)])
+    state.inconsistent?.should be_false
+  end
+
+  it 'should be consistent if does not contain any handle item' do
+    state = Aurum::Builder::State.new(@augmented_grammar, [Aurum::Builder::LRItem.new(@e_productions[1]),
+        Aurum::Builder::LRItem.new(@e_productions[1], 2)])
+    state.inconsistent?.should be_false
+  end
+
+  it 'should be inconsistent if contains handle and other kernels' do
+    state = Aurum::Builder::State.new(@augmented_grammar, [Aurum::Builder::LRItem.new(@e_productions[1], 1),
+        Aurum::Builder::LRItem.new(@production, 1)])
+    state.inconsistent?.should be_true
+  end
+
+  it 'should be inconsistent if contains multi-handles' do
+    state = Aurum::Builder::State.new(@augmented_grammar, [Aurum::Builder::LRItem.new(@e_productions[1], 3),
+        Aurum::Builder::LRItem.new(@production, 1)])
+    state.inconsistent?.should be_true
+  end
+
+  it 'should be read reducable if contains one and only one handle item' do
+    state = Aurum::Builder::State.new(@augmented_grammar, [Aurum::Builder::LRItem.new(@f_productions[1], 1)])
+    state.read_reduce.should == @f_productions[1]
+  end
+=begin
+  it 'should return predecessors according to given symbol list' do
+    state1 = Aurum::Builder::State.new([Aurum::Builder::LRItem.new(@production, 0)])
+    state2 = Aurum::Builder::State.new([Aurum::Builder::LRItem.new(@production, 1)])
+    state1.add_shift(@E, state2)
+    state2.predecessors([@E]).should == [state1]
+  end
+
+  it 'should return all including configs' do
+    production = @E.productions[0]
+    state0 = Aurum::ParsingTableBuilder::State.new([Aurum::ParsingTableBuilder::LRItem.new(production, 0)])
+    state1 = Aurum::ParsingTableBuilder::State.new([Aurum::ParsingTableBuilder::LRItem.new(production, 1)])
+    state2 = Aurum::ParsingTableBuilder::State.new([Aurum::ParsingTableBuilder::LRItem.new(production, 2)])
+    state3 = Aurum::ParsingTableBuilder::State.new([Aurum::ParsingTableBuilder::LRItem.new(production, 3)])
+    state0.add_shift(@E, state1)
+    state1.add_shift(@plus, state2)
+    state2.add_shift(@T, state3)
+    include_configs = []
+    state2.include_each(@T) {|state, symbol| include_configs << [state, symbol]}
+    include_configs.should == [[state0, @E]]
+  end
+=end  
+end

data/spec/grammar/dsl/character_classes_builder_spec.rb

@@ -0,0 +1,50 @@
+helper_dir = File.join(File.dirname(__FILE__), '..', '..')
+$:.unshift(helper_dir) unless $:.include?(helper_dir)
+
+require 'spec_helper'
+
+describe Aurum::Grammar::DSL::CharacterClassesBuilder do
+  before :each do
+    @builder = Aurum::Grammar::DSL::CharacterClassesBuilder.new
+  end
+
+  it 'should define named character classes' do
+    @builder.instance_eval do
+      number range(?0, ?9)
+      number_enum enum('0123456789')
+    end
+    char_set = @builder.__named_character_classes__['number']
+    char_set.should == Aurum::Grammar::LexicalRules::CharacterSet.range(?0, ?9)
+    char_set = @builder.__named_character_classes__['number_enum']
+    char_set.should == Aurum::Grammar::LexicalRules::CharacterSet.enum('0123456789')
+  end
+
+  it 'should define character class by expression' do
+    @builder.instance_eval do
+      number range(?0, ?9)
+      alpha range(?a, ?z)
+      number_and_alpha number + alpha
+    end
+    char_set = @builder.__named_character_classes__['number_and_alpha']
+    char_set.should == (Aurum::Grammar::LexicalRules::CharacterSet.range(?0, ?9) + Aurum::Grammar::LexicalRules::CharacterSet.range(?a, ?z))
+  end
+
+  it 'should contain pre-definied classes' do
+    @builder.__named_character_classes__.should have_key('any')
+    @builder.__named_character_classes__.should have_key('underscore')
+    @builder.__named_character_classes__.should have_key('single_quote')
+    @builder.__named_character_classes__.should have_key('double_quote')
+    @builder.__named_character_classes__.should have_key('ascii')
+    @builder.__named_character_classes__.should have_key('ascii_punctuation')
+    @builder.__named_character_classes__.should have_key('decimal_number')
+  end
+
+  it 'should not re-define character class' do
+    lambda do
+      @builder.instance_eval do
+        number range(?0, ?9)
+        number range(?1, ?9)
+      end
+    end.should raise_error(RuntimeError, 'already defined character class: number')
+  end
+end

data/spec/grammar/dsl/lexical_rules_builder_spec.rb

@@ -0,0 +1,181 @@
+helper_dir = File.join(File.dirname(__FILE__), '..', '..')
+$:.unshift(helper_dir) unless $:.include?(helper_dir)
+
+require 'spec_helper'
+
+describe Aurum::Grammar::DSL::LexicalRulesBuilder, ' pattern definition' do
+  include PatternMatcher
+  before :each do
+    @lexical_rules = Aurum::Grammar::LexicalRules.new
+    @char_classes = Aurum::Grammar::DSL::CharacterClassesBuilder.new
+    @builder = Aurum::Grammar::DSL::LexicalRulesBuilder.new(@lexical_rules, @char_classes)
+  end
+  
+  it 'should create pattern using range' do
+    pattern = @builder.range(?a, ?c)
+    'a'.should be_matched_by('[a-c]', pattern)
+    'b'.should be_matched_by('[a-c]', pattern)
+    'c'.should be_matched_by('[a-c]', pattern)
+  end
+
+  it 'should create pattern using enum' do
+    pattern = @builder.enum('qwe')
+    'q'.should be_matched_by('q|w|e', pattern)
+    'w'.should be_matched_by('q|w|e', pattern)
+    'e'.should be_matched_by('q|w|e', pattern)
+  end
+
+  it 'should create pattern using string literal' do
+    pattern = @builder.string('string')
+    'string'.should be_matched_by('string', pattern)
+  end
+  
+  it 'should create pattern using pre-defined character classes' do
+    @char_classes.instance_eval {qwe enum('qwe')}
+    pattern = @builder.qwe
+    'q'.should be_matched_by('q|w|e', pattern)
+    'w'.should be_matched_by('q|w|e', pattern)
+    'e'.should be_matched_by('q|w|e', pattern)    
+  end
+  
+  it 'should raise error if can not find pattern with specified name' do
+    lambda {@builder.inexist}.should raise_error(RuntimeError, "can not find pattern named 'inexist'")
+  end
+
+  it 'should create pattern by concatenating other patterns' do
+    string_a = @builder.string('string_a')
+    string_b = @builder.string('string_b')
+    pattern = @builder.concat(string_a, string_b)
+    'string_astring_b'.should be_matched_by('string_astring_b', pattern)
+  end
+
+  it 'should create pattern by concatenating patterns and string literals' do
+    string_a = @builder.string('string_a')
+    pattern = @builder.concat(string_a, 'string_b')
+    'string_astring_b'.should be_matched_by('string_astring_b', pattern)
+  end
+
+  it 'should add named pattern to pattern list' do
+    @builder.instance_eval do
+      pattern_a 'abc', string('def').zero_or_more
+    end
+    pattern = @builder.instance_eval {@named_patterns['pattern_a']}
+    'abc'.should be_matched_by('abc(def)*', pattern)
+    'abcdefdef'.should be_matched_by('abc(def)*', pattern)
+  end
+
+  it 'should add state local named pattern' do
+    @builder.instance_eval do
+      pattern_a 'abc', string('def').zero_or_more
+      within :state_a do
+        pattern_a 'def'
+      end
+    end    
+    pattern = @builder.instance_eval {@named_patterns['pattern_a']}
+    'abcdefdef'.should be_matched_by('abc(def)*', pattern)
+  end
+end
+
+describe Aurum::Grammar::DSL::LexicalRulesBuilder, ' leixcal action definition' do
+  before :each do
+    @lexical_rules = Aurum::Grammar::LexicalRules.new
+    @char_classes = Aurum::Grammar::DSL::CharacterClassesBuilder.new
+    @builder = Aurum::Grammar::DSL::LexicalRulesBuilder.new(@lexical_rules, @char_classes)
+  end
+
+  it 'should add pattern to initial state' do
+    @builder._id 'string'
+    actions = @lexical_rules.patterns['initial'].values
+    actions.should == [Aurum::LexicalTable::Action.new('_id', nil, nil, nil)]
+  end
+  
+  it 'should add pattern to new lexical states' do
+    @builder.within :state_a, :state_b do
+      _id 'string'
+    end
+    actions = @lexical_rules.patterns['state_a'].values
+    actions.should == [Aurum::LexicalTable::Action.new( '_id', nil, nil, nil)]
+    actions = @lexical_rules.patterns['state_b'].values
+    actions.should == [Aurum::LexicalTable::Action.new('_id', nil, nil, nil)]
+  end
+  
+  it 'should add pattern to exist lexical states' do
+    @builder.within :state_a, :state_b do
+      _id 'string'
+    end
+    @builder.within :state_a do
+      _id_2 'string_2'
+    end
+    actions = @lexical_rules.patterns['state_a'].values
+    actions.to_set.should == [Aurum::LexicalTable::Action.new('_id', nil, nil, nil), Aurum::LexicalTable::Action.new('_id_2', nil, nil, nil)].to_set
+  end
+
+  it "should define 'recognize' lexical action" do
+    pattern = @builder.pattern 'string'
+    @builder.instance_eval do
+      match(pattern, :recognize => :id)
+    end
+    action = @lexical_rules.patterns['initial'][pattern]
+    action.should == Aurum::LexicalTable::Action.new('id', nil, nil, nil)
+  end
+
+  it "should define 'shift to' lexical action" do
+    pattern = @builder.pattern 'string'
+    @builder.instance_eval do
+      match(pattern, :shift_to => :state)
+    end
+    action = @lexical_rules.patterns['initial'][pattern]
+    action.should == Aurum::LexicalTable::Action.new(nil,'state', nil, nil)
+  end
+
+  it "should define 'recognize and shift to' lexical action" do
+    pattern = @builder.pattern 'string'
+    @builder.instance_eval do
+      match(pattern, :recognize => :id, :shift_to => :state)
+    end
+    action = @lexical_rules.patterns['initial'][pattern]
+    action.should == Aurum::LexicalTable::Action.new('id', 'state', nil, nil)
+  end
+
+  it "should store add lexical event id to lexical action" do
+    pattern = @builder.pattern 'string'
+    @builder.instance_eval do
+      match(pattern, :event => :identifier_recongized)
+    end
+    action = @lexical_rules.patterns['initial'][pattern]
+    action.should == Aurum::LexicalTable::Action.new(nil, nil, 'identifier_recongized', nil)
+  end
+
+  it 'should store semantic action to lexical action' do
+    semantic_action = lambda { ruby action }
+    @builder._id 'string', &semantic_action
+    actions = @lexical_rules.patterns['initial'].values
+    actions.should == [Aurum::LexicalTable::Action.new('_id', nil, nil, semantic_action)]
+  end
+
+  it "should define 'ignore' lexical action" do
+    pattern = @builder.pattern 'string'
+    @builder.instance_eval do
+      ignore(pattern)
+    end
+    actions = @lexical_rules.patterns['initial'].values
+    actions.should == [Aurum::LexicalTable::Action.new('$ignored', nil, nil, nil)]
+  end
+
+  it "should define 'ignore and shif to' lexical action" do
+    pattern = @builder.pattern 'string'
+    @builder.instance_eval do
+      ignore(pattern, :event => :id_ignore, :shift_to => :state)
+    end
+    actions = @lexical_rules.patterns['initial'].values
+    actions.should == [Aurum::LexicalTable::Action.new('$ignored', 'state', 'id_ignore', nil)]
+  end
+
+  it 'should store semantic action to ignored lexical action' do
+    semantic_action = lambda { ruby action }
+    pattern = @builder.pattern 'string'
+    @builder.ignore pattern, &semantic_action
+    actions = @lexical_rules.patterns['initial'].values
+    actions.should == [Aurum::LexicalTable::Action.new('$ignored', nil, nil, semantic_action)]
+  end
+end