aurum 0.1.1 → 0.2.0

metadata

@@ -1,10 +1,10 @@
 --- !ruby/object:Gem::Specification 
-rubygems_version: 0.9.2
+rubygems_version: 0.9.4
 specification_version: 1
 name: aurum
 version: !ruby/object:Gem::Version 
-  version: 0.1.1
-date: 2007-05-26 00:00:00 +10:00
+  version: 0.2.0
+date: 2007-10-11 00:00:00 +10:00
 summary: Aurum is a LALR(n) parser generator written in Ruby.
 require_paths: 
 - lib
@@ -29,43 +29,77 @@ post_install_message:
 authors: 
 - Vincent Xu
 files: 
-- example/expression
-- example/expression/expression.rb
-- example/expression/lisp.rb
+- examples/dangling_else
+- examples/dangling_else/grammar.rb
+- examples/expression
+- examples/expression/grammar.rb
+- examples/smalltalk
+- examples/smalltalk/grammar.rb
+- examples/smalltalk/interpreter.rb
+- examples/yacc
+- examples/yacc/grammar.rb
 - lib/aurum
-- lib/aurum/lexical_table_generator.rb
-- lib/aurum/parsing_table_generator.rb
+- lib/aurum/engine
+- lib/aurum/engine/parsing_facility.rb
+- lib/aurum/engine/tokenization_facility.rb
 - lib/aurum/engine.rb
+- lib/aurum/grammar
+- lib/aurum/grammar/automata.rb
+- lib/aurum/grammar/builder
+- lib/aurum/grammar/builder/augmented_grammar.rb
+- lib/aurum/grammar/builder/dot_logger.rb
+- lib/aurum/grammar/builder/lexical_table_builder.rb
+- lib/aurum/grammar/builder/parsing_table_builder.rb
+- lib/aurum/grammar/builder/set_of_items.rb
+- lib/aurum/grammar/compiled_tables.rb
+- lib/aurum/grammar/dsl
+- lib/aurum/grammar/dsl/lexical_definition.rb
+- lib/aurum/grammar/dsl/syntax_definition.rb
+- lib/aurum/grammar/lexical_rules.rb
+- lib/aurum/grammar/metalang
+- lib/aurum/grammar/metalang/grammar.rb
+- lib/aurum/grammar/syntax_rules.rb
 - lib/aurum/grammar.rb
-- lib/grammars
 - lib/aurum.rb
-- test/parsing_table_generator
-- test/parsing_table_generator/augmented_grammar_test.rb
-- test/parsing_table_generator/precedence_table_test.rb
-- test/parsing_table_generator/production_test.rb
-- test/parsing_table_generator/lr_0_automata_test.rb
-- test/parsing_table_generator/lalr_n_computation_test.rb
-- test/parsing_table_generator/parsing_table_state_test.rb
-- test/parsing_table_generator/lr_item_test.rb
-- test/engine
-- test/engine/lexer_test.rb
-- test/engine/semantic_attributes_test.rb
-- test/grammars
-- test/grammars/ruby
-- test/grammar_definition
-- test/grammar_definition/grammar_definition_test.rb
-- test/grammar_definition/production_definition_test.rb
-- test/grammar_definition/operator_precedence_definition_test.rb
-- test/grammar_definition/character_class_definition_test.rb
-- test/grammar_definition/lexical_definition_test.rb
-- test/lexical_table_generator
-- test/lexical_table_generator/interval_test.rb
-- test/lexical_table_generator/subset_determinizer_test.rb
-- test/lexical_table_generator/character_set_test.rb
-- test/lexical_table_generator/automata_test.rb
-- test/lexical_table_generator/pattern_test.rb
-- test/lexical_table_generator/table_generator_test.rb
-- test/test_helper.rb
+- spec/builder
+- spec/builder/dsl_definition
+- spec/builder/dsl_definition/aurum_grammar_spec.rb
+- spec/engine
+- spec/engine/lexer_spec.rb
+- spec/engine/parser_spec.rb
+- spec/examples
+- spec/examples/dangling_else_example.rb
+- spec/examples/expression_example.rb
+- spec/examples/smalltalk_example.rb
+- spec/examples/yacc_spec.rb
+- spec/grammar
+- spec/grammar/builder
+- spec/grammar/builder/lexical_table
+- spec/grammar/builder/lexical_table/automata_spec.rb
+- spec/grammar/builder/lexical_table/builder_spec.rb
+- spec/grammar/builder/lexical_table/character_set_spec.rb
+- spec/grammar/builder/lexical_table/pattern_spec.rb
+- spec/grammar/builder/lexical_table/regular_expression.rb
+- spec/grammar/builder/parsing_table
+- spec/grammar/builder/parsing_table/augmented_grammar_spec.rb
+- spec/grammar/builder/parsing_table/builder_spec.rb
+- spec/grammar/builder/parsing_table/digraph_traverser_spec.rb
+- spec/grammar/builder/parsing_table/item_spec.rb
+- spec/grammar/builder/parsing_table/sources_spec.rb
+- spec/grammar/builder/parsing_table/state_spec.rb
+- spec/grammar/dsl
+- spec/grammar/dsl/character_classes_builder_spec.rb
+- spec/grammar/dsl/lexical_rules_builder_spec.rb
+- spec/grammar/dsl/precedence_builder_spec.rb
+- spec/grammar/dsl/productions_builder_spec.rb
+- spec/grammar/metalang
+- spec/grammar/metalang/metalang_spec.rb
+- spec/grammar/precedence_spec.rb
+- spec/grammar/syntax_rules_spec.rb
+- spec/parser_matcher.rb
+- spec/pattern_matcher.rb
+- spec/spec_helper.rb
+- Rakefile
 test_files: []
 
 rdoc_options: []

data/example/expression/expression.rb

@@ -1,35 +0,0 @@
-$:.unshift(File.dirname(__FILE__) + '/../../lib')
-require 'aurum'
-
-class ExpressionGrammar < Aurum::Grammar
-	tokens do
-		ignore string(' ').one_or_more 
-		_number range(?0, ?9).one_or_more
-	end
-	
-	precedences do
-		operator '*', '/'
-		operator '+', '-'
-	end
-	
-	productions do
-		expression expression, '+', expression {expression.value = expression1.value + expression2.value}
-		expression expression, '-', expression {expression.value = expression1.value - expression2.value}
-		expression expression, '*', expression {expression.value = expression1.value * expression2.value}
-		expression expression, '/', expression {expression.value = expression1.value / expression2.value}
-		expression '(', expression, ')'
-		expression _number {expression.value = _number.value.to_i}
-		expression '+', _number {expression.value = _number.value.to_i}
-		expression '-', _number {expression.value = -_number.value.to_i}
-	end		
-end
-
-puts ExpressionGrammar.parse_expression('1 + 2').value
-puts ExpressionGrammar.parse_expression('(1 + 2)').value
-puts ExpressionGrammar.parse_expression('-1 - 2').value
-puts ExpressionGrammar.parse_expression('1 + 2 * 3').value
-puts ExpressionGrammar.parse_expression('1 * 2 + 3').value
-puts ExpressionGrammar.parse_expression('1 * (2 + 3)').value
-puts ExpressionGrammar.parse_expression('1 + (2 + 3) * 4').value
-puts ExpressionGrammar.parse_expression('1 * - 5').value
-puts ExpressionGrammar.parse_expression('(1+3) - - 5').value

data/example/expression/lisp.rb

@@ -1,26 +0,0 @@
-$:.unshift(File.dirname(__FILE__) + '/../../lib')
-require 'aurum'
-
-class LispGrammar < Aurum::Grammar
-	tokens do
-		ignore string(' ').one_or_more 
-		_number range(?0, ?9).one_or_more
-	end
-		
-	productions do
-		expression '(', tuple, ')'
-		tuple '+', atom, atom {tuple.value = atom1.value + atom2.value}
-		tuple tuple, atom {tuple.value = tuple1.value + atom.value}
-		atom _number {atom.value = _number.value.to_i}
-		atom expression
-	end		
-end
-
-puts LispGrammar.parse_expression('(+ 1 3 4) ').value
-#puts LispGrammar.parse_expression('-1 - 2').value
-#puts LispGrammar.parse_expression('1 + 2 * 3').value
-#puts LispGrammar.parse_expression('1 * 2 + 3').value
-#puts LispGrammar.parse_expression('1 * (2 + 3)').value
-#puts LispGrammar.parse_expression('1 + (2 + 3) * 4').value
-#puts LispGrammar.parse_expression('1 * - 5').value
-#puts LispGrammar.parse_expression('(1+3) - - 5').value

data/lib/aurum/lexical_table_generator.rb

@@ -1,429 +0,0 @@
-module Aurum
-	RecognizeTokenAction, ChangeStateAction, UserDefinedAction = Struct.new(:token), Struct.new(:state), Struct.new(:action)
-	RecognizeTokenAndChangeStateAction = Struct.new :token, :state
-	IgnoreAction = RecognizeTokenAction.new '$ignore'
-
-    class LexicalTableGenerator
-        attr_reader :lexical_states
-        def initialize specification
-            @specification, @accept_states  = specification, {}
-            @lexical_states = @specification.keys - [:all]
-            @patterns_for_all = specification[:all] ? specification[:all] : {}
-        end
-
-        def lexical_table
-            construct_automata			
-            make_initial_partitions
-            refine_partitions
-            @partitions.size < @lexical_automata.table.size ? construct_minimize_automata : [@lexical_automata.table, @accept_states]
-        end
-
-        private
-        def construct_automata
-            automata, accepts = Automata.new(1), {}, {}
-            @lexical_states.each_with_index do |lexcial_state, index|
-                lexical_state_start = automata.new_state
-                automata.connect 0, CharacterSet::Interval.new(-index - 1).to_char_set, lexical_state_start
-                @patterns_for_all.merge(@specification[lexcial_state]).each do |pattern, action|
-                    pattern_start = automata.merge! pattern.automata
-                    automata.connect lexical_state_start, Epsilon, pattern_start
-                    accepts[pattern_start + pattern.accept] = action
-                end
-            end
-            @lexical_automata, deterministic_accepts = automata.determinize accepts.keys
-            deterministic_accepts.each {|d, n| @accept_states[d] = n.inject([]){|r, x| r << accepts[x]}}
-        end
-
-        def make_initial_partitions
-            partitions = {}
-            @accept_states.each do |state, action|
-                partitions[action] = [] unless partitions.has_key? action
-                partitions[action] << state
-            end
-            @partitions = [[0], @lexical_automata.all_states - @accept_states.keys - [0]] + partitions.values
-            @partitions.delete []
-        end
-
-        def refine_partitions
-            reverse_automata, working_list = @lexical_automata.reverse, @partitions.dup
-            until working_list.empty?
-                reverse_automata.alphabet(working_list.pop) do |ia, symbols|
-                    @partitions.grep_each 'x.size > 1' do |r|
-                        r1, r2 = r & ia, r - ia
-                        unless r2.empty? || r2 == r
-                            replace @partitions, r => [r1, r2]
-                            if working_list.include? r
-                                replace working_list, r => [r1, r2]
-                            else
-                                working_list << (r1.size <= r2.size ? r1 : r2)
-                            end
-                            working_list.uniq!
-                        end
-                    end
-                end
-            end
-        end
-
-        def construct_minimize_automata
-            automata, accepts = Automata.new(@partitions.size), {}
-            choose_representatives do |representative, index|
-                @lexical_automata.table[representative].each do |transition|
-                    automata.connect index, transition.symbols, partition_contains(transition.destination)
-                end
-                accepts[index] = @accept_states[representative] if @accept_states.has_key? representative
-            end
-            return automata.table, accepts
-        end
-
-        def choose_representatives
-            @partitions.each_with_index {|partition, index| yield partition.first, index}
-        end
-
-        def partition_contains state
-            @partitions.each_with_index {|partition, index| return index if partition.include? state}
-        end
-
-        def replace array, replacements
-            replacements.each do |old, new|
-                array.delete old
-                new.each {|x| array << x}
-            end
-        end
-    end
-
-    class Pattern
-        attr_reader :automata, :accept
-        def self.from_string literal
-            automata, index = Automata.new(literal.length + 1), 0
-            literal.each_byte {|byte|automata.connect index, CharacterSet::Interval.new(byte).to_char_set, (index += 1)}
-            new automata, index
-        end
-
-        def self.from_char_set set
-            automata = Automata.new 2
-            automata.connect 0, set, 1
-            new automata, 1
-        end
-		
-		def self.from_enum enum_literal
-			automata = Automata.new enum_literal.length + 2
-			enum_literal.each_byte {|byte| automata.connect 0, CharacterSet::Interval.new(byte).to_char_set, 1}
-			new automata, 1
-		end		
-
-        def self.concat *patterns
-            automata, index = Automata.new, 0
-            patterns.each do |pattern|
-                index = automata.connect(index, Epsilon, automata.merge!(pattern.automata)) + pattern.accept
-            end
-            new automata, index
-        end
-
-        def initialize automata, accept
-            @automata, @accept = automata, accept
-        end
-
-        def kleene
-            kleene_automata = @automata.dup
-            kleene_automata.connect 0, Epsilon, @accept
-            kleene_automata.connect @accept, Epsilon, 0
-            Pattern.new kleene_automata, @accept
-        end
-        alias :zero_or_more :kleene
-
-        def iterate
-            iterate_automata = @automata.dup
-            iterate_automata.connect @accept, Epsilon, 0
-            Pattern.new iterate_automata, @accept
-        end
-		alias :one_or_more :iterate
-
-        def opt
-            opt_automata = @automata.dup
-            opt_automata.connect 0, Epsilon, @accept
-            Pattern.new opt_automata, @accept
-        end
-        alias :zero_or_one :opt
-
-        def negate
-            deterministic, accepts = automata.determinize [@accept]
-            sink = deterministic.new_state
-            deterministic.connect sink, CharacterSet.any, sink
-            sink.times do |state|
-                joint = CharacterSet.any
-                deterministic.table[state].each {|tran| joint.delete tran.symbols}
-                deterministic.connect state, joint, sink unless joint.empty?
-            end
-            accept = deterministic.new_state
-            accept.times {|state| deterministic.connect state, Epsilon, accept unless accepts.include? state }
-            Pattern.new deterministic, accept
-        end
-        alias :not :negate
-
-        def [] least, most = least
-            Pattern.concat *([self] * least + [self.opt] * (most-least))
-        end
-
-        def | other
-            automata = Automata.new 2
-            [self, other].each do |pattern|
-                automata.connect automata.connect(0, Epsilon, automata.merge!(pattern.automata)) + pattern.accept, Epsilon, 1
-            end
-            Pattern.new automata, 1
-        end
-
-        def ~
-            any = Pattern.from_char_set(CharacterSet.any).kleene
-            return Pattern.concat(Pattern.concat(any, self, any).negate, self)
-        end
-    end
-
-    class Automata
-        attr_reader :table
-        Transition = Struct.new(:symbols, :destination)
-
-        def initialize(table=[])
-            case table
-            when Array
-                @table = table
-            when Fixnum
-                @table = []
-                table.times {@table << []}
-            end
-        end
-
-        def connect start, symbols, destination
-            @table[start]  << Transition.new(symbols, destination)
-            destination
-        end
-
-        def merge! other
-            start = @table.length
-            other_table = other.instance_eval{@table}
-            other_table.each do |trans|
-                @table << []
-                trans.each {|tran| @table.last << Transition.new(tran.symbols, tran.destination + start)}
-            end
-            start
-        end
-
-        def reverse
-            reverse = []
-            @table.length.times {reverse << []}
-            @table.each_with_index do |trans, index|
-                trans.each {|tran| reverse[tran.destination] << Transition.new(tran.symbols, index)}
-            end
-            Automata.new reverse
-        end
-
-        def dup
-            dup_table = []
-            @table.each {|x| dup_table << x.dup}
-            Automata.new dup_table
-        end
-
-        def alphabet states
-            points = states.inject([]) do |result, state|
-                @table[state].inject(result){|r, s|r += s.symbols.to_points s.destination}
-            end
-            points.sort! do |x, y|
-                x.char == y.char ? (x.is_start ? (y.is_start ? 0 : -1) : (y.is_start ? 1 : 0)) : (x.char < y.char ? -1 : 1)
-            end
-            reachable_states = []
-            points.each_with_index do |point, index|
-                if point.is_start
-                    reachable_states << point.destination
-                else
-                    reachable_states.delete point.destination
-                    next if reachable_states.empty?
-                end
-                symbols = range(point, points[index + 1])
-                yield reachable_states.uniq, symbols if symbols
-            end
-        end
-
-        def determinize accepts
-            SubsetDeterminizer.new(self, accepts).determinize
-        end
-
-        def new_state
-            @table << []
-            @table.length - 1
-        end
-
-        def all_states
-            (0..table.length - 1).to_a
-        end
-
-        private
-        def range point_a, point_b
-            start_point = point_a.is_start ? point_a.char : (point_a.char + 1)
-            end_point = point_b.is_start ? point_b.char - 1 : point_b.char
-            start_point > end_point ? nil : CharacterSet::Interval.new(start_point, end_point).to_char_set
-        end
-    end
-
-    class SubsetDeterminizer
-        def initialize nondeterministic, accepts
-            @unmarked, @dstates, @accepts = [], [], accepts
-            @nondeterministic, @deterministic, @accept_states = nondeterministic, Automata.new, {}
-            unmark closure([0])
-        end
-
-        def determinize
-            until @unmarked.empty?
-                start = @unmarked.pop
-                @nondeterministic.alphabet(@dstates[start]) do |states, symbols|
-                    destination_state = closure(states)
-                    destination = unmark destination_state unless destination = @dstates.index(destination_state)
-                    @deterministic.connect start, symbols, destination
-                end
-            end
-            return @deterministic, @accept_states
-        end
-        private
-        def unmark states
-            @dstates << states
-            @unmarked.push @deterministic.new_state
-            accepts = states.find_all {|x| @accepts.include? x}
-            @accept_states[@unmarked.last] = accepts unless accepts.empty?
-            @unmarked.last
-        end
-
-        def closure states
-            closure, unvisited = states.dup, states.dup
-            until unvisited.empty? do
-                @nondeterministic.table[unvisited.pop].each do |tran|
-                    if tran.symbols == Epsilon && !closure.include?(tran.destination)
-                        closure << tran.destination
-                        unvisited << tran.destination
-                    end
-                end
-            end
-            closure.sort!
-        end
-    end
-
-    class CharacterSet
-        attr_reader :intervals
-        def self.any
-            Interval.new(0, 65535).to_char_set
-        end
-
-        def + other
-            result = self.dup
-            if (other.kind_of? CharacterSet)
-                for interval in other.intervals
-                    result.add_interval interval.first, interval.last
-                end
-            else
-                other.to_s.each_byte do |byte|
-                    result.add_interval byte
-                end
-            end
-            result
-        end
-
-        def - other
-            result = self.dup
-            if (other.kind_of? CharacterSet)
-                for interval in other.intervals
-                    result.delete_interval interval.first, interval.last
-                end
-            else
-                other.to_s.each_byte do |byte|
-                    result.delete_interval byte
-                end
-            end
-            result
-        end
-
-        def initialize *intervals
-            @intervals = intervals
-        end
-
-        def << obj
-            obj.kind_of?(Range) ? add_interval(obj.first, obj.last) : obj.to_s.each_byte {|x| add_interval x}
-        end
-
-        def delete obj
-            case obj
-            when Range
-                delete_interval obj.first, obj.last
-            when Aurum::CharacterSet
-                obj.intervals.each {|interval| delete_interval interval.first, interval.last}
-            else
-                obj.to_s.each_byte {|x| delete_interval x}
-            end
-        end
-
-        def include? char
-            @intervals.any? {|x| x.include? char}
-        end
-
-        def empty?
-            return @intervals.empty?
-        end
-
-        def to_points destination
-            @intervals.inject [] do |points, interval|
-                points << Point.new(interval.first, true, destination)
-                points << Point.new(interval.last, false, destination)
-            end
-        end
-
-        def dup
-            intervals = []
-            for interval in @intervals
-                intervals << interval.dup
-            end
-            CharacterSet.new *intervals
-        end
-
-        protected
-        def add_interval first, last = first
-            interval = Interval.new first, last
-            @intervals << interval unless @intervals.any? {|x| x.merge! interval}
-        end
-
-        def delete_interval first, last = first
-            interval = Interval.new first, last
-            return unless to_be_replaced = @intervals.find {|x| x.include?(interval.first) || x.include?(interval.last)}
-            @intervals.delete to_be_replaced
-            add_new_interval to_be_replaced.first, interval.first - 1
-            add_new_interval interval.last + 1, to_be_replaced.last
-        end
-
-        def add_new_interval first, last
-            @intervals << Interval.new(first, last) if first <= last
-        end
-
-        Interval, Point = Struct.new(:first, :last), Struct.new(:char, :is_start, :destination)
-
-        Interval.class_eval do
-            def initialize first, last = first
-                super first, last
-            end
-
-            def include? char
-                char = char[0] if char.kind_of? String
-                self.first <= char && char <= self.last
-            end
-
-            def merge! other
-                if include?(other.first) || include?(other.last) || other.first - self.last == 1 || self.first - other.last == 1
-                    self.first = [self.first, other.first].min
-                    self.last = [self.last, other.last].max
-                    return true;
-                end
-                false
-            end
-
-            def to_char_set
-                CharacterSet.new self
-            end
-        end
-    end
-
-    Epsilon = CharacterSet.new
-end