dhaka 2.0.1 → 2.1.0

data/test/parser_state_test.rb

@@ -21,16 +21,4 @@ class TestParserState < Test::Unit::TestCase
     state = ParserState.new(nil, {item1 => item1, item2 => item2})
     assert_equal(["i'm item 1", "i'm item 2"], state.to_s(options).split("\n").sort)
   end
-  
-  def test_to_dot_method
-    options = {:some_option => true}
-    item1 = mock()
-    item1.stubs(:to_s).with(options).returns("i'm item 1")
-    item2 = mock()
-    item2.stubs(:to_s).with(options).returns("i'm item 2")
-    state = ParserState.new(nil, {item1 => item1, item2 => item2})
-    state.stubs(:unique_name).returns("State1")
-    assert(state.to_dot(options) =~ /^State1 \[label="(.+)\\n(.+)"\]/)
-    assert_equal(["i'm item 1", "i'm item 2"], [$1, $2].sort)
-  end
 end

metadata

@@ -3,8 +3,8 @@ rubygems_version: 0.9.0
 specification_version: 1
 name: dhaka
 version: !ruby/object:Gem::Version 
-  version: 2.0.1
-date: 2007-02-11 00:00:00 -05:00
+  version: 2.1.0
+date: 2007-03-11 00:00:00 -05:00
 summary: An LALR1 parser generator written in Ruby
 require_paths: 
 - lib
@@ -30,12 +30,24 @@ authors:
 - Mushfeq Khan
 files: 
 - lib/dhaka.rb
+- lib/dot/dot.rb
 - lib/evaluator/evaluator.rb
 - lib/grammar/closure_hash.rb
 - lib/grammar/grammar.rb
 - lib/grammar/grammar_symbol.rb
 - lib/grammar/precedence.rb
 - lib/grammar/production.rb
+- lib/lexer/compiled_lexer.rb
+- lib/lexer/dfa.rb
+- lib/lexer/lexeme.rb
+- lib/lexer/lexer.rb
+- lib/lexer/lexer_run.rb
+- lib/lexer/regex_grammar.rb
+- lib/lexer/regex_parser.rb
+- lib/lexer/regex_tokenizer.rb
+- lib/lexer/specification.rb
+- lib/lexer/state.rb
+- lib/lexer/state_machine.rb
 - lib/parser/action.rb
 - lib/parser/channel.rb
 - lib/parser/compiled_parser.rb
@@ -57,6 +69,7 @@ files:
 - test/fake_logger.rb
 - test/grammar_test.rb
 - test/lalr_but_not_slr_grammar.rb
+- test/lexer_test.rb
 - test/malformed_grammar.rb
 - test/malformed_grammar_test.rb
 - test/nullable_grammar.rb
@@ -78,8 +91,8 @@ files:
 - test/arithmetic_precedence/arithmetic_precedence_evaluator.rb
 - test/arithmetic_precedence/arithmetic_precedence_grammar.rb
 - test/arithmetic_precedence/arithmetic_precedence_grammar_test.rb
+- test/arithmetic_precedence/arithmetic_precedence_lexer_specification.rb
 - test/arithmetic_precedence/arithmetic_precedence_parser_test.rb
-- test/arithmetic_precedence/arithmetic_precedence_tokenizer.rb
 - test/brackets/bracket_grammar.rb
 - test/brackets/bracket_tokenizer.rb
 - test/brackets/brackets_test.rb
@@ -88,10 +101,13 @@ files:
 - test/chittagong/chittagong_evaluator.rb
 - test/chittagong/chittagong_evaluator_test.rb
 - test/chittagong/chittagong_grammar.rb
+- test/chittagong/chittagong_lexer.rb
+- test/chittagong/chittagong_lexer_specification.rb
+- test/chittagong/chittagong_lexer_test.rb
+- test/chittagong/chittagong_parser.rb
 - test/chittagong/chittagong_parser_test.rb
 - test/chittagong/chittagong_test.rb
-- test/chittagong/chittagong_tokenizer.rb
-- test/chittagong/chittagong_tokenizer_test.rb
+- Rakefile
 test_files: []
 
 rdoc_options: []

data/test/arithmetic_precedence/arithmetic_precedence_tokenizer.rb

@@ -1,39 +0,0 @@
-require File.dirname(__FILE__) + '/arithmetic_precedence_grammar'
-
-class ArithmeticPrecedenceTokenizer < Dhaka::Tokenizer
-  
-  digits        = ('0'..'9').to_a
-  parenths      = %w| ( ) |
-  operators     = %w| - + / * ^ |
-  functions     = %w| h l |
-  arg_separator = %w| , |
-  whitespace    = [' ']
-
-  all_characters = digits + parenths + operators + functions + arg_separator + whitespace
-
-  for_state Dhaka::TOKENIZER_IDLE_STATE do
-    for_characters(all_characters - (digits + whitespace)) do
-      create_token(curr_char, nil)
-      advance
-    end
-    for_characters digits do
-      create_token('n', '')
-      switch_to :get_integer_literal
-    end
-    for_character whitespace do
-      advance
-    end
-  end
-  
-  for_state :get_integer_literal do
-    for_characters all_characters - digits do
-      switch_to Dhaka::TOKENIZER_IDLE_STATE
-    end
-    for_characters digits do
-      curr_token.value << curr_char
-      advance
-    end
-  end
-  
-end
-

data/test/chittagong/chittagong_tokenizer.rb

@@ -1,88 +0,0 @@
-require File.dirname(__FILE__) + '/chittagong_grammar'
-
-class ChittagongTokenizer < Dhaka::Tokenizer
-
-  KEYWORDS      = %w| print if else end while def return |
-  
-  digits        = ('0'..'9').to_a
-  decimal_point = %w| . |
-  letters       = ('a'..'z').to_a
-  parenths      = %w| ( ) |
-  operators     = %w| - + / * ^ ! > < |
-  equal_sign    = %w| = |
-  whitespace    = [' ']
-  arg_separator = %w| , |
-  newline       = ["\n"]
-  
-  all_characters = digits + decimal_point + letters + parenths + operators + whitespace + newline + arg_separator + equal_sign
-
-  for_state Dhaka::TOKENIZER_IDLE_STATE do
-    for_characters(all_characters - (digits + letters + newline + whitespace + equal_sign)) do
-      create_token(curr_char, nil)
-      advance
-    end
-    for_characters(equal_sign) do
-      create_token('=', nil)
-      advance
-      switch_to :get_equal_sign_operator
-    end
-    for_characters digits do
-      create_token('int_literal', '')
-      switch_to :get_integer_literal
-    end
-    for_characters letters do
-      create_token(nil, '')
-      switch_to :get_word_literal
-    end
-    for_character newline do
-      create_token('newline', nil)
-      advance
-    end
-    for_character whitespace do
-      advance
-    end
-  end
-  
-  for_state :get_equal_sign_operator do
-    for_characters all_characters - equal_sign do
-      switch_to Dhaka::TOKENIZER_IDLE_STATE
-    end
-    for_character equal_sign do
-      curr_token.symbol_name << '='
-      advance
-      switch_to Dhaka::TOKENIZER_IDLE_STATE
-    end
-  end
-  
-  for_state :get_integer_literal do
-    for_characters all_characters - digits do
-      switch_to Dhaka::TOKENIZER_IDLE_STATE
-    end
-    for_characters digits do
-      curr_token.value << curr_char
-      advance
-    end
-  end
-  # lipi:keywords
-  for_state :get_word_literal do
-    for_characters all_characters - letters do
-      curr_token.symbol_name = word_literal_symbol(curr_token.value)
-      switch_to Dhaka::TOKENIZER_IDLE_STATE
-    end
-    for_characters letters do
-      curr_token.value << curr_char
-      advance
-      curr_token.symbol_name = word_literal_symbol(curr_token.value) unless curr_char
-    end
-  end
-  
-  def word_literal_symbol(value)
-    if KEYWORDS.include? value
-      value
-    else
-      'word_literal'
-    end
-  end
-  # lipi:keywords
-  
-end