keyword_search 1.1.1 → 1.2.0

data/History.txt

@@ -1,3 +1,7 @@
+= 1.2.0 / 2007-05-09
+
+* Raises KeywordSearch::ParseError instead of returning an empty Hash if an error occurs during parsing
+
 = 1.1.0 / 2007-03-21
 
 * Updated to use Dhaka 2.1.0

data/Rakefile

@@ -23,4 +23,20 @@ task :rebuild_parser do
   end
 end
 
+task :rebuild_lexer do
+  require 'dhaka'
+  lexer = Dhaka::Lexer.new(KeywordSearch::LexerSpec)
+  File.open('lib/keyword_search/lexer.rb', 'w') do |file|
+    file << lexer.compile_to_ruby_source_as('KeywordSearch::Lexer')
+  end
+end
+
+task :rebuild_lexer do
+  require 'dhaka'
+  lexer = Dhaka::Lexer.new(KeywordSearch::LexerSpec)
+  File.open('lib/keyword_search/lexer.rb', 'w') do |file|
+    file << lexer.compile_to_ruby_source_as('KeywordSearch::Lexer')
+  end
+end
+
 # vim: syntax=Ruby

data/lib/keyword_search.rb

@@ -1,20 +1,21 @@
 require 'dhaka'
 
 dirname = File.join(File.dirname(__FILE__), 'keyword_search')
-%w|grammar tokenizer parser evaluator definition|.each do |dependency|
+%w|grammar parser lexer_spec lexer evaluator definition|.each do |dependency|
   require File.join(dirname, dependency)
 end
 
 module KeywordSearch
   
-  VERSION = '1.1.1'
+  class ParseError < ::SyntaxError; end
+  
+  VERSION = '1.2.0'
   
   class << self
     def search(input_string, definition=nil, &block)
       @evaluator ||= Evaluator.new
       definition ||= Definition.new(&block)
-      tokens = Tokenizer.tokenize(input_string)
-      parse_result = Parser.parse(tokens)
+      parse_result = Parser.parse(Lexer.lex(input_string))
       unless parse_result.has_error?
         results = @evaluator.evaluate(parse_result.parse_tree)
         results.each do |key, terms|
@@ -22,7 +23,7 @@ module KeywordSearch
         end
         results
       else
-        {}
+        raise ParseError, "Unexpected token #{parse_result.unexpected_token.inspect}"
       end
     end
   end

data/lib/keyword_search/grammar.rb

@@ -12,7 +12,7 @@ module KeywordSearch
     end
   
     for_symbol 'Pair' do
-      keyword_and_term ['k', 's']
+      keyword_and_term ['s', ':', 's']
       default_keyword_term ['s']
     end
     

data/lib/keyword_search/parser.rb

@@ -4,38 +4,37 @@ class KeywordSearch::Parser < Dhaka::CompiledParser
 
   start_with 0
 
-  at_state(3) {
-    for_symbols("k", "_End_", "s") { reduce_with "one_pair" }
-  }
-
   at_state(1) {
-    for_symbols("s") { shift_to 2 }
+    for_symbols("_End_") { reduce_with "start" }
+    for_symbols("s") { shift_to 3 }
+    for_symbols("Pair") { shift_to 2 }
   }
 
-  at_state(6) {
-    for_symbols("k", "_End_", "s") { reduce_with "multiple_pairs" }
+  at_state(5) {
+    for_symbols("_End_", "s") { reduce_with "keyword_and_term" }
   }
 
-  at_state(5) {
-    for_symbols("s") { shift_to 4 }
-    for_symbols("_End_") { reduce_with "start" }
-    for_symbols("k") { shift_to 1 }
-    for_symbols("Pair") { shift_to 6 }
+  at_state(4) {
+    for_symbols("s") { shift_to 5 }
   }
 
   at_state(2) {
-    for_symbols("k", "_End_", "s") { reduce_with "keyword_and_term" }
+    for_symbols("_End_", "s") { reduce_with "multiple_pairs" }
   }
 
   at_state(0) {
-    for_symbols("s") { shift_to 4 }
-    for_symbols("k") { shift_to 1 }
-    for_symbols("Pair") { shift_to 3 }
-    for_symbols("Pairs") { shift_to 5 }
+    for_symbols("Pair") { shift_to 6 }
+    for_symbols("s") { shift_to 3 }
+    for_symbols("Pairs") { shift_to 1 }
   }
 
-  at_state(4) {
-    for_symbols("k", "_End_", "s") { reduce_with "default_keyword_term" }
+  at_state(6) {
+    for_symbols("_End_", "s") { reduce_with "one_pair" }
+  }
+
+  at_state(3) {
+    for_symbols(":") { shift_to 4 }
+    for_symbols("_End_", "s") { reduce_with "default_keyword_term" }
   }
 
 end

data/lib/keyword_search/tokenizer.rb

@@ -1,105 +0,0 @@
-module KeywordSearch
-  
-  class Tokenizer < Dhaka::Tokenizer
-    
-    def accumulator
-      @accumulator ||= ''
-    end
-    
-    def accumulate(string)
-      accumulator << string
-    end
-    
-    def clear_accumulator
-      @accumulator = ''
-    end
-
-
-    # TODO: Add further character support; this is just for initial release
-    letters = ('a'..'z').to_a + ('A'..'Z').to_a
-    numbers = ('0'..'9').to_a
-    extras = %w|_ - ' / \ [ ] { } 1 @ # $ % ^ & * ( ) . , ? < > |
-    printables = letters + numbers + extras
-    whitespace = [' ']
-    quotes = %w|' "|
-    keyword_separator = [':']
-    all_characters = keyword_separator + printables + whitespace + quotes
-
-    for_state :idle_state do
-    
-      for_characters(printables) do
-        clear_accumulator
-        switch_to :unquoted_literal_state
-      end
-
-      for_characters(quotes) do
-        advance unless accumulator.empty?
-        clear_accumulator
-        case curr_char
-        when %<">
-          advance
-          switch_to :double_quoted_literal_state
-        when %<'>
-          advance
-          switch_to :single_quoted_literal_state
-        end
-      end
-    
-      for_characters whitespace do
-        advance
-      end
-    
-    end
-  
-    for_state :unquoted_literal_state do
-    
-      for_characters(printables) do
-        accumulate curr_char
-        advance
-        create_token('s', accumulator) unless curr_char
-      end
-    
-      for_characters(keyword_separator) do
-        create_token 'k', accumulator
-        clear_accumulator
-        advance
-        switch_to :idle_state
-      end
-    
-      for_characters(whitespace) do
-        create_token 's', accumulator
-        clear_accumulator
-        switch_to :idle_state
-      end
-    
-    end
-  
-    for_state :double_quoted_literal_state do
-      for_characters(all_characters - %w<">) do
-        accumulate curr_char
-        advance
-      end
-      for_characters %w<"> do
-        create_token 's', accumulator
-        clear_accumulator
-        advance
-        switch_to :idle_state
-      end
-    end
-
-    for_state :single_quoted_literal_state do
-      for_characters(all_characters - %w<'>) do
-        accumulate curr_char
-        advance
-      end
-      for_characters %w<'> do
-        create_token 's', accumulator
-        clear_accumulator
-        advance
-        switch_to :idle_state
-      end
-    end
-
-  end
-
-end

data/test/test_keyword_search.rb

@@ -1,5 +1,5 @@
 require 'test/unit'
-require 'lib/keyword_search'
+require File.dirname(__FILE__) + '/../lib/keyword_search'
   
 class TestKeywordSearch < Test::Unit::TestCase
   
@@ -129,6 +129,17 @@ class TestKeywordSearch < Test::Unit::TestCase
     assert_equal %<okay...>, result    
   end
   
+  def test_parse_error_results_in_exception
+    assert_raises(KeywordSearch::ParseError) do
+      KeywordSearch.search(%<we_do_not_allow:! or ::>) do |with|
+        with.default_keyword :text
+        with.keyword :text do |values|
+          result = values.first
+        end
+      end
+    end
+  end
+  
 end
 
 

metadata

@@ -3,8 +3,8 @@ rubygems_version: 0.9.0
 specification_version: 1
 name: keyword_search
 version: !ruby/object:Gem::Version 
-  version: 1.1.1
-date: 2007-03-21 00:00:00 -06:00
+  version: 1.2.0
+date: 2007-05-09 00:00:00 -06:00
 summary: Generic support for extracting GMail-style search keywords/values from strings
 require_paths: 
 - lib