keyword_search 1.0.4 → 1.0.5

data/History.txt

@@ -1,3 +1,8 @@
+= 1.0.5 / 2007-01-29
+
+* Added single quoting support and nested quoting (apostrophes, etc)
+* Added a few more punctuation marks as valid input
+
 = 1.0.4 / 2007-01-15
 
 * Updated Dhaka dependency to lock to version 0.0.6 due to breakage of backwards compatibility in 1.0.0.

data/lib/keyword_search.rb

@@ -7,7 +7,7 @@ end
 
 module KeywordSearch
   
-  VERSION = '1.0.4'
+  VERSION = '1.0.5'
   
   class << self
     def search(input_string, definition=nil, &block)

data/lib/keyword_search/parser.rb

@@ -4,46 +4,46 @@ class KeywordSearch::Parser < Dhaka::CompiledParser
 
   start_with 0
 
-  at_state(3) {
-    for_symbol('k') { reduce_with 'default_keyword_term' }
-    for_symbol('_End_') { reduce_with 'default_keyword_term' }
-    for_symbol('s') { reduce_with 'default_keyword_term' }
-  }
-
-  at_state(0) {
-    for_symbol('k') { shift_to 4 }
-    for_symbol('Pairs') { shift_to 1 }
-    for_symbol('s') { shift_to 3 }
+  at_state(5) {
+    for_symbol('k') { shift_to 2 }
+    for_symbol('_End_') { reduce_with 'start' }
     for_symbol('Pair') { shift_to 6 }
+    for_symbol('s') { shift_to 1 }
   }
 
-  at_state(4) {
-    for_symbol('s') { shift_to 5 }
+  at_state(2) {
+    for_symbol('s') { shift_to 3 }
   }
 
-  at_state(5) {
+  at_state(3) {
     for_symbol('k') { reduce_with 'keyword_and_term' }
     for_symbol('_End_') { reduce_with 'keyword_and_term' }
     for_symbol('s') { reduce_with 'keyword_and_term' }
   }
 
-  at_state(1) {
-    for_symbol('k') { shift_to 4 }
-    for_symbol('_End_') { reduce_with 'start' }
-    for_symbol('s') { shift_to 3 }
-    for_symbol('Pair') { shift_to 2 }
+  at_state(6) {
+    for_symbol('k') { reduce_with 'multiple_pairs' }
+    for_symbol('_End_') { reduce_with 'multiple_pairs' }
+    for_symbol('s') { reduce_with 'multiple_pairs' }
   }
 
-  at_state(6) {
+  at_state(4) {
     for_symbol('k') { reduce_with 'one_pair' }
     for_symbol('_End_') { reduce_with 'one_pair' }
     for_symbol('s') { reduce_with 'one_pair' }
   }
 
-  at_state(2) {
-    for_symbol('k') { reduce_with 'multiple_pairs' }
-    for_symbol('_End_') { reduce_with 'multiple_pairs' }
-    for_symbol('s') { reduce_with 'multiple_pairs' }
+  at_state(1) {
+    for_symbol('k') { reduce_with 'default_keyword_term' }
+    for_symbol('_End_') { reduce_with 'default_keyword_term' }
+    for_symbol('s') { reduce_with 'default_keyword_term' }
+  }
+
+  at_state(0) {
+    for_symbol('k') { shift_to 2 }
+    for_symbol('Pairs') { shift_to 5 }
+    for_symbol('Pair') { shift_to 4 }
+    for_symbol('s') { shift_to 1 }
   }
 
 end

data/lib/keyword_search/tokenizer.rb

@@ -6,12 +6,12 @@ module KeywordSearch
     # TODO: Add further character support; this is just for initial release
     letters = ('a'..'z').to_a
     numbers = ('0'..'9').to_a
-    extras = %w|_ - ' / \ [ ] { } 1 @ # $ % ^ & * ( )|
+    extras = %w|_ - ' / \ [ ] { } 1 @ # $ % ^ & * ( ) . , ? < > |
     printables = letters + numbers + extras
     whitespace = [' ']
-    quotes = ['"']
+    quotes = %w|' "|
     keyword_separator = [':']
-    all_characters = keyword_separator + printables + whitespace
+    all_characters = keyword_separator + printables + whitespace + quotes
 
     for_state :idle_state do
     
@@ -20,10 +20,17 @@ module KeywordSearch
         switch_to :unquoted_literal_state
       end
 
-      for_characters quotes do
-        self.accumulator = ''
-        advance
-        switch_to :quoted_literal_state
+      for_characters(quotes) do
+       advance if self.accumulator && !self.accumulator.empty?
+       self.accumulator = ''
+        case curr_char
+        when %<">
+          advance
+          switch_to :double_quoted_literal_state
+        when %<'>
+          advance
+          switch_to :single_quoted_literal_state
+        end
       end
     
       for_characters whitespace do
@@ -42,24 +49,40 @@ module KeywordSearch
     
       for_characters(keyword_separator) do
         tokens << Dhaka::Token.new(Grammar.symbol_for_name('k'), self.accumulator)
+        self.accumulator = ''
         advance
         switch_to :idle_state
       end
     
       for_characters(whitespace) do
         tokens << Dhaka::Token.new(Grammar.symbol_for_name('s'), self.accumulator)
+        self.accumulator = ''
         switch_to :idle_state
       end
     
     end
   
-    for_state :quoted_literal_state do
-      for_characters(all_characters - quotes) do
+    for_state :double_quoted_literal_state do
+      for_characters(all_characters - %w<">) do
         self.accumulator += curr_char
         advance
       end
-      for_characters quotes do
+      for_characters %w<"> do
         tokens << Dhaka::Token.new(Grammar.symbol_for_name('s'), self.accumulator)
+        self.accumulator = ''
+        advance
+        switch_to :idle_state
+      end
+    end
+
+    for_state :single_quoted_literal_state do
+      for_characters(all_characters - %w<'>) do
+        self.accumulator += curr_char
+        advance
+      end
+      for_characters %w<'> do
+        tokens << Dhaka::Token.new(Grammar.symbol_for_name('s'), self.accumulator)
+        self.accumulator = ''
         advance
         switch_to :idle_state
       end

data/test/test_keyword_search.rb

@@ -5,8 +5,10 @@ class TestKeywordSearch < Test::Unit::TestCase
   
   NAME_AND_AGE = %<bruce williams age:26>
   NAME_QUOTED_AND_AGE = %<"bruce williams" age:26>
-  NAME_AND_QUOTED_AGE = %<bruce williams age:"26">
+  NAME_AND_QUOTED_AGE = %<bruce williams age:"26">  
   DEFAULT_AGE_WITH_QUOTED_AGE = %<26 name:"bruce williams">
+  DEFAULT_AGE_WITH_SINGLE_QUOTED_AGE = %<26 name:'bruce williams'>
+  NAME_WITH_NESTED_SINGLE_QUOTES = %<"d'arcy d'uberville" age:28>
   
   def test_default_keyword
     result = nil
@@ -61,6 +63,72 @@ class TestKeywordSearch < Test::Unit::TestCase
     assert_equal 'bruce williams', result    
   end
   
+  def test_single_quoted_keyword_term_with_whitespace
+    result = nil
+    KeywordSearch.search(DEFAULT_AGE_WITH_SINGLE_QUOTED_AGE) do |with|
+      with.default_keyword :age
+      with.keyword :name do |values|
+        result = values.first
+      end
+    end
+    assert_equal 'bruce williams', result    
+  end
+  
+  def test_nested_single_quote_is_accumulated
+    result = nil
+    KeywordSearch.search(NAME_WITH_NESTED_SINGLE_QUOTES) do |with|
+      with.default_keyword :name
+      with.keyword :name do |values|
+        result = values.first
+      end
+    end
+    assert_equal %<d'arcy d'uberville>, result    
+  end
+  
+  def test_nested_double_quote_is_accumulated
+    result = nil
+    KeywordSearch.search(%<'he was called "jake"'>) do |with|
+      with.default_keyword :text
+      with.keyword :text do |values|
+        result = values.first
+      end
+    end
+    assert_equal %<he was called "jake">, result    
+  end
+  
+  def test_bare_single_quote_in_unquoted_literal_is_accumulated
+    result = nil
+    KeywordSearch.search(%<bruce's age:27>) do |with|
+      with.default_keyword :text
+      with.keyword :text do |values|
+        result = values.first
+      end
+    end
+    assert_equal %<bruce's>, result    
+  end
+  
+  def test_single_quoted_literal_is_accumulated
+    result = nil
+    KeywordSearch.search(%<foo 'bruce williams' age:27>) do |with|
+      with.default_keyword :text
+      with.keyword :text do |values|
+        result = values.last
+      end
+    end
+    assert_equal %<bruce williams>, result    
+  end
+  
+  def test_period_in_literal_is_accumulated
+    result = nil
+    KeywordSearch.search(%<okay... age:27>) do |with|
+      with.default_keyword :text
+      with.keyword :text do |values|
+        result = values.first
+      end
+    end
+    assert_equal %<okay...>, result    
+  end
+  
 end
 
 

metadata

@@ -3,8 +3,8 @@ rubygems_version: 0.9.0
 specification_version: 1
 name: keyword_search
 version: !ruby/object:Gem::Version 
-  version: 1.0.4
-date: 2007-01-15 00:00:00 -07:00
+  version: 1.0.5
+date: 2007-01-29 00:00:00 -07:00
 summary: Generic support for extracting GMail-style search keywords/values from strings
 require_paths: 
 - lib