keyword_search 1.0.0

data/History.txt

@@ -0,0 +1,3 @@
+== 1.0.0 / 2007-01-08
+
+* Initial release

data/Manifest.txt

@@ -0,0 +1,11 @@
+History.txt
+Manifest.txt
+README.txt
+Rakefile
+lib/keyword_search.rb
+lib/keyword_search/definition.rb
+lib/keyword_search/evaluator.rb
+lib/keyword_search/grammar.rb
+lib/keyword_search/parser.rb
+lib/keyword_search/tokenizer.rb
+test/test_keyword_search.rb

data/README.txt

@@ -0,0 +1,96 @@
+keyword_search
+    http://codefluency.rubyforge.org/keyword_search
+    by Bruce Williams
+
+== DESCRIPTION:
+  
+Generic library to parse GMail-style search strings for keyword/value pairs; supports definition of valid keywords and handling of quoted values.
+
+== FEATURES/PROBLEMS:
+  
+The library features a very simple, easy-to-use API.
+* Define handlers for supported keywords with blocks
+* Define the default keyword (values not part of a keyword/value pair)
+
+Various notes:
+* Quoted values are supported.
+* Input is automatically downcased (both keywords and values should be assumed to be in lowercase)
+
+Development Roadmap:
+1.1:: Expand supported character set for keywords and values
+      (currently supports a-z)
+2.0:: Add negation and grouping (will break API backwards compatibility)
+
+== SYNOPSIS:
+
+Here's an example of usage from Rails.  
+* Note that the library is generic, and could presumably be used for any Ruby project.
+
+  # Some variables to build up
+  clauses = []
+  arguments = []
+
+  # Search a string, defining the supported keywords and building up
+  # the variables in the associated closures
+  KeywordSearch.search('account has:attachment since:2006-12-03') do |with|
+
+    with.default_keyword :title
+  
+    with.keyword :title do |values|
+      clauses << "title like ?"
+      arguments << "%#{values.join(' ')}%"
+    end
+  
+    with.keyword :has do |values|
+      clauses << 'has_attachment = true' if values.include?('attachment')
+    end
+  
+    with.keyword :since do |values|
+      date = Date.parse(values.first) # only support one
+      clauses << 'created_on >= ?'
+      arguments << date.to_s
+    end
+  
+  end
+  
+  # Do our search with <tt>clauses</tt> and <tt>arguments</tt>
+  conditions = [clauses.map{|c| "(#{c})"}.join(' AND ')), *arguments] # simplistic example
+  results = Message.find(:all, :conditions => conditions)
+
+== REQUIREMENTS:
+
+* dhaka
+* hoe
+
+== INSTALL:
+
+sudo gem install keyword_search
+
+== LICENSE:
+
+(The MIT License)
+
+Copyright (c) 2007 Bruce Williams
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+'Software'), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+== LEGAL NOTES
+
+GMail is copyright Google, Inc.

data/Rakefile

@@ -0,0 +1,25 @@
+# -*- ruby -*-
+
+require 'rubygems'
+require 'hoe'
+require './lib/keyword_search.rb'
+
+Hoe.new('keyword_search', KeywordSearch::VERSION) do |p|
+  p.rubyforge_name = 'keyword_search'
+  p.summary = 'Generic support for extracting GMail-style search keywords/values from strings'
+  p.description = p.paragraphs_of('README.txt', 2..5).join("\n\n")
+  p.url = p.paragraphs_of('README.txt', 0).first.split(/\n/)[1..-1]
+  p.author = 'Bruce Williams'
+  p.email = 'bruce@codefluency.com'
+  p.changes = p.paragraphs_of('History.txt', 0..1).join("\n\n")
+end
+
+task :rebuild_parser do
+  require 'dhaka'
+  parser = Dhaka::Parser.new(KeywordSearch::Grammar)
+  File.open('lib/keyword_search/parser.rb', 'w') do |file|
+    file << parser.compile_to_ruby_source_as('KeywordSearch::Parser')
+  end
+end
+
+# vim: syntax=Ruby

data/lib/keyword_search.rb

@@ -0,0 +1,25 @@
+require 'dhaka'
+
+dirname = File.join(File.dirname(__FILE__), 'keyword_search')
+%w|grammar tokenizer parser evaluator definition|.each do |dependency|
+  require File.join(dirname, dependency)
+end
+
+module KeywordSearch
+  
+  VERSION = '1.0.0'
+  
+  class << self
+    def search(input_string, definition=nil, &block)
+      definition ||= Definition.new(&block)
+      tokens = Tokenizer.tokenize(input_string.downcase)
+      parse_result = Parser.parse(tokens)
+      unless parse_result.has_error?
+        Evaluator.new(parse_result.syntax_tree).result.each do |key, terms|
+          definition.handle(key, terms)
+        end
+      end
+    end
+  end
+  
+end

data/lib/keyword_search/definition.rb

@@ -0,0 +1,45 @@
+module KeywordSearch
+  
+  class Definition
+    
+    class Keyword
+      
+      attr_reader :name, :description
+      
+      def initialize(name, description=nil, &handler)
+        @name, @description = name, description
+        @handler = handler
+      end
+      
+      def handle(value)
+        @handler.call(value)
+      end
+      
+    end
+
+    attr_reader :keywords
+    def initialize
+      @keywords = []
+      @default_keyword = nil
+      yield self if block_given?
+    end
+
+    def keyword(name, description=nil, &block)
+      @keywords << Keyword.new(name, description, &block)
+    end
+      
+    def default_keyword(name)
+      @default_keyword = name
+    end
+      
+    def handle(key, values)
+      key = @default_keyword if key == :default
+      return false unless key
+      if k = @keywords.detect{|kw| kw.name == key.to_sym}
+        k.handle(values) 
+      end
+    end
+    
+  end
+  
+end

data/lib/keyword_search/evaluator.rb

@@ -0,0 +1,33 @@
+module KeywordSearch
+  
+  class Evaluator < Dhaka::Evaluator
+
+    self.grammar = Grammar
+
+    define_evaluation_rules do
+
+      for_multiple_pairs do
+        child_nodes[1].each do |key, terms|
+          child_nodes[0][key] ||= []
+          child_nodes[0][key] += terms
+        end
+        child_nodes[0]
+      end
+
+      for_one_pair do
+        child_nodes[0]
+      end
+
+      for_keyword_and_term do
+        {child_nodes[0].token.value => [child_nodes[1].token.value]}
+      end
+
+      for_default_keyword_term do
+        {:default => [child_nodes[0].token.value]}
+      end
+
+    end
+
+  end
+  
+end

data/lib/keyword_search/grammar.rb

@@ -0,0 +1,21 @@
+module KeywordSearch
+  
+  class Grammar < Dhaka::Grammar
+  
+    for_symbol(Dhaka::START_SYMBOL_NAME) do
+      start ['Pairs']
+    end
+  
+    for_symbol 'Pairs' do
+      one_pair ['Pair']
+      multiple_pairs ['Pairs', 'Pair']
+    end
+  
+    for_symbol 'Pair' do
+      keyword_and_term ['k', 's']
+      default_keyword_term ['s']
+    end
+    
+  end
+  
+end

data/lib/keyword_search/parser.rb

@@ -0,0 +1,49 @@
+class KeywordSearch::Parser < Dhaka::CompiledParser
+
+  self.grammar = KeywordSearch::Grammar
+
+  start_with 0
+
+  at_state(3) {
+    for_symbol('k') { reduce_with 'default_keyword_term' }
+    for_symbol('_End_') { reduce_with 'default_keyword_term' }
+    for_symbol('s') { reduce_with 'default_keyword_term' }
+  }
+
+  at_state(0) {
+    for_symbol('k') { shift_to 4 }
+    for_symbol('Pairs') { shift_to 1 }
+    for_symbol('s') { shift_to 3 }
+    for_symbol('Pair') { shift_to 6 }
+  }
+
+  at_state(4) {
+    for_symbol('s') { shift_to 5 }
+  }
+
+  at_state(5) {
+    for_symbol('k') { reduce_with 'keyword_and_term' }
+    for_symbol('_End_') { reduce_with 'keyword_and_term' }
+    for_symbol('s') { reduce_with 'keyword_and_term' }
+  }
+
+  at_state(1) {
+    for_symbol('k') { shift_to 4 }
+    for_symbol('_End_') { reduce_with 'start' }
+    for_symbol('s') { shift_to 3 }
+    for_symbol('Pair') { shift_to 2 }
+  }
+
+  at_state(6) {
+    for_symbol('k') { reduce_with 'one_pair' }
+    for_symbol('_End_') { reduce_with 'one_pair' }
+    for_symbol('s') { reduce_with 'one_pair' }
+  }
+
+  at_state(2) {
+    for_symbol('k') { reduce_with 'multiple_pairs' }
+    for_symbol('_End_') { reduce_with 'multiple_pairs' }
+    for_symbol('s') { reduce_with 'multiple_pairs' }
+  }
+
+end

data/lib/keyword_search/tokenizer.rb

@@ -0,0 +1,68 @@
+module KeywordSearch
+  
+  class Tokenizer < Dhaka::Tokenizer
+
+    letters = ('a'..'z').to_a
+    numbers = ('0'..'9').to_a
+    extras = %w|_ - '|
+    printables = letters + numbers + extras
+    whitespace = [' ']
+    quotes = ['"']
+    keyword_separator = [':']
+    all_characters = keyword_separator + printables + whitespace
+
+    for_state :idle_state do
+    
+      for_characters(printables) do
+        self.accumulator = ''
+        switch_to :unquoted_literal_state
+      end
+
+      for_characters quotes do
+        self.accumulator = ''
+        advance
+        switch_to :quoted_literal_state
+      end
+    
+      for_characters whitespace do
+        advance
+      end
+    
+    end
+  
+    for_state :unquoted_literal_state do
+    
+      for_characters(printables) do
+        self.accumulator += curr_char
+        advance
+        tokens << Dhaka::Token.new(Grammar.symbol_for_name('s'), accumulator) unless curr_char
+      end
+    
+      for_characters(keyword_separator) do
+        tokens << Dhaka::Token.new(Grammar.symbol_for_name('k'), self.accumulator)
+        advance
+        switch_to :idle_state
+      end
+    
+      for_characters(whitespace) do
+        tokens << Dhaka::Token.new(Grammar.symbol_for_name('s'), self.accumulator)
+        switch_to :idle_state
+      end
+    
+    end
+  
+    for_state :quoted_literal_state do
+      for_characters(all_characters - quotes) do
+        self.accumulator += curr_char
+        advance
+      end
+      for_characters quotes do
+        tokens << Dhaka::Token.new(Grammar.symbol_for_name('s'), self.accumulator)
+        advance
+        switch_to :idle_state
+      end
+    end
+
+  end
+
+end

data/test/test_keyword_search.rb

@@ -0,0 +1,70 @@
+require 'test/unit'
+require 'lib/keyword_search'
+  
+class TestKeywordSearch < Test::Unit::TestCase
+  
+  NAME_AND_AGE = %<bruce williams age:26>
+  NAME_QUOTED_AND_AGE = %<"bruce williams" age:26>
+  NAME_AND_QUOTED_AGE = %<bruce williams age:"26">
+  DEFAULT_AGE_WITH_QUOTED_AGE = %<26 name:"bruce williams">
+  
+  def test_default_keyword
+    result = nil
+    KeywordSearch.search(NAME_AND_AGE) do |with|
+      with.default_keyword :name
+      with.keyword :name do |values|
+        result = values.join(' ')
+      end
+    end
+    assert_equal 'bruce williams', result
+  end
+  
+  def test_unquoted_keyword_term
+    result = nil
+    KeywordSearch.search(NAME_AND_AGE) do |with|
+      with.keyword :age do |values|
+        result = Integer(values.first)
+      end
+    end
+    assert_equal 26, result    
+  end
+  
+  def test_quoted_default_keyword_term
+    result = nil
+    KeywordSearch.search(NAME_QUOTED_AND_AGE) do |with|
+      with.default_keyword :name
+      with.keyword :name do |values|
+        result = values.join(' ')
+      end
+    end
+    assert_equal 'bruce williams', result    
+  end
+  
+  def test_quoted_keyword_term
+    result = nil
+    KeywordSearch.search(NAME_AND_QUOTED_AGE) do |with|
+      with.keyword :age do |values|
+        result = Integer(values.first)
+      end
+    end
+    assert_equal 26, result    
+  end
+  
+  def test_quoted_keyword_term_with_whitespace
+    result = nil
+    KeywordSearch.search(DEFAULT_AGE_WITH_QUOTED_AGE) do |with|
+      with.default_keyword :age
+      with.keyword :name do |values|
+        result = values.first
+      end
+    end
+    assert_equal 'bruce williams', result    
+  end
+  
+end
+
+
+
+
+
+

metadata

@@ -0,0 +1,64 @@
+--- !ruby/object:Gem::Specification 
+rubygems_version: 0.9.0
+specification_version: 1
+name: keyword_search
+version: !ruby/object:Gem::Version 
+  version: 1.0.0
+date: 2007-01-08 00:00:00 -07:00
+summary: Generic support for extracting GMail-style search keywords/values from strings
+require_paths: 
+- lib
+email: bruce@codefluency.com
+homepage: "    http://codefluency.rubyforge.org/keyword_search"
+rubyforge_project: keyword_search
+description: "== FEATURES/PROBLEMS:  The library features a very simple, easy-to-use API. * Define handlers for supported keywords with blocks * Define the default keyword (values not part of a keyword/value pair)  Various notes: * Quoted values are supported. * Input is automatically downcased (both keywords and values should be assumed to be in lowercase)  Development Roadmap: 1.1:: Expand supported character set for keywords and values (currently supports a-z) 2.0:: Add negation and grouping (will break API backwards compatibility)  == SYNOPSIS:"
+autorequire: 
+default_executable: 
+bindir: bin
+has_rdoc: true
+required_ruby_version: !ruby/object:Gem::Version::Requirement 
+  requirements: 
+  - - ">"
+    - !ruby/object:Gem::Version 
+      version: 0.0.0
+  version: 
+platform: ruby
+signing_key: 
+cert_chain: 
+post_install_message: 
+authors: 
+- Bruce Williams
+files: 
+- History.txt
+- Manifest.txt
+- README.txt
+- Rakefile
+- lib/keyword_search.rb
+- lib/keyword_search/definition.rb
+- lib/keyword_search/evaluator.rb
+- lib/keyword_search/grammar.rb
+- lib/keyword_search/parser.rb
+- lib/keyword_search/tokenizer.rb
+- test/test_keyword_search.rb
+test_files: 
+- test/test_keyword_search.rb
+rdoc_options: []
+
+extra_rdoc_files: []
+
+executables: []
+
+extensions: []
+
+requirements: []
+
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: hoe
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Version::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: 1.1.6
+    version: