gbdev-scoped_search 0.2.0

data/LICENSE

@@ -0,0 +1,20 @@
+Copyright (c) 2008 Willem van Bergen
+ 
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+ 
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+ 
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.rdoc

@@ -0,0 +1,54 @@
+= scoped_search
+
+This simple plugin will make it easy to search your ActiveRecord models. Searching is performed using a query string, which should be passed to the named_scope <tt>search_for</tt> that uses SQL %LIKE% conditions for searching. You can specify what fields should be used for searching.
+
+== Installation
+
+You can enable <tt>scoped_search</tt> as a Ruby gem. You must enable the gem in your <tt>environment.rb</tt> configuration:
+  
+  Rails::Initializer.run do |config|
+    ...
+    config.gem 'wvanbergen-scoped_search', :lib => 'scoped_search', :source => 'http://gems.github.com/'
+  end
+
+Make sure the gem is installed by running <tt>rake gems:install</tt> in you project root. You can also install the gem by running <tt>sudo gem install wvanbergen-scoped_search -s http://gems.github.com</tt>
+
+You can use scoped_search as a Rails plugin as well, but this is deprecated. Simply download or
+clone scoped_search into your +vendor/plugins+ directory of your project.
+
+== Usage
+
+First, you have to specify in what columns should be searched:
+
+  class Project < ActiveRecord::Base
+    searchable_on :name, :description
+  end
+
+Now, the +search_for+ scope is available for queries. You should pass a query string to the scope. This can be empty or nil, in which case all no search conditions are set (and all records will be returned).
+
+  Project.search_for(params[:q]).each { |project| ... }
+
+The search query language is simple. It supports these constructs:
+
+* words: some search keywords
+* phrases: "a single search phrase"
+* negation: "look for this" -"but do not look for this phrase and this" -word
+
+This functionality is build on named_scope. The searchable_on statement creates 
+a named_scope "search_for". Because of this, you can actually chain the call with
+other scopes. For example, this can be very useful if you only want to search in 
+projects that are accessible by a given user.
+
+  class Project < ActiveRecord::Base
+    searchable_on :name, :description
+    named_scope :accessible_by, lambda { |user| ... }
+  end
+  
+  # using chained named_scopes and will_paginate
+  Project.accessible_by(current_user).search_for(params[:q]).paginate(:page => params[:page], :include => :tasks)
+
+== Disclaimer
+
+This Rails plugin is written by Willem van Bergen for the Floorplanner.com website. It is 
+released under the <b>MIT license</b>. Please contact me (willem AT vanbergen DOT org if 
+you have any suggestions or remarks.

data/Rakefile

@@ -0,0 +1,70 @@
+require 'rubygems'
+
+load 'test/tasks.rake'
+ 
+desc 'Default: run unit tests.'
+task :default => :test
+
+namespace :gem do
+
+  desc "Sets the version and date of the scoped_search gem. Requires the VERSION environment variable."
+  task :version => [:manifest] do
+    
+    require 'date'
+    
+    new_version = ENV['VERSION']
+    raise "VERSION is required" unless /\d+(\.\d+)*/ =~ new_version
+    
+    spec_file = Dir['*.gemspec'].first
+    
+    spec = File.read(spec_file)
+    spec.gsub!(/^(\s*s\.version\s*=\s*)('|")(.+)('|")(\s*)$/) { "#{$1}'#{new_version}'#{$5}" }
+    spec.gsub!(/^(\s*s\.date\s*=\s*)('|")(.+)('|")(\s*)$/) { "#{$1}'#{Date.today.strftime('%Y-%m-%d')}'#{$5}" }    
+    File.open(spec_file, 'w') { |f| f << spec }
+  end
+  
+  task :tag => [:version] do
+    
+    new_version = ENV['VERSION']
+    raise "VERSION is required" unless /\d+(\.\d+)*/ =~ new_version
+        
+    sh "git add scoped_search.gemspec .manifest"
+    sh "git commit -m \"Set gem version to #{new_version}\""
+    sh "git push origin"    
+    sh "git tag -a \"scoped_search-#{new_version}\" -m \"Tagged version #{new_version}\""
+    sh "git push --tags"
+  end
+
+  desc "Builds a ruby gem for scoped_search"
+  task :build => [:manifest] do
+    system %[gem build scoped_search.gemspec]
+  end
+
+  desc %{Update ".manifest" with the latest list of project filenames. Respect\
+  .gitignore by excluding everything that git ignores. Update `files` and\
+  `test_files` arrays in "*.gemspec" file if it's present.}
+  task :manifest do
+    list = Dir['**/*'].sort
+    spec_file = Dir['*.gemspec'].first
+    list -= [spec_file] if spec_file
+  
+    File.read('.gitignore').each_line do |glob|
+      glob = glob.chomp.sub(/^\//, '')
+      list -= Dir[glob]
+      list -= Dir["#{glob}/**/*"] if File.directory?(glob) and !File.symlink?(glob)
+      puts "excluding #{glob}"
+    end
+ 
+    if spec_file
+      spec = File.read spec_file
+      spec.gsub! /^(\s* s.(test_)?files \s* = \s* )( \[ [^\]]* \] | %w\( [^)]* \) )/mx do
+        assignment = $1
+        bunch = $2 ? list.grep(/^test.*_test\.rb$/) : list
+        '%s%%w(%s)' % [assignment, bunch.join(' ')]
+      end
+      
+      File.open(spec_file,   'w') {|f| f << spec }
+    end
+    File.open('.manifest', 'w') {|f| f << list.join("\n") }
+  end
+end

data/TODO

@@ -0,0 +1,20 @@
+TODO items for named_scope
+==========================
+Contact willem AT vanbergen DOT org if you want to help out
+
+
+New features:
+- Search fields of associations as well
+- Allow other search operators than %LIKE% 
+
+Refactoring:
+- Create a separate class to build the actual SQL queries.
+- For searchable_on(*fields) make it so that instead of fields it accepts options (a hash) where 
+  :only and :except can be values.  That way it is possible for all fields to be loaded except 
+  the ones specified with :except.
+- Add checks for field types because the latest version of PostgreSQL (version 8.3.3) is more 
+  strict about searching for strings in columns that are not string types.
+
+Documentation & testing:
+- Put something useful in the wiki
+- Add rdoc en comments to code

data/init.rb

@@ -0,0 +1 @@
+require 'scoped_search'

data/lib/scoped_search.rb

@@ -0,0 +1,67 @@
+module ScopedSearch
+  
+  module ClassMethods
+  
+    def self.extended(base)
+      require 'scoped_search/query_language_parser'
+    end
+  
+    # Creates a named scope in the class it was called upon
+    def searchable_on(*fields)
+      self.cattr_accessor :scoped_search_fields
+      self.scoped_search_fields = fields
+      self.named_scope :search_for, lambda { |keywords| self.build_scoped_search_conditions(keywords) }
+    end
+  
+    # Build a hash that is used for the named_scope search_for.
+    # This function will split the search_string into keywords, and search for all the keywords
+    # in the fields that were provided to searchable_on
+    def build_scoped_search_conditions(search_string)
+      if search_string.nil? || search_string.strip.blank?
+        return { :conditions => nil }
+      else
+        conditions = []
+        query_params = {}
+      
+        QueryLanguageParser.parse(search_string).each_with_index do |search_condition, index|
+          keyword_name = "keyword_#{index}".to_sym
+          query_params[keyword_name] = "%#{search_condition.first}%" 
+
+          # a keyword may be found in any of the provided fields, so join the conitions with OR
+          if search_condition.length == 2 && search_condition.last == :not
+            keyword_conditions = self.scoped_search_fields.map do |field| 
+              field_name = connection.quote_table_name(table_name) + "." + connection.quote_column_name(field)
+              "(#{field_name} NOT LIKE :#{keyword_name.to_s} OR #{field_name} IS NULL)"
+            end
+            conditions << "(#{keyword_conditions.join(' AND ')})" 
+          elsif search_condition.length == 2 && search_condition.last == :or
+            word1, word2 = query_params[keyword_name].split(' OR ')
+            
+            query_params.delete(keyword_name)
+            keyword_name_a = "#{keyword_name.to_s}a".to_sym
+            keyword_name_b = "#{keyword_name.to_s}b".to_sym
+            query_params[keyword_name_a] = word1
+            query_params[keyword_name_b] = word2
+            
+            keyword_conditions = self.scoped_search_fields.map do |field| 
+              field_name = connection.quote_table_name(table_name) + "." + connection.quote_column_name(field)
+              "(#{field_name} LIKE :#{keyword_name_a.to_s} OR #{field_name} LIKE :#{keyword_name_b.to_s})"
+            end
+            conditions << "(#{keyword_conditions.join(' OR ')})"            
+          else
+            keyword_conditions = self.scoped_search_fields.map do |field| 
+              field_name = connection.quote_table_name(table_name) + "." + connection.quote_column_name(field)
+              "#{field_name} LIKE :#{keyword_name.to_s}"
+            end
+            conditions << "(#{keyword_conditions.join(' OR ')})"
+          end       
+        end
+      
+        # all keywords must be matched, so join the conditions with AND
+        return { :conditions => [conditions.join(' AND '), query_params] }
+      end
+    end
+  end
+end
+
+ActiveRecord::Base.send(:extend, ScopedSearch::ClassMethods)

data/lib/scoped_search/query_language_parser.rb

@@ -0,0 +1,71 @@
+module ScopedSearch
+  
+  class QueryLanguageParser
+    
+    def parse_query(query = nil)
+      return build_conditions_tree(tokenize(query))
+    end
+    
+    def self.parse(query)
+      self.new.parse_query(query)
+    end
+    
+    protected
+  
+    def build_conditions_tree(tokens)
+      conditions_tree = []
+    
+      negate = false
+      tokens.each do |item|
+        case item
+          when :not
+            negate = true
+          else
+            if /^.+[ ]OR[ ].+$/ =~ item
+              conditions_tree << [item, :or]
+            else
+              conditions_tree << (negate ? [item, :not] : [item, :like])
+              negate = false
+            end
+        end
+      end
+      return conditions_tree
+    end
+  
+    # **Patterns**
+    # Each pattern is sperated by a "|".  With regular expressions the order of the expression does matter.
+    #
+    # ([\w]+[ ]OR[ ][\w]+)
+    # ([\w]+[ ]OR[ ]["][\w ]+["])
+    # (["][\w ]+["][ ]OR[ ][\w]+)
+    # (["][\w ]+["][ ]OR[ ]["][\w ]+["])
+    #   Any two combinations of letters, numbers and underscores that are seperated by " OR " (a single space must 
+    #   be on each side of the "OR"). 
+    #   THESE COULD BE COMBINED BUT BECAUSE OF THE WAY PARSING WORKS THIS IS NOT DONE ON PURPOSE!!
+    #
+    # ([-]?[\w]+) 
+    #   Any combination of letters, numbers and underscores that may or may not have a dash in front.
+    # 
+    # ([-]?["][\w ]+["])
+    #   Any combination of letters, numbers, underscores and spaces within double quotes that may or may not have a dash in front.
+    def tokenize(query)
+      pattern = ['([\w]+[ ]OR[ ][\w]+)',
+                 '([\w]+[ ]OR[ ]["][\w ]+["])',
+                 '(["][\w ]+["][ ]OR[ ][\w]+)',
+                 '(["][\w ]+["][ ]OR[ ]["][\w ]+["])',
+                 '([-]?[\w]+)',
+                 '([-]?["][\w ]+["])']
+      pattern = Regexp.new(pattern.join('|'))
+      
+      tokens = []
+      matches = query.scan(pattern).flatten.compact
+      matches.each { |match|
+        tokens << :not unless match.index('-').nil?
+        # Remove any escaped quotes and any dashes - the dash usually the first character.
+        # Remove any additional spaces - more that one.
+        tokens << match.gsub(/[-"]/,'').gsub(/[ ]{2,}/, ' ')
+      }
+      return tokens
+    end
+  end
+end

data/test/query_language_test.rb

@@ -0,0 +1,140 @@
+require File.dirname(__FILE__) + '/test_helper'
+
+class QueryLanguageTest < Test::Unit::TestCase
+
+  # change this function if you switch to another query language parser
+  def parse_query(query)
+    ScopedSearch::QueryLanguageParser.parse(query)
+  end
+  
+  def test_empty_search_query
+    parsed = parse_query('')
+    assert_equal 0, parsed.length    
+    
+    parsed = parse_query("\t  \n")
+    assert_equal 0, parsed.length    
+  end
+  
+  def test_single_keyword
+    parsed = parse_query('hallo')
+    assert_equal 1, parsed.length
+    assert_equal 'hallo', parsed.first.first
+  
+    parsed = parse_query('  hallo  ')
+    assert_equal 1, parsed.length
+    assert_equal 'hallo', parsed.first.first
+  end
+   
+  def test_multiple_keywords
+    parsed = parse_query('  hallo  willem')
+    assert_equal 2, parsed.length
+    assert_equal 'willem', parsed.last.first
+    
+    parsed = parse_query("  hallo  willem   van\tbergen ")
+    assert_equal 4, parsed.length    
+    assert_equal 'hallo',  parsed[0].first
+    assert_equal 'willem', parsed[1].first
+    assert_equal 'van',    parsed[2].first
+    assert_equal 'bergen', parsed[3].first      
+  end
+  
+  def test_quoted_keywords
+    parsed = parse_query('  "hallo"')    
+    assert_equal 1, parsed.length
+    assert_equal 'hallo', parsed.first.first
+    
+    parsed = parse_query('  "hallo   willem"')
+    assert_equal 1, parsed.length
+    assert_equal 'hallo willem', parsed.first.first
+     
+    parsed = parse_query('  "hallo   willem')
+    assert_equal 2, parsed.length
+    assert_equal 'hallo',  parsed[0].first
+    assert_equal 'willem', parsed[1].first
+
+    parsed = parse_query('  "hallo   wi"llem"')
+    assert_equal 2, parsed.length
+    assert_equal 'hallo wi', parsed[0].first
+    assert_equal 'llem',       parsed[1].first
+  end
+  
+  def test_quote_escaping
+    parsed = parse_query('  "hallo   wi\\"llem"')
+    assert_equal 3, parsed.length  
+    assert_equal 'hallo', parsed[0].first
+    assert_equal 'wi',    parsed[1].first
+    assert_equal 'llem',  parsed[2].first
+  
+    parsed = parse_query('"\\"hallo willem\\""')
+    assert_equal 2, parsed.length
+    assert_equal 'hallo',  parsed[0].first
+    assert_equal 'willem', parsed[1].first
+  end
+  
+  def test_negation
+    parsed = parse_query('-willem')
+    assert_equal 1, parsed.length
+    assert_equal 'willem', parsed[0].first
+    assert_equal :not,     parsed[0].last
+  
+    parsed = parse_query('123 -"456 789"')
+    assert_equal 2, parsed.length
+    assert_equal '123', parsed[0].first
+    assert_equal :like, parsed[0].last    
+    
+    assert_equal '456 789', parsed[1].first
+    assert_equal :not,      parsed[1].last
+  end
+  
+  def test_or
+    parsed = parse_query('Wes OR Hays')
+    assert_equal 1, parsed.length
+    assert_equal 'Wes OR Hays', parsed[0][0]
+    assert_equal :or, parsed[0][1]
+    
+    parsed = parse_query('"Man made" OR Dogs')
+    assert_equal 1, parsed.length
+    assert_equal 'Man made OR Dogs', parsed[0][0]
+    assert_equal :or, parsed[0][1]    
+    
+    parsed = parse_query('Cows OR "Frog Toys"')
+    assert_equal 1, parsed.length
+    assert_equal 'Cows OR Frog Toys', parsed[0][0]
+    assert_equal :or, parsed[0][1]
+    
+    parsed = parse_query('"Happy cow" OR "Sad Frog"')
+    assert_equal 1, parsed.length
+    assert_equal 'Happy cow OR Sad Frog', parsed[0][0]
+    assert_equal :or, parsed[0][1]        
+  end 
+  
+  def test_long_string
+    str = 'Wes -Hays "Hello World" -"Goodnight Moon" Bob OR Wes "Happy cow" OR "Sad Frog" "Man made" OR Dogs Cows OR "Frog Toys"'
+    parsed = parse_query(str)
+    assert_equal 8, parsed.length
+    
+    assert_equal 'Wes', parsed[0].first
+    assert_equal :like,   parsed[0].last
+    
+    assert_equal 'Hays', parsed[1].first
+    assert_equal :not,   parsed[1].last
+    
+    assert_equal 'Hello World', parsed[2].first
+    assert_equal :like, parsed[2].last
+    
+    assert_equal 'Goodnight Moon', parsed[3].first
+    assert_equal :not, parsed[3].last      
+    
+    assert_equal 'Bob OR Wes', parsed[4].first
+    assert_equal :or,   parsed[4].last    
+    
+    assert_equal 'Happy cow OR Sad Frog', parsed[5].first
+    assert_equal :or,   parsed[5].last
+    
+    assert_equal 'Man made OR Dogs', parsed[6].first
+    assert_equal :or,   parsed[6].last
+    
+    assert_equal 'Cows OR Frog Toys', parsed[7].first
+    assert_equal :or,   parsed[7].last                  
+  end
+end

data/test/search_for_test.rb

@@ -0,0 +1,43 @@
+require File.dirname(__FILE__) + '/test_helper'
+
+class ScopedSearchTest < Test::Unit::TestCase
+
+  def setup
+    setup_db
+    SearchTestModel.create_corpus!
+  end
+
+  def teardown
+    teardown_db
+  end
+  
+  def test_enabling
+    assert !SearchTestModel.respond_to?(:search_for)
+    SearchTestModel.searchable_on :string_field, :text_field
+    assert SearchTestModel.respond_to?(:search_for)
+    
+    assert_equal ActiveRecord::NamedScope::Scope, SearchTestModel.search_for('test').class
+    
+  end
+  
+  def test_search
+    SearchTestModel.searchable_on :string_field, :text_field
+    
+    assert_equal 15, SearchTestModel.search_for('').count
+    assert_equal 0, SearchTestModel.search_for('456').count   
+    assert_equal 2, SearchTestModel.search_for('hays').count 
+    assert_equal 1, SearchTestModel.search_for('hay ob').count        
+    assert_equal 13, SearchTestModel.search_for('o').count    
+    assert_equal 2, SearchTestModel.search_for('-o').count
+    assert_equal 13, SearchTestModel.search_for('-Jim').count
+    assert_equal 1, SearchTestModel.search_for('Jim -Bush').count
+    assert_equal 1, SearchTestModel.search_for('"Hello World" -"Goodnight Moon"').count    
+    assert_equal 2, SearchTestModel.search_for('Wes OR Bob').count
+    assert_equal 3, SearchTestModel.search_for('"Happy cow" OR "Sad Frog"').count
+    assert_equal 3, SearchTestModel.search_for('"Man made" OR Dogs').count
+    assert_equal 2, SearchTestModel.search_for('Cows OR "Frog Toys"').count   
+  end
+
+end
+
+

data/test/tasks.rake

@@ -0,0 +1,8 @@
+require 'rake/testtask'
+ 
+desc 'Test the scoped_search plugin.'
+Rake::TestTask.new(:test) do |t|
+  t.pattern = 'test/**/*_test.rb'
+  t.verbose = true
+  t.libs << 'test'
+end

data/test/test_helper.rb

@@ -0,0 +1,41 @@
+require 'test/unit'
+require 'rubygems'
+require 'active_record'
+
+require "#{File.dirname(__FILE__)}/../lib/scoped_search"
+
+def setup_db
+  ActiveRecord::Base.establish_connection(:adapter => "sqlite3", :dbfile => ":memory:")
+  ActiveRecord::Schema.define(:version => 1) do
+    create_table :search_test_models do |t|
+      t.string :string_field
+      t.text :text_field
+      t.string :ignored_field
+      t.timestamps
+    end
+  end
+end
+
+def teardown_db
+  ActiveRecord::Base.connection.tables.each { |table| ActiveRecord::Base.connection.drop_table(table) }
+end
+
+class SearchTestModel < ActiveRecord::Base
+  def self.create_corpus!
+    create!(:string_field => "Programmer 123", :text_field => nil,               :ignored_field => "123456")    
+    create!(:string_field => "Jim",            :text_field => "Henson",          :ignored_field => "123456a")   
+    create!(:string_field => "Jim",            :text_field => "Bush",            :ignored_field => "123456b")    
+    create!(:string_field => "Wes",            :text_field => "Hays",            :ignored_field => "123456c")  
+    create!(:string_field => "Bob",            :text_field => "Hays",            :ignored_field => "123456d")  
+    create!(:string_field => "Dogs",           :text_field => "Pit Bull",        :ignored_field => "123456e") 
+    create!(:string_field => "Dogs",           :text_field => "Eskimo",          :ignored_field => "123456f")
+    create!(:string_field => "Cows",           :text_field => "Farms",           :ignored_field => "123456g")
+    create!(:string_field => "Hello World",    :text_field => "Hello Moon",      :ignored_field => "123456h")   
+    create!(:string_field => "Hello World",    :text_field => "Goodnight Moon",  :ignored_field => "123456i")
+    create!(:string_field => "Happy Cow",      :text_field => "Sad Cow",         :ignored_field => "123456j")
+    create!(:string_field => "Happy Frog",     :text_field => "Sad Frog",        :ignored_field => "123456k")
+    create!(:string_field => "Excited Frog",   :text_field => "Sad Frog",        :ignored_field => "123456l")    
+    create!(:string_field => "Man made",       :text_field => "Woman made",      :ignored_field => "123456m")
+    create!(:string_field => "Cat Toys",       :text_field => "Frog Toys",       :ignored_field => "123456n") 
+  end
+end

metadata

@@ -0,0 +1,70 @@
+--- !ruby/object:Gem::Specification 
+name: gbdev-scoped_search
+version: !ruby/object:Gem::Version 
+  version: 0.2.0
+platform: ruby
+authors: 
+- Willem van Bergen
+- Wes Hays
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2008-09-13 00:00:00 -07:00
+default_executable: 
+dependencies: []
+
+description: Scoped search makes it easy to search your ActiveRecord-based models. It will create a named scope according to a provided query string. The named_scope can be used like any other named_scope, so it can be cchained or combined with will_paginate.
+email: 
+- willem@vanbergen.org
+- weshays@gbdev.com
+executables: []
+
+extensions: []
+
+extra_rdoc_files: []
+
+files: 
+- LICENSE
+- README.rdoc
+- Rakefile
+- TODO
+- init.rb
+- lib
+- lib/scoped_search
+- lib/scoped_search.rb
+- lib/scoped_search/query_language_parser.rb
+- test
+- test/query_language_test.rb
+- test/search_for_test.rb
+- test/tasks.rake
+- test/test_helper.rb
+has_rdoc: false
+homepage: http://github.com/wvanbergen/scoped_search/wikis
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.2.0
+signing_key: 
+specification_version: 2
+summary: A Rails plugin to search your models using a named_scope
+test_files: 
+- test/query_language_test.rb
+- test/search_for_test.rb