awesome_search 1.0.1 → 1.1.0

data/README.rdoc

@@ -2,7 +2,6 @@
 
 * A helpful library to make searching more organized in the controller and views.
 * Using this library will force you to write a class for each kind of search you want to use.
-* It has features that allow easy configuration of searches tailored to work with rsolr.
 
 == Usage
 

data/Rakefile

@@ -22,11 +22,22 @@ begin
      "lib/awesome/definitions/bits.rb",
      "lib/awesome/definitions/filters.rb",
      "lib/awesome/definitions/locales.rb",
+     "lib/awesome/definitions/stopwords.rb",
      "lib/awesome/definitions/types.rb",
      "lib/awesome/search.rb",
      "lib/awesome/super_search.rb",
      "lib/awesome/triage.rb",
      "lib/awesome_search.rb",
+     "test/search_classes/amazon.rb",
+     "test/search_classes/ebay.rb",
+     "test/search_classes/google.rb",
+     "test/search_classes/local.rb",
+     "test/helper.rb",
+     "test/test_awesome_search.rb",
+     "test/test_multiple.rb",
+     "test/test_multiple_filters.rb",
+     "test/test_multiple_locales.rb",
+     "test/test_multiple_types.rb",
      "rails/init.rb",
      "init.rb"
     ]

data/VERSION

@@ -1 +1 @@
-1.0.1
+1.1.0

data/awesome_search.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = %q{awesome_search}
-  s.version = "1.0.1"
+  s.version = "1.1.0"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["pboling"]
-  s.date = %q{2010-04-29}
+  s.date = %q{2010-06-22}
   s.description = %q{Organize complicated search results}
   s.email = %q{peter.boling@peterboling.com}
   s.extra_rdoc_files = [
@@ -26,17 +26,28 @@ Gem::Specification.new do |s|
      "lib/awesome/definitions/bits.rb",
      "lib/awesome/definitions/filters.rb",
      "lib/awesome/definitions/locales.rb",
+     "lib/awesome/definitions/stopwords.rb",
      "lib/awesome/definitions/types.rb",
      "lib/awesome/search.rb",
      "lib/awesome/super_search.rb",
      "lib/awesome/triage.rb",
      "lib/awesome_search.rb",
-     "rails/init.rb"
+     "rails/init.rb",
+     "test/helper.rb",
+     "test/search_classes/amazon.rb",
+     "test/search_classes/ebay.rb",
+     "test/search_classes/google.rb",
+     "test/search_classes/local.rb",
+     "test/test_awesome_search.rb",
+     "test/test_multiple.rb",
+     "test/test_multiple_filters.rb",
+     "test/test_multiple_locales.rb",
+     "test/test_multiple_types.rb"
   ]
   s.homepage = %q{http://github.com/pboling/awesome_search}
   s.rdoc_options = ["--charset=UTF-8"]
   s.require_paths = ["lib"]
-  s.rubygems_version = %q{1.3.6}
+  s.rubygems_version = %q{1.3.7}
   s.summary = %q{Organize complicated search results}
   s.test_files = [
     "test/helper.rb",
@@ -55,7 +66,7 @@ Gem::Specification.new do |s|
     current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
     s.specification_version = 3
 
-    if Gem::Version.new(Gem::RubyGemsVersion) >= Gem::Version.new('1.2.0') then
+    if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
       s.add_development_dependency(%q<shoulda>, [">= 0"])
       s.add_runtime_dependency(%q<activesupport>, [">= 2.1"])
     else

data/lib/awesome/definitions/stopwords.rb

@@ -0,0 +1,125 @@
+module Awesome
+  module Definitions
+    module Stopwords
+
+      QUOTED_REGEX =    /("[^"]*")/
+      UNQUOTED_REGEX =    /"([^"]*)"/
+      RM_QUOTED_REGEX = /"[^"]*"/
+      BEG_OPERATORS =   /^[+-]/
+      END_OPERATORS =   /[,+-]$/
+      EXCLUSION_OPERATORS =   /^[-]/
+      INCLUSION_OPERATORS =   /^[+]/
+
+      def self.included(base)
+        base.extend ClassMethods
+        base.cattr_accessor :search_stopwords
+        base.cattr_accessor :verbose_stopwords
+      end
+
+      module ClassMethods
+        def stopwords(key = :none)
+          case key
+            when :none then
+              self.search_stopwords[:none]
+            when :standard then
+              self.search_stopwords[:standard]
+            when :custom then
+              self.search_stopwords[:custom]
+            when :both then
+              self.search_stopwords[:custom] | self.search_stopwords[:standard]
+            else
+              Rails.logger.warn("AwesomeSearch: Stopwords Key Invalid, defaulting to :both")
+              self.search_stopwords[:custom] | self.search_stopwords[:standard]
+          end
+        end
+      end
+
+      #Instance Methods:
+
+      #remove the stopwords from regular search terms, BUT NOT from exact phrase searches (quoted)
+      #example:
+      # txt = "+hair    \"in the\"  on the grapes,  \"middle fork\" wrath  \"age  of man\"  -end"
+      def process_stopwords(txt = self.search_text)
+        #Needs to be set so highlighting will work properly (can't match quotes)
+        self.highlight_token_array(txt)
+        #Now put humpty dumpty back together without the nasty stopwords, sort the tokens by length
+        self.search_token_array(txt).join(" ")
+      end
+
+      def search_token_array(txt)
+        self.search_tokens ||= (self.quoted_exact_phrases_array(txt) | self.gowords_array(txt)).sort {|a,b| b.length <=> a.length }
+      end
+
+      def highlight_token_array(txt)
+        self.highlight_tokens ||= begin
+          array = (self.unquoted_exact_phrases_array(txt) | self.gowords_array(txt)).sort {|a,b| b.length <=> a.length }
+          remove_exclusions(array)
+        end
+      end
+
+      def set_clean_search_query
+        self.clean_search_query = self.highlight_tokens.join(" ")
+      end
+
+      def remove_exclusions(array)
+        array.map do |tok|
+          tok.match(Awesome::Definitions::Stopwords::EXCLUSION_OPERATORS) ?
+            nil :
+            tok.match(Awesome::Definitions::Stopwords::RM_QUOTED_REGEX) ?
+              tok :
+              tok.gsub(Awesome::Definitions::Stopwords::INCLUSION_OPERATORS, '')
+        end.compact
+      end
+      
+      #All tokens that are quoted
+      def tokenize_quot(txt)
+        self.tokenize_quoted ||= txt.split(Awesome::Definitions::Stopwords::QUOTED_REGEX)
+      end
+
+      #All tokens that are quoted, in their unquoted form
+      def tokenize_unquot(txt)
+        self.tokenize_unquoted ||= txt.split(Awesome::Definitions::Stopwords::UNQUOTED_REGEX)
+      end
+
+      #Remove all tokens that are quoted
+      def tokenize_without_quot(txt)
+        self.tokenize_without_quoted ||= txt.split(Awesome::Definitions::Stopwords::RM_QUOTED_REGEX)
+      end
+
+      # ["\"in the\"", "\"middle fork\"", "\"age  of man\""]
+      def quoted_exact_phrases_array(txt)
+        self.quoted_exact_phrases ||= self.tokenize_quot(txt) - self.tokenize_without_quot(txt) - ['']
+      end
+
+      # ["in the", "middle fork", "age  of man"]
+      def unquoted_exact_phrases_array(txt)
+        self.unquoted_exact_phrases ||= self.tokenize_unquot(txt) - self.tokenize_without_quot(txt) - ['']
+      end
+
+      # "+hair     on the grapes,   wrath    -end"
+      def query_wo_exact_phrases(txt)
+        self.query_without_exact_phrases ||= txt.gsub(Awesome::Definitions::Stopwords::QUOTED_REGEX, '')
+      end
+
+      # ["+hair", "on", "the", "grapes,", "wrath", "-end"]
+      def array_with_stopwords(txt)
+        qa = self.query_wo_exact_phrases(txt).split
+        qa.delete(',') #delete works on self (qa here), so won't work chained onto the statement above!
+        qa
+      end
+
+      # ["+hair", "grapes,", "wrath", "-end"]
+      def gowords_array(txt)
+        self.gowords ||= self.array_with_stopwords(txt).map do |token|
+          cleaned_token = self.clean_token(token)
+          self.stopwords.include?(cleaned_token) ? nil : cleaned_token.blank? ? nil : token
+        end.compact
+      end
+
+      def clean_token(token)
+        token.gsub(Awesome::Definitions::Stopwords::BEG_OPERATORS, '').gsub(Awesome::Definitions::Stopwords::END_OPERATORS, '')
+      end
+
+    end
+  end
+end

data/lib/awesome/search.rb

@@ -15,6 +15,7 @@ module Awesome
 
     #Some defaults if stopwords are set to be used (by default they are turned off)
     @@search_stopwords ||= {:standard => %w(an and are as at be but by for if in into is it no not of on or s such t that the their then there these they this to was will with),
+                            :none => [],
                             :custom => []}
 
     def self.configure_search_types(&block)
@@ -37,6 +38,7 @@ module Awesome
 
     attr_accessor(:search_text,
                   :search_query,
+                  :clean_search_query,
                   :search_type,
                   :search_filters,
                   :search_locale,
@@ -61,19 +63,22 @@ module Awesome
                   :search_tokens,
                   :highlight_tokens)
 
+    alias :query :search_text
+    alias :processed_query :search_query
+    
     #CLASS METHODS
     #Main focus of class methods is determining which sort of AwesomeSearch subclass we need to instantiate for the search
     def initialize(*args)
       @multiple_types_as_one = args.first[:multiple_types_as_one]
-      @stopwords =  args.first[:stopwords] ?
+      @stopwords =  !args.first[:stopwords].blank? ?
                       args.first[:stopwords].is_a?(Array) ?
                         args.first[:stopwords] :
                         args.first[:stopwords].is_a?(Symbol) ?
                           self.class.stopwords(args.first[:stopwords]) :
                           self.class.stopwords(:both) :
-                      args.first[:stopwords] != false ?
+                      args.first[:stopwords] == true ?
                         self.class.stopwords(:standard) :
-                        []
+                        self.class.stopwords(:none)
       @page     = args.first[:page]
       @per_page = args.first[:per_page]
       @search_text =    args.first[:search_text]    # a string
@@ -82,6 +87,7 @@ module Awesome
       @search_tokens =  args.first[:search_tokens]  # an array of the query terms as tokens after being cleaned, unless passed in as param (not sure why this would ever be desired, but why not allow it jic?) When not set in args, will be set by clean_search_text methods
       @highlight_tokens =  args.first[:highlight_tokens]  # an array of the query terms as unquoted tokens after being cleaned, unless passed in as param (not sure why this would ever be desired, but why not allow it jic?) When not set in args, will be set by clean_search_text methods
       @search_query =   self.clean_search_text      # a string to be set based on the search text by removing the search modifiers from the search text
+      @clean_search_query = self.set_clean_search_query # Removed search tokens and modifiers (+) so Regexes can be run on this string
       @search_type =    args.first[:search_type]    # a symring (symring methods are in the Bits mixin)
       @search_locale =  args.first[:search_locale]  # a symring (symring methods are in the Bits mixin)
       @found = nil
@@ -104,9 +110,13 @@ module Awesome
     end
 
     def clean_search_text
+      #puts "search_text: #{self.search_text}" if Awesome::Search.verbose
       txt = Awesome::Triage.clean_search_text(self.search_text)
+      #puts "cleaning1: #{txt}" if Awesome::Search.verbose
       txt = Awesome::Search.clean_search_text(txt, self.multiple_types_as_one)
+      #puts "cleaning2: #{txt}" if Awesome::Search.verbose
       txt = self.process_stopwords(txt)
+      #puts "cleaning3: #{txt}" if Awesome::Search.verbose
       txt
     end
 

data/lib/awesome_search.rb

@@ -1,6 +1,7 @@
 require "awesome/definitions/bits" unless defined?(Awesome::Definitions::Bits)
 require "awesome/definitions/locales" unless defined?(Awesome::Definitions::Locales)
 require "awesome/definitions/filters" unless defined?(Awesome::Definitions::Filters)
+require "awesome/definitions/stopwords" unless defined?(Awesome::Definitions::Stopwords)
 require "awesome/definitions/types" unless defined?(Awesome::Definitions::Types)
 require "awesome/triage" unless defined?(Awesome::Triage)
 require "awesome/search" unless defined?(Awesome::Search)

data/test/helper.rb

@@ -12,7 +12,8 @@ require "test/search_classes/google"
 require "test/search_classes/local"
 
 class Test::Unit::TestCase
-  Awesome::Search.protect_types = true
+  #protect_types is a work in progress, doesn't work now.
+  Awesome::Search.protect_types = false
   Awesome::Search.protect_filters = true
   Awesome::Search.verbose = false
   Awesome::Search.verbose_types = false
@@ -100,7 +101,7 @@ class Test::Unit::TestCase
           [ ":all",
             ":every" ],
         ":spotted" =>
-          [ "spotted" ],
+          [ ":spotted" ],
         ":old" =>
           [ ":old",
             ":ancient",

data/test/test_awesome_search.rb

@@ -7,8 +7,14 @@ class TestAwesomeSearch < Test::Unit::TestCase
       @awesome = Awesome::Search.new({:search_text => "this is a test", :search_type => :isbn, :search_locale => :amazon})
     end
 
+    should "return search_text" do
+      assert_equal 'this is a test', @awesome.search_text
+    end
     should "return search_query" do
-      assert_equal 'this is a test', @awesome.search_query
+      assert_equal 'this test is a', @awesome.search_query
+    end
+    should "return clean_search_query" do
+      assert_equal 'this test is a', @awesome.clean_search_query
     end
     should "return search_type" do
       assert_equal :isbn, @awesome.search_type
@@ -20,11 +26,14 @@ class TestAwesomeSearch < Test::Unit::TestCase
 
   context "An Awesome::Search instance with query modifiers" do
     setup do
-      @awesome = Awesome::Search.new({:search_text => ":ebay :sku this is a test"})
+      @awesome = Awesome::Search.new({:search_text => ":ebay :sku this is a +test"})
     end
 
-    should "return search_query without query modifiers" do
-      assert_equal 'this is a test', @awesome.search_query
+    should "return search_query without query tokens" do
+      assert_equal '+test this is a', @awesome.search_query
+    end
+    should "return clean_search_query without query tokens and modifiers" do
+      assert_equal 'test this is a', @awesome.clean_search_query
     end
     should "search_type should be nil" do
       assert_equal nil, @awesome.search_type

data/test/test_multiple_filters.rb

@@ -4,7 +4,7 @@ class TestMultipleFilters < Test::Unit::TestCase
 
   context "#results_for with multiple filters" do
     setup do
-      @searches = Awesome::Search.results_for(":local :amazon :text :isbn :old this is a test 1234567890", ":text", ":local", [":old",":spotted",":shiny",":new"])
+      @searches = Awesome::Search.results_for(":local :amazon :text :isbn :old :spotted :new this is a test 1234567890", ":text", ":local", [":old",":spotted",":shiny",":new"])
     end
     should "should return an array" do
       assert @searches.is_a?(Array)

metadata

@@ -1,12 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: awesome_search
 version: !ruby/object:Gem::Version 
+  hash: 19
   prerelease: false
   segments: 
   - 1
-  - 0
   - 1
-  version: 1.0.1
+  - 0
+  version: 1.1.0
 platform: ruby
 authors: 
 - pboling
@@ -14,16 +15,18 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-04-29 00:00:00 -04:00
+date: 2010-06-22 00:00:00 -04:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
   name: shoulda
   prerelease: false
   requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
+        hash: 3
         segments: 
         - 0
         version: "0"
@@ -33,9 +36,11 @@ dependencies:
   name: activesupport
   prerelease: false
   requirement: &id002 !ruby/object:Gem::Requirement 
+    none: false
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
+        hash: 1
         segments: 
         - 2
         - 1
@@ -61,12 +66,23 @@ files:
 - lib/awesome/definitions/bits.rb
 - lib/awesome/definitions/filters.rb
 - lib/awesome/definitions/locales.rb
+- lib/awesome/definitions/stopwords.rb
 - lib/awesome/definitions/types.rb
 - lib/awesome/search.rb
 - lib/awesome/super_search.rb
 - lib/awesome/triage.rb
 - lib/awesome_search.rb
 - rails/init.rb
+- test/helper.rb
+- test/search_classes/amazon.rb
+- test/search_classes/ebay.rb
+- test/search_classes/google.rb
+- test/search_classes/local.rb
+- test/test_awesome_search.rb
+- test/test_multiple.rb
+- test/test_multiple_filters.rb
+- test/test_multiple_locales.rb
+- test/test_multiple_types.rb
 has_rdoc: true
 homepage: http://github.com/pboling/awesome_search
 licenses: []
@@ -77,23 +93,27 @@ rdoc_options:
 require_paths: 
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
   requirements: 
   - - ">="
     - !ruby/object:Gem::Version 
+      hash: 3
       segments: 
       - 0
       version: "0"
 required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
   requirements: 
   - - ">="
     - !ruby/object:Gem::Version 
+      hash: 3
       segments: 
       - 0
       version: "0"
 requirements: []
 
 rubyforge_project: 
-rubygems_version: 1.3.6
+rubygems_version: 1.3.7
 signing_key: 
 specification_version: 3
 summary: Organize complicated search results