smart_search 0.0.4 → 0.0.7

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: b4452571bb09d489fdd980a8b151245b891470c2
+  data.tar.gz: d5d8f1b4a301583170703bf6f7c0e322d3b1c59a
+SHA512:
+  metadata.gz: 001028ede264fd11fad4bbf5a2a1c106c28fa3d5581ef23f1c5839e432b4b66286efc55152b35738187da969a606277c86bb2e2bce9b9839e0dfefe7c5ef2697
+  data.tar.gz: 770d3e266a57b8e1adf23893c85893c2d3248e840974cbd050fad9f110930bebd8561365b377287688d2570d02173491e7a9b55b472e9969ce097b1cb06fb311

data/config/database.yml

@@ -0,0 +1,6 @@
+test:
+  adapter: mysql2
+  database: smart_search_test
+  username: travis
+  password: 
+  encoding: utf8

data/config/routes.rb

@@ -2,7 +2,8 @@
 Rails.application.routes.draw do
   
   #=== Search Routing
-  get   "/find",    :controller => "search", :action => "all"
-  post "/find",     :controller => "search", :action => "all"
+  get   "/suche/:query(/:only)",      :controller => "search", :action => "all"
+  post  "/suche/:query(/:only)",      :controller => "search", :action => "all"
+  post  "/suche",                     :controller => "search", :action => "all"
 
 end

data/lib/smart_search/smart_search_engine.rb

@@ -0,0 +1,10 @@
+# :nodoc:
+module SmartSearch
+  require "rails"
+  # :nodoc:
+  class SmartSearchEngine < Rails::Engine
+    isolate_namespace SmartSearch
+    require "friendly_extensions"
+    require "amatch"
+  end  
+end  

data/lib/smart_search.rb

@@ -1,6 +1,13 @@
 # -*- encoding : utf-8 -*-
+require "rails"
+
 require "smart_search"
-require "add_search_tags"
+require "smart_search/smart_search_engine"
+
+require "smart_similarity"
+require "smart_search_history"
+require "smart_search_tag"
+
 
 module SmartSearch
   
@@ -8,148 +15,222 @@ module SmartSearch
     base.extend ClassMethods
   end  
   
+  # Class Methods for ActiveRecord
   module ClassMethods
-    def smart_search(options = {:on => [], :conditions => nil, :group => nil, :order => "created_at"})
+    # Enable SmartSearch for the current ActiveRecord model.
+    # accepts options:
+    # - :on, define which attributes to add to the search index
+    # - :conditions, define default scope for all queries made
+    # - :group, group by column
+    # - :order, order by column
+    # see readme for details
+    def smart_search(options = {:on => [], :conditions => nil, :group => nil, :order => "created_at", :force => false})
       if table_exists?
         # Check if search_tags exists
-        unless is_smart_search?
-          cattr_accessor :condition_default, :group_default, :tags, :order 
+        if !is_smart_search? || options[:force] == true || Rails.env == "test"
+          
+          cattr_accessor :condition_default, :group_default, :tags, :order_default, :enable_similarity, :default_template_path
           send :include, InstanceMethods
-          if self.column_names.index("search_tags").nil?
-            ::AddSearchTags.add_to_table(self.table_name)
-          end
-            self.send(:before_save, :create_search_tags)
-
+            self.send(:after_save, :create_search_tags)
+            self.send(:before_destroy, :clear_search_tags)
+            self.enable_similarity ||= true
+            
+            attr_accessor :query_score
+            
             # options zuweisen
             if options[:conditions].is_a?(String) && !options[:conditions].blank?
               self.condition_default = options[:conditions]
             elsif !options[:conditions].nil?
               raise ArgumentError, ":conditions must be a valid SQL Query"  
-            end
-          
-            if self.column_names.include?("created_at")
-              self.order = options[:order] || "created_at"
-            else  
-              self.order = options[:order] || "id"
+            else
+              self.condition_default = nil
             end  
-
-            if options[:group].is_a?(String) && !options[:group].blank?
-              self.group_default = options[:group]
-            elsif !options[:group].nil?
-              raise ArgumentError, ":group must be a valid SQL Query"  
-            end
+          
+            self.order_default = options[:order]
 
             self.tags = options[:on] || []
         end
       end  
     end
     
+    # Verify if SmartSearch already loaded for this model
     def is_smart_search?
       self.included_modules.include?(InstanceMethods)
     end
     
+    # defines where to look for a partial to load when displaying results for this model
     def result_template_path
       "/search/results/#{self.name.split("::").last.underscore}"
     end  
     
-    def find_by_tags(tags = "", conditions = {}, group_by = nil)
+    # Serach database for given search tags
+    def find_by_tags(tags = "", options = {})
       if self.is_smart_search?
-        tags = tags.split(" ")
+        
+        # Save Data for similarity analysis
+        if tags.size > 3
+          self.connection.execute("INSERT INTO `#{::SmartSearchHistory.table_name}` (`query`) VALUES ('#{tags.gsub(/[^a-zA-ZäöüÖÄÜß\ ]/, '')}');")
+        end  
+        
+        tags = tags.split(/[\ -]/).select {|t| !t.blank?}
         
         # Fallback for Empty String
         tags << "#" if tags.empty?
         
-        tags.map! {|t| "search_tags LIKE '%#{t.downcase}%'"}
-      
-        if self.condition_default
-          default_cond = "#{self.condition_default} AND"
+        # Similarity
+        if self.enable_similarity == true
+          tags.map! do |t|   
+            similars = SmartSimilarity.similars(t, :increment_counter => true).join("|")
+            "search_tags REGEXP '#{similars}'"
+          end  
+          
         else
-          default_cond = ""
-        end    
-      
-        sql = "SELECT * FROM #{self.table_name} WHERE #{default_cond} (#{tags.join(' AND ')})"
-        if conditions.nil?
-          sql << ""
-        elsif conditions.is_a?(String)
-          sql << " AND #{conditions} "
+          tags.map! {|t| "search_tags LIKE '%#{t}%'"}
+        end  
+        
+        # Load ranking from Search tags
+        result_ids = []
+        result_scores = {}
+          SmartSearchTag.connection.select_all("select entry_id, sum(boost) as score, group_concat(search_tags) as grouped_tags 
+        from smart_search_tags where `table_name`= '#{self.table_name}' and 
+        
+        (#{tags.join(' OR ')}) group by entry_id having (#{tags.join(' AND ').gsub('search_tags', 'grouped_tags')}) order by score DESC").each do |r| 
+          result_ids << r["entry_id"].to_i 
+          result_scores[r["entry_id"].to_i] = r['score'].to_f
+        end  
+        
+        # Enable unscoped searching
+        if options[:unscoped] == true
+          results     =  self.unscoped.where(:id => result_ids)
         else  
-          condi = ""
-          conditions.each do |field, value|
-            case value
-              when true
-                value = 1
-              when false 
-                value = 0
-            end
-            if value.is_a?(String) || value.is_a?(Numeric)     
-              value = "'#{value}'" 
-              condi << " AND #{field.to_s} = #{value}"
-            elsif value.is_a?(Array)  
-              condi << " AND #{field.to_s} IN (#{value.join(",")})"
-            end  
-          
-          end  
-          sql << " #{condi}"
-        end
-        if !group_by.nil?
-          sql << " GROUP BY #{group_by}"
-        elsif self.group_default
-          sql << " GROUP BY #{self.group_default}"
+          results     =  self.where(:id => result_ids)
+        end  
+        
+        
+        
+        if options[:conditions]
+          results = results.where(options[:conditions])
         end
         
-        if !self.order.blank?
-          sql << " ORDER BY #{self.order}"
+        if !self.condition_default.blank?
+          results = results.where(self.condition_default)
+        end  
+        
+        if options[:group] 
+          results = results.group(options[:group])
         end  
         
-        puts sql
-        self.find_by_sql(sql)
+        if options[:order] || self.order_default
+          results = results.order(options[:order] || self.order_default)
+        else
+          ordered_results = []
+          results.each do |r| 
+            r.query_score = result_scores[r.id]
+            ordered_results[result_ids.index(r.id)] = r
+          end  
+          
+          results = ordered_results.compact
+        end      
+                
+        return results
       else                      
         raise "#{self.inspect} is not a SmartSearch"
       end  
     end
     
+    # reload search_tags for entire table based on the attributes defined in ':on' option passed to the 'smart_search' method
     def set_search_index
       s = self.all.size.to_f
       self.all.each_with_index do |a, i|
         a.create_search_tags
-        a.send(:update_without_callbacks)
         done = ((i+1).to_f/s)*100
         printf "Set search index for #{self.name}: #{done}%%                  \r"
       end  
     end  
+    
+    # Load all search tags for this table into similarity index
+    def set_similarity_index
+      
+      search_tags_list = self.connection.select_all("SELECT search_tags from #{SmartSearchTag.table_name} where `table_name` = #{self.table_name}").map {|r| r["search_tags"]}
+      
+      SmartSimilarity.create_from_text(search_tags_list.join(" "))
+    end  
          
   end  
   
+  # Instance Methods for ActiveRecord
   module InstanceMethods
     
+    # Load the result template path for this instance
     def result_template_path
       self.class.result_template_path
     end  
     
+    # create search tags for this very record based on the attributes defined in ':on' option passed to the 'Class.smart_search' method
     def create_search_tags
-      tags = []
+      tags      = []
+      
       self.class.tags.each do |tag|
-        if tag.is_a?(Symbol)
-          tags << self.send(tag) rescue ""
-        elsif tag.is_a?(String)
-          tag_methods = tag.split(".")  
+        
+        if !tag.is_a?(Hash)
+          tag = {:field_name => tag, :boost => 1, :search_tags => ""} 
+        else
+          tag[:search_tags] = ""
+          tag[:boost] ||= 1
+        end    
+        
+        if tag[:field_name].is_a?(Symbol)
+          tag[:search_tags] << self.send(tag[:field_name]).to_s
+        elsif tag[:field_name].is_a?(String)
+          tag_methods = tag[:field_name].split(".")  
           tagx = self.send(tag_methods[0])
           tag_methods[1..-1].each do |x|
             tagx = tagx.send(x) rescue ""
           end
-          tags << tagx  
+          tag[:search_tags] << tagx.to_s  
+        end
+        
+        tag[:search_tags] = tag[:search_tags].split(" ").uniq.join(" ").downcase.clear_html   
+        tags << tag
+      end
+      
+      
+      self.clear_search_tags
+      
+      # Merge search tags with same boost
+      @merged_tags = {}
+      
+      tags.each do |t|
+        boost = t[:boost]
+        
+        if @merged_tags[boost]
+        
+          @merged_tags[boost][:field_name] << ",#{t[:field_name]}"
+          @merged_tags[boost][:search_tags] << " #{t[:search_tags]}"
+        else
+          @merged_tags[boost] = {:field_name => "#{t[:field_name]}", :search_tags => t[:search_tags], :boost => boost }
+        end    
+        
+      end  
+      
+      @merged_tags.values.each do |t|
+        if !t[:search_tags].blank? && t[:search_tags].size > 1
+          SmartSearchTag.create(t.merge!(:table_name => self.class.table_name, :entry_id => self.id, :search_tags => t[:search_tags].strip.split(" ").uniq.join(" "))) 
         end  
       end
-      searchtags = tags.join(" ").split(" ")  
-      searchtags = searchtags.uniq.join(" ")
-      search_tags_min = searchtags.gsub(" ", "").downcase
       
-      self.search_tags = "#{searchtags.downcase}"
-    end  
+    end
+    
+    # Remove search data for the instance from the index
+    def clear_search_tags
+      if !self.id.nil?
+        SmartSearchTag.connection.execute("DELETE from #{SmartSearchTag.table_name} where `table_name` = '#{self.class.table_name}' and entry_id = #{self.id}")
+      end  
+    end    
     
   end    
         
-  
+
   class Config
     
     cattr_accessor  :search_models
@@ -169,10 +250,7 @@ module SmartSearch
   end  
   
   
-  class SmartSearchEngine < Rails::Engine
-  end  
-  
 end
 
 
-ActiveRecord::Base.send(:include, SmartSearch)
+ActiveRecord::Base.send(:include, SmartSearch)

data/lib/smart_search_history.rb

@@ -0,0 +1,7 @@
+# Saves all queries made so the data can be used to build a similarity index based on the queries made
+# by actual users of the website
+class SmartSearchHistory < ActiveRecord::Base
+  
+  
+  
+end  

data/lib/smart_search_ignore_word.rb

@@ -0,0 +1,30 @@
+# The keep words out of the index, they can be added into this table
+# TODO: Its not working yet
+class SmartSearchIgnoreWord < ActiveRecord::Base
+
+  #= Configuration
+  self.table_name = "smart_search_ignore_words" 
+      #== Associations
+          # => Stuff in Here
+  
+      #== Plugins and modules
+        #=== PlugIns
+          # => Stuff in Here        
+  
+        #=== include Modules
+          # => Stuff in Here
+  
+      #== Konstanten
+          # => Stuff in Here
+  
+      #== Validation and Callbacks
+        #=== Validation
+          validates_uniqueness_of :word
+          
+        #=== Callbacks
+          # => Stuff in Here
+          
+          
+    # => END
+  
+end  

data/lib/smart_search_tag.rb

@@ -0,0 +1,16 @@
+# Represents the search index
+class SmartSearchTag < ActiveRecord::Base
+  
+  
+  # Get a list of available search tags
+  def self.tags_list(query, table= nil)
+    if query.size < 3
+      return []
+    else  
+      list = sql_query!("select search_tags from #{self.table_name} where search_tags like '%#{query}%' #{"and table_name = '#{table}'" if table} ").map {|r| r['search_tags']}
+      list = list.join(" ").clear_html.split(" ").uniq
+      return list.sort.grep(Regexp.new(query))
+    end  
+  end  
+  
+end  

data/lib/smart_similarity.rb

@@ -0,0 +1,133 @@
+# This class is used to build similiarity index
+class SmartSimilarity < ActiveRecord::Base
+  #= Configuration
+  serialize :similarities, Array
+  self.table_name = "smart_search_similarities"
+  
+      #== Associations
+          # => Stuff in Here
+
+      #== Plugins and modules
+        #=== PlugIns
+          # => Stuff in Here        
+
+        #=== include Modules
+          # => Stuff in Here
+
+      #== Konstanten
+          # Defines the min. result of word simililarity check
+          SIMILARITY_FACTOR = 0.77
+          # Defines first simililarity check method 
+          SIMILARITY_METHOD_1 = :jarowinkler
+          # Defines first simililarity check method 
+          SIMILARITY_METHOD_2 = :levenshtein
+          
+          # An average of both results will generated and compered with 'SIMILARITY_FACTOR'
+          
+          # Limit Number of similar words (still unused)
+          SIMILARITY_LIMIT  = 8
+          
+          # USe this regexp to split texts into words
+          SPLITTING_REGEXP = /\b/
+          
+      #== Validation and Callbacks
+        #=== Validation
+        
+        #=== Callbacks
+          # => Stuff in Here
+    # => END
+  
+    # Create similarity data based on the given text
+    # This method is used to generate date from every source, e.g. file, url, single words etc..
+    def self.create_from_text(text)
+      # prepare text
+      prepared_text = text.downcase.split(SPLITTING_REGEXP).uniq
+      prepared_text = prepared_text.select {|w| w.size >= 3 && !w.match(/[0-9\-_<>\.\/(){}&\?"'@*+$!=,:'#;]/)}
+      list = {}
+      prepared_text.each do |word|
+        # Load index from database
+        words_in_db = self.find_by_phrase(word)
+        if words_in_db.nil?
+          self.connection.execute "INSERT INTO `#{self.table_name}` (`phrase`, `ind`) VALUES ('#{word}', '#{word[0..1]}');"
+          current = []
+        else  
+          current = words_in_db.similarities
+        end  
+          
+        # If word is a substring of similarity word, it must not be saved,
+        # cause it will match anyway:
+        # 'how' will match 'show', so 'show' is not needed in index for 'how'
+        # Vice Versa, 'how' should also be found if query is 'show', so it will be kept in the index
+        current += prepared_text.select {|w| w != word && self.match_words(w,word) >= SIMILARITY_FACTOR && !w.match(word)}
+        
+        list[word] = current.uniq
+      end  
+      
+      # Write to Database
+      list.each do |word, sims|
+        sims = sims.sort_by {|s| self.match_words(s,word) }.reverse.first(SIMILARITY_LIMIT)
+        
+        self.connection.execute 'UPDATE %s set similarities = "%s" where phrase = "%s"' % [self.table_name, sims.to_yaml, word] rescue nil
+      end  
+    end
+    
+    # Add one simgle word to database and check if there are already similars
+    def self.add_word(word)
+      words = [word]
+      phrases = self.connection.select_all("SELECT phrase from smart_search_similarities").map {|r| r["phrase"] }  
+      words +=  phrases.select {|p| self.match_words(p,word) >= SIMILARITY_FACTOR }
+      
+      self.create_from_text(words.join(" "))
+    end
+    
+    # Load an entire file to the index.
+    # Best used for loading big dictionary files.
+    # Uses 'spawnling' to split the data into 8 stacks and load them simultaniously
+    def self.load_file(path)
+      count = %x{wc -l #{path}}.split[0].to_i.max(1)
+      puts "loading file: #{path}"
+      puts "=> #{count} rows"
+      
+      if count == 1
+        File.open(path, "r").read.split(SPLITTING_REGEXP).each {|w| self.add_word(w)}
+      else  
+        File.open(path, "r").read.split(SPLITTING_REGEXP).seperate([8,count].min).each_with_index do |stack, si| 
+            stack.each_with_index do |l,i|
+              self.add_word(l)
+            end
+        end
+      end    
+    end
+    
+    # Load words from website and save them to index
+    def self.load_url(url)
+      self.create_from_text(%x(curl #{url}))
+    end  
+    
+    # Loads your created query history and saves them to the index
+    def self.load_from_query_history
+      queries = ActiveRecord::Base.connection.select_all("SELECT query from `#{::SmartSearchHistory.table_name}`").map {|r| r["query"]}
+      queries.each {|q| self.add_word(q) }
+      
+      self.connection.execute("TRUNCATE `#{::SmartSearchHistory.table_name}`")
+    end
+    
+    # Get array of similar words including orig word
+    def self.similars(word, options = {})      
+      list = self.where(:phrase => word).first
+      if list.nil?
+        return [word]
+      else
+        self.connection.execute("UPDATE `smart_search_similarities` SET `count` = #{list.count+1} where `smart_search_similarities`.`phrase` = '#{list.phrase}'")
+        return [word, list.similarities].flatten
+      end    
+    end
+    
+    # Return match score for two words bases und the two defined similarity methods
+    def self.match_words(word1, word2)
+      x1 = word1.send("#{SIMILARITY_METHOD_1}_similar", word2)
+      x2 = word1.send("#{SIMILARITY_METHOD_2}_similar", word2)
+      return (x1+x2)/2.0
+    end    
+  
+end 

data/lib/tasks/smart_search.rake

@@ -0,0 +1,58 @@
+namespace :smart_search do
+  desc "Load similarity data from query history"
+  task :similarity_from_query_history => :environment do
+    require File.expand_path("../../smart_similarity", __FILE__)
+    SmartSimilarity.load_from_query_history
+  end
+  
+  desc "Load similarity data from file - Use FILE=path/to/file to specify file"
+  task :similarity_from_file => :environment do
+    require File.expand_path("../../smart_similarity", __FILE__)
+    if ENV['FILE_PATH'].nil?
+      raise ArgumentError, "No file specified. "
+    elsif !File.exist?(ENV['FILE_PATH'])
+      raise ArgumentError, "File not found "
+    else    
+      SmartSimilarity.load_file(ENV['FILE_PATH'])
+    end  
+  end
+  
+  desc "Load similarity data from url - Use URL=http://.../ to specify url - Requires 'curl'"
+  task :similarity_from_url => :environment do
+    require File.expand_path("../../smart_similarity", __FILE__)
+    if ENV['URL'].nil?
+      raise ArgumentError, "No URL specified. "
+    else    
+      SmartSimilarity.load_url(ENV['URL'])
+    end  
+  end
+  
+  
+  
+  desc "load ignore words list"
+  task :load_ignore_words => :environment do
+    require File.expand_path("../../smart_search_ignore_word", __FILE__)
+    
+    dic_path = File.expand_path("../../../dictionaries/*", __FILE__)
+    
+    raise dic_path.inspect
+    
+    dic_folders = Dir.glob(dic_path).select {|d| File.directory?(d)}
+    
+    dic_folders.each do |folder|
+      locale = folder.split("/").last
+      word_file = File.join(folder, "#{locale}.ignore_words.dic")
+      if File.exists?(word_file)
+        File.open(word_file, "r").each_line do |word|
+          SmartSearchIgnoreWord.create(:word => word.strip.downcase, :locale => locale)
+        end  
+      end  
+    end  
+  end   
+  
+  
+
+    
+end  
+
+

data/lib/tasks/testing.rake

@@ -0,0 +1,27 @@
+require "active_record"
+namespace :db do
+  desc "Create test database. Overwrite dasebase config with USERNAME=, PASSWORD=, DATABASE="
+  task :create_test_db do
+    config = YAML::load(File.open(File.expand_path("config/database.yml")))["test"]
+    
+    # Overwrite config
+    config.merge!('database' => ENV['DATABASE']) if ENV['DATABASE']
+    config.merge!('username' => ENV['USERNAME']) if ENV['USERNAME']
+    config.merge!('password' => ENV['PASSWORD']) if ENV['PASSWORD']
+    
+    ActiveRecord::Base.establish_connection(config.merge('database' => nil))
+    ActiveRecord::Base.connection.drop_database(config['database']) rescue nil
+    ActiveRecord::Base.connection.create_database(config['database'])
+    ActiveRecord::Base.establish_connection(config)
+  end
+  
+  task :migrate do
+    ActiveRecord::Migrator.migrate(ActiveRecord::Migrator.migrations_paths)
+  end    
+end  
+
+task :test_smart_search do
+  Rake::Task["db:create_test_db"].execute
+  Rake::Task["db:migrate"].execute
+  Rake::Task["test"].execute
+end  

data/test/test_document_multi_line.txt

@@ -0,0 +1,6 @@
+The quick brown fox jumps over the lazy dog.
+The five boxing wizards jump quickly.
+Sphinx of all black quartz judge my vow.
+A quick movement of the enemy will jeopardize six gunboats.
+Five quacking Zephyrs jolt my wax bed.
+Heavy boxes perform waltzes and jigs.

data/test/test_document_one_line.txt

@@ -0,0 +1 @@
+Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.